diff --git a/distrib/sets/lists/base/shl.mi b/distrib/sets/lists/base/shl.mi index 1c005ef945ced..777c2629741da 100644 --- a/distrib/sets/lists/base/shl.mi +++ b/distrib/sets/lists/base/shl.mi @@ -136,6 +136,9 @@ ./lib/npf/ext_rndblock.so base-npf-shlib compatfile,npf ./lib/npf/ext_rndblock.so.0 base-npf-shlib compatfile,npf ./lib/npf/ext_rndblock.so.0.0 base-npf-shlib compatfile,npf +./lib/npf/ext_route.so base-npf-shlib compatfile,npf +./lib/npf/ext_route.so.0 base-npf-shlib compatfile,npf +./lib/npf/ext_route.so.0.0 base-npf-shlib compatfile,npf ./libexec/ld.elf_so base-sys-shlib dynamicroot ./usr/lib/i18n/libBIG5.so base-i18n-shlib compatfile ./usr/lib/i18n/libBIG5.so.5 base-i18n-shlib compatfile diff --git a/distrib/sets/lists/modules/mi b/distrib/sets/lists/modules/mi index 1c0d24477e1ef..5b13c59bcf852 100644 --- a/distrib/sets/lists/modules/mi +++ b/distrib/sets/lists/modules/mi @@ -363,6 +363,8 @@ ./@MODULEDIR@/npf_ext_normalize/npf_ext_normalize.kmod modules-base-kernel kmod ./@MODULEDIR@/npf_ext_rndblock modules-base-kernel kmod ./@MODULEDIR@/npf_ext_rndblock/npf_ext_rndblock.kmod modules-base-kernel kmod +./@MODULEDIR@/npf_ext_route modules-base-kernel kmod +./@MODULEDIR@/npf_ext_route/npf_ext_route.kmod modules-base-kernel kmod ./@MODULEDIR@/ntfs modules-base-kernel kmod ./@MODULEDIR@/ntfs/ntfs.kmod modules-base-kernel kmod ./@MODULEDIR@/null modules-base-kernel kmod diff --git a/lib/npf/Makefile b/lib/npf/Makefile index 1e2b3c3e33b1c..6ff0e1671a764 100644 --- a/lib/npf/Makefile +++ b/lib/npf/Makefile @@ -4,7 +4,7 @@ .if ${MKPIC} != "no" -SUBDIR= ext_log ext_normalize ext_rndblock +SUBDIR= ext_log ext_normalize ext_rndblock ext_route .endif diff --git a/lib/npf/ext_route/Makefile b/lib/npf/ext_route/Makefile new file mode 100644 index 0000000000000..73162657a91be --- /dev/null +++ b/lib/npf/ext_route/Makefile @@ -0,0 +1,4 @@ +# $NetBSD: Makefile,v 1.3 2013/03/10 21:41:06 joe Exp $ + +MOD= ext_route +.include "${.CURDIR}/../mod.mk" diff --git a/lib/npf/ext_route/npfext_route.c b/lib/npf/ext_route/npfext_route.c new file mode 100644 index 0000000000000..f0118d3173c86 --- /dev/null +++ b/lib/npf/ext_route/npfext_route.c @@ -0,0 +1,71 @@ +/*- + * Copyright (c) 2025 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Mindaugas Rasiukevicius. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +__RCSID("$NetBSD$"); + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +int npfext_route_init(void); +nl_ext_t * npfext_route_construct(const char *); +int npfext_route_param(nl_ext_t *, const char *, const char *); + +int +npfext_route_init(void) +{ + /* Nothing to initialise. */ + return 0; +} + +nl_ext_t * +npfext_route_construct(const char *name) +{ + assert(strcmp(name, "route") == 0); + return npf_ext_construct(name); +} + +int +npfext_route_param(nl_ext_t *ext, const char *param, const char *val __unused) +{ + assert(param != NULL); + npf_ext_param_string(ext, "route-interface", param); + return 0; +} diff --git a/lib/npf/ext_route/shlib_version b/lib/npf/ext_route/shlib_version new file mode 100644 index 0000000000000..eecfd70dc57b6 --- /dev/null +++ b/lib/npf/ext_route/shlib_version @@ -0,0 +1,4 @@ +# $NetBSD: shlib_version,v 1.1 2025/08/26 00:32:25 joe Exp $ + +major=0 +minor=0 \ No newline at end of file diff --git a/sys/dev/usb/usb_subr.c b/sys/dev/usb/usb_subr.c deleted file mode 100644 index ff6e45375f85e..0000000000000 --- a/sys/dev/usb/usb_subr.c +++ /dev/null @@ -1,1942 +0,0 @@ -/* $NetBSD: usb_subr.c,v 1.281 2025/10/05 20:41:04 riastradh Exp $ */ -/* $FreeBSD: src/sys/dev/usb/usb_subr.c,v 1.18 1999/11/17 22:33:47 n_hibma Exp $ */ - -/* - * Copyright (c) 1998, 2004 The NetBSD Foundation, Inc. - * All rights reserved. - * - * This code is derived from software contributed to The NetBSD Foundation - * by Lennart Augustsson (lennart@augustsson.net) at - * Carlstedt Research & Technology. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED - * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#include -__KERNEL_RCSID(0, "$NetBSD: usb_subr.c,v 1.281 2025/10/05 20:41:04 riastradh Exp $"); - -#ifdef _KERNEL_OPT -#include "opt_compat_netbsd.h" -#include "opt_usb.h" -#include "opt_usbverbose.h" -#endif - -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "locators.h" - -#define DPRINTF(FMT,A,B,C,D) USBHIST_LOG(usbdebug,FMT,A,B,C,D) -#define DPRINTFN(N,FMT,A,B,C,D) USBHIST_LOGN(usbdebug,N,FMT,A,B,C,D) - -Static void usbd_devinfo(struct usbd_device *, int, char *, size_t); -Static int usbd_getnewaddr(struct usbd_bus *); -Static int usbd_print(void *, const char *); -Static int usbd_ifprint(void *, const char *); -Static void usbd_free_iface_data(struct usbd_device *, int); - -uint32_t usb_cookie_no = 0; - -Static const char * const usbd_error_strs[] = { - "NORMAL_COMPLETION", - "IN_PROGRESS", - "PENDING_REQUESTS", - "NOT_STARTED", - "INVAL", - "NOMEM", - "CANCELLED", - "BAD_ADDRESS", - "IN_USE", - "NO_ADDR", - "SET_ADDR_FAILED", - "NO_POWER", - "TOO_DEEP", - "IOERROR", - "NOT_CONFIGURED", - "TIMEOUT", - "SHORT_XFER", - "STALLED", - "INTERRUPTED", - "XXX", -}; - -DEV_VERBOSE_DEFINE(usb); - -const char * -usbd_errstr(usbd_status err) -{ - static char buffer[5]; - - if (err < USBD_ERROR_MAX) { - return usbd_error_strs[err]; - } else { - snprintf(buffer, sizeof(buffer), "%d", err); - return buffer; - } -} - -static void -usbd_trim_spaces(char *p) -{ - char *q, *e; - - q = e = p; - while (*q == ' ') /* skip leading spaces */ - q++; - while ((*p = *q++)) /* copy string */ - if (*p++ != ' ') /* remember last non-space */ - e = p; - *e = '\0'; /* kill trailing spaces */ -} - -static void -usbd_get_device_string(struct usbd_device *ud, uByte index, char **buf) -{ - char *b; - usbd_status err; - - b = kmem_alloc(USB_MAX_ENCODED_STRING_LEN, KM_SLEEP); - err = usbd_get_string0(ud, index, b, true); - if (err != USBD_NORMAL_COMPLETION) { - kmem_free(b, USB_MAX_ENCODED_STRING_LEN); - b = NULL; - } else { - usbd_trim_spaces(b); - } - - *buf = b; -} - -void -usbd_get_device_strings(struct usbd_device *ud) -{ - usb_device_descriptor_t *udd = &ud->ud_ddesc; - - usbd_get_device_string(ud, udd->iManufacturer, &ud->ud_vendor); - usbd_get_device_string(ud, udd->iProduct, &ud->ud_product); - usbd_get_device_string(ud, udd->iSerialNumber, &ud->ud_serial); -} - -void -usbd_devinfo_vp(struct usbd_device *dev, char *v, size_t vl, char *p, - size_t pl, int usedev, int useencoded) -{ - usb_device_descriptor_t *udd = &dev->ud_ddesc; - if (dev == NULL) - return; - - v[0] = p[0] = '\0'; - - if (usedev) { - if (usbd_get_string0(dev, udd->iManufacturer, v, useencoded) == - USBD_NORMAL_COMPLETION) - usbd_trim_spaces(v); - if (usbd_get_string0(dev, udd->iProduct, p, useencoded) == - USBD_NORMAL_COMPLETION) - usbd_trim_spaces(p); - } else { - if (dev->ud_vendor) { - strlcpy(v, dev->ud_vendor, vl); - } - if (dev->ud_product) { - strlcpy(p, dev->ud_product, pl); - } - } - if (v[0] == '\0') - usb_findvendor(v, vl, UGETW(udd->idVendor)); - if (p[0] == '\0') - usb_findproduct(p, pl, UGETW(udd->idVendor), - UGETW(udd->idProduct)); -} - -int -usbd_printBCD(char *cp, size_t l, int bcd) -{ - return snprintf(cp, l, "%x.%02x", bcd >> 8, bcd & 0xff); -} - -Static void -usbd_devinfo(struct usbd_device *dev, int showclass, char *cp, size_t l) -{ - usb_device_descriptor_t *udd = &dev->ud_ddesc; - char *vendor, *product; - int bcdDevice, bcdUSB; - char *ep; - - vendor = kmem_alloc(USB_MAX_ENCODED_STRING_LEN * 2, KM_SLEEP); - product = &vendor[USB_MAX_ENCODED_STRING_LEN]; - - ep = cp + l; - - usbd_devinfo_vp(dev, vendor, USB_MAX_ENCODED_STRING_LEN, - product, USB_MAX_ENCODED_STRING_LEN, 0, 1); - cp += snprintf(cp, ep - cp, "%s (0x%04x) %s (0x%04x)", vendor, - UGETW(udd->idVendor), product, UGETW(udd->idProduct)); - if (showclass) - cp += snprintf(cp, ep - cp, ", class %d/%d", - udd->bDeviceClass, udd->bDeviceSubClass); - bcdUSB = UGETW(udd->bcdUSB); - bcdDevice = UGETW(udd->bcdDevice); - cp += snprintf(cp, ep - cp, ", rev "); - cp += usbd_printBCD(cp, ep - cp, bcdUSB); - *cp++ = '/'; - cp += usbd_printBCD(cp, ep - cp, bcdDevice); - cp += snprintf(cp, ep - cp, ", addr %d", dev->ud_addr); - *cp = 0; - kmem_free(vendor, USB_MAX_ENCODED_STRING_LEN * 2); -} - -char * -usbd_devinfo_alloc(struct usbd_device *dev, int showclass) -{ - char *devinfop; - - devinfop = kmem_alloc(DEVINFOSIZE, KM_SLEEP); - usbd_devinfo(dev, showclass, devinfop, DEVINFOSIZE); - return devinfop; -} - -void -usbd_devinfo_free(char *devinfop) -{ - kmem_free(devinfop, DEVINFOSIZE); -} - -/* Delay for a certain number of ms */ -void -usb_delay_ms_locked(struct usbd_bus *bus, u_int ms, kmutex_t *lock) -{ - /* Wait at least two clock ticks so we know the time has passed. */ - if (bus->ub_usepolling || cold) - delay((ms+1) * 1000); - else - kpause("usbdly", false, (ms*hz+999)/1000 + 1, lock); -} - -void -usb_delay_ms(struct usbd_bus *bus, u_int ms) -{ - usb_delay_ms_locked(bus, ms, NULL); -} - -/* Delay given a device handle. */ -void -usbd_delay_ms_locked(struct usbd_device *dev, u_int ms, kmutex_t *lock) -{ - usb_delay_ms_locked(dev->ud_bus, ms, lock); -} - -/* Delay given a device handle. */ -void -usbd_delay_ms(struct usbd_device *dev, u_int ms) -{ - usb_delay_ms_locked(dev->ud_bus, ms, NULL); -} - -usbd_status -usbd_reset_port(struct usbd_device *dev, int port, usb_port_status_t *ps) -{ - USBHIST_FUNC(); USBHIST_CALLARGS(usbdebug, "port %jd", port, 0, 0, 0); - usb_device_request_t req; - usbd_status err; - int n; - - req.bmRequestType = UT_WRITE_CLASS_OTHER; - req.bRequest = UR_SET_FEATURE; - USETW(req.wValue, UHF_PORT_RESET); - USETW(req.wIndex, port); - USETW(req.wLength, 0); - err = usbd_do_request(dev, &req, 0); - DPRINTFN(1, "port %jd reset done, error=%jd", port, err, 0, 0); - if (err) - return err; - n = 10; - do { - /* Wait for device to recover from reset. */ - usbd_delay_ms(dev, USB_PORT_RESET_DELAY); - err = usbd_get_port_status(dev, port, ps); - if (err) { - DPRINTF("get status failed %jd", err, 0, 0, 0); - return err; - } - /* If the device disappeared, just give up. */ - if (!(UGETW(ps->wPortStatus) & UPS_CURRENT_CONNECT_STATUS)) - return USBD_NORMAL_COMPLETION; - } while ((UGETW(ps->wPortChange) & UPS_C_PORT_RESET) == 0 && --n > 0); - if (n == 0) - return USBD_TIMEOUT; - err = usbd_clear_port_feature(dev, port, UHF_C_PORT_RESET); -#ifdef USB_DEBUG - if (err) - DPRINTF("clear port feature failed %jd", err, 0, 0, 0); -#endif - - /* Wait for the device to recover from reset. */ - usbd_delay_ms(dev, USB_PORT_RESET_RECOVERY); - return err; -} - -usb_interface_descriptor_t * -usbd_find_idesc(usb_config_descriptor_t *cd, int ifaceidx, int altidx) -{ - USBHIST_FUNC(); - USBHIST_CALLARGS(usbdebug, "iface/alt idx %jd/%jd", - ifaceidx, altidx, 0, 0); - char *p = (char *)cd; - char *end = p + UGETW(cd->wTotalLength); - usb_descriptor_t *desc; - usb_interface_descriptor_t *idesc; - int curidx, lastidx, curaidx = 0; - - for (curidx = lastidx = -1; end - p >= sizeof(*desc);) { - desc = (usb_descriptor_t *)p; - - DPRINTFN(4, "idx=%jd(%jd) altidx=%jd(%jd)", ifaceidx, curidx, - altidx, curaidx); - DPRINTFN(4, "len=%jd type=%jd", desc->bLength, - desc->bDescriptorType, 0, 0); - - if (desc->bLength < USB_DESCRIPTOR_SIZE) - break; - if (desc->bLength > end - p) - break; - p += desc->bLength; - - if (desc->bDescriptorType != UDESC_INTERFACE) - continue; - if (desc->bLength < USB_INTERFACE_DESCRIPTOR_SIZE) - break; - idesc = (usb_interface_descriptor_t *)desc; - - if (idesc->bInterfaceNumber != lastidx) { - lastidx = idesc->bInterfaceNumber; - curidx++; - curaidx = 0; - } else { - curaidx++; - } - if (ifaceidx == curidx && altidx == curaidx) - return idesc; - } - - return NULL; -} - -usb_endpoint_descriptor_t * -usbd_find_edesc(usb_config_descriptor_t *cd, int ifaceidx, int altidx, - int endptidx) -{ - char *p = (char *)cd; - char *end = p + UGETW(cd->wTotalLength); - usb_interface_descriptor_t *idesc; - usb_endpoint_descriptor_t *edesc; - usb_descriptor_t *desc; - int curidx; - - idesc = usbd_find_idesc(cd, ifaceidx, altidx); - if (idesc == NULL) - return NULL; - if (endptidx >= idesc->bNumEndpoints) /* quick exit */ - return NULL; - - curidx = -1; - for (p = (char *)idesc + idesc->bLength; end - p >= sizeof(*edesc);) { - desc = (usb_descriptor_t *)p; - - if (desc->bLength < USB_DESCRIPTOR_SIZE) - break; - if (desc->bLength > end - p) - break; - p += desc->bLength; - - if (desc->bDescriptorType == UDESC_INTERFACE) - break; - if (desc->bDescriptorType != UDESC_ENDPOINT) - continue; - - if (desc->bLength < USB_ENDPOINT_DESCRIPTOR_SIZE) - break; - edesc = (usb_endpoint_descriptor_t *)desc; - - curidx++; - if (curidx == endptidx) - return edesc; - } - return NULL; -} - -static void -usbd_iface_init(struct usbd_device *dev, int ifaceidx) -{ - struct usbd_interface *ifc = &dev->ud_ifaces[ifaceidx]; - - memset(ifc, 0, sizeof(*ifc)); - - ifc->ui_dev = dev; - ifc->ui_idesc = NULL; - ifc->ui_index = 0; - ifc->ui_altindex = 0; - ifc->ui_endpoints = NULL; - ifc->ui_busy = 0; -} - -static void -usbd_iface_fini(struct usbd_device *dev, int ifaceidx) -{ - struct usbd_interface *ifc __diagused = &dev->ud_ifaces[ifaceidx]; - - KASSERT(ifc->ui_dev == dev); - KASSERT(ifc->ui_idesc == NULL); - KASSERT(ifc->ui_index == 0); - KASSERT(ifc->ui_altindex == 0); - KASSERT(ifc->ui_endpoints == NULL); - KASSERTMSG(ifc->ui_busy == 0, "%"PRId64, ifc->ui_busy); -} - -/* - * usbd_iface_lock/locked/unlock, usbd_iface_piperef/pipeunref - * - * We lock the interface while we are setting it, and we acquire a - * reference to the interface for each pipe opened on it. - * - * Setting the interface while pipes are open is forbidden, and - * opening pipes while the interface is being set is forbidden. - */ - -bool -usbd_iface_locked(struct usbd_interface *iface) -{ - bool locked; - - mutex_enter(iface->ui_dev->ud_bus->ub_lock); - locked = (iface->ui_busy == -1); - mutex_exit(iface->ui_dev->ud_bus->ub_lock); - - return locked; -} - -static void -usbd_iface_exlock(struct usbd_interface *iface) -{ - - mutex_enter(iface->ui_dev->ud_bus->ub_lock); - KASSERTMSG(iface->ui_busy == 0, "interface is not idle," - " busy=%"PRId64, iface->ui_busy); - iface->ui_busy = -1; - mutex_exit(iface->ui_dev->ud_bus->ub_lock); -} - -usbd_status -usbd_iface_lock(struct usbd_interface *iface) -{ - usbd_status err; - - mutex_enter(iface->ui_dev->ud_bus->ub_lock); - KASSERTMSG(iface->ui_busy != -1, "interface is locked"); - KASSERTMSG(iface->ui_busy >= 0, "%"PRId64, iface->ui_busy); - if (iface->ui_busy) { - err = USBD_IN_USE; - } else { - iface->ui_busy = -1; - err = 0; - } - mutex_exit(iface->ui_dev->ud_bus->ub_lock); - - return err; -} - -void -usbd_iface_unlock(struct usbd_interface *iface) -{ - - mutex_enter(iface->ui_dev->ud_bus->ub_lock); - KASSERTMSG(iface->ui_busy == -1, "interface is not locked," - " busy=%"PRId64, iface->ui_busy); - iface->ui_busy = 0; - mutex_exit(iface->ui_dev->ud_bus->ub_lock); -} - -usbd_status -usbd_iface_piperef(struct usbd_interface *iface) -{ - usbd_status err; - - mutex_enter(iface->ui_dev->ud_bus->ub_lock); - KASSERTMSG(iface->ui_busy >= -1, "%"PRId64, iface->ui_busy); - if (iface->ui_busy == -1) { - err = USBD_IN_USE; - } else { - iface->ui_busy++; - err = 0; - } - mutex_exit(iface->ui_dev->ud_bus->ub_lock); - - return err; -} - -void -usbd_iface_pipeunref(struct usbd_interface *iface) -{ - - mutex_enter(iface->ui_dev->ud_bus->ub_lock); - KASSERTMSG(iface->ui_busy != -1, "interface is locked"); - KASSERTMSG(iface->ui_busy != 0, "interface not in use"); - KASSERTMSG(iface->ui_busy >= 1, "%"PRId64, iface->ui_busy); - iface->ui_busy--; - mutex_exit(iface->ui_dev->ud_bus->ub_lock); -} - -usbd_status -usbd_fill_iface_data(struct usbd_device *dev, int ifaceidx, int altidx) -{ - USBHIST_FUNC(); - USBHIST_CALLARGS(usbdebug, "ifaceidx=%jd altidx=%jd", - ifaceidx, altidx, 0, 0); - struct usbd_interface *ifc = &dev->ud_ifaces[ifaceidx]; - usb_descriptor_t *desc; - usb_interface_descriptor_t *idesc; - usb_endpoint_descriptor_t *ed; - struct usbd_endpoint *endpoints; - char *p, *end; - int endpt, nendpt; - - KASSERT(ifc->ui_dev == dev); - KASSERT(usbd_iface_locked(ifc)); - - idesc = usbd_find_idesc(dev->ud_cdesc, ifaceidx, altidx); - if (idesc == NULL) - return USBD_INVAL; - - nendpt = idesc->bNumEndpoints; - DPRINTFN(4, "found idesc nendpt=%jd", nendpt, 0, 0, 0); - if (nendpt != 0) { - endpoints = kmem_alloc(nendpt * sizeof(struct usbd_endpoint), - KM_SLEEP); - } else - endpoints = NULL; - - p = (char *)idesc + idesc->bLength; - end = (char *)dev->ud_cdesc + UGETW(dev->ud_cdesc->wTotalLength); - KASSERTMSG((char *)dev->ud_cdesc <= (char *)idesc, "cdesc=%p idesc=%p", - dev->ud_cdesc, idesc); - KASSERTMSG((char *)idesc < end, "idesc=%p end=%p", idesc, end); - for (endpt = 0; endpt < nendpt; endpt++) { - DPRINTFN(10, "endpt=%jd", endpt, 0, 0, 0); - for (; end - p >= sizeof(*desc); p += desc->bLength) { - desc = (usb_descriptor_t *)p; - DPRINTFN(10, "p=%#jx end=%#jx len=%jd type=%jd", - (uintptr_t)p, (uintptr_t)end, desc->bLength, - desc->bDescriptorType); - if (desc->bLength < sizeof(*desc)) { - printf("%s: bad descriptor: too short\n", - __func__); - goto bad; - } else if (desc->bLength > end - p) { - printf("%s: bad descriptor: too long\n", - __func__); - goto bad; - } else if (desc->bDescriptorType == UDESC_INTERFACE) { - printf("%s: bad descriptor: iface desc\n", - __func__); - goto bad; - } - if (desc->bLength >= USB_ENDPOINT_DESCRIPTOR_SIZE && - desc->bDescriptorType == UDESC_ENDPOINT) { - ed = (usb_endpoint_descriptor_t *)p; - goto found; - } - } - printf("%s: no desc found\n", __func__); - goto bad; - found: - endpoints[endpt].ue_edesc = ed; - if (dev->ud_speed == USB_SPEED_HIGH) { - u_int mps; - /* Control and bulk endpoints have max packet limits. */ - switch (UE_GET_XFERTYPE(ed->bmAttributes)) { - case UE_CONTROL: - mps = USB_2_MAX_CTRL_PACKET; - goto check; - case UE_BULK: - mps = USB_2_MAX_BULK_PACKET; - check: - if (UGETW(ed->wMaxPacketSize) != mps) { - USETW(ed->wMaxPacketSize, mps); -#ifdef DIAGNOSTIC - printf("usbd_fill_iface_data: bad max " - "packet size\n"); -#endif - } - break; - default: - break; - } - } - endpoints[endpt].ue_refcnt = 0; - endpoints[endpt].ue_toggle = 0; - KASSERTMSG(end - p >= ed->bLength, "p=%p end=%p length=%u", - p, end, ed->bLength); - p += ed->bLength; - } -#undef ed - - /* Success! Free the old endpoints and commit the changes. */ - if (ifc->ui_endpoints) { - kmem_free(ifc->ui_endpoints, (sizeof(ifc->ui_endpoints[0]) * - ifc->ui_idesc->bNumEndpoints)); - } - - ifc->ui_idesc = idesc; - ifc->ui_index = ifaceidx; - ifc->ui_altindex = altidx; - ifc->ui_endpoints = endpoints; - - return USBD_NORMAL_COMPLETION; - - bad: - if (endpoints) - kmem_free(endpoints, nendpt * sizeof(struct usbd_endpoint)); - return USBD_INVAL; -} - -Static void -usbd_free_iface_data(struct usbd_device *dev, int ifcno) -{ - struct usbd_interface *ifc = &dev->ud_ifaces[ifcno]; - - KASSERT(ifc->ui_dev == dev); - KASSERT(ifc->ui_idesc != NULL); - KASSERT(usbd_iface_locked(ifc)); - - if (ifc->ui_endpoints) { - int nendpt = ifc->ui_idesc->bNumEndpoints; - size_t sz = nendpt * sizeof(struct usbd_endpoint); - kmem_free(ifc->ui_endpoints, sz); - ifc->ui_endpoints = NULL; - } - - ifc->ui_altindex = 0; - ifc->ui_index = 0; - ifc->ui_idesc = NULL; -} - -usbd_status -usbd_set_config_no(struct usbd_device *dev, int no, int msg) -{ - USBHIST_FUNC(); USBHIST_CALLARGS(usbdebug, "%jd", no, 0, 0, 0); - usb_config_descriptor_t cd; - usbd_status err; - int index; - - if (no == USB_UNCONFIG_NO) - return usbd_set_config_index(dev, USB_UNCONFIG_INDEX, msg); - - /* Figure out what config index to use. */ - for (index = 0; index < dev->ud_ddesc.bNumConfigurations; index++) { - err = usbd_get_config_desc(dev, index, &cd); - if (err) - return err; - if (cd.bConfigurationValue == no) - return usbd_set_config_index(dev, index, msg); - } - return USBD_INVAL; -} - -usbd_status -usbd_set_config_index(struct usbd_device *dev, int index, int msg) -{ - USBHIST_FUNC(); - USBHIST_CALLARGS(usbdebug, "dev=%#jx index=%jd", - (uintptr_t)dev, index, 0, 0); - usb_config_descriptor_t cd, *cdp; - usb_bos_descriptor_t *bdp = NULL; - usbd_status err; - int i, ifcidx, nifc, len, selfpowered, power; - - if ((unsigned)index >= dev->ud_ddesc.bNumConfigurations && - index != USB_UNCONFIG_INDEX) { - /* panic? */ - printf("usbd_set_config_index: illegal index\n"); - return USBD_INVAL; - } - - /* XXX check that all interfaces are idle */ - if (dev->ud_configidx != USB_UNCONFIG_INDEX) { - DPRINTF("free old config", 0, 0, 0, 0); - /* Free all configuration data structures. */ - nifc = dev->ud_cdesc->bNumInterface; - for (ifcidx = 0; ifcidx < nifc; ifcidx++) { - usbd_iface_exlock(&dev->ud_ifaces[ifcidx]); - usbd_free_iface_data(dev, ifcidx); - usbd_iface_unlock(&dev->ud_ifaces[ifcidx]); - usbd_iface_fini(dev, ifcidx); - } - kmem_free(dev->ud_ifaces, nifc * sizeof(struct usbd_interface)); - kmem_free(dev->ud_cdesc, UGETW(dev->ud_cdesc->wTotalLength)); - if (dev->ud_bdesc != NULL) - kmem_free(dev->ud_bdesc, - UGETW(dev->ud_bdesc->wTotalLength)); - dev->ud_ifaces = NULL; - dev->ud_cdesc = NULL; - dev->ud_bdesc = NULL; - dev->ud_config = USB_UNCONFIG_NO; - dev->ud_configidx = USB_UNCONFIG_INDEX; - } - KASSERTMSG(dev->ud_config == USB_UNCONFIG_NO, "ud_config=%u", - dev->ud_config); - KASSERTMSG(dev->ud_configidx == USB_UNCONFIG_INDEX, "ud_configidx=%d", - dev->ud_configidx); - - if (index == USB_UNCONFIG_INDEX) { - /* We are unconfiguring the device, so leave unallocated. */ - DPRINTF("set config 0", 0, 0, 0, 0); - err = usbd_set_config(dev, USB_UNCONFIG_NO); - if (err) { - DPRINTF("setting config=0 failed, err = %jd", err, - 0, 0, 0); - } - return err; - } - - /* Get the short descriptor. */ - err = usbd_get_config_desc(dev, index, &cd); - if (err) { - DPRINTF("get_config_desc=%jd", err, 0, 0, 0); - return err; - } - len = UGETW(cd.wTotalLength); - if (len < USB_CONFIG_DESCRIPTOR_SIZE) { - DPRINTF("empty short descriptor", 0, 0, 0, 0); - return USBD_INVAL; - } - cdp = kmem_alloc(len, KM_SLEEP); - - /* Get the full descriptor. Try a few times for slow devices. */ - for (i = 0; i < 3; i++) { - err = usbd_get_desc(dev, UDESC_CONFIG, index, len, cdp); - if (!err) - break; - usbd_delay_ms(dev, 200); - } - if (err) { - DPRINTF("get_desc=%jd", err, 0, 0, 0); - goto bad; - } - if (cdp->bDescriptorType != UDESC_CONFIG) { - DPRINTF("bad desc %jd", cdp->bDescriptorType, 0, 0, 0); - err = USBD_INVAL; - goto bad; - } - if (UGETW(cdp->wTotalLength) != UGETW(cd.wTotalLength)) { - DPRINTF("bad len %jd", UGETW(cdp->wTotalLength), 0, 0, 0); - err = USBD_INVAL; - goto bad; - } - - if (USB_IS_SS(dev->ud_speed)) { - usb_bos_descriptor_t bd; - - /* get short bos desc */ - err = usbd_get_bos_desc(dev, index, &bd); - if (!err) { - int blen = UGETW(bd.wTotalLength); - if (blen < USB_BOS_DESCRIPTOR_SIZE) { - DPRINTF("empty bos descriptor", 0, 0, 0, 0); - err = USBD_INVAL; - goto bad; - } - bdp = kmem_alloc(blen, KM_SLEEP); - - /* Get the full desc */ - for (i = 0; i < 3; i++) { - err = usbd_get_desc(dev, UDESC_BOS, index, blen, - bdp); - if (!err) - break; - usbd_delay_ms(dev, 200); - } - if (err || bdp->bDescriptorType != UDESC_BOS || - UGETW(bdp->wTotalLength) != UGETW(bd.wTotalLength)) { - DPRINTF("error %jd or bad desc %jd", err, - bdp->bDescriptorType, 0, 0); - kmem_free(bdp, blen); - bdp = NULL; - } - } - } - dev->ud_bdesc = bdp; - - /* - * Figure out if the device is self or bus powered. - */ -#if 0 /* XXX various devices don't report the power state correctly */ - selfpowered = 0; - err = usbd_get_device_status(dev, &ds); - if (!err && (UGETW(ds.wStatus) & UDS_SELF_POWERED)) - selfpowered = 1; -#endif - /* - * Use the power state in the configuration we are going - * to set. This doesn't necessarily reflect the actual - * power state of the device; the driver can control this - * by choosing the appropriate configuration. - */ - selfpowered = !!(cdp->bmAttributes & UC_SELF_POWERED); - - DPRINTF("addr %jd cno=%jd attr=0x%02jx, selfpowered=%jd", - dev->ud_addr, cdp->bConfigurationValue, cdp->bmAttributes, - selfpowered); - DPRINTF("max power=%jd", cdp->bMaxPower * 2, 0, 0, 0); - - /* Check if we have enough power. */ -#if 0 /* this is a no-op, see above */ - if ((cdp->bmAttributes & UC_SELF_POWERED) && !selfpowered) { - if (msg) - printf("%s: device addr %d (config %d): " - "can't set self powered configuration\n", - device_xname(dev->ud_bus->bdev), dev->ud_addr, - cdp->bConfigurationValue); - err = USBD_NO_POWER; - goto bad; - } -#endif -#ifdef USB_DEBUG - if (dev->ud_powersrc == NULL) { - DPRINTF("No power source?", 0, 0, 0, 0); - err = USBD_IOERROR; - goto bad; - } -#endif - power = cdp->bMaxPower * 2; - if (power > dev->ud_powersrc->up_power) { - DPRINTF("power exceeded %jd %jd", power, - dev->ud_powersrc->up_power, 0, 0); - /* XXX print nicer message. */ - if (msg) - printf("%s: device addr %d (config %d) exceeds power " - "budget, %d mA > %d mA\n", - device_xname(dev->ud_bus->ub_usbctl), dev->ud_addr, - cdp->bConfigurationValue, - power, dev->ud_powersrc->up_power); - err = USBD_NO_POWER; - goto bad; - } - dev->ud_power = power; - dev->ud_selfpowered = selfpowered; - - /* Set the actual configuration value. */ - DPRINTF("set config %jd", cdp->bConfigurationValue, 0, 0, 0); - err = usbd_set_config(dev, cdp->bConfigurationValue); - if (err) { - DPRINTF("setting config=%jd failed, error=%jd", - cdp->bConfigurationValue, err, 0, 0); - goto bad; - } - - KASSERTMSG(dev->ud_ifaces == NULL, "ud_ifaces=%p", dev->ud_ifaces); - - /* Allocate and fill interface data. */ - nifc = cdp->bNumInterface; - if (nifc == 0) { - DPRINTF("no interfaces", 0, 0, 0, 0); - err = USBD_INVAL; - goto bad; - } - dev->ud_ifaces = kmem_alloc(nifc * sizeof(struct usbd_interface), - KM_SLEEP); - DPRINTFN(5, "dev=%#jx cdesc=%#jx", (uintptr_t)dev, (uintptr_t)cdp, - 0, 0); - dev->ud_cdesc = cdp; - dev->ud_config = cdp->bConfigurationValue; - dev->ud_configidx = index; - for (ifcidx = 0; ifcidx < nifc; ifcidx++) { - usbd_iface_init(dev, ifcidx); - usbd_iface_exlock(&dev->ud_ifaces[ifcidx]); - err = usbd_fill_iface_data(dev, ifcidx, 0); - usbd_iface_unlock(&dev->ud_ifaces[ifcidx]); - if (err) { - while (--ifcidx >= 0) { - usbd_iface_exlock(&dev->ud_ifaces[ifcidx]); - usbd_free_iface_data(dev, ifcidx); - usbd_iface_unlock(&dev->ud_ifaces[ifcidx]); - usbd_iface_fini(dev, ifcidx); - } - kmem_free(dev->ud_ifaces, - nifc * sizeof(struct usbd_interface)); - dev->ud_ifaces = NULL; - goto bad; - } - } - - return USBD_NORMAL_COMPLETION; - -bad: - /* XXX Use usbd_set_config() to reset the config? */ - dev->ud_config = USB_UNCONFIG_NO; - dev->ud_configidx = USB_UNCONFIG_INDEX; - KASSERT(dev->ud_ifaces == NULL); - kmem_free(cdp, len); - dev->ud_cdesc = NULL; - if (bdp != NULL) { - kmem_free(bdp, UGETW(bdp->wTotalLength)); - dev->ud_bdesc = NULL; - } - return err; -} - -/* XXX add function for alternate settings */ - -usbd_status -usbd_setup_pipe(struct usbd_device *dev, struct usbd_interface *iface, - struct usbd_endpoint *ep, int ival, struct usbd_pipe **pipe) -{ - return usbd_setup_pipe_flags(dev, iface, ep, ival, pipe, 0); -} - -usbd_status -usbd_setup_pipe_flags(struct usbd_device *dev, struct usbd_interface *iface, - struct usbd_endpoint *ep, int ival, struct usbd_pipe **pipe, uint8_t flags) -{ - USBHIST_FUNC(); - USBHIST_CALLARGS(usbdebug, "dev=%#jx addr=%jd iface=%#jx ep=%#jx", - (uintptr_t)dev, dev->ud_addr, (uintptr_t)iface, (uintptr_t)ep); - struct usbd_pipe *p = NULL; - bool ep_acquired = false; - usbd_status err; - - /* Block exclusive use of the endpoint by later pipes. */ - err = usbd_endpoint_acquire(dev, ep, flags & USBD_EXCLUSIVE_USE); - if (err) - goto out; - ep_acquired = true; - - p = kmem_alloc(dev->ud_bus->ub_pipesize, KM_SLEEP); - DPRINTFN(1, "pipe=%#jx", (uintptr_t)p, 0, 0, 0); - p->up_dev = dev; - p->up_iface = iface; - p->up_endpoint = ep; - p->up_intrxfer = NULL; - p->up_running = 0; - p->up_aborting = 0; - p->up_serialise = true; - p->up_repeat = 0; - p->up_interval = ival; - p->up_flags = flags; - SIMPLEQ_INIT(&p->up_queue); - p->up_callingxfer = NULL; - cv_init(&p->up_callingcv, "usbpipecb"); - p->up_abortlwp = NULL; - - err = dev->ud_bus->ub_methods->ubm_open(p); - if (err) { - DPRINTF("endpoint=%#jx failed, error=%jd", - (uintptr_t)ep->ue_edesc->bEndpointAddress, err, 0, 0); - goto out; - } - - KASSERT(p->up_methods->upm_start || p->up_serialise == false); - - usb_init_task(&p->up_async_task, usbd_clear_endpoint_stall_task, p, - USB_TASKQ_MPSAFE); - DPRINTFN(1, "pipe=%#jx", (uintptr_t)p, 0, 0, 0); - *pipe = p; - p = NULL; /* handed off to caller */ - ep_acquired = false; /* handed off to pipe */ - err = USBD_NORMAL_COMPLETION; - -out: if (p) { - KASSERT(p->up_abortlwp == NULL); - KASSERT(p->up_callingxfer == NULL); - cv_destroy(&p->up_callingcv); - kmem_free(p, dev->ud_bus->ub_pipesize); - } - if (ep_acquired) - usbd_endpoint_release(dev, ep); - return err; -} - -usbd_status -usbd_endpoint_acquire(struct usbd_device *dev, struct usbd_endpoint *ep, - int flags) -{ - usbd_status err; - - mutex_enter(dev->ud_bus->ub_lock); - if (ep->ue_refcnt == INT_MAX) { - err = USBD_IN_USE; /* XXX rule out or switch to 64-bit */ - } else if ((flags & USBD_EXCLUSIVE_USE) && ep->ue_refcnt) { - err = USBD_IN_USE; - } else { - ep->ue_refcnt++; - err = 0; - } - mutex_exit(dev->ud_bus->ub_lock); - - return err; -} - -void -usbd_endpoint_release(struct usbd_device *dev, struct usbd_endpoint *ep) -{ - - mutex_enter(dev->ud_bus->ub_lock); - KASSERT(ep->ue_refcnt); - ep->ue_refcnt--; - mutex_exit(dev->ud_bus->ub_lock); -} - -/* Abort and close the device control pipe. */ -void -usbd_kill_pipe(struct usbd_pipe *pipe) -{ - - usbd_abort_pipe(pipe); - usbd_close_pipe(pipe); -} - -int -usbd_getnewaddr(struct usbd_bus *bus) -{ - int addr; - - for (addr = 1; addr < USB_MAX_DEVICES; addr++) { - size_t dindex = usb_addr2dindex(addr); - if (bus->ub_devices[dindex] == NULL) - return addr; - } - return -1; -} - -usbd_status -usbd_attach_roothub(device_t parent, struct usbd_device *dev) -{ - struct usb_attach_arg uaa; - usb_device_descriptor_t *dd = &dev->ud_ddesc; - device_t dv; - - uaa.uaa_device = dev; - uaa.uaa_usegeneric = 0; - uaa.uaa_port = 0; - uaa.uaa_vendor = UGETW(dd->idVendor); - uaa.uaa_product = UGETW(dd->idProduct); - uaa.uaa_release = UGETW(dd->bcdDevice); - uaa.uaa_class = dd->bDeviceClass; - uaa.uaa_subclass = dd->bDeviceSubClass; - uaa.uaa_proto = dd->bDeviceProtocol; - - KERNEL_LOCK(1, curlwp); - dv = config_found(parent, &uaa, NULL, - CFARGS(.iattr = "usbroothubif")); - KERNEL_UNLOCK_ONE(curlwp); - if (dv) { - dev->ud_subdevs = kmem_alloc(sizeof(dv), KM_SLEEP); - dev->ud_subdevs[0] = dv; - dev->ud_subdevlen = 1; - } - return USBD_NORMAL_COMPLETION; -} - -static void -usbd_properties(device_t dv, struct usbd_device *dev) -{ - usb_device_descriptor_t *dd = &dev->ud_ddesc; - prop_dictionary_t dict = device_properties(dv); - int class, subclass, release, proto, vendor, product; - - class = dd->bDeviceClass; - subclass = dd->bDeviceSubClass; - release = UGETW(dd->bcdDevice); - proto = dd->bDeviceProtocol; - vendor = UGETW(dd->idVendor); - product = UGETW(dd->idProduct); - - prop_dictionary_set_uint8(dict, "address", dev->ud_addr); - - if (dev->ud_myhub) { - struct usbd_device *hdev = dev->ud_myhub; - struct usbd_hub *hub = hdev->ud_hub; - int p; - - KASSERT(hub != NULL); - - prop_dictionary_set_uint8(dict, "hub-address", hdev->ud_addr); - for (p=1; p <= hub->uh_hubdesc.bNbrPorts; ++p) { - if (hub->uh_ports[p-1].up_dev == dev) { - prop_dictionary_set_uint8(dict, "hub-port", p); - break; - } - } - } - - prop_dictionary_set_uint8(dict, "class", class); - prop_dictionary_set_uint8(dict, "subclass", subclass); - prop_dictionary_set_uint16(dict, "release", release); - prop_dictionary_set_uint8(dict, "proto", proto); - prop_dictionary_set_uint16(dict, "vendor-id", vendor); - prop_dictionary_set_uint16(dict, "product-id", product); - - if (dev->ud_vendor) { - prop_dictionary_set_string(dict, - "vendor-string", dev->ud_vendor); - } - if (dev->ud_product) { - prop_dictionary_set_string(dict, - "product-string", dev->ud_product); - } - if (dev->ud_serial) { - prop_dictionary_set_string(dict, - "serialnumber", dev->ud_serial); - } -} - -static usbd_status -usbd_attachwholedevice(device_t parent, struct usbd_device *dev, int port, - int usegeneric) -{ - struct usb_attach_arg uaa; - usb_device_descriptor_t *dd = &dev->ud_ddesc; - device_t dv; - int dlocs[USBDEVIFCF_NLOCS]; - - KASSERT(usb_in_event_thread(parent)); - - uaa.uaa_device = dev; - uaa.uaa_usegeneric = usegeneric; - uaa.uaa_port = port; - uaa.uaa_vendor = UGETW(dd->idVendor); - uaa.uaa_product = UGETW(dd->idProduct); - uaa.uaa_release = UGETW(dd->bcdDevice); - uaa.uaa_class = dd->bDeviceClass; - uaa.uaa_subclass = dd->bDeviceSubClass; - uaa.uaa_proto = dd->bDeviceProtocol; - - dlocs[USBDEVIFCF_PORT] = uaa.uaa_port; - dlocs[USBDEVIFCF_VENDOR] = uaa.uaa_vendor; - dlocs[USBDEVIFCF_PRODUCT] = uaa.uaa_product; - dlocs[USBDEVIFCF_RELEASE] = uaa.uaa_release; - /* the rest is historical ballast */ - dlocs[USBDEVIFCF_CONFIGURATION] = -1; - dlocs[USBDEVIFCF_INTERFACE] = -1; - - config_pending_incr(parent); - - KERNEL_LOCK(1, curlwp); - dv = config_found(parent, &uaa, usbd_print, - CFARGS(.submatch = config_stdsubmatch, - .iattr = "usbdevif", - .locators = dlocs)); - KERNEL_UNLOCK_ONE(curlwp); - if (dv) { - dev->ud_subdevs = kmem_alloc(sizeof(dv), KM_SLEEP); - dev->ud_subdevs[0] = dv; - dev->ud_subdevlen = 1; - dev->ud_nifaces_claimed = 1; /* XXX */ - usbd_properties(dv, dev); - } - config_pending_decr(parent); - return USBD_NORMAL_COMPLETION; -} - -static usbd_status -usbd_attachinterfaces(device_t parent, struct usbd_device *dev, - int port, const int *locators) -{ - USBHIST_FUNC(); USBHIST_CALLED(usbdebug); - struct usbif_attach_arg uiaa; - int ilocs[USBIFIFCF_NLOCS]; - usb_device_descriptor_t *dd = &dev->ud_ddesc; - int nifaces; - struct usbd_interface **ifaces; - int i, j, loc; - device_t dv; - - KASSERT(usb_in_event_thread(parent)); - - nifaces = dev->ud_cdesc->bNumInterface; - ifaces = kmem_zalloc(nifaces * sizeof(*ifaces), KM_SLEEP); - for (i = 0; i < nifaces; i++) { - if (!dev->ud_subdevs[i]) { - ifaces[i] = &dev->ud_ifaces[i]; - } - DPRINTF("interface %jd %#jx", i, (uintptr_t)ifaces[i], 0, 0); - } - - uiaa.uiaa_device = dev; - uiaa.uiaa_port = port; - uiaa.uiaa_vendor = UGETW(dd->idVendor); - uiaa.uiaa_product = UGETW(dd->idProduct); - uiaa.uiaa_release = UGETW(dd->bcdDevice); - uiaa.uiaa_configno = dev->ud_cdesc->bConfigurationValue; - uiaa.uiaa_ifaces = ifaces; - uiaa.uiaa_nifaces = nifaces; - ilocs[USBIFIFCF_PORT] = uiaa.uiaa_port; - ilocs[USBIFIFCF_VENDOR] = uiaa.uiaa_vendor; - ilocs[USBIFIFCF_PRODUCT] = uiaa.uiaa_product; - ilocs[USBIFIFCF_RELEASE] = uiaa.uiaa_release; - ilocs[USBIFIFCF_CONFIGURATION] = uiaa.uiaa_configno; - - for (i = 0; i < nifaces; i++) { - if (!ifaces[i]) { - DPRINTF("interface %jd claimed", i, 0, 0, 0); - continue; /* interface already claimed */ - } - uiaa.uiaa_iface = ifaces[i]; - uiaa.uiaa_class = ifaces[i]->ui_idesc->bInterfaceClass; - uiaa.uiaa_subclass = ifaces[i]->ui_idesc->bInterfaceSubClass; - uiaa.uiaa_proto = ifaces[i]->ui_idesc->bInterfaceProtocol; - uiaa.uiaa_ifaceno = ifaces[i]->ui_idesc->bInterfaceNumber; - - DPRINTF("searching for interface %jd...", i, 0, 0, 0); - DPRINTF("class %jx subclass %jx proto %jx ifaceno %jd", - uiaa.uiaa_class, uiaa.uiaa_subclass, uiaa.uiaa_proto, - uiaa.uiaa_ifaceno); - ilocs[USBIFIFCF_INTERFACE] = uiaa.uiaa_ifaceno; - if (locators != NULL) { - loc = locators[USBIFIFCF_CONFIGURATION]; - if (loc != USBIFIFCF_CONFIGURATION_DEFAULT && - loc != uiaa.uiaa_configno) - continue; - loc = locators[USBIFIFCF_INTERFACE]; - if (loc != USBIFIFCF_INTERFACE_DEFAULT && - loc != uiaa.uiaa_ifaceno) - continue; - } - KERNEL_LOCK(1, curlwp); - dv = config_found(parent, &uiaa, usbd_ifprint, - CFARGS(.submatch = config_stdsubmatch, - .iattr = "usbifif", - .locators = ilocs)); - KERNEL_UNLOCK_ONE(curlwp); - if (!dv) - continue; - - usbd_properties(dv, dev); - - /* claim */ - ifaces[i] = NULL; - /* account for ifaces claimed by the driver behind our back */ - for (j = 0; j < nifaces; j++) { - - if (!ifaces[j] && !dev->ud_subdevs[j]) { - DPRINTF("interface %jd claimed behind our back", - j, 0, 0, 0); - dev->ud_subdevs[j] = dv; - dev->ud_nifaces_claimed++; - } - } - } - - kmem_free(ifaces, nifaces * sizeof(*ifaces)); - return USBD_NORMAL_COMPLETION; -} - -usbd_status -usbd_probe_and_attach(device_t parent, struct usbd_device *dev, - int port, int addr) -{ - USBHIST_FUNC(); - USBHIST_CALLARGS(usbdebug, "trying device specific drivers", 0, 0, 0, 0); - usb_device_descriptor_t *dd = &dev->ud_ddesc; - int confi, nifaces; - usbd_status err; - - KASSERT(usb_in_event_thread(parent)); - - /* First try with device specific drivers. */ - err = usbd_attachwholedevice(parent, dev, port, 0); - if (dev->ud_nifaces_claimed || err) - return err; - DPRINTF("no device specific driver found", 0, 0, 0, 0); - - DPRINTF("looping over %jd configurations", dd->bNumConfigurations, - 0, 0, 0); - for (confi = 0; confi < dd->bNumConfigurations; confi++) { - DPRINTFN(1, "trying config idx=%jd", confi, 0, 0, 0); - err = usbd_set_config_index(dev, confi, 1); - if (err) { - DPRINTF("port %jd, set config at addr %jd failed, " - "error=%jd", port, addr, err, 0); - printf("%s: port %d, set config at addr %d failed\n", - device_xname(parent), port, addr); - return err; - } - nifaces = dev->ud_cdesc->bNumInterface; - dev->ud_subdevs = kmem_zalloc(nifaces * sizeof(device_t), - KM_SLEEP); - dev->ud_subdevlen = nifaces; - - err = usbd_attachinterfaces(parent, dev, port, NULL); - - if (dev->ud_subdevs && dev->ud_nifaces_claimed == 0) { - kmem_free(dev->ud_subdevs, - dev->ud_subdevlen * sizeof(device_t)); - dev->ud_subdevs = 0; - dev->ud_subdevlen = 0; - } - if (dev->ud_nifaces_claimed || err) - return err; - } - /* No interfaces were attached in any of the configurations. */ - - if (dd->bNumConfigurations > 1) /* don't change if only 1 config */ - usbd_set_config_index(dev, 0, 0); - - DPRINTF("no interface drivers found", 0, 0, 0, 0); - - /* Finally try the generic driver. */ - err = usbd_attachwholedevice(parent, dev, port, 1); - - /* - * The generic attach failed, but leave the device as it is. - * We just did not find any drivers, that's all. The device is - * fully operational and not harming anyone. - */ - DPRINTF("generic attach failed", 0, 0, 0, 0); - - return USBD_NORMAL_COMPLETION; -} - -/** - * Called from uhub_rescan(). usbd_new_device() for the target dev must be - * called before calling this. - */ -usbd_status -usbd_reattach_device(device_t parent, struct usbd_device *dev, - int port, const int *locators) -{ - int i, loc; - - USBHIST_FUNC(); - USBHIST_CALLARGS(usbdebug, "uhub%jd port=%jd", - device_unit(parent), port, 0, 0); - - KASSERT(usb_in_event_thread(parent)); - - if (locators != NULL) { - loc = locators[USBIFIFCF_PORT]; - if (loc != USBIFIFCF_PORT_DEFAULT && loc != port) - return USBD_NORMAL_COMPLETION; - loc = locators[USBIFIFCF_VENDOR]; - if (loc != USBIFIFCF_VENDOR_DEFAULT && - loc != UGETW(dev->ud_ddesc.idVendor)) - return USBD_NORMAL_COMPLETION; - loc = locators[USBIFIFCF_PRODUCT]; - if (loc != USBIFIFCF_PRODUCT_DEFAULT && - loc != UGETW(dev->ud_ddesc.idProduct)) - return USBD_NORMAL_COMPLETION; - loc = locators[USBIFIFCF_RELEASE]; - if (loc != USBIFIFCF_RELEASE_DEFAULT && - loc != UGETW(dev->ud_ddesc.bcdDevice)) - return USBD_NORMAL_COMPLETION; - } - if (dev->ud_subdevlen == 0) { - /* XXX: check USBIFIFCF_CONFIGURATION and - * USBIFIFCF_INTERFACE too */ - return usbd_probe_and_attach(parent, dev, port, dev->ud_addr); - } else if (dev->ud_subdevlen != dev->ud_cdesc->bNumInterface) { - /* device-specific or generic driver is already attached. */ - return USBD_NORMAL_COMPLETION; - } - /* Does the device have unconfigured interfaces? */ - for (i = 0; i < dev->ud_subdevlen; i++) { - if (dev->ud_subdevs[i] == NULL) { - break; - } - } - if (i >= dev->ud_subdevlen) - return USBD_NORMAL_COMPLETION; - return usbd_attachinterfaces(parent, dev, port, locators); -} - -/* - * Called when a new device has been put in the powered state, - * but not yet in the addressed state. - * Get initial descriptor, set the address, get full descriptor, - * and attach a driver. - */ -usbd_status -usbd_new_device(device_t parent, struct usbd_bus *bus, int depth, int speed, - int port, struct usbd_port *up) -{ - USBHIST_FUNC(); - USBHIST_CALLARGS(usbdebug, "bus=%#jx port=%jd depth=%jd speed=%jd", - (uintptr_t)bus, port, depth, speed); - struct usbd_device *dev, *adev; - struct usbd_device *hub; - usb_device_descriptor_t *dd; - usb_port_status_t ps; - usbd_status err; - int addr; - int i; - int p; - - KASSERT(usb_in_event_thread(parent)); - - if (bus->ub_methods->ubm_newdev != NULL) - return (bus->ub_methods->ubm_newdev)(parent, bus, depth, speed, - port, up); - - addr = usbd_getnewaddr(bus); - if (addr < 0) { - printf("%s: No free USB addresses, new device ignored.\n", - device_xname(bus->ub_usbctl)); - return USBD_NO_ADDR; - } - - dev = kmem_zalloc(sizeof(*dev), KM_SLEEP); - dev->ud_bus = bus; - - /* Set up default endpoint handle. */ - dev->ud_ep0.ue_edesc = &dev->ud_ep0desc; - - /* Set up default endpoint descriptor. */ - dev->ud_ep0desc.bLength = USB_ENDPOINT_DESCRIPTOR_SIZE; - dev->ud_ep0desc.bDescriptorType = UDESC_ENDPOINT; - dev->ud_ep0desc.bEndpointAddress = USB_CONTROL_ENDPOINT; - dev->ud_ep0desc.bmAttributes = UE_CONTROL; - /* - * temporary, will be fixed after first descriptor fetch - * (which uses 64 bytes so it shouldn't be less), - * highspeed devices must support 64 byte packets anyway - */ - if (speed == USB_SPEED_HIGH || speed == USB_SPEED_FULL) - USETW(dev->ud_ep0desc.wMaxPacketSize, 64); - else - USETW(dev->ud_ep0desc.wMaxPacketSize, USB_MAX_IPACKET); - - dev->ud_ep0desc.bInterval = 0; - - /* doesn't matter, just don't leave it uninitialized */ - dev->ud_ep0.ue_toggle = 0; - - dev->ud_quirks = &usbd_no_quirk; - dev->ud_addr = USB_START_ADDR; - dev->ud_ddesc.bMaxPacketSize = 0; - dev->ud_config = USB_UNCONFIG_NO; - dev->ud_configidx = USB_UNCONFIG_INDEX; - dev->ud_depth = depth; - dev->ud_powersrc = up; - dev->ud_myhub = up->up_parent; - - up->up_dev = dev; - - /* Locate port on upstream high speed hub */ - for (adev = dev, hub = up->up_parent; - hub != NULL && hub->ud_speed != USB_SPEED_HIGH; - adev = hub, hub = hub->ud_myhub) - ; - if (hub) { - for (p = 1; p <= hub->ud_hub->uh_hubdesc.bNbrPorts; p++) { - if (hub->ud_hub->uh_ports[p - 1].up_dev == adev) { - dev->ud_myhsport = - &hub->ud_hub->uh_ports[p - 1]; - goto found; - } - } - panic("usbd_new_device: cannot find HS port"); - found: - DPRINTFN(1, "high speed port %jd", p, 0, 0, 0); - } else { - dev->ud_myhsport = NULL; - } - dev->ud_speed = speed; - dev->ud_langid = USBD_NOLANG; - dev->ud_cookie.cookie = ++usb_cookie_no; - - /* Establish the default pipe. */ - err = usbd_setup_pipe_flags(dev, 0, &dev->ud_ep0, USBD_DEFAULT_INTERVAL, - &dev->ud_pipe0, USBD_MPSAFE); - if (err) { - usbd_remove_device(dev, up); - return err; - } - - dd = &dev->ud_ddesc; - /* Try a few times in case the device is slow (i.e. outside specs.) */ - for (i = 0; i < 10; i++) { - /* Get the first 8 bytes of the device descriptor. */ - err = usbd_get_initial_ddesc(dev, dd); - if (!err) - break; - /* - * The root hub can never fail to give the initial descriptor, - * but assert it just in case. - */ - KASSERT(up->up_parent); - usbd_delay_ms(dev, 200); - if ((i & 3) == 3) - usbd_reset_port(up->up_parent, port, &ps); - } - if (err) { - DPRINTF("addr=%jd, getting first desc failed: %jd", addr, err, - 0, 0); - usbd_remove_device(dev, up); - return err; - } - - /* Windows resets the port here, do likewise */ - if (up->up_parent) - usbd_reset_port(up->up_parent, port, &ps); - - if (speed == USB_SPEED_HIGH) { - /* Max packet size must be 64 (sec 5.5.3). */ - if (dd->bMaxPacketSize != USB_2_MAX_CTRL_PACKET) { -#ifdef DIAGNOSTIC - printf("usbd_new_device: addr=%d bad max packet " - "size=%d. adjusting to %d.\n", - addr, dd->bMaxPacketSize, USB_2_MAX_CTRL_PACKET); -#endif - dd->bMaxPacketSize = USB_2_MAX_CTRL_PACKET; - } - } - - DPRINTF("adding unit addr=%jd, rev=%02jx, class=%jd, subclass=%jd", - addr, UGETW(dd->bcdUSB), dd->bDeviceClass, dd->bDeviceSubClass); - DPRINTF("protocol=%jd, maxpacket=%jd, len=%jd, speed=%jd", - dd->bDeviceProtocol, dd->bMaxPacketSize, dd->bLength, dev->ud_speed); - - if (dd->bDescriptorType != UDESC_DEVICE) { - /* Illegal device descriptor */ - DPRINTF("illegal descriptor %jd", dd->bDescriptorType, 0, 0, 0); - usbd_remove_device(dev, up); - return USBD_INVAL; - } - - if (dd->bLength < USB_DEVICE_DESCRIPTOR_SIZE) { - DPRINTF("bad length %jd", dd->bLength, 0, 0, 0); - usbd_remove_device(dev, up); - return USBD_INVAL; - } - - USETW(dev->ud_ep0desc.wMaxPacketSize, dd->bMaxPacketSize); - - /* Re-establish the default pipe with the new MPS. */ - usbd_kill_pipe(dev->ud_pipe0); - dev->ud_pipe0 = NULL; - err = usbd_setup_pipe_flags(dev, 0, &dev->ud_ep0, USBD_DEFAULT_INTERVAL, - &dev->ud_pipe0, USBD_MPSAFE); - if (err) { - DPRINTF("setup default pipe failed err %jd", err, 0, 0, 0); - usbd_remove_device(dev, up); - return err; - } - - /* Set the address */ - DPRINTFN(5, "setting device address=%jd", addr, 0, 0, 0); - err = usbd_set_address(dev, addr); - if (err) { - DPRINTF("set address %jd failed, err = %jd", addr, err, 0, 0); - err = USBD_SET_ADDR_FAILED; - usbd_remove_device(dev, up); - return err; - } - - /* Allow device time to set new address */ - usbd_delay_ms(dev, USB_SET_ADDRESS_SETTLE); - dev->ud_addr = addr; /* new device address now */ - bus->ub_devices[usb_addr2dindex(addr)] = dev; - - /* Re-establish the default pipe with the new address. */ - usbd_kill_pipe(dev->ud_pipe0); - dev->ud_pipe0 = NULL; - err = usbd_setup_pipe_flags(dev, 0, &dev->ud_ep0, USBD_DEFAULT_INTERVAL, - &dev->ud_pipe0, USBD_MPSAFE); - if (err) { - DPRINTF("setup default pipe failed, err = %jd", err, 0, 0, 0); - usbd_remove_device(dev, up); - return err; - } - - err = usbd_reload_device_desc(dev); - if (err) { - DPRINTF("addr=%jd, getting full desc failed, err = %jd", addr, - err, 0, 0); - usbd_remove_device(dev, up); - return err; - } - - /* Assume 100mA bus powered for now. Changed when configured. */ - dev->ud_power = USB_MIN_POWER; - dev->ud_selfpowered = 0; - - DPRINTF("new dev (addr %jd), dev=%#jx, parent=%#jx", - addr, (uintptr_t)dev, (uintptr_t)parent, 0); - - usbd_get_device_strings(dev); - - usbd_add_dev_event(USB_EVENT_DEVICE_ATTACH, dev); - - if (port == 0) { /* root hub */ - KASSERT(addr == 1); - usbd_attach_roothub(parent, dev); - return USBD_NORMAL_COMPLETION; - } - - err = usbd_probe_and_attach(parent, dev, port, addr); - if (err) { - usbd_remove_device(dev, up); - return err; - } - - return USBD_NORMAL_COMPLETION; -} - -usbd_status -usbd_reload_device_desc(struct usbd_device *dev) -{ - USBHIST_FUNC(); USBHIST_CALLED(usbdebug); - usb_device_descriptor_t *udd = &dev->ud_ddesc; - usbd_status err; - - /* Get the full device descriptor. */ - err = usbd_get_device_desc(dev, udd); - if (err) - return err; - if (udd->bDescriptorType != UDESC_DEVICE) - return USBD_INVAL; - if (udd->bLength < USB_DEVICE_DESCRIPTOR_SIZE) - return USBD_INVAL; - - DPRINTFN(15, "bLength %5ju", udd->bLength, 0, 0, 0); - DPRINTFN(15, "bDescriptorType %5ju", udd->bDescriptorType, 0, 0, 0); - DPRINTFN(15, "bcdUSB %2jx.%02jx", udd->bcdUSB[1], - udd->bcdUSB[0], 0, 0); - DPRINTFN(15, "bDeviceClass %5ju", udd->bDeviceClass, 0, 0, 0); - DPRINTFN(15, "bDeviceSubClass %5ju", udd->bDeviceSubClass, 0, 0, 0); - DPRINTFN(15, "bDeviceProtocol %5ju", udd->bDeviceProtocol, 0, 0, 0); - DPRINTFN(15, "bMaxPacketSize0 %5ju", udd->bMaxPacketSize, 0, 0, 0); - DPRINTFN(15, "idVendor 0x%02jx 0x%02jx", - udd->idVendor[0], - udd->idVendor[1], 0, 0); - DPRINTFN(15, "idProduct 0x%02jx 0x%02jx", - udd->idProduct[0], - udd->idProduct[1], 0, 0); - DPRINTFN(15, "bcdDevice %2jx.%02jx", udd->bcdDevice[1], - udd->bcdDevice[0], 0, 0); - DPRINTFN(15, "iManufacturer %5ju", udd->iManufacturer, 0, 0, 0); - DPRINTFN(15, "iProduct %5ju", udd->iProduct, 0, 0, 0); - DPRINTFN(15, "iSerial %5ju", udd->iSerialNumber, 0, 0, 0); - DPRINTFN(15, "bNumConfigurations %5ju", udd->bNumConfigurations, 0, 0, - 0); - - /* Figure out what's wrong with this device. */ - dev->ud_quirks = usbd_find_quirk(udd); - - return USBD_NORMAL_COMPLETION; -} - -void -usbd_remove_device(struct usbd_device *dev, struct usbd_port *up) -{ - - USBHIST_FUNC(); - USBHIST_CALLARGS(usbdebug, "dev %#jx up %#jx", - (uintptr_t)dev, (uintptr_t)up, 0, 0); - - if (dev->ud_pipe0 != NULL) - usbd_kill_pipe(dev->ud_pipe0); - up->up_dev = NULL; - dev->ud_bus->ub_devices[usb_addr2dindex(dev->ud_addr)] = NULL; - - if (dev->ud_vendor != NULL) { - kmem_free(dev->ud_vendor, USB_MAX_ENCODED_STRING_LEN); - } - if (dev->ud_product != NULL) { - kmem_free(dev->ud_product, USB_MAX_ENCODED_STRING_LEN); - } - if (dev->ud_serial != NULL) { - kmem_free(dev->ud_serial, USB_MAX_ENCODED_STRING_LEN); - } - kmem_free(dev, sizeof(*dev)); -} - -int -usbd_print(void *aux, const char *pnp) -{ - struct usb_attach_arg *uaa = aux; - - if (pnp) { -#define USB_DEVINFO 1024 - char *devinfo; - if (!uaa->uaa_usegeneric) - return QUIET; - devinfo = kmem_alloc(USB_DEVINFO, KM_SLEEP); - usbd_devinfo(uaa->uaa_device, 1, devinfo, USB_DEVINFO); - aprint_normal("%s, %s", devinfo, pnp); - kmem_free(devinfo, USB_DEVINFO); - } - aprint_normal(" port %d", uaa->uaa_port); -#if 0 - /* - * It gets very crowded with these locators on the attach line. - * They are not really needed since they are printed in the clear - * by each driver. - */ - if (uaa->uaa_vendor != UHUB_UNK_VENDOR) - aprint_normal(" vendor 0x%04x", uaa->uaa_vendor); - if (uaa->uaa_product != UHUB_UNK_PRODUCT) - aprint_normal(" product 0x%04x", uaa->uaa_product); - if (uaa->uaa_release != UHUB_UNK_RELEASE) - aprint_normal(" release 0x%04x", uaa->uaa_release); -#endif - return UNCONF; -} - -int -usbd_ifprint(void *aux, const char *pnp) -{ - struct usbif_attach_arg *uiaa = aux; - - if (pnp) - return QUIET; - aprint_normal(" port %d", uiaa->uiaa_port); - aprint_normal(" configuration %d", uiaa->uiaa_configno); - aprint_normal(" interface %d", uiaa->uiaa_ifaceno); -#if 0 - /* - * It gets very crowded with these locators on the attach line. - * They are not really needed since they are printed in the clear - * by each driver. - */ - if (uaa->uaa_vendor != UHUB_UNK_VENDOR) - aprint_normal(" vendor 0x%04x", uaa->uaa_vendor); - if (uaa->uaa_product != UHUB_UNK_PRODUCT) - aprint_normal(" product 0x%04x", uaa->uaa_product); - if (uaa->uaa_release != UHUB_UNK_RELEASE) - aprint_normal(" release 0x%04x", uaa->uaa_release); -#endif - return UNCONF; -} - -void -usbd_fill_deviceinfo(struct usbd_device *dev, struct usb_device_info *di, - int usedev) -{ - struct usbd_port *p; - int i, j, err; - - di->udi_bus = device_unit(dev->ud_bus->ub_usbctl); - di->udi_addr = dev->ud_addr; - di->udi_cookie = dev->ud_cookie; - usbd_devinfo_vp(dev, di->udi_vendor, sizeof(di->udi_vendor), - di->udi_product, sizeof(di->udi_product), usedev, 1); - usbd_printBCD(di->udi_release, sizeof(di->udi_release), - UGETW(dev->ud_ddesc.bcdDevice)); - if (usedev) { - usbd_status uerr = usbd_get_string(dev, - dev->ud_ddesc.iSerialNumber, di->udi_serial); - if (uerr != USBD_NORMAL_COMPLETION) { - di->udi_serial[0] = '\0'; - } else { - usbd_trim_spaces(di->udi_serial); - } - } else { - di->udi_serial[0] = '\0'; - if (dev->ud_serial) { - strlcpy(di->udi_serial, dev->ud_serial, - sizeof(di->udi_serial)); - } - } - - di->udi_vendorNo = UGETW(dev->ud_ddesc.idVendor); - di->udi_productNo = UGETW(dev->ud_ddesc.idProduct); - di->udi_releaseNo = UGETW(dev->ud_ddesc.bcdDevice); - di->udi_class = dev->ud_ddesc.bDeviceClass; - di->udi_subclass = dev->ud_ddesc.bDeviceSubClass; - di->udi_protocol = dev->ud_ddesc.bDeviceProtocol; - di->udi_config = dev->ud_config; - di->udi_power = dev->ud_selfpowered ? 0 : dev->ud_power; - di->udi_speed = dev->ud_speed; - - if (dev->ud_subdevlen > 0) { - for (i = 0, j = 0; i < dev->ud_subdevlen && - j < USB_MAX_DEVNAMES; i++) { - if (!dev->ud_subdevs[i]) - continue; - strncpy(di->udi_devnames[j], - device_xname(dev->ud_subdevs[i]), USB_MAX_DEVNAMELEN); - di->udi_devnames[j][USB_MAX_DEVNAMELEN-1] = '\0'; - j++; - } - } else { - j = 0; - } - for (/* j is set */; j < USB_MAX_DEVNAMES; j++) - di->udi_devnames[j][0] = 0; /* empty */ - - if (!dev->ud_hub) { - di->udi_nports = 0; - return; - } - - const int nports = dev->ud_hub->uh_hubdesc.bNbrPorts; - for (i = 1; i <= __arraycount(di->udi_ports) && i <= nports; i++) { - p = &dev->ud_hub->uh_ports[i - 1]; - if (p->up_dev) - err = p->up_dev->ud_addr; - else { - const int s = UGETW(p->up_status.wPortStatus); - const bool sshub_p = USB_IS_SS(dev->ud_speed); - if (s & UPS_PORT_ENABLED) - err = USB_PORT_ENABLED; - else if (s & UPS_SUSPEND) - err = USB_PORT_SUSPENDED; - /* - * Note: UPS_PORT_POWER_SS is available only - * on 3.x, and UPS_PORT_POWER is available - * only on 2.0 or 1.1. - */ - else if (sshub_p && (s & UPS_PORT_POWER_SS)) - err = USB_PORT_POWERED; - else if (!sshub_p && (s & UPS_PORT_POWER)) - err = USB_PORT_POWERED; - else - err = USB_PORT_DISABLED; - } - di->udi_ports[i - 1] = err; - } - di->udi_nports = nports; -} - -void -usb_free_device(struct usbd_device *dev) -{ - int ifcidx, nifc; - - if (dev->ud_pipe0 != NULL) - usbd_kill_pipe(dev->ud_pipe0); - if (dev->ud_ifaces != NULL) { - nifc = dev->ud_cdesc->bNumInterface; - for (ifcidx = 0; ifcidx < nifc; ifcidx++) { - usbd_iface_exlock(&dev->ud_ifaces[ifcidx]); - usbd_free_iface_data(dev, ifcidx); - usbd_iface_unlock(&dev->ud_ifaces[ifcidx]); - usbd_iface_fini(dev, ifcidx); - } - kmem_free(dev->ud_ifaces, - nifc * sizeof(struct usbd_interface)); - } - if (dev->ud_cdesc != NULL) - kmem_free(dev->ud_cdesc, UGETW(dev->ud_cdesc->wTotalLength)); - if (dev->ud_bdesc != NULL) - kmem_free(dev->ud_bdesc, UGETW(dev->ud_bdesc->wTotalLength)); - if (dev->ud_subdevlen > 0) { - kmem_free(dev->ud_subdevs, - dev->ud_subdevlen * sizeof(device_t)); - dev->ud_subdevlen = 0; - } - if (dev->ud_vendor) { - kmem_free(dev->ud_vendor, USB_MAX_ENCODED_STRING_LEN); - } - if (dev->ud_product) { - kmem_free(dev->ud_product, USB_MAX_ENCODED_STRING_LEN); - } - if (dev->ud_serial) { - kmem_free(dev->ud_serial, USB_MAX_ENCODED_STRING_LEN); - } - kmem_free(dev, sizeof(*dev)); -} - -/* - * The general mechanism for detaching drivers works as follows: Each - * driver is responsible for maintaining a reference count on the - * number of outstanding references to its softc (e.g. from - * processing hanging in a read or write). The detach method of the - * driver decrements this counter and flags in the softc that the - * driver is dying and then wakes any sleepers. It then sleeps on the - * softc. Each place that can sleep must maintain the reference - * count. When the reference count drops to -1 (0 is the normal value - * of the reference count) then a wakeup on the softc is performed - * signaling to the detach waiter that all references are gone. - */ - -/* - * Called from process context when we discover that a port has - * been disconnected. - */ -int -usb_disconnect_port(struct usbd_port *up, device_t parent, int flags) -{ - struct usbd_device *dev = up->up_dev; - device_t subdev; - char subdevname[16]; - const char *hubname = device_xname(parent); - int i, rc; - - USBHIST_FUNC(); - USBHIST_CALLARGS(usbdebug, "up=%#jx dev=%#jx port=%jd", - (uintptr_t)up, (uintptr_t)dev, up->up_portno, 0); - - if (dev == NULL) { - return 0; - } - - usbd_suspend_pipe(dev->ud_pipe0); - if (dev->ud_subdevlen > 0) { - DPRINTFN(3, "disconnect subdevs", 0, 0, 0, 0); - for (i = 0; i < dev->ud_subdevlen; i++) { - if ((subdev = dev->ud_subdevs[i]) == NULL) - continue; - strlcpy(subdevname, device_xname(subdev), - sizeof(subdevname)); - KERNEL_LOCK(1, curlwp); - rc = config_detach(subdev, flags); - KERNEL_UNLOCK_ONE(curlwp); - if (rc != 0) - return rc; - printf("%s: at %s", subdevname, hubname); - if (up->up_portno != 0) - printf(" port %d", up->up_portno); - printf(" (addr %d) disconnected\n", dev->ud_addr); - } - KASSERT(!dev->ud_nifaces_claimed); - } - - mutex_enter(dev->ud_bus->ub_lock); - dev->ud_bus->ub_devices[usb_addr2dindex(dev->ud_addr)] = NULL; - up->up_dev = NULL; - mutex_exit(dev->ud_bus->ub_lock); - - usbd_add_dev_event(USB_EVENT_DEVICE_DETACH, dev); - - usb_free_device(dev); - - return 0; -} diff --git a/sys/dev/usb/usbdivar.h b/sys/dev/usb/usbdivar.h deleted file mode 100644 index 7618237f5c49c..0000000000000 --- a/sys/dev/usb/usbdivar.h +++ /dev/null @@ -1,429 +0,0 @@ -/* $NetBSD: usbdivar.h,v 1.140 2025/10/05 20:04:30 riastradh Exp $ */ - -/* - * Copyright (c) 1998, 2012 The NetBSD Foundation, Inc. - * All rights reserved. - * - * This code is derived from software contributed to The NetBSD Foundation - * by Lennart Augustsson (lennart@augustsson.net) at - * Carlstedt Research & Technology and Matthew R. Green (mrg@eterna23.net). - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED - * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _DEV_USB_USBDIVAR_H_ -#define _DEV_USB_USBDIVAR_H_ - -/* - * Discussion about locking in the USB code: - * - * The host controller presents one lock at IPL_SOFTUSB (aka IPL_SOFTNET). - * - * List of hardware interface methods, and whether the lock is held - * when each is called by this module: - * - * BUS METHOD LOCK NOTES - * ----------------------- ------- ------------------------- - * ubm_open - might want to take lock? - * ubm_softint x may release/reacquire lock - * ubm_dopoll - might want to take lock? - * ubm_allocx - - * ubm_freex - - * ubm_abortx x must not release/reacquire lock - * ubm_getlock - Called at attach time - * ubm_newdev - Will take lock - * ubm_rhctrl - - * - * PIPE METHOD LOCK NOTES - * ----------------------- ------- ------------------------- - * upm_init - - * upm_fini - - * upm_transfer x - * upm_start x - * upm_abort x - * upm_close x - * upm_cleartoggle - - * upm_done x - * - * The above semantics are likely to change. Little performance - * evaluation has been done on this code and the locking strategy. - * - * USB functions known to expect the lock taken include (this list is - * probably not exhaustive): - * usb_transfer_complete() - * usb_start_next() - * - */ - -#include -#include -#include - -#include - -/* From usb_mem.h */ -struct usb_dma_block; -typedef struct { - struct usb_dma_block *udma_block; - u_int udma_offs; -} usb_dma_t; - -struct usbd_xfer; -struct usbd_pipe; -struct usbd_port; - -struct usbd_endpoint { - usb_endpoint_descriptor_t *ue_edesc; - int ue_refcnt; - int ue_toggle; -}; - -struct usbd_bus_methods { - usbd_status (*ubm_open)(struct usbd_pipe *); - void (*ubm_softint)(void *); - void (*ubm_dopoll)(struct usbd_bus *); - struct usbd_xfer *(*ubm_allocx)(struct usbd_bus *, unsigned int); - void (*ubm_freex)(struct usbd_bus *, struct usbd_xfer *); - void (*ubm_abortx)(struct usbd_xfer *); - bool (*ubm_dying)(struct usbd_bus *); - void (*ubm_getlock)(struct usbd_bus *, kmutex_t **); - usbd_status (*ubm_newdev)(device_t, struct usbd_bus *, int, - int, int, struct usbd_port *); - - int (*ubm_rhctrl)(struct usbd_bus *, - usb_device_request_t *, void *, int); -}; - -struct usbd_pipe_methods { - int (*upm_init)(struct usbd_xfer *); - void (*upm_fini)(struct usbd_xfer *); - usbd_status (*upm_transfer)(struct usbd_xfer *); - usbd_status (*upm_start)(struct usbd_xfer *); - void (*upm_abort)(struct usbd_xfer *); - void (*upm_close)(struct usbd_pipe *); - void (*upm_cleartoggle)(struct usbd_pipe *); - void (*upm_done)(struct usbd_xfer *); -}; - -struct usbd_tt { - struct usbd_hub *utt_hub; -}; - -struct usbd_port { - usb_port_status_t up_status; - uint16_t up_power; /* mA of current on port */ - uint8_t up_portno; - uint8_t up_restartcnt; -#define USBD_RESTART_MAX 5 - uint8_t up_reattach; - struct usbd_device *up_dev; /* Connected device */ - struct usbd_device *up_parent; /* The ports hub */ - struct usbd_tt *up_tt; /* Transaction translator (if any) */ -}; - -struct usbd_hub { - usbd_status (*uh_explore)(struct usbd_device *hub); - void *uh_hubsoftc; - usb_hub_descriptor_t uh_hubdesc; - struct usbd_port uh_ports[1]; -}; - -/*****/ -/* 0, root, and 1->127 */ -#define USB_ROOTHUB_INDEX 1 -#define USB_TOTAL_DEVICES (USB_MAX_DEVICES + 1) - -struct usbd_bus { - /* Filled by HC driver */ - void *ub_hcpriv; - int ub_revision; /* USB revision */ -#define USBREV_UNKNOWN 0 -#define USBREV_PRE_1_0 1 -#define USBREV_1_0 2 -#define USBREV_1_1 3 -#define USBREV_2_0 4 -#define USBREV_3_0 5 -#define USBREV_3_1 6 -#define USBREV_STR { "unknown", "pre 1.0", "1.0", "1.1", "2.0", "3.0", "3.1" } - int ub_hctype; -#define USBHCTYPE_UNKNOWN 0 -#define USBHCTYPE_MOTG 1 -#define USBHCTYPE_OHCI 2 -#define USBHCTYPE_UHCI 3 -#define USBHCTYPE_EHCI 4 -#define USBHCTYPE_XHCI 5 -#define USBHCTYPE_VHCI 6 - int ub_busnum; - const struct usbd_bus_methods - *ub_methods; - uint32_t ub_pipesize; /* size of a pipe struct */ - bool ub_usedma; /* Does this HC support DMA */ - int ub_dmaflags; - bus_dma_tag_t ub_dmatag; /* DMA tag */ - - /* Filled by usb driver */ - kmutex_t *ub_lock; - struct usbd_device *ub_roothub; - struct usbd_xfer *ub_rhxfer; /* roothub xfer in progress */ - kcondvar_t ub_rhxfercv; - uint8_t ub_rhaddr; /* roothub address */ - uint8_t ub_rhconf; /* roothub configuration */ - struct usbd_device *ub_devices[USB_TOTAL_DEVICES]; - kcondvar_t ub_needsexplore_cv; - char ub_needsexplore;/* a hub a signalled a change */ - char ub_usepolling; - device_t ub_usbctl; - struct usb_device_stats ub_stats; - - void *ub_soft; /* soft interrupt cookie */ -}; - -struct usbd_device { - struct usbd_bus *ud_bus; /* our controller */ - struct usbd_pipe *ud_pipe0; /* pipe 0 */ - uint8_t ud_addr; /* device address */ - uint8_t ud_config; /* current configuration # */ - uint8_t ud_depth; /* distance from root hub */ - uint8_t ud_speed; /* low/full/high speed */ - uint8_t ud_selfpowered; /* flag for self powered */ - uint16_t ud_power; /* mA the device uses */ - int16_t ud_langid; /* language for strings */ -#define USBD_NOLANG (-1) - usb_event_cookie_t ud_cookie; /* unique connection id */ - struct usbd_port *ud_powersrc; /* upstream hub port, or 0 */ - struct usbd_device *ud_myhub; /* upstream hub */ - struct usbd_port *ud_myhsport; /* closest high speed port */ - struct usbd_endpoint ud_ep0; /* for pipe 0 */ - usb_endpoint_descriptor_t - ud_ep0desc; /* for pipe 0 */ - struct usbd_interface *ud_ifaces; /* array of all interfaces */ - usb_device_descriptor_t ud_ddesc; /* device descriptor */ - usb_config_descriptor_t *ud_cdesc; /* full config descr */ - usb_bos_descriptor_t *ud_bdesc; /* full BOS descr */ - const struct usbd_quirks - *ud_quirks; /* device quirks, always set */ - struct usbd_hub *ud_hub; /* only if this is a hub */ - u_int ud_subdevlen; /* array length of following */ - device_t *ud_subdevs; /* sub-devices */ - int ud_nifaces_claimed; /* number of ifaces in use */ - void *ud_hcpriv; - - char *ud_serial; /* serial number, can be NULL */ - char *ud_vendor; /* vendor string, can be NULL */ - char *ud_product; /* product string can be NULL */ - - /* - * ud_config above holds a value of bConfigurationValue from - * the config descriptor, or USB_UNCONFIG_NO=0 -- which may - * _also_ be a value of bConfigurationValue. - * - * ud_configidx below holds an index in [0, bNumConfigurations) - * into the list of configuration descriptors, or - * USB_UNCONFIG_INDEX=-1 to denote that the interface is - * unconfigured. Note that ud_config may be USB_UNCONFIG_NO - * even if ud_configidx is not USB_UNCONFIG_INDEX, if a screwy - * device has a config descriptor with bConfigurationValue=0. - * - * This goes at the end, rather than next to ud_config where it - * might properly belong, so the change preserves ABI for - * pullup to release branches. - */ - int16_t ud_configidx; -}; - -struct usbd_interface { - struct usbd_device *ui_dev; - usb_interface_descriptor_t - *ui_idesc; - int ui_index; - int ui_altindex; - struct usbd_endpoint *ui_endpoints; - int64_t ui_busy; /* #pipes, or -1 if setting */ -}; - -struct usbd_pipe { - struct usbd_interface *up_iface; - struct usbd_device *up_dev; - struct usbd_endpoint *up_endpoint; - char up_running; - char up_aborting; - bool up_serialise; - SIMPLEQ_HEAD(, usbd_xfer) - up_queue; - struct usb_task up_async_task; - - struct usbd_xfer *up_intrxfer; /* used for repeating requests */ - char up_repeat; - int up_interval; - uint8_t up_flags; - - struct usbd_xfer *up_callingxfer; /* currently in callback */ - kcondvar_t up_callingcv; - - struct lwp *up_abortlwp; /* lwp currently aborting */ - - /* Filled by HC driver. */ - const struct usbd_pipe_methods - *up_methods; -}; - -struct usbd_xfer { - struct usbd_pipe *ux_pipe; - void *ux_priv; - void *ux_buffer; - kcondvar_t ux_cv; - uint32_t ux_length; - uint32_t ux_actlen; - uint16_t ux_flags; - uint32_t ux_timeout; - usbd_status ux_status; - usbd_callback ux_callback; - volatile uint8_t ux_done; - uint8_t ux_state; /* used for DIAGNOSTIC */ -#define XFER_FREE 0x46 -#define XFER_BUSY 0x55 -#define XFER_ONQU 0x9e - - /* For control pipe */ - usb_device_request_t ux_request; - - /* For isoc */ - uint16_t *ux_frlengths; - int ux_nframes; - - const struct usbd_pipe_methods *ux_methods; - - /* For memory allocation and softc */ - struct usbd_bus *ux_bus; - usb_dma_t ux_dmabuf; - void *ux_buf; - uint32_t ux_bufsize; - - uint8_t ux_rqflags; -#define URQ_REQUEST 0x01 - - SIMPLEQ_ENTRY(usbd_xfer) - ux_next; - - void *ux_hcpriv; /* private use by the HC driver */ - - struct usb_task ux_aborttask; - struct callout ux_callout; - - /* - * Protected by bus lock. - * - * - ux_timeout_set: The timeout is scheduled as a callout or - * usb task, and has not yet acquired the bus lock. - * - * - ux_timeout_reset: The xfer completed, and was resubmitted - * before the callout or task was able to acquire the bus - * lock, so one or the other needs to schedule a new callout. - */ - bool ux_timeout_set; - bool ux_timeout_reset; -}; - -void usbd_init(void); -void usbd_finish(void); - -#if defined(USB_DEBUG) -void usbd_dump_iface(struct usbd_interface *); -void usbd_dump_device(struct usbd_device *); -void usbd_dump_endpoint(struct usbd_endpoint *); -void usbd_dump_queue(struct usbd_pipe *); -void usbd_dump_pipe(struct usbd_pipe *); -#endif - -/* Routines from usb_subr.c */ -int usbctlprint(void *, const char *); -void usbd_get_device_strings(struct usbd_device *); -void usb_delay_ms_locked(struct usbd_bus *, u_int, kmutex_t *); -void usb_delay_ms(struct usbd_bus *, u_int); -void usbd_delay_ms_locked(struct usbd_device *, u_int, kmutex_t *); -void usbd_delay_ms(struct usbd_device *, u_int); -usbd_status usbd_reset_port(struct usbd_device *, int, usb_port_status_t *); -usbd_status usbd_setup_pipe(struct usbd_device *, - struct usbd_interface *, - struct usbd_endpoint *, int, - struct usbd_pipe **); -usbd_status usbd_setup_pipe_flags(struct usbd_device *, - struct usbd_interface *, - struct usbd_endpoint *, int, - struct usbd_pipe **, - uint8_t); -usbd_status usbd_new_device(device_t, struct usbd_bus *, int, int, int, - struct usbd_port *); -usbd_status usbd_reattach_device(device_t, struct usbd_device *, - int, const int *); - -void usbd_remove_device(struct usbd_device *, struct usbd_port *); -bool usbd_iface_locked(struct usbd_interface *); -usbd_status usbd_iface_lock(struct usbd_interface *); -void usbd_iface_unlock(struct usbd_interface *); -usbd_status usbd_iface_piperef(struct usbd_interface *); -void usbd_iface_pipeunref(struct usbd_interface *); -usbd_status usbd_fill_iface_data(struct usbd_device *, int, int); -void usb_free_device(struct usbd_device *); - -void usb_transfer_complete(struct usbd_xfer *); -int usb_disconnect_port(struct usbd_port *, device_t, int); - -usbd_status usbd_endpoint_acquire(struct usbd_device *, - struct usbd_endpoint *, int); -void usbd_endpoint_release(struct usbd_device *, - struct usbd_endpoint *); - -void usbd_kill_pipe(struct usbd_pipe *); -usbd_status usbd_attach_roothub(device_t, struct usbd_device *); -usbd_status usbd_probe_and_attach(device_t, struct usbd_device *, int, int); - -/* Routines from usb.c */ -void usb_needs_explore(struct usbd_device *); -void usb_needs_reattach(struct usbd_device *); -void usb_schedsoftintr(struct usbd_bus *); - -static __inline int -usbd_xfer_isread(struct usbd_xfer *xfer) -{ - if (xfer->ux_rqflags & URQ_REQUEST) - return xfer->ux_request.bmRequestType & UT_READ; - - return xfer->ux_pipe->up_endpoint->ue_edesc->bEndpointAddress & - UE_DIR_IN; -} - -static __inline size_t -usb_addr2dindex(int addr) -{ - - return USB_ROOTHUB_INDEX + addr; -} - -/* - * These macros reflect the current locking scheme. They might change. - */ - -#define usbd_lock_pipe(p) mutex_enter((p)->up_dev->ud_bus->ub_lock) -#define usbd_unlock_pipe(p) mutex_exit((p)->up_dev->ud_bus->ub_lock) - -#endif /* _DEV_USB_USBDIVAR_H_ */ diff --git a/sys/dev/usb/xhci.c b/sys/dev/usb/xhci.c deleted file mode 100644 index d6e89e8620b4a..0000000000000 --- a/sys/dev/usb/xhci.c +++ /dev/null @@ -1,4994 +0,0 @@ -/* $NetBSD: xhci.c,v 1.192 2025/10/10 20:21:05 nat Exp $ */ - -/* - * Copyright (c) 2013 Jonathan A. Kollasch - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED - * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/* - * USB rev 2.0 and rev 3.1 specification - * http://www.usb.org/developers/docs/ - * xHCI rev 1.1 specification - * http://www.intel.com/technology/usb/spec.htm - */ - -#include -__KERNEL_RCSID(0, "$NetBSD: xhci.c,v 1.192 2025/10/10 20:21:05 nat Exp $"); - -#ifdef _KERNEL_OPT -#include "opt_usb.h" -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - - -#ifdef USB_DEBUG -#ifndef XHCI_DEBUG -#define xhcidebug 0 -#else /* !XHCI_DEBUG */ -#define HEXDUMP(a, b, c) \ - do { \ - if (xhcidebug > 0) \ - hexdump(printf, a, b, c); \ - } while (/*CONSTCOND*/0) -static int xhcidebug = 0; - -SYSCTL_SETUP(sysctl_hw_xhci_setup, "sysctl hw.xhci setup") -{ - int err; - const struct sysctlnode *rnode; - const struct sysctlnode *cnode; - - err = sysctl_createv(clog, 0, NULL, &rnode, - CTLFLAG_PERMANENT, CTLTYPE_NODE, "xhci", - SYSCTL_DESCR("xhci global controls"), - NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); - - if (err) - goto fail; - - /* control debugging printfs */ - err = sysctl_createv(clog, 0, &rnode, &cnode, - CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT, - "debug", SYSCTL_DESCR("Enable debugging output"), - NULL, 0, &xhcidebug, sizeof(xhcidebug), CTL_CREATE, CTL_EOL); - if (err) - goto fail; - - return; -fail: - aprint_error("%s: sysctl_createv failed (err = %d)\n", __func__, err); -} - -#endif /* !XHCI_DEBUG */ -#endif /* USB_DEBUG */ - -#ifndef HEXDUMP -#define HEXDUMP(a, b, c) -#endif - -#define DPRINTF(FMT,A,B,C,D) USBHIST_LOG(xhcidebug,FMT,A,B,C,D) -#define DPRINTFN(N,FMT,A,B,C,D) USBHIST_LOGN(xhcidebug,N,FMT,A,B,C,D) -#define XHCIHIST_FUNC() USBHIST_FUNC() -#define XHCIHIST_CALLED(name) USBHIST_CALLED(xhcidebug) -#define XHCIHIST_CALLARGS(FMT,A,B,C,D) \ - USBHIST_CALLARGS(xhcidebug,FMT,A,B,C,D) - -#define XHCI_DCI_SLOT 0 -#define XHCI_DCI_EP_CONTROL 1 - -#define XHCI_ICI_INPUT_CONTROL 0 - -struct xhci_pipe { - struct usbd_pipe xp_pipe; - struct usb_task xp_async_task; - int16_t xp_isoc_next; /* next micro frame */ - uint8_t xp_maxb; /* max burst */ - uint8_t xp_mult; - uint8_t xp_ival; -}; - -#define XHCI_COMMAND_RING_TRBS 256 -#define XHCI_EVENT_RING_TRBS 256 -#define XHCI_EVENT_RING_SEGMENTS 1 -#define XHCI_TRB_3_ED_BIT XHCI_TRB_3_ISP_BIT - -static usbd_status xhci_open(struct usbd_pipe *); -static void xhci_close_pipe(struct usbd_pipe *); -static int xhci_intr1(struct xhci_softc * const); -static void xhci_softintr(void *); -static void xhci_poll(struct usbd_bus *); -static struct usbd_xfer *xhci_allocx(struct usbd_bus *, unsigned int); -static void xhci_freex(struct usbd_bus *, struct usbd_xfer *); -static void xhci_abortx(struct usbd_xfer *); -static bool xhci_dying(struct usbd_bus *); -static void xhci_get_lock(struct usbd_bus *, kmutex_t **); -static usbd_status xhci_new_device(device_t, struct usbd_bus *, int, int, int, - struct usbd_port *); -static int xhci_roothub_ctrl(struct usbd_bus *, usb_device_request_t *, - void *, int); - -static void xhci_pipe_restart(struct usbd_pipe *); -static void xhci_pipe_restart_async_task(void *); -static void xhci_pipe_restart_async(struct usbd_pipe *); - -static usbd_status xhci_configure_endpoint(struct usbd_pipe *); -//static usbd_status xhci_unconfigure_endpoint(struct usbd_pipe *); -static void xhci_reset_endpoint(struct usbd_pipe *); -static usbd_status xhci_stop_endpoint_cmd(struct xhci_softc *, - struct xhci_slot *, u_int, uint32_t); -static usbd_status xhci_stop_endpoint(struct usbd_pipe *); - -static void xhci_host_dequeue(struct xhci_ring * const); -static void xhci_set_dequeue(struct usbd_pipe *); - -static usbd_status xhci_do_command(struct xhci_softc * const, - struct xhci_soft_trb * const, int); -static usbd_status xhci_do_command_locked(struct xhci_softc * const, - struct xhci_soft_trb * const, int); -static usbd_status xhci_init_slot(struct usbd_device *, uint32_t); -static void xhci_free_slot(struct xhci_softc *, struct xhci_slot *); -static usbd_status xhci_set_address(struct usbd_device *, uint32_t, bool); -static usbd_status xhci_enable_slot(struct xhci_softc * const, - uint8_t * const); -static usbd_status xhci_disable_slot(struct xhci_softc * const, uint8_t); -static usbd_status xhci_address_device(struct xhci_softc * const, - uint64_t, uint8_t, bool); -static void xhci_set_dcba(struct xhci_softc * const, uint64_t, int); -static usbd_status xhci_update_ep0_mps(struct xhci_softc * const, - struct xhci_slot * const, u_int); -static usbd_status xhci_ring_init(struct xhci_softc * const, - struct xhci_ring **, size_t, size_t); -static void xhci_ring_free(struct xhci_softc * const, - struct xhci_ring ** const); - -static void xhci_setup_ctx(struct usbd_pipe *); -static void xhci_setup_route(struct usbd_pipe *, uint32_t *); -static void xhci_setup_tthub(struct usbd_pipe *, uint32_t *); -static void xhci_setup_maxburst(struct usbd_pipe *, uint32_t *); -static uint32_t xhci_bival2ival(uint32_t, uint32_t, uint32_t); - -static void xhci_noop(struct usbd_pipe *); - -static usbd_status xhci_root_intr_transfer(struct usbd_xfer *); -static usbd_status xhci_root_intr_start(struct usbd_xfer *); -static void xhci_root_intr_abort(struct usbd_xfer *); -static void xhci_root_intr_close(struct usbd_pipe *); -static void xhci_root_intr_done(struct usbd_xfer *); - -static usbd_status xhci_device_ctrl_transfer(struct usbd_xfer *); -static usbd_status xhci_device_ctrl_start(struct usbd_xfer *); -static void xhci_device_ctrl_abort(struct usbd_xfer *); -static void xhci_device_ctrl_close(struct usbd_pipe *); -static void xhci_device_ctrl_done(struct usbd_xfer *); - -static usbd_status xhci_device_isoc_transfer(struct usbd_xfer *); -static usbd_status xhci_device_isoc_enter(struct usbd_xfer *); -static void xhci_device_isoc_abort(struct usbd_xfer *); -static void xhci_device_isoc_close(struct usbd_pipe *); -static void xhci_device_isoc_done(struct usbd_xfer *); - -static usbd_status xhci_device_intr_transfer(struct usbd_xfer *); -static usbd_status xhci_device_intr_start(struct usbd_xfer *); -static void xhci_device_intr_abort(struct usbd_xfer *); -static void xhci_device_intr_close(struct usbd_pipe *); -static void xhci_device_intr_done(struct usbd_xfer *); - -static usbd_status xhci_device_bulk_transfer(struct usbd_xfer *); -static usbd_status xhci_device_bulk_start(struct usbd_xfer *); -static void xhci_device_bulk_abort(struct usbd_xfer *); -static void xhci_device_bulk_close(struct usbd_pipe *); -static void xhci_device_bulk_done(struct usbd_xfer *); - -static const struct usbd_bus_methods xhci_bus_methods = { - .ubm_open = xhci_open, - .ubm_softint = xhci_softintr, - .ubm_dopoll = xhci_poll, - .ubm_allocx = xhci_allocx, - .ubm_freex = xhci_freex, - .ubm_abortx = xhci_abortx, - .ubm_dying = xhci_dying, - .ubm_getlock = xhci_get_lock, - .ubm_newdev = xhci_new_device, - .ubm_rhctrl = xhci_roothub_ctrl, -}; - -static const struct usbd_pipe_methods xhci_root_intr_methods = { - .upm_transfer = xhci_root_intr_transfer, - .upm_start = xhci_root_intr_start, - .upm_abort = xhci_root_intr_abort, - .upm_close = xhci_root_intr_close, - .upm_cleartoggle = xhci_noop, - .upm_done = xhci_root_intr_done, -}; - - -static const struct usbd_pipe_methods xhci_device_ctrl_methods = { - .upm_transfer = xhci_device_ctrl_transfer, - .upm_start = xhci_device_ctrl_start, - .upm_abort = xhci_device_ctrl_abort, - .upm_close = xhci_device_ctrl_close, - .upm_cleartoggle = xhci_noop, - .upm_done = xhci_device_ctrl_done, -}; - -static const struct usbd_pipe_methods xhci_device_isoc_methods = { - .upm_transfer = xhci_device_isoc_transfer, - .upm_abort = xhci_device_isoc_abort, - .upm_close = xhci_device_isoc_close, - .upm_cleartoggle = xhci_noop, - .upm_done = xhci_device_isoc_done, -}; - -static const struct usbd_pipe_methods xhci_device_bulk_methods = { - .upm_transfer = xhci_device_bulk_transfer, - .upm_start = xhci_device_bulk_start, - .upm_abort = xhci_device_bulk_abort, - .upm_close = xhci_device_bulk_close, - .upm_cleartoggle = xhci_noop, - .upm_done = xhci_device_bulk_done, -}; - -static const struct usbd_pipe_methods xhci_device_intr_methods = { - .upm_transfer = xhci_device_intr_transfer, - .upm_start = xhci_device_intr_start, - .upm_abort = xhci_device_intr_abort, - .upm_close = xhci_device_intr_close, - .upm_cleartoggle = xhci_noop, - .upm_done = xhci_device_intr_done, -}; - -static inline uint32_t -xhci_read_1(const struct xhci_softc * const sc, bus_size_t offset) -{ - if (ISSET(sc->sc_quirks, XHCI_32BIT_ACCESS)) { - uint32_t val; - val = bus_space_read_4(sc->sc_iot, sc->sc_ioh, offset & ~3); - return (val >> ((offset & 3) * NBBY)) & 0xff; - } else { - return bus_space_read_1(sc->sc_iot, sc->sc_ioh, offset); - } -} - -static inline uint32_t -xhci_read_2(const struct xhci_softc * const sc, bus_size_t offset) -{ - if (ISSET(sc->sc_quirks, XHCI_32BIT_ACCESS)) { - uint32_t val; - val = bus_space_read_4(sc->sc_iot, sc->sc_ioh, offset & ~3); - return (val >> ((offset & 3) * NBBY)) & 0xffff; - } else { - return bus_space_read_2(sc->sc_iot, sc->sc_ioh, offset); - } -} - -static inline uint32_t -xhci_read_4(const struct xhci_softc * const sc, bus_size_t offset) -{ - return bus_space_read_4(sc->sc_iot, sc->sc_ioh, offset); -} - -static inline void -xhci_write_1(const struct xhci_softc * const sc, bus_size_t offset, - uint32_t value) -{ - if (ISSET(sc->sc_quirks, XHCI_32BIT_ACCESS)) { - const uint32_t mask = 0xffU << ((offset & 3) * NBBY); - uint32_t val; - val = bus_space_read_4(sc->sc_iot, sc->sc_ioh, offset & ~3); - val &= ~mask; - val |= __SHIFTIN(value, mask); - bus_space_write_4(sc->sc_iot, sc->sc_ioh, offset & ~3, val); - } else { - bus_space_write_1(sc->sc_iot, sc->sc_ioh, offset, value); - } -} - -#if 0 /* unused */ -static inline void -xhci_write_4(const struct xhci_softc * const sc, bus_size_t offset, - uint32_t value) -{ - bus_space_write_4(sc->sc_iot, sc->sc_ioh, offset, value); -} -#endif /* unused */ - -static inline uint32_t -xhci_cap_read_4(const struct xhci_softc * const sc, bus_size_t offset) -{ - return bus_space_read_4(sc->sc_iot, sc->sc_cbh, offset); -} - -static inline uint32_t -xhci_op_read_4(const struct xhci_softc * const sc, bus_size_t offset) -{ - return bus_space_read_4(sc->sc_iot, sc->sc_obh, offset); -} - -static inline void -xhci_op_write_4(const struct xhci_softc * const sc, bus_size_t offset, - uint32_t value) -{ - bus_space_write_4(sc->sc_iot, sc->sc_obh, offset, value); -} - -static inline uint64_t -xhci_op_read_8(const struct xhci_softc * const sc, bus_size_t offset) -{ - uint64_t value; - -#ifdef XHCI_USE_BUS_SPACE_8 - value = bus_space_read_8(sc->sc_iot, sc->sc_obh, offset); -#else - value = bus_space_read_4(sc->sc_iot, sc->sc_obh, offset); - value |= (uint64_t)bus_space_read_4(sc->sc_iot, sc->sc_obh, - offset + 4) << 32; -#endif - - return value; -} - -static inline void -xhci_op_write_8(const struct xhci_softc * const sc, bus_size_t offset, - uint64_t value) -{ -#ifdef XHCI_USE_BUS_SPACE_8 - bus_space_write_8(sc->sc_iot, sc->sc_obh, offset, value); -#else - bus_space_write_4(sc->sc_iot, sc->sc_obh, offset + 0, - (value >> 0) & 0xffffffff); - bus_space_write_4(sc->sc_iot, sc->sc_obh, offset + 4, - (value >> 32) & 0xffffffff); -#endif -} - -static inline uint32_t -xhci_rt_read_4(const struct xhci_softc * const sc, bus_size_t offset) -{ - return bus_space_read_4(sc->sc_iot, sc->sc_rbh, offset); -} - -static inline void -xhci_rt_write_4(const struct xhci_softc * const sc, bus_size_t offset, - uint32_t value) -{ - bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset, value); -} - -static inline uint64_t -xhci_rt_read_8(const struct xhci_softc * const sc, bus_size_t offset) -{ - uint64_t value; - -#ifdef XHCI_USE_BUS_SPACE_8 - value = bus_space_read_8(sc->sc_iot, sc->sc_rbh, offset); -#else - value = bus_space_read_4(sc->sc_iot, sc->sc_rbh, offset); - value |= (uint64_t)bus_space_read_4(sc->sc_iot, sc->sc_rbh, - offset + 4) << 32; -#endif - - return value; -} - -static inline void -xhci_rt_write_8(const struct xhci_softc * const sc, bus_size_t offset, - uint64_t value) -{ -#ifdef XHCI_USE_BUS_SPACE_8 - bus_space_write_8(sc->sc_iot, sc->sc_rbh, offset, value); -#else - bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset + 0, - (value >> 0) & 0xffffffff); - bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset + 4, - (value >> 32) & 0xffffffff); -#endif -} - -#if 0 /* unused */ -static inline uint32_t -xhci_db_read_4(const struct xhci_softc * const sc, bus_size_t offset) -{ - return bus_space_read_4(sc->sc_iot, sc->sc_dbh, offset); -} -#endif /* unused */ - -static inline void -xhci_db_write_4(const struct xhci_softc * const sc, bus_size_t offset, - uint32_t value) -{ - bus_space_write_4(sc->sc_iot, sc->sc_dbh, offset, value); -} - -/* --- */ - -static inline uint8_t -xhci_ep_get_type(usb_endpoint_descriptor_t * const ed) -{ - u_int eptype = 0; - - switch (UE_GET_XFERTYPE(ed->bmAttributes)) { - case UE_CONTROL: - eptype = 0x0; - break; - case UE_ISOCHRONOUS: - eptype = 0x1; - break; - case UE_BULK: - eptype = 0x2; - break; - case UE_INTERRUPT: - eptype = 0x3; - break; - } - - if ((UE_GET_XFERTYPE(ed->bmAttributes) == UE_CONTROL) || - (UE_GET_DIR(ed->bEndpointAddress) == UE_DIR_IN)) - return eptype | 0x4; - else - return eptype; -} - -static u_int -xhci_ep_get_dci(usb_endpoint_descriptor_t * const ed) -{ - /* xHCI 1.0 section 4.5.1 */ - u_int epaddr = UE_GET_ADDR(ed->bEndpointAddress); - u_int in = 0; - - if ((UE_GET_XFERTYPE(ed->bmAttributes) == UE_CONTROL) || - (UE_GET_DIR(ed->bEndpointAddress) == UE_DIR_IN)) - in = 1; - - return epaddr * 2 + in; -} - -static inline u_int -xhci_dci_to_ici(const u_int i) -{ - return i + 1; -} - -static inline void * -xhci_slot_get_dcv(struct xhci_softc * const sc, struct xhci_slot * const xs, - const u_int dci) -{ - return KERNADDR(&xs->xs_dc_dma, sc->sc_ctxsz * dci); -} - -#if 0 /* unused */ -static inline bus_addr_t -xhci_slot_get_dcp(struct xhci_softc * const sc, struct xhci_slot * const xs, - const u_int dci) -{ - return DMAADDR(&xs->xs_dc_dma, sc->sc_ctxsz * dci); -} -#endif /* unused */ - -static inline void * -xhci_slot_get_icv(struct xhci_softc * const sc, struct xhci_slot * const xs, - const u_int ici) -{ - return KERNADDR(&xs->xs_ic_dma, sc->sc_ctxsz * ici); -} - -static inline bus_addr_t -xhci_slot_get_icp(struct xhci_softc * const sc, struct xhci_slot * const xs, - const u_int ici) -{ - return DMAADDR(&xs->xs_ic_dma, sc->sc_ctxsz * ici); -} - -static inline struct xhci_trb * -xhci_ring_trbv(struct xhci_ring * const xr, u_int idx) -{ - return KERNADDR(&xr->xr_dma, XHCI_TRB_SIZE * idx); -} - -static inline bus_addr_t -xhci_ring_trbp(struct xhci_ring * const xr, u_int idx) -{ - return DMAADDR(&xr->xr_dma, XHCI_TRB_SIZE * idx); -} - -static inline void -xhci_xfer_put_trb(struct xhci_xfer * const xx, u_int idx, - uint64_t parameter, uint32_t status, uint32_t control) -{ - KASSERTMSG(idx < xx->xx_ntrb, "idx=%u xx_ntrb=%u", idx, xx->xx_ntrb); - xx->xx_trb[idx].trb_0 = parameter; - xx->xx_trb[idx].trb_2 = status; - xx->xx_trb[idx].trb_3 = control; -} - -static inline void -xhci_trb_put(struct xhci_trb * const trb, uint64_t parameter, uint32_t status, - uint32_t control) -{ - trb->trb_0 = htole64(parameter); - trb->trb_2 = htole32(status); - trb->trb_3 = htole32(control); -} - -static int -xhci_trb_get_idx(struct xhci_ring *xr, uint64_t trb_0, int *idx) -{ - /* base address of TRBs */ - bus_addr_t trbp = xhci_ring_trbp(xr, 0); - - /* trb_0 range sanity check */ - if (trb_0 == 0 || trb_0 < trbp || - (trb_0 - trbp) % sizeof(struct xhci_trb) != 0 || - (trb_0 - trbp) / sizeof(struct xhci_trb) >= xr->xr_ntrb) { - return 1; - } - *idx = (trb_0 - trbp) / sizeof(struct xhci_trb); - return 0; -} - -static unsigned int -xhci_get_epstate(struct xhci_softc * const sc, struct xhci_slot * const xs, - u_int dci) -{ - uint32_t *cp; - - usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD); - cp = xhci_slot_get_dcv(sc, xs, dci); - return XHCI_EPCTX_0_EPSTATE_GET(le32toh(cp[0])); -} - -static inline unsigned int -xhci_ctlrport2bus(struct xhci_softc * const sc, unsigned int ctlrport) -{ - const unsigned int port = ctlrport - 1; - const uint8_t bit = __BIT(port % NBBY); - - return __SHIFTOUT(sc->sc_ctlrportbus[port / NBBY], bit); -} - -/* - * Return the roothub port for a controller port. Both are 1..n. - */ -static inline unsigned int -xhci_ctlrport2rhport(struct xhci_softc * const sc, unsigned int ctrlport) -{ - - return sc->sc_ctlrportmap[ctrlport - 1]; -} - -/* - * Return the controller port for a bus roothub port. Both are 1..n. - */ -static inline unsigned int -xhci_rhport2ctlrport(struct xhci_softc * const sc, unsigned int bn, - unsigned int rhport) -{ - - return sc->sc_rhportmap[bn][rhport - 1]; -} - -/* --- */ - -void -xhci_childdet(device_t self, device_t child) -{ - struct xhci_softc * const sc = device_private(self); - - mutex_enter(&sc->sc_intr_lock); - KASSERT((sc->sc_child == child) || (sc->sc_child2 == child)); - if (child == sc->sc_child2) - sc->sc_child2 = NULL; - else if (child == sc->sc_child) - sc->sc_child = NULL; - mutex_exit(&sc->sc_intr_lock); -} - -int -xhci_detach(struct xhci_softc *sc, int flags) -{ - int rv = 0; - - if (sc->sc_child2 != NULL) { - rv = config_detach(sc->sc_child2, flags); - if (rv != 0) - return rv; - KASSERT(sc->sc_child2 == NULL); - } - - if (sc->sc_child != NULL) { - rv = config_detach(sc->sc_child, flags); - if (rv != 0) - return rv; - KASSERT(sc->sc_child == NULL); - } - - /* XXX unconfigure/free slots */ - - /* verify: */ - xhci_rt_write_4(sc, XHCI_IMAN(0), 0); - xhci_op_write_4(sc, XHCI_USBCMD, 0); - /* do we need to wait for stop? */ - - xhci_op_write_8(sc, XHCI_CRCR, 0); - xhci_ring_free(sc, &sc->sc_cr); - cv_destroy(&sc->sc_command_cv); - cv_destroy(&sc->sc_cmdbusy_cv); - - xhci_rt_write_4(sc, XHCI_ERSTSZ(0), 0); - xhci_rt_write_8(sc, XHCI_ERSTBA(0), 0); - xhci_rt_write_8(sc, XHCI_ERDP(0), 0 | XHCI_ERDP_BUSY); - xhci_ring_free(sc, &sc->sc_er); - - usb_freemem(&sc->sc_eventst_dma); - - xhci_op_write_8(sc, XHCI_DCBAAP, 0); - usb_freemem(&sc->sc_dcbaa_dma); - - kmem_free(sc->sc_slots, sizeof(*sc->sc_slots) * sc->sc_maxslots); - - kmem_free(sc->sc_ctlrportbus, - howmany(sc->sc_maxports * sizeof(uint8_t), NBBY)); - kmem_free(sc->sc_ctlrportmap, sc->sc_maxports * sizeof(int)); - - for (size_t j = 0; j < __arraycount(sc->sc_rhportmap); j++) { - kmem_free(sc->sc_rhportmap[j], sc->sc_maxports * sizeof(int)); - } - - mutex_destroy(&sc->sc_rhlock); - mutex_destroy(&sc->sc_lock); - mutex_destroy(&sc->sc_intr_lock); - - pool_cache_destroy(sc->sc_xferpool); - - return rv; -} - -int -xhci_activate(device_t self, enum devact act) -{ - struct xhci_softc * const sc = device_private(self); - - switch (act) { - case DVACT_DEACTIVATE: - sc->sc_dying = true; - return 0; - default: - return EOPNOTSUPP; - } -} - -bool -xhci_suspend(device_t self, const pmf_qual_t *qual) -{ - struct xhci_softc * const sc = device_private(self); - size_t i, j, bn, dci; - int port; - uint32_t v; - usbd_status err; - bool ok = false; - - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - - /* - * Block issuance of new commands, and wait for all pending - * commands to complete. - */ - mutex_enter(&sc->sc_lock); - KASSERT(sc->sc_suspender == NULL); - sc->sc_suspender = curlwp; - while (sc->sc_command_addr != 0) - cv_wait(&sc->sc_cmdbusy_cv, &sc->sc_lock); - mutex_exit(&sc->sc_lock); - - /* - * Block roothub xfers which might touch portsc registers until - * we're done suspending. - */ - mutex_enter(&sc->sc_rhlock); - - /* - * xHCI Requirements Specification 1.2, May 2019, Sec. 4.23.2: - * xHCI Power Management, p. 342 - * https://www.intel.com/content/dam/www/public/us/en/documents/technical-specifications/extensible-host-controler-interface-usb-xhci.pdf#page=342 - */ - - /* - * `1. Stop all USB activity by issuing Stop Endpoint Commands - * for Busy endpoints in the Running state. If the Force - * Save Context Capability (FSC = ``0'') is not supported, - * then Stop Endpoint Commands shall be issued for all idle - * endpoints in the Running state as well. The Stop - * Endpoint Command causes the xHC to update the respective - * Endpoint or Stream Contexts in system memory, e.g. the - * TR Dequeue Pointer, DCS, etc. fields. Refer to - * Implementation Note "0".' - */ - for (i = 0; i < sc->sc_maxslots; i++) { - struct xhci_slot *xs = &sc->sc_slots[i]; - - /* Skip if the slot is not in use. */ - if (xs->xs_idx == 0) - continue; - - for (dci = XHCI_DCI_SLOT; dci <= XHCI_MAX_DCI; dci++) { - /* Skip if the endpoint is not Running. */ - /* XXX What about Busy? */ - if (xhci_get_epstate(sc, xs, dci) != - XHCI_EPSTATE_RUNNING) - continue; - - /* Stop endpoint. */ - mutex_enter(&sc->sc_lock); - err = xhci_stop_endpoint_cmd(sc, xs, dci, - XHCI_TRB_3_SUSP_EP_BIT); - mutex_exit(&sc->sc_lock); - if (err) { - device_printf(self, "failed to stop endpoint" - " slot %zu dci %zu err %d\n", - i, dci, err); - goto out; - } - } - } - - /* - * Next, suspend all the ports: - * - * xHCI Requirements Specification 1.2, May 2019, Sec. 4.15: - * Suspend-Resume, pp. 276-283 - * https://www.intel.com/content/dam/www/public/us/en/documents/technical-specifications/extensible-host-controler-interface-usb-xhci.pdf#page=276 - */ - for (bn = 0; bn < 2; bn++) { - for (i = 1; i <= sc->sc_rhportcount[bn]; i++) { - /* 4.15.1: Port Suspend. */ - port = XHCI_PORTSC(xhci_rhport2ctlrport(sc, bn, i)); - - /* - * `System software places individual ports - * into suspend mode by writing a ``3'' into - * the appropriate PORTSC register Port Link - * State (PLS) field (refer to Section 5.4.8). - * Software should only set the PLS field to - * ``3'' when the port is in the Enabled - * state.' - * - * `Software should not attempt to suspend a - * port unless the port reports that it is in - * the enabled (PED = ``1''; PLS < ``3'') - * state (refer to Section 5.4.8 for more - * information about PED and PLS).' - */ - v = xhci_op_read_4(sc, port); - if (((v & XHCI_PS_PED) == 0) || - XHCI_PS_PLS_GET(v) >= XHCI_PS_PLS_U3) - continue; - v &= ~(XHCI_PS_PLS_MASK | XHCI_PS_CLEAR); - v |= XHCI_PS_LWS | XHCI_PS_PLS_SET(XHCI_PS_PLS_SETU3); - xhci_op_write_4(sc, port, v); - - /* - * `When the PLS field is written with U3 - * (``3''), the status of the PLS bit will not - * change to the target U state U3 until the - * suspend signaling has completed to the - * attached device (which may be as long as - * 10ms.).' - * - * `Software is required to wait for U3 - * transitions to complete before it puts the - * xHC into a low power state, and before - * resuming the port.' - * - * XXX Take advantage of the technique to - * reduce polling on host controllers that - * support the U3C capability. - */ - for (j = 0; j < XHCI_WAIT_PLS_U3; j++) { - v = xhci_op_read_4(sc, port); - if (XHCI_PS_PLS_GET(v) == XHCI_PS_PLS_U3) - break; - usb_delay_ms(&sc->sc_bus, 1); - } - if (j == XHCI_WAIT_PLS_U3) { - device_printf(self, - "suspend timeout on bus %zu port %zu\n", - bn, i); - goto out; - } - } - } - - /* - * `2. Ensure that the Command Ring is in the Stopped state - * (CRR = ``0'') or Idle (i.e. the Command Transfer Ring is - * empty), and all Command Completion Events associated - * with them have been received.' - * - * XXX - */ - - /* `3. Stop the controller by setting Run/Stop (R/S) = ``0''.' */ - xhci_op_write_4(sc, XHCI_USBCMD, - xhci_op_read_4(sc, XHCI_USBCMD) & ~XHCI_CMD_RS); - - /* - * `4. Read the Operational Runtime, and VTIO registers in the - * following order: USBCMD, DNCTRL, DCBAAP, CONFIG, ERSTSZ, - * ERSTBA, ERDP, IMAN, IMOD, and VTIO and save their - * state.' - * - * (We don't use VTIO here (XXX for now?).) - */ - sc->sc_regs.usbcmd = xhci_op_read_4(sc, XHCI_USBCMD); - sc->sc_regs.dnctrl = xhci_op_read_4(sc, XHCI_DNCTRL); - sc->sc_regs.dcbaap = xhci_op_read_8(sc, XHCI_DCBAAP); - sc->sc_regs.config = xhci_op_read_4(sc, XHCI_CONFIG); - sc->sc_regs.erstsz0 = xhci_rt_read_4(sc, XHCI_ERSTSZ(0)); - sc->sc_regs.erstba0 = xhci_rt_read_8(sc, XHCI_ERSTBA(0)); - sc->sc_regs.erdp0 = xhci_rt_read_8(sc, XHCI_ERDP(0)); - sc->sc_regs.iman0 = xhci_rt_read_4(sc, XHCI_IMAN(0)); - sc->sc_regs.imod0 = xhci_rt_read_4(sc, XHCI_IMOD(0)); - - /* - * `5. Set the Controller Save State (CSS) flag in the USBCMD - * register (5.4.1)...' - */ - xhci_op_write_4(sc, XHCI_USBCMD, - xhci_op_read_4(sc, XHCI_USBCMD) | XHCI_CMD_CSS); - - /* - * `...and wait for the Save State Status (SSS) flag in the - * USBSTS register (5.4.2) to transition to ``0''.' - */ - for (i = 0; i < XHCI_WAIT_SSS; i++) { - if ((xhci_op_read_4(sc, XHCI_USBSTS) & XHCI_STS_SSS) == 0) - break; - usb_delay_ms(&sc->sc_bus, 1); - } - if (i >= XHCI_WAIT_SSS) { - device_printf(self, "suspend timeout, USBSTS.SSS\n"); - /* - * Just optimistically go on and check SRE anyway -- - * what's the worst that could happen? - */ - } - - /* - * `Note: After a Save or Restore operation completes, the - * Save/Restore Error (SRE) flag in the USBSTS register should - * be checked to ensure that the operation completed - * successfully.' - */ - if (xhci_op_read_4(sc, XHCI_USBSTS) & XHCI_STS_SRE) { - device_printf(self, "suspend error, USBSTS.SRE\n"); - goto out; - } - - /* Success! */ - ok = true; - -out: mutex_exit(&sc->sc_rhlock); - if (!ok) { - /* - * If suspend failed, stop holding up command issuance - * and make it fail instead. - */ - mutex_enter(&sc->sc_lock); - KASSERT(sc->sc_suspender == curlwp); - sc->sc_suspender = NULL; - sc->sc_suspendresume_failed = true; - cv_broadcast(&sc->sc_cmdbusy_cv); - mutex_exit(&sc->sc_lock); - } - return ok; -} - -bool -xhci_resume(device_t self, const pmf_qual_t *qual) -{ - struct xhci_softc * const sc = device_private(self); - size_t i, j, bn, dci; - int port; - uint32_t v; - bool ok = false; - - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - - /* - * If resume had previously failed, just try again. Can't make - * things worse, probably. - */ - mutex_enter(&sc->sc_lock); - if (sc->sc_suspendresume_failed) { - KASSERT(sc->sc_suspender == NULL); - sc->sc_suspender = curlwp; - sc->sc_suspendresume_failed = false; - } - KASSERT(sc->sc_suspender); - mutex_exit(&sc->sc_lock); - - /* - * Block roothub xfers which might touch portsc registers until - * we're done resuming. - */ - mutex_enter(&sc->sc_rhlock); - - /* - * xHCI Requirements Specification 1.2, May 2019, Sec. 4.23.2: - * xHCI Power Management, p. 343 - * https://www.intel.com/content/dam/www/public/us/en/documents/technical-specifications/extensible-host-controler-interface-usb-xhci.pdf#page=343 - */ - - /* - * `4. Restore the Operational Runtime, and VTIO registers with - * their previously saved state in the following order: - * DNCTRL, DCBAAP, CONFIG, ERSTSZ, ERSTBA, ERDP, IMAN, - * IMOD, and VTIO.' - * - * (We don't use VTIO here (for now?).) - */ - xhci_op_write_4(sc, XHCI_USBCMD, sc->sc_regs.usbcmd); - xhci_op_write_4(sc, XHCI_DNCTRL, sc->sc_regs.dnctrl); - xhci_op_write_8(sc, XHCI_DCBAAP, sc->sc_regs.dcbaap); - xhci_op_write_4(sc, XHCI_CONFIG, sc->sc_regs.config); - xhci_rt_write_4(sc, XHCI_ERSTSZ(0), sc->sc_regs.erstsz0); - xhci_rt_write_8(sc, XHCI_ERSTBA(0), sc->sc_regs.erstba0); - xhci_rt_write_8(sc, XHCI_ERDP(0), sc->sc_regs.erdp0); - xhci_rt_write_4(sc, XHCI_IMAN(0), sc->sc_regs.iman0); - xhci_rt_write_4(sc, XHCI_IMOD(0), sc->sc_regs.imod0); - - memset(&sc->sc_regs, 0, sizeof(sc->sc_regs)); /* paranoia */ - - /* - * `5. Set the Controller Restore State (CRS) flag in the - * USBCMD register (5.4.1) to ``1''...' - */ - xhci_op_write_4(sc, XHCI_USBCMD, - xhci_op_read_4(sc, XHCI_USBCMD) | XHCI_CMD_CRS); - - /* - * `...and wait for the Restore State Status (RSS) in the - * USBSTS register (5.4.2) to transition to ``0''.' - */ - for (i = 0; i < XHCI_WAIT_RSS; i++) { - if ((xhci_op_read_4(sc, XHCI_USBSTS) & XHCI_STS_RSS) == 0) - break; - usb_delay_ms(&sc->sc_bus, 1); - } - if (i >= XHCI_WAIT_RSS) { - device_printf(self, "resume timeout, USBSTS.RSS\n"); - goto out; - } - - /* - * `6. Reinitialize the Command Ring, i.e. so its Cycle bits - * are consistent with the RCS values to be written to the - * CRCR.' - * - * XXX Hope just zeroing it is good enough! - */ - xhci_host_dequeue(sc->sc_cr); - - /* - * `7. Write the CRCR with the address and RCS value of the - * reinitialized Command Ring. Note that this write will - * cause the Command Ring to restart at the address - * specified by the CRCR.' - */ - xhci_op_write_8(sc, XHCI_CRCR, xhci_ring_trbp(sc->sc_cr, 0) | - sc->sc_cr->xr_cs); - - /* - * `8. Enable the controller by setting Run/Stop (R/S) = - * ``1''.' - */ - xhci_op_write_4(sc, XHCI_USBCMD, - xhci_op_read_4(sc, XHCI_USBCMD) | XHCI_CMD_RS); - - /* - * `9. Software shall walk the USB topology and initialize each - * of the xHC PORTSC, PORTPMSC, and PORTLI registers, and - * external hub ports attached to USB devices.' - * - * This follows the procedure in 4.15 `Suspend-Resume', 4.15.2 - * `Port Resume', 4.15.2.2 `Host Initiated'. - * - * XXX We should maybe batch up initiating the state - * transitions, and then wait for them to complete all at once. - */ - for (bn = 0; bn < 2; bn++) { - for (i = 1; i <= sc->sc_rhportcount[bn]; i++) { - port = XHCI_PORTSC(xhci_rhport2ctlrport(sc, bn, i)); - - /* `When a port is in the U3 state: ...' */ - v = xhci_op_read_4(sc, port); - if (XHCI_PS_PLS_GET(v) != XHCI_PS_PLS_U3) - continue; - - /* - * `For a USB2 protocol port, software shall - * write a ``15'' (Resume) to the PLS field to - * initiate resume signaling. The port shall - * transition to the Resume substate and the - * xHC shall transmit the resume signaling - * within 1ms (T_URSM). Software shall ensure - * that resume is signaled for at least 20ms - * (T_DRSMDN). Software shall start timing - * T_DRSMDN from the write of ``15'' (Resume) - * to PLS.' - */ - if (bn == 1) { - KASSERT(sc->sc_bus2.ub_revision == USBREV_2_0); - v &= ~(XHCI_PS_PLS_MASK | XHCI_PS_CLEAR); - v |= XHCI_PS_LWS; - v |= XHCI_PS_PLS_SET(XHCI_PS_PLS_SETRESUME); - xhci_op_write_4(sc, port, v); - usb_delay_ms(&sc->sc_bus, USB_RESUME_WAIT); - } else { - KASSERT(sc->sc_bus.ub_revision > USBREV_2_0); - } - - /* - * `For a USB3 protocol port [and a USB2 - * protocol port after transitioning to - * Resume], software shall write a ``0'' (U0) - * to the PLS field...' - */ - v = xhci_op_read_4(sc, port); - v &= ~(XHCI_PS_PLS_MASK | XHCI_PS_CLEAR); - v |= XHCI_PS_LWS | XHCI_PS_PLS_SET(XHCI_PS_PLS_SETU0); - xhci_op_write_4(sc, port, v); - - for (j = 0; j < XHCI_WAIT_PLS_U0; j++) { - v = xhci_op_read_4(sc, port); - if (XHCI_PS_PLS_GET(v) == XHCI_PS_PLS_U0) - break; - usb_delay_ms(&sc->sc_bus, 1); - } - if (j == XHCI_WAIT_PLS_U0) { - device_printf(self, - "resume timeout on bus %zu port %zu\n", - bn, i); - goto out; - } - } - } - - /* - * `10. Restart each of the previously Running endpoints by - * ringing their doorbells.' - */ - for (i = 0; i < sc->sc_maxslots; i++) { - struct xhci_slot *xs = &sc->sc_slots[i]; - - /* Skip if the slot is not in use. */ - if (xs->xs_idx == 0) - continue; - - for (dci = XHCI_DCI_SLOT; dci <= XHCI_MAX_DCI; dci++) { - /* Skip if the endpoint is not Running. */ - if (xhci_get_epstate(sc, xs, dci) != - XHCI_EPSTATE_RUNNING) - continue; - - /* Ring the doorbell. */ - xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci); - } - } - - /* - * `Note: After a Save or Restore operation completes, the - * Save/Restore Error (SRE) flag in the USBSTS register should - * be checked to ensure that the operation completed - * successfully.' - */ - if (xhci_op_read_4(sc, XHCI_USBSTS) & XHCI_STS_SRE) { - device_printf(self, "resume error, USBSTS.SRE\n"); - goto out; - } - - /* Success! */ - ok = true; - -out: /* - * Resume command issuance. If the hardware failed to resume, - * well, tough -- deadlocking because everything is held up on - * the suspension, with no opportunity to detach, isn't better - * than timing out waiting for dead hardware. - */ - mutex_enter(&sc->sc_lock); - KASSERT(sc->sc_suspender); - sc->sc_suspender = NULL; - sc->sc_suspendresume_failed = !ok; - cv_broadcast(&sc->sc_cmdbusy_cv); - mutex_exit(&sc->sc_lock); - - mutex_exit(&sc->sc_rhlock); - return ok; -} - -bool -xhci_shutdown(device_t self, int flags) -{ - return false; -} - -static int -xhci_hc_reset(struct xhci_softc * const sc) -{ - uint32_t usbcmd, usbsts; - int i; - - /* Check controller not ready */ - for (i = 0; i < XHCI_WAIT_CNR; i++) { - usbsts = xhci_op_read_4(sc, XHCI_USBSTS); - if ((usbsts & XHCI_STS_CNR) == 0) - break; - usb_delay_ms(&sc->sc_bus, 1); - } - if (i >= XHCI_WAIT_CNR) { - aprint_error_dev(sc->sc_dev, "controller not ready timeout\n"); - return EIO; - } - - /* Halt controller */ - usbcmd = 0; - xhci_op_write_4(sc, XHCI_USBCMD, usbcmd); - usb_delay_ms(&sc->sc_bus, 1); - - /* Reset controller */ - usbcmd = XHCI_CMD_HCRST; - xhci_op_write_4(sc, XHCI_USBCMD, usbcmd); - for (i = 0; i < XHCI_WAIT_HCRST; i++) { - /* - * Wait 1ms first. Existing Intel xHCI requires 1ms delay to - * prevent system hang (Errata). - */ - usb_delay_ms(&sc->sc_bus, 1); - usbcmd = xhci_op_read_4(sc, XHCI_USBCMD); - if ((usbcmd & XHCI_CMD_HCRST) == 0) - break; - } - if (i >= XHCI_WAIT_HCRST) { - aprint_error_dev(sc->sc_dev, "host controller reset timeout\n"); - return EIO; - } - - /* Check controller not ready */ - for (i = 0; i < XHCI_WAIT_CNR; i++) { - usbsts = xhci_op_read_4(sc, XHCI_USBSTS); - if ((usbsts & XHCI_STS_CNR) == 0) - break; - usb_delay_ms(&sc->sc_bus, 1); - } - if (i >= XHCI_WAIT_CNR) { - aprint_error_dev(sc->sc_dev, - "controller not ready timeout after reset\n"); - return EIO; - } - - return 0; -} - -/* 7.2 xHCI Support Protocol Capability */ -static void -xhci_id_protocols(struct xhci_softc *sc, bus_size_t ecp) -{ - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - - /* XXX Cache this lot */ - - const uint32_t w0 = xhci_read_4(sc, ecp); - const uint32_t w4 = xhci_read_4(sc, ecp + 4); - const uint32_t w8 = xhci_read_4(sc, ecp + 8); - const uint32_t wc = xhci_read_4(sc, ecp + 0xc); - - aprint_debug_dev(sc->sc_dev, - " SP: 0x%08x 0x%08x 0x%08x 0x%08x\n", w0, w4, w8, wc); - - if (w4 != XHCI_XECP_USBID) - return; - - const int major = XHCI_XECP_SP_W0_MAJOR(w0); - const int minor = XHCI_XECP_SP_W0_MINOR(w0); - const uint8_t cpo = XHCI_XECP_SP_W8_CPO(w8); - const uint8_t cpc = XHCI_XECP_SP_W8_CPC(w8); - - const uint16_t mm = __SHIFTOUT(w0, __BITS(31, 16)); - switch (mm) { - case 0x0200: - case 0x0300: - case 0x0301: - case 0x0310: - case 0x0320: - aprint_debug_dev(sc->sc_dev, " %s ports %d - %d\n", - major == 3 ? "ss" : "hs", cpo, cpo + cpc - 1); - if (major == 3) - sc->sc_usb3nports += cpo + cpc - 1; - else - sc->sc_usb2nports += cpo + cpc - 1; - break; - default: - aprint_error_dev(sc->sc_dev, " unknown major/minor (%d/%d)\n", - major, minor); - return; - } - - const size_t bus = (major == 3) ? 0 : 1; - - /* Index arrays with 0..n-1 where ports are numbered 1..n */ - for (size_t cp = cpo - 1; cp < cpo + cpc - 1; cp++) { - if (sc->sc_ctlrportmap[cp] != 0) { - aprint_error_dev(sc->sc_dev, "controller port %zu " - "already assigned", cp); - continue; - } - - sc->sc_ctlrportbus[cp / NBBY] |= - bus == 0 ? 0 : __BIT(cp % NBBY); - - const size_t rhp = sc->sc_rhportcount[bus]++; - - KASSERTMSG(sc->sc_rhportmap[bus][rhp] == 0, - "bus %zu rhp %zu is %d", bus, rhp, - sc->sc_rhportmap[bus][rhp]); - - sc->sc_rhportmap[bus][rhp] = cp + 1; - sc->sc_ctlrportmap[cp] = rhp + 1; - } -} - -/* Process extended capabilities */ -static void -xhci_ecp(struct xhci_softc *sc) -{ - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - - bus_size_t ecp = XHCI_HCC_XECP(sc->sc_hcc) * 4; - while (ecp != 0) { - uint32_t ecr = xhci_read_4(sc, ecp); - aprint_debug_dev(sc->sc_dev, "ECR: 0x%08x\n", ecr); - switch (XHCI_XECP_ID(ecr)) { - case XHCI_ID_PROTOCOLS: { - xhci_id_protocols(sc, ecp); - break; - } - case XHCI_ID_USB_LEGACY: { - uint8_t bios_sem; - - /* Take host controller ownership from BIOS */ - bios_sem = xhci_read_1(sc, ecp + XHCI_XECP_BIOS_SEM); - if (bios_sem) { - /* sets xHCI to be owned by OS */ - xhci_write_1(sc, ecp + XHCI_XECP_OS_SEM, 1); - aprint_debug_dev(sc->sc_dev, - "waiting for BIOS to give up control\n"); - for (int i = 0; i < 5000; i++) { - bios_sem = xhci_read_1(sc, ecp + - XHCI_XECP_BIOS_SEM); - if (bios_sem == 0) - break; - DELAY(1000); - } - if (bios_sem) { - aprint_error_dev(sc->sc_dev, - "timed out waiting for BIOS\n"); - } - } - break; - } - default: - break; - } - ecr = xhci_read_4(sc, ecp); - if (XHCI_XECP_NEXT(ecr) == 0) { - ecp = 0; - } else { - ecp += XHCI_XECP_NEXT(ecr) * 4; - } - } -} - -#define XHCI_HCCPREV1_BITS \ - "\177\020" /* New bitmask */ \ - "f\020\020XECP\0" \ - "f\014\4MAXPSA\0" \ - "b\013CFC\0" \ - "b\012SEC\0" \ - "b\011SBD\0" \ - "b\010FSE\0" \ - "b\7NSS\0" \ - "b\6LTC\0" \ - "b\5LHRC\0" \ - "b\4PIND\0" \ - "b\3PPC\0" \ - "b\2CZC\0" \ - "b\1BNC\0" \ - "b\0AC64\0" \ - "\0" -#define XHCI_HCCV1_x_BITS \ - "\177\020" /* New bitmask */ \ - "f\020\020XECP\0" \ - "f\014\4MAXPSA\0" \ - "b\013CFC\0" \ - "b\012SEC\0" \ - "b\011SPC\0" \ - "b\010PAE\0" \ - "b\7NSS\0" \ - "b\6LTC\0" \ - "b\5LHRC\0" \ - "b\4PIND\0" \ - "b\3PPC\0" \ - "b\2CSZ\0" \ - "b\1BNC\0" \ - "b\0AC64\0" \ - "\0" - -#define XHCI_HCC2_BITS \ - "\177\020" /* New bitmask */ \ - "b\7ETC_TSC\0" \ - "b\6ETC\0" \ - "b\5CIC\0" \ - "b\4LEC\0" \ - "b\3CTC\0" \ - "b\2FSC\0" \ - "b\1CMC\0" \ - "b\0U3C\0" \ - "\0" - -void -xhci_start(struct xhci_softc *sc) -{ - xhci_rt_write_4(sc, XHCI_IMAN(0), XHCI_IMAN_INTR_ENA); - if ((sc->sc_quirks & XHCI_QUIRK_INTEL) != 0) - /* Intel xhci needs interrupt rate moderated. */ - xhci_rt_write_4(sc, XHCI_IMOD(0), XHCI_IMOD_DEFAULT_LP); - else - xhci_rt_write_4(sc, XHCI_IMOD(0), 0); - aprint_debug_dev(sc->sc_dev, "current IMOD %u\n", - xhci_rt_read_4(sc, XHCI_IMOD(0))); - - /* Go! */ - xhci_op_write_4(sc, XHCI_USBCMD, XHCI_CMD_INTE|XHCI_CMD_RS); - aprint_debug_dev(sc->sc_dev, "USBCMD 0x%08"PRIx32"\n", - xhci_op_read_4(sc, XHCI_USBCMD)); -} - -int -xhci_init(struct xhci_softc *sc) -{ - bus_size_t bsz; - uint32_t hcs1, hcs2, hcs3, dboff, rtsoff; - uint32_t pagesize, config; - int i = 0; - uint16_t hciversion; - uint8_t caplength; - - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - - /* Set up the bus struct for the usb 3 and usb 2 buses */ - sc->sc_bus.ub_methods = &xhci_bus_methods; - sc->sc_bus.ub_pipesize = sizeof(struct xhci_pipe); - sc->sc_bus.ub_usedma = true; - sc->sc_bus.ub_hcpriv = sc; - - sc->sc_bus2.ub_methods = &xhci_bus_methods; - sc->sc_bus2.ub_pipesize = sizeof(struct xhci_pipe); - sc->sc_bus2.ub_revision = USBREV_2_0; - sc->sc_bus2.ub_usedma = true; - sc->sc_bus2.ub_hcpriv = sc; - sc->sc_bus2.ub_dmatag = sc->sc_bus.ub_dmatag; - - caplength = xhci_read_1(sc, XHCI_CAPLENGTH); - hciversion = xhci_read_2(sc, XHCI_HCIVERSION); - - if (hciversion < XHCI_HCIVERSION_0_96 || - hciversion >= 0x0200) { - aprint_normal_dev(sc->sc_dev, - "xHCI version %x.%x not known to be supported\n", - (hciversion >> 8) & 0xff, (hciversion >> 0) & 0xff); - } else { - aprint_verbose_dev(sc->sc_dev, "xHCI version %x.%x\n", - (hciversion >> 8) & 0xff, (hciversion >> 0) & 0xff); - } - - if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, 0, caplength, - &sc->sc_cbh) != 0) { - aprint_error_dev(sc->sc_dev, "capability subregion failure\n"); - return ENOMEM; - } - - hcs1 = xhci_cap_read_4(sc, XHCI_HCSPARAMS1); - sc->sc_maxslots = XHCI_HCS1_MAXSLOTS(hcs1); - sc->sc_maxintrs = XHCI_HCS1_MAXINTRS(hcs1); - sc->sc_maxports = XHCI_HCS1_MAXPORTS(hcs1); - hcs2 = xhci_cap_read_4(sc, XHCI_HCSPARAMS2); - hcs3 = xhci_cap_read_4(sc, XHCI_HCSPARAMS3); - aprint_debug_dev(sc->sc_dev, - "hcs1=%"PRIx32" hcs2=%"PRIx32" hcs3=%"PRIx32"\n", hcs1, hcs2, hcs3); - - sc->sc_hcc = xhci_cap_read_4(sc, XHCI_HCCPARAMS); - sc->sc_ctxsz = XHCI_HCC_CSZ(sc->sc_hcc) ? 64 : 32; - - char sbuf[128]; - if (hciversion < XHCI_HCIVERSION_1_0) - snprintb(sbuf, sizeof(sbuf), XHCI_HCCPREV1_BITS, sc->sc_hcc); - else - snprintb(sbuf, sizeof(sbuf), XHCI_HCCV1_x_BITS, sc->sc_hcc); - aprint_debug_dev(sc->sc_dev, "hcc=%s\n", sbuf); - aprint_debug_dev(sc->sc_dev, "xECP %" __PRIxBITS "\n", - XHCI_HCC_XECP(sc->sc_hcc) * 4); - if (hciversion >= XHCI_HCIVERSION_1_1) { - sc->sc_hcc2 = xhci_cap_read_4(sc, XHCI_HCCPARAMS2); - snprintb(sbuf, sizeof(sbuf), XHCI_HCC2_BITS, sc->sc_hcc2); - aprint_debug_dev(sc->sc_dev, "hcc2=%s\n", sbuf); - } - - /* default all ports to bus 0, i.e. usb 3 */ - sc->sc_ctlrportbus = kmem_zalloc( - howmany(sc->sc_maxports * sizeof(uint8_t), NBBY), KM_SLEEP); - sc->sc_ctlrportmap = - kmem_zalloc(sc->sc_maxports * sizeof(int), KM_SLEEP); - - /* controller port to bus roothub port map */ - for (size_t j = 0; j < __arraycount(sc->sc_rhportmap); j++) { - sc->sc_rhportmap[j] = - kmem_zalloc(sc->sc_maxports * sizeof(int), KM_SLEEP); - } - - /* - * Process all Extended Capabilities - */ - xhci_ecp(sc); - - bsz = XHCI_PORTSC(sc->sc_maxports); - if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, caplength, bsz, - &sc->sc_obh) != 0) { - aprint_error_dev(sc->sc_dev, "operational subregion failure\n"); - return ENOMEM; - } - - dboff = xhci_cap_read_4(sc, XHCI_DBOFF); - if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, dboff, - sc->sc_maxslots * 4, &sc->sc_dbh) != 0) { - aprint_error_dev(sc->sc_dev, "doorbell subregion failure\n"); - return ENOMEM; - } - - rtsoff = xhci_cap_read_4(sc, XHCI_RTSOFF); - if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, rtsoff, - sc->sc_maxintrs * 0x20, &sc->sc_rbh) != 0) { - aprint_error_dev(sc->sc_dev, "runtime subregion failure\n"); - return ENOMEM; - } - - int rv; - rv = xhci_hc_reset(sc); - if (rv != 0) { - return rv; - } - - if (sc->sc_vendor_init) - sc->sc_vendor_init(sc); - - pagesize = xhci_op_read_4(sc, XHCI_PAGESIZE); - aprint_debug_dev(sc->sc_dev, "PAGESIZE 0x%08x\n", pagesize); - pagesize = ffs(pagesize); - if (pagesize == 0) { - aprint_error_dev(sc->sc_dev, "pagesize is 0\n"); - return EIO; - } - sc->sc_pgsz = 1 << (12 + (pagesize - 1)); - aprint_debug_dev(sc->sc_dev, "sc_pgsz 0x%08x\n", (uint32_t)sc->sc_pgsz); - aprint_debug_dev(sc->sc_dev, "sc_maxslots 0x%08x\n", - (uint32_t)sc->sc_maxslots); - aprint_debug_dev(sc->sc_dev, "sc_maxports %d\n", sc->sc_maxports); - - int err; - sc->sc_maxspbuf = XHCI_HCS2_MAXSPBUF(hcs2); - aprint_debug_dev(sc->sc_dev, "sc_maxspbuf %d\n", sc->sc_maxspbuf); - if (sc->sc_maxspbuf != 0) { - err = usb_allocmem(sc->sc_bus.ub_dmatag, - sizeof(uint64_t) * sc->sc_maxspbuf, sizeof(uint64_t), - USBMALLOC_ZERO, &sc->sc_spbufarray_dma); - if (err) { - aprint_error_dev(sc->sc_dev, - "spbufarray init fail, err %d\n", err); - return ENOMEM; - } - - sc->sc_spbuf_dma = kmem_zalloc(sizeof(*sc->sc_spbuf_dma) * - sc->sc_maxspbuf, KM_SLEEP); - uint64_t *spbufarray = KERNADDR(&sc->sc_spbufarray_dma, 0); - for (i = 0; i < sc->sc_maxspbuf; i++) { - usb_dma_t * const dma = &sc->sc_spbuf_dma[i]; - /* allocate contexts */ - err = usb_allocmem(sc->sc_bus.ub_dmatag, sc->sc_pgsz, - sc->sc_pgsz, USBMALLOC_ZERO, dma); - if (err) { - aprint_error_dev(sc->sc_dev, - "spbufarray_dma init fail, err %d\n", err); - rv = ENOMEM; - goto bad1; - } - spbufarray[i] = htole64(DMAADDR(dma, 0)); - usb_syncmem(dma, 0, sc->sc_pgsz, - BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); - } - - usb_syncmem(&sc->sc_spbufarray_dma, 0, - sizeof(uint64_t) * sc->sc_maxspbuf, BUS_DMASYNC_PREWRITE); - } - - sc->sc_isthresh = XHCI_HCS2_IST(hcs2); - aprint_debug_dev(sc->sc_dev, "sc_isthresh %d\n", sc->sc_isthresh); - - /* - * xHI 5.3.4 - * If bit[3] is 0, IST is number of microframes in bit[2:0] - * If bit[3] is 1, IST is number of frames in bit[2:0] - */ - if (sc->sc_isthresh & 0x8) { - sc->sc_isthresh = (sc->sc_isthresh & 0x7) * - USB_UFRAMES_PER_FRAME; - } - - config = xhci_op_read_4(sc, XHCI_CONFIG); - config &= ~0xFF; - config |= sc->sc_maxslots & 0xFF; - xhci_op_write_4(sc, XHCI_CONFIG, config); - - err = xhci_ring_init(sc, &sc->sc_cr, XHCI_COMMAND_RING_TRBS, - XHCI_COMMAND_RING_SEGMENTS_ALIGN); - if (err) { - aprint_error_dev(sc->sc_dev, "command ring init fail, err %d\n", - err); - rv = ENOMEM; - goto bad1; - } - - err = xhci_ring_init(sc, &sc->sc_er, XHCI_EVENT_RING_TRBS, - XHCI_EVENT_RING_SEGMENTS_ALIGN); - if (err) { - aprint_error_dev(sc->sc_dev, "event ring init fail, err %d\n", - err); - rv = ENOMEM; - goto bad2; - } - - usb_dma_t *dma; - size_t size; - size_t align; - - dma = &sc->sc_eventst_dma; - size = roundup2(XHCI_EVENT_RING_SEGMENTS * XHCI_ERSTE_SIZE, - XHCI_EVENT_RING_SEGMENT_TABLE_ALIGN); - KASSERTMSG(size <= (512 * 1024), "eventst size %zu too large", size); - align = XHCI_EVENT_RING_SEGMENT_TABLE_ALIGN; - err = usb_allocmem(sc->sc_bus.ub_dmatag, size, align, - USBMALLOC_ZERO, dma); - if (err) { - aprint_error_dev(sc->sc_dev, "eventst init fail, err %d\n", - err); - rv = ENOMEM; - goto bad3; - } - - aprint_debug_dev(sc->sc_dev, "eventst: 0x%016jx %p %zx\n", - (uintmax_t)DMAADDR(&sc->sc_eventst_dma, 0), - KERNADDR(&sc->sc_eventst_dma, 0), - sc->sc_eventst_dma.udma_block->size); - - dma = &sc->sc_dcbaa_dma; - size = (1 + sc->sc_maxslots) * sizeof(uint64_t); - KASSERTMSG(size <= 2048, "dcbaa size %zu too large", size); - align = XHCI_DEVICE_CONTEXT_BASE_ADDRESS_ARRAY_ALIGN; - err = usb_allocmem(sc->sc_bus.ub_dmatag, size, align, - USBMALLOC_ZERO, dma); - if (err) { - aprint_error_dev(sc->sc_dev, "dcbaa init fail, err %d\n", err); - rv = ENOMEM; - goto bad4; - } - aprint_debug_dev(sc->sc_dev, "dcbaa: 0x%016jx %p %zx\n", - (uintmax_t)DMAADDR(&sc->sc_dcbaa_dma, 0), - KERNADDR(&sc->sc_dcbaa_dma, 0), - sc->sc_dcbaa_dma.udma_block->size); - - if (sc->sc_maxspbuf != 0) { - /* - * DCBA entry 0 hold the scratchbuf array pointer. - */ - *(uint64_t *)KERNADDR(dma, 0) = - htole64(DMAADDR(&sc->sc_spbufarray_dma, 0)); - usb_syncmem(dma, 0, size, BUS_DMASYNC_PREWRITE); - } - - sc->sc_slots = kmem_zalloc(sizeof(*sc->sc_slots) * sc->sc_maxslots, - KM_SLEEP); - if (sc->sc_slots == NULL) { - aprint_error_dev(sc->sc_dev, "slots init fail, err %d\n", err); - rv = ENOMEM; - goto bad; - } - - sc->sc_xferpool = pool_cache_init(sizeof(struct xhci_xfer), 0, 0, 0, - "xhcixfer", NULL, IPL_USB, NULL, NULL, NULL); - if (sc->sc_xferpool == NULL) { - aprint_error_dev(sc->sc_dev, "pool_cache init fail, err %d\n", - err); - rv = ENOMEM; - goto bad; - } - - cv_init(&sc->sc_command_cv, "xhcicmd"); - cv_init(&sc->sc_cmdbusy_cv, "xhcicmdq"); - mutex_init(&sc->sc_rhlock, MUTEX_DEFAULT, IPL_NONE); - mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_SOFTUSB); - mutex_init(&sc->sc_intr_lock, MUTEX_DEFAULT, IPL_USB); - - struct xhci_erste *erst; - erst = KERNADDR(&sc->sc_eventst_dma, 0); - erst[0].erste_0 = htole64(xhci_ring_trbp(sc->sc_er, 0)); - erst[0].erste_2 = htole32(sc->sc_er->xr_ntrb); - erst[0].erste_3 = htole32(0); - usb_syncmem(&sc->sc_eventst_dma, 0, - XHCI_ERSTE_SIZE * XHCI_EVENT_RING_SEGMENTS, BUS_DMASYNC_PREWRITE); - - xhci_rt_write_4(sc, XHCI_ERSTSZ(0), XHCI_EVENT_RING_SEGMENTS); - xhci_rt_write_8(sc, XHCI_ERSTBA(0), DMAADDR(&sc->sc_eventst_dma, 0)); - xhci_rt_write_8(sc, XHCI_ERDP(0), xhci_ring_trbp(sc->sc_er, 0) | - XHCI_ERDP_BUSY); - - xhci_op_write_8(sc, XHCI_DCBAAP, DMAADDR(&sc->sc_dcbaa_dma, 0)); - xhci_op_write_8(sc, XHCI_CRCR, xhci_ring_trbp(sc->sc_cr, 0) | - sc->sc_cr->xr_cs); - - HEXDUMP("eventst", KERNADDR(&sc->sc_eventst_dma, 0), - XHCI_ERSTE_SIZE * XHCI_EVENT_RING_SEGMENTS); - - if ((sc->sc_quirks & XHCI_DEFERRED_START) == 0) - xhci_start(sc); - - return 0; - - bad: - if (sc->sc_xferpool) { - pool_cache_destroy(sc->sc_xferpool); - sc->sc_xferpool = NULL; - } - - if (sc->sc_slots) { - kmem_free(sc->sc_slots, sizeof(*sc->sc_slots) * - sc->sc_maxslots); - sc->sc_slots = NULL; - } - - usb_freemem(&sc->sc_dcbaa_dma); - bad4: - usb_freemem(&sc->sc_eventst_dma); - bad3: - xhci_ring_free(sc, &sc->sc_er); - bad2: - xhci_ring_free(sc, &sc->sc_cr); - i = sc->sc_maxspbuf; - bad1: - for (int j = 0; j < i; j++) - usb_freemem(&sc->sc_spbuf_dma[j]); - usb_freemem(&sc->sc_spbufarray_dma); - - return rv; -} - -static inline bool -xhci_polling_p(struct xhci_softc * const sc) -{ - return sc->sc_bus.ub_usepolling || sc->sc_bus2.ub_usepolling; -} - -int -xhci_intr(void *v) -{ - struct xhci_softc * const sc = v; - int ret = 0; - - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - - if (sc == NULL) - return 0; - - mutex_spin_enter(&sc->sc_intr_lock); - - if (sc->sc_dying || !device_has_power(sc->sc_dev)) - goto done; - - /* If we get an interrupt while polling, then just ignore it. */ - if (xhci_polling_p(sc)) { -#ifdef DIAGNOSTIC - DPRINTFN(16, "ignored interrupt while polling", 0, 0, 0, 0); -#endif - goto done; - } - - ret = xhci_intr1(sc); - if (ret) { - KASSERT(sc->sc_child || sc->sc_child2); - - /* - * One of child busses could be already detached. It doesn't - * matter on which of the two the softintr is scheduled. - */ - if (sc->sc_child) - usb_schedsoftintr(&sc->sc_bus); - else - usb_schedsoftintr(&sc->sc_bus2); - } -done: - mutex_spin_exit(&sc->sc_intr_lock); - return ret; -} - -int -xhci_intr1(struct xhci_softc * const sc) -{ - uint32_t usbsts; - uint32_t iman; - - XHCIHIST_FUNC(); - - usbsts = xhci_op_read_4(sc, XHCI_USBSTS); - XHCIHIST_CALLARGS("USBSTS 0x%08jx", usbsts, 0, 0, 0); - if ((usbsts & (XHCI_STS_HSE | XHCI_STS_EINT | XHCI_STS_PCD | - XHCI_STS_HCE)) == 0) { - DPRINTFN(16, "ignored intr not for %jd", - device_unit(sc->sc_dev), 0, 0, 0); - return 0; - } - - /* - * Clear EINT and other transient flags, to not misenterpret - * next shared interrupt. Also, to avoid race, EINT must be cleared - * before XHCI_IMAN_INTR_PEND is cleared. - */ - xhci_op_write_4(sc, XHCI_USBSTS, usbsts & ~XHCI_STS_RSVDP0); - -#ifdef XHCI_DEBUG - usbsts = xhci_op_read_4(sc, XHCI_USBSTS); - DPRINTFN(16, "USBSTS 0x%08jx", usbsts, 0, 0, 0); -#endif - - iman = xhci_rt_read_4(sc, XHCI_IMAN(0)); - DPRINTFN(16, "IMAN0 0x%08jx", iman, 0, 0, 0); - iman |= XHCI_IMAN_INTR_PEND; - xhci_rt_write_4(sc, XHCI_IMAN(0), iman); - -#ifdef XHCI_DEBUG - iman = xhci_rt_read_4(sc, XHCI_IMAN(0)); - DPRINTFN(16, "IMAN0 0x%08jx", iman, 0, 0, 0); - usbsts = xhci_op_read_4(sc, XHCI_USBSTS); - DPRINTFN(16, "USBSTS 0x%08jx", usbsts, 0, 0, 0); -#endif - - return 1; -} - -/* - * 3 port speed types used in USB stack - * - * usbdi speed - * definition: USB_SPEED_* in usb.h - * They are used in struct usbd_device in USB stack. - * ioctl interface uses these values too. - * port_status speed - * definition: UPS_*_SPEED in usb.h - * They are used in usb_port_status_t and valid only for USB 2.0. - * Speed value is always 0 for Super Speed or more, and dwExtPortStatus - * of usb_port_status_ext_t indicates port speed. - * Note that some 3.0 values overlap with 2.0 values. - * (e.g. 0x200 means UPS_POER_POWER_SS in SS and - * means UPS_LOW_SPEED in HS.) - * port status returned from hub also uses these values. - * On NetBSD UPS_OTHER_SPEED indicates port speed is super speed - * or more. - * xspeed: - * definition: Protocol Speed ID (PSI) (xHCI 1.1 7.2.1) - * They are used in only slot context and PORTSC reg of xhci. - * The difference between usbdi speed and xspeed is - * that FS and LS values are swapped. - */ - -/* convert usbdi speed to xspeed */ -static int -xhci_speed2xspeed(int speed) -{ - switch (speed) { - case USB_SPEED_LOW: return 2; - case USB_SPEED_FULL: return 1; - default: return speed; - } -} - -#if 0 -/* convert xspeed to usbdi speed */ -static int -xhci_xspeed2speed(int xspeed) -{ - switch (xspeed) { - case 1: return USB_SPEED_FULL; - case 2: return USB_SPEED_LOW; - default: return xspeed; - } -} -#endif - -/* convert xspeed to port status speed */ -static int -xhci_xspeed2psspeed(int xspeed) -{ - switch (xspeed) { - case 0: return 0; - case 1: return UPS_FULL_SPEED; - case 2: return UPS_LOW_SPEED; - case 3: return UPS_HIGH_SPEED; - default: return UPS_OTHER_SPEED; - } -} - -/* - * Construct input contexts and issue TRB to open pipe. - */ -static usbd_status -xhci_configure_endpoint(struct usbd_pipe *pipe) -{ - struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); - struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; -#ifdef USB_DEBUG - const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc); -#endif - struct xhci_soft_trb trb; - usbd_status err; - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("slot %ju dci %ju epaddr 0x%02jx attr 0x%02jx", - xs->xs_idx, dci, pipe->up_endpoint->ue_edesc->bEndpointAddress, - pipe->up_endpoint->ue_edesc->bmAttributes); - - /* XXX ensure input context is available? */ - - memset(xhci_slot_get_icv(sc, xs, 0), 0, sc->sc_pgsz); - - /* set up context */ - xhci_setup_ctx(pipe); - - HEXDUMP("input control context", xhci_slot_get_icv(sc, xs, 0), - sc->sc_ctxsz * 1); - HEXDUMP("input endpoint context", xhci_slot_get_icv(sc, xs, - xhci_dci_to_ici(dci)), sc->sc_ctxsz * 1); - - trb.trb_0 = xhci_slot_get_icp(sc, xs, 0); - trb.trb_2 = 0; - trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | - XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_CONFIGURE_EP); - - err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); - - usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD); - HEXDUMP("output context", xhci_slot_get_dcv(sc, xs, dci), - sc->sc_ctxsz * 1); - - return err; -} - -#if 0 -static usbd_status -xhci_unconfigure_endpoint(struct usbd_pipe *pipe) -{ -#ifdef USB_DEBUG - struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; -#endif - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("slot %ju", xs->xs_idx, 0, 0, 0); - - return USBD_NORMAL_COMPLETION; -} -#endif - -/* 4.6.8, 6.4.3.7 */ -static void -xhci_reset_endpoint(struct usbd_pipe *pipe) -{ - struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); - struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; - const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc); - struct xhci_soft_trb trb; - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("slot %ju dci %ju", xs->xs_idx, dci, 0, 0); - - KASSERT(mutex_owned(&sc->sc_lock)); - - trb.trb_0 = 0; - trb.trb_2 = 0; - trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | - XHCI_TRB_3_EP_SET(dci) | - XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_RESET_EP); - - if (xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT)) { - device_printf(sc->sc_dev, "%s: endpoint 0x%x: timed out\n", - __func__, pipe->up_endpoint->ue_edesc->bEndpointAddress); - } -} - -/* - * 4.6.9, 6.4.3.8 - * Stop execution of TDs on xfer ring. - * Should be called with sc_lock held. - */ -static usbd_status -xhci_stop_endpoint_cmd(struct xhci_softc *sc, struct xhci_slot *xs, u_int dci, - uint32_t trb3flags) -{ - struct xhci_soft_trb trb; - usbd_status err; - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("slot %ju dci %ju", xs->xs_idx, dci, 0, 0); - - KASSERT(mutex_owned(&sc->sc_lock)); - - trb.trb_0 = 0; - trb.trb_2 = 0; - trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | - XHCI_TRB_3_EP_SET(dci) | - XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STOP_EP) | - trb3flags; - - err = xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT); - - return err; -} - -static usbd_status -xhci_stop_endpoint(struct usbd_pipe *pipe) -{ - struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); - struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; - const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc); - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("slot %ju dci %ju", xs->xs_idx, dci, 0, 0); - - KASSERT(mutex_owned(&sc->sc_lock)); - - return xhci_stop_endpoint_cmd(sc, xs, dci, 0); -} - -/* - * Set TR Dequeue Pointer. - * xHCI 1.1 4.6.10 6.4.3.9 - * Purge all of the TRBs on ring and reinitialize ring. - * Set TR dequeue Pointer to 0 and Cycle State to 1. - * EPSTATE of endpoint must be ERROR or STOPPED, otherwise CONTEXT_STATE - * error will be generated. - */ -static void -xhci_set_dequeue(struct usbd_pipe *pipe) -{ - struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); - struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; - const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc); - struct xhci_ring * const xr = xs->xs_xr[dci]; - struct xhci_soft_trb trb; - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("slot %ju dci %ju", xs->xs_idx, dci, 0, 0); - - KASSERT(mutex_owned(&sc->sc_lock)); - KASSERT(xr != NULL); - - xhci_host_dequeue(xr); - - /* set DCS */ - trb.trb_0 = xhci_ring_trbp(xr, 0) | 1; /* XXX */ - trb.trb_2 = 0; - trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | - XHCI_TRB_3_EP_SET(dci) | - XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SET_TR_DEQUEUE); - - if (xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT)) { - device_printf(sc->sc_dev, "%s: endpoint 0x%x: timed out\n", - __func__, pipe->up_endpoint->ue_edesc->bEndpointAddress); - } -} - -/* - * Open new pipe: called from usbd_setup_pipe_flags. - * Fills methods of pipe. - * If pipe is not for ep0, calls configure_endpoint. - */ -static usbd_status -xhci_open(struct usbd_pipe *pipe) -{ - struct usbd_device * const dev = pipe->up_dev; - struct xhci_pipe * const xpipe = (struct xhci_pipe *)pipe; - struct xhci_softc * const sc = XHCI_BUS2SC(dev->ud_bus); - struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; - usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc; - const u_int dci = xhci_ep_get_dci(ed); - const uint8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes); - usbd_status err; - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("addr %jd depth %jd port %jd speed %jd", dev->ud_addr, - dev->ud_depth, dev->ud_powersrc->up_portno, dev->ud_speed); - DPRINTFN(1, " dci %ju type 0x%02jx epaddr 0x%02jx attr 0x%02jx", - xhci_ep_get_dci(ed), ed->bDescriptorType, ed->bEndpointAddress, - ed->bmAttributes); - DPRINTFN(1, " mps %ju ival %ju", UGETW(ed->wMaxPacketSize), - ed->bInterval, 0, 0); - - if (sc->sc_dying) - return USBD_IOERROR; - - /* Root Hub */ - if (dev->ud_depth == 0 && dev->ud_powersrc->up_portno == 0) { - switch (ed->bEndpointAddress) { - case USB_CONTROL_ENDPOINT: - pipe->up_methods = &roothub_ctrl_methods; - break; - case UE_DIR_IN | USBROOTHUB_INTR_ENDPT: - pipe->up_methods = &xhci_root_intr_methods; - break; - default: - pipe->up_methods = NULL; - DPRINTFN(0, "bad bEndpointAddress 0x%02jx", - ed->bEndpointAddress, 0, 0, 0); - return USBD_INVAL; - } - return USBD_NORMAL_COMPLETION; - } - - usb_init_task(&xpipe->xp_async_task, xhci_pipe_restart_async_task, - pipe, USB_TASKQ_MPSAFE); - - switch (xfertype) { - case UE_CONTROL: - pipe->up_methods = &xhci_device_ctrl_methods; - break; - case UE_ISOCHRONOUS: - pipe->up_methods = &xhci_device_isoc_methods; - pipe->up_serialise = false; - xpipe->xp_isoc_next = -1; - break; - case UE_BULK: - pipe->up_methods = &xhci_device_bulk_methods; - break; - case UE_INTERRUPT: - pipe->up_methods = &xhci_device_intr_methods; - break; - default: - return USBD_IOERROR; - break; - } - - KASSERT(xs != NULL); - KASSERT(xs->xs_xr[dci] == NULL); - - /* allocate transfer ring */ - err = xhci_ring_init(sc, &xs->xs_xr[dci], XHCI_TRANSFER_RING_TRBS, - XHCI_TRB_ALIGN); - if (err) { - DPRINTFN(1, "ring alloc failed %jd", err, 0, 0, 0); - return err; - } - - if (ed->bEndpointAddress != USB_CONTROL_ENDPOINT) - return xhci_configure_endpoint(pipe); - - return USBD_NORMAL_COMPLETION; -} - -/* - * Closes pipe, called from usbd_kill_pipe via close methods. - * If the endpoint to be closed is ep0, disable_slot. - * Should be called with sc_lock held. - */ -static void -xhci_close_pipe(struct usbd_pipe *pipe) -{ - struct xhci_pipe * const xp = - container_of(pipe, struct xhci_pipe, xp_pipe); - struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); - struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; - usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc; - const u_int dci = xhci_ep_get_dci(ed); - struct xhci_soft_trb trb; - uint32_t *cp; - - XHCIHIST_FUNC(); - - usb_rem_task_wait(pipe->up_dev, &xp->xp_async_task, USB_TASKQ_HC, - &sc->sc_lock); - - if (sc->sc_dying) - return; - - /* xs is uninitialized before xhci_init_slot */ - if (xs == NULL || xs->xs_idx == 0) - return; - - XHCIHIST_CALLARGS("pipe %#jx slot %ju dci %ju", - (uintptr_t)pipe, xs->xs_idx, dci, 0); - - KASSERTMSG(!cpu_intr_p() && !cpu_softintr_p(), "called from intr ctx"); - KASSERT(mutex_owned(&sc->sc_lock)); - - if (pipe->up_dev->ud_depth == 0) - return; - - if (dci == XHCI_DCI_EP_CONTROL) { - DPRINTFN(4, "closing ep0", 0, 0, 0, 0); - /* This frees all rings */ - xhci_disable_slot(sc, xs->xs_idx); - return; - } - - if (xhci_get_epstate(sc, xs, dci) != XHCI_EPSTATE_STOPPED) - (void)xhci_stop_endpoint(pipe); - - /* - * set appropriate bit to be dropped. - * don't set DC bit to 1, otherwise all endpoints - * would be deconfigured. - */ - cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL); - cp[0] = htole32(XHCI_INCTX_0_DROP_MASK(dci)); - cp[1] = htole32(0); - - /* XXX should be most significant one, not dci? */ - cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_SLOT)); - cp[0] = htole32(XHCI_SCTX_0_CTX_NUM_SET(dci)); - - /* configure ep context performs an implicit dequeue */ - xhci_host_dequeue(xs->xs_xr[dci]); - - /* sync input contexts before they are read from memory */ - usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE); - - trb.trb_0 = xhci_slot_get_icp(sc, xs, 0); - trb.trb_2 = 0; - trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | - XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_CONFIGURE_EP); - - (void)xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT); - usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD); - - xhci_ring_free(sc, &xs->xs_xr[dci]); - xs->xs_xr[dci] = NULL; -} - -/* - * Abort transfer. Must be called with sc_lock held. Releases and - * reacquires sc_lock to sleep until hardware acknowledges abort. - */ -static void -xhci_abortx(struct usbd_xfer *xfer) -{ - XHCIHIST_FUNC(); - struct xhci_softc * const sc = XHCI_XFER2SC(xfer); - - XHCIHIST_CALLARGS("xfer %#jx pipe %#jx", - (uintptr_t)xfer, (uintptr_t)xfer->ux_pipe, 0, 0); - - KASSERT(mutex_owned(&sc->sc_lock)); - KASSERTMSG((xfer->ux_status == USBD_CANCELLED || - xfer->ux_status == USBD_TIMEOUT), - "bad abort status: %d", xfer->ux_status); - - xhci_pipe_restart(xfer->ux_pipe); - - DPRINTFN(14, "end", 0, 0, 0, 0); -} - -static void -xhci_host_dequeue(struct xhci_ring * const xr) -{ - /* When dequeueing the controller, update our struct copy too */ - memset(xr->xr_trb, 0, xr->xr_ntrb * XHCI_TRB_SIZE); - usb_syncmem(&xr->xr_dma, 0, xr->xr_ntrb * XHCI_TRB_SIZE, - BUS_DMASYNC_PREWRITE); - memset(xr->xr_cookies, 0, xr->xr_ntrb * sizeof(*xr->xr_cookies)); - - xr->xr_ep = 0; - xr->xr_cs = 1; -} - -/* - * Recover STALLed endpoint, or stop endpoint to abort a pipe. - * xHCI 1.1 sect 4.10.2.1 - * Issue RESET_EP to recover halt condition and SET_TR_DEQUEUE to remove - * all transfers on transfer ring. - */ -static void -xhci_pipe_restart(struct usbd_pipe *pipe) -{ - struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); - struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; - const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc); - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("pipe %#jx slot %ju dci %ju", - (uintptr_t)pipe, xs->xs_idx, dci, 0); - - KASSERT(xhci_polling_p(sc) || mutex_owned(&sc->sc_lock)); - - /* - * - If the endpoint is halted, indicating a stall, reset it. - * - If the endpoint is stopped, we're already good. - * - Otherwise, someone wanted to abort the pipe, so stop the - * endpoint. - * - * In any case, clear the ring. - */ - switch (xhci_get_epstate(sc, xs, dci)) { - case XHCI_EPSTATE_HALTED: - xhci_reset_endpoint(pipe); - break; - case XHCI_EPSTATE_STOPPED: - break; - default: - xhci_stop_endpoint(pipe); - break; - } - - switch (xhci_get_epstate(sc, xs, dci)) { - case XHCI_EPSTATE_STOPPED: - break; - case XHCI_EPSTATE_ERROR: - device_printf(sc->sc_dev, "endpoint 0x%x error\n", - pipe->up_endpoint->ue_edesc->bEndpointAddress); - break; - default: - device_printf(sc->sc_dev, "endpoint 0x%x failed to stop\n", - pipe->up_endpoint->ue_edesc->bEndpointAddress); - } - - xhci_set_dequeue(pipe); - - DPRINTFN(4, "ends", 0, 0, 0, 0); -} - -static void -xhci_pipe_restart_async_task(void *cookie) -{ - struct usbd_pipe * const pipe = cookie; - struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); - struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; - const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc); - struct xhci_ring * const tr = xs->xs_xr[dci]; - struct usbd_xfer *xfer; - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("sc=%#jx pipe=%#jx", - (uintptr_t)sc, (uintptr_t)pipe, 0, 0); - - mutex_enter(&sc->sc_lock); - - xhci_pipe_restart(pipe); - - /* - * We halted our own queue because it stalled. Mark it no - * longer halted and start issuing queued transfers again. - */ - tr->is_halted = false; - xfer = SIMPLEQ_FIRST(&pipe->up_queue); - if (xfer) { - /* - * If the first xfer of the queue is not in progress, - * though, there may be a concurrent software abort - * that has already cancelled it and is now in the - * middle of a concurrent xhci_pipe_restart waiting to - * reacquire the pipe (bus) lock. So only restart the - * xfer if it's still USBD_IN_PROGRESS. - * - * Either way, xfers on the queue can't be in - * USBD_NOT_STARTED. - */ - KASSERT(xfer->ux_status != USBD_NOT_STARTED); - if (xfer->ux_status == USBD_IN_PROGRESS) { - if (pipe->up_methods->upm_start != NULL) - (*pipe->up_methods->upm_start)(xfer); - } else { - DPRINTF("pipe restart race xfer=%#jx status=%jd", - (uintptr_t)xfer, xfer->ux_status, 0, 0); - } - } - - mutex_exit(&sc->sc_lock); -} - -static void -xhci_pipe_restart_async(struct usbd_pipe *pipe) -{ - struct xhci_pipe * const xp = - container_of(pipe, struct xhci_pipe, xp_pipe); - struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); - struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; - const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc); - struct xhci_ring * const tr = xs->xs_xr[dci]; - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("pipe %#jx", (uintptr_t)pipe, 0, 0, 0); - - KASSERT(xhci_polling_p(sc) || mutex_owned(&sc->sc_lock)); - - tr->is_halted = true; - usb_add_task(pipe->up_dev, &xp->xp_async_task, USB_TASKQ_HC); - - DPRINTFN(4, "ends", 0, 0, 0, 0); -} - -/* Process roothub port status/change events and notify to uhub_intr. */ -static void -xhci_rhpsc(struct xhci_softc * const sc, u_int ctlrport) -{ - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("xhci%jd: port %ju status change", - device_unit(sc->sc_dev), ctlrport, 0, 0); - - if (ctlrport > sc->sc_maxports) - return; - - const size_t bn = xhci_ctlrport2bus(sc, ctlrport); - const size_t rhp = xhci_ctlrport2rhport(sc, ctlrport); - struct usbd_xfer * const xfer = sc->sc_intrxfer[bn]; - - DPRINTFN(4, "xhci%jd: bus %jd bp %ju xfer %#jx status change", - device_unit(sc->sc_dev), bn, rhp, (uintptr_t)xfer); - - if (xfer == NULL) - return; - KASSERT(xfer->ux_status == USBD_IN_PROGRESS); - - uint8_t *p = xfer->ux_buf; - if (!xhci_polling_p(sc) || !sc->sc_intrxfer_deferred[bn]) - memset(p, 0, xfer->ux_length); - p[rhp / NBBY] |= 1 << (rhp % NBBY); - xfer->ux_actlen = xfer->ux_length; - xfer->ux_status = USBD_NORMAL_COMPLETION; - if (xhci_polling_p(sc)) - sc->sc_intrxfer_deferred[bn] = true; - else - usb_transfer_complete(xfer); -} - -/* Process Transfer Events */ -static void -xhci_event_transfer(struct xhci_softc * const sc, - const struct xhci_trb * const trb) -{ - uint64_t trb_0; - uint32_t trb_2, trb_3; - uint8_t trbcode; - u_int slot, dci; - struct xhci_slot *xs; - struct xhci_ring *xr; - struct xhci_xfer *xx; - struct usbd_xfer *xfer; - usbd_status err; - - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - - trb_0 = le64toh(trb->trb_0); - trb_2 = le32toh(trb->trb_2); - trb_3 = le32toh(trb->trb_3); - trbcode = XHCI_TRB_2_ERROR_GET(trb_2); - slot = XHCI_TRB_3_SLOT_GET(trb_3); - dci = XHCI_TRB_3_EP_GET(trb_3); - xs = &sc->sc_slots[slot]; - xr = xs->xs_xr[dci]; - - /* sanity check */ - KASSERT(xr != NULL); - KASSERTMSG(xs->xs_idx != 0 && xs->xs_idx <= sc->sc_maxslots, - "invalid xs_idx %u slot %u", xs->xs_idx, slot); - - int idx = 0; - if ((trb_3 & XHCI_TRB_3_ED_BIT) == 0) { - if (xhci_trb_get_idx(xr, trb_0, &idx)) { - DPRINTFN(0, "invalid trb_0 %#jx", trb_0, 0, 0, 0); - return; - } - xx = xr->xr_cookies[idx]; - - /* clear cookie of consumed TRB */ - xr->xr_cookies[idx] = NULL; - - /* - * xx is NULL if pipe is opened but xfer is not started. - * It happens when stopping idle pipe. - */ - if (xx == NULL || trbcode == XHCI_TRB_ERROR_LENGTH) { - DPRINTFN(1, "Ignore #%ju: cookie %#jx cc %ju dci %ju", - idx, (uintptr_t)xx, trbcode, dci); - DPRINTFN(1, " orig TRB %#jx type %ju", trb_0, - XHCI_TRB_3_TYPE_GET(le32toh(xr->xr_trb[idx].trb_3)), - 0, 0); - return; - } - } else { - /* When ED != 0, trb_0 is virtual addr of struct xhci_xfer. */ - xx = (void *)(uintptr_t)(trb_0 & ~0x3); - } - /* XXX this may not happen */ - if (xx == NULL) { - DPRINTFN(1, "xfer done: xx is NULL", 0, 0, 0, 0); - return; - } - xfer = &xx->xx_xfer; - /* XXX this may happen when detaching */ - if (xfer == NULL) { - DPRINTFN(1, "xx(%#jx)->xx_xfer is NULL trb_0 %#jx", - (uintptr_t)xx, trb_0, 0, 0); - return; - } - DPRINTFN(14, "xfer %#jx", (uintptr_t)xfer, 0, 0, 0); - /* XXX I dunno why this happens */ - KASSERTMSG(xfer->ux_pipe != NULL, "xfer(%p)->ux_pipe is NULL", xfer); - - if (!xfer->ux_pipe->up_repeat && - SIMPLEQ_EMPTY(&xfer->ux_pipe->up_queue)) { - DPRINTFN(1, "xfer(%#jx)->pipe not queued", (uintptr_t)xfer, - 0, 0, 0); - return; - } - - const uint8_t xfertype = - UE_GET_XFERTYPE(xfer->ux_pipe->up_endpoint->ue_edesc->bmAttributes); - - /* 4.11.5.2 Event Data TRB */ - if ((trb_3 & XHCI_TRB_3_ED_BIT) != 0) { - DPRINTFN(14, "transfer Event Data: 0x%016jx 0x%08jx" - " %02jx", trb_0, XHCI_TRB_2_REM_GET(trb_2), trbcode, 0); - if ((trb_0 & 0x3) == 0x3) { - xfer->ux_actlen = XHCI_TRB_2_REM_GET(trb_2); - } - } - - switch (trbcode) { - case XHCI_TRB_ERROR_SHORT_PKT: - case XHCI_TRB_ERROR_SUCCESS: - /* - * A ctrl transfer can generate two events if it has a Data - * stage. A short data stage can be OK and should not - * complete the transfer as the status stage needs to be - * performed. - * - * Note: Data and Status stage events point at same xfer. - * ux_actlen and ux_dmabuf will be passed to - * usb_transfer_complete after the Status stage event. - * - * It can be distinguished which stage generates the event: - * + by checking least 3 bits of trb_0 if ED==1. - * (see xhci_device_ctrl_start). - * + by checking the type of original TRB if ED==0. - * - * In addition, intr, bulk, and isoc transfer currently - * consists of single TD, so the "skip" is not needed. - * ctrl xfer uses EVENT_DATA, and others do not. - * Thus driver can switch the flow by checking ED bit. - */ - if (xfertype == UE_ISOCHRONOUS) { - xfer->ux_frlengths[xx->xx_isoc_done] -= - XHCI_TRB_2_REM_GET(trb_2); - xfer->ux_actlen += xfer->ux_frlengths[xx->xx_isoc_done]; - } else if ((trb_3 & XHCI_TRB_3_ED_BIT) == 0) { - if (xfer->ux_actlen == 0) - xfer->ux_actlen = xfer->ux_length - - XHCI_TRB_2_REM_GET(trb_2); - if (XHCI_TRB_3_TYPE_GET(le32toh(xr->xr_trb[idx].trb_3)) - == XHCI_TRB_TYPE_DATA_STAGE) { - return; - } - } else if ((trb_0 & 0x3) == 0x3) { - return; - } - err = USBD_NORMAL_COMPLETION; - break; - case XHCI_TRB_ERROR_STOPPED: - case XHCI_TRB_ERROR_LENGTH: - case XHCI_TRB_ERROR_STOPPED_SHORT: - err = USBD_IOERROR; - break; - case XHCI_TRB_ERROR_STALL: - case XHCI_TRB_ERROR_BABBLE: - DPRINTFN(1, "ERR %ju slot %ju dci %ju", trbcode, slot, dci, 0); - xhci_pipe_restart_async(xfer->ux_pipe); - err = USBD_STALLED; - break; - default: - DPRINTFN(1, "ERR %ju slot %ju dci %ju", trbcode, slot, dci, 0); - err = USBD_IOERROR; - break; - } - - if (xfertype == UE_ISOCHRONOUS) { - switch (trbcode) { - case XHCI_TRB_ERROR_SHORT_PKT: - case XHCI_TRB_ERROR_SUCCESS: - break; - case XHCI_TRB_ERROR_MISSED_SERVICE: - case XHCI_TRB_ERROR_RING_UNDERRUN: - case XHCI_TRB_ERROR_RING_OVERRUN: - default: - xfer->ux_frlengths[xx->xx_isoc_done] = 0; - break; - } - if (++xx->xx_isoc_done < xfer->ux_nframes) - return; - } - - /* - * If next event will be from zero-length packet, - * suppress notification of first event. - */ - if (xfertype == UE_BULK && - err == USBD_NORMAL_COMPLETION && - (xfer->ux_flags & USBD_FORCE_SHORT_XFER) && - XHCI_TRB_2_REM_GET(le32toh(xr->xr_trb[idx].trb_2)) != 0) { - DPRINTFN(100, "short xfer %#jx: suppress notification status " - "%ju pipe %#jx", (uintptr_t)xfer, xfer->ux_status, - (uintptr_t)xfer->ux_pipe, 0); - return; - } - - if ((trb_3 & XHCI_TRB_3_ED_BIT) == 0 || - (trb_0 & 0x3) == 0x0) { - /* - * Try to claim this xfer for completion. If it has - * already completed or aborted, drop it on the floor. - */ - if (!usbd_xfer_trycomplete(xfer)) - return; - - /* Set the status. */ - xfer->ux_status = err; - - usb_transfer_complete(xfer); - } -} - -/* Process Command complete events */ -static void -xhci_event_cmd(struct xhci_softc * const sc, const struct xhci_trb * const trb) -{ - uint64_t trb_0; - uint32_t trb_2, trb_3; - - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - - KASSERT(xhci_polling_p(sc) || mutex_owned(&sc->sc_lock)); - - trb_0 = le64toh(trb->trb_0); - trb_2 = le32toh(trb->trb_2); - trb_3 = le32toh(trb->trb_3); - - if (trb_0 == sc->sc_command_addr) { - sc->sc_resultpending = false; - - sc->sc_result_trb.trb_0 = trb_0; - sc->sc_result_trb.trb_2 = trb_2; - sc->sc_result_trb.trb_3 = trb_3; - if (XHCI_TRB_2_ERROR_GET(trb_2) != - XHCI_TRB_ERROR_SUCCESS) { - DPRINTFN(1, "command completion " - "failure: 0x%016jx 0x%08jx 0x%08jx", - trb_0, trb_2, trb_3, 0); - } - cv_signal(&sc->sc_command_cv); - } else { - DPRINTFN(1, "spurious event: %#jx 0x%016jx " - "0x%08jx 0x%08jx", (uintptr_t)trb, trb_0, trb_2, trb_3); - } -} - -/* - * Process events. - * called from xhci_softintr - */ -static void -xhci_handle_event(struct xhci_softc * const sc, - const struct xhci_trb * const trb) -{ - uint64_t trb_0; - uint32_t trb_2, trb_3; - - XHCIHIST_FUNC(); - - trb_0 = le64toh(trb->trb_0); - trb_2 = le32toh(trb->trb_2); - trb_3 = le32toh(trb->trb_3); - - XHCIHIST_CALLARGS("event: %#jx 0x%016jx 0x%08jx 0x%08jx", - (uintptr_t)trb, trb_0, trb_2, trb_3); - - /* - * 4.11.3.1, 6.4.2.1 - * TRB Pointer is invalid for these completion codes. - */ - switch (XHCI_TRB_2_ERROR_GET(trb_2)) { - case XHCI_TRB_ERROR_RING_UNDERRUN: - case XHCI_TRB_ERROR_RING_OVERRUN: - case XHCI_TRB_ERROR_VF_RING_FULL: - return; - default: - if (trb_0 == 0) { - return; - } - break; - } - - switch (XHCI_TRB_3_TYPE_GET(trb_3)) { - case XHCI_TRB_EVENT_TRANSFER: - xhci_event_transfer(sc, trb); - break; - case XHCI_TRB_EVENT_CMD_COMPLETE: - xhci_event_cmd(sc, trb); - break; - case XHCI_TRB_EVENT_PORT_STS_CHANGE: - xhci_rhpsc(sc, (uint32_t)((trb_0 >> 24) & 0xff)); - break; - default: - break; - } -} - -static void -xhci_softintr(void *v) -{ - struct usbd_bus * const bus = v; - struct xhci_softc * const sc = XHCI_BUS2SC(bus); - struct xhci_ring * const er = sc->sc_er; - struct xhci_trb *trb; - int i, j, k, bn; - - XHCIHIST_FUNC(); - - KASSERT(xhci_polling_p(sc) || mutex_owned(&sc->sc_lock)); - - i = er->xr_ep; - j = er->xr_cs; - - XHCIHIST_CALLARGS("er: xr_ep %jd xr_cs %jd", i, j, 0, 0); - - /* - * Handle deferred root intr xfer, in case we just switched off - * polling. It's not safe to complete root intr xfers while - * polling -- too much kernel machinery gets involved. - */ - if (!xhci_polling_p(sc)) { - for (bn = 0; bn < 2; bn++) { - if (__predict_false(sc->sc_intrxfer_deferred[bn])) { - sc->sc_intrxfer_deferred[bn] = false; - usb_transfer_complete(sc->sc_intrxfer[bn]); - } - } - } - - while (1) { - usb_syncmem(&er->xr_dma, XHCI_TRB_SIZE * i, XHCI_TRB_SIZE, - BUS_DMASYNC_POSTREAD); - trb = &er->xr_trb[i]; - k = (le32toh(trb->trb_3) & XHCI_TRB_3_CYCLE_BIT) ? 1 : 0; - - if (j != k) - break; - - xhci_handle_event(sc, trb); - - i++; - if (i == er->xr_ntrb) { - i = 0; - j ^= 1; - } - } - - er->xr_ep = i; - er->xr_cs = j; - - xhci_rt_write_8(sc, XHCI_ERDP(0), xhci_ring_trbp(er, er->xr_ep) | - XHCI_ERDP_BUSY); - - DPRINTFN(16, "ends", 0, 0, 0, 0); - - return; -} - -static void -xhci_poll(struct usbd_bus *bus) -{ - struct xhci_softc * const sc = XHCI_BUS2SC(bus); - - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - - mutex_enter(&sc->sc_intr_lock); - int ret = xhci_intr1(sc); - if (ret) { - xhci_softintr(bus); - } - mutex_exit(&sc->sc_intr_lock); - - return; -} - -static struct usbd_xfer * -xhci_allocx(struct usbd_bus *bus, unsigned int nframes) -{ - struct xhci_softc * const sc = XHCI_BUS2SC(bus); - struct xhci_xfer *xx; - u_int ntrbs; - - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - - ntrbs = uimax(3, nframes); - const size_t trbsz = sizeof(*xx->xx_trb) * ntrbs; - - xx = pool_cache_get(sc->sc_xferpool, PR_WAITOK); - if (xx != NULL) { - memset(xx, 0, sizeof(*xx)); - if (ntrbs > 0) { - xx->xx_trb = kmem_alloc(trbsz, KM_SLEEP); - xx->xx_ntrb = ntrbs; - } -#ifdef DIAGNOSTIC - xx->xx_xfer.ux_state = XFER_BUSY; -#endif - } - - return &xx->xx_xfer; -} - -static void -xhci_freex(struct usbd_bus *bus, struct usbd_xfer *xfer) -{ - struct xhci_softc * const sc = XHCI_BUS2SC(bus); - struct xhci_xfer * const xx = XHCI_XFER2XXFER(xfer); - - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - -#ifdef DIAGNOSTIC - if (xfer->ux_state != XFER_BUSY && - xfer->ux_status != USBD_NOT_STARTED) { - DPRINTFN(0, "xfer=%#jx not busy, 0x%08jx", - (uintptr_t)xfer, xfer->ux_state, 0, 0); - } - xfer->ux_state = XFER_FREE; -#endif - if (xx->xx_ntrb > 0) { - kmem_free(xx->xx_trb, xx->xx_ntrb * sizeof(*xx->xx_trb)); - xx->xx_trb = NULL; - xx->xx_ntrb = 0; - } - pool_cache_put(sc->sc_xferpool, xx); -} - -static bool -xhci_dying(struct usbd_bus *bus) -{ - struct xhci_softc * const sc = XHCI_BUS2SC(bus); - - return sc->sc_dying; -} - -static void -xhci_get_lock(struct usbd_bus *bus, kmutex_t **lock) -{ - struct xhci_softc * const sc = XHCI_BUS2SC(bus); - - *lock = &sc->sc_lock; -} - -extern uint32_t usb_cookie_no; - -/* - * xHCI 4.3 - * Called when uhub_explore finds a new device (via usbd_new_device). - * Port initialization and speed detection (4.3.1) are already done in uhub.c. - * This function does: - * Allocate and construct dev structure of default endpoint (ep0). - * Allocate and open pipe of ep0. - * Enable slot and initialize slot context. - * Set Address. - * Read initial device descriptor. - * Determine initial MaxPacketSize (mps) by speed. - * Read full device descriptor. - * Register this device. - * Finally state of device transitions ADDRESSED. - */ -static usbd_status -xhci_new_device(device_t parent, struct usbd_bus *bus, int depth, - int speed, int port, struct usbd_port *up) -{ - struct xhci_softc * const sc = XHCI_BUS2SC(bus); - struct usbd_device *dev; - usbd_status err; - usb_device_descriptor_t *dd; - struct xhci_slot *xs; - uint32_t *cp; - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("port %ju depth %ju speed %ju up %#jx", - port, depth, speed, (uintptr_t)up); - - KASSERT(KERNEL_LOCKED_P()); - - dev = kmem_zalloc(sizeof(*dev), KM_SLEEP); - dev->ud_bus = bus; - dev->ud_quirks = &usbd_no_quirk; - dev->ud_addr = 0; - dev->ud_ddesc.bMaxPacketSize = 0; - dev->ud_config = USB_UNCONFIG_NO; - dev->ud_configidx = USB_UNCONFIG_INDEX; - dev->ud_depth = depth; - dev->ud_powersrc = up; - dev->ud_myhub = up->up_parent; - dev->ud_speed = speed; - dev->ud_langid = USBD_NOLANG; - dev->ud_cookie.cookie = ++usb_cookie_no; - - /* Set up default endpoint handle. */ - dev->ud_ep0.ue_edesc = &dev->ud_ep0desc; - /* doesn't matter, just don't let it uninitialized */ - dev->ud_ep0.ue_toggle = 0; - - /* Set up default endpoint descriptor. */ - dev->ud_ep0desc.bLength = USB_ENDPOINT_DESCRIPTOR_SIZE; - dev->ud_ep0desc.bDescriptorType = UDESC_ENDPOINT; - dev->ud_ep0desc.bEndpointAddress = USB_CONTROL_ENDPOINT; - dev->ud_ep0desc.bmAttributes = UE_CONTROL; - dev->ud_ep0desc.bInterval = 0; - - /* 4.3, 4.8.2.1 */ - switch (speed) { - case USB_SPEED_SUPER: - case USB_SPEED_SUPER_PLUS: - USETW(dev->ud_ep0desc.wMaxPacketSize, USB_3_MAX_CTRL_PACKET); - break; - case USB_SPEED_FULL: - /* XXX using 64 as initial mps of ep0 in FS */ - case USB_SPEED_HIGH: - USETW(dev->ud_ep0desc.wMaxPacketSize, USB_2_MAX_CTRL_PACKET); - break; - case USB_SPEED_LOW: - default: - USETW(dev->ud_ep0desc.wMaxPacketSize, USB_MAX_IPACKET); - break; - } - - up->up_dev = dev; - - dd = &dev->ud_ddesc; - - if (depth == 0 && port == 0) { - KASSERT(bus->ub_devices[USB_ROOTHUB_INDEX] == NULL); - bus->ub_devices[USB_ROOTHUB_INDEX] = dev; - - /* Establish the default pipe. */ - err = usbd_setup_pipe(dev, 0, &dev->ud_ep0, - USBD_DEFAULT_INTERVAL, &dev->ud_pipe0); - if (err) { - DPRINTFN(1, "setup default pipe failed %jd", err,0,0,0); - goto bad; - } - err = usbd_get_initial_ddesc(dev, dd); - if (err) { - DPRINTFN(1, "get_initial_ddesc %ju", err, 0, 0, 0); - goto bad; - } - } else { - uint8_t slot = 0; - - /* 4.3.2 */ - err = xhci_enable_slot(sc, &slot); - if (err) { - DPRINTFN(1, "enable slot %ju", err, 0, 0, 0); - goto bad; - } - - xs = &sc->sc_slots[slot]; - dev->ud_hcpriv = xs; - - /* 4.3.3 initialize slot structure */ - err = xhci_init_slot(dev, slot); - if (err) { - DPRINTFN(1, "init slot %ju", err, 0, 0, 0); - dev->ud_hcpriv = NULL; - /* - * We have to disable_slot here because - * xs->xs_idx == 0 when xhci_init_slot fails, - * in that case usbd_remove_dev won't work. - */ - mutex_enter(&sc->sc_lock); - xhci_disable_slot(sc, slot); - mutex_exit(&sc->sc_lock); - goto bad; - } - - /* - * We have to establish the default pipe _after_ slot - * structure has been prepared. - */ - err = usbd_setup_pipe(dev, 0, &dev->ud_ep0, - USBD_DEFAULT_INTERVAL, &dev->ud_pipe0); - if (err) { - DPRINTFN(1, "setup default pipe failed %jd", err, 0, 0, - 0); - goto bad; - } - - /* 4.3.4 Address Assignment */ - err = xhci_set_address(dev, slot, false); - if (err) { - DPRINTFN(1, "failed! to set address: %ju", err, 0, 0, 0); - goto bad; - } - - /* Allow device time to set new address */ - usbd_delay_ms(dev, USB_SET_ADDRESS_SETTLE); - - usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD); - cp = xhci_slot_get_dcv(sc, xs, XHCI_DCI_SLOT); - HEXDUMP("slot context", cp, sc->sc_ctxsz); - uint8_t addr = XHCI_SCTX_3_DEV_ADDR_GET(le32toh(cp[3])); - DPRINTFN(4, "device address %ju", addr, 0, 0, 0); - /* - * XXX ensure we know when the hardware does something - * we can't yet cope with - */ - KASSERTMSG(addr >= 1 && addr <= 127, "addr %d", addr); - dev->ud_addr = addr; - - KASSERTMSG(bus->ub_devices[usb_addr2dindex(dev->ud_addr)] == NULL, - "addr %d already allocated", dev->ud_addr); - /* - * The root hub is given its own slot - */ - bus->ub_devices[usb_addr2dindex(dev->ud_addr)] = dev; - - err = usbd_get_initial_ddesc(dev, dd); - if (err) { - DPRINTFN(1, "get_initial_ddesc %ju", err, 0, 0, 0); - goto bad; - } - - /* 4.8.2.1 */ - if (USB_IS_SS(speed)) { - if (dd->bMaxPacketSize != 9) { - printf("%s: invalid mps 2^%u for SS ep0," - " using 512\n", - device_xname(sc->sc_dev), - dd->bMaxPacketSize); - dd->bMaxPacketSize = 9; - } - USETW(dev->ud_ep0desc.wMaxPacketSize, - (1 << dd->bMaxPacketSize)); - } else - USETW(dev->ud_ep0desc.wMaxPacketSize, - dd->bMaxPacketSize); - DPRINTFN(4, "bMaxPacketSize %ju", dd->bMaxPacketSize, 0, 0, 0); - err = xhci_update_ep0_mps(sc, xs, - UGETW(dev->ud_ep0desc.wMaxPacketSize)); - if (err) { - DPRINTFN(1, "update mps of ep0 %ju", err, 0, 0, 0); - goto bad; - } - } - - err = usbd_reload_device_desc(dev); - if (err) { - DPRINTFN(1, "reload desc %ju", err, 0, 0, 0); - goto bad; - } - - DPRINTFN(1, "adding unit addr=%jd, rev=%02jx,", - dev->ud_addr, UGETW(dd->bcdUSB), 0, 0); - DPRINTFN(1, " class=%jd, subclass=%jd, protocol=%jd,", - dd->bDeviceClass, dd->bDeviceSubClass, - dd->bDeviceProtocol, 0); - DPRINTFN(1, " mps=%jd, len=%jd, noconf=%jd, speed=%jd", - dd->bMaxPacketSize, dd->bLength, dd->bNumConfigurations, - dev->ud_speed); - - usbd_get_device_strings(dev); - - usbd_add_dev_event(USB_EVENT_DEVICE_ATTACH, dev); - - if (depth == 0 && port == 0) { - usbd_attach_roothub(parent, dev); - DPRINTFN(1, "root hub %#jx", (uintptr_t)dev, 0, 0, 0); - return USBD_NORMAL_COMPLETION; - } - - err = usbd_probe_and_attach(parent, dev, port, dev->ud_addr); - bad: - if (err != USBD_NORMAL_COMPLETION) { - if (depth == 0 && port == 0 && dev->ud_pipe0) - usbd_kill_pipe(dev->ud_pipe0); - usbd_remove_device(dev, up); - } - - return err; -} - -static usbd_status -xhci_ring_init(struct xhci_softc * const sc, struct xhci_ring **xrp, - size_t ntrb, size_t align) -{ - size_t size = ntrb * XHCI_TRB_SIZE; - struct xhci_ring *xr; - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("xr %#jx ntrb %#jx align %#jx", - (uintptr_t)*xrp, ntrb, align, 0); - - xr = kmem_zalloc(sizeof(struct xhci_ring), KM_SLEEP); - DPRINTFN(1, "ring %#jx", (uintptr_t)xr, 0, 0, 0); - - int err = usb_allocmem(sc->sc_bus.ub_dmatag, size, align, - USBMALLOC_ZERO, &xr->xr_dma); - if (err) { - kmem_free(xr, sizeof(struct xhci_ring)); - DPRINTFN(1, "alloc xr_dma failed %jd", err, 0, 0, 0); - return err; - } - mutex_init(&xr->xr_lock, MUTEX_DEFAULT, IPL_SOFTUSB); - xr->xr_cookies = kmem_zalloc(sizeof(*xr->xr_cookies) * ntrb, KM_SLEEP); - xr->xr_trb = xhci_ring_trbv(xr, 0); - xr->xr_ntrb = ntrb; - xr->is_halted = false; - xhci_host_dequeue(xr); - *xrp = xr; - - return USBD_NORMAL_COMPLETION; -} - -static void -xhci_ring_free(struct xhci_softc * const sc, struct xhci_ring ** const xr) -{ - if (*xr == NULL) - return; - - usb_freemem(&(*xr)->xr_dma); - mutex_destroy(&(*xr)->xr_lock); - kmem_free((*xr)->xr_cookies, - sizeof(*(*xr)->xr_cookies) * (*xr)->xr_ntrb); - kmem_free(*xr, sizeof(struct xhci_ring)); - *xr = NULL; -} - -static void -xhci_ring_put(struct xhci_softc * const sc, struct xhci_ring * const xr, - void *cookie, struct xhci_soft_trb * const trbs, size_t ntrbs) -{ - size_t i; - u_int ri; - u_int cs; - uint64_t parameter; - uint32_t status; - uint32_t control; - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("%#jx xr_ep %#jx xr_cs %ju", - (uintptr_t)xr, xr->xr_ep, xr->xr_cs, 0); - - KASSERTMSG(ntrbs < xr->xr_ntrb, "ntrbs %zu, xr->xr_ntrb %u", - ntrbs, xr->xr_ntrb); - for (i = 0; i < ntrbs; i++) { - DPRINTFN(12, "xr %#jx trbs %#jx num %ju", (uintptr_t)xr, - (uintptr_t)trbs, i, 0); - DPRINTFN(12, " 0x%016jx 0x%08jx 0x%08jx", - trbs[i].trb_0, trbs[i].trb_2, trbs[i].trb_3, 0); - KASSERTMSG(XHCI_TRB_3_TYPE_GET(trbs[i].trb_3) != - XHCI_TRB_TYPE_LINK, "trbs[%zu].trb3 %#x", i, trbs[i].trb_3); - } - - ri = xr->xr_ep; - cs = xr->xr_cs; - - /* - * Although the xhci hardware can do scatter/gather dma from - * arbitrary sized buffers, there is a non-obvious restriction - * that a LINK trb is only allowed at the end of a burst of - * transfers - which might be 16kB. - * Arbitrary aligned LINK trb definitely fail on Ivy bridge. - * The simple solution is not to allow a LINK trb in the middle - * of anything - as here. - * XXX: (dsl) There are xhci controllers out there (eg some made by - * ASMedia) that seem to lock up if they process a LINK trb but - * cannot process the linked-to trb yet. - * The code should write the 'cycle' bit on the link trb AFTER - * adding the other trb. - */ - u_int firstep = xr->xr_ep; - u_int firstcs = xr->xr_cs; - - for (i = 0; i < ntrbs; ) { - u_int oldri = ri; - u_int oldcs = cs; - - if (ri >= (xr->xr_ntrb - 1)) { - /* Put Link TD at the end of ring */ - parameter = xhci_ring_trbp(xr, 0); - status = 0; - control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK) | - XHCI_TRB_3_TC_BIT; - xr->xr_cookies[ri] = NULL; - xr->xr_ep = 0; - xr->xr_cs ^= 1; - ri = xr->xr_ep; - cs = xr->xr_cs; - } else { - parameter = trbs[i].trb_0; - status = trbs[i].trb_2; - control = trbs[i].trb_3; - - xr->xr_cookies[ri] = cookie; - ri++; - i++; - } - /* - * If this is a first TRB, mark it invalid to prevent - * xHC from running it immediately. - */ - if (oldri == firstep) { - if (oldcs) { - control &= ~XHCI_TRB_3_CYCLE_BIT; - } else { - control |= XHCI_TRB_3_CYCLE_BIT; - } - } else { - if (oldcs) { - control |= XHCI_TRB_3_CYCLE_BIT; - } else { - control &= ~XHCI_TRB_3_CYCLE_BIT; - } - } - xhci_trb_put(&xr->xr_trb[oldri], parameter, status, control); - usb_syncmem(&xr->xr_dma, XHCI_TRB_SIZE * oldri, - XHCI_TRB_SIZE * 1, BUS_DMASYNC_PREWRITE); - } - - /* Now invert cycle bit of first TRB */ - if (firstcs) { - xr->xr_trb[firstep].trb_3 |= htole32(XHCI_TRB_3_CYCLE_BIT); - } else { - xr->xr_trb[firstep].trb_3 &= ~htole32(XHCI_TRB_3_CYCLE_BIT); - } - usb_syncmem(&xr->xr_dma, XHCI_TRB_SIZE * firstep, - XHCI_TRB_SIZE * 1, BUS_DMASYNC_PREWRITE); - - xr->xr_ep = ri; - xr->xr_cs = cs; - - DPRINTFN(12, "%#jx xr_ep %#jx xr_cs %ju", (uintptr_t)xr, xr->xr_ep, - xr->xr_cs, 0); -} - -static inline void -xhci_ring_put_xfer(struct xhci_softc * const sc, struct xhci_ring * const tr, - struct xhci_xfer *xx, u_int ntrb) -{ - KASSERT(ntrb <= xx->xx_ntrb); - xhci_ring_put(sc, tr, xx, xx->xx_trb, ntrb); -} - -/* - * Stop execution commands, purge all commands on command ring, and - * rewind dequeue pointer. - */ -static void -xhci_abort_command(struct xhci_softc *sc) -{ - struct xhci_ring * const cr = sc->sc_cr; - uint64_t crcr; - int i; - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("command %#jx timeout, aborting", - sc->sc_command_addr, 0, 0, 0); - - mutex_enter(&cr->xr_lock); - - /* 4.6.1.2 Aborting a Command */ - crcr = xhci_op_read_8(sc, XHCI_CRCR); - xhci_op_write_8(sc, XHCI_CRCR, crcr | XHCI_CRCR_LO_CA); - - for (i = 0; i < 500; i++) { - crcr = xhci_op_read_8(sc, XHCI_CRCR); - if ((crcr & XHCI_CRCR_LO_CRR) == 0) - break; - usb_delay_ms(&sc->sc_bus, 1); - } - if ((crcr & XHCI_CRCR_LO_CRR) != 0) { - DPRINTFN(1, "Command Abort timeout", 0, 0, 0, 0); - /* reset HC here? */ - } - - /* reset command ring dequeue pointer */ - cr->xr_ep = 0; - cr->xr_cs = 1; - xhci_op_write_8(sc, XHCI_CRCR, xhci_ring_trbp(cr, 0) | cr->xr_cs); - - mutex_exit(&cr->xr_lock); -} - -/* - * Put a command on command ring, ring bell, set timer, and cv_timedwait. - * Command completion is notified by cv_signal from xhci_event_cmd() - * (called from xhci_softint), or timed-out. - * The completion code is copied to sc->sc_result_trb in xhci_event_cmd(), - * then do_command examines it. - */ -static usbd_status -xhci_do_command_locked(struct xhci_softc * const sc, - struct xhci_soft_trb * const trb, int timeout) -{ - struct xhci_ring * const cr = sc->sc_cr; - usbd_status err; - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("input: 0x%016jx 0x%08jx 0x%08jx", - trb->trb_0, trb->trb_2, trb->trb_3, 0); - - KASSERTMSG(!cpu_intr_p() && !cpu_softintr_p(), "called from intr ctx"); - KASSERT(mutex_owned(&sc->sc_lock)); - - while (sc->sc_command_addr != 0 || - (sc->sc_suspender != NULL && sc->sc_suspender != curlwp)) - cv_wait(&sc->sc_cmdbusy_cv, &sc->sc_lock); - if (sc->sc_suspendresume_failed) - return USBD_IOERROR; - - /* - * If enqueue pointer points at last of ring, it's Link TRB, - * command TRB will be stored in 0th TRB. - */ - if (cr->xr_ep == cr->xr_ntrb - 1) - sc->sc_command_addr = xhci_ring_trbp(cr, 0); - else - sc->sc_command_addr = xhci_ring_trbp(cr, cr->xr_ep); - - sc->sc_resultpending = true; - - mutex_enter(&cr->xr_lock); - xhci_ring_put(sc, cr, NULL, trb, 1); - mutex_exit(&cr->xr_lock); - - xhci_db_write_4(sc, XHCI_DOORBELL(0), 0); - - while (sc->sc_resultpending) { - if (cv_timedwait(&sc->sc_command_cv, &sc->sc_lock, - MAX(1, mstohz(timeout))) == EWOULDBLOCK) { - xhci_abort_command(sc); - err = USBD_TIMEOUT; - goto timedout; - } - } - - trb->trb_0 = sc->sc_result_trb.trb_0; - trb->trb_2 = sc->sc_result_trb.trb_2; - trb->trb_3 = sc->sc_result_trb.trb_3; - - DPRINTFN(12, "output: 0x%016jx 0x%08jx 0x%08jx", - trb->trb_0, trb->trb_2, trb->trb_3, 0); - - switch (XHCI_TRB_2_ERROR_GET(trb->trb_2)) { - case XHCI_TRB_ERROR_SUCCESS: - err = USBD_NORMAL_COMPLETION; - break; - default: - case 192 ... 223: - DPRINTFN(5, "error %#jx", - XHCI_TRB_2_ERROR_GET(trb->trb_2), 0, 0, 0); - err = USBD_IOERROR; - break; - case 224 ... 255: - err = USBD_NORMAL_COMPLETION; - break; - } - -timedout: - sc->sc_resultpending = false; - sc->sc_command_addr = 0; - cv_broadcast(&sc->sc_cmdbusy_cv); - - return err; -} - -static usbd_status -xhci_do_command(struct xhci_softc * const sc, struct xhci_soft_trb * const trb, - int timeout) -{ - - mutex_enter(&sc->sc_lock); - usbd_status ret = xhci_do_command_locked(sc, trb, timeout); - mutex_exit(&sc->sc_lock); - - return ret; -} - -static usbd_status -xhci_enable_slot(struct xhci_softc * const sc, uint8_t * const slotp) -{ - struct xhci_soft_trb trb; - usbd_status err; - - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - - trb.trb_0 = 0; - trb.trb_2 = 0; - trb.trb_3 = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ENABLE_SLOT); - - err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); - if (err != USBD_NORMAL_COMPLETION) { - return err; - } - - *slotp = XHCI_TRB_3_SLOT_GET(trb.trb_3); - - return err; -} - -/* - * xHCI 4.6.4 - * Deallocate ring and device/input context DMA buffers, and disable_slot. - * All endpoints in the slot should be stopped. - * Should be called with sc_lock held. - */ -static usbd_status -xhci_disable_slot(struct xhci_softc * const sc, uint8_t slot) -{ - struct xhci_soft_trb trb; - struct xhci_slot *xs; - usbd_status err; - - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - - if (sc->sc_dying) - return USBD_IOERROR; - - trb.trb_0 = 0; - trb.trb_2 = 0; - trb.trb_3 = XHCI_TRB_3_SLOT_SET(slot) | - XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DISABLE_SLOT); - - err = xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT); - - if (!err) { - xs = &sc->sc_slots[slot]; - if (xs->xs_idx != 0) { - xhci_free_slot(sc, xs); - xhci_set_dcba(sc, 0, slot); - memset(xs, 0, sizeof(*xs)); - } - } - - return err; -} - -/* - * Set address of device and transition slot state from ENABLED to ADDRESSED - * if Block Setaddress Request (BSR) is false. - * If BSR==true, transition slot state from ENABLED to DEFAULT. - * see xHCI 1.1 4.5.3, 3.3.4 - * Should be called without sc_lock held. - */ -static usbd_status -xhci_address_device(struct xhci_softc * const sc, - uint64_t icp, uint8_t slot_id, bool bsr) -{ - struct xhci_soft_trb trb; - usbd_status err; - - XHCIHIST_FUNC(); - if (bsr) { - XHCIHIST_CALLARGS("icp %#jx slot %#jx with bsr", - icp, slot_id, 0, 0); - } else { - XHCIHIST_CALLARGS("icp %#jx slot %#jx nobsr", - icp, slot_id, 0, 0); - } - - trb.trb_0 = icp; - trb.trb_2 = 0; - trb.trb_3 = XHCI_TRB_3_SLOT_SET(slot_id) | - XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ADDRESS_DEVICE) | - (bsr ? XHCI_TRB_3_BSR_BIT : 0); - - err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); - - if (XHCI_TRB_2_ERROR_GET(trb.trb_2) == XHCI_TRB_ERROR_NO_SLOTS) - err = USBD_NO_ADDR; - - return err; -} - -static usbd_status -xhci_update_ep0_mps(struct xhci_softc * const sc, - struct xhci_slot * const xs, u_int mps) -{ - struct xhci_soft_trb trb; - usbd_status err; - uint32_t * cp; - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("slot %ju mps %ju", xs->xs_idx, mps, 0, 0); - - cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL); - cp[0] = htole32(0); - cp[1] = htole32(XHCI_INCTX_1_ADD_MASK(XHCI_DCI_EP_CONTROL)); - - cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_EP_CONTROL)); - cp[1] &= ~htole32(XHCI_EPCTX_1_MAXP_SIZE_MASK); - cp[1] |= htole32(XHCI_EPCTX_1_MAXP_SIZE_SET(mps)); - - /* sync input contexts before they are read from memory */ - usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE); - HEXDUMP("input context", xhci_slot_get_icv(sc, xs, 0), - sc->sc_ctxsz * 4); - - trb.trb_0 = xhci_slot_get_icp(sc, xs, 0); - trb.trb_2 = 0; - trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | - XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_EVALUATE_CTX); - - err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); - return err; -} - -static void -xhci_set_dcba(struct xhci_softc * const sc, uint64_t dcba, int si) -{ - uint64_t * const dcbaa = KERNADDR(&sc->sc_dcbaa_dma, 0); - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("dcbaa %#jx dc 0x%016jx slot %jd", - (uintptr_t)&dcbaa[si], dcba, si, 0); - - dcbaa[si] = htole64(dcba); - usb_syncmem(&sc->sc_dcbaa_dma, si * sizeof(uint64_t), sizeof(uint64_t), - BUS_DMASYNC_PREWRITE); -} - -/* - * Allocate device and input context DMA buffer, and - * TRB DMA buffer for each endpoint. - */ -static usbd_status -xhci_init_slot(struct usbd_device *dev, uint32_t slot) -{ - struct xhci_softc * const sc = XHCI_BUS2SC(dev->ud_bus); - struct xhci_slot *xs; - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("slot %ju", slot, 0, 0, 0); - - xs = &sc->sc_slots[slot]; - - /* allocate contexts */ - int err = usb_allocmem(sc->sc_bus.ub_dmatag, sc->sc_pgsz, sc->sc_pgsz, - USBMALLOC_ZERO, &xs->xs_dc_dma); - if (err) { - DPRINTFN(1, "failed to allocmem output device context %jd", - err, 0, 0, 0); - return USBD_NOMEM; - } - - err = usb_allocmem(sc->sc_bus.ub_dmatag, sc->sc_pgsz, sc->sc_pgsz, - USBMALLOC_ZERO, &xs->xs_ic_dma); - if (err) { - DPRINTFN(1, "failed to allocmem input device context %jd", - err, 0, 0, 0); - goto bad1; - } - - memset(&xs->xs_xr[0], 0, sizeof(xs->xs_xr)); - xs->xs_idx = slot; - - return USBD_NORMAL_COMPLETION; - -bad1: - usb_freemem(&xs->xs_dc_dma); - xs->xs_idx = 0; - return USBD_NOMEM; -} - -static void -xhci_free_slot(struct xhci_softc *sc, struct xhci_slot *xs) -{ - u_int dci; - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("slot %ju", xs->xs_idx, 0, 0, 0); - - /* deallocate all allocated rings in the slot */ - for (dci = XHCI_DCI_SLOT; dci <= XHCI_MAX_DCI; dci++) { - if (xs->xs_xr[dci] != NULL) - xhci_ring_free(sc, &xs->xs_xr[dci]); - } - usb_freemem(&xs->xs_ic_dma); - usb_freemem(&xs->xs_dc_dma); - xs->xs_idx = 0; -} - -/* - * Setup slot context, set Device Context Base Address, and issue - * Set Address Device command. - */ -static usbd_status -xhci_set_address(struct usbd_device *dev, uint32_t slot, bool bsr) -{ - struct xhci_softc * const sc = XHCI_BUS2SC(dev->ud_bus); - struct xhci_slot *xs; - usbd_status err; - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("slot %ju bsr %ju", slot, bsr, 0, 0); - - xs = &sc->sc_slots[slot]; - - xhci_setup_ctx(dev->ud_pipe0); - - HEXDUMP("input context", xhci_slot_get_icv(sc, xs, 0), - sc->sc_ctxsz * 3); - - xhci_set_dcba(sc, DMAADDR(&xs->xs_dc_dma, 0), slot); - - err = xhci_address_device(sc, xhci_slot_get_icp(sc, xs, 0), slot, bsr); - - usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD); - HEXDUMP("output context", xhci_slot_get_dcv(sc, xs, 0), - sc->sc_ctxsz * 2); - - return err; -} - -/* - * 4.8.2, 6.2.3.2 - * construct slot/endpoint context parameters and do syncmem - */ -static void -xhci_setup_ctx(struct usbd_pipe *pipe) -{ - struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); - struct usbd_device *dev = pipe->up_dev; - struct xhci_slot * const xs = dev->ud_hcpriv; - usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc; - const u_int dci = xhci_ep_get_dci(ed); - const uint8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes); - uint32_t *cp; - uint8_t speed = dev->ud_speed; - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("pipe %#jx: slot %ju dci %ju speed %ju", - (uintptr_t)pipe, xs->xs_idx, dci, speed); - - /* set up initial input control context */ - cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL); - cp[0] = htole32(0); - cp[1] = htole32(XHCI_INCTX_1_ADD_MASK(dci)); - cp[1] |= htole32(XHCI_INCTX_1_ADD_MASK(XHCI_DCI_SLOT)); - cp[7] = htole32(0); - - /* set up input slot context */ - cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_SLOT)); - cp[0] = - XHCI_SCTX_0_CTX_NUM_SET(dci) | - XHCI_SCTX_0_SPEED_SET(xhci_speed2xspeed(speed)); - cp[1] = 0; - cp[2] = XHCI_SCTX_2_IRQ_TARGET_SET(0); - cp[3] = 0; - xhci_setup_route(pipe, cp); - xhci_setup_tthub(pipe, cp); - - cp[0] = htole32(cp[0]); - cp[1] = htole32(cp[1]); - cp[2] = htole32(cp[2]); - cp[3] = htole32(cp[3]); - - /* set up input endpoint context */ - cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(dci)); - cp[0] = - XHCI_EPCTX_0_EPSTATE_SET(0) | - XHCI_EPCTX_0_MULT_SET(0) | - XHCI_EPCTX_0_MAXP_STREAMS_SET(0) | - XHCI_EPCTX_0_LSA_SET(0) | - XHCI_EPCTX_0_MAX_ESIT_PAYLOAD_HI_SET(0); - cp[1] = - XHCI_EPCTX_1_EPTYPE_SET(xhci_ep_get_type(ed)) | - XHCI_EPCTX_1_HID_SET(0) | - XHCI_EPCTX_1_MAXB_SET(0); - - if (xfertype != UE_ISOCHRONOUS) - cp[1] |= XHCI_EPCTX_1_CERR_SET(3); - - xhci_setup_maxburst(pipe, cp); - - DPRINTFN(4, "setting on dci %ju ival %ju mult %ju mps %#jx", - dci, XHCI_EPCTX_0_IVAL_GET(cp[0]), XHCI_EPCTX_0_MULT_GET(cp[0]), - XHCI_EPCTX_1_MAXP_SIZE_GET(cp[1])); - DPRINTFN(4, " maxburst %ju mep %#jx atl %#jx", - XHCI_EPCTX_1_MAXB_GET(cp[1]), - (XHCI_EPCTX_0_MAX_ESIT_PAYLOAD_HI_GET(cp[0]) << 16) + - XHCI_EPCTX_4_MAX_ESIT_PAYLOAD_GET(cp[4]), - XHCI_EPCTX_4_AVG_TRB_LEN_GET(cp[4]), 0); - - /* rewind TR dequeue pointer in xHC */ - /* can't use xhci_ep_get_dci() yet? */ - *(uint64_t *)(&cp[2]) = htole64( - xhci_ring_trbp(xs->xs_xr[dci], 0) | - XHCI_EPCTX_2_DCS_SET(1)); - - cp[0] = htole32(cp[0]); - cp[1] = htole32(cp[1]); - cp[4] = htole32(cp[4]); - - /* rewind TR dequeue pointer in driver */ - struct xhci_ring *xr = xs->xs_xr[dci]; - mutex_enter(&xr->xr_lock); - xhci_host_dequeue(xr); - mutex_exit(&xr->xr_lock); - - /* sync input contexts before they are read from memory */ - usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE); -} - -/* - * Setup route string and roothub port of given device for slot context - */ -static void -xhci_setup_route(struct usbd_pipe *pipe, uint32_t *cp) -{ - struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); - struct usbd_device *dev = pipe->up_dev; - struct usbd_port *up = dev->ud_powersrc; - struct usbd_device *hub; - struct usbd_device *adev; - uint8_t rhport = 0; - uint32_t route = 0; - - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - - /* Locate root hub port and Determine route string */ - /* 4.3.3 route string does not include roothub port */ - for (hub = dev; hub != NULL; hub = hub->ud_myhub) { - uint32_t dep; - - DPRINTFN(4, "hub %#jx depth %jd upport %#jx upportno %jd", - (uintptr_t)hub, hub->ud_depth, (uintptr_t)hub->ud_powersrc, - hub->ud_powersrc ? (uintptr_t)hub->ud_powersrc->up_portno : - -1); - - if (hub->ud_powersrc == NULL) - break; - dep = hub->ud_depth; - if (dep == 0) - break; - rhport = hub->ud_powersrc->up_portno; - if (dep > USB_HUB_MAX_DEPTH) - continue; - - route |= - (rhport > UHD_SS_NPORTS_MAX ? UHD_SS_NPORTS_MAX : rhport) - << ((dep - 1) * 4); - } - route = route >> 4; - size_t bn = hub == sc->sc_bus.ub_roothub ? 0 : 1; - - /* Locate port on upstream high speed hub */ - for (adev = dev, hub = up->up_parent; - hub != NULL && hub->ud_speed != USB_SPEED_HIGH; - adev = hub, hub = hub->ud_myhub) - ; - if (hub) { - int p; - for (p = 1; p <= hub->ud_hub->uh_hubdesc.bNbrPorts; p++) { - if (hub->ud_hub->uh_ports[p - 1].up_dev == adev) { - dev->ud_myhsport = &hub->ud_hub->uh_ports[p - 1]; - goto found; - } - } - panic("%s: cannot find HS port", __func__); - found: - DPRINTFN(4, "high speed port %jd", p, 0, 0, 0); - } else { - dev->ud_myhsport = NULL; - } - - const size_t ctlrport = xhci_rhport2ctlrport(sc, bn, rhport); - - DPRINTFN(4, "rhport %ju ctlrport %ju Route %05jx hub %#jx", rhport, - ctlrport, route, (uintptr_t)hub); - - cp[0] |= XHCI_SCTX_0_ROUTE_SET(route); - cp[1] |= XHCI_SCTX_1_RH_PORT_SET(ctlrport); -} - -/* - * Setup whether device is hub, whether device uses MTT, and - * TT informations if it uses MTT. - */ -static void -xhci_setup_tthub(struct usbd_pipe *pipe, uint32_t *cp) -{ - struct usbd_device *dev = pipe->up_dev; - struct usbd_port *myhsport = dev->ud_myhsport; - usb_device_descriptor_t * const dd = &dev->ud_ddesc; - uint32_t speed = dev->ud_speed; - uint8_t rhaddr = dev->ud_bus->ub_rhaddr; - uint8_t tthubslot, ttportnum; - bool ishub; - bool usemtt; - - XHCIHIST_FUNC(); - - /* - * 6.2.2, Table 57-60, 6.2.2.1, 6.2.2.2 - * tthubslot: - * This is the slot ID of parent HS hub - * if LS/FS device is connected && connected through HS hub. - * This is 0 if device is not LS/FS device || - * parent hub is not HS hub || - * attached to root hub. - * ttportnum: - * This is the downstream facing port of parent HS hub - * if LS/FS device is connected. - * This is 0 if device is not LS/FS device || - * parent hub is not HS hub || - * attached to root hub. - */ - if (myhsport && - myhsport->up_parent->ud_addr != rhaddr && - (speed == USB_SPEED_LOW || speed == USB_SPEED_FULL)) { - ttportnum = myhsport->up_portno; - tthubslot = myhsport->up_parent->ud_addr; - } else { - ttportnum = 0; - tthubslot = 0; - } - XHCIHIST_CALLARGS("myhsport %#jx ttportnum=%jd tthubslot=%jd", - (uintptr_t)myhsport, ttportnum, tthubslot, 0); - - /* ishub is valid after reading UDESC_DEVICE */ - ishub = (dd->bDeviceClass == UDCLASS_HUB); - - /* dev->ud_hub is valid after reading UDESC_HUB */ - if (ishub && dev->ud_hub) { - usb_hub_descriptor_t *hd = &dev->ud_hub->uh_hubdesc; - uint8_t ttt = - __SHIFTOUT(UGETW(hd->wHubCharacteristics), UHD_TT_THINK); - - cp[1] |= XHCI_SCTX_1_NUM_PORTS_SET(hd->bNbrPorts); - cp[2] |= XHCI_SCTX_2_TT_THINK_TIME_SET(ttt); - DPRINTFN(4, "nports=%jd ttt=%jd", hd->bNbrPorts, ttt, 0, 0); - } - -#define IS_MTTHUB(dd) \ - ((dd)->bDeviceProtocol == UDPROTO_HSHUBMTT) - - /* - * MTT flag is set if - * 1. this is HS hub && MTTs are supported and enabled; or - * 2. this is LS or FS device && there is a parent HS hub where MTTs - * are supported and enabled. - * - * XXX enabled is not tested yet - */ - if (ishub && speed == USB_SPEED_HIGH && IS_MTTHUB(dd)) - usemtt = true; - else if ((speed == USB_SPEED_LOW || speed == USB_SPEED_FULL) && - myhsport && - myhsport->up_parent->ud_addr != rhaddr && - IS_MTTHUB(&myhsport->up_parent->ud_ddesc)) - usemtt = true; - else - usemtt = false; - DPRINTFN(4, "class %ju proto %ju ishub %jd usemtt %jd", - dd->bDeviceClass, dd->bDeviceProtocol, ishub, usemtt); - -#undef IS_MTTHUB - - cp[0] |= - XHCI_SCTX_0_HUB_SET(ishub ? 1 : 0) | - XHCI_SCTX_0_MTT_SET(usemtt ? 1 : 0); - cp[2] |= - XHCI_SCTX_2_TT_HUB_SID_SET(tthubslot) | - XHCI_SCTX_2_TT_PORT_NUM_SET(ttportnum); -} - -static const usb_endpoint_ss_comp_descriptor_t * -xhci_get_essc_desc(struct usbd_pipe *pipe) -{ - struct usbd_device *dev = pipe->up_dev; - usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc; - const usb_cdc_descriptor_t *cdcd; - usbd_desc_iter_t iter; - uint8_t ep; - - /* config desc is NULL when opening ep0 */ - if (dev == NULL || dev->ud_cdesc == NULL) - return NULL; - - cdcd = (const usb_cdc_descriptor_t *)usb_find_desc(dev, - UDESC_INTERFACE, USBD_CDCSUBTYPE_ANY); - if (cdcd == NULL) - return NULL; - - usb_desc_iter_init(dev, &iter); - iter.cur = (const void *)cdcd; - - /* find endpoint_ss_comp desc for ep of this pipe */ - for (ep = 0;;) { - cdcd = (const usb_cdc_descriptor_t *)usb_desc_iter_next(&iter); - if (cdcd == NULL) - break; - if (ep == 0 && cdcd->bDescriptorType == UDESC_ENDPOINT) { - ep = ((const usb_endpoint_descriptor_t *)cdcd)-> - bEndpointAddress; - if (UE_GET_ADDR(ep) == - UE_GET_ADDR(ed->bEndpointAddress)) { - cdcd = (const usb_cdc_descriptor_t *) - usb_desc_iter_next(&iter); - break; - } - ep = 0; - } - } - if (cdcd != NULL && cdcd->bDescriptorType == UDESC_ENDPOINT_SS_COMP) { - return (const usb_endpoint_ss_comp_descriptor_t *)cdcd; - } - return NULL; -} - -/* set up params for periodic endpoint */ -static void -xhci_setup_maxburst(struct usbd_pipe *pipe, uint32_t *cp) -{ - struct xhci_pipe * const xpipe = (struct xhci_pipe *)pipe; - struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); - struct usbd_device * const dev = pipe->up_dev; - usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc; - const uint8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes); - uint16_t mps = UGETW(ed->wMaxPacketSize); - uint8_t speed = dev->ud_speed; - uint32_t maxb, mep, atl; - uint8_t ival, mult; - - const usb_endpoint_ss_comp_descriptor_t * esscd = - xhci_get_essc_desc(pipe); - - /* USB 2.0 9.6.6, xHCI 4.8.2.4, 6.2.3.2 - 6.2.3.8 */ - switch (xfertype) { - case UE_ISOCHRONOUS: - case UE_INTERRUPT: - if (USB_IS_SS(speed)) { - maxb = esscd ? esscd->bMaxBurst : UE_GET_TRANS(mps); - mep = esscd ? UGETW(esscd->wBytesPerInterval) : - UE_GET_SIZE(mps) * (maxb + 1); - if (esscd && xfertype == UE_ISOCHRONOUS && - XHCI_HCC2_LEC(sc->sc_hcc2) == 0) { - mult = UE_GET_SS_ISO_MULT(esscd->bmAttributes); - mult = (mult > 2) ? 2 : mult; - } else - mult = 0; - - } else { - switch (speed) { - case USB_SPEED_HIGH: - maxb = UE_GET_TRANS(mps); - mep = UE_GET_SIZE(mps) * (maxb + 1); - break; - case USB_SPEED_FULL: - maxb = 0; - mep = UE_GET_SIZE(mps); - break; - default: - maxb = 0; - mep = 0; - break; - } - mult = 0; - } - mps = UE_GET_SIZE(mps); - - if (pipe->up_interval == USBD_DEFAULT_INTERVAL) - ival = ed->bInterval; - else - ival = pipe->up_interval; - - ival = xhci_bival2ival(ival, speed, xfertype); - atl = mep; - break; - case UE_CONTROL: - case UE_BULK: - default: - if (USB_IS_SS(speed)) { - maxb = esscd ? esscd->bMaxBurst : 0; - } else - maxb = 0; - - mps = UE_GET_SIZE(mps); - mep = 0; - mult = 0; - ival = 0; - if (xfertype == UE_CONTROL) - atl = 8; /* 6.2.3 */ - else - atl = mps; - break; - } - - switch (speed) { - case USB_SPEED_LOW: - break; - case USB_SPEED_FULL: - if (xfertype == UE_INTERRUPT) - if (mep > XHCI_EPCTX_MEP_FS_INTR) - mep = XHCI_EPCTX_MEP_FS_INTR; - if (xfertype == UE_ISOCHRONOUS) - if (mep > XHCI_EPCTX_MEP_FS_ISOC) - mep = XHCI_EPCTX_MEP_FS_ISOC; - break; - case USB_SPEED_HIGH: - if (xfertype == UE_INTERRUPT) - if (mep > XHCI_EPCTX_MEP_HS_INTR) - mep = XHCI_EPCTX_MEP_HS_INTR; - if (xfertype == UE_ISOCHRONOUS) - if (mep > XHCI_EPCTX_MEP_HS_ISOC) - mep = XHCI_EPCTX_MEP_HS_ISOC; - break; - case USB_SPEED_SUPER: - case USB_SPEED_SUPER_PLUS: - default: - if (xfertype == UE_INTERRUPT) - if (mep > XHCI_EPCTX_MEP_SS_INTR) - mep = XHCI_EPCTX_MEP_SS_INTR; - if (xfertype == UE_ISOCHRONOUS) { - if (speed == USB_SPEED_SUPER || - XHCI_HCC2_LEC(sc->sc_hcc2) == 0) { - if (mep > XHCI_EPCTX_MEP_SS_ISOC) - mep = XHCI_EPCTX_MEP_SS_ISOC; - } else { - if (mep > XHCI_EPCTX_MEP_SS_ISOC_LEC) - mep = XHCI_EPCTX_MEP_SS_ISOC_LEC; - } - } - break; - } - - xpipe->xp_ival = ival; - xpipe->xp_maxb = maxb + 1; - xpipe->xp_mult = mult + 1; - - cp[0] |= XHCI_EPCTX_0_MAX_ESIT_PAYLOAD_HI_SET(mep >> 16); - cp[0] |= XHCI_EPCTX_0_IVAL_SET(ival); - cp[0] |= XHCI_EPCTX_0_MULT_SET(mult); - cp[1] |= XHCI_EPCTX_1_MAXP_SIZE_SET(mps); - cp[1] |= XHCI_EPCTX_1_MAXB_SET(maxb); - cp[4] |= XHCI_EPCTX_4_MAX_ESIT_PAYLOAD_SET(mep & 0xffff); - cp[4] |= XHCI_EPCTX_4_AVG_TRB_LEN_SET(atl); -} - -/* - * Convert usbdi bInterval value to xhci endpoint context interval value - * for periodic pipe. - * xHCI 6.2.3.6 Table 65, USB 2.0 9.6.6 - */ -static uint32_t -xhci_bival2ival(uint32_t ival, uint32_t speed, uint32_t xfertype) -{ - if (xfertype != UE_INTERRUPT && xfertype != UE_ISOCHRONOUS) - return 0; - - if (xfertype == UE_INTERRUPT && - (speed == USB_SPEED_LOW || speed == USB_SPEED_FULL)) { - u_int i; - - /* - * round ival down to "the nearest base 2 multiple of - * bInterval * 8". - * bInterval is at most 255 as its type is uByte. - * 255(ms) = 2040(x 125us) < 2^11, so start with 10. - */ - for (i = 10; i > 0; i--) { - if ((ival * 8) >= (1 << i)) - break; - } - ival = i; - - /* 3 - 10 */ - ival = (ival < 3) ? 3 : ival; - } else if (speed == USB_SPEED_FULL) { - /* FS isoc */ - ival += 3; /* 1ms -> 125us */ - ival--; /* Interval = bInterval-1 */ - /* 3 - 18 */ - ival = (ival > 18) ? 18 : ival; - ival = (ival < 3) ? 3 : ival; - } else { - /* SS/HS intr/isoc */ - if (ival > 0) - ival--; /* Interval = bInterval-1 */ - /* 0 - 15 */ - ival = (ival > 15) ? 15 : ival; - } - - return ival; -} - -/* ----- */ - -static void -xhci_noop(struct usbd_pipe *pipe) -{ - XHCIHIST_FUNC(); XHCIHIST_CALLED(); -} - -/* - * Process root hub request. - */ -static int -xhci_roothub_ctrl_locked(struct usbd_bus *bus, usb_device_request_t *req, - void *buf, int buflen) -{ - struct xhci_softc * const sc = XHCI_BUS2SC(bus); - usb_port_status_t ps; - int l, totlen = 0; - uint16_t len, value, index; - int port, i; - uint32_t v; - - XHCIHIST_FUNC(); - - KASSERT(mutex_owned(&sc->sc_rhlock)); - - if (sc->sc_dying) - return -1; - - size_t bn = bus == &sc->sc_bus ? 0 : 1; - - len = UGETW(req->wLength); - value = UGETW(req->wValue); - index = UGETW(req->wIndex); - - XHCIHIST_CALLARGS("rhreq: %04jx %04jx %04jx %04jx", - req->bmRequestType | (req->bRequest << 8), value, index, len); - -#define C(x,y) ((x) | ((y) << 8)) - switch (C(req->bRequest, req->bmRequestType)) { - case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE): - DPRINTFN(8, "getdesc: wValue=0x%04jx", value, 0, 0, 0); - if (len == 0) - break; - switch (value) { -#define sd ((usb_string_descriptor_t *)buf) - case C(2, UDESC_STRING): - /* Product */ - totlen = usb_makestrdesc(sd, len, "xHCI root hub"); - break; -#undef sd - default: - /* default from usbroothub */ - return buflen; - } - break; - - /* Hub requests */ - case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE): - break; - /* Clear Port Feature request */ - case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER): { - const size_t cp = xhci_rhport2ctlrport(sc, bn, index); - - DPRINTFN(4, "UR_CLEAR_PORT_FEAT bp=%jd feat=%jd bus=%jd cp=%jd", - index, value, bn, cp); - if (index < 1 || index > sc->sc_rhportcount[bn]) { - return -1; - } - port = XHCI_PORTSC(cp); - v = xhci_op_read_4(sc, port); - DPRINTFN(4, "portsc=0x%08jx", v, 0, 0, 0); - v &= ~XHCI_PS_CLEAR; - switch (value) { - case UHF_PORT_ENABLE: - xhci_op_write_4(sc, port, v & ~XHCI_PS_PED); - break; - case UHF_PORT_SUSPEND: - return -1; - case UHF_PORT_POWER: - break; - case UHF_PORT_TEST: - case UHF_PORT_INDICATOR: - return -1; - case UHF_C_PORT_CONNECTION: - xhci_op_write_4(sc, port, v | XHCI_PS_CSC); - break; - case UHF_C_PORT_ENABLE: - case UHF_C_PORT_SUSPEND: - case UHF_C_PORT_OVER_CURRENT: - return -1; - case UHF_C_BH_PORT_RESET: - xhci_op_write_4(sc, port, v | XHCI_PS_WRC); - break; - case UHF_C_PORT_RESET: - xhci_op_write_4(sc, port, v | XHCI_PS_PRC); - break; - case UHF_C_PORT_LINK_STATE: - xhci_op_write_4(sc, port, v | XHCI_PS_PLC); - break; - case UHF_C_PORT_CONFIG_ERROR: - xhci_op_write_4(sc, port, v | XHCI_PS_CEC); - break; - default: - return -1; - } - break; - } - case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE): - if (len == 0) - break; - if ((value & 0xff) != 0) { - return -1; - } - usb_hub_descriptor_t hubd; - - totlen = uimin(buflen, sizeof(hubd)); - memcpy(&hubd, buf, totlen); - hubd.bNbrPorts = sc->sc_rhportcount[bn]; - USETW(hubd.wHubCharacteristics, UHD_PWR_NO_SWITCH); - hubd.bPwrOn2PwrGood = 200; - for (i = 0, l = sc->sc_rhportcount[bn]; l > 0; i++, l -= 8) { - /* XXX can't find out? */ - hubd.DeviceRemovable[i++] = 0; - } - hubd.bDescLength = USB_HUB_DESCRIPTOR_SIZE + i; - totlen = uimin(totlen, hubd.bDescLength); - memcpy(buf, &hubd, totlen); - break; - case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE): - if (len != 4) { - return -1; - } - memset(buf, 0, len); /* ? XXX */ - totlen = len; - break; - /* Get Port Status request */ - case C(UR_GET_STATUS, UT_READ_CLASS_OTHER): { - const size_t cp = xhci_rhport2ctlrport(sc, bn, index); - - DPRINTFN(8, "get port status bn=%jd i=%jd cp=%ju", - bn, index, cp, 0); - if (index < 1 || index > sc->sc_rhportcount[bn]) { - DPRINTFN(5, "bad get port status: index=%jd bn=%jd " - "portcount=%jd", - index, bn, sc->sc_rhportcount[bn], 0); - return -1; - } - if (len != 4) { - DPRINTFN(5, "bad get port status: len %jd != 4", - len, 0, 0, 0); - return -1; - } - v = xhci_op_read_4(sc, XHCI_PORTSC(cp)); - DPRINTFN(4, "getrhportsc %jd 0x%08jx", cp, v, 0, 0); - i = xhci_xspeed2psspeed(XHCI_PS_SPEED_GET(v)); - if (v & XHCI_PS_CCS) i |= UPS_CURRENT_CONNECT_STATUS; - if (v & XHCI_PS_PED) i |= UPS_PORT_ENABLED; - if (v & XHCI_PS_OCA) i |= UPS_OVERCURRENT_INDICATOR; - //if (v & XHCI_PS_SUSP) i |= UPS_SUSPEND; - if (v & XHCI_PS_PR) i |= UPS_RESET; - if (v & XHCI_PS_PP) { - if (i & UPS_OTHER_SPEED) - i |= UPS_PORT_POWER_SS; - else - i |= UPS_PORT_POWER; - } - if (i & UPS_OTHER_SPEED) - i |= UPS_PORT_LS_SET(XHCI_PS_PLS_GET(v)); - if (sc->sc_vendor_port_status) - i = sc->sc_vendor_port_status(sc, v, i); - USETW(ps.wPortStatus, i); - i = 0; - if (v & XHCI_PS_CSC) i |= UPS_C_CONNECT_STATUS; - if (v & XHCI_PS_PEC) i |= UPS_C_PORT_ENABLED; - if (v & XHCI_PS_OCC) i |= UPS_C_OVERCURRENT_INDICATOR; - if (v & XHCI_PS_PRC) i |= UPS_C_PORT_RESET; - if (v & XHCI_PS_WRC) i |= UPS_C_BH_PORT_RESET; - if (v & XHCI_PS_PLC) i |= UPS_C_PORT_LINK_STATE; - if (v & XHCI_PS_CEC) i |= UPS_C_PORT_CONFIG_ERROR; - USETW(ps.wPortChange, i); - totlen = uimin(len, sizeof(ps)); - memcpy(buf, &ps, totlen); - DPRINTFN(5, "get port status: wPortStatus %#jx wPortChange %#jx" - " totlen %jd", - UGETW(ps.wPortStatus), UGETW(ps.wPortChange), totlen, 0); - break; - } - case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE): - return -1; - case C(UR_SET_HUB_DEPTH, UT_WRITE_CLASS_DEVICE): - break; - case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE): - break; - /* Set Port Feature request */ - case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER): { - int optval = (index >> 8) & 0xff; - index &= 0xff; - if (index < 1 || index > sc->sc_rhportcount[bn]) { - return -1; - } - - const size_t cp = xhci_rhport2ctlrport(sc, bn, index); - - port = XHCI_PORTSC(cp); - v = xhci_op_read_4(sc, port); - DPRINTFN(4, "index %jd cp %jd portsc=0x%08jx", index, cp, v, 0); - v &= ~XHCI_PS_CLEAR; - switch (value) { - case UHF_PORT_ENABLE: - xhci_op_write_4(sc, port, v | XHCI_PS_PED); - break; - case UHF_PORT_SUSPEND: - /* XXX suspend */ - break; - case UHF_PORT_RESET: - xhci_op_write_4(sc, port, v | XHCI_PS_PR); - /* Wait for reset to complete. */ - for (i = 0; i < USB_PORT_ROOT_RESET_DELAY / 10; i++) { - if (sc->sc_dying) { - return -1; - } - v = xhci_op_read_4(sc, port); - if ((v & XHCI_PS_PR) == 0) { - break; - } - usb_delay_ms(&sc->sc_bus, 10); - } - break; - case UHF_PORT_POWER: - /* XXX power control */ - break; - /* XXX more */ - case UHF_C_PORT_RESET: - xhci_op_write_4(sc, port, v | XHCI_PS_PRC); - break; - case UHF_PORT_U1_TIMEOUT: - if (XHCI_PS_SPEED_GET(v) < XHCI_PS_SPEED_SS) { - return -1; - } - port = XHCI_PORTPMSC(cp); - v = xhci_op_read_4(sc, port); - DPRINTFN(4, "index %jd cp %jd portpmsc=0x%08jx", - index, cp, v, 0); - v &= ~XHCI_PM3_U1TO_SET(0xff); - v |= XHCI_PM3_U1TO_SET(optval); - xhci_op_write_4(sc, port, v); - break; - case UHF_PORT_U2_TIMEOUT: - if (XHCI_PS_SPEED_GET(v) < XHCI_PS_SPEED_SS) { - return -1; - } - port = XHCI_PORTPMSC(cp); - v = xhci_op_read_4(sc, port); - DPRINTFN(4, "index %jd cp %jd portpmsc=0x%08jx", - index, cp, v, 0); - v &= ~XHCI_PM3_U2TO_SET(0xff); - v |= XHCI_PM3_U2TO_SET(optval); - xhci_op_write_4(sc, port, v); - break; - default: - return -1; - } - } - break; - case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER): - case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER): - case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER): - case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER): - break; - default: - /* default from usbroothub */ - return buflen; - } - - return totlen; -} - -static int -xhci_roothub_ctrl(struct usbd_bus *bus, usb_device_request_t *req, - void *buf, int buflen) -{ - struct xhci_softc *sc = XHCI_BUS2SC(bus); - int actlen; - - mutex_enter(&sc->sc_rhlock); - actlen = xhci_roothub_ctrl_locked(bus, req, buf, buflen); - mutex_exit(&sc->sc_rhlock); - - return actlen; -} - -/* root hub interrupt */ - -static usbd_status -xhci_root_intr_transfer(struct usbd_xfer *xfer) -{ - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - - /* Pipe isn't running, start first */ - return xhci_root_intr_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue)); -} - -/* Wait for roothub port status/change */ -static usbd_status -xhci_root_intr_start(struct usbd_xfer *xfer) -{ - struct xhci_softc * const sc = XHCI_XFER2SC(xfer); - const size_t bn = XHCI_XFER2BUS(xfer) == &sc->sc_bus ? 0 : 1; - - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - - KASSERT(xhci_polling_p(sc) || mutex_owned(&sc->sc_lock)); - - if (sc->sc_dying) - return USBD_IOERROR; - - KASSERT(sc->sc_intrxfer[bn] == NULL); - sc->sc_intrxfer[bn] = xfer; - xfer->ux_status = USBD_IN_PROGRESS; - - return USBD_IN_PROGRESS; -} - -static void -xhci_root_intr_abort(struct usbd_xfer *xfer) -{ - struct xhci_softc * const sc = XHCI_XFER2SC(xfer); - const size_t bn = XHCI_XFER2BUS(xfer) == &sc->sc_bus ? 0 : 1; - - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - - KASSERT(mutex_owned(&sc->sc_lock)); - KASSERT(xfer->ux_pipe->up_intrxfer == xfer); - - /* If xfer has already completed, nothing to do here. */ - if (sc->sc_intrxfer[bn] == NULL) - return; - - /* - * Otherwise, sc->sc_intrxfer[bn] had better be this transfer. - * Cancel it. - */ - KASSERT(sc->sc_intrxfer[bn] == xfer); - xfer->ux_status = USBD_CANCELLED; - usb_transfer_complete(xfer); -} - -static void -xhci_root_intr_close(struct usbd_pipe *pipe) -{ - struct xhci_softc * const sc __diagused = XHCI_PIPE2SC(pipe); - const struct usbd_xfer *xfer __diagused = pipe->up_intrxfer; - const size_t bn __diagused = XHCI_XFER2BUS(xfer) == &sc->sc_bus ? 0 : 1; - - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - - KASSERT(mutex_owned(&sc->sc_lock)); - - /* - * Caller must guarantee the xfer has completed first, by - * closing the pipe only after normal completion or an abort. - */ - KASSERT(sc->sc_intrxfer[bn] == NULL); -} - -static void -xhci_root_intr_done(struct usbd_xfer *xfer) -{ - struct xhci_softc * const sc = XHCI_XFER2SC(xfer); - const size_t bn = XHCI_XFER2BUS(xfer) == &sc->sc_bus ? 0 : 1; - - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - - KASSERT(mutex_owned(&sc->sc_lock)); - - /* Claim the xfer so it doesn't get completed again. */ - KASSERT(sc->sc_intrxfer[bn] == xfer); - KASSERT(xfer->ux_status != USBD_IN_PROGRESS); - sc->sc_intrxfer[bn] = NULL; -} - -/* -------------- */ -/* device control */ - -static usbd_status -xhci_device_ctrl_transfer(struct usbd_xfer *xfer) -{ - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - - /* Pipe isn't running, start first */ - return xhci_device_ctrl_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue)); -} - -static usbd_status -xhci_device_ctrl_start(struct usbd_xfer *xfer) -{ - struct xhci_softc * const sc = XHCI_XFER2SC(xfer); - struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; - const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); - struct xhci_ring * const tr = xs->xs_xr[dci]; - struct xhci_xfer * const xx = XHCI_XFER2XXFER(xfer); - usb_device_request_t * const req = &xfer->ux_request; - const bool isread = usbd_xfer_isread(xfer); - const uint32_t len = UGETW(req->wLength); - usb_dma_t * const dma = &xfer->ux_dmabuf; - uint64_t parameter; - uint32_t status; - uint32_t control; - u_int i; - const bool polling = xhci_polling_p(sc); - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("req: %04jx %04jx %04jx %04jx", - req->bmRequestType | (req->bRequest << 8), UGETW(req->wValue), - UGETW(req->wIndex), UGETW(req->wLength)); - - KASSERT(polling || mutex_owned(&sc->sc_lock)); - - /* we rely on the bottom bits for extra info */ - KASSERTMSG(((uintptr_t)xfer & 0x3) == 0x0, "xfer %p", xfer); - - KASSERT((xfer->ux_rqflags & URQ_REQUEST) != 0); - - if (tr->is_halted) - goto out; - - i = 0; - - /* setup phase */ - parameter = le64dec(req); /* to keep USB endian after xhci_trb_put() */ - status = XHCI_TRB_2_IRQ_SET(0) | XHCI_TRB_2_BYTES_SET(sizeof(*req)); - control = ((len == 0) ? XHCI_TRB_3_TRT_NONE : - (isread ? XHCI_TRB_3_TRT_IN : XHCI_TRB_3_TRT_OUT)) | - XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SETUP_STAGE) | - XHCI_TRB_3_IDT_BIT; - xhci_xfer_put_trb(xx, i++, parameter, status, control); - - if (len != 0) { - /* data phase */ - parameter = DMAADDR(dma, 0); - KASSERTMSG(len <= 0x10000, "len %d", len); - status = XHCI_TRB_2_IRQ_SET(0) | - XHCI_TRB_2_TDSZ_SET(0) | - XHCI_TRB_2_BYTES_SET(len); - control = (isread ? XHCI_TRB_3_DIR_IN : 0) | - XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DATA_STAGE) | - (isread ? XHCI_TRB_3_ISP_BIT : 0) | - XHCI_TRB_3_IOC_BIT; - xhci_xfer_put_trb(xx, i++, parameter, status, control); - - usb_syncmem(dma, 0, len, - isread ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); - } - - parameter = 0; - status = XHCI_TRB_2_IRQ_SET(0); - /* the status stage has inverted direction */ - control = ((isread && (len > 0)) ? 0 : XHCI_TRB_3_DIR_IN) | - XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STATUS_STAGE) | - XHCI_TRB_3_IOC_BIT; - xhci_xfer_put_trb(xx, i++, parameter, status, control); - - if (!polling) - mutex_enter(&tr->xr_lock); - xhci_ring_put_xfer(sc, tr, xx, i); - if (!polling) - mutex_exit(&tr->xr_lock); - - xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci); - -out: if (xfer->ux_status == USBD_NOT_STARTED) { - xfer->ux_status = USBD_IN_PROGRESS; - usbd_xfer_schedule_timeout(xfer); - } else { - /* - * We must be coming from xhci_pipe_restart -- timeout - * already set up, nothing to do. - */ - } - KASSERT(xfer->ux_status == USBD_IN_PROGRESS); - - return USBD_IN_PROGRESS; -} - -static void -xhci_device_ctrl_done(struct usbd_xfer *xfer) -{ - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - usb_device_request_t *req = &xfer->ux_request; - int len = UGETW(req->wLength); - int rd = req->bmRequestType & UT_READ; - - if (len) - usb_syncmem(&xfer->ux_dmabuf, 0, len, - rd ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); -} - -static void -xhci_device_ctrl_abort(struct usbd_xfer *xfer) -{ - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - - usbd_xfer_abort(xfer); -} - -static void -xhci_device_ctrl_close(struct usbd_pipe *pipe) -{ - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - - xhci_close_pipe(pipe); -} - -/* ------------------ */ -/* device isochronous */ - -static usbd_status -xhci_device_isoc_transfer(struct usbd_xfer *xfer) -{ - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - - return xhci_device_isoc_enter(xfer); -} - -static usbd_status -xhci_device_isoc_enter(struct usbd_xfer *xfer) -{ - struct xhci_softc * const sc = XHCI_XFER2SC(xfer); - struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; - const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); - struct xhci_ring * const tr = xs->xs_xr[dci]; - struct xhci_xfer * const xx = XHCI_XFER2XXFER(xfer); - struct xhci_pipe * const xpipe = (struct xhci_pipe *)xfer->ux_pipe; - usb_dma_t * const dma = &xfer->ux_dmabuf; - uint64_t parameter; - uint32_t status; - uint32_t control; - uint32_t offs; - int i, ival; - const bool polling = xhci_polling_p(sc); - const uint16_t MPS = UGETW(xfer->ux_pipe->up_endpoint->ue_edesc->wMaxPacketSize); - const uint16_t mps = UE_GET_SIZE(MPS); - const uint8_t maxb = xpipe->xp_maxb; - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("%#jx slot %ju dci %ju", - (uintptr_t)xfer, xs->xs_idx, dci, 0); - - KASSERT(polling || mutex_owned(&sc->sc_lock)); - - if (sc->sc_dying) - return USBD_IOERROR; - - KASSERT(xfer->ux_nframes != 0 && xfer->ux_frlengths); - KASSERT((xfer->ux_rqflags & URQ_REQUEST) == 0); - - const bool isread = usbd_xfer_isread(xfer); - if (xfer->ux_length) - usb_syncmem(dma, 0, xfer->ux_length, - isread ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); - - ival = xpipe->xp_ival; - if (ival >= 0 && ival <= 15) - ival = 1 << ival; - else - ival = 1; /* fake something up */ - - const unsigned mfmask = XHCI_MFINDEX_GET(~(uint32_t)0); - - if (xpipe->xp_isoc_next == -1) { - uint32_t mfindex = xhci_rt_read_4(sc, XHCI_MFINDEX); - - DPRINTF("mfindex %jx", (uintmax_t)mfindex, 0, 0, 0); - mfindex = XHCI_MFINDEX_GET(mfindex); - - /* Start Frame = MFINDEX + IST + 1 */ - mfindex += sc->sc_isthresh + 1; - xpipe->xp_isoc_next = roundup2(mfindex, ival) & mfmask; - } - - offs = 0; - for (i = 0; i < xfer->ux_nframes; i++) { - const uint32_t len = xfer->ux_frlengths[i]; - const unsigned tdpc = howmany(len, mps); - const unsigned tbc = howmany(tdpc, maxb) - 1; - const unsigned tlbpc1 = tdpc % maxb; - const unsigned tlbpc = tlbpc1 ? tlbpc1 - 1 : maxb - 1; - const unsigned frid = xpipe->xp_isoc_next / - USB_UFRAMES_PER_FRAME; - - KASSERTMSG(len <= 0x10000, "len %d", len); - parameter = DMAADDR(dma, offs); - status = XHCI_TRB_2_IRQ_SET(0) | - XHCI_TRB_2_TDSZ_SET(0) | - XHCI_TRB_2_BYTES_SET(len); - control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ISOCH) | - (isread ? XHCI_TRB_3_ISP_BIT : 0) | - XHCI_TRB_3_TBC_SET(tbc) | - XHCI_TRB_3_TLBPC_SET(tlbpc) | - XHCI_TRB_3_IOC_BIT; - if (XHCI_HCC_CFC(sc->sc_hcc)) { - control |= XHCI_TRB_3_FRID_SET(frid); -#if 0 - } else if (xpipe->xp_isoc_next == -1) { - control |= XHCI_TRB_3_FRID_SET(frid); -#endif - } else { - control |= XHCI_TRB_3_ISO_SIA_BIT; - } -#if 0 - if (i != xfer->ux_nframes - 1) - control |= XHCI_TRB_3_BEI_BIT; -#endif - xhci_xfer_put_trb(xx, i, parameter, status, control); - - xpipe->xp_isoc_next = (xpipe->xp_isoc_next + ival) & mfmask; - offs += len; - } - - xx->xx_isoc_done = 0; - - if (!polling) - mutex_enter(&tr->xr_lock); - xhci_ring_put_xfer(sc, tr, xx, i); - if (!polling) - mutex_exit(&tr->xr_lock); - - xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci); - xfer->ux_status = USBD_IN_PROGRESS; - usbd_xfer_schedule_timeout(xfer); - - return USBD_IN_PROGRESS; -} - -static void -xhci_device_isoc_abort(struct usbd_xfer *xfer) -{ - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - - usbd_xfer_abort(xfer); -} - -static void -xhci_device_isoc_close(struct usbd_pipe *pipe) -{ - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - - xhci_close_pipe(pipe); -} - -static void -xhci_device_isoc_done(struct usbd_xfer *xfer) -{ -#ifdef USB_DEBUG - struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; - const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); -#endif - const bool isread = usbd_xfer_isread(xfer); - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("%#jx slot %ju dci %ju", - (uintptr_t)xfer, xs->xs_idx, dci, 0); - - usb_syncmem(&xfer->ux_dmabuf, 0, xfer->ux_length, - isread ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); -} - -/* ----------- */ -/* device bulk */ - -static usbd_status -xhci_device_bulk_transfer(struct usbd_xfer *xfer) -{ - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - - /* Pipe isn't running, so start it first. */ - return xhci_device_bulk_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue)); -} - -static usbd_status -xhci_device_bulk_start(struct usbd_xfer *xfer) -{ - struct xhci_softc * const sc = XHCI_XFER2SC(xfer); - struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; - const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); - struct xhci_ring * const tr = xs->xs_xr[dci]; - struct xhci_xfer * const xx = XHCI_XFER2XXFER(xfer); - const uint32_t len = xfer->ux_length; - usb_dma_t * const dma = &xfer->ux_dmabuf; - uint64_t parameter; - uint32_t status; - uint32_t control; - u_int i = 0; - const bool polling = xhci_polling_p(sc); - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("%#jx slot %ju dci %ju", - (uintptr_t)xfer, xs->xs_idx, dci, 0); - - KASSERT(polling || mutex_owned(&sc->sc_lock)); - - if (sc->sc_dying) - return USBD_IOERROR; - - KASSERT((xfer->ux_rqflags & URQ_REQUEST) == 0); - - if (tr->is_halted) - goto out; - - parameter = DMAADDR(dma, 0); - const bool isread = usbd_xfer_isread(xfer); - if (len) - usb_syncmem(dma, 0, len, - isread ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); - - /* - * XXX: (dsl) The physical buffer must not cross a 64k boundary. - * If the user supplied buffer crosses such a boundary then 2 - * (or more) TRB should be used. - * If multiple TRB are used the td_size field must be set correctly. - * For v1.0 devices (like ivy bridge) this is the number of usb data - * blocks needed to complete the transfer. - * Setting it to 1 in the last TRB causes an extra zero-length - * data block be sent. - * The earlier documentation differs, I don't know how it behaves. - */ - KASSERTMSG(len <= 0x10000, "len %d", len); - status = XHCI_TRB_2_IRQ_SET(0) | - XHCI_TRB_2_TDSZ_SET(0) | - XHCI_TRB_2_BYTES_SET(len); - control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL) | - (isread ? XHCI_TRB_3_ISP_BIT : 0) | - XHCI_TRB_3_IOC_BIT; - xhci_xfer_put_trb(xx, i++, parameter, status, control); - - if (!isread && (xfer->ux_flags & USBD_FORCE_SHORT_XFER)) { - status = XHCI_TRB_2_IRQ_SET(0) | - XHCI_TRB_2_TDSZ_SET(0) | - XHCI_TRB_2_BYTES_SET(0); - xhci_xfer_put_trb(xx, i++, parameter, status, control); - } - - if (!polling) - mutex_enter(&tr->xr_lock); - xhci_ring_put_xfer(sc, tr, xx, i); - if (!polling) - mutex_exit(&tr->xr_lock); - - xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci); - -out: if (xfer->ux_status == USBD_NOT_STARTED) { - xfer->ux_status = USBD_IN_PROGRESS; - usbd_xfer_schedule_timeout(xfer); - } else { - /* - * We must be coming from xhci_pipe_restart -- timeout - * already set up, nothing to do. - */ - } - KASSERT(xfer->ux_status == USBD_IN_PROGRESS); - - return USBD_IN_PROGRESS; -} - -static void -xhci_device_bulk_done(struct usbd_xfer *xfer) -{ -#ifdef USB_DEBUG - struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; - const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); -#endif - const bool isread = usbd_xfer_isread(xfer); - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("%#jx slot %ju dci %ju", - (uintptr_t)xfer, xs->xs_idx, dci, 0); - - usb_syncmem(&xfer->ux_dmabuf, 0, xfer->ux_length, - isread ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); -} - -static void -xhci_device_bulk_abort(struct usbd_xfer *xfer) -{ - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - - usbd_xfer_abort(xfer); -} - -static void -xhci_device_bulk_close(struct usbd_pipe *pipe) -{ - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - - xhci_close_pipe(pipe); -} - -/* ---------------- */ -/* device interrupt */ - -static usbd_status -xhci_device_intr_transfer(struct usbd_xfer *xfer) -{ - XHCIHIST_FUNC(); XHCIHIST_CALLED(); - - /* Pipe isn't running, so start it first. */ - return xhci_device_intr_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue)); -} - -static usbd_status -xhci_device_intr_start(struct usbd_xfer *xfer) -{ - struct xhci_softc * const sc = XHCI_XFER2SC(xfer); - struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; - const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); - struct xhci_ring * const tr = xs->xs_xr[dci]; - struct xhci_xfer * const xx = XHCI_XFER2XXFER(xfer); - const uint32_t len = xfer->ux_length; - const bool polling = xhci_polling_p(sc); - usb_dma_t * const dma = &xfer->ux_dmabuf; - uint64_t parameter; - uint32_t status; - uint32_t control; - u_int i = 0; - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("%#jx slot %ju dci %ju", - (uintptr_t)xfer, xs->xs_idx, dci, 0); - - KASSERT(polling || mutex_owned(&sc->sc_lock)); - - if (sc->sc_dying) - return USBD_IOERROR; - - if (tr->is_halted) - goto out; - - KASSERT((xfer->ux_rqflags & URQ_REQUEST) == 0); - - const bool isread = usbd_xfer_isread(xfer); - if (len) - usb_syncmem(dma, 0, len, - isread ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); - - parameter = DMAADDR(dma, 0); - KASSERTMSG(len <= 0x10000, "len %d", len); - status = XHCI_TRB_2_IRQ_SET(0) | - XHCI_TRB_2_TDSZ_SET(0) | - XHCI_TRB_2_BYTES_SET(len); - control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL) | - (isread ? XHCI_TRB_3_ISP_BIT : 0) | XHCI_TRB_3_IOC_BIT; - xhci_xfer_put_trb(xx, i++, parameter, status, control); - - if (!polling) - mutex_enter(&tr->xr_lock); - xhci_ring_put_xfer(sc, tr, xx, i); - if (!polling) - mutex_exit(&tr->xr_lock); - - xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci); - -out: if (xfer->ux_status == USBD_NOT_STARTED) { - xfer->ux_status = USBD_IN_PROGRESS; - usbd_xfer_schedule_timeout(xfer); - } else { - /* - * We must be coming from xhci_pipe_restart -- timeout - * already set up, nothing to do. - */ - } - KASSERT(xfer->ux_status == USBD_IN_PROGRESS); - - return USBD_IN_PROGRESS; -} - -static void -xhci_device_intr_done(struct usbd_xfer *xfer) -{ - struct xhci_softc * const sc __diagused = XHCI_XFER2SC(xfer); -#ifdef USB_DEBUG - struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; - const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); -#endif - const bool isread = usbd_xfer_isread(xfer); - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("%#jx slot %ju dci %ju", - (uintptr_t)xfer, xs->xs_idx, dci, 0); - - KASSERT(xhci_polling_p(sc) || mutex_owned(&sc->sc_lock)); - - usb_syncmem(&xfer->ux_dmabuf, 0, xfer->ux_length, - isread ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); -} - -static void -xhci_device_intr_abort(struct usbd_xfer *xfer) -{ - struct xhci_softc * const sc __diagused = XHCI_XFER2SC(xfer); - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("%#jx", (uintptr_t)xfer, 0, 0, 0); - - KASSERT(mutex_owned(&sc->sc_lock)); - usbd_xfer_abort(xfer); -} - -static void -xhci_device_intr_close(struct usbd_pipe *pipe) -{ - //struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); - - XHCIHIST_FUNC(); - XHCIHIST_CALLARGS("%#jx", (uintptr_t)pipe, 0, 0, 0); - - xhci_close_pipe(pipe); -} diff --git a/sys/modules/Makefile b/sys/modules/Makefile index e8760aca44fe6..b4832954750df 100644 --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -144,6 +144,7 @@ SUBDIR+= npf_alg_icmp SUBDIR+= npf_ext_log SUBDIR+= npf_ext_normalize SUBDIR+= npf_ext_rndblock +SUBDIR+= npf_ext_route SUBDIR+= ntfs SUBDIR+= null SUBDIR+= onewire diff --git a/sys/modules/npf_ext_route/Makefile b/sys/modules/npf_ext_route/Makefile new file mode 100644 index 0000000000000..2fc562f27cd53 --- /dev/null +++ b/sys/modules/npf_ext_route/Makefile @@ -0,0 +1,11 @@ +.include "../Makefile.inc" + +.PATH: ${S}/net/npf + +KMOD= npf_ext_route + +SRCS= npf_ext_route.c + +CPPFLAGS+= -I${NETBSDSRCDIR}/sys/external/bsd/libnv/dist + +.include diff --git a/sys/net/npf/files.npf b/sys/net/npf/files.npf index 6a9db26830f25..e9dc6af970995 100644 --- a/sys/net/npf/files.npf +++ b/sys/net/npf/files.npf @@ -44,6 +44,7 @@ file net/npf/lpm.c npf file net/npf/npf_ext_log.c npf file net/npf/npf_ext_normalize.c npf file net/npf/npf_ext_rndblock.c npf +file net/npf/npf_ext_route.c npf # ALGs file net/npf/npf_alg_icmp.c npf diff --git a/sys/net/npf/npf.h b/sys/net/npf/npf.h index 47f9d97a1b8c2..f630a1fd6b857 100644 --- a/sys/net/npf/npf.h +++ b/sys/net/npf/npf.h @@ -373,8 +373,12 @@ typedef enum { NPF_STAT_RACE_NAT, /* Fragments. */ NPF_STAT_FRAGMENTS, + NPF_STAT_NOFRAGMENT, NPF_STAT_REASSEMBLY, NPF_STAT_REASSFAIL, + /* routing */ + NPF_STAT_REROUTE, + NPF_STAT_NOREROUTE, /* Other errors. */ NPF_STAT_ERROR, /* nbuf non-contiguous cases. */ diff --git a/sys/net/npf/npf_ext_route.c b/sys/net/npf/npf_ext_route.c new file mode 100644 index 0000000000000..8252e1a2c877e --- /dev/null +++ b/sys/net/npf/npf_ext_route.c @@ -0,0 +1,493 @@ +/*- + * Copyright (c) 2025 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Emmanuel Nyarko. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * NPF route extension. + */ + + #ifdef _KERNEL +#include +__KERNEL_RCSID(0, "$NetBSD$"); + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +#include "npf_impl.h" + +NPF_EXT_MODULE(npf_ext_route, ""); + +#define NPFEXT_ROUTE_VER 1 + +#define NPF_LOGADDR(buf, a) \ + snprintf(buf, sizeof(buf), "%u.%u.%u.%u", \ + ((a)->s_addr >> 24) & 0xFF, \ + ((a)->s_addr >> 16) & 0xFF, \ + ((a)->s_addr >> 8) & 0xFF, \ + (a)->s_addr & 0xFF) + +static void * npf_ext_route_id; + +typedef struct { + char ifname[IFNAMSIZ]; +} npf_ext_route_t; + +static int +npf_route_ctor(npf_rproc_t *rp, const nvlist_t* params) +{ + npf_ext_route_t *meta; + const char *ifname; + + meta = kmem_zalloc(sizeof(*meta), KM_SLEEP); + ifname = nvlist_get_string(params, "route-interface"); + + if (!ifname) + return EINVAL; + + /* XXX use something like npf_ifmap */ + strlcpy(meta->ifname, ifname, IFNAMSIZ); + npf_rproc_assign(rp, meta); + return 0; +} + +static void +npf_route_dtor(npf_rproc_t *rp, void *meta) +{ + kmem_free(meta, sizeof(npf_ext_route_t)); +} + +static void +npf_chcksum(struct ifnet *ifp1, struct mbuf *m0, struct ip *ip1, int *sw_csum) +{ + int hlen; + struct mbuf *m = m0; + struct ip *ip = ip1; + struct ifnet *ifp = ifp1; + int csum; + + /* NB: This code is copied from ip_output */ + hlen = ip->ip_hl << 2; + ip->ip_sum = 0; + m->m_pkthdr.csum_data |= hlen << 16; + + /* Maybe skip checksums on loopback interfaces. */ + if (IN_NEED_CHECKSUM(ifp, M_CSUM_IPv4)) { + m->m_pkthdr.csum_flags |= M_CSUM_IPv4; + } + + *sw_csum = m->m_pkthdr.csum_flags & ~ifp->if_csum_flags_tx; + csum = *sw_csum; + + if ((m->m_pkthdr.csum_flags & M_CSUM_TSOv4) == 0) { + /* + * Perform any checksums that the hardware can't do + * for us. + * + * XXX Does any hardware require the {th,uh}_sum + * XXX fields to be 0? + */ + if (csum & M_CSUM_IPv4) { + KASSERT(IN_NEED_CHECKSUM(ifp, M_CSUM_IPv4)); + ip->ip_sum = in_cksum(m, hlen); + m->m_pkthdr.csum_flags &= ~M_CSUM_IPv4; + } + if (csum & (M_CSUM_TCPv4|M_CSUM_UDPv4)) { + + if (IN_NEED_CHECKSUM(ifp, + csum & (M_CSUM_TCPv4|M_CSUM_UDPv4))) { + in_undefer_cksum_tcpudp(m); + } + m->m_pkthdr.csum_flags &= + ~(M_CSUM_TCPv4|M_CSUM_UDPv4); + + } + } +} + +static int +npf_fragment(npf_t *npf, struct ifnet *ifp, struct ip *ip1, + struct mbuf **m0, struct sockaddr_in *dst) +{ + int error; + struct ip *ip = ip1; + struct mbuf *m = *m0; + + /* + * Too large for interface; fragment if possible. + * Must be able to put at least 8 bytes per fragment. + */ + if (ntohs(ip->ip_off) & IP_DF) { + npf_stats_inc(npf, NPF_STAT_NOFRAGMENT); + icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_NEEDFRAG, 0, + ifp->if_mtu); + error = -1; + return error; + } + /* + * We can't use HW checksumming if we're about to fragment the packet. + * + * XXX Some hardware can do this. + */ + if (m->m_pkthdr.csum_flags & (M_CSUM_TCPv4|M_CSUM_UDPv4)) { + if (IN_NEED_CHECKSUM(ifp, + m->m_pkthdr.csum_flags & (M_CSUM_TCPv4|M_CSUM_UDPv4))) { + in_undefer_cksum_tcpudp(m); + } + m->m_pkthdr.csum_flags &= ~(M_CSUM_TCPv4|M_CSUM_UDPv4); + } + + error = ip_fragment(m, ifp, ifp->if_mtu); + if (error) { + m = NULL; + return error; + } + + for (; m; m = *m0) { + *m0 = m->m_nextpkt; + m->m_nextpkt = NULL; + if (error) { + m_freem(m); + continue; + } + + KASSERT((m->m_pkthdr.csum_flags & + (M_CSUM_UDPv4 | M_CSUM_TCPv4)) == 0); + error = ip_if_output(ifp, m, sintocsa(dst), NULL); + } + + if (error == 0) { + npf_stats_inc(npf, NPF_STAT_FRAGMENTS); + } + + return error; +} + +/* + * Ensure sending address is valid. + * Returns 0 on success, -1 if an error should be sent back or 1 + * if the packet could be dropped without error (protocol dependent). + */ +static int +ip_ifaddrvalid(const struct in_ifaddr *ia) +{ + + if (ia->ia_addr.sin_addr.s_addr == INADDR_ANY) + return 0; + + if (ia->ia4_flags & IN_IFF_DUPLICATED) + return -1; + else if (ia->ia4_flags & (IN_IFF_TENTATIVE | IN_IFF_DETACHED)) + return 1; + + return 0; +} + +/* + * search for the source address structure to + * maintain output statistics, and verify address + * validity + */ +static int +npf_validate_saddr(npf_t * npf, struct ip *ip, struct ifnet* ifp) +{ + int error = 0; + struct in_ifaddr *ia = NULL; + union { + struct sockaddr sa; + struct sockaddr_in sin; + } usrc; + + struct psref psref_ia; + + KASSERT(ia == NULL); + sockaddr_in_init(&usrc.sin, &ip->ip_src, 0); + ia = ifatoia(ifaof_ifpforaddr_psref(&usrc.sa, ifp, &psref_ia)); + + /* + * Ensure we only send from a valid address. + * A NULL address is valid because the packet could be + * generated from a packet filter. + */ + if (ia != NULL && + (error = ip_ifaddrvalid(ia)) != 0) + { + char buf[32]; + NPF_LOGADDR(buf, &ip->ip_src); + log(LOG_ERR, + "refusing to send from invalid address %s (pid %d)\n", + buf, curproc->p_pid); + + npf_stats_inc(npf, NPF_STAT_NOREROUTE); + if (error == 1) + /* + * Address exists, but is tentative or detached. + * We can't send from it because it's invalid, + * so we drop the packet. + */ + error = 0; + else + error = EADDRNOTAVAIL; + + } + ia4_release(ia, &psref_ia); + return error; +} + +#if defined(INET6) +/* this code is copied from ip6_output*/ +static void +npf_validate_s6addr(struct mbuf *m0, struct ifnet *ifp1, int *sw_csum) +{ + struct in6_ifaddr *ia6; + struct mbuf *m = m0; + struct ifnet *ifp = ifp1; + struct ip6_hdr *ip6; + int csum; + int s; + + ip6 = mtod(m, struct ip6_hdr *); + s = pserialize_read_enter(); + ia6 = in6_ifawithifp(ifp, &ip6->ip6_src); + if (ia6) { + /* Record statistics for this interface address. */ + ia6->ia_ifa.ifa_data.ifad_outbytes += m->m_pkthdr.len; + } + pserialize_read_exit(s); + + /* check sum */ + *sw_csum = m->m_pkthdr.csum_flags & ~ifp->if_csum_flags_tx; + csum = *sw_csum; + + if ((csum & (M_CSUM_UDPv6|M_CSUM_TCPv6)) != 0) { + if (IN6_NEED_CHECKSUM(ifp, + csum & (M_CSUM_UDPv6|M_CSUM_TCPv6))) { + in6_undefer_cksum_tcpudp(m); + } + m->m_pkthdr.csum_flags &= ~(M_CSUM_UDPv6|M_CSUM_TCPv6); + } +} +#endif + +/* main routing function for kernel module */ +static bool +npf_route(npf_cache_t *npc, void *meta, const npf_match_info_t __unused *mi, int *decision) +{ + struct mbuf *m0 = nbuf_head_mbuf(npc->npc_nbuf); + const npf_ext_route_t *route = meta; + npf_t *npf = npf_getkernctx(); + struct ifnet *ifp; + int error = -1; + int sw_csum; + + union { + struct sockaddr_in v4; + struct sockaddr_in6 v6; + } dst; + + /* Skip, if already blocking. + * also when routing is applied to a statful rule, incoming packets + * are routed since the procedure becomes attached to the connection + * and hence will not be desirable + */ + if (*decision == NPF_DECISION_BLOCK || + (mi->mi_di == PFIL_IN)) { + return true; + } + + /* global lock for interface lookup */ + KERNEL_LOCK(1, NULL); + ifp = ifunit(route->ifname); + if (ifp == NULL) { + /* XXX: oops */ + goto bad; + } + + if (npf_iscached(npc, NPC_IP6)) { +#if defined(INET6) + struct ip6_hdr *ip6 = npc->npc_ip.v6; + sockaddr_in6_init(&dst.v6, &ip6->ip6_dst, 0, 0, 0); + + if (IN6_IS_SCOPE_EMBEDDABLE(&dst.v6.sin6_addr)) { + error = in6_setscope(&dst.v6.sin6_addr, ifp, NULL); + if (error) { + goto bad; + } + } + + npf_validate_s6addr(m0, ifp, &sw_csum); + + if (m0->m_pkthdr.len <= ifp->if_mtu) { + if (__predict_false(sw_csum & M_CSUM_TSOv6)) { + /* + * TSO6 is required by a packet, but disabled for + * the interface. + */ + error = ip6_tso_output(ifp, ifp, m0, &dst.v6, NULL); + } else + error = ip6_if_output(ifp, ifp, m0, &dst.v6, NULL); + + } else { + /* router not allowed to fragmenrt */ + npf_stats_inc(npf, NPF_STAT_NOFRAGMENT); + icmp6_error(m0, ICMP6_PACKET_TOO_BIG, 0, ifp->if_mtu); + } +#endif + } else if (npf_iscached(npc, NPC_IP4)) { + struct ip *ip = npc->npc_ip.v4; + struct mbuf *m = m0; + + KASSERT(ip != NULL); + KASSERT(m != NULL); + + /* + * NB: This code is copied from ip_output and re-arranged + * checks fragmentation, checksum and source address validity + */ + sockaddr_in_init(&dst.v4, &ip->ip_dst, 0); + + error = npf_validate_saddr(npf, ip, ifp); + if (error) + goto bad; + + if (ntohs(ip->ip_len) > ifp->if_mtu) + goto fragment; + + npf_chcksum(ifp, m, ip, &sw_csum); + + /* Send it */ + if (__predict_false(sw_csum & M_CSUM_TSOv4)) { + /* + * TSO4 is required by a packet, but disabled for + * the interface. + */ + error = ip_tso_output(ifp, m, sintocsa(&dst.v4), NULL); + } else + error = ip_if_output(ifp, m, sintocsa(&dst.v4), NULL); + + if (error) { + goto bad; + } + goto done; + +fragment: + error = npf_fragment(npf, ifp, ip, &m0, &dst.v4); + + } + + /* check for any error and exit */ + if (error) { + goto bad; + } + +/* + * if (error || m == NULL) { + * IP_STATINC(IP_STAT_PFILDROP_OUT); + * goto done; + * } + * for routing procedures, we reverse the returns + * because we need the kernel to stop processing the mbuf + * after we leave the filtering context + */ +done: + npf_stats_inc(npf, NPF_STAT_REROUTE); + m0 = NULL; + KERNEL_UNLOCK_ONE(NULL); + return false; + +bad: + npf_stats_inc(npf, NPF_STAT_NOREROUTE); + m_freem(m0); + m0 = NULL; + KERNEL_UNLOCK_ONE(NULL); + return true; +} + +__dso_public int +npf_ext_route_init(npf_t *npf) +{ + static const npf_ext_ops_t npf_route_ops = { + .version = NPFEXT_ROUTE_VER, + .ctx = NULL, + .ctor = npf_route_ctor, + .dtor = npf_route_dtor, + .proc = npf_route + }; + npf_ext_route_id = npf_ext_register(npf, "route", &npf_route_ops); + return npf_ext_route_id ? 0 : EEXIST; +} + +__dso_public int +npf_ext_route_fini(npf_t *npf) +{ + return npf_ext_unregister(npf, npf_ext_route_id); +} + +#ifdef _KERNEL +static int +npf_ext_route_modcmd(modcmd_t cmd, void *arg) +{ + npf_t *npf = npf_getkernctx(); + + switch (cmd) { + case MODULE_CMD_INIT: + return npf_ext_route_init(npf); + case MODULE_CMD_FINI: + return npf_ext_route_fini(npf); + case MODULE_CMD_AUTOUNLOAD: + /* Allow auto-unload only if NPF permits it. */ + return npf_autounload_p() ? 0 : EBUSY; + default: + return ENOTTY; + } + return 0; +} +#endif diff --git a/sys/net/npf/npf_handler.c b/sys/net/npf/npf_handler.c index ee56caee4e2da..6488714ea7449 100644 --- a/sys/net/npf/npf_handler.c +++ b/sys/net/npf/npf_handler.c @@ -292,8 +292,14 @@ npfk_packet_handler(npf_t *npf, struct mbuf **mp, ifnet_t *ifp, int di) if (rp && !npf_rproc_run(&npc, rp, &mi, &decision)) { if (con) { npf_conn_release(con); + + /* ensure that reference is released only on connection in stateful rules */ + if (is_rproc_route(rp)) + return 0; } + npf_rproc_release(rp); + /* mbuf already freed */ return 0; } diff --git a/sys/net/npf/npf_impl.h b/sys/net/npf/npf_impl.h index b545a33bc9102..314c878e0d9be 100644 --- a/sys/net/npf/npf_impl.h +++ b/sys/net/npf/npf_impl.h @@ -457,6 +457,7 @@ void npf_rprocset_destroy(npf_rprocset_t *); npf_rproc_t * npf_rprocset_lookup(npf_rprocset_t *, const char *); void npf_rprocset_insert(npf_rprocset_t *, npf_rproc_t *); int npf_rprocset_export(const npf_rprocset_t *, nvlist_t *); +int is_rproc_route(npf_rproc_t *); npf_rproc_t * npf_rproc_create(const nvlist_t *); void npf_rproc_acquire(npf_rproc_t *); diff --git a/sys/net/npf/npf_rproc.c b/sys/net/npf/npf_rproc.c index 675cf4ec2418e..407a1b6e51cce 100644 --- a/sys/net/npf/npf_rproc.c +++ b/sys/net/npf/npf_rproc.c @@ -388,3 +388,9 @@ npf_rproc_run(npf_cache_t *npc, npf_rproc_t *rp, const npf_match_info_t *mi, return true; } + +int +is_rproc_route(npf_rproc_t *rp) +{ + return !strcmp(rp->rp_name, "route"); +} diff --git a/usr.sbin/npf/npfctl/npfctl.c b/usr.sbin/npf/npfctl/npfctl.c index dc11be377c516..bcb304de5fdb8 100644 --- a/usr.sbin/npf/npfctl/npfctl.c +++ b/usr.sbin/npf/npfctl/npfctl.c @@ -171,9 +171,13 @@ npfctl_print_stats(int fd) { -1, "Fragmentation" }, { NPF_STAT_FRAGMENTS, "fragments" }, + { NPF_STAT_NOFRAGMENT, "failed fragmentation" }, { NPF_STAT_REASSEMBLY, "reassembled" }, { NPF_STAT_REASSFAIL, "failed reassembly" }, + { -1, "Routing" }, + { NPF_STAT_REROUTE, "re-routing" }, + { NPF_STAT_NOREROUTE, "failed re-routing" }, { -1, "Other" }, { NPF_STAT_ERROR, "unexpected errors" }, };