Index: net/if.c =================================================================== RCS file: /home/kmacy/devel/ncvs/src/sys/net/if.c,v retrieving revision 1.272 diff -d -u -r1.272 if.c --- net/if.c 16 May 2007 19:59:01 -0000 1.272 +++ net/if.c 31 Jul 2007 00:04:39 -0000 @@ -2706,6 +2706,41 @@ (*(ifp)->if_start)(ifp); } +#ifdef IFNET_MULTIQUEUE +int ifnet_multiqueue = 1; + +int +if_mq_start(struct ifnet *ifp, int32_t cookie, struct mbuf *m) +{ + + KASSERT((ifp->if_flags & IFF_NEEDSGIANT) == 0, ("IFF_NEEDSGIANT set on multi queue interface")); + + return (*(ifp)->if_mq_start)(ifp, cookie, m); +} + + +int +if_mq_enqueue_packet(struct ifnet *ifp, int32_t cookie, struct mbuf *m) +{ + + KASSERT((ifp->if_flags & IFF_NEEDSGIANT) == 0, ("IFF_NEEDSGIANT set on multi queue interface")); + + return (*(ifp)->if_mq_enqueue_packet)(ifp, cookie, m); +} + +int32_t +if_mq_get_cookie(struct ifnet *ifp, struct in6_addr *lip, uint16_t lport, struct in6_addr *rip, uint16_t rport, int ipv6) +{ + + KASSERT((ifp->if_flags & IFF_NEEDSGIANT) == 0, ("IFF_NEEDSGIANT set on multi queue interface")); + + return (*(ifp)->if_mq_get_cookie)(ifp, lip, lport, rip, rport, ipv6); +} +#else +int ifnet_multiqueue = 1; +#endif + + static void if_start_deferred(void *context, int pending) { Index: net/if.h =================================================================== RCS file: /home/kmacy/devel/ncvs/src/sys/net/if.h,v retrieving revision 1.108 diff -d -u -r1.108 if.h --- net/if.h 11 Jun 2007 20:08:11 -0000 1.108 +++ net/if.h 28 Jul 2007 21:42:19 -0000 @@ -150,6 +150,8 @@ #define IFF_MONITOR 0x40000 /* (n) user-requested monitor mode */ #define IFF_STATICARP 0x80000 /* (n) static ARP */ #define IFF_NEEDSGIANT 0x100000 /* (i) hold Giant over if_start calls */ +#define IFF_MULTIQ 0x200000 /* (i) driver has multiple queues and manages them privately */ + /* * Old names for driver flags so that user space tools can continue to use Index: net/if_var.h =================================================================== RCS file: /home/kmacy/devel/ncvs/src/sys/net/if_var.h,v retrieving revision 1.115 diff -d -u -r1.115 if_var.h --- net/if_var.h 16 May 2007 18:37:37 -0000 1.115 +++ net/if_var.h 31 Jul 2007 06:55:48 -0000 @@ -86,6 +86,10 @@ #define IF_DUNIT_NONE -1 #include +#ifdef IFNET_MULTIQUEUE +#include +#endif + TAILQ_HEAD(ifnethead, ifnet); /* we use TAILQs so that the order of */ TAILQ_HEAD(ifaddrhead, ifaddr); /* instantiation is preserved in the list */ @@ -152,6 +156,14 @@ (struct ifnet *, struct mbuf *); void (*if_start) /* initiate output routine */ (struct ifnet *); +#ifdef IFNET_MULTIQUEUE + int (*if_mq_start) /* initiate output routine with immediate */ + (struct ifnet *, int32_t, struct mbuf *); + int (*if_mq_enqueue_packet) /* enqueue packet to the appropriate queue */ + (struct ifnet *, int32_t, struct mbuf *); + int32_t (*if_mq_get_cookie) /* calculate the txq cookie for this connection */ + (struct ifnet *, struct in6_addr *, uint16_t, struct in6_addr *, uint16_t, int); +#endif int (*if_ioctl) /* ioctl routine */ (struct ifnet *, u_long, caddr_t); void (*if_watchdog) /* timer routine */ @@ -379,6 +391,13 @@ void if_start(struct ifnet *); +#ifdef IFNET_MULTIQUEUE +int if_mq_start(struct ifnet *, int32_t, struct mbuf *); +int if_mq_enqueue_packet(struct ifnet *, int32_t, struct mbuf *); +int32_t if_mq_get_cookie(struct ifnet *ifp, struct in6_addr *lip, uint16_t lport, struct in6_addr *rip, uint16_t rport, int ipv6); +extern int ifnet_multiqueue; /* allow driver module to confirm that multiqueue is supported */ +#endif + #define IFQ_ENQUEUE(ifq, m, err) \ do { \ IF_LOCK(ifq); \ @@ -459,26 +478,70 @@ #define IFQ_INC_DROPS(ifq) ((ifq)->ifq_drops++) #define IFQ_SET_MAXLEN(ifq, len) ((ifq)->ifq_maxlen = (len)) +#ifdef IFNET_MULTIQUEUE +#define IFQ_HANDOFF_ADJ_MQ(ifp, m, adj, err, cookie) \ +do { \ + int len; \ + short mflags; \ + \ + len = (m)->m_pkthdr.len; \ + mflags = (m)->m_flags; \ + err = if_mq_start((ifp), cookie, m); \ + if ((err) == 0) { \ + (ifp)->if_obytes += len + (adj); \ + if (mflags & M_MCAST) \ + (ifp)->if_omcasts++; \ + } \ +} while (0) +#define IFQ_HANDOFF_MQ(ifp, m, err, cookie) \ + IFQ_HANDOFF_ADJ_MQ(ifp, m, 0, err, cookie) +#endif + /* * The IFF_DRV_OACTIVE test should really occur in the device driver, not in * the handoff logic, as that flag is locked by the device driver. */ +#ifdef IFNET_MULTIQUEUE #define IFQ_HANDOFF_ADJ(ifp, m, adj, err) \ do { \ int len; \ short mflags; \ - \ + \ len = (m)->m_pkthdr.len; \ mflags = (m)->m_flags; \ - IFQ_ENQUEUE(&(ifp)->if_snd, m, err); \ + if ((ifp)->if_flags & IFF_MULTIQ) \ + err = if_mq_start((ifp), -1, m); \ + else \ + IFQ_ENQUEUE(&(ifp)->if_snd, m, err); \ if ((err) == 0) { \ (ifp)->if_obytes += len + (adj); \ if (mflags & M_MCAST) \ (ifp)->if_omcasts++; \ - if (((ifp)->if_drv_flags & IFF_DRV_OACTIVE) == 0) \ + if (((ifp)->if_drv_flags & IFF_DRV_OACTIVE) == 0 && \ + ((ifp)->if_flags & IFF_MULTIQ) == 0) \ + if_start(ifp); \ + } \ +} while (0) +#else +#define IFQ_HANDOFF_ADJ(ifp, m, adj, err) \ +do { \ + int len; \ + short mflags; \ + \ + len = (m)->m_pkthdr.len; \ + mflags = (m)->m_flags; \ + IFQ_ENQUEUE(&(ifp)->if_snd, m, err); \ + if ((err) == 0) { \ + (ifp)->if_obytes += len + (adj); \ + if (mflags & M_MCAST) \ + (ifp)->if_omcasts++; \ + if (((ifp)->if_drv_flags & IFF_DRV_OACTIVE) == 0 && \ + ((ifp)->if_flags & IFF_MULTIQ) == 0) \ if_start(ifp); \ } \ } while (0) +#endif + #define IFQ_HANDOFF(ifp, m, err) \ IFQ_HANDOFF_ADJ(ifp, m, 0, err) Index: conf/options =================================================================== RCS file: /home/kmacy/devel/ncvs/src/sys/conf/options,v retrieving revision 1.603 diff -d -u -r1.603 options --- conf/options 24 Jul 2007 15:35:01 -0000 1.603 +++ conf/options 30 Jul 2007 20:42:42 -0000 @@ -547,6 +547,7 @@ RWLOCK_NOINLINE opt_global.h SX_NOINLINE opt_global.h VFS_BIO_DEBUG opt_global.h +IFNET_MULTIQUEUE opt_global.h # These are VM related options VM_KMEM_SIZE opt_vm.h Index: modules/cxgb/Makefile =================================================================== RCS file: /home/kmacy/devel/ncvs/src/sys/modules/cxgb/Makefile,v retrieving revision 1.9 diff -d -u -r1.9 Makefile --- modules/cxgb/Makefile 17 Jul 2007 06:50:35 -0000 1.9 +++ modules/cxgb/Makefile 1 Aug 2007 07:34:38 -0000 @@ -11,7 +11,11 @@ SRCS+= uipc_mvec.c CFLAGS+= -DCONFIG_CHELSIO_T3_CORE -g -DCONFIG_DEFINED -DDEFAULT_JUMBO -I${CXGB} -#CFLAGS+= -DINVARIANT_SUPPORT -DINVARIANTS +CFLAGS+= -DIFNET_MULTIQUEUE +CFLAGS+= -DINVARIANT_SUPPORT -DINVARIANTS + +#CFLAGS+= -DDEBUG -DDEBUG_PRINT + .if ${MACHINE_ARCH} != "ia64" # ld is broken on ia64 Index: dev/cxgb/cxgb_adapter.h =================================================================== RCS file: /home/kmacy/devel/ncvs/src/sys/dev/cxgb/cxgb_adapter.h,v retrieving revision 1.14 diff -d -u -r1.14 cxgb_adapter.h --- dev/cxgb/cxgb_adapter.h 17 Jul 2007 06:50:33 -0000 1.14 +++ dev/cxgb/cxgb_adapter.h 1 Aug 2007 02:02:33 -0000 @@ -117,9 +117,10 @@ #else struct mtx lock; #endif - int port; + int port_id; uint8_t hw_addr[ETHER_ADDR_LEN]; - uint8_t nqsets; + uint8_t tx_chan; + uint8_t nqsets; uint8_t first_qset; struct taskqueue *tq; struct task start_task; @@ -267,6 +268,8 @@ bus_dmamap_t desc_map; bus_dma_tag_t entry_tag; struct mbuf_head sendq; + struct mbuf_ring txq_mr; + int32_t txq_drops; struct mtx lock; #define TXQ_NAME_LEN 32 char lockbuf[TXQ_NAME_LEN]; @@ -286,6 +289,9 @@ #define SGE_PSTAT_MAX (SGE_PSTATS_LRO_X_STREAMS+1) +#define QS_EXITING 0x1 +#define QS_RUNNING 0x2 + struct sge_qset { struct sge_rspq rspq; struct sge_fl fl[SGE_RXQ_PER_SET]; @@ -294,7 +300,9 @@ uint32_t txq_stopped; /* which Tx queues are stopped */ uint64_t port_stats[SGE_PSTAT_MAX]; struct port_info *port; - int idx; /* qset # */ + int idx; /* qset # */ + int cpuid; + int flags; }; struct sge { @@ -334,9 +342,10 @@ struct resource *msix_irq_res[SGE_QSETS]; int msix_irq_rid[SGE_QSETS]; void *msix_intr_tag[SGE_QSETS]; - uint8_t rxpkt_map[8]; /* maps RX_PKT interface values to port ids */ - uint8_t rrss_map[SGE_QSETS]; /* revers RSS map table */ - + uint8_t rxpkt_map[8]; /* maps RX_PKT interface values to port ids */ + uint8_t rrss_map[SGE_QSETS]; /* reverse RSS map table - maps qidx to smallest cookie */ + uint16_t rspq_map[RSS_TABLE_SIZE]; /* maps 7-bit cookie to qidx */ + struct filter_info *filters; /* Tasks */ @@ -387,7 +396,7 @@ struct t3_rx_mode { uint32_t idx; - struct port_info *port; + struct port_info *port; }; @@ -466,7 +475,7 @@ uint8_t *macaddr = NULL; if (rm->idx == 0) - macaddr = rm->port->hw_addr; + macaddr = (uint8_t *)rm->port->hw_addr; rm->idx++; return (macaddr); @@ -507,7 +516,7 @@ void t3b_intr(void *data); void t3_intr_msi(void *data); void t3_intr_msix(void *data); -int t3_encap(struct port_info *, struct mbuf **); +int t3_encap(struct sge_txq *, struct mbuf **); int t3_sge_init_adapter(adapter_t *); int t3_sge_init_port(struct port_info *); Index: dev/cxgb/cxgb_main.c =================================================================== RCS file: /home/kmacy/devel/ncvs/src/sys/dev/cxgb/cxgb_main.c,v retrieving revision 1.28 diff -d -u -r1.28 cxgb_main.c --- dev/cxgb/cxgb_main.c 17 Jul 2007 06:50:33 -0000 1.28 +++ dev/cxgb/cxgb_main.c 1 Aug 2007 07:42:36 -0000 @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -47,10 +48,13 @@ #include #include #include +#include +#include #include #include #include #include +#include #include #include @@ -64,10 +68,13 @@ #include #include #include -#include +#include +#include +#include #include #include + #include #include #include @@ -100,6 +107,7 @@ static void cxgb_down_locked(struct adapter *sc); static void cxgb_tick(void *); static void setup_rss(adapter_t *sc); +static inline int cxgb_tx_common(struct ifnet *ifp, struct sge_txq *txq); /* Attachment glue for the PCI controller end of the device. Each port of * the device is attached separately, as defined later. @@ -117,6 +125,16 @@ static int offload_close(struct toedev *tdev); #endif +#ifdef IFNET_MULTIQUEUE +static inline int cxgb_pcpu_enqueue_packet(struct ifnet *ifp, int32_t cpuid, struct mbuf *m); +static int cxgb_pcpu_start(struct ifnet *ifp, int32_t cpuid, struct mbuf *m); +static inline int32_t cxgb_pcpu_calc_cookie(struct ifnet *ifp, struct mbuf *immpkt); +static int32_t cxgb_pcpu_get_cookie(struct ifnet *ifp, struct in6_addr *lip, uint16_t lport, + struct in6_addr *rip, uint16_t rport, int ipv6); +static void cxgb_pcpu_start_proc(void *arg); +static int cxgb_pcpu_cookie_to_qidx(struct port_info *, int32_t cookie); +#endif + static device_method_t cxgb_controller_methods[] = { DEVMETHOD(device_probe, cxgb_controller_probe), @@ -208,11 +226,40 @@ * The driver uses an auto-queue algorithm by default. * To disable it and force a single queue-set per port, use singleq = 1. */ +#ifdef IFNET_MULTIQUEUE +static int singleq = 0; +#else static int singleq = 1; +#endif TUNABLE_INT("hw.cxgb.singleq", &singleq); SYSCTL_UINT(_hw_cxgb, OID_AUTO, singleq, CTLFLAG_RDTUN, &singleq, 0, "use a single queue-set per port"); + +#ifdef IFNET_MULTIQUEUE +static int cxgb_pcpu_tx_coalesce = 0; +TUNABLE_INT("hw.cxgb.tx_coalesce", &cxgb_pcpu_tx_coalesce); +SYSCTL_UINT(_hw_cxgb, OID_AUTO, tx_coalesce, CTLFLAG_RDTUN, &cxgb_pcpu_tx_coalesce, 0, + "coalesce small packets into a single work request"); + +static int sleep_ticks = 1; +TUNABLE_INT("hw.cxgb.sleep_ticks", &sleep_ticks); +SYSCTL_UINT(_hw_cxgb, OID_AUTO, sleep_ticks, CTLFLAG_RDTUN, &sleep_ticks, 0, + "ticks to sleep between checking pcpu queues"); + +int cxgb_txq_mbuf_ring_size = 2048; +TUNABLE_INT("hw.cxgb.txq_mr_size", &cxgb_txq_mbuf_ring_size); +SYSCTL_UINT(_hw_cxgb, OID_AUTO, txq_mr_size, CTLFLAG_RDTUN, &cxgb_txq_mbuf_ring_size, 0, + "size of per-queue mbuf ring"); +#else +int cxgb_txq_mbuf_ring_size = 0; +#endif + + +/* + * Add tunable + */ + enum { MAX_TXQ_ENTRIES = 16384, MAX_CTRL_TXQ_ENTRIES = 1024, @@ -557,7 +604,8 @@ sc->port[i].adapter = sc; sc->port[i].nqsets = port_qsets; sc->port[i].first_qset = i*port_qsets; - sc->port[i].port = i; + sc->port[i].port_id = i; + sc->port[i].tx_chan = i >= ai->nports0; sc->portdev[i] = child; device_set_softc(child, &sc->port[i]); } @@ -588,6 +636,7 @@ G_FW_VERSION_MICRO(vers)); t3_add_sysctls(sc); + sleep_ticks = hz >> 4; /* default value */ out: if (error) cxgb_free(sc); @@ -611,13 +660,32 @@ cxgb_free(struct adapter *sc) { int i; - - ADAPTER_LOCK(sc); +#ifdef IFNET_MULTIQUEUE + int j; + for (i = 0; i < sc->params.nports; i++) { + struct port_info *pi = &sc->port[i]; + int first = pi->first_qset; + for (j = 0; j < pi->nqsets; j++) { + struct sge_qset *qs = &sc->sge.qs[first + j]; + wakeup(qs); + qs->flags |= QS_EXITING; + tsleep(&sc, 0, "cxgb unload 0", hz>>2); + while (qs->flags & QS_RUNNING) { + device_printf(sc->dev, "qset thread %d still running - sleeping\n", first + j); + tsleep(&sc, 0, "cxgb unload 1", 2*hz); + } + } + } + wakeup(sc); /* - * drops the lock + * Wait for pcpu threads to exit */ + +#endif + + ADAPTER_LOCK(sc); cxgb_down_locked(sc); - + #ifdef MSI_SUPPORTED if (sc->flags & (USING_MSI | USING_MSIX)) { device_printf(sc->dev, "releasing msi message(s)\n"); @@ -630,7 +698,7 @@ bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid, sc->msix_regs_res); } - + t3_sge_deinit_sw(sc); if (sc->tq != NULL) { @@ -639,7 +707,7 @@ taskqueue_free(sc->tq); } - tsleep(&sc, 0, "cxgb unload", hz); + tsleep(&sc, 0, "cxgb unload", 2*hz); for (i = 0; i < (sc)->params.nports; ++i) { if (sc->portdev[i] != NULL) @@ -668,7 +736,6 @@ MTX_DESTROY(&sc->sge.reg_lock); MTX_DESTROY(&sc->elmer_lock); ADAPTER_LOCK_DEINIT(sc); - return; } @@ -905,7 +972,7 @@ p = device_get_softc(dev); - snprintf(buf, sizeof(buf), "Port %d %s", p->port, p->port_type->desc); + snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, p->port_type->desc); device_set_desc_copy(dev, buf); return (0); } @@ -936,6 +1003,7 @@ /* Don't enable TSO6 yet */ #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU) #define IFCAP_TSO4 0x0 +#define IFCAP_TSO6 0x0 #define CSUM_TSO 0x0 #endif @@ -950,7 +1018,7 @@ p = device_get_softc(dev); snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d", - device_get_unit(device_get_parent(dev)), p->port); + device_get_unit(device_get_parent(dev)), p->port_id); PORT_LOCK_INIT(p, p->lockbuf); /* Allocate an ifnet object and set it up */ @@ -969,6 +1037,16 @@ ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_ioctl = cxgb_ioctl; ifp->if_start = cxgb_start; +#ifdef IFNET_MULTIQUEUE + if (ifnet_multiqueue == 0) { + device_printf(dev, "cxgb compiled with multi-queue support, but kernel doesn't support it\n"); + return (EDOOFUS); + } + ifp->if_flags |= IFF_MULTIQ; + ifp->if_mq_start = cxgb_pcpu_start; + ifp->if_mq_enqueue_packet = cxgb_pcpu_enqueue_packet; + ifp->if_mq_get_cookie = cxgb_pcpu_get_cookie; +#endif ifp->if_timer = 0; /* Disable ifnet watchdog */ ifp->if_watchdog = NULL; @@ -1032,7 +1110,7 @@ } - snprintf(p->taskqbuf, TASKQ_NAME_LEN, "cxgb_port_taskq%d", p->port); + snprintf(p->taskqbuf, TASKQ_NAME_LEN, "cxgb_port_taskq%d", p->port_id); #ifdef TASKQUEUE_CURRENT /* Create a port for handling TX without starvation */ p->tq = taskqueue_create(p->taskqbuf, M_NOWAIT, @@ -1274,27 +1352,36 @@ int i; u_int nq[2]; uint8_t cpus[SGE_QSETS + 1]; - uint16_t rspq_map[RSS_TABLE_SIZE]; - - nq[0] = adap->port[0].nqsets; - nq[1] = max((u_int)adap->port[1].nqsets, 1U); for (i = 0; i < SGE_QSETS; ++i) cpus[i] = i; - cpus[SGE_QSETS] = 0xff; + cpus[SGE_QSETS] = 0xff; /* terminator */ + nq[0] = nq[1] = 0; + for_each_port(adap, i) { + const struct port_info *pi = adap2pinfo(adap, i); + + nq[pi->tx_chan] += pi->nqsets; + } + for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) { - rspq_map[i] = nq[0] ? i % nq[0] : 0; - rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0; + adap->rspq_map[i] = nq[0] ? i % nq[0] : 0; + adap->rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0; + } + device_printf(adap->dev, "RSS TABLE:\n"); + for (i = 0; i < RSS_TABLE_SIZE; i += 8) { + device_printf(adap->dev, "%2d %2d %2d %2d %2d %2d %2d %2d\n", + adap->rspq_map[i], adap->rspq_map[i+1], adap->rspq_map[i+2], adap->rspq_map[i+3], + adap->rspq_map[i+4], adap->rspq_map[i+5], adap->rspq_map[i+6], adap->rspq_map[i+7]); } /* Calculate the reverse RSS map table */ for (i = 0; i < RSS_TABLE_SIZE; ++i) - if (adap->rrss_map[rspq_map[i]] == 0xff) - adap->rrss_map[rspq_map[i]] = i; + if (adap->rrss_map[adap->rspq_map[i]] == 0xff) + adap->rrss_map[adap->rspq_map[i]] = i; t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN | F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN | - F_RRCPLMAPEN | V_RRCPLCPUSIZE(6), cpus, rspq_map); + F_RRCPLMAPEN | V_RRCPLCPUSIZE(6), cpus, adap->rspq_map); } @@ -1384,13 +1471,30 @@ bind_qsets(adapter_t *sc) { int i, j; + struct proc *p; + if (singleq) + return; + for (i = 0; i < (sc)->params.nports; ++i) { - const struct port_info *pi = adap2pinfo(sc, i); + struct port_info *pi = adap2pinfo(sc, i); + + for (j = 0; j < pi->nqsets; ++j) { +#ifdef IFNET_MULTIQUEUE + struct sge_qset *qs; - for (j = 0; j < pi->nqsets; ++j) + qs = &sc->sge.qs[pi->first_qset + j]; + qs->port = pi; + qs->cpuid = ((pi->first_qset + j) % mp_ncpus); + device_printf(sc->dev, "starting thread for %d\n", qs->cpuid); + + kthread_create(cxgb_pcpu_start_proc, qs, &p, + RFNOWAIT, 0, "cxgbsp"); + DELAY(200); +#endif send_pktsched_cmd(sc, 1, pi->first_qset + j, -1, -1, i); + } } } @@ -1704,7 +1808,7 @@ t3_intr_clear(sc); t3_sge_init_adapter(sc); } - setbit(&p->adapter->open_device_map, p->port); + setbit(&p->adapter->open_device_map, p->port_id); ADAPTER_UNLOCK(p->adapter); if (is_offload(sc) && !ofld_disable) { @@ -1714,10 +1818,10 @@ "Could not initialize offload capabilities\n"); } cxgb_link_start(p); - t3_link_changed(sc, p->port); + t3_link_changed(sc, p->port_id); ifp->if_baudrate = p->link_config.speed * 1000000; - t3_port_intr_enable(sc, p->port); + t3_port_intr_enable(sc, p->port_id); callout_reset(&sc->cxgb_tick_ch, sc->params.stats_update_period * hz, cxgb_tick, sc); @@ -1748,13 +1852,13 @@ ifp = p->ifp; - t3_port_intr_disable(p->adapter, p->port); + t3_port_intr_disable(p->adapter, p->port_id); ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); p->phy.ops->power_down(&p->phy, 1); t3_mac_disable(&p->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX); ADAPTER_LOCK(p->adapter); - clrbit(&p->adapter->open_device_map, p->port); + clrbit(&p->adapter->open_device_map, p->port_id); if (p->adapter->open_device_map == 0) { @@ -1882,34 +1986,484 @@ return (error); } +#ifdef IFNET_MULTIQUEUE +static inline int +cxgb_pcpu_enqueue_packet_(struct sge_qset *qs, struct mbuf *m) +{ + struct sge_txq *txq; + struct mbuf_ring *mr; + struct mbuf_head *mbq; + int err = 0; + + if (qs->flags & QS_EXITING) { + m_freem(m); + return (0); + } + txq = &qs->txq[TXQ_ETH]; + + DPRINTF("enqueueing packet to cpuid %d\n", qs->cpuid); + + + mbq = &txq->sendq; + if (curcpu == qs->cpuid) { + critical_enter(); + mbufq_tail(mbq, m); + critical_exit(); + } else { + int prod, cons, mask; + + mr = &txq->txq_mr; + mtx_lock(&txq->lock); + cons = mr->mr_cons; + prod = mr->mr_prod; + mask = mr->mr_size - 1; + if (((prod + 1) & mask) != cons) { + mr->mr_ring[prod] = m; + mr->mr_prod = (prod + 1) & mask; + } else { + err = ENOMEM; + txq->txq_drops++; + } + mtx_unlock(&txq->lock); + if (err) + m_freem(m); + } + + return (err); +} + static int -cxgb_start_tx(struct ifnet *ifp, uint32_t txmax) +cxgb_pcpu_enqueue_packet(struct ifnet *ifp, int32_t cookie, struct mbuf *m) +{ + struct port_info *pi; + struct sge_qset *qs; + int err, qidx; + int32_t calc_cookie; + + pi = ifp->if_softc; + err = 0; + + calc_cookie = (cookie != -1) ? cookie : cxgb_pcpu_calc_cookie(ifp, m); + qidx = cxgb_pcpu_cookie_to_qidx(pi, calc_cookie); + qs = &pi->adapter->sge.qs[qidx]; + + err = cxgb_pcpu_enqueue_packet_(qs, m); + + return (err); +} + +static struct mbuf * +cxgb_dequeue_packet(struct ifnet *unused, struct sge_txq *txq) { + struct mbuf *m, *tail, *head; + struct sge_qset *qs; + int count, size; + struct mbuf_head *mbq; + + mbq = &txq->sendq; + count = size = 0; + tail = NULL; + + qs = txq_to_qset(txq, TXQ_ETH); + if (qs->flags & QS_EXITING) + return (NULL); + + + critical_enter(); + for (head = m = mbufq_dequeue(mbq); m != NULL; m = mbufq_dequeue(mbq)) { + size += m->m_pkthdr.len; + count++; + + if (tail) + tail->m_nextpkt = m; + + if (count == 7 || size + mbufq_head_size(mbq) > 11*1024 || cxgb_pcpu_tx_coalesce == 0) + break; + tail = m; + } + critical_exit(); + + return (head); +} + + +static int32_t +cxgb_pcpu_get_cookie(struct ifnet *ifp, struct in6_addr *lip, uint16_t lport, struct in6_addr *rip, uint16_t rport, int ipv6) +{ + uint32_t base; + uint8_t buf[36]; + int count; + int32_t cookie; + + /* + * Can definitely bypass bcopy XXX + */ + if (ipv6 == 0) { + count = 12; + bcopy(rip, &buf[0], 4); + bcopy(lip, &buf[4], 4); + bcopy(&rport, &buf[8], 2); + bcopy(&lport, &buf[10], 2); + } else { + count = 36; + bcopy(rip, &buf[0], 16); + bcopy(lip, &buf[16], 16); + bcopy(&rport, &buf[32], 2); + bcopy(&lport, &buf[34], 2); + } + + base = 0xffffffff; + base = update_crc32(base, buf, count); + base = sctp_csum_finalize(base); + + /* + * Indirection table is 128 bits + * -> cookie indexes into indirection table which maps connection to queue + * -> RSS map maps queue to CPU + */ + cookie = (base & (RSS_TABLE_SIZE-1)); + + return (cookie); +} + +static int32_t +cxgb_pcpu_calc_cookie(struct ifnet *ifp, struct mbuf *immpkt) +{ + struct in6_addr lip, rip; + uint16_t lport, rport; + struct ether_header *eh; + int32_t cookie; + struct ip *ip; + struct ip6_hdr *ip6; + struct tcphdr *th; + struct udphdr *uh; + struct sctphdr *sh; + uint8_t *next, proto; + int etype; + + if (immpkt == NULL) + return -1; + + rport = lport = 0; + cookie = -1; + next = NULL; + eh = mtod(immpkt, struct ether_header *); + etype = ntohs(eh->ether_type); + + switch (etype) { + case ETHERTYPE_IP: + ip = (struct ip *)(eh + 1); + next = (uint8_t *)(ip + 1); + bcopy(&ip->ip_src, &lip, 4); + bcopy(&ip->ip_dst, &rip, 4); + proto = ip->ip_p; + break; + case ETHERTYPE_IPV6: + ip6 = (struct ip6_hdr *)(eh + 1); + next = (uint8_t *)(ip6 + 1); + bcopy(&ip6->ip6_src, &lip, sizeof(struct in6_addr)); + bcopy(&ip6->ip6_dst, &rip, sizeof(struct in6_addr)); + if (ip6->ip6_nxt == IPPROTO_HOPOPTS) { + struct ip6_hbh *hbh; + + hbh = (struct ip6_hbh *)(ip6 + 1); + proto = hbh->ip6h_nxt; + } else + proto = ip6->ip6_nxt; + break; + case ETHERTYPE_ARP: + default: + /* + * Default to queue zero + */ + proto = cookie = 0; + } + if (proto) { + switch (proto) { + case IPPROTO_TCP: + th = (struct tcphdr *)next; + lport = th->th_sport; + rport = th->th_dport; + break; + case IPPROTO_UDP: + uh = (struct udphdr *)next; + lport = uh->uh_sport; + rport = uh->uh_dport; + break; + case IPPROTO_SCTP: + sh = (struct sctphdr *)next; + lport = sh->src_port; + rport = sh->dest_port; + break; + default: + /* nothing to do */ + break; + } + } + + if (cookie) + cookie = cxgb_pcpu_get_cookie(ifp, &lip, lport, &rip, rport, (etype == ETHERTYPE_IPV6)); + + return (cookie); +} + +static inline int +cxgb_pcpu_pkt_coalesce(struct sge_txq *txq, struct mbuf *imm, int *complete) +{ + int prod, cons, mask, transferred; + struct mbuf_head *mbq; + struct mbuf_ring *mr; + struct mbuf **ring, *m; + + + mr = &txq->txq_mr; + ring = mr->mr_ring; + mask = mr->mr_size - 1; + + critical_enter(); + mbq = &txq->sendq; + cons = mr->mr_cons; + prod = mr->mr_prod; + while (cons != prod) { + m = ring[cons]; + cons = (cons + 1) & mask; + mbufq_tail(mbq, m); + transferred++; + } + mr->mr_cons = cons; + if (imm) + mbufq_tail(mbq, imm); + *complete = ((mbufq_size(mbq) > TX_WR_SIZE_MAX) || (mbufq_len(mbq) >= TX_WR_COUNT_MAX)); + critical_exit(); + + return (transferred); +} + + +static int +cxgb_pcpu_start_(struct port_info *pi, int32_t cookie, struct mbuf *immpkt, int tx_flush) +{ + int32_t calc_cookie; + int err, flush, complete, qidx; struct sge_qset *qs; struct sge_txq *txq; - struct port_info *p = ifp->if_softc; - struct mbuf *m0, *m = NULL; - int err, in_use_init; + adapter_t *sc; - if (!p->link_config.link_ok) + if (!pi->link_config.link_ok) { return (ENXIO); + } + sc = pi->adapter; + calc_cookie = ((cookie == -1) && immpkt) ? cxgb_pcpu_calc_cookie(pi->ifp, immpkt) : cookie; + qidx = cxgb_pcpu_cookie_to_qidx(pi, calc_cookie); + qs = &pi->adapter->sge.qs[qidx]; + + if (qs->flags & QS_EXITING) { + printf("exting\n"); + if (immpkt) + m_freem(immpkt); + return (0); + } + if (immpkt != NULL && tx_flush == 0) + DPRINTF("calc_cookie=%d\n", calc_cookie); - if (IFQ_DRV_IS_EMPTY(&ifp->if_snd)) - return (ENOBUFS); - qs = &p->adapter->sge.qs[p->first_qset]; - txq = &qs->txq[TXQ_ETH]; err = 0; + txq = &qs->txq[TXQ_ETH]; + /* + * A transmitter is already running on the current cpu + */ if (txq->flags & TXQ_TRANSMITTING) return (EINPROGRESS); + if (immpkt != NULL && tx_flush == 0) { + DPRINTF("immpkt=%p qidx=%d\n", immpkt, qidx); + DPRINTF("curcpu=%d sc->sge.qs[%d].cpuid=%d\n", curcpu, + qidx, sc->sge.qs[qidx].cpuid); + } + /* + * If we're passed a packet and it isn't outbound from this cpu + * we need to enqueue it be transmitted from the appropriate CPU + */ + if (immpkt && qs->cpuid != curcpu) { + cxgb_pcpu_enqueue_packet_(qs, immpkt); + immpkt = NULL; + } + + /* + * XXX Hard coding cpu to qset mapping + */ + if (curcpu > SGE_QSETS) + return (0); + + cxgb_pcpu_pkt_coalesce(txq, immpkt, &complete); + /* + * If coalescing is disabled OR a complete packet is ready OR we're being called from the flush thread + */ + flush = (cxgb_pcpu_tx_coalesce == 0) || complete || tx_flush; + + if (flush) + err = cxgb_tx_common(qs->port->ifp, txq); + + return (err); +} + +static int +cxgb_pcpu_start(struct ifnet *ifp, int32_t cookie, struct mbuf *immpkt) +{ + int err; + struct port_info *pi; + + pi = ifp->if_softc; + sched_pin(); + err = cxgb_pcpu_start_(pi, cookie, immpkt, FALSE); + sched_unpin(); + + return (err); +} + +static void +cxgb_start(struct ifnet *ifp) +{ + struct port_info *pi = ifp->if_softc; + struct sge_qset *qs; + struct mbuf *m, *head, *tail, *lhead, *ltail; + int calc_cookie, qidx; + + IFQ_LOCK(&ifp->if_snd); + IFQ_DEQUEUE_NOLOCK(&ifp->if_snd, m); + head = tail = m; + while (m != NULL) { + IFQ_DEQUEUE_NOLOCK(&ifp->if_snd, m); + tail->m_nextpkt = m; + tail = m; + } + IFQ_UNLOCK(&ifp->if_snd); + lhead = ltail = NULL; + sched_pin(); + for (m = head; m != NULL; m = head->m_nextpkt) { + calc_cookie = cxgb_pcpu_calc_cookie(ifp, m); + qidx = cxgb_pcpu_cookie_to_qidx(pi, calc_cookie); + qs = &pi->adapter->sge.qs[qidx]; + critical_enter(); + if (qs->cpuid == curcpu) { + if (lhead == NULL) + lhead = m; + else + ltail->m_nextpkt = m; + ltail = m; + critical_exit(); + } else { + critical_exit(); + cxgb_pcpu_enqueue_packet_(qs, m); + } + } + if (curcpu < SGE_QSETS) { + int32_t cookie; + + /* + * Assume one-to-one mapping of qset to CPU for now XXX + */ + cookie = pi->adapter->rrss_map[curcpu]; + (void)cxgb_pcpu_start_(pi, cookie, lhead, 0); + } + sched_unpin(); +} + +static void +cxgb_pcpu_start_proc(void *arg) +{ + struct sge_qset *qs = arg; + struct thread *td; + + td = curthread; + + qs->flags |= QS_RUNNING; + + thread_lock(td); + sched_bind(td, qs->cpuid); + thread_unlock(td); + + for (;;) { + if (qs->flags & QS_EXITING) + break; + + cxgb_pcpu_start_(qs->port, qs->cpuid, NULL, TRUE); + tsleep(qs, 0, "cxgbidle", sleep_ticks); + } + thread_lock(td); + sched_unbind(td); + thread_unlock(td); + + device_printf(qs->port->adapter->dev, "exiting thread for cpu%d\n", qs->cpuid); + qs->flags &= ~QS_RUNNING; + kthread_exit(0); +} + +static int +cxgb_pcpu_cookie_to_qidx(struct port_info *pi, int32_t cookie) +{ + int qidx, tmp; + + /* + * Will probably need to be changed for 4-port XXX + */ + tmp = pi->tx_chan ? cookie : cookie & ((RSS_TABLE_SIZE>>1)-1); + qidx = pi->adapter->rspq_map[tmp]; + + return (qidx); +} + +#else + +static inline struct mbuf * +cxgb_dequeue_packet(struct ifnet *ifp, struct sge_txq *unused) +{ + struct mbuf *m; + + IFQ_DRV_DEQUEUE(&ifp->if_snd, m); + return (m); +} + + +static void +cxgb_start(struct ifnet *ifp) +{ + struct port_info *pi = ifp->if_softc; + struct sge_qset *qs; + struct sge_txq *txq; + int err; + + qs = &pi->adapter->sge.qs[pi->first_qset]; + txq = &qs->txq[TXQ_ETH]; + + if (desc_reclaimable(txq) > TX_CLEAN_MAX_DESC >> 2) + taskqueue_enqueue(pi->tq, + &txq->qreclaim_task); + + err = cxgb_start_tx(ifp); + + if (err == 0) + taskqueue_enqueue(pi->tq, &pi->start_task); +} + +#endif /* IFNET_MULTIQUEUE */ + +static inline int +cxgb_tx_common(struct ifnet *ifp, struct sge_txq *txq) +{ + int err, in_use_init; + struct mbuf *m0, *m; + uint32_t txmax; + + txmax = TX_START_MAX_DESC; + err = 0; - mtx_lock(&txq->lock); txq->flags |= TXQ_TRANSMITTING; in_use_init = txq->in_use; while ((txq->in_use - in_use_init < txmax) && (txq->size > txq->in_use + TX_MAX_DESC)) { - IFQ_DRV_DEQUEUE(&ifp->if_snd, m); + + m = cxgb_dequeue_packet(ifp, txq); if (m == NULL) break; /* @@ -1930,6 +2484,7 @@ m_sanity(m0, 0); m0 = m; #endif +#ifdef notyet if (collapse_mbufs && m->m_pkthdr.len > MCLBYTES && m_collapse(m, TX_MAX_SEGS, &m0) == EFBIG) { if ((m0 = m_defrag(m, M_NOWAIT)) != NULL) { @@ -1939,24 +2494,51 @@ break; } m = m0; - if ((err = t3_encap(p, &m)) != 0) +#endif + if ((err = t3_encap(txq, &m)) != 0) break; BPF_MTAP(ifp, m); } txq->flags &= ~TXQ_TRANSMITTING; + + return (err); +} + +static int +cxgb_start_tx(struct ifnet *ifp) +{ + struct sge_qset *qs; + struct sge_txq *txq; + struct port_info *p = ifp->if_softc; + int err; + + if (!p->link_config.link_ok) + return (ENXIO); + + if (IFQ_DRV_IS_EMPTY(&ifp->if_snd)) + return (ENOBUFS); + + qs = &p->adapter->sge.qs[p->first_qset]; + txq = &qs->txq[TXQ_ETH]; + err = 0; + + if (txq->flags & TXQ_TRANSMITTING) + return (EINPROGRESS); + + mtx_lock(&txq->lock); + err = cxgb_tx_common(ifp, txq); mtx_unlock(&txq->lock); if (__predict_false(err)) { if (err == ENOMEM) { ifp->if_drv_flags |= IFF_DRV_OACTIVE; +#ifdef notyet IFQ_LOCK(&ifp->if_snd); IFQ_DRV_PREPEND(&ifp->if_snd, m); IFQ_UNLOCK(&ifp->if_snd); +#endif } - } - if (err == 0 && m == NULL) - err = ENOBUFS; - else if ((err == 0) && (txq->size <= txq->in_use + TX_MAX_DESC) && + } else if ((err == 0) && (txq->size <= txq->in_use + TX_MAX_DESC) && (ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) { ifp->if_drv_flags |= IFF_DRV_OACTIVE; err = ENOSPC; @@ -1980,32 +2562,10 @@ if (desc_reclaimable(txq) > TX_CLEAN_MAX_DESC >> 2) taskqueue_enqueue(pi->tq, &txq->qreclaim_task); - error = cxgb_start_tx(ifp, TX_START_MAX_DESC); + error = cxgb_start_tx(ifp); } while (error == 0); } -static void -cxgb_start(struct ifnet *ifp) -{ - struct port_info *pi = ifp->if_softc; - struct sge_qset *qs; - struct sge_txq *txq; - int err; - - qs = &pi->adapter->sge.qs[pi->first_qset]; - txq = &qs->txq[TXQ_ETH]; - - if (desc_reclaimable(txq) > TX_CLEAN_MAX_DESC >> 2) - taskqueue_enqueue(pi->tq, - &txq->qreclaim_task); - - err = cxgb_start_tx(ifp, TX_START_MAX_DESC); - - if (err == 0) - taskqueue_enqueue(pi->tq, &pi->start_task); -} - - static int cxgb_media_change(struct ifnet *ifp) { @@ -2119,7 +2679,7 @@ cxgb_set_rxmode(p); t3_link_start(&p->phy, mac, &p->link_config); t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); - t3_port_intr_enable(adapter, p->port); + t3_port_intr_enable(adapter, p->port_id); p->mac.stats.num_resets++; } PORT_UNLOCK(p); @@ -2527,7 +3087,7 @@ } case CHELSIO_SET_QSET_NUM: { struct ch_reg *edata = (struct ch_reg *)data; - unsigned int port_idx = pi->port; + unsigned int port_idx = pi->port_id; if (sc->flags & FULL_INIT_DONE) return (EBUSY); @@ -2856,3 +3416,4 @@ reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1), XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1)); } + Index: dev/cxgb/cxgb_offload.c =================================================================== RCS file: /home/kmacy/devel/ncvs/src/sys/dev/cxgb/cxgb_offload.c,v retrieving revision 1.6 diff -d -u -r1.6 cxgb_offload.c --- dev/cxgb/cxgb_offload.c 17 Jul 2007 06:50:33 -0000 1.6 +++ dev/cxgb/cxgb_offload.c 1 Aug 2007 02:24:46 -0000 @@ -1250,7 +1250,7 @@ } /* Add new L2T entry */ - e = t3_l2t_get(tdev, new, ((struct port_info *)new->rt_ifp->if_softc)->port); + e = t3_l2t_get(tdev, new, ((struct port_info *)new->rt_ifp->if_softc)->port_id); if (!e) { log(LOG_ERR, "%s: couldn't allocate new l2t entry!\n", __FUNCTION__); Index: dev/cxgb/cxgb_osdep.h =================================================================== RCS file: /home/kmacy/devel/ncvs/src/sys/dev/cxgb/cxgb_osdep.h,v retrieving revision 1.12 diff -d -u -r1.12 cxgb_osdep.h --- dev/cxgb/cxgb_osdep.h 17 Jul 2007 06:50:33 -0000 1.12 +++ dev/cxgb/cxgb_osdep.h 30 Jul 2007 20:32:57 -0000 @@ -52,6 +52,14 @@ typedef struct adapter adapter_t; struct sge_rspq; +struct mbuf_ring { + struct mbuf **mr_ring; + volatile uint32_t mr_cons; + volatile uint32_t mr_prod; + int mr_size; +}; + + #define PANIC_IF(exp) do { \ if (exp) \ panic("BUG: %s", exp); \ @@ -101,12 +109,7 @@ #define TX_MAX_SEGS 36 /* maximum supported by card */ #define TX_MAX_DESC 4 /* max descriptors per packet */ -#define TX_START_MIN_DESC (TX_MAX_DESC << 2) - -#if 0 -#define TX_START_MAX_DESC (TX_ETH_Q_SIZE >> 2) /* maximum number of descriptors */ -#endif - +#define TX_START_MIN_DESC (TX_MAX_DESC << 2) #define TX_START_MAX_DESC (TX_MAX_DESC << 3) /* maximum number of descriptors * call to start used per */ @@ -114,6 +117,13 @@ * to clean per iteration */ +#define TX_WR_SIZE_MAX 11*1024 /* the maximum total size of packets aggregated into a single + * TX WR + */ +#define TX_WR_COUNT_MAX 7 /* the maximum total number of packets that can be + * aggregated into a single TX WR + */ + #if defined(__i386__) || defined(__amd64__) #define mb() __asm volatile("mfence":::"memory") #define rmb() __asm volatile("lfence":::"memory") @@ -174,8 +184,6 @@ #define net_device ifnet #define cpu_to_be32 htobe32 - - /* Standard PHY definitions */ #define BMCR_LOOPBACK BMCR_LOOP #define BMCR_ISOLATE BMCR_ISO Index: dev/cxgb/cxgb_sge.c =================================================================== RCS file: /home/kmacy/devel/ncvs/src/sys/dev/cxgb/cxgb_sge.c,v retrieving revision 1.24 diff -d -u -r1.24 cxgb_sge.c --- dev/cxgb/cxgb_sge.c 17 Jul 2007 06:50:33 -0000 1.24 +++ dev/cxgb/cxgb_sge.c 1 Aug 2007 02:03:07 -0000 @@ -68,6 +68,7 @@ uint32_t mb_free_vec_free = 0; int txq_fills = 0; int collapse_mbufs = 0; +extern int cxgb_txq_mbuf_ring_size; static int recycle_enable = 1; static int bogus_imm = 0; @@ -304,7 +305,8 @@ len = G_RSPD_LEN(ntohl(resp->len_cq)); if (sopeop == RSPQ_NSOP_NEOP || sopeop == RSPQ_SOP) { if (cxgb_debug) - device_printf(sc->dev, "unexpected value sopeop=%d flags=0x%x len=%din get_imm_packet\n", sopeop, flags, len); + device_printf(sc->dev, "unexpected value sopeop=%d flags=0x%x len=%din get_imm_packet\n", + sopeop, flags, len); bogus_imm++; return (EINVAL); } @@ -897,6 +899,10 @@ qs->txq[TXQ_OFLD].cntxt_id = FW_OFLD_SGEEC_START + id; qs->txq[TXQ_CTRL].cntxt_id = FW_CTRL_SGEEC_START + id; qs->txq[TXQ_CTRL].token = FW_CTRL_TID_START + id; + + mbufq_init(&qs->txq[TXQ_ETH].sendq); + mbufq_init(&qs->txq[TXQ_OFLD].sendq); + mbufq_init(&qs->txq[TXQ_CTRL].sendq); } @@ -1160,12 +1166,10 @@ #define TCPPKTHDRSIZE (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + 20 + 20) int -t3_encap(struct port_info *p, struct mbuf **m) +t3_encap(struct sge_txq *txq, struct mbuf **m) { adapter_t *sc; struct mbuf *m0; - struct sge_qset *qs; - struct sge_txq *txq; struct tx_sw_desc *stx; struct txq_state txqs; unsigned int nsegs, ndesc, flits, cntrl, mlen; @@ -1179,24 +1183,22 @@ struct tx_desc *txd; struct cpl_tx_pkt *cpl; - + DPRINTF("t3_encap "); m0 = *m; - sc = p->adapter; - qs = &sc->sge.qs[p->first_qset]; - txq = &qs->txq[TXQ_ETH]; stx = &txq->sdesc[txq->pidx]; txd = &txq->desc[txq->pidx]; cpl = (struct cpl_tx_pkt *)txd; mlen = m0->m_pkthdr.len; cpl->len = htonl(mlen | 0x80000000); + sc = txq->port->adapter; DPRINTF("mlen=%d\n", mlen); /* * XXX handle checksum, TSO, and VLAN here * */ - cntrl = V_TXPKT_INTF(p->port); + cntrl = V_TXPKT_INTF(txq->port->port_id); /* * XXX need to add VLAN support for 6.x @@ -1292,7 +1294,7 @@ m_set_priority(m0, txqs.pidx); write_wr_hdr_sgl(ndesc, txd, &txqs, txq, sgl, flits, sgl_flits, wr_hi, wr_lo); - check_ring_tx_db(p->adapter, txq); + check_ring_tx_db(txq->port->adapter, txq); return (0); } @@ -1519,6 +1521,10 @@ { int i; + for (i = 0; i < SGE_TXQ_PER_SET; i++) + if (q->txq[i].txq_mr.mr_ring != NULL) + free(q->txq[i].txq_mr.mr_ring, M_DEVBUF); + for (i = 0; i < SGE_RXQ_PER_SET; ++i) { if (q->fl[i].desc) { mtx_lock(&sc->sge.reg_lock); @@ -2034,6 +2040,13 @@ struct sge_qset *q = &sc->sge.qs[id]; int i, ret = 0; + for (i = 0; i < SGE_TXQ_PER_SET; i++) { + if ((q->txq[i].txq_mr.mr_ring = malloc(cxgb_txq_mbuf_ring_size*sizeof(struct mbuf *), M_DEVBUF, M_WAITOK|M_ZERO)) == NULL) { + device_printf(sc->dev, "failed to allocate mbuf ring\n"); + goto err; + } + } + init_qset_cntxt(q, id); if ((ret = alloc_ring(sc, p->fl_size, sizeof(struct rx_desc), @@ -2041,7 +2054,7 @@ &q->fl[0].desc, &q->fl[0].sdesc, &q->fl[0].desc_tag, &q->fl[0].desc_map, sc->rx_dmat, &q->fl[0].entry_tag)) != 0) { - printf("error %d from alloc ring fl0\n", ret); + device_printf(sc->dev, "error %d from alloc ring fl0\n", ret); goto err; } @@ -2050,7 +2063,7 @@ &q->fl[1].desc, &q->fl[1].sdesc, &q->fl[1].desc_tag, &q->fl[1].desc_map, sc->rx_jumbo_dmat, &q->fl[1].entry_tag)) != 0) { - printf("error %d from alloc ring fl1\n", ret); + device_printf(sc->dev, "error %d from alloc ring fl1\n", ret); goto err; } @@ -2058,7 +2071,7 @@ &q->rspq.phys_addr, &q->rspq.desc, NULL, &q->rspq.desc_tag, &q->rspq.desc_map, NULL, NULL)) != 0) { - printf("error %d from alloc ring rspq\n", ret); + device_printf(sc->dev, "error %d from alloc ring rspq\n", ret); goto err; } @@ -2076,7 +2089,7 @@ &q->txq[i].sdesc, &q->txq[i].desc_tag, &q->txq[i].desc_map, sc->tx_dmat, &q->txq[i].entry_tag)) != 0) { - printf("error %d from alloc ring tx %i\n", ret, i); + device_printf(sc->dev, "error %d from alloc ring tx %i\n", ret, i); goto err; } mbufq_init(&q->txq[i].sendq); Index: dev/cxgb/sys/mbufq.h =================================================================== RCS file: /home/kmacy/devel/ncvs/src/sys/dev/cxgb/sys/mbufq.h,v retrieving revision 1.2 diff -d -u -r1.2 mbufq.h --- dev/cxgb/sys/mbufq.h 28 May 2007 22:57:27 -0000 1.2 +++ dev/cxgb/sys/mbufq.h 31 Jul 2007 22:03:15 -0000 @@ -36,6 +36,7 @@ struct mbuf *head; struct mbuf *tail; uint32_t qlen; + uint32_t qsize; struct mtx lock; }; @@ -43,6 +44,7 @@ mbufq_init(struct mbuf_head *l) { l->head = l->tail = NULL; + l->qlen = l->qsize = 0; } static __inline int @@ -57,13 +59,28 @@ return (l->qlen); } +static __inline int +mbufq_size(struct mbuf_head *l) +{ + return (l->qsize); +} + +static __inline int +mbufq_head_size(struct mbuf_head *l) +{ + return (l->head ? l->head->m_pkthdr.len : 0); +} static __inline void mbufq_tail(struct mbuf_head *l, struct mbuf *m) { l->qlen++; - l->tail->m_nextpkt = m; + if (l->tail) + l->tail->m_nextpkt = m; + else + l->head = m; l->tail = m; + l->qsize += m->m_pkthdr.len; } static __inline struct mbuf * @@ -76,7 +93,9 @@ if (m == l->tail) l->tail = NULL; l->head = m->m_nextpkt; + m->m_nextpkt = NULL; l->qlen--; + l->qsize -= m->m_pkthdr.len; } return (m);