=== sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c ================================================================== --- sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c (revision 14749) +++ sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c (local) @@ -85,6 +85,10 @@ #include #include +#define TIMEWAIT 1 +#define TCP_CLOSE 2 +#define TCP_DROP 3 + /* * For ULP connections HW may add headers, e.g., for digests, that aren't part * of the messages sent by the host but that are part of the TCP payload and @@ -792,7 +796,6 @@ * XXX how do we handle teardown in the SYN_SENT state? * */ - INP_INFO_WLOCK(&tcbinfo); inp_wlock_assert(tp->t_inpcb); toep = tp->t_toe; toep->tp_tp = NULL; @@ -802,7 +805,6 @@ */ tp->t_flags &= ~TF_TOE; tp->t_toe = NULL; - INP_INFO_WUNLOCK(&tcbinfo); } @@ -1368,7 +1370,6 @@ struct cpl_act_open_rpl *rpl = cplhdr(m); struct inpcb *inp; - INP_INFO_WLOCK(&tcbinfo); if (toep->tp_tp == NULL) goto done; @@ -1391,7 +1392,6 @@ fail_act_open(toep, act_open_rpl_status_to_errno(rpl->status)); inp_wunlock(inp); done: - INP_INFO_WUNLOCK(&tcbinfo); m_free(m); } @@ -2312,6 +2312,39 @@ return (0); } +static void +toedrv_tcp_twstart(struct tcpcb *tp) +{ + INP_INFO_WLOCK(&tcbinfo); + inp_wlock(tp->t_inpcb); + tcp_twstart(tp); + INP_INFO_WUNLOCK(&tcbinfo); +} + +static void +toedrv_tcp_close(struct tcpcb *tp) +{ + + INP_INFO_WLOCK(&tcbinfo); + INP_LOCK(tp->t_inpcb); + tp = tcp_close(tp); + INP_INFO_WUNLOCK(&tcbinfo); + if (tp) + INP_UNLOCK(tp->t_inpcb); +} + +static void +toedrv_tcp_drop(struct tcpcb *tp, int error) +{ + + INP_INFO_WLOCK(&tcbinfo); + INP_LOCK(tp->t_inpcb); + tp = tcp_drop(tp, error); + INP_INFO_WUNLOCK(&tcbinfo); + if (tp) + INP_UNLOCK(tp->t_inpcb); +} + /* * Move a socket to TIME_WAIT state. We need to make some adjustments to the * socket state before calling tcp_time_wait to comply with its expectations. @@ -2321,7 +2354,7 @@ { struct tcpcb *tp = sototcpcb(so); - inp_wlock_assert(tp->t_inpcb); + inp_wlock(tp->t_inpcb); /* * Bump rcv_nxt for the peer FIN. We don't do this at the time we * process peer_close because we don't want to carry the peer FIN in @@ -2333,7 +2366,8 @@ tp->ts_recent_age = 0; /* defeat recycling */ tp->t_srtt = 0; /* defeat tcp_update_metrics */ - tcp_twstart(tp); + inp_wunlock(tp->t_inpcb); + toedrv_tcp_twstart(tp); } /* @@ -2408,6 +2442,7 @@ struct tcpcb *tp = sototcpcb(so); struct toepcb *toep = tp->t_toe; int keep = 0; + int shutdown = 0; DPRINTF("do_peer_fin state=%d\n", tp->t_state); #ifdef T3_TRACE @@ -2419,12 +2454,10 @@ goto out; } - INP_INFO_WLOCK(&tcbinfo); inp_wlock(tp->t_inpcb); if (toep->tp_ulp_mode == ULP_MODE_TCPDDP) { keep = handle_peer_close_data(so, m); if (keep < 0) { - INP_INFO_WUNLOCK(&tcbinfo); inp_wunlock(tp->t_inpcb); return; } @@ -2467,20 +2500,24 @@ */ t3_release_offload_resources(toep); if (toep->tp_flags & TP_ABORT_RPL_PENDING) { - tp = tcp_close(tp); + shutdown = TCP_CLOSE; } else { - enter_timewait(so); + shutdown = TIMEWAIT; } + cxgb_toe_detach(tp); break; default: log(LOG_ERR, "%s: TID %u received PEER_CLOSE in bad state %d\n", TOE_DEV(so)->tod_name, toep->tp_tid, tp->t_state); } - INP_INFO_WUNLOCK(&tcbinfo); - if (tp) - inp_wunlock(tp->t_inpcb); + inp_wunlock(tp->t_inpcb); + if (shutdown == TCP_CLOSE) + toedrv_tcp_close(tp); + else if (shutdown == TIMEWAIT) + enter_timewait(so); + DPRINTF("waking up waiters on %p rcv_notify=%d flags=0x%x\n", so, sb_notify(&so->so_rcv), so->so_rcv.sb_flags); #ifdef notyet @@ -2518,7 +2555,8 @@ struct tcpcb *tp = sototcpcb(so); struct cpl_close_con_rpl *rpl = cplhdr(m); struct toepcb *toep = tp->t_toe; - + int shutdown = 0; + tp->snd_una = ntohl(rpl->snd_nxt) - 1; /* exclude FIN */ DPRINTF("process_close_con_rpl(%p) state=%d dead=%d\n", so, tp->t_state, @@ -2526,18 +2564,17 @@ if (!is_t3a(TOE_DEV(so)) && (toep->tp_flags & TP_ABORT_RPL_PENDING)) goto out; - INP_INFO_WLOCK(&tcbinfo); inp_wlock(tp->t_inpcb); switch (tp->t_state) { case TCPS_CLOSING: /* see FIN_WAIT2 case in do_peer_fin */ t3_release_offload_resources(toep); if (toep->tp_flags & TP_ABORT_RPL_PENDING) { - tp = tcp_close(tp); - + shutdown = TCP_CLOSE; } else { - enter_timewait(so); + shutdown = TIMEWAIT; soisdisconnected(so); } + cxgb_toe_detach(tp); break; case TCPS_LAST_ACK: /* @@ -2546,7 +2583,8 @@ * late, this close_con_rpl is the actual last message. */ t3_release_offload_resources(toep); - tp = tcp_close(tp); + cxgb_toe_detach(tp); + shutdown = TCP_CLOSE; break; case TCPS_FIN_WAIT_1: /* @@ -2571,7 +2609,8 @@ tp->t_state = TCPS_FIN_WAIT_2; if ((so->so_options & SO_LINGER) && so->so_linger == 0 && (toep->tp_flags & TP_ABORT_SHUTDOWN) == 0) { - tp = tcp_drop(tp, 0); + shutdown = TCP_DROP; + cxgb_toe_detach(tp); } break; @@ -2581,9 +2620,15 @@ TOE_DEV(so)->tod_name, toep->tp_tid, tp->t_state); } - INP_INFO_WUNLOCK(&tcbinfo); - if (tp) - inp_wunlock(tp->t_inpcb); + inp_wunlock(tp->t_inpcb); + + if (shutdown == TIMEWAIT) + enter_timewait(so); + else if (shutdown == TCP_CLOSE) + toedrv_tcp_close(tp); + else if (shutdown == TCP_DROP) + toedrv_tcp_drop(tp, 0); + out: m_freem(m); } @@ -2621,10 +2666,7 @@ "process_abort_rpl: GTS rpl pending %d", sock_flag(sk, ABORT_RPL_PENDING)); #endif - - INP_INFO_WLOCK(&tcbinfo); inp_wlock(tp->t_inpcb); - if (toep->tp_flags & TP_ABORT_RPL_PENDING) { /* * XXX panic on tcpdrop @@ -2638,14 +2680,12 @@ if (toep->tp_flags & TP_ABORT_REQ_RCVD) panic("TP_ABORT_REQ_RCVD set"); t3_release_offload_resources(toep); - tp = tcp_close(tp); + cxgb_toe_detach(tp); } } } - if (tp) - inp_wunlock(tp->t_inpcb); - INP_INFO_WUNLOCK(&tcbinfo); - + inp_wunlock(tp->t_inpcb); + toedrv_tcp_close(tp); m_free(m); } @@ -2829,13 +2869,13 @@ */ if (__predict_false(parenttp->t_state == TCPS_LISTEN)) { cleanup_syn_rcv_conn(child, parent); - INP_INFO_WLOCK(&tcbinfo); inp_wlock(childtp->t_inpcb); t3_release_offload_resources(childtp->t_toe); - childtp = tcp_close(childtp); - INP_INFO_WUNLOCK(&tcbinfo); - if (childtp) - inp_wunlock(childtp->t_inpcb); + cxgb_toe_detach(childtp); + inp_wunlock(childtp->t_inpcb); + + toedrv_tcp_close(childtp); + } } #endif @@ -2882,7 +2922,8 @@ const struct cpl_abort_req_rss *req = cplhdr(m); struct tcpcb *tp = sototcpcb(so); struct toepcb *toep = tp->t_toe; - + int shutdown_now = 0; + inp_wlock(tp->t_inpcb); if ((toep->tp_flags & TP_ABORT_REQ_RCVD) == 0) { toep->tp_flags |= (TP_ABORT_REQ_RCVD|TP_ABORT_SHUTDOWN); @@ -2914,10 +2955,14 @@ goto skip; t3_release_offload_resources(toep); - tp = tcp_close(tp); + cxgb_toe_detach(tp); + shutdown_now = 1; } - if (tp) - inp_wunlock(tp->t_inpcb); + inp_wunlock(tp->t_inpcb); + + if (shutdown_now) + toedrv_tcp_close(tp); + send_abort_rpl(m, tdev, rst_status); return; @@ -2978,9 +3023,7 @@ VALIDATE_SOCK(so); toepcb_hold(toep); - INP_INFO_WLOCK(&tcbinfo); process_abort_req(so, m, TOE_DEV(so)); - INP_INFO_WUNLOCK(&tcbinfo); toepcb_release(toep); return (0); } @@ -3158,8 +3201,6 @@ to.to_mss = mss; to.to_wscale = wsf; to.to_flags = (mss ? TOF_MSS : 0) | (wsf ? TOF_SCALE : 0) | (ts ? TOF_TS : 0) | (sack ? TOF_SACKPERM : 0); - INP_INFO_WLOCK(&tcbinfo); - inp_wlock(inp); syncache_offload_add(&inc, &to, &th, inp, &lso, &cxgb_toe_usrreqs, toep); } @@ -3311,7 +3352,6 @@ DPRINTF("opt0l_status=%08x\n", rpl->opt0l_status); m_set_priority(reply_mbuf, mkprio(CPL_PRIORITY_SETUP, newtoep)); - l2t_send(cdev, reply_mbuf, e); m_free(m); if (newtoep->tp_ulp_mode) { @@ -3496,7 +3536,7 @@ ntohl(req->local_ip), ntohs(req->local_port), ntohl(req->peer_ip), ntohs(req->peer_port), mss, wsf, ts, sack); - return syncache_expand(&inc, &to, &th, so, m); + return syncache_offload_expand(&inc, &to, &th, so, m); } @@ -3523,7 +3563,6 @@ LIST_REMOVE(toep, synq_entry); SOCK_UNLOCK(so); - INP_INFO_WLOCK(&tcbinfo); if (!syncache_expand_establish_req(req, &so, toep)) { /* * No entry @@ -3567,7 +3606,6 @@ */ make_established(so, ntohl(req->snd_isn), ntohs(req->tcp_opt)); - INP_INFO_WUNLOCK(&tcbinfo); inp_wunlock(tp->t_inpcb); CTR1(KTR_TOM, "do_pass_establish tid=%u", toep->tp_tid); === sys/netinet/tcp_syncache.c ================================================================== --- sys/netinet/tcp_syncache.c (revision 14749) +++ sys/netinet/tcp_syncache.c (local) @@ -952,6 +952,14 @@ return (0); } +int +syncache_offload_expand(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, + struct socket **lsop, struct mbuf *m) +{ + INP_INFO_WLOCK(&tcbinfo); + return syncache_expand(inc, to, th, lsop, m); +} + /* * Given a LISTEN socket and an inbound SYN request, add * this to the syn cache, and send back a segment: @@ -1422,6 +1430,8 @@ struct tcphdr *th, struct inpcb *inp, struct socket **lsop, struct toe_usrreqs *tu, void *toepcb) { + INP_INFO_WLOCK(&tcbinfo); + INP_LOCK(inp); _syncache_add(inc, to, th, inp, lsop, NULL, tu, toepcb); } === sys/netinet/tcp_syncache.h ================================================================== --- sys/netinet/tcp_syncache.h (revision 14749) +++ sys/netinet/tcp_syncache.h (local) @@ -38,6 +38,8 @@ void syncache_unreach(struct in_conninfo *, struct tcphdr *); int syncache_expand(struct in_conninfo *, struct tcpopt *, struct tcphdr *, struct socket **, struct mbuf *); +int syncache_offload_expand(struct in_conninfo *, struct tcpopt *, + struct tcphdr *, struct socket **, struct mbuf *); void syncache_add(struct in_conninfo *, struct tcpopt *, struct tcphdr *, struct inpcb *, struct socket **, struct mbuf *); void syncache_offload_add(struct in_conninfo *, struct tcpopt *,