==== //depot/vendor/freebsd/src/sys/sys/callout.h#12 (text+ko) - //depot/projects/ethng/src/sys/sys/callout.h#3 (text+ko) ==== content @@ -54,6 +54,7 @@ void *c_arg; /* function argument */ void (*c_func)(void *); /* function to call */ struct mtx *c_mtx; /* mutex to lock */ + struct rwlock *c_rwlock; /* rwlock to lock */ int c_flags; /* state of this entry */ }; @@ -62,6 +63,7 @@ #define CALLOUT_PENDING 0x0004 /* callout is waiting for timeout */ #define CALLOUT_MPSAFE 0x0008 /* callout handler is mp safe */ #define CALLOUT_RETURNUNLOCKED 0x0010 /* handler returns with mtx unlocked */ +#define CALLOUT_RETURNUNLOCKED_RW 0x0040 /* handler returns with rwlock unlocked */ struct callout_handle { struct callout *callout; @@ -80,6 +82,7 @@ #define callout_drain(c) _callout_stop_safe(c, 1) void callout_init(struct callout *, int); void callout_init_mtx(struct callout *, struct mtx *, int); +void callout_init_rwlock(struct callout *, struct rwlock *, int); #define callout_pending(c) ((c)->c_flags & CALLOUT_PENDING) int callout_reset(struct callout *, int, void (*)(void *), void *); #define callout_stop(c) _callout_stop_safe(c, 0) ==== //depot/vendor/freebsd/src/sys/sys/interrupt.h#20 (text+ko) - //depot/projects/ethng/src/sys/sys/interrupt.h#2 (text+ko) ==== content @@ -73,6 +73,7 @@ void *ie_source; /* Cookie used by MD code. */ struct intr_thread *ie_thread; /* Thread we are connected to. */ void (*ie_enable)(void *); + int (*ie_assign_cpu)(void *, u_char); #ifdef INTR_FILTER void (*ie_eoi)(void *); void (*ie_disab)(void *); @@ -81,6 +82,7 @@ int ie_count; /* Loop counter. */ int ie_warncnt; /* Rate-check interrupt storm warns. */ struct timeval ie_warntm; + u_char ie_cpu; }; /* Interrupt event flags kept in ie_flags. */ @@ -127,15 +129,16 @@ int intr_event_add_handler(struct intr_event *ie, const char *name, driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri, enum intr_type flags, void **cookiep); +int intr_event_bind(struct intr_event *ie, u_char cpu); #ifndef INTR_FILTER int intr_event_create(struct intr_event **event, void *source, - int flags, void (*enable)(void *), const char *fmt, ...) - __printflike(5, 6); + int flags, void (*enable)(void *), int (*assign_cpu)(void *, u_char), const char *fmt, ...) + __printflike(6, 7); #else int intr_event_create(struct intr_event **event, void *source, int flags, void (*enable)(void *), void (*eoi)(void *), - void (*disab)(void *), const char *fmt, ...) - __printflike(7, 8); + void (*disab)(void *), int (*assign_cpu)(void *, u_char), const char *fmt, ...) + __printflike(8, 9); #endif int intr_event_destroy(struct intr_event *ie); int intr_event_remove_handler(void *cookie); ==== //depot/vendor/freebsd/src/sys/sys/kernel.h#45 (text+ko) - //depot/projects/ethng/src/sys/sys/kernel.h#2 (text+ko) ==== content @@ -351,5 +351,6 @@ int config_intrhook_establish(struct intr_config_hook *hook); void config_intrhook_disestablish(struct intr_config_hook *hook); +int check_subsystem(int); #endif /* !_SYS_KERNEL_H_*/ ==== - //depot/projects/ethng/src/sys/sys/linux_atomic.h#1 ==== ==== - //depot/projects/ethng/src/sys/sys/linux_compat.h#8 ==== ==== - //depot/projects/ethng/src/sys/sys/linux_pci.h#2 ==== ==== //depot/vendor/freebsd/src/sys/sys/mbuf.h#136 (text+ko) - //depot/projects/ethng/src/sys/sys/mbuf.h#10 (text+ko) ==== content @@ -44,6 +44,48 @@ #endif #endif +/*- + * mbuf external reference count management macros. + * + * MEXT_IS_REF(m): true if (m) is not the only mbuf referencing + * the external buffer ext_buf. + * + * MEXT_REM_REF(m): remove reference to m_ext object. + * + * MEXT_ADD_REF(m): add reference to m_ext object already + * referred to by (m). XXX Note that it is VERY important that you + * always set the second mbuf's m_ext.ref_cnt to point to the first + * one's (i.e., n->m_ext.ref_cnt = m->m_ext.ref_cnt) AFTER you run + * MEXT_ADD_REF(m). This is because m might have a lazy initialized + * ref_cnt (NULL) before this is run and it will only be looked up + * from here. We should make MEXT_ADD_REF() always take two mbufs + * as arguments so that it can take care of this itself. + */ +#define MEXT_IS_REF(m) (((m)->m_ext.ref_cnt != NULL) \ + && (*((m)->m_ext.ref_cnt) > 1)) + +#define MEXT_REM_REF(m) do { \ + KASSERT((m)->m_ext.ref_cnt != NULL, ("m_ext refcnt lazy NULL")); \ + KASSERT(*((m)->m_ext.ref_cnt) > 0, ("m_ext refcnt < 0")); \ + atomic_subtract_int((m)->m_ext.ref_cnt, 1); \ +} while(0) + +#define MEXT_ADD_REF(m) do { \ + if ((m)->m_ext.ref_cnt == NULL) { \ + KASSERT((m)->m_ext.ext_type == EXT_CLUSTER || \ + (m)->m_ext.ext_type == EXT_PACKET || \ + (m)->m_ext.ext_type == EXT_JUMBOP || \ + (m)->m_ext.ext_type == EXT_JUMBO9 || \ + (m)->m_ext.ext_type == EXT_JUMBO16, \ + ("Unexpected mbuf type has lazy refcnt %d", \ + (m)->m_ext.ext_type)); \ + (m)->m_ext.ref_cnt = (u_int *)uma_find_refcnt( \ + m_getzone((m)->m_ext.ext_size), (m)->m_ext.ext_buf); \ + *((m)->m_ext.ref_cnt) = 2; \ + } else \ + atomic_add_int((m)->m_ext.ref_cnt, 1); \ +} while (0) + /* * Mbufs are of a single size, MSIZE (sys/param.h), which includes overhead. * An mbuf may add a single "mbuf cluster" of size MCLBYTES (also in @@ -55,8 +97,10 @@ */ #define MLEN (MSIZE - sizeof(struct m_hdr)) /* normal data len */ #define MHLEN (MLEN - sizeof(struct pkthdr)) /* data len w/pkthdr */ +#define M_MAXCOMPRESS (MHLEN / 2) /* max amount to copy for compression */ +#ifndef MINCLSIZE #define MINCLSIZE (MHLEN + 1) /* smallest amount to put in cluster */ -#define M_MAXCOMPRESS (MHLEN / 2) /* max amount to copy for compression */ +#endif #ifdef _KERNEL /*- @@ -78,7 +122,7 @@ #endif /* _KERNEL */ #if defined(__LP64__) -#define M_HDR_PAD 6 +#define M_HDR_PAD 2 #else #define M_HDR_PAD 2 #endif @@ -92,7 +136,7 @@ caddr_t mh_data; /* location of data */ int mh_len; /* amount of data in this mbuf */ int mh_flags; /* flags; see below */ - short mh_type; /* type of data in this mbuf */ + short mh_type; /* type of data in this mbuf */ uint8_t pad[M_HDR_PAD];/* word align */ }; @@ -121,19 +165,20 @@ u_int16_t tso_segsz; /* TSO segment size */ u_int16_t ether_vtag; /* Ethernet 802.1p+q vlan tag */ SLIST_HEAD(packet_tags, m_tag) tags; /* list of packet tags */ + uint32_t rss_hash; /* RSS hash for this 4-tuple */ }; /* * Description of external storage mapped into mbuf; valid only if M_EXT is * set. */ -struct m_ext { +struct m_ext_ { caddr_t ext_buf; /* start of buffer */ void (*ext_free) /* free routine if not the usual */ (void *, void *); void *ext_args; /* optional argument pointer */ + volatile u_int *ref_cnt; /* pointer to ref count info */ u_int ext_size; /* size of buffer, for ext_free */ - volatile u_int *ref_cnt; /* pointer to ref count info */ int ext_type; /* type of external storage */ }; @@ -147,7 +192,7 @@ struct { struct pkthdr MH_pkthdr; /* M_PKTHDR set */ union { - struct m_ext MH_ext; /* M_EXT set */ + struct m_ext_ MH_ext; /* M_EXT set */ char MH_databuf[MHLEN]; } MH_dat; } MH; @@ -161,6 +206,7 @@ #define m_flags m_hdr.mh_flags #define m_nextpkt m_hdr.mh_nextpkt #define m_act m_nextpkt +#define m_rss_hash m_hdr.mh_rss_hash #define m_pkthdr M_dat.MH.MH_pkthdr #define m_ext M_dat.MH.MH_dat.MH_ext #define m_pktdat M_dat.MH.MH_dat.MH_databuf @@ -192,7 +238,7 @@ #define M_LASTFRAG 0x2000 /* packet is last fragment */ #define M_VLANTAG 0x10000 /* ether_vtag is valid */ #define M_PROMISC 0x20000 /* packet was not for us */ -#define M_NOFREE 0x40000 /* do not free mbuf - it is embedded in the cluster */ +#define M_NOFREE 0x40000 /* do not free the mbuf - its embedded in a cluster */ /* * External buffer types: identify ext_buf type. @@ -204,6 +250,9 @@ #define EXT_JUMBO16 5 /* jumbo cluster 16184 bytes */ #define EXT_PACKET 6 /* mbuf+cluster from packet zone */ #define EXT_MBUF 7 /* external mbuf reference (M_IOVEC) */ +#define EXT_IOVEC 8 +#define EXT_CLIOVEC 9 +#define EXT_JMPIOVEC 10 #define EXT_NET_DRV 100 /* custom ext_buf provided by net driver(s) */ #define EXT_MOD_TYPE 200 /* custom module's ext_buf type */ #define EXT_DISPOSABLE 300 /* can throw this buffer away w/page flipping */ @@ -248,6 +297,8 @@ #define MT_DATA 1 /* dynamic (data) allocation */ #define MT_HEADER MT_DATA /* packet header, use M_PKTHDR instead */ #define MT_SONAME 8 /* socket name */ +#define MT_IOVEC 9 +#define MT_CLIOVEC 10 #define MT_CONTROL 14 /* extra-data protocol message */ #define MT_OOBDATA 15 /* expedited data */ #define MT_NTYPES 16 /* number of mbuf types for mbtypes[] */ @@ -338,7 +389,6 @@ extern uma_zone_t zone_mbuf; extern uma_zone_t zone_clust; -extern uma_zone_t zone_pack; extern uma_zone_t zone_jumbop; extern uma_zone_t zone_jumbo9; extern uma_zone_t zone_jumbo16; @@ -457,11 +507,7 @@ static __inline struct mbuf * m_getcl(int how, short type, int flags) { - struct mb_args args; - - args.flags = flags; - args.type = type; - return ((struct mbuf *)(uma_zalloc_arg(zone_pack, &args, how))); + return ((struct mbuf *)(m_getjcl(how, type, flags, MCLBYTES))); } /* @@ -496,8 +542,10 @@ static __inline void m_free_fast(struct mbuf *m) { - KASSERT(SLIST_EMPTY(&m->m_pkthdr.tags), ("doing fast free of mbuf with tags")); - +#ifdef INVARIANTS + if (m->m_flags & M_PKTHDR) + KASSERT(SLIST_EMPTY(&m->m_pkthdr.tags), ("doing fast free of mbuf with tags")); +#endif uma_zfree_arg(zone_mbuf, m, (void *)MB_NOTAGS); } @@ -512,7 +560,6 @@ uma_zfree(zone_mbuf, m); return (n); } - static __inline void m_clget(struct mbuf *m, int how) { @@ -521,14 +568,6 @@ printf("%s: %p mbuf already has cluster\n", __func__, m); m->m_ext.ext_buf = (char *)NULL; uma_zalloc_arg(zone_clust, m, how); - /* - * On a cluster allocation failure, drain the packet zone and retry, - * we might be able to loosen a few clusters up on the drain. - */ - if ((how & M_NOWAIT) && (m->m_ext.ext_buf == NULL)) { - zone_drain(zone_pack); - uma_zalloc_arg(zone_clust, m, how); - } } /* @@ -538,6 +577,7 @@ * the cluster attached to it and the return value can be safely ignored. * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES. */ + static __inline void * m_cljget(struct mbuf *m, int how, int size) { @@ -553,7 +593,7 @@ } static __inline void -m_cljset(struct mbuf *m, void *cl, int type) +m_cljset(struct mbuf *m, void *cl, int type, volatile uint32_t *ref) { uma_zone_t zone; int size; @@ -586,7 +626,7 @@ m->m_ext.ext_free = m->m_ext.ext_args = NULL; m->m_ext.ext_size = size; m->m_ext.ext_type = type; - m->m_ext.ref_cnt = uma_find_refcnt(zone, cl); + m->m_ext.ref_cnt = ref; m->m_flags |= M_EXT; } @@ -627,7 +667,7 @@ */ #define M_WRITABLE(m) (!((m)->m_flags & M_RDONLY) && \ (!(((m)->m_flags & M_EXT)) || \ - (*((m)->m_ext.ref_cnt) == 1)) ) \ + !MEXT_IS_REF(m))) \ /* Check if the supplied mbuf has a packet header, or else panic. */ #define M_ASSERTPKTHDR(m) \ @@ -732,6 +772,7 @@ extern int max_protohdr; /* Largest protocol header */ extern struct mbstat mbstat; /* General mbuf stats/infos */ extern int nmbclusters; /* Maximum number of clusters */ +extern int jumbo_phys_contig; struct uio; ==== //depot/vendor/freebsd/src/sys/sys/param.h#218 (text+ko) - //depot/projects/ethng/src/sys/sys/param.h#8 (text+ko) ==== content @@ -142,7 +142,7 @@ #define MCLBYTES (1 << MCLSHIFT) /* size of an mbuf cluster */ #define MJUMPAGESIZE PAGE_SIZE /* jumbo cluster 4k */ -#define MJUM9BYTES (9 * 1024) /* jumbo cluster 9k */ +#define MJUM9BYTES (10 * 1024) /* jumbo cluster 9k */ #define MJUM16BYTES (16 * 1024) /* jumbo cluster 16k */ /* ==== //depot/vendor/freebsd/src/sys/sys/proc.h#323 (text+ko) - //depot/projects/ethng/src/sys/sys/proc.h#6 (text+ko) ==== content @@ -914,6 +914,46 @@ struct thread *thread_find(struct proc *p, lwpid_t tid); void thr_exit1(void); +#include +#include +#include "opt_sched.h" + +/* + * Kernel thread preemption implementation. Critical sections mark + * regions of code in which preemptions are not allowed. + */ +static __inline void +critical_enter(void) +{ + struct thread *td; + + td = curthread; + td->td_critnest++; + CTR4(KTR_CRITICAL, "critical_enter by thread %p (%ld, %s) to %d", td, + (long)td->td_proc->p_pid, td->td_proc->p_comm, td->td_critnest); +} + +static __inline void +critical_exit(void) +{ + struct thread *td; + + td = curthread; + KASSERT(td->td_critnest != 0, + ("critical_exit: td_critnest == 0")); +#ifdef PREEMPTION + if (td->td_critnest == 1) { + td->td_critnest = 0; + if (td->td_owepreempt) + critical_exit_owepreempt(td); + } else +#endif + td->td_critnest--; + + CTR4(KTR_CRITICAL, "critical_exit by thread %p (%ld, %s) to %d", td, + (long)td->td_proc->p_pid, td->td_proc->p_comm, td->td_critnest); +} + #endif /* _KERNEL */ #endif /* !_SYS_PROC_H_ */ ==== //depot/vendor/freebsd/src/sys/sys/systm.h#119 (text+ko) - //depot/projects/ethng/src/sys/sys/systm.h#3 (text+ko) ==== content @@ -151,8 +151,7 @@ void cpu_boot(int); void cpu_rootconf(void); -void critical_enter(void); -void critical_exit(void); +void critical_exit_owepreempt(struct thread *td); void init_param1(void); void init_param2(long physpages); void init_param3(long kmempages); @@ -184,6 +183,7 @@ #define HD_OMIT_HEX (1 << 17) #define HD_OMIT_CHARS (1 << 18) + #define ovbcopy(f, t, l) bcopy((f), (t), (l)) void bcopy(const void *from, void *to, size_t len) __nonnull(1) __nonnull(2); void bzero(void *buf, size_t len) __nonnull(1);