Index: sys/kern/kern_mutex.c =================================================================== RCS file: /usr/home/kmacy/devel/cxgb/ncvs/src/sys/kern/kern_mutex.c,v retrieving revision 1.198 diff -d -u -r1.198 kern_mutex.c --- sys/kern/kern_mutex.c 18 Jul 2007 20:46:05 -0000 1.198 +++ sys/kern/kern_mutex.c 19 Jul 2007 06:47:25 -0000 @@ -121,17 +121,18 @@ struct mtx Giant; #ifdef LOCK_PROFILING -static inline void lock_profile_init(void) +#include + + + +static void +lock_profile_init(void *unused) { - int i; - /* Initialize the mutex profiling locks */ - for (i = 0; i < LPROF_LOCK_SIZE; i++) { - mtx_init(&lprof_locks[i], "mprof lock", - NULL, MTX_SPIN|MTX_QUIET|MTX_NOPROFILE); - } + if (lock_profile_init_rings() == 0) + lock_profile_start(); } #else -static inline void lock_profile_init(void) {;} +static void lock_profile_init(void *foo) {;} #endif void @@ -781,7 +782,6 @@ mtx_init(&devmtx, "cdev", NULL, MTX_DEF); mtx_lock(&Giant); - lock_profile_init(); } #ifdef DDB @@ -825,3 +825,5 @@ } } #endif + +SYSINIT(lock_prof_init, SI_SUB_KTHREAD_IDLE, SI_ORDER_ANY, lock_profile_init, 0); Index: sys/kern/kern_thread.c =================================================================== RCS file: /usr/home/kmacy/devel/cxgb/ncvs/src/sys/kern/kern_thread.c,v retrieving revision 1.250 diff -d -u -r1.250 kern_thread.c --- sys/kern/kern_thread.c 12 Jun 2007 19:49:39 -0000 1.250 +++ sys/kern/kern_thread.c 18 Jul 2007 07:29:23 -0000 @@ -531,6 +531,10 @@ td->td_flags = 0; LIST_INIT(&td->td_contested); +#ifdef LOCK_PROFILING + LIST_INIT(&td->td_lprofile); + LIST_INIT(&td->td_slprofile); +#endif sigqueue_init(&td->td_sigqueue, p); callout_init(&td->td_slpcallout, CALLOUT_MPSAFE); TAILQ_INSERT_HEAD(&p->p_threads, td, td_plist); Index: sys/kern/sched_ule.c =================================================================== RCS file: /usr/home/kmacy/devel/cxgb/ncvs/src/sys/kern/sched_ule.c,v retrieving revision 1.200 diff -d -u -r1.200 sched_ule.c --- sys/kern/sched_ule.c 17 Jul 2007 22:53:23 -0000 1.200 +++ sys/kern/sched_ule.c 18 Jul 2007 06:24:35 -0000 @@ -1069,8 +1069,7 @@ * If we're being awoken by an interrupt thread or the waker * is going right to sleep run here as well. */ - if ((TDQ_SELF()->tdq_load <= 1) && (flags & (SRQ_YIELDING) || - curthread->td_pri_class == PRI_ITHD)) { + if ((TDQ_SELF()->tdq_load <= 1) && (flags & (SRQ_YIELDING))) { CTR2(KTR_ULE, "tryself load %d flags %d", TDQ_SELF()->tdq_load, flags); return (self); Index: sys/kern/subr_lock.c =================================================================== RCS file: /usr/home/kmacy/devel/cxgb/ncvs/src/sys/kern/subr_lock.c,v retrieving revision 1.16 diff -d -u -r1.16 subr_lock.c --- sys/kern/subr_lock.c 3 Jun 2007 18:24:31 -0000 1.16 +++ sys/kern/subr_lock.c 19 Jul 2007 17:05:42 -0000 @@ -40,12 +40,20 @@ #include #include +#include +#include #include +#include #include #include +#include #include #include +#include #include +#include +#include + #ifdef DDB #include @@ -66,9 +74,12 @@ SYSCTL_NODE(_debug, OID_AUTO, lock, CTLFLAG_RD, NULL, "lock debugging"); SYSCTL_NODE(_debug_lock, OID_AUTO, prof, CTLFLAG_RD, NULL, "lock profiling"); +SYSCTL_NODE(_debug_lock_prof, OID_AUTO, knob, CTLFLAG_RD, NULL, "tunables"); +SYSCTL_NODE(_debug_lock_prof, OID_AUTO, internal, CTLFLAG_RD, NULL, "internal statistics"); int lock_prof_enable = 0; SYSCTL_INT(_debug_lock_prof, OID_AUTO, enable, CTLFLAG_RW, &lock_prof_enable, 0, "Enable lock profiling"); +static int hash_update_running = 0; /* * lprof_buf is a static pool of profiling records to avoid possible @@ -78,30 +89,34 @@ */ struct lock_prof lprof_buf[LPROF_HASH_SIZE]; static int allocated_lprof_buf; -struct mtx lprof_locks[LPROF_LOCK_SIZE]; - /* SWAG: sbuf size = avg stat. line size * number of locks */ #define LPROF_SBUF_SIZE 256 * 400 static int lock_prof_acquisitions; -SYSCTL_INT(_debug_lock_prof, OID_AUTO, acquisitions, CTLFLAG_RD, +SYSCTL_INT(_debug_lock_prof_internal, OID_AUTO, acquisitions, CTLFLAG_RD, &lock_prof_acquisitions, 0, "Number of lock acquistions recorded"); static int lock_prof_records; -SYSCTL_INT(_debug_lock_prof, OID_AUTO, records, CTLFLAG_RD, +SYSCTL_INT(_debug_lock_prof_internal, OID_AUTO, records, CTLFLAG_RD, &lock_prof_records, 0, "Number of profiling records"); static int lock_prof_maxrecords = LPROF_HASH_SIZE; -SYSCTL_INT(_debug_lock_prof, OID_AUTO, maxrecords, CTLFLAG_RD, +SYSCTL_INT(_debug_lock_prof_internal, OID_AUTO, maxrecords, CTLFLAG_RD, &lock_prof_maxrecords, 0, "Maximum number of profiling records"); static int lock_prof_rejected; -SYSCTL_INT(_debug_lock_prof, OID_AUTO, rejected, CTLFLAG_RD, +SYSCTL_INT(_debug_lock_prof_internal, OID_AUTO, rejected, CTLFLAG_RD, &lock_prof_rejected, 0, "Number of rejected profiling records"); static int lock_prof_hashsize = LPROF_HASH_SIZE; -SYSCTL_INT(_debug_lock_prof, OID_AUTO, hashsize, CTLFLAG_RD, +SYSCTL_INT(_debug_lock_prof_internal, OID_AUTO, hashsize, CTLFLAG_RD, &lock_prof_hashsize, 0, "Hash size"); static int lock_prof_collisions = 0; -SYSCTL_INT(_debug_lock_prof, OID_AUTO, collisions, CTLFLAG_RD, +SYSCTL_INT(_debug_lock_prof_internal, OID_AUTO, collisions, CTLFLAG_RD, &lock_prof_collisions, 0, "Number of hash collisions"); +static int lock_prof_iterations = 0; +SYSCTL_INT(_debug_lock_prof_internal, OID_AUTO, lock_prof_iterations, CTLFLAG_RD, + &lock_prof_iterations, 0, "passes through update loop"); +static int sleep_ticks = 1; +SYSCTL_INT(_debug_lock_prof_knob, OID_AUTO, sleep_ticks, CTLFLAG_RW, + &sleep_ticks, 0, "ticks to sleep between iterations"); #ifndef USE_CPU_NANOSECONDS u_int64_t @@ -215,7 +230,6 @@ lock->lo_flags |= flags | LO_INITIALIZED; LOCK_LOG_INIT(lock, 0); WITNESS_INIT(lock); - lock_profile_object_init(lock, class, name); } void @@ -223,7 +237,6 @@ { KASSERT(lock_initalized(lock), ("lock %p is not initialized", lock)); - lock_profile_object_destroy(lock); WITNESS_DESTROY(lock); LOCK_LOG_DESTROY(lock, 0); lock->lo_flags &= ~LO_INITIALIZED; @@ -252,105 +265,328 @@ #endif #ifdef LOCK_PROFILING -void _lock_profile_obtain_lock_success(struct lock_object *lo, int contested, uint64_t waittime, const char *file, int line) + +static __inline struct lock_profile_object * +lock_profile_alloc_lpo(struct lock_object *lo) { - struct lock_profile_object *l = &lo->lo_profile_obj; + struct lock_profile_object *lpo; + struct lock_profile_ring *lpr; + int spinlock; - lo->lo_profile_obj.lpo_contest_holding = 0; + spinlock = (LOCK_CLASS(lo) == &lock_class_mtx_spin); + critical_enter(); + lpr = spinlock ? PCPU_GET(slpr) : PCPU_GET(lpr); + lpr->lpr_lpo_alloc_count++; + if ((lpo = LIST_FIRST(&lpr->lpr_lpo_free)) != NULL) + LIST_REMOVE(lpo, lpo_entry); + else + lpr->lpr_lpo_free_empty++; + critical_exit(); + + return (lpo); +} + +/* + * Update per-cpu prod-cons ring with our lpo if there is space + * if there is an old lpo return it the local cpu's free list + */ +static __inline void +lock_profile_append_lpo(struct lock_object *lo, struct lock_profile_object *lpo) +{ + int spinlock; + struct lock_profile_ring *lpr; + struct lock_profile_object *lpo_old; + + spinlock = (LOCK_CLASS(lo) == &lock_class_mtx_spin); - if (contested) - lo->lo_profile_obj.lpo_contest_locking++; + critical_enter(); + lpr = spinlock ? PCPU_GET(slpr) : PCPU_GET(lpr); + if (!(lpr->lpr_prod == lpr->lpr_cons - 1 || + (lpr->lpr_prod == lpr->lpr_size-1 && + lpr->lpr_cons == 0))) { + + lpo->lpo_object = lo; + lpo_old = lpr->lpr_lpo_ring[lpr->lpr_prod]; + lpr->lpr_lpo_ring[lpr->lpr_prod] = lpo; + + if (++(lpr->lpr_prod) == lpr->lpr_size) + lpr->lpr_prod = 0; + } else { + lpo_old = lpo; + lpr->lpr_lpo_ring_full++; + } + + if (lpo_old) + LIST_INSERT_HEAD(&lpr->lpr_lpo_free, lpo_old, lpo_entry); + critical_exit(); +} + +void +_lock_profile_obtain_lock_success(struct lock_object *lo, int contested, uint64_t waittime, const char *file, int line) +{ + struct lock_profile_object *l; + struct lock_profile_object_head *lpoh; + int spinlock; + + if (lo->lo_flags & LO_NOPROFILE) + return; + l = lock_profile_alloc_lpo(lo); + if (l == NULL) + return; + + spinlock = (LOCK_CLASS(lo) == &lock_class_mtx_spin); + l->lpo_acqtime = nanoseconds(); l->lpo_filename = file; l->lpo_lineno = line; - l->lpo_acqtime = nanoseconds(); + l->lpo_type = LOCK_CLASS(lo)->lc_name; + l->lpo_name = lo->lo_name; + l->lpo_contest_locking = contested; + l->lpo_object = lo; if (waittime && (l->lpo_acqtime > waittime)) l->lpo_waittime = l->lpo_acqtime - waittime; else l->lpo_waittime = 0; + + lpoh = spinlock ? &curthread->td_slprofile : &curthread->td_lprofile; + LIST_INSERT_HEAD(lpoh, l, lpo_entry); } -void _lock_profile_release_lock(struct lock_object *lo) +/* + * find and return the thread's lock_profile_object for this lock_object + * + */ +struct lock_profile_object * +lock_profile_find_lpo(struct lock_object *lo) { - struct lock_profile_object *l = &lo->lo_profile_obj; + int spinlock; + lpoh_t *lpoh; + struct thread *td = curthread; + struct lock_profile_object *lpo; + + spinlock = (LOCK_CLASS(lo) == &lock_class_mtx_spin); + lpoh = spinlock ? &td->td_slprofile : &td->td_lprofile; + lpo = NULL; + + LIST_FOREACH(lpo, lpoh, lpo_entry) + if (lpo->lpo_object == lo) { + LIST_REMOVE(lpo, lpo_entry); + break; + } - if (l->lpo_acqtime && !(lo->lo_flags & LO_NOPROFILE)) { - const char *unknown = "(unknown)"; - u_int64_t acqtime, now, waittime; - struct lock_prof *mpp; - u_int hash; - const char *p = l->lpo_filename; - int collision = 0; + return (lpo); +} - now = nanoseconds(); - acqtime = l->lpo_acqtime; - waittime = l->lpo_waittime; - if (now <= acqtime) - return; - if (p == NULL || *p == '\0') - p = unknown; - hash = (l->lpo_namehash * 31 * 31 + (uintptr_t)p * 31 + l->lpo_lineno) & LPROF_HASH_MASK; - mpp = &lprof_buf[hash]; - while (mpp->name != NULL) { - if (mpp->line == l->lpo_lineno && - mpp->file == p && - mpp->namehash == l->lpo_namehash) - break; - /* If the lprof_hash entry is allocated to someone - * else, try the next one - */ - collision = 1; - hash = (hash + 1) & LPROF_HASH_MASK; - mpp = &lprof_buf[hash]; - } - if (mpp->name == NULL) { - int buf; +/* + * if a corresponding lpo exists add it the profile queue + * + */ +void +_lock_profile_release_lock(struct lock_object *lo, struct lock_profile_object *lpo) +{ + + lock_profile_append_lpo(lo, lpo); +} - buf = atomic_fetchadd_int(&allocated_lprof_buf, 1); - /* Just exit if we cannot get a trace buffer */ - if (buf >= LPROF_HASH_SIZE) { - ++lock_prof_rejected; - return; - } - mpp->file = p; - mpp->line = l->lpo_lineno; - mpp->namehash = l->lpo_namehash; - mpp->type = l->lpo_type; - mpp->name = lo->lo_name; +static void +lock_profile_update_hash(struct lock_profile_object *lpo) +{ + const char *unknown = "(unknown)"; + u_int64_t acqtime, now, waittime; + struct lock_prof *mpp; + u_int hash = 0; + const char *p, *p2; + int collision = 0; - if (collision) - ++lock_prof_collisions; - - /* - * We might have raced someone else but who cares, - * they'll try again next time - */ - ++lock_prof_records; - } - LPROF_LOCK(hash); - /* - * Record if the lock has been held longer now than ever - * before. - */ - if (now - acqtime > mpp->cnt_max) - mpp->cnt_max = now - acqtime; - mpp->cnt_tot += now - acqtime; - mpp->cnt_wait += waittime; - mpp->cnt_cur++; - /* + p = p2 = lpo->lpo_filename; + now = nanoseconds(); + acqtime = lpo->lpo_acqtime; + waittime = lpo->lpo_waittime; + if (now <= acqtime) + return; + + if (p2 == NULL || *p2 == '\0') + p = unknown; + + for (; *p2 != '\0'; p2++) + hash = 31 * hash + *p2; + + lpo->lpo_namehash = hash; + hash = (lpo->lpo_namehash * 31 * 31 + (uintptr_t)p * 31 + lpo->lpo_lineno) & LPROF_HASH_MASK; + mpp = &lprof_buf[hash]; + while (mpp->name != NULL) { + if (mpp->line == lpo->lpo_lineno && + mpp->file == p && + mpp->namehash == lpo->lpo_namehash) + break; + /* If the lprof_hash entry is allocated to someone + * else, try the next one + */ + collision = 1; + hash = (hash + 1) & LPROF_HASH_MASK; + mpp = &lprof_buf[hash]; + } + if (mpp->name == NULL) { + int buf; + + buf = allocated_lprof_buf++; + /* Just exit if we cannot get a trace buffer */ + if (buf >= LPROF_HASH_SIZE) { + ++lock_prof_rejected; + return; + } + mpp->file = p; + mpp->line = lpo->lpo_lineno; + mpp->namehash = lpo->lpo_namehash; + mpp->type = lpo->lpo_type; + mpp->name = lpo->lpo_name; + + if (collision) + ++lock_prof_collisions; + /* + * We might have raced someone else but who cares, + * they'll try again next time + */ + ++lock_prof_records; + } + /* + * Record if the lock has been held longer now than ever + * before. + */ + if (now - acqtime > mpp->cnt_max) + mpp->cnt_max = now - acqtime; + mpp->cnt_tot += now - acqtime; + mpp->cnt_wait += waittime; + mpp->cnt_cur++; + /* * There's a small race, really we should cmpxchg * 0 with the current value, but that would bill * the contention to the wrong lock instance if * it followed this also. */ - mpp->cnt_contest_holding += l->lpo_contest_holding; - mpp->cnt_contest_locking += l->lpo_contest_locking; - LPROF_UNLOCK(hash); + mpp->cnt_contest_holding += lpo->lpo_contest_holding; + mpp->cnt_contest_locking += lpo->lpo_contest_locking; +} - } - l->lpo_acqtime = 0; - l->lpo_waittime = 0; - l->lpo_contest_locking = 0; - l->lpo_contest_holding = 0; +static void +lock_profile_update_cpu(struct lock_profile_ring *lpr) +{ + int prod, cons; + struct lock_profile_object *lpo; + + prod = lpr->lpr_prod; + cons = lpr->lpr_cons; + + while (cons != prod) { + lock_prof_acquisitions++; + lpo = lpr->lpr_lpo_ring[cons]; + lock_profile_update_hash(lpo); + if (++cons == lpr->lpr_size) + cons = 0; + } + critical_enter(); + /* + * XXX this is a gross and pointlest atomic op + * but FreeBSD has no MI way to do memory barriers + * we need to ensure that the store buffer is flushed + * to memory before we get scheduled on another cpu + * KMM 07.07.18 + */ + atomic_cmpset_acq_int(&lpr->lpr_cons, lpr->lpr_cons, cons); + critical_exit(); +} + +static void +lock_profile_update_proc(void *unused) +{ + int i; + struct pcpu *pc; + + printf("lock_profile_update started\n"); + for (;;) { + lock_prof_iterations++; + hash_update_running = 1; + for (i = 0; i <= mp_maxid; i++) { + if ((pc = pcpu_find(i)) == NULL) + continue; + lock_profile_update_cpu(pc->pc_lpr); + lock_profile_update_cpu(pc->pc_slpr); + } + hash_update_running = 0; + tsleep(&hash_update_running, 0, "lpidle", sleep_ticks + 1); + } +} + +void +lock_profile_start(void) +{ + struct proc *p; + + sleep_ticks = hz >> 4; /* default value */ + kthread_create(lock_profile_update_proc, NULL, &p, + RFNOWAIT, 0, "lprofd"); +} + +void +lock_prof_ring_init(struct lock_profile_ring **plpr, int size) +{ + int i; + struct lock_profile_ring *lpr; + struct lock_profile_object *lpo; + + lpr = *plpr = malloc(sizeof(struct lock_profile_ring), + M_DEVBUF, M_WAITOK | M_ZERO); + lpr->lpr_lpo_ring = malloc(size*sizeof(struct lock_profile_object *), + M_DEVBUF, M_WAITOK | M_ZERO); + lpr->lpr_cons = lpr->lpr_prod = 0; + lpr->lpr_size = size; + LIST_INIT(&lpr->lpr_lpo_free); + + for (i = 0; i < size*2; i++) { + lpo = malloc(sizeof(struct lock_profile_object), M_DEVBUF, M_WAITOK|M_ZERO); + LIST_INSERT_HEAD(&lpr->lpr_lpo_free, lpo, lpo_entry); + } } + +int +lock_profile_init_rings(void) +{ + int i; + struct pcpu *pc; + + for (i = 0; i <= mp_maxid; i++) { + char field_name[32]; + + if ((pc = pcpu_find(i)) == NULL) + continue; + printf("initializing profile ring for cpu%d\n", i); + + lock_prof_ring_init(&pc->pc_lpr, 2048); + + snprintf(field_name, sizeof(field_name), "cpu%d_ring_full", i); + SYSCTL_ADD_UINT(NULL, SYSCTL_STATIC_CHILDREN(_debug_lock_prof_internal), OID_AUTO, + field_name, CTLFLAG_RD, &pc->pc_lpr->lpr_lpo_ring_full, 0, NULL); + snprintf(field_name, sizeof(field_name), "cpu%d_free_list_empty", i); + SYSCTL_ADD_UINT(NULL, SYSCTL_STATIC_CHILDREN(_debug_lock_prof_internal), OID_AUTO, + field_name, CTLFLAG_RD, &pc->pc_lpr->lpr_lpo_free_empty, 0, NULL); + snprintf(field_name, sizeof(field_name), "cpu%d_alloc_count", i); + SYSCTL_ADD_UINT(NULL, SYSCTL_STATIC_CHILDREN(_debug_lock_prof_internal), OID_AUTO, + field_name, CTLFLAG_RD, &pc->pc_lpr->lpr_lpo_alloc_count, 0, NULL); + + lock_prof_ring_init(&pc->pc_slpr, 512); + + snprintf(field_name, sizeof(field_name), "cpu%d_spin_ring_full", i); + SYSCTL_ADD_UINT(NULL, SYSCTL_STATIC_CHILDREN(_debug_lock_prof_internal), OID_AUTO, + field_name, CTLFLAG_RD, &pc->pc_slpr->lpr_lpo_ring_full, 0, NULL); + snprintf(field_name, sizeof(field_name), "cpu%d_spin_free_list_empty", i); + SYSCTL_ADD_UINT(NULL, SYSCTL_STATIC_CHILDREN(_debug_lock_prof_internal), OID_AUTO, + field_name, CTLFLAG_RD, &pc->pc_slpr->lpr_lpo_free_empty, 0, NULL); + snprintf(field_name, sizeof(field_name), "cpu%d_spin_alloc_count", i); + SYSCTL_ADD_UINT(NULL, SYSCTL_STATIC_CHILDREN(_debug_lock_prof_internal), OID_AUTO, + field_name, CTLFLAG_RD, &pc->pc_slpr->lpr_lpo_alloc_count, 0, NULL); + + } + return (0); +} + #endif Index: sys/sys/_lock.h =================================================================== RCS file: /usr/home/kmacy/devel/cxgb/ncvs/src/sys/sys/_lock.h,v retrieving revision 1.14 diff -d -u -r1.14 _lock.h --- sys/sys/_lock.h 13 Nov 2006 05:41:26 -0000 1.14 +++ sys/sys/_lock.h 19 Jul 2007 16:11:15 -0000 @@ -31,17 +31,24 @@ #ifndef _SYS__LOCK_H_ #define _SYS__LOCK_H_ +struct lock_object { + const char *lo_name; /* Individual lock name. */ + const char *lo_type; /* General lock type. */ + u_int lo_flags; + union { /* Data for witness. */ + STAILQ_ENTRY(lock_object) lod_list; + struct witness *lod_witness; + } lo_witness_data; +}; + struct lock_profile_object { - /* - * This does not result in variant structure sizes because - * MUTEX_PROFILING is in opt_global.h - */ u_int64_t lpo_acqtime; u_int64_t lpo_waittime; const char *lpo_filename; u_int lpo_namehash; int lpo_lineno; const char *lpo_type; + const char *lpo_name; /* * Fields relating to measuring contention on mutexes. * holding must be accessed atomically since it's @@ -51,19 +58,9 @@ */ u_int lpo_contest_holding; u_int lpo_contest_locking; -}; -struct lock_object { - const char *lo_name; /* Individual lock name. */ - const char *lo_type; /* General lock type. */ - u_int lo_flags; -#ifdef LOCK_PROFILING - struct lock_profile_object lo_profile_obj; -#endif - union { /* Data for witness. */ - STAILQ_ENTRY(lock_object) lod_list; - struct witness *lod_witness; - } lo_witness_data; + struct lock_object *lpo_object; /* pointer to profiled lock */ + LIST_ENTRY(lock_profile_object) lpo_entry; /* list entry for thread & free list */ }; #endif /* !_SYS__LOCK_H_ */ Index: sys/sys/filedesc.h =================================================================== RCS file: /usr/home/kmacy/devel/cxgb/ncvs/src/sys/sys/filedesc.h,v retrieving revision 1.78 diff -d -u -r1.78 filedesc.h --- sys/sys/filedesc.h 31 May 2007 11:51:52 -0000 1.78 +++ sys/sys/filedesc.h 18 Jul 2007 06:51:24 -0000 @@ -37,6 +37,7 @@ #include #include #include +#include #include #include Index: sys/sys/lock_profile.h =================================================================== RCS file: /usr/home/kmacy/devel/cxgb/ncvs/src/sys/sys/lock_profile.h,v retrieving revision 1.14 diff -d -u -r1.14 lock_profile.h --- sys/sys/lock_profile.h 11 Jul 2007 18:51:31 -0000 1.14 +++ sys/sys/lock_profile.h 18 Jul 2007 10:54:54 -0000 @@ -34,8 +34,6 @@ #ifdef LOCK_PROFILING #include #include -#include -#include #include #ifndef LPROF_HASH_SIZE @@ -47,11 +45,25 @@ u_int64_t nanoseconds(void); #endif +LIST_HEAD(lock_profile_object_head, lock_profile_object); +typedef struct lock_profile_object_head lpoh_t; + +struct lock_profile_ring { + struct lock_profile_object **lpr_lpo_ring; /* pointer to profiled locks array */ + int lpr_prod; + int lpr_cons; + int lpr_size; + lpoh_t lpr_lpo_free; /* head of the per-cpu free list */ + uint32_t lpr_lpo_alloc_count; + uint32_t lpr_lpo_ring_full; + uint32_t lpr_lpo_free_empty; +}; + struct lock_prof { const char *name; const char *type; const char *file; - u_int namehash; + u_int namehash; int line; uintmax_t cnt_max; uintmax_t cnt_tot; @@ -64,65 +76,29 @@ extern struct lock_prof lprof_buf[LPROF_HASH_SIZE]; #define LPROF_SBUF_SIZE 256 * 400 -/* We keep a smaller pool of spin mutexes for protecting the lprof hash entries */ -#define LPROF_LOCK_SIZE 16 -#define LPROF_LOCK_MASK (LPROF_LOCK_SIZE - 1) #define LPROF_LHASH(hash) ((hash) & LPROF_LOCK_MASK) -#define LPROF_LOCK(hash) mtx_lock_spin(&lprof_locks[LPROF_LHASH(hash)]) -#define LPROF_UNLOCK(hash) mtx_unlock_spin(&lprof_locks[LPROF_LHASH(hash)]) - #ifdef _KERNEL -extern struct mtx lprof_locks[LPROF_LOCK_SIZE]; extern int lock_prof_enable; void _lock_profile_obtain_lock_success(struct lock_object *lo, int contested, uint64_t waittime, const char *file, int line); -void _lock_profile_update_wait(struct lock_object *lo, uint64_t waitstart); -void _lock_profile_release_lock(struct lock_object *lo); - -static inline void lock_profile_object_init(struct lock_object *lo, struct lock_class *class, const char *name) { - const char *p; - u_int hash = 0; - struct lock_profile_object *l = &lo->lo_profile_obj; - - l->lpo_acqtime = 0; - l->lpo_waittime = 0; - l->lpo_filename = NULL; - l->lpo_lineno = 0; - l->lpo_contest_holding = 0; - l->lpo_contest_locking = 0; - l->lpo_type = class->lc_name; - - /* Hash the mutex name to an int so we don't have to strcmp() it repeatedly */ - for (p = name; *p != '\0'; p++) - hash = 31 * hash + *p; - l->lpo_namehash = hash; -#if 0 - if (opts & MTX_PROFILE) - l->lpo_stack = stack_create(); -#endif -} - - -static inline void -lock_profile_object_destroy(struct lock_object *lo) -{ -#if 0 - struct lock_profile_object *l = &lo->lo_profile_obj; - if (lo->lo_flags & LO_PROFILE) - stack_destroy(l->lpo_stack); -#endif -} +void _lock_profile_release_lock(struct lock_object *lo, struct lock_profile_object *lpo); +struct lock_profile_object *lock_profile_find_lpo(struct lock_object *lo); +void lock_profile_start(void); +void lock_prof_ring_init(struct lock_profile_ring **plpr, int size); +int lock_profile_init_rings(void); static inline void lock_profile_obtain_lock_failed(struct lock_object *lo, int *contested, uint64_t *waittime) { - struct lock_profile_object *l = &lo->lo_profile_obj; - if (lock_prof_enable && *contested == 0) { + if (lock_prof_enable && *contested == 0 && (lo->lo_flags & LO_NOPROFILE) == 0) { *waittime = nanoseconds(); - atomic_add_int(&l->lpo_contest_holding, 1); *contested = 1; +#if 0 + if (l) + atomic_add_int(&l->lpo_contest_holding, 1); +#endif } } @@ -130,20 +106,16 @@ { /* don't reset the timer when/if recursing */ - if (lock_prof_enable && lo->lo_profile_obj.lpo_acqtime == 0) { -#ifdef LOCK_PROFILING_FAST - if (contested == 0) - return; -#endif + if (lock_prof_enable) _lock_profile_obtain_lock_success(lo, contested, waittime, file, line); - } } + static inline void lock_profile_release_lock(struct lock_object *lo) { - struct lock_profile_object *l = &lo->lo_profile_obj; - - if (l->lpo_acqtime) - _lock_profile_release_lock(lo); + struct lock_profile_object *lpo; + + if (lock_prof_enable && (lpo = lock_profile_find_lpo(lo)) != NULL) + _lock_profile_release_lock(lo, lpo); } #endif /* _KERNEL */ @@ -151,15 +123,9 @@ #else /* !LOCK_PROFILING */ #ifdef _KERNEL -static inline void lock_profile_update_wait(struct lock_object *lo, uint64_t waitstart) {;} -static inline void lock_profile_update_contest_locking(struct lock_object *lo, int contested) {;} static inline void lock_profile_release_lock(struct lock_object *lo) {;} static inline void lock_profile_obtain_lock_failed(struct lock_object *lo, int *contested, uint64_t *waittime) {;} -static inline void lock_profile_obtain_lock_success(struct lock_object *lo, int contested, uint64_t waittime, - const char *file, int line) {;} -static inline void lock_profile_object_destroy(struct lock_object *lo) {;} -static inline void lock_profile_object_init(struct lock_object *lo, struct lock_class *class, const char *name) {;} - +static inline void lock_profile_obtain_lock_success(struct lock_object *lo, int contested, uint64_t waittime, const char *file, int line) {;} #endif /* _KERNEL */ #endif /* !LOCK_PROFILING */ Index: sys/sys/pcpu.h =================================================================== RCS file: /usr/home/kmacy/devel/cxgb/ncvs/src/sys/sys/pcpu.h,v retrieving revision 1.22 diff -d -u -r1.22 pcpu.h --- sys/sys/pcpu.h 6 Jun 2007 07:35:08 -0000 1.22 +++ sys/sys/pcpu.h 18 Jul 2007 08:27:40 -0000 @@ -45,6 +45,11 @@ #include #include +#ifdef LOCK_PROFILING +#include +#include +#endif + struct pcb; struct thread; @@ -67,13 +72,17 @@ cpumask_t pc_other_cpus; /* Mask of all other cpus */ SLIST_ENTRY(pcpu) pc_allcpu; struct lock_list_entry *pc_spinlocks; + PCPU_MD_FIELDS; + struct vmmeter pc_cnt; /* VM stats counters */ + struct device *pc_device; +#ifdef LOCK_PROFILING + struct lock_profile_ring *pc_slpr; /* profiled spin locks ring */ + struct lock_profile_ring *pc_lpr; /* profiled non-spin locks ring */ +#endif /* LOCK_PROFILING */ #ifdef KTR_PERCPU int pc_ktr_idx; /* Index into trace table */ char *pc_ktr_buf; #endif - PCPU_MD_FIELDS; - struct vmmeter pc_cnt; /* VM stats counters */ - struct device *pc_device; }; SLIST_HEAD(cpuhead, pcpu); Index: sys/sys/proc.h =================================================================== RCS file: /usr/home/kmacy/devel/cxgb/ncvs/src/sys/sys/proc.h,v retrieving revision 1.486 diff -d -u -r1.486 proc.h --- sys/sys/proc.h 12 Jun 2007 20:22:06 -0000 1.486 +++ sys/sys/proc.h 18 Jul 2007 06:59:50 -0000 @@ -263,6 +263,10 @@ u_long td_profil_addr; /* (k) Temporary addr until AST. */ u_int td_profil_ticks; /* (k) Temporary ticks until AST. */ char td_name[MAXCOMLEN + 1]; /* (*) Thread name. */ +#ifdef LOCK_PROFILING + lpoh_t td_lprofile; /* profiling information for regular locks */ + lpoh_t td_slprofile; /* profiling information for spin locks */ +#endif #define td_endzero td_base_pri /* Copied during fork1() or thread_sched_upcall(). */