--- a/usr/src/uts/common/disp/cpupart.c Tue Apr 13 09:45:48 2010 +0800
+++ b/usr/src/uts/common/disp/cpupart.c Mon Apr 12 22:00:39 2010 -0700
@@ -19,8 +19,7 @@
* CDDL HEADER END
*/
/*
- * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 1996, 2010, Oracle and/or its affiliates. All rights reserved.
*/
#include <sys/types.h>
@@ -97,7 +96,6 @@
#define PSTOCP(psid) ((cpupartid_t)((psid) == PS_NONE ? CP_DEFAULT : (psid)))
#define CPTOPS(cpid) ((psetid_t)((cpid) == CP_DEFAULT ? PS_NONE : (cpid)))
-
static int cpupart_unbind_threads(cpupart_t *, boolean_t);
/*
@@ -312,7 +310,8 @@
t0.t_lpl = &cp_default.cp_lgrploads[LGRP_ROOTID];
bitset_init(&cp_default.cp_cmt_pgs);
- bitset_init(&cp_default.cp_haltset);
+ bitset_init_fanout(&cp_default.cp_haltset, cp_haltset_fanout);
+
bitset_resize(&cp_default.cp_haltset, max_ncpus);
}
@@ -856,9 +855,9 @@
bitset_init(&pp->cp_cmt_pgs);
/*
- * Initialize and size the partition's bitset of halted CPUs
+ * Initialize and size the partition's bitset of halted CPUs.
*/
- bitset_init(&pp->cp_haltset);
+ bitset_init_fanout(&pp->cp_haltset, cp_haltset_fanout);
bitset_resize(&pp->cp_haltset, max_ncpus);
/*
--- a/usr/src/uts/common/os/bitset.c Tue Apr 13 09:45:48 2010 +0800
+++ b/usr/src/uts/common/os/bitset.c Mon Apr 12 22:00:39 2010 -0700
@@ -19,8 +19,7 @@
* CDDL HEADER END
*/
/*
- * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
*/
#include <sys/bitset.h>
@@ -41,6 +40,17 @@
}
/*
+ * Initialize a bitset_t using a fanout. The fanout factor is platform
+ * specific and passed in as a power of two.
+ */
+void
+bitset_init_fanout(bitset_t *b, uint_t fanout)
+{
+ bzero(b, sizeof (bitset_t));
+ b->bs_fanout = fanout;
+}
+
+/*
* Uninitialize a bitset_t.
* This will free the bitset's data, leaving it zero sized.
*/
@@ -52,18 +62,18 @@
}
/*
- * Resize a bitset to where it can hold sz number of bits.
+ * Resize a bitset to where it can hold els number of elements.
* This can either grow or shrink the bitset holding capacity.
* In the case of shrinkage, elements that reside outside the new
* holding capacity of the bitset are lost.
*/
void
-bitset_resize(bitset_t *b, uint_t sz)
+bitset_resize(bitset_t *b, uint_t els)
{
uint_t nwords;
ulong_t *bset_new, *bset_tmp;
- nwords = BT_BITOUL(sz);
+ nwords = BT_BITOUL(els << b->bs_fanout);
if (b->bs_words == nwords)
return; /* already properly sized */
@@ -87,11 +97,12 @@
/* free up the old array */
if (b->bs_words > 0)
kmem_free(bset_tmp, b->bs_words * sizeof (ulong_t));
+
b->bs_words = nwords;
}
/*
- * Returns the current holding capacity of the bitset
+ * Returns the current holding capacity of the bitset.
*/
uint_t
bitset_capacity(bitset_t *b)
@@ -100,35 +111,33 @@
}
/*
- * Add and delete bits in the bitset.
+ * Add (set) and delete (clear) bits in the bitset.
*
- * Adding a bit that is already set, and clearing a bit that's already clear
+ * Adding a bit that is already set, or removing a bit that's already clear
* is legal.
*
* Adding or deleting an element that falls outside the bitset's current
* holding capacity is illegal.
*/
-
-/*
- * Set a bit
- */
void
bitset_add(bitset_t *b, uint_t elt)
{
- ASSERT(b->bs_words * BT_NBIPUL > elt);
+ uint_t pos = (elt << b->bs_fanout);
- BT_SET(b->bs_set, elt);
+ ASSERT(b->bs_words * BT_NBIPUL > pos);
+ BT_SET(b->bs_set, pos);
}
/*
- * Set a bit in an atomically safe way
+ * Set a bit in an atomically safe way.
*/
void
bitset_atomic_add(bitset_t *b, uint_t elt)
{
- ASSERT(b->bs_words * BT_NBIPUL > elt);
+ uint_t pos = (elt << b->bs_fanout);
- BT_ATOMIC_SET(b->bs_set, elt);
+ ASSERT(b->bs_words * BT_NBIPUL > pos);
+ BT_ATOMIC_SET(b->bs_set, pos);
}
/*
@@ -138,34 +147,37 @@
int
bitset_atomic_test_and_add(bitset_t *b, uint_t elt)
{
- int r;
+ uint_t pos = (elt << b->bs_fanout);
+ int ret;
- ASSERT(b->bs_words * BT_NBIPUL > elt);
- BT_ATOMIC_SET_EXCL(b->bs_set, elt, r);
+ ASSERT(b->bs_words * BT_NBIPUL > pos);
+ BT_ATOMIC_SET_EXCL(b->bs_set, pos, ret);
- return (r);
+ return (ret);
}
/*
- * Clear a bit
+ * Clear a bit.
*/
void
bitset_del(bitset_t *b, uint_t elt)
{
- ASSERT(b->bs_words * BT_NBIPUL > elt);
+ uint_t pos = (elt << b->bs_fanout);
- BT_CLEAR(b->bs_set, elt);
+ ASSERT(b->bs_words * BT_NBIPUL > pos);
+ BT_CLEAR(b->bs_set, pos);
}
/*
- * Clear a bit in an atomically safe way
+ * Clear a bit in an atomically safe way.
*/
void
bitset_atomic_del(bitset_t *b, uint_t elt)
{
- ASSERT(b->bs_words * BT_NBIPUL > elt);
+ uint_t pos = (elt << b->bs_fanout);
- BT_ATOMIC_CLEAR(b->bs_set, elt);
+ ASSERT(b->bs_words * BT_NBIPUL > pos);
+ BT_ATOMIC_CLEAR(b->bs_set, pos);
}
/*
@@ -175,33 +187,36 @@
int
bitset_atomic_test_and_del(bitset_t *b, uint_t elt)
{
- int r;
+ uint_t pos = (elt << b->bs_fanout);
+ int ret;
- ASSERT(b->bs_words * BT_NBIPUL > elt);
- BT_ATOMIC_CLEAR_EXCL(b->bs_set, elt, r);
+ ASSERT(b->bs_words * BT_NBIPUL > pos);
+ BT_ATOMIC_CLEAR_EXCL(b->bs_set, pos, ret);
- return (r);
+ return (ret);
}
/*
- * Return non-zero if the bit is present in the set
+ * Return non-zero if the bit is present in the set.
*/
int
bitset_in_set(bitset_t *b, uint_t elt)
{
- if (elt >= b->bs_words * BT_NBIPUL)
+ uint_t pos = (elt << b->bs_fanout);
+
+ if (pos >= b->bs_words * BT_NBIPUL)
return (0);
- return (BT_TEST(b->bs_set, elt));
+ return (BT_TEST(b->bs_set, pos));
}
/*
- * Return non-zero if the bitset is empty
+ * Return non-zero if the bitset is empty.
*/
int
bitset_is_null(bitset_t *b)
{
- int i;
+ int i;
for (i = 0; i < b->bs_words; i++)
if (b->bs_set[i] != 0)
@@ -210,9 +225,9 @@
}
/*
- * Perform a non-victimizing search for a set bit in a word
+ * Perform a non-victimizing search for a set bit in a word.
* A "seed" is passed to pseudo-randomize the search.
- * Return -1 if no set bit was found
+ * Return -1 if no set bit was found.
*/
static uint_t
bitset_find_in_word(ulong_t w, uint_t seed)
@@ -245,6 +260,8 @@
uint_t seed;
seed = CPU_PSEUDO_RANDOM();
+
+ ASSERT(b->bs_words > 0);
start = seed % b->bs_words;
i = start;
@@ -252,7 +269,7 @@
elt = bitset_find_in_word(b->bs_set[i], seed);
if (elt != (uint_t)-1) {
elt += i * BT_NBIPUL;
- break;
+ return (elt >> b->bs_fanout);
}
if (++i == b->bs_words)
i = 0;
@@ -262,14 +279,17 @@
}
/*
- * AND, OR, and XOR bitset computations
- * Returns 1 if resulting bitset has any set bits
+ * AND, OR, and XOR bitset computations, returns 1 if resulting bitset has any
+ * set bits. Operands must have the same fanout, if any.
*/
int
bitset_and(bitset_t *bs1, bitset_t *bs2, bitset_t *res)
{
int i, anyset;
+ ASSERT(bs1->bs_fanout == bs2->bs_fanout);
+ ASSERT(bs1->bs_fanout == res->bs_fanout);
+
for (anyset = 0, i = 0; i < bs1->bs_words; i++) {
if ((res->bs_set[i] = (bs1->bs_set[i] & bs2->bs_set[i])) != 0)
anyset = 1;
@@ -282,6 +302,9 @@
{
int i, anyset;
+ ASSERT(bs1->bs_fanout == bs2->bs_fanout);
+ ASSERT(bs1->bs_fanout == res->bs_fanout);
+
for (anyset = 0, i = 0; i < bs1->bs_words; i++) {
if ((res->bs_set[i] = (bs1->bs_set[i] | bs2->bs_set[i])) != 0)
anyset = 1;
@@ -292,8 +315,10 @@
int
bitset_xor(bitset_t *bs1, bitset_t *bs2, bitset_t *res)
{
- int i;
- int anyset = 0;
+ int i, anyset = 0;
+
+ ASSERT(bs1->bs_fanout == bs2->bs_fanout);
+ ASSERT(bs1->bs_fanout == res->bs_fanout);
for (i = 0; i < bs1->bs_words; i++) {
if ((res->bs_set[i] = (bs1->bs_set[i] ^ bs2->bs_set[i])) != 0)
@@ -303,7 +328,7 @@
}
/*
- * return 1 if bitmaps are identical
+ * Return 1 if bitmaps are identical.
*/
int
bitset_match(bitset_t *bs1, bitset_t *bs2)
@@ -320,7 +345,7 @@
}
/*
- * Zero a bitset_t
+ * Zero a bitset_t.
*/
void
bitset_zero(bitset_t *b)
@@ -328,12 +353,12 @@
bzero(b->bs_set, sizeof (ulong_t) * b->bs_words);
}
-
/*
- * Copy a bitset_t
+ * Copy a bitset_t.
*/
void
bitset_copy(bitset_t *src, bitset_t *dest)
{
+ ASSERT(src->bs_fanout == dest->bs_fanout);
bcopy(src->bs_set, dest->bs_set, sizeof (ulong_t) * src->bs_words);
}
--- a/usr/src/uts/common/sys/bitset.h Tue Apr 13 09:45:48 2010 +0800
+++ b/usr/src/uts/common/sys/bitset.h Mon Apr 12 22:00:39 2010 -0700
@@ -19,8 +19,7 @@
* CDDL HEADER END
*/
/*
- * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
*/
#ifndef _BITSET_H
@@ -37,12 +36,14 @@
typedef struct bitset {
ulong_t *bs_set;
uint_t bs_words;
+ uint_t bs_fanout;
} bitset_t;
/*
* Bitset initialiation / teardown
*/
void bitset_init(bitset_t *);
+void bitset_init_fanout(bitset_t *, uint_t);
void bitset_fini(bitset_t *);
/*
--- a/usr/src/uts/common/sys/cpupart.h Tue Apr 13 09:45:48 2010 +0800
+++ b/usr/src/uts/common/sys/cpupart.h Mon Apr 12 22:00:39 2010 -0700
@@ -19,8 +19,7 @@
* CDDL HEADER END
*/
/*
- * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 1996, 2010, Oracle and/or its affiliates. All rights reserved.
*/
#ifndef _SYS_CPUPART_H
@@ -125,6 +124,15 @@
extern uint_t cp_numparts;
extern uint_t cp_numparts_nonempty;
+/*
+ * Each partition contains a bitset that indicates which CPUs are halted and
+ * which ones are running. Given the growing number of CPUs in current and
+ * future platforms, it's important to fanout each CPU within its partition's
+ * haltset to prevent contention due to false sharing. The fanout factor
+ * is platform specific, and declared accordingly.
+ */
+extern uint_t cp_haltset_fanout;
+
extern void cpupart_initialize_default();
extern cpupart_t *cpupart_find(psetid_t);
extern int cpupart_create(psetid_t *);
--- a/usr/src/uts/i86pc/os/mp_machdep.c Tue Apr 13 09:45:48 2010 +0800
+++ b/usr/src/uts/i86pc/os/mp_machdep.c Mon Apr 12 22:00:39 2010 -0700
@@ -20,8 +20,7 @@
* CDDL HEADER END
*/
/*
- * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
*/
/*
* Copyright (c) 2009-2010, Intel Corporation.
@@ -220,6 +219,8 @@
#endif /* ifndef __xpv */
+uint_t cp_haltset_fanout = 0;
+
/*ARGSUSED*/
int
pg_plat_hw_shared(cpu_t *cp, pghw_type_t hw)
--- a/usr/src/uts/sun4u/os/mach_startup.c Tue Apr 13 09:45:48 2010 +0800
+++ b/usr/src/uts/sun4u/os/mach_startup.c Mon Apr 12 22:00:39 2010 -0700
@@ -19,8 +19,7 @@
* CDDL HEADER END
*/
/*
- * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
*/
#include <sys/machsystm.h>
@@ -103,6 +102,8 @@
int enable_halt_idle_cpus = 1; /* global switch */
+uint_t cp_haltset_fanout = 3;
+
void
setup_trap_table(void)
{
--- a/usr/src/uts/sun4v/os/mach_startup.c Tue Apr 13 09:45:48 2010 +0800
+++ b/usr/src/uts/sun4v/os/mach_startup.c Mon Apr 12 22:00:39 2010 -0700
@@ -20,8 +20,7 @@
*/
/*
- * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
*/
#include <sys/machsystm.h>
@@ -77,6 +76,8 @@
extern int clock_tick_threshold;
extern int clock_tick_ncpus;
+uint_t cp_haltset_fanout = 3;
+
void
setup_trap_table(void)
{