usr/src/uts/common/os/bitset.c
author Rafael Vanoni <rafael.vanoni@sun.com>
Mon, 12 Apr 2010 22:00:39 -0700
changeset 12149 607008ac563e
parent 10696 cd0f390dd9e2
permissions -rw-r--r--
6900377 atomic operations on cp_haltset limit IOPs
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
3434
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
     1
/*
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
     2
 * CDDL HEADER START
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
     3
 *
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
     4
 * The contents of this file are subject to the terms of the
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
     5
 * Common Development and Distribution License (the "License").
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
     6
 * You may not use this file except in compliance with the License.
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
     7
 *
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
     8
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
     9
 * or http://www.opensolaris.org/os/licensing.
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    10
 * See the License for the specific language governing permissions
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    11
 * and limitations under the License.
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    12
 *
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    13
 * When distributing Covered Code, include this CDDL HEADER in each
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    14
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    15
 * If applicable, add the following below this CDDL HEADER, with the
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    16
 * fields enclosed by brackets "[]" replaced with your own identifying
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    17
 * information: Portions Copyright [yyyy] [name of copyright owner]
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    18
 *
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    19
 * CDDL HEADER END
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    20
 */
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    21
/*
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
    22
 * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
3434
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    23
 */
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    24
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    25
#include <sys/bitset.h>
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    26
#include <sys/kmem.h>
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    27
#include <sys/systm.h>
8408
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
    28
#include <sys/cpuvar.h>
3434
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    29
#include <sys/cmn_err.h>
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    30
#include <sys/sysmacros.h>
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    31
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    32
/*
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    33
 * Initialize a bitset_t.
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    34
 * After bitset_init(), the bitset will be zero sized.
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    35
 */
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    36
void
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    37
bitset_init(bitset_t *b)
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    38
{
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    39
	bzero(b, sizeof (bitset_t));
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    40
}
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    41
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    42
/*
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
    43
 * Initialize a bitset_t using a fanout. The fanout factor is platform
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
    44
 * specific and passed in as a power of two.
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
    45
 */
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
    46
void
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
    47
bitset_init_fanout(bitset_t *b, uint_t fanout)
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
    48
{
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
    49
	bzero(b, sizeof (bitset_t));
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
    50
	b->bs_fanout = fanout;
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
    51
}
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
    52
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
    53
/*
3434
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    54
 * Uninitialize a bitset_t.
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    55
 * This will free the bitset's data, leaving it zero sized.
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    56
 */
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    57
void
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    58
bitset_fini(bitset_t *b)
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    59
{
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    60
	if (b->bs_words > 0)
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    61
		kmem_free(b->bs_set, b->bs_words * sizeof (ulong_t));
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    62
}
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    63
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    64
/*
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
    65
 * Resize a bitset to where it can hold els number of elements.
3434
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    66
 * This can either grow or shrink the bitset holding capacity.
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    67
 * In the case of shrinkage, elements that reside outside the new
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    68
 * holding capacity of the bitset are lost.
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    69
 */
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    70
void
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
    71
bitset_resize(bitset_t *b, uint_t els)
3434
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    72
{
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    73
	uint_t	nwords;
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    74
	ulong_t	*bset_new, *bset_tmp;
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    75
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
    76
	nwords = BT_BITOUL(els << b->bs_fanout);
3434
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    77
	if (b->bs_words == nwords)
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    78
		return;	/* already properly sized */
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    79
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    80
	/*
8408
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
    81
	 * Allocate the new ulong_t array, and copy the old one, if there
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
    82
	 * was an old one.
3434
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    83
	 */
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    84
	if (nwords > 0) {
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    85
		bset_new = kmem_zalloc(nwords * sizeof (ulong_t), KM_SLEEP);
8408
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
    86
		if (b->bs_words > 0)
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
    87
			bcopy(b->bs_set, bset_new,
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
    88
			    MIN(b->bs_words, nwords) * sizeof (ulong_t));
3434
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    89
	} else {
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    90
		bset_new = NULL;
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    91
	}
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    92
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    93
	/* swap out the old ulong_t array for new one */
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    94
	bset_tmp = b->bs_set;
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    95
	b->bs_set = bset_new;
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    96
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
    97
	/* free up the old array */
8408
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
    98
	if (b->bs_words > 0)
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
    99
		kmem_free(bset_tmp, b->bs_words * sizeof (ulong_t));
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   100
3434
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   101
	b->bs_words = nwords;
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   102
}
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   103
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   104
/*
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   105
 * Returns the current holding capacity of the bitset.
3434
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   106
 */
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   107
uint_t
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   108
bitset_capacity(bitset_t *b)
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   109
{
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   110
	return (b->bs_words * BT_NBIPUL);
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   111
}
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   112
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   113
/*
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   114
 * Add (set) and delete (clear) bits in the bitset.
3434
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   115
 *
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   116
 * Adding a bit that is already set, or removing a bit that's already clear
3434
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   117
 * is legal.
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   118
 *
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   119
 * Adding or deleting an element that falls outside the bitset's current
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   120
 * holding capacity is illegal.
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   121
 */
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   122
void
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   123
bitset_add(bitset_t *b, uint_t elt)
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   124
{
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   125
	uint_t pos = (elt << b->bs_fanout);
3434
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   126
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   127
	ASSERT(b->bs_words * BT_NBIPUL > pos);
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   128
	BT_SET(b->bs_set, pos);
3434
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   129
}
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   130
8408
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   131
/*
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   132
 * Set a bit in an atomically safe way.
8408
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   133
 */
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   134
void
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   135
bitset_atomic_add(bitset_t *b, uint_t elt)
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   136
{
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   137
	uint_t pos = (elt << b->bs_fanout);
8408
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   138
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   139
	ASSERT(b->bs_words * BT_NBIPUL > pos);
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   140
	BT_ATOMIC_SET(b->bs_set, pos);
8408
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   141
}
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   142
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   143
/*
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   144
 * Atomically test that a given bit isn't set, and set it.
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   145
 * Returns -1 if the bit was already set.
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   146
 */
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   147
int
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   148
bitset_atomic_test_and_add(bitset_t *b, uint_t elt)
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   149
{
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   150
	uint_t pos = (elt << b->bs_fanout);
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   151
	int ret;
8408
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   152
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   153
	ASSERT(b->bs_words * BT_NBIPUL > pos);
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   154
	BT_ATOMIC_SET_EXCL(b->bs_set, pos, ret);
8408
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   155
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   156
	return (ret);
8408
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   157
}
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   158
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   159
/*
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   160
 * Clear a bit.
8408
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   161
 */
3434
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   162
void
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   163
bitset_del(bitset_t *b, uint_t elt)
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   164
{
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   165
	uint_t pos = (elt << b->bs_fanout);
3434
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   166
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   167
	ASSERT(b->bs_words * BT_NBIPUL > pos);
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   168
	BT_CLEAR(b->bs_set, pos);
3434
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   169
}
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   170
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   171
/*
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   172
 * Clear a bit in an atomically safe way.
8408
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   173
 */
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   174
void
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   175
bitset_atomic_del(bitset_t *b, uint_t elt)
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   176
{
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   177
	uint_t pos = (elt << b->bs_fanout);
8408
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   178
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   179
	ASSERT(b->bs_words * BT_NBIPUL > pos);
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   180
	BT_ATOMIC_CLEAR(b->bs_set, pos);
8408
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   181
}
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   182
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   183
/*
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   184
 * Atomically test that a bit is set, and clear it.
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   185
 * Returns -1 if the bit was already clear.
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   186
 */
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   187
int
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   188
bitset_atomic_test_and_del(bitset_t *b, uint_t elt)
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   189
{
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   190
	uint_t pos = (elt << b->bs_fanout);
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   191
	int ret;
8408
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   192
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   193
	ASSERT(b->bs_words * BT_NBIPUL > pos);
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   194
	BT_ATOMIC_CLEAR_EXCL(b->bs_set, pos, ret);
8408
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   195
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   196
	return (ret);
8408
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   197
}
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   198
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   199
/*
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   200
 * Return non-zero if the bit is present in the set.
3434
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   201
 */
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   202
int
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   203
bitset_in_set(bitset_t *b, uint_t elt)
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   204
{
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   205
	uint_t pos = (elt << b->bs_fanout);
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   206
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   207
	if (pos >= b->bs_words * BT_NBIPUL)
3693
0500530f987d 6525824 assertion failed: b->bs_words * (1 << 6) > elt
esaxe
parents: 3434
diff changeset
   208
		return (0);
3434
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   209
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   210
	return (BT_TEST(b->bs_set, pos));
3434
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   211
}
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   212
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   213
/*
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   214
 * Return non-zero if the bitset is empty.
3434
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   215
 */
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   216
int
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   217
bitset_is_null(bitset_t *b)
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   218
{
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   219
	int i;
3434
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   220
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   221
	for (i = 0; i < b->bs_words; i++)
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   222
		if (b->bs_set[i] != 0)
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   223
			return (0);
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   224
	return (1);
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   225
}
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   226
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   227
/*
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   228
 * Perform a non-victimizing search for a set bit in a word.
8408
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   229
 * A "seed" is passed to pseudo-randomize the search.
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   230
 * Return -1 if no set bit was found.
8408
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   231
 */
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   232
static uint_t
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   233
bitset_find_in_word(ulong_t w, uint_t seed)
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   234
{
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   235
	uint_t rotate_bit, elt = (uint_t)-1;
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   236
	ulong_t rotated_word;
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   237
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   238
	if (w == (ulong_t)0)
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   239
		return (elt);
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   240
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   241
	rotate_bit = seed % BT_NBIPUL;
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   242
	rotated_word = (w >> rotate_bit) | (w << (BT_NBIPUL - rotate_bit));
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   243
	elt = (uint_t)(lowbit(rotated_word) - 1);
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   244
	if (elt != (uint_t)-1)
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   245
		elt = ((elt + rotate_bit) % BT_NBIPUL);
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   246
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   247
	return (elt);
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   248
}
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   249
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   250
/*
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   251
 * Select a bit that is set in the bitset in a non-victimizing fashion
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   252
 * (e.g. doesn't bias the low/high order bits/words).
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   253
 * Return -1 if no set bit was found
3434
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   254
 */
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   255
uint_t
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   256
bitset_find(bitset_t *b)
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   257
{
8408
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   258
	uint_t start, i;
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   259
	uint_t elt = (uint_t)-1;
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   260
	uint_t seed;
3434
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   261
8408
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   262
	seed = CPU_PSEUDO_RANDOM();
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   263
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   264
	ASSERT(b->bs_words > 0);
8408
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   265
	start = seed % b->bs_words;
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   266
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   267
	i = start;
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   268
	do {
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   269
		elt = bitset_find_in_word(b->bs_set[i], seed);
3434
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   270
		if (elt != (uint_t)-1) {
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   271
			elt += i * BT_NBIPUL;
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   272
			return (elt >> b->bs_fanout);
3434
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   273
		}
8408
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   274
		if (++i == b->bs_words)
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   275
			i = 0;
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   276
	} while (i != start);
7b4e48a75d0c 6724307 CPU selection bias observed on NUMA/CMT systems
Eric Saxe <Eric.Saxe@Sun.COM>
parents: 3693
diff changeset
   277
3434
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   278
	return (elt);
5142e1d7d0bc 6461311 multi-level CMT scheduling optimizations
esaxe
parents:
diff changeset
   279
}
10696
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   280
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   281
/*
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   282
 * AND, OR, and XOR bitset computations, returns 1 if resulting bitset has any
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   283
 * set bits. Operands must have the same fanout, if any.
10696
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   284
 */
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   285
int
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   286
bitset_and(bitset_t *bs1, bitset_t *bs2, bitset_t *res)
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   287
{
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   288
	int i, anyset;
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   289
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   290
	ASSERT(bs1->bs_fanout == bs2->bs_fanout);
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   291
	ASSERT(bs1->bs_fanout == res->bs_fanout);
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   292
10696
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   293
	for (anyset = 0, i = 0; i < bs1->bs_words; i++) {
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   294
		if ((res->bs_set[i] = (bs1->bs_set[i] & bs2->bs_set[i])) != 0)
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   295
			anyset = 1;
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   296
	}
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   297
	return (anyset);
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   298
}
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   299
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   300
int
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   301
bitset_or(bitset_t *bs1, bitset_t *bs2, bitset_t *res)
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   302
{
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   303
	int i, anyset;
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   304
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   305
	ASSERT(bs1->bs_fanout == bs2->bs_fanout);
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   306
	ASSERT(bs1->bs_fanout == res->bs_fanout);
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   307
10696
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   308
	for (anyset = 0, i = 0; i < bs1->bs_words; i++) {
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   309
		if ((res->bs_set[i] = (bs1->bs_set[i] | bs2->bs_set[i])) != 0)
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   310
			anyset = 1;
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   311
	}
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   312
	return (anyset);
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   313
}
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   314
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   315
int
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   316
bitset_xor(bitset_t *bs1, bitset_t *bs2, bitset_t *res)
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   317
{
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   318
	int i, anyset = 0;
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   319
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   320
	ASSERT(bs1->bs_fanout == bs2->bs_fanout);
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   321
	ASSERT(bs1->bs_fanout == res->bs_fanout);
10696
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   322
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   323
	for (i = 0; i < bs1->bs_words; i++) {
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   324
		if ((res->bs_set[i] = (bs1->bs_set[i] ^ bs2->bs_set[i])) != 0)
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   325
			anyset = 1;
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   326
	}
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   327
	return (anyset);
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   328
}
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   329
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   330
/*
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   331
 * Return 1 if bitmaps are identical.
10696
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   332
 */
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   333
int
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   334
bitset_match(bitset_t *bs1, bitset_t *bs2)
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   335
{
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   336
	int i;
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   337
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   338
	if (bs1->bs_words != bs2->bs_words)
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   339
		return (0);
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   340
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   341
	for (i = 0; i < bs1->bs_words; i++)
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   342
		if (bs1->bs_set[i] != bs2->bs_set[i])
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   343
			return (0);
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   344
	return (1);
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   345
}
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   346
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   347
/*
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   348
 * Zero a bitset_t.
10696
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   349
 */
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   350
void
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   351
bitset_zero(bitset_t *b)
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   352
{
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   353
	bzero(b->bs_set, sizeof (ulong_t) * b->bs_words);
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   354
}
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   355
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   356
/*
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   357
 * Copy a bitset_t.
10696
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   358
 */
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   359
void
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   360
bitset_copy(bitset_t *src, bitset_t *dest)
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   361
{
12149
607008ac563e 6900377 atomic operations on cp_haltset limit IOPs
Rafael Vanoni <rafael.vanoni@sun.com>
parents: 10696
diff changeset
   362
	ASSERT(src->bs_fanout == dest->bs_fanout);
10696
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   363
	bcopy(src->bs_set, dest->bs_set, sizeof (ulong_t) * src->bs_words);
cd0f390dd9e2 PSARC 2008/672 thebe SAS/SATA driver
dh142964 <David.Hollister@Sun.COM>
parents: 8408
diff changeset
   364
}