# HG changeset patch # User Danek Duvall # Date 1414014312 25200 # Node ID ef55c867b23fd0fe91dbc27a4fb4dcfb0691178d # Parent 15f86a2c5b18cace1b00459f329513fbcc186225 19878891 problem in SERVICE/SWIFT diff -r 15f86a2c5b18 -r ef55c867b23f components/openstack/swift/Makefile --- a/components/openstack/swift/Makefile Wed Nov 19 02:05:10 2014 -0800 +++ b/components/openstack/swift/Makefile Wed Oct 22 14:45:12 2014 -0700 @@ -49,6 +49,7 @@ COMPONENT_TEST_DIR = $(SOURCE_DIR) COMPONENT_TEST_CMD = nosetests +COMPONENT_TEST_ENV += SWIFT_TEST_CONFIG_FILE=$(SWIFT_TEST_CONFIG_FILE) COMPONENT_TEST_ARGS = --with-xunit --xunit-file=$(BUILD_DIR)/nosetests-$(MACH).xml COMPONENT_TEST_ARGS += test/unit diff -r 15f86a2c5b18 -r ef55c867b23f components/openstack/swift/patches/CVE-2014-7960.patch --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/components/openstack/swift/patches/CVE-2014-7960.patch Wed Oct 22 14:45:12 2014 -0700 @@ -0,0 +1,971 @@ +This is a backport to Havana of the Icehouse backport of the fix to bug 1365350, +plus a fix to the testsuite to actually test the problem (bug 1381159). Because +Havana is out of support, it is not a candidate for upstream submission, +though a re-work of the testsuite changes to trunk Swift is. + +================================================================================ + +From 2c4622a28ea04e1c6b2382189b0a1f6cccdc9c0f Mon Sep 17 00:00:00 2001 +From: "Richard (Rick) Hawkins" +Date: Wed, 1 Oct 2014 09:37:47 -0400 +Subject: [PATCH] Fix metadata overall limits bug + +Currently metadata limits are checked on a per request basis. If +multiple requests are sent within the per request limits, it is +possible to exceed the overall limits. This patch adds an overall +metadata check to ensure that multiple requests to add metadata to +an account/container will check overall limits before adding +the additional metadata. + +This is a backport to the stable/icehouse branch for commit SHA +5b2c27a5874c2b5b0a333e4955b03544f6a8119f. + +Closes-Bug: 1365350 + +Conflicts: + swift/common/db.py + swift/container/server.py + +Change-Id: Id9fca209c9c1216f1949de7108bbe332808f1045 +--- + swift/account/server.py | 4 +- + swift/common/constraints.py | 5 ++- + swift/common/db.py | 34 +++++++++++++- + swift/container/server.py | 4 +- + test/functional/test_account.py | 66 +++++++++++++++++++++++++++ + test/functional/test_container.py | 20 +++++++++ + test/unit/common/test_db.py | 90 ++++++++++++++++++++++++++++++++++++- + 7 files changed, 216 insertions(+), 7 deletions(-) + +--- a/swift/account/server.py ++++ b/swift/account/server.py +@@ -156,7 +156,7 @@ class AccountController(object): + for key, value in req.headers.iteritems() + if key.lower().startswith('x-account-meta-')) + if metadata: +- broker.update_metadata(metadata) ++ broker.update_metadata(metadata, validate_metadata=True) + if created: + return HTTPCreated(request=req) + else: +@@ -262,7 +262,7 @@ class AccountController(object): + for key, value in req.headers.iteritems() + if key.lower().startswith('x-account-meta-')) + if metadata: +- broker.update_metadata(metadata) ++ broker.update_metadata(metadata, validate_metadata=True) + return HTTPNoContent(request=req) + + def __call__(self, env, start_response): +--- a/swift/common/constraints.py ++++ b/swift/common/constraints.py +@@ -68,7 +68,10 @@ FORMAT2CONTENT_TYPE = {'plain': 'text/pl + + def check_metadata(req, target_type): + """ +- Check metadata sent in the request headers. ++ Check metadata sent in the request headers. This should only check ++ that the metadata in the request given is valid. Checks against ++ account/container overall metadata should be forwarded on to its ++ respective server to be checked. + + :param req: request object + :param target_type: str: one of: object, container, or account: indicates +--- a/swift/common/db.py ++++ b/swift/common/db.py +@@ -32,7 +32,9 @@ import sqlite3 + + from swift.common.utils import json, normalize_timestamp, renamer, \ + mkdirs, lock_parent_directory, fallocate ++from swift.common.constraints import MAX_META_COUNT, MAX_META_OVERALL_SIZE + from swift.common.exceptions import LockTimeout ++from swift.common.swob import HTTPBadRequest + + + #: Whether calls will be made to preallocate disk space for database files. +@@ -615,7 +617,35 @@ class DatabaseBroker(object): + metadata = {} + return metadata + +- def update_metadata(self, metadata_updates): ++ @staticmethod ++ def validate_metadata(metadata): ++ """ ++ Validates that metadata_falls within acceptable limits. ++ ++ :param metadata: to be validated ++ :raises: HTTPBadRequest if MAX_META_COUNT or MAX_META_OVERALL_SIZE ++ is exceeded ++ """ ++ meta_count = 0 ++ meta_size = 0 ++ for key, (value, timestamp) in metadata.iteritems(): ++ key = key.lower() ++ if value != '' and (key.startswith('x-account-meta') or ++ key.startswith('x-container-meta')): ++ prefix = 'x-account-meta-' ++ if key.startswith('x-container-meta-'): ++ prefix = 'x-container-meta-' ++ key = key[len(prefix):] ++ meta_count = meta_count + 1 ++ meta_size = meta_size + len(key) + len(value) ++ if meta_count > MAX_META_COUNT: ++ raise HTTPBadRequest('Too many metadata items; max %d' ++ % MAX_META_COUNT) ++ if meta_size > MAX_META_OVERALL_SIZE: ++ raise HTTPBadRequest('Total metadata too large; max %d' ++ % MAX_META_OVERALL_SIZE) ++ ++ def update_metadata(self, metadata_updates, validate_metadata=False): + """ + Updates the metadata dict for the database. The metadata dict values + are tuples of (value, timestamp) where the timestamp indicates when +@@ -648,6 +678,8 @@ class DatabaseBroker(object): + value, timestamp = value_timestamp + if key not in md or timestamp > md[key][1]: + md[key] = value_timestamp ++ if validate_metadata: ++ DatabaseBroker.validate_metadata(md) + conn.execute('UPDATE %s_stat SET metadata = ?' % self.db_type, + (json.dumps(md),)) + conn.commit() +--- a/swift/container/server.py ++++ b/swift/container/server.py +@@ -275,7 +275,7 @@ class ContainerController(object): + metadata['X-Container-Sync-To'][0] != \ + broker.metadata['X-Container-Sync-To'][0]: + broker.set_x_container_sync_points(-1, -1) +- broker.update_metadata(metadata) ++ broker.update_metadata(metadata, validate_metadata=True) + resp = self.account_update(req, account, container, broker) + if resp: + return resp +@@ -461,7 +461,7 @@ class ContainerController(object): + metadata['X-Container-Sync-To'][0] != \ + broker.metadata['X-Container-Sync-To'][0]: + broker.set_x_container_sync_points(-1, -1) +- broker.update_metadata(metadata) ++ broker.update_metadata(metadata, validate_metadata=True) + return HTTPNoContent(request=req) + + def __call__(self, env, start_response): +--- a/test/functional/swift_testing.py ++++ b/test/functional/swift_testing.py +@@ -0,0 +1,231 @@ ++# Copyright (c) 2010-2012 OpenStack Foundation ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); ++# you may not use this file except in compliance with the License. ++# You may obtain a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, ++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or ++# implied. ++# See the License for the specific language governing permissions and ++# limitations under the License. ++ ++from httplib import HTTPException ++import os ++import socket ++import sys ++from time import sleep ++from urlparse import urlparse ++import functools ++from nose import SkipTest ++ ++from test import get_config ++ ++from swiftclient import get_auth, http_connection ++from test.functional.swift_test_client import Connection ++ ++conf = get_config('func_test') ++web_front_end = conf.get('web_front_end', 'integral') ++normalized_urls = conf.get('normalized_urls', False) ++ ++# If no conf was read, we will fall back to old school env vars ++swift_test_auth = os.environ.get('SWIFT_TEST_AUTH') ++swift_test_user = [os.environ.get('SWIFT_TEST_USER'), None, None] ++swift_test_key = [os.environ.get('SWIFT_TEST_KEY'), None, None] ++swift_test_tenant = ['', '', ''] ++swift_test_perm = ['', '', ''] ++ ++if conf: ++ swift_test_auth_version = str(conf.get('auth_version', '1')) ++ ++ swift_test_auth = 'http' ++ if conf.get('auth_ssl', 'no').lower() in ('yes', 'true', 'on', '1'): ++ swift_test_auth = 'https' ++ if 'auth_prefix' not in conf: ++ conf['auth_prefix'] = '/' ++ try: ++ suffix = '://%(auth_host)s:%(auth_port)s%(auth_prefix)s' % conf ++ swift_test_auth += suffix ++ except KeyError: ++ pass # skip ++ ++ if swift_test_auth_version == "1": ++ swift_test_auth += 'v1.0' ++ ++ if 'account' in conf: ++ swift_test_user[0] = '%(account)s:%(username)s' % conf ++ else: ++ swift_test_user[0] = '%(username)s' % conf ++ swift_test_key[0] = conf['password'] ++ try: ++ swift_test_user[1] = '%s%s' % ( ++ '%s:' % conf['account2'] if 'account2' in conf else '', ++ conf['username2']) ++ swift_test_key[1] = conf['password2'] ++ except KeyError as err: ++ pass # old conf, no second account tests can be run ++ try: ++ swift_test_user[2] = '%s%s' % ('%s:' % conf['account'] if 'account' ++ in conf else '', conf['username3']) ++ swift_test_key[2] = conf['password3'] ++ except KeyError as err: ++ pass # old conf, no third account tests can be run ++ ++ for _ in range(3): ++ swift_test_perm[_] = swift_test_user[_] ++ ++ else: ++ swift_test_user[0] = conf['username'] ++ swift_test_tenant[0] = conf['account'] ++ swift_test_key[0] = conf['password'] ++ swift_test_user[1] = conf['username2'] ++ swift_test_tenant[1] = conf['account2'] ++ swift_test_key[1] = conf['password2'] ++ swift_test_user[2] = conf['username3'] ++ swift_test_tenant[2] = conf['account'] ++ swift_test_key[2] = conf['password3'] ++ ++ for _ in range(3): ++ swift_test_perm[_] = swift_test_tenant[_] + ':' \ ++ + swift_test_user[_] ++ ++skip = not all([swift_test_auth, swift_test_user[0], swift_test_key[0]]) ++if skip: ++ print >>sys.stderr, 'SKIPPING FUNCTIONAL TESTS DUE TO NO CONFIG' ++ ++skip2 = not all([not skip, swift_test_user[1], swift_test_key[1]]) ++if not skip and skip2: ++ print >>sys.stderr, \ ++ 'SKIPPING SECOND ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM' ++ ++skip3 = not all([not skip, swift_test_user[2], swift_test_key[2]]) ++if not skip and skip3: ++ print >>sys.stderr, \ ++ 'SKIPPING THIRD ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM' ++ ++ ++class AuthError(Exception): ++ pass ++ ++ ++class InternalServerError(Exception): ++ pass ++ ++ ++url = [None, None, None] ++token = [None, None, None] ++parsed = [None, None, None] ++conn = [None, None, None] ++ ++ ++def retry(func, *args, **kwargs): ++ """ ++ You can use the kwargs to override: ++ 'retries' (default: 5) ++ 'use_account' (default: 1) - which user's token to pass ++ 'url_account' (default: matches 'use_account') - which user's storage URL ++ 'resource' (default: url[url_account] - URL to connect to; retry() ++ will interpolate the variable :storage_url: if present ++ """ ++ global url, token, parsed, conn ++ retries = kwargs.get('retries', 5) ++ attempts, backoff = 0, 1 ++ ++ # use account #1 by default; turn user's 1-indexed account into 0-indexed ++ use_account = kwargs.pop('use_account', 1) - 1 ++ ++ # access our own account by default ++ url_account = kwargs.pop('url_account', use_account + 1) - 1 ++ ++ while attempts <= retries: ++ attempts += 1 ++ try: ++ if not url[use_account] or not token[use_account]: ++ url[use_account], token[use_account] = \ ++ get_auth(swift_test_auth, swift_test_user[use_account], ++ swift_test_key[use_account], ++ snet=False, ++ tenant_name=swift_test_tenant[use_account], ++ auth_version=swift_test_auth_version, ++ os_options={}) ++ parsed[use_account] = conn[use_account] = None ++ if not parsed[use_account] or not conn[use_account]: ++ parsed[use_account], conn[use_account] = \ ++ http_connection(url[use_account]) ++ ++ # default resource is the account url[url_account] ++ resource = kwargs.pop('resource', '%(storage_url)s') ++ template_vars = {'storage_url': url[url_account]} ++ parsed_result = urlparse(resource % template_vars) ++ return func(url[url_account], token[use_account], ++ parsed_result, conn[url_account], ++ *args, **kwargs) ++ except (socket.error, HTTPException): ++ if attempts > retries: ++ raise ++ parsed[use_account] = conn[use_account] = None ++ except AuthError: ++ url[use_account] = token[use_account] = None ++ continue ++ except InternalServerError: ++ pass ++ if attempts <= retries: ++ sleep(backoff) ++ backoff *= 2 ++ raise Exception('No result after %s retries.' % retries) ++ ++ ++def check_response(conn): ++ resp = conn.getresponse() ++ if resp.status == 401: ++ resp.read() ++ raise AuthError() ++ elif resp.status // 100 == 5: ++ resp.read() ++ raise InternalServerError() ++ return resp ++ ++cluster_info = {} ++ ++ ++def get_cluster_info(): ++ conn = Connection(conf) ++ conn.authenticate() ++ global cluster_info ++ cluster_info = conn.cluster_info() ++ ++ ++def reset_acl(): ++ def post(url, token, parsed, conn): ++ conn.request('POST', parsed.path, '', { ++ 'X-Auth-Token': token, ++ 'X-Account-Access-Control': '{}' ++ }) ++ return check_response(conn) ++ resp = retry(post, use_account=1) ++ resp.read() ++ ++ ++def requires_acls(f): ++ @functools.wraps(f) ++ def wrapper(*args, **kwargs): ++ if skip: ++ raise SkipTest ++ if not cluster_info: ++ get_cluster_info() ++ # Determine whether this cluster has account ACLs; if not, skip test ++ if not cluster_info.get('tempauth', {}).get('account_acls'): ++ raise SkipTest ++ if 'keystoneauth' in cluster_info: ++ # remove when keystoneauth supports account acls ++ raise SkipTest ++ reset_acl() ++ try: ++ rv = f(*args, **kwargs) ++ finally: ++ reset_acl() ++ return rv ++ return wrapper +--- a/test/functional/tests.py ++++ b/test/functional/tests.py +@@ -1624,5 +1624,466 @@ class TestFileComparison(Base): + class TestFileComparisonUTF8(Base2, TestFileComparison): + set_up = False + ++import json ++from uuid import uuid4 ++from swift_testing import (check_response, retry, skip, ++ web_front_end, requires_acls) ++import swift_testing ++ ++class TestCVE20147960Container(unittest.TestCase): ++ ++ def setUp(self): ++ if skip: ++ raise SkipTest ++ self.name = uuid4().hex ++ # this container isn't created by default, but will be cleaned up ++ self.container = uuid4().hex ++ ++ def put(url, token, parsed, conn): ++ conn.request('PUT', parsed.path + '/' + self.name, '', ++ {'X-Auth-Token': token}) ++ return check_response(conn) ++ ++ resp = retry(put) ++ resp.read() ++ self.assertEqual(resp.status, 201) ++ ++ self.max_meta_count = load_constraint('max_meta_count') ++ self.max_meta_name_length = load_constraint('max_meta_name_length') ++ self.max_meta_overall_size = load_constraint('max_meta_overall_size') ++ self.max_meta_value_length = load_constraint('max_meta_value_length') ++ ++ def tearDown(self): ++ if skip: ++ raise SkipTest ++ ++ def get(url, token, parsed, conn, container): ++ conn.request( ++ 'GET', parsed.path + '/' + container + '?format=json', '', ++ {'X-Auth-Token': token}) ++ return check_response(conn) ++ ++ def delete(url, token, parsed, conn, container, obj): ++ conn.request( ++ 'DELETE', '/'.join([parsed.path, container, obj['name']]), '', ++ {'X-Auth-Token': token}) ++ return check_response(conn) ++ ++ for container in (self.name, self.container): ++ while True: ++ resp = retry(get, container) ++ body = resp.read() ++ if resp.status == 404: ++ break ++ self.assert_(resp.status // 100 == 2, resp.status) ++ objs = json.loads(body) ++ if not objs: ++ break ++ for obj in objs: ++ resp = retry(delete, container, obj) ++ resp.read() ++ self.assertEqual(resp.status, 204) ++ ++ def delete(url, token, parsed, conn, container): ++ conn.request('DELETE', parsed.path + '/' + container, '', ++ {'X-Auth-Token': token}) ++ return check_response(conn) ++ ++ resp = retry(delete, self.name) ++ resp.read() ++ self.assertEqual(resp.status, 204) ++ ++ # container may have not been created ++ resp = retry(delete, self.container) ++ resp.read() ++ self.assert_(resp.status in (204, 404)) ++ ++ def test_POST_bad_metadata(self): ++ if skip: ++ raise SkipTest ++ ++ def post(url, token, parsed, conn, extra_headers): ++ headers = {'X-Auth-Token': token} ++ headers.update(extra_headers) ++ conn.request('POST', parsed.path + '/' + self.name, '', headers) ++ return check_response(conn) ++ ++ resp = retry( ++ post, ++ {'X-Container-Meta-' + ('k' * self.max_meta_name_length): 'v'}) ++ resp.read() ++ self.assertEqual(resp.status, 204) ++ resp = retry( ++ post, ++ {'X-Container-Meta-' + ( ++ 'k' * (self.max_meta_name_length + 1)): 'v'}) ++ resp.read() ++ self.assertEqual(resp.status, 400) ++ ++ resp = retry( ++ post, ++ {'X-Container-Meta-Too-Long': 'k' * self.max_meta_value_length}) ++ resp.read() ++ self.assertEqual(resp.status, 204) ++ resp = retry( ++ post, ++ {'X-Container-Meta-Too-Long': 'k' * ( ++ self.max_meta_value_length + 1)}) ++ resp.read() ++ self.assertEqual(resp.status, 400) ++ ++ def test_POST_maxcount_metadata(self): ++ if skip: ++ raise SkipTest ++ ++ def post(url, token, parsed, conn, extra_headers): ++ headers = {'X-Auth-Token': token} ++ headers.update(extra_headers) ++ conn.request('POST', parsed.path + '/' + self.name, '', headers) ++ return check_response(conn) ++ ++ headers = {} ++ for x in xrange(self.max_meta_count): ++ headers['X-Container-Meta-%d' % x] = 'v' ++ resp = retry(post, headers) ++ resp.read() ++ self.assertEqual(resp.status, 204) ++ ++ def test_POST_maxcount_metadata_over_one(self): ++ if skip: ++ raise SkipTest ++ ++ def post(url, token, parsed, conn, extra_headers): ++ headers = {'X-Auth-Token': token} ++ headers.update(extra_headers) ++ conn.request('POST', parsed.path + '/' + self.name, '', headers) ++ return check_response(conn) ++ ++ headers = {} ++ for x in xrange(self.max_meta_count + 1): ++ headers['X-Container-Meta-%d' % x] = 'v' ++ resp = retry(post, headers) ++ resp.read() ++ self.assertEqual(resp.status, 400) ++ ++ def test_POST_maxcount_metadata_over_two(self): ++ if skip: ++ raise SkipTest ++ ++ def post(url, token, parsed, conn, extra_headers): ++ headers = {'X-Auth-Token': token} ++ headers.update(extra_headers) ++ conn.request('POST', parsed.path + '/' + self.name, '', headers) ++ return check_response(conn) ++ ++ headers = {} ++ for x in xrange(self.max_meta_count - 1): ++ headers['X-Container-Meta-%d' % x] = 'v' ++ resp = retry(post, headers) ++ resp.read() ++ self.assertEqual(resp.status, 204) ++ ++ headers = {} ++ for x in xrange(self.max_meta_count - 1, self.max_meta_count + 1): ++ headers['X-Container-Meta-%d' % x] = 'v' ++ resp = retry(post, headers) ++ resp.read() ++ self.assertEqual(resp.status, 400) ++ ++ def test_POST_maxsize_metadata(self): ++ if skip: ++ raise SkipTest ++ ++ def post(url, token, parsed, conn, extra_headers): ++ headers = {'X-Auth-Token': token} ++ headers.update(extra_headers) ++ conn.request('POST', parsed.path + '/' + self.name, '', headers) ++ return check_response(conn) ++ ++ headers = {} ++ header_value = 'k' * self.max_meta_value_length ++ size = 0 ++ x = 0 ++ while size < (self.max_meta_overall_size - 4 ++ - self.max_meta_value_length): ++ size += 4 + self.max_meta_value_length ++ headers['X-Container-Meta-%04d' % x] = header_value ++ x += 1 ++ if self.max_meta_overall_size - size > 1: ++ headers['X-Container-Meta-k'] = \ ++ 'v' * (self.max_meta_overall_size - size - 1) ++ resp = retry(post, headers) ++ resp.read() ++ self.assertEqual(resp.status, 204) ++ ++ def test_POST_maxsize_metadata_over_one(self): ++ if skip: ++ raise SkipTest ++ ++ def post(url, token, parsed, conn, extra_headers): ++ headers = {'X-Auth-Token': token} ++ headers.update(extra_headers) ++ conn.request('POST', parsed.path + '/' + self.name, '', headers) ++ return check_response(conn) ++ ++ headers = {} ++ header_value = 'k' * self.max_meta_value_length ++ size = 0 ++ x = 0 ++ while size < (self.max_meta_overall_size - 4 ++ - self.max_meta_value_length): ++ size += 4 + self.max_meta_value_length ++ headers['X-Container-Meta-%04d' % x] = header_value ++ x += 1 ++ if self.max_meta_overall_size >= size: ++ headers['X-Container-Meta-k'] = \ ++ 'v' * (self.max_meta_overall_size - size) ++ resp = retry(post, headers) ++ resp.read() ++ self.assertEqual(resp.status, 400) ++ ++ def test_POST_maxsize_metadata_over_two(self): ++ if skip: ++ raise SkipTest ++ ++ def post(url, token, parsed, conn, extra_headers): ++ headers = {'X-Auth-Token': token} ++ headers.update(extra_headers) ++ conn.request('POST', parsed.path + '/' + self.name, '', headers) ++ return check_response(conn) ++ ++ headers = {} ++ header_value = 'k' * self.max_meta_value_length ++ size = 0 ++ x = 0 ++ while size < (self.max_meta_overall_size - 4 ++ - self.max_meta_value_length): ++ size += 4 + self.max_meta_value_length ++ headers['X-Container-Meta-%04d' % x] = header_value ++ x += 1 ++ if self.max_meta_overall_size - size > 1: ++ headers['X-Container-Meta-k'] = \ ++ 'v' * (self.max_meta_overall_size - size - 1) ++ resp = retry(post, headers) ++ resp.read() ++ self.assertEqual(resp.status, 204) ++ headers = {} ++ headers['X-Container-Meta-k2'] = 'v' ++ resp = retry(post, headers) ++ resp.read() ++ self.assertEqual(resp.status, 400) ++ ++ ++class TestCVE20147960Account(unittest.TestCase): ++ ++ def setUp(self): ++ self.max_meta_count = load_constraint('max_meta_count') ++ self.max_meta_name_length = load_constraint('max_meta_name_length') ++ self.max_meta_overall_size = load_constraint('max_meta_overall_size') ++ self.max_meta_value_length = load_constraint('max_meta_value_length') ++ ++ def head(url, token, parsed, conn): ++ conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) ++ return check_response(conn) ++ resp = retry(head) ++ self.existing_metadata = set([ ++ k for k, v in resp.getheaders() if ++ k.lower().startswith('x-account-meta')]) ++ ++ def tearDown(self): ++ def head(url, token, parsed, conn): ++ conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) ++ return check_response(conn) ++ resp = retry(head) ++ resp.read() ++ new_metadata = set( ++ [k for k, v in resp.getheaders() if ++ k.lower().startswith('x-account-meta')]) ++ ++ def clear_meta(url, token, parsed, conn, remove_metadata_keys): ++ headers = {'X-Auth-Token': token} ++ headers.update((k, '') for k in remove_metadata_keys) ++ conn.request('POST', parsed.path, '', headers) ++ return check_response(conn) ++ extra_metadata = list(self.existing_metadata ^ new_metadata) ++ for i in range(0, len(extra_metadata), 90): ++ batch = extra_metadata[i:i + 90] ++ resp = retry(clear_meta, batch) ++ resp.read() ++ self.assertEqual(resp.status // 100, 2) ++ ++ def test_maxcount_metadata(self): ++ if skip: ++ raise SkipTest ++ ++ def post(url, token, parsed, conn, extra_headers): ++ headers = {'X-Auth-Token': token} ++ headers.update(extra_headers) ++ conn.request('POST', parsed.path, '', headers) ++ return check_response(conn) ++ ++ # TODO: Find the test that adds these and remove them. ++ headers = {'x-remove-account-meta-temp-url-key': 'remove', ++ 'x-remove-account-meta-temp-url-key-2': 'remove'} ++ resp = retry(post, headers) ++ ++ headers = {} ++ for x in xrange(MAX_META_COUNT): ++ headers['X-Account-Meta-%d' % x] = 'v' ++ resp = retry(post, headers) ++ resp.read() ++ self.assertEqual(resp.status, 204) ++ ++ def test_maxcount_metadata_over_one(self): ++ if skip: ++ raise SkipTest ++ ++ def post(url, token, parsed, conn, extra_headers): ++ headers = {'X-Auth-Token': token} ++ headers.update(extra_headers) ++ conn.request('POST', parsed.path, '', headers) ++ return check_response(conn) ++ ++ # TODO: Find the test that adds these and remove them. ++ headers = {'x-remove-account-meta-temp-url-key': 'remove', ++ 'x-remove-account-meta-temp-url-key-2': 'remove'} ++ resp = retry(post, headers) ++ ++ headers = {} ++ for x in xrange(MAX_META_COUNT + 1): ++ headers['X-Account-Meta-%d' % x] = 'v' ++ resp = retry(post, headers) ++ resp.read() ++ self.assertEqual(resp.status, 400) ++ ++ def test_maxcount_metadata_over_two(self): ++ if skip: ++ raise SkipTest ++ ++ def post(url, token, parsed, conn, extra_headers): ++ headers = {'X-Auth-Token': token} ++ headers.update(extra_headers) ++ conn.request('POST', parsed.path, '', headers) ++ return check_response(conn) ++ ++ # TODO: Find the test that adds these and remove them. ++ headers = {'x-remove-account-meta-temp-url-key': 'remove', ++ 'x-remove-account-meta-temp-url-key-2': 'remove'} ++ resp = retry(post, headers) ++ ++ headers = {} ++ for x in xrange(MAX_META_COUNT - 1): ++ headers['X-Account-Meta-%d' % x] = 'v' ++ resp = retry(post, headers) ++ resp.read() ++ self.assertEqual(resp.status, 204) ++ ++ headers = {} ++ for x in xrange(MAX_META_COUNT - 1, MAX_META_COUNT + 1): ++ headers['X-Account-Meta-%d' % x] = 'v' ++ resp = retry(post, headers) ++ resp.read() ++ self.assertEqual(resp.status, 400) ++ ++ def test_maxsize_metadata(self): ++ if skip: ++ raise SkipTest ++ ++ def post(url, token, parsed, conn, extra_headers): ++ headers = {'X-Auth-Token': token} ++ headers.update(extra_headers) ++ conn.request('POST', parsed.path, '', headers) ++ return check_response(conn) ++ ++ # TODO: Find the test that adds these and remove them. ++ headers = {'x-remove-account-meta-temp-url-key': 'remove', ++ 'x-remove-account-meta-temp-url-key-2': 'remove'} ++ resp = retry(post, headers) ++ ++ headers = {} ++ header_value = 'k' * MAX_META_VALUE_LENGTH ++ size = 0 ++ x = 0 ++ while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH: ++ size += 4 + MAX_META_VALUE_LENGTH ++ headers['X-Account-Meta-%04d' % x] = header_value ++ x += 1 ++ if MAX_META_OVERALL_SIZE - size > 1: ++ headers['X-Account-Meta-k'] = \ ++ 'v' * (MAX_META_OVERALL_SIZE - size - 1) ++ resp = retry(post, headers) ++ resp.read() ++ self.assertEqual(resp.status, 204) ++ headers['X-Account-Meta-k'] = \ ++ 'v' * (MAX_META_OVERALL_SIZE - size) ++ resp = retry(post, headers) ++ resp.read() ++ self.assertEqual(resp.status, 400) ++ ++ def test_maxsize_metadata_over_one(self): ++ if skip: ++ raise SkipTest ++ ++ def post(url, token, parsed, conn, extra_headers): ++ headers = {'X-Auth-Token': token} ++ headers.update(extra_headers) ++ conn.request('POST', parsed.path, '', headers) ++ return check_response(conn) ++ ++ # TODO: Find the test that adds these and remove them. ++ headers = {'x-remove-account-meta-temp-url-key': 'remove', ++ 'x-remove-account-meta-temp-url-key-2': 'remove'} ++ resp = retry(post, headers) ++ ++ headers = {} ++ header_value = 'k' * MAX_META_VALUE_LENGTH ++ size = 0 ++ x = 0 ++ while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH: ++ size += 4 + MAX_META_VALUE_LENGTH ++ headers['X-Account-Meta-%04d' % x] = header_value ++ x += 1 ++ if MAX_META_OVERALL_SIZE >= size: ++ headers['X-Account-Meta-k'] = \ ++ 'v' * (MAX_META_OVERALL_SIZE - size) ++ resp = retry(post, headers) ++ resp.read() ++ self.assertEqual(resp.status, 400) ++ ++ def test_maxsize_metadata_over_two(self): ++ if skip: ++ raise SkipTest ++ ++ def post(url, token, parsed, conn, extra_headers): ++ headers = {'X-Auth-Token': token} ++ headers.update(extra_headers) ++ conn.request('POST', parsed.path, '', headers) ++ return check_response(conn) ++ ++ # TODO: Find the test that adds these and remove them. ++ headers = {'x-remove-account-meta-temp-url-key': 'remove', ++ 'x-remove-account-meta-temp-url-key-2': 'remove'} ++ resp = retry(post, headers) ++ ++ headers = {} ++ header_value = 'k' * MAX_META_VALUE_LENGTH ++ size = 0 ++ x = 0 ++ while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH: ++ size += 4 + MAX_META_VALUE_LENGTH ++ headers['X-Account-Meta-%04d' % x] = header_value ++ x += 1 ++ if MAX_META_OVERALL_SIZE - size > 1: ++ headers['X-Account-Meta-k'] = \ ++ 'v' * (MAX_META_OVERALL_SIZE - size - 1) ++ resp = retry(post, headers) ++ resp.read() ++ self.assertEqual(resp.status, 204) ++ headers = {} ++ headers['X-Account-Meta-k2'] = 'v' ++ resp = retry(post, headers) ++ resp.read() ++ self.assertEqual(resp.status, 400) ++ + if __name__ == '__main__': + unittest.main() +--- a/test/unit/common/test_db.py ++++ b/test/unit/common/test_db.py +@@ -26,10 +26,13 @@ import sqlite3 + from mock import patch + + import swift.common.db ++from swift.common.constraints import \ ++ MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE + from swift.common.db import chexor, dict_factory, get_db_connection, \ + DatabaseBroker, DatabaseConnectionError, DatabaseAlreadyExists + from swift.common.utils import normalize_timestamp + from swift.common.exceptions import LockTimeout ++from swift.common.swob import HTTPException + + + class TestDatabaseConnectionError(unittest.TestCase): +@@ -181,7 +184,7 @@ class TestDatabaseBroker(unittest.TestCa + conn.execute('CREATE TABLE test (one TEXT)') + conn.execute('CREATE TABLE test_stat (id TEXT)') + conn.execute('INSERT INTO test_stat (id) VALUES (?)', +- (str(uuid4),)) ++ (str(uuid4),)) + conn.execute('INSERT INTO test (one) VALUES ("1")') + conn.commit() + stub_called = [False] +@@ -632,6 +635,91 @@ class TestDatabaseBroker(unittest.TestCa + [first_value, first_timestamp]) + self.assert_('Second' not in broker.metadata) + ++ @patch.object(DatabaseBroker, 'validate_metadata') ++ def test_validate_metadata_is_called_from_update_metadata(self, mock): ++ broker = self.get_replication_info_tester(metadata=True) ++ first_timestamp = normalize_timestamp(1) ++ first_value = '1' ++ metadata = {'First': [first_value, first_timestamp]} ++ broker.update_metadata(metadata, validate_metadata=True) ++ self.assertTrue(mock.called) ++ ++ @patch.object(DatabaseBroker, 'validate_metadata') ++ def test_validate_metadata_is_not_called_from_update_metadata(self, mock): ++ broker = self.get_replication_info_tester(metadata=True) ++ first_timestamp = normalize_timestamp(1) ++ first_value = '1' ++ metadata = {'First': [first_value, first_timestamp]} ++ broker.update_metadata(metadata) ++ self.assertFalse(mock.called) ++ ++ def test_metadata_with_max_count(self): ++ metadata = {} ++ for c in xrange(MAX_META_COUNT): ++ key = 'X-Account-Meta-F{0}'.format(c) ++ metadata[key] = ('B', normalize_timestamp(1)) ++ key = 'X-Account-Meta-Foo'.format(c) ++ metadata[key] = ('', normalize_timestamp(1)) ++ try: ++ DatabaseBroker.validate_metadata(metadata) ++ except HTTPException: ++ self.fail('Unexpected HTTPException') ++ ++ def test_metadata_raises_exception_over_max_count(self): ++ metadata = {} ++ for c in xrange(MAX_META_COUNT + 1): ++ key = 'X-Account-Meta-F{0}'.format(c) ++ metadata[key] = ('B', normalize_timestamp(1)) ++ message = '' ++ try: ++ DatabaseBroker.validate_metadata(metadata) ++ except HTTPException as e: ++ message = str(e) ++ self.assertEqual(message, '400 Bad Request') ++ ++ def test_metadata_with_max_overall_size(self): ++ metadata = {} ++ metadata_value = 'v' * MAX_META_VALUE_LENGTH ++ size = 0 ++ x = 0 ++ while size < (MAX_META_OVERALL_SIZE - 4 ++ - MAX_META_VALUE_LENGTH): ++ size += 4 + MAX_META_VALUE_LENGTH ++ metadata['X-Account-Meta-%04d' % x] = (metadata_value, ++ normalize_timestamp(1)) ++ x += 1 ++ if MAX_META_OVERALL_SIZE - size > 1: ++ metadata['X-Account-Meta-k'] = ( ++ 'v' * (MAX_META_OVERALL_SIZE - size - 1), ++ normalize_timestamp(1)) ++ try: ++ DatabaseBroker.validate_metadata(metadata) ++ except HTTPException: ++ self.fail('Unexpected HTTPException') ++ ++ def test_metadata_raises_exception_over_max_overall_size(self): ++ metadata = {} ++ metadata_value = 'k' * MAX_META_VALUE_LENGTH ++ size = 0 ++ x = 0 ++ while size < (MAX_META_OVERALL_SIZE - 4 ++ - MAX_META_VALUE_LENGTH): ++ size += 4 + MAX_META_VALUE_LENGTH ++ metadata['X-Account-Meta-%04d' % x] = (metadata_value, ++ normalize_timestamp(1)) ++ x += 1 ++ if MAX_META_OVERALL_SIZE - size > 1: ++ metadata['X-Account-Meta-k'] = ( ++ 'v' * (MAX_META_OVERALL_SIZE - size - 1), ++ normalize_timestamp(1)) ++ metadata['X-Account-Meta-k2'] = ('v', normalize_timestamp(1)) ++ message = '' ++ try: ++ DatabaseBroker.validate_metadata(metadata) ++ except HTTPException as e: ++ message = str(e) ++ self.assertEqual(message, '400 Bad Request') ++ + + if __name__ == '__main__': + unittest.main() diff -r 15f86a2c5b18 -r ef55c867b23f components/openstack/swift/patches/test.patch --- a/components/openstack/swift/patches/test.patch Wed Nov 19 02:05:10 2014 -0800 +++ b/components/openstack/swift/patches/test.patch Wed Oct 22 14:45:12 2014 -0700 @@ -5,10 +5,21 @@ - Solaris doesn't yet support syslog logging to /dev/log. + - Solaris exclusive file locks don't fail when applied multiple times + from a single process (it has to happen in a separate process). + + - Uncomment portions of test/sample.conf to match what we ship in + /etc/swift/swift.conf, since the latter can't be read without + sufficient privileges. This allows us to set SWIFT_TEST_CONFIG_FILE + and run the functional tests. + The first, while potentially useful elsewhere, is really only an issue on Solaris because Linux runs almost exclusively 64-bit, which makes this a -non-issue. The last is Solaris-only -- though clearly a similar problem -exists on MacOS -- and we will want to fix this in our Python. +non-issue. The second is Solaris-only -- though clearly a similar problem +exists on MacOS -- and we will want to fix this in our Python. The third +is not in a form that is worth sending upstream. To test this properly, +the test should fork a separate process to test the lock, which should work +regardless of the OS. diff --git a/test/unit/__init__.py b/test/unit/__init__.py --- a/test/unit/__init__.py @@ -58,3 +69,53 @@ os.path.isfile('/dev/log') or \ os.path.isdir('/dev/log'): # Since socket on OSX is in /var/run/syslog, there will be +diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py +--- a/test/unit/common/test_utils.py ++++ b/test/unit/common/test_utils.py +@@ -46,6 +46,7 @@ + from tempfile import TemporaryFile, NamedTemporaryFile, mkdtemp + from netifaces import AF_INET6 + from mock import MagicMock, patch ++from nose import SkipTest + + from swift.common.exceptions import (Timeout, MessageTimeout, + ConnectionTimeout, LockTimeout) +@@ -1406,6 +1407,8 @@ + MagicMock(side_effect=BaseException('test3'))) + + def test_lock_file(self): ++ if sys.platform == 'sunos5': ++ raise SkipTest + flags = os.O_CREAT | os.O_RDWR + with NamedTemporaryFile(delete=False) as nt: + nt.write("test string") +diff --git a/test/sample.conf b/test/unit/sample.conf +--- a/test/sample.conf ++++ b/test/sample.conf +@@ -29,16 +29,16 @@ password3 = testing3 + # to set them from /etc/swift/swift.conf. If that file isn't found, + # the test runner will skip tests that depend on these values. + # Note that the cluster must have "sane" values for the test suite to pass. +-#max_file_size = 5368709122 +-#max_meta_name_length = 128 +-#max_meta_value_length = 256 +-#max_meta_count = 90 +-#max_meta_overall_size = 4096 +-#max_object_name_length = 1024 +-#container_listing_limit = 10000 +-#account_listing_limit = 10000 +-#max_account_name_length = 256 +-#max_container_name_length = 256 ++max_file_size = 5368709122 ++max_meta_name_length = 128 ++max_meta_value_length = 256 ++max_meta_count = 90 ++max_meta_overall_size = 4096 ++max_object_name_length = 1024 ++container_listing_limit = 10000 ++account_listing_limit = 10000 ++max_account_name_length = 256 ++max_container_name_length = 256 + + collate = C +