--- a/components/openstack/swift/patches/CVE-2014-7960.patch Fri Mar 20 03:13:26 2015 -0700
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,971 +0,0 @@
-This is a backport to Havana of the Icehouse backport of the fix to bug 1365350,
-plus a fix to the testsuite to actually test the problem (bug 1381159). Because
-Havana is out of support, it is not a candidate for upstream submission,
-though a re-work of the testsuite changes to trunk Swift is.
-
-================================================================================
-
-From 2c4622a28ea04e1c6b2382189b0a1f6cccdc9c0f Mon Sep 17 00:00:00 2001
-From: "Richard (Rick) Hawkins" <[email protected]>
-Date: Wed, 1 Oct 2014 09:37:47 -0400
-Subject: [PATCH] Fix metadata overall limits bug
-
-Currently metadata limits are checked on a per request basis. If
-multiple requests are sent within the per request limits, it is
-possible to exceed the overall limits. This patch adds an overall
-metadata check to ensure that multiple requests to add metadata to
-an account/container will check overall limits before adding
-the additional metadata.
-
-This is a backport to the stable/icehouse branch for commit SHA
-5b2c27a5874c2b5b0a333e4955b03544f6a8119f.
-
-Closes-Bug: 1365350
-
-Conflicts:
- swift/common/db.py
- swift/container/server.py
-
-Change-Id: Id9fca209c9c1216f1949de7108bbe332808f1045
----
- swift/account/server.py | 4 +-
- swift/common/constraints.py | 5 ++-
- swift/common/db.py | 34 +++++++++++++-
- swift/container/server.py | 4 +-
- test/functional/test_account.py | 66 +++++++++++++++++++++++++++
- test/functional/test_container.py | 20 +++++++++
- test/unit/common/test_db.py | 90 ++++++++++++++++++++++++++++++++++++-
- 7 files changed, 216 insertions(+), 7 deletions(-)
-
---- a/swift/account/server.py
-+++ b/swift/account/server.py
-@@ -156,7 +156,7 @@ class AccountController(object):
- for key, value in req.headers.iteritems()
- if key.lower().startswith('x-account-meta-'))
- if metadata:
-- broker.update_metadata(metadata)
-+ broker.update_metadata(metadata, validate_metadata=True)
- if created:
- return HTTPCreated(request=req)
- else:
-@@ -262,7 +262,7 @@ class AccountController(object):
- for key, value in req.headers.iteritems()
- if key.lower().startswith('x-account-meta-'))
- if metadata:
-- broker.update_metadata(metadata)
-+ broker.update_metadata(metadata, validate_metadata=True)
- return HTTPNoContent(request=req)
-
- def __call__(self, env, start_response):
---- a/swift/common/constraints.py
-+++ b/swift/common/constraints.py
-@@ -68,7 +68,10 @@ FORMAT2CONTENT_TYPE = {'plain': 'text/pl
-
- def check_metadata(req, target_type):
- """
-- Check metadata sent in the request headers.
-+ Check metadata sent in the request headers. This should only check
-+ that the metadata in the request given is valid. Checks against
-+ account/container overall metadata should be forwarded on to its
-+ respective server to be checked.
-
- :param req: request object
- :param target_type: str: one of: object, container, or account: indicates
---- a/swift/common/db.py
-+++ b/swift/common/db.py
-@@ -32,7 +32,9 @@ import sqlite3
-
- from swift.common.utils import json, normalize_timestamp, renamer, \
- mkdirs, lock_parent_directory, fallocate
-+from swift.common.constraints import MAX_META_COUNT, MAX_META_OVERALL_SIZE
- from swift.common.exceptions import LockTimeout
-+from swift.common.swob import HTTPBadRequest
-
-
- #: Whether calls will be made to preallocate disk space for database files.
-@@ -615,7 +617,35 @@ class DatabaseBroker(object):
- metadata = {}
- return metadata
-
-- def update_metadata(self, metadata_updates):
-+ @staticmethod
-+ def validate_metadata(metadata):
-+ """
-+ Validates that metadata_falls within acceptable limits.
-+
-+ :param metadata: to be validated
-+ :raises: HTTPBadRequest if MAX_META_COUNT or MAX_META_OVERALL_SIZE
-+ is exceeded
-+ """
-+ meta_count = 0
-+ meta_size = 0
-+ for key, (value, timestamp) in metadata.iteritems():
-+ key = key.lower()
-+ if value != '' and (key.startswith('x-account-meta') or
-+ key.startswith('x-container-meta')):
-+ prefix = 'x-account-meta-'
-+ if key.startswith('x-container-meta-'):
-+ prefix = 'x-container-meta-'
-+ key = key[len(prefix):]
-+ meta_count = meta_count + 1
-+ meta_size = meta_size + len(key) + len(value)
-+ if meta_count > MAX_META_COUNT:
-+ raise HTTPBadRequest('Too many metadata items; max %d'
-+ % MAX_META_COUNT)
-+ if meta_size > MAX_META_OVERALL_SIZE:
-+ raise HTTPBadRequest('Total metadata too large; max %d'
-+ % MAX_META_OVERALL_SIZE)
-+
-+ def update_metadata(self, metadata_updates, validate_metadata=False):
- """
- Updates the metadata dict for the database. The metadata dict values
- are tuples of (value, timestamp) where the timestamp indicates when
-@@ -648,6 +678,8 @@ class DatabaseBroker(object):
- value, timestamp = value_timestamp
- if key not in md or timestamp > md[key][1]:
- md[key] = value_timestamp
-+ if validate_metadata:
-+ DatabaseBroker.validate_metadata(md)
- conn.execute('UPDATE %s_stat SET metadata = ?' % self.db_type,
- (json.dumps(md),))
- conn.commit()
---- a/swift/container/server.py
-+++ b/swift/container/server.py
-@@ -275,7 +275,7 @@ class ContainerController(object):
- metadata['X-Container-Sync-To'][0] != \
- broker.metadata['X-Container-Sync-To'][0]:
- broker.set_x_container_sync_points(-1, -1)
-- broker.update_metadata(metadata)
-+ broker.update_metadata(metadata, validate_metadata=True)
- resp = self.account_update(req, account, container, broker)
- if resp:
- return resp
-@@ -461,7 +461,7 @@ class ContainerController(object):
- metadata['X-Container-Sync-To'][0] != \
- broker.metadata['X-Container-Sync-To'][0]:
- broker.set_x_container_sync_points(-1, -1)
-- broker.update_metadata(metadata)
-+ broker.update_metadata(metadata, validate_metadata=True)
- return HTTPNoContent(request=req)
-
- def __call__(self, env, start_response):
---- a/test/functional/swift_testing.py
-+++ b/test/functional/swift_testing.py
-@@ -0,0 +1,231 @@
-+# Copyright (c) 2010-2012 OpenStack Foundation
-+#
-+# Licensed under the Apache License, Version 2.0 (the "License");
-+# you may not use this file except in compliance with the License.
-+# You may obtain a copy of the License at
-+#
-+# http://www.apache.org/licenses/LICENSE-2.0
-+#
-+# Unless required by applicable law or agreed to in writing, software
-+# distributed under the License is distributed on an "AS IS" BASIS,
-+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-+# implied.
-+# See the License for the specific language governing permissions and
-+# limitations under the License.
-+
-+from httplib import HTTPException
-+import os
-+import socket
-+import sys
-+from time import sleep
-+from urlparse import urlparse
-+import functools
-+from nose import SkipTest
-+
-+from test import get_config
-+
-+from swiftclient import get_auth, http_connection
-+from test.functional.swift_test_client import Connection
-+
-+conf = get_config('func_test')
-+web_front_end = conf.get('web_front_end', 'integral')
-+normalized_urls = conf.get('normalized_urls', False)
-+
-+# If no conf was read, we will fall back to old school env vars
-+swift_test_auth = os.environ.get('SWIFT_TEST_AUTH')
-+swift_test_user = [os.environ.get('SWIFT_TEST_USER'), None, None]
-+swift_test_key = [os.environ.get('SWIFT_TEST_KEY'), None, None]
-+swift_test_tenant = ['', '', '']
-+swift_test_perm = ['', '', '']
-+
-+if conf:
-+ swift_test_auth_version = str(conf.get('auth_version', '1'))
-+
-+ swift_test_auth = 'http'
-+ if conf.get('auth_ssl', 'no').lower() in ('yes', 'true', 'on', '1'):
-+ swift_test_auth = 'https'
-+ if 'auth_prefix' not in conf:
-+ conf['auth_prefix'] = '/'
-+ try:
-+ suffix = '://%(auth_host)s:%(auth_port)s%(auth_prefix)s' % conf
-+ swift_test_auth += suffix
-+ except KeyError:
-+ pass # skip
-+
-+ if swift_test_auth_version == "1":
-+ swift_test_auth += 'v1.0'
-+
-+ if 'account' in conf:
-+ swift_test_user[0] = '%(account)s:%(username)s' % conf
-+ else:
-+ swift_test_user[0] = '%(username)s' % conf
-+ swift_test_key[0] = conf['password']
-+ try:
-+ swift_test_user[1] = '%s%s' % (
-+ '%s:' % conf['account2'] if 'account2' in conf else '',
-+ conf['username2'])
-+ swift_test_key[1] = conf['password2']
-+ except KeyError as err:
-+ pass # old conf, no second account tests can be run
-+ try:
-+ swift_test_user[2] = '%s%s' % ('%s:' % conf['account'] if 'account'
-+ in conf else '', conf['username3'])
-+ swift_test_key[2] = conf['password3']
-+ except KeyError as err:
-+ pass # old conf, no third account tests can be run
-+
-+ for _ in range(3):
-+ swift_test_perm[_] = swift_test_user[_]
-+
-+ else:
-+ swift_test_user[0] = conf['username']
-+ swift_test_tenant[0] = conf['account']
-+ swift_test_key[0] = conf['password']
-+ swift_test_user[1] = conf['username2']
-+ swift_test_tenant[1] = conf['account2']
-+ swift_test_key[1] = conf['password2']
-+ swift_test_user[2] = conf['username3']
-+ swift_test_tenant[2] = conf['account']
-+ swift_test_key[2] = conf['password3']
-+
-+ for _ in range(3):
-+ swift_test_perm[_] = swift_test_tenant[_] + ':' \
-+ + swift_test_user[_]
-+
-+skip = not all([swift_test_auth, swift_test_user[0], swift_test_key[0]])
-+if skip:
-+ print >>sys.stderr, 'SKIPPING FUNCTIONAL TESTS DUE TO NO CONFIG'
-+
-+skip2 = not all([not skip, swift_test_user[1], swift_test_key[1]])
-+if not skip and skip2:
-+ print >>sys.stderr, \
-+ 'SKIPPING SECOND ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM'
-+
-+skip3 = not all([not skip, swift_test_user[2], swift_test_key[2]])
-+if not skip and skip3:
-+ print >>sys.stderr, \
-+ 'SKIPPING THIRD ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM'
-+
-+
-+class AuthError(Exception):
-+ pass
-+
-+
-+class InternalServerError(Exception):
-+ pass
-+
-+
-+url = [None, None, None]
-+token = [None, None, None]
-+parsed = [None, None, None]
-+conn = [None, None, None]
-+
-+
-+def retry(func, *args, **kwargs):
-+ """
-+ You can use the kwargs to override:
-+ 'retries' (default: 5)
-+ 'use_account' (default: 1) - which user's token to pass
-+ 'url_account' (default: matches 'use_account') - which user's storage URL
-+ 'resource' (default: url[url_account] - URL to connect to; retry()
-+ will interpolate the variable :storage_url: if present
-+ """
-+ global url, token, parsed, conn
-+ retries = kwargs.get('retries', 5)
-+ attempts, backoff = 0, 1
-+
-+ # use account #1 by default; turn user's 1-indexed account into 0-indexed
-+ use_account = kwargs.pop('use_account', 1) - 1
-+
-+ # access our own account by default
-+ url_account = kwargs.pop('url_account', use_account + 1) - 1
-+
-+ while attempts <= retries:
-+ attempts += 1
-+ try:
-+ if not url[use_account] or not token[use_account]:
-+ url[use_account], token[use_account] = \
-+ get_auth(swift_test_auth, swift_test_user[use_account],
-+ swift_test_key[use_account],
-+ snet=False,
-+ tenant_name=swift_test_tenant[use_account],
-+ auth_version=swift_test_auth_version,
-+ os_options={})
-+ parsed[use_account] = conn[use_account] = None
-+ if not parsed[use_account] or not conn[use_account]:
-+ parsed[use_account], conn[use_account] = \
-+ http_connection(url[use_account])
-+
-+ # default resource is the account url[url_account]
-+ resource = kwargs.pop('resource', '%(storage_url)s')
-+ template_vars = {'storage_url': url[url_account]}
-+ parsed_result = urlparse(resource % template_vars)
-+ return func(url[url_account], token[use_account],
-+ parsed_result, conn[url_account],
-+ *args, **kwargs)
-+ except (socket.error, HTTPException):
-+ if attempts > retries:
-+ raise
-+ parsed[use_account] = conn[use_account] = None
-+ except AuthError:
-+ url[use_account] = token[use_account] = None
-+ continue
-+ except InternalServerError:
-+ pass
-+ if attempts <= retries:
-+ sleep(backoff)
-+ backoff *= 2
-+ raise Exception('No result after %s retries.' % retries)
-+
-+
-+def check_response(conn):
-+ resp = conn.getresponse()
-+ if resp.status == 401:
-+ resp.read()
-+ raise AuthError()
-+ elif resp.status // 100 == 5:
-+ resp.read()
-+ raise InternalServerError()
-+ return resp
-+
-+cluster_info = {}
-+
-+
-+def get_cluster_info():
-+ conn = Connection(conf)
-+ conn.authenticate()
-+ global cluster_info
-+ cluster_info = conn.cluster_info()
-+
-+
-+def reset_acl():
-+ def post(url, token, parsed, conn):
-+ conn.request('POST', parsed.path, '', {
-+ 'X-Auth-Token': token,
-+ 'X-Account-Access-Control': '{}'
-+ })
-+ return check_response(conn)
-+ resp = retry(post, use_account=1)
-+ resp.read()
-+
-+
-+def requires_acls(f):
-+ @functools.wraps(f)
-+ def wrapper(*args, **kwargs):
-+ if skip:
-+ raise SkipTest
-+ if not cluster_info:
-+ get_cluster_info()
-+ # Determine whether this cluster has account ACLs; if not, skip test
-+ if not cluster_info.get('tempauth', {}).get('account_acls'):
-+ raise SkipTest
-+ if 'keystoneauth' in cluster_info:
-+ # remove when keystoneauth supports account acls
-+ raise SkipTest
-+ reset_acl()
-+ try:
-+ rv = f(*args, **kwargs)
-+ finally:
-+ reset_acl()
-+ return rv
-+ return wrapper
---- a/test/functional/tests.py
-+++ b/test/functional/tests.py
-@@ -1624,5 +1624,466 @@ class TestFileComparison(Base):
- class TestFileComparisonUTF8(Base2, TestFileComparison):
- set_up = False
-
-+import json
-+from uuid import uuid4
-+from swift_testing import (check_response, retry, skip,
-+ web_front_end, requires_acls)
-+import swift_testing
-+
-+class TestCVE20147960Container(unittest.TestCase):
-+
-+ def setUp(self):
-+ if skip:
-+ raise SkipTest
-+ self.name = uuid4().hex
-+ # this container isn't created by default, but will be cleaned up
-+ self.container = uuid4().hex
-+
-+ def put(url, token, parsed, conn):
-+ conn.request('PUT', parsed.path + '/' + self.name, '',
-+ {'X-Auth-Token': token})
-+ return check_response(conn)
-+
-+ resp = retry(put)
-+ resp.read()
-+ self.assertEqual(resp.status, 201)
-+
-+ self.max_meta_count = load_constraint('max_meta_count')
-+ self.max_meta_name_length = load_constraint('max_meta_name_length')
-+ self.max_meta_overall_size = load_constraint('max_meta_overall_size')
-+ self.max_meta_value_length = load_constraint('max_meta_value_length')
-+
-+ def tearDown(self):
-+ if skip:
-+ raise SkipTest
-+
-+ def get(url, token, parsed, conn, container):
-+ conn.request(
-+ 'GET', parsed.path + '/' + container + '?format=json', '',
-+ {'X-Auth-Token': token})
-+ return check_response(conn)
-+
-+ def delete(url, token, parsed, conn, container, obj):
-+ conn.request(
-+ 'DELETE', '/'.join([parsed.path, container, obj['name']]), '',
-+ {'X-Auth-Token': token})
-+ return check_response(conn)
-+
-+ for container in (self.name, self.container):
-+ while True:
-+ resp = retry(get, container)
-+ body = resp.read()
-+ if resp.status == 404:
-+ break
-+ self.assert_(resp.status // 100 == 2, resp.status)
-+ objs = json.loads(body)
-+ if not objs:
-+ break
-+ for obj in objs:
-+ resp = retry(delete, container, obj)
-+ resp.read()
-+ self.assertEqual(resp.status, 204)
-+
-+ def delete(url, token, parsed, conn, container):
-+ conn.request('DELETE', parsed.path + '/' + container, '',
-+ {'X-Auth-Token': token})
-+ return check_response(conn)
-+
-+ resp = retry(delete, self.name)
-+ resp.read()
-+ self.assertEqual(resp.status, 204)
-+
-+ # container may have not been created
-+ resp = retry(delete, self.container)
-+ resp.read()
-+ self.assert_(resp.status in (204, 404))
-+
-+ def test_POST_bad_metadata(self):
-+ if skip:
-+ raise SkipTest
-+
-+ def post(url, token, parsed, conn, extra_headers):
-+ headers = {'X-Auth-Token': token}
-+ headers.update(extra_headers)
-+ conn.request('POST', parsed.path + '/' + self.name, '', headers)
-+ return check_response(conn)
-+
-+ resp = retry(
-+ post,
-+ {'X-Container-Meta-' + ('k' * self.max_meta_name_length): 'v'})
-+ resp.read()
-+ self.assertEqual(resp.status, 204)
-+ resp = retry(
-+ post,
-+ {'X-Container-Meta-' + (
-+ 'k' * (self.max_meta_name_length + 1)): 'v'})
-+ resp.read()
-+ self.assertEqual(resp.status, 400)
-+
-+ resp = retry(
-+ post,
-+ {'X-Container-Meta-Too-Long': 'k' * self.max_meta_value_length})
-+ resp.read()
-+ self.assertEqual(resp.status, 204)
-+ resp = retry(
-+ post,
-+ {'X-Container-Meta-Too-Long': 'k' * (
-+ self.max_meta_value_length + 1)})
-+ resp.read()
-+ self.assertEqual(resp.status, 400)
-+
-+ def test_POST_maxcount_metadata(self):
-+ if skip:
-+ raise SkipTest
-+
-+ def post(url, token, parsed, conn, extra_headers):
-+ headers = {'X-Auth-Token': token}
-+ headers.update(extra_headers)
-+ conn.request('POST', parsed.path + '/' + self.name, '', headers)
-+ return check_response(conn)
-+
-+ headers = {}
-+ for x in xrange(self.max_meta_count):
-+ headers['X-Container-Meta-%d' % x] = 'v'
-+ resp = retry(post, headers)
-+ resp.read()
-+ self.assertEqual(resp.status, 204)
-+
-+ def test_POST_maxcount_metadata_over_one(self):
-+ if skip:
-+ raise SkipTest
-+
-+ def post(url, token, parsed, conn, extra_headers):
-+ headers = {'X-Auth-Token': token}
-+ headers.update(extra_headers)
-+ conn.request('POST', parsed.path + '/' + self.name, '', headers)
-+ return check_response(conn)
-+
-+ headers = {}
-+ for x in xrange(self.max_meta_count + 1):
-+ headers['X-Container-Meta-%d' % x] = 'v'
-+ resp = retry(post, headers)
-+ resp.read()
-+ self.assertEqual(resp.status, 400)
-+
-+ def test_POST_maxcount_metadata_over_two(self):
-+ if skip:
-+ raise SkipTest
-+
-+ def post(url, token, parsed, conn, extra_headers):
-+ headers = {'X-Auth-Token': token}
-+ headers.update(extra_headers)
-+ conn.request('POST', parsed.path + '/' + self.name, '', headers)
-+ return check_response(conn)
-+
-+ headers = {}
-+ for x in xrange(self.max_meta_count - 1):
-+ headers['X-Container-Meta-%d' % x] = 'v'
-+ resp = retry(post, headers)
-+ resp.read()
-+ self.assertEqual(resp.status, 204)
-+
-+ headers = {}
-+ for x in xrange(self.max_meta_count - 1, self.max_meta_count + 1):
-+ headers['X-Container-Meta-%d' % x] = 'v'
-+ resp = retry(post, headers)
-+ resp.read()
-+ self.assertEqual(resp.status, 400)
-+
-+ def test_POST_maxsize_metadata(self):
-+ if skip:
-+ raise SkipTest
-+
-+ def post(url, token, parsed, conn, extra_headers):
-+ headers = {'X-Auth-Token': token}
-+ headers.update(extra_headers)
-+ conn.request('POST', parsed.path + '/' + self.name, '', headers)
-+ return check_response(conn)
-+
-+ headers = {}
-+ header_value = 'k' * self.max_meta_value_length
-+ size = 0
-+ x = 0
-+ while size < (self.max_meta_overall_size - 4
-+ - self.max_meta_value_length):
-+ size += 4 + self.max_meta_value_length
-+ headers['X-Container-Meta-%04d' % x] = header_value
-+ x += 1
-+ if self.max_meta_overall_size - size > 1:
-+ headers['X-Container-Meta-k'] = \
-+ 'v' * (self.max_meta_overall_size - size - 1)
-+ resp = retry(post, headers)
-+ resp.read()
-+ self.assertEqual(resp.status, 204)
-+
-+ def test_POST_maxsize_metadata_over_one(self):
-+ if skip:
-+ raise SkipTest
-+
-+ def post(url, token, parsed, conn, extra_headers):
-+ headers = {'X-Auth-Token': token}
-+ headers.update(extra_headers)
-+ conn.request('POST', parsed.path + '/' + self.name, '', headers)
-+ return check_response(conn)
-+
-+ headers = {}
-+ header_value = 'k' * self.max_meta_value_length
-+ size = 0
-+ x = 0
-+ while size < (self.max_meta_overall_size - 4
-+ - self.max_meta_value_length):
-+ size += 4 + self.max_meta_value_length
-+ headers['X-Container-Meta-%04d' % x] = header_value
-+ x += 1
-+ if self.max_meta_overall_size >= size:
-+ headers['X-Container-Meta-k'] = \
-+ 'v' * (self.max_meta_overall_size - size)
-+ resp = retry(post, headers)
-+ resp.read()
-+ self.assertEqual(resp.status, 400)
-+
-+ def test_POST_maxsize_metadata_over_two(self):
-+ if skip:
-+ raise SkipTest
-+
-+ def post(url, token, parsed, conn, extra_headers):
-+ headers = {'X-Auth-Token': token}
-+ headers.update(extra_headers)
-+ conn.request('POST', parsed.path + '/' + self.name, '', headers)
-+ return check_response(conn)
-+
-+ headers = {}
-+ header_value = 'k' * self.max_meta_value_length
-+ size = 0
-+ x = 0
-+ while size < (self.max_meta_overall_size - 4
-+ - self.max_meta_value_length):
-+ size += 4 + self.max_meta_value_length
-+ headers['X-Container-Meta-%04d' % x] = header_value
-+ x += 1
-+ if self.max_meta_overall_size - size > 1:
-+ headers['X-Container-Meta-k'] = \
-+ 'v' * (self.max_meta_overall_size - size - 1)
-+ resp = retry(post, headers)
-+ resp.read()
-+ self.assertEqual(resp.status, 204)
-+ headers = {}
-+ headers['X-Container-Meta-k2'] = 'v'
-+ resp = retry(post, headers)
-+ resp.read()
-+ self.assertEqual(resp.status, 400)
-+
-+
-+class TestCVE20147960Account(unittest.TestCase):
-+
-+ def setUp(self):
-+ self.max_meta_count = load_constraint('max_meta_count')
-+ self.max_meta_name_length = load_constraint('max_meta_name_length')
-+ self.max_meta_overall_size = load_constraint('max_meta_overall_size')
-+ self.max_meta_value_length = load_constraint('max_meta_value_length')
-+
-+ def head(url, token, parsed, conn):
-+ conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token})
-+ return check_response(conn)
-+ resp = retry(head)
-+ self.existing_metadata = set([
-+ k for k, v in resp.getheaders() if
-+ k.lower().startswith('x-account-meta')])
-+
-+ def tearDown(self):
-+ def head(url, token, parsed, conn):
-+ conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token})
-+ return check_response(conn)
-+ resp = retry(head)
-+ resp.read()
-+ new_metadata = set(
-+ [k for k, v in resp.getheaders() if
-+ k.lower().startswith('x-account-meta')])
-+
-+ def clear_meta(url, token, parsed, conn, remove_metadata_keys):
-+ headers = {'X-Auth-Token': token}
-+ headers.update((k, '') for k in remove_metadata_keys)
-+ conn.request('POST', parsed.path, '', headers)
-+ return check_response(conn)
-+ extra_metadata = list(self.existing_metadata ^ new_metadata)
-+ for i in range(0, len(extra_metadata), 90):
-+ batch = extra_metadata[i:i + 90]
-+ resp = retry(clear_meta, batch)
-+ resp.read()
-+ self.assertEqual(resp.status // 100, 2)
-+
-+ def test_maxcount_metadata(self):
-+ if skip:
-+ raise SkipTest
-+
-+ def post(url, token, parsed, conn, extra_headers):
-+ headers = {'X-Auth-Token': token}
-+ headers.update(extra_headers)
-+ conn.request('POST', parsed.path, '', headers)
-+ return check_response(conn)
-+
-+ # TODO: Find the test that adds these and remove them.
-+ headers = {'x-remove-account-meta-temp-url-key': 'remove',
-+ 'x-remove-account-meta-temp-url-key-2': 'remove'}
-+ resp = retry(post, headers)
-+
-+ headers = {}
-+ for x in xrange(MAX_META_COUNT):
-+ headers['X-Account-Meta-%d' % x] = 'v'
-+ resp = retry(post, headers)
-+ resp.read()
-+ self.assertEqual(resp.status, 204)
-+
-+ def test_maxcount_metadata_over_one(self):
-+ if skip:
-+ raise SkipTest
-+
-+ def post(url, token, parsed, conn, extra_headers):
-+ headers = {'X-Auth-Token': token}
-+ headers.update(extra_headers)
-+ conn.request('POST', parsed.path, '', headers)
-+ return check_response(conn)
-+
-+ # TODO: Find the test that adds these and remove them.
-+ headers = {'x-remove-account-meta-temp-url-key': 'remove',
-+ 'x-remove-account-meta-temp-url-key-2': 'remove'}
-+ resp = retry(post, headers)
-+
-+ headers = {}
-+ for x in xrange(MAX_META_COUNT + 1):
-+ headers['X-Account-Meta-%d' % x] = 'v'
-+ resp = retry(post, headers)
-+ resp.read()
-+ self.assertEqual(resp.status, 400)
-+
-+ def test_maxcount_metadata_over_two(self):
-+ if skip:
-+ raise SkipTest
-+
-+ def post(url, token, parsed, conn, extra_headers):
-+ headers = {'X-Auth-Token': token}
-+ headers.update(extra_headers)
-+ conn.request('POST', parsed.path, '', headers)
-+ return check_response(conn)
-+
-+ # TODO: Find the test that adds these and remove them.
-+ headers = {'x-remove-account-meta-temp-url-key': 'remove',
-+ 'x-remove-account-meta-temp-url-key-2': 'remove'}
-+ resp = retry(post, headers)
-+
-+ headers = {}
-+ for x in xrange(MAX_META_COUNT - 1):
-+ headers['X-Account-Meta-%d' % x] = 'v'
-+ resp = retry(post, headers)
-+ resp.read()
-+ self.assertEqual(resp.status, 204)
-+
-+ headers = {}
-+ for x in xrange(MAX_META_COUNT - 1, MAX_META_COUNT + 1):
-+ headers['X-Account-Meta-%d' % x] = 'v'
-+ resp = retry(post, headers)
-+ resp.read()
-+ self.assertEqual(resp.status, 400)
-+
-+ def test_maxsize_metadata(self):
-+ if skip:
-+ raise SkipTest
-+
-+ def post(url, token, parsed, conn, extra_headers):
-+ headers = {'X-Auth-Token': token}
-+ headers.update(extra_headers)
-+ conn.request('POST', parsed.path, '', headers)
-+ return check_response(conn)
-+
-+ # TODO: Find the test that adds these and remove them.
-+ headers = {'x-remove-account-meta-temp-url-key': 'remove',
-+ 'x-remove-account-meta-temp-url-key-2': 'remove'}
-+ resp = retry(post, headers)
-+
-+ headers = {}
-+ header_value = 'k' * MAX_META_VALUE_LENGTH
-+ size = 0
-+ x = 0
-+ while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH:
-+ size += 4 + MAX_META_VALUE_LENGTH
-+ headers['X-Account-Meta-%04d' % x] = header_value
-+ x += 1
-+ if MAX_META_OVERALL_SIZE - size > 1:
-+ headers['X-Account-Meta-k'] = \
-+ 'v' * (MAX_META_OVERALL_SIZE - size - 1)
-+ resp = retry(post, headers)
-+ resp.read()
-+ self.assertEqual(resp.status, 204)
-+ headers['X-Account-Meta-k'] = \
-+ 'v' * (MAX_META_OVERALL_SIZE - size)
-+ resp = retry(post, headers)
-+ resp.read()
-+ self.assertEqual(resp.status, 400)
-+
-+ def test_maxsize_metadata_over_one(self):
-+ if skip:
-+ raise SkipTest
-+
-+ def post(url, token, parsed, conn, extra_headers):
-+ headers = {'X-Auth-Token': token}
-+ headers.update(extra_headers)
-+ conn.request('POST', parsed.path, '', headers)
-+ return check_response(conn)
-+
-+ # TODO: Find the test that adds these and remove them.
-+ headers = {'x-remove-account-meta-temp-url-key': 'remove',
-+ 'x-remove-account-meta-temp-url-key-2': 'remove'}
-+ resp = retry(post, headers)
-+
-+ headers = {}
-+ header_value = 'k' * MAX_META_VALUE_LENGTH
-+ size = 0
-+ x = 0
-+ while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH:
-+ size += 4 + MAX_META_VALUE_LENGTH
-+ headers['X-Account-Meta-%04d' % x] = header_value
-+ x += 1
-+ if MAX_META_OVERALL_SIZE >= size:
-+ headers['X-Account-Meta-k'] = \
-+ 'v' * (MAX_META_OVERALL_SIZE - size)
-+ resp = retry(post, headers)
-+ resp.read()
-+ self.assertEqual(resp.status, 400)
-+
-+ def test_maxsize_metadata_over_two(self):
-+ if skip:
-+ raise SkipTest
-+
-+ def post(url, token, parsed, conn, extra_headers):
-+ headers = {'X-Auth-Token': token}
-+ headers.update(extra_headers)
-+ conn.request('POST', parsed.path, '', headers)
-+ return check_response(conn)
-+
-+ # TODO: Find the test that adds these and remove them.
-+ headers = {'x-remove-account-meta-temp-url-key': 'remove',
-+ 'x-remove-account-meta-temp-url-key-2': 'remove'}
-+ resp = retry(post, headers)
-+
-+ headers = {}
-+ header_value = 'k' * MAX_META_VALUE_LENGTH
-+ size = 0
-+ x = 0
-+ while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH:
-+ size += 4 + MAX_META_VALUE_LENGTH
-+ headers['X-Account-Meta-%04d' % x] = header_value
-+ x += 1
-+ if MAX_META_OVERALL_SIZE - size > 1:
-+ headers['X-Account-Meta-k'] = \
-+ 'v' * (MAX_META_OVERALL_SIZE - size - 1)
-+ resp = retry(post, headers)
-+ resp.read()
-+ self.assertEqual(resp.status, 204)
-+ headers = {}
-+ headers['X-Account-Meta-k2'] = 'v'
-+ resp = retry(post, headers)
-+ resp.read()
-+ self.assertEqual(resp.status, 400)
-+
- if __name__ == '__main__':
- unittest.main()
---- a/test/unit/common/test_db.py
-+++ b/test/unit/common/test_db.py
-@@ -26,10 +26,13 @@ import sqlite3
- from mock import patch
-
- import swift.common.db
-+from swift.common.constraints import \
-+ MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE
- from swift.common.db import chexor, dict_factory, get_db_connection, \
- DatabaseBroker, DatabaseConnectionError, DatabaseAlreadyExists
- from swift.common.utils import normalize_timestamp
- from swift.common.exceptions import LockTimeout
-+from swift.common.swob import HTTPException
-
-
- class TestDatabaseConnectionError(unittest.TestCase):
-@@ -181,7 +184,7 @@ class TestDatabaseBroker(unittest.TestCa
- conn.execute('CREATE TABLE test (one TEXT)')
- conn.execute('CREATE TABLE test_stat (id TEXT)')
- conn.execute('INSERT INTO test_stat (id) VALUES (?)',
-- (str(uuid4),))
-+ (str(uuid4),))
- conn.execute('INSERT INTO test (one) VALUES ("1")')
- conn.commit()
- stub_called = [False]
-@@ -632,6 +635,91 @@ class TestDatabaseBroker(unittest.TestCa
- [first_value, first_timestamp])
- self.assert_('Second' not in broker.metadata)
-
-+ @patch.object(DatabaseBroker, 'validate_metadata')
-+ def test_validate_metadata_is_called_from_update_metadata(self, mock):
-+ broker = self.get_replication_info_tester(metadata=True)
-+ first_timestamp = normalize_timestamp(1)
-+ first_value = '1'
-+ metadata = {'First': [first_value, first_timestamp]}
-+ broker.update_metadata(metadata, validate_metadata=True)
-+ self.assertTrue(mock.called)
-+
-+ @patch.object(DatabaseBroker, 'validate_metadata')
-+ def test_validate_metadata_is_not_called_from_update_metadata(self, mock):
-+ broker = self.get_replication_info_tester(metadata=True)
-+ first_timestamp = normalize_timestamp(1)
-+ first_value = '1'
-+ metadata = {'First': [first_value, first_timestamp]}
-+ broker.update_metadata(metadata)
-+ self.assertFalse(mock.called)
-+
-+ def test_metadata_with_max_count(self):
-+ metadata = {}
-+ for c in xrange(MAX_META_COUNT):
-+ key = 'X-Account-Meta-F{0}'.format(c)
-+ metadata[key] = ('B', normalize_timestamp(1))
-+ key = 'X-Account-Meta-Foo'.format(c)
-+ metadata[key] = ('', normalize_timestamp(1))
-+ try:
-+ DatabaseBroker.validate_metadata(metadata)
-+ except HTTPException:
-+ self.fail('Unexpected HTTPException')
-+
-+ def test_metadata_raises_exception_over_max_count(self):
-+ metadata = {}
-+ for c in xrange(MAX_META_COUNT + 1):
-+ key = 'X-Account-Meta-F{0}'.format(c)
-+ metadata[key] = ('B', normalize_timestamp(1))
-+ message = ''
-+ try:
-+ DatabaseBroker.validate_metadata(metadata)
-+ except HTTPException as e:
-+ message = str(e)
-+ self.assertEqual(message, '400 Bad Request')
-+
-+ def test_metadata_with_max_overall_size(self):
-+ metadata = {}
-+ metadata_value = 'v' * MAX_META_VALUE_LENGTH
-+ size = 0
-+ x = 0
-+ while size < (MAX_META_OVERALL_SIZE - 4
-+ - MAX_META_VALUE_LENGTH):
-+ size += 4 + MAX_META_VALUE_LENGTH
-+ metadata['X-Account-Meta-%04d' % x] = (metadata_value,
-+ normalize_timestamp(1))
-+ x += 1
-+ if MAX_META_OVERALL_SIZE - size > 1:
-+ metadata['X-Account-Meta-k'] = (
-+ 'v' * (MAX_META_OVERALL_SIZE - size - 1),
-+ normalize_timestamp(1))
-+ try:
-+ DatabaseBroker.validate_metadata(metadata)
-+ except HTTPException:
-+ self.fail('Unexpected HTTPException')
-+
-+ def test_metadata_raises_exception_over_max_overall_size(self):
-+ metadata = {}
-+ metadata_value = 'k' * MAX_META_VALUE_LENGTH
-+ size = 0
-+ x = 0
-+ while size < (MAX_META_OVERALL_SIZE - 4
-+ - MAX_META_VALUE_LENGTH):
-+ size += 4 + MAX_META_VALUE_LENGTH
-+ metadata['X-Account-Meta-%04d' % x] = (metadata_value,
-+ normalize_timestamp(1))
-+ x += 1
-+ if MAX_META_OVERALL_SIZE - size > 1:
-+ metadata['X-Account-Meta-k'] = (
-+ 'v' * (MAX_META_OVERALL_SIZE - size - 1),
-+ normalize_timestamp(1))
-+ metadata['X-Account-Meta-k2'] = ('v', normalize_timestamp(1))
-+ message = ''
-+ try:
-+ DatabaseBroker.validate_metadata(metadata)
-+ except HTTPException as e:
-+ message = str(e)
-+ self.assertEqual(message, '400 Bad Request')
-+
-
- if __name__ == '__main__':
- unittest.main()