1 This is a backport to Havana of the Icehouse backport of the fix to bug 1365350,

     2 plus a fix to the testsuite to actually test the problem (bug 1381159).  Because

     3 Havana is out of support, it is not a candidate for upstream submission,

     4 though a re-work of the testsuite changes to trunk Swift is.

     5 

     6 ================================================================================

     7 

     8 From 2c4622a28ea04e1c6b2382189b0a1f6cccdc9c0f Mon Sep 17 00:00:00 2001

     9 From: "Richard (Rick) Hawkins" <[email protected]>

    10 Date: Wed, 1 Oct 2014 09:37:47 -0400

    11 Subject: [PATCH] Fix metadata overall limits bug

    12 

    13 Currently metadata limits are checked on a per request basis. If

    14 multiple requests are sent within the per request limits, it is

    15 possible to exceed the overall limits.  This patch adds an overall

    16 metadata check to ensure that multiple requests to add metadata to

    17 an account/container will check overall limits before adding

    18 the additional metadata.

    19 

    20 This is a backport to the stable/icehouse branch for commit SHA

    21 5b2c27a5874c2b5b0a333e4955b03544f6a8119f.

    22 

    23 Closes-Bug: 1365350

    24 

    25 Conflicts:

    26 	swift/common/db.py

    27 	swift/container/server.py

    28 

    29 Change-Id: Id9fca209c9c1216f1949de7108bbe332808f1045

    30 ---

    31  swift/account/server.py           |    4 +-

    32  swift/common/constraints.py       |    5 ++-

    33  swift/common/db.py                |   34 +++++++++++++-

    34  swift/container/server.py         |    4 +-

    35  test/functional/test_account.py   |   66 +++++++++++++++++++++++++++

    36  test/functional/test_container.py |   20 +++++++++

    37  test/unit/common/test_db.py       |   90 ++++++++++++++++++++++++++++++++++++-

    38  7 files changed, 216 insertions(+), 7 deletions(-)

    39 

    40 --- a/swift/account/server.py

    41 +++ b/swift/account/server.py

    42 @@ -156,7 +156,7 @@ class AccountController(object):

    43                              for key, value in req.headers.iteritems()

    44                              if key.lower().startswith('x-account-meta-'))

    45              if metadata:

    46 -                broker.update_metadata(metadata)

    47 +                broker.update_metadata(metadata, validate_metadata=True)

    48              if created:

    49                  return HTTPCreated(request=req)

    50              else:

    51 @@ -262,7 +262,7 @@ class AccountController(object):

    52                          for key, value in req.headers.iteritems()

    53                          if key.lower().startswith('x-account-meta-'))

    54          if metadata:

    55 -            broker.update_metadata(metadata)

    56 +            broker.update_metadata(metadata, validate_metadata=True)

    57          return HTTPNoContent(request=req)

    58  

    59      def __call__(self, env, start_response):

    60 --- a/swift/common/constraints.py

    61 +++ b/swift/common/constraints.py

    62 @@ -68,7 +68,10 @@ FORMAT2CONTENT_TYPE = {'plain': 'text/pl

    63  

    64  def check_metadata(req, target_type):

    65      """

    66 -    Check metadata sent in the request headers.

    67 +    Check metadata sent in the request headers.  This should only check

    68 +    that the metadata in the request given is valid.  Checks against

    69 +    account/container overall metadata should be forwarded on to its

    70 +    respective server to be checked.

    71  

    72      :param req: request object

    73      :param target_type: str: one of: object, container, or account: indicates

    74 --- a/swift/common/db.py

    75 +++ b/swift/common/db.py

    76 @@ -32,7 +32,9 @@ import sqlite3

    77  

    78  from swift.common.utils import json, normalize_timestamp, renamer, \

    79      mkdirs, lock_parent_directory, fallocate

    80 +from swift.common.constraints import MAX_META_COUNT, MAX_META_OVERALL_SIZE

    81  from swift.common.exceptions import LockTimeout

    82 +from swift.common.swob import HTTPBadRequest

    83  

    84  

    85  #: Whether calls will be made to preallocate disk space for database files.

    86 @@ -615,7 +617,35 @@ class DatabaseBroker(object):

    87              metadata = {}

    88          return metadata

    89  

    90 -    def update_metadata(self, metadata_updates):

    91 +    @staticmethod

    92 +    def validate_metadata(metadata):

    93 +        """

    94 +        Validates that metadata_falls within acceptable limits.

    95 +

    96 +        :param metadata: to be validated

    97 +        :raises: HTTPBadRequest if MAX_META_COUNT or MAX_META_OVERALL_SIZE

    98 +                 is exceeded

    99 +        """

   100 +        meta_count = 0

   101 +        meta_size = 0

   102 +        for key, (value, timestamp) in metadata.iteritems():

   103 +            key = key.lower()

   104 +            if value != '' and (key.startswith('x-account-meta') or

   105 +                                key.startswith('x-container-meta')):

   106 +                prefix = 'x-account-meta-'

   107 +                if key.startswith('x-container-meta-'):

   108 +                    prefix = 'x-container-meta-'

   109 +                key = key[len(prefix):]

   110 +                meta_count = meta_count + 1

   111 +                meta_size = meta_size + len(key) + len(value)

   112 +        if meta_count > MAX_META_COUNT:

   113 +            raise HTTPBadRequest('Too many metadata items; max %d'

   114 +                                 % MAX_META_COUNT)

   115 +        if meta_size > MAX_META_OVERALL_SIZE:

   116 +            raise HTTPBadRequest('Total metadata too large; max %d'

   117 +                                 % MAX_META_OVERALL_SIZE)

   118 +

   119 +    def update_metadata(self, metadata_updates, validate_metadata=False):

   120          """

   121          Updates the metadata dict for the database. The metadata dict values

   122          are tuples of (value, timestamp) where the timestamp indicates when

   123 @@ -648,6 +678,8 @@ class DatabaseBroker(object):

   124                  value, timestamp = value_timestamp

   125                  if key not in md or timestamp > md[key][1]:

   126                      md[key] = value_timestamp

   127 +            if validate_metadata:

   128 +                DatabaseBroker.validate_metadata(md)

   129              conn.execute('UPDATE %s_stat SET metadata = ?' % self.db_type,

   130                           (json.dumps(md),))

   131              conn.commit()

   132 --- a/swift/container/server.py

   133 +++ b/swift/container/server.py

   134 @@ -275,7 +275,7 @@ class ContainerController(object):

   135                              metadata['X-Container-Sync-To'][0] != \

   136                              broker.metadata['X-Container-Sync-To'][0]:

   137                          broker.set_x_container_sync_points(-1, -1)

   138 -                broker.update_metadata(metadata)

   139 +                broker.update_metadata(metadata, validate_metadata=True)

   140              resp = self.account_update(req, account, container, broker)

   141              if resp:

   142                  return resp

   143 @@ -461,7 +461,7 @@ class ContainerController(object):

   144                          metadata['X-Container-Sync-To'][0] != \

   145                          broker.metadata['X-Container-Sync-To'][0]:

   146                      broker.set_x_container_sync_points(-1, -1)

   147 -            broker.update_metadata(metadata)

   148 +            broker.update_metadata(metadata, validate_metadata=True)

   149          return HTTPNoContent(request=req)

   150  

   151      def __call__(self, env, start_response):

   152 --- a/test/functional/swift_testing.py

   153 +++ b/test/functional/swift_testing.py

   154 @@ -0,0 +1,231 @@

   155 +# Copyright (c) 2010-2012 OpenStack Foundation

   156 +#

   157 +# Licensed under the Apache License, Version 2.0 (the "License");

   158 +# you may not use this file except in compliance with the License.

   159 +# You may obtain a copy of the License at

   160 +#

   161 +#    http://www.apache.org/licenses/LICENSE-2.0

   162 +#

   163 +# Unless required by applicable law or agreed to in writing, software

   164 +# distributed under the License is distributed on an "AS IS" BASIS,

   165 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or

   166 +# implied.

   167 +# See the License for the specific language governing permissions and

   168 +# limitations under the License.

   169 +

   170 +from httplib import HTTPException

   171 +import os

   172 +import socket

   173 +import sys

   174 +from time import sleep

   175 +from urlparse import urlparse

   176 +import functools

   177 +from nose import SkipTest

   178 +

   179 +from test import get_config

   180 +

   181 +from swiftclient import get_auth, http_connection

   182 +from test.functional.swift_test_client import Connection

   183 +

   184 +conf = get_config('func_test')

   185 +web_front_end = conf.get('web_front_end', 'integral')

   186 +normalized_urls = conf.get('normalized_urls', False)

   187 +

   188 +# If no conf was read, we will fall back to old school env vars

   189 +swift_test_auth = os.environ.get('SWIFT_TEST_AUTH')

   190 +swift_test_user = [os.environ.get('SWIFT_TEST_USER'), None, None]

   191 +swift_test_key = [os.environ.get('SWIFT_TEST_KEY'), None, None]

   192 +swift_test_tenant = ['', '', '']

   193 +swift_test_perm = ['', '', '']

   194 +

   195 +if conf:

   196 +    swift_test_auth_version = str(conf.get('auth_version', '1'))

   197 +

   198 +    swift_test_auth = 'http'

   199 +    if conf.get('auth_ssl', 'no').lower() in ('yes', 'true', 'on', '1'):

   200 +        swift_test_auth = 'https'

   201 +    if 'auth_prefix' not in conf:

   202 +        conf['auth_prefix'] = '/'

   203 +    try:

   204 +        suffix = '://%(auth_host)s:%(auth_port)s%(auth_prefix)s' % conf

   205 +        swift_test_auth += suffix

   206 +    except KeyError:

   207 +        pass  # skip

   208 +

   209 +    if swift_test_auth_version == "1":

   210 +        swift_test_auth += 'v1.0'

   211 +

   212 +        if 'account' in conf:

   213 +            swift_test_user[0] = '%(account)s:%(username)s' % conf

   214 +        else:

   215 +            swift_test_user[0] = '%(username)s' % conf

   216 +        swift_test_key[0] = conf['password']

   217 +        try:

   218 +            swift_test_user[1] = '%s%s' % (

   219 +                '%s:' % conf['account2'] if 'account2' in conf else '',

   220 +                conf['username2'])

   221 +            swift_test_key[1] = conf['password2']

   222 +        except KeyError as err:

   223 +            pass  # old conf, no second account tests can be run

   224 +        try:

   225 +            swift_test_user[2] = '%s%s' % ('%s:' % conf['account'] if 'account'

   226 +                                           in conf else '', conf['username3'])

   227 +            swift_test_key[2] = conf['password3']

   228 +        except KeyError as err:

   229 +            pass  # old conf, no third account tests can be run

   230 +

   231 +        for _ in range(3):

   232 +            swift_test_perm[_] = swift_test_user[_]

   233 +

   234 +    else:

   235 +        swift_test_user[0] = conf['username']

   236 +        swift_test_tenant[0] = conf['account']

   237 +        swift_test_key[0] = conf['password']

   238 +        swift_test_user[1] = conf['username2']

   239 +        swift_test_tenant[1] = conf['account2']

   240 +        swift_test_key[1] = conf['password2']

   241 +        swift_test_user[2] = conf['username3']

   242 +        swift_test_tenant[2] = conf['account']

   243 +        swift_test_key[2] = conf['password3']

   244 +

   245 +        for _ in range(3):

   246 +            swift_test_perm[_] = swift_test_tenant[_] + ':' \

   247 +                + swift_test_user[_]

   248 +

   249 +skip = not all([swift_test_auth, swift_test_user[0], swift_test_key[0]])

   250 +if skip:

   251 +    print >>sys.stderr, 'SKIPPING FUNCTIONAL TESTS DUE TO NO CONFIG'

   252 +

   253 +skip2 = not all([not skip, swift_test_user[1], swift_test_key[1]])

   254 +if not skip and skip2:

   255 +    print >>sys.stderr, \

   256 +        'SKIPPING SECOND ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM'

   257 +

   258 +skip3 = not all([not skip, swift_test_user[2], swift_test_key[2]])

   259 +if not skip and skip3:

   260 +    print >>sys.stderr, \

   261 +        'SKIPPING THIRD ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM'

   262 +

   263 +

   264 +class AuthError(Exception):

   265 +    pass

   266 +

   267 +

   268 +class InternalServerError(Exception):

   269 +    pass

   270 +

   271 +

   272 +url = [None, None, None]

   273 +token = [None, None, None]

   274 +parsed = [None, None, None]

   275 +conn = [None, None, None]

   276 +

   277 +

   278 +def retry(func, *args, **kwargs):

   279 +    """

   280 +    You can use the kwargs to override:

   281 +      'retries' (default: 5)

   282 +      'use_account' (default: 1) - which user's token to pass

   283 +      'url_account' (default: matches 'use_account') - which user's storage URL

   284 +      'resource' (default: url[url_account] - URL to connect to; retry()

   285 +          will interpolate the variable :storage_url: if present

   286 +    """

   287 +    global url, token, parsed, conn

   288 +    retries = kwargs.get('retries', 5)

   289 +    attempts, backoff = 0, 1

   290 +

   291 +    # use account #1 by default; turn user's 1-indexed account into 0-indexed

   292 +    use_account = kwargs.pop('use_account', 1) - 1

   293 +

   294 +    # access our own account by default

   295 +    url_account = kwargs.pop('url_account', use_account + 1) - 1

   296 +

   297 +    while attempts <= retries:

   298 +        attempts += 1

   299 +        try:

   300 +            if not url[use_account] or not token[use_account]:

   301 +                url[use_account], token[use_account] = \

   302 +                    get_auth(swift_test_auth, swift_test_user[use_account],

   303 +                             swift_test_key[use_account],

   304 +                             snet=False,

   305 +                             tenant_name=swift_test_tenant[use_account],

   306 +                             auth_version=swift_test_auth_version,

   307 +                             os_options={})

   308 +                parsed[use_account] = conn[use_account] = None

   309 +            if not parsed[use_account] or not conn[use_account]:

   310 +                parsed[use_account], conn[use_account] = \

   311 +                    http_connection(url[use_account])

   312 +

   313 +            # default resource is the account url[url_account]

   314 +            resource = kwargs.pop('resource', '%(storage_url)s')

   315 +            template_vars = {'storage_url': url[url_account]}

   316 +            parsed_result = urlparse(resource % template_vars)

   317 +            return func(url[url_account], token[use_account],

   318 +                        parsed_result, conn[url_account],

   319 +                        *args, **kwargs)

   320 +        except (socket.error, HTTPException):

   321 +            if attempts > retries:

   322 +                raise

   323 +            parsed[use_account] = conn[use_account] = None

   324 +        except AuthError:

   325 +            url[use_account] = token[use_account] = None

   326 +            continue

   327 +        except InternalServerError:

   328 +            pass

   329 +        if attempts <= retries:

   330 +            sleep(backoff)

   331 +            backoff *= 2

   332 +    raise Exception('No result after %s retries.' % retries)

   333 +

   334 +

   335 +def check_response(conn):

   336 +    resp = conn.getresponse()

   337 +    if resp.status == 401:

   338 +        resp.read()

   339 +        raise AuthError()

   340 +    elif resp.status // 100 == 5:

   341 +        resp.read()

   342 +        raise InternalServerError()

   343 +    return resp

   344 +

   345 +cluster_info = {}

   346 +

   347 +

   348 +def get_cluster_info():

   349 +    conn = Connection(conf)

   350 +    conn.authenticate()

   351 +    global cluster_info

   352 +    cluster_info = conn.cluster_info()

   353 +

   354 +

   355 +def reset_acl():

   356 +    def post(url, token, parsed, conn):

   357 +        conn.request('POST', parsed.path, '', {

   358 +            'X-Auth-Token': token,

   359 +            'X-Account-Access-Control': '{}'

   360 +        })

   361 +        return check_response(conn)

   362 +    resp = retry(post, use_account=1)

   363 +    resp.read()

   364 +

   365 +

   366 +def requires_acls(f):

   367 +    @functools.wraps(f)

   368 +    def wrapper(*args, **kwargs):

   369 +        if skip:

   370 +            raise SkipTest

   371 +        if not cluster_info:

   372 +            get_cluster_info()

   373 +        # Determine whether this cluster has account ACLs; if not, skip test

   374 +        if not cluster_info.get('tempauth', {}).get('account_acls'):

   375 +            raise SkipTest

   376 +        if 'keystoneauth' in cluster_info:

   377 +            # remove when keystoneauth supports account acls

   378 +            raise SkipTest

   379 +        reset_acl()

   380 +        try:

   381 +            rv = f(*args, **kwargs)

   382 +        finally:

   383 +            reset_acl()

   384 +        return rv

   385 +    return wrapper

   386 --- a/test/functional/tests.py

   387 +++ b/test/functional/tests.py

   388 @@ -1624,5 +1624,466 @@ class TestFileComparison(Base):

   389  class TestFileComparisonUTF8(Base2, TestFileComparison):

   390      set_up = False

   391  

   392 +import json

   393 +from uuid import uuid4

   394 +from swift_testing import (check_response, retry, skip,

   395 +                           web_front_end, requires_acls)

   396 +import swift_testing

   397 +

   398 +class TestCVE20147960Container(unittest.TestCase):

   399 +

   400 +    def setUp(self):

   401 +        if skip:

   402 +            raise SkipTest

   403 +        self.name = uuid4().hex

   404 +        # this container isn't created by default, but will be cleaned up

   405 +        self.container = uuid4().hex

   406 +

   407 +        def put(url, token, parsed, conn):

   408 +            conn.request('PUT', parsed.path + '/' + self.name, '',

   409 +                         {'X-Auth-Token': token})

   410 +            return check_response(conn)

   411 +

   412 +        resp = retry(put)

   413 +        resp.read()

   414 +        self.assertEqual(resp.status, 201)

   415 +

   416 +        self.max_meta_count = load_constraint('max_meta_count')

   417 +        self.max_meta_name_length = load_constraint('max_meta_name_length')

   418 +        self.max_meta_overall_size = load_constraint('max_meta_overall_size')

   419 +        self.max_meta_value_length = load_constraint('max_meta_value_length')

   420 +

   421 +    def tearDown(self):

   422 +        if skip:

   423 +            raise SkipTest

   424 +

   425 +        def get(url, token, parsed, conn, container):

   426 +            conn.request(

   427 +                'GET', parsed.path + '/' + container + '?format=json', '',

   428 +                {'X-Auth-Token': token})

   429 +            return check_response(conn)

   430 +

   431 +        def delete(url, token, parsed, conn, container, obj):

   432 +            conn.request(

   433 +                'DELETE', '/'.join([parsed.path, container, obj['name']]), '',

   434 +                {'X-Auth-Token': token})

   435 +            return check_response(conn)

   436 +

   437 +        for container in (self.name, self.container):

   438 +            while True:

   439 +                resp = retry(get, container)

   440 +                body = resp.read()

   441 +                if resp.status == 404:

   442 +                    break

   443 +                self.assert_(resp.status // 100 == 2, resp.status)

   444 +                objs = json.loads(body)

   445 +                if not objs:

   446 +                    break

   447 +                for obj in objs:

   448 +                    resp = retry(delete, container, obj)

   449 +                    resp.read()

   450 +                    self.assertEqual(resp.status, 204)

   451 +

   452 +        def delete(url, token, parsed, conn, container):

   453 +            conn.request('DELETE', parsed.path + '/' + container, '',

   454 +                         {'X-Auth-Token': token})

   455 +            return check_response(conn)

   456 +

   457 +        resp = retry(delete, self.name)

   458 +        resp.read()

   459 +        self.assertEqual(resp.status, 204)

   460 +

   461 +        # container may have not been created

   462 +        resp = retry(delete, self.container)

   463 +        resp.read()

   464 +        self.assert_(resp.status in (204, 404))

   465 +

   466 +    def test_POST_bad_metadata(self):

   467 +        if skip:

   468 +            raise SkipTest

   469 +

   470 +        def post(url, token, parsed, conn, extra_headers):

   471 +            headers = {'X-Auth-Token': token}

   472 +            headers.update(extra_headers)

   473 +            conn.request('POST', parsed.path + '/' + self.name, '', headers)

   474 +            return check_response(conn)

   475 +

   476 +        resp = retry(

   477 +            post,

   478 +            {'X-Container-Meta-' + ('k' * self.max_meta_name_length): 'v'})

   479 +        resp.read()

   480 +        self.assertEqual(resp.status, 204)

   481 +        resp = retry(

   482 +            post,

   483 +            {'X-Container-Meta-' + (

   484 +                'k' * (self.max_meta_name_length + 1)): 'v'})

   485 +        resp.read()

   486 +        self.assertEqual(resp.status, 400)

   487 +

   488 +        resp = retry(

   489 +            post,

   490 +            {'X-Container-Meta-Too-Long': 'k' * self.max_meta_value_length})

   491 +        resp.read()

   492 +        self.assertEqual(resp.status, 204)

   493 +        resp = retry(

   494 +            post,

   495 +            {'X-Container-Meta-Too-Long': 'k' * (

   496 +                self.max_meta_value_length + 1)})

   497 +        resp.read()

   498 +        self.assertEqual(resp.status, 400)

   499 +

   500 +    def test_POST_maxcount_metadata(self):

   501 +        if skip:

   502 +            raise SkipTest

   503 +

   504 +        def post(url, token, parsed, conn, extra_headers):

   505 +            headers = {'X-Auth-Token': token}

   506 +            headers.update(extra_headers)

   507 +            conn.request('POST', parsed.path + '/' + self.name, '', headers)

   508 +            return check_response(conn)

   509 +

   510 +        headers = {}

   511 +        for x in xrange(self.max_meta_count):

   512 +            headers['X-Container-Meta-%d' % x] = 'v'

   513 +        resp = retry(post, headers)

   514 +        resp.read()

   515 +        self.assertEqual(resp.status, 204)

   516 +

   517 +    def test_POST_maxcount_metadata_over_one(self):

   518 +        if skip:

   519 +            raise SkipTest

   520 +

   521 +        def post(url, token, parsed, conn, extra_headers):

   522 +            headers = {'X-Auth-Token': token}

   523 +            headers.update(extra_headers)

   524 +            conn.request('POST', parsed.path + '/' + self.name, '', headers)

   525 +            return check_response(conn)

   526 +

   527 +        headers = {}

   528 +        for x in xrange(self.max_meta_count + 1):

   529 +            headers['X-Container-Meta-%d' % x] = 'v'

   530 +        resp = retry(post, headers)

   531 +        resp.read()

   532 +        self.assertEqual(resp.status, 400)

   533 +

   534 +    def test_POST_maxcount_metadata_over_two(self):

   535 +        if skip:

   536 +            raise SkipTest

   537 +

   538 +        def post(url, token, parsed, conn, extra_headers):

   539 +            headers = {'X-Auth-Token': token}

   540 +            headers.update(extra_headers)

   541 +            conn.request('POST', parsed.path + '/' + self.name, '', headers)

   542 +            return check_response(conn)

   543 +

   544 +        headers = {}

   545 +        for x in xrange(self.max_meta_count - 1):

   546 +            headers['X-Container-Meta-%d' % x] = 'v'

   547 +        resp = retry(post, headers)

   548 +        resp.read()

   549 +        self.assertEqual(resp.status, 204)

   550 +

   551 +        headers = {}

   552 +        for x in xrange(self.max_meta_count - 1, self.max_meta_count + 1):

   553 +            headers['X-Container-Meta-%d' % x] = 'v'

   554 +        resp = retry(post, headers)

   555 +        resp.read()

   556 +        self.assertEqual(resp.status, 400)

   557 +

   558 +    def test_POST_maxsize_metadata(self):

   559 +        if skip:

   560 +            raise SkipTest

   561 +

   562 +        def post(url, token, parsed, conn, extra_headers):

   563 +            headers = {'X-Auth-Token': token}

   564 +            headers.update(extra_headers)

   565 +            conn.request('POST', parsed.path + '/' + self.name, '', headers)

   566 +            return check_response(conn)

   567 +

   568 +        headers = {}

   569 +        header_value = 'k' * self.max_meta_value_length

   570 +        size = 0

   571 +        x = 0

   572 +        while size < (self.max_meta_overall_size - 4

   573 +                      - self.max_meta_value_length):

   574 +            size += 4 + self.max_meta_value_length

   575 +            headers['X-Container-Meta-%04d' % x] = header_value

   576 +            x += 1

   577 +        if self.max_meta_overall_size - size > 1:

   578 +            headers['X-Container-Meta-k'] = \

   579 +                'v' * (self.max_meta_overall_size - size - 1)

   580 +        resp = retry(post, headers)

   581 +        resp.read()

   582 +        self.assertEqual(resp.status, 204)

   583 +

   584 +    def test_POST_maxsize_metadata_over_one(self):

   585 +        if skip:

   586 +            raise SkipTest

   587 +

   588 +        def post(url, token, parsed, conn, extra_headers):

   589 +            headers = {'X-Auth-Token': token}

   590 +            headers.update(extra_headers)

   591 +            conn.request('POST', parsed.path + '/' + self.name, '', headers)

   592 +            return check_response(conn)

   593 +

   594 +        headers = {}

   595 +        header_value = 'k' * self.max_meta_value_length

   596 +        size = 0

   597 +        x = 0

   598 +        while size < (self.max_meta_overall_size - 4

   599 +                      - self.max_meta_value_length):

   600 +            size += 4 + self.max_meta_value_length

   601 +            headers['X-Container-Meta-%04d' % x] = header_value

   602 +            x += 1

   603 +        if self.max_meta_overall_size >= size:

   604 +            headers['X-Container-Meta-k'] = \

   605 +                'v' * (self.max_meta_overall_size - size)

   606 +        resp = retry(post, headers)

   607 +        resp.read()

   608 +        self.assertEqual(resp.status, 400)

   609 +

   610 +    def test_POST_maxsize_metadata_over_two(self):

   611 +        if skip:

   612 +            raise SkipTest

   613 +

   614 +        def post(url, token, parsed, conn, extra_headers):

   615 +            headers = {'X-Auth-Token': token}

   616 +            headers.update(extra_headers)

   617 +            conn.request('POST', parsed.path + '/' + self.name, '', headers)

   618 +            return check_response(conn)

   619 +

   620 +        headers = {}

   621 +        header_value = 'k' * self.max_meta_value_length

   622 +        size = 0

   623 +        x = 0

   624 +        while size < (self.max_meta_overall_size - 4

   625 +                      - self.max_meta_value_length):

   626 +            size += 4 + self.max_meta_value_length

   627 +            headers['X-Container-Meta-%04d' % x] = header_value

   628 +            x += 1

   629 +        if self.max_meta_overall_size - size > 1:

   630 +            headers['X-Container-Meta-k'] = \

   631 +                'v' * (self.max_meta_overall_size - size - 1)

   632 +        resp = retry(post, headers)

   633 +        resp.read()

   634 +        self.assertEqual(resp.status, 204)

   635 +        headers = {}

   636 +        headers['X-Container-Meta-k2'] = 'v'

   637 +        resp = retry(post, headers)

   638 +        resp.read()

   639 +        self.assertEqual(resp.status, 400)

   640 +

   641 +

   642 +class TestCVE20147960Account(unittest.TestCase):

   643 +

   644 +    def setUp(self):

   645 +        self.max_meta_count = load_constraint('max_meta_count')

   646 +        self.max_meta_name_length = load_constraint('max_meta_name_length')

   647 +        self.max_meta_overall_size = load_constraint('max_meta_overall_size')

   648 +        self.max_meta_value_length = load_constraint('max_meta_value_length')

   649 +

   650 +        def head(url, token, parsed, conn):

   651 +            conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token})

   652 +            return check_response(conn)

   653 +        resp = retry(head)

   654 +        self.existing_metadata = set([

   655 +            k for k, v in resp.getheaders() if

   656 +            k.lower().startswith('x-account-meta')])

   657 +

   658 +    def tearDown(self):

   659 +        def head(url, token, parsed, conn):

   660 +            conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token})

   661 +            return check_response(conn)

   662 +        resp = retry(head)

   663 +        resp.read()

   664 +        new_metadata = set(

   665 +            [k for k, v in resp.getheaders() if

   666 +             k.lower().startswith('x-account-meta')])

   667 +

   668 +        def clear_meta(url, token, parsed, conn, remove_metadata_keys):

   669 +            headers = {'X-Auth-Token': token}

   670 +            headers.update((k, '') for k in remove_metadata_keys)

   671 +            conn.request('POST', parsed.path, '', headers)

   672 +            return check_response(conn)

   673 +        extra_metadata = list(self.existing_metadata ^ new_metadata)

   674 +        for i in range(0, len(extra_metadata), 90):

   675 +            batch = extra_metadata[i:i + 90]

   676 +            resp = retry(clear_meta, batch)

   677 +            resp.read()

   678 +            self.assertEqual(resp.status // 100, 2)

   679 +

   680 +    def test_maxcount_metadata(self):

   681 +        if skip:

   682 +            raise SkipTest

   683 +

   684 +        def post(url, token, parsed, conn, extra_headers):

   685 +            headers = {'X-Auth-Token': token}

   686 +            headers.update(extra_headers)

   687 +            conn.request('POST', parsed.path, '', headers)

   688 +            return check_response(conn)

   689 +

   690 +        # TODO: Find the test that adds these and remove them.

   691 +        headers = {'x-remove-account-meta-temp-url-key': 'remove',

   692 +                   'x-remove-account-meta-temp-url-key-2': 'remove'}

   693 +        resp = retry(post, headers)

   694 +

   695 +        headers = {}

   696 +        for x in xrange(MAX_META_COUNT):

   697 +            headers['X-Account-Meta-%d' % x] = 'v'

   698 +        resp = retry(post, headers)

   699 +        resp.read()

   700 +        self.assertEqual(resp.status, 204)

   701 +

   702 +    def test_maxcount_metadata_over_one(self):

   703 +        if skip:

   704 +            raise SkipTest

   705 +

   706 +        def post(url, token, parsed, conn, extra_headers):

   707 +            headers = {'X-Auth-Token': token}

   708 +            headers.update(extra_headers)

   709 +            conn.request('POST', parsed.path, '', headers)

   710 +            return check_response(conn)

   711 +

   712 +        # TODO: Find the test that adds these and remove them.

   713 +        headers = {'x-remove-account-meta-temp-url-key': 'remove',

   714 +                   'x-remove-account-meta-temp-url-key-2': 'remove'}

   715 +        resp = retry(post, headers)

   716 +

   717 +        headers = {}

   718 +        for x in xrange(MAX_META_COUNT + 1):

   719 +            headers['X-Account-Meta-%d' % x] = 'v'

   720 +        resp = retry(post, headers)

   721 +        resp.read()

   722 +        self.assertEqual(resp.status, 400)

   723 +

   724 +    def test_maxcount_metadata_over_two(self):

   725 +        if skip:

   726 +            raise SkipTest

   727 +

   728 +        def post(url, token, parsed, conn, extra_headers):

   729 +            headers = {'X-Auth-Token': token}

   730 +            headers.update(extra_headers)

   731 +            conn.request('POST', parsed.path, '', headers)

   732 +            return check_response(conn)

   733 +

   734 +        # TODO: Find the test that adds these and remove them.

   735 +        headers = {'x-remove-account-meta-temp-url-key': 'remove',

   736 +                   'x-remove-account-meta-temp-url-key-2': 'remove'}

   737 +        resp = retry(post, headers)

   738 +

   739 +        headers = {}

   740 +        for x in xrange(MAX_META_COUNT - 1):

   741 +            headers['X-Account-Meta-%d' % x] = 'v'

   742 +        resp = retry(post, headers)

   743 +        resp.read()

   744 +        self.assertEqual(resp.status, 204)

   745 +

   746 +        headers = {}

   747 +        for x in xrange(MAX_META_COUNT - 1, MAX_META_COUNT + 1):

   748 +            headers['X-Account-Meta-%d' % x] = 'v'

   749 +        resp = retry(post, headers)

   750 +        resp.read()

   751 +        self.assertEqual(resp.status, 400)

   752 +

   753 +    def test_maxsize_metadata(self):

   754 +        if skip:

   755 +            raise SkipTest

   756 +

   757 +        def post(url, token, parsed, conn, extra_headers):

   758 +            headers = {'X-Auth-Token': token}

   759 +            headers.update(extra_headers)

   760 +            conn.request('POST', parsed.path, '', headers)

   761 +            return check_response(conn)

   762 +

   763 +        # TODO: Find the test that adds these and remove them.

   764 +        headers = {'x-remove-account-meta-temp-url-key': 'remove',

   765 +                   'x-remove-account-meta-temp-url-key-2': 'remove'}

   766 +        resp = retry(post, headers)

   767 +

   768 +        headers = {}

   769 +        header_value = 'k' * MAX_META_VALUE_LENGTH

   770 +        size = 0

   771 +        x = 0

   772 +        while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH:

   773 +            size += 4 + MAX_META_VALUE_LENGTH

   774 +            headers['X-Account-Meta-%04d' % x] = header_value

   775 +            x += 1

   776 +        if MAX_META_OVERALL_SIZE - size > 1:

   777 +            headers['X-Account-Meta-k'] = \

   778 +                'v' * (MAX_META_OVERALL_SIZE - size - 1)

   779 +        resp = retry(post, headers)

   780 +        resp.read()

   781 +        self.assertEqual(resp.status, 204)

   782 +        headers['X-Account-Meta-k'] = \

   783 +            'v' * (MAX_META_OVERALL_SIZE - size)

   784 +        resp = retry(post, headers)

   785 +        resp.read()

   786 +        self.assertEqual(resp.status, 400)

   787 +

   788 +    def test_maxsize_metadata_over_one(self):

   789 +        if skip:

   790 +            raise SkipTest

   791 +

   792 +        def post(url, token, parsed, conn, extra_headers):

   793 +            headers = {'X-Auth-Token': token}

   794 +            headers.update(extra_headers)

   795 +            conn.request('POST', parsed.path, '', headers)

   796 +            return check_response(conn)

   797 +

   798 +        # TODO: Find the test that adds these and remove them.

   799 +        headers = {'x-remove-account-meta-temp-url-key': 'remove',

   800 +                   'x-remove-account-meta-temp-url-key-2': 'remove'}

   801 +        resp = retry(post, headers)

   802 +

   803 +        headers = {}

   804 +        header_value = 'k' * MAX_META_VALUE_LENGTH

   805 +        size = 0

   806 +        x = 0

   807 +        while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH:

   808 +            size += 4 + MAX_META_VALUE_LENGTH

   809 +            headers['X-Account-Meta-%04d' % x] = header_value

   810 +            x += 1

   811 +        if MAX_META_OVERALL_SIZE >= size:

   812 +            headers['X-Account-Meta-k'] = \

   813 +                'v' * (MAX_META_OVERALL_SIZE - size)

   814 +        resp = retry(post, headers)

   815 +        resp.read()

   816 +        self.assertEqual(resp.status, 400)

   817 +

   818 +    def test_maxsize_metadata_over_two(self):

   819 +        if skip:

   820 +            raise SkipTest

   821 +

   822 +        def post(url, token, parsed, conn, extra_headers):

   823 +            headers = {'X-Auth-Token': token}

   824 +            headers.update(extra_headers)

   825 +            conn.request('POST', parsed.path, '', headers)

   826 +            return check_response(conn)

   827 +

   828 +        # TODO: Find the test that adds these and remove them.

   829 +        headers = {'x-remove-account-meta-temp-url-key': 'remove',

   830 +                   'x-remove-account-meta-temp-url-key-2': 'remove'}

   831 +        resp = retry(post, headers)

   832 +

   833 +        headers = {}

   834 +        header_value = 'k' * MAX_META_VALUE_LENGTH

   835 +        size = 0

   836 +        x = 0

   837 +        while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH:

   838 +            size += 4 + MAX_META_VALUE_LENGTH

   839 +            headers['X-Account-Meta-%04d' % x] = header_value

   840 +            x += 1

   841 +        if MAX_META_OVERALL_SIZE - size > 1:

   842 +            headers['X-Account-Meta-k'] = \

   843 +                'v' * (MAX_META_OVERALL_SIZE - size - 1)

   844 +        resp = retry(post, headers)

   845 +        resp.read()

   846 +        self.assertEqual(resp.status, 204)

   847 +        headers = {}

   848 +        headers['X-Account-Meta-k2'] = 'v'

   849 +        resp = retry(post, headers)

   850 +        resp.read()

   851 +        self.assertEqual(resp.status, 400)

   852 +

   853  if __name__ == '__main__':

   854      unittest.main()

   855 --- a/test/unit/common/test_db.py

   856 +++ b/test/unit/common/test_db.py

   857 @@ -26,10 +26,13 @@ import sqlite3

   858  from mock import patch

   859  

   860  import swift.common.db

   861 +from swift.common.constraints import \

   862 +    MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE

   863  from swift.common.db import chexor, dict_factory, get_db_connection, \

   864      DatabaseBroker, DatabaseConnectionError, DatabaseAlreadyExists

   865  from swift.common.utils import normalize_timestamp

   866  from swift.common.exceptions import LockTimeout

   867 +from swift.common.swob import HTTPException

   868  

   869  

   870  class TestDatabaseConnectionError(unittest.TestCase):

   871 @@ -181,7 +184,7 @@ class TestDatabaseBroker(unittest.TestCa

   872              conn.execute('CREATE TABLE test (one TEXT)')

   873              conn.execute('CREATE TABLE test_stat (id TEXT)')

   874              conn.execute('INSERT INTO test_stat (id) VALUES (?)',

   875 -                        (str(uuid4),))

   876 +                         (str(uuid4),))

   877              conn.execute('INSERT INTO test (one) VALUES ("1")')

   878              conn.commit()

   879          stub_called = [False]

   880 @@ -632,6 +635,91 @@ class TestDatabaseBroker(unittest.TestCa

   881                            [first_value, first_timestamp])

   882          self.assert_('Second' not in broker.metadata)

   883  

   884 +    @patch.object(DatabaseBroker, 'validate_metadata')

   885 +    def test_validate_metadata_is_called_from_update_metadata(self, mock):

   886 +        broker = self.get_replication_info_tester(metadata=True)

   887 +        first_timestamp = normalize_timestamp(1)

   888 +        first_value = '1'

   889 +        metadata = {'First': [first_value, first_timestamp]}

   890 +        broker.update_metadata(metadata, validate_metadata=True)

   891 +        self.assertTrue(mock.called)

   892 +

   893 +    @patch.object(DatabaseBroker, 'validate_metadata')

   894 +    def test_validate_metadata_is_not_called_from_update_metadata(self, mock):

   895 +        broker = self.get_replication_info_tester(metadata=True)

   896 +        first_timestamp = normalize_timestamp(1)

   897 +        first_value = '1'

   898 +        metadata = {'First': [first_value, first_timestamp]}

   899 +        broker.update_metadata(metadata)

   900 +        self.assertFalse(mock.called)

   901 +

   902 +    def test_metadata_with_max_count(self):

   903 +        metadata = {}

   904 +        for c in xrange(MAX_META_COUNT):

   905 +            key = 'X-Account-Meta-F{0}'.format(c)

   906 +            metadata[key] = ('B', normalize_timestamp(1))

   907 +        key = 'X-Account-Meta-Foo'.format(c)

   908 +        metadata[key] = ('', normalize_timestamp(1))

   909 +        try:

   910 +            DatabaseBroker.validate_metadata(metadata)

   911 +        except HTTPException:

   912 +            self.fail('Unexpected HTTPException')

   913 +

   914 +    def test_metadata_raises_exception_over_max_count(self):

   915 +        metadata = {}

   916 +        for c in xrange(MAX_META_COUNT + 1):

   917 +            key = 'X-Account-Meta-F{0}'.format(c)

   918 +            metadata[key] = ('B', normalize_timestamp(1))

   919 +        message = ''

   920 +        try:

   921 +            DatabaseBroker.validate_metadata(metadata)

   922 +        except HTTPException as e:

   923 +            message = str(e)

   924 +        self.assertEqual(message, '400 Bad Request')

   925 +

   926 +    def test_metadata_with_max_overall_size(self):

   927 +        metadata = {}

   928 +        metadata_value = 'v' * MAX_META_VALUE_LENGTH

   929 +        size = 0

   930 +        x = 0

   931 +        while size < (MAX_META_OVERALL_SIZE - 4

   932 +                      - MAX_META_VALUE_LENGTH):

   933 +            size += 4 + MAX_META_VALUE_LENGTH

   934 +            metadata['X-Account-Meta-%04d' % x] = (metadata_value,

   935 +                                                   normalize_timestamp(1))

   936 +            x += 1

   937 +        if MAX_META_OVERALL_SIZE - size > 1:

   938 +            metadata['X-Account-Meta-k'] = (

   939 +                'v' * (MAX_META_OVERALL_SIZE - size - 1),

   940 +                normalize_timestamp(1))

   941 +        try:

   942 +            DatabaseBroker.validate_metadata(metadata)

   943 +        except HTTPException:

   944 +            self.fail('Unexpected HTTPException')

   945 +

   946 +    def test_metadata_raises_exception_over_max_overall_size(self):

   947 +        metadata = {}

   948 +        metadata_value = 'k' * MAX_META_VALUE_LENGTH

   949 +        size = 0

   950 +        x = 0

   951 +        while size < (MAX_META_OVERALL_SIZE - 4

   952 +                      - MAX_META_VALUE_LENGTH):

   953 +            size += 4 + MAX_META_VALUE_LENGTH

   954 +            metadata['X-Account-Meta-%04d' % x] = (metadata_value,

   955 +                                                   normalize_timestamp(1))

   956 +            x += 1

   957 +        if MAX_META_OVERALL_SIZE - size > 1:

   958 +            metadata['X-Account-Meta-k'] = (

   959 +                'v' * (MAX_META_OVERALL_SIZE - size - 1),

   960 +                normalize_timestamp(1))

   961 +        metadata['X-Account-Meta-k2'] = ('v', normalize_timestamp(1))

   962 +        message = ''

   963 +        try:

   964 +            DatabaseBroker.validate_metadata(metadata)

   965 +        except HTTPException as e:

   966 +            message = str(e)

   967 +        self.assertEqual(message, '400 Bad Request')

   968 +

   969  

   970  if __name__ == '__main__':

   971      unittest.main()