22575858 problem in SERVICE/SWIFT
authordavid.comay@oracle.com
Thu, 11 Feb 2016 16:42:18 -0800
changeset 5445 9b7b887c6955
parent 5444 9f72ce29e7f4
child 5446 27d201e3362b
22575858 problem in SERVICE/SWIFT
components/openstack/swift/patches/CVE-2016-0737.patch
components/openstack/swift/patches/CVE-2016-0738.patch
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/components/openstack/swift/patches/CVE-2016-0737.patch	Thu Feb 11 16:42:18 2016 -0800
@@ -0,0 +1,476 @@
+This upstream patch addresses CVE-2016-0737 which is filed under
+Launchpad bug 1466549. The issue is addressed in Swift 2.3.1 as well as
+2.5.1 or later.
+
+From 036c2f348d24c01c7a4deba3e44889c45270b46d Mon Sep 17 00:00:00 2001
+From: Samuel Merritt <[email protected]>
+Date: Thu, 18 Jun 2015 12:58:03 -0700
+Subject: Get better at closing WSGI iterables.
+
+PEP 333 (WSGI) says: "If the iterable returned by the application has
+a close() method, the server or gateway must call that method upon
+completion of the current request[.]"
+
+There's a bunch of places where we weren't doing that; some of them
+matter more than others. Calling .close() can prevent a connection
+leak in some cases. In others, it just provides a certain pedantic
+smugness. Either way, we should do what WSGI requires.
+
+Noteworthy goofs include:
+
+  * If a client is downloading a large object and disconnects halfway
+    through, a proxy -> obj connection may be leaked. In this case,
+    the WSGI iterable is a SegmentedIterable, which lacked a close()
+    method. Thus, when the WSGI server noticed the client disconnect,
+    it had no way of telling the SegmentedIterable about it, and so
+    the underlying iterable for the segment's data didn't get
+    closed.
+
+    Here, it seems likely (though unproven) that the object server
+    would time out and kill the connection, or that a
+    ChunkWriteTimeout would fire down in the proxy server, so the
+    leaked connection would eventually go away. However, a flurry of
+    client disconnects could leave a big pile of useless connections.
+
+  * If a conditional request receives a 304 or 412, the underlying
+    app_iter is not closed. This mostly affects conditional requests
+    for large objects.
+
+The leaked connections were noticed by this patch's co-author, who
+made the changes to SegmentedIterable. Those changes helped, but did
+not completely fix, the issue. The rest of the patch is an attempt to
+plug the rest of the holes.
+
+Co-Authored-By: Romain LE DISEZ <[email protected]>
+
+Closes-Bug: #1466549
+
+Change-Id: I168e147aae7c1728e7e3fdabb7fba6f2d747d937
+(cherry picked from commit 12d8a53fffea6e4bed8ba3d502ce625f5c6710b9
+with fixed import conflicts)
+---
+ swift/common/middleware/dlo.py          |  8 ++++++--
+ swift/common/middleware/slo.py          | 10 ++++++----
+ swift/common/request_helpers.py         | 35 ++++++++++++---------------------
+ swift/common/swob.py                    |  9 ++++++++-
+ swift/common/utils.py                   | 22 +++++++++++++++++++++
+ swift/proxy/controllers/obj.py          |  4 ++--
+ test/unit/common/middleware/helpers.py  | 32 +++++++++++++++++++++++++++++-
+ test/unit/common/middleware/test_dlo.py | 10 ++++++++--
+ test/unit/common/middleware/test_slo.py | 13 ++++++++----
+ 9 files changed, 105 insertions(+), 38 deletions(-)
+
+diff --git a/swift/common/middleware/dlo.py b/swift/common/middleware/dlo.py
+index d2761ac..9330ccb 100644
+--- a/swift/common/middleware/dlo.py
++++ b/swift/common/middleware/dlo.py
[email protected]@ -22,7 +22,8 @@ from swift.common.http import is_success
+ from swift.common.swob import Request, Response, \
+     HTTPRequestedRangeNotSatisfiable, HTTPBadRequest, HTTPConflict
+ from swift.common.utils import get_logger, json, \
+-    RateLimitedIterator, read_conf_dir, quote
++    RateLimitedIterator, read_conf_dir, quote, close_if_possible, \
++    closing_if_possible
+ from swift.common.request_helpers import SegmentedIterable
+ from swift.common.wsgi import WSGIContext, make_subrequest
+ from urllib import unquote
[email protected]@ -48,7 +49,8 @@ class GetContext(WSGIContext):
+         con_resp = con_req.get_response(self.dlo.app)
+         if not is_success(con_resp.status_int):
+             return con_resp, None
+-        return None, json.loads(''.join(con_resp.app_iter))
++        with closing_if_possible(con_resp.app_iter):
++            return None, json.loads(''.join(con_resp.app_iter))
+ 
+     def _segment_listing_iterator(self, req, version, account, container,
+                                   prefix, segments, first_byte=None,
[email protected]@ -107,6 +109,7 @@ class GetContext(WSGIContext):
+                 # we've already started sending the response body to the
+                 # client, so all we can do is raise an exception to make the
+                 # WSGI server close the connection early
++                close_if_possible(error_response.app_iter)
+                 raise ListingIterError(
+                     "Got status %d listing container /%s/%s" %
+                     (error_response.status_int, account, container))
[email protected]@ -233,6 +236,7 @@ class GetContext(WSGIContext):
+         # make sure this response is for a dynamic large object manifest
+         for header, value in self._response_headers:
+             if (header.lower() == 'x-object-manifest'):
++                close_if_possible(resp_iter)
+                 response = self.get_or_head_response(req, value)
+                 return response(req.environ, start_response)
+         else:
+diff --git a/swift/common/middleware/slo.py b/swift/common/middleware/slo.py
+index e8f1707..c7a97a6 100644
+--- a/swift/common/middleware/slo.py
++++ b/swift/common/middleware/slo.py
[email protected]@ -147,9 +147,9 @@ from swift.common.swob import Request, HTTPBadRequest, HTTPServerError, \
+     Response
+ from swift.common.utils import json, get_logger, config_true_value, \
+     get_valid_utf8_str, override_bytes_from_content_type, split_path, \
+-    register_swift_info, RateLimitedIterator, quote
+-from swift.common.request_helpers import SegmentedIterable, \
+-    closing_if_possible, close_if_possible
++    register_swift_info, RateLimitedIterator, quote, close_if_possible, \
++    closing_if_possible
++from swift.common.request_helpers import SegmentedIterable
+ from swift.common.constraints import check_utf8, MAX_BUFFERED_SLO_SEGMENTS
+ from swift.common.http import HTTP_NOT_FOUND, HTTP_UNAUTHORIZED, is_success
+ from swift.common.wsgi import WSGIContext, make_subrequest
[email protected]@ -227,6 +227,7 @@ class SloGetContext(WSGIContext):
+         sub_resp = sub_req.get_response(self.slo.app)
+ 
+         if not is_success(sub_resp.status_int):
++            close_if_possible(sub_resp.app_iter)
+             raise ListingIterError(
+                 'ERROR: while fetching %s, GET of submanifest %s '
+                 'failed with status %d' % (req.path, sub_req.path,
[email protected]@ -400,7 +401,8 @@ class SloGetContext(WSGIContext):
+         return response(req.environ, start_response)
+ 
+     def get_or_head_response(self, req, resp_headers, resp_iter):
+-        resp_body = ''.join(resp_iter)
++        with closing_if_possible(resp_iter):
++            resp_body = ''.join(resp_iter)
+         try:
+             segments = json.loads(resp_body)
+         except ValueError:
+diff --git a/swift/common/request_helpers.py b/swift/common/request_helpers.py
+index 14b9fd8..8aa8457 100644
+--- a/swift/common/request_helpers.py
++++ b/swift/common/request_helpers.py
[email protected]@ -23,7 +23,6 @@ from swob in here without creating circular imports.
+ import hashlib
+ import itertools
+ import time
+-from contextlib import contextmanager
+ from urllib import unquote
+ from swift import gettext_ as _
+ from swift.common.storage_policy import POLICIES
[email protected]@ -32,7 +31,8 @@ from swift.common.exceptions import ListingIterError, SegmentError
+ from swift.common.http import is_success
+ from swift.common.swob import (HTTPBadRequest, HTTPNotAcceptable,
+                                HTTPServiceUnavailable)
+-from swift.common.utils import split_path, validate_device_partition
++from swift.common.utils import split_path, validate_device_partition, \
++    close_if_possible
+ from swift.common.wsgi import make_subrequest
+ 
+ 
[email protected]@ -249,26 +249,6 @@ def copy_header_subset(from_r, to_r, condition):
+             to_r.headers[k] = v
+ 
+ 
+-def close_if_possible(maybe_closable):
+-    close_method = getattr(maybe_closable, 'close', None)
+-    if callable(close_method):
+-        return close_method()
+-
+-
[email protected]
+-def closing_if_possible(maybe_closable):
+-    """
+-    Like contextlib.closing(), but doesn't crash if the object lacks a close()
+-    method.
+-
+-    PEP 333 (WSGI) says: "If the iterable returned by the application has a
+-    close() method, the server or gateway must call that method upon
+-    completion of the current request[.]" This function makes that easier.
+-    """
+-    yield maybe_closable
+-    close_if_possible(maybe_closable)
+-
+-
+ class SegmentedIterable(object):
+     """
+     Iterable that returns the object contents for a large object.
[email protected]@ -304,6 +284,7 @@ class SegmentedIterable(object):
+         self.peeked_chunk = None
+         self.app_iter = self._internal_iter()
+         self.validated_first_segment = False
++        self.current_resp = None
+ 
+     def _internal_iter(self):
+         start_time = time.time()
[email protected]@ -360,6 +341,8 @@ class SegmentedIterable(object):
+                          'r_size': seg_resp.content_length,
+                          's_etag': seg_etag,
+                          's_size': seg_size})
++                else:
++                    self.current_resp = seg_resp
+ 
+                 seg_hash = hashlib.md5()
+                 for chunk in seg_resp.app_iter:
[email protected]@ -431,3 +414,11 @@ class SegmentedIterable(object):
+             return itertools.chain([pc], self.app_iter)
+         else:
+             return self.app_iter
++
++    def close(self):
++        """
++        Called when the client disconnect. Ensure that the connection to the
++        backend server is closed.
++        """
++        if self.current_resp:
++            close_if_possible(self.current_resp.app_iter)
+diff --git a/swift/common/swob.py b/swift/common/swob.py
+index c2e3afb..a1bd9f6 100644
+--- a/swift/common/swob.py
++++ b/swift/common/swob.py
[email protected]@ -49,7 +49,8 @@ import random
+ import functools
+ import inspect
+ 
+-from swift.common.utils import reiterate, split_path, Timestamp, pairs
++from swift.common.utils import reiterate, split_path, Timestamp, pairs, \
++    close_if_possible
+ from swift.common.exceptions import InvalidTimestamp
+ 
+ 
[email protected]@ -1203,12 +1204,14 @@ class Response(object):
+                     etag in self.request.if_none_match:
+                 self.status = 304
+                 self.content_length = 0
++                close_if_possible(app_iter)
+                 return ['']
+ 
+             if etag and self.request.if_match and \
+                etag not in self.request.if_match:
+                 self.status = 412
+                 self.content_length = 0
++                close_if_possible(app_iter)
+                 return ['']
+ 
+             if self.status_int == 404 and self.request.if_match \
[email protected]@ -1219,18 +1222,21 @@ class Response(object):
+                 # Failed) response. [RFC 2616 section 14.24]
+                 self.status = 412
+                 self.content_length = 0
++                close_if_possible(app_iter)
+                 return ['']
+ 
+             if self.last_modified and self.request.if_modified_since \
+                and self.last_modified <= self.request.if_modified_since:
+                 self.status = 304
+                 self.content_length = 0
++                close_if_possible(app_iter)
+                 return ['']
+ 
+             if self.last_modified and self.request.if_unmodified_since \
+                and self.last_modified > self.request.if_unmodified_since:
+                 self.status = 412
+                 self.content_length = 0
++                close_if_possible(app_iter)
+                 return ['']
+ 
+         if self.request and self.request.method == 'HEAD':
[email protected]@ -1244,6 +1250,7 @@ class Response(object):
+             if ranges == []:
+                 self.status = 416
+                 self.content_length = 0
++                close_if_possible(app_iter)
+                 return ['']
+             elif ranges:
+                 range_size = len(ranges)
+diff --git a/swift/common/utils.py b/swift/common/utils.py
+index 19dcfd3..85c3824 100644
+--- a/swift/common/utils.py
++++ b/swift/common/utils.py
[email protected]@ -3143,6 +3143,28 @@ def ismount_raw(path):
+     return False
+ 
+ 
++def close_if_possible(maybe_closable):
++    close_method = getattr(maybe_closable, 'close', None)
++    if callable(close_method):
++        return close_method()
++
++
[email protected]
++def closing_if_possible(maybe_closable):
++    """
++    Like contextlib.closing(), but doesn't crash if the object lacks a close()
++    method.
++
++    PEP 333 (WSGI) says: "If the iterable returned by the application has a
++    close() method, the server or gateway must call that method upon
++    completion of the current request[.]" This function makes that easier.
++    """
++    try:
++        yield maybe_closable
++    finally:
++        close_if_possible(maybe_closable)
++
++
+ _rfc_token = r'[^()<>@,;:\"/\[\]?={}\x00-\x20\x7f]+'
+ _rfc_extension_pattern = re.compile(
+     r'(?:\s*;\s*(' + _rfc_token + r")\s*(?:=\s*(" + _rfc_token +
+diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py
+index a83242b..784dfef 100644
+--- a/swift/proxy/controllers/obj.py
++++ b/swift/proxy/controllers/obj.py
[email protected]@ -43,7 +43,7 @@ from swift.common.utils import (
+     clean_content_type, config_true_value, ContextPool, csv_append,
+     GreenAsyncPile, GreenthreadSafeIterator, json, Timestamp,
+     normalize_delete_at_timestamp, public, get_expirer_container,
+-    quorum_size)
++    quorum_size, close_if_possible)
+ from swift.common.bufferedhttp import http_connect
+ from swift.common.constraints import check_metadata, check_object_creation, \
+     check_copy_from_header, check_destination_header, \
[email protected]@ -68,7 +68,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPNotFound, \
+     HTTPServerError, HTTPServiceUnavailable, Request, HeaderKeyDict, \
+     HTTPClientDisconnect, HTTPUnprocessableEntity, Response, HTTPException
+ from swift.common.request_helpers import is_sys_or_user_meta, is_sys_meta, \
+-    remove_items, copy_header_subset, close_if_possible
++    remove_items, copy_header_subset
+ 
+ 
+ def copy_headers_into(from_r, to_r):
+diff --git a/test/unit/common/middleware/helpers.py b/test/unit/common/middleware/helpers.py
+index 68a4bfe..7c1b455 100644
+--- a/test/unit/common/middleware/helpers.py
++++ b/test/unit/common/middleware/helpers.py
[email protected]@ -15,6 +15,7 @@
+ 
+ # This stuff can't live in test/unit/__init__.py due to its swob dependency.
+ 
++from collections import defaultdict
+ from copy import deepcopy
+ from hashlib import md5
+ from swift.common import swob
[email protected]@ -23,6 +24,20 @@ from swift.common.utils import split_path
+ from test.unit import FakeLogger, FakeRing
+ 
+ 
++class LeakTrackingIter(object):
++    def __init__(self, inner_iter, fake_swift, path):
++        self.inner_iter = inner_iter
++        self.fake_swift = fake_swift
++        self.path = path
++
++    def __iter__(self):
++        for x in self.inner_iter:
++            yield x
++
++    def close(self):
++        self.fake_swift.mark_closed(self.path)
++
++
+ class FakeSwift(object):
+     """
+     A good-enough fake Swift proxy server to use in testing middleware.
[email protected]@ -30,6 +45,7 @@ class FakeSwift(object):
+ 
+     def __init__(self):
+         self._calls = []
++        self._unclosed_req_paths = defaultdict(int)
+         self.req_method_paths = []
+         self.swift_sources = []
+         self.uploaded = {}
[email protected]@ -105,7 +121,21 @@ class FakeSwift(object):
+         req = swob.Request(env)
+         resp = resp_class(req=req, headers=headers, body=body,
+                           conditional_response=True)
+-        return resp(env, start_response)
++        wsgi_iter = resp(env, start_response)
++        self.mark_opened(path)
++        return LeakTrackingIter(wsgi_iter, self, path)
++
++    def mark_opened(self, path):
++        self._unclosed_req_paths[path] += 1
++
++    def mark_closed(self, path):
++        self._unclosed_req_paths[path] -= 1
++
++    @property
++    def unclosed_requests(self):
++        return {path: count
++                for path, count in self._unclosed_req_paths.items()
++                if count > 0}
+ 
+     @property
+     def calls(self):
+diff --git a/test/unit/common/middleware/test_dlo.py b/test/unit/common/middleware/test_dlo.py
+index 16237eb..119e4ab 100644
+--- a/test/unit/common/middleware/test_dlo.py
++++ b/test/unit/common/middleware/test_dlo.py
[email protected]@ -26,6 +26,7 @@ import unittest
+ 
+ from swift.common import exceptions, swob
+ from swift.common.middleware import dlo
++from swift.common.utils import closing_if_possible
+ from test.unit.common.middleware.helpers import FakeSwift
+ 
+ 
[email protected]@ -54,8 +55,10 @@ class DloTestCase(unittest.TestCase):
+         body = ''
+         caught_exc = None
+         try:
+-            for chunk in body_iter:
+-                body += chunk
++            # appease the close-checker
++            with closing_if_possible(body_iter):
++                for chunk in body_iter:
++                    body += chunk
+         except Exception as exc:
+             if expect_exception:
+                 caught_exc = exc
[email protected]@ -279,6 +282,9 @@ class TestDloHeadManifest(DloTestCase):
+ 
+ 
+ class TestDloGetManifest(DloTestCase):
++    def tearDown(self):
++        self.assertEqual(self.app.unclosed_requests, {})
++
+     def test_get_manifest(self):
+         expected_etag = '"%s"' % md5hex(
+             md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") +
+diff --git a/test/unit/common/middleware/test_slo.py b/test/unit/common/middleware/test_slo.py
+index 4160d91..4d483c8 100644
+--- a/test/unit/common/middleware/test_slo.py
++++ b/test/unit/common/middleware/test_slo.py
[email protected]@ -24,7 +24,7 @@ from swift.common import swob, utils
+ from swift.common.exceptions import ListingIterError, SegmentError
+ from swift.common.middleware import slo
+ from swift.common.swob import Request, Response, HTTPException
+-from swift.common.utils import json
++from swift.common.utils import json, closing_if_possible
+ from test.unit.common.middleware.helpers import FakeSwift
+ 
+ 
[email protected]@ -74,8 +74,10 @@ class SloTestCase(unittest.TestCase):
+         body = ''
+         caught_exc = None
+         try:
+-            for chunk in body_iter:
+-                body += chunk
++            # appease the close-checker
++            with closing_if_possible(body_iter):
++                for chunk in body_iter:
++                    body += chunk
+         except Exception as exc:
+             if expect_exception:
+                 caught_exc = exc
[email protected]@ -222,7 +224,7 @@ class TestSloPutManifest(SloTestCase):
+             '/?multipart-manifest=put',
+             environ={'REQUEST_METHOD': 'PUT'}, body=test_json_data)
+         self.assertEquals(
+-            self.slo.handle_multipart_put(req, fake_start_response),
++            list(self.slo.handle_multipart_put(req, fake_start_response)),
+             ['passed'])
+ 
+     def test_handle_multipart_put_success(self):
[email protected]@ -844,6 +846,9 @@ class TestSloGetManifest(SloTestCase):
+                           'X-Object-Meta-Fish': 'Bass'},
+             "[not {json (at ++++all")
+ 
++    def tearDown(self):
++        self.assertEqual(self.app.unclosed_requests, {})
++
+     def test_get_manifest_passthrough(self):
+         req = Request.blank(
+             '/v1/AUTH_test/gettest/manifest-bc?multipart-manifest=get',
+-- 
+cgit v0.11.2
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/components/openstack/swift/patches/CVE-2016-0738.patch	Thu Feb 11 16:42:18 2016 -0800
@@ -0,0 +1,164 @@
+This upstream patch addresses CVE-2016-0738 which is filed under
+Launchpad bug 1493303. The issue is addressed in Swift 2.3.1 as well as
+2.5.1 or later.
+
+From a4c1825a026655b7ed21d779824ae7c25318fd52 Mon Sep 17 00:00:00 2001
+From: Samuel Merritt <[email protected]>
+Date: Tue, 8 Dec 2015 16:36:05 -0800
+Subject: Fix memory/socket leak in proxy on truncated SLO/DLO GET
+
+When a client disconnected while consuming an SLO or DLO GET response,
+the proxy would leak a socket. This could be observed via strace as a
+socket that had shutdown() called on it, but was never closed. It
+could also be observed by counting entries in /proc/<pid>/fd, where
+<pid> is the pid of a proxy server worker process.
+
+This is due to a memory leak in SegmentedIterable. A SegmentedIterable
+has an 'app_iter' attribute, which is a generator. That generator
+references 'self' (the SegmentedIterable object). This creates a
+cyclic reference: the generator refers to the SegmentedIterable, and
+the SegmentedIterable refers to the generator.
+
+Python can normally handle cyclic garbage; reference counting won't
+reclaim it, but the garbage collector will. However, objects with
+finalizers will stop the garbage collector from collecting them* and
+the cycle of which they are part.
+
+For most objects, "has finalizer" is synonymous with "has a __del__
+method". However, a generator has a finalizer once it's started
+running and before it finishes: basically, while it has stack frames
+associated with it**.
+
+When a client disconnects mid-stream, we get a memory leak. We have
+our SegmentedIterable object (call it "si"), and its associated
+generator. si.app_iter is the generator, and the generator closes over
+si, so we have a cycle; and the generator has started but not yet
+finished, so the generator needs finalization; hence, the garbage
+collector won't ever clean it up.
+
+The socket leak comes in because the generator *also* refers to the
+request's WSGI environment, which contains wsgi.input, which
+ultimately refers to a _socket object from the standard
+library. Python's _socket objects only close their underlying file
+descriptor when their reference counts fall to 0***.
+
+This commit makes SegmentedIterable.close() call
+self.app_iter.close(), thereby unwinding its generator's stack and
+making it eligible for garbage collection.
+
+* in Python < 3.4, at least. See PEP 442.
+
+** see PyGen_NeedsFinalizing() in Objects/genobject.c and also
+   has_finalizer() in Modules/gcmodule.c in Python.
+
+*** see sock_dealloc() in Modules/socketmodule.c in Python. See
+    sock_close() in the same file for the other half of the sad story.
+
+This closes CVE-2016-0738.
+
+Closes-Bug: 1493303
+
+Change-Id: I9b617bfc152dca40d1750131d1d814d85c0a88dd
+Co-Authored-By: Kota Tsuyuzaki <[email protected]>
+---
+ swift/common/request_helpers.py         |  6 ++--
+ test/unit/common/middleware/test_slo.py | 62 +++++++++++++++++++++++++++++++++
+ 2 files changed, 66 insertions(+), 2 deletions(-)
+
+diff --git a/swift/common/request_helpers.py b/swift/common/request_helpers.py
+index 8aa8457..611ee83 100644
+--- a/swift/common/request_helpers.py
++++ b/swift/common/request_helpers.py
[email protected]@ -378,6 +378,9 @@ class SegmentedIterable(object):
+             self.logger.exception(_('ERROR: An error occurred '
+                                     'while retrieving segments'))
+             raise
++        finally:
++            if self.current_resp:
++                close_if_possible(self.current_resp.app_iter)
+ 
+     def app_iter_range(self, *a, **kw):
+         """
[email protected]@ -420,5 +423,4 @@ class SegmentedIterable(object):
+         Called when the client disconnect. Ensure that the connection to the
+         backend server is closed.
+         """
+-        if self.current_resp:
+-            close_if_possible(self.current_resp.app_iter)
++        close_if_possible(self.app_iter)
+diff --git a/test/unit/common/middleware/test_slo.py b/test/unit/common/middleware/test_slo.py
+index 4d483c8..8119b5f 100644
+--- a/test/unit/common/middleware/test_slo.py
++++ b/test/unit/common/middleware/test_slo.py
[email protected]@ -1253,6 +1253,68 @@ class TestSloGetManifest(SloTestCase):
+         self.assertEqual(headers['X-Object-Meta-Fish'], 'Bass')
+         self.assertEqual(body, '')
+ 
++    def test_generator_closure(self):
++        # Test that the SLO WSGI iterable closes its internal .app_iter when
++        # it receives a close() message.
++        #
++        # This is sufficient to fix a memory leak. The memory leak arises
++        # due to cyclic references involving a running generator; a running
++        # generator sometimes preventes the GC from collecting it in the
++        # same way that an object with a defined __del__ does.
++        #
++        # There are other ways to break the cycle and fix the memory leak as
++        # well; calling .close() on the generator is sufficient, but not
++        # necessary. However, having this test is better than nothing for
++        # preventing regressions.
++        leaks = [0]
++
++        class LeakTracker(object):
++            def __init__(self, inner_iter):
++                leaks[0] += 1
++                self.inner_iter = iter(inner_iter)
++
++            def __iter__(self):
++                return self
++
++            def next(self):
++                return next(self.inner_iter)
++
++            def close(self):
++                leaks[0] -= 1
++                self.inner_iter.close()
++
++        class LeakTrackingSegmentedIterable(slo.SegmentedIterable):
++            def _internal_iter(self, *a, **kw):
++                it = super(
++                    LeakTrackingSegmentedIterable, self)._internal_iter(
++                        *a, **kw)
++                return LeakTracker(it)
++
++        status = [None]
++        headers = [None]
++
++        def start_response(s, h, ei=None):
++            status[0] = s
++            headers[0] = h
++
++        req = Request.blank(
++            '/v1/AUTH_test/gettest/manifest-abcd',
++            environ={'REQUEST_METHOD': 'GET',
++                     'HTTP_ACCEPT': 'application/json'})
++
++        # can't self.call_slo() here since we don't want to consume the
++        # whole body
++        with patch.object(slo, 'SegmentedIterable',
++                          LeakTrackingSegmentedIterable):
++            app_resp = self.slo(req.environ, start_response)
++        self.assertEqual(status[0], '200 OK')  # sanity check
++        body_iter = iter(app_resp)
++        chunk = next(body_iter)
++        self.assertEqual(chunk, 'aaaaa')  # sanity check
++
++        app_resp.close()
++        self.assertEqual(0, leaks[0])
++
+     def test_head_manifest_is_efficient(self):
+         req = Request.blank(
+             '/v1/AUTH_test/gettest/manifest-abcd',
+-- 
+cgit v0.11.2
+