upstream/oracle/userland-gate: comparison components/openstack/swift/patches/recon.patch

equal deleted inserted replaced

-:1ff833d174d4
+:77584387a894
 Linux-specific.  Make the tests acknowledge these changes.
 diff --git a/bin/swift-recon-cron b/bin/swift-recon-cron
 --- a/bin/swift-recon-cron
 +++ b/bin/swift-recon-cron
-@@ -34,8 +34,8 @@ def main():
+@@ -49,8 +49,8 @@ def main():
 sys.exit(1)
 conf = dict(c.items('filter:recon'))
 device_dir = conf.get('devices', '/srv/node')
 -    recon_cache_path = conf.get('recon_cache_path', '/var/cache/swift')
 -    recon_lock_path = conf.get('recon_lock_path', '/var/lock')
 lock_dir = os.path.join(recon_lock_path, "swift-recon-object-cron")
 conf['log_name'] = conf.get('log_name', 'recon-cron')
 diff --git a/etc/account-server.conf-sample b/etc/account-server.conf-sample
 --- a/etc/account-server.conf-sample
 +++ b/etc/account-server.conf-sample
-@@ -56,7 +56,7 @@ use = egg:swift#healthcheck
+@@ -78,7 +78,7 @@ use = egg:swift#healthcheck
 [filter:recon]
 use = egg:swift#recon
 -# recon_cache_path = /var/cache/swift
 +# recon_cache_path = /var/lib/swift/recon-cache
 [account-replicator]
 # You can override the default log routing for this app here (don't use set!):
-@@ -80,7 +80,7 @@ use = egg:swift#recon
+@@ -109,7 +109,7 @@ use = egg:swift#recon
-# reclaim_age = 604800
 # Time in seconds to wait between replication passes
 # run_pause = 30
+#
 -# recon_cache_path = /var/cache/swift
 +# recon_cache_path = /var/lib/swift/recon-cache
 [account-auditor]
 # You can override the default log routing for this app here (don't use set!):
-@@ -93,7 +93,7 @@ use = egg:swift#recon
+@@ -124,7 +124,7 @@ use = egg:swift#recon
 # log_facility = LOG_LOCAL0
 # log_level = INFO
 # accounts_per_second = 200
 -# recon_cache_path = /var/cache/swift
 +# recon_cache_path = /var/lib/swift/recon-cache
 [account-reaper]
 # You can override the default log routing for this app here (don't use set!):
 diff --git a/etc/container-server.conf-sample b/etc/container-server.conf-sample
 --- a/etc/container-server.conf-sample
 +++ b/etc/container-server.conf-sample
-@@ -62,7 +62,7 @@ use = egg:swift#healthcheck
+@@ -85,7 +85,7 @@ use = egg:swift#healthcheck
 [filter:recon]
 use = egg:swift#recon
 -#recon_cache_path = /var/cache/swift
 +#recon_cache_path = /var/lib/swift/recon-cache
 [container-replicator]
 # You can override the default log routing for this app here (don't use set!):
-@@ -81,7 +81,7 @@ use = egg:swift#recon
+@@ -108,7 +108,7 @@ use = egg:swift#recon
-# reclaim_age = 604800
 # Time in seconds to wait between replication passes
 # run_pause = 30
+#
 -# recon_cache_path = /var/cache/swift
 +# recon_cache_path = /var/lib/swift/recon-cache
 [container-updater]
 # You can override the default log routing for this app here (don't use set!):
-@@ -97,7 +97,7 @@ use = egg:swift#recon
+@@ -128,7 +128,7 @@ use = egg:swift#recon
-# slowdown = 0.01
 # Seconds to suppress updating an account that has generated an error
 # account_suppression_time = 60
+#
 -# recon_cache_path = /var/cache/swift
 +# recon_cache_path = /var/lib/swift/recon-cache
 [container-auditor]
 # You can override the default log routing for this app here (don't use set!):
-@@ -108,7 +108,7 @@ use = egg:swift#recon
+@@ -141,7 +141,7 @@ use = egg:swift#recon
-# Will audit each container at most once per interval
 # interval = 1800
+#
 # containers_per_second = 200
 -# recon_cache_path = /var/cache/swift
 +# recon_cache_path = /var/lib/swift/recon-cache
 [container-sync]
 # You can override the default log routing for this app here (don't use set!):
 diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample
 --- a/etc/object-server.conf-sample
 +++ b/etc/object-server.conf-sample
-@@ -71,8 +71,8 @@ use = egg:swift#healthcheck
+@@ -99,8 +99,8 @@ use = egg:swift#healthcheck
 [filter:recon]
 use = egg:swift#recon
 -#recon_cache_path = /var/cache/swift
 -#recon_lock_path = /var/lock
 +#recon_cache_path = /var/lib/swift/recon-cache
 +#recon_lock_path = /var/run/swift
 [object-replicator]
 # You can override the default log routing for this app here (don't use set!):
-@@ -96,7 +96,7 @@ use = egg:swift#recon
+@@ -134,7 +134,7 @@ use = egg:swift#recon
-# The replicator also performs reclamation
 # reclaim_age = 604800
+#
 # ring_check_interval = 15
 -# recon_cache_path = /var/cache/swift
 +# recon_cache_path = /var/lib/swift/recon-cache
+#
-[object-updater]
+# limits how long rsync error log lines are
-# You can override the default log routing for this app here (don't use set!):
+# 0 means to log the entire line
-@@ -110,7 +110,7 @@ use = egg:swift#recon
+@@ -155,7 +155,7 @@ use = egg:swift#recon
-# conn_timeout = 0.5
 # slowdown will sleep that amount between objects
 # slowdown = 0.01
+#
 -# recon_cache_path = /var/cache/swift
 +# recon_cache_path = /var/lib/swift/recon-cache
 [object-auditor]
 # You can override the default log routing for this app here (don't use set!):
-@@ -122,4 +122,4 @@ use = egg:swift#recon
+@@ -168,7 +168,7 @@ use = egg:swift#recon
 # bytes_per_second = 10000000
 # log_time = 3600
 # zero_byte_files_per_second = 50
 -# recon_cache_path = /var/cache/swift
 +# recon_cache_path = /var/lib/swift/recon-cache
+# Takes a comma separated list of ints. If set, the object auditor will
+# increment a counter for every object whose size is <= to the given break
 diff --git a/swift/account/auditor.py b/swift/account/auditor.py
 --- a/swift/account/auditor.py
 +++ b/swift/account/auditor.py
-@@ -44,7 +44,7 @@ class AccountAuditor(Daemon):
+@@ -45,7 +45,7 @@ class AccountAuditor(Daemon):
 swift.common.db.DB_PREALLOCATION = \
 config_true_value(conf.get('db_preallocation', 'f'))
 self.recon_cache_path = conf.get('recon_cache_path',
 -                                         '/var/cache/swift')
 +                                         '/var/lib/swift/recon-cache')
 def _one_audit_pass(self, reported):
 diff --git a/swift/common/db_replicator.py b/swift/common/db_replicator.py
 --- a/swift/common/db_replicator.py
 +++ b/swift/common/db_replicator.py
-@@ -166,7 +166,7 @@ class Replicator(Daemon):
+@@ -169,7 +169,7 @@ class Replicator(Daemon):
 config_true_value(conf.get('db_preallocation', 'f'))
 self._zero_stats()
 self.recon_cache_path = conf.get('recon_cache_path',
 -                                         '/var/cache/swift')
 +                                         '/var/lib/swift/recon-cache')
 @@ -15,6 +15,7 @@
 import errno
 import os
 +import sys
+from swift import gettext_ as _
-from swift.common.swob import Request, Response
-from swift.common.utils import get_logger, config_true_value, json
+from swift import __version__ as swiftver
-@@ -43,7 +44,7 @@ class ReconMiddleware(object):
+@@ -45,7 +46,7 @@ class ReconMiddleware(object):
 swift_dir = conf.get('swift_dir', '/etc/swift')
 self.logger = get_logger(conf, log_route='recon')
 self.recon_cache_path = conf.get('recon_cache_path',
 -                                         '/var/cache/swift')
 +                                         '/var/lib/swift/recon-cache')
 self.object_recon_cache = os.path.join(self.recon_cache_path,
 'object.recon')
 self.container_recon_cache = os.path.join(self.recon_cache_path,
-@@ -80,28 +81,43 @@ class ReconMiddleware(object):
+@@ -87,28 +88,43 @@ class ReconMiddleware(object):
 def get_mounted(self, openr=open):
 """get ALL mounted fs from /proc/mounts"""
 mounts = []
 -        with openr('/proc/mounts', 'r') as procmounts:
 +        if sys.platform == 'sunos5':
 with openr('/proc/meminfo', 'r') as memlines:
 for i in memlines:
 diff --git a/swift/container/auditor.py b/swift/container/auditor.py
 --- a/swift/container/auditor.py
 +++ b/swift/container/auditor.py
-@@ -44,7 +44,7 @@ class ContainerAuditor(Daemon):
+@@ -45,7 +45,7 @@ class ContainerAuditor(Daemon):
 swift.common.db.DB_PREALLOCATION = \
 config_true_value(conf.get('db_preallocation', 'f'))
 self.recon_cache_path = conf.get('recon_cache_path',
 -                                         '/var/cache/swift')
 +                                         '/var/lib/swift/recon-cache')
 def _one_audit_pass(self, reported):
 diff --git a/swift/container/updater.py b/swift/container/updater.py
 --- a/swift/container/updater.py
 +++ b/swift/container/updater.py
-@@ -59,7 +59,7 @@ class ContainerUpdater(Daemon):
+@@ -60,7 +60,7 @@ class ContainerUpdater(Daemon):
 swift.common.db.DB_PREALLOCATION = \
 config_true_value(conf.get('db_preallocation', 'f'))
 self.recon_cache_path = conf.get('recon_cache_path',
 -                                         '/var/cache/swift')
 +                                         '/var/lib/swift/recon-cache')
 self.rcache = os.path.join(self.recon_cache_path, "container.recon")
+self.user_agent = 'container-updater %s' % os.getpid()
-def get_account_ring(self):
 diff --git a/swift/obj/auditor.py b/swift/obj/auditor.py
 --- a/swift/obj/auditor.py
 +++ b/swift/obj/auditor.py
-@@ -54,7 +54,7 @@ class AuditorWorker(object):
+@@ -56,7 +56,7 @@ class AuditorWorker(object):
 self.quarantines = 0
 self.errors = 0
 self.recon_cache_path = conf.get('recon_cache_path',
 -                                         '/var/cache/swift')
 +                                         '/var/lib/swift/recon-cache')
 self.rcache = os.path.join(self.recon_cache_path, "object.recon")
+self.stats_sizes = sorted(
-def audit_all_objects(self, mode='once'):
+[int(s) for s in list_from_csv(conf.get('object_size_stats'))])
 diff --git a/swift/obj/expirer.py b/swift/obj/expirer.py
 --- a/swift/obj/expirer.py
 +++ b/swift/obj/expirer.py
-@@ -50,7 +50,7 @@ class ObjectExpirer(Daemon):
+@@ -54,7 +54,7 @@ class ObjectExpirer(Daemon):
 self.report_first_time = self.report_last_time = time()
 self.report_objects = 0
 self.recon_cache_path = conf.get('recon_cache_path',
 -                                         '/var/cache/swift')
 +                                         '/var/lib/swift/recon-cache')
 self.rcache = join(self.recon_cache_path, 'object.recon')
+self.concurrency = int(conf.get('concurrency', 1))
-def report(self, final=False):
+if self.concurrency < 1:
 diff --git a/swift/obj/replicator.py b/swift/obj/replicator.py
 --- a/swift/obj/replicator.py
 +++ b/swift/obj/replicator.py
-@@ -265,7 +265,7 @@ class ObjectReplicator(Daemon):
+@@ -76,7 +76,7 @@ class ObjectReplicator(Daemon):
 self.http_timeout = int(conf.get('http_timeout', 60))
 self.lockup_timeout = int(conf.get('lockup_timeout', 1800))
 self.recon_cache_path = conf.get('recon_cache_path',
 -                                         '/var/cache/swift')
 +                                         '/var/lib/swift/recon-cache')
 self.rcache = os.path.join(self.recon_cache_path, "object.recon")
+self.headers = {
-def _rsync(self, args):
+'Content-Length': '0',
 diff --git a/swift/obj/updater.py b/swift/obj/updater.py
 --- a/swift/obj/updater.py
 +++ b/swift/obj/updater.py
-@@ -51,7 +51,7 @@ class ObjectUpdater(Daemon):
+@@ -52,7 +52,7 @@ class ObjectUpdater(Daemon):
 self.successes = 0
 self.failures = 0
 self.recon_cache_path = conf.get('recon_cache_path',
 -                                         '/var/cache/swift')
 +                                         '/var/lib/swift/recon-cache')
 def get_container_ring(self):
 diff --git a/test/unit/common/middleware/test_recon.py b/test/unit/common/middleware/test_recon.py
 --- a/test/unit/common/middleware/test_recon.py
 +++ b/test/unit/common/middleware/test_recon.py
-@@ -18,6 +18,8 @@ from unittest import TestCase
+@@ -18,7 +18,9 @@ from unittest import TestCase
 from contextlib import contextmanager
 from posix import stat_result, statvfs_result
 import os
 +import sys
+import mock
 +from nose import SkipTest
 import swift.common.constraints
-from swift.common.swob import Request
+from swift import __version__ as swiftver
-@@ -272,10 +274,12 @@ class TestReconSuccess(TestCase):
+@@ -277,10 +279,12 @@ class TestReconSuccess(TestCase):
 {'device': 'none', 'path': '/proc/fs/vmblock/mountPoint'}]
 oart = OpenAndReadTester(mounts_content)
 rv = self.app.get_mounted(openr=oart.open)
 -        self.assertEquals(oart.open_calls, [(('/proc/mounts', 'r'), {})])
 +        self.assertEquals(oart.open_calls, [(('/etc/mnttab', 'r'), {})])
 self.assertEquals(rv, mounted_resp)
 +        if sys.platform == 'sunos5':
 +            raise SkipTest
 oart = OpenAndReadTester(['0.03 0.03 0.00 1/220 16306'])
 rv = self.app.get_load(openr=oart.open)
 self.assertEquals(oart.read_calls, [((), {})])
-@@ -285,6 +289,8 @@ class TestReconSuccess(TestCase):
+@@ -290,6 +294,8 @@ class TestReconSuccess(TestCase):
 '1m': 0.029999999999999999})
 def test_get_mem(self):
 +        if sys.platform == 'sunos5':
 +            raise SkipTest
 meminfo_content = ['MemTotal:         505840 kB',
 'MemFree:           26588 kB',
 'Buffers:           44948 kB',
-@@ -395,7 +401,7 @@ class TestReconSuccess(TestCase):
+@@ -401,7 +407,7 @@ class TestReconSuccess(TestCase):
 self.assertEquals(self.fakecache.fakeout_calls,
 [((['replication_time', 'replication_stats',
 'replication_last'],
--                                '/var/cache/swift/account.recon'), {})])
+-                              '/var/cache/swift/account.recon'), {})])
-+                                '/var/lib/swift/recon-cache/account.recon'), {})])
++                              '/var/lib/swift/recon-cache/account.recon'), {})])
-self.assertEquals(rv, {"replication_stats": {
+self.assertEquals(rv, {
-"attempted": 1, "diff": 0,
+"replication_stats": {
-"diff_capped": 0, "empty": 0,
+"attempted": 1, "diff": 0,
-@@ -424,7 +430,7 @@ class TestReconSuccess(TestCase):
+@@ -432,7 +438,7 @@ class TestReconSuccess(TestCase):
 self.assertEquals(self.fakecache.fakeout_calls,
 [((['replication_time', 'replication_stats',
 'replication_last'],
--                                '/var/cache/swift/container.recon'), {})])
+-                              '/var/cache/swift/container.recon'), {})])
-+                                '/var/lib/swift/recon-cache/container.recon'), {})])
++                              '/var/lib/swift/recon-cache/container.recon'), {})])
-self.assertEquals(rv, {"replication_time": 200.0,
+self.assertEquals(rv, {
-"replication_stats": {
+"replication_time": 200.0,
-"attempted": 179, "diff": 0,
+"replication_stats": {
-@@ -445,7 +451,7 @@ class TestReconSuccess(TestCase):
+@@ -454,7 +460,7 @@ class TestReconSuccess(TestCase):
 self.assertEquals(self.fakecache.fakeout_calls,
 [((['object_replication_time',
 'object_replication_last'],
--                                '/var/cache/swift/object.recon'), {})])
+-                              '/var/cache/swift/object.recon'), {})])
-+                                '/var/lib/swift/recon-cache/object.recon'), {})])
++                              '/var/lib/swift/recon-cache/object.recon'), {})])
 self.assertEquals(rv, {'object_replication_time': 200.0,
 'object_replication_last': 1357962809.15})
-@@ -456,7 +462,7 @@ class TestReconSuccess(TestCase):
+@@ -465,7 +471,7 @@ class TestReconSuccess(TestCase):
 rv = self.app.get_updater_info('container')
 self.assertEquals(self.fakecache.fakeout_calls,
 [((['container_updater_sweep'],
--                            '/var/cache/swift/container.recon'), {})])
+-                             '/var/cache/swift/container.recon'), {})])
-+                            '/var/lib/swift/recon-cache/container.recon'), {})])
++                             '/var/lib/swift/recon-cache/container.recon'), {})])
 self.assertEquals(rv, {"container_updater_sweep": 18.476239919662476})
 def test_get_updater_info_object(self):
-@@ -466,7 +472,7 @@ class TestReconSuccess(TestCase):
+@@ -475,7 +481,7 @@ class TestReconSuccess(TestCase):
 rv = self.app.get_updater_info('object')
 self.assertEquals(self.fakecache.fakeout_calls,
 [((['object_updater_sweep'],
--                            '/var/cache/swift/object.recon'), {})])
+-                             '/var/cache/swift/object.recon'), {})])
-+                            '/var/lib/swift/recon-cache/object.recon'), {})])
++                             '/var/lib/swift/recon-cache/object.recon'), {})])
 self.assertEquals(rv, {"object_updater_sweep": 0.79848217964172363})
 def test_get_auditor_info_account(self):
-@@ -482,7 +488,7 @@ class TestReconSuccess(TestCase):
+@@ -491,7 +497,7 @@ class TestReconSuccess(TestCase):
 'account_auditor_pass_completed',
 'account_audits_since',
 'account_audits_failed'],
--                                '/var/cache/swift/account.recon'), {})])
+-                              '/var/cache/swift/account.recon'), {})])
-+                                '/var/lib/swift/recon-cache/account.recon'), {})])
++                              '/var/lib/swift/recon-cache/account.recon'), {})])
 self.assertEquals(rv, {"account_auditor_pass_completed": 0.24,
 "account_audits_failed": 0,
 "account_audits_passed": 6,
-@@ -501,7 +507,7 @@ class TestReconSuccess(TestCase):
+@@ -510,7 +516,7 @@ class TestReconSuccess(TestCase):
 'container_auditor_pass_completed',
 'container_audits_since',
 'container_audits_failed'],
--                                '/var/cache/swift/container.recon'), {})])
+-                              '/var/cache/swift/container.recon'), {})])
-+                                '/var/lib/swift/recon-cache/container.recon'), {})])
++                              '/var/lib/swift/recon-cache/container.recon'), {})])
 self.assertEquals(rv, {"container_auditor_pass_completed": 0.24,
 "container_audits_failed": 0,
 "container_audits_passed": 6,
-@@ -528,7 +534,7 @@ class TestReconSuccess(TestCase):
+@@ -538,7 +544,7 @@ class TestReconSuccess(TestCase):
 self.assertEquals(self.fakecache.fakeout_calls,
 [((['object_auditor_stats_ALL',
 'object_auditor_stats_ZBF'],
--                            '/var/cache/swift/object.recon'), {})])
+-                              '/var/cache/swift/object.recon'), {})])
-+                            '/var/lib/swift/recon-cache/object.recon'), {})])
++                              '/var/lib/swift/recon-cache/object.recon'), {})])
-self.assertEquals(rv, {"object_auditor_stats_ALL": {
+self.assertEquals(rv, {
-"audit_time": 115.14418768882751,
+"object_auditor_stats_ALL": {
-"bytes_processed": 234660,
+"audit_time": 115.14418768882751,

branch	s11-update
changeset 3178	77584387a894
parent 3135	0a68f5e884e0
child 3998	5bd484384122