components/openstack/swift/patches/recon.patch
author Drew Fisher <drew.fisher@oracle.com>
Fri, 13 Jun 2014 09:10:23 -0700
branchs11-update
changeset 3178 77584387a894
parent 3135 0a68f5e884e0
child 3998 5bd484384122
permissions -rw-r--r--
PSARC/2014/207 OpenStack Glance Update to Havana PSARC/2014/208 OpenStack Cinder Update to Havana PSARC/2014/209 OpenStack Keystone Update to Havana PSARC/2014/210 OpenStack Nova Update to Havana 18416146 Neutron agents (L3 and DHCP) should cleanup resources when they are disabled 18562372 Failed to create a new project under Horizon 18645763 ZFSSA Cinder Driver support 18686327 evs agent silently ignores user-specified pool allocation ranges 18702697 fibre channel volumes should be supported in the cinder volume driver 18734289 nova won't terminate failed kz deployments 18738371 cinder-volume:setup should account for commented-out zfs_volume_base 18738374 cinder-volume:setup should check for existence of configuration file 18826190 nova-compute fails due to nova.utils.to_bytes 18855698 Update OpenStack to Havana 2013.2.3 18855710 Update python-cinderclient to 1.0.9 18855743 Update python-keystoneclient to 0.8.0 18855754 Update python-neutronclient to 2.3.4 18855764 Update python-novaclient to 2.17.0 18855793 Update python-swiftclient to 2.1.0 18856992 External networks can be deleted even when floating IP addresses are in use 18857784 bake in some more openstack configuration 18884923 Incorrect locale facets in python modules for openstack 18913890 the error in _get_view_and_lun may cause the failure of deleting volumes 18943044 Disable 'Security Groups' tab in Horizon dashboard 18969275 problem in SERVICE/KEYSTONE

Move recon's locks to /var/run/swift and caches to /var/lib/swift/recon-cache.
Change some of the ways we gather information, since those are normally
Linux-specific.  Make the tests acknowledge these changes.

diff --git a/bin/swift-recon-cron b/bin/swift-recon-cron
--- a/bin/swift-recon-cron
+++ b/bin/swift-recon-cron
@@ -49,8 +49,8 @@ def main():
         sys.exit(1)
     conf = dict(c.items('filter:recon'))
     device_dir = conf.get('devices', '/srv/node')
-    recon_cache_path = conf.get('recon_cache_path', '/var/cache/swift')
-    recon_lock_path = conf.get('recon_lock_path', '/var/lock')
+    recon_cache_path = conf.get('recon_cache_path', '/var/lib/swift/recon-cache')
+    recon_lock_path = conf.get('recon_lock_path', '/var/run/swift')
     cache_file = os.path.join(recon_cache_path, "object.recon")
     lock_dir = os.path.join(recon_lock_path, "swift-recon-object-cron")
     conf['log_name'] = conf.get('log_name', 'recon-cron')
diff --git a/etc/account-server.conf-sample b/etc/account-server.conf-sample
--- a/etc/account-server.conf-sample
+++ b/etc/account-server.conf-sample
@@ -78,7 +78,7 @@ use = egg:swift#healthcheck
 
 [filter:recon]
 use = egg:swift#recon
-# recon_cache_path = /var/cache/swift
+# recon_cache_path = /var/lib/swift/recon-cache
 
 [account-replicator]
 # You can override the default log routing for this app here (don't use set!):
@@ -109,7 +109,7 @@ use = egg:swift#recon
 # Time in seconds to wait between replication passes
 # run_pause = 30
 #
-# recon_cache_path = /var/cache/swift
+# recon_cache_path = /var/lib/swift/recon-cache
 
 [account-auditor]
 # You can override the default log routing for this app here (don't use set!):
@@ -124,7 +124,7 @@ use = egg:swift#recon
 # log_facility = LOG_LOCAL0
 # log_level = INFO
 # accounts_per_second = 200
-# recon_cache_path = /var/cache/swift
+# recon_cache_path = /var/lib/swift/recon-cache
 
 [account-reaper]
 # You can override the default log routing for this app here (don't use set!):
diff --git a/etc/container-server.conf-sample b/etc/container-server.conf-sample
--- a/etc/container-server.conf-sample
+++ b/etc/container-server.conf-sample
@@ -85,7 +85,7 @@ use = egg:swift#healthcheck
 
 [filter:recon]
 use = egg:swift#recon
-#recon_cache_path = /var/cache/swift
+#recon_cache_path = /var/lib/swift/recon-cache
 
 [container-replicator]
 # You can override the default log routing for this app here (don't use set!):
@@ -108,7 +108,7 @@ use = egg:swift#recon
 # Time in seconds to wait between replication passes
 # run_pause = 30
 #
-# recon_cache_path = /var/cache/swift
+# recon_cache_path = /var/lib/swift/recon-cache
 
 [container-updater]
 # You can override the default log routing for this app here (don't use set!):
@@ -128,7 +128,7 @@ use = egg:swift#recon
 # Seconds to suppress updating an account that has generated an error
 # account_suppression_time = 60
 #
-# recon_cache_path = /var/cache/swift
+# recon_cache_path = /var/lib/swift/recon-cache
 
 [container-auditor]
 # You can override the default log routing for this app here (don't use set!):
@@ -141,7 +141,7 @@ use = egg:swift#recon
 # interval = 1800
 #
 # containers_per_second = 200
-# recon_cache_path = /var/cache/swift
+# recon_cache_path = /var/lib/swift/recon-cache
 
 [container-sync]
 # You can override the default log routing for this app here (don't use set!):
diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample
--- a/etc/object-server.conf-sample
+++ b/etc/object-server.conf-sample
@@ -99,8 +99,8 @@ use = egg:swift#healthcheck
 
 [filter:recon]
 use = egg:swift#recon
-#recon_cache_path = /var/cache/swift
-#recon_lock_path = /var/lock
+#recon_cache_path = /var/lib/swift/recon-cache
+#recon_lock_path = /var/run/swift
 
 [object-replicator]
 # You can override the default log routing for this app here (don't use set!):
@@ -134,7 +134,7 @@ use = egg:swift#recon
 # reclaim_age = 604800
 #
 # ring_check_interval = 15
-# recon_cache_path = /var/cache/swift
+# recon_cache_path = /var/lib/swift/recon-cache
 #
 # limits how long rsync error log lines are
 # 0 means to log the entire line
@@ -155,7 +155,7 @@ use = egg:swift#recon
 # slowdown will sleep that amount between objects
 # slowdown = 0.01
 #
-# recon_cache_path = /var/cache/swift
+# recon_cache_path = /var/lib/swift/recon-cache
 
 [object-auditor]
 # You can override the default log routing for this app here (don't use set!):
@@ -168,7 +168,7 @@ use = egg:swift#recon
 # bytes_per_second = 10000000
 # log_time = 3600
 # zero_byte_files_per_second = 50
-# recon_cache_path = /var/cache/swift
+# recon_cache_path = /var/lib/swift/recon-cache
 
 # Takes a comma separated list of ints. If set, the object auditor will
 # increment a counter for every object whose size is <= to the given break
diff --git a/swift/account/auditor.py b/swift/account/auditor.py
--- a/swift/account/auditor.py
+++ b/swift/account/auditor.py
@@ -45,7 +45,7 @@ class AccountAuditor(Daemon):
         swift.common.db.DB_PREALLOCATION = \
             config_true_value(conf.get('db_preallocation', 'f'))
         self.recon_cache_path = conf.get('recon_cache_path',
-                                         '/var/cache/swift')
+                                         '/var/lib/swift/recon-cache')
         self.rcache = os.path.join(self.recon_cache_path, "account.recon")
 
     def _one_audit_pass(self, reported):
diff --git a/swift/common/db_replicator.py b/swift/common/db_replicator.py
--- a/swift/common/db_replicator.py
+++ b/swift/common/db_replicator.py
@@ -169,7 +169,7 @@ class Replicator(Daemon):
             config_true_value(conf.get('db_preallocation', 'f'))
         self._zero_stats()
         self.recon_cache_path = conf.get('recon_cache_path',
-                                         '/var/cache/swift')
+                                         '/var/lib/swift/recon-cache')
         self.recon_replicator = '%s.recon' % self.server_type
         self.rcache = os.path.join(self.recon_cache_path,
                                    self.recon_replicator)
diff --git a/swift/common/middleware/recon.py b/swift/common/middleware/recon.py
--- a/swift/common/middleware/recon.py
+++ b/swift/common/middleware/recon.py
@@ -15,6 +15,7 @@
 
 import errno
 import os
+import sys
 from swift import gettext_ as _
 
 from swift import __version__ as swiftver
@@ -45,7 +46,7 @@ class ReconMiddleware(object):
         swift_dir = conf.get('swift_dir', '/etc/swift')
         self.logger = get_logger(conf, log_route='recon')
         self.recon_cache_path = conf.get('recon_cache_path',
-                                         '/var/cache/swift')
+                                         '/var/lib/swift/recon-cache')
         self.object_recon_cache = os.path.join(self.recon_cache_path,
                                                'object.recon')
         self.container_recon_cache = os.path.join(self.recon_cache_path,
@@ -87,28 +88,43 @@ class ReconMiddleware(object):
     def get_mounted(self, openr=open):
         """get ALL mounted fs from /proc/mounts"""
         mounts = []
-        with openr('/proc/mounts', 'r') as procmounts:
+        if sys.platform == 'sunos5':
+            mountsfile = '/etc/mnttab'
+        else:
+            mountsfile = '/proc/mounts'
+        with openr(mountsfile, 'r') as procmounts:
             for line in procmounts:
                 mount = {}
-                mount['device'], mount['path'], opt1, opt2, opt3, \
-                    opt4 = line.rstrip().split()
+                mount['device'], mount['path'] = line.rstrip().split()[:2]
                 mounts.append(mount)
         return mounts
 
     def get_load(self, openr=open):
         """get info from /proc/loadavg"""
         loadavg = {}
-        with openr('/proc/loadavg', 'r') as f:
-            onemin, fivemin, ftmin, tasks, procs = f.read().rstrip().split()
+        if sys.platform == 'sunos5':
+            onemin, fivemin, ftmin = os.getloadavg()
+        else:
+            with openr('/proc/loadavg', 'r') as f:
+                onemin, fivemin, ftmin, tasks, procs = f.read().rstrip().split()
         loadavg['1m'] = float(onemin)
         loadavg['5m'] = float(fivemin)
         loadavg['15m'] = float(ftmin)
-        loadavg['tasks'] = tasks
-        loadavg['processes'] = int(procs)
+        if sys.platform == 'sunos5':
+            # The next two fields don't seem to be used.
+            # executing threads / total threads
+            loadavg['tasks'] = "0/0"
+            # pid of newest process
+            loadavg['processes'] = 0
+        else:
+            loadavg['tasks'] = tasks
+            loadavg['processes'] = int(procs)
         return loadavg
 
     def get_mem(self, openr=open):
         """get info from /proc/meminfo"""
+        if sys.platform == 'sunos5':
+            return {}
         meminfo = {}
         with openr('/proc/meminfo', 'r') as memlines:
             for i in memlines:
diff --git a/swift/container/auditor.py b/swift/container/auditor.py
--- a/swift/container/auditor.py
+++ b/swift/container/auditor.py
@@ -45,7 +45,7 @@ class ContainerAuditor(Daemon):
         swift.common.db.DB_PREALLOCATION = \
             config_true_value(conf.get('db_preallocation', 'f'))
         self.recon_cache_path = conf.get('recon_cache_path',
-                                         '/var/cache/swift')
+                                         '/var/lib/swift/recon-cache')
         self.rcache = os.path.join(self.recon_cache_path, "container.recon")
 
     def _one_audit_pass(self, reported):
diff --git a/swift/container/updater.py b/swift/container/updater.py
--- a/swift/container/updater.py
+++ b/swift/container/updater.py
@@ -60,7 +60,7 @@ class ContainerUpdater(Daemon):
         swift.common.db.DB_PREALLOCATION = \
             config_true_value(conf.get('db_preallocation', 'f'))
         self.recon_cache_path = conf.get('recon_cache_path',
-                                         '/var/cache/swift')
+                                         '/var/lib/swift/recon-cache')
         self.rcache = os.path.join(self.recon_cache_path, "container.recon")
         self.user_agent = 'container-updater %s' % os.getpid()
 
diff --git a/swift/obj/auditor.py b/swift/obj/auditor.py
--- a/swift/obj/auditor.py
+++ b/swift/obj/auditor.py
@@ -56,7 +56,7 @@ class AuditorWorker(object):
         self.quarantines = 0
         self.errors = 0
         self.recon_cache_path = conf.get('recon_cache_path',
-                                         '/var/cache/swift')
+                                         '/var/lib/swift/recon-cache')
         self.rcache = os.path.join(self.recon_cache_path, "object.recon")
         self.stats_sizes = sorted(
             [int(s) for s in list_from_csv(conf.get('object_size_stats'))])
diff --git a/swift/obj/expirer.py b/swift/obj/expirer.py
--- a/swift/obj/expirer.py
+++ b/swift/obj/expirer.py
@@ -54,7 +54,7 @@ class ObjectExpirer(Daemon):
         self.report_first_time = self.report_last_time = time()
         self.report_objects = 0
         self.recon_cache_path = conf.get('recon_cache_path',
-                                         '/var/cache/swift')
+                                         '/var/lib/swift/recon-cache')
         self.rcache = join(self.recon_cache_path, 'object.recon')
         self.concurrency = int(conf.get('concurrency', 1))
         if self.concurrency < 1:
diff --git a/swift/obj/replicator.py b/swift/obj/replicator.py
--- a/swift/obj/replicator.py
+++ b/swift/obj/replicator.py
@@ -76,7 +76,7 @@ class ObjectReplicator(Daemon):
         self.http_timeout = int(conf.get('http_timeout', 60))
         self.lockup_timeout = int(conf.get('lockup_timeout', 1800))
         self.recon_cache_path = conf.get('recon_cache_path',
-                                         '/var/cache/swift')
+                                         '/var/lib/swift/recon-cache')
         self.rcache = os.path.join(self.recon_cache_path, "object.recon")
         self.headers = {
             'Content-Length': '0',
diff --git a/swift/obj/updater.py b/swift/obj/updater.py
--- a/swift/obj/updater.py
+++ b/swift/obj/updater.py
@@ -52,7 +52,7 @@ class ObjectUpdater(Daemon):
         self.successes = 0
         self.failures = 0
         self.recon_cache_path = conf.get('recon_cache_path',
-                                         '/var/cache/swift')
+                                         '/var/lib/swift/recon-cache')
         self.rcache = os.path.join(self.recon_cache_path, 'object.recon')
 
     def get_container_ring(self):
diff --git a/test/unit/common/middleware/test_recon.py b/test/unit/common/middleware/test_recon.py
--- a/test/unit/common/middleware/test_recon.py
+++ b/test/unit/common/middleware/test_recon.py
@@ -18,7 +18,9 @@ from unittest import TestCase
 from contextlib import contextmanager
 from posix import stat_result, statvfs_result
 import os
+import sys
 import mock
+from nose import SkipTest
 
 import swift.common.constraints
 from swift import __version__ as swiftver
@@ -277,10 +279,12 @@ class TestReconSuccess(TestCase):
             {'device': 'none', 'path': '/proc/fs/vmblock/mountPoint'}]
         oart = OpenAndReadTester(mounts_content)
         rv = self.app.get_mounted(openr=oart.open)
-        self.assertEquals(oart.open_calls, [(('/proc/mounts', 'r'), {})])
+        self.assertEquals(oart.open_calls, [(('/etc/mnttab', 'r'), {})])
         self.assertEquals(rv, mounted_resp)
 
     def test_get_load(self):
+        if sys.platform == 'sunos5':
+            raise SkipTest
         oart = OpenAndReadTester(['0.03 0.03 0.00 1/220 16306'])
         rv = self.app.get_load(openr=oart.open)
         self.assertEquals(oart.read_calls, [((), {})])
@@ -290,6 +294,8 @@ class TestReconSuccess(TestCase):
                                '1m': 0.029999999999999999})
 
     def test_get_mem(self):
+        if sys.platform == 'sunos5':
+            raise SkipTest
         meminfo_content = ['MemTotal:         505840 kB',
                            'MemFree:           26588 kB',
                            'Buffers:           44948 kB',
@@ -401,7 +407,7 @@ class TestReconSuccess(TestCase):
         self.assertEquals(self.fakecache.fakeout_calls,
                           [((['replication_time', 'replication_stats',
                               'replication_last'],
-                              '/var/cache/swift/account.recon'), {})])
+                              '/var/lib/swift/recon-cache/account.recon'), {})])
         self.assertEquals(rv, {
             "replication_stats": {
                 "attempted": 1, "diff": 0,
@@ -432,7 +438,7 @@ class TestReconSuccess(TestCase):
         self.assertEquals(self.fakecache.fakeout_calls,
                           [((['replication_time', 'replication_stats',
                               'replication_last'],
-                              '/var/cache/swift/container.recon'), {})])
+                              '/var/lib/swift/recon-cache/container.recon'), {})])
         self.assertEquals(rv, {
             "replication_time": 200.0,
             "replication_stats": {
@@ -454,7 +460,7 @@ class TestReconSuccess(TestCase):
         self.assertEquals(self.fakecache.fakeout_calls,
                           [((['object_replication_time',
                               'object_replication_last'],
-                              '/var/cache/swift/object.recon'), {})])
+                              '/var/lib/swift/recon-cache/object.recon'), {})])
         self.assertEquals(rv, {'object_replication_time': 200.0,
                                'object_replication_last': 1357962809.15})
 
@@ -465,7 +471,7 @@ class TestReconSuccess(TestCase):
         rv = self.app.get_updater_info('container')
         self.assertEquals(self.fakecache.fakeout_calls,
                           [((['container_updater_sweep'],
-                             '/var/cache/swift/container.recon'), {})])
+                             '/var/lib/swift/recon-cache/container.recon'), {})])
         self.assertEquals(rv, {"container_updater_sweep": 18.476239919662476})
 
     def test_get_updater_info_object(self):
@@ -475,7 +481,7 @@ class TestReconSuccess(TestCase):
         rv = self.app.get_updater_info('object')
         self.assertEquals(self.fakecache.fakeout_calls,
                           [((['object_updater_sweep'],
-                             '/var/cache/swift/object.recon'), {})])
+                             '/var/lib/swift/recon-cache/object.recon'), {})])
         self.assertEquals(rv, {"object_updater_sweep": 0.79848217964172363})
 
     def test_get_auditor_info_account(self):
@@ -491,7 +497,7 @@ class TestReconSuccess(TestCase):
                               'account_auditor_pass_completed',
                               'account_audits_since',
                               'account_audits_failed'],
-                              '/var/cache/swift/account.recon'), {})])
+                              '/var/lib/swift/recon-cache/account.recon'), {})])
         self.assertEquals(rv, {"account_auditor_pass_completed": 0.24,
                                "account_audits_failed": 0,
                                "account_audits_passed": 6,
@@ -510,7 +516,7 @@ class TestReconSuccess(TestCase):
                               'container_auditor_pass_completed',
                               'container_audits_since',
                               'container_audits_failed'],
-                              '/var/cache/swift/container.recon'), {})])
+                              '/var/lib/swift/recon-cache/container.recon'), {})])
         self.assertEquals(rv, {"container_auditor_pass_completed": 0.24,
                                "container_audits_failed": 0,
                                "container_audits_passed": 6,
@@ -538,7 +544,7 @@ class TestReconSuccess(TestCase):
         self.assertEquals(self.fakecache.fakeout_calls,
                           [((['object_auditor_stats_ALL',
                               'object_auditor_stats_ZBF'],
-                              '/var/cache/swift/object.recon'), {})])
+                              '/var/lib/swift/recon-cache/object.recon'), {})])
         self.assertEquals(rv, {
             "object_auditor_stats_ALL": {
                 "audit_time": 115.14418768882751,