23480374 network_info is incorrectly checked against None when it can be [] as well
23480293 need a way to recover ports from binding_failed status
--- a/components/openstack/neutron/files/agent/evs_l3_agent.py Thu Jun 02 09:32:04 2016 -0700
+++ b/components/openstack/neutron/files/agent/evs_l3_agent.py Thu Jun 02 10:56:17 2016 -0700
@@ -103,7 +103,8 @@
internal_dlname = self.get_internal_device_name(port['id'])
# driver just returns if datalink and IP interface already exists
self.driver.plug(port['tenant_id'], port['network_id'], port['id'],
- internal_dlname, port['mac_address'])
+ internal_dlname, port['mac_address'],
+ vif_type=port['binding:vif_type'])
fixed_ips = port['fixed_ips']
ip_cidrs = common_utils.fixed_ip_cidrs(fixed_ips)
self.driver.init_l3(internal_dlname, ip_cidrs)
@@ -360,7 +361,8 @@
self.driver.plug(ex_gw_port['tenant_id'], ex_gw_port['network_id'],
ex_gw_port['id'], external_dlname,
ex_gw_port['mac_address'],
- bridge=self.agent_conf.external_network_bridge)
+ bridge=self.agent_conf.external_network_bridge,
+ vif_type=ex_gw_port['binding:vif_type'])
ip_cidrs = common_utils.fixed_ip_cidrs(ex_gw_port['fixed_ips'])
self.driver.init_l3(external_dlname, ip_cidrs)
for fixed_ip in ex_gw_port['fixed_ips']:
--- a/components/openstack/neutron/files/agent/solaris/dhcp.py Thu Jun 02 09:32:04 2016 -0700
+++ b/components/openstack/neutron/files/agent/solaris/dhcp.py Thu Jun 02 10:56:17 2016 -0700
@@ -255,7 +255,8 @@
else:
self.driver.plug(network.tenant_id, network.id,
port.id, interface_name, port.mac_address,
- network=network)
+ network=network,
+ vif_type=port['binding:vif_type'])
ip_cidrs = []
addrconf = False
for fixed_ip in port.fixed_ips:
--- a/components/openstack/neutron/files/agent/solaris/interface.py Thu Jun 02 09:32:04 2016 -0700
+++ b/components/openstack/neutron/files/agent/solaris/interface.py Thu Jun 02 10:56:17 2016 -0700
@@ -235,7 +235,7 @@
def plug(self, tenant_id, network_id, port_id, datalink_name, mac_address,
network=None, bridge=None, namespace=None, prefix=None,
- protection=False):
+ protection=False, vif_type=None):
"""Plug in the interface."""
if net_lib.Datalink.datalink_exists(datalink_name):
@@ -302,6 +302,14 @@
(datalink_name, phys_network))
LOG.error(msg)
raise exceptions.Invalid(message=msg)
+
+ if vif_type == 'binding_failed':
+ msg = (_('Port binding has failed for %s. Ensure that '
+ 'OVS agent is running and/or bridge_mappings are '
+ 'correctly configured. Port will not have network '
+ 'connectivity') % datalink_name)
+ LOG.error(msg)
+
dl = net_lib.Datalink(datalink_name)
dl.create_vnic(lower_link, mac_address, vid, temp=True)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/components/openstack/neutron/patches/08-ovs-binding-failed-fix.patch Thu Jun 02 10:56:17 2016 -0700
@@ -0,0 +1,284 @@
+Patches to recover from ovs port binding_failed error by correcting the
+configuration and/or restarting neutron-openvswitch-agent. Without this patch,
+port will have to be deleted and recreated to recover from the error.
+ https://bugs.launchpad.net/neutron/+bug/1399249
+ https://review.openstack.org/#/c/212854/
+
+*** neutron-2015.1.2/neutron/plugins/ml2/plugin.py Tue Oct 13 10:35:16 2015
+--- new/neutron/plugins/ml2/plugin.py Wed May 25 17:49:35 2016
+***************
+*** 246,313 ****
+
+ def _bind_port_if_needed(self, context, allow_notify=False,
+ need_notify=False):
+! plugin_context = context._plugin_context
+! port_id = context._port['id']
+
+! # Since the mechanism driver bind_port() calls must be made
+! # outside a DB transaction locking the port state, it is
+! # possible (but unlikely) that the port's state could change
+! # concurrently while these calls are being made. If another
+! # thread or process succeeds in binding the port before this
+! # thread commits its results, the already committed results are
+! # used. If attributes such as binding:host_id,
+! # binding:profile, or binding:vnic_type are updated
+! # concurrently, this loop retries binding using the new
+! # values.
+! count = 0
+! while True:
+! # First, determine whether it is necessary and possible to
+! # bind the port.
+! binding = context._binding
+! if (binding.vif_type != portbindings.VIF_TYPE_UNBOUND
+! or not binding.host):
+! # We either don't need to bind the port, or can't, so
+! # notify if needed and return.
+ if allow_notify and need_notify:
+ self._notify_port_updated(context)
+ return context
+
+! # Limit binding attempts to avoid any possibility of
+! # infinite looping and to ensure an error is logged
+! # instead. This does not need to be tunable because no
+! # more than a couple attempts should ever be required in
+! # normal operation. Log at info level if not 1st attempt.
+! count += 1
+! if count > MAX_BIND_TRIES:
+! LOG.error(_LE("Failed to commit binding results for %(port)s "
+! "after %(max)s tries"),
+! {'port': port_id, 'max': MAX_BIND_TRIES})
+! return context
+! if count > 1:
+! greenthread.sleep(0) # yield
+! LOG.info(_LI("Attempt %(count)s to bind port %(port)s"),
+! {'count': count, 'port': port_id})
+
+! # The port isn't already bound and the necessary
+! # information is available, so attempt to bind the port.
+ bind_context = self._bind_port(context)
+
+! # Now try to commit result of attempting to bind the port.
+! new_context, did_commit = self._commit_port_binding(
+! plugin_context, port_id, binding, bind_context)
+! if not new_context:
+! # The port has been deleted concurrently, so just
+! # return the unbound result from the initial
+! # transaction that completed before the deletion.
+! LOG.debug("Port %s has been deleted concurrently",
+! port_id)
+! return context
+! # Need to notify if we succeed and our results were
+! # committed.
+! if did_commit and (new_context._binding.vif_type !=
+! portbindings.VIF_TYPE_BINDING_FAILED):
+ need_notify = True
+! context = new_context
+
+ def _bind_port(self, orig_context):
+ # Construct a new PortContext from the one from the previous
+--- 246,308 ----
+
+ def _bind_port_if_needed(self, context, allow_notify=False,
+ need_notify=False):
+! for count in range(1, MAX_BIND_TRIES + 1):
+! if count > 1:
+! # yield for binding retries so that we give other threads a
+! # chance to do their work
+! greenthread.sleep(0)
+!
+! # multiple attempts shouldn't happen very often so we log each
+! # attempt after the 1st.
+! LOG.info(_LI("Attempt %(count)s to bind port %(port)s"),
+! {'count': count, 'port': context._port['id']})
+!
+! bind_context, need_notify, try_again = self._attempt_binding(
+! context, need_notify)
+!
+! if count == MAX_BIND_TRIES or not try_again:
+! if self._should_bind_port(context):
+! # At this point, we attempted to bind a port and reached
+! # its final binding state. Binding either succeeded or
+! # exhausted all attempts, thus no need to try again.
+! # Now, the port and its binding state should be committed.
+! context, need_notify, try_again = (
+! self._commit_port_binding(context, bind_context,
+! need_notify, try_again))
+! else:
+! context = bind_context
+
+! if not try_again:
+ if allow_notify and need_notify:
+ self._notify_port_updated(context)
+ return context
+
+! LOG.error(_LE("Failed to commit binding results for %(port)s "
+! "after %(max)s tries"),
+! {'port': context._port['id'], 'max': MAX_BIND_TRIES})
+! return context
+!
+! def _should_bind_port(self, context):
+! return (context._binding.host and context._binding.vif_type
+! in (portbindings.VIF_TYPE_UNBOUND,
+! portbindings.VIF_TYPE_BINDING_FAILED))
+!
+! def _attempt_binding(self, context, need_notify):
+! try_again = False
+
+! if self._should_bind_port(context):
+ bind_context = self._bind_port(context)
+
+! if bind_context._binding.vif_type != \
+! portbindings.VIF_TYPE_BINDING_FAILED:
+! # Binding succeeded. Suggest notifying of successful binding.
+ need_notify = True
+! else:
+! # Current attempt binding failed, try to bind again.
+! try_again = True
+! context = bind_context
+!
+! return context, need_notify, try_again
+
+ def _bind_port(self, orig_context):
+ # Construct a new PortContext from the one from the previous
+***************
+*** 331,362 ****
+ self.mechanism_manager.bind_port(new_context)
+ return new_context
+
+! def _commit_port_binding(self, plugin_context, port_id, orig_binding,
+! new_context):
+ session = plugin_context.session
+! new_binding = new_context._binding
+
+ # After we've attempted to bind the port, we begin a
+ # transaction, get the current port state, and decide whether
+ # to commit the binding results.
+! #
+! # REVISIT: Serialize this operation with a semaphore to
+! # prevent deadlock waiting to acquire a DB lock held by
+! # another thread in the same process, leading to 'lock wait
+! # timeout' errors.
+! with contextlib.nested(lockutils.lock('db-access'),
+! session.begin(subtransactions=True)):
+ # Get the current port state and build a new PortContext
+ # reflecting this state as original state for subsequent
+ # mechanism driver update_port_*commit() calls.
+ port_db, cur_binding = db.get_locked_port_and_binding(session,
+ port_id)
+ if not port_db:
+! # The port has been deleted concurrently.
+! return (None, None)
+ oport = self._make_port_dict(port_db)
+ port = self._make_port_dict(port_db)
+! network = new_context.network.current
+ if port['device_owner'] == const.DEVICE_OWNER_DVR_INTERFACE:
+ # REVISIT(rkukura): The PortBinding instance from the
+ # ml2_port_bindings table, returned as cur_binding
+--- 326,366 ----
+ self.mechanism_manager.bind_port(new_context)
+ return new_context
+
+! def _commit_port_binding(self, orig_context, bind_context,
+! need_notify, try_again):
+! port_id = orig_context._port['id']
+! plugin_context = orig_context._plugin_context
+ session = plugin_context.session
+! orig_binding = orig_context._binding
+! new_binding = bind_context._binding
+
+ # After we've attempted to bind the port, we begin a
+ # transaction, get the current port state, and decide whether
+ # to commit the binding results.
+! with session.begin(subtransactions=True):
+ # Get the current port state and build a new PortContext
+ # reflecting this state as original state for subsequent
+ # mechanism driver update_port_*commit() calls.
+ port_db, cur_binding = db.get_locked_port_and_binding(session,
+ port_id)
++ # Since the mechanism driver bind_port() calls must be made
++ # outside a DB transaction locking the port state, it is
++ # possible (but unlikely) that the port's state could change
++ # concurrently while these calls are being made. If another
++ # thread or process succeeds in binding the port before this
++ # thread commits its results, the already committed results are
++ # used. If attributes such as binding:host_id, binding:profile,
++ # or binding:vnic_type are updated concurrently, the try_again
++ # flag is returned to indicate that the commit was unsuccessful.
+ if not port_db:
+! # The port has been deleted concurrently, so just
+! # return the unbound result from the initial
+! # transaction that completed before the deletion.
+! LOG.debug("Port %s has been deleted concurrently", port_id)
+! return orig_context, False, False
+ oport = self._make_port_dict(port_db)
+ port = self._make_port_dict(port_db)
+! network = bind_context.network.current
+ if port['device_owner'] == const.DEVICE_OWNER_DVR_INTERFACE:
+ # REVISIT(rkukura): The PortBinding instance from the
+ # ml2_port_bindings table, returned as cur_binding
+***************
+*** 400,428 ****
+ cur_binding.vif_type = new_binding.vif_type
+ cur_binding.vif_details = new_binding.vif_details
+ db.clear_binding_levels(session, port_id, cur_binding.host)
+! db.set_binding_levels(session, new_context._binding_levels)
+! cur_context._binding_levels = new_context._binding_levels
+
+ # Update PortContext's port dictionary to reflect the
+ # updated binding state.
+ self._update_port_dict_binding(port, cur_binding)
+
+ # Update the port status if requested by the bound driver.
+! if (new_context._binding_levels and
+! new_context._new_port_status):
+! port_db.status = new_context._new_port_status
+! port['status'] = new_context._new_port_status
+
+ # Call the mechanism driver precommit methods, commit
+ # the results, and call the postcommit methods.
+ self.mechanism_manager.update_port_precommit(cur_context)
+ if commit:
+ self.mechanism_manager.update_port_postcommit(cur_context)
+
+! # Continue, using the port state as of the transaction that
+! # just finished, whether that transaction committed new
+! # results or discovered concurrent port state changes.
+! return (cur_context, commit)
+
+ def _update_port_dict_binding(self, port, binding):
+ port[portbindings.HOST_ID] = binding.host
+--- 404,437 ----
+ cur_binding.vif_type = new_binding.vif_type
+ cur_binding.vif_details = new_binding.vif_details
+ db.clear_binding_levels(session, port_id, cur_binding.host)
+! db.set_binding_levels(session, bind_context._binding_levels)
+! cur_context._binding_levels = bind_context._binding_levels
+
+ # Update PortContext's port dictionary to reflect the
+ # updated binding state.
+ self._update_port_dict_binding(port, cur_binding)
+
+ # Update the port status if requested by the bound driver.
+! if (bind_context._binding_levels and
+! bind_context._new_port_status):
+! port_db.status = bind_context._new_port_status
+! port['status'] = bind_context._new_port_status
+
+ # Call the mechanism driver precommit methods, commit
+ # the results, and call the postcommit methods.
+ self.mechanism_manager.update_port_precommit(cur_context)
+ if commit:
++ # Continue, using the port state as of the transaction that
++ # just finished, whether that transaction committed new
++ # results or discovered concurrent port state changes.
++ # Also, Trigger notification for successful binding commit.
+ self.mechanism_manager.update_port_postcommit(cur_context)
++ need_notify = True
++ try_again = False
++ else:
++ try_again = True
+
+! return cur_context, need_notify, try_again
+
+ def _update_port_dict_binding(self, port, binding):
+ port[portbindings.HOST_ID] = binding.host
--- a/components/openstack/nova/files/solariszones/driver.py Thu Jun 02 09:32:04 2016 -0700
+++ b/components/openstack/nova/files/solariszones/driver.py Thu Jun 02 10:56:17 2016 -0700
@@ -1346,7 +1346,7 @@
def _plug_vifs(self, instance, network_info):
# if the VIF is of EVS type (i.e., vif['type'] is ''),
# then nothing to do
- if network_info is None or not network_info[0]['type']:
+ if not network_info or not network_info[0]['type']:
LOG.debug(_("VIF is an EVS type. Nothing to plug."))
return
@@ -1373,9 +1373,11 @@
# we need to map the VNIC to the bridge
bridge = CONF.neutron.ovs_bridge
for vif in network_info:
- if vif['type'] != 'ovs':
- LOG.debug(_("VIF %s is not OVS type") % vif)
- continue
+ if vif['type'] == 'binding_failed':
+ LOG.error(_('Port binding has failed for VIF %s. Ensure that '
+ 'OVS agent is running and/or bridge_mappings are '
+ 'correctly configured. VM will not have network '
+ 'connectivity') % vif)
vif_maddr = ''.join(['%02x' % int(b, 16) for b in
vif['address'].split(':')])
anet = anetdict.get(vif_maddr)