--- a/components/openstack/neutron/files/agent/evs_l3_agent.py	Fri Oct 03 10:13:24 2014 -0700
+++ b/components/openstack/neutron/files/agent/evs_l3_agent.py	Fri Oct 03 10:20:17 2014 -0700
@@ -260,81 +260,157 @@
                    gw_ip]
             utils.execute(cmd, check_exit_code=False)
 
+            # for each of the internal ports, add Policy Based
+            # Routing (PBR) rule
+            for port in ri.internal_ports:
+                internal_dlname = self.get_internal_device_name(port['id'])
+                rules = ['pass in on %s to %s:%s from any to any' %
+                         (internal_dlname, external_dlname, gw_ip)]
+                ipversion = netaddr.IPNetwork(port['subnet']['cidr']).version
+                ri.ipfilters_manager.add_ipf_rules(rules, ipversion)
+
     def external_gateway_removed(self, ri, ex_gw_port,
                                  external_dlname, internal_cidrs):
 
-        if net_lib.Datalink.datalink_exists(external_dlname):
-            self.driver.fini_l3(external_dlname)
-            self.driver.unplug(external_dlname)
         gw_ip = ex_gw_port['subnet']['gateway_ip']
         if gw_ip:
+            # remove PBR rules
+            for port in ri.internal_ports:
+                internal_dlname = self.get_internal_device_name(port['id'])
+                rules = ['pass in on %s to %s:%s from any to any' %
+                         (internal_dlname, external_dlname, gw_ip)]
+                ipversion = netaddr.IPNetwork(port['subnet']['cidr']).version
+                ri.ipfilters_manager.remove_ipf_rules(rules, ipversion)
+
             cmd = ['/usr/bin/pfexec', '/usr/sbin/route', 'delete', 'default',
                    gw_ip]
             utils.execute(cmd, check_exit_code=False)
 
-    def _get_ippool_name(self, mac_address):
-        # generate a unique-name for ippool(1m) from that last 3
-        # bytes of mac-address
-        mac_suffix = mac_address.split(':')[3:]
-        return int("".join(mac_suffix), 16)
+        if net_lib.Datalink.datalink_exists(external_dlname):
+            self.driver.fini_l3(external_dlname)
+            self.driver.unplug(external_dlname)
+
+    def _get_ippool_name(self, mac_address, suffix=None):
+        # Generate a unique-name for ippool(1m) from that last 3
+        # bytes of mac-address. It is called pool name, but it is
+        # actually a 32 bit integer
+        name = mac_address.split(':')[3:]
+        if suffix:
+            name.append(suffix)
+        return int("".join(name), 16)
 
     def internal_network_added(self, ri, port):
-
         internal_dlname = self.get_internal_device_name(port['id'])
-        if not net_lib.Datalink.datalink_exists(internal_dlname):
-            self.driver.plug(port['tenant_id'], port['network_id'], port['id'],
-                             internal_dlname)
+        # driver just returns if datalink and IP interface already exists
+        self.driver.plug(port['tenant_id'], port['network_id'], port['id'],
+                         internal_dlname)
         self.driver.init_l3(internal_dlname, [port['ip_cidr']])
 
-        # add ippool(1m) for the new internal port
-        new_ippool_name = self._get_ippool_name(port['mac_address'])
-        ri.ipfilters_manager.add_ippool(new_ippool_name, None)
+        # Since we support shared router model, we need to block the new
+        # internal port from reaching other tenant's ports
+        block_pname = self._get_ippool_name(port['mac_address'])
+        ri.ipfilters_manager.add_ippool(block_pname, None)
+        if self.conf.allow_forwarding_between_networks:
+            # If allow_forwarding_between_networks is set, then we need to
+            # allow forwarding of packets between same tenant's ports.
+            allow_pname = self._get_ippool_name(port['mac_address'], '0')
+            ri.ipfilters_manager.add_ippool(allow_pname, None)
 
         # walk through the other internal ports and retrieve their
         # cidrs and at the same time add the new internal port's
         # cidr to them
-        subnet_cidr = port['subnet']['cidr']
-        other_subnet_cidrs = []
-        for oip in ri.internal_ports:
-            if oip['mac_address'] != port['mac_address']:
-                if (self.conf.allow_forwarding_between_networks and
-                        oip['tenant_id'] == port['tenant_id']):
-                    continue
-                other_subnet_cidrs.append(oip['subnet']['cidr'])
-                ippool_name = self._get_ippool_name(oip['mac_address'])
-                ri.ipfilters_manager.add_ippool(ippool_name, [subnet_cidr])
-        # update the new port's pool with other port's cidrs
-        ri.ipfilters_manager.add_ippool(new_ippool_name, other_subnet_cidrs)
+        port_subnet = port['subnet']['cidr']
+        block_subnets = []
+        allow_subnets = []
+        for internal_port in ri.internal_ports:
+            if internal_port['mac_address'] == port['mac_address']:
+                continue
+            if (self.conf.allow_forwarding_between_networks and
+                    internal_port['tenant_id'] == port['tenant_id']):
+                allow_subnets.append(internal_port['subnet']['cidr'])
+                # we need to add the port's subnet to this internal_port's
+                # allowed_subnet_pool
+                iport_allow_pname = \
+                    self._get_ippool_name(internal_port['mac_address'], '0')
+                ri.ipfilters_manager.add_ippool(iport_allow_pname,
+                                                [port_subnet])
+            else:
+                block_subnets.append(internal_port['subnet']['cidr'])
+                iport_block_pname = \
+                    self._get_ippool_name(internal_port['mac_address'])
+                ri.ipfilters_manager.add_ippool(iport_block_pname,
+                                                [port_subnet])
+        # update the new port's pool with other ports' subnet
+        ri.ipfilters_manager.add_ippool(block_pname, block_subnets)
+        if self.conf.allow_forwarding_between_networks:
+            ri.ipfilters_manager.add_ippool(allow_pname, allow_subnets)
 
-        # now setup the IPF rule
+        # now setup the IPF rules
         rules = ['block in quick on %s from %s to pool/%d' %
-                 (internal_dlname, subnet_cidr, new_ippool_name)]
-        ipversion = netaddr.IPNetwork(subnet_cidr).version
+                 (internal_dlname, port_subnet, block_pname)]
+        # pass in packets between networks that belong to same tenant
+        if self.conf.allow_forwarding_between_networks:
+            rules.append('pass in quick on %s from %s to pool/%d' %
+                         (internal_dlname, port_subnet, allow_pname))
+        # if the external gateway is already setup for the shared router,
+        # then we need to add Policy Based Routing (PBR) for this internal
+        # network
+        ex_gw_port = ri.ex_gw_port
+        ex_gw_ip = (ex_gw_port['subnet']['gateway_ip'] if ex_gw_port else None)
+        if ex_gw_ip:
+            external_dlname = self.get_external_device_name(ex_gw_port['id'])
+            rules.append('pass in on %s to %s:%s from any to any' %
+                         (internal_dlname, external_dlname, ex_gw_ip))
+
+        ipversion = netaddr.IPNetwork(port_subnet).version
         ri.ipfilters_manager.add_ipf_rules(rules, ipversion)
 
     def internal_network_removed(self, ri, port):
         internal_dlname = self.get_internal_device_name(port['id'])
+        port_subnet = port['subnet']['cidr']
+        # remove all the IP filter rules that we added during
+        # internal network addition
+        block_pname = self._get_ippool_name(port['mac_address'])
+        rules = ['block in quick on %s from %s to pool/%d' %
+                 (internal_dlname, port_subnet, block_pname)]
+        if self.conf.allow_forwarding_between_networks:
+            allow_pname = self._get_ippool_name(port['mac_address'], '0')
+            rules.append('pass in quick on %s from %s to pool/%d' %
+                         (internal_dlname, port_subnet, allow_pname))
+
+        # remove all the IP filter rules that we added during
+        # external network addition
+        ex_gw_port = ri.ex_gw_port
+        ex_gw_ip = (ex_gw_port['subnet']['gateway_ip'] if ex_gw_port else None)
+        if ex_gw_ip:
+            external_dlname = self.get_external_device_name(ex_gw_port['id'])
+            rules.append('pass in on %s to %s:%s from any to any' %
+                         (internal_dlname, external_dlname, ex_gw_ip))
+        ipversion = netaddr.IPNetwork(port['subnet']['cidr']).version
+        ri.ipfilters_manager.remove_ipf_rules(rules, ipversion)
+
+        # remove the ippool
+        ri.ipfilters_manager.remove_ippool(block_pname, None)
+        if self.conf.allow_forwarding_between_networks:
+            ri.ipfilters_manager.remove_ippool(allow_pname, None)
+
+        for internal_port in ri.internal_ports:
+            if (self.conf.allow_forwarding_between_networks and
+                    internal_port['tenant_id'] == port['tenant_id']):
+                iport_allow_pname = \
+                    self._get_ippool_name(internal_port['mac_address'], '0')
+                ri.ipfilters_manager.remove_ippool(iport_allow_pname,
+                                                   [port_subnet])
+            else:
+                iport_block_pname = \
+                    self._get_ippool_name(internal_port['mac_address'])
+                ri.ipfilters_manager.remove_ippool(iport_block_pname,
+                                                   [port_subnet])
+
         if net_lib.Datalink.datalink_exists(internal_dlname):
             self.driver.fini_l3(internal_dlname)
             self.driver.unplug(internal_dlname)
 
-        # remove all the IP filter rules that we added during addition.
-        ippool_name = self._get_ippool_name(port['mac_address'])
-        rules = ['block in quick on %s from %s to pool/%d' %
-                 (internal_dlname, port['subnet']['cidr'], ippool_name)]
-        ipversion = netaddr.IPNetwork(port['subnet']['cidr']).version
-        ri.ipfilters_manager.remove_ipf_rules(rules, ipversion)
-        # remove the ippool
-        ri.ipfilters_manager.remove_ippool(ippool_name, None)
-        for internal_port in ri.internal_ports:
-            if (self.conf.allow_forwarding_between_networks and
-                    internal_port['tenant_id'] == port['tenant_id']):
-                continue
-            ippool_name = \
-                self._get_ippool_name(internal_port['mac_address'])
-            subnet_cidr = internal_port['subnet']['cidr']
-            ri.ipfilters_manager.remove_ippool(ippool_name, [subnet_cidr])
-
     def routers_updated(self, context, routers):
         super(EVSL3NATAgent, self).routers_updated(context, routers)
         if routers:

--- a/components/openstack/neutron/files/agent/solaris/interface.py	Fri Oct 03 10:13:24 2014 -0700
+++ b/components/openstack/neutron/files/agent/solaris/interface.py	Fri Oct 03 10:20:17 2014 -0700
@@ -83,13 +83,18 @@
         return vnicname.replace('-', '_')
 
     def plug(self, tenant_id, network_id, port_id, datalink_name,
-             namespace=None, prefix=None):
+             namespace=None, prefix=None, protection=False):
         """Plug in the interface."""
 
         evs_vport = ('%s/%s') % (network_id, port_id)
         dl = net_lib.Datalink(datalink_name)
         dl.connect_vnic(evs_vport, tenant_id)
 
+        if not protection:
+            cmd = ['/usr/sbin/evsadm', 'set-vportprop', '-T', tenant_id,
+                   '-p', 'protection=none', evs_vport]
+            utils.execute(cmd)
+
     def unplug(self, device_name, namespace=None, prefix=None):
         """Unplug the interface."""
 

--- a/components/openstack/neutron/files/agent/solaris/ipfilters_manager.py	Fri Oct 03 10:13:24 2014 -0700
+++ b/components/openstack/neutron/files/agent/solaris/ipfilters_manager.py	Fri Oct 03 10:20:17 2014 -0700
@@ -44,36 +44,32 @@
         else:
             ippool.remove_pool()
 
-    def add_nat_rules(self, rules, version='4'):
-        # Solaris doesn't support IPv6 NAT rules
-        assert version == '4'
+    def add_nat_rules(self, rules):
         ipnat = net_lib.IPnatCommand()
         ipnat.add_rules(rules)
         # we successfully added the nat rules, update the local copy
         for rule in rules:
             self.ipv4['nat'].append(rule)
 
-    def remove_nat_rules(self, rules, version='4'):
-        # Solaris doesn't support IPv6 NAT rules
-        assert version == '4'
+    def remove_nat_rules(self, rules):
         ipnat = net_lib.IPnatCommand()
         ipnat.remove_rules(rules)
         # we successfully removed the nat rules, update the local copy
         for rule in rules:
             self.ipv4['nat'].remove(rule)
 
-    def add_ipf_rules(self, rules, version='4'):
+    def add_ipf_rules(self, rules, version=4):
         ipf = net_lib.IPfilterCommand()
         ipf.add_rules(rules, version)
-        version_rules = (self.ipv4['filter'] if version == '4' else
+        version_rules = (self.ipv4['filter'] if version == 4 else
                          self.ipv6['filter'])
         for rule in rules:
             version_rules.append(rule)
 
-    def remove_ipf_rules(self, rules, version='4'):
+    def remove_ipf_rules(self, rules, version=4):
         ipf = net_lib.IPfilterCommand()
         ipf.remove_rules(rules, version)
-        version_rules = (self.ipv4['filter'] if version == '4' else
+        version_rules = (self.ipv4['filter'] if version == 4 else
                          self.ipv6['filter'])
         for rule in rules:
             version_rules.remove(rule)

--- a/components/openstack/neutron/files/agent/solaris/net_lib.py	Fri Oct 03 10:13:24 2014 -0700
+++ b/components/openstack/neutron/files/agent/solaris/net_lib.py	Fri Oct 03 10:20:17 2014 -0700
@@ -109,7 +109,7 @@
         if temp:
             cmd.append('-t')
 
-        return self.execute_with_pfexec(cmd)
+        self.execute_with_pfexec(cmd)
 
     def delete_address(self, ipaddr):
         if not self.ipaddr_exists(self._ifname, ipaddr):
@@ -166,7 +166,7 @@
             cmd.append('-T')
             cmd.append(tenantname)
 
-        return self.execute_with_pfexec(cmd)
+        self.execute_with_pfexec(cmd)
 
     def create_vnic(self, lower_link, mac_address=None, vid=None, temp=True):
         if self.datalink_exists(self._dlname):
@@ -187,14 +187,14 @@
         if temp:
             cmd.append('-t')
 
-        return self.execute_with_pfexec(cmd)
+        self.execute_with_pfexec(cmd)
 
     def delete_vnic(self):
         if not self.datalink_exists(self._dlname):
             return
 
         cmd = ['/usr/sbin/dladm', 'delete-vnic', self._dlname]
-        return self.execute_with_pfexec(cmd)
+        self.execute_with_pfexec(cmd)
 
 
 class IPpoolCommand(CommandBase):
@@ -280,30 +280,38 @@
 
     def add_rules(self, rules, version=4):
         rules = self._split_rules(rules, version)[1]
-        process_input = '\n'.join(rules)
+        if not rules:
+            return
+        process_input = '\n'.join(rules) + '\n'
         cmd = ['/usr/sbin/ipf', '-f', '-']
         if version == 6:
             cmd.insert(1, '-6')
-        return self.execute_with_pfexec(cmd, process_input=process_input)
+        self.execute_with_pfexec(cmd, process_input=process_input)
 
     def remove_rules(self, rules, version=4):
         rules = self._split_rules(rules, version)[0]
-        process_input = '\n'.join(rules)
+        if not rules:
+            return
+        process_input = '\n'.join(rules) + '\n'
         cmd = ['/usr/sbin/ipf', '-r', '-f', '-']
         if version == 6:
             cmd.insert(1, '-6')
-        return self.execute_with_pfexec(cmd, process_input=process_input)
+        self.execute_with_pfexec(cmd, process_input=process_input)
 
 
 class IPnatCommand(CommandBase):
     '''Wrapper around Solaris ipnat(1m) command'''
 
     def add_rules(self, rules):
-        process_input = '\n'.join(rules)
+        if not rules:
+            return
+        process_input = '\n'.join(rules) + '\n'
         cmd = ['/usr/sbin/ipnat', '-f', '-']
-        return self.execute_with_pfexec(cmd, process_input=process_input)
+        self.execute_with_pfexec(cmd, process_input=process_input)
 
     def remove_rules(self, rules):
-        process_input = '\n'.join(rules)
+        if not rules:
+            return
+        process_input = '\n'.join(rules) + '\n'
         cmd = ['/usr/sbin/ipnat', '-r', '-f', '-']
-        return self.execute_with_pfexec(cmd, process_input=process_input)
+        self.execute_with_pfexec(cmd, process_input=process_input)

--- a/components/openstack/neutron/files/neutron-l3-agent	Fri Oct 03 10:13:24 2014 -0700
+++ b/components/openstack/neutron/files/neutron-l3-agent	Fri Oct 03 10:20:17 2014 -0700
@@ -81,7 +81,8 @@
         if not prog.search(ipf):
             continue
         # capture the IP pool name
-        ippool_names.append(ipf.split('pool/')[1])
+        if 'pool/' in ipf:
+            ippool_names.append(ipf.split('pool/')[1])
 
         try:
             # remove the IP Filter rule
@@ -112,32 +113,9 @@
     except CalledProcessError as err:
         print "failed to kill the SMF contract: %s" % (err)
         return smf_include.SMF_EXIT_ERR_FATAL
-    # remove VNICs associated with L3 agent
-    cmd = ["/usr/sbin/ipadm", "show-if", "-p", "-o", "ifname"]
-    p = Popen(cmd, stdout=PIPE, stderr=PIPE)
-    output, error = p.communicate()
-    if p.returncode != 0:
-        print "failed to retrieve IP interface names"
-        return smf_include.SMF_EXIT_ERR_CONFIG
 
-    ifnames = output.splitlines()
-    # L3 agent datalinks are always 15 characters in length. They start
-    # with either 'l3i' or 'l3e', end with '_0', and in between they are
-    # hexadecimal digits.
-    prog = re.compile('l3[ie][0-9A-Fa-f\_]{10}_0')
-    for ifname in ifnames:
-        if not prog.search(ifname):
-            continue
-        try:
-            # first remove the IP
-            check_call(["/usr/bin/pfexec", "/usr/sbin/ipadm", "delete-ip",
-                        ifname])
-            # next remove the VNIC
-            check_call(["/usr/bin/pfexec", "/usr/sbin/dladm", "delete-vnic",
-                        ifname])
-        except CalledProcessError as err:
-            print "failed to remove datalinks used by L3 agent: %s" % (err)
-            return smf_include.SMF_EXIT_ERR_FATAL
+    # We need to first remove the IP filter rules and then remove
+    # the IP interfaces on which the rules were applied.
 
     # remove IPv4 Filter rules added by neutron-l3-agent
     rv = remove_ipfilter_rules(4)
@@ -173,6 +151,33 @@
             print "failed to remove IP NAT rule %s: %s" % (ipnat_rule, err)
             return smf_include.SMF_EXIT_ERR_FATAL
 
+    # remove VNICs associated with L3 agent
+    cmd = ["/usr/sbin/ipadm", "show-if", "-p", "-o", "ifname"]
+    p = Popen(cmd, stdout=PIPE, stderr=PIPE)
+    output, error = p.communicate()
+    if p.returncode != 0:
+        print "failed to retrieve IP interface names"
+        return smf_include.SMF_EXIT_ERR_CONFIG
+
+    ifnames = output.splitlines()
+    # L3 agent datalinks are always 15 characters in length. They start
+    # with either 'l3i' or 'l3e', end with '_0', and in between they are
+    # hexadecimal digits.
+    prog = re.compile('l3[ie][0-9A-Fa-f\_]{10}_0')
+    for ifname in ifnames:
+        if not prog.search(ifname):
+            continue
+        try:
+            # first remove the IP
+            check_call(["/usr/bin/pfexec", "/usr/sbin/ipadm", "delete-ip",
+                        ifname])
+            # next remove the VNIC
+            check_call(["/usr/bin/pfexec", "/usr/sbin/dladm", "delete-vnic",
+                        ifname])
+        except CalledProcessError as err:
+            print "failed to remove datalinks used by L3 agent: %s" % (err)
+            return smf_include.SMF_EXIT_ERR_FATAL
+
     return smf_include.SMF_EXIT_OK
 
 if __name__ == "__main__":