This patchset is for bug:

22726251 - Neutron needs to support MySQL Cluster

This fixes the following aspects of Neutron:

1. Implementation of an oslo.db configuration parameter to specify the MySQL

   storage engine (mysql_storage_engine).

2. Replacement of hardcoded SQL statements that set the engine to "InnoDB"

   to the above configuration value.

3. Logic to handle SQL differences between MySQL InnoDB and MySQL Cluster (NDB).

   This includes column lengths, constraints, foreign keys, and indexes.

This has not been committed upstream, but has been filed in launchpad:

https://bugs.launchpad.net/neutron/+bug/1564110

--- neutron-2015.1.2/neutron/db/model_base.py.orig	2016-07-29 13:15:07.909806102 -0600

+++ neutron-2015.1.2/neutron/db/model_base.py	2016-07-29 13:27:41.817140969 -0600

@@ -13,15 +13,24 @@

 # See the License for the specific language governing permissions and

 # limitations under the License.

+from oslo_config import cfg

 from oslo_db.sqlalchemy import models

 from sqlalchemy.ext import declarative

 from sqlalchemy import orm

+# Attempt to determine the context that this module is being used.

+# If via neutron-db-manage and cli.py, import global variable. If not,

+# use oslo_config.

+try:

+     from neutron.db.migration.cli import mysql_storage_eng_type

+except ImportError:

+     CONF = cfg.CONF

+     mysql_storage_eng_type = CONF.database.mysql_storage_engine

 class NeutronBase(models.ModelBase):

     """Base class for Neutron Models."""

-    __table_args__ = {'mysql_engine': 'InnoDB'}

+    __table_args__ = {'mysql_engine': mysql_storage_eng_type}

     def __iter__(self):

         self._i = iter(orm.object_mapper(self).columns)

--- neutron-2015.1.2/neutron/db/api.py.orig	2016-07-29 13:15:14.380329905 -0600

+++ neutron-2015.1.2/neutron/db/api.py	2016-07-28 14:15:31.186601900 -0600

@@ -19,7 +19,6 @@ from oslo_config import cfg

 from oslo_db.sqlalchemy import session

 from sqlalchemy import exc

-

 _FACADE = None

 MAX_RETRIES = 10

@@ -50,10 +49,30 @@ def get_session(autocommit=True, expire_

 @contextlib.contextmanager

 def autonested_transaction(sess):

     """This is a convenience method to not bother with 'nested' parameter."""

+    # Attempt to determine the context that this module is being used.

+    # If via neutron-db-manage and cli.py, import global variable. If not,

+    # use oslo_config.

     try:

-        session_context = sess.begin_nested()

-    except exc.InvalidRequestError:

-        session_context = sess.begin(subtransactions=True)

-    finally:

-        with session_context as tx:

-            yield tx

+        from neutron.db.migration.cli import mysql_storage_eng_type

+    except ImportError:

+        CONF = cfg.CONF

+        mysql_storage_eng_type = CONF.database.mysql_storage_engine

+

+    # MySQL Cluster NDB does not support nested transactions

+    # TODO (oorgeron) Look into making this workable.

+    if mysql_storage_eng_type == "NDBCLUSTER":

+        try:

+            session_context = sess.begin(subtransactions=True, nested=False)

+        except exc.InvalidRequestError:

+            session_context = sess.begin(subtransactions=False, nested=False)

+        finally:

+            with session_context as tx:

+                yield tx

+    else:

+        try:

+            session_context = sess.begin_nested()

+        except exc.InvalidRequestError:

+            session_context = sess.begin(subtransactions=True)

+        finally:

+            with session_context as tx:

+                yield tx

--- neutron-2015.1.2/neutron/db/db_base_plugin_v2.py.orig	2016-07-29 13:15:20.388338380 -0600

+++ neutron-2015.1.2/neutron/db/db_base_plugin_v2.py	2016-07-28 14:15:31.187095505 -0600

@@ -46,6 +46,15 @@ from neutron.plugins.common import const

 LOG = logging.getLogger(__name__)

+# Attempt to determine the context that this module is being used.

+# If via neutron-db-manage and cli.py, import global variable. If not,

+# use oslo_config.

+try:

+     from neutron.db.migration.cli import mysql_storage_eng_type

+except ImportError:

+     CONF = cfg.CONF

+     mysql_storage_eng_type = CONF.database.mysql_storage_engine

+

 # Ports with the following 'device_owner' values will not prevent

 # network deletion.  If delete_network() finds that all ports on a

 # network have these owners, it will explicitly delete each port

@@ -1443,16 +1452,26 @@ class NeutronDbPluginV2(neutron_plugin_b

                                                    port_id=port['id'],

                                                    ip_address=ip_address,

                                                    subnet_id=subnet['id'])

-                try:

-                    # Do the insertion of each IP allocation entry within

-                    # the context of a nested transaction, so that the entry

-                    # is rolled back independently of other entries whenever

-                    # the corresponding port has been deleted.

-                    with context.session.begin_nested():

-                        context.session.add(allocated)

-                except db_exc.DBReferenceError:

-                    LOG.debug("Port %s was deleted while updating it with an "

-                              "IPv6 auto-address. Ignoring.", port['id'])

+                # MySQL Cluster NDB does not support nested transactions

+                # TODO (oorgeron) Look into making this workable.

+                if mysql_storage_eng_type == "NDBCLUSTER":

+                    try:

+                        with context.session.begin(subtransactions=True):

+                            context.session.add(allocated)

+                    except db_exc.DBReferenceError:

+                        LOG.debug("Port %s was deleted while updating it with an "

+                                  "IPv6 auto-address. Ignoring.", port['id'])

+                else:

+                    try:

+                        # Do the insertion of each IP allocation entry within

+                        # the context of a nested transaction, so that the entry

+                        # is rolled back independently of other entries whenever

+                        # the corresponding port has been deleted.

+                        with context.session.begin_nested():

+                            context.session.add(allocated)

+                    except db_exc.DBReferenceError:

+                        LOG.debug("Port %s was deleted while updating it with an "

+                                  "IPv6 auto-address. Ignoring.", port['id'])

     def _update_subnet_dns_nameservers(self, context, id, s):

         old_dns_list = self._get_dns_by_subnet(context, id)

@@ -1797,8 +1816,16 @@ class NeutronDbPluginV2(neutron_plugin_b

                 # within a transaction, so that it can be rolled back to the

                 # point before its failure while maintaining the enclosing

                 # transaction

-                return self._create_port_with_mac(

-                    context, network_id, port_data, mac, nested=True)

+

+                # MySQL Cluster NDB does not support nested transactions

+                # TODO (oorgeron) Look into making this workable.

+                if mysql_storage_eng_type == "NDBCLUSTER":

+                    return self._create_port_with_mac(

+                        context, network_id, port_data, mac, nested=False)

+                else:

+                    return self._create_port_with_mac(

+                        context, network_id, port_data, mac, nested=True)

+

             except n_exc.MacAddressInUse:

                 LOG.debug('Generated mac %(mac_address)s exists on '

                           'network %(network_id)s',

--- neutron-2015.1.2/neutron/db/migration/cli.py.orig	2016-07-29 13:15:27.041205822 -0600

+++ neutron-2015.1.2/neutron/db/migration/cli.py	2016-07-28 14:15:31.187312338 -0600

@@ -59,6 +59,9 @@ _db_opts = [

     cfg.StrOpt('engine',

                default='',

                help=_('Database engine')),

+    cfg.StrOpt('mysql_storage_engine',

+               default='',

+               help=_('MySQL Storage Engine')),

 ]

 CONF = cfg.ConfigOpts()

@@ -234,5 +237,11 @@ def main():

     config = get_alembic_config()

     config.neutron_config = CONF

+    # Make global variable available for modules that will be called

+    # by neutron-db-manage. This will prevent namespace collisions

+    # between oslo_config and albemic_config

+    global mysql_storage_eng_type

+    mysql_storage_eng_type = CONF.database.mysql_storage_engine

+

     #TODO(gongysh) enable logging

     CONF.command.func(config, CONF.command.name)

--- neutron-2015.1.2/neutron/db/migration/models/frozen.py.orig	2016-07-29 13:15:33.940472370 -0600

+++ neutron-2015.1.2/neutron/db/migration/models/frozen.py	2016-07-29 13:29:33.819709706 -0600

@@ -22,7 +22,8 @@ Based on this comparison database can be

 Current HEAD commit is 59da928e945ec58836d34fd561d30a8a446e2728

 """

-

+from alembic import context

+from oslo_config import cfg

 import sqlalchemy as sa

 from sqlalchemy.ext import declarative

 from sqlalchemy.ext.orderinglist import ordering_list

@@ -32,6 +33,8 @@ from sqlalchemy import schema

 from neutron.db import model_base

 from neutron.openstack.common import uuidutils

+config = context.config

+CONF = config.neutron_config

 # Dictionary of all tables that was renamed:

 # {new_table_name: old_table_name}

@@ -77,6 +80,10 @@ DHCPV6_STATELESS = 'dhcpv6-stateless'

 BASEV2 = declarative.declarative_base(cls=model_base.NeutronBaseV2)

+if CONF.database.mysql_storage_engine == "NDBCLUSTER":

+   db_string_length = 128

+else:

+   db_string_length = 256

 #neutron/db/models_v2.py

 class HasTenant(object):

@@ -218,17 +225,30 @@ class Agent(BASEV2, HasId):

                             name='uniq_agents0agent_type0host'),

     )

-    agent_type = sa.Column(sa.String(255), nullable=False)

-    binary = sa.Column(sa.String(255), nullable=False)

-    topic = sa.Column(sa.String(255), nullable=False)

-    host = sa.Column(sa.String(255), nullable=False)

-    admin_state_up = sa.Column(sa.Boolean, default=True,

-                               server_default=sa.sql.true(), nullable=False)

-    created_at = sa.Column(sa.DateTime, nullable=False)

-    started_at = sa.Column(sa.DateTime, nullable=False)

-    heartbeat_timestamp = sa.Column(sa.DateTime, nullable=False)

-    description = sa.Column(sa.String(255))

-    configurations = sa.Column(sa.String(4095), nullable=False)

+    if CONF.database.mysql_storage_engine == "NDBCLUSTER":

+        agent_type = sa.Column(sa.String(db_string_length), nullable=False)

+        binary = sa.Column(sa.String(db_string_length), nullable=False)

+        topic = sa.Column(sa.String(db_string_length), nullable=False)

+        host = sa.Column(sa.String(db_string_length), nullable=False)

+        admin_state_up = sa.Column(sa.Boolean, default=True,

+                                   server_default=sa.sql.true(), nullable=False)

+        created_at = sa.Column(sa.DateTime, nullable=False)

+        started_at = sa.Column(sa.DateTime, nullable=False)

+        heartbeat_timestamp = sa.Column(sa.DateTime, nullable=False)

+        description = sa.Column(sa.String(db_string_length))

+        configurations = sa.Column(sa.Text(4095), nullable=False)

+    else:

+        agent_type = sa.Column(sa.String(db_string_length), nullable=False)

+        binary = sa.Column(sa.String(db_string_length), nullable=False)

+        topic = sa.Column(sa.String(db_string_length), nullable=False)

+        host = sa.Column(sa.String(db_string_length), nullable=False)

+        admin_state_up = sa.Column(sa.Boolean, default=True,

+                                   server_default=sa.sql.true(), nullable=False)

+        created_at = sa.Column(sa.DateTime, nullable=False)

+        started_at = sa.Column(sa.DateTime, nullable=False)

+        heartbeat_timestamp = sa.Column(sa.DateTime, nullable=False)

+        description = sa.Column(sa.String(db_string_length))

+        configurations = sa.Column(sa.String(4095), nullable=False)

 #neutron/db/agentschedulers_db.py

@@ -431,10 +451,15 @@ class Vip(BASEV2, HasId, HasTenant, HasS

 #neutron/db/loadbalancer/loadbalancer_db.py

 class Member(BASEV2, HasId, HasTenant, HasStatusDescription):

-    __table_args__ = (

-        sa.schema.UniqueConstraint('pool_id', 'address', 'protocol_port',

-                                   name='uniq_member0pool_id0address0port'),

-    )

+    

+    # MySQL Cluster NDB does not support this constraint.

+    # TODO (oorgeron) Look into making this workable.

+    if CONF.database.mysql_storage_engine != "NDBCLUSTER":

+        __table_args__ = (

+            sa.schema.UniqueConstraint('pool_id', 'address', 'protocol_port',

+                                       name='uniq_member0pool_id0address0port'),

+        )

+

     pool_id = sa.Column(sa.String(36), sa.ForeignKey("pools.id"),

                         nullable=False)

     address = sa.Column(sa.String(64), nullable=False)

@@ -1229,8 +1254,15 @@ class PortBinding(BASEV2):

     profile = sa.Column(sa.String(BINDING_PROFILE_LEN), nullable=False,

                         default='', server_default='')

     vif_type = sa.Column(sa.String(64), nullable=False)

-    vif_details = sa.Column(sa.String(4095), nullable=False, default='',

+    if CONF.database.mysql_storage_engine == "NDBCLUSTER":

+        vif_details = sa.Column(sa.Text(4095), nullable=False)

+        profile = sa.Column(sa.Text(length=4095),nullable=False)

+    else:

+        vif_details = sa.Column(sa.String(4095), nullable=False,

+                                server_default='')

+        profile = sa.Column(sa.String(length=4095),nullable=False,

                             server_default='')

+

     driver = sa.Column(sa.String(64))

     segment = sa.Column(sa.String(36),

                         sa.ForeignKey('ml2_network_segments.id',

@@ -1827,10 +1859,17 @@ class PoolPort(BASEV2):

 class IdentifierMap(BASEV2, HasTenant):

     __tablename__ = 'cisco_csr_identifier_map'

-    ipsec_site_conn_id = sa.Column(sa.String(64),

-                                   sa.ForeignKey('ipsec_site_connections.id',

-                                                 ondelete="CASCADE"),

-                                   primary_key=True)

+    if CONF.database.mysql_storage_engine == "NDBCLUSTER":

+        ipsec_site_conn_id = sa.Column(sa.String(36),

+                                       sa.ForeignKey('ipsec_site_connections.id',

+                                                     ondelete="CASCADE"),

+                                       primary_key=True)

+    else:

+        ipsec_site_conn_id = sa.Column(sa.String(64),

+                                       sa.ForeignKey('ipsec_site_connections.id',

+                                                     ondelete="CASCADE"),

+                                       primary_key=True)

+

     csr_tunnel_id = sa.Column(sa.Integer, nullable=False)

     csr_ike_policy_id = sa.Column(sa.Integer, nullable=False)

     csr_ipsec_policy_id = sa.Column(sa.Integer, nullable=False)

--- neutron-2015.1.2/neutron/db/migration/alembic_migrations/agent_init_ops.py.orig	2016-07-29 13:15:57.799898995 -0600

+++ neutron-2015.1.2/neutron/db/migration/alembic_migrations/agent_init_ops.py	2016-07-29 13:30:30.011137004 -0600

@@ -17,26 +17,49 @@

 # This module only manages the 'agents' table. Binding tables are created

 # in the modules for relevant resources

-

+from alembic import context

 from alembic import op

 import sqlalchemy as sa

+config = context.config

+CONF = config.neutron_config

+

+if CONF.database.mysql_storage_engine == "NDBCLUSTER":

+   db_string_length = 128

+else:

+   db_string_length = 256

 def upgrade():

     bind = op.get_bind()

     insp = sa.engine.reflection.Inspector.from_engine(bind)

     if 'agents' not in insp.get_table_names():

-        op.create_table(

-        'agents',

-        sa.Column('id', sa.String(length=36), nullable=False),

-        sa.Column('agent_type', sa.String(length=255), nullable=False),

-        sa.Column('binary', sa.String(length=255), nullable=False),

-        sa.Column('topic', sa.String(length=255), nullable=False),

-        sa.Column('host', sa.String(length=255), nullable=False),

-        sa.Column('admin_state_up', sa.Boolean(), nullable=False),

-        sa.Column('created_at', sa.DateTime(), nullable=False),

-        sa.Column('started_at', sa.DateTime(), nullable=False),

-        sa.Column('heartbeat_timestamp', sa.DateTime(), nullable=False),

-        sa.Column('description', sa.String(length=255), nullable=True),

-        sa.Column('configurations', sa.String(length=4095), nullable=False),

-        sa.PrimaryKeyConstraint('id'))

+        if CONF.database.mysql_storage_engine == "NDBCLUSTER":

+            op.create_table(

+                'agents',

+                sa.Column('id', sa.String(length=36), nullable=False),

+                sa.Column('agent_type', sa.String(length=db_string_length), nullable=False),

+                sa.Column('binary', sa.String(length=db_string_length), nullable=False),

+                sa.Column('topic', sa.String(length=db_string_length), nullable=False),

+                sa.Column('host', sa.String(length=db_string_length), nullable=False),

+                sa.Column('admin_state_up', sa.Boolean(), nullable=False),

+                sa.Column('created_at', sa.DateTime(), nullable=False),

+                sa.Column('started_at', sa.DateTime(), nullable=False),

+                sa.Column('heartbeat_timestamp', sa.DateTime(), nullable=False),

+                sa.Column('description', sa.String(length=db_string_length), nullable=True),

+                sa.Column('configurations', sa.Text(length=4095), nullable=False),

+                sa.PrimaryKeyConstraint('id'))

+        else:

+            op.create_table(

+                'agents',

+                sa.Column('id', sa.String(length=36), nullable=False),

+                sa.Column('agent_type', sa.String(length=db_string_length), nullable=False),

+                sa.Column('binary', sa.String(length=db_string_length), nullable=False),

+                sa.Column('topic', sa.String(length=db_string_length), nullable=False),

+                sa.Column('host', sa.String(length=db_string_length), nullable=False),

+                sa.Column('admin_state_up', sa.Boolean(), nullable=False),

+                sa.Column('created_at', sa.DateTime(), nullable=False),

+                sa.Column('started_at', sa.DateTime(), nullable=False),

+                sa.Column('heartbeat_timestamp', sa.DateTime(), nullable=False),

+                sa.Column('description', sa.String(length=db_string_length), nullable=True),

+                sa.Column('configurations', sa.String(length=4095), nullable=False),

+                sa.PrimaryKeyConstraint('id'))

--- neutron-2015.1.2/neutron/db/migration/alembic_migrations/heal_script.py.orig	2016-07-29 13:16:06.570595984 -0600

+++ neutron-2015.1.2/neutron/db/migration/alembic_migrations/heal_script.py	2016-07-29 13:31:36.373228770 -0600

@@ -19,6 +19,7 @@ import alembic

 from alembic import autogenerate as autogen

 from alembic import context

 from alembic import op

+from oslo_config import cfg

 import sqlalchemy

 from sqlalchemy import schema as sa_schema

@@ -29,6 +30,9 @@ from sqlalchemy import types

 from neutron.db.migration.models import frozen as frozen_models

 from neutron.i18n import _LI, _LW

+config = context.config

+CONF = config.neutron_config

+

 LOG = logging.getLogger(__name__)

 METHODS = {}

@@ -70,7 +74,10 @@ def heal():

         'compare_server_default': _compare_server_default,

     }

     mc = alembic.migration.MigrationContext.configure(op.get_bind(), opts=opts)

-    set_storage_engine(op.get_bind(), "InnoDB")

+    if CONF.database.mysql_storage_engine == "NDBCLUSTER":

+        set_storage_engine(op.get_bind(), 'NDBCLUSTER')

+    else:

+        set_storage_engine(op.get_bind(), 'InnoDB')

     diff = autogen.compare_metadata(mc, models_metadata)

     for el in diff:

         execute_alembic_command(el)

@@ -286,4 +293,6 @@ def set_storage_engine(bind, engine):

     if bind.dialect.name == 'mysql':

         for table in insp.get_table_names():

             if insp.get_table_options(table)['mysql_engine'] != engine:

-                op.execute("ALTER TABLE %s ENGINE=%s" % (table, engine))

+                op.execute("ALTER TABLE %(db_table)s Engine=%(mysql_storage_engine)s"

+                           % dict(db_table=table,

+                           mysql_storage_engine=CONF.database.mysql_storage_engine))

--- neutron-2015.1.2/neutron/db/migration/alembic_migrations/versions/157a5d299379_ml2_binding_profile.py.orig	2016-07-29 13:16:13.971296279 -0600

+++ neutron-2015.1.2/neutron/db/migration/alembic_migrations/versions/157a5d299379_ml2_binding_profile.py	2016-07-29 13:32:20.859598698 -0600

@@ -25,14 +25,24 @@ Create Date: 2014-02-13 23:48:25.147279

 revision = '157a5d299379'

 down_revision = '50d5ba354c23'

+from alembic import context

 from alembic import op

 import sqlalchemy as sa

 from neutron.db import migration

+config = context.config

+CONF = config.neutron_config

 def upgrade():

     if migration.schema_has_table('ml2_port_bindings'):

-        op.add_column('ml2_port_bindings',

-                      sa.Column('profile', sa.String(length=4095),

-                                nullable=False, server_default=''))

+        # MySQL Cluster (NDB) does not support rows longer than 14000.

+        # This configures the profile column as TEXT to keep the row size down.

+        if CONF.database.mysql_storage_engine == "NDBCLUSTER":

+            op.add_column('ml2_port_bindings',

+                          sa.Column('profile', sa.Text(length=4095),

+                                    nullable=False))

+        else:

+            op.add_column('ml2_port_bindings',

+                          sa.Column('profile', sa.String(length=4095),

+                                    nullable=False, server_default=''))

--- neutron-2015.1.2/neutron/db/migration/alembic_migrations/versions/50d5ba354c23_ml2_binding_vif_details.py.orig	2016-07-29 13:16:20.121107758 -0600

+++ neutron-2015.1.2/neutron/db/migration/alembic_migrations/versions/50d5ba354c23_ml2_binding_vif_details.py	2016-07-29 13:32:42.870041087 -0600

@@ -25,11 +25,14 @@ Create Date: 2014-02-11 23:21:59.577972

 revision = '50d5ba354c23'

 down_revision = '27cc183af192'

+from alembic import context

 from alembic import op

 import sqlalchemy as sa

 from neutron.db import migration

+config = context.config

+CONF = config.neutron_config

 def upgrade():

@@ -38,9 +41,17 @@ def upgrade():

         # did not create the ml2_port_bindings table.

         return

-    op.add_column('ml2_port_bindings',

-                  sa.Column('vif_details', sa.String(length=4095),

-                            nullable=False, server_default=''))

+    # MySQL Cluster (NDB) does not support rows longer than 14000.

+    # This converts vif_details to TEXT to keep the row size down.

+    if CONF.database.mysql_storage_engine == "NDBCLUSTER":

+        op.add_column('ml2_port_bindings',

+                      sa.Column('vif_details', sa.Text(length=4095),

+                                nullable=False))

+    else:

+        op.add_column('ml2_port_bindings',

+                      sa.Column('vif_details', sa.String(length=4095),

+                                nullable=False, server_default=''))

+

     if op.get_bind().engine.name == 'ibm_db_sa':

         op.execute(

             "UPDATE ml2_port_bindings SET"

--- neutron-2015.1.2/neutron/db/migration/alembic_migrations/versions/e197124d4b9_add_unique_constrain.py.orig	2016-07-29 13:16:26.541620109 -0600

+++ neutron-2015.1.2/neutron/db/migration/alembic_migrations/versions/e197124d4b9_add_unique_constrain.py	2016-07-29 13:33:26.620005420 -0600

@@ -25,16 +25,27 @@ Create Date: 2013-11-17 10:09:37.728903

 revision = 'e197124d4b9'

 down_revision = 'havana'

+from alembic import context

 from alembic import op

+from oslo_config import cfg

 from neutron.db import migration

+config = context.config

+CONF = config.neutron_config

 CONSTRAINT_NAME = 'uniq_member0pool_id0address0port'

 TABLE_NAME = 'members'

 def upgrade():

+

+    # MySQL Cluster, a.k.a. NDB, does not support this migration step.

+    # This test will skip this migration.

+    # TODO (oorgeron) Look into making this workable.

+    if CONF.database.mysql_storage_engine == "NDBCLUSTER":

+        return

+

     if migration.schema_has_table(TABLE_NAME):

         op.create_unique_constraint(

             name=CONSTRAINT_NAME,

--- neutron-2015.1.2/neutron/db/migration/alembic_migrations/versions/86d6d9776e2b_cisco_apic_driver_update_l3.py.orig	2016-07-29 13:16:33.520954213 -0600