This patchset is for bug:

24394587 Mitaka Nova should support MySQL Cluster

This fixes the following aspects of Nova:

1. Implementation of an oslo.db configuration parameter to specify the MySQL

   storage engine (mysql_storage_engine).

2. Replacement of hardcoded SQL statements that set the engine to "InnoDB"

   to the above configuration value.

3. Logic to handle SQL differences between MySQL InnoDB and MySQL Cluster (NDB).

   This includes column lengths, constraints, foreign keys, and indexes.

This has not been committed upstream, but has been filed in launchpad:

https://bugs.launchpad.net/nova/+bug/1564110

--- nova-13.1.0/nova/db/sqlalchemy/api_migrations/migrate_repo/versions/001_cell_mapping.py.orig	2016-08-24 14:22:51.920016201 -0700

+++ nova-13.1.0/nova/db/sqlalchemy/api_migrations/migrate_repo/versions/001_cell_mapping.py	2016-08-24 14:22:34.974838825 -0700

@@ -11,6 +11,7 @@

 #    under the License.

 from migrate import UniqueConstraint

+from oslo_config import cfg

 from sqlalchemy import Column

 from sqlalchemy import DateTime

 from sqlalchemy import Index

@@ -20,6 +21,8 @@ from sqlalchemy import String

 from sqlalchemy import Table

 from sqlalchemy import Text

+CONF = cfg.CONF

+

 def upgrade(migrate_engine):

     meta = MetaData()

@@ -34,7 +37,7 @@ def upgrade(migrate_engine):

         Column('transport_url', Text()),

         Column('database_connection', Text()),

         UniqueConstraint('uuid', name='uniq_cell_mappings0uuid'),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

--- nova-13.1.0/nova/db/sqlalchemy/api_migrations/migrate_repo/versions/002_instance_mapping.py.orig	2016-08-24 14:22:59.979100128 -0700

+++ nova-13.1.0/nova/db/sqlalchemy/api_migrations/migrate_repo/versions/002_instance_mapping.py	2016-08-24 14:22:34.976482225 -0700

@@ -12,6 +12,7 @@

 from migrate.changeset.constraint import ForeignKeyConstraint

 from migrate import UniqueConstraint

+from oslo_config import cfg

 from sqlalchemy import Column

 from sqlalchemy import DateTime

 from sqlalchemy import Index

@@ -20,6 +21,8 @@ from sqlalchemy import MetaData

 from sqlalchemy import String

 from sqlalchemy import Table

+CONF = cfg.CONF

+

 def upgrade(migrate_engine):

     meta = MetaData()

@@ -38,7 +41,7 @@ def upgrade(migrate_engine):

         Index('project_id_idx', 'project_id'),

         ForeignKeyConstraint(columns=['cell_id'],

             refcolumns=[cell_mappings.c.id]),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

--- nova-13.1.0/nova/db/sqlalchemy/api_migrations/migrate_repo/versions/003_host_mapping.py.orig	2016-08-24 14:23:07.246721338 -0700

+++ nova-13.1.0/nova/db/sqlalchemy/api_migrations/migrate_repo/versions/003_host_mapping.py	2016-08-24 14:22:34.977720063 -0700

@@ -12,6 +12,7 @@

 from migrate.changeset.constraint import ForeignKeyConstraint

 from migrate import UniqueConstraint

+from oslo_config import cfg

 from sqlalchemy import Column

 from sqlalchemy import DateTime

 from sqlalchemy import Index

@@ -20,6 +21,8 @@ from sqlalchemy import MetaData

 from sqlalchemy import String

 from sqlalchemy import Table

+CONF = cfg.CONF

+

 def upgrade(migrate_engine):

     meta = MetaData()

@@ -36,7 +39,7 @@ def upgrade(migrate_engine):

             name='uniq_host_mappings0host'),

         ForeignKeyConstraint(columns=['cell_id'],

             refcolumns=[cell_mappings.c.id]),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

--- nova-13.1.0/nova/db/sqlalchemy/api_migrations/migrate_repo/versions/004_add_request_spec.py.orig	2016-08-24 14:23:13.631755716 -0700

+++ nova-13.1.0/nova/db/sqlalchemy/api_migrations/migrate_repo/versions/004_add_request_spec.py	2016-08-24 14:22:34.978932430 -0700

@@ -11,6 +11,7 @@

 #    under the License.

 from migrate import UniqueConstraint

+from oslo_config import cfg

 from sqlalchemy import Column

 from sqlalchemy import DateTime

 from sqlalchemy import Index

@@ -20,6 +21,8 @@ from sqlalchemy import String

 from sqlalchemy import Table

 from sqlalchemy import Text

+CONF = cfg.CONF

+

 def upgrade(migrate_engine):

     meta = MetaData()

@@ -33,7 +36,7 @@ def upgrade(migrate_engine):

         Column('spec', Text, nullable=False),

         UniqueConstraint('instance_uuid',

             name='uniq_request_specs0instance_uuid'),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

--- nova-13.1.0/nova/db/sqlalchemy/api_migrations/migrate_repo/versions/005_flavors.py.orig	2016-08-24 14:23:20.047706518 -0700

+++ nova-13.1.0/nova/db/sqlalchemy/api_migrations/migrate_repo/versions/005_flavors.py	2016-08-24 14:22:34.980360970 -0700

@@ -12,6 +12,7 @@

 from migrate.changeset.constraint import ForeignKeyConstraint

 from migrate import UniqueConstraint

+from oslo_config import cfg

 from sqlalchemy import Boolean

 from sqlalchemy import Column

 from sqlalchemy import DateTime

@@ -22,6 +23,8 @@ from sqlalchemy import MetaData

 from sqlalchemy import String

 from sqlalchemy import Table

+CONF = cfg.CONF

+

 def upgrade(migrate_engine):

     meta = MetaData()

@@ -44,7 +47,7 @@ def upgrade(migrate_engine):

         Column('is_public', Boolean),

         UniqueConstraint("flavorid", name="uniq_flavors0flavorid"),

         UniqueConstraint("name", name="uniq_flavors0name"),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

     flavors.create(checkfirst=True)

@@ -59,7 +62,7 @@ def upgrade(migrate_engine):

         UniqueConstraint('flavor_id', 'key',

             name='uniq_flavor_extra_specs0flavor_id0key'),

         ForeignKeyConstraint(columns=['flavor_id'], refcolumns=[flavors.c.id]),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

@@ -82,7 +85,7 @@ def upgrade(migrate_engine):

             name='uniq_flavor_projects0flavor_id0project_id'),

         ForeignKeyConstraint(columns=['flavor_id'],

             refcolumns=[flavors.c.id]),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

     flavor_projects.create(checkfirst=True)

--- nova-13.1.0/nova/db/sqlalchemy/api_migrations/migrate_repo/versions/006_build_request.py.orig	2016-08-24 14:23:28.058484501 -0700

+++ nova-13.1.0/nova/db/sqlalchemy/api_migrations/migrate_repo/versions/006_build_request.py	2016-08-24 14:22:34.981684038 -0700

@@ -12,6 +12,7 @@

 from migrate.changeset.constraint import ForeignKeyConstraint

 from migrate import UniqueConstraint

+from oslo_config import cfg

 from sqlalchemy import Boolean

 from sqlalchemy import Column

 from sqlalchemy import DateTime

@@ -24,6 +25,8 @@ from sqlalchemy import String

 from sqlalchemy import Table

 from sqlalchemy import Text

+CONF = cfg.CONF

+

 def InetSmall():

     return String(length=39).with_variant(dialects.postgresql.INET(),

@@ -61,7 +64,7 @@ def upgrade(migrate_engine):

         Index('build_requests_project_id_idx', 'project_id'),

         ForeignKeyConstraint(columns=['request_spec_id'],

             refcolumns=[request_specs.c.id]),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

--- nova-13.1.0/nova/db/sqlalchemy/migrate_repo/versions/216_havana.py.orig	2016-08-24 14:23:37.472011893 -0700

+++ nova-13.1.0/nova/db/sqlalchemy/migrate_repo/versions/216_havana.py	2016-08-24 14:22:34.987060556 -0700

@@ -14,6 +14,7 @@

 from migrate.changeset import UniqueConstraint

 from migrate import ForeignKeyConstraint

+from oslo_config import cfg

 from oslo_log import log as logging

 from sqlalchemy import Boolean, BigInteger, Column, DateTime, Enum, Float

 from sqlalchemy import dialects

@@ -25,6 +26,8 @@ from nova.i18n import _LE

 LOG = logging.getLogger(__name__)

+CONF = cfg.CONF

+

 # Note on the autoincrement flag: this is defaulted for primary key columns

 # of integral type, so is no longer set explicitly in such cases.

@@ -37,6 +40,10 @@ def MediumText():

     return Text().with_variant(dialects.mysql.MEDIUMTEXT(), 'mysql')

+def TinyText():

+    return Text().with_variant(dialects.mysql.TINYTEXT(), 'mysql')

+

+

 def Inet():

     return String(length=43).with_variant(dialects.postgresql.INET(),

                   'postgresql')

@@ -76,7 +83,7 @@ def _create_shadow_tables(migrate_engine

         shadow_table_name = 'shadow_' + table_name

         shadow_table = Table(shadow_table_name, meta, *columns,

-                             mysql_engine='InnoDB')

+                             mysql_engine=CONF.database.mysql_storage_engine)

         try:

             shadow_table.create()

         except Exception:

@@ -167,7 +174,7 @@ def _create_dump_tables(migrate_engine):

             else:

                 columns.append(column.copy())

         table_dump = Table(dump_table_name, meta, *columns,

-                           mysql_engine='InnoDB')

+                           mysql_engine=CONF.database.mysql_storage_engine)

         table_dump.create()

@@ -187,7 +194,7 @@ def upgrade(migrate_engine):

         Column('url', String(length=255)),

         Column('md5hash', String(length=255)),

         Column('deleted', Integer),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

@@ -200,7 +207,7 @@ def upgrade(migrate_engine):

         Column('aggregate_id', Integer, ForeignKey('aggregates.id'),

               nullable=False),

         Column('deleted', Integer),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

@@ -214,7 +221,7 @@ def upgrade(migrate_engine):

         Column('key', String(length=255), nullable=False),

         Column('value', String(length=255), nullable=False),

         Column('deleted', Integer),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

@@ -225,7 +232,7 @@ def upgrade(migrate_engine):

         Column('id', Integer, primary_key=True, nullable=False),

         Column('name', String(length=255)),

         Column('deleted', Integer),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

@@ -250,7 +257,7 @@ def upgrade(migrate_engine):

         Column('disk_bus', String(length=255), nullable=True),

         Column('boot_index', Integer),

         Column('image_id', String(length=36), nullable=True),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

@@ -268,7 +275,7 @@ def upgrade(migrate_engine):

         Column('last_ctr_in', BigInteger()),

         Column('last_ctr_out', BigInteger()),

         Column('deleted', Integer),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

@@ -284,7 +291,7 @@ def upgrade(migrate_engine):

         Column('is_parent', Boolean),

         Column('deleted', Integer),

         Column('transport_url', String(length=255), nullable=False),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

@@ -297,7 +304,7 @@ def upgrade(migrate_engine):

         Column('project_id', String(length=255)),

         Column('file_name', String(length=255)),

         Column('deleted', Integer),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

@@ -310,7 +317,7 @@ def upgrade(migrate_engine):

         Column('key', String(length=255), nullable=False),

         Column('value', String(length=255)),

         Column('deleted', Integer),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

@@ -339,7 +346,7 @@ def upgrade(migrate_engine):

         Column('host_ip', InetSmall()),

         Column('supported_instances', Text),

         Column('pci_stats', Text, nullable=True),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

@@ -356,7 +363,7 @@ def upgrade(migrate_engine):

         Column('host', String(length=255)),

         Column('compute_host', String(length=255)),

         Column('deleted', Integer),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

@@ -379,7 +386,7 @@ def upgrade(migrate_engine):

         Column('pool_id', Integer, ForeignKey('console_pools.id')),

         Column(*consoles_instance_uuid_column_args),

         Column('deleted', Integer),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

@@ -392,7 +399,7 @@ def upgrade(migrate_engine):

         Column('scope', String(length=255)),

         Column('availability_zone', String(length=255)),

         Column('project_id', String(length=255)),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

@@ -410,7 +417,7 @@ def upgrade(migrate_engine):

         Column('host', String(length=255)),

         Column('instance_uuid', String(length=36)),

         Column('deleted', Integer),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

@@ -427,7 +434,7 @@ def upgrade(migrate_engine):

         Column('pool', String(length=255)),

         Column('interface', String(length=255)),

         Column('deleted', Integer),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

@@ -442,7 +449,7 @@ def upgrade(migrate_engine):

         Column('details', MediumText()),

         Column('host', String(length=255)),

         Column('deleted', Integer),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

@@ -453,7 +460,7 @@ def upgrade(migrate_engine):

         Column('id', Integer, primary_key=True, nullable=False),

         Column('uuid', String(36), nullable=False),

         Column('deleted', Integer),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

@@ -465,7 +472,7 @@ def upgrade(migrate_engine):

         Column('network_info', MediumText()),

         Column('instance_uuid', String(length=36), nullable=False),

         Column('deleted', Integer),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

@@ -481,7 +488,7 @@ def upgrade(migrate_engine):

         Column('name', String(length=255)),

         UniqueConstraint('uuid', 'deleted',

                          name='uniq_instance_groups0uuid0deleted'),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8',

     )

@@ -495,7 +502,7 @@ def upgrade(migrate_engine):

         Column('value', String(length=255)),

         Column('group_id', Integer, ForeignKey('instance_groups.id'),

                nullable=False),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8',

     )

@@ -508,7 +515,7 @@ def upgrade(migrate_engine):

         Column('policy', String(length=255)),

         Column('group_id', Integer, ForeignKey('instance_groups.id'),

                nullable=False),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8',

     )

@@ -521,7 +528,7 @@ def upgrade(migrate_engine):

         Column('instance_id', String(length=255)),

         Column('group_id', Integer, ForeignKey('instance_groups.id'),

                nullable=False),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8',

     )

@@ -534,7 +541,7 @@ def upgrade(migrate_engine):

         Column('value', String(length=255)),

         Column('instance_uuid', String(length=36), nullable=True),

         Column('deleted', Integer),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

@@ -547,7 +554,7 @@ def upgrade(migrate_engine):

         Column('key', String(length=255), nullable=False),

         Column('value', String(length=255)),

         Column('deleted', Integer),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

@@ -561,7 +568,7 @@ def upgrade(migrate_engine):

         Column('key', String(length=255)),

         Column('value', String(length=255)),

         Column('deleted', Integer),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

@@ -573,7 +580,7 @@ def upgrade(migrate_engine):

         Column('instance_type_id', Integer, nullable=False),

         Column('project_id', String(length=255)),

         Column('deleted', Integer),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

@@ -594,67 +601,131 @@ def upgrade(migrate_engine):

         Column('disabled', Boolean),

         Column('is_public', Boolean),

         Column('deleted', Integer),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8'

     )

-    inst_lock_enum = Enum('owner', 'admin', name='instances0locked_by')

-    instances = Table('instances', meta,

-        Column('created_at', DateTime),

-        Column('updated_at', DateTime),

-        Column('deleted_at', DateTime),

-        Column('id', Integer, primary_key=True, nullable=False),

-        Column('internal_id', Integer),

-        Column('user_id', String(length=255)),

-        Column('project_id', String(length=255)),

-        Column('image_ref', String(length=255)),

-        Column('kernel_id', String(length=255)),

-        Column('ramdisk_id', String(length=255)),

-        Column('launch_index', Integer),

-        Column('key_name', String(length=255)),

-        Column('key_data', MediumText()),

-        Column('power_state', Integer),

-        Column('vm_state', String(length=255)),

-        Column('memory_mb', Integer),

-        Column('vcpus', Integer),

-        Column('hostname', String(length=255)),

-        Column('host', String(length=255)),

-        Column('user_data', MediumText()),

-        Column('reservation_id', String(length=255)),

-        Column('scheduled_at', DateTime),

-        Column('launched_at', DateTime),

-        Column('terminated_at', DateTime),

-        Column('display_name', String(length=255)),

-        Column('display_description', String(length=255)),

-        Column('availability_zone', String(length=255)),

-        Column('locked', Boolean),

-        Column('os_type', String(length=255)),

-        Column('launched_on', MediumText()),

-        Column('instance_type_id', Integer),

-        Column('vm_mode', String(length=255)),

-        Column('uuid', String(length=36)),