This patchset is for bug:

24394567 Mitaka Keystone should support MySQL Cluster

This fixes the following aspects of Keystone:

1. Implementation of an oslo.db configuration parameter to specify the MySQL

   storage engine (mysql_storage_engine).

2. Replacement of hardcoded SQL statements that set the engine to "InnoDB"

   to the above configuration value.

3. Logic to handle SQL differences between MySQL InnoDB and MySQL Cluster (NDB).

   This includes column lengths, constraints, foreign keys, and indexes.

This has not been committed upstream, but has been filed in launchpad:

https://bugs.launchpad.net/keystone/+bug/1564110

--- keystone-9.1.0/keystone/common/sql/migrate_repo/versions/067_kilo.py.orig	2016-08-04 10:10:34.947867201 -0700

+++ keystone-9.1.0/keystone/common/sql/migrate_repo/versions/067_kilo.py	2016-08-04 10:16:37.481333376 -0700

@@ -12,6 +12,7 @@

 import migrate

+from oslo_config import cfg

 from oslo_log import log

 import sqlalchemy as sql

@@ -21,6 +22,7 @@ from keystone.identity.mapping_backends

 LOG = log.getLogger(__name__)

+CONF = cfg.CONF

 def upgrade(migrate_engine):

@@ -44,7 +46,7 @@ def upgrade(migrate_engine):

         sql.Column('blob', ks_sql.JsonBlob, nullable=False),

         sql.Column('type', sql.String(length=255), nullable=False),

         sql.Column('extra', ks_sql.JsonBlob.impl),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8')

     domain = sql.Table(

@@ -53,7 +55,7 @@ def upgrade(migrate_engine):

         sql.Column('name', sql.String(length=64), nullable=False),

         sql.Column('enabled', sql.Boolean, default=True, nullable=False),

         sql.Column('extra', ks_sql.JsonBlob.impl),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8')

     endpoint = sql.Table(

@@ -67,7 +69,7 @@ def upgrade(migrate_engine):

         sql.Column('enabled', sql.Boolean, nullable=False, default=True,

                    server_default='1'),

         sql.Column('region_id', sql.String(length=255), nullable=True),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8')

     group = sql.Table(

@@ -77,7 +79,7 @@ def upgrade(migrate_engine):

         sql.Column('name', sql.String(length=64), nullable=False),

         sql.Column('description', sql.Text),

         sql.Column('extra', ks_sql.JsonBlob.impl),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8')

     policy = sql.Table(

@@ -86,7 +88,7 @@ def upgrade(migrate_engine):

         sql.Column('type', sql.String(length=255), nullable=False),

         sql.Column('blob', ks_sql.JsonBlob, nullable=False),

         sql.Column('extra', ks_sql.JsonBlob.impl),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8')

     project = sql.Table(

@@ -98,7 +100,7 @@ def upgrade(migrate_engine):

         sql.Column('enabled', sql.Boolean),

         sql.Column('domain_id', sql.String(length=64), nullable=False),

         sql.Column('parent_id', sql.String(64), nullable=True),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8')

     role = sql.Table(

@@ -106,7 +108,7 @@ def upgrade(migrate_engine):

         sql.Column('id', sql.String(length=64), primary_key=True),

         sql.Column('name', sql.String(length=255), nullable=False),

         sql.Column('extra', ks_sql.JsonBlob.impl),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8')

     service = sql.Table(

@@ -116,7 +118,7 @@ def upgrade(migrate_engine):

         sql.Column('enabled', sql.Boolean, nullable=False, default=True,

                    server_default='1'),

         sql.Column('extra', ks_sql.JsonBlob.impl),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8')

     token = sql.Table(

@@ -127,7 +129,7 @@ def upgrade(migrate_engine):

         sql.Column('valid', sql.Boolean, default=True, nullable=False),

         sql.Column('trust_id', sql.String(length=64)),

         sql.Column('user_id', sql.String(length=64)),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8')

     trust = sql.Table(

@@ -141,7 +143,7 @@ def upgrade(migrate_engine):

         sql.Column('expires_at', sql.DateTime),

         sql.Column('remaining_uses', sql.Integer, nullable=True),

         sql.Column('extra', ks_sql.JsonBlob.impl),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8')

     trust_role = sql.Table(

@@ -150,7 +152,7 @@ def upgrade(migrate_engine):

                    nullable=False),

         sql.Column('role_id', sql.String(length=64), primary_key=True,

                    nullable=False),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8')

     user = sql.Table(

@@ -162,14 +164,14 @@ def upgrade(migrate_engine):

         sql.Column('enabled', sql.Boolean),

         sql.Column('domain_id', sql.String(length=64), nullable=False),

         sql.Column('default_project_id', sql.String(length=64)),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8')

     user_group_membership = sql.Table(

         'user_group_membership', meta,

         sql.Column('user_id', sql.String(length=64), primary_key=True),

         sql.Column('group_id', sql.String(length=64), primary_key=True),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8')

     region = sql.Table(

@@ -179,7 +181,7 @@ def upgrade(migrate_engine):

         sql.Column('description', sql.String(255), nullable=False),

         sql.Column('parent_region_id', sql.String(255), nullable=True),

         sql.Column('extra', sql.Text()),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8')

     assignment = sql.Table(

@@ -197,7 +199,7 @@ def upgrade(migrate_engine):

         sql.Column('role_id', sql.String(64), nullable=False),

         sql.Column('inherited', sql.Boolean, default=False, nullable=False),

         sql.PrimaryKeyConstraint('type', 'actor_id', 'target_id', 'role_id'),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8')

     mapping = sql.Table(

@@ -211,7 +213,7 @@ def upgrade(migrate_engine):

             mapping_backend.EntityType.GROUP,

             name='entity_type'),

             nullable=False),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8')

     domain_config_whitelist = sql.Table(

@@ -221,7 +223,7 @@ def upgrade(migrate_engine):

         sql.Column('group', sql.String(255), primary_key=True),

         sql.Column('option', sql.String(255), primary_key=True),

         sql.Column('value', ks_sql.JsonBlob.impl, nullable=False),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8')

     domain_config_sensitive = sql.Table(

@@ -231,7 +233,7 @@ def upgrade(migrate_engine):

         sql.Column('group', sql.String(255), primary_key=True),

         sql.Column('option', sql.String(255), primary_key=True),

         sql.Column('value', ks_sql.JsonBlob.impl, nullable=False),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8')

     # create all tables

--- keystone-9.1.0/keystone/common/sql/migrate_repo/versions/073_insert_assignment_inherited_pk.py.orig	2016-08-04 10:10:40.619013471 -0700

+++ keystone-9.1.0/keystone/common/sql/migrate_repo/versions/073_insert_assignment_inherited_pk.py	2016-08-04 10:18:06.349087340 -0700

@@ -11,11 +11,13 @@

 # under the License.

 import migrate

+from oslo_config import cfg

 import sqlalchemy as sql

 from sqlalchemy.orm import sessionmaker

 from keystone.assignment.backends import sql as assignment_sql

+CONF = cfg.CONF

 def upgrade(migrate_engine):

     """Inserts inherited column to assignment table PK constraints.

@@ -57,7 +59,7 @@ def upgrade(migrate_engine):

                        nullable=False),

             sql.PrimaryKeyConstraint('type', 'actor_id', 'target_id',

                                      'role_id', 'inherited'),

-            mysql_engine='InnoDB',

+            mysql_engine=CONF.database.mysql_storage_engine,

             mysql_charset='utf8')

         # Create the new assignment table

--- keystone-9.1.0/keystone/common/sql/migrate_repo/versions/075_confirm_config_registration.py.orig	2016-08-04 10:10:47.746776861 -0700

+++ keystone-9.1.0/keystone/common/sql/migrate_repo/versions/075_confirm_config_registration.py	2016-08-04 10:19:03.802564813 -0700

@@ -10,9 +10,11 @@

 #    License for the specific language governing permissions and limitations

 #    under the License.

+from oslo_config import cfg

 import sqlalchemy as sql

 REGISTRATION_TABLE = 'config_register'

+CONF = cfg.CONF

 def upgrade(migrate_engine):

@@ -24,6 +26,6 @@ def upgrade(migrate_engine):

         meta,

         sql.Column('type', sql.String(64), primary_key=True),

         sql.Column('domain_id', sql.String(64), nullable=False),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8')

     registration_table.create(migrate_engine, checkfirst=True)

--- keystone-9.1.0/keystone/common/sql/migrate_repo/versions/081_add_endpoint_policy_table.py.orig	2016-08-04 10:10:53.258335033 -0700

+++ keystone-9.1.0/keystone/common/sql/migrate_repo/versions/081_add_endpoint_policy_table.py	2016-08-04 10:20:06.575865756 -0700

@@ -12,10 +12,12 @@

 # License for the specific language governing permissions and limitations

 # under the License.

+from oslo_config import cfg

 import sqlalchemy as sql

 from keystone.common.sql import migration_helpers

+CONF = cfg.CONF

 def upgrade(migrate_engine):

     try:

@@ -48,7 +50,7 @@ def upgrade(migrate_engine):

         sql.Column('region_id', sql.String(64),

                    nullable=True),

         sql.UniqueConstraint('endpoint_id', 'service_id', 'region_id'),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8')

     endpoint_policy_table.create(migrate_engine, checkfirst=True)

--- keystone-9.1.0/keystone/common/sql/migrate_repo/versions/082_add_federation_tables.py.orig	2016-08-04 10:10:59.098720956 -0700

+++ keystone-9.1.0/keystone/common/sql/migrate_repo/versions/082_add_federation_tables.py	2016-08-04 10:22:41.994100588 -0700

@@ -45,7 +45,7 @@ def upgrade(migrate_engine):

         sql.Column('id', sql.String(64), primary_key=True),

         sql.Column('enabled', sql.Boolean, nullable=False),

         sql.Column('description', sql.Text(), nullable=True),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8')

     idp_table.create(migrate_engine, checkfirst=True)

@@ -57,7 +57,7 @@ def upgrade(migrate_engine):

                    sql.ForeignKey('identity_provider.id', ondelete='CASCADE'),

                    primary_key=True),

         sql.Column('mapping_id', sql.String(64), nullable=False),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8')

     federation_protocol_table.create(migrate_engine, checkfirst=True)

@@ -66,7 +66,7 @@ def upgrade(migrate_engine):

         meta,

         sql.Column('id', sql.String(64), primary_key=True),

         sql.Column('rules', sql.Text(), nullable=False),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8')

     mapping_table.create(migrate_engine, checkfirst=True)

@@ -81,7 +81,7 @@ def upgrade(migrate_engine):

         sql.Column('sp_url', sql.String(256), nullable=False),

         sql.Column(_RELAY_STATE_PREFIX, sql.String(256), nullable=False,

                    server_default=relay_state_prefix_default),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8')

     sp_table.create(migrate_engine, checkfirst=True)

@@ -92,6 +92,6 @@ def upgrade(migrate_engine):

         sql.Column('idp_id', sql.String(64),

                    sql.ForeignKey('identity_provider.id', ondelete='CASCADE')),

         sql.Column('remote_id', sql.String(255), primary_key=True),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8')

     remote_id_table.create(migrate_engine, checkfirst=True)

--- keystone-9.1.0/keystone/common/sql/migrate_repo/versions/087_implied_roles.py.orig	2016-08-04 10:11:04.322293468 -0700

+++ keystone-9.1.0/keystone/common/sql/migrate_repo/versions/087_implied_roles.py	2016-08-04 10:23:34.919749416 -0700

@@ -12,10 +12,12 @@

 import migrate

+from oslo_config import cfg

 import sqlalchemy as sql

 ROLE_TABLE = 'role'

+CONF = cfg.CONF

 def upgrade(migrate_engine):

@@ -27,7 +29,7 @@ def upgrade(migrate_engine):

         sql.Column('prior_role_id', sql.String(length=64), primary_key=True),

         sql.Column(

             'implied_role_id', sql.String(length=64), primary_key=True),

-        mysql_engine='InnoDB',

+        mysql_engine=CONF.database.mysql_storage_engine,

         mysql_charset='utf8')

     implied_role.create()

     role = sql.Table(ROLE_TABLE, meta, autoload=True)