Switch from MySQL-python to PyMySQL

As discussed in the Liberty Design Summit "Moving apps to Python 3"
cross-project workshop, the way forward in the near future is to
switch to the pure-python PyMySQL library as a default.

https://etherpad.openstack.org/p/liberty-cross-project-python3

BaseMySqlRootAccess.enable_root(): catch also InternalError because
the PyMySQL error is not wrapped into a SQLAlchemy OperationalError,
but a generic SQLAlchemy InternalError. Similar change is made in
026_datastore_versions_unique_fix.py.

This change requires a trove integration change to add the PyMySQL to
the guest image: Id4d013d174ba40a453819f900aaa316a93e59b48.

Partially implements: blueprint trove-python3
Co-Authored-By: Victor Stinner <vstinner@redhat.com>
Depends-On: Id4d013d174ba40a453819f900aaa316a93e59b48
Change-Id: I65e8a8d5dc251a8b00529cdfb1a6ada3d5720f68

doc/source/dev/manual_install.rst

@@ -71,7 +71,7 @@ List of packages to be installed:
 .. code-block:: bash
 
    $ sudo apt-get install build-essential libxslt1-dev qemu-utils mysql-client \
-     git python-dev python-pexpect python-mysqldb libmysqlclient-dev
+     git python-dev python-pexpect python-pymysql libmysqlclient-dev
 
 Python settings
 ---------------
@@ -254,7 +254,7 @@ Create the Trove database schema:
 
   - Connect to the storage backend (MySQL, PostgreSQL)
   - Create a database called `trove` (this database will be used for storing Trove ORM)
-  - Compose connection string. Example: mysql://<user>:<password>@<backend_host>:<backend_port>/<database_name>
+  - Compose connection string. Example: mysql+pymysql://<user>:<password>@<backend_host>:<backend_port>/<database_name>
 
 Initialize the database
 =======================

etc/trove/trove-conductor.conf.sample

@@ -27,7 +27,7 @@ control_exchange = trove
 #trace_sqlalchemy = True
 
 [database]
-connection = mysql://root:e1a2c042c828d3566d0a@localhost/trove
+connection = mysql+pymysql://root:e1a2c042c828d3566d0a@localhost/trove
 
 [oslo_messaging_rabbit]
 # The RabbitMQ broker address where a single node is used. (string value)

etc/trove/trove-taskmanager.conf.sample

@@ -163,8 +163,8 @@ pydev_debug = disabled
 # SQLAlchemy connection string for the reference implementation
 # registry server. Any valid SQLAlchemy connection string is fine.
 # See: http://www.sqlalchemy.org/docs/05/reference/sqlalchemy/connections.html#sqlalchemy.create_engine
-connection = mysql://root:e1a2c042c828d3566d0a@localhost/trove
+connection = mysql+pymysql://root:e1a2c042c828d3566d0a@localhost/trove
-# connection = mysql://root:root@localhost/trove
+# connection = mysql+pymysql://root:root@localhost/trove
 
 # Period in seconds after which SQLAlchemy should reestablish its connection
 # to the database.

etc/trove/trove.conf.sample

@@ -146,7 +146,7 @@ api_paste_config = api-paste.ini
 # registry server. Any valid SQLAlchemy connection string is fine.
 # See: http://www.sqlalchemy.org/docs/05/reference/sqlalchemy/connections.html#sqlalchemy.create_engine
 # connection = sqlite:///trove_test.sqlite
-connection = mysql://root:e1a2c042c828d3566d0a@localhost/trove
+connection = mysql+pymysql://root:e1a2c042c828d3566d0a@localhost/trove
 #connection = postgresql://trove:trove@localhost/trove
 
 # Period in seconds after which SQLAlchemy should reestablish its connection

etc/trove/trove.conf.test

@@ -147,7 +147,7 @@ device_path = /dev/vdb
 # registry server. Any valid SQLAlchemy connection string is fine.
 # See: http://www.sqlalchemy.org/docs/05/reference/sqlalchemy/connections.html#sqlalchemy.create_engine
 connection = sqlite:///trove_test.sqlite
-#connection = mysql://root:e1a2c042c828d3566d0a@localhost/trove
+#connection = mysql+pymysql://root:e1a2c042c828d3566d0a@localhost/trove
 #connection = postgresql://trove:trove@localhost/trove
 
 # Period in seconds after which SQLAlchemy should reestablish its connection

install-guide/source/common_configure.txt

@@ -15,7 +15,7 @@
         notifier_queue_hostname = controller
         ...
         [database]
-        connection = mysql://trove:TROVE_DBPASS@controller/trove
+        connection = mysql+pymysql://trove:TROVE_DBPASS@controller/trove
 
    * Configure the Database service to use the ``RabbitMQ`` message broker
      by setting the following options in each file:
@@ -98,7 +98,7 @@
       # su -s /bin/sh -c "trove-manage db_sync" trove
         ...
         2016-04-06 22:00:17.771 10706 INFO trove.db.sqlalchemy.migration [-]
-        Upgrading mysql://trove:dbaasdb@controller/trove to version latest
+        Upgrading mysql+pymysql://trove:dbaasdb@controller/trove to version latest
 
    .. note::
 

requirements.txt

@@ -35,7 +35,7 @@ oslo.serialization>=1.10.0 # Apache-2.0
 oslo.service>=1.10.0 # Apache-2.0
 oslo.utils>=3.11.0 # Apache-2.0
 oslo.concurrency>=3.8.0 # Apache-2.0
-MySQL-python;python_version=='2.7' # GPL with FOSS exception
+PyMySQL>=0.6.2  # MIT License
 Babel>=2.3.4 # BSD
 six>=1.9.0 # MIT
 stevedore>=1.10.0 # Apache-2.0

trove/db/sqlalchemy/migrate_repo/versions/026_datastore_versions_unique_fix.py

@@ -14,6 +14,7 @@
 
 from migrate.changeset import UniqueConstraint
 from oslo_log import log as logging
+from sqlalchemy.exc import InternalError
 from sqlalchemy.exc import OperationalError
 from sqlalchemy.schema import MetaData
 
@@ -38,7 +39,7 @@ def upgrade(migrate_engine):
     if uc:
         try:
             uc.drop()
-        except OperationalError as e:
+        except (OperationalError, InternalError) as e:
             logger.info(e)
 
 

trove/guestagent/datastore/experimental/pxc/service.py

@@ -56,8 +56,8 @@ class PXCApp(galera_service.GaleraApp):
         LOG.info(_("Generating admin password."))
         admin_password = utils.generate_random_password()
         mysql_service.clear_expired_password()
-        engine = sqlalchemy.create_engine("mysql://root:@localhost:3306",
+        uri = "mysql+pymysql://root:@localhost:3306"
-                                          echo=True)
+        engine = sqlalchemy.create_engine(uri, echo=True)
         with self.local_sql_client(engine) as client:
             self._remove_anonymous_user(client)
             self._create_admin_user(client, admin_password)

trove/guestagent/datastore/galera_common/service.py

@@ -37,8 +37,8 @@ class GaleraApp(service.BaseMySqlApp):
                                         keep_alive_connection_cls)
 
     def _test_mysql(self):
-        engine = sqlalchemy.create_engine("mysql://root:@localhost:3306",
+        uri = "mysql+pymysql://root:@localhost:3306"
-                                          echo=True)
+        engine = sqlalchemy.create_engine(uri, echo=True)
         try:
             with self.local_sql_client(engine) as client:
                 out = client.execute(text("select 1;"))

trove/guestagent/datastore/mysql_common/service.py

@@ -606,8 +606,9 @@ class BaseMySqlApp(object):
             return ENGINE
 
         pwd = self.get_auth_password()
-        ENGINE = sqlalchemy.create_engine("mysql://%s:%s@localhost:3306" %
+        uri = ("mysql+pymysql://%s:%s@localhost:3306"
-                                          (ADMIN_USER_NAME, pwd.strip()),
+               % (ADMIN_USER_NAME, pwd.strip()))
+        ENGINE = sqlalchemy.create_engine(uri,
                                           pool_recycle=7200,
                                           echo=CONF.sql_query_logging,
                                           listeners=[
@@ -682,8 +683,8 @@ class BaseMySqlApp(object):
         LOG.info(_("Generating admin password."))
         admin_password = utils.generate_random_password()
         clear_expired_password()
-        engine = sqlalchemy.create_engine("mysql://root:@localhost:3306",
+        uri = "mysql+pymysql://root:@localhost:3306"
-                                          echo=True)
+        engine = sqlalchemy.create_engine(uri, echo=True)
         with self.local_sql_client(engine) as client:
             self._remove_anonymous_user(client)
             self._create_admin_user(client, admin_password)
@@ -1037,7 +1038,7 @@ class BaseMySqlRootAccess(object):
                 cu = sql_query.CreateUser(user.name, host=user.host)
                 t = text(str(cu))
                 client.execute(t, **cu.keyArgs)
-            except exc.OperationalError as err:
+            except (exc.OperationalError, exc.InternalError) as err:
                 # Ignore, user is already created, just reset the password
                 # TODO(rnirmal): More fine grained error checking later on
                 LOG.debug(err)

trove/tests/db/migrations.py

@@ -150,7 +150,7 @@ class TestTroveMigrations(object):
 
     @test
     def test_mysql_migration(self):
-        db_backend = "mysql+mysqldb"
+        db_backend = "mysql+pymysql"
         # Gracefully skip this test if the developer do not have
         # MySQL running. MySQL should always be available on
         # the infrastructure

trove/tests/unittests/guestagent/test_dbmodels.py

@@ -40,7 +40,7 @@ class MySQLDatabaseTest(trove_testtools.TestCase):
         self.assertEqual(test_name, self.mysqlDb.name)
 
     def test_is_valid_positive(self):
-        self.assertTrue(self.mysqlDb._is_valid('mysqldb'))
+        self.assertTrue(self.mysqlDb._is_valid('pymysql'))
 
     def test_is_valid_negative(self):
         self.assertFalse(self.mysqlDb._is_valid('mysql'))

trove/tests/util/__init__.py

@@ -295,6 +295,6 @@ class LocalSqlClient(object):
 
     @staticmethod
     def init_engine(user, password, host):
-        return create_engine("mysql://%s:%s@%s:3306" %
+        return create_engine("mysql+pymysql://%s:%s@%s:3306" %
                              (user, password, host),
                              pool_recycle=1800, echo=True)

trove/tests/util/mysql.py

@@ -115,7 +115,7 @@ class SqlAlchemyConnection(object):
     @staticmethod
     def _init_engine(user, password, host):
         return session.EngineFacade(
-            "mysql://%s:%s@%s:3306" % (user, password, host),
+            "mysql+pymysql://%s:%s@%s:3306" % (user, password, host),
             pool_recycle=1800, echo=True
         ).get_engine()