MariaDB dialect implementation

Fixes: #5459

Closes: #5515
Pull-request: https://github.com/sqlalchemy/sqlalchemy/pull/5515
Pull-request-sha: 760090b906

Change-Id: I30e8fbc02b7b5329ca228cd39f6fb7cfd0e43092

README.unittests.rst

@@ -85,6 +85,7 @@ a pre-set URL.  These can be seen using --dbs::
     Available --db options (use --dburi to override)
                  default    sqlite:///:memory:
                 firebird    firebird://sysdba:masterkey@localhost//Users/classic/foo.fdb
+                 mariadb    mariadb://scott:tiger@192.168.0.199:3307/test
                    mssql    mssql+pyodbc://scott:tiger^5HHH@mssql2017:1433/test?driver=ODBC+Driver+13+for+SQL+Server
            mssql_pymssql    mssql+pymssql://scott:tiger@ms_2008
                    mysql    mysql://scott:tiger@127.0.0.1:3306/test?charset=utf8mb4
@@ -260,6 +261,21 @@ intended for production use!
     # To stop the container. It will also remove it.
     docker stop mysql
 
+**MariaDB configuration**::
+
+    # only needed if a local image of MariaDB is not already present
+    docker pull mariadb
+
+    # create the container with the proper configuration for sqlalchemy
+    docker run --rm -e MYSQL_USER='scott' -e MYSQL_PASSWORD='tiger' -e MYSQL_DATABASE='test' -e MYSQL_ROOT_PASSWORD='password' -p 127.0.0.1:3306:3306 -d --name mariadb mariadb --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci
+
+    # configure the database
+    sleep 20
+    docker exec -ti mariadb mysql -u root -ppassword -D test -w -e "GRANT ALL ON *.* TO scott@'%'; CREATE DATABASE test_schema CHARSET utf8mb4; CREATE DATABASE test_schema_2 CHARSET utf8mb4;"
+
+    # To stop the container. It will also remove it.
+    docker stop mariadb
+
 **MSSQL configuration**::
 
     # only needed if a local image of mssql is not already present

doc/build/changelog/unreleased_14/5459.rst

@@ -0,0 +1,6 @@
+.. change::
+    :tags: engine, change
+    :tickets: 5459
+
+    Added support for MariaDB Connector/Python to the mysql dialect. Original
+    pull request courtesy Georg Richter.

lib/sqlalchemy/dialects/mysql/__init__.py

@@ -7,6 +7,7 @@
 
 from . import base  # noqa
 from . import cymysql  # noqa
+from . import mariadbconnector  # noqa
 from . import mysqlconnector  # noqa
 from . import mysqldb  # noqa
 from . import oursql  # noqa

lib/sqlalchemy/dialects/mysql/mariadbconnector.py

@@ -0,0 +1,231 @@
+# mysql/mariadbconnector.py
+# Copyright (C) 2020 the SQLAlchemy authors and contributors
+# <see AUTHORS file>
+#
+# This module is part of SQLAlchemy and is released under
+# the MIT License: http://www.opensource.org/licenses/mit-license.php
+
+"""
+
+.. dialect:: mysql+mariadbconnector
+    :name: MariaDB Connector/Python
+    :dbapi: mariadb
+    :connectstring: mariadb+mariadbconnector://<user>:<password>@<host>[:<port>]/<dbname>
+    :url: https://pypi.org/project/mariadb/
+
+Driver Status
+-------------
+
+MariaDB Connector/Python enables Python programs to access MariaDB and MySQL
+databases using an API which is compliant with the Python DB API 2.0 (PEP-249).
+It is written in C and uses MariaDB Connector/C client library for client server
+communication.
+
+Note that the default driver for a ``mariadb://`` connection URI continues to
+be ``mysqldb``. ``mariadb+mariadbconnector://`` is required to use this driver.
+
+.. mariadb: https://github.com/mariadb-corporation/mariadb-connector-python
+
+"""  # noqa
+import re
+
+from .base import MySQLCompiler
+from .base import MySQLDialect
+from .base import MySQLExecutionContext
+from .base import MySQLIdentifierPreparer
+from ... import sql
+from ... import util
+
+mariadb_cpy_minimum_version = (1, 0, 1)
+
+
+class MySQLExecutionContext_mariadbconnector(MySQLExecutionContext):
+    pass
+
+
+class MySQLCompiler_mariadbconnector(MySQLCompiler):
+    pass
+
+
+class MySQLIdentifierPreparer_mariadbconnector(MySQLIdentifierPreparer):
+    pass
+
+
+class MySQLDialect_mariadbconnector(MySQLDialect):
+    driver = "mariadbconnector"
+
+    # set this to True at the module level to prevent the driver from running
+    # against a backend that server detects as MySQL. currently this appears to
+    # be unnecessary as MariaDB client libraries have always worked against
+    # MySQL databases.   However, if this changes at some point, this can be
+    # adjusted, but PLEASE ADD A TEST in test/dialect/mysql/test_dialect.py if
+    # this change is made at some point to ensure the correct exception
+    # is raised at the correct point when running the driver against
+    # a MySQL backend.
+    # is_mariadb = True
+
+    supports_unicode_statements = True
+    encoding = "utf8mb4"
+    convert_unicode = True
+    supports_sane_rowcount = True
+    supports_sane_multi_rowcount = True
+    supports_native_decimal = True
+    default_paramstyle = "qmark"
+    execution_ctx_cls = MySQLExecutionContext_mariadbconnector
+    statement_compiler = MySQLCompiler_mariadbconnector
+    preparer = MySQLIdentifierPreparer_mariadbconnector
+
+    @util.memoized_property
+    def _dbapi_version(self):
+        if self.dbapi and hasattr(self.dbapi, "__version__"):
+            return tuple(
+                [
+                    int(x)
+                    for x in re.findall(
+                        r"(\d+)(?:[-\.]?|$)", self.dbapi.__version__
+                    )
+                ]
+            )
+        else:
+            return (99, 99, 99)
+
+    def __init__(self, server_side_cursors=False, **kwargs):
+        super(MySQLDialect_mariadbconnector, self).__init__(**kwargs)
+        self.server_side_cursors = True
+        self.paramstyle = "qmark"
+        if self.dbapi is not None:
+            if self._dbapi_version < mariadb_cpy_minimum_version:
+                raise NotImplementedError(
+                    "The minimum required version for MariaDB "
+                    "Connector/Python is %s"
+                    % ".".join(str(x) for x in mariadb_cpy_minimum_version)
+                )
+
+    @classmethod
+    def dbapi(cls):
+        return __import__("mariadb")
+
+    def is_disconnect(self, e, connection, cursor):
+        if super(MySQLDialect_mariadbconnector, self).is_disconnect(
+            e, connection, cursor
+        ):
+            return True
+        elif isinstance(e, self.dbapi.Error):
+            str_e = str(e).lower()
+            return "not connected" in str_e or "isn't valid" in str_e
+        else:
+            return False
+
+    def create_connect_args(self, url):
+        opts = url.translate_connect_args()
+
+        int_params = [
+            "connect_timeout",
+            "read_timeout",
+            "write_timeout",
+            "client_flag",
+            "port",
+            "pool_size",
+        ]
+        bool_params = [
+            "local_infile",
+            "ssl_verify_cert",
+            "ssl",
+            "pool_reset_connection",
+        ]
+
+        for key in int_params:
+            util.coerce_kw_type(opts, key, int)
+        for key in bool_params:
+            util.coerce_kw_type(opts, key, bool)
+
+        # FOUND_ROWS must be set in CLIENT_FLAGS to enable
+        # supports_sane_rowcount.
+        client_flag = opts.get("client_flag", 0)
+        if self.dbapi is not None:
+            try:
+                CLIENT_FLAGS = __import__(
+                    self.dbapi.__name__ + ".constants.CLIENT"
+                ).constants.CLIENT
+                client_flag |= CLIENT_FLAGS.FOUND_ROWS
+            except (AttributeError, ImportError):
+                self.supports_sane_rowcount = False
+            opts["client_flag"] = client_flag
+        return [[], opts]
+
+    def _extract_error_code(self, exception):
+        try:
+            rc = exception.errno
+        except:
+            rc = -1
+        return rc
+
+    def _detect_charset(self, connection):
+        return "utf8mb4"
+
+    _isolation_lookup = set(
+        [
+            "SERIALIZABLE",
+            "READ UNCOMMITTED",
+            "READ COMMITTED",
+            "REPEATABLE READ",
+            "AUTOCOMMIT",
+        ]
+    )
+
+    def _set_isolation_level(self, connection, level):
+        if level == "AUTOCOMMIT":
+            connection.autocommit = True
+        else:
+            connection.autocommit = False
+            super(MySQLDialect_mariadbconnector, self)._set_isolation_level(
+                connection, level
+            )
+
+    def do_begin_twophase(self, connection, xid):
+        connection.execute(
+            sql.text("XA BEGIN :xid").bindparams(
+                sql.bindparam("xid", xid, literal_execute=True)
+            )
+        )
+
+    def do_prepare_twophase(self, connection, xid):
+        connection.execute(
+            sql.text("XA END :xid").bindparams(
+                sql.bindparam("xid", xid, literal_execute=True)
+            )
+        )
+        connection.execute(
+            sql.text("XA PREPARE :xid").bindparams(
+                sql.bindparam("xid", xid, literal_execute=True)
+            )
+        )
+
+    def do_rollback_twophase(
+        self, connection, xid, is_prepared=True, recover=False
+    ):
+        if not is_prepared:
+            connection.execute(
+                sql.text("XA END :xid").bindparams(
+                    sql.bindparam("xid", xid, literal_execute=True)
+                )
+            )
+        connection.execute(
+            sql.text("XA ROLLBACK :xid").bindparams(
+                sql.bindparam("xid", xid, literal_execute=True)
+            )
+        )
+
+    def do_commit_twophase(
+        self, connection, xid, is_prepared=True, recover=False
+    ):
+        if not is_prepared:
+            self.do_prepare_twophase(connection, xid)
+        connection.execute(
+            sql.text("XA COMMIT :xid").bindparams(
+                sql.bindparam("xid", xid, literal_execute=True)
+            )
+        )
+
+
+dialect = MySQLDialect_mariadbconnector

lib/sqlalchemy/dialects/mysql/provision.py

@@ -10,6 +10,18 @@ from ...testing.provision import temp_table_keyword_args
 def generate_driver_url(url, driver, query_str):
     backend = url.get_backend_name()
 
+    # NOTE: at the moment, tests are running mariadbconnector
+    # against both mariadb and mysql backends.   if we want this to be
+    # limited, do the decisionmaking here to reject a "mysql+mariadbconnector"
+    # URL.  Optionally also re-enable the module level
+    # MySQLDialect_mariadbconnector.is_mysql flag as well, which must include
+    # a unit and/or functional test.
+
+    # all the Jenkins tests have been running mysqlclient Python library
+    # built against mariadb client drivers for years against all MySQL /
+    # MariaDB versions going back to MySQL 5.6, currently they can talk
+    # to MySQL databases without problems.
+
     if backend == "mysql":
         dialect_cls = url.get_dialect()
         if dialect_cls._is_mariadb_from_url(url):

setup.cfg

@@ -50,6 +50,10 @@ mssql_pyodbc = pyodbc
 mysql =
     mysqlclient>=1.4.0,<2;python_version<"3"
     mysqlclient>=1.4.0;python_version>="3"
+mysql_connector =
+    mysqlconnector
+mariadb_connector =
+    mariadb>=1.0.1;python_version>="3"
 oracle =
     cx_oracle>=7,<8;python_version<"3"
     cx_oracle>=7;python_version>="3"
@@ -120,6 +124,7 @@ pg8000 = postgresql+pg8000://scott:tiger@127.0.0.1:5432/test
 postgresql_psycopg2cffi = postgresql+psycopg2cffi://scott:tiger@127.0.0.1:5432/test
 mysql = mysql://scott:tiger@127.0.0.1:3306/test?charset=utf8mb4
 pymysql = mysql+pymysql://scott:tiger@127.0.0.1:3306/test?charset=utf8mb4
+mariadb = mariadb://scott:tiger@127.0.0.1:3306/test
 mssql = mssql+pyodbc://scott:tiger^5HHH@mssql2017:1433/test?driver=ODBC+Driver+13+for+SQL+Server
 mssql_pymssql = mssql+pymssql://scott:tiger@ms_2008
 docker_mssql = mssql+pymssql://scott:tiger^5HHH@127.0.0.1:1433/test

test/engine/test_execute.py

@@ -453,7 +453,7 @@ class ExecuteTest(fixtures.TablesTest):
             try:
                 cursor = raw.cursor()
                 cursor.execute("SELECTINCORRECT")
-            except testing.db.dialect.dbapi.DatabaseError as orig:
+            except testing.db.dialect.dbapi.Error as orig:
                 # py3k has "orig" in local scope...
                 the_orig = orig
         finally:
@@ -1506,6 +1506,7 @@ class EngineEventsTest(fixtures.TestBase):
         with e1.connect() as conn:
 
             result = conn.exec_driver_sql(stmt)
+            eq_(result.scalar(), 1)
 
         ctx = result.context
         eq_(

test/engine/test_reconnect.py

@@ -1356,6 +1356,7 @@ class InvalidateDuringResultTest(fixtures.TestBase):
     )
     @testing.fails_if(
         [
+            "+mariadbconnector",
             "+mysqlconnector",
             "+mysqldb",
             "+cymysql",

tox.ini

@@ -25,6 +25,12 @@ deps=pytest!=3.9.1,!=3.9.2
      postgresql: .[postgresql_pg8000]; python_version >= '3'
      mysql: .[mysql]
      mysql: .[pymysql]
+     mysql: .[mariadb_connector]; python_version >= '3'
+
+     # we should probably try to get mysql_connector back in the mix
+     # as well
+     # mysql: .[mysql_connector]; python_version >= '3'
+
      oracle: .[oracle]
      mssql: .[mssql]
 
@@ -71,6 +77,7 @@ setenv=
 
     mysql: MYSQL={env:TOX_MYSQL:--db mysql}
     mysql: EXTRA_MYSQL_DRIVERS={env:EXTRA_MYSQL_DRIVERS:--dbdriver mysqldb --dbdriver pymysql}
+    py3{,5,6,7,8,9,10,11}-mysql: EXTRA_MYSQL_DRIVERS={env:EXTRA_MYSQL_DRIVERS:--dbdriver mysqldb --dbdriver pymysql --dbdriver mariadbconnector}
 
 
     mssql: MSSQL={env:TOX_MSSQL:--db mssql}