From 7168c6f77d5da48b9d1ccb7a252c518c8d5b80fe Mon Sep 17 00:00:00 2001 From: Matthew Wardrop Date: Thu, 24 Mar 2016 13:02:51 -0700 Subject: [PATCH] Initial support for Python 3 with Presto. Closes #45 (modified by jingw) --- .travis.yml | 39 +++++++++++++++++--------- TCLIService/TCLIService.py | 2 +- TCLIService/constants.py | 2 +- pyhive/__init__.py | 2 ++ pyhive/common.py | 22 ++++++++++----- pyhive/exc.py | 4 +-- pyhive/presto.py | 7 ++++- pyhive/sqlalchemy_presto.py | 2 +- pyhive/tests/dbapi_test_case.py | 13 +++++---- pyhive/tests/sqlachemy_test_case.py | 6 ++-- pyhive/tests/test_hive.py | 2 ++ pyhive/tests/test_presto.py | 2 +- pyhive/tests/test_sqlalchemy_hive.py | 3 ++ pyhive/tests/test_sqlalchemy_presto.py | 3 +- scripts/travis-install.sh | 3 +- setup.py | 3 ++ 16 files changed, 77 insertions(+), 38 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4666c4aa..ccef46cc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,18 +1,31 @@ sudo: required language: python -env: - # newest everything - - CDH=cdh5 PRESTO=0.142 SQLALCHEMY=1.0.12 - # stale stuff we're still using / supporting - - CDH=cdh5 PRESTO=0.118 SQLALCHEMY=0.5.8 - # every version of sqlalchemy with special code - - CDH=cdh4 PRESTO=0.118 SQLALCHEMY=0.5.8 - - CDH=cdh4 PRESTO=0.118 SQLALCHEMY=0.6.9 - - CDH=cdh4 PRESTO=0.118 SQLALCHEMY=0.7.10 - - CDH=cdh4 PRESTO=0.118 SQLALCHEMY=0.8.7 - - CDH=cdh4 PRESTO=0.118 SQLALCHEMY=1.0.8 -python: - - "2.7" +matrix: + include: + # newest dependencies + a few python versions + - python: 3.5 + env: CDH=cdh5 PRESTO=0.142 SQLALCHEMY=1.0.12 + - python: 3.4 + env: CDH=cdh5 PRESTO=0.142 SQLALCHEMY=1.0.12 + - python: 3.3 + env: CDH=cdh5 PRESTO=0.142 SQLALCHEMY=1.0.12 + - python: 2.7 + env: CDH=cdh5 PRESTO=0.142 SQLALCHEMY=1.0.12 + # stale stuff we're still using / supporting + - python: 2.7 + env: CDH=cdh5 PRESTO=0.118 SQLALCHEMY=0.5.8 + # old python + every version of sqlalchemy with special code + - python: 2.7 + env: CDH=cdh4 PRESTO=0.118 SQLALCHEMY=0.5.8 + - python: 2.7 + env: CDH=cdh4 PRESTO=0.118 SQLALCHEMY=0.6.9 + - python: 2.7 + env: CDH=cdh4 PRESTO=0.118 SQLALCHEMY=0.7.10 + - python: 2.7 + env: CDH=cdh4 PRESTO=0.118 SQLALCHEMY=0.8.7 + - python: 2.7 + env: CDH=cdh4 PRESTO=0.118 SQLALCHEMY=1.0.8 + # exclude: python 3 against old libries before_install: - ./scripts/travis-install.sh - pip install codecov diff --git a/TCLIService/TCLIService.py b/TCLIService/TCLIService.py index b938513c..3479b230 100644 --- a/TCLIService/TCLIService.py +++ b/TCLIService/TCLIService.py @@ -7,7 +7,7 @@ # from thrift.Thrift import TType, TMessageType, TException, TApplicationException -from ttypes import * +from .ttypes import * from thrift.Thrift import TProcessor from thrift.transport import TTransport from thrift.protocol import TBinaryProtocol, TProtocol diff --git a/TCLIService/constants.py b/TCLIService/constants.py index eab0b1bd..2f34533e 100644 --- a/TCLIService/constants.py +++ b/TCLIService/constants.py @@ -7,7 +7,7 @@ # from thrift.Thrift import TType, TMessageType, TException, TApplicationException -from ttypes import * +from .ttypes import * PRIMITIVE_TYPES = set([ 0, diff --git a/pyhive/__init__.py b/pyhive/__init__.py index 124e4620..2461d3c4 100644 --- a/pyhive/__init__.py +++ b/pyhive/__init__.py @@ -1 +1,3 @@ +from __future__ import absolute_import +from __future__ import unicode_literals __version__ = '0.1.7' diff --git a/pyhive/common.py b/pyhive/common.py index 539e79be..5671ceec 100644 --- a/pyhive/common.py +++ b/pyhive/common.py @@ -5,15 +5,21 @@ from __future__ import absolute_import from __future__ import unicode_literals +from builtins import bytes +from builtins import int +from builtins import object +from builtins import range +from past.builtins import basestring from pyhive import exc import abc import collections +import sys import time +from future.utils import with_metaclass -class DBAPICursor(object): +class DBAPICursor(with_metaclass(abc.ABCMeta, object)): """Base class for some common DB-API logic""" - __metaclass__ = abc.ABCMeta _STATE_NONE = 0 _STATE_RUNNING = 1 @@ -119,7 +125,7 @@ def fetchmany(self, size=None): if size is None: size = self.arraysize result = [] - for _ in xrange(size): + for _ in range(size): one = self.fetchone() if one is None: break @@ -176,7 +182,7 @@ def rownumber(self): """ return self._rownumber - def next(self): + def __next__(self): """Return the next row from the currently executing SQL statement using the same semantics as :py:meth:`fetchone`. A ``StopIteration`` exception is raised when the result set is exhausted. @@ -187,6 +193,8 @@ def next(self): else: return one + next = __next__ + def __iter__(self): """Return self to make cursors compatible to the iteration protocol.""" return self @@ -209,7 +217,7 @@ def __cmp__(self, other): class ParamEscaper(object): def escape_args(self, parameters): if isinstance(parameters, dict): - return {k: self.escape_item(v) for k, v in parameters.iteritems()} + return {k: self.escape_item(v) for k, v in parameters.items()} elif isinstance(parameters, (list, tuple)): return tuple(self.escape_item(x) for x in parameters) else: @@ -223,7 +231,7 @@ def escape_string(self, item): # Newer SQLAlchemy checks dialect.supports_unicode_binds before encoding Unicode strings # as byte strings. The old version always encodes Unicode as byte strings, which breaks # string formatting here. - if isinstance(item, str): + if isinstance(item, bytes): item = item.decode('utf-8') # This is good enough when backslashes are literal, newlines are just followed, and the way # to escape a single quote is to put two single quotes. @@ -231,7 +239,7 @@ def escape_string(self, item): return "'{}'".format(item.replace("'", "''")) def escape_item(self, item): - if isinstance(item, (int, long, float)): + if isinstance(item, (int, float)): return self.escape_number(item) elif isinstance(item, basestring): return self.escape_string(item) diff --git a/pyhive/exc.py b/pyhive/exc.py index f4b74fb8..931cf211 100644 --- a/pyhive/exc.py +++ b/pyhive/exc.py @@ -10,7 +10,7 @@ ] -class Error(StandardError): +class Error(Exception): """Exception that is the base class of all other error exceptions. You can use this to catch all errors with one single except statement. @@ -18,7 +18,7 @@ class Error(StandardError): pass -class Warning(StandardError): +class Warning(Exception): """Exception raised for important warnings like data truncations while inserting, etc.""" pass diff --git a/pyhive/presto.py b/pyhive/presto.py index fd3fbfdc..cef13d8b 100644 --- a/pyhive/presto.py +++ b/pyhive/presto.py @@ -7,6 +7,7 @@ from __future__ import absolute_import from __future__ import unicode_literals +from builtins import object from pyhive import common from pyhive.common import DBAPITypeObject # Make all exceptions visible in this module per DB-API @@ -15,7 +16,11 @@ import getpass import logging import requests -import urlparse + +try: # Python 3 + import urllib.parse as urlparse +except ImportError: # Python 2 + import urlparse # PEP 249 module globals diff --git a/pyhive/sqlalchemy_presto.py b/pyhive/sqlalchemy_presto.py index 28ecaf64..ef45ab4b 100644 --- a/pyhive/sqlalchemy_presto.py +++ b/pyhive/sqlalchemy_presto.py @@ -97,7 +97,7 @@ def _get_table_columns(self, connection, table_name, schema): # call. SQLAlchemy doesn't handle this. Thus, we catch the unwrapped # presto.DatabaseError here. # Does the table exist? - msg = e.message.get('message') if isinstance(e.message, dict) else None + msg = e.args[0].get('message') if len(e.args) > 0 and isinstance(e.args[0], dict) else None regex = r"Table\ \'.*{}\'\ does\ not\ exist".format(re.escape(table_name)) if msg and re.search(regex, msg): raise exc.NoSuchTableError(table_name) diff --git a/pyhive/tests/dbapi_test_case.py b/pyhive/tests/dbapi_test_case.py index aefcbbbc..fc1dd3a9 100644 --- a/pyhive/tests/dbapi_test_case.py +++ b/pyhive/tests/dbapi_test_case.py @@ -3,6 +3,9 @@ from __future__ import absolute_import from __future__ import unicode_literals +from builtins import object +from builtins import range +from future.utils import with_metaclass from pyhive import exc import abc import contextlib @@ -22,9 +25,7 @@ def wrapped_fn(self, *args, **kwargs): return wrapped_fn -class DBAPITestCase(object): - __metaclass__ = abc.ABCMeta - +class DBAPITestCase(with_metaclass(abc.ABCMeta, object)): @abc.abstractmethod def connect(self): raise NotImplementedError # pragma: no cover @@ -42,13 +43,13 @@ def test_fetchall(self, cursor): cursor.execute('SELECT * FROM one_row') self.assertEqual(cursor.fetchall(), [[1]]) cursor.execute('SELECT a FROM many_rows ORDER BY a') - self.assertEqual(cursor.fetchall(), [[i] for i in xrange(10000)]) + self.assertEqual(cursor.fetchall(), [[i] for i in range(10000)]) @with_cursor def test_iterator(self, cursor): cursor.execute('SELECT * FROM one_row') self.assertEqual(list(cursor), [[1]]) - self.assertRaises(StopIteration, cursor.next) + self.assertRaises(StopIteration, cursor.__next__) @with_cursor def test_description_initial(self, cursor): @@ -80,7 +81,7 @@ def test_executemany(self, cursor): for length in 1, 2: cursor.executemany( 'SELECT %(x)d FROM one_row', - [{'x': i} for i in xrange(1, length + 1)] + [{'x': i} for i in range(1, length + 1)] ) self.assertEqual(cursor.fetchall(), [[length]]) diff --git a/pyhive/tests/sqlachemy_test_case.py b/pyhive/tests/sqlachemy_test_case.py index 2f704d0c..bca8beea 100644 --- a/pyhive/tests/sqlachemy_test_case.py +++ b/pyhive/tests/sqlachemy_test_case.py @@ -1,7 +1,9 @@ # coding: utf-8 from __future__ import absolute_import from __future__ import unicode_literals +from builtins import object from distutils.version import StrictVersion +from future.utils import with_metaclass import sqlalchemy from sqlalchemy.exc import NoSuchTableError from sqlalchemy.schema import Index @@ -30,9 +32,7 @@ def wrapped_fn(self, *args, **kwargs): return wrapped_fn -class SqlAlchemyTestCase(object): - __metaclass__ = abc.ABCMeta - +class SqlAlchemyTestCase(with_metaclass(abc.ABCMeta, object)): @with_engine_connection def test_basic_query(self, engine, connection): rows = connection.execute('SELECT * FROM one_row').fetchall() diff --git a/pyhive/tests/test_hive.py b/pyhive/tests/test_hive.py index 7daea404..55443161 100644 --- a/pyhive/tests/test_hive.py +++ b/pyhive/tests/test_hive.py @@ -14,10 +14,12 @@ import mock import os import unittest +import sys _HOST = 'localhost' +@unittest.skipIf(sys.version_info.major == 3, 'Hive not yet supported on Python 3') class TestHive(unittest.TestCase, DBAPITestCase): __test__ = True diff --git a/pyhive/tests/test_presto.py b/pyhive/tests/test_presto.py index 63963b7b..6d444ec7 100644 --- a/pyhive/tests/test_presto.py +++ b/pyhive/tests/test_presto.py @@ -66,7 +66,7 @@ def test_complex(self, cursor): 0.25, 'a string', '1970-01-01 00:00:00.000', - '123', + b'123', [1, 2], {"1": 2, "3": 4}, # Presto converts all keys to strings so that they're valid JSON [1, 2], # struct is returned as a list of elements diff --git a/pyhive/tests/test_sqlalchemy_hive.py b/pyhive/tests/test_sqlalchemy_hive.py index 13825b69..404ef0e0 100644 --- a/pyhive/tests/test_sqlalchemy_hive.py +++ b/pyhive/tests/test_sqlalchemy_hive.py @@ -1,5 +1,6 @@ from __future__ import absolute_import from __future__ import unicode_literals +from builtins import str from distutils.version import StrictVersion from pyhive.sqlalchemy_hive import HiveDate from pyhive.sqlalchemy_hive import HiveDecimal @@ -15,6 +16,7 @@ import decimal import os import sqlalchemy.types +import sys import unittest _ONE_ROW_COMPLEX_CONTENTS = [ @@ -36,6 +38,7 @@ ] +@unittest.skipIf(sys.version_info.major == 3, 'Hive not yet supported on Python 3') class TestSqlAlchemyHive(unittest.TestCase, SqlAlchemyTestCase): def create_engine(self): return create_engine('hive://localhost:10000/default') diff --git a/pyhive/tests/test_sqlalchemy_presto.py b/pyhive/tests/test_sqlalchemy_presto.py index a87679a0..41826540 100644 --- a/pyhive/tests/test_sqlalchemy_presto.py +++ b/pyhive/tests/test_sqlalchemy_presto.py @@ -1,5 +1,6 @@ from __future__ import absolute_import from __future__ import unicode_literals +from builtins import str from pyhive.tests.sqlachemy_test_case import SqlAlchemyTestCase from pyhive.tests.sqlachemy_test_case import with_engine_connection from sqlalchemy.engine import create_engine @@ -40,7 +41,7 @@ def test_reflect_select(self, engine, connection): 0.25, 'a string', '1970-01-01 00:00:00.000', - '123', + b'123', [1, 2], {"1": 2, "3": 4}, # Presto converts all keys to strings so that they're valid JSON [1, 2], # struct is returned as a list of elements diff --git a/scripts/travis-install.sh b/scripts/travis-install.sh index 95ac48b1..78546336 100755 --- a/scripts/travis-install.sh +++ b/scripts/travis-install.sh @@ -22,6 +22,7 @@ sudo -Eu hive $(dirname $0)/make_test_tables.sh # Presto # +sudo apt-get install -y python # Use python2 for presto server sudo apt-get install -y oracle-java8-installer sudo update-java-alternatives -s java-8-oracle @@ -36,4 +37,4 @@ then sed -i '/query.max-memory/d' presto-server-$PRESTO/etc/config.properties fi -./presto-server-$PRESTO/bin/launcher start +/usr/bin/python2.7 presto-server-$PRESTO/bin/launcher.py start diff --git a/setup.py b/setup.py index 724da08e..40cff6aa 100755 --- a/setup.py +++ b/setup.py @@ -37,6 +37,9 @@ def run_tests(self): "Operating System :: OS Independent", "Topic :: Database :: Front-Ends", ], + install_requires=[ + 'future', + ], extras_require={ "Presto": ['requests>=1.0.0'], "Hive": ['sasl>=0.1.3', 'thrift>=0.8.0', 'thrift_sasl>=0.1.0'],