From cb3d37c57afa5ee641d01aded9fe5864530fbdc4 Mon Sep 17 00:00:00 2001 From: Maxim Avanov Date: Sat, 29 Jul 2017 11:59:20 +0100 Subject: [PATCH 1/8] Make aiopg installable in PyPy environment --- aiopg/__init__.py | 11 +++++++++++ requirements.txt | 1 - setup.py | 11 +++++++++-- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/aiopg/__init__.py b/aiopg/__init__.py index bf467c93..8847a2f0 100644 --- a/aiopg/__init__.py +++ b/aiopg/__init__.py @@ -1,7 +1,18 @@ +import platform import re import sys from collections import namedtuple + +PY_IMPL = platform.python_implementation() + +# map psycopg2cffi to psycopg2, so any code +# that imports psycopg2 will use psycopg2cffi +if PY_IMPL == 'PyPy': + from psycopg2cffi import compat + compat.register() + + from .connection import connect, Connection, TIMEOUT as DEFAULT_TIMEOUT from .cursor import Cursor from .pool import create_pool, Pool diff --git a/requirements.txt b/requirements.txt index eac3aae0..1c155c21 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,4 +15,3 @@ pytest-sugar==0.8.0 pytest-timeout==1.2.0 sphinxcontrib-asyncio==0.2.0 sqlalchemy==1.1.9 -psycopg2==2.6.2 diff --git a/setup.py b/setup.py index 599d9c0b..19e34e14 100644 --- a/setup.py +++ b/setup.py @@ -1,17 +1,23 @@ import os +import platform import re import sys from setuptools import setup -install_requires = ['psycopg2>=2.5.2'] - PY_VER = sys.version_info +PY_IMPL = platform.python_implementation() if PY_VER < (3, 4): raise RuntimeError("aiopg doesn't suppport Python earlier than 3.4") +if PY_IMPL == 'PyPy': + install_requires = ['psycopg2cffi>=2.7.5'] +else: + install_requires = ['psycopg2>=2.5.2'] + + def read(f): return open(os.path.join(os.path.dirname(__file__), f)).read().strip() @@ -35,6 +41,7 @@ def read_version(): 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: Implementation :: PyPy', 'Operating System :: POSIX', 'Operating System :: MacOS :: MacOS X', 'Operating System :: Microsoft :: Windows', From 61a7e6b84cdb07ccda9d12bffa81ac11c6030871 Mon Sep 17 00:00:00 2001 From: Maxim Avanov Date: Sat, 29 Jul 2017 13:02:38 +0100 Subject: [PATCH 2/8] Add PyPy to TravisCI config, fix flake8 reports --- .travis.yml | 1 + aiopg/__init__.py | 14 ++------------ aiopg/utils.py | 9 +++++++++ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index 848f4238..326956ec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ python: - "3.4" - "3.5" - "3.6" +- "pypy3.5-5.8.0" install: - python setup.py install diff --git a/aiopg/__init__.py b/aiopg/__init__.py index 8847a2f0..dafefffb 100644 --- a/aiopg/__init__.py +++ b/aiopg/__init__.py @@ -1,18 +1,8 @@ -import platform import re import sys from collections import namedtuple - -PY_IMPL = platform.python_implementation() - -# map psycopg2cffi to psycopg2, so any code -# that imports psycopg2 will use psycopg2cffi -if PY_IMPL == 'PyPy': - from psycopg2cffi import compat - compat.register() - - +from .utils import PY_IMPL from .connection import connect, Connection, TIMEOUT as DEFAULT_TIMEOUT from .cursor import Cursor from .pool import create_pool, Pool @@ -53,4 +43,4 @@ def _parse_version(ver): # make pyflakes happy -(connect, create_pool, Connection, Cursor, Pool, DEFAULT_TIMEOUT) +(connect, create_pool, Connection, Cursor, Pool, DEFAULT_TIMEOUT, PY_IMPL) diff --git a/aiopg/utils.py b/aiopg/utils.py index 84a768d2..fcfa09f4 100644 --- a/aiopg/utils.py +++ b/aiopg/utils.py @@ -1,9 +1,11 @@ import asyncio +import platform import sys PY_35 = sys.version_info >= (3, 5) PY_352 = sys.version_info >= (3, 5, 2) +PY_IMPL = platform.python_implementation() if PY_35: from collections.abc import Coroutine @@ -231,3 +233,10 @@ def __exit__(self, *args): coroutines._COROUTINE_TYPES += (_ContextManager,) except: pass + + +# map psycopg2cffi to psycopg2, so any code +# that imports psycopg2 will use psycopg2cffi +if PY_IMPL == 'PyPy': + from psycopg2cffi import compat + compat.register() From 376b6748a3dc7b81ea0c97667926083369b498d8 Mon Sep 17 00:00:00 2001 From: Maxim Avanov Date: Sun, 30 Jul 2017 16:53:09 +0100 Subject: [PATCH 3/8] Add sub-package for PyPy compatibility through psycopg2cffi --- CHANGES.txt | 6 ++++++ aiopg/connection.py | 6 +++--- aiopg/cursor.py | 2 +- aiopg/pool.py | 2 +- aiopg/psycopg2_compat/__init__.py | 7 +++++++ aiopg/psycopg2_compat/extensions.py | 7 +++++++ aiopg/psycopg2_compat/extras.py | 7 +++++++ aiopg/sa/engine.py | 9 +++++++-- aiopg/utils.py | 8 +------- tests/conftest.py | 2 +- tests/test_connection.py | 4 +--- tests/test_cursor.py | 3 +-- tests/test_extended_types.py | 2 +- tests/test_pool.py | 2 +- tests/test_sa_connection.py | 2 +- tests/test_sa_types.py | 2 +- 16 files changed, 47 insertions(+), 24 deletions(-) create mode 100644 aiopg/psycopg2_compat/__init__.py create mode 100644 aiopg/psycopg2_compat/extensions.py create mode 100644 aiopg/psycopg2_compat/extras.py diff --git a/CHANGES.txt b/CHANGES.txt index f3a77af3..9d400c9f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,6 +1,12 @@ CHANGES ------- +release-candidate +^^^^^^^^^^^^^^^^^ + +* Add PyPy support through `psycopg2cffi` #361 + + 0.13.0 (2016-12-02) ^^^^^^^^^^^^^^^^^^^ diff --git a/aiopg/connection.py b/aiopg/connection.py index 8718bec4..6bbe662e 100755 --- a/aiopg/connection.py +++ b/aiopg/connection.py @@ -8,10 +8,10 @@ import weakref import platform -import psycopg2 -from psycopg2.extensions import ( +from . import psycopg2_compat as psycopg2 +from .psycopg2_compat.extensions import ( POLL_OK, POLL_READ, POLL_WRITE, POLL_ERROR) -from psycopg2 import extras +from .psycopg2_compat import extras from .cursor import Cursor from .utils import _ContextManager, PY_35, create_future diff --git a/aiopg/cursor.py b/aiopg/cursor.py index f3db3304..ce0b8b05 100644 --- a/aiopg/cursor.py +++ b/aiopg/cursor.py @@ -1,7 +1,7 @@ import asyncio import warnings -import psycopg2 +from . import psycopg2_compat as psycopg2 from .log import logger from .utils import PY_35, PY_352 diff --git a/aiopg/pool.py b/aiopg/pool.py index e3912834..c9005d0d 100644 --- a/aiopg/pool.py +++ b/aiopg/pool.py @@ -4,7 +4,7 @@ import warnings -from psycopg2.extensions import TRANSACTION_STATUS_IDLE +from .psycopg2_compat.extensions import TRANSACTION_STATUS_IDLE from .connection import connect, TIMEOUT from .log import logger diff --git a/aiopg/psycopg2_compat/__init__.py b/aiopg/psycopg2_compat/__init__.py new file mode 100644 index 00000000..dee1bfc7 --- /dev/null +++ b/aiopg/psycopg2_compat/__init__.py @@ -0,0 +1,7 @@ +from ..utils import IS_PYPY + + +if IS_PYPY: + from psycopg2cffi import * # NOQA +else: + from psycopg2 import * # NOQA diff --git a/aiopg/psycopg2_compat/extensions.py b/aiopg/psycopg2_compat/extensions.py new file mode 100644 index 00000000..9b2a0432 --- /dev/null +++ b/aiopg/psycopg2_compat/extensions.py @@ -0,0 +1,7 @@ +from ..utils import IS_PYPY + + +if IS_PYPY: + from psycopg2cffi.extensions import * # NOQA +else: + from psycopg2.extensions import * # NOQA diff --git a/aiopg/psycopg2_compat/extras.py b/aiopg/psycopg2_compat/extras.py new file mode 100644 index 00000000..551045ee --- /dev/null +++ b/aiopg/psycopg2_compat/extras.py @@ -0,0 +1,7 @@ +from ..utils import IS_PYPY + + +if IS_PYPY: + from psycopg2cffi.extras import * # NOQA +else: + from psycopg2.extras import * # NOQA \ No newline at end of file diff --git a/aiopg/sa/engine.py b/aiopg/sa/engine.py index 6b1cbd49..a3e0f669 100644 --- a/aiopg/sa/engine.py +++ b/aiopg/sa/engine.py @@ -6,10 +6,15 @@ from .connection import SAConnection from .exc import InvalidRequestError from ..connection import TIMEOUT -from ..utils import PY_35, _PoolContextManager, _PoolAcquireContextManager +from ..utils import \ + PY_35, _PoolContextManager, _PoolAcquireContextManager, IS_PYPY try: - from sqlalchemy.dialects.postgresql.psycopg2 import PGDialect_psycopg2 + if IS_PYPY: + from sqlalchemy.dialects.postgresql.psycopg2cffi import \ + PGDialect_psycopg2cffi as PGDialect_psycopg2 + else: + from sqlalchemy.dialects.postgresql.psycopg2 import PGDialect_psycopg2 from sqlalchemy.dialects.postgresql.psycopg2 import PGCompiler_psycopg2 except ImportError: # pragma: no cover raise ImportError('aiopg.sa requires sqlalchemy') diff --git a/aiopg/utils.py b/aiopg/utils.py index fcfa09f4..e60608df 100644 --- a/aiopg/utils.py +++ b/aiopg/utils.py @@ -6,6 +6,7 @@ PY_35 = sys.version_info >= (3, 5) PY_352 = sys.version_info >= (3, 5, 2) PY_IMPL = platform.python_implementation() +IS_PYPY = (PY_IMPL == 'PyPy') if PY_35: from collections.abc import Coroutine @@ -233,10 +234,3 @@ def __exit__(self, *args): coroutines._COROUTINE_TYPES += (_ContextManager,) except: pass - - -# map psycopg2cffi to psycopg2, so any code -# that imports psycopg2 will use psycopg2cffi -if PY_IMPL == 'PyPy': - from psycopg2cffi import compat - compat.register() diff --git a/tests/conftest.py b/tests/conftest.py index 6be30172..95868875 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,7 +3,6 @@ import contextlib import gc import logging -import psycopg2 import pytest import re import socket @@ -15,6 +14,7 @@ from docker import Client as DockerClient import aiopg +from aiopg import psycopg2_compat as psycopg2 from aiopg import sa diff --git a/tests/test_connection.py b/tests/test_connection.py index faab1349..138c2d88 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -1,14 +1,12 @@ import asyncio import aiopg import gc -import psycopg2 -import psycopg2.extras -import psycopg2.extensions import pytest import socket import time import sys +from aiopg import psycopg2_compat as psycopg2 from aiopg.connection import Connection, TIMEOUT from aiopg.cursor import Cursor from aiopg.utils import ensure_future, create_future diff --git a/tests/test_cursor.py b/tests/test_cursor.py index 4d332aba..0ef48968 100644 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -1,9 +1,8 @@ import asyncio -import psycopg2 -import psycopg2.tz import pytest import time +from aiopg import psycopg2_compat as psycopg2 from aiopg.connection import TIMEOUT diff --git a/tests/test_extended_types.py b/tests/test_extended_types.py index c5921e1a..5a3e42b7 100644 --- a/tests/test_extended_types.py +++ b/tests/test_extended_types.py @@ -1,7 +1,7 @@ import asyncio import uuid -from psycopg2.extras import Json +from aiopg.psycopg2_compat.extras import Json @asyncio.coroutine diff --git a/tests/test_pool.py b/tests/test_pool.py index 501e761b..68686993 100644 --- a/tests/test_pool.py +++ b/tests/test_pool.py @@ -3,7 +3,7 @@ import pytest import sys -from psycopg2.extensions import TRANSACTION_STATUS_INTRANS +from aiopg.psycopg2_compat.extensions import TRANSACTION_STATUS_INTRANS import aiopg from aiopg.connection import Connection, TIMEOUT diff --git a/tests/test_sa_connection.py b/tests/test_sa_connection.py index 86631786..c242fde6 100644 --- a/tests/test_sa_connection.py +++ b/tests/test_sa_connection.py @@ -10,7 +10,7 @@ from sqlalchemy import MetaData, Table, Column, Integer, String from sqlalchemy.schema import DropTable, CreateTable -import psycopg2 +from aiopg import psycopg2_compat as psycopg2 meta = MetaData() diff --git a/tests/test_sa_types.py b/tests/test_sa_types.py index 81ae9b8a..d3a44191 100644 --- a/tests/test_sa_types.py +++ b/tests/test_sa_types.py @@ -1,7 +1,7 @@ import asyncio from enum import Enum -import psycopg2 +from aiopg import psycopg2_compat as psycopg2 import pytest sa = pytest.importorskip("aiopg.sa") # noqa From 73b20c257abfc67b6609a11af55c11893b0bd72c Mon Sep 17 00:00:00 2001 From: Maxim Avanov Date: Wed, 2 Aug 2017 00:24:40 +0100 Subject: [PATCH 4/8] Un-pin strict psycopg2 dependency in requirements.txt --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 21d993da..69e7ddc9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,4 +12,3 @@ pytest-sugar==0.8.0 pytest-timeout==1.2.0 sphinxcontrib-asyncio==0.2.0 sqlalchemy==1.1.12 -psycopg2==2.6.2 From d457b264e7749136b804fcbbd8085a01f793bad8 Mon Sep 17 00:00:00 2001 From: Maxim Avanov Date: Wed, 2 Aug 2017 03:05:30 +0100 Subject: [PATCH 5/8] Fix driver name test for PyPy --- tests/test_sa_engine.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_sa_engine.py b/tests/test_sa_engine.py index 2493b813..c3fed2b1 100644 --- a/tests/test_sa_engine.py +++ b/tests/test_sa_engine.py @@ -1,5 +1,6 @@ import asyncio from aiopg.connection import TIMEOUT +from aiopg.utils import IS_PYPY import pytest sa = pytest.importorskip("aiopg.sa") # noqa @@ -35,7 +36,11 @@ def test_name(engine): def test_driver(engine): - assert 'psycopg2' == engine.driver + if IS_PYPY: + driver = 'psycopg2cffi' + else: + driver = 'psycopg2' + assert driver == engine.driver def test_dsn(engine, pg_params): From 83bfd6bd009eaf7ab87fd1fa3dee9ef4dcc1d5d5 Mon Sep 17 00:00:00 2001 From: Maxim Avanov Date: Sun, 20 Aug 2017 00:10:37 +0100 Subject: [PATCH 6/8] Cursor.nextset() raises NotImplementedError with psycopg2cffi --- aiopg/cursor.py | 4 +++- tests/test_cursor.py | 8 +++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/aiopg/cursor.py b/aiopg/cursor.py index ce0b8b05..1572bdfe 100644 --- a/aiopg/cursor.py +++ b/aiopg/cursor.py @@ -338,7 +338,9 @@ def tzinfo_factory(self, val): @asyncio.coroutine def nextset(self): # Not supported - self._impl.nextset() # raises psycopg2.NotSupportedError + # raises psycopg2.NotSupportedError on CPython + # raises NotImplementedError on PyPy + self._impl.nextset() @asyncio.coroutine def setoutputsize(self, size, column=None): diff --git a/tests/test_cursor.py b/tests/test_cursor.py index 0ef48968..a8a910e9 100644 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -3,6 +3,7 @@ import time from aiopg import psycopg2_compat as psycopg2 +from aiopg.utils import IS_PYPY from aiopg.connection import TIMEOUT @@ -248,9 +249,14 @@ def test_tzinfo_factory(connect): @asyncio.coroutine def test_nextset(connect): + if IS_PYPY: + exception_type = NotImplementedError + else: + exception_type = psycopg2.NotSupportedError + conn = yield from connect() cur = yield from conn.cursor() - with pytest.raises(psycopg2.NotSupportedError): + with pytest.raises(exception_type): yield from cur.nextset() From 04d84a5e45bca4a0bc9a83527b3518292e0b85b5 Mon Sep 17 00:00:00 2001 From: Maxim Avanov Date: Sun, 20 Aug 2017 00:25:35 +0100 Subject: [PATCH 7/8] psycopg2cffi doesn't obfuscate passwords in dsn at the moment --- tests/test_sa_engine.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_sa_engine.py b/tests/test_sa_engine.py index c3fed2b1..7624400b 100644 --- a/tests/test_sa_engine.py +++ b/tests/test_sa_engine.py @@ -44,10 +44,13 @@ def test_driver(engine): def test_dsn(engine, pg_params): - pg_params['password'] = 'x' * len(pg_params['password']) + if not IS_PYPY: + pg_params['password'] = 'x' * len(pg_params['password']) + dsn = ('dbname={database} user={user} password={password} ' 'host={host} port={port}').format_map(pg_params) - assert dsn == engine.dsn + + assert sorted(dsn.split()) == sorted(engine.dsn.split()) def test_minsize(engine): From 8535cbec3400329a00d1a8e79ddbe717f891f00c Mon Sep 17 00:00:00 2001 From: Maxim Avanov Date: Sun, 20 Aug 2017 01:11:35 +0100 Subject: [PATCH 8/8] Fix pool.__del__ check on PyPy --- tests/test_pool.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_pool.py b/tests/test_pool.py index 68686993..6fd0ece2 100644 --- a/tests/test_pool.py +++ b/tests/test_pool.py @@ -2,12 +2,14 @@ from unittest import mock import pytest import sys +import gc from aiopg.psycopg2_compat.extensions import TRANSACTION_STATUS_INTRANS import aiopg from aiopg.connection import Connection, TIMEOUT from aiopg.pool import Pool +from aiopg.utils import IS_PYPY @asyncio.coroutine @@ -474,6 +476,11 @@ def test___del__(loop, pg_params, warning): pool = yield from aiopg.create_pool(loop=loop, **pg_params) with warning(ResourceWarning): del pool + if IS_PYPY: + # PyPy's GC is not based on reference counting, and the objects + # are not freed instantly when they are no longer reachable. + # Therefore, we explicitly collect unreachable objects here. + gc.collect() @asyncio.coroutine