diff --git a/docs/blobs.rst b/docs/blobs.rst deleted file mode 100644 index 365865eb..00000000 --- a/docs/blobs.rst +++ /dev/null @@ -1,162 +0,0 @@ -.. _blobs: - -===== -Blobs -===== - -The CrateDB Python client library provides full access to the powerful -:ref:`blob storage capabilities ` of your -CrateDB cluster. - -.. rubric:: Table of contents - -.. contents:: - :local: - -Get a blob container -==================== - -The first thing you will need to do is connect to CrateDB. Follow the -instructions in the :ref:`connection document ` for more detailed -information. - -For the sake of this example, we will do the following: - - >>> from crate import client - >>> connection = client.connect("http://localhost:4200/") - -This is a simple connection that connects to a CrateDB node running on -the local host with the :ref:`crate-reference:interface-http` listening -on port 4200 (the default). - -To work with blobs in CrateDB, you must specifically create -:ref:`blob tables `. - -The CrateDB Python client allows you to interact with these blob tables via a -blob container, which you can create like this: - - >>> blob_container = connection.get_blob_container('my_blobs') - >>> blob_container - - -Here, we have created a ``BlobContainer`` for the ``my_blobs`` table, using -``connection`` object. - -Now we can start working with our blob container. - -Working with the blob container -=============================== - -Upload blobs ------------- - -The blob container can work with files or *file-like objects*, as long as -produce bytes when read. - -What is a file-like object? Well, to put it simply, any object that provides a -``read()`` method. - -The stream objects provided by the Python standard library :mod:`py:io` and -:mod:`py:tempfile` modules are the most commonly used file-like objects. - -The :class:`py:io.StringIO` class is not suitable, as it produces Unicode strings when -read. But you can easily encode a Unicode string and feed it to a :class:`py:io.BytesIO` -object. - -Here's a trivial example: - - >>> import io - >>> bytestream = "An example sentence.".encode("utf8") - >>> file = io.BytesIO(bytestream) - -This file can then be uploaded to the blob table using the ``put`` method: - - >>> blob_container.put(file) - '6f10281ad07d4a35c6ec2f993e6376032b77181d' - -Notice that this method computes and returns an `SHA-1 digest`_. This is -necessary for attempting to save the blob to CrateDB. - -If you already have the SHA-1 digest computed, or are able to compute it as part -of an existing read, this may improve the performance of your application. - -If you pass in a SHA-1 digest, it will not be recomputed: - - >>> file.seek(0) # seek to the beginning before attempting to re-upload - - >>> digest = "6f10281ad07d4a35c6ec2f993e6376032b77181d" - >>> blob_container.put(file, digest=digest) - False - -Notice that the method returned ``False`` this time. If you specify a digest, -the return value of the ``put`` method is a boolean indicating whether the -object was written or not. In this instance, it was not written, because the -digest is the same as an existing object. - -Let's make a new object: - - >>> bytestream = "Another example sentence.".encode("utf8") - >>> digest = hashlib.sha1(bytestream).hexdigest() - >>> another_file = io.BytesIO(bytestream) - -And upload it: - - >>> blob_container.put(another_file, digest) - True - -The ``put`` method returns ``True``, indicating that the object has been -written to the blob container. - -Retrieve blobs --------------- - -To retrieve a blob, you need to know its digest. - -Let's use the ``digest`` variable we created before to check whether that object -exists with the ``exists`` method: - - >>> blob_container.exists(digest) - True - -This method returns a boolean value. And in this instance, ``True`` indicates -that the blob we're interested in is contained within the blob container. - -You can get the blob, with the ``get`` method, like so: - - >>> blob_generator = blob_container.get(digest) - -Blobs are read in chunks. The default size of these chunks is 128 kilobytes, -but this can be changed by supplying the desired chunk size to the ``get`` -method, like so: - - >>> res = blob_container.get(digest, 1024 * 128) - -The ``blob`` object is a Python :term:`py:generator`, meaning that you can call -``next(blob)`` for each new chunk you want to read, until you encounter a -``StopIteration`` exception. - -Instead of calling ``next()`` manually, the idiomatic way to iterate over a -generator is like so: - - >>> blob_content = b'' - >>> for chunk in blob_container.get(digest): - ... blob_content += chunk - - -Delete blobs ------------- - -You can delete a blob with the ``delete`` method and the blob digest, like so: - - >>> blob_container.delete(digest) - True - -This method returns a boolean status value. In this instance, ``True`` -indicates that the blob was deleted. - -We can verify that, like so: - - >>> blob_container.exists(digest) - False - -.. _SHA-1 digest: https://en.wikipedia.org/wiki/SHA-1 diff --git a/docs/by-example/blob.rst b/docs/by-example/blob.rst deleted file mode 100644 index 3b5f6ca3..00000000 --- a/docs/by-example/blob.rst +++ /dev/null @@ -1,100 +0,0 @@ -================== -Blob container API -================== - -The connection object provides a convenience API for easy access to -:ref:`blob tables `. - - -Get blob container handle -========================= - -Create a connection: - - >>> from crate.client import connect - >>> client = connect([crate_host]) - -Get a blob container: - - >>> container = client.get_blob_container('myfiles') - - -Store blobs -=========== - -The container allows to store a blob without explicitly providing the hash -for the blob. This feature is possible if the blob is provided as a seekable -stream like object. - -Store a ``StringIO`` stream: - - >>> from io import BytesIO - >>> f = BytesIO(b'StringIO data') - >>> stringio_bob = container.put(f) - >>> stringio_bob - '0cd4511d696823779692484029f234471cd21f28' - -Store from a file: - - >>> from tempfile import TemporaryFile - >>> f = TemporaryFile() - >>> _ = f.write(b'File data') - >>> _ = f.seek(0) - >>> file_blob = container.put(f) - >>> file_blob - 'ea6e03a4a4ee8a2366fe5a88af2bde61797973ea' - >>> f.close() - -If the blob data is not provided as a seekable stream the hash must be -provided explicitly: - - >>> import hashlib - >>> string_data = b'String data' - >>> string_blob = hashlib.sha1(string_data).hexdigest() - >>> container.put(string_data, string_blob) - True - - -Check for existence -=================== - - >>> container.exists(string_blob) - True - >>> container.exists('unknown') - False - - -Retrieve blobs -============== - -Blobs can be retrieved using its hash: - - >>> blob_stream = container.get(string_blob) - >>> blob_stream - - >>> data = next(blob_stream) - >>> data == string_data - True - - -Delete blobs -============ - -Blobs can be deleted using its hash: - - >>> container.delete(string_blob) - True - >>> container.exists(string_blob) - False - -Trying to delete a not existing blob: - - >>> container.delete(string_blob) - False - -Close connection -================ - -Close the connection to clear the connection pool: - - >>> client.close() diff --git a/docs/by-example/client.rst b/docs/by-example/client.rst deleted file mode 100644 index e053d73f..00000000 --- a/docs/by-example/client.rst +++ /dev/null @@ -1,308 +0,0 @@ -=============== -Database client -=============== - -``crate.client.connect`` is the primary method to connect to CrateDB using -Python. This section of the documentation outlines different methods to connect -to the database cluster, as well as how to run basic inquiries to the database, -and closing the connection again. - -.. rubric:: Table of Contents - -.. contents:: - :local: - - -Connect to a database -===================== - -Before we can start we have to import the client: - - >>> from crate import client - -The client provides a ``connect()`` function which is used to establish a -connection, the first argument is the url of the server to connect to: - - >>> connection = client.connect(crate_host) - >>> connection.close() - -CrateDB is a clustered database providing high availability through -replication. In order for clients to make use of this property it is -recommended to specify all hosts of the cluster. This way if a server does not -respond, the request is automatically routed to the next server: - - >>> invalid_host = 'http://not_responding_host:4200' - >>> connection = client.connect([invalid_host, crate_host]) - >>> connection.close() - -If no ``servers`` are given, the default one ``http://127.0.0.1:4200`` is used: - - >>> connection = client.connect() - >>> connection.client._active_servers - ['http://127.0.0.1:4200'] - >>> connection.close() - -If the option ``error_trace`` is set to ``True``, the client will print a whole -traceback if a server error occurs: - - >>> connection = client.connect([crate_host], error_trace=True) - >>> connection.close() - -It's possible to define a default timeout value in seconds for all servers -using the optional parameter ``timeout``: - - >>> connection = client.connect([crate_host, invalid_host], timeout=5) - >>> connection.close() - -Authentication --------------- - -Users that are trusted as by definition of the ``auth.host_based.config`` -setting do not need a password, but only require the ``username`` argument to -connect: - - >>> connection = client.connect([crate_host], - ... username='trusted_me') - >>> connection.client.username - 'trusted_me' - >>> connection.client.password - >>> connection.close() - -The username for trusted users can also be provided in the URL: - - >>> connection = client.connect(['http://trusted_me@' + crate_host]) - >>> connection.client.username - 'trusted_me' - >>> connection.client.password - >>> connection.close() - -To connect to CrateDB with as a user that requires password authentication, you -also need to provide ``password`` as argument for the ``connect()`` call: - - >>> connection = client.connect([crate_host], - ... username='me', - ... password='my_secret_pw') - >>> connection.client.username - 'me' - >>> connection.client.password - 'my_secret_pw' - >>> connection.close() - -The authentication credentials can also be provided in the URL: - - >>> connection = client.connect(['http://me:my_secret_pw@' + crate_host]) - >>> connection.client.username - 'me' - >>> connection.client.password - 'my_secret_pw' - >>> connection.close() - - -Default Schema --------------- - -To connect to CrateDB and use a different default schema than ``doc``, you can -provide the ``schema`` keyword argument in the ``connect()`` method, like so: - - >>> connection = client.connect([crate_host], - ... schema='custom_schema') - >>> connection.close() - -Inserting Data -============== - -Use user "crate" for rest of the tests: - - >>> connection = client.connect([crate_host]) - -Before executing any statement, a cursor has to be opened to perform -database operations: - - >>> cursor = connection.cursor() - >>> cursor.execute("""INSERT INTO locations - ... (name, date, kind, position) VALUES (?, ?, ?, ?)""", - ... ('Einstein Cross', '2007-03-11', 'Quasar', 7)) - -To bulk insert data you can use the ``executemany`` function: - - >>> cursor.executemany("""INSERT INTO locations - ... (name, date, kind, position) VALUES (?, ?, ?, ?)""", - ... [('Cloverleaf', '2007-03-11', 'Quasar', 7), - ... ('Old Faithful', '2007-03-11', 'Quasar', 7)]) - [{'rowcount': 1}, {'rowcount': 1}] - -``executemany`` returns a list of results for every parameter. Each result -contains a rowcount. If an error occurs, the rowcount is ``-2`` and the result -may contain an ``error_message`` depending on the error. - -Refresh locations: - - >>> cursor.execute("REFRESH TABLE locations") - -Updating Data -============= - -Values for ``TIMESTAMP`` columns can be obtained as a string literal, ``date``, -or ``datetime`` object. If it contains timezone information, it is converted to -UTC, and the timezone information is discarded. - - >>> import datetime as dt - >>> timestamp_full = "2023-06-26T09:24:00.123+02:00" - >>> timestamp_date = "2023-06-26" - >>> datetime_aware = dt.datetime.fromisoformat("2023-06-26T09:24:00.123+02:00") - >>> datetime_naive = dt.datetime.fromisoformat("2023-06-26T09:24:00.123") - >>> datetime_date = dt.date.fromisoformat("2023-06-26") - >>> cursor.execute("UPDATE locations SET date=? WHERE name='Cloverleaf'", (timestamp_full, )) - >>> cursor.execute("UPDATE locations SET date=? WHERE name='Cloverleaf'", (timestamp_date, )) - >>> cursor.execute("UPDATE locations SET date=? WHERE name='Cloverleaf'", (datetime_aware, )) - >>> cursor.execute("UPDATE locations SET date=? WHERE name='Cloverleaf'", (datetime_naive, )) - >>> cursor.execute("UPDATE locations SET date=? WHERE name='Cloverleaf'", (datetime_date, )) - -Selecting Data -============== - -To perform the select operation simply execute the statement on the -open cursor: - - >>> cursor.execute("SELECT name FROM locations where name = ?", ('Algol',)) - -To retrieve a row we can use one of the cursor's fetch functions (described below). - -fetchone() ----------- - -``fetchone()`` with each call returns the next row from the results: - - >>> result = cursor.fetchone() - >>> pprint(result) - ['Algol'] - -If no more data is available, an empty result is returned: - - >>> while cursor.fetchone(): - ... pass - >>> cursor.fetchone() - -fetchmany() ------------ - -``fetch_many()`` returns a list of all remaining rows, containing no more than -the specified size of rows: - - >>> cursor.execute("SELECT name FROM locations order by name") - >>> result = cursor.fetchmany(2) - >>> pprint(result) - [['Aldebaran'], ['Algol']] - -If a size is not given, the cursor's arraysize, which defaults to '1', -determines the number of rows to be fetched: - - >>> cursor.fetchmany() - [['Allosimanius Syneca']] - -It's also possible to change the cursors arraysize to an other value: - - >>> cursor.arraysize = 3 - >>> cursor.fetchmany() - [['Alpha Centauri'], ['Altair'], ['Argabuthon']] - -fetchall() ----------- - -``fetchall()`` returns a list of all remaining rows: - - >>> cursor.execute("SELECT name FROM locations order by name") - >>> result = cursor.fetchall() - >>> pprint(result) - [['Aldebaran'], - ['Algol'], - ['Allosimanius Syneca'], - ['Alpha Centauri'], - ['Altair'], - ['Argabuthon'], - ['Arkintoofle Minor'], - ['Bartledan'], - ['Cloverleaf'], - ['Einstein Cross'], - ['Folfanga'], - ['Galactic Sector QQ7 Active J Gamma'], - ['Galaxy'], - ['North West Ripple'], - ['Old Faithful'], - ['Outer Eastern Rim']] - -Cursor Description -================== - -The ``description`` property of the cursor returns a sequence of 7-item -sequences containing the column name as first parameter. Just the name field is -supported, all other fields are 'None': - - >>> cursor.execute("SELECT * FROM locations order by name") - >>> result = cursor.fetchone() - >>> pprint(result) - ['Aldebaran', - 1658167836758, - 1658167836758, - 1658167836758, - None, - None, - 'Star System', - None, - 1, - 'Max Quordlepleen claims that the only thing left after the end of the ' - 'Universe will be the sweets trolley and a fine selection of Aldebaran ' - 'liqueurs.', - None] - - >>> result = cursor.description - >>> pprint(result) - (('name', None, None, None, None, None, None), - ('date', None, None, None, None, None, None), - ('datetime_tz', None, None, None, None, None, None), - ('datetime_notz', None, None, None, None, None, None), - ('nullable_datetime', None, None, None, None, None, None), - ('nullable_date', None, None, None, None, None, None), - ('kind', None, None, None, None, None, None), - ('flag', None, None, None, None, None, None), - ('position', None, None, None, None, None, None), - ('description', None, None, None, None, None, None), - ('details', None, None, None, None, None, None)) - -Closing the Cursor -================== - -The following command closes the cursor: - - >>> cursor.close() - -If a cursor is closed, it will be unusable from this point forward. - -If any operation is attempted to a closed cursor an ``ProgrammingError`` will -be raised. - - >>> cursor.execute("SELECT * FROM locations") - Traceback (most recent call last): - ... - crate.client.exceptions.ProgrammingError: Cursor closed - -Closing the Connection -====================== - -The following command closes the connection: - - >>> connection.close() - -If a connection is closed, it will be unusable from this point forward. If any -operation using the connection is attempted to a closed connection an -``ProgrammingError`` will be raised: - - >>> cursor.execute("SELECT * FROM locations") - Traceback (most recent call last): - ... - crate.client.exceptions.ProgrammingError: Connection closed - - >>> cursor = connection.cursor() - Traceback (most recent call last): - ... - crate.client.exceptions.ProgrammingError: Connection closed diff --git a/docs/by-example/connection.rst b/docs/by-example/connection.rst deleted file mode 100644 index 4b89db7d..00000000 --- a/docs/by-example/connection.rst +++ /dev/null @@ -1,68 +0,0 @@ -===================== -The Connection object -===================== - -This documentation section outlines different attributes, methods, and -behaviors of the ``crate.client.connection.Connection`` object. - -The examples use an instance of ``ClientMocked`` instead of a real ``Client`` -instance. This allows us to verify the examples without needing a real database -connection. - -.. rubric:: Table of Contents - -.. contents:: - :local: - - -connect() -========= - -This section sets up a connection object, and inspects some of its attributes. - - >>> from crate.client import connect - >>> from crate.client.test_util import ClientMocked - - >>> connection = connect(client=ClientMocked()) - >>> connection.lowest_server_version.version - (2, 0, 0) - -cursor() -======== - -Calling the ``cursor()`` function on the connection will -return a cursor object: - - >>> cursor = connection.cursor() - -Now we are able to perform any operation provided by the -cursor object: - - >>> cursor.rowcount - -1 - -close() -======= - -Now we close the connection: - - >>> connection.close() - -The connection will be unusable from this point. Any -operation attempted with the closed connection will -raise a ``ProgrammingError``: - - >>> cursor = connection.cursor() - Traceback (most recent call last): - ... - crate.client.exceptions.ProgrammingError: Connection closed - - >>> cursor.execute('') - Traceback (most recent call last): - ... - crate.client.exceptions.ProgrammingError: Connection closed - - >>> connection.commit() - Traceback (most recent call last): - ... - crate.client.exceptions.ProgrammingError: Connection closed diff --git a/docs/by-example/cursor.rst b/docs/by-example/cursor.rst deleted file mode 100644 index 7fc7da7d..00000000 --- a/docs/by-example/cursor.rst +++ /dev/null @@ -1,443 +0,0 @@ -================= -The Cursor object -================= - -This documentation section outlines different attributes, methods, and -behaviors of the ``crate.client.cursor.Cursor`` object. - -The example code uses ``ClientMocked`` and ``set_next_response`` for -demonstration purposes, so they don't need a real database connection. - -.. rubric:: Table of Contents - -.. contents:: - :local: - - -Introduction -============ - -This section sets up a cursor object, inspects some of its attributes, and sets -up the response for subsequent cursor operations. - - >>> from crate.client import connect - >>> from crate.client.converter import DefaultTypeConverter - >>> from crate.client.cursor import Cursor - >>> from crate.client.test_util import ClientMocked - - >>> connection = connect(client=ClientMocked()) - >>> cursor = connection.cursor() - -The ``rowcount`` and ``duration`` attributes are ``-1``, in case no ``execute()`` has -been performed on the cursor yet. - - >>> cursor.rowcount - -1 - - >>> cursor.duration - -1 - -Define the response of the mocked connection client. It will be returned on -request without needing to execute an SQL statement. - - >>> connection.client.set_next_response({ - ... "rows":[ [ "North West Ripple", 1 ], [ "Arkintoofle Minor", 3 ], [ "Alpha Centauri", 3 ] ], - ... "cols":[ "name", "position" ], - ... "rowcount":3, - ... "duration":123 - ... }) - -fetchone() -========== - -Calling ``fetchone()`` on the cursor object the first time after an execute returns the first row: - - >>> cursor.execute('') - - >>> cursor.fetchone() - ['North West Ripple', 1] - -Each call to ``fetchone()`` increments the cursor and returns the next row: - - >>> cursor.fetchone() - ['Arkintoofle Minor', 3] - -One more iteration: - - >>> cursor.next() - ['Alpha Centauri', 3] - -The iteration is stopped after the last row is returned. -A further call to ``fetchone()`` returns an empty result: - - >>> cursor.fetchone() - -Using ``fetchone()`` on a cursor before issuing a database statement results -in an error: - - >>> new_cursor = connection.cursor() - >>> new_cursor.fetchone() - Traceback (most recent call last): - ... - crate.client.exceptions.ProgrammingError: No result available. execute() or executemany() must be called first. - - -fetchmany() -=========== - -``fetchmany()`` takes an argument which specifies the number of rows we want to fetch: - - >>> cursor.execute('') - - >>> cursor.fetchmany(2) - [['North West Ripple', 1], ['Arkintoofle Minor', 3]] - -If the specified number of rows not being available, fewer rows may returned: - - >>> cursor.fetchmany(2) - [['Alpha Centauri', 3]] - - >>> cursor.execute('') - -If no number of rows are specified it defaults to the current ``cursor.arraysize``: - - >>> cursor.arraysize - 1 - - >>> cursor.fetchmany() - [['North West Ripple', 1]] - - >>> cursor.execute('') - >>> cursor.arraysize = 2 - >>> cursor.fetchmany() - [['North West Ripple', 1], ['Arkintoofle Minor', 3]] - -If zero number of rows are specified, all rows left are returned: - - >>> cursor.fetchmany(0) - [['Alpha Centauri', 3]] - -fetchall() -========== - -``fetchall()`` fetches all (remaining) rows of a query result: - - >>> cursor.execute('') - - >>> cursor.fetchall() - [['North West Ripple', 1], ['Arkintoofle Minor', 3], ['Alpha Centauri', 3]] - -Since all data was fetched 'None' is returned by ``fetchone()``: - - >>> cursor.fetchone() - -And each other call returns an empty sequence: - - >>> cursor.fetchmany(2) - [] - - >>> cursor.fetchmany() - [] - - >>> cursor.fetchall() - [] - -iteration -========= - -The cursor supports the iterator interface and can be iterated upon: - - >>> cursor.execute('') - >>> [row for row in cursor] - [['North West Ripple', 1], ['Arkintoofle Minor', 3], ['Alpha Centauri', 3]] - -When no other call to execute has been done, it will raise StopIteration on -subsequent iterations: - - >>> next(cursor) - Traceback (most recent call last): - ... - StopIteration - - >>> cursor.execute('') - >>> for row in cursor: - ... row - ['North West Ripple', 1] - ['Arkintoofle Minor', 3] - ['Alpha Centauri', 3] - -Iterating over a new cursor without results will immediately raise a ProgrammingError: - - >>> new_cursor = connection.cursor() - >>> next(new_cursor) - Traceback (most recent call last): - ... - crate.client.exceptions.ProgrammingError: No result available. execute() or executemany() must be called first. - -description -=========== - - >>> cursor.description - (('name', None, None, None, None, None, None), ('position', None, None, None, None, None, None)) - -rowcount -======== - -The ``rowcount`` property specifies the number of rows that the last ``execute()`` produced: - - >>> cursor.execute('') - >>> cursor.rowcount - 3 - -The attribute is ``-1``, in case the cursor has been closed: - - >>> cursor.close() - >>> cursor.rowcount - -1 - -If the last response does not contain the rowcount attribute, ``-1`` is returned: - - >>> cursor = connection.cursor() - >>> connection.client.set_next_response({ - ... "rows":[], - ... "cols":[], - ... "duration":123 - ... }) - - >>> cursor.execute('') - >>> cursor.rowcount - -1 - - >>> connection.client.set_next_response({ - ... "rows":[ [ "North West Ripple", 1 ], [ "Arkintoofle Minor", 3 ], [ "Alpha Centauri", 3 ] ], - ... "cols":[ "name", "position" ], - ... "rowcount":3, - ... "duration":123 - ... }) - -duration -======== - -The ``duration`` property specifies the server-side duration in milliseconds of the last query -issued by ``execute()``: - - >>> cursor = connection.cursor() - >>> cursor.execute('') - >>> cursor.duration - 123 - -The attribute is ``-1``, in case the cursor has been closed: - - >>> cursor.close() - >>> cursor.duration - -1 - - >>> connection.client.set_next_response({ - ... "results": [ - ... { - ... "rowcount": 3 - ... }, - ... { - ... "rowcount": 2 - ... } - ... ], - ... "duration":123, - ... "cols":[ "name", "position" ], - ... }) - -executemany() -============= - -``executemany()`` allows to execute a single sql statement against a sequence -of parameters: - - >>> cursor = connection.cursor() - - >>> cursor.executemany('', (1,2,3)) - [{'rowcount': 3}, {'rowcount': 2}] - - >>> cursor.rowcount - 5 - >>> cursor.duration - 123 - -``executemany()`` is not intended to be used with statements returning result -sets. The result will always be empty: - - >>> cursor.fetchall() - [] - -For completeness' sake the cursor description is updated nonetheless: - - >>> [ desc[0] for desc in cursor.description ] - ['name', 'position'] - - >>> connection.client.set_next_response({ - ... "rows":[ [ "North West Ripple", 1 ], [ "Arkintoofle Minor", 3 ], [ "Alpha Centauri", 3 ] ], - ... "cols":[ "name", "position" ], - ... "rowcount":3, - ... "duration":123 - ... }) - - -close() -======= - -After closing a cursor the connection will be unusable. If any operation is attempted with the -closed connection an ``ProgrammingError`` exception will be raised: - - >>> cursor = connection.cursor() - >>> cursor.execute('') - >>> cursor.fetchone() - ['North West Ripple', 1] - - >>> cursor.close() - >>> cursor.fetchone() - Traceback (most recent call last): - ... - crate.client.exceptions.ProgrammingError: Cursor closed - - >>> cursor.fetchmany() - Traceback (most recent call last): - ... - crate.client.exceptions.ProgrammingError: Cursor closed - - >>> cursor.fetchall() - Traceback (most recent call last): - ... - crate.client.exceptions.ProgrammingError: Cursor closed - - >>> cursor.next() - Traceback (most recent call last): - ... - crate.client.exceptions.ProgrammingError: Cursor closed - - -Python data type conversion -=========================== - -The cursor object can optionally convert database types to native Python data -types. Currently, this is implemented for the CrateDB data types ``IP`` and -``TIMESTAMP`` on behalf of the ``DefaultTypeConverter``. - - >>> cursor = connection.cursor(converter=DefaultTypeConverter()) - - >>> connection.client.set_next_response({ - ... "col_types": [4, 5, 11], - ... "rows":[ [ "foo", "10.10.10.1", 1658167836758 ] ], - ... "cols":[ "name", "address", "timestamp" ], - ... "rowcount":1, - ... "duration":123 - ... }) - - >>> cursor.execute('') - - >>> cursor.fetchone() - ['foo', IPv4Address('10.10.10.1'), datetime.datetime(2022, 7, 18, 18, 10, 36, 758000)] - - -Custom data type conversion -=========================== - -By providing a custom converter instance, you can define your own data type -conversions. For investigating the list of available data types, please either -inspect the ``DataType`` enum, or the documentation about the list of available -:ref:`CrateDB data type identifiers for the HTTP interface -`. - -To create a simple converter for converging CrateDB's ``BIT`` type to Python's -``int`` type. - - >>> from crate.client.converter import Converter, DataType - - >>> converter = Converter({DataType.BIT: lambda value: int(value[2:-1], 2)}) - >>> cursor = connection.cursor(converter=converter) - -Proof that the converter works correctly, ``B\'0110\'`` should be converted to -``6``. CrateDB's ``BIT`` data type has the numeric identifier ``25``. - - >>> connection.client.set_next_response({ - ... "col_types": [25], - ... "rows":[ [ "B'0110'" ] ], - ... "cols":[ "value" ], - ... "rowcount":1, - ... "duration":123 - ... }) - - >>> cursor.execute('') - - >>> cursor.fetchone() - [6] - - -``TIMESTAMP`` conversion with time zone -======================================= - -Based on the data type converter functionality, the driver offers a convenient -interface to make it return timezone-aware ``datetime`` objects, using the -desired time zone. - -For your reference, in the following examples, epoch 1658167836758 is -``Mon, 18 Jul 2022 18:10:36 GMT``. - - >>> import datetime - >>> tz_mst = datetime.timezone(datetime.timedelta(hours=7), name="MST") - >>> cursor = connection.cursor(time_zone=tz_mst) - - >>> connection.client.set_next_response({ - ... "col_types": [4, 11], - ... "rows":[ [ "foo", 1658167836758 ] ], - ... "cols":[ "name", "timestamp" ], - ... "rowcount":1, - ... "duration":123 - ... }) - - >>> cursor.execute('') - - >>> cursor.fetchone() - ['foo', datetime.datetime(2022, 7, 19, 1, 10, 36, 758000, tzinfo=datetime.timezone(datetime.timedelta(seconds=25200), 'MST'))] - -For the ``time_zone`` keyword argument, different data types are supported. -The available options are: - -- ``datetime.timezone.utc`` -- ``datetime.timezone(datetime.timedelta(hours=7), name="MST")`` -- ``pytz.timezone("Australia/Sydney")`` -- ``zoneinfo.ZoneInfo("Australia/Sydney")`` -- ``+0530`` (UTC offset in string format) - -Let's exercise all of them: - - >>> cursor.time_zone = datetime.timezone.utc - >>> cursor.execute('') - >>> cursor.fetchone() - ['foo', datetime.datetime(2022, 7, 18, 18, 10, 36, 758000, tzinfo=datetime.timezone.utc)] - - >>> import pytz - >>> cursor.time_zone = pytz.timezone("Australia/Sydney") - >>> cursor.execute('') - >>> cursor.fetchone() - ['foo', datetime.datetime(2022, 7, 19, 4, 10, 36, 758000, tzinfo=)] - - >>> try: - ... import zoneinfo - ... except ImportError: - ... from backports import zoneinfo - >>> cursor.time_zone = zoneinfo.ZoneInfo("Australia/Sydney") - >>> cursor.execute('') - >>> record = cursor.fetchone() - >>> record - ['foo', datetime.datetime(2022, 7, 19, 4, 10, 36, 758000, ...zoneinfo.ZoneInfo(key='Australia/Sydney'))] - - >>> record[1].tzname() - 'AEST' - - >>> cursor.time_zone = "+0530" - >>> cursor.execute('') - >>> cursor.fetchone() - ['foo', datetime.datetime(2022, 7, 18, 23, 40, 36, 758000, tzinfo=datetime.timezone(datetime.timedelta(seconds=19800), '+0530'))] - - -.. Hidden: close connection - - >>> connection.close() diff --git a/docs/by-example/http.rst b/docs/by-example/http.rst deleted file mode 100644 index 494e7b65..00000000 --- a/docs/by-example/http.rst +++ /dev/null @@ -1,235 +0,0 @@ -=========== -HTTP client -=========== - -.. rubric:: Table of Contents - -.. contents:: - :local: - - -Introduction -============ - -The CrateDB Python driver package offers an HTTP client API object. - - >>> from crate.client import http - >>> HttpClient = http.Client - - -Server configuration -==================== - -A list of servers can be passed while creating an instance of the http client: - - >>> http_client = HttpClient([crate_host]) - >>> http_client.close() - -Its also possible to pass a single server as a string: - - >>> http_client = HttpClient(crate_host) - >>> http_client.close() - -If no ``server`` argument (or no argument at all) is passed, the default one -``127.0.0.1:4200`` is used: - - >>> http_client = HttpClient() - >>> http_client._active_servers - ['http://127.0.0.1:4200'] - >>> http_client.close() - -When using a list of servers, the servers are selected by round-robin: - - >>> invalid_host = "invalid_host:9999" - >>> even_more_invalid_host = "even_more_invalid_host:9999" - >>> http_client = HttpClient([crate_host, invalid_host, even_more_invalid_host]) - >>> http_client._get_server() - 'http://127.0.0.1:44209' - - >>> http_client._get_server() - 'http://invalid_host:9999' - - >>> http_client._get_server() - 'http://even_more_invalid_host:9999' - - >>> http_client.close() - -Servers with connection errors will be removed from the active server list: - - >>> http_client = HttpClient([invalid_host, even_more_invalid_host, crate_host]) - >>> result = http_client.sql('select name from locations') - >>> http_client._active_servers - ['http://127.0.0.1:44209'] - -Inactive servers will be re-added after a given time interval. -To validate this, set the interval very short and sleep for that interval: - - >>> http_client.retry_interval = 1 - >>> import time; time.sleep(1) - >>> result = http_client.sql('select name from locations') - >>> http_client._active_servers - ['http://invalid_host:9999', - 'http://even_more_invalid_host:9999', - 'http://127.0.0.1:44209'] - >>> http_client.close() - -If no active servers are available and the retry interval is not reached, just use the oldest -inactive one: - - >>> http_client = HttpClient([invalid_host, even_more_invalid_host, crate_host]) - >>> result = http_client.sql('select name from locations') - >>> http_client._active_servers = [] - >>> http_client._get_server() - 'http://invalid_host:9999' - >>> http_client.close() - -SQL Statements -============== - -Issue a select statement against our with test data pre-filled crate instance: - - >>> http_client = HttpClient(crate_host) - >>> result = http_client.sql('select name from locations order by name') - >>> pprint(result) - {'col_types': [4], - 'cols': ['name'], - 'duration': ..., - 'rowcount': 13, - 'rows': [['Aldebaran'], - ['Algol'], - ['Allosimanius Syneca'], - ['Alpha Centauri'], - ['Altair'], - ['Argabuthon'], - ['Arkintoofle Minor'], - ['Bartledan'], - ['Folfanga'], - ['Galactic Sector QQ7 Active J Gamma'], - ['Galaxy'], - ['North West Ripple'], - ['Outer Eastern Rim']]} - -Blobs -===== - -Check if a blob exists: - - >>> http_client.blob_exists('myfiles', '040f06fd774092478d450774f5ba30c5da78acc8') - False - -Trying to get a non-existing blob throws an exception: - - >>> http_client.blob_get('myfiles', '041f06fd774092478d450774f5ba30c5da78acc8') - Traceback (most recent call last): - ... - crate.client.exceptions.DigestNotFoundException: myfiles/041f06fd774092478d450774f5ba30c5da78acc8 - -Creating a new blob - this method returns ``True`` if the blob was newly created: - - >>> from tempfile import TemporaryFile - >>> f = TemporaryFile() - >>> _ = f.write(b'content') - >>> _ = f.seek(0) - >>> http_client.blob_put( - ... 'myfiles', '040f06fd774092478d450774f5ba30c5da78acc8', f) - True - -Uploading the same content again returns ``False``: - - >>> _ = f.seek(0) - >>> http_client.blob_put( - ... 'myfiles', '040f06fd774092478d450774f5ba30c5da78acc8', f) - False - -Now the blob exist: - - >>> http_client.blob_exists('myfiles', '040f06fd774092478d450774f5ba30c5da78acc8') - True - -Blobs are returned as generators, generating a chunk on each call: - - >>> g = http_client.blob_get('myfiles', '040f06fd774092478d450774f5ba30c5da78acc8') - >>> print(next(g)) - content - -The chunk_size can be set explicitly on get: - - >>> g = http_client.blob_get( - ... 'myfiles', '040f06fd774092478d450774f5ba30c5da78acc8', 5) - >>> print(next(g)) - conte - - >>> print(next(g)) - nt - -Deleting a blob - this method returns true if the blob existed: - - >>> http_client.blob_del('myfiles', '040f06fd774092478d450774f5ba30c5da78acc8') - True - - >>> http_client.blob_del('myfiles', '040f06fd774092478d450774f5ba30c5da78acc8') - False - -Uploading a blob to a table with disabled blob support throws an exception: - - >>> _ = f.seek(0) - >>> http_client.blob_put( - ... 'locations', '040f06fd774092478d450774f5ba30c5da78acc8', f) - Traceback (most recent call last): - ... - crate.client.exceptions.BlobLocationNotFoundException: locations/040f06fd774092478d450774f5ba30c5da78acc8 - - >>> http_client.close() - >>> f.close() - - -Error Handling -============== - -Create a function that takes a lot of time to return so we can run into a -timeout exception: - - >>> http_client = HttpClient(crate_host) - >>> http_client.sql(''' - ... CREATE FUNCTION fib(LONG) RETURNS LONG - ... LANGUAGE JAVASCRIPT AS ' - ... var fib = function fib(n) { return n < 2 ? n : fib(n-1) + fib(n-2); } - ... ' - ... ''') - {...} - >>> http_client.close() - -It's possible to define a HTTP timeout in seconds on client instantiation, so -an exception is raised when the timeout is reached: - - >>> http_client = HttpClient(crate_host, timeout=0.01) - >>> http_client.sql('select fib(32)') - Traceback (most recent call last): - ... - crate.client.exceptions.ConnectionError: No more Servers available, exception from last server: ... - >>> http_client.close() - -When connecting to non-CrateDB servers, the HttpClient will raise a ConnectionError like this: - - >>> http_client = HttpClient(["https://example.org/"]) - >>> http_client.server_infos(http_client._get_server()) - Traceback (most recent call last): - ... - crate.client.exceptions.ProgrammingError: Invalid server response of content-type 'text/html; charset=UTF-8': - ... - >>> http_client.close() - -When using the ``error_trace`` kwarg a full traceback of the server exception -will be provided: - - >>> from crate.client.exceptions import ProgrammingError - >>> http_client = HttpClient([crate_host], error_trace=True) - >>> try: - ... http_client.sql("select grmpf form error arrrggghh") - ... except ProgrammingError as e: - ... trace = 'TRACE: ' + str(e.error_trace) - - >>> print(trace) - TRACE: ... mismatched input 'error' expecting {, ... - at io.crate... - >>> http_client.close() diff --git a/docs/by-example/https.rst b/docs/by-example/https.rst deleted file mode 100644 index 4bbd408e..00000000 --- a/docs/by-example/https.rst +++ /dev/null @@ -1,127 +0,0 @@ -.. _https_connection: - -======================== -HTTPS connection support -======================== - -This documentation section outlines different options to connect to CrateDB -using SSL/TLS. - -.. rubric:: Table of Contents - -.. contents:: - :local: - - -Introduction -============ - -The CrateDB client is able to connect via HTTPS. - -A check against a specific CA certificate can be made by creating the client -with the path to the CA certificate file using the keyword argument -``ca_cert``. - -.. note:: - - By default, SSL server certificates are verified. To disable verification, - use the keyword argument ``verify_ssl_cert``. If it is set to ``False``, - server certificate validation will be skipped. - -All the following examples will connect to a host using a self-signed -certificate. - -The CrateDB Python driver package offers a HTTP client API object. - - >>> from crate.client import http - >>> HttpClient = http.Client - - -With certificate verification -============================= - -When using a valid CA certificate, the connection will be successful: - - >>> client = HttpClient([crate_host], ca_cert=cacert_valid) - >>> client.server_infos(client._get_server()) - ('https://localhost:65534', 'test', '0.0.0') - -When not providing a ``ca_cert`` file, the connection will fail: - - >>> client = HttpClient([crate_host]) - >>> client.server_infos(crate_host) - Traceback (most recent call last): - ... - crate.client.exceptions.ConnectionError: Server not available, ...certificate verify failed... - -Also, when providing an invalid ``ca_cert``, an error is raised: - - >>> client = HttpClient([crate_host], ca_cert=cacert_invalid) - >>> client.server_infos(crate_host) - Traceback (most recent call last): - ... - crate.client.exceptions.ConnectionError: Server not available, ...certificate verify failed... - - -Without certificate verification -================================ - -When turning off certificate verification, calling the server will succeed, -even when not providing a valid CA certificate: - - >>> client = HttpClient([crate_host], verify_ssl_cert=False) - >>> client.server_infos(crate_host) - ('https://localhost:65534', 'test', '0.0.0') - -Without verification, calling the server will even work when using an invalid -``ca_cert``: - - >>> client = HttpClient([crate_host], verify_ssl_cert=False, ca_cert=cacert_invalid) - >>> client.server_infos(crate_host) - ('https://localhost:65534', 'test', '0.0.0') - - - -X.509 client certificate -======================== - -The CrateDB driver also supports client certificates. - -The ``HttpClient`` constructor takes two keyword arguments: ``cert_file`` and -``key_file``. Both should be strings pointing to the path of the client -certificate and key file: - - >>> client = HttpClient([crate_host], ca_cert=cacert_valid, cert_file=clientcert_valid, key_file=clientcert_valid) - >>> client.server_infos(crate_host) - ('https://localhost:65534', 'test', '0.0.0') - -When using an invalid client certificate, the connection will fail: - - >>> client = HttpClient([crate_host], ca_cert=cacert_valid, cert_file=clientcert_invalid, key_file=clientcert_invalid) - >>> client.server_infos(crate_host) - Traceback (most recent call last): - ... - crate.client.exceptions.ConnectionError: Server not available, exception: HTTPSConnectionPool... - -The connection will also fail when providing an invalid CA certificate: - - >>> client = HttpClient([crate_host], ca_cert=cacert_invalid, cert_file=clientcert_valid, key_file=clientcert_valid) - >>> client.server_infos(crate_host) - Traceback (most recent call last): - ... - crate.client.exceptions.ConnectionError: Server not available, exception: HTTPSConnectionPool... - - -Relaxing minimum SSL version -============================ - -urrlib3 v2 dropped support for TLS 1.0 and TLS 1.1 by default, see `Modern security by default - -HTTPS requires TLS 1.2+`_. If you need to re-enable it, use the ``ssl_relax_minimum_version`` flag, -which will configure ``kwargs["ssl_minimum_version"] = ssl.TLSVersion.MINIMUM_SUPPORTED``. - - >>> client = HttpClient([crate_host], ssl_relax_minimum_version=True, verify_ssl_cert=False) - >>> client.server_infos(crate_host) - ('https://localhost:65534', 'test', '0.0.0') - - -.. _Modern security by default - HTTPS requires TLS 1.2+: https://urllib3.readthedocs.io/en/latest/v2-migration-guide.html#https-requires-tls-1-2 diff --git a/docs/by-example/index.rst b/docs/by-example/index.rst index 39c503e4..301cf699 100644 --- a/docs/by-example/index.rst +++ b/docs/by-example/index.rst @@ -7,26 +7,6 @@ By example This part of the documentation enumerates different kinds of examples how to use the CrateDB Python client. - -DB API, HTTP, and BLOB interfaces -================================= - -The examples in this section are all about CrateDB's `Python DB API`_ interface, -the plain HTTP API interface, and a convenience interface for working with -:ref:`blob tables `. It details attributes, -methods, and behaviors of the ``Connection`` and ``Cursor`` objects. - -.. toctree:: - :maxdepth: 1 - - client - connection - cursor - http - https - blob - - .. _sqlalchemy-by-example: SQLAlchemy by example diff --git a/docs/connect.rst b/docs/connect.rst deleted file mode 100644 index 944fe263..00000000 --- a/docs/connect.rst +++ /dev/null @@ -1,286 +0,0 @@ -.. _connect: - -================== -Connect to CrateDB -================== - -.. NOTE:: - - This page documents the CrateDB client, implementing the - `Python Database API Specification v2.0`_ (PEP 249). - - For help using the `SQLAlchemy`_ dialect, consult the - :ref:`SQLAlchemy dialect documentation `. - -.. SEEALSO:: - - Supplementary information about the CrateDB Database API client can be found - in the :ref:`data types appendix `. - -.. rubric:: Table of contents - -.. contents:: - :local: - -.. _single-node: - -Connect to a single node -======================== - -To connect to a single CrateDB node, use the ``connect()`` function, like so: - - >>> connection = client.connect("", username="") - -Here, replace ```` with a URL pointing to the -:ref:`crate-reference:interface-http` of a CrateDB node. Replace ```` -with the username you are authenticating as. - -.. NOTE:: - - This example authenticates as ``crate``, which is the default - :ref:`database user `. - - Consult the `Authentication`_ section for more information. - -Example node URLs: - -- ``http://localhost:4200/`` -- ``http://crate-1.vm.example.com:4200/`` -- ``http://198.51.100.1:4200/`` - -If the CrateDB hostname is ``crate-1.vm.example.com`` and CrateDB is listening -for HTTP requests on port 4200, the node URL would be -``http://crate-1.vm.example.com:4200/``. - -.. TIP:: - - If a ```` argument is not provided, the library will attempt - to connect to CrateDB on the local host with the default HTTP port number, - i.e. ``http://localhost:4200/``. - - If you're just getting started with CrateDB, the first time you connect, - you can probably omit this argument. - -.. _multiple-nodes: - -Connect to multiple nodes -========================= - -To connect to one of multiple nodes, pass a list of database URLs to the -connect() function, like so: - - >>> connection = client.connect(["", ""], ...) - -Here, ```` and ```` correspond to two node URLs, as -described in the previous section. - -You can pass in as many node URLs as you like. - -.. TIP:: - - For every query, the client will attempt to connect to each node in sequence - until a successful connection is made. Nodes are moved to the end of the - list each time they are tried. - - Over multiple query executions, this behaviour functions as client-side - *round-robin* load balancing. (This is analogous to `round-robin DNS`_.) - -.. _connection-options: - -Connection options -================== - -HTTPS ------ - -You can connect to a CrateDB client via HTTPS by specifying ``https`` in the -URL: - - >>> connection = client.connect('https://localhost:4200/', ...) - -.. SEEALSO:: - - The CrateDB reference has a section on :ref:`setting up SSL - `. This will be a useful background reading for - the following two subsections. - -Server verification -................... - -Server certificates are verified by default. In order to connect to a -SSL-enabled host using self-signed certificates, you will need to provide the -CA certificate file used to sign the server SSL certificate: - - >>> connection = client.connect(..., ca_cert="") - -Here, replace ```` with the path to the CA certificate file. - -You can disable server SSL certificate verification by using the -``verify_ssl_cert`` keyword argument and setting it to ``False``: - - >>> connection = client.connect(..., verify_ssl_cert=False) - - -Client verification -................... - -The client also supports client verification via client certificates. - -Here's how you might do that: - - >>> connection = client.connect(..., cert_file="", key_file="") - -Here, replace ```` with the path to the client certificate file, and -```` with the path to the client private key file. - -.. TIP:: - - Often, you will want to perform server verification *and* client - verification. In such circumstances, you can combine the two methods above - to do both at once. - -Relaxing minimum SSL version -............................ - -urrlib3 v2 dropped support for TLS 1.0 and TLS 1.1 by default, see `Modern security by default - -HTTPS requires TLS 1.2+`_. If you need to re-enable it, use the ``ssl_relax_minimum_version`` flag, -which will configure ``kwargs["ssl_minimum_version"] = ssl.TLSVersion.MINIMUM_SUPPORTED``. - - >>> connection = client.connect(..., ssl_relax_minimum_version=True) - - -Timeout -------- - -Connection timeouts (in seconds) can be configured with the optional -``timeout`` argument: - - >>> connection = client.connect(..., timeout=5) - -Here, replace ``...`` with the rest of your arguments. - -.. NOTE:: - - If no timeout is specified, the client will use the default Python - :func:`socket timeout `. - -Tracebacks ----------- - -In the event of a connection error, a :mod:`py:traceback` will be printed, if -you set the optional ``error_trace`` argument to ``True``, like so: - - >>> connection = client.connect(..., error_trace=True) - -Backoff Factor --------------- - -When attempting to make a request, the connection can be configured so that -retries are made in increasing time intervals. This can be configured like so: - - >>> connection = client.connect(..., backoff_factor=0.1) - -If ``backoff_factor`` is set to 0.1, then the delay between retries will be 0.0, -0.1, 0.2, 0.4 etc. The maximum backoff factor cannot exceed 120 seconds and by -default its value is 0. - -Socket Options --------------- - -Creating connections uses :class:`urllib3 default socket options -` but additionally enables TCP -keepalive by setting ``socket.SO_KEEPALIVE`` to ``1``. - -Keepalive can be disabled using the ``socket_keepalive`` argument, like so: - - >>> connection = client.connect(..., socket_keepalive=False) - -If keepalive is enabled (default), there are three additional, optional socket -options that can be configured via connection arguments. - -:``socket_tcp_keepidle``: - - Set the ``TCP_KEEPIDLE`` socket option, which overrides - ``net.ipv4.tcp_keepalive_time`` kernel setting if ``socket_keepalive`` is - ``True``. - -:``socket_tcp_keepintvl``: - - Set the ``TCP_KEEPINTVL`` socket option, which overrides - ``net.ipv4.tcp_keepalive_intvl`` kernel setting if ``socket_keepalive`` is - ``True``. - -:``socket_tcp_keepcnt``: - - Set the ``TCP_KEEPCNT`` socket option, which overrides - ``net.ipv4.tcp_keepalive_probes`` kernel setting if ``socket_keepalive`` is - ``True``. - -.. _authentication: - -Authentication -============== - -.. NOTE:: - - Authentication was introduced in CrateDB versions 2.1.x. - - If you are using CrateDB 2.1.x or later, you must supply a username. If you - are using earlier versions of CrateDB, this argument is not supported. - -You can authenticate with CrateDB like so: - - >>> connection = client.connect(..., username="", password="") - -At your disposal, you can also embed the credentials into the URI, like so: - - >>> connection = client.connect("https://:@cratedb.example.org:4200") - -Here, replace ```` and ```` with the appropriate username -and password. - -.. TIP:: - - If you have not configured a custom :ref:`database user - `, you probably want to - authenticate as the CrateDB superuser, which is ``crate``. The superuser - does not have a password, so you can omit the ``password`` argument. - -.. _schema-selection: - -Schema selection -================ - -You can select a schema using the optional ``schema`` argument, like so: - - >>> connection = client.connect(..., schema="") - -Here, replace ```` with the name of your schema, and replace ``...`` -with the rest of your arguments. - -.. TIP:: - - The default CrateDB schema is ``doc``, and if you do not specify a schema, - this is what will be used. - - However, you can query any schema you like by specifying it in the query. - -Next steps -========== - -Once you're connected, you can :ref:`query CrateDB `. - -.. SEEALSO:: - - Check out the `sample application`_ (and the corresponding `sample - application documentation`_) for a practical demonstration of this driver - in use. - - -.. _client-side random load balancing: https://en.wikipedia.org/wiki/Load_balancing_(computing)#Client-side_random_load_balancing -.. _Modern security by default - HTTPS requires TLS 1.2+: https://urllib3.readthedocs.io/en/latest/v2-migration-guide.html#https-requires-tls-1-2 -.. _Python Database API Specification v2.0: https://www.python.org/dev/peps/pep-0249/ -.. _round-robin DNS: https://en.wikipedia.org/wiki/Round-robin_DNS -.. _sample application: https://github.com/crate/crate-sample-apps/tree/main/python-flask -.. _sample application documentation: https://github.com/crate/crate-sample-apps/blob/main/python-flask/documentation.md -.. _SQLAlchemy: https://www.sqlalchemy.org/ diff --git a/docs/data-types.rst b/docs/data-types.rst index 2c55e7a7..51a3e659 100644 --- a/docs/data-types.rst +++ b/docs/data-types.rst @@ -4,94 +4,6 @@ Data types ========== -The :ref:`Database API client ` and the :ref:`SQLAlchemy dialect -` use different Python data types. Consult the corresponding -section for further information. - -.. rubric:: Table of contents - -.. contents:: - :local: - -.. _data-types-db-api: - -Database API client -=================== - -This section documents data types for the CrateDB :ref:`Database API client -`. - -.. CAUTION:: - - The CrateDB Database API client implementation is incomplete. For the time - being, the client uses native Python types. - -In general, types are mapped as follows: - -============= =========== -CrateDB Python -============= =========== -`boolean`__ `boolean`__ -`string`__ `str`__ -`int`__ `int`__ -`long`__ `int`__ -`short`__ `int`__ -`double`__ `float`__ - -`float`__ `float`__ -`byte`__ `int`__ -`geo_point`__ `list`__ -`geo_shape`__ `dict`__ -`object`__ `dict`__ -`array`__ `list`__ -============= =========== - -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#boolean -__ https://docs.python.org/3/library/stdtypes.html#boolean-type-bool -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#character-data -__ https://docs.python.org/3/library/stdtypes.html#str -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data -__ https://docs.python.org/3/library/functions.html#int -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data -__ https://docs.python.org/3/library/functions.html#int -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data -__ https://docs.python.org/3/library/functions.html#int -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data -__ https://docs.python.org/3/library/functions.html#float -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data -__ https://docs.python.org/3/library/functions.html#float -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data -__ https://docs.python.org/3/library/functions.html#int -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#geo-point -__ https://docs.python.org/3/library/stdtypes.html#list -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#geo-shape -__ https://docs.python.org/3/library/stdtypes.html#dict -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#object -__ https://docs.python.org/3/library/stdtypes.html#dict -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#array -__ https://docs.python.org/3/library/stdtypes.html#list - -When writing to CrateDB, the following conversions take place: - -============= ==================================== -Python CrateDB -============= ==================================== -`Decimal`__ `string`__ -`date`__ `integer`__, `long`__, or `string`__ -`datetime`__ `integer`__, `long`__, or `string`__ -============= ==================================== - -__ https://docs.python.org/3/library/decimal.html -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#character-data -__ https://docs.python.org/3/library/datetime.html#date-objects -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#character-data -__ https://docs.python.org/3/library/datetime.html#datetime-objects -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#character-data - .. NOTE:: The type that ``date`` and ``datetime`` objects are mapped to, depends on the diff --git a/docs/getting-started.rst b/docs/getting-started.rst index a0ae8d09..1125bece 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -17,44 +17,23 @@ Install .. highlight:: sh -The CrateDB Python client is available as package `crate`_ on `PyPI`_. +The CrateDB Python client is available as package ``sqlalchemy-cratedb`` on `PyPI`_. To install the most recent driver version, including the SQLAlchemy dialect extension, run:: - pip install "crate[sqlalchemy]" --upgrade + pip install --upgrade sqlalchemy-cratedb After that is done, you can import the library, like so: .. code-block:: python - >>> from crate import client - -Interactive use -=============== - -Python provides a REPL_, also known as an interactive language shell. It's a -handy way to experiment with code and try out new libraries. We recommend -`IPython`_, which you can install, like so:: - - pip install ipython - -Once installed, you can start it up, like this:: - - ipython - -From there, try importing the CrateDB Python client library and seeing how far -you get with the built-in ``help()`` function (that can be called on any -object), IPython's autocompletion, and many other features. - -.. SEEALSO:: - - `The IPython Documentation`_ + >>> from sqlalchemy_cratedb import CrateDialect Set up as a dependency ====================== -There are `many ways`_ to add the ``crate`` package as a dependency to your +There are `many ways`_ to add the ``sqlalchemy-cratedb`` package as a dependency to your project. All of them work equally well. Please note that you may want to employ package version pinning in order to keep the environment of your project stable and reproducible, achieving `repeatable installations`_. @@ -66,11 +45,8 @@ Next steps Learn how to :ref:`connect to CrateDB `. -.. _crate: https://pypi.org/project/crate/ +.. _sqlalchemy-cratedb: https://pypi.org/project/sqlalchemy-cratedb/ .. _CrateDB: https://crate.io/products/cratedb/ -.. _IPython: https://ipython.org/ .. _many ways: https://packaging.python.org/key_projects/ .. _PyPI: https://pypi.org/ .. _repeatable installations: https://pip.pypa.io/en/latest/topics/repeatable-installs/ -.. _REPL: https://en.wikipedia.org/wiki/Read%E2%80%93eval%E2%80%93print_loop -.. _The IPython Documentation: https://ipython.readthedocs.io/ diff --git a/docs/index-all.rst b/docs/index-all.rst index 85a508e9..8ef1b682 100644 --- a/docs/index-all.rst +++ b/docs/index-all.rst @@ -13,10 +13,6 @@ CrateDB Python Client -- all pages :maxdepth: 2 getting-started - connect - query - blobs sqlalchemy data-types by-example/index - other-options diff --git a/docs/index.rst b/docs/index.rst index 27e4752e..7cf2a23f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,8 +1,8 @@ .. _index: -##################### -CrateDB Python Client -##################### +############################## +SQLAlchemy dialect for CrateDB +############################## .. rubric:: Table of contents @@ -15,26 +15,21 @@ CrateDB Python Client Introduction ************ -The Python client library for `CrateDB`_ implements the Python Database API -Specification v2.0 (`PEP 249`_), and also includes the :ref:`CrateDB dialect -` for `SQLAlchemy`_. +The :ref:`CrateDB dialect ` for `SQLAlchemy`_ provides adapters +for `CrateDB`_ and SQLAlchemy. The supported versions are 1.3, 1.4, and 2.0. -The Python driver can be used to connect to both `CrateDB`_ and `CrateDB +The connector can be used to connect to both `CrateDB`_ and `CrateDB Cloud`_, and is verified to work on Linux, macOS, and Windows. It is used by -the `Crash CLI`_, as well as other libraries and applications connecting to +pandas, Dask, and many other libraries and applications connecting to CrateDB from the Python ecosystem. It is verified to work with CPython, but it has also been tested successfully with `PyPy`_. -Please make sure to also visit the section about :ref:`other-options`, using -the :ref:`crate-reference:interface-postgresql` interface of `CrateDB`_. - ************* Documentation ************* -For general help about the Python Database API, or SQLAlchemy, please consult -`PEP 249`_, the `SQLAlchemy tutorial`_, and the general `SQLAlchemy +Please consult the `SQLAlchemy tutorial`_, and the general `SQLAlchemy documentation`_. For more detailed information about how to install the client driver, how to connect to a CrateDB cluster, and how to run queries, consult the resources @@ -44,50 +39,15 @@ referenced below. :titlesonly: getting-started - connect - query - blobs - -DB API -====== +SQLAlchemy +========== Install package from PyPI. .. code-block:: shell - pip install crate - -Connect to CrateDB instance running on ``localhost``. - -.. code-block:: python - - # Connect using DB API. - from crate import client - from pprint import pp - - query = "SELECT country, mountain, coordinates, height FROM sys.summits ORDER BY country;" - - with client.connect("localhost:4200", username="crate") as connection: - cursor = connection.cursor() - cursor.execute(query) - pp(cursor.fetchall()) - cursor.close() - -Connect to `CrateDB Cloud`_. - -.. code-block:: python - - # Connect using DB API. - from crate import client - connection = client.connect( - servers="https://example.aks1.westeurope.azure.cratedb.net:4200", - username="admin", - password="") - - -SQLAlchemy -========== + pip install sqlalchemy-cratedb The CrateDB dialect for `SQLAlchemy`_ offers convenient ORM access and supports CrateDB's ``OBJECT``, ``ARRAY``, and geospatial data types using `GeoJSON`_, @@ -102,7 +62,7 @@ Install package from PyPI with DB API and SQLAlchemy support. .. code-block:: shell - pip install 'crate[sqlalchemy]' pandas + pip install sqlalchemy-cratedb pandas Connect to CrateDB instance running on ``localhost``. diff --git a/docs/other-options.rst b/docs/other-options.rst deleted file mode 100644 index 95e5a0ad..00000000 --- a/docs/other-options.rst +++ /dev/null @@ -1,55 +0,0 @@ -.. _other-options: - -##################################### -Other connectivity options for Python -##################################### - - -************ -Introduction -************ - -Using the :ref:`crate-reference:interface-postgresql` interface of `CrateDB`_, -there are a few other connectivity options for Python. This section enumerates -the verified drivers, together with some example and test case code using them. - - -******* -Details -******* - -- `asyncpg`_, see `testing CrateDB with asyncpg`_. - -- `psycopg2`_ - - The `CrateDB Astronomer/Airflow tutorials`_ repository includes a few - orchestration examples implemented using `Apache Airflow`_ DAGs for different - import and export tasks, and for automating recurrent queries. It accompanies - a series of articles starting with `CrateDB and Apache Airflow » Automating - Data Export to S3`_. - -- `psycopg3`_, see `testing CrateDB with psycopg3`_. - -- ODBC connectivity is offered by `pyodbc`_ and `turbodbc`_, see - `testing CrateDB with pyodbc`_ and `using CrateDB with turbodbc`_. - -- `connector-x`_ promises to be the fastest library to load data from DB to - DataFrames in Rust and Python. It is the designated database connector - library for `Apache Arrow DataFusion`_. - - -.. _asyncpg: https://github.com/MagicStack/asyncpg -.. _Apache Airflow: https://github.com/apache/airflow -.. _Apache Arrow DataFusion: https://github.com/apache/arrow-datafusion -.. _connector-x: https://github.com/sfu-db/connector-x -.. _CrateDB: https://github.com/crate/crate -.. _CrateDB Astronomer/Airflow tutorials: https://github.com/crate/crate-airflow-tutorial -.. _CrateDB and Apache Airflow » Automating Data Export to S3: https://community.crate.io/t/cratedb-and-apache-airflow-automating-data-export-to-s3/901 -.. _pyodbc: https://github.com/mkleehammer/pyodbc -.. _psycopg2: https://github.com/psycopg/psycopg2 -.. _psycopg3: https://github.com/psycopg/psycopg -.. _Testing CrateDB with asyncpg: https://github.com/crate/crate-qa/blob/master/tests/client_tests/python/asyncpg/test_asyncpg.py -.. _Testing CrateDB with psycopg3: https://github.com/crate/crate-qa/blob/master/tests/client_tests/python/psycopg3/test_psycopg3.py -.. _Testing CrateDB with pyodbc: https://github.com/crate/crate-qa/blob/master/tests/client_tests/odbc/test_pyodbc.py -.. _turbodbc: https://github.com/blue-yonder/turbodbc -.. _Using CrateDB with turbodbc: https://github.com/crate/cratedb-examples/pull/18 diff --git a/docs/query.rst b/docs/query.rst deleted file mode 100644 index a408f369..00000000 --- a/docs/query.rst +++ /dev/null @@ -1,303 +0,0 @@ -.. _query: - -============= -Query CrateDB -============= - -.. NOTE:: - - This page documents the CrateDB client, implementing the - `Python Database API Specification v2.0`_ (PEP 249). - - For help using the `SQLAlchemy`_ dialect, consult - :ref:`the SQLAlchemy dialect documentation `. - -.. SEEALSO:: - - Supplementary information about the CrateDB Database API client can be found - in the :ref:`data types appendix `. - -.. rubric:: Table of contents - -.. contents:: - :local: - -.. _cursor: - -Using a cursor -============== - -After :ref:`connecting to CrateDB `, you can execute queries via a -`database cursor`_. - -Open a cursor like so: - - >>> cursor = connection.cursor() - -.. _inserts: - -Inserting data -============== - -Regular inserts ---------------- - -Regular inserts are possible with the ``execute()`` method, like so: - - >>> cursor.execute( - ... "INSERT INTO locations (name, date, kind, position) VALUES (?, ?, ?, ?)", - ... ("Einstein Cross", "2007-03-11", "Quasar", 7)) - -Here, the values of the :class:`py:tuple` (the second argument) are safely -interpolated into the query string (the first argument) where the ``?`` -characters appear, in the order they appear. - -.. WARNING:: - - Never use string concatenation to build query strings. - - Always use the parameter interpolation feature of the client library to - guard against malicious input, as demonstrated in the example above. - -Bulk inserts ------------- - -:ref:`Bulk inserts ` are possible with the -``executemany()`` method, which takes a :class:`py:list` of tuples to insert: - - >>> cursor.executemany( - ... "INSERT INTO locations (name, date, kind, position) VALUES (?, ?, ?, ?)", - ... [('Cloverleaf', '2007-03-11', 'Quasar', 7), - ... ('Old Faithful', '2007-03-11', 'Quasar', 7)]) - [{'rowcount': 1}, {'rowcount': 1}] - -The ``executemany()`` method returns a result :class:`dictionary ` -for every tuple. This dictionary always has a ``rowcount`` key, indicating -how many rows were inserted. If an error occurs, the ``rowcount`` value is -``-2``, and the dictionary may additionally have an ``error_message`` key. - -.. _selects: - -Selecting data -============== - -Executing a query ------------------ - -Selects can be performed with the ``execute()`` method, like so: - - >>> cursor.execute("SELECT name FROM locations WHERE name = ?", ("Algol",)) - -Like with :ref:`inserts `, here, the single value of the tuple (the -second argument) is safely interpolated into the query string (the first -argument) where the ``?`` character appears. - -.. WARNING:: - - As with :ref:`inserts `, always use parameter interpolation. - -After executing a query, you can fetch the results using one of three fetch -methods, detailed below. - -Fetching results ----------------- - -.. _fetchone: - -``fetchone()`` -.............. - -After executing a query, a ``fetchone()`` call on the cursor returns an list -representing the next row from the result set: - - >>> result = cursor.fetchone() - ['Algol'] - -You can call ``fetchone()`` multiple times to return multiple rows. - -If no more rows are available, ``None`` is returned. - -.. TIP:: - - The ``cursor`` object is an :term:`py:iterator`, and the ``fetchone()`` - method is an alias for ``next()``. - -.. _fetchmany: - -``fetchmany()`` -............... - -After executing a query, a ``fetch_many()`` call with a numeric argument -returns the specified number of result rows: - - >>> cursor.execute("SELECT name FROM locations order by name") - >>> result = cursor.fetchmany(2) - >>> pprint(result) - [['Aldebaran'], ['Algol']] - -If a number is not given as an argument, ``fetch_many()`` will return a result -list with one result row: - - >>> cursor.fetchmany() - [['Allosimanius Syneca']] - -.. _fetchall: - -``fetchall()`` -.............. - -After executing a query, a ``fetchall()`` call on the cursor returns all -remaining rows: - - >>> cursor.execute("SELECT name FROM locations ORDER BY name") - >>> cursor.fetchall() - [['Aldebaran'], - ['Algol'], - ['Allosimanius Syneca'], - ... - ['Old Faithful'], - ['Outer Eastern Rim']] - -Accessing column names -====================== - -Result rows are lists, not dictionaries. Which means that they do use contain -column names for keys. If you want to access column names, you must use -``cursor.description``. - -The `Python Database API Specification v2.0`_ `defines`_ seven description -attributes per column, but only the first one (column name) is supported by -this library. All other attributes are ``None``. - -Let's say you have a query like this: - - >>> cursor.execute("SELECT * FROM locations ORDER BY name") - >>> cursor.fetchone() - [1373932800000, - None, - 'Max Quordlepleen claims that the only thing left ...', - ... - None, - 1] - -The cursor ``description`` might look like this: - - >>> cursor.description - (('date', None, None, None, None, None, None), - ('datetime_tz', None, None, None, None, None, None), - ('datetime_notz', None, None, None, None, None, None), - ('description', None, None, None, None, None, None), - ... - ('nullable_datetime', None, None, None, None, None, None), - ('position', None, None, None, None, None, None)) - -You can turn this into something more manageable with :ref:`py:tut-listcomps`: - - >>> [column[0] for column in cursor.description] - ['date', 'datetime_tz', 'datetime_notz', ..., 'nullable_datetime', 'position'] - - -Data type conversion -==================== - -The cursor object can optionally convert database types to native Python data -types. There is a default implementation for the CrateDB data types ``IP`` and -``TIMESTAMP`` on behalf of the ``DefaultTypeConverter``. - - >>> from crate.client.converter import DefaultTypeConverter - >>> from crate.client.cursor import Cursor - >>> cursor = connection.cursor(converter=DefaultTypeConverter()) - - >>> cursor.execute("SELECT datetime_tz, datetime_notz FROM locations ORDER BY name") - - >>> cursor.fetchone() - [datetime.datetime(2022, 7, 18, 18, 10, 36, 758000), datetime.datetime(2022, 7, 18, 18, 10, 36, 758000)] - - -Custom data type conversion -=========================== - -By providing a custom converter instance, you can define your own data type -conversions. For investigating the list of available data types, please either -inspect the ``DataType`` enum, or the documentation about the list of available -:ref:`CrateDB data type identifiers for the HTTP interface -`. - -This example creates and applies a simple custom converter for converging -CrateDB's ``BOOLEAN`` type to Python's ``str`` type. It is using a simple -converter function defined as ``lambda``, which assigns ``yes`` for boolean -``True``, and ``no`` otherwise. - - >>> from crate.client.converter import Converter, DataType - - >>> converter = Converter() - >>> converter.set(DataType.BOOLEAN, lambda value: value is True and "yes" or "no") - >>> cursor = connection.cursor(converter=converter) - - >>> cursor.execute("SELECT flag FROM locations ORDER BY name") - - >>> cursor.fetchone() - ['no'] - - -``TIMESTAMP`` conversion with time zone -======================================= - -Based on the data type converter functionality, the driver offers a convenient -interface to make it return timezone-aware ``datetime`` objects, using the -desired time zone. - -For your reference, in the following examples, epoch 1658167836758 is -``Mon, 18 Jul 2022 18:10:36 GMT``. - - >>> import datetime - >>> tz_mst = datetime.timezone(datetime.timedelta(hours=7), name="MST") - >>> cursor = connection.cursor(time_zone=tz_mst) - - >>> cursor.execute("SELECT datetime_tz FROM locations ORDER BY name") - - >>> cursor.fetchone() - [datetime.datetime(2022, 7, 19, 1, 10, 36, 758000, tzinfo=datetime.timezone(datetime.timedelta(seconds=25200), 'MST'))] - -For the ``time_zone`` keyword argument, different data types are supported. -The available options are: - -- ``datetime.timezone.utc`` -- ``datetime.timezone(datetime.timedelta(hours=7), name="MST")`` -- ``pytz.timezone("Australia/Sydney")`` -- ``zoneinfo.ZoneInfo("Australia/Sydney")`` -- ``+0530`` (UTC offset in string format) - -Let's exercise all of them. - - >>> cursor.time_zone = datetime.timezone.utc - >>> cursor.execute("SELECT datetime_tz FROM locations ORDER BY name") - >>> cursor.fetchone() - [datetime.datetime(2022, 7, 18, 18, 10, 36, 758000, tzinfo=datetime.timezone.utc)] - - >>> import pytz - >>> cursor.time_zone = pytz.timezone("Australia/Sydney") - >>> cursor.execute("SELECT datetime_tz FROM locations ORDER BY name") - >>> cursor.fetchone() - ['foo', datetime.datetime(2022, 7, 19, 4, 10, 36, 758000, tzinfo=)] - - >>> try: - ... import zoneinfo - ... except ImportError: - ... from backports import zoneinfo - - >>> cursor.time_zone = zoneinfo.ZoneInfo("Australia/Sydney") - >>> cursor.execute("SELECT datetime_tz FROM locations ORDER BY name") - >>> cursor.fetchone() - [datetime.datetime(2022, 7, 19, 4, 10, 36, 758000, tzinfo=zoneinfo.ZoneInfo(key='Australia/Sydney'))] - - >>> cursor.time_zone = "+0530" - >>> cursor.execute("SELECT datetime_tz FROM locations ORDER BY name") - >>> cursor.fetchone() - [datetime.datetime(2022, 7, 18, 23, 40, 36, 758000, tzinfo=datetime.timezone(datetime.timedelta(seconds=19800), '+0530'))] - - -.. _database cursor: https://en.wikipedia.org/wiki/Cursor_(databases) -.. _defines: https://legacy.python.org/dev/peps/pep-0249/#description -.. _Python Database API Specification v2.0: https://www.python.org/dev/peps/pep-0249/ -.. _SQLAlchemy: https://www.sqlalchemy.org/ diff --git a/src/crate/__init__.py b/src/crate/__init__.py index 1fcff2bb..bbe84413 100644 --- a/src/crate/__init__.py +++ b/src/crate/__init__.py @@ -18,11 +18,3 @@ # However, if you have executed another commercial license agreement # with Crate these terms will supersede the license and you may use the # software solely pursuant to the terms of the relevant commercial agreement. - -# this is a namespace package -try: - import pkg_resources - pkg_resources.declare_namespace(__name__) -except ImportError: - import pkgutil - __path__ = pkgutil.extend_path(__path__, __name__) diff --git a/src/crate/client/__init__.py b/src/crate/client/__init__.py index 3d67a541..e69de29b 100644 --- a/src/crate/client/__init__.py +++ b/src/crate/client/__init__.py @@ -1,36 +0,0 @@ -# -*- coding: utf-8; -*- -# -# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor -# license agreements. See the NOTICE file distributed with this work for -# additional information regarding copyright ownership. Crate licenses -# this file to you under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. You may -# obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# However, if you have executed another commercial license agreement -# with Crate these terms will supersede the license and you may use the -# software solely pursuant to the terms of the relevant commercial agreement. - -from .connection import Connection as connect -from .exceptions import Error - -__all__ = [ - connect, - Error, -] - -# version string read from setup.py using a regex. Take care not to break the -# regex! -__version__ = "0.34.0" - -apilevel = "2.0" -threadsafety = 2 -paramstyle = "qmark" diff --git a/src/crate/client/blob.py b/src/crate/client/blob.py deleted file mode 100644 index 73d733ef..00000000 --- a/src/crate/client/blob.py +++ /dev/null @@ -1,103 +0,0 @@ -# -*- coding: utf-8; -*- -# -# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor -# license agreements. See the NOTICE file distributed with this work for -# additional information regarding copyright ownership. Crate licenses -# this file to you under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. You may -# obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# However, if you have executed another commercial license agreement -# with Crate these terms will supersede the license and you may use the -# software solely pursuant to the terms of the relevant commercial agreement. - -import hashlib - - -class BlobContainer(object): - """ class that represents a blob collection in crate. - - can be used to download, upload and delete blobs - """ - - def __init__(self, container_name, connection): - self.container_name = container_name - self.conn = connection - - def _compute_digest(self, f): - f.seek(0) - m = hashlib.sha1() - while True: - d = f.read(1024 * 32) - if not d: - break - m.update(d) - f.seek(0) - return m.hexdigest() - - def put(self, f, digest=None): - """ - Upload a blob - - :param f: - File object to be uploaded (required to support seek if digest is - not provided). - :param digest: - Optional SHA-1 hex digest of the file contents. Gets computed - before actual upload if not provided, which requires an extra file - read. - :return: - The hex digest of the uploaded blob if not provided in the call. - Otherwise a boolean indicating if the blob has been newly created. - """ - - if digest: - actual_digest = digest - else: - actual_digest = self._compute_digest(f) - - created = self.conn.client.blob_put(self.container_name, - actual_digest, f) - if digest: - return created - return actual_digest - - def get(self, digest, chunk_size=1024 * 128): - """ - Return the contents of a blob - - :param digest: the hex digest of the blob to return - :param chunk_size: the size of the chunks returned on each iteration - :return: generator returning chunks of data - """ - return self.conn.client.blob_get(self.container_name, digest, - chunk_size) - - def delete(self, digest): - """ - Delete a blob - - :param digest: the hex digest of the blob to be deleted - :return: True if blob existed - """ - return self.conn.client.blob_del(self.container_name, digest) - - def exists(self, digest): - """ - Check if a blob exists - - :param digest: Hex digest of the blob - :return: Boolean indicating existence of the blob - """ - return self.conn.client.blob_exists(self.container_name, digest) - - def __repr__(self): - return "".format(self.container_name) diff --git a/src/crate/client/connection.py b/src/crate/client/connection.py deleted file mode 100644 index 9e72b2f7..00000000 --- a/src/crate/client/connection.py +++ /dev/null @@ -1,215 +0,0 @@ -# -*- coding: utf-8; -*- -# -# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor -# license agreements. See the NOTICE file distributed with this work for -# additional information regarding copyright ownership. Crate licenses -# this file to you under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. You may -# obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# However, if you have executed another commercial license agreement -# with Crate these terms will supersede the license and you may use the -# software solely pursuant to the terms of the relevant commercial agreement. - -from .cursor import Cursor -from .exceptions import ProgrammingError, ConnectionError -from .http import Client -from .blob import BlobContainer -from verlib2 import Version - - -class Connection(object): - - def __init__(self, - servers=None, - timeout=None, - backoff_factor=0, - client=None, - verify_ssl_cert=True, - ca_cert=None, - error_trace=False, - cert_file=None, - key_file=None, - ssl_relax_minimum_version=False, - username=None, - password=None, - schema=None, - pool_size=None, - socket_keepalive=True, - socket_tcp_keepidle=None, - socket_tcp_keepintvl=None, - socket_tcp_keepcnt=None, - converter=None, - time_zone=None, - ): - """ - :param servers: - either a string in the form of ':' - or a list of servers in the form of [':', '...'] - :param timeout: - (optional) - define the retry timeout for unreachable servers in seconds - :param backoff_factor: - (optional) - define the retry interval for unreachable servers in seconds - :param client: - (optional - for testing) - client used to communicate with crate. - :param verify_ssl_cert: - if set to ``False``, disable SSL server certificate verification. - defaults to ``True`` - :param ca_cert: - a path to a CA certificate to use when verifying the SSL server - certificate. - :param error_trace: - if set to ``True`` return a whole stacktrace of any server error if - one occurs - :param cert_file: - a path to the client certificate to present to the server. - :param key_file: - a path to the client key to use when communicating with the server. - :param username: - the username in the database. - :param password: - the password of the user in the database. - :param pool_size: - (optional) - Number of connections to save that can be reused. - More than 1 is useful in multithreaded situations. - :param socket_keepalive: - (optional, defaults to ``True``) - Enable TCP keepalive on socket level. - :param socket_tcp_keepidle: - (optional) - Set the ``TCP_KEEPIDLE`` socket option, which overrides - ``net.ipv4.tcp_keepalive_time`` kernel setting if ``socket_keepalive`` - is ``True``. - :param socket_tcp_keepintvl: - (optional) - Set the ``TCP_KEEPINTVL`` socket option, which overrides - ``net.ipv4.tcp_keepalive_intvl`` kernel setting if ``socket_keepalive`` - is ``True``. - :param socket_tcp_keepcnt: - (optional) - Set the ``TCP_KEEPCNT`` socket option, which overrides - ``net.ipv4.tcp_keepalive_probes`` kernel setting if ``socket_keepalive`` - is ``True``. - :param converter: - (optional, defaults to ``None``) - A `Converter` object to propagate to newly created `Cursor` objects. - :param time_zone: - (optional, defaults to ``None``) - A time zone specifier used for returning `TIMESTAMP` types as - timezone-aware native Python `datetime` objects. - - Different data types are supported. Available options are: - - - ``datetime.timezone.utc`` - - ``datetime.timezone(datetime.timedelta(hours=7), name="MST")`` - - ``pytz.timezone("Australia/Sydney")`` - - ``zoneinfo.ZoneInfo("Australia/Sydney")`` - - ``+0530`` (UTC offset in string format) - - When `time_zone` is `None`, the returned `datetime` objects are - "naive", without any `tzinfo`, converted using ``datetime.utcfromtimestamp(...)``. - - When `time_zone` is given, the returned `datetime` objects are "aware", - with `tzinfo` set, converted using ``datetime.fromtimestamp(..., tz=...)``. - """ - - self._converter = converter - self.time_zone = time_zone - - if client: - self.client = client - else: - self.client = Client(servers, - timeout=timeout, - backoff_factor=backoff_factor, - verify_ssl_cert=verify_ssl_cert, - ca_cert=ca_cert, - error_trace=error_trace, - cert_file=cert_file, - key_file=key_file, - ssl_relax_minimum_version=ssl_relax_minimum_version, - username=username, - password=password, - schema=schema, - pool_size=pool_size, - socket_keepalive=socket_keepalive, - socket_tcp_keepidle=socket_tcp_keepidle, - socket_tcp_keepintvl=socket_tcp_keepintvl, - socket_tcp_keepcnt=socket_tcp_keepcnt, - ) - self.lowest_server_version = self._lowest_server_version() - self._closed = False - - def cursor(self, **kwargs) -> Cursor: - """ - Return a new Cursor Object using the connection. - """ - converter = kwargs.pop("converter", self._converter) - time_zone = kwargs.pop("time_zone", self.time_zone) - if not self._closed: - return Cursor( - connection=self, - converter=converter, - time_zone=time_zone, - ) - else: - raise ProgrammingError("Connection closed") - - def close(self): - """ - Close the connection now - """ - self._closed = True - self.client.close() - - def commit(self): - """ - Transactions are not supported, so ``commit`` is not implemented. - """ - if self._closed: - raise ProgrammingError("Connection closed") - - def get_blob_container(self, container_name): - """ Retrieve a BlobContainer for `container_name` - - :param container_name: the name of the BLOB container. - :returns: a :class:ContainerObject - """ - return BlobContainer(container_name, self) - - def _lowest_server_version(self): - lowest = None - for server in self.client.active_servers: - try: - _, _, version = self.client.server_infos(server) - version = Version(version) - except (ValueError, ConnectionError): - continue - if not lowest or version < lowest: - lowest = version - return lowest or Version('0.0.0') - - def __repr__(self): - return ''.format(repr(self.client)) - - def __enter__(self): - return self - - def __exit__(self, *excs): - self.close() - - -# For backwards compatibility and not to break existing imports -connect = Connection diff --git a/src/crate/client/converter.py b/src/crate/client/converter.py deleted file mode 100644 index c4dbf598..00000000 --- a/src/crate/client/converter.py +++ /dev/null @@ -1,138 +0,0 @@ -# -*- coding: utf-8; -*- -# -# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor -# license agreements. See the NOTICE file distributed with this work for -# additional information regarding copyright ownership. Crate licenses -# this file to you under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. You may -# obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# However, if you have executed another commercial license agreement -# with Crate these terms will supersede the license and you may use the -# software solely pursuant to the terms of the relevant commercial agreement. -""" -Machinery for converting CrateDB database types to native Python data types. - -https://crate.io/docs/crate/reference/en/latest/interfaces/http.html#column-types -""" -import ipaddress -from copy import deepcopy -from datetime import datetime -from enum import Enum -from typing import Any, Callable, Dict, List, Optional, Union - -ConverterFunction = Callable[[Optional[Any]], Optional[Any]] -ColTypesDefinition = Union[int, List[Union[int, "ColTypesDefinition"]]] - - -def _to_ipaddress(value: Optional[str]) -> Optional[Union[ipaddress.IPv4Address, ipaddress.IPv6Address]]: - """ - https://docs.python.org/3/library/ipaddress.html - """ - if value is None: - return None - return ipaddress.ip_address(value) - - -def _to_datetime(value: Optional[float]) -> Optional[datetime]: - """ - https://docs.python.org/3/library/datetime.html - """ - if value is None: - return None - return datetime.utcfromtimestamp(value / 1e3) - - -def _to_default(value: Optional[Any]) -> Optional[Any]: - return value - - -# Symbolic aliases for the numeric data type identifiers defined by the CrateDB HTTP interface. -# https://crate.io/docs/crate/reference/en/latest/interfaces/http.html#column-types -class DataType(Enum): - NULL = 0 - NOT_SUPPORTED = 1 - CHAR = 2 - BOOLEAN = 3 - TEXT = 4 - IP = 5 - DOUBLE = 6 - REAL = 7 - SMALLINT = 8 - INTEGER = 9 - BIGINT = 10 - TIMESTAMP_WITH_TZ = 11 - OBJECT = 12 - GEOPOINT = 13 - GEOSHAPE = 14 - TIMESTAMP_WITHOUT_TZ = 15 - UNCHECKED_OBJECT = 16 - REGPROC = 19 - TIME = 20 - OIDVECTOR = 21 - NUMERIC = 22 - REGCLASS = 23 - DATE = 24 - BIT = 25 - JSON = 26 - CHARACTER = 27 - ARRAY = 100 - - -ConverterMapping = Dict[DataType, ConverterFunction] - - -# Map data type identifier to converter function. -_DEFAULT_CONVERTERS: ConverterMapping = { - DataType.IP: _to_ipaddress, - DataType.TIMESTAMP_WITH_TZ: _to_datetime, - DataType.TIMESTAMP_WITHOUT_TZ: _to_datetime, -} - - -class Converter: - def __init__( - self, - mappings: Optional[ConverterMapping] = None, - default: ConverterFunction = _to_default, - ) -> None: - self._mappings = mappings or {} - self._default = default - - def get(self, type_: ColTypesDefinition) -> ConverterFunction: - if isinstance(type_, int): - return self._mappings.get(DataType(type_), self._default) - type_, inner_type = type_ - if DataType(type_) is not DataType.ARRAY: - raise ValueError(f"Data type {type_} is not implemented as collection type") - - inner_convert = self.get(inner_type) - - def convert(value: Any) -> Optional[List[Any]]: - if value is None: - return None - return [inner_convert(x) for x in value] - - return convert - - def set(self, type_: DataType, converter: ConverterFunction): - self._mappings[type_] = converter - - -class DefaultTypeConverter(Converter): - def __init__(self, more_mappings: Optional[ConverterMapping] = None) -> None: - mappings: ConverterMapping = {} - mappings.update(deepcopy(_DEFAULT_CONVERTERS)) - if more_mappings: - mappings.update(deepcopy(more_mappings)) - super().__init__( - mappings=mappings, default=_to_default - ) diff --git a/src/crate/client/cursor.py b/src/crate/client/cursor.py deleted file mode 100644 index c458ae1b..00000000 --- a/src/crate/client/cursor.py +++ /dev/null @@ -1,317 +0,0 @@ -# -*- coding: utf-8; -*- -# -# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor -# license agreements. See the NOTICE file distributed with this work for -# additional information regarding copyright ownership. Crate licenses -# this file to you under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. You may -# obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# However, if you have executed another commercial license agreement -# with Crate these terms will supersede the license and you may use the -# software solely pursuant to the terms of the relevant commercial agreement. -from datetime import datetime, timedelta, timezone - -from .converter import DataType -import warnings -import typing as t - -from .converter import Converter -from .exceptions import ProgrammingError - - -class Cursor(object): - """ - not thread-safe by intention - should not be shared between different threads - """ - lastrowid = None # currently not supported - - def __init__(self, connection, converter: Converter, **kwargs): - self.arraysize = 1 - self.connection = connection - self._converter = converter - self._closed = False - self._result = None - self.rows = None - self._time_zone = None - self.time_zone = kwargs.get("time_zone") - - def execute(self, sql, parameters=None, bulk_parameters=None): - """ - Prepare and execute a database operation (query or command). - """ - if self.connection._closed: - raise ProgrammingError("Connection closed") - - if self._closed: - raise ProgrammingError("Cursor closed") - - self._result = self.connection.client.sql(sql, parameters, - bulk_parameters) - if "rows" in self._result: - if self._converter is None: - self.rows = iter(self._result["rows"]) - else: - self.rows = iter(self._convert_rows()) - - def executemany(self, sql, seq_of_parameters): - """ - Prepare a database operation (query or command) and then execute it - against all parameter sequences or mappings found in the sequence - ``seq_of_parameters``. - """ - row_counts = [] - durations = [] - self.execute(sql, bulk_parameters=seq_of_parameters) - - for result in self._result.get('results', []): - if result.get('rowcount') > -1: - row_counts.append(result.get('rowcount')) - if self.duration > -1: - durations.append(self.duration) - - self._result = { - "rowcount": sum(row_counts) if row_counts else -1, - "duration": sum(durations) if durations else -1, - "rows": [], - "cols": self._result.get("cols", []), - "col_types": self._result.get("col_types", []), - "results": self._result.get("results") - } - if self._converter is None: - self.rows = iter(self._result["rows"]) - else: - self.rows = iter(self._convert_rows()) - return self._result["results"] - - def fetchone(self): - """ - Fetch the next row of a query result set, returning a single sequence, - or None when no more data is available. - Alias for ``next()``. - """ - try: - return self.next() - except StopIteration: - return None - - def __iter__(self): - """ - support iterator interface: - http://legacy.python.org/dev/peps/pep-0249/#iter - - This iterator is shared. Advancing this iterator will advance other - iterators created from this cursor. - """ - warnings.warn("DB-API extension cursor.__iter__() used") - return self - - def fetchmany(self, count=None): - """ - Fetch the next set of rows of a query result, returning a sequence of - sequences (e.g. a list of tuples). An empty sequence is returned when - no more rows are available. - """ - if count is None: - count = self.arraysize - if count == 0: - return self.fetchall() - result = [] - for i in range(count): - try: - result.append(self.next()) - except StopIteration: - pass - return result - - def fetchall(self): - """ - Fetch all (remaining) rows of a query result, returning them as a - sequence of sequences (e.g. a list of tuples). Note that the cursor's - arraysize attribute can affect the performance of this operation. - """ - result = [] - iterate = True - while iterate: - try: - result.append(self.next()) - except StopIteration: - iterate = False - return result - - def close(self): - """ - Close the cursor now - """ - self._closed = True - self._result = None - - def setinputsizes(self, sizes): - """ - Not supported method. - """ - pass - - def setoutputsize(self, size, column=None): - """ - Not supported method. - """ - pass - - @property - def rowcount(self): - """ - This read-only attribute specifies the number of rows that the last - .execute*() produced (for DQL statements like ``SELECT``) or affected - (for DML statements like ``UPDATE`` or ``INSERT``). - """ - if (self._closed or not self._result or "rows" not in self._result): - return -1 - return self._result.get("rowcount", -1) - - def next(self): - """ - Return the next row of a query result set, respecting if cursor was - closed. - """ - if self.rows is None: - raise ProgrammingError( - "No result available. " + - "execute() or executemany() must be called first." - ) - elif not self._closed: - return next(self.rows) - else: - raise ProgrammingError("Cursor closed") - - __next__ = next - - @property - def description(self): - """ - This read-only attribute is a sequence of 7-item sequences. - """ - if self._closed: - return - - description = [] - for col in self._result["cols"]: - description.append((col, - None, - None, - None, - None, - None, - None)) - return tuple(description) - - @property - def duration(self): - """ - This read-only attribute specifies the server-side duration of a query - in milliseconds. - """ - if self._closed or \ - not self._result or \ - "duration" not in self._result: - return -1 - return self._result.get("duration", 0) - - def _convert_rows(self): - """ - Iterate rows, apply type converters, and generate converted rows. - """ - assert "col_types" in self._result and self._result["col_types"], \ - "Unable to apply type conversion without `col_types` information" - - # Resolve `col_types` definition to converter functions. Running the lookup - # redundantly on each row loop iteration would be a huge performance hog. - types = self._result["col_types"] - converters = [ - self._converter.get(type) for type in types - ] - - # Process result rows with conversion. - for row in self._result["rows"]: - yield [ - convert(value) - for convert, value in zip(converters, row) - ] - - @property - def time_zone(self): - """ - Get the current time zone. - """ - return self._time_zone - - @time_zone.setter - def time_zone(self, tz): - """ - Set the time zone. - - Different data types are supported. Available options are: - - - ``datetime.timezone.utc`` - - ``datetime.timezone(datetime.timedelta(hours=7), name="MST")`` - - ``pytz.timezone("Australia/Sydney")`` - - ``zoneinfo.ZoneInfo("Australia/Sydney")`` - - ``+0530`` (UTC offset in string format) - - When `time_zone` is `None`, the returned `datetime` objects are - "naive", without any `tzinfo`, converted using ``datetime.utcfromtimestamp(...)``. - - When `time_zone` is given, the returned `datetime` objects are "aware", - with `tzinfo` set, converted using ``datetime.fromtimestamp(..., tz=...)``. - """ - - # Do nothing when time zone is reset. - if tz is None: - self._time_zone = None - return - - # Requesting datetime-aware `datetime` objects needs the data type converter. - # Implicitly create one, when needed. - if self._converter is None: - self._converter = Converter() - - # When the time zone is given as a string, assume UTC offset format, e.g. `+0530`. - if isinstance(tz, str): - tz = self._timezone_from_utc_offset(tz) - - self._time_zone = tz - - def _to_datetime_with_tz(value: t.Optional[float]) -> t.Optional[datetime]: - """ - Convert CrateDB's `TIMESTAMP` value to a native Python `datetime` - object, with timezone-awareness. - """ - if value is None: - return None - return datetime.fromtimestamp(value / 1e3, tz=self._time_zone) - - # Register converter function for `TIMESTAMP` type. - self._converter.set(DataType.TIMESTAMP_WITH_TZ, _to_datetime_with_tz) - self._converter.set(DataType.TIMESTAMP_WITHOUT_TZ, _to_datetime_with_tz) - - @staticmethod - def _timezone_from_utc_offset(tz) -> timezone: - """ - Convert UTC offset in string format (e.g. `+0530`) into `datetime.timezone` object. - """ - assert len(tz) == 5, f"Time zone '{tz}' is given in invalid UTC offset format" - try: - hours = int(tz[:3]) - minutes = int(tz[0] + tz[3:]) - return timezone(timedelta(hours=hours, minutes=minutes), name=tz) - except Exception as ex: - raise ValueError(f"Time zone '{tz}' is given in invalid UTC offset format: {ex}") diff --git a/src/crate/client/exceptions.py b/src/crate/client/exceptions.py deleted file mode 100644 index 71bf5d8d..00000000 --- a/src/crate/client/exceptions.py +++ /dev/null @@ -1,94 +0,0 @@ -# -*- coding: utf-8; -*- -# -# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor -# license agreements. See the NOTICE file distributed with this work for -# additional information regarding copyright ownership. Crate licenses -# this file to you under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. You may -# obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# However, if you have executed another commercial license agreement -# with Crate these terms will supersede the license and you may use the -# software solely pursuant to the terms of the relevant commercial agreement. - - -class Error(Exception): - - def __init__(self, msg=None, error_trace=None): - # for compatibility reasons we want to keep the exception message - # attribute because clients may depend on it - if msg: - self.message = msg - super(Error, self).__init__(msg) - self.error_trace = error_trace - - -class Warning(Exception): - pass - - -class InterfaceError(Error): - pass - - -class DatabaseError(Error): - pass - - -class InternalError(DatabaseError): - pass - - -class OperationalError(DatabaseError): - pass - - -class ProgrammingError(DatabaseError): - pass - - -class IntegrityError(DatabaseError): - pass - - -class DataError(DatabaseError): - pass - - -class NotSupportedError(DatabaseError): - pass - - -# exceptions not in db api - -class ConnectionError(OperationalError): - pass - - -class BlobException(Exception): - def __init__(self, table, digest): - self.table = table - self.digest = digest - - def __str__(self): - return "{table}/{digest}".format(table=self.table, digest=self.digest) - - -class DigestNotFoundException(BlobException): - pass - - -class BlobLocationNotFoundException(BlobException): - pass - - -class TimezoneUnawareException(Error): - pass diff --git a/src/crate/client/http.py b/src/crate/client/http.py deleted file mode 100644 index a22a1ff0..00000000 --- a/src/crate/client/http.py +++ /dev/null @@ -1,650 +0,0 @@ -# -*- coding: utf-8; -*- -# -# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor -# license agreements. See the NOTICE file distributed with this work for -# additional information regarding copyright ownership. Crate licenses -# this file to you under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. You may -# obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# However, if you have executed another commercial license agreement -# with Crate these terms will supersede the license and you may use the -# software solely pursuant to the terms of the relevant commercial agreement. - - -import calendar -import heapq -import io -import json -import logging -import os -import re -import socket -import ssl -import threading -from urllib.parse import urlparse -from base64 import b64encode -from time import time -from datetime import datetime, date, timezone -from decimal import Decimal -from uuid import UUID - -import urllib3 -from urllib3 import connection_from_url -from urllib3.connection import HTTPConnection -from urllib3.exceptions import ( - HTTPError, - MaxRetryError, - ProtocolError, - ProxyError, - ReadTimeoutError, - SSLError, -) -from urllib3.util.retry import Retry -from verlib2 import Version - -from crate.client.exceptions import ( - ConnectionError, - BlobLocationNotFoundException, - DigestNotFoundException, - ProgrammingError, - IntegrityError, -) - - -logger = logging.getLogger(__name__) - - -_HTTP_PAT = pat = re.compile('https?://.+', re.I) -SRV_UNAVAILABLE_STATUSES = set((502, 503, 504, 509)) -PRESERVE_ACTIVE_SERVER_EXCEPTIONS = set((ConnectionResetError, BrokenPipeError)) -SSL_ONLY_ARGS = set(('ca_certs', 'cert_reqs', 'cert_file', 'key_file')) - - -def super_len(o): - if hasattr(o, '__len__'): - return len(o) - if hasattr(o, 'len'): - return o.len - if hasattr(o, 'fileno'): - try: - fileno = o.fileno() - except io.UnsupportedOperation: - pass - else: - return os.fstat(fileno).st_size - if hasattr(o, 'getvalue'): - # e.g. BytesIO, cStringIO.StringI - return len(o.getvalue()) - - -class CrateJsonEncoder(json.JSONEncoder): - - epoch_aware = datetime(1970, 1, 1, tzinfo=timezone.utc) - epoch_naive = datetime(1970, 1, 1) - - def default(self, o): - if isinstance(o, (Decimal, UUID)): - return str(o) - if isinstance(o, datetime): - if o.tzinfo is not None: - delta = o - self.epoch_aware - else: - delta = o - self.epoch_naive - return int(delta.microseconds / 1000.0 + - (delta.seconds + delta.days * 24 * 3600) * 1000.0) - if isinstance(o, date): - return calendar.timegm(o.timetuple()) * 1000 - return json.JSONEncoder.default(self, o) - - -class Server(object): - - def __init__(self, server, **pool_kw): - socket_options = _get_socket_opts( - pool_kw.pop('socket_keepalive', False), - pool_kw.pop('socket_tcp_keepidle', None), - pool_kw.pop('socket_tcp_keepintvl', None), - pool_kw.pop('socket_tcp_keepcnt', None), - ) - self.pool = connection_from_url( - server, - socket_options=socket_options, - **pool_kw, - ) - - def request(self, - method, - path, - data=None, - stream=False, - headers=None, - username=None, - password=None, - schema=None, - backoff_factor=0, - **kwargs): - """Send a request - - Always set the Content-Length and the Content-Type header. - """ - if headers is None: - headers = {} - if 'Content-Length' not in headers: - length = super_len(data) - if length is not None: - headers['Content-Length'] = length - - # Authentication credentials - if username is not None: - if 'Authorization' not in headers and username is not None: - credentials = username + ':' - if password is not None: - credentials += password - headers['Authorization'] = 'Basic %s' % b64encode(credentials.encode('utf-8')).decode('utf-8') - # For backwards compatibility with Crate <= 2.2 - if 'X-User' not in headers: - headers['X-User'] = username - - if schema is not None: - headers['Default-Schema'] = schema - headers['Accept'] = 'application/json' - headers['Content-Type'] = 'application/json' - kwargs['assert_same_host'] = False - kwargs['redirect'] = False - kwargs['retries'] = Retry(read=0, backoff_factor=backoff_factor) - return self.pool.urlopen( - method, - path, - body=data, - preload_content=not stream, - headers=headers, - **kwargs - ) - - def close(self): - self.pool.close() - - -def _json_from_response(response): - try: - return json.loads(response.data.decode('utf-8')) - except ValueError: - raise ProgrammingError( - "Invalid server response of content-type '{}':\n{}" - .format(response.headers.get("content-type", "unknown"), response.data.decode('utf-8'))) - - -def _blob_path(table, digest): - return '/_blobs/{table}/{digest}'.format(table=table, digest=digest) - - -def _ex_to_message(ex): - return getattr(ex, 'message', None) or str(ex) or repr(ex) - - -def _raise_for_status(response): - """ - Properly raise `IntegrityError` exceptions for CrateDB's `DuplicateKeyException` errors. - """ - try: - return _raise_for_status_real(response) - except ProgrammingError as ex: - if "DuplicateKeyException" in ex.message: - raise IntegrityError(ex.message, error_trace=ex.error_trace) from ex - raise - - -def _raise_for_status_real(response): - """ make sure that only crate.exceptions are raised that are defined in - the DB-API specification """ - message = '' - if 400 <= response.status < 500: - message = '%s Client Error: %s' % (response.status, response.reason) - elif 500 <= response.status < 600: - message = '%s Server Error: %s' % (response.status, response.reason) - else: - return - if response.status == 503: - raise ConnectionError(message) - if response.headers.get("content-type", "").startswith("application/json"): - data = json.loads(response.data.decode('utf-8')) - error = data.get('error', {}) - error_trace = data.get('error_trace', None) - if "results" in data: - errors = [res["error_message"] for res in data["results"] - if res.get("error_message")] - if errors: - raise ProgrammingError("\n".join(errors)) - if isinstance(error, dict): - raise ProgrammingError(error.get('message', ''), - error_trace=error_trace) - raise ProgrammingError(error, error_trace=error_trace) - raise ProgrammingError(message) - - -def _server_url(server): - """ - Normalizes a given server string to an url - - >>> print(_server_url('a')) - http://a - >>> print(_server_url('a:9345')) - http://a:9345 - >>> print(_server_url('https://a:9345')) - https://a:9345 - >>> print(_server_url('https://a')) - https://a - >>> print(_server_url('demo.crate.io')) - http://demo.crate.io - """ - if not _HTTP_PAT.match(server): - server = 'http://%s' % server - parsed = urlparse(server) - url = '%s://%s' % (parsed.scheme, parsed.netloc) - return url - - -def _to_server_list(servers): - if isinstance(servers, str): - servers = servers.split() - return [_server_url(s) for s in servers] - - -def _pool_kw_args(verify_ssl_cert, ca_cert, client_cert, client_key, - timeout=None, pool_size=None): - ca_cert = ca_cert or os.environ.get('REQUESTS_CA_BUNDLE', None) - if ca_cert and not os.path.exists(ca_cert): - # Sanity check - raise IOError('CA bundle file "{}" does not exist.'.format(ca_cert)) - - kw = { - 'ca_certs': ca_cert, - 'cert_reqs': ssl.CERT_REQUIRED if verify_ssl_cert else ssl.CERT_NONE, - 'cert_file': client_cert, - 'key_file': client_key, - } - if timeout is not None: - kw['timeout'] = float(timeout) - if pool_size is not None: - kw['maxsize'] = int(pool_size) - return kw - - -def _remove_certs_for_non_https(server, kwargs): - if server.lower().startswith('https'): - return kwargs - used_ssl_args = SSL_ONLY_ARGS & set(kwargs.keys()) - if used_ssl_args: - kwargs = kwargs.copy() - for arg in used_ssl_args: - kwargs.pop(arg) - return kwargs - - -def _update_pool_kwargs_for_ssl_minimum_version(server, kwargs): - """ - On urllib3 v2, re-add support for TLS 1.0 and TLS 1.1. - - https://urllib3.readthedocs.io/en/latest/v2-migration-guide.html#https-requires-tls-1-2 - """ - if Version(urllib3.__version__) >= Version("2"): - from urllib3.util import parse_url - scheme, _, host, port, *_ = parse_url(server) - if scheme == "https": - kwargs["ssl_minimum_version"] = ssl.TLSVersion.MINIMUM_SUPPORTED - - -def _create_sql_payload(stmt, args, bulk_args): - if not isinstance(stmt, str): - raise ValueError('stmt is not a string') - if args and bulk_args: - raise ValueError('Cannot provide both: args and bulk_args') - - data = { - 'stmt': stmt - } - if args: - data['args'] = args - if bulk_args: - data['bulk_args'] = bulk_args - return json.dumps(data, cls=CrateJsonEncoder) - - -def _get_socket_opts(keepalive=True, - tcp_keepidle=None, - tcp_keepintvl=None, - tcp_keepcnt=None): - """ - Return an optional list of socket options for urllib3's HTTPConnection - constructor. - """ - if not keepalive: - return None - - # always use TCP keepalive - opts = [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)] - - # hasattr check because some options depend on system capabilities - # see https://docs.python.org/3/library/socket.html#socket.SOMAXCONN - if hasattr(socket, 'TCP_KEEPIDLE') and tcp_keepidle is not None: - opts.append((socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, tcp_keepidle)) - if hasattr(socket, 'TCP_KEEPINTVL') and tcp_keepintvl is not None: - opts.append((socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, tcp_keepintvl)) - if hasattr(socket, 'TCP_KEEPCNT') and tcp_keepcnt is not None: - opts.append((socket.IPPROTO_TCP, socket.TCP_KEEPCNT, tcp_keepcnt)) - - # additionally use urllib3's default socket options - return HTTPConnection.default_socket_options + opts - - -class Client(object): - """ - Crate connection client using CrateDB's HTTP API. - """ - - SQL_PATH = '/_sql?types=true' - """Crate URI path for issuing SQL statements.""" - - retry_interval = 30 - """Retry interval for failed servers in seconds.""" - - default_server = "http://127.0.0.1:4200" - """Default server to use if no servers are given on instantiation.""" - - def __init__(self, - servers=None, - timeout=None, - backoff_factor=0, - verify_ssl_cert=True, - ca_cert=None, - error_trace=False, - cert_file=None, - key_file=None, - ssl_relax_minimum_version=False, - username=None, - password=None, - schema=None, - pool_size=None, - socket_keepalive=True, - socket_tcp_keepidle=None, - socket_tcp_keepintvl=None, - socket_tcp_keepcnt=None, - ): - if not servers: - servers = [self.default_server] - else: - servers = _to_server_list(servers) - - # Try to derive credentials from first server argument if not - # explicitly given. - if servers and not username: - try: - url = urlparse(servers[0]) - if url.username is not None: - username = url.username - if url.password is not None: - password = url.password - except Exception as ex: - logger.warning("Unable to decode credentials from database " - "URI, so connecting to CrateDB without " - "authentication: {ex}" - .format(ex=ex)) - - self._active_servers = servers - self._inactive_servers = [] - pool_kw = _pool_kw_args( - verify_ssl_cert, ca_cert, cert_file, key_file, timeout, pool_size, - ) - pool_kw.update({ - 'socket_keepalive': socket_keepalive, - 'socket_tcp_keepidle': socket_tcp_keepidle, - 'socket_tcp_keepintvl': socket_tcp_keepintvl, - 'socket_tcp_keepcnt': socket_tcp_keepcnt, - }) - self.ssl_relax_minimum_version = ssl_relax_minimum_version - self.backoff_factor = backoff_factor - self.server_pool = {} - self._update_server_pool(servers, **pool_kw) - self._pool_kw = pool_kw - self._lock = threading.RLock() - self._local = threading.local() - self.username = username - self.password = password - self.schema = schema - - self.path = self.SQL_PATH - if error_trace: - self.path += '&error_trace=true' - - def close(self): - for server in self.server_pool.values(): - server.close() - - def _create_server(self, server, **pool_kw): - kwargs = _remove_certs_for_non_https(server, pool_kw) - # After updating to urllib3 v2, optionally retain support for TLS 1.0 and TLS 1.1, - # in order to support connectivity to older versions of CrateDB. - if self.ssl_relax_minimum_version: - _update_pool_kwargs_for_ssl_minimum_version(server, kwargs) - self.server_pool[server] = Server(server, **kwargs) - - def _update_server_pool(self, servers, **pool_kw): - for server in servers: - self._create_server(server, **pool_kw) - - def sql(self, stmt, parameters=None, bulk_parameters=None): - """ - Execute SQL stmt against the crate server. - """ - if stmt is None: - return None - - data = _create_sql_payload(stmt, parameters, bulk_parameters) - logger.debug( - 'Sending request to %s with payload: %s', self.path, data) - content = self._json_request('POST', self.path, data=data) - logger.debug("JSON response for stmt(%s): %s", stmt, content) - - return content - - def server_infos(self, server): - response = self._request('GET', '/', server=server) - _raise_for_status(response) - content = _json_from_response(response) - node_name = content.get("name") - node_version = content.get('version', {}).get('number', '0.0.0') - return server, node_name, node_version - - def blob_put(self, table, digest, data): - """ - Stores the contents of the file like @data object in a blob under the - given table and digest. - """ - response = self._request('PUT', _blob_path(table, digest), - data=data) - if response.status == 201: - # blob created - return True - if response.status == 409: - # blob exists - return False - if response.status in (400, 404): - raise BlobLocationNotFoundException(table, digest) - _raise_for_status(response) - - def blob_del(self, table, digest): - """ - Deletes the blob with given digest under the given table. - """ - response = self._request('DELETE', _blob_path(table, digest)) - if response.status == 204: - return True - if response.status == 404: - return False - _raise_for_status(response) - - def blob_get(self, table, digest, chunk_size=1024 * 128): - """ - Returns a file like object representing the contents of the blob - with the given digest. - """ - response = self._request('GET', _blob_path(table, digest), stream=True) - if response.status == 404: - raise DigestNotFoundException(table, digest) - _raise_for_status(response) - return response.stream(amt=chunk_size) - - def blob_exists(self, table, digest): - """ - Returns true if the blob with the given digest exists - under the given table. - """ - response = self._request('HEAD', _blob_path(table, digest)) - if response.status == 200: - return True - elif response.status == 404: - return False - _raise_for_status(response) - - def _add_server(self, server): - with self._lock: - if server not in self.server_pool: - self._create_server(server, **self._pool_kw) - - def _request(self, method, path, server=None, **kwargs): - """Execute a request to the cluster - - A server is selected from the server pool. - """ - while True: - next_server = server or self._get_server() - try: - response = self.server_pool[next_server].request( - method, - path, - username=self.username, - password=self.password, - backoff_factor=self.backoff_factor, - schema=self.schema, - **kwargs - ) - redirect_location = response.get_redirect_location() - if redirect_location and 300 <= response.status <= 308: - redirect_server = _server_url(redirect_location) - self._add_server(redirect_server) - return self._request( - method, path, server=redirect_server, **kwargs) - if not server and response.status in SRV_UNAVAILABLE_STATUSES: - with self._lock: - # drop server from active ones - self._drop_server(next_server, response.reason) - else: - return response - except (MaxRetryError, - ReadTimeoutError, - SSLError, - HTTPError, - ProxyError,) as ex: - ex_message = _ex_to_message(ex) - if server: - raise ConnectionError( - "Server not available, exception: %s" % ex_message - ) - preserve_server = False - if isinstance(ex, ProtocolError): - preserve_server = any( - t in [type(arg) for arg in ex.args] - for t in PRESERVE_ACTIVE_SERVER_EXCEPTIONS - ) - if (not preserve_server): - with self._lock: - # drop server from active ones - self._drop_server(next_server, ex_message) - except Exception as e: - raise ProgrammingError(_ex_to_message(e)) - - def _json_request(self, method, path, data): - """ - Issue request against the crate HTTP API. - """ - - response = self._request(method, path, data=data) - _raise_for_status(response) - if len(response.data) > 0: - return _json_from_response(response) - return response.data - - def _get_server(self): - """ - Get server to use for request. - Also process inactive server list, re-add them after given interval. - """ - with self._lock: - inactive_server_count = len(self._inactive_servers) - for i in range(inactive_server_count): - try: - ts, server, message = heapq.heappop(self._inactive_servers) - except IndexError: - pass - else: - if (ts + self.retry_interval) > time(): - # Not yet, put it back - heapq.heappush(self._inactive_servers, - (ts, server, message)) - else: - self._active_servers.append(server) - logger.warning("Restored server %s into active pool", - server) - - # if none is old enough, use oldest - if not self._active_servers: - ts, server, message = heapq.heappop(self._inactive_servers) - self._active_servers.append(server) - logger.info("Restored server %s into active pool", server) - - server = self._active_servers[0] - self._roundrobin() - - return server - - @property - def active_servers(self): - """get the active servers for this client""" - with self._lock: - return list(self._active_servers) - - def _drop_server(self, server, message): - """ - Drop server from active list and adds it to the inactive ones. - """ - try: - self._active_servers.remove(server) - except ValueError: - pass - else: - heapq.heappush(self._inactive_servers, (time(), server, message)) - logger.warning("Removed server %s from active pool", server) - - # if this is the last server raise exception, otherwise try next - if not self._active_servers: - raise ConnectionError( - ("No more Servers available, " - "exception from last server: %s") % message) - - def _roundrobin(self): - """ - Very simple round-robin implementation - """ - self._active_servers.append(self._active_servers.pop(0)) - - def __repr__(self): - return ''.format(str(self._active_servers)) diff --git a/src/crate/client/pki/cacert_invalid.pem b/src/crate/client/pki/cacert_invalid.pem deleted file mode 100644 index eef32c22..00000000 --- a/src/crate/client/pki/cacert_invalid.pem +++ /dev/null @@ -1,34 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAMWp9xnQ06EvXkZO -54y55VOZqzRi8cjsBTsg2s9KogvrXKg+o36M1W1LhHWy7RdDpHSmA9ysCWj5WQm1 -f6izZsQmUu3lApW7sRKbWmh6Uz+Ij0n0EySWDemetfYCrOXpsVLL7sEtbSGSX6lc -gKcpncShpDXxFDVspnt2o6UJLs5XAgMBAAECgYBei2yQ4YzIVTLfqFAixKwHL8Jv -HUCPHS1nWdCvRAB8eQhpxQRIvK49R9jEv1+eOBGKZfoE7BjYKCRJe5HjLGRjZ3Aq -YL0vZXeXNsOgulYrbJI0bzJG/aJpcAnZVj+pvTAFUHl1ipr2V2O1vp5VzOTjXoHU -9hYLSgohP84ItJ11uQJBAPutFu9kuhtJybq5CRjGVtE2C/FLQi/r/yCrXf8qfPro -tL9Ro36CvqsAHlNPytCV1MgPlrbQdnPpiTKQrei3paUCQQDJD1JXi+gRNaJQ0znY -8IUHLCdziCy/tY1XAKCe3RWVyytglcg8LZ1Zgok+pivXD7Ft0eirw8Np4jNDy0pn -r3tLAkEAsQew8VWl/1no91n+xxmEqgbdYa5xcSoMvNst9DXb0dZshYMUgQHQwpID -wtCGtjAei1dDyXZbS++C3TmdyuubLQJAN1qThezLwGkuvefZZkOZrEbYubME9ubJ -+ej/T514gtwDnjy+euroBiutE2V2bmgwphaDYz6rPyx6hrCiVHd4jwJAfRnt/qQu -zlafmZicK65nGj08FPdK0ngccLaZZZ7T1G2sFa7tapXxdp9bMu2gOgASF90OxbPq -U2Ro6iFRFxGoAg== ------END PRIVATE KEY----- ------BEGIN CERTIFICATE----- -MIIC4DCCAkmgAwIBAgIJAJquZfuKfTejMA0GCSqGSIb3DQEBBQUAMIGIMQswCQYD -VQQGEwJERTEPMA0GA1UECAwGQmVybGluMQ8wDQYDVQQHDAZCZXJsaW4xIjAgBgNV -BAoMGUNyw4PCpHRlIFRlY2hub2xvZ2llIEdtYkgxEzARBgNVBAMMCmxvY2FsaG9y -c3QxHjAcBgkqhkiG9w0BCQEWD25vYm9keUBjcmF0ZS5pbzAeFw0xNDAzMTcxODE0 -MjZaFw0xNTAzMTcxODE0MjZaMIGIMQswCQYDVQQGEwJERTEPMA0GA1UECAwGQmVy -bGluMQ8wDQYDVQQHDAZCZXJsaW4xIjAgBgNVBAoMGUNyw4PCpHRlIFRlY2hub2xv -Z2llIEdtYkgxEzARBgNVBAMMCmxvY2FsaG9yc3QxHjAcBgkqhkiG9w0BCQEWD25v -Ym9keUBjcmF0ZS5pbzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxan3GdDT -oS9eRk7njLnlU5mrNGLxyOwFOyDaz0qiC+tcqD6jfozVbUuEdbLtF0OkdKYD3KwJ -aPlZCbV/qLNmxCZS7eUClbuxEptaaHpTP4iPSfQTJJYN6Z619gKs5emxUsvuwS1t -IZJfqVyApymdxKGkNfEUNWyme3ajpQkuzlcCAwEAAaNQME4wHQYDVR0OBBYEFCFp -JcTjkew8eIFxDqQbwFmt8rGRMB8GA1UdIwQYMBaAFCFpJcTjkew8eIFxDqQbwFmt -8rGRMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAdC0wK/DXhVjUI+OC -oOJX2tO/xvfN/iWObYHuDkw+ofSarP0l1HaBYJR05RMg8+B2/y4EJTNAHmeP127B -QAKXxZHkYlC/TdlZWQiwJ8DA6Fr+E2/3NbYiFtBMpXcgBtLs6TRDOhh/nLUrcguz -xggFgG3NE/mH2VNraP7Pk7DNyOE= ------END CERTIFICATE----- diff --git a/src/crate/client/pki/cacert_valid.pem b/src/crate/client/pki/cacert_valid.pem deleted file mode 100644 index e169a7fd..00000000 --- a/src/crate/client/pki/cacert_valid.pem +++ /dev/null @@ -1,49 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDN4PgIebnsjVfY -cwop20PNA2w8vRZvU3M9Ust7YP6ZrOaGFrg0l4uipDSyJa8G4jlbmIpr0ypwf7bZ -2uE8ICLQ0QBRmywA6KsOoiC3JP1/c6AD0veiAl8587S4i8+HtKd5CVjxsZkHmvqo -fuLK4/xporLES1+RU5E6d64gafbSI0dUJFIVfUySTaDvzJIPl6kfZFJqA1NQNJ24 -B0aa6y0dbXfUtMialRNCJyIuVny0kWKyPIVho+pVgJxEp8xyrkDld0urOG8EkQkU -Hk9MHcPaeKwQ0T0l+qzKRjyTG3ymx+af68xU6s4dM2g9YM0+2tkZCeDoWkTA9OQE -+GmOJAc/AgMBAAECggEAFHQoUDc/uHemZZOwS85D4ydW5oXmp7LDvTDvlFdjlALw -eBvjux3fOo5Tyesig22QQ0BZaDL3gWu+z9AGFoIe014gSPtAmOqErBSBaZCcOsBT -Am5AOfFAYrKKntcNDC9vf//kvUZmrLHB+2F3yK5z0k7esc/HM9n4kLV5MDE221OG -EsEx4peGpizFn5K7O9Ek4caVTYcDVMjBp2dug6N626cMBfcIdSiZKkdGOROHEWZk -DdGf3oWoGGVQ8wzMYyw4ZV2B6TNDFB0afaEkF5Z7Yj86Z1KC+uK8nJMVtqJmZbtt -DfjvrflB4uJf91ddDpq0o15AxyczGtCMfWvRnk0HoQKBgQD+sWfouFMX7mtaY47G -rV7M9RFHKsrdYhHqiMdtef1cIIO1srYhX3tjUis90WJdDWyEqC6xbMZhpgafCeXE -MwPTcOswrP6irITaTRzwPE7as7nrj2axbTRJUYxktWFruYE8uPWidGz/g0vfDhu8 -kIP5Bfee2Qe9kmH3MK3zFgP/jwKBgQDO729cwE8Tld6+sLbw6S+r4slZVAv1Shja -au5OQSF19h1lTWZRgooBmx5GkNz/ur7l0OXur+F7FE+rWtn1sRMR5lZs53QssqEY -iKlPTk0phpQbCI5GJwiy17DoHZMt5iqZVWgK28aXHikBoIg+fOL0wL5Y92DoiG3q -MjWpdcelUQKBgQCS9F3WI3SeXEzI7KTW8fW/ILAFdiVzM0DPKHiZLEgJviEA18rK -2sLg/epBUu0Eb9hrenbmnLKiaR9s6FMQr7bHa2HoxghuaEiHhPLrkoCVJBpkVmuU -eEQxAcKV4SoC9BgjpzzjrXWuwF0oqIVMeb4ME2ta1jLnKO0pqYbUuaE49QKBgBHg -wcBDpRFOG2ZiIgwCOWoiN78N8dKJSkhkgJ4mJlvonXWJEFPucTneSulRzqYRXjjA -qXzLmTFm+dMWEEqXt8wOGF1kSbcq35wdAnOlkikKRXVocdJBwRCibdg/5d1LS1bf -+BMoFaosouJPGjY71+fJVyichrTQRJ69I8G2OT9hAoGBAKSTyncv8s8Z1M44PCZA -IozDM+tj1I6ipZv2QihweI1goATDcaRhqLGlS++WI5SOCtXcW57VzLBXi05eFJoK -ylYmuLOFzELE8J9l5uERzEag6ioov69hne4drHxpY7Tr2zmEE6TKKczIKtx+oipD -wNTpdf2QRgs5LjpQ7UqM4n9B ------END PRIVATE KEY----- ------BEGIN CERTIFICATE----- -MIIDhTCCAm2gAwIBAgIJALDywhQYMNucMA0GCSqGSIb3DQEBCwUAMFkxCzAJBgNV -BAYTAkFUMRMwEQYDVQQIDApWb3JhcmxiZXJnMREwDwYDVQQHDAhEb3JuYmlybjEO -MAwGA1UECgwFQ3JhdGUxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0xNjA1MTMxNTM1 -NDdaFw0zNjA1MDgxNTM1NDdaMFkxCzAJBgNVBAYTAkFUMRMwEQYDVQQIDApWb3Jh -cmxiZXJnMREwDwYDVQQHDAhEb3JuYmlybjEOMAwGA1UECgwFQ3JhdGUxEjAQBgNV -BAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM3g -+Ah5ueyNV9hzCinbQ80DbDy9Fm9Tcz1Sy3tg/pms5oYWuDSXi6KkNLIlrwbiOVuY -imvTKnB/ttna4TwgItDRAFGbLADoqw6iILck/X9zoAPS96ICXznztLiLz4e0p3kJ -WPGxmQea+qh+4srj/GmissRLX5FTkTp3riBp9tIjR1QkUhV9TJJNoO/Mkg+XqR9k -UmoDU1A0nbgHRprrLR1td9S0yJqVE0InIi5WfLSRYrI8hWGj6lWAnESnzHKuQOV3 -S6s4bwSRCRQeT0wdw9p4rBDRPSX6rMpGPJMbfKbH5p/rzFTqzh0zaD1gzT7a2RkJ -4OhaRMD05AT4aY4kBz8CAwEAAaNQME4wHQYDVR0OBBYEFBQy5IK2vjhzQHSPkONV -qG1AhiV5MB8GA1UdIwQYMBaAFBQy5IK2vjhzQHSPkONVqG1AhiV5MAwGA1UdEwQF -MAMBAf8wDQYJKoZIhvcNAQELBQADggEBAMk8XevGvMzpr1CitS7+lpHuL97pjNAe -/a5xWNRDkjlppyD1MybaiwwTzlIIfdL35mK2LkzVaT+0q0TzK5aSspMx3/KeM9P9 -A9a4cGcY7qYIabEz1m3etqHse1SvBA/GhxfPL7/xHILhFP2fL1Ds2bSxREbQTP1M -O3nWPlgW3TWOPGnHYpUpbqBT2LdGBaA6H/abycvAcV9ihCy2+fMupvhqiA0ARqQt -yWyX4OEXcCEaIHHobhpXzu9qNLoi9IP1SaqUHZ1w8ave/URP+gwMAc6J+QTc06xI -9hg0DKKizjNgnjmzPgHh7M8B7OHStO4BeWyMy7Kp9mcqU9lEVcILUPU= ------END CERTIFICATE----- diff --git a/src/crate/client/pki/client_invalid.pem b/src/crate/client/pki/client_invalid.pem deleted file mode 100644 index 78547390..00000000 --- a/src/crate/client/pki/client_invalid.pem +++ /dev/null @@ -1,49 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDN4lSw+BESq83X -tfWcFfvXSKrErh+DpN06XpL0tCP/wGobnHtjzEMXSrFC+zO2bqU8HwmdjuAGSVEy -3Gtn1krBr1LUhMDNXKywRlh69JylAFgXhllCCe7e4U06cV1Mn3TgMKeFr8TEZJIp -EKP74HxCKYgnp068h0P/JpDZDjUpjoOHRYR3Fw6PlCR6Y8N4jkOxz6p7M9gBumx8 -cIZDI4wEIZ1nWGj4Sw+x54EdrmBweFYb8jTSOcjsPbEhcPcyP2uxo2lcaXtjz3/F -an278pGXsdfDCS5L5XCdxTcppDqMFd7WH1309H767mjxFlI1wqA3GF7ilEjlSVIr -MOREzQEJAgMBAAECggEAW1kupo6KVLRx0mHjpeuDIBQX4Mg5J1nA5qMLpRNAXbtr -2PBnNvJsWitD1ypJ2YniOniy9XttHpztMBnoddv4s1Ms9yonuXaDEHtFytg5oGQ0 -sctkUw7BM5bXgzTFZrfhTY+I5KIGNfVBfILrn1gNCfYPeTickL2bh9v+rK/HGrZa -xnCTZRVsaYyZohpNcSrGj8QIKg/4LjLY/Y1dxPkZ7nck6CwZJhLakuefeUWwkfkK -R9gO6eIdpopDS7t0e5RQ35NtAMlWIMUv5qwjKz80ueyhsTwfJZE0W5EI5eAaRJ9y -P4iJZIZDUY2A4DsadoxX/oryygDUQ2iZ2+kXXbVquQKBgQDzZyF8DubkQrDVmPSt -H4LyDY3/31yZEyB2D3DeWs7X8IAEmjDFeVRqq09BHvt9jEBGyBxlUtLZLoci5cYk -Fi8OgLwDBgIxQmFhQUCXH0OVuW4iZbyLzwLhpIUaUjgQi+k9hHrXioo9odThuy7F -81k1wZI7hDC6G+qXp/FS/+aLrwKBgQDYihw76a+4fhNC6zBxVQKWe8DZKbkv6T1V -xMC7Re0k10egYPatY9jvPqyDY8Zlqu008pWSEIRB8jhqJ5r5yO4u6yyg6rYKkMC2 -J+8BC/quGieBDYUeAmbmRDHUIX1iHELleFr4FRczWfkOOsgzKr1h+17KY1cNyief -X+IqFI9UxwKBgQDyAaNoXsSxRaHe7jKwgzlGA7YhJ2tBA6Rt3hJh8rXgPE58xPYj -EeyeFnA5ll2EydMKzWJ2V/AuYjWYvA7SyH/HErZc5zd81LxP33oiB8LB9lmLt83M -0GnUAikZL5Bw2ztvn+4nqqaieupX+i2aQcd8TFdh96AfGyyX1zJ5TNhkHwKBgEaE -A1nHaf/snp0mNepSQrt6pXySx8nAbMbngdP6m5VpvduOeAZTA6w1frxy24L0PLcH -YInmcwt+s7xuFVvOgTIqR6hHhuy94uPu8TgoDIRx4/d0zarOIXBPOOLZ3Rj8FxTf -MtCjHaENZbuqjNOM0Yt87ot9+jV1ZZ3S/bWyaFK1AoGBAJREK4guQ0qmTjzWuT6S -U/3SodR0kiRkfEM7u73dGMrRKk+9m1C0z7wY4hN8BIjvBzzwABXzwScuSFhIH0rY -B4GNWdzAOZwKRyV9YjtM+pK6bY+Kx5hRnGmTWsaAEt14SjGYOdU6lwdhuP5EWdoc -MU0GzRm9x98HSQD1dyrtT4MX ------END PRIVATE KEY----- ------BEGIN CERTIFICATE----- -MIIDdTCCAl2gAwIBAgIUNtmyUs7LRi9mvG1Y7NtDAbHlyJMwDQYJKoZIhvcNAQEL -BQAwWTELMAkGA1UEBhMCQVQxEzARBgNVBAgMClZvcmFybGJlcmcxETAPBgNVBAcM -CERvcm5iaXJuMQ4wDAYDVQQKDAVDcmF0ZTESMBAGA1UEAwwJbG9jYWxob3N0MCAX -DTIzMDEyNjE1MzQxOFoYDzE5MjMwMjIwMTUzNDE4WjB4MQswCQYDVQQGEwJBVDET -MBEGA1UECAwKVm9yYXJsYmVyZzERMA8GA1UEBwwIRG9ybmJpcm4xETAPBgNVBAoM -CENyYXRlLmlvMQ4wDAYDVQQDDAVob3JzdDEeMBwGCSqGSIb3DQEJARYPbm9ib2R5 -QGNyYXRlLmlvMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzeJUsPgR -EqvN17X1nBX710iqxK4fg6TdOl6S9LQj/8BqG5x7Y8xDF0qxQvsztm6lPB8JnY7g -BklRMtxrZ9ZKwa9S1ITAzVyssEZYevScpQBYF4ZZQgnu3uFNOnFdTJ904DCnha/E -xGSSKRCj++B8QimIJ6dOvIdD/yaQ2Q41KY6Dh0WEdxcOj5QkemPDeI5Dsc+qezPY -AbpsfHCGQyOMBCGdZ1ho+EsPseeBHa5gcHhWG/I00jnI7D2xIXD3Mj9rsaNpXGl7 -Y89/xWp9u/KRl7HXwwkuS+VwncU3KaQ6jBXe1h9d9PR++u5o8RZSNcKgNxhe4pRI -5UlSKzDkRM0BCQIDAQABoxQwEjAQBgNVHREECTAHggVob3JzdDANBgkqhkiG9w0B -AQsFAAOCAQEAJvqhvFiMpXTzH5dE/t2UFqMy7UPd4mypI2YqBelvN7pw/wQI1OIZ -N9bk52N6M9CuaENpgxkUAFVuPFSOa9Bp2/qA+TysyWC4+iSukL9+pQg8fmd3Ul7e -DYVHsOLjB2DwiK+og/P/kUvBEJ2z13EmtjNr4id1cWAD9r2Eh+dAbKS1MtvXCFMc -USJzAJ/QKw5h1x+ddub38zxgXgIQiDlWt4uwN9M6d/T+dAMm5FIU3MHQsdADBMtG -iXT5F/wz7f7UBbGK3kt3EuhBXHN8i84eTxZKb4a9g8L6AIzuSx4dg4q6vg9lee7P -umKNSySbfHF8ex/R2CLoDPW8NbXzOoBdCg== ------END CERTIFICATE----- diff --git a/src/crate/client/pki/client_valid.pem b/src/crate/client/pki/client_valid.pem deleted file mode 100644 index 8473b9dd..00000000 --- a/src/crate/client/pki/client_valid.pem +++ /dev/null @@ -1,49 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCguOpnDbcZvnH2 -trOBWGCkwTfEjLSHYA3S+f799Kiyc33CrvqRjxH9m+0N2Lt5JIrV2ToiRS49lxc0 -nEhbGAIXSjLVq1UMhTeXTWP1Rkl2v7Q01dQc1Oe/895mjGMrKoA+Mj+x4mkJfiAx -HDIhZ+YDs/DCR6ONKb/3xpr85YuBDEpXNINYAov/B6DSH4/N57F/5o6/PQhaa51b -8eMhF90aPPWNGxLBOzOJPGbp+ci9gee7tQimihtk5UDhFJn1CXRd7+3VoL4PM4JI -rq7veZDEA4JFEqcBPcYuwd+R+MdIK3kPy+Y+tXIqAL9weC66xImUvafRBH9Bae1o -hzOzNXItAgMBAAECggEAepjzPI7Kt5l9BwuQW77VYXd9mbEW2BjeEqvd5UvmDtFo -AcRFoBi5SXHSXniPcLX+LWeZH6ETx6lj6x63Vr02gxt3MWOS6Y0IvaIr+GGYEjvj -M1ZUiXWiHdrhL+owjzHJRyg/S+p/4tzRo4R1fOPrIbH1mczZpglNxKw7d2OFiXZn -5Y4RcFn2rx3O6Qzldo6Am+pPm0FLugvzQ8pFlWyO1OI9NcAJ9fMjMNjXwmis6bUj -gZTXESsBLHaljefWC5CK2iDUo0VwZKJG15b2rskPShStlSIwhmnP46osizfZ2gCK -9C0GJ0PVYbHp15SnbWzoYx3Ny3sXGA6PxpRl7Xj7xQKBgQDSn9rXhxBqSdXFqAUn -6bGuhumN2KmezQOSGiMFVjZLZT6DR/fyh+5pah8v7okZHUxI+pTzQu0aBzWMpZqp -dPuVPyJyfyK/J8c5MiFbf7jqKhlEBTFDkhDvtoTTXc1+iSpusz76WMwnm5yJNq9/ -M/uj0z23ufUgl/+rHScZX/jNDwKBgQDDWOkRNqKRVp550QP1TFr37LVvm2QVuJux -JpEwJhLjMFipQjJPlli7FiruPPsKnMXmqKAEJuzFreGCkRnsI+fCcJHuiZ/KV6zf -jTxL0rCoY0EkTYNztJpWPpY46uwpz7YsPjQ5tYInDibdlHz/liqR3WMnUqhtEkCZ -4xeRR5pFAwKBgHCEGOI+MtS08NAQPADgZJz5UVcHQUWl+5xW/hJhxcttIIH9NkWH -vCLwIAz4/qA9+Hyb8GorfIIFC+RAq2iPJ77I5VwI8sTvOQwi7ZL1nhDpwGmH5JNW -Oln0ROytFZPdLp/IfYI7YYRfKrZaUlI/sNQJitTVME/jIx+ECVkS3dSdAoGAUGqy -j36BT/lrhdRQn9OOA0/zpP1AJ1z24udwj3StA8+sQAlbMr4+ys2mYKrD4auGIJ08 -OllX0Uzyb9CR7k8dokK7IIqRODf9l43Jy6DxTnCFqY8rVR99BZIAP3AeRlwWr4Hv -9+3LpY5C26a8Cm9kGOYdYlu5sCT6aR8+XXUvgccCgYEArsqN2ialQsUseutRn/kf -apBHxGm9ihPDZ9csnnX7OPlQL3Y4X4B5p3k98pwZ9dJlm/HAm0cM2KoXo7DdR2O+ -XDBkTvPLPJaNA9y/AS2Lam9IVqVfx5vkOSwLdWXh1wgNDnSn4Q3x63MbCkFbNEuH -jZ3cOn6SqwD0tiV+Dg45NRM= ------END PRIVATE KEY----- ------BEGIN CERTIFICATE----- -MIIDejCCAmKgAwIBAgIUNtmyUs7LRi9mvG1Y7NtDAbHlyI4wDQYJKoZIhvcNAQEL -BQAwWTELMAkGA1UEBhMCQVQxEzARBgNVBAgMClZvcmFybGJlcmcxETAPBgNVBAcM -CERvcm5iaXJuMQ4wDAYDVQQKDAVDcmF0ZTESMBAGA1UEAwwJbG9jYWxob3N0MCAX -DTIzMDEyNjE1MTMyMVoYDzMwMDMwMzMwMTUxMzIxWjB5MQswCQYDVQQGEwJBVDET -MBEGA1UECAwKVm9yYXJsYmVyZzERMA8GA1UEBwwIRG9ybmJpcm4xDjAMBgNVBAoM -BUNyYXRlMRIwEAYDVQQDDAlsb2NhbGhvc3QxHjAcBgkqhkiG9w0BCQEWD25vYm9k -eUBjcmF0ZS5pbzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKC46mcN -txm+cfa2s4FYYKTBN8SMtIdgDdL5/v30qLJzfcKu+pGPEf2b7Q3Yu3kkitXZOiJF -Lj2XFzScSFsYAhdKMtWrVQyFN5dNY/VGSXa/tDTV1BzU57/z3maMYysqgD4yP7Hi -aQl+IDEcMiFn5gOz8MJHo40pv/fGmvzli4EMSlc0g1gCi/8HoNIfj83nsX/mjr89 -CFprnVvx4yEX3Ro89Y0bEsE7M4k8Zun5yL2B57u1CKaKG2TlQOEUmfUJdF3v7dWg -vg8zgkiuru95kMQDgkUSpwE9xi7B35H4x0greQ/L5j61cioAv3B4LrrEiZS9p9EE -f0Fp7WiHM7M1ci0CAwEAAaMYMBYwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqG -SIb3DQEBCwUAA4IBAQBCTZ3nMg+Y2ymj9DgNPW5KAMGwdphv8ugO5fCRoGUtYc1B -Yz6ZGYUIbIDImgSr/czE2O7BVOwOkWmeXOCTRL8n30Wm2yVT90NZ9jG6dOX2eF2M -7Lyh7+Vy4XuDcura+/5y3PjTsApNUeCZWQgwrLSV8xNvrSH8Cbv3yS4b3rzMVb4l -RipVO9V75SNcduvLDR3VNK3c+mlhX03FYuJ6XZjgX/hvf8fZdCrUqfmM2NSwvQdj -QH3m1Fh5rh3xi+ReiBVP4R4uF2mSDqaqd+iTpLzV6VSwfT58m+AgRss5xgzCWxlf -Xwwb1pa5q7eZXxZOjnfWaIgJmkdGYLco/ZVvWV11 ------END CERTIFICATE----- diff --git a/src/crate/client/pki/readme.rst b/src/crate/client/pki/readme.rst deleted file mode 100644 index 74c75e1a..00000000 --- a/src/crate/client/pki/readme.rst +++ /dev/null @@ -1,91 +0,0 @@ -######################### -Generate new certificates -######################### - - -***** -About -***** - -For conducting TLS connectivity tests, there are a few X.509 certificates at -`src/crate/client/pki/*.pem`_. The instructions here outline how to renew them. - -In order to invoke the corresponding test cases, run:: - - ./bin/test -t https.rst - - -******* -Details -******* - - -``*_valid.pem`` -=============== - -By example, this will renew the ``client_valid.pem`` X.509 certificate. The -``server_valid.pem`` certificate can be generated in the same manner. - -Create RSA private key and certificate request:: - - openssl req -nodes \ - -newkey rsa:2048 -keyout key.pem -out request.csr \ - -subj "/C=AT/ST=Vorarlberg/L=Dornbirn/O=Crate.io/CN=localhost/emailAddress=nobody@crate.io" - -Display the certificate request:: - - openssl req -in request.csr -text - -Sign certificate request:: - - openssl x509 -req -in request.csr \ - -CA cacert_valid.pem -CAkey cacert_valid.pem -CAcreateserial -sha256 \ - -days 358000 -extfile <(printf "subjectAltName=DNS:localhost") -out client.pem - -Display the certificate:: - - openssl x509 -in client.pem -text - -Combine private key and certificate into single PEM file:: - - cat key.pem > client_valid.pem - cat client.pem >> client_valid.pem - - -``client_invalid.pem`` -====================== - -This will renew the ``client_invalid.pem`` X.509 certificate. Please note that, -in order to create an invalid certificate, two attributes are used: - -- ``CN=horst`` and ``subjectAltName=DNS:horst`` do not match ``localhost``. -- The validity end date will be adjusted a few years into the past, by using - ``-days -36500``. - -Create RSA private key and certificate request:: - - openssl req -nodes \ - -newkey rsa:2048 -keyout invalid_key.pem -out invalid.csr \ - -subj "/C=AT/ST=Vorarlberg/L=Dornbirn/O=Crate.io/CN=horst/emailAddress=nobody@crate.io" - -Display the certificate request:: - - openssl req -in invalid.csr -text - -Sign certificate request:: - - openssl x509 -req -in invalid.csr \ - -CA cacert_valid.pem -CAkey cacert_valid.pem -CAcreateserial -sha256 \ - -days -36500 -extfile <(printf "subjectAltName=DNS:horst") -out invalid_cert.pem - -Display the certificate:: - - openssl x509 -in invalid_cert.pem -text - -Combine private key and certificate into single PEM file:: - - cat invalid_key.pem > client_invalid.pem - cat invalid_cert.pem >> client_invalid.pem - - -.. _src/crate/client/pki/*.pem: https://github.com/crate/crate-python/tree/master/src/crate/client/pki diff --git a/src/crate/client/pki/server_valid.pem b/src/crate/client/pki/server_valid.pem deleted file mode 100644 index 7b1cf8f4..00000000 --- a/src/crate/client/pki/server_valid.pem +++ /dev/null @@ -1,49 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDOORP/4SVgETya -ve7+HQqJa/Aye6qw9K1YL1t86FL82TIUwyAk86YQ8yrdCVzrGb52ZVF/4YDgNBpV -SHfCgLhGUSjcdmLxpKtKoTiwnb7z3KqdRUPFF7LegN0MHX6fCAalGr89pAE1AcUb -MQ57FV2uwVikq3DFRAufoA7xNXrOVc5N0Q+r4GEHp3HPCxd1bqneSCmDZe3nZwp9 -y8nllOPChB9CcUXz2yY0ZkJUd+2CwYXjfKR+09UnaoYlmkSm6+8BlzHOvneBp37z -68m9Dsuref10L7nOpH7nKUrYoJZy6+hq+pzNvtFJbrdgqfUmxseFtfYZH6+5kmcM -CpATIvFVAgMBAAECggEAUO3Xamh52WyKQxPckX7mHq3sUnNztgQfbucO2UL3JmE6 -JSm6GKZBeo9jN+EvxNeShjYWuL/Paq0n1GYfEYagSoAZMAOJqtj7m8sPS0hsopjr -n9KJ2PQG7wjVNqbwhQqKSQrpGBCfpKSwLI6g5y6oWCdqWR0qoU+l3BvdIU3ihswz -UgDCw1boSoki9paZsdHNiYfTNON6Wpx6yibmYBt9lJgHsmqds/a1XEU1j09un4MN -MkCc9MsIPSj9qKYdxGllMeg/Kmfg4qXZvKeK0AS8NCeWqVhoJLPmmt+WPC6LhhOb -4glnOjHJ2KnTEqPbiCXLaO+Izxbmo/tTN/eO9LOzQQKBgQDozxshxDBIKFR7i2NK -A3RVUBa4mhP+zyy77/W5VsIkBKbqFA+tRaQXp1XOjTeHJ14jskDzUo0loJ8ZL82S -yQr2LqxUtvEwDctp+/kkN6WkXtEEDjP0ACVtsBgR5kZygi6pWT44Ae8nUjmhZ/M5 -/JD2KoXH8aAHtJ8kRE7aVFlFMQKBgQDiw/9uFAZraguCVf6du5nD/nzNqcI4Nr9f -0UabNA5261i4fw2Yft77piv+9MeugI8GDC/x0YIW1ZshxfRJXgBo5l9hnEaP+uyr -wzXPSAphv7UUbGScR1B1+BF103U6OHl2PIOiiwrHWsIF+KUSTL8Virzb4WhJo6SV -pu2PGi+1ZQKBgQDWQj0LL37r+cn+xcLETDeViJbQoGUEnnDiKi6wuysDcRCY34uq -ASzK5CMxbIANL+sQ2S2zgmcKmS+zQ25jyAkBluTdNlz0x81Mpiyd62TTyLt+iv+/ -cR8BOw578r0lB7CgBNUhQI50VtVZOcz8sfhLxcjHwhVw4geQnhkgEH70EQKBgFHf -jiPCWycBHLKsNcfhaf0XrxvaRONi8Om5d5Kl0usgweGrDc+XTw7wykW9PzND+1+l -mtHmYN+5s88X18F9jQxS0PE/KULmx/ij/JOgYQ811j1PfWvnW6ecL0GpXVPt+/yy -kJxpMzUTEaZyRbc7umoes114Ht0nlk7p/C+EtuD5AoGAYKMYspyKgYeiPS9nfdP2 -nzfNjCx4WiPMw3zvQ7QF3T+icu4+oTPU8aYBEHZc3lnhFdGzIm27ww/pmAbEqiJ9 -72QM9jFlGObGVm+Dypbrj1eFMxm0tcAyKg4f2nYC1CH2n8JMiLERFpmXZQacLHZx -O7sBCdj90ycqRiUcRDXvzlk= ------END PRIVATE KEY----- ------BEGIN CERTIFICATE----- -MIIDfTCCAmWgAwIBAgIUNtmyUs7LRi9mvG1Y7NtDAbHlyJQwDQYJKoZIhvcNAQEL -BQAwWTELMAkGA1UEBhMCQVQxEzARBgNVBAgMClZvcmFybGJlcmcxETAPBgNVBAcM -CERvcm5iaXJuMQ4wDAYDVQQKDAVDcmF0ZTESMBAGA1UEAwwJbG9jYWxob3N0MCAX -DTIzMDEyNjE2MjAyN1oYDzMwMDMwMzMwMTYyMDI3WjB8MQswCQYDVQQGEwJBVDET -MBEGA1UECAwKVm9yYXJsYmVyZzERMA8GA1UEBwwIRG9ybmJpcm4xETAPBgNVBAoM -CENyYXRlLmlvMRIwEAYDVQQDDAlsb2NhbGhvc3QxHjAcBgkqhkiG9w0BCQEWD25v -Ym9keUBjcmF0ZS5pbzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM45 -E//hJWARPJq97v4dColr8DJ7qrD0rVgvW3zoUvzZMhTDICTzphDzKt0JXOsZvnZl -UX/hgOA0GlVId8KAuEZRKNx2YvGkq0qhOLCdvvPcqp1FQ8UXst6A3Qwdfp8IBqUa -vz2kATUBxRsxDnsVXa7BWKSrcMVEC5+gDvE1es5Vzk3RD6vgYQencc8LF3Vuqd5I -KYNl7ednCn3LyeWU48KEH0JxRfPbJjRmQlR37YLBheN8pH7T1SdqhiWaRKbr7wGX -Mc6+d4GnfvPryb0Oy6t5/XQvuc6kfucpStiglnLr6Gr6nM2+0Ulut2Cp9SbGx4W1 -9hkfr7mSZwwKkBMi8VUCAwEAAaMYMBYwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA0G -CSqGSIb3DQEBCwUAA4IBAQBOM7AfDYDzPGOsNya5s6+58PHhPEfRFixCSLGSK5Pm -sfvg7rRre5P+I37863B1S52E5QWzOlVJM+POKiNKp64846eWvk4TYenW0KOxjL75 -R0Y5LQVNM80x1rw9j5iBdMSYgkMPwSccO6WGOdTV+6X077QgLpmqnEgmmfZj0CMz -+k33sbJ4H7HC7bl6+bSQBwxSQIVmuXTTmHIpv6Kz4lLIezjuYikkeiEMBPp+XF9Q -ZqaBfGvnvUE9KBUoxQZe0jzTTQE31FsnKtDyaMcyV3rMoBDmD6B6CaSo7yfj2fpI -EueW/Mx4EtMLTU4QY5DJsXsszBpB3+8YhuWFpHqP5jpu ------END CERTIFICATE----- diff --git a/src/crate/client/test_connection.py b/src/crate/client/test_connection.py deleted file mode 100644 index 3b5c294c..00000000 --- a/src/crate/client/test_connection.py +++ /dev/null @@ -1,74 +0,0 @@ -import datetime - -from .connection import Connection -from .http import Client -from crate.client import connect -from unittest import TestCase - -from ..testing.settings import crate_host - - -class ConnectionTest(TestCase): - - def test_connection_mock(self): - """ - For testing purposes it is often useful to replace the client used for - communication with the CrateDB server with a stub or mock. - - This can be done by passing an object of the Client class when calling the - ``connect`` method. - """ - - class MyConnectionClient: - active_servers = ["localhost:4200"] - - def __init__(self): - pass - - def server_infos(self, server): - return ("localhost:4200", "my server", "0.42.0") - - connection = connect([crate_host], client=MyConnectionClient()) - self.assertIsInstance(connection, Connection) - self.assertEqual(connection.client.server_infos("foo"), ('localhost:4200', 'my server', '0.42.0')) - - def test_lowest_server_version(self): - infos = [(None, None, '0.42.3'), - (None, None, '0.41.8'), - (None, None, 'not a version')] - - client = Client(servers="localhost:4200 localhost:4201 localhost:4202") - client.server_infos = lambda server: infos.pop() - connection = connect(client=client) - self.assertEqual((0, 41, 8), connection.lowest_server_version.version) - connection.close() - - def test_invalid_server_version(self): - client = Client(servers="localhost:4200") - client.server_infos = lambda server: (None, None, "No version") - connection = connect(client=client) - self.assertEqual((0, 0, 0), connection.lowest_server_version.version) - connection.close() - - def test_context_manager(self): - with connect('localhost:4200') as conn: - pass - self.assertEqual(conn._closed, True) - - def test_with_timezone(self): - """ - Verify the cursor objects will return timezone-aware `datetime` objects when requested to. - When switching the time zone at runtime on the connection object, only new cursor objects - will inherit the new time zone. - """ - - tz_mst = datetime.timezone(datetime.timedelta(hours=7), name="MST") - connection = connect('localhost:4200', time_zone=tz_mst) - cursor = connection.cursor() - self.assertEqual(cursor.time_zone.tzname(None), "MST") - self.assertEqual(cursor.time_zone.utcoffset(None), datetime.timedelta(seconds=25200)) - - connection.time_zone = datetime.timezone.utc - cursor = connection.cursor() - self.assertEqual(cursor.time_zone.tzname(None), "UTC") - self.assertEqual(cursor.time_zone.utcoffset(None), datetime.timedelta(0)) diff --git a/src/crate/client/test_cursor.py b/src/crate/client/test_cursor.py deleted file mode 100644 index 79e7ddd6..00000000 --- a/src/crate/client/test_cursor.py +++ /dev/null @@ -1,341 +0,0 @@ -# -*- coding: utf-8; -*- -# -# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor -# license agreements. See the NOTICE file distributed with this work for -# additional information regarding copyright ownership. Crate licenses -# this file to you under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. You may -# obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# However, if you have executed another commercial license agreement -# with Crate these terms will supersede the license and you may use the -# software solely pursuant to the terms of the relevant commercial agreement. - -import datetime -from ipaddress import IPv4Address -from unittest import TestCase -from unittest.mock import MagicMock -try: - import zoneinfo -except ImportError: - from backports import zoneinfo - -import pytz - -from crate.client import connect -from crate.client.converter import DataType, DefaultTypeConverter -from crate.client.http import Client -from crate.client.test_util import ClientMocked - - -class CursorTest(TestCase): - - @staticmethod - def get_mocked_connection(): - client = MagicMock(spec=Client) - return connect(client=client) - - def test_create_with_timezone_as_datetime_object(self): - """ - Verify the cursor returns timezone-aware `datetime` objects when requested to. - Switching the time zone at runtime on the cursor object is possible. - Here: Use a `datetime.timezone` instance. - """ - - connection = self.get_mocked_connection() - - tz_mst = datetime.timezone(datetime.timedelta(hours=7), name="MST") - cursor = connection.cursor(time_zone=tz_mst) - - self.assertEqual(cursor.time_zone.tzname(None), "MST") - self.assertEqual(cursor.time_zone.utcoffset(None), datetime.timedelta(seconds=25200)) - - cursor.time_zone = datetime.timezone.utc - self.assertEqual(cursor.time_zone.tzname(None), "UTC") - self.assertEqual(cursor.time_zone.utcoffset(None), datetime.timedelta(0)) - - def test_create_with_timezone_as_pytz_object(self): - """ - Verify the cursor returns timezone-aware `datetime` objects when requested to. - Here: Use a `pytz.timezone` instance. - """ - connection = self.get_mocked_connection() - cursor = connection.cursor(time_zone=pytz.timezone('Australia/Sydney')) - self.assertEqual(cursor.time_zone.tzname(None), "Australia/Sydney") - - # Apparently, when using `pytz`, the timezone object does not return an offset. - # Nevertheless, it works, as demonstrated per doctest in `cursor.txt`. - self.assertEqual(cursor.time_zone.utcoffset(None), None) - - def test_create_with_timezone_as_zoneinfo_object(self): - """ - Verify the cursor returns timezone-aware `datetime` objects when requested to. - Here: Use a `zoneinfo.ZoneInfo` instance. - """ - connection = self.get_mocked_connection() - cursor = connection.cursor(time_zone=zoneinfo.ZoneInfo('Australia/Sydney')) - self.assertEqual(cursor.time_zone.key, 'Australia/Sydney') - - def test_create_with_timezone_as_utc_offset_success(self): - """ - Verify the cursor returns timezone-aware `datetime` objects when requested to. - Here: Use a UTC offset in string format. - """ - connection = self.get_mocked_connection() - cursor = connection.cursor(time_zone="+0530") - self.assertEqual(cursor.time_zone.tzname(None), "+0530") - self.assertEqual(cursor.time_zone.utcoffset(None), datetime.timedelta(seconds=19800)) - - connection = self.get_mocked_connection() - cursor = connection.cursor(time_zone="-1145") - self.assertEqual(cursor.time_zone.tzname(None), "-1145") - self.assertEqual(cursor.time_zone.utcoffset(None), datetime.timedelta(days=-1, seconds=44100)) - - def test_create_with_timezone_as_utc_offset_failure(self): - """ - Verify the cursor croaks when trying to create it with invalid UTC offset strings. - """ - connection = self.get_mocked_connection() - with self.assertRaises(AssertionError) as ex: - connection.cursor(time_zone="foobar") - self.assertEqual(str(ex.exception), "Time zone 'foobar' is given in invalid UTC offset format") - - connection = self.get_mocked_connection() - with self.assertRaises(ValueError) as ex: - connection.cursor(time_zone="+abcd") - self.assertEqual(str(ex.exception), "Time zone '+abcd' is given in invalid UTC offset format: " - "invalid literal for int() with base 10: '+ab'") - - def test_create_with_timezone_connection_cursor_precedence(self): - """ - Verify that the time zone specified on the cursor object instance - takes precedence over the one specified on the connection instance. - """ - client = MagicMock(spec=Client) - connection = connect(client=client, time_zone=pytz.timezone('Australia/Sydney')) - cursor = connection.cursor(time_zone="+0530") - self.assertEqual(cursor.time_zone.tzname(None), "+0530") - self.assertEqual(cursor.time_zone.utcoffset(None), datetime.timedelta(seconds=19800)) - - def test_execute_with_args(self): - client = MagicMock(spec=Client) - conn = connect(client=client) - c = conn.cursor() - statement = 'select * from locations where position = ?' - c.execute(statement, 1) - client.sql.assert_called_once_with(statement, 1, None) - conn.close() - - def test_execute_with_bulk_args(self): - client = MagicMock(spec=Client) - conn = connect(client=client) - c = conn.cursor() - statement = 'select * from locations where position = ?' - c.execute(statement, bulk_parameters=[[1]]) - client.sql.assert_called_once_with(statement, None, [[1]]) - conn.close() - - def test_execute_with_converter(self): - client = ClientMocked() - conn = connect(client=client) - - # Use the set of data type converters from `DefaultTypeConverter` - # and add another custom converter. - converter = DefaultTypeConverter( - {DataType.BIT: lambda value: value is not None and int(value[2:-1], 2) or None}) - - # Create a `Cursor` object with converter. - c = conn.cursor(converter=converter) - - # Make up a response using CrateDB data types `TEXT`, `IP`, - # `TIMESTAMP`, `BIT`. - conn.client.set_next_response({ - "col_types": [4, 5, 11, 25], - "cols": ["name", "address", "timestamp", "bitmask"], - "rows": [ - ["foo", "10.10.10.1", 1658167836758, "B'0110'"], - [None, None, None, None], - ], - "rowcount": 1, - "duration": 123 - }) - - c.execute("") - result = c.fetchall() - self.assertEqual(result, [ - ['foo', IPv4Address('10.10.10.1'), datetime.datetime(2022, 7, 18, 18, 10, 36, 758000), 6], - [None, None, None, None], - ]) - - conn.close() - - def test_execute_with_converter_and_invalid_data_type(self): - client = ClientMocked() - conn = connect(client=client) - converter = DefaultTypeConverter() - - # Create a `Cursor` object with converter. - c = conn.cursor(converter=converter) - - # Make up a response using CrateDB data types `TEXT`, `IP`, - # `TIMESTAMP`, `BIT`. - conn.client.set_next_response({ - "col_types": [999], - "cols": ["foo"], - "rows": [ - ["n/a"], - ], - "rowcount": 1, - "duration": 123 - }) - - c.execute("") - with self.assertRaises(ValueError) as ex: - c.fetchone() - self.assertEqual(ex.exception.args, ("999 is not a valid DataType",)) - - def test_execute_array_with_converter(self): - client = ClientMocked() - conn = connect(client=client) - converter = DefaultTypeConverter() - cursor = conn.cursor(converter=converter) - - conn.client.set_next_response({ - "col_types": [4, [100, 5]], - "cols": ["name", "address"], - "rows": [["foo", ["10.10.10.1", "10.10.10.2"]]], - "rowcount": 1, - "duration": 123 - }) - - cursor.execute("") - result = cursor.fetchone() - self.assertEqual(result, [ - 'foo', - [IPv4Address('10.10.10.1'), IPv4Address('10.10.10.2')], - ]) - - def test_execute_array_with_converter_and_invalid_collection_type(self): - client = ClientMocked() - conn = connect(client=client) - converter = DefaultTypeConverter() - cursor = conn.cursor(converter=converter) - - # Converting collections only works for `ARRAY`s. (ID=100). - # When using `DOUBLE` (ID=6), it should croak. - conn.client.set_next_response({ - "col_types": [4, [6, 5]], - "cols": ["name", "address"], - "rows": [["foo", ["10.10.10.1", "10.10.10.2"]]], - "rowcount": 1, - "duration": 123 - }) - - cursor.execute("") - - with self.assertRaises(ValueError) as ex: - cursor.fetchone() - self.assertEqual(ex.exception.args, ("Data type 6 is not implemented as collection type",)) - - def test_execute_nested_array_with_converter(self): - client = ClientMocked() - conn = connect(client=client) - converter = DefaultTypeConverter() - cursor = conn.cursor(converter=converter) - - conn.client.set_next_response({ - "col_types": [4, [100, [100, 5]]], - "cols": ["name", "address_buckets"], - "rows": [["foo", [["10.10.10.1", "10.10.10.2"], ["10.10.10.3"], [], None]]], - "rowcount": 1, - "duration": 123 - }) - - cursor.execute("") - result = cursor.fetchone() - self.assertEqual(result, [ - 'foo', - [[IPv4Address('10.10.10.1'), IPv4Address('10.10.10.2')], [IPv4Address('10.10.10.3')], [], None], - ]) - - def test_executemany_with_converter(self): - client = ClientMocked() - conn = connect(client=client) - converter = DefaultTypeConverter() - cursor = conn.cursor(converter=converter) - - conn.client.set_next_response({ - "col_types": [4, 5], - "cols": ["name", "address"], - "rows": [["foo", "10.10.10.1"]], - "rowcount": 1, - "duration": 123 - }) - - cursor.executemany("", []) - result = cursor.fetchall() - - # ``executemany()`` is not intended to be used with statements returning result - # sets. The result will always be empty. - self.assertEqual(result, []) - - def test_execute_with_timezone(self): - client = ClientMocked() - conn = connect(client=client) - - # Create a `Cursor` object with `time_zone`. - tz_mst = datetime.timezone(datetime.timedelta(hours=7), name="MST") - c = conn.cursor(time_zone=tz_mst) - - # Make up a response using CrateDB data type `TIMESTAMP`. - conn.client.set_next_response({ - "col_types": [4, 11], - "cols": ["name", "timestamp"], - "rows": [ - ["foo", 1658167836758], - [None, None], - ], - }) - - # Run execution and verify the returned `datetime` object is timezone-aware, - # using the designated timezone object. - c.execute("") - result = c.fetchall() - self.assertEqual(result, [ - [ - 'foo', - datetime.datetime(2022, 7, 19, 1, 10, 36, 758000, - tzinfo=datetime.timezone(datetime.timedelta(seconds=25200), 'MST')), - ], - [ - None, - None, - ], - ]) - self.assertEqual(result[0][1].tzname(), "MST") - - # Change timezone and verify the returned `datetime` object is using it. - c.time_zone = datetime.timezone.utc - c.execute("") - result = c.fetchall() - self.assertEqual(result, [ - [ - 'foo', - datetime.datetime(2022, 7, 18, 18, 10, 36, 758000, tzinfo=datetime.timezone.utc), - ], - [ - None, - None, - ], - ]) - self.assertEqual(result[0][1].tzname(), "UTC") - - conn.close() diff --git a/src/crate/client/test_http.py b/src/crate/client/test_http.py deleted file mode 100644 index 8e547963..00000000 --- a/src/crate/client/test_http.py +++ /dev/null @@ -1,678 +0,0 @@ -# -*- coding: utf-8; -*- -# -# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor -# license agreements. See the NOTICE file distributed with this work for -# additional information regarding copyright ownership. Crate licenses -# this file to you under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. You may -# obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# However, if you have executed another commercial license agreement -# with Crate these terms will supersede the license and you may use the -# software solely pursuant to the terms of the relevant commercial agreement. - -import json -import time -import socket -import multiprocessing -import sys -import os -import queue -import random -import traceback -from http.server import BaseHTTPRequestHandler, HTTPServer -from multiprocessing.context import ForkProcess -from unittest import TestCase -from unittest.mock import patch, MagicMock -from threading import Thread, Event -from decimal import Decimal -import datetime as dt - -import urllib3.exceptions -from base64 import b64decode -from urllib.parse import urlparse, parse_qs - -import uuid -import certifi - -from .http import Client, CrateJsonEncoder, _get_socket_opts, _remove_certs_for_non_https -from .exceptions import ConnectionError, ProgrammingError, IntegrityError - -REQUEST = 'crate.client.http.Server.request' -CA_CERT_PATH = certifi.where() - - -def fake_request(response=None): - def request(*args, **kwargs): - if isinstance(response, list): - resp = response.pop(0) - response.append(resp) - return resp - elif response: - return response - else: - return MagicMock(spec=urllib3.response.HTTPResponse) - return request - - -def fake_response(status, reason=None, content_type='application/json'): - m = MagicMock(spec=urllib3.response.HTTPResponse) - m.status = status - m.reason = reason or '' - m.headers = {'content-type': content_type} - return m - - -def fake_redirect(location): - m = fake_response(307) - m.get_redirect_location.return_value = location - return m - - -def bad_bulk_response(): - r = fake_response(400, 'Bad Request') - r.data = json.dumps({ - "results": [ - {"rowcount": 1}, - {"error_message": "an error occured"}, - {"error_message": "another error"}, - {"error_message": ""}, - {"error_message": None} - ]}).encode() - return r - - -def duplicate_key_exception(): - r = fake_response(409, 'Conflict') - r.data = json.dumps({ - "error": { - "code": 4091, - "message": "DuplicateKeyException[A document with the same primary key exists already]" - } - }).encode() - return r - - -def fail_sometimes(*args, **kwargs): - if random.randint(1, 100) % 10 == 0: - raise urllib3.exceptions.MaxRetryError(None, '/_sql', '') - return fake_response(200) - - -class HttpClientTest(TestCase): - - @patch(REQUEST, fake_request([fake_response(200), - fake_response(104, 'Connection reset by peer'), - fake_response(503, 'Service Unavailable')])) - def test_connection_reset_exception(self): - client = Client(servers="localhost:4200") - client.sql('select 1') - client.sql('select 2') - self.assertEqual(['http://localhost:4200'], list(client._active_servers)) - try: - client.sql('select 3') - except ProgrammingError: - self.assertEqual([], list(client._active_servers)) - else: - self.assertTrue(False) - finally: - client.close() - - def test_no_connection_exception(self): - client = Client() - self.assertRaises(ConnectionError, client.sql, 'select foo') - client.close() - - @patch(REQUEST) - def test_http_error_is_re_raised(self, request): - request.side_effect = Exception - - client = Client() - self.assertRaises(ProgrammingError, client.sql, 'select foo') - client.close() - - @patch(REQUEST) - def test_programming_error_contains_http_error_response_content(self, request): - request.side_effect = Exception("this shouldn't be raised") - - client = Client() - try: - client.sql('select 1') - except ProgrammingError as e: - self.assertEqual("this shouldn't be raised", e.message) - else: - self.assertTrue(False) - finally: - client.close() - - @patch(REQUEST, fake_request([fake_response(200), - fake_response(503, 'Service Unavailable')])) - def test_server_error_50x(self): - client = Client(servers="localhost:4200 localhost:4201") - client.sql('select 1') - client.sql('select 2') - try: - client.sql('select 3') - except ProgrammingError as e: - self.assertEqual("No more Servers available, " + - "exception from last server: Service Unavailable", - e.message) - self.assertEqual([], list(client._active_servers)) - else: - self.assertTrue(False) - finally: - client.close() - - def test_connect(self): - client = Client(servers="localhost:4200 localhost:4201") - self.assertEqual(client._active_servers, - ["http://localhost:4200", "http://localhost:4201"]) - client.close() - - client = Client(servers="localhost:4200") - self.assertEqual(client._active_servers, ["http://localhost:4200"]) - client.close() - - client = Client(servers=["localhost:4200"]) - self.assertEqual(client._active_servers, ["http://localhost:4200"]) - client.close() - - client = Client(servers=["localhost:4200", "127.0.0.1:4201"]) - self.assertEqual(client._active_servers, - ["http://localhost:4200", "http://127.0.0.1:4201"]) - client.close() - - @patch(REQUEST, fake_request(fake_redirect('http://localhost:4201'))) - def test_redirect_handling(self): - client = Client(servers='localhost:4200') - try: - client.blob_get('blobs', 'fake_digest') - except ProgrammingError: - # 4201 gets added to serverpool but isn't available - # that's why we run into an infinite recursion - # exception message is: maximum recursion depth exceeded - pass - self.assertEqual( - ['http://localhost:4200', 'http://localhost:4201'], - sorted(list(client.server_pool.keys())) - ) - # the new non-https server must not contain any SSL only arguments - # regression test for github issue #179/#180 - self.assertEqual( - {'socket_options': _get_socket_opts(keepalive=True)}, - client.server_pool['http://localhost:4201'].pool.conn_kw - ) - client.close() - - @patch(REQUEST) - def test_server_infos(self, request): - request.side_effect = urllib3.exceptions.MaxRetryError( - None, '/', "this shouldn't be raised") - client = Client(servers="localhost:4200 localhost:4201") - self.assertRaises( - ConnectionError, client.server_infos, 'http://localhost:4200') - client.close() - - @patch(REQUEST, fake_request(fake_response(503))) - def test_server_infos_503(self): - client = Client(servers="localhost:4200") - self.assertRaises( - ConnectionError, client.server_infos, 'http://localhost:4200') - client.close() - - @patch(REQUEST, fake_request( - fake_response(401, 'Unauthorized', 'text/html'))) - def test_server_infos_401(self): - client = Client(servers="localhost:4200") - try: - client.server_infos('http://localhost:4200') - except ProgrammingError as e: - self.assertEqual("401 Client Error: Unauthorized", e.message) - else: - self.assertTrue(False, msg="Exception should have been raised") - finally: - client.close() - - @patch(REQUEST, fake_request(bad_bulk_response())) - def test_bad_bulk_400(self): - client = Client(servers="localhost:4200") - try: - client.sql("Insert into users (name) values(?)", - bulk_parameters=[["douglas"], ["monthy"]]) - except ProgrammingError as e: - self.assertEqual("an error occured\nanother error", e.message) - else: - self.assertTrue(False, msg="Exception should have been raised") - finally: - client.close() - - @patch(REQUEST, autospec=True) - def test_decimal_serialization(self, request): - client = Client(servers="localhost:4200") - request.return_value = fake_response(200) - - dec = Decimal(0.12) - client.sql('insert into users (float_col) values (?)', (dec,)) - - data = json.loads(request.call_args[1]['data']) - self.assertEqual(data['args'], [str(dec)]) - client.close() - - @patch(REQUEST, autospec=True) - def test_datetime_is_converted_to_ts(self, request): - client = Client(servers="localhost:4200") - request.return_value = fake_response(200) - - datetime = dt.datetime(2015, 2, 28, 7, 31, 40) - client.sql('insert into users (dt) values (?)', (datetime,)) - - # convert string to dict - # because the order of the keys isn't deterministic - data = json.loads(request.call_args[1]['data']) - self.assertEqual(data['args'], [1425108700000]) - client.close() - - @patch(REQUEST, autospec=True) - def test_date_is_converted_to_ts(self, request): - client = Client(servers="localhost:4200") - request.return_value = fake_response(200) - - day = dt.date(2016, 4, 21) - client.sql('insert into users (dt) values (?)', (day,)) - data = json.loads(request.call_args[1]['data']) - self.assertEqual(data['args'], [1461196800000]) - client.close() - - def test_socket_options_contain_keepalive(self): - server = 'http://localhost:4200' - client = Client(servers=server) - conn_kw = client.server_pool[server].pool.conn_kw - self.assertIn( - (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1), conn_kw['socket_options'] - ) - client.close() - - @patch(REQUEST, autospec=True) - def test_uuid_serialization(self, request): - client = Client(servers="localhost:4200") - request.return_value = fake_response(200) - - uid = uuid.uuid4() - client.sql('insert into my_table (str_col) values (?)', (uid,)) - - data = json.loads(request.call_args[1]['data']) - self.assertEqual(data['args'], [str(uid)]) - client.close() - - @patch(REQUEST, fake_request(duplicate_key_exception())) - def test_duplicate_key_error(self): - """ - Verify that an `IntegrityError` is raised on duplicate key errors, - instead of the more general `ProgrammingError`. - """ - client = Client(servers="localhost:4200") - with self.assertRaises(IntegrityError) as cm: - client.sql('INSERT INTO testdrive (foo) VALUES (42)') - self.assertEqual(cm.exception.message, - "DuplicateKeyException[A document with the same primary key exists already]") - - -@patch(REQUEST, fail_sometimes) -class ThreadSafeHttpClientTest(TestCase): - """ - Using a pool of 5 Threads to emit commands to the multiple servers through - one Client-instance - - check if number of servers in _inactive_servers and _active_servers always - equals the number of servers initially given. - """ - servers = [ - "127.0.0.1:44209", - "127.0.0.2:44209", - "127.0.0.3:44209", - ] - num_threads = 5 - num_commands = 1000 - thread_timeout = 5.0 # seconds - - def __init__(self, *args, **kwargs): - self.event = Event() - self.err_queue = queue.Queue() - super(ThreadSafeHttpClientTest, self).__init__(*args, **kwargs) - - def setUp(self): - self.client = Client(self.servers) - self.client.retry_interval = 0.2 # faster retry - - def tearDown(self): - self.client.close() - - def _run(self): - self.event.wait() # wait for the others - expected_num_servers = len(self.servers) - for x in range(self.num_commands): - try: - self.client.sql('select name from sys.cluster') - except ConnectionError: - pass - try: - with self.client._lock: - num_servers = len(self.client._active_servers) + \ - len(self.client._inactive_servers) - self.assertEqual( - expected_num_servers, - num_servers, - "expected %d but got %d" % (expected_num_servers, - num_servers) - ) - except AssertionError: - self.err_queue.put(sys.exc_info()) - - def test_client_threaded(self): - """ - Testing if lists of servers is handled correctly when client is used - from multiple threads with some requests failing. - - **ATTENTION:** this test is probabilistic and does not ensure that the - client is indeed thread-safe in all cases, it can only show that it - withstands this scenario. - """ - threads = [ - Thread(target=self._run, name=str(x)) - for x in range(self.num_threads) - ] - for thread in threads: - thread.start() - - self.event.set() - for t in threads: - t.join(self.thread_timeout) - - if not self.err_queue.empty(): - self.assertTrue(False, "".join( - traceback.format_exception(*self.err_queue.get(block=False)))) - - -class ClientAddressRequestHandler(BaseHTTPRequestHandler): - """ - http handler for use with HTTPServer - - returns client host and port in crate-conform-responses - """ - protocol_version = 'HTTP/1.1' - - def do_GET(self): - content_length = self.headers.get("content-length") - if content_length: - self.rfile.read(int(content_length)) - response = json.dumps({ - "cols": ["host", "port"], - "rows": [ - self.client_address[0], - self.client_address[1] - ], - "rowCount": 1, - }) - self.send_response(200) - self.send_header("Content-Length", len(response)) - self.send_header("Content-Type", "application/json; charset=UTF-8") - self.end_headers() - self.wfile.write(response.encode('UTF-8')) - - do_POST = do_PUT = do_DELETE = do_HEAD = do_GET - - -class KeepAliveClientTest(TestCase): - - server_address = ("127.0.0.1", 65535) - - def __init__(self, *args, **kwargs): - super(KeepAliveClientTest, self).__init__(*args, **kwargs) - self.server_process = ForkProcess(target=self._run_server) - - def setUp(self): - super(KeepAliveClientTest, self).setUp() - self.client = Client(["%s:%d" % self.server_address]) - self.server_process.start() - time.sleep(.10) - - def tearDown(self): - self.server_process.terminate() - self.client.close() - super(KeepAliveClientTest, self).tearDown() - - def _run_server(self): - self.server = HTTPServer(self.server_address, - ClientAddressRequestHandler) - self.server.handle_request() - - def test_client_keepalive(self): - for x in range(10): - result = self.client.sql("select * from fake") - - another_result = self.client.sql("select again from fake") - self.assertEqual(result, another_result) - - -class ParamsTest(TestCase): - - def test_params(self): - client = Client(['127.0.0.1:4200'], error_trace=True) - parsed = urlparse(client.path) - params = parse_qs(parsed.query) - self.assertEqual(params["error_trace"], ["true"]) - client.close() - - def test_no_params(self): - client = Client() - self.assertEqual(client.path, "/_sql?types=true") - client.close() - - -class RequestsCaBundleTest(TestCase): - - def test_open_client(self): - os.environ["REQUESTS_CA_BUNDLE"] = CA_CERT_PATH - try: - Client('http://127.0.0.1:4200') - except ProgrammingError: - self.fail("HTTP not working with REQUESTS_CA_BUNDLE") - finally: - os.unsetenv('REQUESTS_CA_BUNDLE') - os.environ["REQUESTS_CA_BUNDLE"] = '' - - def test_remove_certs_for_non_https(self): - d = _remove_certs_for_non_https('https', {"ca_certs": 1}) - self.assertIn('ca_certs', d) - - kwargs = {'ca_certs': 1, 'foobar': 2, 'cert_file': 3} - d = _remove_certs_for_non_https('http', kwargs) - self.assertNotIn('ca_certs', d) - self.assertNotIn('cert_file', d) - self.assertIn('foobar', d) - - -class TimeoutRequestHandler(BaseHTTPRequestHandler): - """ - HTTP handler for use with TestingHTTPServer - updates the shared counter and waits so that the client times out - """ - - def do_POST(self): - self.server.SHARED['count'] += 1 - time.sleep(5) - - -class SharedStateRequestHandler(BaseHTTPRequestHandler): - """ - HTTP handler for use with TestingHTTPServer - sets the shared state of the server and returns an empty response - """ - - def do_POST(self): - self.server.SHARED['count'] += 1 - self.server.SHARED['schema'] = self.headers.get('Default-Schema') - - if self.headers.get('Authorization') is not None: - auth_header = self.headers['Authorization'].replace('Basic ', '') - credentials = b64decode(auth_header).decode('utf-8').split(":", 1) - self.server.SHARED['username'] = credentials[0] - if len(credentials) > 1 and credentials[1]: - self.server.SHARED['password'] = credentials[1] - else: - self.server.SHARED['password'] = None - else: - self.server.SHARED['username'] = None - - if self.headers.get('X-User') is not None: - self.server.SHARED['usernameFromXUser'] = self.headers['X-User'] - else: - self.server.SHARED['usernameFromXUser'] = None - - # send empty response - response = '{}' - self.send_response(200) - self.send_header("Content-Length", len(response)) - self.send_header("Content-Type", "application/json; charset=UTF-8") - self.end_headers() - self.wfile.write(response.encode('utf-8')) - - -class TestingHTTPServer(HTTPServer): - """ - http server providing a shared dict - """ - manager = multiprocessing.Manager() - SHARED = manager.dict() - SHARED['count'] = 0 - SHARED['usernameFromXUser'] = None - SHARED['username'] = None - SHARED['password'] = None - SHARED['schema'] = None - - @classmethod - def run_server(cls, server_address, request_handler_cls): - cls(server_address, request_handler_cls).serve_forever() - - -class TestingHttpServerTestCase(TestCase): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.assertIsNotNone(self.request_handler) - self.server_address = ('127.0.0.1', random.randint(65000, 65535)) - self.server_process = ForkProcess(target=TestingHTTPServer.run_server, - args=(self.server_address, self.request_handler)) - - def setUp(self): - self.server_process.start() - self.wait_for_server() - - def wait_for_server(self): - while True: - try: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.connect(self.server_address) - except Exception: - time.sleep(.25) - else: - break - - def tearDown(self): - self.server_process.terminate() - - def clientWithKwargs(self, **kwargs): - return Client(["%s:%d" % self.server_address], timeout=1, **kwargs) - - -class RetryOnTimeoutServerTest(TestingHttpServerTestCase): - - request_handler = TimeoutRequestHandler - - def setUp(self): - super().setUp() - self.client = self.clientWithKwargs() - - def tearDown(self): - super().tearDown() - self.client.close() - - def test_no_retry_on_read_timeout(self): - try: - self.client.sql("select * from fake") - except ConnectionError as e: - self.assertIn('Read timed out', e.message, - msg='Error message must contain: Read timed out') - self.assertEqual(TestingHTTPServer.SHARED['count'], 1) - - -class TestDefaultSchemaHeader(TestingHttpServerTestCase): - - request_handler = SharedStateRequestHandler - - def setUp(self): - super().setUp() - self.client = self.clientWithKwargs(schema='my_custom_schema') - - def tearDown(self): - self.client.close() - super().tearDown() - - def test_default_schema(self): - self.client.sql('SELECT 1') - self.assertEqual(TestingHTTPServer.SHARED['schema'], 'my_custom_schema') - - -class TestUsernameSentAsHeader(TestingHttpServerTestCase): - - request_handler = SharedStateRequestHandler - - def setUp(self): - super().setUp() - self.clientWithoutUsername = self.clientWithKwargs() - self.clientWithUsername = self.clientWithKwargs(username='testDBUser') - self.clientWithUsernameAndPassword = self.clientWithKwargs(username='testDBUser', - password='test:password') - - def tearDown(self): - self.clientWithoutUsername.close() - self.clientWithUsername.close() - self.clientWithUsernameAndPassword.close() - super().tearDown() - - def test_username(self): - self.clientWithoutUsername.sql("select * from fake") - self.assertEqual(TestingHTTPServer.SHARED['usernameFromXUser'], None) - self.assertEqual(TestingHTTPServer.SHARED['username'], None) - self.assertEqual(TestingHTTPServer.SHARED['password'], None) - - self.clientWithUsername.sql("select * from fake") - self.assertEqual(TestingHTTPServer.SHARED['usernameFromXUser'], 'testDBUser') - self.assertEqual(TestingHTTPServer.SHARED['username'], 'testDBUser') - self.assertEqual(TestingHTTPServer.SHARED['password'], None) - - self.clientWithUsernameAndPassword.sql("select * from fake") - self.assertEqual(TestingHTTPServer.SHARED['usernameFromXUser'], 'testDBUser') - self.assertEqual(TestingHTTPServer.SHARED['username'], 'testDBUser') - self.assertEqual(TestingHTTPServer.SHARED['password'], 'test:password') - - -class TestCrateJsonEncoder(TestCase): - - def test_naive_datetime(self): - data = dt.datetime.fromisoformat("2023-06-26T09:24:00.123") - result = json.dumps(data, cls=CrateJsonEncoder) - self.assertEqual(result, "1687771440123") - - def test_aware_datetime(self): - data = dt.datetime.fromisoformat("2023-06-26T09:24:00.123+02:00") - result = json.dumps(data, cls=CrateJsonEncoder) - self.assertEqual(result, "1687764240123") diff --git a/src/crate/client/tests.py b/src/crate/client/tests.py index 026fb56f..e242f54f 100644 --- a/src/crate/client/tests.py +++ b/src/crate/client/tests.py @@ -338,53 +338,7 @@ def test_suite(): flags = (doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS) # Unit tests. - suite.addTest(makeSuite(CursorTest)) - suite.addTest(makeSuite(HttpClientTest)) - suite.addTest(makeSuite(KeepAliveClientTest)) - suite.addTest(makeSuite(ThreadSafeHttpClientTest)) - suite.addTest(makeSuite(ParamsTest)) - suite.addTest(makeSuite(ConnectionTest)) - suite.addTest(makeSuite(RetryOnTimeoutServerTest)) - suite.addTest(makeSuite(RequestsCaBundleTest)) - suite.addTest(makeSuite(TestUsernameSentAsHeader)) - suite.addTest(makeSuite(TestCrateJsonEncoder)) - suite.addTest(makeSuite(TestDefaultSchemaHeader)) suite.addTest(sqlalchemy_test_suite_unit()) - suite.addTest(doctest.DocTestSuite('crate.client.connection')) - suite.addTest(doctest.DocTestSuite('crate.client.http')) - - s = doctest.DocFileSuite( - 'docs/by-example/connection.rst', - 'docs/by-example/cursor.rst', - module_relative=False, - optionflags=flags, - encoding='utf-8' - ) - suite.addTest(s) - - s = doctest.DocFileSuite( - 'docs/by-example/https.rst', - module_relative=False, - setUp=setUpWithHttps, - optionflags=flags, - encoding='utf-8' - ) - s.layer = HttpsTestServerLayer() - suite.addTest(s) - - # Integration tests. - s = doctest.DocFileSuite( - 'docs/by-example/http.rst', - 'docs/by-example/client.rst', - 'docs/by-example/blob.rst', - module_relative=False, - setUp=setUpCrateLayerBaseline, - tearDown=tearDownDropEntitiesBaseline, - optionflags=flags, - encoding='utf-8' - ) - s.layer = ensure_cratedb_layer() - suite.addTest(s) sqlalchemy_integration_tests = [ 'docs/by-example/sqlalchemy/getting-started.rst', diff --git a/src/crate/testing/__init__.py b/src/crate/testing/__init__.py index 5bb534f7..e69de29b 100644 --- a/src/crate/testing/__init__.py +++ b/src/crate/testing/__init__.py @@ -1 +0,0 @@ -# package diff --git a/src/crate/testing/layer.py b/src/crate/testing/layer.py deleted file mode 100644 index ef8bfe2b..00000000 --- a/src/crate/testing/layer.py +++ /dev/null @@ -1,390 +0,0 @@ -# -*- coding: utf-8; -*- -# -# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor -# license agreements. See the NOTICE file distributed with this work for -# additional information regarding copyright ownership. Crate licenses -# this file to you under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. You may -# obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# However, if you have executed another commercial license agreement -# with Crate these terms will supersede the license and you may use the -# software solely pursuant to the terms of the relevant commercial agreement. - -import os -import re -import sys -import time -import json -import urllib3 -import tempfile -import shutil -import subprocess -import tarfile -import io -import threading -import logging - -try: - from urllib.request import urlopen -except ImportError: - from urllib import urlopen - - -log = logging.getLogger(__name__) - - -CRATE_CONFIG_ERROR = 'crate_config must point to a folder or to a file named "crate.yml"' -HTTP_ADDRESS_RE = re.compile( - r'.*\[(http|.*HttpServer.*)\s*] \[.*\] .*' - 'publish_address {' - r'(?:inet\[[\w\d\.-]*/|\[)?' - r'(?:[\w\d\.-]+/)?' - r'(?P[\d\.:]+)' - r'(?:\])?' - '}' -) - - -def http_url_from_host_port(host, port): - if host and port: - if not isinstance(port, int): - try: - port = int(port) - except ValueError: - return None - return '{}:{}'.format(prepend_http(host), port) - return None - - -def prepend_http(host): - if not re.match(r'^https?\:\/\/.*', host): - return 'http://{}'.format(host) - return host - - -def _download_and_extract(uri, directory): - sys.stderr.write("\nINFO: Downloading CrateDB archive from {} into {}".format(uri, directory)) - sys.stderr.flush() - with io.BytesIO(urlopen(uri).read()) as tmpfile: - with tarfile.open(fileobj=tmpfile) as t: - t.extractall(directory) - - -def wait_for_http_url(log, timeout=30, verbose=False): - start = time.monotonic() - while True: - line = log.readline().decode('utf-8').strip() - elapsed = time.monotonic() - start - if verbose: - sys.stderr.write('[{:>4.1f}s]{}\n'.format(elapsed, line)) - m = HTTP_ADDRESS_RE.match(line) - if m: - return prepend_http(m.group('addr')) - elif elapsed > timeout: - return None - - -class OutputMonitor: - - def __init__(self): - self.consumers = [] - - def consume(self, iterable): - for line in iterable: - for consumer in self.consumers: - consumer.send(line) - - def start(self, proc): - self._stop_out_thread = threading.Event() - self._out_thread = threading.Thread(target=self.consume, args=(proc.stdout,)) - self._out_thread.daemon = True - self._out_thread.start() - - def stop(self): - if self._out_thread is not None: - self._stop_out_thread.set() - self._out_thread.join() - - -class LineBuffer: - - def __init__(self): - self.lines = [] - - def send(self, line): - self.lines.append(line.strip()) - - -class CrateLayer(object): - """ - This layer starts a Crate server. - """ - - __bases__ = () - - tmpdir = tempfile.gettempdir() - wait_interval = 0.2 - - @staticmethod - def from_uri(uri, - name, - http_port='4200-4299', - transport_port='4300-4399', - settings=None, - directory=None, - cleanup=True, - verbose=False): - """Download the Crate tarball from a URI and create a CrateLayer - - :param uri: The uri that points to the Crate tarball - :param name: layer and cluster name - :param http_port: The http port on which Crate will listen - :param transport_port: the transport port on which Crate will listen - :param settings: A dictionary that contains Crate settings - :param directory: Where the tarball will be extracted to. - If this is None a temporary directory will be created. - :param clean: a boolean indicating if the directory should be removed - on teardown. - :param verbose: Set the log verbosity of the test layer - """ - directory = directory or tempfile.mkdtemp() - filename = os.path.basename(uri) - crate_dir = re.sub(r'\.tar(\.gz)?$', '', filename) - crate_home = os.path.join(directory, crate_dir) - - if os.path.exists(crate_home): - sys.stderr.write("\nWARNING: Not extracting Crate tarball because folder already exists") - sys.stderr.flush() - else: - _download_and_extract(uri, directory) - - layer = CrateLayer( - name=name, - crate_home=crate_home, - port=http_port, - transport_port=transport_port, - settings=settings, - verbose=verbose) - if cleanup: - tearDown = layer.tearDown - - def new_teardown(*args, **kws): - shutil.rmtree(directory) - tearDown(*args, **kws) - layer.tearDown = new_teardown - return layer - - def __init__(self, - name, - crate_home, - crate_config=None, - port=None, - keepRunning=False, - transport_port=None, - crate_exec=None, - cluster_name=None, - host="127.0.0.1", - settings=None, - verbose=False, - env=None): - """ - :param name: layer name, is also used as the cluser name - :param crate_home: path to home directory of the crate installation - :param port: port on which crate should run - :param transport_port: port on which transport layer for crate should - run - :param crate_exec: alternative executable command - :param crate_config: alternative crate config file location. - Must be a directory or a file named 'crate.yml' - :param cluster_name: the name of the cluster to join/build. Will be - generated automatically if omitted. - :param host: the host to bind to. defaults to 'localhost' - :param settings: further settings that do not deserve a keyword - argument will be prefixed with ``es.``. - :param verbose: Set the log verbosity of the test layer - :param env: Set environment variables. - """ - self.__name__ = name - if settings and isinstance(settings, dict): - # extra settings may override host/port specification! - self.http_url = http_url_from_host_port(settings.get('network.host', host), - settings.get('http.port', port)) - else: - self.http_url = http_url_from_host_port(host, port) - - self.process = None - self.verbose = verbose - self.env = env or {} - self.env.setdefault('CRATE_USE_IPV4', 'true') - self.env.setdefault('JAVA_HOME', os.environ.get('JAVA_HOME', '')) - self._stdout_consumers = [] - self.conn_pool = urllib3.PoolManager(num_pools=1) - - crate_home = os.path.abspath(crate_home) - if crate_exec is None: - start_script = 'crate.bat' if sys.platform == 'win32' else 'crate' - crate_exec = os.path.join(crate_home, 'bin', start_script) - if crate_config is None: - crate_config = os.path.join(crate_home, 'config', 'crate.yml') - elif (os.path.isfile(crate_config) and - os.path.basename(crate_config) != 'crate.yml'): - raise ValueError(CRATE_CONFIG_ERROR) - if cluster_name is None: - cluster_name = "Testing{0}".format(port or 'Dynamic') - settings = self.create_settings(crate_config, - cluster_name, - name, - host, - port or '4200-4299', - transport_port or '4300-4399', - settings) - # ES 5 cannot parse 'True'/'False' as booleans so convert to lowercase - start_cmd = (crate_exec, ) + tuple(["-C%s=%s" % ((key, str(value).lower()) if isinstance(value, bool) else (key, value)) - for key, value in settings.items()]) - - self._wd = wd = os.path.join(CrateLayer.tmpdir, 'crate_layer', name) - self.start_cmd = start_cmd + ('-Cpath.data=%s' % wd,) - - def create_settings(self, - crate_config, - cluster_name, - node_name, - host, - http_port, - transport_port, - further_settings=None): - settings = { - "discovery.type": "zen", - "discovery.initial_state_timeout": 0, - "node.name": node_name, - "cluster.name": cluster_name, - "network.host": host, - "http.port": http_port, - "path.conf": os.path.dirname(crate_config), - "transport.tcp.port": transport_port, - } - if further_settings: - settings.update(further_settings) - return settings - - def wdPath(self): - return self._wd - - @property - def crate_servers(self): - if self.http_url: - return [self.http_url] - return [] - - def setUp(self): - self.start() - - def _clean(self): - if os.path.exists(self._wd): - shutil.rmtree(self._wd) - - def start(self): - self._clean() - self.process = subprocess.Popen(self.start_cmd, - env=self.env, - stdout=subprocess.PIPE) - returncode = self.process.poll() - if returncode is not None: - raise SystemError( - 'Failed to start server rc={0} cmd={1}'.format(returncode, - self.start_cmd) - ) - - if not self.http_url: - # try to read http_url from startup logs - # this is necessary if no static port is assigned - self.http_url = wait_for_http_url(self.process.stdout, verbose=self.verbose) - - self.monitor = OutputMonitor() - self.monitor.start(self.process) - - if not self.http_url: - self.stop() - else: - sys.stderr.write('HTTP: {}\n'.format(self.http_url)) - self._wait_for_start() - self._wait_for_master() - sys.stderr.write('\nCrate instance ready.\n') - - def stop(self): - self.conn_pool.clear() - if self.process: - self.process.terminate() - self.process.communicate(timeout=10) - self.process.stdout.close() - self.process = None - self.monitor.stop() - self._clean() - - def tearDown(self): - self.stop() - - def _wait_for(self, validator): - start = time.monotonic() - - line_buf = LineBuffer() - self.monitor.consumers.append(line_buf) - - while True: - wait_time = time.monotonic() - start - try: - if validator(): - break - except Exception as e: - self.stop() - raise e - - if wait_time > 30: - for line in line_buf.lines: - log.error(line) - self.stop() - raise SystemError('Failed to start Crate instance in time.') - else: - sys.stderr.write('.') - time.sleep(self.wait_interval) - - self.monitor.consumers.remove(line_buf) - - def _wait_for_start(self): - """Wait for instance to be started""" - - # since crate 0.10.0 http may be ready before the cluster - # is fully available block here so that early requests - # after the layer starts don't result in 503 - def validator(): - try: - resp = self.conn_pool.request('HEAD', self.http_url) - return resp.status == 200 - except Exception: - return False - - self._wait_for(validator) - - def _wait_for_master(self): - """Wait for master node""" - - def validator(): - resp = self.conn_pool.urlopen( - 'POST', - '{server}/_sql'.format(server=self.http_url), - headers={'Content-Type': 'application/json'}, - body='{"stmt": "select master_node from sys.cluster"}' - ) - data = json.loads(resp.data.decode('utf-8')) - return resp.status == 200 and data['rows'][0][0] - - self._wait_for(validator) diff --git a/src/crate/testing/settings.py b/src/crate/testing/settings.py deleted file mode 100644 index 34793cc6..00000000 --- a/src/crate/testing/settings.py +++ /dev/null @@ -1,51 +0,0 @@ -# vi: set encoding=utf-8 -# -*- coding: utf-8; -*- -# -# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor -# license agreements. See the NOTICE file distributed with this work for -# additional information regarding copyright ownership. Crate licenses -# this file to you under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. You may -# obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# However, if you have executed another commercial license agreement -# with Crate these terms will supersede the license and you may use the -# software solely pursuant to the terms of the relevant commercial agreement. -from __future__ import absolute_import - -import os - - -def docs_path(*parts): - return os.path.abspath( - os.path.join( - os.path.dirname(os.path.dirname(__file__)), *parts - ) - ) - - -def project_root(*parts): - return os.path.abspath( - os.path.join(docs_path("..", ".."), *parts) - ) - - -def crate_path(*parts): - return os.path.abspath( - project_root("parts", "crate", *parts) - ) - - -crate_port = 44209 -crate_transport_port = 44309 -localhost = '127.0.0.1' -crate_host = "{host}:{port}".format(host=localhost, port=crate_port) -crate_uri = "http://%s" % crate_host diff --git a/src/crate/testing/test_layer.py b/src/crate/testing/test_layer.py deleted file mode 100644 index aaeca336..00000000 --- a/src/crate/testing/test_layer.py +++ /dev/null @@ -1,290 +0,0 @@ -# -*- coding: utf-8; -*- -# -# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor -# license agreements. See the NOTICE file distributed with this work for -# additional information regarding copyright ownership. Crate licenses -# this file to you under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. You may -# obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# However, if you have executed another commercial license agreement -# with Crate these terms will supersede the license and you may use the -# software solely pursuant to the terms of the relevant commercial agreement. -import json -import os -import tempfile -import urllib -from verlib2 import Version -from unittest import TestCase, mock -from io import BytesIO - -import urllib3 - -import crate -from .layer import CrateLayer, prepend_http, http_url_from_host_port, wait_for_http_url -from .settings import crate_path - - -class LayerUtilsTest(TestCase): - - def test_prepend_http(self): - host = prepend_http('localhost') - self.assertEqual('http://localhost', host) - host = prepend_http('http://localhost') - self.assertEqual('http://localhost', host) - host = prepend_http('https://localhost') - self.assertEqual('https://localhost', host) - host = prepend_http('http') - self.assertEqual('http://http', host) - - def test_http_url(self): - url = http_url_from_host_port(None, None) - self.assertEqual(None, url) - url = http_url_from_host_port('localhost', None) - self.assertEqual(None, url) - url = http_url_from_host_port(None, 4200) - self.assertEqual(None, url) - url = http_url_from_host_port('localhost', 4200) - self.assertEqual('http://localhost:4200', url) - url = http_url_from_host_port('https://crate', 4200) - self.assertEqual('https://crate:4200', url) - - def test_wait_for_http(self): - log = BytesIO(b'[i.c.p.h.CrateNettyHttpServerTransport] [crate] publish_address {127.0.0.1:4200}') - addr = wait_for_http_url(log) - self.assertEqual('http://127.0.0.1:4200', addr) - log = BytesIO(b'[i.c.p.h.CrateNettyHttpServerTransport] [crate] publish_address {}') - addr = wait_for_http_url(log=log, timeout=1) - self.assertEqual(None, addr) - - @mock.patch.object(crate.testing.layer, "_download_and_extract", lambda uri, directory: None) - def test_layer_from_uri(self): - """ - The CrateLayer can also be created by providing an URI that points to - a CrateDB tarball. - """ - with urllib.request.urlopen("https://crate.io/versions.json") as response: - versions = json.loads(response.read().decode()) - version = versions["crate_testing"] - - self.assertGreaterEqual(Version(version), Version("4.5.0")) - - uri = "https://cdn.crate.io/downloads/releases/crate-{}.tar.gz".format(version) - layer = CrateLayer.from_uri(uri, name="crate-by-uri", http_port=42203) - self.assertIsInstance(layer, CrateLayer) - - @mock.patch.dict('os.environ', {}, clear=True) - def test_java_home_env_not_set(self): - with tempfile.TemporaryDirectory() as tmpdir: - layer = CrateLayer('java-home-test', tmpdir) - # JAVA_HOME must not be set to `None`, since it would be interpreted as a - # string 'None', and therefore intepreted as a path - self.assertEqual(layer.env['JAVA_HOME'], '') - - @mock.patch.dict('os.environ', {}, clear=True) - def test_java_home_env_set(self): - java_home = '/usr/lib/jvm/java-11-openjdk-amd64' - with tempfile.TemporaryDirectory() as tmpdir: - os.environ['JAVA_HOME'] = java_home - layer = CrateLayer('java-home-test', tmpdir) - self.assertEqual(layer.env['JAVA_HOME'], java_home) - - @mock.patch.dict('os.environ', {}, clear=True) - def test_java_home_env_override(self): - java_11_home = '/usr/lib/jvm/java-11-openjdk-amd64' - java_12_home = '/usr/lib/jvm/java-12-openjdk-amd64' - with tempfile.TemporaryDirectory() as tmpdir: - os.environ['JAVA_HOME'] = java_11_home - layer = CrateLayer('java-home-test', tmpdir, env={'JAVA_HOME': java_12_home}) - self.assertEqual(layer.env['JAVA_HOME'], java_12_home) - - -class LayerTest(TestCase): - - def test_basic(self): - """ - This layer starts and stops a ``Crate`` instance on a given host, port, - a given crate node name and, optionally, a given cluster name:: - """ - - port = 44219 - transport_port = 44319 - - layer = CrateLayer('crate', - crate_home=crate_path(), - host='127.0.0.1', - port=port, - transport_port=transport_port, - cluster_name='my_cluster' - ) - - # The working directory is defined on layer instantiation. - # It is sometimes required to know it before starting the layer. - self.assertRegex(layer.wdPath(), r".+/crate_layer/crate") - - # Start the layer. - layer.start() - - # The urls of the crate servers to be instantiated can be obtained - # via `crate_servers`. - self.assertEqual(layer.crate_servers, ["http://127.0.0.1:44219"]) - - # Access the CrateDB instance on the HTTP interface. - - http = urllib3.PoolManager() - - stats_uri = "http://127.0.0.1:{0}/".format(port) - response = http.request('GET', stats_uri) - self.assertEqual(response.status, 200) - - # The layer can be shutdown using its `stop()` method. - layer.stop() - - def test_dynamic_http_port(self): - """ - It is also possible to define a port range instead of a static HTTP port for the layer. - - Crate will start with the first available port in the given range and the test - layer obtains the chosen port from the startup logs of the Crate process. - Note, that this feature requires a logging configuration with at least loglevel - ``INFO`` on ``http``. - """ - port = '44200-44299' - layer = CrateLayer('crate', crate_home=crate_path(), port=port) - layer.start() - self.assertRegex(layer.crate_servers[0], r"http://127.0.0.1:442\d\d") - layer.stop() - - def test_default_settings(self): - """ - Starting a CrateDB layer leaving out optional parameters will apply the following - defaults. - - The default http port is the first free port in the range of ``4200-4299``, - the default transport port is the first free port in the range of ``4300-4399``, - the host defaults to ``127.0.0.1``. - - The command to call is ``bin/crate`` inside the ``crate_home`` path. - The default config file is ``config/crate.yml`` inside ``crate_home``. - The default cluster name will be auto generated using the HTTP port. - """ - layer = CrateLayer('crate_defaults', crate_home=crate_path()) - layer.start() - self.assertEqual(layer.crate_servers[0], "http://127.0.0.1:4200") - layer.stop() - - def test_additional_settings(self): - """ - The ``Crate`` layer can be started with additional settings as well. - Add a dictionary for keyword argument ``settings`` which contains your settings. - Those additional setting will override settings given as keyword argument. - - The settings will be handed over to the ``Crate`` process with the ``-C`` flag. - So the setting ``threadpool.bulk.queue_size: 100`` becomes - the command line flag: ``-Cthreadpool.bulk.queue_size=100``:: - """ - layer = CrateLayer( - 'custom', - crate_path(), - port=44401, - settings={ - "cluster.graceful_stop.min_availability": "none", - "http.port": 44402 - } - ) - layer.start() - self.assertEqual(layer.crate_servers[0], "http://127.0.0.1:44402") - self.assertIn("-Ccluster.graceful_stop.min_availability=none", layer.start_cmd) - layer.stop() - - def test_verbosity(self): - """ - The test layer hides the standard output of Crate per default. To increase the - verbosity level the additional keyword argument ``verbose`` needs to be set - to ``True``:: - """ - layer = CrateLayer('crate', - crate_home=crate_path(), - verbose=True) - layer.start() - self.assertTrue(layer.verbose) - layer.stop() - - def test_environment_variables(self): - """ - It is possible to provide environment variables for the ``Crate`` testing - layer. - """ - layer = CrateLayer('crate', - crate_home=crate_path(), - env={"CRATE_HEAP_SIZE": "300m"}) - - layer.start() - - sql_uri = layer.crate_servers[0] + "/_sql" - - http = urllib3.PoolManager() - response = http.urlopen('POST', sql_uri, - body='{"stmt": "select heap[\'max\'] from sys.nodes"}') - json_response = json.loads(response.data.decode('utf-8')) - - self.assertEqual(json_response["rows"][0][0], 314572800) - - layer.stop() - - def test_cluster(self): - """ - To start a cluster of ``Crate`` instances, give each instance the same - ``cluster_name``. If you want to start instances on the same machine then - use value ``_local_`` for ``host`` and give every node different ports:: - """ - cluster_layer1 = CrateLayer( - 'crate1', - crate_path(), - host='_local_', - cluster_name='my_cluster', - ) - cluster_layer2 = CrateLayer( - 'crate2', - crate_path(), - host='_local_', - cluster_name='my_cluster', - settings={"discovery.initial_state_timeout": "10s"} - ) - - # If we start both layers, they will, after a small amount of time, find each other - # and form a cluster. - cluster_layer1.start() - cluster_layer2.start() - - # We can verify that by checking the number of nodes a node knows about. - http = urllib3.PoolManager() - - def num_cluster_nodes(crate_layer): - sql_uri = crate_layer.crate_servers[0] + "/_sql" - response = http.urlopen('POST', sql_uri, body='{"stmt":"select count(*) from sys.nodes"}') - json_response = json.loads(response.data.decode('utf-8')) - return json_response["rows"][0][0] - - # We might have to wait a moment before the cluster is finally created. - num_nodes = num_cluster_nodes(cluster_layer1) - import time - retries = 0 - while num_nodes < 2: # pragma: no cover - time.sleep(1) - num_nodes = num_cluster_nodes(cluster_layer1) - retries += 1 - if retries == 30: - break - self.assertEqual(num_nodes, 2) - - cluster_layer1.stop() - cluster_layer2.stop() diff --git a/src/crate/testing/util.py b/src/crate/testing/util.py deleted file mode 100644 index 3e9885d6..00000000 --- a/src/crate/testing/util.py +++ /dev/null @@ -1,20 +0,0 @@ -class ExtraAssertions: - """ - Additional assert methods for unittest. - - - https://github.com/python/cpython/issues/71339 - - https://bugs.python.org/issue14819 - - https://bugs.python.org/file43047/extra_assertions.patch - """ - - def assertIsSubclass(self, cls, superclass, msg=None): - try: - r = issubclass(cls, superclass) - except TypeError: - if not isinstance(cls, type): - self.fail(self._formatMessage(msg, - '%r is not a class' % (cls,))) - raise - if not r: - self.fail(self._formatMessage(msg, - '%r is not a subclass of %r' % (cls, superclass)))