diff --git a/.gitignore b/.gitignore index aeca2d6b..3b32ddeb 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ eggs/ htmlcov/ out/ parts/ +tmp/ diff --git a/DEVELOP.rst b/DEVELOP.rst index 80589a26..31903943 100644 --- a/DEVELOP.rst +++ b/DEVELOP.rst @@ -36,7 +36,12 @@ Ignore specific test directories:: ./bin/test -vvvv --ignore_dir=testing -Invoke all tests without integration tests (~5 seconds runtime):: +The ``LayerTest`` test cases have quite some overhead. Omitting them will save +a few cycles (~90 seconds runtime):: + + ./bin/test -t '!LayerTest' + +Invoke all tests without integration tests (~3 seconds runtime):: ./bin/test \ -t '!LayerTest' -t '!docs/by-example' \ diff --git a/docs/by-example/connection.rst b/docs/by-example/connection.rst index 22230436..81f5a756 100644 --- a/docs/by-example/connection.rst +++ b/docs/by-example/connection.rst @@ -5,6 +5,10 @@ The Connection object This documentation section outlines different attributes, methods, and behaviors of the ``crate.client.connection.Connection`` object. +To improve focus and reduce boilerplate, the example code uses both +``ClientMocked``. It is required for demonstration purposes, so the example +does not need a real database connection. + .. rubric:: Table of Contents .. contents:: @@ -13,13 +17,13 @@ behaviors of the ``crate.client.connection.Connection`` object. connect() ========= -:: - >>> from crate.client import connect +This section sets up a connection object, and inspects some of its attributes. -We create a new connection object:: + >>> from crate.client import connect + >>> from crate.client.test_util import ClientMocked - >>> connection = connect(client=connection_client_mocked) + >>> connection = connect(client=ClientMocked()) >>> connection.lowest_server_version.version (2, 0, 0) diff --git a/docs/by-example/cursor.rst b/docs/by-example/cursor.rst index 17605131..c7a46928 100644 --- a/docs/by-example/cursor.rst +++ b/docs/by-example/cursor.rst @@ -5,8 +5,9 @@ The Cursor object This documentation section outlines different attributes, methods, and behaviors of the ``crate.client.cursor.Cursor`` object. -To improve focus and reduce boilerplate, it uses a ``set_next_response`` method -only intended for testing. +To improve focus and reduce boilerplate, the example code uses both +``ClientMocked`` and ``set_next_response``. They are required for demonstration +purposes, so the example does not need a real database connection. .. rubric:: Table of Contents @@ -14,24 +15,21 @@ only intended for testing. :local: -Setup -===== +Introduction +============ This section sets up a cursor object, inspects some of its attributes, and sets up the response for subsequent cursor operations. -:: - >>> from crate.client import connect >>> from crate.client.cursor import Cursor + >>> from crate.client.test_util import ClientMocked - >>> connection = connect(client=connection_client_mocked) + >>> connection = connect(client=ClientMocked()) >>> cursor = connection.cursor() The rowcount and duration attribute is ``-1``, in case no ``execute()`` has -been performed on the cursor. - -:: +been performed on the cursor yet. >>> cursor.rowcount -1 @@ -39,8 +37,8 @@ been performed on the cursor. >>> cursor.duration -1 -Hardcode the next response of the mocked connection client, so we won't need a sql statement -to execute:: +Define the response of the mocked connection client. It will be returned on +request without needing to execute an SQL statement. >>> connection.client.set_next_response({ ... "rows":[ [ "North West Ripple", 1 ], [ "Arkintoofle Minor", 3 ], [ "Alpha Centauri", 3 ] ], @@ -179,8 +177,6 @@ Iterating over a new cursor without results will immediately raise a Programming description =========== -:: - >>> cursor.description (('name', None, None, None, None, None, None), ('position', None, None, None, None, None, None)) @@ -324,8 +320,6 @@ The cursor object can optionally convert database types to native Python data types. Currently, this is implemented for the CrateDB data types ``IP`` and ``TIMESTAMP`` on behalf of the ``DefaultTypeConverter``. -:: - >>> cursor = connection.cursor(converter=Cursor.get_default_converter()) >>> connection.client.set_next_response({ @@ -351,7 +345,7 @@ inspect the ``DataType`` enum, or the documentation about the list of available `CrateDB data type identifiers for the HTTP interface`_. To create a simple converter for converging CrateDB's ``BIT`` type to Python's -``int`` type:: +``int`` type. >>> from crate.client.converter import Converter, DataType @@ -360,7 +354,7 @@ To create a simple converter for converging CrateDB's ``BIT`` type to Python's >>> cursor = connection.cursor(converter=converter) Proof that the converter works correctly, ``B\'0110\'`` should be converted to -``6``. CrateDB's ``BIT`` data type has the numeric identifier ``25``:: +``6``. CrateDB's ``BIT`` data type has the numeric identifier ``25``. >>> connection.client.set_next_response({ ... "col_types": [25], @@ -386,8 +380,6 @@ desired time zone. For your reference, in the following examples, epoch 1658167836758 is ``Mon, 18 Jul 2022 18:10:36 GMT``. -:: - >>> import datetime >>> tz_mst = datetime.timezone(datetime.timedelta(hours=7), name="MST") >>> cursor = connection.cursor(time_zone=tz_mst) @@ -414,7 +406,7 @@ The available options are: - ``zoneinfo.ZoneInfo("Australia/Sydney")`` - ``+0530`` (UTC offset in string format) -Let's exercise all of them:: +Let's exercise all of them: >>> cursor.time_zone = datetime.timezone.utc >>> cursor.execute('') diff --git a/docs/by-example/sqlalchemy/cru.rst b/docs/by-example/sqlalchemy/cru.rst index 16231e11..285bb74e 100644 --- a/docs/by-example/sqlalchemy/cru.rst +++ b/docs/by-example/sqlalchemy/cru.rst @@ -13,13 +13,42 @@ and updating complex data types with nested Python dictionaries. :local: -Setup -===== +Introduction +============ + +Import the relevant symbols: + + >>> import sqlalchemy as sa + >>> from datetime import datetime + >>> from sqlalchemy.ext.declarative import declarative_base + >>> from sqlalchemy.orm import sessionmaker + >>> from sqlalchemy.sql import text + >>> from crate.client.sqlalchemy.types import ObjectArray Establish a connection to the database: + >>> engine = sa.create_engine(f"crate://{crate_host}") >>> connection = engine.connect() +Define the ORM schema for the ``Location`` entity: + + >>> Base = declarative_base(bind=engine) + + >>> class Location(Base): + ... __tablename__ = 'locations' + ... name = sa.Column(sa.String, primary_key=True) + ... kind = sa.Column(sa.String) + ... date = sa.Column(sa.Date, default=lambda: datetime.utcnow().date()) + ... datetime_tz = sa.Column(sa.DateTime, default=datetime.utcnow) + ... datetime_notz = sa.Column(sa.DateTime, default=datetime.utcnow) + ... nullable_datetime = sa.Column(sa.DateTime) + ... nullable_date = sa.Column(sa.Date) + ... flag = sa.Column(sa.Boolean) + ... details = sa.Column(ObjectArray) + +Create a session with SQLAlchemy: + + >>> session = sessionmaker(bind=engine)() Retrieve ======== @@ -247,8 +276,7 @@ Refresh "characters" table: >>> _ = connection.execute("REFRESH TABLE characters") >>> session.refresh(char) - >>> import pprint - >>> pprint.pprint(char.details) + >>> pprint(char.details) {'name': {'first': 'Trillian', 'last': 'Dent'}, 'size': 45} .. Hidden: close connection diff --git a/docs/by-example/sqlalchemy/getting-started.rst b/docs/by-example/sqlalchemy/getting-started.rst index 39bb986f..52b3f4f9 100644 --- a/docs/by-example/sqlalchemy/getting-started.rst +++ b/docs/by-example/sqlalchemy/getting-started.rst @@ -13,6 +13,28 @@ as well as the use of complex and geospatial data types. :local: +Introduction +============ + +Import the relevant symbols: + + >>> import sqlalchemy as sa + >>> from sqlalchemy.ext.declarative import declarative_base + >>> from sqlalchemy.orm import sessionmaker + >>> from sqlalchemy.sql import text + +Establish a connection to the database: + + >>> engine = sa.create_engine(f"crate://{crate_host}") + >>> connection = engine.connect() + + >>> Base = declarative_base(bind=engine) + +Create a session with SQLAlchemy: + + >>> session = sessionmaker(bind=engine)() + + Connection String ================= @@ -116,9 +138,9 @@ defined in the schema: After ``INSERT`` statements are sent to the database the newly inserted rows aren't immediately available for search because the index is only updated -periodically: +periodically. In order to synchronize that, refresh the table: - >>> refresh("characters") + >>> _ = connection.execute(text("REFRESH TABLE characters")) A regular select query will then fetch the whole documents: @@ -180,7 +202,7 @@ This will generate an UPDATE statement roughly like this: "UPDATE characters set more_details = ? ...", ([{'foo': 1, 'bar': 10}, {'foo': 2}, {'foo': 3}],) - >>> refresh("characters") + >>> _ = connection.execute(text("REFRESH TABLE characters")) To do queries against fields of ``ObjectArray``s you have to use the ``.any(value, operator=operators.eq)`` method on a subscript, because accessing @@ -241,10 +263,10 @@ session: >>> tokyo = City(coordinate=point, area=area, name='Tokyo') >>> session.add(tokyo) >>> session.commit() + >>> _ = connection.execute(text("REFRESH TABLE cities")) When retrieved, they are retrieved as the corresponding geojson objects: - >>> refresh("cities") >>> query = session.query(City.name, City.coordinate, City.area) >>> query.all() [('Tokyo', (139.75999999791384, 35.67999996710569), {"coordinates": [[[139.806, 35.515], [139.919, 35.703], [139.768, 35.817], [139.575, 35.76], [139.584, 35.619], [139.806, 35.515]]], "type": "Polygon"})] @@ -425,7 +447,7 @@ Let's add a task to the ``Todo`` table: >>> task = Todos(content='Write Tests', status='done') >>> session.add(task) >>> session.commit() - >>> refresh("todos") + >>> _ = connection.execute(text("REFRESH TABLE todos")) Using ``insert().from_select()`` to archive the task in ``ArchivedTasks`` table: @@ -434,7 +456,7 @@ table: >>> ins = insert(ArchivedTasks).from_select(['id','content'], sel) >>> result = session.execute(ins) >>> session.commit() - >>> refresh("archived_tasks") + >>> _ = connection.execute(text("REFRESH TABLE archived_tasks")) This will result in the following query: diff --git a/docs/by-example/sqlalchemy/inspection-reflection.rst b/docs/by-example/sqlalchemy/inspection-reflection.rst index e935fc7b..92611325 100644 --- a/docs/by-example/sqlalchemy/inspection-reflection.rst +++ b/docs/by-example/sqlalchemy/inspection-reflection.rst @@ -19,6 +19,9 @@ A low level interface which provides a backend-agnostic system of loading lists of schema, table, column, and constraint descriptions from a given database is available. This is known as the `SQLAlchemy inspector`_. + >>> import sqlalchemy as sa + + >>> engine = sa.create_engine(f"crate://{crate_host}") >>> inspector = sa.inspect(engine) diff --git a/docs/by-example/sqlalchemy/internals.rst b/docs/by-example/sqlalchemy/internals.rst index 94486c45..9d03f12c 100644 --- a/docs/by-example/sqlalchemy/internals.rst +++ b/docs/by-example/sqlalchemy/internals.rst @@ -9,24 +9,34 @@ focuses on showing specific internals. CrateDialect ============ -The initialize method sets the default schema name and version info: +Import the relevant symbols: + >>> import sqlalchemy as sa + >>> from crate.client.sqlalchemy.dialect import CrateDialect + +Establish a connection to the database: + + >>> engine = sa.create_engine(f"crate://{crate_host}") >>> connection = engine.connect() + +After initializing the dialect instance with a connection instance, + >>> dialect = CrateDialect() >>> dialect.initialize(connection) +the database server version and default schema name can be inquired. >>> dialect.server_version_info >= (1, 0, 0) True -Check if table exists: +Check if schema exists: - >>> dialect.has_table(connection, 'locations') + >>> dialect.has_schema(connection, 'doc') True -Check if schema exists: +Check if table exists: - >>> dialect.has_schema(connection, 'doc') + >>> dialect.has_table(connection, 'locations') True .. Hidden: close connection diff --git a/docs/sqlalchemy.rst b/docs/sqlalchemy.rst index a750f9b5..491e14ab 100644 --- a/docs/sqlalchemy.rst +++ b/docs/sqlalchemy.rst @@ -166,7 +166,7 @@ Table definition ---------------- Here is an example SQLAlchemy table definition using the `declarative -system`_:: +system`_: >>> from sqlalchemy.ext import declarative >>> from crate.client.sqlalchemy import types @@ -402,7 +402,8 @@ default, which is a short time for me and you, but a long time for your code). You can request a `table refresh`_ to update the index manually:: - >>> refresh("characters") + >>> connection = engine.connect() + >>> _ = connection.execute(text("REFRESH TABLE characters")) .. NOTE:: diff --git a/src/crate/client/tests.py b/src/crate/client/tests.py index 71cf6066..638e54f8 100644 --- a/src/crate/client/tests.py +++ b/src/crate/client/tests.py @@ -27,7 +27,6 @@ import unittest import doctest from pprint import pprint -from datetime import datetime from http.server import HTTPServer, BaseHTTPRequestHandler import ssl import time @@ -41,8 +40,6 @@ crate_host, crate_path, crate_port, \ crate_transport_port, docs_path, localhost from crate.client import connect -from crate.client.sqlalchemy.dialect import CrateDialect -from crate.client.test_util import ClientMocked from .test_cursor import CursorTest from .test_connection import ConnectionTest @@ -57,7 +54,6 @@ TestDefaultSchemaHeader, ) from .sqlalchemy.tests import test_suite as sqlalchemy_test_suite -from .sqlalchemy.types import ObjectArray log = logging.getLogger('crate.testing.layer') ch = logging.StreamHandler() @@ -71,10 +67,6 @@ def cprint(s): print(s) -def setUpMocked(test): - test.globs['connection_client_mocked'] = ClientMocked() - - settings = { 'udc.enabled': 'false', 'lang.js.enabled': 'true', @@ -115,17 +107,10 @@ def ensure_cratedb_layer(): return crate_layer -def refresh(table): - with connect(crate_host) as conn: - cursor = conn.cursor() - cursor.execute("refresh table %s" % table) - - def setUpWithCrateLayer(test): test.globs['crate_host'] = crate_host test.globs['pprint'] = pprint test.globs['print'] = cprint - test.globs["refresh"] = refresh with connect(crate_host) as conn: cursor = conn.cursor() @@ -154,54 +139,33 @@ def setUpWithCrateLayer(test): def setUpCrateLayerAndSqlAlchemy(test): setUpWithCrateLayer(test) import sqlalchemy as sa - from sqlalchemy.ext.declarative import declarative_base - from sqlalchemy.orm import sessionmaker - with connect(crate_host) as conn: - cursor = conn.cursor() - cursor.execute("""create table characters ( - id string primary key, - name string, - quote string, - details object, - more_details array(object), - INDEX name_ft using fulltext(name) with (analyzer = 'english'), - INDEX quote_ft using fulltext(quote) with (analyzer = 'english') - )""") - cursor.execute("CREATE VIEW characters_view AS SELECT * FROM characters") + ddl_statements = [ + """ + CREATE TABLE characters ( + id STRING PRIMARY KEY, + name STRING, + quote STRING, + details OBJECT, + more_details ARRAY(OBJECT), + INDEX name_ft USING fulltext(name) WITH (analyzer = 'english'), + INDEX quote_ft USING fulltext(quote) WITH (analyzer = 'english') + )""", + """ + CREATE VIEW characters_view + AS SELECT * FROM characters + """, + """ + CREATE TABLE cities ( + name STRING PRIMARY KEY, + coordinate GEO_POINT, + area GEO_SHAPE + )""" + ] - with connect(crate_host) as conn: - cursor = conn.cursor() - cursor.execute("""create table cities ( - name string primary key, - coordinate geo_point, - area geo_shape - ) """) - - engine = sa.create_engine('crate://{0}'.format(crate_host)) - Base = declarative_base() - - class Location(Base): - __tablename__ = 'locations' - name = sa.Column(sa.String, primary_key=True) - kind = sa.Column(sa.String) - date = sa.Column(sa.Date, default=lambda: datetime.utcnow().date()) - datetime_tz = sa.Column(sa.DateTime, default=datetime.utcnow) - datetime_notz = sa.Column(sa.DateTime, default=datetime.utcnow) - nullable_datetime = sa.Column(sa.DateTime) - nullable_date = sa.Column(sa.Date) - flag = sa.Column(sa.Boolean) - details = sa.Column(ObjectArray) - - Session = sessionmaker(engine) - session = Session() - test.globs['sa'] = sa - test.globs['engine'] = engine - test.globs['Location'] = Location - test.globs['Base'] = Base - test.globs['session'] = session - test.globs['Session'] = Session - test.globs['CrateDialect'] = CrateDialect + engine = sa.create_engine(f"crate://{crate_host}") + for ddl_statement in ddl_statements: + engine.execute(sa.text(ddl_statement)) class HttpsTestServerLayer: @@ -350,7 +314,6 @@ def test_suite(): 'docs/by-example/connection.rst', 'docs/by-example/cursor.rst', module_relative=False, - setUp=setUpMocked, optionflags=flags, encoding='utf-8' ) diff --git a/src/crate/testing/tests.py b/src/crate/testing/tests.py index b647bc13..186b6dc0 100644 --- a/src/crate/testing/tests.py +++ b/src/crate/testing/tests.py @@ -20,18 +20,12 @@ # with Crate these terms will supersede the license and you may use the # software solely pursuant to the terms of the relevant commercial agreement. -import os import unittest -import tempfile from .test_layer import LayerUtilsTest, LayerTest -from .settings import project_root, crate_path def setUp(test): - test.globs['project_root'] = project_root - test.globs['crate_path'] = crate_path - test.globs['tempfile'] = tempfile - test.globs['os'] = os + pass def test_suite():