From 841f06e74c9bf6e06cd3de9bb1967ae817c18f86 Mon Sep 17 00:00:00 2001 From: Pamphile Roy Date: Tue, 8 Feb 2022 23:05:24 +0100 Subject: [PATCH 01/21] ENH: add support for list to execute --- rbc/omniscidb.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/rbc/omniscidb.py b/rbc/omniscidb.py index 702e8d05e..8400ae708 100644 --- a/rbc/omniscidb.py +++ b/rbc/omniscidb.py @@ -1290,6 +1290,38 @@ def get_types(self, *values): """ types = [] for value in values: + + if isinstance(value, list): + items_types = set(map(typesystem.Type.fromvalue, value)) + + has_int, has_float, has_other = False, False, False + + for item_type in items_types: + is_int = item_type.is_int or item_type.is_uint + if is_int: + has_int = True + if item_type.is_float: + has_float = True + if not (is_int or item_type.is_float): + has_other = True + + if (has_int or has_float) and (not has_other) \ + and len(items_types) != 1: + + if has_int and (not has_float): + items_types = {typesystem.Type.fromstring('int64')} + else: + items_types = {typesystem.Type.fromstring('float64')} + + if len(items_types) == 1: + array_type = f'Array<{items_types.pop().tostring()}>' + array_type = typesystem.Type.fromstring(array_type) + types.append(array_type) + else: + raise NotImplementedError(...) + else: + types.append(typesystem.Type.fromvalue(value)) + if isinstance(value, RemoteCallCapsule): typ = value.__typesystem_type__ if typ.is_struct and typ._params.get('struct_is_tuple'): From 10b20487b71a958b82f2e2709760ebea800d88c7 Mon Sep 17 00:00:00 2001 From: Pamphile Roy Date: Sat, 12 Feb 2022 12:46:59 +0100 Subject: [PATCH 02/21] Update rbc/omniscidb.py Co-authored-by: Pearu Peterson --- rbc/omniscidb.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/rbc/omniscidb.py b/rbc/omniscidb.py index 8400ae708..ed5fa3a5d 100644 --- a/rbc/omniscidb.py +++ b/rbc/omniscidb.py @@ -1319,10 +1319,7 @@ def get_types(self, *values): types.append(array_type) else: raise NotImplementedError(...) - else: - types.append(typesystem.Type.fromvalue(value)) - - if isinstance(value, RemoteCallCapsule): + elif isinstance(value, RemoteCallCapsule): typ = value.__typesystem_type__ if typ.is_struct and typ._params.get('struct_is_tuple'): types.extend(typ) From 2e002cd0be05c460ce72990e7447e822d09a2c06 Mon Sep 17 00:00:00 2001 From: Pamphile Roy Date: Wed, 16 Feb 2022 16:28:26 +0100 Subject: [PATCH 03/21] Add fromtypes to reduce a list of types --- rbc/omniscidb.py | 30 ++++------------------------- rbc/tests/test_typesystem.py | 24 ++++++++++++++++++++++- rbc/typesystem.py | 37 ++++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 27 deletions(-) diff --git a/rbc/omniscidb.py b/rbc/omniscidb.py index ed5fa3a5d..b681f0f32 100644 --- a/rbc/omniscidb.py +++ b/rbc/omniscidb.py @@ -1293,32 +1293,10 @@ def get_types(self, *values): if isinstance(value, list): items_types = set(map(typesystem.Type.fromvalue, value)) - - has_int, has_float, has_other = False, False, False - - for item_type in items_types: - is_int = item_type.is_int or item_type.is_uint - if is_int: - has_int = True - if item_type.is_float: - has_float = True - if not (is_int or item_type.is_float): - has_other = True - - if (has_int or has_float) and (not has_other) \ - and len(items_types) != 1: - - if has_int and (not has_float): - items_types = {typesystem.Type.fromstring('int64')} - else: - items_types = {typesystem.Type.fromstring('float64')} - - if len(items_types) == 1: - array_type = f'Array<{items_types.pop().tostring()}>' - array_type = typesystem.Type.fromstring(array_type) - types.append(array_type) - else: - raise NotImplementedError(...) + com_type = typesystem.Type.fromtypes(items_types) + array_type = f'Array<{com_type.tostring()}>' + array_type = typesystem.Type.fromstring(array_type) + types.append(array_type) elif isinstance(value, RemoteCallCapsule): typ = value.__typesystem_type__ if typ.is_struct and typ._params.get('struct_is_tuple'): diff --git a/rbc/tests/test_typesystem.py b/rbc/tests/test_typesystem.py index 73b484dfb..6b19ba47f 100644 --- a/rbc/tests/test_typesystem.py +++ b/rbc/tests/test_typesystem.py @@ -18,7 +18,7 @@ np = None import pytest -from rbc.typesystem import Type, get_signature +from rbc.typesystem import Type, get_signature, TypeParseError from rbc.utils import get_datamodel from rbc.targetinfo import TargetInfo @@ -82,6 +82,28 @@ def test_commasplit(): assert '^'.join(commasplit('a[:, :], b[:, :, :]')) == 'a[:, :]^b[:, :, :]' +def test_fromtypes(): + + def from_list(values): + return set(map(Type.fromvalue, values)) + + assert Type.fromtypes(from_list([1, 3., 6])) == Type('float64') + assert Type.fromtypes(from_list([1, 3, 6])) == Type('int64') + + types = from_list([np.int8(1), np.int8(3), np.int8(6)]) + assert Type.fromtypes(types) == Type('int8') + + types = from_list([np.int8(1), np.int32(3), np.int16(6)]) + assert Type.fromtypes(types) == Type('int32') + + types = from_list([np.uint8(1), np.uint32(3), np.int8(6)]) + assert Type.fromtypes(types) == Type('int32') + + msg = "Failed to cast" + with pytest.raises(TypeParseError, match=msg): + Type.fromtypes(from_list([1, 'a', 6])) + + def test_fromstring(target_info): assert Type.fromstring('void') == Type() diff --git a/rbc/typesystem.py b/rbc/typesystem.py index f855e7c52..109c046b9 100644 --- a/rbc/typesystem.py +++ b/rbc/typesystem.py @@ -1226,6 +1226,43 @@ def fromobject(cls, obj): return cls.fromcallable(obj) raise NotImplementedError(repr((type(obj)))) + @classmethod + def fromtypes(cls, t): + """Consolidate a list of types into a single type.""" + has_uint, has_int, has_float, has_other = True, False, False, False + bit_int, bit_float = 8, 8 + + for item_type in t: + curr_type = item_type.tostring() + + if 'int' in curr_type: + has_int = True + if 'uint' not in curr_type: + has_uint = False + curr_bit = int(curr_type.split('int')[1]) + bit_int = max(bit_int, curr_bit) + elif 'float' in curr_type: + has_float = True + curr_bit = int(curr_type.split('float')[1]) + bit_float = max(bit_float, curr_bit) + else: + has_other = True + + if (has_int or has_float) and (not has_other) and len(t) != 1: + if not has_float: + com_type = f'int{bit_int}' + if has_uint: + com_type = 'u' + com_type + else: + com_type = f'float{bit_float}' + + t = [Type.fromstring(com_type)] + + if len(t) == 1: + return t.pop() + else: + raise TypeParseError(f"Failed to cast {repr(t)!r} to a single type") + def _normalize(self): """Return new Type instance with atomic types normalized. """ From 02e4e67532bc1d6c168893412b611bfbed7d42d9 Mon Sep 17 00:00:00 2001 From: Pearu Peterson Date: Thu, 17 Feb 2022 15:43:57 +0200 Subject: [PATCH 04/21] Fix converting list to SQL array literal. Add test_direct_call_array. --- rbc/omniscidb.py | 8 ++++++-- rbc/tests/test_omnisci.py | 13 ++++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/rbc/omniscidb.py b/rbc/omniscidb.py index b681f0f32..fe1869d02 100644 --- a/rbc/omniscidb.py +++ b/rbc/omniscidb.py @@ -272,6 +272,7 @@ def type_to_type_name(typ: typesystem.Type): ).get(styp) if type_name is not None: return type_name + raise NotImplementedError(f'converting `{styp}` to DatumType not supported') @@ -1294,8 +1295,7 @@ def get_types(self, *values): if isinstance(value, list): items_types = set(map(typesystem.Type.fromvalue, value)) com_type = typesystem.Type.fromtypes(items_types) - array_type = f'Array<{com_type.tostring()}>' - array_type = typesystem.Type.fromstring(array_type) + array_type = OmnisciArrayType((com_type,)) types.append(array_type) elif isinstance(value, RemoteCallCapsule): typ = value.__typesystem_type__ @@ -1401,6 +1401,10 @@ def remote_call(self, func, ftype: typesystem.Type, arguments: tuple, hold=False elif isinstance(a, str): a = repr(a) args.append(f'{a}') + elif isinstance(atype, OmnisciArrayType): + element_type_name = type_to_type_name(atype.element_type) + astr = ", ".join([f'CAST({a_} AS {element_type_name})' for a_ in a]) + args.append(f'ARRAY[{astr}]') else: args.append(f'CAST({a} AS {type_to_type_name(atype)})') args = ', '.join(args) diff --git a/rbc/tests/test_omnisci.py b/rbc/tests/test_omnisci.py index 099bab8d0..f01bfff25 100644 --- a/rbc/tests/test_omnisci.py +++ b/rbc/tests/test_omnisci.py @@ -67,7 +67,7 @@ def omnisci(): yield o -def test_direct_call(omnisci): +def test_direct_call_scalar(omnisci): omnisci.reset() @omnisci('double(double)') @@ -77,6 +77,17 @@ def farhenheit2celcius(f): assert_equal(farhenheit2celcius(40).execute(), np.float32(40 / 9)) +def test_direct_call_array(omnisci): + from rbc.omnisci_backend import mean + omnisci.reset() + + @omnisci('double(double[])') + def farhenheit2celcius(f): + return (mean(f)-32) * 5 / 9 + + assert_equal(farhenheit2celcius([30.0, 50.0]).execute(), np.float32(40 / 9)) + + def test_local_caller(omnisci): omnisci.reset() From 7839af413ec3dfb4f752b151cc8efbf947034da5 Mon Sep 17 00:00:00 2001 From: Pamphile Roy Date: Thu, 17 Feb 2022 16:20:54 +0100 Subject: [PATCH 05/21] Refactor fromtypes to reducetypes and fix int16/float16 --- rbc/omniscidb.py | 4 ++-- rbc/tests/test_typesystem.py | 15 +++++++++------ rbc/typesystem.py | 21 +++++++++++++++++++-- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/rbc/omniscidb.py b/rbc/omniscidb.py index fe1869d02..fff3876ea 100644 --- a/rbc/omniscidb.py +++ b/rbc/omniscidb.py @@ -1292,9 +1292,9 @@ def get_types(self, *values): types = [] for value in values: - if isinstance(value, list): + if isinstance(value, (list, numpy.ndarray)): items_types = set(map(typesystem.Type.fromvalue, value)) - com_type = typesystem.Type.fromtypes(items_types) + com_type = typesystem.Type.reducetypes(items_types) array_type = OmnisciArrayType((com_type,)) types.append(array_type) elif isinstance(value, RemoteCallCapsule): diff --git a/rbc/tests/test_typesystem.py b/rbc/tests/test_typesystem.py index 6b19ba47f..f86c375ef 100644 --- a/rbc/tests/test_typesystem.py +++ b/rbc/tests/test_typesystem.py @@ -87,21 +87,24 @@ def test_fromtypes(): def from_list(values): return set(map(Type.fromvalue, values)) - assert Type.fromtypes(from_list([1, 3., 6])) == Type('float64') - assert Type.fromtypes(from_list([1, 3, 6])) == Type('int64') + assert Type.reducetypes(from_list([1, 3., 6])) == Type('float64') + assert Type.reducetypes(from_list([1, 3, 6])) == Type('int64') types = from_list([np.int8(1), np.int8(3), np.int8(6)]) - assert Type.fromtypes(types) == Type('int8') + assert Type.reducetypes(types) == Type('int8') types = from_list([np.int8(1), np.int32(3), np.int16(6)]) - assert Type.fromtypes(types) == Type('int32') + assert Type.reducetypes(types) == Type('int32') types = from_list([np.uint8(1), np.uint32(3), np.int8(6)]) - assert Type.fromtypes(types) == Type('int32') + assert Type.reducetypes(types) == Type('int32') + + types = from_list([np.int16(1), np.float16(3), np.int8(6)]) + assert Type.reducetypes(types) == Type('float32') msg = "Failed to cast" with pytest.raises(TypeParseError, match=msg): - Type.fromtypes(from_list([1, 'a', 6])) + Type.reducetypes(from_list([1, 'a', 6])) def test_fromstring(target_info): diff --git a/rbc/typesystem.py b/rbc/typesystem.py index 109c046b9..c17042988 100644 --- a/rbc/typesystem.py +++ b/rbc/typesystem.py @@ -1227,8 +1227,23 @@ def fromobject(cls, obj): raise NotImplementedError(repr((type(obj)))) @classmethod - def fromtypes(cls, t): - """Consolidate a list of types into a single type.""" + def reducetypes(cls, t, *, method='highest'): + """Reduce a list of types into a single type. + + Parameters + ---------- + t : list(Type) + List of types to reduce to a type. + method : {common} + Reduction method. Can be: + + * 'common': reduce to the highest common denominator. + + Returns + ------- + t : Type + New Type instance. + """ has_uint, has_int, has_float, has_other = True, False, False, False bit_int, bit_float = 8, 8 @@ -1254,6 +1269,8 @@ def fromtypes(cls, t): if has_uint: com_type = 'u' + com_type else: + if bit_float == 16 and bit_int >= 16: + bit_float = 32 com_type = f'float{bit_float}' t = [Type.fromstring(com_type)] From bab148d60c32dc669f23588db0937df02f4a0866 Mon Sep 17 00:00:00 2001 From: Pamphile Roy Date: Thu, 17 Feb 2022 16:21:08 +0100 Subject: [PATCH 06/21] Parametrize test with dtype --- rbc/tests/test_omnisci.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/rbc/tests/test_omnisci.py b/rbc/tests/test_omnisci.py index f01bfff25..e6ae173dd 100644 --- a/rbc/tests/test_omnisci.py +++ b/rbc/tests/test_omnisci.py @@ -77,15 +77,18 @@ def farhenheit2celcius(f): assert_equal(farhenheit2celcius(40).execute(), np.float32(40 / 9)) -def test_direct_call_array(omnisci): +@pytest.mark.parametrize('dtype', ('float32', 'float64', 'int32', 'int64')) +def test_direct_call_array(omnisci, dtype): from rbc.omnisci_backend import mean omnisci.reset() - @omnisci('double(double[])') + @omnisci('T(T[])', T=['float32', 'float64', 'int64', 'int32'], devices=['cpu']) def farhenheit2celcius(f): return (mean(f)-32) * 5 / 9 - assert_equal(farhenheit2celcius([30.0, 50.0]).execute(), np.float32(40 / 9)) + ref_value = np.dtype(dtype).type(40 / 9) + inp_array = np.array([30, 50], dtype=dtype) + assert farhenheit2celcius(inp_array).execute() == pytest.approx(ref_value) def test_local_caller(omnisci): From 526643f150f65386621c4172997606ccc8f82258 Mon Sep 17 00:00:00 2001 From: Pamphile Roy Date: Thu, 17 Feb 2022 16:48:03 +0100 Subject: [PATCH 07/21] Fix error with ints --- rbc/tests/test_omnisci.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rbc/tests/test_omnisci.py b/rbc/tests/test_omnisci.py index e6ae173dd..534087398 100644 --- a/rbc/tests/test_omnisci.py +++ b/rbc/tests/test_omnisci.py @@ -81,6 +81,7 @@ def farhenheit2celcius(f): def test_direct_call_array(omnisci, dtype): from rbc.omnisci_backend import mean omnisci.reset() + omnisci.unregister() @omnisci('T(T[])', T=['float32', 'float64', 'int64', 'int32'], devices=['cpu']) def farhenheit2celcius(f): From fb6252f93067bff073502401d7c08c28abac921c Mon Sep 17 00:00:00 2001 From: Pamphile Roy Date: Thu, 17 Feb 2022 17:34:15 +0100 Subject: [PATCH 08/21] Unregister in tests --- rbc/tests/test_omnisci.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rbc/tests/test_omnisci.py b/rbc/tests/test_omnisci.py index 534087398..d737b8ab3 100644 --- a/rbc/tests/test_omnisci.py +++ b/rbc/tests/test_omnisci.py @@ -68,7 +68,7 @@ def omnisci(): def test_direct_call_scalar(omnisci): - omnisci.reset() + omnisci.unregister() @omnisci('double(double)') def farhenheit2celcius(f): @@ -80,7 +80,6 @@ def farhenheit2celcius(f): @pytest.mark.parametrize('dtype', ('float32', 'float64', 'int32', 'int64')) def test_direct_call_array(omnisci, dtype): from rbc.omnisci_backend import mean - omnisci.reset() omnisci.unregister() @omnisci('T(T[])', T=['float32', 'float64', 'int64', 'int32'], devices=['cpu']) From 7455b866ade2e720d44a947ecc5cf1b63ce2a3af Mon Sep 17 00:00:00 2001 From: Pamphile Roy Date: Thu, 17 Feb 2022 17:53:59 +0100 Subject: [PATCH 09/21] Disable fail-fast --- .github/workflows/rbc_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rbc_test.yml b/.github/workflows/rbc_test.yml index 5e05e741b..205491fea 100644 --- a/.github/workflows/rbc_test.yml +++ b/.github/workflows/rbc_test.yml @@ -105,7 +105,7 @@ jobs: runs-on: ${{ matrix.os }} timeout-minutes: 20 strategy: - fail-fast: true + fail-fast: false matrix: os: [ubuntu-latest] python-version: ['3.9', '3.8', '3.7'] From f0fdfd6398b2656cabb75c7d4abbe1133a7118da Mon Sep 17 00:00:00 2001 From: Pamphile Roy Date: Mon, 21 Feb 2022 10:38:57 +0100 Subject: [PATCH 10/21] Change function name for array tests --- rbc/tests/test_omnisci.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rbc/tests/test_omnisci.py b/rbc/tests/test_omnisci.py index d737b8ab3..4930f875f 100644 --- a/rbc/tests/test_omnisci.py +++ b/rbc/tests/test_omnisci.py @@ -83,12 +83,12 @@ def test_direct_call_array(omnisci, dtype): omnisci.unregister() @omnisci('T(T[])', T=['float32', 'float64', 'int64', 'int32'], devices=['cpu']) - def farhenheit2celcius(f): + def farhenheit2celcius_arr(f): return (mean(f)-32) * 5 / 9 ref_value = np.dtype(dtype).type(40 / 9) inp_array = np.array([30, 50], dtype=dtype) - assert farhenheit2celcius(inp_array).execute() == pytest.approx(ref_value) + assert farhenheit2celcius_arr(inp_array).execute() == pytest.approx(ref_value) def test_local_caller(omnisci): From 5360a55e84b761ac6961294a8114bb5bf2f88ebb Mon Sep 17 00:00:00 2001 From: Pamphile Roy Date: Thu, 24 Feb 2022 12:35:20 +0100 Subject: [PATCH 11/21] REVERT THIS COMMIT: Disable vectorization in tests --- .github/workflows/rbc_test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/rbc_test.yml b/.github/workflows/rbc_test.yml index 205491fea..39abf4806 100644 --- a/.github/workflows/rbc_test.yml +++ b/.github/workflows/rbc_test.yml @@ -7,6 +7,11 @@ on: branches: - master +env: + NUMBA_LOOP_VECTORIZE: 0 + NUMBA_SLP_VECTORIZE: 0 + + # kill any previous running job on a new commit concurrency: group: build-and-test-rbc-${{ github.head_ref }} From 993fb5c44839b198e336aa354d102a82afaf3f4e Mon Sep 17 00:00:00 2001 From: Pamphile Roy Date: Thu, 24 Feb 2022 13:59:41 +0100 Subject: [PATCH 12/21] Fix mean from array_api --- rbc/tests/test_omnisci.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rbc/tests/test_omnisci.py b/rbc/tests/test_omnisci.py index 4930f875f..0f43244c2 100644 --- a/rbc/tests/test_omnisci.py +++ b/rbc/tests/test_omnisci.py @@ -79,12 +79,12 @@ def farhenheit2celcius(f): @pytest.mark.parametrize('dtype', ('float32', 'float64', 'int32', 'int64')) def test_direct_call_array(omnisci, dtype): - from rbc.omnisci_backend import mean + from rbc.omnisci_backend import array_api omnisci.unregister() @omnisci('T(T[])', T=['float32', 'float64', 'int64', 'int32'], devices=['cpu']) def farhenheit2celcius_arr(f): - return (mean(f)-32) * 5 / 9 + return (array_api.mean(f)-32) * 5 / 9 ref_value = np.dtype(dtype).type(40 / 9) inp_array = np.array([30, 50], dtype=dtype) From 37d003c69d07988a8dcc74194049f5de835eb2a4 Mon Sep 17 00:00:00 2001 From: Pamphile Roy Date: Thu, 24 Feb 2022 18:55:42 +0100 Subject: [PATCH 13/21] Rename functions to less than 22 chars --- rbc/tests/test_omnisci.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rbc/tests/test_omnisci.py b/rbc/tests/test_omnisci.py index 0f43244c2..7b0d43fa3 100644 --- a/rbc/tests/test_omnisci.py +++ b/rbc/tests/test_omnisci.py @@ -83,12 +83,12 @@ def test_direct_call_array(omnisci, dtype): omnisci.unregister() @omnisci('T(T[])', T=['float32', 'float64', 'int64', 'int32'], devices=['cpu']) - def farhenheit2celcius_arr(f): + def func(f): return (array_api.mean(f)-32) * 5 / 9 ref_value = np.dtype(dtype).type(40 / 9) inp_array = np.array([30, 50], dtype=dtype) - assert farhenheit2celcius_arr(inp_array).execute() == pytest.approx(ref_value) + assert func(inp_array).execute() == pytest.approx(ref_value) def test_local_caller(omnisci): From 84bafcaaa0afb110ba1fa7434a073ad1e820f9bc Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Wed, 12 Jul 2023 21:35:41 -0300 Subject: [PATCH 14/21] Rename OmnisciArrayType -> HeavyDBArrayType --- rbc/heavydb/remoteheavydb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rbc/heavydb/remoteheavydb.py b/rbc/heavydb/remoteheavydb.py index 20265f41a..3670c78e1 100644 --- a/rbc/heavydb/remoteheavydb.py +++ b/rbc/heavydb/remoteheavydb.py @@ -1565,7 +1565,7 @@ def remote_call(self, func, ftype: typesystem.Type, arguments: tuple, hold=False elif isinstance(a, str): a = repr(a) args.append(f'{a}') - elif isinstance(atype, OmnisciArrayType): + elif isinstance(atype, HeavyDBArrayType): element_type_name = type_to_type_name(atype.element_type) astr = ", ".join([f'CAST({a_} AS {element_type_name})' for a_ in a]) args.append(f'ARRAY[{astr}]') From 9b5e5713ce7d9672c6b588508dc883d2be0f6a41 Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Wed, 12 Jul 2023 21:47:14 -0300 Subject: [PATCH 15/21] Test requires most recent version of HeavyDB --- rbc/tests/heavydb/test_heavydb.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rbc/tests/heavydb/test_heavydb.py b/rbc/tests/heavydb/test_heavydb.py index acba80d1f..7e42172ce 100644 --- a/rbc/tests/heavydb/test_heavydb.py +++ b/rbc/tests/heavydb/test_heavydb.py @@ -80,6 +80,9 @@ def farhenheit2celcius(f): @pytest.mark.parametrize('dtype', ('float32', 'float64', 'int32', 'int64')) def test_direct_call_array(heavydb, dtype): + if heavydb.version[:2] < (7, 0): + continue + heavydb.unregister() @heavydb('T(T[])', T=['float32', 'float64', 'int64', 'int32'], devices=['cpu']) From d3f94994d957f77cdd248552c597ee2ca4f992d9 Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Wed, 12 Jul 2023 21:47:42 -0300 Subject: [PATCH 16/21] add pytest.skip --- rbc/tests/heavydb/test_heavydb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rbc/tests/heavydb/test_heavydb.py b/rbc/tests/heavydb/test_heavydb.py index 7e42172ce..8f709cebb 100644 --- a/rbc/tests/heavydb/test_heavydb.py +++ b/rbc/tests/heavydb/test_heavydb.py @@ -81,7 +81,7 @@ def farhenheit2celcius(f): @pytest.mark.parametrize('dtype', ('float32', 'float64', 'int32', 'int64')) def test_direct_call_array(heavydb, dtype): if heavydb.version[:2] < (7, 0): - continue + pytest.skip('Test requires HeavyDB 7.0 or newer') heavydb.unregister() From a4a0389d06413a2902c9d597a27485ee859d8d26 Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Wed, 12 Jul 2023 21:51:04 -0300 Subject: [PATCH 17/21] OmnisciArrayType -> HeavyDBArrayType --- rbc/heavydb/remoteheavydb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rbc/heavydb/remoteheavydb.py b/rbc/heavydb/remoteheavydb.py index 3670c78e1..d78fe6cec 100644 --- a/rbc/heavydb/remoteheavydb.py +++ b/rbc/heavydb/remoteheavydb.py @@ -1428,7 +1428,7 @@ def get_types(self, *values): if isinstance(value, (list, numpy.ndarray)): items_types = set(map(typesystem.Type.fromvalue, value)) com_type = typesystem.Type.reducetypes(items_types) - array_type = OmnisciArrayType((com_type,)) + array_type = HeavyDBArrayType((com_type,)) types.append(array_type) elif isinstance(value, RemoteCallCapsule): typ = value.__typesystem_type__ From d879ab08434ab8733a2b496e44fa94963b17258b Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Wed, 12 Jul 2023 22:01:01 -0300 Subject: [PATCH 18/21] add remote list evaluation test --- rbc/tests/heavydb/test_caller.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/rbc/tests/heavydb/test_caller.py b/rbc/tests/heavydb/test_caller.py index 814624b5e..335e1e558 100644 --- a/rbc/tests/heavydb/test_caller.py +++ b/rbc/tests/heavydb/test_caller.py @@ -47,6 +47,10 @@ def myupper(s): r[i] = c return r + @heavydb('int64(T[])', T=['int64', 'double'], devices=['CPU']) + def mylength(lst): + return len(lst) + def test_udf_string_repr(heavydb): myincr = heavydb.get_caller('myincr') @@ -119,6 +123,17 @@ def test_remote_TextEncodingNone_evaluation(heavydb): assert str(myupper(b"abc").execute()) == 'ABC' +def test_remote_list_evaluation(heavydb): + mylength = heavydb.get_caller('mylength') + assert str(mylength) == "mylength['int64(T[]), T=int64|double, device=CPU']" + assert str(mylength([1, 2])) == \ + "SELECT mylength(ARRAY[CAST(1 AS BIGINT), CAST(2 AS BIGINT)])" + assert mylength([1, 2]).execute() == 2 + assert str(mylength([1.0, 2])) == \ + "SELECT mylength(ARRAY[CAST(1.0 AS DOUBLE), CAST(2 AS DOUBLE)])" + assert mylength([1.0, 2]).execute() == 2 + + def test_remote_composite_udf_evaluation(heavydb): myincr = heavydb.get_caller('myincr') From dc693362c70cedfba288b6e1dfe6ac1f65cbe53f Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Wed, 12 Jul 2023 22:20:26 -0300 Subject: [PATCH 19/21] skip test when HeavyDB version < 7.0 --- rbc/tests/heavydb/test_caller.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rbc/tests/heavydb/test_caller.py b/rbc/tests/heavydb/test_caller.py index 335e1e558..e5419f05d 100644 --- a/rbc/tests/heavydb/test_caller.py +++ b/rbc/tests/heavydb/test_caller.py @@ -124,6 +124,9 @@ def test_remote_TextEncodingNone_evaluation(heavydb): def test_remote_list_evaluation(heavydb): + if heavydb.version[:2] < (7, 0): + pytest.skip('Test requires HeavyDB 7.0 or newer') + mylength = heavydb.get_caller('mylength') assert str(mylength) == "mylength['int64(T[]), T=int64|double, device=CPU']" assert str(mylength([1, 2])) == \ From 857db58f642c5957b270d69d55bbec8ed358fe55 Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Fri, 14 Jul 2023 12:27:17 -0300 Subject: [PATCH 20/21] add test for ndarray and TextEncodingNone --- rbc/tests/heavydb/test_caller.py | 33 ++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/rbc/tests/heavydb/test_caller.py b/rbc/tests/heavydb/test_caller.py index e5419f05d..e4ef147e2 100644 --- a/rbc/tests/heavydb/test_caller.py +++ b/rbc/tests/heavydb/test_caller.py @@ -51,6 +51,10 @@ def myupper(s): def mylength(lst): return len(lst) + @heavydb('T(T[])', T=['TextEncodingNone'], devices=['CPU']) + def myfirst(lst): + return lst[0] + def test_udf_string_repr(heavydb): myincr = heavydb.get_caller('myincr') @@ -137,6 +141,35 @@ def test_remote_list_evaluation(heavydb): assert mylength([1.0, 2]).execute() == 2 +def test_remote_ndarray_evaluation(heavydb): + if heavydb.version[:2] < (7, 0): + pytest.skip('Test requires HeavyDB 7.0 or newer') + + arr = np.asarray([1, 2]) + farr = np.asarray([1, 2], dtype=np.float64) + + mylength = heavydb.get_caller('mylength') + assert str(mylength) == "mylength['int64(T[]), T=int64|double, device=CPU']" + assert str(mylength(arr)) == \ + "SELECT mylength(ARRAY[CAST(1 AS BIGINT), CAST(2 AS BIGINT)])" + assert mylength(arr).execute() == 2 + assert str(mylength(farr)) == \ + "SELECT mylength(ARRAY[CAST(1.0 AS DOUBLE), CAST(2.0 AS DOUBLE)])" + assert mylength(farr).execute() == 2 + + +def test_remote_text_list_evaluation(heavydb): + if heavydb.version[:2] < (7, 0): + pytest.skip('Test requires HeavyDB 7.0 or newer') + pytest.xfail('Array is not supported') + + myfirst = heavydb.get_caller('myfirst') + assert str(myfirst) == "myfirst['T(T[]), T=TextEncodingNone, device=CPU']" + assert str(myfirst(['one', 'two'])) == \ + "SELECT myfirst(ARRAY['one', 'two'])" + assert myfirst(['one', 'two']).execute() == 'one' + + def test_remote_composite_udf_evaluation(heavydb): myincr = heavydb.get_caller('myincr') From 4d4ffce799111f354616f1b3631aed03f989e785 Mon Sep 17 00:00:00 2001 From: Guilherme Leobas Date: Fri, 14 Jul 2023 17:10:19 -0300 Subject: [PATCH 21/21] Revert "add test for ndarray and TextEncodingNone" This reverts commit 857db58f642c5957b270d69d55bbec8ed358fe55. --- rbc/tests/heavydb/test_caller.py | 33 -------------------------------- 1 file changed, 33 deletions(-) diff --git a/rbc/tests/heavydb/test_caller.py b/rbc/tests/heavydb/test_caller.py index e4ef147e2..e5419f05d 100644 --- a/rbc/tests/heavydb/test_caller.py +++ b/rbc/tests/heavydb/test_caller.py @@ -51,10 +51,6 @@ def myupper(s): def mylength(lst): return len(lst) - @heavydb('T(T[])', T=['TextEncodingNone'], devices=['CPU']) - def myfirst(lst): - return lst[0] - def test_udf_string_repr(heavydb): myincr = heavydb.get_caller('myincr') @@ -141,35 +137,6 @@ def test_remote_list_evaluation(heavydb): assert mylength([1.0, 2]).execute() == 2 -def test_remote_ndarray_evaluation(heavydb): - if heavydb.version[:2] < (7, 0): - pytest.skip('Test requires HeavyDB 7.0 or newer') - - arr = np.asarray([1, 2]) - farr = np.asarray([1, 2], dtype=np.float64) - - mylength = heavydb.get_caller('mylength') - assert str(mylength) == "mylength['int64(T[]), T=int64|double, device=CPU']" - assert str(mylength(arr)) == \ - "SELECT mylength(ARRAY[CAST(1 AS BIGINT), CAST(2 AS BIGINT)])" - assert mylength(arr).execute() == 2 - assert str(mylength(farr)) == \ - "SELECT mylength(ARRAY[CAST(1.0 AS DOUBLE), CAST(2.0 AS DOUBLE)])" - assert mylength(farr).execute() == 2 - - -def test_remote_text_list_evaluation(heavydb): - if heavydb.version[:2] < (7, 0): - pytest.skip('Test requires HeavyDB 7.0 or newer') - pytest.xfail('Array is not supported') - - myfirst = heavydb.get_caller('myfirst') - assert str(myfirst) == "myfirst['T(T[]), T=TextEncodingNone, device=CPU']" - assert str(myfirst(['one', 'two'])) == \ - "SELECT myfirst(ARRAY['one', 'two'])" - assert myfirst(['one', 'two']).execute() == 'one' - - def test_remote_composite_udf_evaluation(heavydb): myincr = heavydb.get_caller('myincr')