From 08cc8ed87fa77b904a4c44e659681f6608925315 Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Tue, 3 Jan 2023 10:12:52 +0300 Subject: [PATCH 01/21] ENH: introduce scalar type hierarchy Try to duck-type NumPy, only without array scalars, so that * dtype('float32').type --> np.float32 and * np.float32(3) --> array(3.0, dtype='float32) IOW, the attempt is to only have zero-dim arrays (we have them anyway) and have them everywhere where NumPy creates a scalar. --- torch_np/__init__.py | 1 + torch_np/_dtypes.py | 26 +++----- torch_np/_scalar_types.py | 137 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 148 insertions(+), 16 deletions(-) create mode 100644 torch_np/_scalar_types.py diff --git a/torch_np/__init__.py b/torch_np/__init__.py index c581279c..38712ea3 100644 --- a/torch_np/__init__.py +++ b/torch_np/__init__.py @@ -1,4 +1,5 @@ from ._dtypes import * +from ._scalar_types import * from ._wrapper import * from . import testing diff --git a/torch_np/_dtypes.py b/torch_np/_dtypes.py index 8d4176d6..658b76b0 100644 --- a/torch_np/_dtypes.py +++ b/torch_np/_dtypes.py @@ -8,6 +8,8 @@ import builtins import torch +from . import _scalar_types + # Define analogs of numpy dtypes supported by pytorch. class dtype: @@ -22,6 +24,9 @@ def __init__(self, name): _name = typecode_chars_dict[name] elif name in dt_aliases_dict: _name = dt_aliases_dict[name] + # the check must come last, so that 'name' is not a string + elif issubclass(name, _scalar_types.generic): + _name = name.name else: raise TypeError(f"data type '{name}' not understood") self._name = _name @@ -30,6 +35,10 @@ def __init__(self, name): def name(self): return self._name + @property + def type(self): + return _scalar_types._typemap[self._name] + @property def typecode(self): return _typecodes_from_dtype_dict[self._name] @@ -104,21 +113,6 @@ def __repr__(self): } -float16 = dtype("float16") -float32 = dtype("float32") -float64 = dtype("float64") -complex64 = dtype("complex64") -complex128 = dtype("complex128") -uint8 = dtype("uint8") -int8 = dtype("int8") -int16 = dtype("int16") -int32 = dtype("int32") -int64 = dtype("int64") -bool = dtype("bool") - -intp = int64 # XXX -int_ = int64 - # Map the torch-suppored subset dtypes to local analogs # "quantized" types not available in numpy, skip _dtype_from_torch_dict = { @@ -217,5 +211,5 @@ def is_integer(dtyp): ########################## end autogenerated part -__all__ = ['dtype_from_torch', 'dtype', 'typecodes'] + dt_names + ['intp', 'int_'] +__all__ = ['dtype_from_torch', 'dtype', 'typecodes'] diff --git a/torch_np/_scalar_types.py b/torch_np/_scalar_types.py new file mode 100644 index 00000000..700638e9 --- /dev/null +++ b/torch_np/_scalar_types.py @@ -0,0 +1,137 @@ +"""Replicate the NumPy scalar type hierarchy +""" + +import abc +import torch + +class generic(abc.ABC): + @property + @abc.abstractmethod + def name(self): + return self.__class__.__name__ + + def __new__(self, value): + # + # Yes, a call to np.float32(4) produces a zero-dim array. + # + from . import _dtypes + from . import _ndarray + + torch_dtype = _dtypes.torch_dtype_from(self.name) + tensor = torch.as_tensor(value, dtype=torch_dtype) + return _ndarray.ndarray._from_tensor_and_base(tensor, None) + + +##### these are abstract types + +class number(generic): + pass + + +class integer(generic): + pass + + +class inexact(generic): + pass + + +class signedinteger(generic): + pass + + +class unsignedinteger(generic): + pass + + +class inexact(generic): + pass + + +class floating(generic): + pass + + +class complexfloating(generic): + pass + + +# ##### concrete types + +# signed integers + +class int8(signedinteger): + name = 'int8' + + +class int16(signedinteger): + name = 'int16' + + +class int32(signedinteger): + name = 'int32' + + +class int64(signedinteger): + name = 'int64' + + +# unsigned integers + +class uint8(unsignedinteger): + name = 'uint8' + + +# floating point + +class float16(floating): + name = 'float16' + + +class float32(floating): + name = 'float32' + + +class float64(floating): + name = 'float64' + + +class complex64(complexfloating): + name = 'complex64' + + +class complex128(complexfloating): + name = 'complex128' + + +class bool_(generic): + name = 'bool' + + +# name aliases : FIXME (OS, bitness) +intp = int64 +int_ = int64 + + +_typemap ={ + 'int8' : int8, + 'int16' : int16, + 'int32' : int32, + 'int64' : int64, + 'uint8' : uint8, + 'float16': float16, + 'float32': float32, + 'float64': float64, + 'complex64': complex64, + 'complex128': complex128, + 'bool': bool_ +} + + +__all__ = list(_typemap.keys()) +__all__.remove('bool') + +__all__ += ['bool_', 'intp', 'int_'] +__all__ += ['generic', 'number', + 'signedinteger', 'unsignedinteger', + 'inexact', 'floating', 'complexfloating'] From fd4a4d9286ae1348a69f8a3407a69231a61461bf Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Tue, 3 Jan 2023 15:48:54 +0300 Subject: [PATCH 02/21] TST: undo (some) test skips of test with scalars --- torch_np/_helpers.py | 6 ++++++ torch_np/_wrapper.py | 3 ++- torch_np/tests/test_function_base.py | 5 ++--- torch_np/tests/test_shape_base.py | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/torch_np/_helpers.py b/torch_np/_helpers.py index 7016e4e0..00b8faa7 100644 --- a/torch_np/_helpers.py +++ b/torch_np/_helpers.py @@ -125,3 +125,9 @@ def allow_only_single_axis(axis): if len(axis) != 1: raise NotImplementedError("does not handle tuple axis") return axis[0] + + +def to_tensors(*inputs): + """Convert all ndarrays from `inputs` to tensors.""" + return tuple([value.get() if isinstance(value, ndarray) else value + for value in inputs]) diff --git a/torch_np/_wrapper.py b/torch_np/_wrapper.py index f71c2d63..639f2d01 100644 --- a/torch_np/_wrapper.py +++ b/torch_np/_wrapper.py @@ -193,8 +193,9 @@ def arange(start=None, stop=None, step=1, dtype=None, *, like=None): if dtype is None: dtype = _dtypes.default_int_type() dtype = result_type(start, stop, step, dtype) - torch_dtype = _dtypes.torch_dtype_from(dtype) + start, stop, step = _helpers.to_tensors(start, stop, step) + try: return asarray(torch.arange(start, stop, step, dtype=torch_dtype)) except RuntimeError: diff --git a/torch_np/tests/test_function_base.py b/torch_np/tests/test_function_base.py index 3d35ab0a..79aece1b 100644 --- a/torch_np/tests/test_function_base.py +++ b/torch_np/tests/test_function_base.py @@ -96,16 +96,15 @@ def test_arange_booleans(self): with pytest.raises(TypeError): np.arange(3, dtype="bool") - @pytest.mark.skip(reason='XXX: python scalars from array scalars') @pytest.mark.parametrize("which", [0, 1, 2]) def test_error_paths_and_promotion(self, which): - args = [0, 1, 2] # start, stop, and step + args = [0, 10, 2] # start, stop, and step args[which] = np.float64(2.) # should ensure float64 output assert np.arange(*args).dtype == np.float64 # Cover stranger error path, test only to achieve code coverage! args[which] = [None, []] - with pytest.raises(ValueError): + with pytest.raises((ValueError, RuntimeError)): # Fails discovering start dtype np.arange(*args) diff --git a/torch_np/tests/test_shape_base.py b/torch_np/tests/test_shape_base.py index ceb7c724..b15eed93 100644 --- a/torch_np/tests/test_shape_base.py +++ b/torch_np/tests/test_shape_base.py @@ -464,7 +464,7 @@ def test_stack(): # 0d input for input_ in [(1, 2, 3), -### [np.int32(1), np.int32(2), np.int32(3)], # XXX: numpy scalars? + [np.int32(1), np.int32(2), np.int32(3)], [np.array(1), np.array(2), np.array(3)]]: assert_array_equal(stack(input_), [1, 2, 3]) # 1d input examples From 5b4d716c4ac50dd26c20e531e50ff4354088ee66 Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Tue, 3 Jan 2023 17:43:43 +0300 Subject: [PATCH 03/21] ENH: add np.issubdtype checker to mimic numpy --- torch_np/_dtypes.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/torch_np/_dtypes.py b/torch_np/_dtypes.py index 658b76b0..1bca51da 100644 --- a/torch_np/_dtypes.py +++ b/torch_np/_dtypes.py @@ -10,6 +10,10 @@ from . import _scalar_types + +__all__ = ['dtype_from_torch', 'dtype', 'typecodes', 'issubdtype'] + + # Define analogs of numpy dtypes supported by pytorch. class dtype: @@ -177,6 +181,23 @@ def is_integer(dtyp): return dtyp.typecode in typecodes['AllInteger'] + +def issubclass_(arg, klass): + try: + return issubclass(arg, klass) + except TypeError: + return False + + +def issubdtype(arg1, arg2): + # cf https://github.com/numpy/numpy/blob/v1.24.0/numpy/core/numerictypes.py#L356-L420 + if not issubclass_(arg1, _scalar_types.generic): + arg1 = dtype(arg1).type + if not issubclass_(arg2, _scalar_types.generic): + arg2 = dtype(arg2).type + return issubclass(arg1, arg2) + + # The casting below is defined *with dtypes only*, so no value-based casting! # These two dicts are autogenerated with autogen/gen_dtypes.py, @@ -210,6 +231,3 @@ def is_integer(dtyp): ########################## end autogenerated part - -__all__ = ['dtype_from_torch', 'dtype', 'typecodes'] - From c5f4949106cffb6111206cbfd8c77e041651b823 Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Tue, 3 Jan 2023 10:12:52 +0300 Subject: [PATCH 04/21] ENH: introduce scalar type hierarchy Try to duck-type NumPy, only without array scalars, so that * dtype('float32').type --> np.float32 and * np.float32(3) --> array(3.0, dtype='float32) IOW, the attempt is to only have zero-dim arrays (we have them anyway) and have them everywhere where NumPy creates a scalar. --- torch_np/__init__.py | 1 + torch_np/_dtypes.py | 26 +++----- torch_np/_scalar_types.py | 137 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 148 insertions(+), 16 deletions(-) create mode 100644 torch_np/_scalar_types.py diff --git a/torch_np/__init__.py b/torch_np/__init__.py index c581279c..38712ea3 100644 --- a/torch_np/__init__.py +++ b/torch_np/__init__.py @@ -1,4 +1,5 @@ from ._dtypes import * +from ._scalar_types import * from ._wrapper import * from . import testing diff --git a/torch_np/_dtypes.py b/torch_np/_dtypes.py index 8d4176d6..658b76b0 100644 --- a/torch_np/_dtypes.py +++ b/torch_np/_dtypes.py @@ -8,6 +8,8 @@ import builtins import torch +from . import _scalar_types + # Define analogs of numpy dtypes supported by pytorch. class dtype: @@ -22,6 +24,9 @@ def __init__(self, name): _name = typecode_chars_dict[name] elif name in dt_aliases_dict: _name = dt_aliases_dict[name] + # the check must come last, so that 'name' is not a string + elif issubclass(name, _scalar_types.generic): + _name = name.name else: raise TypeError(f"data type '{name}' not understood") self._name = _name @@ -30,6 +35,10 @@ def __init__(self, name): def name(self): return self._name + @property + def type(self): + return _scalar_types._typemap[self._name] + @property def typecode(self): return _typecodes_from_dtype_dict[self._name] @@ -104,21 +113,6 @@ def __repr__(self): } -float16 = dtype("float16") -float32 = dtype("float32") -float64 = dtype("float64") -complex64 = dtype("complex64") -complex128 = dtype("complex128") -uint8 = dtype("uint8") -int8 = dtype("int8") -int16 = dtype("int16") -int32 = dtype("int32") -int64 = dtype("int64") -bool = dtype("bool") - -intp = int64 # XXX -int_ = int64 - # Map the torch-suppored subset dtypes to local analogs # "quantized" types not available in numpy, skip _dtype_from_torch_dict = { @@ -217,5 +211,5 @@ def is_integer(dtyp): ########################## end autogenerated part -__all__ = ['dtype_from_torch', 'dtype', 'typecodes'] + dt_names + ['intp', 'int_'] +__all__ = ['dtype_from_torch', 'dtype', 'typecodes'] diff --git a/torch_np/_scalar_types.py b/torch_np/_scalar_types.py new file mode 100644 index 00000000..700638e9 --- /dev/null +++ b/torch_np/_scalar_types.py @@ -0,0 +1,137 @@ +"""Replicate the NumPy scalar type hierarchy +""" + +import abc +import torch + +class generic(abc.ABC): + @property + @abc.abstractmethod + def name(self): + return self.__class__.__name__ + + def __new__(self, value): + # + # Yes, a call to np.float32(4) produces a zero-dim array. + # + from . import _dtypes + from . import _ndarray + + torch_dtype = _dtypes.torch_dtype_from(self.name) + tensor = torch.as_tensor(value, dtype=torch_dtype) + return _ndarray.ndarray._from_tensor_and_base(tensor, None) + + +##### these are abstract types + +class number(generic): + pass + + +class integer(generic): + pass + + +class inexact(generic): + pass + + +class signedinteger(generic): + pass + + +class unsignedinteger(generic): + pass + + +class inexact(generic): + pass + + +class floating(generic): + pass + + +class complexfloating(generic): + pass + + +# ##### concrete types + +# signed integers + +class int8(signedinteger): + name = 'int8' + + +class int16(signedinteger): + name = 'int16' + + +class int32(signedinteger): + name = 'int32' + + +class int64(signedinteger): + name = 'int64' + + +# unsigned integers + +class uint8(unsignedinteger): + name = 'uint8' + + +# floating point + +class float16(floating): + name = 'float16' + + +class float32(floating): + name = 'float32' + + +class float64(floating): + name = 'float64' + + +class complex64(complexfloating): + name = 'complex64' + + +class complex128(complexfloating): + name = 'complex128' + + +class bool_(generic): + name = 'bool' + + +# name aliases : FIXME (OS, bitness) +intp = int64 +int_ = int64 + + +_typemap ={ + 'int8' : int8, + 'int16' : int16, + 'int32' : int32, + 'int64' : int64, + 'uint8' : uint8, + 'float16': float16, + 'float32': float32, + 'float64': float64, + 'complex64': complex64, + 'complex128': complex128, + 'bool': bool_ +} + + +__all__ = list(_typemap.keys()) +__all__.remove('bool') + +__all__ += ['bool_', 'intp', 'int_'] +__all__ += ['generic', 'number', + 'signedinteger', 'unsignedinteger', + 'inexact', 'floating', 'complexfloating'] From e1fa959c497d3ec73aeaf4b4d7f57daed92ecda8 Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Tue, 3 Jan 2023 15:48:54 +0300 Subject: [PATCH 05/21] TST: undo (some) test skips of test with scalars --- torch_np/_helpers.py | 6 ++++++ torch_np/_wrapper.py | 3 ++- torch_np/tests/test_function_base.py | 5 ++--- torch_np/tests/test_shape_base.py | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/torch_np/_helpers.py b/torch_np/_helpers.py index 7016e4e0..00b8faa7 100644 --- a/torch_np/_helpers.py +++ b/torch_np/_helpers.py @@ -125,3 +125,9 @@ def allow_only_single_axis(axis): if len(axis) != 1: raise NotImplementedError("does not handle tuple axis") return axis[0] + + +def to_tensors(*inputs): + """Convert all ndarrays from `inputs` to tensors.""" + return tuple([value.get() if isinstance(value, ndarray) else value + for value in inputs]) diff --git a/torch_np/_wrapper.py b/torch_np/_wrapper.py index f71c2d63..639f2d01 100644 --- a/torch_np/_wrapper.py +++ b/torch_np/_wrapper.py @@ -193,8 +193,9 @@ def arange(start=None, stop=None, step=1, dtype=None, *, like=None): if dtype is None: dtype = _dtypes.default_int_type() dtype = result_type(start, stop, step, dtype) - torch_dtype = _dtypes.torch_dtype_from(dtype) + start, stop, step = _helpers.to_tensors(start, stop, step) + try: return asarray(torch.arange(start, stop, step, dtype=torch_dtype)) except RuntimeError: diff --git a/torch_np/tests/test_function_base.py b/torch_np/tests/test_function_base.py index 3d35ab0a..79aece1b 100644 --- a/torch_np/tests/test_function_base.py +++ b/torch_np/tests/test_function_base.py @@ -96,16 +96,15 @@ def test_arange_booleans(self): with pytest.raises(TypeError): np.arange(3, dtype="bool") - @pytest.mark.skip(reason='XXX: python scalars from array scalars') @pytest.mark.parametrize("which", [0, 1, 2]) def test_error_paths_and_promotion(self, which): - args = [0, 1, 2] # start, stop, and step + args = [0, 10, 2] # start, stop, and step args[which] = np.float64(2.) # should ensure float64 output assert np.arange(*args).dtype == np.float64 # Cover stranger error path, test only to achieve code coverage! args[which] = [None, []] - with pytest.raises(ValueError): + with pytest.raises((ValueError, RuntimeError)): # Fails discovering start dtype np.arange(*args) diff --git a/torch_np/tests/test_shape_base.py b/torch_np/tests/test_shape_base.py index ceb7c724..b15eed93 100644 --- a/torch_np/tests/test_shape_base.py +++ b/torch_np/tests/test_shape_base.py @@ -464,7 +464,7 @@ def test_stack(): # 0d input for input_ in [(1, 2, 3), -### [np.int32(1), np.int32(2), np.int32(3)], # XXX: numpy scalars? + [np.int32(1), np.int32(2), np.int32(3)], [np.array(1), np.array(2), np.array(3)]]: assert_array_equal(stack(input_), [1, 2, 3]) # 1d input examples From 5c2e6f9b85f69fddf332eeca7ddba083ba4795e7 Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Tue, 3 Jan 2023 17:43:43 +0300 Subject: [PATCH 06/21] ENH: add np.issubdtype checker to mimic numpy --- torch_np/_dtypes.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/torch_np/_dtypes.py b/torch_np/_dtypes.py index 658b76b0..1bca51da 100644 --- a/torch_np/_dtypes.py +++ b/torch_np/_dtypes.py @@ -10,6 +10,10 @@ from . import _scalar_types + +__all__ = ['dtype_from_torch', 'dtype', 'typecodes', 'issubdtype'] + + # Define analogs of numpy dtypes supported by pytorch. class dtype: @@ -177,6 +181,23 @@ def is_integer(dtyp): return dtyp.typecode in typecodes['AllInteger'] + +def issubclass_(arg, klass): + try: + return issubclass(arg, klass) + except TypeError: + return False + + +def issubdtype(arg1, arg2): + # cf https://github.com/numpy/numpy/blob/v1.24.0/numpy/core/numerictypes.py#L356-L420 + if not issubclass_(arg1, _scalar_types.generic): + arg1 = dtype(arg1).type + if not issubclass_(arg2, _scalar_types.generic): + arg2 = dtype(arg2).type + return issubclass(arg1, arg2) + + # The casting below is defined *with dtypes only*, so no value-based casting! # These two dicts are autogenerated with autogen/gen_dtypes.py, @@ -210,6 +231,3 @@ def is_integer(dtyp): ########################## end autogenerated part - -__all__ = ['dtype_from_torch', 'dtype', 'typecodes'] - From 7622143801e7f4defed2353fa2be6b9b9d35477f Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Wed, 4 Jan 2023 12:06:25 +0300 Subject: [PATCH 07/21] MAINT: adapt assert_equal, assert_array_equal So that current tests pass. This involves fixing a slew of small issues all around but hey. --- .gitignore | 1 + torch_np/__init__.py | 3 ++- torch_np/_ndarray.py | 12 +++++++++++- torch_np/_scalar_types.py | 15 ++++++++++++++- torch_np/_wrapper.py | 7 +++++++ torch_np/testing/__init__.py | 1 - torch_np/tests/test_ndarray_methods.py | 3 ++- 7 files changed, 37 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index e94b8427..cfd46377 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,6 @@ autogen/__pycache__ torch_np/__pycache__/* torch_np/tests/__pycache__/* torch_np/tests/numpy_tests/core/__pycache__/* +torch_np/testing/__pycache__/* .coverage diff --git a/torch_np/__init__.py b/torch_np/__init__.py index 38712ea3..6c4ce60a 100644 --- a/torch_np/__init__.py +++ b/torch_np/__init__.py @@ -1,7 +1,7 @@ from ._dtypes import * from ._scalar_types import * from ._wrapper import * -from . import testing +#from . import testing from ._unary_ufuncs import * from ._binary_ufuncs import * @@ -11,3 +11,4 @@ inf = float('inf') nan = float('nan') + diff --git a/torch_np/_ndarray.py b/torch_np/_ndarray.py index 4a3ba360..87a65719 100644 --- a/torch_np/_ndarray.py +++ b/torch_np/_ndarray.py @@ -162,6 +162,15 @@ def __truediv__(self, other): other_tensor = asarray(other).get() return asarray(self._tensor.__truediv__(other_tensor)) + def __or__(self, other): + other_tensor = asarray(other).get() + return asarray(self._tensor.__or__(other_tensor)) + + def __ior__(self, other): + other_tensor = asarray(other).get() + return asarray(self._tensor.__ior__(other_tensor)) + + def __invert__(self): return asarray(self._tensor.__invert__()) @@ -307,7 +316,8 @@ def sum(self, axis=None, dtype=None, out=None, keepdims=NoValue, ### indexing ### def __getitem__(self, *args, **kwds): - return ndarray._from_tensor_and_base(self._tensor.__getitem__(*args, **kwds), self) + t_args = _helpers.to_tensors(*args) + return ndarray._from_tensor_and_base(self._tensor.__getitem__(*t_args, **kwds), self) def __setitem__(self, index, value): value = asarray(value).get() diff --git a/torch_np/_scalar_types.py b/torch_np/_scalar_types.py index 700638e9..79455ef3 100644 --- a/torch_np/_scalar_types.py +++ b/torch_np/_scalar_types.py @@ -18,7 +18,20 @@ def __new__(self, value): from . import _ndarray torch_dtype = _dtypes.torch_dtype_from(self.name) - tensor = torch.as_tensor(value, dtype=torch_dtype) + if isinstance(value, _ndarray.ndarray): + tensor = value.get() + else: + tensor = torch.as_tensor(value, dtype=torch_dtype) + # + # With numpy: + # >>> a = np.ones(3) + # >>> np.float64(a) is a # True + # >>> np.float64(a[0]) is a[0] # False + # + # A reasonable assumption is that the second case is more common, + # and here we follow the second approach and create a new object + # *for all inputs*. + # return _ndarray.ndarray._from_tensor_and_base(tensor, None) diff --git a/torch_np/_wrapper.py b/torch_np/_wrapper.py index 639f2d01..9cabbba9 100644 --- a/torch_np/_wrapper.py +++ b/torch_np/_wrapper.py @@ -765,6 +765,13 @@ def i0(x): return torch.special.i0(x) +def isscalar(a): + # XXX: this is a stub + try: + arr = asarray(a) + return arr.size == 1 + except Exception: + return False ###### mapping from numpy API objects to wrappers from this module ###### diff --git a/torch_np/testing/__init__.py b/torch_np/testing/__init__.py index b9f7581e..45399ce2 100644 --- a/torch_np/testing/__init__.py +++ b/torch_np/testing/__init__.py @@ -1,6 +1,5 @@ from .utils import (assert_equal, assert_array_equal, assert_almost_equal, assert_warns, assert_) - from .testing import assert_allclose # FIXME diff --git a/torch_np/tests/test_ndarray_methods.py b/torch_np/tests/test_ndarray_methods.py index ec927ba9..7a880d8d 100644 --- a/torch_np/tests/test_ndarray_methods.py +++ b/torch_np/tests/test_ndarray_methods.py @@ -6,7 +6,8 @@ #import numpy as np import torch_np as np -assert_equal = np.testing.assert_equal +from torch_np.testing import assert_equal + class TestIndexing: From 0a391da695d6c643adae4a3c0adec80287e9bfea Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Wed, 4 Jan 2023 22:01:42 +0300 Subject: [PATCH 08/21] TST: fix test_scalar_ctors from numpy --- torch_np/_dtypes.py | 5 ++++- torch_np/_ndarray.py | 6 ++++++ torch_np/_scalar_types.py | 17 ++++++++++++++++- torch_np/testing/__init__.py | 1 + 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/torch_np/_dtypes.py b/torch_np/_dtypes.py index 1bca51da..67955b93 100644 --- a/torch_np/_dtypes.py +++ b/torch_np/_dtypes.py @@ -51,7 +51,10 @@ def __eq__(self, other): if isinstance(other, dtype): return self._name == other.name else: - other_instance = dtype(other) + try: + other_instance = dtype(other) + except TypeError: + return False return self._name == other_instance.name def __repr__(self): diff --git a/torch_np/_ndarray.py b/torch_np/_ndarray.py index 87a65719..9b399485 100644 --- a/torch_np/_ndarray.py +++ b/torch_np/_ndarray.py @@ -330,6 +330,8 @@ def asarray(a, dtype=None, order=None, *, like=None): raise NotImplementedError if isinstance(a, ndarray): + if dtype != a.dtype: + a = a.astype(dtype) return a if isinstance(a, (list, tuple)): @@ -366,6 +368,10 @@ def array(object, dtype=None, *, copy=True, order='K', subok=False, ndmin=0, if isinstance(object, ndarray): result = object._tensor + + if dtype != object.dtype: + torch_dtype = _dtypes.torch_dtype_from(dtype) + result = result.to(torch_dtype) else: torch_dtype = _dtypes.torch_dtype_from(dtype) result = torch.as_tensor(object, dtype=torch_dtype) diff --git a/torch_np/_scalar_types.py b/torch_np/_scalar_types.py index 79455ef3..193d57cb 100644 --- a/torch_np/_scalar_types.py +++ b/torch_np/_scalar_types.py @@ -124,6 +124,20 @@ class bool_(generic): # name aliases : FIXME (OS, bitness) intp = int64 int_ = int64 +intc = int32 + +byte = int8 +short = int16 +longlong = int64 # XXX: is this correct? + +ubyte = uint8 + +half = float16 +single = float32 +double = float64 + +csingle = complex64 +cdouble = complex128 _typemap ={ @@ -144,7 +158,8 @@ class bool_(generic): __all__ = list(_typemap.keys()) __all__.remove('bool') -__all__ += ['bool_', 'intp', 'int_'] +__all__ += ['bool_', 'intp', 'int_', 'intc', 'byte', 'short', 'longlong', 'ubyte', 'half', 'single', 'double', +'csingle', 'cdouble'] __all__ += ['generic', 'number', 'signedinteger', 'unsignedinteger', 'inexact', 'floating', 'complexfloating'] diff --git a/torch_np/testing/__init__.py b/torch_np/testing/__init__.py index 45399ce2..b9f7581e 100644 --- a/torch_np/testing/__init__.py +++ b/torch_np/testing/__init__.py @@ -1,5 +1,6 @@ from .utils import (assert_equal, assert_array_equal, assert_almost_equal, assert_warns, assert_) + from .testing import assert_allclose # FIXME From 1c8900e616b766b86fb934e4a10930b82eb548c9 Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Thu, 5 Jan 2023 10:47:38 +0300 Subject: [PATCH 09/21] MAINT: test_scalar_methods from numpy This is a stub, will need to decide and clearly articulate what exactly is a scalar. --- torch_np/_dtypes.py | 1 + torch_np/_ndarray.py | 9 +++++++++ torch_np/_scalar_types.py | 18 +++++++++++++++++- 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/torch_np/_dtypes.py b/torch_np/_dtypes.py index 67955b93..8b44201e 100644 --- a/torch_np/_dtypes.py +++ b/torch_np/_dtypes.py @@ -117,6 +117,7 @@ def __repr__(self): typecodes = {'All': 'efdFDBbhil?', 'AllFloat': 'efdFD', 'AllInteger': 'Bbhil', + 'Float': 'efd', } diff --git a/torch_np/_ndarray.py b/torch_np/_ndarray.py index 9b399485..51f0447e 100644 --- a/torch_np/_ndarray.py +++ b/torch_np/_ndarray.py @@ -131,6 +131,15 @@ def __hash__(self): def __float__(self): return float(self._tensor) + # XXX : are single-element ndarrays scalars? + def is_integer(self): + if self.shape == (): + if _dtypes.is_integer(self.dtype): + return True + return self._tensor.item().is_integer() + else: + return False + ### sequence ### def __len__(self): diff --git a/torch_np/_scalar_types.py b/torch_np/_scalar_types.py index 193d57cb..779ed6b4 100644 --- a/torch_np/_scalar_types.py +++ b/torch_np/_scalar_types.py @@ -18,6 +18,10 @@ def __new__(self, value): from . import _ndarray torch_dtype = _dtypes.torch_dtype_from(self.name) + + if isinstance(value, str) and value in ['inf', 'nan']: + value = {'inf': torch.inf, 'nan': torch.nan}[value] + if isinstance(value, _ndarray.ndarray): tensor = value.get() else: @@ -155,11 +159,23 @@ class bool_(generic): } +# Replicate this -- yet another --- NumPy-defined way of grouping scalar types, +# cf tests/core/test_scalar_methods.py +sctypes = { + 'int': [int8, int16, int32, int64], + 'uint': [uint8,], + 'float': [float16, float32, float64], + 'complex': [complex64, complex128], + 'others': [bool], +} + + __all__ = list(_typemap.keys()) __all__.remove('bool') __all__ += ['bool_', 'intp', 'int_', 'intc', 'byte', 'short', 'longlong', 'ubyte', 'half', 'single', 'double', 'csingle', 'cdouble'] +__all__ += ['sctypes'] __all__ += ['generic', 'number', - 'signedinteger', 'unsignedinteger', + 'integer', 'signedinteger', 'unsignedinteger', 'inexact', 'floating', 'complexfloating'] From 43a894d1fb12c1f3c824d3216033e7c5da7652bc Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Thu, 5 Jan 2023 14:55:53 +0300 Subject: [PATCH 10/21] MAINT: numpy-vendored tests get through the collection stage --- torch_np/_dtypes.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/torch_np/_dtypes.py b/torch_np/_dtypes.py index 8b44201e..f43e1fdc 100644 --- a/torch_np/_dtypes.py +++ b/torch_np/_dtypes.py @@ -117,7 +117,10 @@ def __repr__(self): typecodes = {'All': 'efdFDBbhil?', 'AllFloat': 'efdFD', 'AllInteger': 'Bbhil', + 'Integer': 'bhil', + 'UnsignedInteger': 'B', 'Float': 'efd', + 'Complex': 'FD', } From c0d511358e65cd234331cedb839fdb6208065606 Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Thu, 5 Jan 2023 17:29:24 +0300 Subject: [PATCH 11/21] BUG: np.asarray(arr) returns arr not a copy --- torch_np/_ndarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torch_np/_ndarray.py b/torch_np/_ndarray.py index 51f0447e..f8f0c174 100644 --- a/torch_np/_ndarray.py +++ b/torch_np/_ndarray.py @@ -339,7 +339,7 @@ def asarray(a, dtype=None, order=None, *, like=None): raise NotImplementedError if isinstance(a, ndarray): - if dtype != a.dtype: + if dtype is not None and dtype != a.dtype: a = a.astype(dtype) return a From 09ce7e08c00d269e901741c808eb8e1ee4cd914f Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Thu, 5 Jan 2023 17:32:32 +0300 Subject: [PATCH 12/21] BUG: fix import in test_ufunc_basic --- torch_np/tests/test_ufuncs_basic.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/torch_np/tests/test_ufuncs_basic.py b/torch_np/tests/test_ufuncs_basic.py index c72c20fe..e4bd5f6f 100644 --- a/torch_np/tests/test_ufuncs_basic.py +++ b/torch_np/tests/test_ufuncs_basic.py @@ -10,10 +10,8 @@ import pytest from pytest import raises as assert_raises -#import numpy as np import torch_np as np - -assert_equal = np.testing.assert_equal +from torch_np.testing import assert_equal parametrize_unary_ufuncs = pytest.mark.parametrize('ufunc', [np.sin]) From 0b5e9a73f4f0dbdfcc2e028a864493a4c68c62f5 Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Thu, 5 Jan 2023 18:25:59 +0300 Subject: [PATCH 13/21] API: add tests to stipulate equivalence of arrays scalars and 0D arrays --- torch_np/_ndarray.py | 10 +++ torch_np/tests/test_scalars_0D_arrays.py | 78 ++++++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 torch_np/tests/test_scalars_0D_arrays.py diff --git a/torch_np/_ndarray.py b/torch_np/_ndarray.py index f8f0c174..7b1d532c 100644 --- a/torch_np/_ndarray.py +++ b/torch_np/_ndarray.py @@ -110,6 +110,16 @@ def __neq__(self, other): def __gt__(self, other): return asarray(self._tensor > asarray(other).get()) + def __lt__(self, other): + return asarray(self._tensor < asarray(other).get()) + + def __ge__(self, other): + return asarray(self._tensor >= asarray(other).get()) + + def __le__(self, other): + return asarray(self._tensor <= asarray(other).get()) + + def __bool__(self): try: return bool(self._tensor) diff --git a/torch_np/tests/test_scalars_0D_arrays.py b/torch_np/tests/test_scalars_0D_arrays.py new file mode 100644 index 00000000..4dc24f61 --- /dev/null +++ b/torch_np/tests/test_scalars_0D_arrays.py @@ -0,0 +1,78 @@ +""" +Basic tests to fix and illustrate the behavior around the decision to use 0D +arrays in place of array scalars. Also fix what is a scalar really. + +Extensive tests of this sort of functionality is in numpy_tests/core/*scalar* +""" +import pytest + +import torch_np as np +from torch_np.testing import assert_equal + + +@pytest.mark.parametrize('value', + [np.int64(42), np.array(42), np.asarray(42), np.asarray(np.int64(42))]) +class TestArrayScalars: + def test_array_scalar_basic(self, value): + assert value.ndim == 0 + assert value.shape == () + assert value.size == 1 + assert value.dtype == np.dtype('int64') + + def test_conversion_to_int(self, value): + py_scalar = int(value) + assert py_scalar == 42 + assert isinstance(py_scalar, int) + assert not isinstance(value, int) + + def test_decay_to_py_scalar(self, value): + # NumPy distinguishes array scalars and 0D arrays. For instance + # `scalar * list` is equivalent to `int(scalar) * list`, but + # `0D array * list` is equivalent to `0D array * np.asarray(list)`. + # Our scalars follow 0D array behavior (because they are 0D arrays) + lst = [1, 2, 3] + + product = value * lst + assert isinstance(product, np.ndarray) + assert product.shape == (3,) + assert_equal(product, [42, 42*2, 42*3]) + + # repeat with right-mulitply + product = lst * value + assert isinstance(product, np.ndarray) + assert product.shape == (3,) + assert_equal(product, [42, 42*2, 42*3]) + + +def test_scalar_comparisons(): + scalar = np.int64(42) + arr = np.array(42) + + assert arr == scalar + assert arr >= scalar + assert arr <= scalar + + assert scalar == 42 + assert arr == 42 + + +class TestIsScalar: + # + # np.isscalar(...) checks that its argument is a numeric object with exactly one element. + # + # This differs from NumPy which also requires that shape == (). + # + scalars = [42, int(42.), np.float32(42), np.array(42), [42], [[42]], + np.array([42]), np.array([[42]])] + + import math + not_scalars = [int, np.float32, 's', 'string', (), [], math.sin, np, + np.transpose, [1, 2], np.asarray([1, 2]), np.float32([1, 2])] + + @pytest.mark.parametrize('value', scalars) + def test_is_scalar(self, value): + assert np.isscalar(value) + + @pytest.mark.parametrize('value', not_scalars) + def test_is_not_scalar(self, value): + assert not np.isscalar(value) From 5be93f1a73f4ce89f4d4efedeff663e3a9816ab6 Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Thu, 5 Jan 2023 18:51:33 +0300 Subject: [PATCH 14/21] TST: test_numerictypes: remove definitely unsupported things Structured dtypes, longdoubles and all that. --- .../numpy_tests/core/test_numerictypes.py | 366 +----------------- .../numpy_tests/core/test_scalar_methods.py | 7 +- .../numpy_tests/core/test_scalarinherit.py | 2 +- torch_np/tests/test_scalars_0D_arrays.py | 6 +- 4 files changed, 17 insertions(+), 364 deletions(-) diff --git a/torch_np/tests/numpy_tests/core/test_numerictypes.py b/torch_np/tests/numpy_tests/core/test_numerictypes.py index 072cd65f..f22e47fa 100644 --- a/torch_np/tests/numpy_tests/core/test_numerictypes.py +++ b/torch_np/tests/numpy_tests/core/test_numerictypes.py @@ -2,340 +2,12 @@ import itertools import pytest -import numpy as np -from numpy.testing import assert_, assert_equal, assert_raises, IS_PYPY - -# This is the structure of the table used for plain objects: -# -# +-+-+-+ -# |x|y|z| -# +-+-+-+ - -# Structure of a plain array description: -Pdescr = [ - ('x', 'i4', (2,)), - ('y', 'f8', (2, 2)), - ('z', 'u1')] - -# A plain list of tuples with values for testing: -PbufferT = [ - # x y z - ([3, 2], [[6., 4.], [6., 4.]], 8), - ([4, 3], [[7., 5.], [7., 5.]], 9), - ] - +from pytest import raises as assert_raises -# This is the structure of the table used for nested objects (DON'T PANIC!): -# -# +-+---------------------------------+-----+----------+-+-+ -# |x|Info |color|info |y|z| -# | +-----+--+----------------+----+--+ +----+-----+ | | -# | |value|y2|Info2 |name|z2| |Name|Value| | | -# | | | +----+-----+--+--+ | | | | | | | -# | | | |name|value|y3|z3| | | | | | | | -# +-+-----+--+----+-----+--+--+----+--+-----+----+-----+-+-+ -# - -# The corresponding nested array description: -Ndescr = [ - ('x', 'i4', (2,)), - ('Info', [ - ('value', 'c16'), - ('y2', 'f8'), - ('Info2', [ - ('name', 'S2'), - ('value', 'c16', (2,)), - ('y3', 'f8', (2,)), - ('z3', 'u4', (2,))]), - ('name', 'S2'), - ('z2', 'b1')]), - ('color', 'S2'), - ('info', [ - ('Name', 'U8'), - ('Value', 'c16')]), - ('y', 'f8', (2, 2)), - ('z', 'u1')] - -NbufferT = [ - # x Info color info y z - # value y2 Info2 name z2 Name Value - # name value y3 z3 - ([3, 2], (6j, 6., (b'nn', [6j, 4j], [6., 4.], [1, 2]), b'NN', True), - b'cc', ('NN', 6j), [[6., 4.], [6., 4.]], 8), - ([4, 3], (7j, 7., (b'oo', [7j, 5j], [7., 5.], [2, 1]), b'OO', False), - b'dd', ('OO', 7j), [[7., 5.], [7., 5.]], 9), - ] +import torch_np as np +from torch_np.testing import assert_, assert_equal -byteorder = {'little':'<', 'big':'>'}[sys.byteorder] - -def normalize_descr(descr): - "Normalize a description adding the platform byteorder." - - out = [] - for item in descr: - dtype = item[1] - if isinstance(dtype, str): - if dtype[0] not in ['|', '<', '>']: - onebyte = dtype[1:] == "1" - if onebyte or dtype[0] in ['S', 'V', 'b']: - dtype = "|" + dtype - else: - dtype = byteorder + dtype - if len(item) > 2 and np.prod(item[2]) > 1: - nitem = (item[0], dtype, item[2]) - else: - nitem = (item[0], dtype) - out.append(nitem) - elif isinstance(dtype, list): - l = normalize_descr(dtype) - out.append((item[0], l)) - else: - raise ValueError("Expected a str or list and got %s" % - (type(item))) - return out - - -############################################################ -# Creation tests -############################################################ - -class CreateZeros: - """Check the creation of heterogeneous arrays zero-valued""" - - def test_zeros0D(self): - """Check creation of 0-dimensional objects""" - h = np.zeros((), dtype=self._descr) - assert_(normalize_descr(self._descr) == h.dtype.descr) - assert_(h.dtype.fields['x'][0].name[:4] == 'void') - assert_(h.dtype.fields['x'][0].char == 'V') - assert_(h.dtype.fields['x'][0].type == np.void) - # A small check that data is ok - assert_equal(h['z'], np.zeros((), dtype='u1')) - - def test_zerosSD(self): - """Check creation of single-dimensional objects""" - h = np.zeros((2,), dtype=self._descr) - assert_(normalize_descr(self._descr) == h.dtype.descr) - assert_(h.dtype['y'].name[:4] == 'void') - assert_(h.dtype['y'].char == 'V') - assert_(h.dtype['y'].type == np.void) - # A small check that data is ok - assert_equal(h['z'], np.zeros((2,), dtype='u1')) - - def test_zerosMD(self): - """Check creation of multi-dimensional objects""" - h = np.zeros((2, 3), dtype=self._descr) - assert_(normalize_descr(self._descr) == h.dtype.descr) - assert_(h.dtype['z'].name == 'uint8') - assert_(h.dtype['z'].char == 'B') - assert_(h.dtype['z'].type == np.uint8) - # A small check that data is ok - assert_equal(h['z'], np.zeros((2, 3), dtype='u1')) - - -class TestCreateZerosPlain(CreateZeros): - """Check the creation of heterogeneous arrays zero-valued (plain)""" - _descr = Pdescr - -class TestCreateZerosNested(CreateZeros): - """Check the creation of heterogeneous arrays zero-valued (nested)""" - _descr = Ndescr - - -class CreateValues: - """Check the creation of heterogeneous arrays with values""" - - def test_tuple(self): - """Check creation from tuples""" - h = np.array(self._buffer, dtype=self._descr) - assert_(normalize_descr(self._descr) == h.dtype.descr) - if self.multiple_rows: - assert_(h.shape == (2,)) - else: - assert_(h.shape == ()) - - def test_list_of_tuple(self): - """Check creation from list of tuples""" - h = np.array([self._buffer], dtype=self._descr) - assert_(normalize_descr(self._descr) == h.dtype.descr) - if self.multiple_rows: - assert_(h.shape == (1, 2)) - else: - assert_(h.shape == (1,)) - - def test_list_of_list_of_tuple(self): - """Check creation from list of list of tuples""" - h = np.array([[self._buffer]], dtype=self._descr) - assert_(normalize_descr(self._descr) == h.dtype.descr) - if self.multiple_rows: - assert_(h.shape == (1, 1, 2)) - else: - assert_(h.shape == (1, 1)) - - -class TestCreateValuesPlainSingle(CreateValues): - """Check the creation of heterogeneous arrays (plain, single row)""" - _descr = Pdescr - multiple_rows = 0 - _buffer = PbufferT[0] - -class TestCreateValuesPlainMultiple(CreateValues): - """Check the creation of heterogeneous arrays (plain, multiple rows)""" - _descr = Pdescr - multiple_rows = 1 - _buffer = PbufferT - -class TestCreateValuesNestedSingle(CreateValues): - """Check the creation of heterogeneous arrays (nested, single row)""" - _descr = Ndescr - multiple_rows = 0 - _buffer = NbufferT[0] - -class TestCreateValuesNestedMultiple(CreateValues): - """Check the creation of heterogeneous arrays (nested, multiple rows)""" - _descr = Ndescr - multiple_rows = 1 - _buffer = NbufferT - - -############################################################ -# Reading tests -############################################################ - -class ReadValuesPlain: - """Check the reading of values in heterogeneous arrays (plain)""" - - def test_access_fields(self): - h = np.array(self._buffer, dtype=self._descr) - if not self.multiple_rows: - assert_(h.shape == ()) - assert_equal(h['x'], np.array(self._buffer[0], dtype='i4')) - assert_equal(h['y'], np.array(self._buffer[1], dtype='f8')) - assert_equal(h['z'], np.array(self._buffer[2], dtype='u1')) - else: - assert_(len(h) == 2) - assert_equal(h['x'], np.array([self._buffer[0][0], - self._buffer[1][0]], dtype='i4')) - assert_equal(h['y'], np.array([self._buffer[0][1], - self._buffer[1][1]], dtype='f8')) - assert_equal(h['z'], np.array([self._buffer[0][2], - self._buffer[1][2]], dtype='u1')) - - -class TestReadValuesPlainSingle(ReadValuesPlain): - """Check the creation of heterogeneous arrays (plain, single row)""" - _descr = Pdescr - multiple_rows = 0 - _buffer = PbufferT[0] - -class TestReadValuesPlainMultiple(ReadValuesPlain): - """Check the values of heterogeneous arrays (plain, multiple rows)""" - _descr = Pdescr - multiple_rows = 1 - _buffer = PbufferT - -class ReadValuesNested: - """Check the reading of values in heterogeneous arrays (nested)""" - - def test_access_top_fields(self): - """Check reading the top fields of a nested array""" - h = np.array(self._buffer, dtype=self._descr) - if not self.multiple_rows: - assert_(h.shape == ()) - assert_equal(h['x'], np.array(self._buffer[0], dtype='i4')) - assert_equal(h['y'], np.array(self._buffer[4], dtype='f8')) - assert_equal(h['z'], np.array(self._buffer[5], dtype='u1')) - else: - assert_(len(h) == 2) - assert_equal(h['x'], np.array([self._buffer[0][0], - self._buffer[1][0]], dtype='i4')) - assert_equal(h['y'], np.array([self._buffer[0][4], - self._buffer[1][4]], dtype='f8')) - assert_equal(h['z'], np.array([self._buffer[0][5], - self._buffer[1][5]], dtype='u1')) - - def test_nested1_acessors(self): - """Check reading the nested fields of a nested array (1st level)""" - h = np.array(self._buffer, dtype=self._descr) - if not self.multiple_rows: - assert_equal(h['Info']['value'], - np.array(self._buffer[1][0], dtype='c16')) - assert_equal(h['Info']['y2'], - np.array(self._buffer[1][1], dtype='f8')) - assert_equal(h['info']['Name'], - np.array(self._buffer[3][0], dtype='U2')) - assert_equal(h['info']['Value'], - np.array(self._buffer[3][1], dtype='c16')) - else: - assert_equal(h['Info']['value'], - np.array([self._buffer[0][1][0], - self._buffer[1][1][0]], - dtype='c16')) - assert_equal(h['Info']['y2'], - np.array([self._buffer[0][1][1], - self._buffer[1][1][1]], - dtype='f8')) - assert_equal(h['info']['Name'], - np.array([self._buffer[0][3][0], - self._buffer[1][3][0]], - dtype='U2')) - assert_equal(h['info']['Value'], - np.array([self._buffer[0][3][1], - self._buffer[1][3][1]], - dtype='c16')) - - def test_nested2_acessors(self): - """Check reading the nested fields of a nested array (2nd level)""" - h = np.array(self._buffer, dtype=self._descr) - if not self.multiple_rows: - assert_equal(h['Info']['Info2']['value'], - np.array(self._buffer[1][2][1], dtype='c16')) - assert_equal(h['Info']['Info2']['z3'], - np.array(self._buffer[1][2][3], dtype='u4')) - else: - assert_equal(h['Info']['Info2']['value'], - np.array([self._buffer[0][1][2][1], - self._buffer[1][1][2][1]], - dtype='c16')) - assert_equal(h['Info']['Info2']['z3'], - np.array([self._buffer[0][1][2][3], - self._buffer[1][1][2][3]], - dtype='u4')) - - def test_nested1_descriptor(self): - """Check access nested descriptors of a nested array (1st level)""" - h = np.array(self._buffer, dtype=self._descr) - assert_(h.dtype['Info']['value'].name == 'complex128') - assert_(h.dtype['Info']['y2'].name == 'float64') - assert_(h.dtype['info']['Name'].name == 'str256') - assert_(h.dtype['info']['Value'].name == 'complex128') - - def test_nested2_descriptor(self): - """Check access nested descriptors of a nested array (2nd level)""" - h = np.array(self._buffer, dtype=self._descr) - assert_(h.dtype['Info']['Info2']['value'].name == 'void256') - assert_(h.dtype['Info']['Info2']['z3'].name == 'void64') - - -class TestReadValuesNestedSingle(ReadValuesNested): - """Check the values of heterogeneous arrays (nested, single row)""" - _descr = Ndescr - multiple_rows = False - _buffer = NbufferT[0] - -class TestReadValuesNestedMultiple(ReadValuesNested): - """Check the values of heterogeneous arrays (nested, multiple rows)""" - _descr = Ndescr - multiple_rows = True - _buffer = NbufferT - -class TestEmptyField: - def test_assign(self): - a = np.arange(10, dtype=np.float32) - a.dtype = [("int", "<0i4"), ("float", "<2f4")] - assert_(a['int'].shape == (5, 0)) - assert_(a['float'].shape == (5, 2)) class TestCommonType: def test_scalar_loses1(self): @@ -358,20 +30,6 @@ def test_scalar_wins3(self): # doesn't go up to 'f16' on purpose res = np.find_common_type(['u8', 'i8', 'i8'], ['f8']) assert_(res == 'f8') -class TestMultipleFields: - def setup_method(self): - self.ary = np.array([(1, 2, 3, 4), (5, 6, 7, 8)], dtype='i4,f4,i2,c8') - - def _bad_call(self): - return self.ary['f0', 'f1'] - - def test_no_tuple(self): - assert_raises(IndexError, self._bad_call) - - def test_return(self): - res = self.ary[['f0', 'f2']].tolist() - assert_(res == [(1, 3), (5, 7)]) - class TestIsSubDType: # scalar types can be promoted into dtypes @@ -461,19 +119,19 @@ class TestMaximumSctype: def test_int(self, t): assert_equal(np.maximum_sctype(t), np.sctypes['int'][-1]) - @pytest.mark.parametrize('t', [np.ubyte, np.ushort, np.uintc, np.uint, np.ulonglong]) + @pytest.mark.parametrize('t', [np.ubyte]) def test_uint(self, t): assert_equal(np.maximum_sctype(t), np.sctypes['uint'][-1]) - @pytest.mark.parametrize('t', [np.half, np.single, np.double, np.longdouble]) + @pytest.mark.parametrize('t', [np.half, np.single, np.double]) def test_float(self, t): assert_equal(np.maximum_sctype(t), np.sctypes['float'][-1]) - @pytest.mark.parametrize('t', [np.csingle, np.cdouble, np.clongdouble]) + @pytest.mark.parametrize('t', [np.csingle, np.cdouble]) def test_complex(self, t): assert_equal(np.maximum_sctype(t), np.sctypes['complex'][-1]) - @pytest.mark.parametrize('t', [np.bool_, np.object_, np.unicode_, np.bytes_, np.void]) + @pytest.mark.parametrize('t', [np.bool_,]) def test_other(self, t): assert_equal(np.maximum_sctype(t), t) @@ -513,8 +171,6 @@ def test_non_type(self): (1.1, False), (str, True), (np.dtype(np.float64), True), - (np.dtype((np.int16, (3, 4))), True), - (np.dtype([('a', np.int8)]), True), ]) def test_issctype(rep, expected): # ensure proper identification of scalar @@ -525,8 +181,6 @@ def test_issctype(rep, expected): @pytest.mark.skipif(sys.flags.optimize > 1, reason="no docstrings present to inspect when PYTHONOPTIMIZE/Py_OptimizeFlag > 1") -@pytest.mark.xfail(IS_PYPY, - reason="PyPy cannot modify tp_doc after PyType_Ready") class TestDocStrings: def test_platform_dependent_aliases(self): if np.int64 is np.int_: @@ -540,9 +194,9 @@ class TestScalarTypeNames: numeric_types = [ np.byte, np.short, np.intc, np.int_, np.longlong, - np.ubyte, np.ushort, np.uintc, np.uint, np.ulonglong, - np.half, np.single, np.double, np.longdouble, - np.csingle, np.cdouble, np.clongdouble, + np.ubyte, + np.half, np.single, np.double, + np.csingle, np.cdouble, ] def test_names_are_unique(self): diff --git a/torch_np/tests/numpy_tests/core/test_scalar_methods.py b/torch_np/tests/numpy_tests/core/test_scalar_methods.py index dc92b795..fdbd1e8d 100644 --- a/torch_np/tests/numpy_tests/core/test_scalar_methods.py +++ b/torch_np/tests/numpy_tests/core/test_scalar_methods.py @@ -7,13 +7,10 @@ import types from typing import Any, Type -import pytest -#import numpy as np - -#from numpy.testing import assert_equal, assert_raises - import torch_np as np from torch_np.testing import assert_equal + +import pytest from pytest import raises as assert_raises diff --git a/torch_np/tests/numpy_tests/core/test_scalarinherit.py b/torch_np/tests/numpy_tests/core/test_scalarinherit.py index 8b5c5cea..a88f6765 100644 --- a/torch_np/tests/numpy_tests/core/test_scalarinherit.py +++ b/torch_np/tests/numpy_tests/core/test_scalarinherit.py @@ -31,7 +31,7 @@ class B1(np.float64, HasNew): pass -@pytest.mark.xfail(reason='scalar repr: numpy plain to make more explicit') +@pytest.mark.skip(reason='scalar repr: numpy plans to make it more explicit') class TestInherit: def test_init(self): x = B(1.0) diff --git a/torch_np/tests/test_scalars_0D_arrays.py b/torch_np/tests/test_scalars_0D_arrays.py index 4dc24f61..17435b94 100644 --- a/torch_np/tests/test_scalars_0D_arrays.py +++ b/torch_np/tests/test_scalars_0D_arrays.py @@ -1,8 +1,10 @@ """ -Basic tests to fix and illustrate the behavior around the decision to use 0D -arrays in place of array scalars. Also fix what is a scalar really. +Basic tests to assert and illustrate the behavior around the decision to use 0D +arrays in place of array scalars. Extensive tests of this sort of functionality is in numpy_tests/core/*scalar* + +Also test the isscalar function (which is deliberately a bit more lax). """ import pytest From a07fab697163479a1004373c647186bbf8e4d840 Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Thu, 5 Jan 2023 19:27:18 +0300 Subject: [PATCH 15/21] BUG: fix the scalar type hierarchy, so that issubdtype works. Also make test_numerictypes from numpy pass. Remove weird things, xfail the "scalar does not upcast array" test, fix up the rest. --- torch_np/_dtypes.py | 7 +- torch_np/_scalar_types.py | 16 ++-- .../numpy_tests/core/test_numerictypes.py | 95 ++----------------- 3 files changed, 18 insertions(+), 100 deletions(-) diff --git a/torch_np/_dtypes.py b/torch_np/_dtypes.py index f43e1fdc..c4b37e82 100644 --- a/torch_np/_dtypes.py +++ b/torch_np/_dtypes.py @@ -91,7 +91,12 @@ def __repr__(self): python_types_dict = { int: 'int64', float: 'float64', - builtins.bool: 'bool' + complex: 'complex128', + builtins.bool: 'bool', + # also allow stringified names of python types + int.__name__ : 'int64', + float.__name__ : 'float64', + complex.__name__: 'complex128', } diff --git a/torch_np/_scalar_types.py b/torch_np/_scalar_types.py index 779ed6b4..0c4429b3 100644 --- a/torch_np/_scalar_types.py +++ b/torch_np/_scalar_types.py @@ -45,31 +45,27 @@ class number(generic): pass -class integer(generic): +class integer(number): pass -class inexact(generic): +class inexact(number): pass -class signedinteger(generic): +class signedinteger(integer): pass -class unsignedinteger(generic): +class unsignedinteger(integer): pass -class inexact(generic): +class floating(inexact): pass -class floating(generic): - pass - - -class complexfloating(generic): +class complexfloating(inexact): pass diff --git a/torch_np/tests/numpy_tests/core/test_numerictypes.py b/torch_np/tests/numpy_tests/core/test_numerictypes.py index f22e47fa..e366043f 100644 --- a/torch_np/tests/numpy_tests/core/test_numerictypes.py +++ b/torch_np/tests/numpy_tests/core/test_numerictypes.py @@ -8,7 +8,8 @@ from torch_np.testing import assert_, assert_equal - +@pytest.mark.xfail(reason="We do not disctinguish between scalar and array types." + " Thus, scalars can upcast arrays.") class TestCommonType: def test_scalar_loses1(self): res = np.find_common_type(['f4', 'f4', 'i2'], ['f8']) @@ -66,9 +67,7 @@ def test_nondtype_nonscalartype(self): # this. These tests are directly taken from gh-9505 assert not np.issubdtype(np.float32, 'float64') assert not np.issubdtype(np.float32, 'f8') - assert not np.issubdtype(np.int32, str) assert not np.issubdtype(np.int32, 'int64') - assert not np.issubdtype(np.str_, 'void') # for the following the correct spellings are # np.integer, np.floating, or np.complexfloating respectively: assert not np.issubdtype(np.int8, int) # np.int8 is never np.int_ @@ -81,9 +80,7 @@ def test_nondtype_nonscalartype(self): # in the case of int, float, complex: assert np.issubdtype(np.float64, 'float64') assert np.issubdtype(np.float64, 'f8') - assert np.issubdtype(np.str_, str) assert np.issubdtype(np.int64, 'int64') - assert np.issubdtype(np.void, 'void') assert np.issubdtype(np.int8, np.integer) assert np.issubdtype(np.float32, np.floating) assert np.issubdtype(np.complex64, np.complexfloating) @@ -91,94 +88,14 @@ def test_nondtype_nonscalartype(self): assert np.issubdtype(np.float32, "f") -class TestSctypeDict: - def test_longdouble(self): - assert_(np.sctypeDict['f8'] is not np.longdouble) - assert_(np.sctypeDict['c16'] is not np.clongdouble) - - def test_ulong(self): - # Test that 'ulong' behaves like 'long'. np.sctypeDict['long'] is an - # alias for np.int_, but np.long is not supported for historical - # reasons (gh-21063) - assert_(np.sctypeDict['ulong'] is np.uint) - with pytest.warns(FutureWarning): - # We will probably allow this in the future: - assert not hasattr(np, 'ulong') - +@pytest.mark.xfail(reason="We do not have (or need) np.core.numerictypes." + " Our type aliases are in _dtypes.py.") class TestBitName: def test_abstract(self): assert_raises(ValueError, np.core.numerictypes.bitname, np.floating) -class TestMaximumSctype: - - # note that parametrizing with sctype['int'] and similar would skip types - # with the same size (gh-11923) - - @pytest.mark.parametrize('t', [np.byte, np.short, np.intc, np.int_, np.longlong]) - def test_int(self, t): - assert_equal(np.maximum_sctype(t), np.sctypes['int'][-1]) - - @pytest.mark.parametrize('t', [np.ubyte]) - def test_uint(self, t): - assert_equal(np.maximum_sctype(t), np.sctypes['uint'][-1]) - - @pytest.mark.parametrize('t', [np.half, np.single, np.double]) - def test_float(self, t): - assert_equal(np.maximum_sctype(t), np.sctypes['float'][-1]) - - @pytest.mark.parametrize('t', [np.csingle, np.cdouble]) - def test_complex(self, t): - assert_equal(np.maximum_sctype(t), np.sctypes['complex'][-1]) - - @pytest.mark.parametrize('t', [np.bool_,]) - def test_other(self, t): - assert_equal(np.maximum_sctype(t), t) - - -class Test_sctype2char: - # This function is old enough that we're really just documenting the quirks - # at this point. - - def test_scalar_type(self): - assert_equal(np.sctype2char(np.double), 'd') - assert_equal(np.sctype2char(np.int_), 'l') - assert_equal(np.sctype2char(np.unicode_), 'U') - assert_equal(np.sctype2char(np.bytes_), 'S') - - def test_other_type(self): - assert_equal(np.sctype2char(float), 'd') - assert_equal(np.sctype2char(list), 'O') - assert_equal(np.sctype2char(np.ndarray), 'O') - - def test_third_party_scalar_type(self): - from numpy.core._rational_tests import rational - assert_raises(KeyError, np.sctype2char, rational) - assert_raises(KeyError, np.sctype2char, rational(1)) - - def test_array_instance(self): - assert_equal(np.sctype2char(np.array([1.0, 2.0])), 'd') - - def test_abstract_type(self): - assert_raises(KeyError, np.sctype2char, np.floating) - - def test_non_type(self): - assert_raises(ValueError, np.sctype2char, 1) - -@pytest.mark.parametrize("rep, expected", [ - (np.int32, True), - (list, False), - (1.1, False), - (str, True), - (np.dtype(np.float64), True), - ]) -def test_issctype(rep, expected): - # ensure proper identification of scalar - # data-types by issctype() - actual = np.issctype(rep) - assert_equal(actual, expected) - - +@pytest.mark.skip(reason="Docstrings for scalar types, not yet.") @pytest.mark.skipif(sys.flags.optimize > 1, reason="no docstrings present to inspect when PYTHONOPTIMIZE/Py_OptimizeFlag > 1") class TestDocStrings: @@ -193,7 +110,7 @@ class TestScalarTypeNames: # gh-9799 numeric_types = [ - np.byte, np.short, np.intc, np.int_, np.longlong, + np.byte, np.short, np.intc, np.int_, #, np.longlong, NB: torch does not properly have longlong np.ubyte, np.half, np.single, np.double, np.csingle, np.cdouble, From eec3bbae1ee9f7cd0ae7a1ed3016825ce3a25d68 Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Thu, 5 Jan 2023 21:32:09 +0300 Subject: [PATCH 16/21] ENH: add dtype.itemsize, rm a buch of tests of timedelta, dtype(str) etc etc --- torch_np/_dtypes.py | 8 + torch_np/tests/numpy_tests/core/test_dtype.py | 157 ++---------------- 2 files changed, 25 insertions(+), 140 deletions(-) diff --git a/torch_np/_dtypes.py b/torch_np/_dtypes.py index c4b37e82..a624f0da 100644 --- a/torch_np/_dtypes.py +++ b/torch_np/_dtypes.py @@ -57,11 +57,19 @@ def __eq__(self, other): return False return self._name == other_instance.name + def __hash__(self): + return hash(self._name) + def __repr__(self): return f'dtype("{self.name}")' __str__ = __repr__ + def itemsize(self): + elem = self.type(1) + return elem.get().element_size() + + dt_names = ['float16', 'float32', 'float64', 'complex64', 'complex128', diff --git a/torch_np/tests/numpy_tests/core/test_dtype.py b/torch_np/tests/numpy_tests/core/test_dtype.py index 81d69f43..1248a665 100644 --- a/torch_np/tests/numpy_tests/core/test_dtype.py +++ b/torch_np/tests/numpy_tests/core/test_dtype.py @@ -5,28 +5,27 @@ from typing import Any import torch_np as np -from torch_np.testing import ( - assert_, assert_equal, assert_array_equal) +from torch_np.testing import (assert_, assert_equal, assert_array_equal) from pytest import raises as assert_raises - import pickle from itertools import permutations import random - def assert_dtype_equal(a, b): assert_equal(a, b) assert_equal(hash(a), hash(b), "two equivalent types do not hash to the same value !") + def assert_dtype_not_equal(a, b): assert_(a != b) assert_(hash(a) != hash(b), "two different types hash to the same value !") + class TestBuiltin: @pytest.mark.parametrize('t', [int, float, complex, np.int32]) def test_run(self, t): @@ -36,13 +35,13 @@ def test_run(self, t): def test_equivalent_dtype_hashing(self): # Make sure equivalent dtypes with different type num hash equal - uintp = np.dtype(np.uintp) - if uintp.itemsize == 4: - left = uintp - right = np.dtype(np.uint32) + intp = np.dtype(np.intp) + if intp.itemsize == 4: + left = intp + right = np.dtype(np.int32) else: - left = uintp - right = np.dtype(np.ulonglong) + left = intp + right = np.dtype(np.int64) assert_(left == right) assert_(hash(left) == hash(right)) @@ -58,11 +57,6 @@ def test_invalid_types(self): assert_raises(TypeError, np.dtype, 'e3') assert_raises(TypeError, np.dtype, 'f5') - if np.dtype('g').itemsize == 8 or np.dtype('g').itemsize == 16: - assert_raises(TypeError, np.dtype, 'g12') - elif np.dtype('g').itemsize == 12: - assert_raises(TypeError, np.dtype, 'g16') - if np.dtype('l').itemsize == 8: assert_raises(TypeError, np.dtype, 'l4') assert_raises(TypeError, np.dtype, 'L4') @@ -70,12 +64,13 @@ def test_invalid_types(self): assert_raises(TypeError, np.dtype, 'l8') assert_raises(TypeError, np.dtype, 'L8') - if np.dtype('q').itemsize == 8: - assert_raises(TypeError, np.dtype, 'q4') - assert_raises(TypeError, np.dtype, 'Q4') - else: - assert_raises(TypeError, np.dtype, 'q8') - assert_raises(TypeError, np.dtype, 'Q8') + # XXX: what is 'q'? on my 64-bit ubuntu maching it's int64, same as 'l' + # if np.dtype('q').itemsize == 8: + # assert_raises(TypeError, np.dtype, 'q4') + # assert_raises(TypeError, np.dtype, 'Q4') + # else: + # assert_raises(TypeError, np.dtype, 'q8') + # assert_raises(TypeError, np.dtype, 'Q8') def test_richcompare_invalid_dtype_equality(self): # Make sure objects that cannot be converted to valid @@ -107,83 +102,8 @@ def test_numeric_style_types_are_invalid(self, dtype): with assert_raises(TypeError): np.dtype(dtype) - def test_remaining_dtypes_with_bad_bytesize(self): - # The np. aliases were deprecated, these probably should be too - assert np.dtype("int0") is np.dtype("intp") - assert np.dtype("uint0") is np.dtype("uintp") - assert np.dtype("bool8") is np.dtype("bool") - assert np.dtype("bytes0") is np.dtype("bytes") - assert np.dtype("str0") is np.dtype("str") - assert np.dtype("object0") is np.dtype("object") - - @pytest.mark.parametrize( - 'value', - [ - 'i4, f4' - ]) - def test_dtype_bytes_str_equivalence(self, value): - bytes_value = value.encode('ascii') - from_bytes = np.dtype(bytes_value) - from_str = np.dtype(value) - assert_dtype_equal(from_bytes, from_str) - - def test_dtype_from_bytes(self): - # Single character where value is a valid type code - assert_dtype_equal(np.dtype(b'f'), np.dtype('float32')) - - # Bytes with non-ascii values raise errors - assert_raises(TypeError, np.dtype, b'\xff') - assert_raises(TypeError, np.dtype, b's\xff') - - - - - - - - -def iter_struct_object_dtypes(): - """ - Iterates over a few complex dtypes and object pattern which - fill the array with a given object (defaults to a singleton). - - Yields - ------ - dtype : dtype - pattern : tuple - Structured tuple for use with `np.array`. - count : int - Number of objects stored in the dtype. - singleton : object - A singleton object. The returned pattern is constructed so that - all objects inside the datatype are set to the singleton. - """ - obj = object() - - dt = np.dtype([('b', 'O', (2, 3))]) - p = ([[obj] * 3] * 2,) - yield pytest.param(dt, p, 6, obj, id="") - - dt = np.dtype([('a', 'i4'), ('b', 'O', (2, 3))]) - p = (0, [[obj] * 3] * 2) - yield pytest.param(dt, p, 6, obj, id="") - - dt = np.dtype([('a', 'i4'), - ('b', [('ba', 'O'), ('bb', 'i1')], (2, 3))]) - p = (0, [[(obj, 0)] * 3] * 2) - yield pytest.param(dt, p, 6, obj, id="") - - dt = np.dtype([('a', 'i4'), - ('b', [('ba', 'O'), ('bb', 'O')], (2, 3))]) - p = (0, [[(obj, obj)] * 3] * 2) - yield pytest.param(dt, p, 12, obj, id="") - - - - - - +@pytest.mark.skip(reason='dtype attributes not yet implemented') class TestDtypeAttributeDeletion: def test_dtype_non_writable_attributes_deletion(self): @@ -202,9 +122,6 @@ def test_dtype_writable_attributes_deletion(self): assert_raises(AttributeError, delattr, dt, s) - - - class TestPickling: def check_pickling(self, dtype): @@ -356,38 +273,6 @@ class dt: np.dtype(dt_instance) -class TestDTypeClasses: - @pytest.mark.parametrize("dtype", list(np.typecodes['All'])) - def test_basic_dtypes_subclass_properties(self, dtype): - # Note: Except for the isinstance and type checks, these attributes - # are considered currently private and may change. - dtype = np.dtype(dtype) - assert isinstance(dtype, np.dtype) - assert type(dtype) is not np.dtype - assert type(dtype).__name__ == f"dtype[{dtype.type.__name__}]" - assert type(dtype).__module__ == "numpy" - assert not type(dtype)._abstract - - # the flexible dtypes and datetime/timedelta have additional parameters - # which are more than just storage information, these would need to be - # given when creating a dtype: - parametric = (np.void, np.str_, np.bytes_, np.datetime64, np.timedelta64) - if dtype.type not in parametric: - assert not type(dtype)._parametric - assert type(dtype)() is dtype - else: - assert type(dtype)._parametric - with assert_raises(TypeError): - type(dtype)() - - def test_dtype_superclass(self): - assert type(np.dtype) is not type - assert isinstance(np.dtype, type) - - assert type(np.dtype).__name__ == "_DTypeMeta" - assert type(np.dtype).__module__ == "numpy" - assert np.dtype._abstract - @pytest.mark.skipif(sys.version_info < (3, 9), reason="Requires python 3.9") @@ -417,14 +302,6 @@ def test_subscript_scalar(self) -> None: assert np.dtype[Any] -def test_result_type_integers_and_unitless_timedelta64(): - # Regression test for gh-20077. The following call of `result_type` - # would cause a seg. fault. - td = np.timedelta64(4) - result = np.result_type(0, td) - assert_dtype_equal(result, td.dtype) - - @pytest.mark.skipif(sys.version_info >= (3, 9), reason="Requires python 3.8") def test_class_getitem_38() -> None: match = "Type subscription requires python >= 3.9" From adf9c73b07fa87924405233f6271097d328eb555 Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Fri, 6 Jan 2023 08:53:08 +0300 Subject: [PATCH 17/21] ENH: dtypes pickle/unpickle --- torch_np/_dtypes.py | 6 ++++++ torch_np/testing/utils.py | 3 +++ torch_np/tests/numpy_tests/core/test_dtype.py | 13 +++++++------ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/torch_np/_dtypes.py b/torch_np/_dtypes.py index a624f0da..348b0463 100644 --- a/torch_np/_dtypes.py +++ b/torch_np/_dtypes.py @@ -69,6 +69,12 @@ def itemsize(self): elem = self.type(1) return elem.get().element_size() + def __getstate__(self): + return self._name + + def __setstate__(self, value): + self._name = value + dt_names = ['float16', 'float32', 'float64', diff --git a/torch_np/testing/utils.py b/torch_np/testing/utils.py index 41e89f47..8f69032a 100644 --- a/torch_np/testing/utils.py +++ b/torch_np/testing/utils.py @@ -186,6 +186,9 @@ def assert_equal(actual, desired, err_msg='', verbose=True): return assert_array_equal(actual, desired, err_msg, verbose) msg = build_err_msg([actual, desired], err_msg, verbose=verbose) + if isinstance(actual, np.dtype) and isinstance(desired, np.dtype): + return actual == desired + # Handle complex numbers: separate into real/imag to handle # nan/inf/negative zero correctly # XXX: catch ValueError for subclasses of ndarray where iscomplex fail diff --git a/torch_np/tests/numpy_tests/core/test_dtype.py b/torch_np/tests/numpy_tests/core/test_dtype.py index 1248a665..33e400b8 100644 --- a/torch_np/tests/numpy_tests/core/test_dtype.py +++ b/torch_np/tests/numpy_tests/core/test_dtype.py @@ -133,10 +133,13 @@ def check_pickling(self, dtype): assert b"dtype" in buf pickled = pickle.loads(buf) assert_equal(pickled, dtype) - assert_equal(pickled.descr, dtype.descr) - if dtype.metadata is not None: - assert_equal(pickled.metadata, dtype.metadata) + + ## XXX: out dtypes do not have .descr + ## assert_equal(pickled.descr, dtype.descr) + ## if dtype.metadata is not None: + ## assert_equal(pickled.metadata, dtype.metadata) # Check the reconstructed dtype is functional + x = np.zeros(3, dtype=dtype) y = np.zeros(3, dtype=pickled) assert_equal(x, y) @@ -146,10 +149,8 @@ def check_pickling(self, dtype): def test_builtin(self, t): self.check_pickling(np.dtype(t)) - @pytest.mark.parametrize("DType", - [type(np.dtype(t)) for t in np.typecodes['All']] + - [np.dtype]) + [type(np.dtype(t)) for t in np.typecodes['All']] + [np.dtype]) def test_pickle_types(self, DType): # Check that DTypes (the classes/types) roundtrip when pickling for proto in range(pickle.HIGHEST_PROTOCOL + 1): From 2d7d9321e896b13e472b4855af202b2c65d29817 Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Fri, 6 Jan 2023 09:01:38 +0300 Subject: [PATCH 18/21] TST: test_dtype from NumPy passes (with skips/fails, of course) The largest difference is that our type promotions are way simpler and more straighforward. No "weak promotion", no values. --- torch_np/_dtypes.py | 4 +++- torch_np/tests/numpy_tests/core/test_dtype.py | 7 +++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/torch_np/_dtypes.py b/torch_np/_dtypes.py index 348b0463..948df8bb 100644 --- a/torch_np/_dtypes.py +++ b/torch_np/_dtypes.py @@ -17,9 +17,11 @@ # Define analogs of numpy dtypes supported by pytorch. class dtype: - def __init__(self, name): + def __init__(self, name, /): if isinstance(name, dtype): _name = name.name + elif hasattr(name, 'dtype'): + _name = name.dtype.name elif name in python_types_dict: _name = python_types_dict[name] elif name in dt_names: diff --git a/torch_np/tests/numpy_tests/core/test_dtype.py b/torch_np/tests/numpy_tests/core/test_dtype.py index 33e400b8..892bb18e 100644 --- a/torch_np/tests/numpy_tests/core/test_dtype.py +++ b/torch_np/tests/numpy_tests/core/test_dtype.py @@ -158,6 +158,7 @@ def test_pickle_types(self, DType): assert roundtrip_DType is DType +@pytest.mark.skip(reason="XXX: value-based promotions, we don't have.") class TestPromotion: """Test cases related to more complex DType promotions. Further promotion tests are defined in `test_numeric.py` @@ -246,7 +247,7 @@ def test_dtypes_are_true(): - +@pytest.mark.xfail(reason="No keyword arg for dtype ctor.") def test_keyword_argument(): # test for https://github.com/numpy/numpy/pull/16574#issuecomment-642660971 assert np.dtype(dtype=np.float64) == np.dtype(np.float64) @@ -260,6 +261,8 @@ class dt: assert np.dtype(dt) == np.float64 assert np.dtype(dt()) == np.float64 + @pytest.mark.skip(reason="We simply require the .name attribute, so this " + "fails with an AttributeError.") def test_recursion(self): class dt: pass @@ -275,7 +278,7 @@ class dt: - +@pytest.mark.skip(reason="Parameteric dtypes, our stuff is simpler.") @pytest.mark.skipif(sys.version_info < (3, 9), reason="Requires python 3.9") class TestClassGetItem: def test_dtype(self) -> None: From 699321569848929d9506583e138aee2e9df542b8 Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Fri, 6 Jan 2023 09:57:23 +0300 Subject: [PATCH 19/21] ENH: add iinfo, finfo --- torch_np/__init__.py | 3 +- torch_np/_dtypes.py | 1 + torch_np/_getlimits.py | 19 ++++++ .../tests/numpy_tests/core/test_getlimits.py | 61 +++++++++++++------ 4 files changed, 63 insertions(+), 21 deletions(-) create mode 100644 torch_np/_getlimits.py diff --git a/torch_np/__init__.py b/torch_np/__init__.py index 6c4ce60a..5060b762 100644 --- a/torch_np/__init__.py +++ b/torch_np/__init__.py @@ -7,7 +7,8 @@ from ._binary_ufuncs import * from ._ndarray import can_cast, result_type, newaxis from ._util import AxisError - +from ._getlimits import iinfo, finfo +from ._getlimits import errstate inf = float('inf') nan = float('nan') diff --git a/torch_np/_dtypes.py b/torch_np/_dtypes.py index 948df8bb..6777d2df 100644 --- a/torch_np/_dtypes.py +++ b/torch_np/_dtypes.py @@ -90,6 +90,7 @@ def __setstate__(self, value): dt_aliases_dict = { + 'u1' : 'uint8', 'i1' : 'int8', 'i2' : 'int16', 'i4' : 'int32', diff --git a/torch_np/_getlimits.py b/torch_np/_getlimits.py new file mode 100644 index 00000000..9076b32e --- /dev/null +++ b/torch_np/_getlimits.py @@ -0,0 +1,19 @@ +import torch +from . import _dtypes + +def finfo(dtyp): + torch_dtype = _dtypes.torch_dtype_from(dtyp) + return torch.finfo(torch_dtype) + + +def iinfo(dtyp): + torch_dtype = _dtypes.torch_dtype_from(dtyp) + return torch.iinfo(torch_dtype) + + +import contextlib + +# FIXME: this is only a stub +@contextlib.contextmanager +def errstate(*args, **kwds): + yield diff --git a/torch_np/tests/numpy_tests/core/test_getlimits.py b/torch_np/tests/numpy_tests/core/test_getlimits.py index b8aaba38..bbf3971f 100644 --- a/torch_np/tests/numpy_tests/core/test_getlimits.py +++ b/torch_np/tests/numpy_tests/core/test_getlimits.py @@ -2,43 +2,49 @@ """ import warnings -import numpy as np -from numpy.core import finfo, iinfo -from numpy import half, single, double, longdouble -from numpy.testing import assert_equal, assert_, assert_raises -from numpy.core.getlimits import _discovered_machar, _float_ma + +import pytest +from pytest import raises as assert_raises + +import torch_np as np +from torch_np import finfo, iinfo +from torch_np import half, single, double +from torch_np.testing import assert_equal, assert_ +#from numpy.core.getlimits import _discovered_machar, _float_ma ################################################## +@pytest.mark.skip(reason='torch.finfo is not a singleton. Why demanding it is?') class TestPythonFloat: def test_singleton(self): ftype = finfo(float) ftype2 = finfo(float) assert_equal(id(ftype), id(ftype2)) + +@pytest.mark.skip(reason='torch.finfo is not a singleton. Why demanding it is?') class TestHalf: def test_singleton(self): ftype = finfo(half) ftype2 = finfo(half) assert_equal(id(ftype), id(ftype2)) + +@pytest.mark.skip(reason='torch.finfo is not a singleton. Why demanding it is?') class TestSingle: def test_singleton(self): ftype = finfo(single) ftype2 = finfo(single) assert_equal(id(ftype), id(ftype2)) + +@pytest.mark.skip(reason='torch.finfo is not a singleton. Why demanding it is?') class TestDouble: def test_singleton(self): ftype = finfo(double) ftype2 = finfo(double) assert_equal(id(ftype), id(ftype2)) -class TestLongdouble: - def test_singleton(self): - ftype = finfo(longdouble) - ftype2 = finfo(longdouble) - assert_equal(id(ftype), id(ftype2)) class TestFinfo: def test_basic(self): @@ -46,25 +52,34 @@ def test_basic(self): [np.float16, np.float32, np.float64, np.complex64, np.complex128])) for dt1, dt2 in dts: - for attr in ('bits', 'eps', 'epsneg', 'iexp', 'machep', - 'max', 'maxexp', 'min', 'minexp', 'negep', 'nexp', - 'nmant', 'precision', 'resolution', 'tiny', - 'smallest_normal', 'smallest_subnormal'): + for attr in ('bits', 'eps', + 'max', 'min', + 'resolution', 'tiny', + 'smallest_normal',): assert_equal(getattr(finfo(dt1), attr), getattr(finfo(dt2), attr), attr) - assert_raises(ValueError, finfo, 'i4') + with assert_raises((TypeError, ValueError)): + finfo('i4') + + @pytest.mark.xfail(reason="These attributes are not implemented yet.") + def test_basic_missing(self): + dt = np.float32 + for attr in ['epsneg', 'iexp', 'machep', 'maxexp', 'minexp', 'negep', + 'nexp', 'nmant', 'precision', 'smallest_subnormal']: + getattr(finfo(dt), attr) class TestIinfo: def test_basic(self): dts = list(zip(['i1', 'i2', 'i4', 'i8', 'u1', 'u2', 'u4', 'u8'], [np.int8, np.int16, np.int32, np.int64, - np.uint8, np.uint16, np.uint32, np.uint64])) + np.uint8,])) for dt1, dt2 in dts: for attr in ('bits', 'min', 'max'): assert_equal(getattr(iinfo(dt1), attr), getattr(iinfo(dt2), attr), attr) - assert_raises(ValueError, iinfo, 'f4') + with assert_raises((TypeError, ValueError)): + iinfo('f4') def test_unsigned_max(self): types = np.sctypes['uint'] @@ -73,22 +88,25 @@ def test_unsigned_max(self): max_calculated = T(0) - T(1) assert_equal(iinfo(T).max, max_calculated) + class TestRepr: def test_iinfo_repr(self): expected = "iinfo(min=-32768, max=32767, dtype=int16)" assert_equal(repr(np.iinfo(np.int16)), expected) def test_finfo_repr(self): - expected = "finfo(resolution=1e-06, min=-3.4028235e+38," + \ - " max=3.4028235e+38, dtype=float32)" - assert_equal(repr(np.finfo(np.float32)), expected) + repr_f32 = repr(np.finfo(np.float32)) + assert "finfo(resolution=1e-06, min=-3.40282e+38," in repr_f32 + assert "dtype=float32" in repr_f32 +@pytest.mark.skip(reason="Instantiate {i,f}info from dtypes.") def test_instances(): iinfo(10) finfo(3.0) +@pytest.mark.skip(reason='MachAr no implemented (does it need to be)?') def assert_ma_equal(discovered, ma_like): # Check MachAr-like objects same as calculated MachAr instances for key, value in discovered.__dict__.items(): @@ -98,6 +116,7 @@ def assert_ma_equal(discovered, ma_like): assert_equal(value.dtype, getattr(ma_like, key).dtype) +@pytest.mark.skip(reason='MachAr no implemented (does it need to)?') def test_known_types(): # Test we are correctly compiling parameters for known types for ftype, ma_like in ((np.float16, _float_ma[16]), @@ -116,6 +135,7 @@ def test_known_types(): assert_ma_equal(ld_ma, _float_ma[128]) +@pytest.mark.skip(reason='MachAr no implemented (does it need to be)?') def test_subnormal_warning(): """Test that the subnormal is zero warning is not being raised.""" with np.errstate(all='ignore'): @@ -138,6 +158,7 @@ def test_subnormal_warning(): assert len(w) == 0 +@pytest.mark.xfail(reason="None of nmant, minexp, maxexp is implemented.") def test_plausible_finfo(): # Assert that finfo returns reasonable results for all types for ftype in np.sctypes['float'] + np.sctypes['complex']: From 2830ada06f1eea2616e3ef5cc6e6e3f0c4293110 Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Fri, 6 Jan 2023 21:16:41 +0300 Subject: [PATCH 20/21] MAINT: update .gitignore --- .gitignore | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index cfd46377..56438c6d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,5 @@ -__pycache__/* -autogen/__pycache__ -torch_np/__pycache__/* -torch_np/tests/__pycache__/* -torch_np/tests/numpy_tests/core/__pycache__/* -torch_np/testing/__pycache__/* +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] .coverage From 91b3cc6bbd8ce74ca16455a8b58e97893ff20d80 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Mon, 9 Jan 2023 14:17:23 +0000 Subject: [PATCH 21/21] Rudiementary autogen binary ufuncs input type fix --- autogen/gen_ufuncs.py | 11 ++- torch_np/tests/test_binary_ufuncs.py | 105 ++++++++++++--------------- 2 files changed, 52 insertions(+), 64 deletions(-) diff --git a/autogen/gen_ufuncs.py b/autogen/gen_ufuncs.py index f0aa9580..d275bb1e 100644 --- a/autogen/gen_ufuncs.py +++ b/autogen/gen_ufuncs.py @@ -1,4 +1,6 @@ -from dump_namespace import grab_namespace, get_signature +from collections import defaultdict +from warnings import warn +from .dump_namespace import grab_namespace, get_signature import numpy as np @@ -138,7 +140,7 @@ def test_{np_name}(): -test_header = header + """\ +test_header = """\ import numpy as np import torch @@ -168,14 +170,15 @@ def {np_name}(x1, x2, /, out=None, *, where=True, casting='same_kind', order='K' test_template = """ def test_{np_name}(): - assert_allclose(np.{np_name}(0.5, 0.6), - {np_name}(0.5, 0.6), atol=1e-7, check_dtype=False) + assert_allclose(np.{np_name}({args}), + np.{np_name}({args}), atol=1e-7, check_dtype=False) """ skip = {np.divmod, # two outputs + np.matmul, # array inputs } diff --git a/torch_np/tests/test_binary_ufuncs.py b/torch_np/tests/test_binary_ufuncs.py index a1a62980..ab3394e7 100644 --- a/torch_np/tests/test_binary_ufuncs.py +++ b/torch_np/tests/test_binary_ufuncs.py @@ -1,247 +1,232 @@ -# this file is autogenerated via gen_ufuncs.py -# do not edit manually! import numpy as np import torch -from .._binary_ufuncs import * from ..testing import assert_allclose def test_add(): assert_allclose(np.add(0.5, 0.6), - add(0.5, 0.6), atol=1e-7, check_dtype=False) + np.add(0.5, 0.6), atol=1e-7, check_dtype=False) def test_arctan2(): assert_allclose(np.arctan2(0.5, 0.6), - arctan2(0.5, 0.6), atol=1e-7, check_dtype=False) + np.arctan2(0.5, 0.6), atol=1e-7, check_dtype=False) def test_bitwise_and(): - assert_allclose(np.bitwise_and(0.5, 0.6), - bitwise_and(0.5, 0.6), atol=1e-7, check_dtype=False) + assert_allclose(np.bitwise_and(5, 6), + np.bitwise_and(5, 6), atol=1e-7, check_dtype=False) def test_bitwise_or(): - assert_allclose(np.bitwise_or(0.5, 0.6), - bitwise_or(0.5, 0.6), atol=1e-7, check_dtype=False) + assert_allclose(np.bitwise_or(5, 6), + np.bitwise_or(5, 6), atol=1e-7, check_dtype=False) def test_bitwise_xor(): - assert_allclose(np.bitwise_xor(0.5, 0.6), - bitwise_xor(0.5, 0.6), atol=1e-7, check_dtype=False) + assert_allclose(np.bitwise_xor(5, 6), + np.bitwise_xor(5, 6), atol=1e-7, check_dtype=False) def test_copysign(): assert_allclose(np.copysign(0.5, 0.6), - copysign(0.5, 0.6), atol=1e-7, check_dtype=False) + np.copysign(0.5, 0.6), atol=1e-7, check_dtype=False) def test_divide(): assert_allclose(np.divide(0.5, 0.6), - divide(0.5, 0.6), atol=1e-7, check_dtype=False) + np.divide(0.5, 0.6), atol=1e-7, check_dtype=False) def test_equal(): assert_allclose(np.equal(0.5, 0.6), - equal(0.5, 0.6), atol=1e-7, check_dtype=False) + np.equal(0.5, 0.6), atol=1e-7, check_dtype=False) def test_float_power(): assert_allclose(np.float_power(0.5, 0.6), - float_power(0.5, 0.6), atol=1e-7, check_dtype=False) + np.float_power(0.5, 0.6), atol=1e-7, check_dtype=False) def test_floor_divide(): assert_allclose(np.floor_divide(0.5, 0.6), - floor_divide(0.5, 0.6), atol=1e-7, check_dtype=False) + np.floor_divide(0.5, 0.6), atol=1e-7, check_dtype=False) def test_fmax(): assert_allclose(np.fmax(0.5, 0.6), - fmax(0.5, 0.6), atol=1e-7, check_dtype=False) + np.fmax(0.5, 0.6), atol=1e-7, check_dtype=False) def test_fmin(): assert_allclose(np.fmin(0.5, 0.6), - fmin(0.5, 0.6), atol=1e-7, check_dtype=False) + np.fmin(0.5, 0.6), atol=1e-7, check_dtype=False) def test_fmod(): assert_allclose(np.fmod(0.5, 0.6), - fmod(0.5, 0.6), atol=1e-7, check_dtype=False) + np.fmod(0.5, 0.6), atol=1e-7, check_dtype=False) def test_gcd(): - assert_allclose(np.gcd(0.5, 0.6), - gcd(0.5, 0.6), atol=1e-7, check_dtype=False) + assert_allclose(np.gcd(5, 6), + np.gcd(5, 6), atol=1e-7, check_dtype=False) def test_greater(): assert_allclose(np.greater(0.5, 0.6), - greater(0.5, 0.6), atol=1e-7, check_dtype=False) + np.greater(0.5, 0.6), atol=1e-7, check_dtype=False) def test_greater_equal(): assert_allclose(np.greater_equal(0.5, 0.6), - greater_equal(0.5, 0.6), atol=1e-7, check_dtype=False) + np.greater_equal(0.5, 0.6), atol=1e-7, check_dtype=False) def test_heaviside(): assert_allclose(np.heaviside(0.5, 0.6), - heaviside(0.5, 0.6), atol=1e-7, check_dtype=False) + np.heaviside(0.5, 0.6), atol=1e-7, check_dtype=False) def test_hypot(): assert_allclose(np.hypot(0.5, 0.6), - hypot(0.5, 0.6), atol=1e-7, check_dtype=False) + np.hypot(0.5, 0.6), atol=1e-7, check_dtype=False) def test_lcm(): - assert_allclose(np.lcm(0.5, 0.6), - lcm(0.5, 0.6), atol=1e-7, check_dtype=False) - - - -def test_ldexp(): - assert_allclose(np.ldexp(0.5, 0.6), - ldexp(0.5, 0.6), atol=1e-7, check_dtype=False) + assert_allclose(np.lcm(5, 6), + np.lcm(5, 6), atol=1e-7, check_dtype=False) def test_left_shift(): - assert_allclose(np.left_shift(0.5, 0.6), - left_shift(0.5, 0.6), atol=1e-7, check_dtype=False) + assert_allclose(np.left_shift(5, 6), + np.left_shift(5, 6), atol=1e-7, check_dtype=False) def test_less(): assert_allclose(np.less(0.5, 0.6), - less(0.5, 0.6), atol=1e-7, check_dtype=False) + np.less(0.5, 0.6), atol=1e-7, check_dtype=False) def test_less_equal(): assert_allclose(np.less_equal(0.5, 0.6), - less_equal(0.5, 0.6), atol=1e-7, check_dtype=False) + np.less_equal(0.5, 0.6), atol=1e-7, check_dtype=False) def test_logaddexp(): assert_allclose(np.logaddexp(0.5, 0.6), - logaddexp(0.5, 0.6), atol=1e-7, check_dtype=False) + np.logaddexp(0.5, 0.6), atol=1e-7, check_dtype=False) def test_logaddexp2(): assert_allclose(np.logaddexp2(0.5, 0.6), - logaddexp2(0.5, 0.6), atol=1e-7, check_dtype=False) + np.logaddexp2(0.5, 0.6), atol=1e-7, check_dtype=False) def test_logical_and(): assert_allclose(np.logical_and(0.5, 0.6), - logical_and(0.5, 0.6), atol=1e-7, check_dtype=False) + np.logical_and(0.5, 0.6), atol=1e-7, check_dtype=False) def test_logical_or(): assert_allclose(np.logical_or(0.5, 0.6), - logical_or(0.5, 0.6), atol=1e-7, check_dtype=False) + np.logical_or(0.5, 0.6), atol=1e-7, check_dtype=False) def test_logical_xor(): assert_allclose(np.logical_xor(0.5, 0.6), - logical_xor(0.5, 0.6), atol=1e-7, check_dtype=False) - - - -def test_matmul(): - assert_allclose(np.matmul(0.5, 0.6), - matmul(0.5, 0.6), atol=1e-7, check_dtype=False) + np.logical_xor(0.5, 0.6), atol=1e-7, check_dtype=False) def test_maximum(): assert_allclose(np.maximum(0.5, 0.6), - maximum(0.5, 0.6), atol=1e-7, check_dtype=False) + np.maximum(0.5, 0.6), atol=1e-7, check_dtype=False) def test_minimum(): assert_allclose(np.minimum(0.5, 0.6), - minimum(0.5, 0.6), atol=1e-7, check_dtype=False) + np.minimum(0.5, 0.6), atol=1e-7, check_dtype=False) def test_remainder(): assert_allclose(np.remainder(0.5, 0.6), - remainder(0.5, 0.6), atol=1e-7, check_dtype=False) + np.remainder(0.5, 0.6), atol=1e-7, check_dtype=False) def test_multiply(): assert_allclose(np.multiply(0.5, 0.6), - multiply(0.5, 0.6), atol=1e-7, check_dtype=False) + np.multiply(0.5, 0.6), atol=1e-7, check_dtype=False) def test_nextafter(): assert_allclose(np.nextafter(0.5, 0.6), - nextafter(0.5, 0.6), atol=1e-7, check_dtype=False) + np.nextafter(0.5, 0.6), atol=1e-7, check_dtype=False) def test_not_equal(): assert_allclose(np.not_equal(0.5, 0.6), - not_equal(0.5, 0.6), atol=1e-7, check_dtype=False) + np.not_equal(0.5, 0.6), atol=1e-7, check_dtype=False) def test_power(): assert_allclose(np.power(0.5, 0.6), - power(0.5, 0.6), atol=1e-7, check_dtype=False) + np.power(0.5, 0.6), atol=1e-7, check_dtype=False) def test_remainder(): assert_allclose(np.remainder(0.5, 0.6), - remainder(0.5, 0.6), atol=1e-7, check_dtype=False) + np.remainder(0.5, 0.6), atol=1e-7, check_dtype=False) def test_right_shift(): - assert_allclose(np.right_shift(0.5, 0.6), - right_shift(0.5, 0.6), atol=1e-7, check_dtype=False) + assert_allclose(np.right_shift(5, 6), + np.right_shift(5, 6), atol=1e-7, check_dtype=False) def test_subtract(): assert_allclose(np.subtract(0.5, 0.6), - subtract(0.5, 0.6), atol=1e-7, check_dtype=False) + np.subtract(0.5, 0.6), atol=1e-7, check_dtype=False) def test_divide(): assert_allclose(np.divide(0.5, 0.6), - divide(0.5, 0.6), atol=1e-7, check_dtype=False) + np.divide(0.5, 0.6), atol=1e-7, check_dtype=False)