From 665c4de271703a90b063ce955992d1c74fcca832 Mon Sep 17 00:00:00 2001 From: Stainless Bot <107565488+stainless-bot@users.noreply.github.com> Date: Mon, 4 Mar 2024 13:52:38 +0100 Subject: [PATCH 1/3] chore(internal): split up transforms into sync / async (#304) --- src/finch/_utils/__init__.py | 2 + src/finch/_utils/_transform.py | 128 +++++++++- src/finch/resources/hris/benefits/benefits.py | 9 +- src/finch/resources/jobs/automated.py | 7 +- src/finch/resources/request_forwarding.py | 7 +- src/finch/resources/sandbox/company.py | 7 +- .../resources/sandbox/connections/accounts.py | 11 +- .../sandbox/connections/connections.py | 7 +- src/finch/resources/sandbox/directory.py | 7 +- src/finch/resources/sandbox/employment.py | 7 +- src/finch/resources/sandbox/individual.py | 7 +- .../resources/sandbox/jobs/configuration.py | 7 +- src/finch/resources/sandbox/jobs/jobs.py | 7 +- src/finch/resources/sandbox/payment.py | 7 +- tests/test_transform.py | 220 ++++++++++++------ 15 files changed, 339 insertions(+), 101 deletions(-) diff --git a/src/finch/_utils/__init__.py b/src/finch/_utils/__init__.py index b5790a87..56978941 100644 --- a/src/finch/_utils/__init__.py +++ b/src/finch/_utils/__init__.py @@ -44,5 +44,7 @@ from ._transform import ( PropertyInfo as PropertyInfo, transform as transform, + async_transform as async_transform, maybe_transform as maybe_transform, + async_maybe_transform as async_maybe_transform, ) diff --git a/src/finch/_utils/_transform.py b/src/finch/_utils/_transform.py index 2cb7726c..9c769306 100644 --- a/src/finch/_utils/_transform.py +++ b/src/finch/_utils/_transform.py @@ -180,11 +180,7 @@ def _transform_recursive( if isinstance(data, pydantic.BaseModel): return model_dump(data, exclude_unset=True) - return _transform_value(data, annotation) - - -def _transform_value(data: object, type_: type) -> object: - annotated_type = _get_annotated_type(type_) + annotated_type = _get_annotated_type(annotation) if annotated_type is None: return data @@ -222,3 +218,125 @@ def _transform_typeddict( else: result[_maybe_transform_key(key, type_)] = _transform_recursive(value, annotation=type_) return result + + +async def async_maybe_transform( + data: object, + expected_type: object, +) -> Any | None: + """Wrapper over `async_transform()` that allows `None` to be passed. + + See `async_transform()` for more details. + """ + if data is None: + return None + return await async_transform(data, expected_type) + + +async def async_transform( + data: _T, + expected_type: object, +) -> _T: + """Transform dictionaries based off of type information from the given type, for example: + + ```py + class Params(TypedDict, total=False): + card_id: Required[Annotated[str, PropertyInfo(alias="cardID")]] + + + transformed = transform({"card_id": ""}, Params) + # {'cardID': ''} + ``` + + Any keys / data that does not have type information given will be included as is. + + It should be noted that the transformations that this function does are not represented in the type system. + """ + transformed = await _async_transform_recursive(data, annotation=cast(type, expected_type)) + return cast(_T, transformed) + + +async def _async_transform_recursive( + data: object, + *, + annotation: type, + inner_type: type | None = None, +) -> object: + """Transform the given data against the expected type. + + Args: + annotation: The direct type annotation given to the particular piece of data. + This may or may not be wrapped in metadata types, e.g. `Required[T]`, `Annotated[T, ...]` etc + + inner_type: If applicable, this is the "inside" type. This is useful in certain cases where the outside type + is a container type such as `List[T]`. In that case `inner_type` should be set to `T` so that each entry in + the list can be transformed using the metadata from the container type. + + Defaults to the same value as the `annotation` argument. + """ + if inner_type is None: + inner_type = annotation + + stripped_type = strip_annotated_type(inner_type) + if is_typeddict(stripped_type) and is_mapping(data): + return await _async_transform_typeddict(data, stripped_type) + + if ( + # List[T] + (is_list_type(stripped_type) and is_list(data)) + # Iterable[T] + or (is_iterable_type(stripped_type) and is_iterable(data) and not isinstance(data, str)) + ): + inner_type = extract_type_arg(stripped_type, 0) + return [await _async_transform_recursive(d, annotation=annotation, inner_type=inner_type) for d in data] + + if is_union_type(stripped_type): + # For union types we run the transformation against all subtypes to ensure that everything is transformed. + # + # TODO: there may be edge cases where the same normalized field name will transform to two different names + # in different subtypes. + for subtype in get_args(stripped_type): + data = await _async_transform_recursive(data, annotation=annotation, inner_type=subtype) + return data + + if isinstance(data, pydantic.BaseModel): + return model_dump(data, exclude_unset=True) + + annotated_type = _get_annotated_type(annotation) + if annotated_type is None: + return data + + # ignore the first argument as it is the actual type + annotations = get_args(annotated_type)[1:] + for annotation in annotations: + if isinstance(annotation, PropertyInfo) and annotation.format is not None: + return await _async_format_data(data, annotation.format, annotation.format_template) + + return data + + +async def _async_format_data(data: object, format_: PropertyFormat, format_template: str | None) -> object: + if isinstance(data, (date, datetime)): + if format_ == "iso8601": + return data.isoformat() + + if format_ == "custom" and format_template is not None: + return data.strftime(format_template) + + return data + + +async def _async_transform_typeddict( + data: Mapping[str, object], + expected_type: type, +) -> Mapping[str, object]: + result: dict[str, object] = {} + annotations = get_type_hints(expected_type, include_extras=True) + for key, value in data.items(): + type_ = annotations.get(key) + if type_ is None: + # we do not have a type annotation for this field, leave it as is + result[key] = value + else: + result[_maybe_transform_key(key, type_)] = await _async_transform_recursive(value, annotation=type_) + return result diff --git a/src/finch/resources/hris/benefits/benefits.py b/src/finch/resources/hris/benefits/benefits.py index 5e712445..ae7e1179 100644 --- a/src/finch/resources/hris/benefits/benefits.py +++ b/src/finch/resources/hris/benefits/benefits.py @@ -8,7 +8,10 @@ from .... import _legacy_response from ...._types import NOT_GIVEN, Body, Query, Headers, NotGiven -from ...._utils import maybe_transform +from ...._utils import ( + maybe_transform, + async_maybe_transform, +) from ...._compat import cached_property from .individuals import ( Individuals, @@ -267,7 +270,7 @@ async def create( """ return await self._post( "/employer/benefits", - body=maybe_transform( + body=await async_maybe_transform( { "description": description, "frequency": frequency, @@ -348,7 +351,7 @@ async def update( raise ValueError(f"Expected a non-empty value for `benefit_id` but received {benefit_id!r}") return await self._post( f"/employer/benefits/{benefit_id}", - body=maybe_transform({"description": description}, benefit_update_params.BenefitUpdateParams), + body=await async_maybe_transform({"description": description}, benefit_update_params.BenefitUpdateParams), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), diff --git a/src/finch/resources/jobs/automated.py b/src/finch/resources/jobs/automated.py index 13b460ff..d2818e0b 100644 --- a/src/finch/resources/jobs/automated.py +++ b/src/finch/resources/jobs/automated.py @@ -8,7 +8,10 @@ from ... import _legacy_response from ..._types import NOT_GIVEN, Body, Query, Headers, NotGiven -from ..._utils import maybe_transform +from ..._utils import ( + maybe_transform, + async_maybe_transform, +) from ..._compat import cached_property from ..._resource import SyncAPIResource, AsyncAPIResource from ..._response import to_streamed_response_wrapper, async_to_streamed_response_wrapper @@ -205,7 +208,7 @@ async def create( """ return await self._post( "/jobs/automated", - body=maybe_transform({"type": type}, automated_create_params.AutomatedCreateParams), + body=await async_maybe_transform({"type": type}, automated_create_params.AutomatedCreateParams), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), diff --git a/src/finch/resources/request_forwarding.py b/src/finch/resources/request_forwarding.py index 09595451..31e9472d 100644 --- a/src/finch/resources/request_forwarding.py +++ b/src/finch/resources/request_forwarding.py @@ -9,7 +9,10 @@ from .. import _legacy_response from ..types import RequestForwardingForwardResponse, request_forwarding_forward_params from .._types import NOT_GIVEN, Body, Query, Headers, NotGiven -from .._utils import maybe_transform +from .._utils import ( + maybe_transform, + async_maybe_transform, +) from .._compat import cached_property from .._resource import SyncAPIResource, AsyncAPIResource from .._response import to_streamed_response_wrapper, async_to_streamed_response_wrapper @@ -155,7 +158,7 @@ async def forward( """ return await self._post( "/forward", - body=maybe_transform( + body=await async_maybe_transform( { "method": method, "route": route, diff --git a/src/finch/resources/sandbox/company.py b/src/finch/resources/sandbox/company.py index 30a1f699..267055b4 100644 --- a/src/finch/resources/sandbox/company.py +++ b/src/finch/resources/sandbox/company.py @@ -9,7 +9,10 @@ from ... import _legacy_response from ...types import LocationParam from ..._types import NOT_GIVEN, Body, Query, Headers, NotGiven -from ..._utils import maybe_transform +from ..._utils import ( + maybe_transform, + async_maybe_transform, +) from ..._compat import cached_property from ..._resource import SyncAPIResource, AsyncAPIResource from ..._response import to_streamed_response_wrapper, async_to_streamed_response_wrapper @@ -151,7 +154,7 @@ async def update( """ return await self._put( "/sandbox/company", - body=maybe_transform( + body=await async_maybe_transform( { "accounts": accounts, "departments": departments, diff --git a/src/finch/resources/sandbox/connections/accounts.py b/src/finch/resources/sandbox/connections/accounts.py index 5de0e2b6..46cc1b77 100644 --- a/src/finch/resources/sandbox/connections/accounts.py +++ b/src/finch/resources/sandbox/connections/accounts.py @@ -9,7 +9,10 @@ from .... import _legacy_response from ...._types import NOT_GIVEN, Body, Query, Headers, NotGiven -from ...._utils import maybe_transform +from ...._utils import ( + maybe_transform, + async_maybe_transform, +) from ...._compat import cached_property from ...._resource import SyncAPIResource, AsyncAPIResource from ...._response import to_streamed_response_wrapper, async_to_streamed_response_wrapper @@ -157,7 +160,7 @@ async def create( """ return await self._post( "/sandbox/connections/accounts", - body=maybe_transform( + body=await async_maybe_transform( { "company_id": company_id, "provider_id": provider_id, @@ -199,7 +202,9 @@ async def update( """ return await self._put( "/sandbox/connections/accounts", - body=maybe_transform({"connection_status": connection_status}, account_update_params.AccountUpdateParams), + body=await async_maybe_transform( + {"connection_status": connection_status}, account_update_params.AccountUpdateParams + ), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), diff --git a/src/finch/resources/sandbox/connections/connections.py b/src/finch/resources/sandbox/connections/connections.py index eaecbcbe..9a5f4b83 100644 --- a/src/finch/resources/sandbox/connections/connections.py +++ b/src/finch/resources/sandbox/connections/connections.py @@ -17,7 +17,10 @@ AsyncAccountsWithStreamingResponse, ) from ...._types import NOT_GIVEN, Body, Query, Headers, NotGiven -from ...._utils import maybe_transform +from ...._utils import ( + maybe_transform, + async_maybe_transform, +) from ...._compat import cached_property from ...._resource import SyncAPIResource, AsyncAPIResource from ...._response import to_streamed_response_wrapper, async_to_streamed_response_wrapper @@ -133,7 +136,7 @@ async def create( """ return await self._post( "/sandbox/connections", - body=maybe_transform( + body=await async_maybe_transform( { "provider_id": provider_id, "authentication_type": authentication_type, diff --git a/src/finch/resources/sandbox/directory.py b/src/finch/resources/sandbox/directory.py index 27968f85..1de04caa 100644 --- a/src/finch/resources/sandbox/directory.py +++ b/src/finch/resources/sandbox/directory.py @@ -8,7 +8,10 @@ from ... import _legacy_response from ..._types import NOT_GIVEN, Body, Query, Headers, NotGiven -from ..._utils import maybe_transform +from ..._utils import ( + maybe_transform, + async_maybe_transform, +) from ..._compat import cached_property from ..._resource import SyncAPIResource, AsyncAPIResource from ..._response import to_streamed_response_wrapper, async_to_streamed_response_wrapper @@ -102,7 +105,7 @@ async def create( """ return await self._post( "/sandbox/directory", - body=maybe_transform(body, directory_create_params.DirectoryCreateParams), + body=await async_maybe_transform(body, directory_create_params.DirectoryCreateParams), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), diff --git a/src/finch/resources/sandbox/employment.py b/src/finch/resources/sandbox/employment.py index a4707fc9..0c71d804 100644 --- a/src/finch/resources/sandbox/employment.py +++ b/src/finch/resources/sandbox/employment.py @@ -9,7 +9,10 @@ from ... import _legacy_response from ...types import IncomeParam, LocationParam from ..._types import NOT_GIVEN, Body, Query, Headers, NotGiven -from ..._utils import maybe_transform +from ..._utils import ( + maybe_transform, + async_maybe_transform, +) from ..._compat import cached_property from ..._resource import SyncAPIResource, AsyncAPIResource from ..._response import to_streamed_response_wrapper, async_to_streamed_response_wrapper @@ -217,7 +220,7 @@ async def update( raise ValueError(f"Expected a non-empty value for `individual_id` but received {individual_id!r}") return await self._put( f"/sandbox/employment/{individual_id}", - body=maybe_transform( + body=await async_maybe_transform( { "class_code": class_code, "custom_fields": custom_fields, diff --git a/src/finch/resources/sandbox/individual.py b/src/finch/resources/sandbox/individual.py index fee47057..68d0fb4a 100644 --- a/src/finch/resources/sandbox/individual.py +++ b/src/finch/resources/sandbox/individual.py @@ -10,7 +10,10 @@ from ... import _legacy_response from ...types import LocationParam from ..._types import NOT_GIVEN, Body, Query, Headers, NotGiven -from ..._utils import maybe_transform +from ..._utils import ( + maybe_transform, + async_maybe_transform, +) from ..._compat import cached_property from ..._resource import SyncAPIResource, AsyncAPIResource from ..._response import to_streamed_response_wrapper, async_to_streamed_response_wrapper @@ -210,7 +213,7 @@ async def update( raise ValueError(f"Expected a non-empty value for `individual_id` but received {individual_id!r}") return await self._put( f"/sandbox/individual/{individual_id}", - body=maybe_transform( + body=await async_maybe_transform( { "dob": dob, "emails": emails, diff --git a/src/finch/resources/sandbox/jobs/configuration.py b/src/finch/resources/sandbox/jobs/configuration.py index de8e1c18..9f6e53d8 100644 --- a/src/finch/resources/sandbox/jobs/configuration.py +++ b/src/finch/resources/sandbox/jobs/configuration.py @@ -8,7 +8,10 @@ from .... import _legacy_response from ...._types import NOT_GIVEN, Body, Query, Headers, NotGiven -from ...._utils import maybe_transform +from ...._utils import ( + maybe_transform, + async_maybe_transform, +) from ...._compat import cached_property from ...._resource import SyncAPIResource, AsyncAPIResource from ...._response import to_streamed_response_wrapper, async_to_streamed_response_wrapper @@ -142,7 +145,7 @@ async def update( """ return await self._put( "/sandbox/jobs/configuration", - body=maybe_transform( + body=await async_maybe_transform( { "completion_status": completion_status, "type": type, diff --git a/src/finch/resources/sandbox/jobs/jobs.py b/src/finch/resources/sandbox/jobs/jobs.py index a65d5bb2..cd7b2f3f 100644 --- a/src/finch/resources/sandbox/jobs/jobs.py +++ b/src/finch/resources/sandbox/jobs/jobs.py @@ -8,7 +8,10 @@ from .... import _legacy_response from ...._types import NOT_GIVEN, Body, Query, Headers, NotGiven -from ...._utils import maybe_transform +from ...._utils import ( + maybe_transform, + async_maybe_transform, +) from ...._compat import cached_property from ...._resource import SyncAPIResource, AsyncAPIResource from ...._response import to_streamed_response_wrapper, async_to_streamed_response_wrapper @@ -118,7 +121,7 @@ async def create( """ return await self._post( "/sandbox/jobs", - body=maybe_transform({"type": type}, job_create_params.JobCreateParams), + body=await async_maybe_transform({"type": type}, job_create_params.JobCreateParams), options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), diff --git a/src/finch/resources/sandbox/payment.py b/src/finch/resources/sandbox/payment.py index f23e7249..471e67c9 100644 --- a/src/finch/resources/sandbox/payment.py +++ b/src/finch/resources/sandbox/payment.py @@ -8,7 +8,10 @@ from ... import _legacy_response from ..._types import NOT_GIVEN, Body, Query, Headers, NotGiven -from ..._utils import maybe_transform +from ..._utils import ( + maybe_transform, + async_maybe_transform, +) from ..._compat import cached_property from ..._resource import SyncAPIResource, AsyncAPIResource from ..._response import to_streamed_response_wrapper, async_to_streamed_response_wrapper @@ -107,7 +110,7 @@ async def create( """ return await self._post( "/sandbox/payment", - body=maybe_transform( + body=await async_maybe_transform( { "end_date": end_date, "pay_statements": pay_statements, diff --git a/tests/test_transform.py b/tests/test_transform.py index 5853a968..da9a5f15 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -1,22 +1,45 @@ from __future__ import annotations -from typing import Any, List, Union, Iterable, Optional, cast +from typing import Any, List, Union, TypeVar, Iterable, Optional, cast from datetime import date, datetime from typing_extensions import Required, Annotated, TypedDict import pytest -from finch._utils import PropertyInfo, transform, parse_datetime +from finch._utils import ( + PropertyInfo, + transform as _transform, + parse_datetime, + async_transform as _async_transform, +) from finch._compat import PYDANTIC_V2 from finch._models import BaseModel +_T = TypeVar("_T") + + +async def transform( + data: _T, + expected_type: object, + use_async: bool, +) -> _T: + if use_async: + return await _async_transform(data, expected_type=expected_type) + + return _transform(data, expected_type=expected_type) + + +parametrize = pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) + class Foo1(TypedDict): foo_bar: Annotated[str, PropertyInfo(alias="fooBar")] -def test_top_level_alias() -> None: - assert transform({"foo_bar": "hello"}, expected_type=Foo1) == {"fooBar": "hello"} +@parametrize +@pytest.mark.asyncio +async def test_top_level_alias(use_async: bool) -> None: + assert await transform({"foo_bar": "hello"}, expected_type=Foo1, use_async=use_async) == {"fooBar": "hello"} class Foo2(TypedDict): @@ -32,9 +55,11 @@ class Baz2(TypedDict): my_baz: Annotated[str, PropertyInfo(alias="myBaz")] -def test_recursive_typeddict() -> None: - assert transform({"bar": {"this_thing": 1}}, Foo2) == {"bar": {"this__thing": 1}} - assert transform({"bar": {"baz": {"my_baz": "foo"}}}, Foo2) == {"bar": {"Baz": {"myBaz": "foo"}}} +@parametrize +@pytest.mark.asyncio +async def test_recursive_typeddict(use_async: bool) -> None: + assert await transform({"bar": {"this_thing": 1}}, Foo2, use_async) == {"bar": {"this__thing": 1}} + assert await transform({"bar": {"baz": {"my_baz": "foo"}}}, Foo2, use_async) == {"bar": {"Baz": {"myBaz": "foo"}}} class Foo3(TypedDict): @@ -45,8 +70,10 @@ class Bar3(TypedDict): my_field: Annotated[str, PropertyInfo(alias="myField")] -def test_list_of_typeddict() -> None: - result = transform({"things": [{"my_field": "foo"}, {"my_field": "foo2"}]}, expected_type=Foo3) +@parametrize +@pytest.mark.asyncio +async def test_list_of_typeddict(use_async: bool) -> None: + result = await transform({"things": [{"my_field": "foo"}, {"my_field": "foo2"}]}, Foo3, use_async) assert result == {"things": [{"myField": "foo"}, {"myField": "foo2"}]} @@ -62,10 +89,14 @@ class Baz4(TypedDict): foo_baz: Annotated[str, PropertyInfo(alias="fooBaz")] -def test_union_of_typeddict() -> None: - assert transform({"foo": {"foo_bar": "bar"}}, Foo4) == {"foo": {"fooBar": "bar"}} - assert transform({"foo": {"foo_baz": "baz"}}, Foo4) == {"foo": {"fooBaz": "baz"}} - assert transform({"foo": {"foo_baz": "baz", "foo_bar": "bar"}}, Foo4) == {"foo": {"fooBaz": "baz", "fooBar": "bar"}} +@parametrize +@pytest.mark.asyncio +async def test_union_of_typeddict(use_async: bool) -> None: + assert await transform({"foo": {"foo_bar": "bar"}}, Foo4, use_async) == {"foo": {"fooBar": "bar"}} + assert await transform({"foo": {"foo_baz": "baz"}}, Foo4, use_async) == {"foo": {"fooBaz": "baz"}} + assert await transform({"foo": {"foo_baz": "baz", "foo_bar": "bar"}}, Foo4, use_async) == { + "foo": {"fooBaz": "baz", "fooBar": "bar"} + } class Foo5(TypedDict): @@ -80,9 +111,11 @@ class Baz5(TypedDict): foo_baz: Annotated[str, PropertyInfo(alias="fooBaz")] -def test_union_of_list() -> None: - assert transform({"foo": {"foo_bar": "bar"}}, Foo5) == {"FOO": {"fooBar": "bar"}} - assert transform( +@parametrize +@pytest.mark.asyncio +async def test_union_of_list(use_async: bool) -> None: + assert await transform({"foo": {"foo_bar": "bar"}}, Foo5, use_async) == {"FOO": {"fooBar": "bar"}} + assert await transform( { "foo": [ {"foo_baz": "baz"}, @@ -90,6 +123,7 @@ def test_union_of_list() -> None: ] }, Foo5, + use_async, ) == {"FOO": [{"fooBaz": "baz"}, {"fooBaz": "baz"}]} @@ -97,8 +131,10 @@ class Foo6(TypedDict): bar: Annotated[str, PropertyInfo(alias="Bar")] -def test_includes_unknown_keys() -> None: - assert transform({"bar": "bar", "baz_": {"FOO": 1}}, Foo6) == { +@parametrize +@pytest.mark.asyncio +async def test_includes_unknown_keys(use_async: bool) -> None: + assert await transform({"bar": "bar", "baz_": {"FOO": 1}}, Foo6, use_async) == { "Bar": "bar", "baz_": {"FOO": 1}, } @@ -113,9 +149,11 @@ class Bar7(TypedDict): foo: str -def test_ignores_invalid_input() -> None: - assert transform({"bar": ""}, Foo7) == {"bAr": ""} - assert transform({"foo": ""}, Foo7) == {"foo": ""} +@parametrize +@pytest.mark.asyncio +async def test_ignores_invalid_input(use_async: bool) -> None: + assert await transform({"bar": ""}, Foo7, use_async) == {"bAr": ""} + assert await transform({"foo": ""}, Foo7, use_async) == {"foo": ""} class DatetimeDict(TypedDict, total=False): @@ -134,52 +172,66 @@ class DateDict(TypedDict, total=False): foo: Annotated[date, PropertyInfo(format="iso8601")] -def test_iso8601_format() -> None: +@parametrize +@pytest.mark.asyncio +async def test_iso8601_format(use_async: bool) -> None: dt = datetime.fromisoformat("2023-02-23T14:16:36.337692+00:00") - assert transform({"foo": dt}, DatetimeDict) == {"foo": "2023-02-23T14:16:36.337692+00:00"} # type: ignore[comparison-overlap] + assert await transform({"foo": dt}, DatetimeDict, use_async) == {"foo": "2023-02-23T14:16:36.337692+00:00"} # type: ignore[comparison-overlap] dt = dt.replace(tzinfo=None) - assert transform({"foo": dt}, DatetimeDict) == {"foo": "2023-02-23T14:16:36.337692"} # type: ignore[comparison-overlap] + assert await transform({"foo": dt}, DatetimeDict, use_async) == {"foo": "2023-02-23T14:16:36.337692"} # type: ignore[comparison-overlap] - assert transform({"foo": None}, DateDict) == {"foo": None} # type: ignore[comparison-overlap] - assert transform({"foo": date.fromisoformat("2023-02-23")}, DateDict) == {"foo": "2023-02-23"} # type: ignore[comparison-overlap] + assert await transform({"foo": None}, DateDict, use_async) == {"foo": None} # type: ignore[comparison-overlap] + assert await transform({"foo": date.fromisoformat("2023-02-23")}, DateDict, use_async) == {"foo": "2023-02-23"} # type: ignore[comparison-overlap] -def test_optional_iso8601_format() -> None: +@parametrize +@pytest.mark.asyncio +async def test_optional_iso8601_format(use_async: bool) -> None: dt = datetime.fromisoformat("2023-02-23T14:16:36.337692+00:00") - assert transform({"bar": dt}, DatetimeDict) == {"bar": "2023-02-23T14:16:36.337692+00:00"} # type: ignore[comparison-overlap] + assert await transform({"bar": dt}, DatetimeDict, use_async) == {"bar": "2023-02-23T14:16:36.337692+00:00"} # type: ignore[comparison-overlap] - assert transform({"bar": None}, DatetimeDict) == {"bar": None} + assert await transform({"bar": None}, DatetimeDict, use_async) == {"bar": None} -def test_required_iso8601_format() -> None: +@parametrize +@pytest.mark.asyncio +async def test_required_iso8601_format(use_async: bool) -> None: dt = datetime.fromisoformat("2023-02-23T14:16:36.337692+00:00") - assert transform({"required": dt}, DatetimeDict) == {"required": "2023-02-23T14:16:36.337692+00:00"} # type: ignore[comparison-overlap] + assert await transform({"required": dt}, DatetimeDict, use_async) == { + "required": "2023-02-23T14:16:36.337692+00:00" + } # type: ignore[comparison-overlap] - assert transform({"required": None}, DatetimeDict) == {"required": None} + assert await transform({"required": None}, DatetimeDict, use_async) == {"required": None} -def test_union_datetime() -> None: +@parametrize +@pytest.mark.asyncio +async def test_union_datetime(use_async: bool) -> None: dt = datetime.fromisoformat("2023-02-23T14:16:36.337692+00:00") - assert transform({"union": dt}, DatetimeDict) == { # type: ignore[comparison-overlap] + assert await transform({"union": dt}, DatetimeDict, use_async) == { # type: ignore[comparison-overlap] "union": "2023-02-23T14:16:36.337692+00:00" } - assert transform({"union": "foo"}, DatetimeDict) == {"union": "foo"} + assert await transform({"union": "foo"}, DatetimeDict, use_async) == {"union": "foo"} -def test_nested_list_iso6801_format() -> None: +@parametrize +@pytest.mark.asyncio +async def test_nested_list_iso6801_format(use_async: bool) -> None: dt1 = datetime.fromisoformat("2023-02-23T14:16:36.337692+00:00") dt2 = parse_datetime("2022-01-15T06:34:23Z") - assert transform({"list_": [dt1, dt2]}, DatetimeDict) == { # type: ignore[comparison-overlap] + assert await transform({"list_": [dt1, dt2]}, DatetimeDict, use_async) == { # type: ignore[comparison-overlap] "list_": ["2023-02-23T14:16:36.337692+00:00", "2022-01-15T06:34:23+00:00"] } -def test_datetime_custom_format() -> None: +@parametrize +@pytest.mark.asyncio +async def test_datetime_custom_format(use_async: bool) -> None: dt = parse_datetime("2022-01-15T06:34:23Z") - result = transform(dt, Annotated[datetime, PropertyInfo(format="custom", format_template="%H")]) + result = await transform(dt, Annotated[datetime, PropertyInfo(format="custom", format_template="%H")], use_async) assert result == "06" # type: ignore[comparison-overlap] @@ -187,47 +239,59 @@ class DateDictWithRequiredAlias(TypedDict, total=False): required_prop: Required[Annotated[date, PropertyInfo(format="iso8601", alias="prop")]] -def test_datetime_with_alias() -> None: - assert transform({"required_prop": None}, DateDictWithRequiredAlias) == {"prop": None} # type: ignore[comparison-overlap] - assert transform({"required_prop": date.fromisoformat("2023-02-23")}, DateDictWithRequiredAlias) == { - "prop": "2023-02-23" - } # type: ignore[comparison-overlap] +@parametrize +@pytest.mark.asyncio +async def test_datetime_with_alias(use_async: bool) -> None: + assert await transform({"required_prop": None}, DateDictWithRequiredAlias, use_async) == {"prop": None} # type: ignore[comparison-overlap] + assert await transform( + {"required_prop": date.fromisoformat("2023-02-23")}, DateDictWithRequiredAlias, use_async + ) == {"prop": "2023-02-23"} # type: ignore[comparison-overlap] class MyModel(BaseModel): foo: str -def test_pydantic_model_to_dictionary() -> None: - assert transform(MyModel(foo="hi!"), Any) == {"foo": "hi!"} - assert transform(MyModel.construct(foo="hi!"), Any) == {"foo": "hi!"} +@parametrize +@pytest.mark.asyncio +async def test_pydantic_model_to_dictionary(use_async: bool) -> None: + assert await transform(MyModel(foo="hi!"), Any, use_async) == {"foo": "hi!"} + assert await transform(MyModel.construct(foo="hi!"), Any, use_async) == {"foo": "hi!"} -def test_pydantic_empty_model() -> None: - assert transform(MyModel.construct(), Any) == {} +@parametrize +@pytest.mark.asyncio +async def test_pydantic_empty_model(use_async: bool) -> None: + assert await transform(MyModel.construct(), Any, use_async) == {} -def test_pydantic_unknown_field() -> None: - assert transform(MyModel.construct(my_untyped_field=True), Any) == {"my_untyped_field": True} +@parametrize +@pytest.mark.asyncio +async def test_pydantic_unknown_field(use_async: bool) -> None: + assert await transform(MyModel.construct(my_untyped_field=True), Any, use_async) == {"my_untyped_field": True} -def test_pydantic_mismatched_types() -> None: +@parametrize +@pytest.mark.asyncio +async def test_pydantic_mismatched_types(use_async: bool) -> None: model = MyModel.construct(foo=True) if PYDANTIC_V2: with pytest.warns(UserWarning): - params = transform(model, Any) + params = await transform(model, Any, use_async) else: - params = transform(model, Any) + params = await transform(model, Any, use_async) assert params == {"foo": True} -def test_pydantic_mismatched_object_type() -> None: +@parametrize +@pytest.mark.asyncio +async def test_pydantic_mismatched_object_type(use_async: bool) -> None: model = MyModel.construct(foo=MyModel.construct(hello="world")) if PYDANTIC_V2: with pytest.warns(UserWarning): - params = transform(model, Any) + params = await transform(model, Any, use_async) else: - params = transform(model, Any) + params = await transform(model, Any, use_async) assert params == {"foo": {"hello": "world"}} @@ -235,10 +299,12 @@ class ModelNestedObjects(BaseModel): nested: MyModel -def test_pydantic_nested_objects() -> None: +@parametrize +@pytest.mark.asyncio +async def test_pydantic_nested_objects(use_async: bool) -> None: model = ModelNestedObjects.construct(nested={"foo": "stainless"}) assert isinstance(model.nested, MyModel) - assert transform(model, Any) == {"nested": {"foo": "stainless"}} + assert await transform(model, Any, use_async) == {"nested": {"foo": "stainless"}} class ModelWithDefaultField(BaseModel): @@ -247,24 +313,26 @@ class ModelWithDefaultField(BaseModel): with_str_default: str = "foo" -def test_pydantic_default_field() -> None: +@parametrize +@pytest.mark.asyncio +async def test_pydantic_default_field(use_async: bool) -> None: # should be excluded when defaults are used model = ModelWithDefaultField.construct() assert model.with_none_default is None assert model.with_str_default == "foo" - assert transform(model, Any) == {} + assert await transform(model, Any, use_async) == {} # should be included when the default value is explicitly given model = ModelWithDefaultField.construct(with_none_default=None, with_str_default="foo") assert model.with_none_default is None assert model.with_str_default == "foo" - assert transform(model, Any) == {"with_none_default": None, "with_str_default": "foo"} + assert await transform(model, Any, use_async) == {"with_none_default": None, "with_str_default": "foo"} # should be included when a non-default value is explicitly given model = ModelWithDefaultField.construct(with_none_default="bar", with_str_default="baz") assert model.with_none_default == "bar" assert model.with_str_default == "baz" - assert transform(model, Any) == {"with_none_default": "bar", "with_str_default": "baz"} + assert await transform(model, Any, use_async) == {"with_none_default": "bar", "with_str_default": "baz"} class TypedDictIterableUnion(TypedDict): @@ -279,21 +347,33 @@ class Baz8(TypedDict): foo_baz: Annotated[str, PropertyInfo(alias="fooBaz")] -def test_iterable_of_dictionaries() -> None: - assert transform({"foo": [{"foo_baz": "bar"}]}, TypedDictIterableUnion) == {"FOO": [{"fooBaz": "bar"}]} - assert cast(Any, transform({"foo": ({"foo_baz": "bar"},)}, TypedDictIterableUnion)) == {"FOO": [{"fooBaz": "bar"}]} +@parametrize +@pytest.mark.asyncio +async def test_iterable_of_dictionaries(use_async: bool) -> None: + assert await transform({"foo": [{"foo_baz": "bar"}]}, TypedDictIterableUnion, use_async) == { + "FOO": [{"fooBaz": "bar"}] + } + assert cast(Any, await transform({"foo": ({"foo_baz": "bar"},)}, TypedDictIterableUnion, use_async)) == { + "FOO": [{"fooBaz": "bar"}] + } def my_iter() -> Iterable[Baz8]: yield {"foo_baz": "hello"} yield {"foo_baz": "world"} - assert transform({"foo": my_iter()}, TypedDictIterableUnion) == {"FOO": [{"fooBaz": "hello"}, {"fooBaz": "world"}]} + assert await transform({"foo": my_iter()}, TypedDictIterableUnion, use_async) == { + "FOO": [{"fooBaz": "hello"}, {"fooBaz": "world"}] + } class TypedDictIterableUnionStr(TypedDict): foo: Annotated[Union[str, Iterable[Baz8]], PropertyInfo(alias="FOO")] -def test_iterable_union_str() -> None: - assert transform({"foo": "bar"}, TypedDictIterableUnionStr) == {"FOO": "bar"} - assert cast(Any, transform(iter([{"foo_baz": "bar"}]), Union[str, Iterable[Baz8]])) == [{"fooBaz": "bar"}] +@parametrize +@pytest.mark.asyncio +async def test_iterable_union_str(use_async: bool) -> None: + assert await transform({"foo": "bar"}, TypedDictIterableUnionStr, use_async) == {"FOO": "bar"} + assert cast(Any, await transform(iter([{"foo_baz": "bar"}]), Union[str, Iterable[Baz8]], use_async)) == [ + {"fooBaz": "bar"} + ] From 5c3f6355d8595517b16f2da8e82a71dea7a0e130 Mon Sep 17 00:00:00 2001 From: Stainless Bot <107565488+stainless-bot@users.noreply.github.com> Date: Mon, 4 Mar 2024 14:39:54 +0100 Subject: [PATCH 2/3] chore(internal): support more input types (#306) --- src/finch/_files.py | 5 +++++ src/finch/_types.py | 2 ++ src/finch/_utils/_transform.py | 39 +++++++++++++++++++++++++++++++++- tests/sample_file.txt | 1 + tests/test_transform.py | 29 +++++++++++++++++++++++++ 5 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 tests/sample_file.txt diff --git a/src/finch/_files.py b/src/finch/_files.py index b6e8af8b..0d2022ae 100644 --- a/src/finch/_files.py +++ b/src/finch/_files.py @@ -13,12 +13,17 @@ FileContent, RequestFiles, HttpxFileTypes, + Base64FileInput, HttpxFileContent, HttpxRequestFiles, ) from ._utils import is_tuple_t, is_mapping_t, is_sequence_t +def is_base64_file_input(obj: object) -> TypeGuard[Base64FileInput]: + return isinstance(obj, io.IOBase) or isinstance(obj, os.PathLike) + + def is_file_content(obj: object) -> TypeGuard[FileContent]: return ( isinstance(obj, bytes) or isinstance(obj, tuple) or isinstance(obj, io.IOBase) or isinstance(obj, os.PathLike) diff --git a/src/finch/_types.py b/src/finch/_types.py index 1ccf4a5e..037cd3e9 100644 --- a/src/finch/_types.py +++ b/src/finch/_types.py @@ -41,8 +41,10 @@ ProxiesDict = Dict["str | URL", Union[None, str, URL, Proxy]] ProxiesTypes = Union[str, Proxy, ProxiesDict] if TYPE_CHECKING: + Base64FileInput = Union[IO[bytes], PathLike[str]] FileContent = Union[IO[bytes], bytes, PathLike[str]] else: + Base64FileInput = Union[IO[bytes], PathLike] FileContent = Union[IO[bytes], bytes, PathLike] # PathLike is not subscriptable in Python 3.8. FileTypes = Union[ # file (or bytes) diff --git a/src/finch/_utils/_transform.py b/src/finch/_utils/_transform.py index 9c769306..1bd1330c 100644 --- a/src/finch/_utils/_transform.py +++ b/src/finch/_utils/_transform.py @@ -1,9 +1,13 @@ from __future__ import annotations +import io +import base64 +import pathlib from typing import Any, Mapping, TypeVar, cast from datetime import date, datetime from typing_extensions import Literal, get_args, override, get_type_hints +import anyio import pydantic from ._utils import ( @@ -11,6 +15,7 @@ is_mapping, is_iterable, ) +from .._files import is_base64_file_input from ._typing import ( is_list_type, is_union_type, @@ -29,7 +34,7 @@ # TODO: ensure works correctly with forward references in all cases -PropertyFormat = Literal["iso8601", "custom"] +PropertyFormat = Literal["iso8601", "base64", "custom"] class PropertyInfo: @@ -201,6 +206,22 @@ def _format_data(data: object, format_: PropertyFormat, format_template: str | N if format_ == "custom" and format_template is not None: return data.strftime(format_template) + if format_ == "base64" and is_base64_file_input(data): + binary: str | bytes | None = None + + if isinstance(data, pathlib.Path): + binary = data.read_bytes() + elif isinstance(data, io.IOBase): + binary = data.read() + + if isinstance(binary, str): # type: ignore[unreachable] + binary = binary.encode() + + if not isinstance(binary, bytes): + raise RuntimeError(f"Could not read bytes from {data}; Received {type(binary)}") + + return base64.b64encode(binary).decode("ascii") + return data @@ -323,6 +344,22 @@ async def _async_format_data(data: object, format_: PropertyFormat, format_templ if format_ == "custom" and format_template is not None: return data.strftime(format_template) + if format_ == "base64" and is_base64_file_input(data): + binary: str | bytes | None = None + + if isinstance(data, pathlib.Path): + binary = await anyio.Path(data).read_bytes() + elif isinstance(data, io.IOBase): + binary = data.read() + + if isinstance(binary, str): # type: ignore[unreachable] + binary = binary.encode() + + if not isinstance(binary, bytes): + raise RuntimeError(f"Could not read bytes from {data}; Received {type(binary)}") + + return base64.b64encode(binary).decode("ascii") + return data diff --git a/tests/sample_file.txt b/tests/sample_file.txt new file mode 100644 index 00000000..af5626b4 --- /dev/null +++ b/tests/sample_file.txt @@ -0,0 +1 @@ +Hello, world! diff --git a/tests/test_transform.py b/tests/test_transform.py index da9a5f15..187d119e 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -1,11 +1,14 @@ from __future__ import annotations +import io +import pathlib from typing import Any, List, Union, TypeVar, Iterable, Optional, cast from datetime import date, datetime from typing_extensions import Required, Annotated, TypedDict import pytest +from finch._types import Base64FileInput from finch._utils import ( PropertyInfo, transform as _transform, @@ -17,6 +20,8 @@ _T = TypeVar("_T") +SAMPLE_FILE_PATH = pathlib.Path(__file__).parent.joinpath("sample_file.txt") + async def transform( data: _T, @@ -377,3 +382,27 @@ async def test_iterable_union_str(use_async: bool) -> None: assert cast(Any, await transform(iter([{"foo_baz": "bar"}]), Union[str, Iterable[Baz8]], use_async)) == [ {"fooBaz": "bar"} ] + + +class TypedDictBase64Input(TypedDict): + foo: Annotated[Union[str, Base64FileInput], PropertyInfo(format="base64")] + + +@parametrize +@pytest.mark.asyncio +async def test_base64_file_input(use_async: bool) -> None: + # strings are left as-is + assert await transform({"foo": "bar"}, TypedDictBase64Input, use_async) == {"foo": "bar"} + + # pathlib.Path is automatically converted to base64 + assert await transform({"foo": SAMPLE_FILE_PATH}, TypedDictBase64Input, use_async) == { + "foo": "SGVsbG8sIHdvcmxkIQo=" + } # type: ignore[comparison-overlap] + + # io instances are automatically converted to base64 + assert await transform({"foo": io.StringIO("Hello, world!")}, TypedDictBase64Input, use_async) == { + "foo": "SGVsbG8sIHdvcmxkIQ==" + } # type: ignore[comparison-overlap] + assert await transform({"foo": io.BytesIO(b"Hello, world!")}, TypedDictBase64Input, use_async) == { + "foo": "SGVsbG8sIHdvcmxkIQ==" + } # type: ignore[comparison-overlap] From a728fec5b99a3a6b3a68e30a43927c9a54314218 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 13:40:08 +0000 Subject: [PATCH 3/3] release: 0.16.1 --- .release-please-manifest.json | 2 +- CHANGELOG.md | 9 +++++++++ pyproject.toml | 2 +- src/finch/_version.py | 2 +- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index b4e9013b..dc93a589 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "0.16.0" + ".": "0.16.1" } \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f425764..0fd5d8a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 0.16.1 (2024-03-04) + +Full Changelog: [v0.16.0...v0.16.1](https://github.com/Finch-API/finch-api-python/compare/v0.16.0...v0.16.1) + +### Chores + +* **internal:** split up transforms into sync / async ([#304](https://github.com/Finch-API/finch-api-python/issues/304)) ([665c4de](https://github.com/Finch-API/finch-api-python/commit/665c4de271703a90b063ce955992d1c74fcca832)) +* **internal:** support more input types ([#306](https://github.com/Finch-API/finch-api-python/issues/306)) ([5c3f635](https://github.com/Finch-API/finch-api-python/commit/5c3f6355d8595517b16f2da8e82a71dea7a0e130)) + ## 0.16.0 (2024-03-01) Full Changelog: [v0.15.0...v0.16.0](https://github.com/Finch-API/finch-api-python/compare/v0.15.0...v0.16.0) diff --git a/pyproject.toml b/pyproject.toml index adc8b3c0..c4d41f32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "finch-api" -version = "0.16.0" +version = "0.16.1" description = "The official Python library for the Finch API" readme = "README.md" license = "Apache-2.0" diff --git a/src/finch/_version.py b/src/finch/_version.py index 0056b453..7fdfe6da 100644 --- a/src/finch/_version.py +++ b/src/finch/_version.py @@ -1,4 +1,4 @@ # File generated from our OpenAPI spec by Stainless. __title__ = "finch" -__version__ = "0.16.0" # x-release-please-version +__version__ = "0.16.1" # x-release-please-version