diff --git a/mypy.ini b/mypy.ini index 8ea2d5af..116e626a 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,7 +1,11 @@ [mypy] pretty = True show_error_codes = True -exclude = _dev + +# Exclude _files.py because mypy isn't smart enough to apply +# the correct type narrowing and as this is an internal module +# it's fine to just use Pyright. +exclude = ^(src/finch/_files\.py|_dev/.*\.py)$ strict_equality = True implicit_reexport = True diff --git a/pyproject.toml b/pyproject.toml index 4f84ce66..2b28fabb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ dev-dependencies = [ "isort==5.10.1", "time-machine==2.9.0", "nox==2023.4.22", + "dirty-equals>=0.6.0", ] @@ -53,6 +54,15 @@ format = { chain = [ "format:ruff" = "ruff --fix ." "format:isort" = "isort ." +typecheck = { chain = [ + "typecheck:pyright", + "typecheck:verify-types", + "typecheck:mypy" +]} +"typecheck:pyright" = "pyright" +"typecheck:verify-types" = "pyright --verifytypes finch --ignoreexternal" +"typecheck:mypy" = "mypy --enable-incomplete-feature=Unpack ." + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/requirements-dev.lock b/requirements-dev.lock index 505b6ccf..7e517132 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -15,6 +15,7 @@ black==23.3.0 certifi==2023.7.22 click==8.1.7 colorlog==6.7.0 +dirty-equals==0.6.0 distlib==0.3.7 distro==1.8.0 exceptiongroup==1.1.3 @@ -40,6 +41,7 @@ pyright==1.1.332 pytest==7.1.1 pytest-asyncio==0.21.1 python-dateutil==2.8.2 +pytz==2023.3.post1 respx==0.19.2 rfc3986==1.5.0 ruff==0.0.282 diff --git a/src/finch/_base_client.py b/src/finch/_base_client.py index c49cc042..44065d1c 100644 --- a/src/finch/_base_client.py +++ b/src/finch/_base_client.py @@ -40,6 +40,7 @@ from . import _exceptions from ._qs import Querystring +from ._files import to_httpx_files, async_to_httpx_files from ._types import ( NOT_GIVEN, Body, @@ -1088,7 +1089,9 @@ def post( stream: bool = False, stream_cls: type[_StreamT] | None = None, ) -> ResponseT | _StreamT: - opts = FinalRequestOptions.construct(method="post", url=path, json_data=body, files=files, **options) + opts = FinalRequestOptions.construct( + method="post", url=path, json_data=body, files=to_httpx_files(files), **options + ) return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls)) def patch( @@ -1111,7 +1114,9 @@ def put( files: RequestFiles | None = None, options: RequestOptions = {}, ) -> ResponseT: - opts = FinalRequestOptions.construct(method="put", url=path, json_data=body, files=files, **options) + opts = FinalRequestOptions.construct( + method="put", url=path, json_data=body, files=to_httpx_files(files), **options + ) return self.request(cast_to, opts) def delete( @@ -1491,7 +1496,9 @@ async def post( stream: bool = False, stream_cls: type[_AsyncStreamT] | None = None, ) -> ResponseT | _AsyncStreamT: - opts = FinalRequestOptions.construct(method="post", url=path, json_data=body, files=files, **options) + opts = FinalRequestOptions.construct( + method="post", url=path, json_data=body, files=await async_to_httpx_files(files), **options + ) return await self.request(cast_to, opts, stream=stream, stream_cls=stream_cls) async def patch( @@ -1514,7 +1521,9 @@ async def put( files: RequestFiles | None = None, options: RequestOptions = {}, ) -> ResponseT: - opts = FinalRequestOptions.construct(method="put", url=path, json_data=body, files=files, **options) + opts = FinalRequestOptions.construct( + method="put", url=path, json_data=body, files=await async_to_httpx_files(files), **options + ) return await self.request(cast_to, opts) async def delete( diff --git a/src/finch/_files.py b/src/finch/_files.py new file mode 100644 index 00000000..b6e8af8b --- /dev/null +++ b/src/finch/_files.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +import io +import os +import pathlib +from typing import overload +from typing_extensions import TypeGuard + +import anyio + +from ._types import ( + FileTypes, + FileContent, + RequestFiles, + HttpxFileTypes, + HttpxFileContent, + HttpxRequestFiles, +) +from ._utils import is_tuple_t, is_mapping_t, is_sequence_t + + +def is_file_content(obj: object) -> TypeGuard[FileContent]: + return ( + isinstance(obj, bytes) or isinstance(obj, tuple) or isinstance(obj, io.IOBase) or isinstance(obj, os.PathLike) + ) + + +def assert_is_file_content(obj: object, *, key: str | None = None) -> None: + if not is_file_content(obj): + prefix = f"Expected entry at `{key}`" if key is not None else f"Expected file input `{obj!r}`" + raise RuntimeError( + f"{prefix} to be bytes, an io.IOBase instance, PathLike or a tuple but received {type(obj)} instead." + ) from None + + +@overload +def to_httpx_files(files: None) -> None: + ... + + +@overload +def to_httpx_files(files: RequestFiles) -> HttpxRequestFiles: + ... + + +def to_httpx_files(files: RequestFiles | None) -> HttpxRequestFiles | None: + if files is None: + return None + + if is_mapping_t(files): + files = {key: _transform_file(file) for key, file in files.items()} + elif is_sequence_t(files): + files = [(key, _transform_file(file)) for key, file in files] + else: + raise TypeError(f"Unexpected file type input {type(files)}, expected mapping or sequence") + + return files + + +def _transform_file(file: FileTypes) -> HttpxFileTypes: + if is_file_content(file): + if isinstance(file, os.PathLike): + path = pathlib.Path(file) + return (path.name, path.read_bytes()) + + return file + + if is_tuple_t(file): + return (file[0], _read_file_content(file[1]), *file[2:]) + + raise TypeError(f"Expected file types input to be a FileContent type or to be a tuple") + + +def _read_file_content(file: FileContent) -> HttpxFileContent: + if isinstance(file, os.PathLike): + return pathlib.Path(file).read_bytes() + return file + + +@overload +async def async_to_httpx_files(files: None) -> None: + ... + + +@overload +async def async_to_httpx_files(files: RequestFiles) -> HttpxRequestFiles: + ... + + +async def async_to_httpx_files(files: RequestFiles | None) -> HttpxRequestFiles | None: + if files is None: + return None + + if is_mapping_t(files): + files = {key: await _async_transform_file(file) for key, file in files.items()} + elif is_sequence_t(files): + files = [(key, await _async_transform_file(file)) for key, file in files] + else: + raise TypeError("Unexpected file type input {type(files)}, expected mapping or sequence") + + return files + + +async def _async_transform_file(file: FileTypes) -> HttpxFileTypes: + if is_file_content(file): + if isinstance(file, os.PathLike): + path = anyio.Path(file) + return (path.name, await path.read_bytes()) + + return file + + if is_tuple_t(file): + return (file[0], await _async_read_file_content(file[1]), *file[2:]) + + raise TypeError(f"Expected file types input to be a FileContent type or to be a tuple") + + +async def _async_read_file_content(file: FileContent) -> HttpxFileContent: + if isinstance(file, os.PathLike): + return await anyio.Path(file).read_bytes() + + return file diff --git a/src/finch/_models.py b/src/finch/_models.py index f663155c..4e4107f5 100644 --- a/src/finch/_models.py +++ b/src/finch/_models.py @@ -3,7 +3,16 @@ import inspect from typing import TYPE_CHECKING, Any, Type, Union, Generic, TypeVar, cast from datetime import date, datetime -from typing_extensions import Literal, ClassVar, Protocol, final, runtime_checkable +from typing_extensions import ( + Unpack, + Literal, + ClassVar, + Protocol, + Required, + TypedDict, + final, + runtime_checkable, +) import pydantic import pydantic.generics @@ -18,7 +27,7 @@ Timeout, NotGiven, AnyMapping, - RequestFiles, + HttpxRequestFiles, ) from ._utils import is_list, is_mapping, parse_date, parse_datetime, strip_not_given from ._compat import PYDANTIC_V2, ConfigDict @@ -363,6 +372,19 @@ def _create_pydantic_model(type_: _T) -> Type[RootModel[_T]]: return RootModel[type_] # type: ignore +class FinalRequestOptionsInput(TypedDict, total=False): + method: Required[str] + url: Required[str] + params: Query + headers: Headers + max_retries: int + timeout: float | Timeout | None + files: HttpxRequestFiles | None + idempotency_key: str + json_data: Body + extra_json: AnyMapping + + @final class FinalRequestOptions(pydantic.BaseModel): method: str @@ -371,7 +393,7 @@ class FinalRequestOptions(pydantic.BaseModel): headers: Union[Headers, NotGiven] = NotGiven() max_retries: Union[int, NotGiven] = NotGiven() timeout: Union[float, Timeout, None, NotGiven] = NotGiven() - files: Union[RequestFiles, None] = None + files: Union[HttpxRequestFiles, None] = None idempotency_key: Union[str, None] = None # It should be noted that we cannot use `json` here as that would override @@ -395,11 +417,13 @@ def get_max_retries(self, max_retries: int) -> int: # this is necessary as we don't want to do any actual runtime type checking # (which means we can't use validators) but we do want to ensure that `NotGiven` # values are not present + # + # type ignore required because we're adding explicit types to `**values` @classmethod - def construct( + def construct( # type: ignore cls, _fields_set: set[str] | None = None, - **values: Any, + **values: Unpack[FinalRequestOptionsInput], ) -> FinalRequestOptions: kwargs: dict[str, Any] = { # we unconditionally call `strip_not_given` on any value diff --git a/src/finch/_types.py b/src/finch/_types.py index 45e89382..6a8b9c5a 100644 --- a/src/finch/_types.py +++ b/src/finch/_types.py @@ -1,5 +1,6 @@ from __future__ import annotations +from os import PathLike from typing import ( IO, TYPE_CHECKING, @@ -32,9 +33,10 @@ _T = TypeVar("_T") # Approximates httpx internal ProxiesTypes and RequestFiles types +# while adding support for `PathLike` instances ProxiesDict = Dict["str | URL", Union[None, str, URL, Proxy]] ProxiesTypes = Union[str, Proxy, ProxiesDict] -FileContent = Union[IO[bytes], bytes] +FileContent = Union[IO[bytes], bytes, PathLike[str]] FileTypes = Union[ # file (or bytes) FileContent, @@ -47,6 +49,19 @@ ] RequestFiles = Union[Mapping[str, FileTypes], Sequence[Tuple[str, FileTypes]]] +# duplicate of the above but without our custom file support +HttpxFileContent = Union[IO[bytes], bytes] +HttpxFileTypes = Union[ + # file (or bytes) + HttpxFileContent, + # (filename, file (or bytes)) + Tuple[Optional[str], HttpxFileContent], + # (filename, file (or bytes), content_type) + Tuple[Optional[str], HttpxFileContent, Optional[str]], + # (filename, file (or bytes), content_type, headers) + Tuple[Optional[str], HttpxFileContent, Optional[str], Mapping[str, str]], +] +HttpxRequestFiles = Union[Mapping[str, HttpxFileTypes], Sequence[Tuple[str, HttpxFileTypes]]] # Workaround to support (cast_to: Type[ResponseT]) -> ResponseT # where ResponseT includes `None`. In order to support directly diff --git a/src/finch/_utils/__init__.py b/src/finch/_utils/__init__.py index 26dc560b..d3397212 100644 --- a/src/finch/_utils/__init__.py +++ b/src/finch/_utils/__init__.py @@ -3,13 +3,18 @@ from ._utils import is_dict as is_dict from ._utils import is_list as is_list from ._utils import is_given as is_given +from ._utils import is_tuple as is_tuple from ._utils import is_mapping as is_mapping +from ._utils import is_tuple_t as is_tuple_t from ._utils import parse_date as parse_date +from ._utils import is_sequence as is_sequence from ._utils import coerce_float as coerce_float from ._utils import is_list_type as is_list_type +from ._utils import is_mapping_t as is_mapping_t from ._utils import removeprefix as removeprefix from ._utils import removesuffix as removesuffix from ._utils import extract_files as extract_files +from ._utils import is_sequence_t as is_sequence_t from ._utils import is_union_type as is_union_type from ._utils import required_args as required_args from ._utils import coerce_boolean as coerce_boolean diff --git a/src/finch/_utils/_utils.py b/src/finch/_utils/_utils.py index cb660d16..4b51dcb2 100644 --- a/src/finch/_utils/_utils.py +++ b/src/finch/_utils/_utils.py @@ -1,11 +1,20 @@ from __future__ import annotations -import io import os import re import inspect import functools -from typing import Any, Mapping, TypeVar, Callable, Iterable, Sequence, cast, overload +from typing import ( + Any, + Tuple, + Mapping, + TypeVar, + Callable, + Iterable, + Sequence, + cast, + overload, +) from pathlib import Path from typing_extensions import Required, Annotated, TypeGuard, get_args, get_origin @@ -15,6 +24,9 @@ from .._compat import parse_datetime as parse_datetime _T = TypeVar("_T") +_TupleT = TypeVar("_TupleT", bound=Tuple[object, ...]) +_MappingT = TypeVar("_MappingT", bound=Mapping[str, object]) +_SequenceT = TypeVar("_SequenceT", bound=Sequence[object]) CallableT = TypeVar("CallableT", bound=Callable[..., Any]) @@ -55,13 +67,11 @@ def _extract_items( # no value was provided - we can safely ignore return [] - # We have exhausted the path, return the entry we found. - if not isinstance(obj, bytes) and not isinstance(obj, tuple) and not isinstance(obj, io.IOBase): - raise RuntimeError( - f"Expected entry at {flattened_key} to be bytes, an io.IOBase instance or a tuple but received {type(obj)} instead." - ) from None + # cyclical import + from .._files import assert_is_file_content - # TODO: validate obj more? + # We have exhausted the path, return the entry we found. + assert_is_file_content(obj, key=flattened_key) assert flattened_key is not None return [(flattened_key, cast(FileTypes, obj))] @@ -116,12 +126,36 @@ def is_given(obj: NotGivenOr[_T]) -> TypeGuard[_T]: # The default narrowing for isinstance(obj, dict) is dict[unknown, unknown], # however this cause Pyright to rightfully report errors. As we know we don't # care about the contained types we can safely use `object` in it's place. +# +# There are two separate functions defined, `is_*` and `is_*_t` for different use cases. +# `is_*` is for when you're dealing with an unknown input +# `is_*_t` is for when you're narrowing a known union type to a specific subset + + +def is_tuple(obj: object) -> TypeGuard[tuple[object, ...]]: + return isinstance(obj, tuple) + + +def is_tuple_t(obj: _TupleT | object) -> TypeGuard[_TupleT]: + return isinstance(obj, tuple) + + +def is_sequence(obj: object) -> TypeGuard[Sequence[object]]: + return isinstance(obj, Sequence) + + +def is_sequence_t(obj: _SequenceT | object) -> TypeGuard[_SequenceT]: + return isinstance(obj, Sequence) def is_mapping(obj: object) -> TypeGuard[Mapping[str, object]]: return isinstance(obj, Mapping) +def is_mapping_t(obj: _MappingT | object) -> TypeGuard[_MappingT]: + return isinstance(obj, Mapping) + + def is_dict(obj: object) -> TypeGuard[dict[object, object]]: return isinstance(obj, dict) diff --git a/tests/test_files.py b/tests/test_files.py new file mode 100644 index 00000000..0ed7d371 --- /dev/null +++ b/tests/test_files.py @@ -0,0 +1,51 @@ +from pathlib import Path + +import anyio +import pytest +from dirty_equals import IsDict, IsList, IsBytes, IsTuple + +from finch._files import to_httpx_files, async_to_httpx_files + +readme_path = Path(__file__).parent.parent.joinpath("README.md") + + +def test_pathlib_includes_file_name() -> None: + result = to_httpx_files({"file": readme_path}) + print(result) + assert result == IsDict({"file": IsTuple("README.md", IsBytes())}) + + +def test_tuple_input() -> None: + result = to_httpx_files([("file", readme_path)]) + print(result) + assert result == IsList(IsTuple("file", IsTuple("README.md", IsBytes()))) + + +@pytest.mark.asyncio +async def test_async_pathlib_includes_file_name() -> None: + result = await async_to_httpx_files({"file": readme_path}) + print(result) + assert result == IsDict({"file": IsTuple("README.md", IsBytes())}) + + +@pytest.mark.asyncio +async def test_async_supports_anyio_path() -> None: + result = await async_to_httpx_files({"file": anyio.Path(readme_path)}) + print(result) + assert result == IsDict({"file": IsTuple("README.md", IsBytes())}) + + +@pytest.mark.asyncio +async def test_async_tuple_input() -> None: + result = await async_to_httpx_files([("file", readme_path)]) + print(result) + assert result == IsList(IsTuple("file", IsTuple("README.md", IsBytes()))) + + +def test_string_not_allowed() -> None: + with pytest.raises(TypeError, match="Expected file types input to be a FileContent type or to be a tuple"): + to_httpx_files( + { + "file": "foo", # type: ignore + } + )