Merge main-v.0.7.3"

Author: forest-benchling

openapi_python_client/parser/properties/__init__.py

@@ -272,13 +272,23 @@ def build_model_property(
     required_properties: List[Property] = []
     optional_properties: List[Property] = []
     relative_imports: Set[str] = set()
+    references: List[oai.Reference] = []
 
     class_name = data.title or name
     if parent_name:
         class_name = f"{utils.pascal_case(parent_name)}{utils.pascal_case(class_name)}"
     ref = Reference.from_ref(class_name)
 
-    for key, value in (data.properties or {}).items():
+    all_props = data.properties or {}
+    if not isinstance(data, oai.Reference) and data.allOf:
+        for sub_prop in data.allOf:
+            if isinstance(sub_prop, oai.Reference):
+                references += [sub_prop]
+            else:
+                all_props.update(sub_prop.properties or {})
+                required_set.update(sub_prop.required or [])
+
+    for key, value in all_props.items():
         prop_required = key in required_set
         prop, schemas = property_from_data(
             name=key, required=prop_required, data=value, schemas=schemas, parent_name=class_name
@@ -314,6 +324,7 @@ def build_model_property(
 
     prop = ModelProperty(
         reference=ref,
+        references=references,
         required_properties=required_properties,
         optional_properties=optional_properties,
         relative_imports=relative_imports,
@@ -471,9 +482,6 @@ def _property_from_data(
         )
     if data.anyOf or data.oneOf:
         return build_union_property(data=data, name=name, required=required, schemas=schemas, parent_name=parent_name)
-    if not data.type:
-        return NoneProperty(name=name, required=required, nullable=False, default=None), schemas
-
     if data.type == "string":
         return _string_based_property(name=name, required=required, data=data), schemas
     elif data.type == "number":
@@ -508,8 +516,10 @@ def _property_from_data(
         )
     elif data.type == "array":
         return build_list_property(data=data, name=name, required=required, schemas=schemas, parent_name=parent_name)
-    elif data.type == "object":
+    elif data.type == "object" or data.allOf:
         return build_model_property(data=data, name=name, schemas=schemas, required=required, parent_name=parent_name)
+    elif not data.type:
+        return NoneProperty(name=name, required=required, nullable=False, default=None), schemas
     return PropertyError(data=data, detail=f"unknown type {data.type}"), schemas
 
 
@@ -566,6 +576,16 @@ def build_schemas(*, components: Dict[str, Union[oai.Reference, oai.Schema]]) ->
                 schemas = schemas_or_err
                 processing = True  # We made some progress this round, do another after it's done
         to_process = next_round
-    schemas.errors.extend(errors)
 
+    resolve_errors: List[PropertyError] = []
+    models = list(schemas.models.values())
+    for model in models:
+        schemas_or_err = model.resolve_references(components=components, schemas=schemas)
+        if isinstance(schemas_or_err, PropertyError):
+            resolve_errors.append(schemas_or_err)
+        else:
+            schemas = schemas_or_err
+
+    schemas.errors.extend(errors)
+    schemas.errors.extend(resolve_errors)
     return schemas

openapi_python_client/parser/properties/model_property.py

@@ -1,17 +1,25 @@
-from typing import ClassVar, List, Set, Union
+from __future__ import annotations
+
+from collections.abc import Iterable
+from typing import TYPE_CHECKING, ClassVar, Dict, List, Set, Union
 
 import attr
 
+from ... import schema as oai
+from ..errors import PropertyError
 from ..reference import Reference
 from .property import Property
 
+if TYPE_CHECKING:
+    from .schemas import Schemas
+
 
 @attr.s(auto_attribs=True, frozen=True)
 class ModelProperty(Property):
     """ A property which refers to another Schema """
 
     reference: Reference
-
+    references: List[oai.Reference]
     required_properties: List[Property]
     optional_properties: List[Property]
     description: str
@@ -21,6 +29,49 @@ class ModelProperty(Property):
 
     template: ClassVar[str] = "model_property.pyi"
 
+    def resolve_references(
+        self, components: Dict[str, Union[oai.Reference, oai.Schema]], schemas: Schemas
+    ) -> Union[Schemas, PropertyError]:
+        from ..properties import property_from_data
+
+        required_set = set()
+        props = {}
+        while self.references:
+            reference = self.references.pop()
+            source_name = Reference.from_ref(reference.ref).class_name
+            referenced_prop = components[source_name]
+            assert isinstance(referenced_prop, oai.Schema)
+            for p, val in (referenced_prop.properties or {}).items():
+                props[p] = (val, source_name)
+            for sub_prop in referenced_prop.allOf or []:
+                if isinstance(sub_prop, oai.Reference):
+                    self.references.append(sub_prop)
+                else:
+                    for p, val in (sub_prop.properties or {}).items():
+                        props[p] = (val, source_name)
+            if isinstance(referenced_prop.required, Iterable):
+                for sub_prop_name in referenced_prop.required:
+                    required_set.add(sub_prop_name)
+
+        for key, (value, source_name) in (props or {}).items():
+            required = key in required_set
+            prop, schemas = property_from_data(
+                name=key, required=required, data=value, schemas=schemas, parent_name=source_name
+            )
+            if isinstance(prop, PropertyError):
+                return prop
+            if required:
+                self.required_properties.append(prop)
+                # Remove the optional version
+                new_optional_props = [op for op in self.optional_properties if op.name != prop.name]
+                self.optional_properties.clear()
+                self.optional_properties.extend(new_optional_props)
+            elif not any(ep for ep in (self.optional_properties + self.required_properties) if ep.name == prop.name):
+                self.optional_properties.append(prop)
+            self.relative_imports.update(prop.get_imports(prefix=".."))
+
+        return schemas
+
     def get_base_type_string(self) -> str:
         return self.reference.class_name
 

pyproject.toml

@@ -67,7 +67,12 @@ task regen\
 && task regen_custom\
 && task e2e\
 """
-docs = "typer openapi_python_client/cli.py utils docs > usage.md"
+gen-setuppy = """
+poetry build \
+&& tar --strip-components=1 -xvf "$(ls -1 dist/*tar.gz | tail -1)" '*/setup.py' \
+&& isort setup.py \
+&& black setup.py
+"""
 
 [tool.black]
 line-length = 120
@@ -95,5 +100,8 @@ skip = [".venv", "tests/test_templates"]
 omit = ["openapi_python_client/templates/*"]
 
 [build-system]
-requires = ["poetry>=1.0"]
+requires = [
+    "setuptools>=30.3.0,<50",
+    "poetry>=1.0"
+]
 build-backend = "poetry.masonry.api"

setup.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+from setuptools import setup
+
+packages = [
+    "openapi_python_client",
+    "openapi_python_client.parser",
+    "openapi_python_client.parser.properties",
+    "openapi_python_client.schema",
+    "openapi_python_client.templates",
+]
+
+package_data = {"": ["*"], "openapi_python_client.templates": ["property_templates/*"]}
+
+install_requires = [
+    "attrs>=20.1.0,<21.0.0",
+    "autoflake>=1.4,<2.0",
+    "black>=20.8b1",
+    "httpx>=0.15.4,<0.17.0",
+    "isort>=5.0.5,<6.0.0",
+    "jinja2>=2.11.1,<3.0.0",
+    "pydantic>=1.6.1,<2.0.0",
+    "python-dateutil>=2.8.1,<3.0.0",
+    "pyyaml>=5.3.1,<6.0.0",
+    "shellingham>=1.3.2,<2.0.0",
+    "stringcase>=1.2.0,<2.0.0",
+    "typer>=0.3,<0.4",
+]
+
+extras_require = {
+    ':python_version < "3.8"': ["importlib_metadata>=2.0.0,<3.0.0"],
+    ':sys_platform == "win32"': ["colorama>=0.4.3,<0.5.0"],
+}
+
+entry_points = {"console_scripts": ["openapi-python-client = openapi_python_client.cli:app"]}
+
+setup_kwargs = {
+    "name": "openapi-python-client",
+    "version": "0.7.0",
+    "description": "Generate modern Python clients from OpenAPI",
+    "long_description": '[![triaxtec](https://circleci.com/gh/triaxtec/openapi-python-client.svg?style=svg)](https://circleci.com/gh/triaxtec/openapi-python-client)\n[![codecov](https://codecov.io/gh/triaxtec/openapi-python-client/branch/main/graph/badge.svg)](https://codecov.io/gh/triaxtec/openapi-python-client)\n[![MIT license](https://img.shields.io/badge/License-MIT-blue.svg)](https://lbesson.mit-license.org/)\n[![Generic badge](https://img.shields.io/badge/type_checked-mypy-informational.svg)](https://mypy.readthedocs.io/en/stable/introduction.html)\n[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/ambv/black)\n[![PyPI version shields.io](https://img.shields.io/pypi/v/openapi-python-client.svg)](https://pypi.python.org/pypi/openapi-python-client/)\n[![Downloads](https://static.pepy.tech/personalized-badge/openapi-python-client?period=total&units=international_system&left_color=blue&right_color=green&left_text=Downloads)](https://pepy.tech/project/openapi-python-client)\n\n# openapi-python-client\n\nGenerate modern Python clients from OpenAPI 3.x documents.\n\n_This generator does not support OpenAPI 2.x FKA Swagger. If you need to use an older document, try upgrading it to\nversion 3 first with one of many available converters._\n\n**This project is still in development and does not support all OpenAPI features**\n\n## Why This?\n\nThe Python clients generated by openapi-generator support Python 2 and therefore come with a lot of baggage. This tool\naims to generate clients which:\n\n1. Use all the latest and greatest Python features like type annotations and dataclasses\n1. Don\'t carry around a bunch of compatibility code for older version of Python (e.g. the `six` package)\n1. Have better documentation and more obvious usage instructions\n\nAdditionally, because this generator is written in Python, it should be more accessible to contribution by the people\nusing it (Python developers).\n\n## Installation\n\nI recommend you install with [pipx](https://pipxproject.github.io/pipx/) so you don\'t conflict with any other packages\nyou might have: `pipx install openapi-python-client`.\n\nBetter yet, use `pipx run openapi-python-client <normal params / options>` to always use the latest version of the generator.\n\nYou can install with normal pip if you want to though: `pip install openapi-python-client`\n\nThen, if you want tab completion: `openapi-python-client --install-completion`\n\n## Usage\n\n### Create a new client\n\n`openapi-python-client generate --url https://my.api.com/openapi.json`\n\nThis will generate a new client library named based on the title in your OpenAPI spec. For example, if the title\nof your API is "My API", the expected output will be "my-api-client". If a folder already exists by that name, you\'ll\nget an error.\n\n### Update an existing client\n\n`openapi-python-client update --url https://my.api.com/openapi.json`\n\n> For more usage details run `openapi-python-client --help` or read [usage](usage.md)\n\n### Using custom templates\n\nThis feature leverages Jinja2\'s [ChoiceLoader](https://jinja.palletsprojects.com/en/2.11.x/api/#jinja2.ChoiceLoader) and [FileSystemLoader](https://jinja.palletsprojects.com/en/2.11.x/api/#jinja2.FileSystemLoader). This means you do _not_ need to customize every template. Simply copy the template(s) you want to customize from [the default template directory](openapi_python_client/templates) to your own custom template directory (file names _must_ match exactly) and pass the template directory through the `custom-template-path` flag to the `generate` and `update` commands. For instance,\n\n```\nopenapi-python-client update \\\n  --url https://my.api.com/openapi.json \\\n  --custom-template-path=relative/path/to/mytemplates\n```\n\n_Be forewarned, this is a beta-level feature in the sense that the API exposed in the templates is undocumented and unstable._\n\n## What You Get\n\n1. A `pyproject.toml` file with some basic metadata intended to be used with [Poetry].\n1. A `README.md` you\'ll most definitely need to update with your project\'s details\n1. A Python module named just like the auto-generated project name (e.g. "my_api_client") which contains:\n   1. A `client` module which will have both a `Client` class and an `AuthenticatedClient` class. You\'ll need these\n      for calling the functions in the `api` module.\n   1. An `api` module which will contain one module for each tag in your OpenAPI spec, as well as a `default` module\n      for endpoints without a tag. Each of these modules in turn contains one function for calling each endpoint.\n   1. A `models` module which has all the classes defined by the various schemas in your OpenAPI spec\n\nFor a full example you can look at the `end_to_end_tests` directory which has an `openapi.json` file.\n"golden-record" in that same directory is the generated client from that OpenAPI document.\n\n## OpenAPI features supported\n\n1. All HTTP Methods\n1. JSON and form bodies, path and query parameters\n1. File uploads with multipart/form-data bodies\n1. float, string, int, date, datetime, string enums, and custom schemas or lists containing any of those\n1. html/text or application/json responses containing any of the previous types\n1. Bearer token security\n\n## Configuration\n\nYou can pass a YAML (or JSON) file to openapi-python-client with the `--config` option in order to change some behavior.\nThe following parameters are supported:\n\n### class_overrides\n\nUsed to change the name of generated model classes. This param should be a mapping of existing class name\n(usually a key in the "schemas" section of your OpenAPI document) to class_name and module_name. As an example, if the\nname of the a model in OpenAPI (and therefore the generated class name) was something like "\\_PrivateInternalLongName"\nand you want the generated client\'s model to be called "ShortName" in a module called "short_name" you could do this:\n\nExample:\n\n```yaml\nclass_overrides:\n  _PrivateInternalLongName:\n    class_name: ShortName\n    module_name: short_name\n```\n\nThe easiest way to find what needs to be overridden is probably to generate your client and go look at everything in the\nmodels folder.\n\n### project_name_override and package_name_override\n\nUsed to change the name of generated client library project/package. If the project name is changed but an override for the package name\nisn\'t provided, the package name will be converted from the project name using the standard convention (replacing `-`\'s with `_`\'s).\n\nExample:\n\n```yaml\nproject_name_override: my-special-project-name\npackage_name_override: my_extra_special_package_name\n```\n\n### field_prefix\n\nWhen generating properties, the `name` attribute of the OpenAPI schema will be used. When the `name` is not a valid\nPython identifier (e.g. begins with a number) this string will be prepended. Defaults to "field\\_".\n\nExample:\n\n```yaml\nfield_prefix: attr_\n```\n\n### package_version_override\n\nSpecify the package version of the generated client. If unset, the client will use the version of the OpenAPI spec.\n\nExample:\n\n```yaml\npackage_version_override: 1.2.3\n```\n\n[changelog.md]: CHANGELOG.md\n[poetry]: https://python-poetry.org/\n',
+    "author": "Dylan Anthony",
+    "author_email": "[email protected]",
+    "maintainer": None,
+    "maintainer_email": None,
+    "url": "https://github.com/triaxtec/openapi-python-client",
+    "packages": packages,
+    "package_data": package_data,
+    "install_requires": install_requires,
+    "extras_require": extras_require,
+    "entry_points": entry_points,
+    "python_requires": ">=3.6,<4.0",
+}
+
+
+setup(**setup_kwargs)

tests/test_parser/test_properties/test_init.py

@@ -584,6 +584,7 @@ def test_property_from_data_ref_model(self):
             nullable=False,
             default=None,
             reference=Reference(class_name=class_name, module_name="my_model"),
+            references=[],
             required_properties=[],
             optional_properties=[],
             description="",
@@ -600,6 +601,7 @@ def test_property_from_data_ref_model(self):
             nullable=False,
             default=None,
             reference=Reference(class_name=class_name, module_name="my_model"),
+            references=[],
             required_properties=[],
             optional_properties=[],
             description="",
@@ -986,19 +988,25 @@ def test__string_based_property_unsupported_format(self, mocker):
 def test_build_schemas(mocker):
     build_model_property = mocker.patch(f"{MODULE_NAME}.build_model_property")
     in_data = {"1": mocker.MagicMock(enum=None), "2": mocker.MagicMock(enum=None), "3": mocker.MagicMock(enum=None)}
+
     model_1 = mocker.MagicMock()
     schemas_1 = mocker.MagicMock()
     model_2 = mocker.MagicMock()
     schemas_2 = mocker.MagicMock(errors=[])
-    error = PropertyError()
+    schemas_2.models = {"1": model_1, "2": model_2}
+    error_1 = PropertyError()
     schemas_3 = mocker.MagicMock()
+    schemas_4 = mocker.MagicMock(errors=[])
+    model_1.resolve_references.return_value = schemas_4
+    error_2 = PropertyError()
+    model_2.resolve_references.return_value = error_2
 
     # This loops through one for each, then again to retry the error
     build_model_property.side_effect = [
         (model_1, schemas_1),
         (model_2, schemas_2),
-        (error, schemas_3),
-        (error, schemas_3),
+        (error_1, schemas_3),
+        (error_1, schemas_3),
     ]
 
     from openapi_python_client.parser.properties import Schemas, build_schemas
@@ -1014,8 +1022,12 @@ def test_build_schemas(mocker):
         ]
     )
     # schemas_3 was the last to come back from build_model_property, but it should be ignored because it's an error
-    assert result == schemas_2
-    assert result.errors == [error]
+    model_1.resolve_references.assert_called_once_with(components=in_data, schemas=schemas_2)
+    # schemas_4 came from resolving model_1
+    model_2.resolve_references.assert_called_once_with(components=in_data, schemas=schemas_4)
+    # resolving model_2 resulted in err, so no schemas_5
+    assert result == schemas_4
+    assert result.errors == [error_1, error_2]
 
 
 def test_build_parse_error_on_reference():
@@ -1089,6 +1101,7 @@ def test_build_model_property(additional_properties_schema, expected_additional_
         nullable=False,
         default=None,
         reference=Reference(class_name="ParentMyModel", module_name="parent_my_model"),
+        references=[],
         required_properties=[StringProperty(name="req", required=True, nullable=False, default=None)],
         optional_properties=[DateTimeProperty(name="opt", required=False, nullable=False, default=None)],
         description=data.description,