Improve field value parsing by adding NoDecode and ForceDecode annotations (#492)

Co-authored-by: hyperlint-ai[bot] <154288675+hyperlint-ai[bot]@users.noreply.github.com>

Author: hramezani

docs/index.md

@@ -371,6 +371,100 @@ print(Settings().model_dump())
 #> {'numbers': [1, 2, 3]}
 ```
 
+### Disabling JSON parsing
+
+pydantic-settings by default parses complex types from environment variables as JSON strings. If you want to disable
+this behavior for a field and parse the value in your own validator, you can annotate the field with
+[`NoDecode`](../api/pydantic_settings.md#pydantic_settings.NoDecode):
+
+```py
+import os
+from typing import List
+
+from pydantic import field_validator
+from typing_extensions import Annotated
+
+from pydantic_settings import BaseSettings, NoDecode
+
+
+class Settings(BaseSettings):
+    numbers: Annotated[List[int], NoDecode]  # (1)!
+
+    @field_validator('numbers', mode='before')
+    @classmethod
+    def decode_numbers(cls, v: str) -> List[int]:
+        return [int(x) for x in v.split(',')]
+
+
+os.environ['numbers'] = '1,2,3'
+print(Settings().model_dump())
+#> {'numbers': [1, 2, 3]}
+```
+
+1. The `NoDecode` annotation disables JSON parsing for the `numbers` field. The `decode_numbers` field validator
+   will be called to parse the value.
+
+You can also disable JSON parsing for all fields by setting the `enable_decoding` config setting to `False`:
+
+```py
+import os
+from typing import List
+
+from pydantic import field_validator
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(enable_decoding=False)
+
+    numbers: List[int]
+
+    @field_validator('numbers', mode='before')
+    @classmethod
+    def decode_numbers(cls, v: str) -> List[int]:
+        return [int(x) for x in v.split(',')]
+
+
+os.environ['numbers'] = '1,2,3'
+print(Settings().model_dump())
+#> {'numbers': [1, 2, 3]}
+```
+
+You can force JSON parsing for a field by annotating it with [`ForceDecode`](../api/pydantic_settings.md#pydantic_settings.ForceDecode).
+This will bypass the `enable_decoding` config setting:
+
+```py
+import os
+from typing import List
+
+from pydantic import field_validator
+from typing_extensions import Annotated
+
+from pydantic_settings import BaseSettings, ForceDecode, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(enable_decoding=False)
+
+    numbers: Annotated[List[int], ForceDecode]
+    numbers1: List[int]  # (1)!
+
+    @field_validator('numbers1', mode='before')
+    @classmethod
+    def decode_numbers1(cls, v: str) -> List[int]:
+        return [int(x) for x in v.split(',')]
+
+
+os.environ['numbers'] = '["1","2","3"]'
+os.environ['numbers1'] = '1,2,3'
+print(Settings().model_dump())
+#> {'numbers': [1, 2, 3], 'numbers1': [1, 2, 3]}
+```
+
+1. The `numbers1` field is not annotated with `ForceDecode`, so it will not be parsed as JSON.
+   and we have to provide a custom validator to parse the value.
+
 ## Nested model default partial updates
 
 By default, Pydantic settings does not allow partial updates to nested model default objects. This behavior can be

pydantic_settings/__init__.py

@@ -11,8 +11,10 @@
     CliSuppress,
     DotEnvSettingsSource,
     EnvSettingsSource,
+    ForceDecode,
     InitSettingsSource,
     JsonConfigSettingsSource,
+    NoDecode,
     PydanticBaseSettingsSource,
     PyprojectTomlConfigSettingsSource,
     SecretsSettingsSource,
@@ -38,6 +40,8 @@
     'CliMutuallyExclusiveGroup',
     'InitSettingsSource',
     'JsonConfigSettingsSource',
+    'NoDecode',
+    'ForceDecode',
     'PyprojectTomlConfigSettingsSource',
     'PydanticBaseSettingsSource',
     'SecretsSettingsSource',

pydantic_settings/main.py

@@ -79,6 +79,7 @@ class SettingsConfigDict(ConfigDict, total=False):
     """
 
     toml_file: PathType | None
+    enable_decoding: bool
 
 
 # Extend `config_keys` by pydantic settings config keys to
@@ -433,6 +434,7 @@ def _settings_build_values(
         toml_file=None,
         secrets_dir=None,
         protected_namespaces=('model_validate', 'model_dump', 'settings_customise_sources'),
+        enable_decoding=True,
     )
 
 

pydantic_settings/sources.py

@@ -118,6 +118,18 @@ def import_azure_key_vault() -> None:
 ENV_FILE_SENTINEL: DotenvType = Path('')
 
 
+class NoDecode:
+    """Annotation to prevent decoding of a field value."""
+
+    pass
+
+
+class ForceDecode:
+    """Annotation to force decoding of a field value."""
+
+    pass
+
+
 class SettingsError(ValueError):
     pass
 
@@ -312,6 +324,12 @@ def decode_complex_value(self, field_name: str, field: FieldInfo, value: Any) ->
         Returns:
             The decoded value for further preparation
         """
+        if field and (
+            NoDecode in field.metadata
+            or (self.config.get('enable_decoding') is False and ForceDecode not in field.metadata)
+        ):
+            return value
+
         return json.loads(value)
 
     @abstractmethod

tests/test_settings.py

@@ -1,4 +1,5 @@
 import dataclasses
+import json
 import os
 import pathlib
 import sys
@@ -26,6 +27,7 @@
     SecretStr,
     Tag,
     ValidationError,
+    field_validator,
 )
 from pydantic import (
     dataclasses as pydantic_dataclasses,
@@ -37,7 +39,9 @@
     BaseSettings,
     DotEnvSettingsSource,
     EnvSettingsSource,
+    ForceDecode,
     InitSettingsSource,
+    NoDecode,
     PydanticBaseSettingsSource,
     SecretsSettingsSource,
     SettingsConfigDict,
@@ -2873,3 +2877,69 @@ class Settings(BaseSettings):
     s = Settings()
     assert s.foo.get_secret_value() == 123
     assert s.bar.get_secret_value() == PostgresDsn('postgres://user:password@localhost/dbname')
+
+
+def test_field_annotated_no_decode(env):
+    class Settings(BaseSettings):
+        a: List[str]  # this field will be decoded because of default `enable_decoding=True`
+        b: Annotated[List[str], NoDecode]
+
+        # decode the value here. the field value won't be decoded because of NoDecode
+        @field_validator('b', mode='before')
+        @classmethod
+        def decode_b(cls, v: str) -> List[str]:
+            return json.loads(v)
+
+    env.set('a', '["one", "two"]')
+    env.set('b', '["1", "2"]')
+
+    s = Settings()
+    assert s.model_dump() == {'a': ['one', 'two'], 'b': ['1', '2']}
+
+
+def test_field_annotated_no_decode_and_disable_decoding(env):
+    class Settings(BaseSettings):
+        model_config = SettingsConfigDict(enable_decoding=False)
+
+        a: Annotated[List[str], NoDecode]
+
+        # decode the value here. the field value won't be decoded because of NoDecode
+        @field_validator('a', mode='before')
+        @classmethod
+        def decode_b(cls, v: str) -> List[str]:
+            return json.loads(v)
+
+    env.set('a', '["one", "two"]')
+
+    s = Settings()
+    assert s.model_dump() == {'a': ['one', 'two']}
+
+
+def test_field_annotated_disable_decoding(env):
+    class Settings(BaseSettings):
+        model_config = SettingsConfigDict(enable_decoding=False)
+
+        a: List[str]
+
+        # decode the value here. the field value won't be decoded because of `enable_decoding=False`
+        @field_validator('a', mode='before')
+        @classmethod
+        def decode_b(cls, v: str) -> List[str]:
+            return json.loads(v)
+
+    env.set('a', '["one", "two"]')
+
+    s = Settings()
+    assert s.model_dump() == {'a': ['one', 'two']}
+
+
+def test_field_annotated_force_decode_disable_decoding(env):
+    class Settings(BaseSettings):
+        model_config = SettingsConfigDict(enable_decoding=False)
+
+        a: Annotated[List[str], ForceDecode]
+
+    env.set('a', '["one", "two"]')
+
+    s = Settings()
+    assert s.model_dump() == {'a': ['one', 'two']}