fix: allow Protobuf 6.x (#1445)

* fix: allow Protobuf 6.x

* add prerelease nox session

* add python 3.13

* lint

* add test dependencies

* add test dependencies

* fix(deps): require google-crc32c >= 1.1.3

* fix(deps): require requests >= 2.22.0

* add dependencies for system tests

* clean up

* clean up

* clean up

* clean up

* clean up

* fix cover

* clean up

* Install dependencies needed for system tests

* add dependencies for system test

* update noxfile config

* add credentials

* exclude .kokoro/presubmit/prerelease-deps.cfg template

* remove obsolete excludes

* clean up

* clean up

* exclude .kokoro/continuous/prerelease-deps.cfg from templates; remove obsolete replacement

* migrate prerelease test from presubmit to continuous build

Author: parthea

.kokoro/continuous/prerelease-deps.cfg

@@ -5,3 +5,9 @@ env_vars: {
     key: "NOX_SESSION"
     value: "prerelease_deps"
 }
+
+# Credentials needed to test universe domain.
+env_vars: {
+  key: "SECRET_MANAGER_KEYS"
+  value: "client-library-test-universe-domain-credential"
+}

.kokoro/presubmit/prerelease-deps.cfg

@@ -22,7 +22,7 @@ In order to add a feature:
   documentation.
 
 - The feature must work fully on the following CPython versions:
-  3.7, 3.8, 3.9, 3.10, 3.11 and 3.12 on both UNIX and Windows.
+  3.7, 3.8, 3.9, 3.10, 3.11, 3.12 and 3.13 on both UNIX and Windows.
 
 - The feature must not add unnecessary dependencies (where
   "unnecessary" is of course subjective, but new dependencies should

CONTRIBUTING.rst

@@ -14,11 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Generated by synthtool. DO NOT EDIT!
-
 from __future__ import absolute_import
 import os
 import pathlib
+import re
 import shutil
 
 import nox
@@ -29,16 +28,27 @@
 
 DEFAULT_PYTHON_VERSION = "3.8"
 SYSTEM_TEST_PYTHON_VERSIONS = ["3.8"]
-UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
 CONFORMANCE_TEST_PYTHON_VERSIONS = ["3.8"]
 
-_DEFAULT_STORAGE_HOST = "https://storage.googleapis.com"
-
 CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()
 
 # Error if a python version is missing
 nox.options.error_on_missing_interpreters = True
 
+nox.options.sessions = [
+    "blacken",
+    "conftest_retry",
+    "docfx",
+    "docs",
+    "lint",
+    "lint_setup_py",
+    "system",
+    "unit",
+    # cover must be last to avoid error `No data to report`
+    "cover",
+]
+
 
 @nox.session(python=DEFAULT_PYTHON_VERSION)
 def lint(session):
@@ -159,8 +169,8 @@ def system(session):
     session.install(
         "google-cloud-testutils",
         "google-cloud-iam",
-        "google-cloud-pubsub < 2.0.0",
-        "google-cloud-kms < 2.0dev",
+        "google-cloud-pubsub",
+        "google-cloud-kms",
         "brotli",
         "-c",
         constraints_path,
@@ -300,3 +310,81 @@ def docfx(session):
         os.path.join("docs", ""),
         os.path.join("docs", "_build", "html", ""),
     )
+
+
+@nox.session(python=UNIT_TEST_PYTHON_VERSIONS[-1])
+@nox.parametrize(
+    "protobuf_implementation",
+    ["python", "upb"],
+)
+def prerelease_deps(session, protobuf_implementation):
+    """Run all tests with prerelease versions of dependencies installed."""
+
+    # Install all test dependencies
+    session.install("mock", "pytest", "pytest-cov", "brotli")
+
+    # Install dependencies needed for system tests
+    session.install(
+        "google-cloud-pubsub",
+        "google-cloud-kms",
+        "google-cloud-testutils",
+        "google-cloud-iam",
+    )
+
+    # Install all dependencies
+    session.install("-e", ".[protobuf, tracing]")
+
+    prerel_deps = [
+        "google-api-core",
+        "google-auth",
+        "google-cloud-core",
+        "google-crc32c",
+        "google-resumable-media",
+        "opentelemetry-api",
+        "protobuf",
+    ]
+
+    package_namespaces = {
+        "google-api-core": "google.api_core",
+        "google-auth": "google.auth",
+        "google-cloud-core": "google.cloud.version",
+        "opentelemetry-api": "opentelemetry.version",
+        "protobuf": "google.protobuf",
+    }
+
+    for dep in prerel_deps:
+        session.install("--pre", "--no-deps", "--upgrade", dep)
+        print(f"Installed {dep}")
+
+        version_namespace = package_namespaces.get(dep)
+
+        if version_namespace:
+            session.run(
+                "python",
+                "-c",
+                f"import {version_namespace}; print({version_namespace}.__version__)",
+            )
+    # Remaining dependencies
+    other_deps = [
+        "requests",
+    ]
+    session.install(*other_deps)
+
+    session.run(
+        "py.test",
+        "tests/unit",
+        env={
+            "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation,
+        },
+    )
+
+    session.run(
+        "py.test",
+        "--verbose",
+        f"--junitxml=system_{session.python}_sponge_log.xml",
+        os.path.join("tests", "system"),
+        *session.posargs,
+        env={
+            "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation,
+        },
+    )

noxfile.py

@@ -26,12 +26,6 @@
 templated_files = common.py_library(
     cov_level=100,
     split_system_tests=True,
-    system_test_external_dependencies=[
-        "google-cloud-iam",
-        "google-cloud-pubsub < 2.0.0",
-        # See: https://github.com/googleapis/python-storage/issues/226
-        "google-cloud-kms < 2.0dev",
-    ],
     intersphinx_dependencies={
         # python-requests url temporary change related to
         # https://github.com/psf/requests/issues/6140#issuecomment-1135071992
@@ -48,7 +42,8 @@
         "README.rst",
         ".kokoro/continuous/continuous.cfg",
         ".kokoro/presubmit/system-3.8.cfg",
-        ".kokoro/samples/python3.6", # remove python 3.6 support
+        ".kokoro/presubmit/prerelease-deps.cfg",
+        ".kokoro/continuous/prerelease-deps.cfg",
         ".github/blunderbuss.yml", # blunderbuss assignment to python squad
         ".github/workflows", # exclude gh actions as credentials are needed for tests
         ".github/release-please.yml", # special support for a python2 branch in this repo
@@ -84,12 +79,6 @@
     """omit =
   .nox/*""")
 
-s.replace(
-    ".kokoro/release/common.cfg",
-    'value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem"',
-    'value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem, client-library-test-universe-domain-credential"'
-)
-
 python.py_samples(skip_readmes=True)
 
 s.shell.run(["nox", "-s", "blacken"], hide_output=False)

owlbot.py

@@ -73,12 +73,15 @@ def get_cloud_kms_key():
     if session == 'py-3.12':
         return ('projects/python-docs-samples-tests-312/locations/us/'
                 'keyRings/gcs-kms-key-ring/cryptoKeys/gcs-kms-key')
+    if session == 'py-3.13':
+        return ('projects/python-docs-samples-tests-313/locations/us/'
+                'keyRings/gcs-kms-key-ring/cryptoKeys/gcs-kms-key')
     return os.environ['CLOUD_KMS_KEY']
 
 
 TEST_CONFIG_OVERRIDE = {
     # You can opt out from the test for specific Python versions.
-    'ignored_versions': ["2.7", "3.6", "3.7", "3.11", "3.12"],
+    'ignored_versions': ["2.7", "3.6", "3.7", "3.11", "3.12", "3.13"],
 
     # An envvar key for determining the project id to use. Change it
     # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a

samples/snippets/noxfile_config.py

@@ -28,24 +28,24 @@
 # 'Development Status :: 5 - Production/Stable'
 release_status = "Development Status :: 5 - Production/Stable"
 dependencies = [
-    "google-auth >= 2.26.1, < 3.0dev",
-    "google-api-core >= 2.15.0, <3.0.0dev",
-    "google-cloud-core >= 2.4.2, < 3.0dev",
+    "google-auth >= 2.26.1, < 3.0.0",
+    "google-api-core >= 2.15.0, < 3.0.0",
+    "google-cloud-core >= 2.4.2, < 3.0.0",
     # The dependency "google-resumable-media" is no longer used. However, the
     # dependency is still included here to accommodate users who may be
     # importing exception classes from the google-resumable-media without
     # installing it explicitly. See the python-storage README for details on
     # exceptions and importing. Users who are not importing
     # google-resumable-media classes in their application can safely disregard
     # this dependency.
-    "google-resumable-media >= 2.7.2",
-    "requests >= 2.18.0, < 3.0.0dev",
-    "google-crc32c >= 1.0, < 2.0dev",
+    "google-resumable-media >= 2.7.2, < 3.0.0",
+    "requests >= 2.22.0, < 3.0.0",
+    "google-crc32c >= 1.1.3, < 2.0.0",
 ]
 extras = {
-    "protobuf": ["protobuf<6.0.0dev"],
+    "protobuf": ["protobuf >= 3.20.2, < 7.0.0"],
     "tracing": [
-        "opentelemetry-api >= 1.1.0",
+        "opentelemetry-api >= 1.1.0, < 2.0.0",
     ],
 }
 
@@ -93,6 +93,7 @@
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: 3.12",
+        "Programming Language :: Python :: 3.13",
         "Operating System :: OS Independent",
         "Topic :: Internet",
     ],

setup.py

@@ -0,0 +1,15 @@
+# This constraints file is used to check that lower bounds
+# are correct in setup.py
+# List all library dependencies and extras in this file.
+# Pin the version to the lower bound.
+# e.g., if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0",
+# Then this file should have google-cloud-foo==1.14.0
+google-auth==2.26.1
+google-api-core==2.15.0
+google-cloud-core==2.4.2
+google-resumable-media==2.7.2
+requests==2.22.0
+google-crc32c==1.1.3
+protobuf==3.20.2
+opentelemetry-api==1.1.0
+

testing/constraints-3.13.txt

@@ -59,14 +59,14 @@ def topic_path(storage_client, topic_name):
 
 @pytest.fixture(scope="session")
 def notification_topic(storage_client, publisher_client, topic_path, no_mtls):
-    _helpers.retry_429(publisher_client.create_topic)(topic_path)
-    policy = publisher_client.get_iam_policy(topic_path)
+    _helpers.retry_429(publisher_client.create_topic)(request={"name": topic_path})
+    policy = publisher_client.get_iam_policy(request={"resource": topic_path})
     binding = policy.bindings.add()
     binding.role = "roles/pubsub.publisher"
     binding.members.append(
         f"serviceAccount:{storage_client.get_service_account_email()}"
     )
-    publisher_client.set_iam_policy(topic_path, policy)
+    publisher_client.set_iam_policy(request={"resource": topic_path, "policy": policy})
 
 
 def test_notification_create_minimal(