[DRAFT] feat: delegate boilerplate helpers to google-api-core

.github/workflows/gapic-generator-tests.yml

@@ -85,6 +85,7 @@ jobs:
           curl --location https://github.com/google/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip --output /usr/src/protoc/protoc.zip
           cd /usr/src/protoc/ && unzip protoc.zip
           sudo ln -s /usr/src/protoc/bin/protoc /usr/local/bin/protoc
+
       - name: Run Nox
         env:
           GOOGLE_SDK_PYTHON_LOGGING_SCOPE: ${{ matrix.logging_scope }}
@@ -167,11 +168,13 @@ jobs:
       - name: Install System Deps
         run: sudo apt-get update && sudo apt-get install -y pandoc
       - name: Run Goldens (Prerelease)
+        env:
+          PRERELEASE_PYTHON: ${{ needs.python_config.outputs.prerelease_python }}
         run: |
           pip install nox
           cd packages/gapic-generator
           for pkg in credentials eventarc logging redis; do
-            nox -f tests/integration/goldens/$pkg/noxfile.py -s core_deps_from_source prerelease_deps
+            nox -f tests/integration/goldens/$pkg/noxfile.py -s core_deps_from_source-${PRERELEASE_PYTHON} prerelease_deps
           done
 
   fragment-snippet:
@@ -201,6 +204,7 @@ jobs:
           curl --location https://github.com/google/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip --output /usr/src/protoc/protoc.zip
           cd /usr/src/protoc/ && unzip protoc.zip
           sudo ln -s /usr/src/protoc/bin/protoc /usr/local/bin/protoc
+
       - name: Run Tests
         run: |
           pip install nox
@@ -232,6 +236,7 @@ jobs:
           # Ensure CACHE_VERSION is defined in the mono-repo secrets!
           key: ${{ runner.os }}-bazel-20210105-${{ secrets.CACHE_VERSION }}
 
+
       - name: Run Bazel Integration Tests
         run: |
           # We need to move into the package directory if the 

.gitignore

@@ -72,3 +72,4 @@ bazel-bin
 bazel-gapic-generator
 bazel-out
 bazel-testlogs
+packages/gapic-generator/api-common-protos/

packages/gapic-generator/.python-version

@@ -0,0 +1,5 @@
+3.10.0
+3.11.0
+3.12.0
+3.13.0
+3.14.0

packages/gapic-generator/WORKSPACE

@@ -2,6 +2,31 @@ workspace(name = "gapic_generator_python")
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
+# Override google-api-core dependency with git archive of packages/google-api-core directory to avoid local paths
+http_archive(
+    name = "gapic_generator_python_pip_deps_google_api_core",
+    urls = ["https://github.com/googleapis/google-cloud-python/archive/e0f3f85e0430e284c0cb50f0186f24c15e499707.tar.gz"],
+    sha256 = "bf184f6b26ed613ab0aec3453ece922e4adb5a4d4fd0a6db99ae5e0c9b371db4",
+    strip_prefix = "google-cloud-python-e0f3f85e0430e284c0cb50f0186f24c15e499707/packages/google-api-core",
+    build_file_content = """
+py_library(
+    name = "pkg",
+    srcs = glob(["google/**/*.py"]),
+    imports = ["."],
+    visibility = ["//visibility:public"],
+    deps = [
+        "@gapic_generator_python_pip_deps_googleapis_common_protos//:pkg",
+        "@gapic_generator_python_pip_deps_protobuf//:pkg",
+        "@gapic_generator_python_pip_deps_proto_plus//:pkg",
+        "@gapic_generator_python_pip_deps_google_auth//:pkg",
+        "@gapic_generator_python_pip_deps_requests//:pkg",
+    ],
+)
+""",
+)
+
+
+
 _bazel_skylib_version = "1.4.0"
 
 _bazel_skylib_sha256 = "f24ab666394232f834f74d19e2ff142b0af17466ea0c69a3f4c276ee75f6efce"
@@ -42,11 +67,12 @@ load("@rules_python//python:pip.bzl", "pip_parse")
 
 pip_parse(
     name = "gapic_generator_python_pip_deps",
-	requirements_lock = "//:requirements.txt",
+    requirements_lock = "//:requirements.txt",
 )
 load("@gapic_generator_python_pip_deps//:requirements.bzl", "install_deps")
 
 install_deps()
+
 #
 # Import gapic-generator-python specific dependencies
 #
@@ -123,3 +149,13 @@ switched_rules_by_language(
     gapic = True,
     grpc = True,
 )
+
+# BEGIN workaround
+load("@rules_python//python:repositories.bzl", "python_register_toolchains")
+
+python_register_toolchains(
+    name = "python311",
+    python_version = "3.11",
+    ignore_root_user_error = True,
+)
+# END workaround

packages/gapic-generator/gapic/ads-templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2

@@ -103,6 +103,35 @@ def test__get_default_mtls_endpoint():
     assert {{ service.client_name }}._get_default_mtls_endpoint(custom_endpoint) == custom_endpoint
 
 
+import asyncio
+
+@pytest.fixture(autouse=True)
+def set_event_loop():
+    try:
+        asyncio.get_running_loop()
+        yield
+    except RuntimeError:
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        try:
+            yield
+        finally:
+            loop.close()
+            asyncio.set_event_loop(None)
+
+
+@pytest.fixture(scope="module", autouse=True)
+def mock_mtls_env():
+    with mock.patch.dict(
+        os.environ,
+        {
+            "GOOGLE_API_USE_CLIENT_CERTIFICATE": "false",
+            "CLOUDSDK_CONTEXT_AWARE_USE_CLIENT_CERTIFICATE": "false",
+        },
+    ):
+        yield
+
+
 @pytest.mark.parametrize("client_class,transport_name", [
     {% if 'grpc' in opts.transport %}
     ({{ service.client_name }}, "grpc"),

packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2

@@ -6,6 +6,14 @@
 {% import "%namespace/%name_%version/%sub/services/%service/_client_macros.j2" as macros %}
 {% import "%namespace/%name_%version/%sub/services/%service/_shared_macros.j2" as shared_macros %}
 
+{% set ns = namespace(has_auto_populated=false) %}
+{% for method in service.methods.values() %}
+    {% set method_settings = api.all_method_settings.get(method.meta.address.proto) %}
+    {% if method_settings is not none and method_settings.auto_populated_fields %}
+        {% set ns.has_auto_populated = true %}
+    {% endif %}
+{% endfor %}
+
 from collections import OrderedDict
 {% if service.any_extended_operations_methods %}
 import functools
@@ -16,8 +24,8 @@ import logging as std_logging
 import os
 import re
 from typing import Dict, Callable, Mapping, MutableMapping, MutableSequence, Optional, {% if service.any_server_streaming %}Iterable, {% endif %}{% if service.any_client_streaming %}Iterator, {% endif %}Sequence, Tuple, Type, Union, cast
-{% if api.all_method_settings.values()|map(attribute="auto_populated_fields", default=[])|select|list %}
-import uuid
+{% if ns.has_auto_populated %}
+from google.api_core.gapic_v1 import _method_helpers
 {% endif %}
 import warnings
 
@@ -30,6 +38,9 @@ from google.api_core import exceptions as core_exceptions
 from google.api_core import extended_operation
 {% endif %}
 from google.api_core import gapic_v1
+from google.api_core.gapic_v1 import _client_cert
+from google.api_core.gapic_v1 import _config_helpers
+from google.api_core.gapic_v1 import _routing
 from google.api_core import retry as retries
 from google.auth import credentials as ga_credentials             # type: ignore
 from google.auth.transport import mtls                            # type: ignore
@@ -154,28 +165,7 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
         Returns:
             Optional[str]: converted mTLS api endpoint.
         """
-        if not api_endpoint:
-            return api_endpoint
-
-        mtls_endpoint_re = re.compile(
-            r"(?P<name>[^.]+)(?P<mtls>\.mtls)?(?P<sandbox>\.sandbox)?(?P<googledomain>\.googleapis\.com)?"
-        )
-
-        m = mtls_endpoint_re.match(api_endpoint)
-        if m is None:
-            # Could not parse api_endpoint; return as-is.
-            return api_endpoint
-
-        name, mtls, sandbox, googledomain = m.groups()
-        if mtls or not googledomain:
-            return api_endpoint
-
-        if sandbox:
-            return api_endpoint.replace(
-                "sandbox.googleapis.com", "mtls.sandbox.googleapis.com"
-            )
-
-        return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com")
+        return _routing.get_default_mtls_endpoint(api_endpoint)
 
     # Note: DEFAULT_ENDPOINT is deprecated. Use _DEFAULT_ENDPOINT_TEMPLATE instead.
     DEFAULT_ENDPOINT = {% if service.host %}"{{ service.host }}"{% else %}None{% endif %}
@@ -201,18 +191,7 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
             ValueError: (If using a version of google-auth without should_use_client_cert and
 	    GOOGLE_API_USE_CLIENT_CERTIFICATE is set to an unexpected value.)
         """
-        # check if google-auth version supports should_use_client_cert for automatic mTLS enablement
-        if hasattr(mtls, "should_use_client_cert"):  # pragma: NO COVER
-            return mtls.should_use_client_cert()
-        else: # pragma: NO COVER
-            # if unsupported, fallback to reading from env var
-            use_client_cert_str = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false").lower()
-            if use_client_cert_str not in ("true", "false"):
-                raise ValueError(
-                    "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be"
-                    " either `true` or `false`"
-                )
-            return use_client_cert_str == "true"
+        return _client_cert.use_client_cert_effective()
 
     @classmethod
     def from_service_account_info(cls, info: dict, *args, **kwargs):
@@ -365,12 +344,7 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
             google.auth.exceptions.MutualTLSChannelError: If GOOGLE_API_USE_MTLS_ENDPOINT
                 is not any of ["auto", "never", "always"].
         """
-        use_client_cert = {{ service.client_name }}._use_client_cert_effective()
-        use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto").lower()
-        universe_domain_env = os.getenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN")
-        if use_mtls_endpoint not in ("auto", "never", "always"):
-            raise MutualTLSChannelError("Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`")
-        return use_client_cert, use_mtls_endpoint, universe_domain_env
+        return _config_helpers.read_environment_variables()
 
     @staticmethod
     def _get_client_cert_source(provided_cert_source, use_cert_flag):
@@ -383,13 +357,7 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
         Returns:
             bytes or None: The client cert source to be used by the client.
         """
-        client_cert_source = None
-        if use_cert_flag:
-            if provided_cert_source:
-                client_cert_source = provided_cert_source
-            elif mtls.has_default_client_cert_source():
-                client_cert_source = mtls.default_client_cert_source()
-        return client_cert_source
+        return _client_cert.get_client_cert_source(provided_cert_source, use_cert_flag)
 
     @staticmethod
     def _get_api_endpoint(api_override, client_cert_source, universe_domain, use_mtls_endpoint) -> str:
@@ -406,16 +374,18 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
         Returns:
             str: The API endpoint to be used by the client.
         """
-        if api_override is not None:
-            api_endpoint = api_override
-        elif use_mtls_endpoint == "always" or (use_mtls_endpoint == "auto" and client_cert_source):
-            _default_universe = {{ service.client_name }}._DEFAULT_UNIVERSE
-            if universe_domain != _default_universe:
-                raise MutualTLSChannelError(f"mTLS is not supported in any universe other than {_default_universe}.")
-            api_endpoint = {{ service.client_name }}.DEFAULT_MTLS_ENDPOINT
-        else:
-            api_endpoint = {{ service.client_name }}._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=universe_domain)
-        return api_endpoint
+        return cast(
+            str,
+            _routing.get_api_endpoint(
+                api_override,
+                client_cert_source,
+                universe_domain,
+                use_mtls_endpoint,
+                {{ service.client_name }}._DEFAULT_UNIVERSE,
+                {{ service.client_name }}.DEFAULT_MTLS_ENDPOINT,
+                {{ service.client_name }}._DEFAULT_ENDPOINT_TEMPLATE,
+            ),
+        )
 
     @staticmethod
     def _get_universe_domain(client_universe_domain: Optional[str], universe_domain_env: Optional[str]) -> str:
@@ -431,14 +401,11 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
         Raises:
             ValueError: If the universe domain is an empty string.
         """
-        universe_domain = {{ service.client_name }}._DEFAULT_UNIVERSE
-        if client_universe_domain is not None:
-            universe_domain = client_universe_domain
-        elif universe_domain_env is not None:
-            universe_domain = universe_domain_env
-        if len(universe_domain.strip()) == 0:
-            raise ValueError("Universe Domain cannot be an empty string.") 
-        return universe_domain
+        return _routing.get_universe_domain(
+            client_universe_domain,
+            universe_domain_env,
+            {{ service.client_name }}._DEFAULT_UNIVERSE,
+        )
 
     def _validate_universe_domain(self):
         """Validates client's and credentials' universe domains are consistent.
@@ -453,7 +420,7 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
         # NOTE (b/349488459): universe validation is disabled until further notice.
         return True
 
-    {% if api.all_method_settings.values()|map(attribute="auto_populated_fields", default=[])|select|list %}
+    {% if ns.has_auto_populated %}
     @staticmethod
     def _setup_request_id(request, field_name: str, is_proto3_optional: bool):
         """Populate a UUID4 field in the request if it is not already set.
@@ -463,26 +430,7 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
             field_name (str): The name of the field to populate.
             is_proto3_optional (bool): Whether the field is proto3 optional.
         """
-        if isinstance(request, dict):
-            if is_proto3_optional:
-                if field_name not in request:
-                    request[field_name] = str(uuid.uuid4())
-            elif not request.get(field_name):
-                request[field_name] = str(uuid.uuid4())
-            return
-
-        if is_proto3_optional:
-            try:
-                # Pure protobuf messages
-                if not request.HasField(field_name):
-                    setattr(request, field_name, str(uuid.uuid4()))
-            except (AttributeError, ValueError):
-                # Proto-plus messages or other objects
-                if field_name not in request:
-                    setattr(request, field_name, str(uuid.uuid4()))
-        else:
-            if not getattr(request, field_name):
-                setattr(request, field_name, str(uuid.uuid4()))
+        _method_helpers.setup_request_id(request, field_name, is_proto3_optional)
     {% endif %}
 
     def _add_cred_info_for_auth_errors(
@@ -588,12 +536,11 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
             google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport
                 creation failed for any reason.
         """
-        self._client_options = client_options
-        if isinstance(self._client_options, dict):
-            self._client_options = client_options_lib.from_dict(self._client_options)
-        if self._client_options is None:
-            self._client_options = client_options_lib.ClientOptions()
-        self._client_options = cast(client_options_lib.ClientOptions, self._client_options)
+        if isinstance(client_options, dict):
+            client_options = client_options_lib.from_dict(client_options)
+        if client_options is None:
+            client_options = client_options_lib.ClientOptions()
+        self._client_options = cast(client_options_lib.ClientOptions, client_options)
 
         universe_domain_opt = getattr(self._client_options, 'universe_domain', None)