feat: delegate generated client boilerplate helpers to google-api-core

.github/workflows/gapic-generator-tests.yml

@@ -85,6 +85,10 @@ jobs:
           curl --location https://github.com/google/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip --output /usr/src/protoc/protoc.zip
           cd /usr/src/protoc/ && unzip protoc.zip
           sudo ln -s /usr/src/protoc/bin/protoc /usr/local/bin/protoc
+      - name: Configure Git Redirect
+        run: |
+          git config --global url."${{ github.workspace }}".insteadOf "https://github.com/googleapis/google-cloud-python"
+          git config --global url."${{ github.workspace }}".insteadOf "https://github.com/googleapis/google-cloud-python.git"
       - name: Run Nox
         env:
           GOOGLE_SDK_PYTHON_LOGGING_SCOPE: ${{ matrix.logging_scope }}
@@ -126,6 +130,10 @@ jobs:
           python-version: ${{ needs.python_config.outputs.latest_stable_python }}
       - name: Install System Deps
         run: sudo apt-get update && sudo apt-get install -y pandoc
+      - name: Configure Git Redirect
+        run: |
+          git config --global url."${{ github.workspace }}".insteadOf "https://github.com/googleapis/google-cloud-python"
+          git config --global url."${{ github.workspace }}".insteadOf "https://github.com/googleapis/google-cloud-python.git"
       - name: Run Goldens
         env:
           LATEST_STABLE_PYTHON: ${{ needs.python_config.outputs.latest_stable_python }}
@@ -201,6 +209,10 @@ jobs:
           curl --location https://github.com/google/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip --output /usr/src/protoc/protoc.zip
           cd /usr/src/protoc/ && unzip protoc.zip
           sudo ln -s /usr/src/protoc/bin/protoc /usr/local/bin/protoc
+      - name: Configure Git Redirect
+        run: |
+          git config --global url."${{ github.workspace }}".insteadOf "https://github.com/googleapis/google-cloud-python"
+          git config --global url."${{ github.workspace }}".insteadOf "https://github.com/googleapis/google-cloud-python.git"
       - name: Run Tests
         run: |
           pip install nox
@@ -232,6 +244,10 @@ jobs:
           # Ensure CACHE_VERSION is defined in the mono-repo secrets!
           key: ${{ runner.os }}-bazel-20210105-${{ secrets.CACHE_VERSION }}
 
+      - name: Configure Git Redirect
+        run: |
+          git config --global url."${{ github.workspace }}".insteadOf "https://github.com/googleapis/google-cloud-python"
+          git config --global url."${{ github.workspace }}".insteadOf "https://github.com/googleapis/google-cloud-python.git"
       - name: Run Bazel Integration Tests
         run: |
           # We need to move into the package directory if the 

packages/gapic-generator/WORKSPACE

@@ -42,7 +42,8 @@ load("@rules_python//python:pip.bzl", "pip_parse")
 
 pip_parse(
     name = "gapic_generator_python_pip_deps",
-	requirements_lock = "//:requirements.txt",
+    requirements_lock = "//:requirements.txt",
+    extra_pip_args = ["--index-url=https://pypi.org/simple"],
 )
 load("@gapic_generator_python_pip_deps//:requirements.bzl", "install_deps")
 
@@ -123,3 +124,13 @@ switched_rules_by_language(
     gapic = True,
     grpc = True,
 )
+
+# BEGIN workaround
+load("@rules_python//python:repositories.bzl", "python_register_toolchains")
+
+python_register_toolchains(
+    name = "python311",
+    python_version = "3.11",
+)
+# END workaround
+

packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2

@@ -6,6 +6,14 @@
 {% import "%namespace/%name_%version/%sub/services/%service/_client_macros.j2" as macros %}
 {% import "%namespace/%name_%version/%sub/services/%service/_shared_macros.j2" as shared_macros %}
 
+{% set ns = namespace(has_auto_populated=false) %}
+{% for method in service.methods.values() %}
+    {% set method_settings = api.all_method_settings.get(method.meta.address.proto) %}
+    {% if method_settings is not none and method_settings.auto_populated_fields %}
+        {% set ns.has_auto_populated = true %}
+    {% endif %}
+{% endfor %}
+
 from collections import OrderedDict
 {% if service.any_extended_operations_methods %}
 import functools
@@ -16,8 +24,8 @@ import logging as std_logging
 import os
 import re
 from typing import Dict, Callable, Mapping, MutableMapping, MutableSequence, Optional, {% if service.any_server_streaming %}Iterable, {% endif %}{% if service.any_client_streaming %}Iterator, {% endif %}Sequence, Tuple, Type, Union, cast
-{% if api.all_method_settings.values()|map(attribute="auto_populated_fields", default=[])|select|list %}
-import uuid
+{% if ns.has_auto_populated %}
+from google.api_core.gapic_v1 import _method_helpers
 {% endif %}
 import warnings
 
@@ -30,6 +38,9 @@ from google.api_core import exceptions as core_exceptions
 from google.api_core import extended_operation
 {% endif %}
 from google.api_core import gapic_v1
+from google.api_core.gapic_v1 import _client_cert
+from google.api_core.gapic_v1 import _config_helpers
+from google.api_core.gapic_v1 import _routing
 from google.api_core import retry as retries
 from google.auth import credentials as ga_credentials             # type: ignore
 from google.auth.transport import mtls                            # type: ignore
@@ -154,28 +165,7 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
         Returns:
             Optional[str]: converted mTLS api endpoint.
         """
-        if not api_endpoint:
-            return api_endpoint
-
-        mtls_endpoint_re = re.compile(
-            r"(?P<name>[^.]+)(?P<mtls>\.mtls)?(?P<sandbox>\.sandbox)?(?P<googledomain>\.googleapis\.com)?"
-        )
-
-        m = mtls_endpoint_re.match(api_endpoint)
-        if m is None:
-            # Could not parse api_endpoint; return as-is.
-            return api_endpoint
-
-        name, mtls, sandbox, googledomain = m.groups()
-        if mtls or not googledomain:
-            return api_endpoint
-
-        if sandbox:
-            return api_endpoint.replace(
-                "sandbox.googleapis.com", "mtls.sandbox.googleapis.com"
-            )
-
-        return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com")
+        return _routing.get_default_mtls_endpoint(api_endpoint)
 
     # Note: DEFAULT_ENDPOINT is deprecated. Use _DEFAULT_ENDPOINT_TEMPLATE instead.
     DEFAULT_ENDPOINT = {% if service.host %}"{{ service.host }}"{% else %}None{% endif %}
@@ -201,18 +191,7 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
             ValueError: (If using a version of google-auth without should_use_client_cert and
 	    GOOGLE_API_USE_CLIENT_CERTIFICATE is set to an unexpected value.)
         """
-        # check if google-auth version supports should_use_client_cert for automatic mTLS enablement
-        if hasattr(mtls, "should_use_client_cert"):  # pragma: NO COVER
-            return mtls.should_use_client_cert()
-        else: # pragma: NO COVER
-            # if unsupported, fallback to reading from env var
-            use_client_cert_str = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false").lower()
-            if use_client_cert_str not in ("true", "false"):
-                raise ValueError(
-                    "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be"
-                    " either `true` or `false`"
-                )
-            return use_client_cert_str == "true"
+        return _client_cert.use_client_cert_effective()
 
     @classmethod
     def from_service_account_info(cls, info: dict, *args, **kwargs):
@@ -365,12 +344,7 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
             google.auth.exceptions.MutualTLSChannelError: If GOOGLE_API_USE_MTLS_ENDPOINT
                 is not any of ["auto", "never", "always"].
         """
-        use_client_cert = {{ service.client_name }}._use_client_cert_effective()
-        use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto").lower()
-        universe_domain_env = os.getenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN")
-        if use_mtls_endpoint not in ("auto", "never", "always"):
-            raise MutualTLSChannelError("Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`")
-        return use_client_cert, use_mtls_endpoint, universe_domain_env
+        return _config_helpers.read_environment_variables()
 
     @staticmethod
     def _get_client_cert_source(provided_cert_source, use_cert_flag):
@@ -383,13 +357,7 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
         Returns:
             bytes or None: The client cert source to be used by the client.
         """
-        client_cert_source = None
-        if use_cert_flag:
-            if provided_cert_source:
-                client_cert_source = provided_cert_source
-            elif mtls.has_default_client_cert_source():
-                client_cert_source = mtls.default_client_cert_source()
-        return client_cert_source
+        return _client_cert.get_client_cert_source(provided_cert_source, use_cert_flag)
 
     @staticmethod
     def _get_api_endpoint(api_override, client_cert_source, universe_domain, use_mtls_endpoint) -> str:
@@ -406,16 +374,18 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
         Returns:
             str: The API endpoint to be used by the client.
         """
-        if api_override is not None:
-            api_endpoint = api_override
-        elif use_mtls_endpoint == "always" or (use_mtls_endpoint == "auto" and client_cert_source):
-            _default_universe = {{ service.client_name }}._DEFAULT_UNIVERSE
-            if universe_domain != _default_universe:
-                raise MutualTLSChannelError(f"mTLS is not supported in any universe other than {_default_universe}.")
-            api_endpoint = {{ service.client_name }}.DEFAULT_MTLS_ENDPOINT
-        else:
-            api_endpoint = {{ service.client_name }}._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=universe_domain)
-        return api_endpoint
+        return cast(
+            str,
+            _routing.get_api_endpoint(
+                api_override,
+                client_cert_source,
+                universe_domain,
+                use_mtls_endpoint,
+                {{ service.client_name }}._DEFAULT_UNIVERSE,
+                {{ service.client_name }}.DEFAULT_MTLS_ENDPOINT,
+                {{ service.client_name }}._DEFAULT_ENDPOINT_TEMPLATE,
+            ),
+        )
 
     @staticmethod
     def _get_universe_domain(client_universe_domain: Optional[str], universe_domain_env: Optional[str]) -> str:
@@ -431,14 +401,11 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
         Raises:
             ValueError: If the universe domain is an empty string.
         """
-        universe_domain = {{ service.client_name }}._DEFAULT_UNIVERSE
-        if client_universe_domain is not None:
-            universe_domain = client_universe_domain
-        elif universe_domain_env is not None:
-            universe_domain = universe_domain_env
-        if len(universe_domain.strip()) == 0:
-            raise ValueError("Universe Domain cannot be an empty string.") 
-        return universe_domain
+        return _routing.get_universe_domain(
+            client_universe_domain,
+            universe_domain_env,
+            {{ service.client_name }}._DEFAULT_UNIVERSE,
+        )
 
     def _validate_universe_domain(self):
         """Validates client's and credentials' universe domains are consistent.
@@ -453,7 +420,7 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
         # NOTE (b/349488459): universe validation is disabled until further notice.
         return True
 
-    {% if api.all_method_settings.values()|map(attribute="auto_populated_fields", default=[])|select|list %}
+    {% if ns.has_auto_populated %}
     @staticmethod
     def _setup_request_id(request, field_name: str, is_proto3_optional: bool):
         """Populate a UUID4 field in the request if it is not already set.
@@ -463,26 +430,7 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
             field_name (str): The name of the field to populate.
             is_proto3_optional (bool): Whether the field is proto3 optional.
         """
-        if isinstance(request, dict):
-            if is_proto3_optional:
-                if field_name not in request:
-                    request[field_name] = str(uuid.uuid4())
-            elif not request.get(field_name):
-                request[field_name] = str(uuid.uuid4())
-            return
-
-        if is_proto3_optional:
-            try:
-                # Pure protobuf messages
-                if not request.HasField(field_name):
-                    setattr(request, field_name, str(uuid.uuid4()))
-            except (AttributeError, ValueError):
-                # Proto-plus messages or other objects
-                if field_name not in request:
-                    setattr(request, field_name, str(uuid.uuid4()))
-        else:
-            if not getattr(request, field_name):
-                setattr(request, field_name, str(uuid.uuid4()))
+        _method_helpers.setup_request_id(request, field_name, is_proto3_optional)
     {% endif %}
 
     def _add_cred_info_for_auth_errors(

packages/gapic-generator/gapic/templates/setup.py.j2

@@ -36,7 +36,7 @@ else:
     release_status = "Development Status :: 5 - Production/Stable"
 
 dependencies = [
-    "google-api-core[grpc] >= 2.25.0, <3.0.0",
+    "google-api-core[grpc] @ git+https://github.com/googleapis/google-cloud-python.git@feat/gapic-centralization-api-core#egg=google-api-core&subdirectory=packages/google-api-core",
     # Exclude incompatible versions of `google-auth`
     # See https://github.com/googleapis/google-cloud-python/issues/12364
     "google-auth >= 2.14.1, <3.0.0,!=2.24.0,!=2.25.0",

packages/gapic-generator/gapic/templates/testing/_default_constraints.j2

@@ -1,7 +1,7 @@
 {% from '_pypi_packages.j2' import pypi_packages %}
 # This constraints file is required for unit tests.
 # List all library dependencies and extras in this file.
-google-api-core
+google-api-core @ git+https://github.com/googleapis/google-cloud-python.git@feat/gapic-centralization-api-core#egg=google-api-core&subdirectory=packages/google-api-core
 google-auth
 grpcio
 proto-plus

packages/gapic-generator/gapic/templates/testing/constraints-3.10-async-rest.txt.j2

@@ -8,7 +8,7 @@
 # pinning their versions to their lower bounds.
 # For example, if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0",
 # then this file should have google-cloud-foo==1.14.0
-google-api-core==2.25.0
+google-api-core @ git+https://github.com/googleapis/google-cloud-python.git@feat/gapic-centralization-api-core#egg=google-api-core&subdirectory=packages/google-api-core
 google-auth==2.35.0
 grpcio==1.59.0
 proto-plus==1.26.1

packages/gapic-generator/gapic/templates/testing/constraints-3.10.txt.j2

@@ -5,7 +5,7 @@
 # pinning their versions to their lower bounds.
 # For example, if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0",
 # then this file should have google-cloud-foo==1.14.0
-google-api-core==2.25.0
+google-api-core @ git+https://github.com/googleapis/google-cloud-python.git@feat/gapic-centralization-api-core#egg=google-api-core&subdirectory=packages/google-api-core
 google-auth==2.14.1
 grpcio==1.59.0
 proto-plus==1.26.1

packages/gapic-generator/gapic/templates/testing/constraints-3.13.txt.j2

@@ -6,7 +6,7 @@
 # Require the latest major version be installed for each dependency.
 # e.g., if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0",
 # Then this file should have google-cloud-foo>=1
-google-api-core>=2
+google-api-core @ git+https://github.com/googleapis/google-cloud-python.git@feat/gapic-centralization-api-core#egg=google-api-core&subdirectory=packages/google-api-core
 google-auth>=2
 grpcio>=1
 proto-plus>=1

packages/gapic-generator/gapic/templates/testing/constraints-3.14.txt.j2

@@ -6,7 +6,7 @@
 # Require the latest major version be installed for each dependency.
 # e.g., if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0",
 # Then this file should have google-cloud-foo>=1
-google-api-core>=2
+google-api-core @ git+https://github.com/googleapis/google-cloud-python.git@feat/gapic-centralization-api-core#egg=google-api-core&subdirectory=packages/google-api-core
 google-auth>=2
 grpcio>=1
 proto-plus>=1

packages/gapic-generator/gapic/templates/testing/constraints-3.15.txt.j2

@@ -6,7 +6,7 @@
 # Require the latest major version be installed for each dependency.
 # e.g., if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0",
 # Then this file should have google-cloud-foo>=1
-google-api-core>=2
+google-api-core @ git+https://github.com/googleapis/google-cloud-python.git@feat/gapic-centralization-api-core#egg=google-api-core&subdirectory=packages/google-api-core
 google-auth>=2
 grpcio>=1
 proto-plus>=1