diff --git a/sdk/ml/azure-ai-ml/CHANGELOG.md b/sdk/ml/azure-ai-ml/CHANGELOG.md index 73d658a90ffb..14f8f3092753 100644 --- a/sdk/ml/azure-ai-ml/CHANGELOG.md +++ b/sdk/ml/azure-ai-ml/CHANGELOG.md @@ -3,6 +3,7 @@ ## 1.35.0 (unreleased) ### Features Added +- Added support for asset-backed default values on non-primitive component inputs of type `uri_file`, `uri_folder` and `mltable`. A default such as `default: azureml:my_data_asset:1` is now accepted by `load_component()` instead of failing with `Non-primitive type Input has no default value`, and is preserved through YAML, SDK and REST serialization. Defaults for these types must be a string asset or path reference. ### Bugs Fixed - Fixed internal pipeline `Command` node dropping node-level interactive `services` (SSH, JupyterLab, TensorBoard, VS Code, etc.) during serialization, which prevented interactive endpoints from being created for Singularity jobs. The `services` are now serialized into the pipeline REST request and round-tripped on deserialization, matching the public `Command` node behavior. diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/constants/_component.py b/sdk/ml/azure-ai-ml/azure/ai/ml/constants/_component.py index 44cb0b9b3488..9026b67ef2e7 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/constants/_component.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/constants/_component.py @@ -131,11 +131,13 @@ class IOConstants: ComponentParameterTypes.BOOLEAN: lambda v: str(v).lower() == "true", ComponentParameterTypes.NUMBER: float, } + # Non-primitive (asset-backed) input types that accept an asset/path reference string as default value. + ASSET_TYPES_SUPPORTING_DEFAULT = ["uri_folder", "uri_file", "mltable"] # For validation, indicates specific parameters combination for each type INPUT_TYPE_COMBINATION = { - "uri_folder": ["path", "mode"], - "uri_file": ["path", "mode"], - "mltable": ["path", "mode"], + "uri_folder": ["path", "mode", "default"], + "uri_file": ["path", "mode", "default"], + "mltable": ["path", "mode", "default"], "mlflow_model": ["path", "mode"], "custom_model": ["path", "mode"], "integer": ["default", "min", "max"], diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_inputs_outputs/input.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_inputs_outputs/input.py index 4a9451084bc3..09c6c1e9f3c2 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_inputs_outputs/input.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_inputs_outputs/input.py @@ -43,6 +43,8 @@ class Input(_InputOutputBase): # pylint: disable=too-many-instance-attributes :keyword path_on_compute: The access path of the data input for compute :paramtype path_on_compute: Optional[str] :keyword default: The default value of the input. If a default is set, the input data will be optional. + For the asset-backed types 'uri_file', 'uri_folder' and 'mltable', the default must be a string asset or + path reference, eg: 'azureml:my_data_asset:1'. :paramtype default: Union[str, int, float, bool] :keyword min: The minimum value for the input. If a value smaller than the minimum is passed to the job, the job execution will fail. @@ -93,6 +95,7 @@ def __init__( type: str, path: Optional[str] = None, mode: Optional[str] = None, + default: Optional[str] = None, optional: Optional[bool] = None, description: Optional[str] = None, **kwargs: Any, @@ -289,6 +292,15 @@ def _multiple_types(self) -> bool: """ return isinstance(self.type, list) + @property + def _supports_asset_default(self) -> bool: + """Whether this non-primitive input supports an asset/path reference string as default value. + + :return: True if the input type is an asset-backed type supporting default value. + :rtype: bool + """ + return not self._multiple_types and self.type in IOConstants.ASSET_TYPES_SUPPORTING_DEFAULT + def _is_literal(self) -> bool: """Whether this input is a literal @@ -373,8 +385,19 @@ def _update_default(self, default_value: Any) -> None: msg_prefix = f"Default value of Input {name}" if not self._is_primitive_type and default_value is not None: - msg = f"{msg_prefix}cannot be set: Non-primitive type Input has no default value." - raise UserErrorException(msg) + if not self._supports_asset_default: + msg = f"{msg_prefix}cannot be set: Non-primitive type Input has no default value." + raise UserErrorException(msg) + # Asset-backed inputs accept an asset/path reference string as default value, + # eg: "azureml:my_data_asset:1"; it is kept as-is on serialization. + if not isinstance(default_value, str): + msg = ( + f"{msg_prefix}cannot be set: default value of {self.type!r} Input must be a string " + f"asset or path reference, got '{default_value}', type = {type(default_value)!r}." + ) + raise UserErrorException(msg) + self.default = default_value + return if isinstance(default_value, float) and not math.isfinite(default_value): # Since nan/inf cannot be stored in the backend, just ignore them. # logger.warning("Float default value %r is not allowed, ignored." % default_value) diff --git a/sdk/ml/azure-ai-ml/tests/component/unittests/test_input_defaults.py b/sdk/ml/azure-ai-ml/tests/component/unittests/test_input_defaults.py new file mode 100644 index 000000000000..56672933ee25 --- /dev/null +++ b/sdk/ml/azure-ai-ml/tests/component/unittests/test_input_defaults.py @@ -0,0 +1,164 @@ +import tempfile +from pathlib import Path + +import pytest +from marshmallow import ValidationError + +from azure.ai.ml import Input, load_component +from azure.ai.ml.entities import CommandComponent, Component, PipelineComponent +from azure.ai.ml.exceptions import UserErrorException + +from .._util import _COMPONENT_TIMEOUT_SECOND + +components_dir = "./tests/test_configs/components/" + + +@pytest.mark.timeout(_COMPONENT_TIMEOUT_SECOND) +@pytest.mark.unittest +@pytest.mark.pipeline_test +class TestAssetBackedInputDefaults: + @staticmethod + def _dump_and_reload(component: Component) -> Component: + with tempfile.TemporaryDirectory() as tmp_dir: + dump_path = Path(tmp_dir) / "component.yml" + component.dump(dump_path) + return load_component(source=dump_path) + + @pytest.mark.parametrize("input_type", ["uri_file", "uri_folder", "mltable"]) + def test_input_with_asset_backed_default(self, input_type: str): + input_obj = Input(type=input_type, mode="ro_mount", default="azureml:my_asset:1") + + assert input_obj.type == input_type + assert input_obj.mode == "ro_mount" + assert input_obj.default == "azureml:my_asset:1" + assert input_obj._to_dict() == { + "type": input_type, + "mode": "ro_mount", + "default": "azureml:my_asset:1", + } + + def test_input_with_datastore_uri_default(self): + default = "azureml://datastores/workspaceblobstore/paths/data/file.csv" + input_obj = Input(type="uri_file", default=default) + + assert input_obj.default == default + assert input_obj._to_dict() == {"type": "uri_file", "default": default} + + def test_input_with_optional_asset_backed_default(self): + input_obj = Input(type="uri_file", mode="ro_mount", default="azureml:my_asset:1", optional=True) + + assert input_obj.optional is True + assert input_obj._to_dict() == { + "type": "uri_file", + "mode": "ro_mount", + "default": "azureml:my_asset:1", + "optional": True, + } + + def test_input_asset_backed_default_rest_round_trip(self): + input_obj = Input(type="uri_file", mode="ro_mount", default="azureml:my_asset:1") + + rest_obj = input_obj._to_rest_object() + assert rest_obj["type"] == "uri_file" + assert rest_obj["default"] == "azureml:my_asset:1" + + from_rest = Input._from_rest_object(dict(rest_obj)) + assert from_rest.type == "uri_file" + assert from_rest.mode == "ro_mount" + assert from_rest.default == "azureml:my_asset:1" + + def test_input_non_string_default_raises(self): + with pytest.raises(UserErrorException, match="must be a string asset or path reference"): + Input(type="uri_file", default=123) + + with pytest.raises(UserErrorException, match="must be a string asset or path reference"): + Input(type="uri_folder", default=True) + + with pytest.raises(UserErrorException, match="cannot be set"): + Input(type="uri_file", default=Input(type="uri_file", path="azureml:my_asset:1")) + + def test_input_unsupported_type_default_raises(self): + with pytest.raises(UserErrorException, match="Non-primitive type Input has no default value"): + Input(type="mlflow_model", default="azureml:my_model:1") + + def test_primitive_defaults_not_impacted(self): + assert Input(type="integer", default=1, min=0, max=10).default == 1 + assert Input(type="number", default="10.99").default == 10.99 + assert Input(type="string", default="value").default == "value" + assert Input(type="boolean", default=True).default is True + assert Input(type="uri_file", mode="ro_mount").default is None + + with pytest.raises(UserErrorException): + Input(type="integer", default=[1]) + + def test_load_command_component_with_asset_backed_defaults(self): + component: CommandComponent = load_component(source=components_dir + "input_asset_defaults_component.yml") + + assert component.inputs["spaceship_data"].type == "uri_file" + assert component.inputs["spaceship_data"].mode == "ro_mount" + assert ( + component.inputs["spaceship_data"].default + == "azureml:dsp_da_test_use_case_spaceships_uri_file:50322a7173b6976c" + ) + assert component.inputs["folder_data"].default == "azureml:folder_asset:1" + assert component.inputs["table_data"].default == "azureml:table_asset:1" + assert component.inputs["year"].default == 2025 + + # yaml round trip + component_dict = component._to_dict() + assert component_dict["inputs"]["spaceship_data"] == { + "type": "uri_file", + "mode": "ro_mount", + "default": "azureml:dsp_da_test_use_case_spaceships_uri_file:50322a7173b6976c", + } + reloaded = self._dump_and_reload(component) + assert reloaded.inputs["spaceship_data"]._to_dict() == component.inputs["spaceship_data"]._to_dict() + assert reloaded.inputs["year"]._to_dict() == component.inputs["year"]._to_dict() + + # rest round trip + rest_object = component._to_rest_object() + assert rest_object.properties.component_spec["inputs"]["spaceship_data"] == { + "type": "uri_file", + "mode": "ro_mount", + "default": "azureml:dsp_da_test_use_case_spaceships_uri_file:50322a7173b6976c", + } + from_rest = Component._from_rest_object(rest_object) + assert ( + from_rest.inputs["spaceship_data"].default + == "azureml:dsp_da_test_use_case_spaceships_uri_file:50322a7173b6976c" + ) + assert from_rest.inputs["spaceship_data"].mode == "ro_mount" + + def test_load_pipeline_component_with_asset_backed_default(self): + component: PipelineComponent = load_component( + source=components_dir + "input_asset_defaults_pipeline_component.yml" + ) + + spaceship_data = component.inputs["spaceship_data"] + assert spaceship_data.type == "uri_file" + assert spaceship_data.mode == "ro_mount" + assert spaceship_data.default == "azureml:dsp_da_test_use_case_spaceships_uri_file:50322a7173b6976c" + assert component._to_dict()["inputs"]["spaceship_data"]["default"] == ( + "azureml:dsp_da_test_use_case_spaceships_uri_file:50322a7173b6976c" + ) + + def test_load_component_with_invalid_default(self): + with pytest.raises(ValidationError): + load_component(source=components_dir + "invalid/input_asset_defaults_invalid_component.yml") + + def test_component_call_with_and_without_default(self): + component: CommandComponent = load_component(source=components_dir + "input_asset_defaults_component.yml") + expected_default = "azureml:dsp_da_test_use_case_spaceships_uri_file:50322a7173b6976c" + + # input with default is not required, so omitting it is valid + node = component(year=2025) + validation_result = node._validate_inputs() + assert "inputs.spaceship_data" not in validation_result.error_messages + assert node._component.inputs["spaceship_data"].default == expected_default + + # explicit value overrides the default without mutating the component default + override = Input(type="uri_file", path="azureml:other_asset:1", mode="ro_mount") + other_node = component(year=2025, spaceship_data=override) + assert other_node.inputs["spaceship_data"]._data.path == "azureml:other_asset:1" + assert component.inputs["spaceship_data"].default == expected_default + assert node._component.inputs["spaceship_data"].default == expected_default diff --git a/sdk/ml/azure-ai-ml/tests/test_configs/components/input_asset_defaults_component.yml b/sdk/ml/azure-ai-ml/tests/test_configs/components/input_asset_defaults_component.yml new file mode 100644 index 000000000000..e79a97d97a30 --- /dev/null +++ b/sdk/ml/azure-ai-ml/tests/test_configs/components/input_asset_defaults_component.yml @@ -0,0 +1,35 @@ +$schema: https://azuremlschemas.azureedge.net/development/commandComponent.schema.json +type: command + +name: input_asset_defaults_component +display_name: Component with asset-backed input defaults +description: This component has default values for non-primitive inputs + +version: 0.0.1 + +inputs: + spaceship_data: + type: uri_file + mode: ro_mount + default: azureml:dsp_da_test_use_case_spaceships_uri_file:50322a7173b6976c + folder_data: + type: uri_folder + default: azureml:folder_asset:1 + table_data: + type: mltable + default: azureml:table_asset:1 + year: + type: integer + default: 2025 + +outputs: + output_path: + type: uri_folder + +command: >- + echo ${{inputs.spaceship_data}} & + echo ${{inputs.folder_data}} & + echo ${{inputs.table_data}} & + echo ${{inputs.year}} > ${{outputs.output_path}}/year + +environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu:33 diff --git a/sdk/ml/azure-ai-ml/tests/test_configs/components/input_asset_defaults_pipeline_component.yml b/sdk/ml/azure-ai-ml/tests/test_configs/components/input_asset_defaults_pipeline_component.yml new file mode 100644 index 000000000000..168cc8389562 --- /dev/null +++ b/sdk/ml/azure-ai-ml/tests/test_configs/components/input_asset_defaults_pipeline_component.yml @@ -0,0 +1,30 @@ +$schema: https://azuremlschemas.azureedge.net/development/pipelineComponent.schema.json +type: pipeline + +name: dsp_pcp_test_use_case_pcp1 +display_name: "Test Use Case: pipeline component" +description: Pipeline component with an asset-backed default on a uri_file input + +version: 1 + +inputs: + year: + type: integer + spaceship_data: + type: uri_file + mode: ro_mount + default: azureml:dsp_da_test_use_case_spaceships_uri_file:50322a7173b6976c + +outputs: + ship_data: + type: uri_folder + +jobs: + component_a_job: + type: command + component: file:./input_asset_defaults_component.yml + inputs: + year: ${{parent.inputs.year}} + spaceship_data: ${{parent.inputs.spaceship_data}} + outputs: + output_path: ${{parent.outputs.ship_data}} diff --git a/sdk/ml/azure-ai-ml/tests/test_configs/components/invalid/input_asset_defaults_invalid_component.yml b/sdk/ml/azure-ai-ml/tests/test_configs/components/invalid/input_asset_defaults_invalid_component.yml new file mode 100644 index 000000000000..c4d40f60ea37 --- /dev/null +++ b/sdk/ml/azure-ai-ml/tests/test_configs/components/invalid/input_asset_defaults_invalid_component.yml @@ -0,0 +1,23 @@ +$schema: https://azuremlschemas.azureedge.net/development/commandComponent.schema.json +type: command + +name: input_asset_defaults_invalid_component +display_name: Component with an invalid default on a non-primitive input +description: The default of a uri_file input must be a string asset or path reference + +version: 0.0.1 + +inputs: + spaceship_data: + type: uri_file + mode: ro_mount + default: 123 + +outputs: + output_path: + type: uri_folder + +command: >- + echo ${{inputs.spaceship_data}} > ${{outputs.output_path}}/out + +environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu:33