From e9ab09cefc9a20c733f407c7b745491ad596d563 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Oct 2026 16:21:39 +0200 Subject: [PATCH 01/12] diag: use structpath for diagnostic paths Diagnostic.Paths is []*structpath.PathNode; producers convert dyn paths with dyn.ToStructPath(s) or use structpath.MustParsePaths. Map keys that are not identifiers are now quoted in rendered paths. Co-authored-by: Isaac --- .nextchanges/bundles/6974.md | 1 + .../variable_in_resource_key/output.txt | 6 +- bundle/apps/validate.go | 2 +- bundle/apps/validate_test.go | 2 +- bundle/artifacts/expand_globs.go | 2 +- bundle/config/loader/process_include.go | 9 ++- bundle/config/loader/process_include_test.go | 75 ++++++++++--------- .../apply_source_linked_deployment_preset.go | 26 +++---- bundle/config/mutator/compute_id_compat.go | 2 +- bundle/config/mutator/load_dbalert_files.go | 8 +- .../python/apply_python_output_test.go | 4 +- .../mutator/python/python_diagnostics.go | 5 +- .../mutator/python/python_diagnostics_test.go | 3 +- .../config/mutator/python/python_mutator.go | 19 ++--- .../mutator/python/python_mutator_test.go | 11 +-- .../mutator/reject_internal_resources.go | 4 +- .../expand_pipeline_glob_paths_test.go | 8 +- .../config/mutator/resourcemutator/run_as.go | 3 +- .../resourcemutator/secret_scope_fixups.go | 2 +- .../resourcemutator/validate_target_mode.go | 5 +- .../mutator/rewrite_workspace_prefix.go | 2 +- bundle/config/mutator/sync_infer_root.go | 2 +- bundle/config/mutator/translate_paths.go | 5 +- .../validate_secret_value_is_variable.go | 8 +- .../validate/all_resources_have_values.go | 2 +- bundle/config/validate/enum.go | 2 +- bundle/config/validate/files_to_sync.go | 2 +- .../validate/interpolation_in_auth_config.go | 2 +- .../validate/job_cluster_key_defined.go | 2 +- .../config/validate/job_task_cluster_spec.go | 2 +- .../no_interpolation_in_bundle_name.go | 4 +- .../no_variable_reference_in_resource_key.go | 2 +- .../pipeline_development_deprecated.go | 4 +- bundle/config/validate/required.go | 19 ++--- bundle/config/validate/scripts.go | 6 +- bundle/config/validate/single_node_cluster.go | 2 +- .../validate/single_node_cluster_test.go | 11 +-- .../config/validate/unique_resource_keys.go | 7 +- .../config/validate/validate_artifact_path.go | 5 +- .../validate/validate_artifact_path_test.go | 13 ++-- .../validate/validate_dashboard_etags.go | 2 +- .../validate/validate_deployment_fields.go | 2 +- bundle/config/validate/validate_engine.go | 3 +- .../validate/validate_genie_space_etags.go | 2 +- .../validate_job_run_idempotency_token.go | 2 +- .../config/validate/validate_sync_patterns.go | 2 +- .../config/validate/validate_volume_path.go | 2 +- .../check_dashboards_modified_remotely.go | 4 +- bundle/internal/schema/annotations_file.go | 3 +- bundle/libraries/expand_glob_references.go | 2 +- bundle/libraries/same_name_libraries.go | 7 +- bundle/libraries/upload.go | 1 - bundle/render/render_text_output_test.go | 21 +++--- bundle/tests/validate_test.go | 47 ++++++------ cmd/apps/import.go | 3 +- libs/diag/diagnostic.go | 3 +- libs/dyn/convert/normalize.go | 16 ++-- libs/dyn/convert/normalize_test.go | 52 ++++++------- libs/dyn/dynvar/ref.go | 21 ++++++ libs/dyn/dynvar/ref_structpath_test.go | 56 ++++++++++++++ libs/dyn/structpath.go | 25 +++++++ libs/dyn/structpath_test.go | 29 +++++++ libs/flags/json_flag_test.go | 6 +- libs/structs/structpath/path.go | 32 +++----- libs/structs/structpath/path_test.go | 57 ++------------ 65 files changed, 383 insertions(+), 316 deletions(-) create mode 100644 .nextchanges/bundles/6974.md create mode 100644 libs/dyn/dynvar/ref_structpath_test.go create mode 100644 libs/dyn/structpath.go create mode 100644 libs/dyn/structpath_test.go diff --git a/.nextchanges/bundles/6974.md b/.nextchanges/bundles/6974.md new file mode 100644 index 00000000000..bf4b18fc997 --- /dev/null +++ b/.nextchanges/bundles/6974.md @@ -0,0 +1 @@ +* Diagnostic paths now quote map keys that are not identifiers, e.g. `resources.jobs['${var.env}_job']`. ([#6974](https://github.com/databricks/cli/pull/6974)) diff --git a/acceptance/bundle/variables/variable_in_resource_key/output.txt b/acceptance/bundle/variables/variable_in_resource_key/output.txt index 6fc17dc855d..841db2e41c5 100644 --- a/acceptance/bundle/variables/variable_in_resource_key/output.txt +++ b/acceptance/bundle/variables/variable_in_resource_key/output.txt @@ -1,13 +1,13 @@ Warning: unknown field: ${var.env} - at targets.dev.resources.jobs.${var.env}_job_2 + at targets.dev.resources.jobs['${var.env}_job_2'] in databricks.yml:21:11 Error: resource key "${var.env}_job" must not contain variable references - at resources.jobs.${var.env}_job + at resources.jobs['${var.env}_job'] in databricks.yml:12:7 Error: resource key "${var.env}_job_2" must not contain variable references - at targets.dev.resources.jobs.${var.env}_job_2 + at targets.dev.resources.jobs['${var.env}_job_2'] in databricks.yml:20:11 diff --git a/bundle/apps/validate.go b/bundle/apps/validate.go index 45a5b49c672..f7a2fee7103 100644 --- a/bundle/apps/validate.go +++ b/bundle/apps/validate.go @@ -172,7 +172,7 @@ func warnForAppResourcePermissions(b *bundle.Bundle, appKey string, app *resourc ref.permission, appKey, ), - Paths: []dyn.Path{dyn.MustPathFromString(appPath)}, + Paths: dyn.ToStructPaths(dyn.MustPathFromString(appPath)), Locations: b.Config.GetLocations(appPath), }) } diff --git a/bundle/apps/validate_test.go b/bundle/apps/validate_test.go index 5edae10a884..3285851ce91 100644 --- a/bundle/apps/validate_test.go +++ b/bundle/apps/validate_test.go @@ -213,7 +213,7 @@ func TestAppsValidateResourcePermissionsWarning(t *testing.T) { require.Len(t, warnings, 1) require.Contains(t, warnings[0].Summary, tc.wantSummary) require.Contains(t, warnings[0].Detail, "service_principal_name: ${resources.apps.my_app.service_principal_client_id}") - require.Equal(t, dyn.MustPathFromString("resources.apps.my_app"), warnings[0].Paths[0]) + require.Equal(t, "resources.apps.my_app", warnings[0].Paths[0].String()) } else { require.Empty(t, warnings) } diff --git a/bundle/artifacts/expand_globs.go b/bundle/artifacts/expand_globs.go index af358d3dda9..e7cd5cc1308 100644 --- a/bundle/artifacts/expand_globs.go +++ b/bundle/artifacts/expand_globs.go @@ -27,7 +27,7 @@ func createGlobError(v dyn.Value, p dyn.Path, message string) diag.Diagnostic { Severity: diag.Error, Summary: fmt.Sprintf("%s: %s", source, message), Locations: []dyn.Location{v.Location()}, - Paths: []dyn.Path{p}, + Paths: dyn.ToStructPaths(p), } } diff --git a/bundle/config/loader/process_include.go b/bundle/config/loader/process_include.go index 3a64297814c..bc82bee1867 100644 --- a/bundle/config/loader/process_include.go +++ b/bundle/config/loader/process_include.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) func validateFileFormat(configRoot dyn.Value, filePath string) diag.Diagnostics { @@ -108,16 +109,16 @@ func validateSingleResourceDefined(configRoot dyn.Value, ext, typ string) diag.D } var locations []dyn.Location - var paths []dyn.Path + var paths []*structpath.PathNode for _, rr := range resources { locations = append(locations, rr.value.Locations()...) - paths = append(paths, rr.path) + paths = append(paths, dyn.ToStructPath(rr.path)) } // Sort the locations and paths to make the output deterministic. slices.SortFunc(locations, func(a, b dyn.Location) int { return cmp.Compare(a.String(), b.String()) }) - slices.SortFunc(paths, func(a, b dyn.Path) int { + slices.SortFunc(paths, func(a, b *structpath.PathNode) int { return cmp.Compare(a.String(), b.String()) }) @@ -168,7 +169,7 @@ func (m *processInclude) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos Detail: `An include section is defined in a file that is not databricks.yml. Only includes defined in databricks.yml are applied.`, Locations: this.GetLocations("include"), - Paths: []dyn.Path{dyn.MustPathFromString("include")}, + Paths: structpath.MustParsePaths("include"), }) } diff --git a/bundle/config/loader/process_include_test.go b/bundle/config/loader/process_include_test.go index c768dc57dc9..28c95ac905b 100644 --- a/bundle/config/loader/process_include_test.go +++ b/bundle/config/loader/process_include_test.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/config/loader" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -71,10 +72,10 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) { {File: "testdata/format_not_match/single_job.pipeline.yaml", Line: 11, Column: 11}, {File: "testdata/format_not_match/single_job.pipeline.yaml", Line: 4, Column: 7}, }, - Paths: []dyn.Path{ - dyn.MustPathFromString("resources.jobs.job1"), - dyn.MustPathFromString("targets.target1.resources.jobs.job1"), - }, + Paths: structpath.MustParsePaths( + "resources.jobs.job1", + "targets.target1.resources.jobs.job1", + ), }, }, "job_and_pipeline.job.yml": { @@ -86,10 +87,10 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) { {File: "testdata/format_not_match/job_and_pipeline.job.yml", Line: 11, Column: 11}, {File: "testdata/format_not_match/job_and_pipeline.job.yml", Line: 4, Column: 7}, }, - Paths: []dyn.Path{ - dyn.MustPathFromString("resources.pipelines.pipeline1"), - dyn.MustPathFromString("targets.target1.resources.jobs.job1"), - }, + Paths: structpath.MustParsePaths( + "resources.pipelines.pipeline1", + "targets.target1.resources.jobs.job1", + ), }, }, "job_and_pipeline.experiment.yml": { @@ -101,10 +102,10 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) { {File: "testdata/format_not_match/job_and_pipeline.experiment.yml", Line: 11, Column: 11}, {File: "testdata/format_not_match/job_and_pipeline.experiment.yml", Line: 4, Column: 7}, }, - Paths: []dyn.Path{ - dyn.MustPathFromString("resources.pipelines.pipeline1"), - dyn.MustPathFromString("targets.target1.resources.jobs.job1"), - }, + Paths: structpath.MustParsePaths( + "resources.pipelines.pipeline1", + "targets.target1.resources.jobs.job1", + ), }, }, "two_jobs.job.yml": { @@ -116,10 +117,10 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) { {File: "testdata/format_not_match/two_jobs.job.yml", Line: 4, Column: 7}, {File: "testdata/format_not_match/two_jobs.job.yml", Line: 7, Column: 7}, }, - Paths: []dyn.Path{ - dyn.MustPathFromString("resources.jobs.job1"), - dyn.MustPathFromString("resources.jobs.job2"), - }, + Paths: structpath.MustParsePaths( + "resources.jobs.job1", + "resources.jobs.job2", + ), }, }, "second_job_in_target.job.yml": { @@ -131,10 +132,10 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) { {File: "testdata/format_not_match/second_job_in_target.job.yml", Line: 11, Column: 11}, {File: "testdata/format_not_match/second_job_in_target.job.yml", Line: 4, Column: 7}, }, - Paths: []dyn.Path{ - dyn.MustPathFromString("resources.jobs.job1"), - dyn.MustPathFromString("targets.target1.resources.jobs.job2"), - }, + Paths: structpath.MustParsePaths( + "resources.jobs.job1", + "targets.target1.resources.jobs.job2", + ), }, }, "two_jobs_in_target.job.yml": { @@ -146,10 +147,10 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) { {File: "testdata/format_not_match/two_jobs_in_target.job.yml", Line: 6, Column: 11}, {File: "testdata/format_not_match/two_jobs_in_target.job.yml", Line: 8, Column: 11}, }, - Paths: []dyn.Path{ - dyn.MustPathFromString("targets.target1.resources.jobs.job1"), - dyn.MustPathFromString("targets.target1.resources.jobs.job2"), - }, + Paths: structpath.MustParsePaths( + "targets.target1.resources.jobs.job1", + "targets.target1.resources.jobs.job2", + ), }, }, "multiple_resources.model_serving_endpoint.yml": { @@ -182,19 +183,19 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) { {File: "testdata/format_not_match/multiple_resources.model_serving_endpoint.yml", Line: 4, Column: 7}, {File: "testdata/format_not_match/multiple_resources.model_serving_endpoint.yml", Line: 8, Column: 7}, }, - Paths: []dyn.Path{ - dyn.MustPathFromString("resources.experiments.experiment1"), - dyn.MustPathFromString("resources.jobs.job1"), - dyn.MustPathFromString("resources.jobs.job2"), - dyn.MustPathFromString("resources.model_serving_endpoints.model_serving_endpoint1"), - dyn.MustPathFromString("resources.models.model1"), - dyn.MustPathFromString("resources.pipelines.pipeline1"), - dyn.MustPathFromString("resources.pipelines.pipeline2"), - dyn.MustPathFromString("resources.schemas.schema1"), - dyn.MustPathFromString("targets.target1.resources.jobs.job3"), - dyn.MustPathFromString("targets.target1.resources.quality_monitors.quality_monitor1"), - dyn.MustPathFromString("targets.target1.resources.registered_models.registered_model1"), - }, + Paths: structpath.MustParsePaths( + "resources.experiments.experiment1", + "resources.jobs.job1", + "resources.jobs.job2", + "resources.model_serving_endpoints.model_serving_endpoint1", + "resources.models.model1", + "resources.pipelines.pipeline1", + "resources.pipelines.pipeline2", + "resources.schemas.schema1", + "targets.target1.resources.jobs.job3", + "targets.target1.resources.quality_monitors.quality_monitor1", + "targets.target1.resources.registered_models.registered_model1", + ), }, }, } { diff --git a/bundle/config/mutator/apply_source_linked_deployment_preset.go b/bundle/config/mutator/apply_source_linked_deployment_preset.go index c1e817673d9..74293566779 100644 --- a/bundle/config/mutator/apply_source_linked_deployment_preset.go +++ b/bundle/config/mutator/apply_source_linked_deployment_preset.go @@ -36,11 +36,9 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment")) diags = diags.Append( diag.Diagnostic{ - Severity: diag.Warning, - Summary: "source-linked deployment is available only in the Databricks Workspace", - Paths: []dyn.Path{ - path, - }, + Severity: diag.Warning, + Summary: "source-linked deployment is available only in the Databricks Workspace", + Paths: dyn.ToStructPaths(path), Locations: b.Config.GetLocations(path[2:].String()), }, ) @@ -56,11 +54,9 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment")) diags = diags.Append( diag.Diagnostic{ - Severity: diag.Warning, - Summary: "source-linked deployment in non-development mode is deprecated and will not be supported in a future release", - Paths: []dyn.Path{ - path, - }, + Severity: diag.Warning, + Summary: "source-linked deployment in non-development mode is deprecated and will not be supported in a future release", + Paths: dyn.ToStructPaths(path), Locations: b.Config.GetLocations(path[2:].String()), }, ) @@ -77,12 +73,10 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle path := dyn.NewPath(dyn.Key("workspace"), dyn.Key("file_path")) diags = diags.Append( diag.Diagnostic{ - Severity: diag.Warning, - Summary: "workspace.file_path setting will be ignored in source-linked deployment mode", - Detail: "In source-linked deployment files are not copied to the destination and resources use source files instead", - Paths: []dyn.Path{ - path, - }, + Severity: diag.Warning, + Summary: "workspace.file_path setting will be ignored in source-linked deployment mode", + Detail: "In source-linked deployment files are not copied to the destination and resources use source files instead", + Paths: dyn.ToStructPaths(path), Locations: b.Config.GetLocations(path.String()), }, ) diff --git a/bundle/config/mutator/compute_id_compat.go b/bundle/config/mutator/compute_id_compat.go index 8f1ff5868e8..c29b01cbd1c 100644 --- a/bundle/config/mutator/compute_id_compat.go +++ b/bundle/config/mutator/compute_id_compat.go @@ -55,7 +55,7 @@ func rewriteComputeIdToClusterId(v dyn.Value, p dyn.Path) (dyn.Value, diag.Diagn Severity: diag.Warning, Summary: "compute_id is deprecated, please use cluster_id instead", Locations: computeId.Locations(), - Paths: []dyn.Path{computeIdPath}, + Paths: dyn.ToStructPaths(computeIdPath), }) clusterIdPath := p.Append(dyn.Key("cluster_id")) diff --git a/bundle/config/mutator/load_dbalert_files.go b/bundle/config/mutator/load_dbalert_files.go index 5c3e6921b93..a8571620f44 100644 --- a/bundle/config/mutator/load_dbalert_files.go +++ b/bundle/config/mutator/load_dbalert_files.go @@ -85,7 +85,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia Severity: diag.Error, Summary: fmt.Sprintf("field %s is not allowed in the bundle configuration.", k), Detail: "When a .dbalert.json is specified, only the following fields are allowed in the bundle configuration: " + strings.Join(allowedInYAML, ", "), - Paths: []dyn.Path{dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.%s", alertKey, k))}, + Paths: dyn.ToStructPaths(dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.%s", alertKey, k))), Locations: v.Locations(), }, } @@ -107,7 +107,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia Severity: diag.Error, Summary: fmt.Sprintf("failed to read .dbalert.json file %s: %s", alert.FilePath, err), Detail: "", - Paths: []dyn.Path{dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.file_path", alertKey))}, + Paths: dyn.ToStructPaths(dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.file_path", alertKey))), Locations: alertV.Get("file_path").Locations(), }, } @@ -122,7 +122,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia Severity: diag.Error, Summary: fmt.Sprintf("failed to parse .dbalert.json file %s: %s", alert.FilePath, err), Detail: "", - Paths: []dyn.Path{dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.file_path", alertKey))}, + Paths: dyn.ToStructPaths(dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.file_path", alertKey))), Locations: alertV.Get("file_path").Locations(), }, } @@ -136,7 +136,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia Severity: diag.Error, Summary: fmt.Sprintf(".alert file %s must not contain variable interpolations.", alert.FilePath), Detail: "Please inline the alert configuration in the bundle configuration to use variables", - Paths: []dyn.Path{dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.file_path", alertKey))}, + Paths: dyn.ToStructPaths(dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.file_path", alertKey))), Locations: alertV.Get("file_path").Locations(), }, } diff --git a/bundle/config/mutator/python/apply_python_output_test.go b/bundle/config/mutator/python/apply_python_output_test.go index 2d64d6c4bbc..32bc409c74c 100644 --- a/bundle/config/mutator/python/apply_python_output_test.go +++ b/bundle/config/mutator/python/apply_python_output_test.go @@ -4,10 +4,8 @@ import ( "testing" "github.com/databricks/cli/bundle/config/mutator/resourcemutator" - - "github.com/databricks/cli/libs/dyn/merge" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/merge" "github.com/stretchr/testify/assert" ) diff --git a/bundle/config/mutator/python/python_diagnostics.go b/bundle/config/mutator/python/python_diagnostics.go index 7a1e13b4e19..b947b3f6ad3 100644 --- a/bundle/config/mutator/python/python_diagnostics.go +++ b/bundle/config/mutator/python/python_diagnostics.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) // pythonDiagnostic is a single entry in diagnostics.json @@ -55,9 +56,9 @@ func parsePythonDiagnostics(input io.Reader) (diag.Diagnostics, error) { if err != nil { return nil, fmt.Errorf("failed to parse path: %s", err) } - var paths []dyn.Path + var paths []*structpath.PathNode if path != nil { - paths = []dyn.Path{path} + paths = dyn.ToStructPaths(path) } var locations []dyn.Location diff --git a/bundle/config/mutator/python/python_diagnostics_test.go b/bundle/config/mutator/python/python_diagnostics_test.go index 6fef9eca9cb..c00650ea717 100644 --- a/bundle/config/mutator/python/python_diagnostics_test.go +++ b/bundle/config/mutator/python/python_diagnostics_test.go @@ -6,6 +6,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" ) @@ -55,7 +56,7 @@ func TestParsePythonDiagnostics(t *testing.T) { { Severity: diag.Error, Summary: "error summary", - Paths: []dyn.Path{dyn.MustPathFromString("resources.jobs.job0.name")}, + Paths: structpath.MustParsePaths("resources.jobs.job0.name"), }, }, }, diff --git a/bundle/config/mutator/python/python_mutator.go b/bundle/config/mutator/python/python_mutator.go index 9112aaa808d..a3e3d94279a 100644 --- a/bundle/config/mutator/python/python_mutator.go +++ b/bundle/config/mutator/python/python_mutator.go @@ -13,25 +13,20 @@ import ( "reflect" "strings" - "github.com/databricks/cli/bundle/config/mutator/resourcemutator" - - "github.com/databricks/cli/libs/log" - "github.com/databricks/cli/libs/logdiag" - - "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/databricks-sdk-go/logger" - - "github.com/databricks/cli/libs/python" - - "github.com/databricks/cli/bundle/env" - "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator/resourcemutator" + "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/dyn/yamlloader" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/process" + "github.com/databricks/cli/libs/python" + "github.com/databricks/databricks-sdk-go/logger" ) type phase string diff --git a/bundle/config/mutator/python/python_mutator_test.go b/bundle/config/mutator/python/python_mutator_test.go index 9b106f727bf..b30272dc21f 100644 --- a/bundle/config/mutator/python/python_mutator_test.go +++ b/bundle/config/mutator/python/python_mutator_test.go @@ -11,19 +11,16 @@ import ( "slices" "testing" - "github.com/databricks/cli/libs/dyn/convert" - - "github.com/databricks/cli/bundle/env" - "github.com/stretchr/testify/require" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/process" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestPythonMutator_Name_loadResources(t *testing.T) { diff --git a/bundle/config/mutator/reject_internal_resources.go b/bundle/config/mutator/reject_internal_resources.go index 5b1d38f8175..5948f048d93 100644 --- a/bundle/config/mutator/reject_internal_resources.go +++ b/bundle/config/mutator/reject_internal_resources.go @@ -5,7 +5,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) type rejectInternalResources struct{} @@ -26,7 +26,7 @@ func (m *rejectInternalResources) Apply(ctx context.Context, b *bundle.Bundle) d diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "Internal resources cannot be set in bundle configuration", - Paths: []dyn.Path{dyn.MustPathFromString("resources")}, + Paths: structpath.MustParsePaths("resources"), Locations: b.Config.GetLocations("resources"), }) } diff --git a/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go b/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go index 56440f7cd1c..7341d747570 100644 --- a/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go +++ b/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go @@ -5,18 +5,16 @@ import ( "path/filepath" "testing" - "github.com/databricks/cli/bundle/config/mutator" - "github.com/databricks/cli/bundle/config/mutator/resourcemutator" - - "github.com/stretchr/testify/assert" - "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/mutator/resourcemutator" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/bundle/config/mutator/resourcemutator/run_as.go b/bundle/config/mutator/resourcemutator/run_as.go index b06acc8b46e..66f1bd7e7d1 100644 --- a/bundle/config/mutator/resourcemutator/run_as.go +++ b/bundle/config/mutator/resourcemutator/run_as.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/databricks/databricks-sdk-go/service/sql" @@ -238,7 +239,7 @@ func (m *setRunAs) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { { Severity: diag.Warning, Summary: "You are using the legacy mode of run_as. The support for this mode is experimental and might be removed in a future release of the CLI. In order to run the pipelines in your DABs project as the run_as user this mode changes the owners of the pipelines to the run_as identity, which requires the user deploying the bundle to be a workspace admin, and also a Metastore admin if the pipeline target is in UC.", - Paths: []dyn.Path{dyn.MustPathFromString("experimental.use_legacy_run_as")}, + Paths: structpath.MustParsePaths("experimental.use_legacy_run_as"), Locations: b.Config.GetLocations("experimental.use_legacy_run_as"), }, } diff --git a/bundle/config/mutator/resourcemutator/secret_scope_fixups.go b/bundle/config/mutator/resourcemutator/secret_scope_fixups.go index 4b1330d220a..66010596dab 100644 --- a/bundle/config/mutator/resourcemutator/secret_scope_fixups.go +++ b/bundle/config/mutator/resourcemutator/secret_scope_fixups.go @@ -138,7 +138,7 @@ func (m *secretScopeFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Di Severity: diag.Error, Summary: "Failed to collapse permissions for secret scope", Detail: err.Error(), - Paths: []dyn.Path{dyn.MustPathFromString("resources.secret_scopes." + key)}, + Paths: dyn.ToStructPaths(dyn.MustPathFromString("resources.secret_scopes." + key)), Locations: []dyn.Location{b.Config.GetLocation("resources.secret_scopes." + key)}, }, } diff --git a/bundle/config/mutator/resourcemutator/validate_target_mode.go b/bundle/config/mutator/resourcemutator/validate_target_mode.go index d100c327fe8..bab38d94700 100644 --- a/bundle/config/mutator/resourcemutator/validate_target_mode.go +++ b/bundle/config/mutator/resourcemutator/validate_target_mode.go @@ -5,12 +5,11 @@ import ( "fmt" "strings" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/iamutil" - "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/iamutil" ) type validateTargetMode struct{} diff --git a/bundle/config/mutator/rewrite_workspace_prefix.go b/bundle/config/mutator/rewrite_workspace_prefix.go index b67f773f207..30914fd149e 100644 --- a/bundle/config/mutator/rewrite_workspace_prefix.go +++ b/bundle/config/mutator/rewrite_workspace_prefix.go @@ -65,7 +65,7 @@ func (m *rewriteWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) di Summary: fmt.Sprintf("substring %q found in %q. Please update this to %q.", rewrite.pattern, vv, newPath), Detail: "For more information, please refer to: https://docs.databricks.com/en/release-notes/dev-tools/bundles.html#workspace-paths", Locations: v.Locations(), - Paths: []dyn.Path{p}, + Paths: dyn.ToStructPaths(p), }) } diff --git a/bundle/config/mutator/sync_infer_root.go b/bundle/config/mutator/sync_infer_root.go index 373a3701dcc..9fc58ee992f 100644 --- a/bundle/config/mutator/sync_infer_root.go +++ b/bundle/config/mutator/sync_infer_root.go @@ -94,7 +94,7 @@ func (m *syncInferRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno Severity: diag.Error, Summary: fmt.Sprintf("invalid sync path %q", path), Locations: b.Config.GetLocations(fmt.Sprintf("sync.paths[%d]", i)), - Paths: []dyn.Path{dyn.NewPath(dyn.Key("sync"), dyn.Key("paths"), dyn.Index(i))}, + Paths: dyn.ToStructPaths(dyn.NewPath(dyn.Key("sync"), dyn.Key("paths"), dyn.Index(i))), }) } diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index 65a903bf5a3..1794da522a4 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -11,12 +11,11 @@ import ( "path/filepath" "strings" - "github.com/databricks/cli/bundle/config/mutator/paths" - "github.com/databricks/cli/bundle/libraries" - "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator/paths" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/notebook" diff --git a/bundle/config/mutator/validate_secret_value_is_variable.go b/bundle/config/mutator/validate_secret_value_is_variable.go index 39b7822168a..1f979e15f74 100644 --- a/bundle/config/mutator/validate_secret_value_is_variable.go +++ b/bundle/config/mutator/validate_secret_value_is_variable.go @@ -33,7 +33,7 @@ func (v *validateSecretValueIsVariable) Apply(ctx context.Context, b *bundle.Bun Summary: "Secret value must be a string", Detail: fmt.Sprintf(`The secret value for "%s" must be a string.`, key), Locations: val.Locations(), - Paths: []dyn.Path{p}, + Paths: dyn.ToStructPaths(p), }) continue } @@ -48,7 +48,7 @@ func (v *validateSecretValueIsVariable) Apply(ctx context.Context, b *bundle.Bun Summary: "Secret value must be a string", Detail: fmt.Sprintf(`The secret value for "%s" must be a string.`, key), Locations: val.Locations(), - Paths: []dyn.Path{p}, + Paths: dyn.ToStructPaths(p), }) continue } @@ -62,7 +62,7 @@ func (v *validateSecretValueIsVariable) Apply(ctx context.Context, b *bundle.Bun Plain text secret values are not allowed to prevent leaking secrets in configuration files. Use bundle variables to pass secret values at deployment time.`, key), Locations: val.Locations(), - Paths: []dyn.Path{p}, + Paths: dyn.ToStructPaths(p), }) continue } @@ -106,6 +106,6 @@ func (v *validateSecretValueIsVariable) checkVariableDefault(b *bundle.Bundle, s A default value is stored in plain text in the configuration file, which defeats the purpose of using a variable reference for a secret. Remove the default value and pass the secret value at deployment time using "--var", the BUNDLE_VAR_%s environment variable, or a variable overrides file.`, varName, secretKey, varName), Locations: locations, - Paths: []dyn.Path{p}, + Paths: dyn.ToStructPaths(p), }} } diff --git a/bundle/config/validate/all_resources_have_values.go b/bundle/config/validate/all_resources_have_values.go index 7f96e529a74..ae5a4ba8bca 100644 --- a/bundle/config/validate/all_resources_have_values.go +++ b/bundle/config/validate/all_resources_have_values.go @@ -43,7 +43,7 @@ func (m *allResourcesHaveValues) Apply(ctx context.Context, b *bundle.Bundle) di Severity: diag.Error, Summary: fmt.Sprintf("%s %s is not defined", rType, rName), Locations: v.Locations(), - Paths: []dyn.Path{slices.Clone(p)}, + Paths: dyn.ToStructPaths(slices.Clone(p)), }) return v, nil diff --git a/bundle/config/validate/enum.go b/bundle/config/validate/enum.go index e6266163bc8..5f046ac7c68 100644 --- a/bundle/config/validate/enum.go +++ b/bundle/config/validate/enum.go @@ -75,7 +75,7 @@ func (f *enum) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { Severity: diag.Warning, Summary: fmt.Sprintf("invalid value %q for enum field. Valid values are %v", strValue, validValues), Locations: v.Locations(), - Paths: []dyn.Path{cloneP}, + Paths: dyn.ToStructPaths(cloneP), }) } diff --git a/bundle/config/validate/files_to_sync.go b/bundle/config/validate/files_to_sync.go index b3779016d5f..7ce1771b5c8 100644 --- a/bundle/config/validate/files_to_sync.go +++ b/bundle/config/validate/files_to_sync.go @@ -63,7 +63,7 @@ func (v *filesToSync) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost // Show all locations where sync.exclude is defined, since merging // sync.exclude is additive. Locations: b.Config.GetLocations(path), - Paths: []dyn.Path{dyn.MustPathFromString(path)}, + Paths: dyn.ToStructPaths(dyn.MustPathFromString(path)), }) } diff --git a/bundle/config/validate/interpolation_in_auth_config.go b/bundle/config/validate/interpolation_in_auth_config.go index 1598ef49d88..f749f9e0613 100644 --- a/bundle/config/validate/interpolation_in_auth_config.go +++ b/bundle/config/validate/interpolation_in_auth_config.go @@ -78,7 +78,7 @@ func (f *noInterpolationInAuthConfig) Apply(ctx context.Context, b *bundle.Bundl Detail: fmt.Sprintf(`Interpolation is not supported for the field %s. Please set the %s environment variable if you wish to configure this field at runtime.`, p.String(), envVar), Locations: v.Locations(), - Paths: []dyn.Path{p}, + Paths: dyn.ToStructPaths(p), }) } } diff --git a/bundle/config/validate/job_cluster_key_defined.go b/bundle/config/validate/job_cluster_key_defined.go index 019a785658a..1f145867288 100644 --- a/bundle/config/validate/job_cluster_key_defined.go +++ b/bundle/config/validate/job_cluster_key_defined.go @@ -61,6 +61,6 @@ func checkJobClusterKey(b *bundle.Bundle, jobClusterKeys map[string]bool, jobClu // Other associated locations are not relevant since they are // overridden during merging. Locations: b.Config.GetLocations(path), - Paths: []dyn.Path{dyn.MustPathFromString(path)}, + Paths: dyn.ToStructPaths(dyn.MustPathFromString(path)), }} } diff --git a/bundle/config/validate/job_task_cluster_spec.go b/bundle/config/validate/job_task_cluster_spec.go index 79672be63e6..14fa8918bfe 100644 --- a/bundle/config/validate/job_task_cluster_spec.go +++ b/bundle/config/validate/job_task_cluster_spec.go @@ -93,7 +93,7 @@ func validateJobTask(b *bundle.Bundle, task jobs.Task, taskPath dyn.Path) diag.D Summary: "Missing required cluster or environment settings", Detail: detail, Locations: b.Config.GetLocations(taskPath.String()), - Paths: []dyn.Path{taskPath}, + Paths: dyn.ToStructPaths(taskPath), }) } } diff --git a/bundle/config/validate/no_interpolation_in_bundle_name.go b/bundle/config/validate/no_interpolation_in_bundle_name.go index c4238ffe25a..4febeb826cd 100644 --- a/bundle/config/validate/no_interpolation_in_bundle_name.go +++ b/bundle/config/validate/no_interpolation_in_bundle_name.go @@ -5,9 +5,9 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structpath" ) type noInterpolationInBundleName struct{} @@ -29,7 +29,7 @@ is a part of the path at which your bundle state is stored at by default. Parame runtime can have unexpected consequences like duplicate deployments or resources not being cleaned up during bundle destroy.`, Locations: b.Config.GetLocations("bundle.name"), - Paths: []dyn.Path{dyn.MustPathFromString("bundle.name")}, + Paths: structpath.MustParsePaths("bundle.name"), }) } diff --git a/bundle/config/validate/no_variable_reference_in_resource_key.go b/bundle/config/validate/no_variable_reference_in_resource_key.go index 5ad5b58f200..75b8361ce97 100644 --- a/bundle/config/validate/no_variable_reference_in_resource_key.go +++ b/bundle/config/validate/no_variable_reference_in_resource_key.go @@ -41,7 +41,7 @@ func (m *noVariableReferenceInResourceKey) Apply(_ context.Context, b *bundle.Bu Severity: diag.Error, Summary: fmt.Sprintf("resource key %q must not contain variable references", key), Locations: v.Locations(), - Paths: []dyn.Path{p}, + Paths: dyn.ToStructPaths(p), }) } return v, nil diff --git a/bundle/config/validate/pipeline_development_deprecated.go b/bundle/config/validate/pipeline_development_deprecated.go index d726ad3efb1..3eb3c77c5f0 100644 --- a/bundle/config/validate/pipeline_development_deprecated.go +++ b/bundle/config/validate/pipeline_development_deprecated.go @@ -43,7 +43,7 @@ func (v *pipelineDevelopmentDeprecated) Apply(_ context.Context, b *bundle.Bundl Severity: diag.Warning, Summary: pipelineDevelopmentDeprecatedSummary, Locations: v.Locations(), - Paths: []dyn.Path{p}, + Paths: dyn.ToStructPaths(p), }) // The preset overwrites YAML values with true, so a false here was set by a Python @@ -54,7 +54,7 @@ func (v *pipelineDevelopmentDeprecated) Apply(_ context.Context, b *bundle.Bundl Summary: pipelineDevelopmentIgnoredSummary, Detail: pipelineDevelopmentIgnoredDetail, Locations: v.Locations(), - Paths: []dyn.Path{p}, + Paths: dyn.ToStructPaths(p), }) } return v, nil diff --git a/bundle/config/validate/required.go b/bundle/config/validate/required.go index b886c2c1d73..b82a920e5a3 100644 --- a/bundle/config/validate/required.go +++ b/bundle/config/validate/required.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/bundle/internal/validation/generated" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) type required struct{} @@ -59,7 +60,7 @@ func warnForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnostic Severity: diag.Warning, Summary: fmt.Sprintf("required field %q is not set", field), Locations: v.Locations(), - Paths: []dyn.Path{cloneP}, + Paths: dyn.ToStructPaths(cloneP), }) } } @@ -97,19 +98,19 @@ func sortDiagnostics(diags diag.Diagnostics) { func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // Dashboards should always have a name and warehouse_id. var nameLocations []dyn.Location - var namePaths []dyn.Path + var namePaths []*structpath.PathNode var warehouseIdLocations []dyn.Location - var warehouseIdPaths []dyn.Path + var warehouseIdPaths []*structpath.PathNode diags := diag.Diagnostics{} for key, dashboard := range b.Config.Resources.Dashboards { if dashboard.DisplayName == "" { nameLocations = append(nameLocations, b.Config.GetLocations("resources.dashboards."+key)...) - namePaths = append(namePaths, dyn.MustPathFromString("resources.dashboards."+key)) + namePaths = append(namePaths, dyn.ToStructPath(dyn.MustPathFromString("resources.dashboards."+key))) } if dashboard.WarehouseId == "" { warehouseIdLocations = append(warehouseIdLocations, b.Config.GetLocations("resources.dashboards."+key)...) - warehouseIdPaths = append(warehouseIdPaths, dyn.MustPathFromString("resources.dashboards."+key)) + warehouseIdPaths = append(warehouseIdPaths, dyn.ToStructPath(dyn.MustPathFromString("resources.dashboards."+key))) } } @@ -139,7 +140,7 @@ func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnosti Severity: diag.Error, Summary: "sql_warehouse name is required", Locations: b.Config.GetLocations(path), - Paths: []dyn.Path{dyn.MustPathFromString(path)}, + Paths: dyn.ToStructPaths(dyn.MustPathFromString(path)), }) } } @@ -166,7 +167,7 @@ func errorForInvalidGrants(ctx context.Context, b *bundle.Bundle) diag.Diagnosti Severity: diag.Error, Summary: "grant principal is required", Locations: v.Locations(), - Paths: []dyn.Path{slices.Clone(p)}, + Paths: dyn.ToStructPaths(slices.Clone(p)), }) } if isMissingOrEmptySequence(v.Get("privileges")) { @@ -174,7 +175,7 @@ func errorForInvalidGrants(ctx context.Context, b *bundle.Bundle) diag.Diagnosti Severity: diag.Error, Summary: "grant privileges is required", Locations: v.Locations(), - Paths: []dyn.Path{slices.Clone(p)}, + Paths: dyn.ToStructPaths(slices.Clone(p)), }) } return v, nil @@ -207,7 +208,7 @@ func errorForInvalidSecretScopePermissions(ctx context.Context, b *bundle.Bundle Summary: "secret scope permission principal is required", Detail: "Set one of user_name, group_name or service_principal_name", Locations: b.Config.GetLocations("resources.secret_scopes." + key), - Paths: []dyn.Path{dyn.MustPathFromString(path)}, + Paths: dyn.ToStructPaths(dyn.MustPathFromString(path)), }) } } diff --git a/bundle/config/validate/scripts.go b/bundle/config/validate/scripts.go index b0f25ee5368..323db769d23 100644 --- a/bundle/config/validate/scripts.go +++ b/bundle/config/validate/scripts.go @@ -44,7 +44,7 @@ func (f *validateScripts) Apply(ctx context.Context, b *bundle.Bundle) diag.Diag diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("Script %s has no content", k), - Paths: []dyn.Path{contentPath}, + Paths: dyn.ToStructPaths(contentPath), }) continue } @@ -80,7 +80,7 @@ from "content" with $NAME: MY_VAR: ${var.foo} content: echo "$MY_VAR"`, Locations: v.Locations(), - Paths: []dyn.Path{p}, + Paths: dyn.ToStructPaths(p), }} } @@ -105,7 +105,7 @@ func validateScriptEnv(b *bundle.Bundle, key string, env map[string]string) diag Severity: diag.Error, Summary: fmt.Sprintf("${%s} cannot be used in scripts.%s.env.%s; only ${bundle.*}, ${workspace.*}, and ${var.*} are resolved before scripts execute", refPath, key, name), Locations: v.Locations(), - Paths: []dyn.Path{envValuePath}, + Paths: dyn.ToStructPaths(envValuePath), }) } } diff --git a/bundle/config/validate/single_node_cluster.go b/bundle/config/validate/single_node_cluster.go index f4f6f0e5954..88b7536f80b 100644 --- a/bundle/config/validate/single_node_cluster.go +++ b/bundle/config/validate/single_node_cluster.go @@ -128,7 +128,7 @@ func (m *singleNodeCluster) Apply(ctx context.Context, b *bundle.Bundle) diag.Di Summary: singleNodeWarningSummary, Detail: singleNodeWarningDetail, Locations: v.Locations(), - Paths: []dyn.Path{p}, + Paths: dyn.ToStructPaths(p), } if showSingleNodeClusterWarning(ctx, v) { diff --git a/bundle/config/validate/single_node_cluster_test.go b/bundle/config/validate/single_node_cluster_test.go index 1e622a3a0e1..46bab0084a1 100644 --- a/bundle/config/validate/single_node_cluster_test.go +++ b/bundle/config/validate/single_node_cluster_test.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/pipelines" @@ -122,7 +123,7 @@ func TestValidateSingleNodeClusterFailForInteractiveClusters(t *testing.T) { Summary: singleNodeWarningSummary, Detail: singleNodeWarningDetail, Locations: []dyn.Location{{File: "a.yml", Line: 1, Column: 1}}, - Paths: []dyn.Path{dyn.NewPath(dyn.Key("resources"), dyn.Key("clusters"), dyn.Key("foo"))}, + Paths: dyn.ToStructPaths(dyn.NewPath(dyn.Key("resources"), dyn.Key("clusters"), dyn.Key("foo"))), }, }, diags) }) @@ -171,7 +172,7 @@ func TestValidateSingleNodeClusterFailForJobClusters(t *testing.T) { Summary: singleNodeWarningSummary, Detail: singleNodeWarningDetail, Locations: []dyn.Location{{File: "b.yml", Line: 1, Column: 1}}, - Paths: []dyn.Path{dyn.MustPathFromString("resources.jobs.foo.job_clusters[0].new_cluster")}, + Paths: structpath.MustParsePaths("resources.jobs.foo.job_clusters[0].new_cluster"), }, }, diags) }) @@ -220,7 +221,7 @@ func TestValidateSingleNodeClusterFailForJobTaskClusters(t *testing.T) { Summary: singleNodeWarningSummary, Detail: singleNodeWarningDetail, Locations: []dyn.Location{{File: "c.yml", Line: 1, Column: 1}}, - Paths: []dyn.Path{dyn.MustPathFromString("resources.jobs.foo.tasks[0].new_cluster")}, + Paths: structpath.MustParsePaths("resources.jobs.foo.tasks[0].new_cluster"), }, }, diags) }) @@ -266,7 +267,7 @@ func TestValidateSingleNodeClusterFailForPipelineClusters(t *testing.T) { Summary: singleNodeWarningSummary, Detail: singleNodeWarningDetail, Locations: []dyn.Location{{File: "d.yml", Line: 1, Column: 1}}, - Paths: []dyn.Path{dyn.MustPathFromString("resources.pipelines.foo.clusters[0]")}, + Paths: structpath.MustParsePaths("resources.pipelines.foo.clusters[0]"), }, }, diags) }) @@ -319,7 +320,7 @@ func TestValidateSingleNodeClusterFailForJobForEachTaskCluster(t *testing.T) { Summary: singleNodeWarningSummary, Detail: singleNodeWarningDetail, Locations: []dyn.Location{{File: "e.yml", Line: 1, Column: 1}}, - Paths: []dyn.Path{dyn.MustPathFromString("resources.jobs.foo.tasks[0].for_each_task.task.new_cluster")}, + Paths: structpath.MustParsePaths("resources.jobs.foo.tasks[0].for_each_task.task.new_cluster"), }, }, diags) }) diff --git a/bundle/config/validate/unique_resource_keys.go b/bundle/config/validate/unique_resource_keys.go index 12c13fd1a86..cff5ef3ccff 100644 --- a/bundle/config/validate/unique_resource_keys.go +++ b/bundle/config/validate/unique_resource_keys.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) // This mutator validates that: @@ -36,7 +37,7 @@ func (m *uniqueResourceKeys) Apply(ctx context.Context, b *bundle.Bundle) diag.D type metadata struct { locations []dyn.Location - paths []dyn.Path + paths []*structpath.PathNode } // Maps of key to the paths and locations the resource / script is defined at. @@ -50,7 +51,7 @@ func (m *uniqueResourceKeys) Apply(ctx context.Context, b *bundle.Bundle) diag.D } } - mv.paths = append(mv.paths, dyn.NewPath(dyn.Key(prefix)).Append(p...)) + mv.paths = append(mv.paths, dyn.ToStructPath(dyn.NewPath(dyn.Key(prefix)).Append(p...))) mv.locations = append(mv.locations, v.Locations()...) resourceAndScriptMetadata[k] = mv @@ -109,7 +110,7 @@ func (m *uniqueResourceKeys) Apply(ctx context.Context, b *bundle.Bundle) diag.D } return cmp.Compare(a.Column, b.Column) }) - slices.SortFunc(v.paths, func(a, b dyn.Path) int { + slices.SortFunc(v.paths, func(a, b *structpath.PathNode) int { return cmp.Compare(a.String(), b.String()) }) diff --git a/bundle/config/validate/validate_artifact_path.go b/bundle/config/validate/validate_artifact_path.go index e7070b20b02..93f38b272a9 100644 --- a/bundle/config/validate/validate_artifact_path.go +++ b/bundle/config/validate/validate_artifact_path.go @@ -13,6 +13,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/apierr" ) @@ -86,7 +87,7 @@ func (v *validateArtifactPath) Apply(ctx context.Context, b *bundle.Bundle) diag Summary: s, Severity: diag.Error, Locations: b.Config.GetLocations("workspace.artifact_path"), - Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, + Paths: structpath.MustParsePaths("workspace.artifact_path"), }, } } @@ -118,7 +119,7 @@ this bundle but which has not been deployed yet. Please first deploy the volume using 'bundle deploy' and then switch over to using it in the artifact_path.`, Locations: slices.Concat(b.Config.GetLocations("workspace.artifact_path"), locations), - Paths: append([]dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, path), + Paths: append(structpath.MustParsePaths("workspace.artifact_path"), dyn.ToStructPath(path)), }} } diff --git a/bundle/config/validate/validate_artifact_path_test.go b/bundle/config/validate/validate_artifact_path_test.go index 08f50e39fb8..4d85dcd9176 100644 --- a/bundle/config/validate/validate_artifact_path_test.go +++ b/bundle/config/validate/validate_artifact_path_test.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/databricks/databricks-sdk-go/service/catalog" @@ -56,10 +57,10 @@ func TestValidateArtifactPathWithVolumeInBundle(t *testing.T) { {File: "file", Line: 1, Column: 1}, {File: "file", Line: 2, Column: 2}, }, - Paths: []dyn.Path{ - dyn.MustPathFromString("workspace.artifact_path"), - dyn.MustPathFromString("resources.volumes.foo"), - }, + Paths: structpath.MustParsePaths( + "workspace.artifact_path", + "resources.volumes.foo", + ), Detail: `You are using a volume in your artifact_path that is managed by this bundle but which has not been deployed yet. Please first deploy the volume using 'bundle deploy' and then switch over to using it in @@ -83,7 +84,7 @@ func TestValidateArtifactPath(t *testing.T) { Severity: diag.Error, Summary: expected, Locations: []dyn.Location{{File: "file", Line: 1, Column: 1}}, - Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, + Paths: structpath.MustParsePaths("workspace.artifact_path"), }}, diags) } @@ -170,7 +171,7 @@ func TestValidateArtifactPathWithInvalidPaths(t *testing.T) { Severity: diag.Error, Summary: "expected UC volume path to be in the format /Volumes////..., got " + p, Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}, - Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, + Paths: structpath.MustParsePaths("workspace.artifact_path"), }}, diags) } } diff --git a/bundle/config/validate/validate_dashboard_etags.go b/bundle/config/validate/validate_dashboard_etags.go index 53c428af1dd..ca323c7d388 100644 --- a/bundle/config/validate/validate_dashboard_etags.go +++ b/bundle/config/validate/validate_dashboard_etags.go @@ -27,7 +27,7 @@ func (v *validateDashboardEtags) Apply(ctx context.Context, b *bundle.Bundle) di { Severity: diag.Error, Summary: fmt.Sprintf("dashboard %q has an etag set. Etags must not be set in bundle configuration", dashboard.DisplayName), - Paths: []dyn.Path{dyn.MustPathFromString("resources.dashboards." + k)}, + Paths: dyn.ToStructPaths(dyn.MustPathFromString("resources.dashboards." + k)), Locations: b.Config.GetLocations("resources.dashboards." + k), }, } diff --git a/bundle/config/validate/validate_deployment_fields.go b/bundle/config/validate/validate_deployment_fields.go index f551c54661b..cb65f5135ce 100644 --- a/bundle/config/validate/validate_deployment_fields.go +++ b/bundle/config/validate/validate_deployment_fields.go @@ -34,7 +34,7 @@ func (v *validateDeploymentFields) Apply(_ context.Context, b *bundle.Bundle) di diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: field + " must not be set in bundle configuration; it is managed by Declarative Automation Bundles", - Paths: []dyn.Path{dyn.MustPathFromString(path)}, + Paths: dyn.ToStructPaths(dyn.MustPathFromString(path)), Locations: b.Config.GetLocations(path), }) } diff --git a/bundle/config/validate/validate_engine.go b/bundle/config/validate/validate_engine.go index 056e0bd50a6..2b9d00971b4 100644 --- a/bundle/config/validate/validate_engine.go +++ b/bundle/config/validate/validate_engine.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) type validateEngine struct{ bundle.RO } @@ -30,7 +31,7 @@ func (v *validateEngine) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos Severity: diag.Warning, Summary: "bundle.terraform is deprecated and has no effect: " + engine.TerraformRemovedSummary, Locations: tf.Locations(), - Paths: []dyn.Path{dyn.MustPathFromString("bundle.terraform")}, + Paths: structpath.MustParsePaths("bundle.terraform"), }) } diff --git a/bundle/config/validate/validate_genie_space_etags.go b/bundle/config/validate/validate_genie_space_etags.go index 84e9626c0f4..c7682f08e9e 100644 --- a/bundle/config/validate/validate_genie_space_etags.go +++ b/bundle/config/validate/validate_genie_space_etags.go @@ -29,7 +29,7 @@ func (v *validateGenieSpaceEtags) Apply(ctx context.Context, b *bundle.Bundle) d { Severity: diag.Error, Summary: fmt.Sprintf("genie space %q has an etag set. Etags must not be set in bundle configuration", genieSpace.Title), - Paths: []dyn.Path{dyn.MustPathFromString("resources.genie_spaces." + k)}, + Paths: dyn.ToStructPaths(dyn.MustPathFromString("resources.genie_spaces." + k)), Locations: b.Config.GetLocations("resources.genie_spaces." + k), }, } diff --git a/bundle/config/validate/validate_job_run_idempotency_token.go b/bundle/config/validate/validate_job_run_idempotency_token.go index ea0d608f990..caf8b6b3cd5 100644 --- a/bundle/config/validate/validate_job_run_idempotency_token.go +++ b/bundle/config/validate/validate_job_run_idempotency_token.go @@ -36,7 +36,7 @@ func (v *validateJobRunIdempotencyToken) Apply(_ context.Context, b *bundle.Bund diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: "idempotency_token must not be set in bundle configuration; the CLI sets it on each run-now request", - Paths: []dyn.Path{dyn.MustPathFromString(path)}, + Paths: dyn.ToStructPaths(dyn.MustPathFromString(path)), Locations: b.Config.GetLocations(path), }) } diff --git a/bundle/config/validate/validate_sync_patterns.go b/bundle/config/validate/validate_sync_patterns.go index 8488aefebd2..8810f2e1f24 100644 --- a/bundle/config/validate/validate_sync_patterns.go +++ b/bundle/config/validate/validate_sync_patterns.go @@ -62,7 +62,7 @@ func checkPatterns(ctx context.Context, patterns []string, path string, b *bundl Severity: diag.Warning, Summary: fmt.Sprintf("Pattern %s does not match any files", pattern), Locations: b.Config.GetLocations(path), - Paths: []dyn.Path{dyn.MustPathFromString(path)}, + Paths: dyn.ToStructPaths(dyn.MustPathFromString(path)), }) } return nil diff --git a/bundle/config/validate/validate_volume_path.go b/bundle/config/validate/validate_volume_path.go index 265a0bfd53b..9a8c1b090cd 100644 --- a/bundle/config/validate/validate_volume_path.go +++ b/bundle/config/validate/validate_volume_path.go @@ -39,7 +39,7 @@ func (m *validateVolumePath) Apply(ctx context.Context, b *bundle.Bundle) diag.D Summary: fmt.Sprintf("%s %s starts with /Volumes. /Volumes can only be used with workspace.artifact_path.", check.configName, check.path), Detail: "For more information, see https://docs.databricks.com/aws/en/dev-tools/bundles/settings#workspace", Locations: b.Config.GetLocations(check.configName), - Paths: []dyn.Path{dyn.MustPathFromString(check.configName)}, + Paths: dyn.ToStructPaths(dyn.MustPathFromString(check.configName)), }) // Return early for root path validation diff --git a/bundle/deploy/check_dashboards_modified_remotely.go b/bundle/deploy/check_dashboards_modified_remotely.go index 559206b876c..34fe393062c 100644 --- a/bundle/deploy/check_dashboards_modified_remotely.go +++ b/bundle/deploy/check_dashboards_modified_remotely.go @@ -80,7 +80,7 @@ func (l *checkDashboardsModifiedRemotely) Apply(ctx context.Context, b *bundle.B Severity: diag.Error, Summary: fmt.Sprintf("failed to get dashboard %q", dashboard.Name), Detail: err.Error(), - Paths: []dyn.Path{path}, + Paths: dyn.ToStructPaths(path), Locations: []dyn.Location{loc}, }) continue @@ -109,7 +109,7 @@ func (l *checkDashboardsModifiedRemotely) Apply(ctx context.Context, b *bundle.B "\n" + "To overwrite the remote changes with your local version, use --force.\n" + "The remote modifications will be lost." + agent.AgentNotice(), - Paths: []dyn.Path{path}, + Paths: dyn.ToStructPaths(path), Locations: []dyn.Location{loc}, }) } diff --git a/bundle/internal/schema/annotations_file.go b/bundle/internal/schema/annotations_file.go index 5c93610036a..bd106b136b2 100644 --- a/bundle/internal/schema/annotations_file.go +++ b/bundle/internal/schema/annotations_file.go @@ -8,13 +8,12 @@ import ( "slices" "strings" - yaml3 "go.yaml.in/yaml/v3" - "github.com/databricks/cli/bundle/internal/annotation" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/cli/libs/dyn/yamlsaver" + yaml3 "go.yaml.in/yaml/v3" ) // fieldsKey nests a type's block of field nodes inside the node of a field diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go index ab1da3df68d..2bdc81b60a2 100644 --- a/bundle/libraries/expand_glob_references.go +++ b/bundle/libraries/expand_glob_references.go @@ -19,7 +19,7 @@ func matchError(p dyn.Path, l []dyn.Location, message string) diag.Diagnostic { Severity: diag.Error, Summary: message, Locations: l, - Paths: []dyn.Path{p}, + Paths: dyn.ToStructPaths(p), } } diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go index 8fb140d7aa4..1d50c946a27 100644 --- a/bundle/libraries/same_name_libraries.go +++ b/bundle/libraries/same_name_libraries.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) type checkForSameNameLibraries struct{} @@ -26,7 +27,7 @@ var patterns = []dyn.Pattern{ type libData struct { fullPath string locations []dyn.Location - paths []dyn.Path + paths []*structpath.PathNode otherPaths []string } @@ -56,12 +57,12 @@ func (c checkForSameNameLibraries) Apply(ctx context.Context, b *bundle.Bundle) libs[lib] = &libData{ fullPath: libPath, locations: []dyn.Location{libraryValue.Location()}, - paths: []dyn.Path{p}, + paths: dyn.ToStructPaths(p), otherPaths: []string{}, } } else if lp.fullPath != libPath { lp.locations = append(lp.locations, libraryValue.Location()) - lp.paths = append(lp.paths, p) + lp.paths = append(lp.paths, dyn.ToStructPath(p)) lp.otherPaths = append(lp.otherPaths, libPath) } diff --git a/bundle/libraries/upload.go b/bundle/libraries/upload.go index dabd4d8860d..5cf3cfa19b3 100644 --- a/bundle/libraries/upload.go +++ b/bundle/libraries/upload.go @@ -15,7 +15,6 @@ import ( "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" - "golang.org/x/sync/errgroup" ) diff --git a/bundle/render/render_text_output_test.go b/bundle/render/render_text_output_test.go index 80c5eb417c5..0d631ec2ac6 100644 --- a/bundle/render/render_text_output_test.go +++ b/bundle/render/render_text_output_test.go @@ -13,6 +13,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -203,7 +204,7 @@ func TestRenderDiagnostics(t *testing.T) { Severity: diag.Error, Detail: "'name' is required", Summary: "failed to load xxx", - Paths: []dyn.Path{dyn.MustPathFromString("resources.jobs.xxx")}, + Paths: structpath.MustParsePaths("resources.jobs.xxx"), }, }, expected: "Error: failed to load xxx\n" + @@ -218,11 +219,11 @@ func TestRenderDiagnostics(t *testing.T) { Severity: diag.Error, Detail: "'name' is required", Summary: "failed to load xxx", - Paths: []dyn.Path{ - dyn.MustPathFromString("resources.jobs.xxx"), - dyn.MustPathFromString("resources.jobs.yyy"), - dyn.MustPathFromString("resources.jobs.zzz"), - }, + Paths: structpath.MustParsePaths( + "resources.jobs.xxx", + "resources.jobs.yyy", + "resources.jobs.zzz", + ), }, }, expected: "Error: failed to load xxx\n" + @@ -239,10 +240,10 @@ func TestRenderDiagnostics(t *testing.T) { Severity: diag.Recommendation, Summary: "summary", Detail: "detail", - Paths: []dyn.Path{ - dyn.MustPathFromString("resources.jobs.xxx"), - dyn.MustPathFromString("resources.jobs.yyy"), - }, + Paths: structpath.MustParsePaths( + "resources.jobs.xxx", + "resources.jobs.yyy", + ), Locations: []dyn.Location{ {File: "foo.yaml", Line: 1, Column: 2}, {File: "bar.yaml", Line: 3, Column: 4}, diff --git a/bundle/tests/validate_test.go b/bundle/tests/validate_test.go index e10f240aaff..d2a3a127843 100644 --- a/bundle/tests/validate_test.go +++ b/bundle/tests/validate_test.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -27,10 +28,10 @@ func TestValidateUniqueResourceIdentifiers(t *testing.T) { {File: "validate/duplicate_resource_names_in_root_job_and_pipeline/databricks.yml", Line: 10, Column: 7}, {File: "validate/duplicate_resource_names_in_root_job_and_pipeline/databricks.yml", Line: 13, Column: 7}, }, - Paths: []dyn.Path{ - dyn.MustPathFromString("resources.jobs.foo"), - dyn.MustPathFromString("resources.pipelines.foo"), - }, + Paths: structpath.MustParsePaths( + "resources.jobs.foo", + "resources.pipelines.foo", + ), }, }, }, @@ -44,10 +45,10 @@ func TestValidateUniqueResourceIdentifiers(t *testing.T) { {File: "validate/duplicate_resource_names_in_root_job_and_experiment/databricks.yml", Line: 10, Column: 7}, {File: "validate/duplicate_resource_names_in_root_job_and_experiment/databricks.yml", Line: 18, Column: 7}, }, - Paths: []dyn.Path{ - dyn.MustPathFromString("resources.experiments.foo"), - dyn.MustPathFromString("resources.jobs.foo"), - }, + Paths: structpath.MustParsePaths( + "resources.experiments.foo", + "resources.jobs.foo", + ), }, }, }, @@ -61,10 +62,10 @@ func TestValidateUniqueResourceIdentifiers(t *testing.T) { {File: "validate/duplicate_resource_name_in_subconfiguration/databricks.yml", Line: 13, Column: 7}, {File: "validate/duplicate_resource_name_in_subconfiguration/resources.yml", Line: 4, Column: 7}, }, - Paths: []dyn.Path{ - dyn.MustPathFromString("resources.jobs.foo"), - dyn.MustPathFromString("resources.pipelines.foo"), - }, + Paths: structpath.MustParsePaths( + "resources.jobs.foo", + "resources.pipelines.foo", + ), }, }, }, @@ -78,9 +79,7 @@ func TestValidateUniqueResourceIdentifiers(t *testing.T) { {File: "validate/duplicate_resource_name_in_subconfiguration_job_and_job/databricks.yml", Line: 13, Column: 7}, {File: "validate/duplicate_resource_name_in_subconfiguration_job_and_job/resources.yml", Line: 4, Column: 7}, }, - Paths: []dyn.Path{ - dyn.MustPathFromString("resources.jobs.foo"), - }, + Paths: structpath.MustParsePaths("resources.jobs.foo"), }, }, }, @@ -94,10 +93,10 @@ func TestValidateUniqueResourceIdentifiers(t *testing.T) { {File: "validate/duplicate_resource_names_in_different_subconfiguations/resources1.yml", Line: 4, Column: 7}, {File: "validate/duplicate_resource_names_in_different_subconfiguations/resources2.yml", Line: 4, Column: 7}, }, - Paths: []dyn.Path{ - dyn.MustPathFromString("resources.jobs.foo"), - dyn.MustPathFromString("resources.pipelines.foo"), - }, + Paths: structpath.MustParsePaths( + "resources.jobs.foo", + "resources.pipelines.foo", + ), }, }, }, @@ -114,11 +113,11 @@ func TestValidateUniqueResourceIdentifiers(t *testing.T) { {File: "validate/duplicate_resource_name_in_multiple_locations/resources2.yml", Line: 4, Column: 7}, {File: "validate/duplicate_resource_name_in_multiple_locations/resources2.yml", Line: 8, Column: 7}, }, - Paths: []dyn.Path{ - dyn.MustPathFromString("resources.experiments.foo"), - dyn.MustPathFromString("resources.jobs.foo"), - dyn.MustPathFromString("resources.pipelines.foo"), - }, + Paths: structpath.MustParsePaths( + "resources.experiments.foo", + "resources.jobs.foo", + "resources.pipelines.foo", + ), }, }, }, diff --git a/cmd/apps/import.go b/cmd/apps/import.go index f6089870aec..1c780729d86 100644 --- a/cmd/apps/import.go +++ b/cmd/apps/import.go @@ -12,8 +12,6 @@ import ( "slices" "strings" - "go.yaml.in/yaml/v3" - "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/generate" "github.com/databricks/cli/bundle/phases" @@ -36,6 +34,7 @@ import ( "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/workspace" "github.com/spf13/cobra" + "go.yaml.in/yaml/v3" ) func newImportCommand() *cobra.Command { diff --git a/libs/diag/diagnostic.go b/libs/diag/diagnostic.go index fe7090462d4..f189cf0ac6c 100644 --- a/libs/diag/diagnostic.go +++ b/libs/diag/diagnostic.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) type Diagnostic struct { @@ -24,7 +25,7 @@ type Diagnostic struct { // Paths are paths to the values in the configuration tree that the diagnostic is associated with. // It may be nil if there are no associated paths. - Paths []dyn.Path + Paths []*structpath.PathNode // A diagnostic ID. Only used for select diagnostic messages. ID ID diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go index a503fd1afc9..379baf439d1 100644 --- a/libs/dyn/convert/normalize.go +++ b/libs/dyn/convert/normalize.go @@ -85,7 +85,7 @@ func nullWarning(expected dyn.Kind, src dyn.Value, path dyn.Path) diag.Diagnosti Severity: diag.Warning, Summary: fmt.Sprintf("expected a %s value, found null", expected), Locations: []dyn.Location{src.Location()}, - Paths: []dyn.Path{path}, + Paths: dyn.ToStructPaths(path), } } @@ -94,7 +94,7 @@ func typeMismatch(expected dyn.Kind, src dyn.Value, path dyn.Path) diag.Diagnost Severity: diag.Warning, Summary: fmt.Sprintf("expected %s, found %s", expected, src.Kind()), Locations: []dyn.Location{src.Location()}, - Paths: []dyn.Path{path}, + Paths: dyn.ToStructPaths(path), } } @@ -151,7 +151,7 @@ func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen Summary: "Use 'value_from' instead of 'valueFrom'", Detail: "The field 'valueFrom' should be 'value_from' (snake_case). The 'valueFrom' field will be ignored.", Locations: pk.Locations(), - Paths: []dyn.Path{path}, + Paths: dyn.ToStructPaths(path), }) continue } @@ -161,7 +161,7 @@ func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen Severity: diag.Warning, Summary: "unknown field: " + fieldName, Locations: pk.Locations(), - Paths: []dyn.Path{path}, + Paths: dyn.ToStructPaths(path), }) } continue @@ -399,7 +399,7 @@ func (n normalizeOptions) normalizeInt(typ reflect.Type, src dyn.Value, path dyn Severity: diag.Warning, Summary: fmt.Sprintf(`cannot accurately represent "%g" as integer due to precision loss`, src.MustFloat()), Locations: []dyn.Location{src.Location()}, - Paths: []dyn.Path{path}, + Paths: dyn.ToStructPaths(path), }) } return dyn.NewValue(out, src.Locations()), nil @@ -416,7 +416,7 @@ func (n normalizeOptions) normalizeInt(typ reflect.Type, src dyn.Value, path dyn Severity: diag.Warning, Summary: fmt.Sprintf("cannot parse %q as an integer", src.MustString()), Locations: []dyn.Location{src.Location()}, - Paths: []dyn.Path{path}, + Paths: dyn.ToStructPaths(path), }) } return dyn.NewValue(out, src.Locations()), nil @@ -443,7 +443,7 @@ func (n normalizeOptions) normalizeFloat(typ reflect.Type, src dyn.Value, path d Severity: diag.Warning, Summary: fmt.Sprintf(`cannot accurately represent "%d" as floating point number due to precision loss`, src.MustInt()), Locations: []dyn.Location{src.Location()}, - Paths: []dyn.Path{path}, + Paths: dyn.ToStructPaths(path), }) } return dyn.NewValue(out, src.Locations()), nil @@ -460,7 +460,7 @@ func (n normalizeOptions) normalizeFloat(typ reflect.Type, src dyn.Value, path d Severity: diag.Warning, Summary: fmt.Sprintf("cannot parse %q as a floating point number", src.MustString()), Locations: []dyn.Location{src.Location()}, - Paths: []dyn.Path{path}, + Paths: dyn.ToStructPaths(path), }) } return dyn.NewValue(out, src.Locations()), nil diff --git a/libs/dyn/convert/normalize_test.go b/libs/dyn/convert/normalize_test.go index f5f3073aeae..08b89d835ba 100644 --- a/libs/dyn/convert/normalize_test.go +++ b/libs/dyn/convert/normalize_test.go @@ -44,7 +44,7 @@ func TestNormalizeStructElementDiagnostic(t *testing.T) { Severity: diag.Warning, Summary: `expected string, found map`, Locations: []dyn.Location{{}}, - Paths: []dyn.Path{dyn.NewPath(dyn.Key("bar"))}, + Paths: dyn.ToStructPaths(dyn.NewPath(dyn.Key("bar"))), }, diags[0]) // Elements that encounter an error during normalization are dropped. @@ -81,7 +81,7 @@ func TestNormalizeStructUnknownField(t *testing.T) { {File: "hello.yaml", Line: 1, Column: 1}, {File: "world.yaml", Line: 2, Column: 2}, }, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, diags[0]) // The field that can be mapped to the struct field is retained. @@ -115,7 +115,7 @@ func TestNormalizeStructError(t *testing.T) { Severity: diag.Warning, Summary: `expected map, found string`, Locations: []dyn.Location{vin.Get("foo").Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -260,7 +260,7 @@ func TestNormalizeStructRandomStringError(t *testing.T) { Severity: diag.Warning, Summary: `expected map, found string`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -277,7 +277,7 @@ func TestNormalizeStructIntError(t *testing.T) { Severity: diag.Warning, Summary: `expected map, found int`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -306,7 +306,7 @@ func TestNormalizeMapElementDiagnostic(t *testing.T) { Severity: diag.Warning, Summary: `expected string, found map`, Locations: []dyn.Location{{}}, - Paths: []dyn.Path{dyn.NewPath(dyn.Key("bar"))}, + Paths: dyn.ToStructPaths(dyn.NewPath(dyn.Key("bar"))), }, err[0]) // Elements that encounter an error during normalization are dropped. @@ -332,7 +332,7 @@ func TestNormalizeMapError(t *testing.T) { Severity: diag.Warning, Summary: `expected map, found string`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -387,7 +387,7 @@ func TestNormalizeMapRandomStringError(t *testing.T) { Severity: diag.Warning, Summary: `expected map, found string`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -400,7 +400,7 @@ func TestNormalizeMapIntError(t *testing.T) { Severity: diag.Warning, Summary: `expected map, found int`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -430,7 +430,7 @@ func TestNormalizeSliceElementDiagnostic(t *testing.T) { Severity: diag.Warning, Summary: `expected string, found map`, Locations: []dyn.Location{{}}, - Paths: []dyn.Path{dyn.NewPath(dyn.Index(2))}, + Paths: dyn.ToStructPaths(dyn.NewPath(dyn.Index(2))), }, err[0]) // Elements that encounter an error during normalization are dropped. @@ -454,7 +454,7 @@ func TestNormalizeSliceError(t *testing.T) { Severity: diag.Warning, Summary: `expected sequence, found string`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -509,7 +509,7 @@ func TestNormalizeSliceRandomStringError(t *testing.T) { Severity: diag.Warning, Summary: `expected sequence, found string`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -522,7 +522,7 @@ func TestNormalizeSliceIntError(t *testing.T) { Severity: diag.Warning, Summary: `expected sequence, found int`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -543,7 +543,7 @@ func TestNormalizeStringNil(t *testing.T) { Severity: diag.Warning, Summary: `expected a string value, found null`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -588,7 +588,7 @@ func TestNormalizeStringError(t *testing.T) { Severity: diag.Warning, Summary: `expected string, found map`, Locations: []dyn.Location{{}}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -609,7 +609,7 @@ func TestNormalizeBoolNil(t *testing.T) { Severity: diag.Warning, Summary: `expected a bool value, found null`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -655,7 +655,7 @@ func TestNormalizeBoolFromStringError(t *testing.T) { Severity: diag.Warning, Summary: `expected bool, found string`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -668,7 +668,7 @@ func TestNormalizeBoolError(t *testing.T) { Severity: diag.Warning, Summary: `expected bool, found map`, Locations: []dyn.Location{{}}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -689,7 +689,7 @@ func TestNormalizeIntNil(t *testing.T) { Severity: diag.Warning, Summary: `expected a int value, found null`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -710,7 +710,7 @@ func TestNormalizeIntFromFloatError(t *testing.T) { Severity: diag.Warning, Summary: `cannot accurately represent "1.5" as integer due to precision loss`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -739,7 +739,7 @@ func TestNormalizeIntFromStringError(t *testing.T) { Severity: diag.Warning, Summary: `cannot parse "abc" as an integer`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -752,7 +752,7 @@ func TestNormalizeIntError(t *testing.T) { Severity: diag.Warning, Summary: `expected int, found map`, Locations: []dyn.Location{{}}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -773,7 +773,7 @@ func TestNormalizeFloatNil(t *testing.T) { Severity: diag.Warning, Summary: `expected a float value, found null`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -798,7 +798,7 @@ func TestNormalizeFloatFromIntError(t *testing.T) { Severity: diag.Warning, Summary: `cannot accurately represent "9007199254740993" as floating point number due to precision loss`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -827,7 +827,7 @@ func TestNormalizeFloatFromStringError(t *testing.T) { Severity: diag.Warning, Summary: `cannot parse "abc" as a floating point number`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -840,7 +840,7 @@ func TestNormalizeFloatError(t *testing.T) { Severity: diag.Warning, Summary: `expected float, found map`, Locations: []dyn.Location{{}}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } diff --git a/libs/dyn/dynvar/ref.go b/libs/dyn/dynvar/ref.go index de25f44a212..7fd61073f0a 100644 --- a/libs/dyn/dynvar/ref.go +++ b/libs/dyn/dynvar/ref.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) // BaseVarDef matches a single dot-separated path segment in ${...} references. @@ -181,3 +182,23 @@ func PureReferenceToPath(s string) (dyn.Path, bool) { return p, true } + +// PureReferenceToStructPath returns a PathNode if s is a pure variable reference, otherwise false. +// This function is similar to PureReferenceToPath but returns a *structpath.PathNode instead of dyn.Path. +func PureReferenceToStructPath(s string) (*structpath.PathNode, bool) { + ref, ok := NewRef(dyn.V(s)) + if !ok { + return nil, false + } + + if !ref.IsPure() { + return nil, false + } + + p, err := structpath.ParsePath(ref.References()[0]) + if err != nil { + return nil, false + } + + return p, true +} diff --git a/libs/dyn/dynvar/ref_structpath_test.go b/libs/dyn/dynvar/ref_structpath_test.go new file mode 100644 index 00000000000..1ca2acf4ceb --- /dev/null +++ b/libs/dyn/dynvar/ref_structpath_test.go @@ -0,0 +1,56 @@ +package dynvar + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPureReferenceToStructPath(t *testing.T) { + tests := []struct { + name string + input string + expected string + ok bool + }{ + { + name: "simple reference", + input: "${resources.jobs.foo.id}", + expected: "resources.jobs.foo.id", + ok: true, + }, + { + name: "simple reference", + input: "${resources.jobs.foo.tasks[1].env.key}", + expected: "resources.jobs.foo.tasks[1].env.key", + ok: true, + }, + { + name: "complex nested reference", + input: "${var.resources.jobs['my_job'].tasks[0]}", + // we use regex from dyn module which only support integers inside brackets: + // expected: "resources.jobs['my_job'].tasks[0]", + }, + { + name: "not a pure reference", + input: "prefix_${var.field}", + }, + { + name: "not a variable reference", + input: "plain_string", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pathNode, ok := PureReferenceToStructPath(tt.input) + assert.Equal(t, tt.ok, ok) + if tt.ok { + assert.NotNil(t, pathNode) + assert.Equal(t, tt.expected, pathNode.String()) + } else { + assert.Nil(t, pathNode) + } + }) + } +} diff --git a/libs/dyn/structpath.go b/libs/dyn/structpath.go new file mode 100644 index 00000000000..ca9afb5c7b7 --- /dev/null +++ b/libs/dyn/structpath.go @@ -0,0 +1,25 @@ +package dyn + +import "github.com/databricks/cli/libs/structs/structpath" + +// ToStructPath converts a [Path] to a [structpath.PathNode]. +func ToStructPath(p Path) *structpath.PathNode { + var result *structpath.PathNode + for _, c := range p { + if c.isKey() { + result = structpath.NewStringKey(result, c.key) + } else { + result = structpath.NewIndex(result, c.index) + } + } + return result +} + +// ToStructPaths converts each of paths with [ToStructPath], e.g. for diag.Diagnostic.Paths. +func ToStructPaths(paths ...Path) []*structpath.PathNode { + out := make([]*structpath.PathNode, len(paths)) + for i, p := range paths { + out[i] = ToStructPath(p) + } + return out +} diff --git a/libs/dyn/structpath_test.go b/libs/dyn/structpath_test.go new file mode 100644 index 00000000000..0e24f8ea51a --- /dev/null +++ b/libs/dyn/structpath_test.go @@ -0,0 +1,29 @@ +package dyn_test + +import ( + "testing" + + "github.com/databricks/cli/libs/dyn" + "github.com/stretchr/testify/assert" +) + +func TestToStructPath(t *testing.T) { + assert.Nil(t, dyn.ToStructPath(nil)) + + p := dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"), dyn.Key("a_job"), dyn.Key("tasks"), dyn.Index(1), dyn.Key("task_key")) + assert.Equal(t, "resources.jobs.a_job.tasks[1].task_key", dyn.ToStructPath(p).String()) + + p = dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"), dyn.Key("${var.env}_job")) + assert.Equal(t, "resources.jobs['${var.env}_job']", dyn.ToStructPath(p).String()) +} + +func TestToStructPaths(t *testing.T) { + paths := dyn.ToStructPaths( + dyn.MustPathFromString("a.b"), + dyn.NewPath(dyn.Key("c"), dyn.Index(1)), + ) + assert.Len(t, paths, 2) + assert.Equal(t, "a.b", paths[0].String()) + assert.Equal(t, "c[1]", paths[1].String()) + assert.Empty(t, dyn.ToStructPaths()) +} diff --git a/libs/flags/json_flag_test.go b/libs/flags/json_flag_test.go index 975976de34b..99b83a80a28 100644 --- a/libs/flags/json_flag_test.go +++ b/libs/flags/json_flag_test.go @@ -201,7 +201,7 @@ func TestJsonUnmarshalRequestMismatch(t *testing.T) { Column: 6, }, }, - Paths: []dyn.Path{{}}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }) } @@ -259,7 +259,7 @@ func TestJsonUnmarshalWrongTypeReportsCorrectLocation(t *testing.T) { Column: 15, }, }, - Paths: []dyn.Path{dyn.NewPath(dyn.Key("job_id"))}, + Paths: dyn.ToStructPaths(dyn.NewPath(dyn.Key("job_id"))), }) } @@ -284,7 +284,7 @@ func TestJsonUnmarshalArrayInsteadOfIntReportsCorrectLocation(t *testing.T) { Column: 40, }, }, - Paths: []dyn.Path{dyn.NewPath(dyn.Key("new_settings"), dyn.Key("timeout_seconds"))}, + Paths: dyn.ToStructPaths(dyn.NewPath(dyn.Key("new_settings"), dyn.Key("timeout_seconds"))), }) } diff --git a/libs/structs/structpath/path.go b/libs/structs/structpath/path.go index 92715dc09d6..436a063d865 100644 --- a/libs/structs/structpath/path.go +++ b/libs/structs/structpath/path.go @@ -5,9 +5,6 @@ import ( "fmt" "strconv" "strings" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" ) const ( @@ -637,6 +634,15 @@ func MustParsePath(s string) *PathNode { return path } +// MustParsePaths parses each of paths like [MustParsePath], e.g. for diag.Diagnostic.Paths. +func MustParsePaths(paths ...string) []*PathNode { + out := make([]*PathNode, len(paths)) + for i, s := range paths { + out[i] = MustParsePath(s) + } + return out +} + // isReservedFieldChar checks if character is reserved and cannot be used in field names func isReservedFieldChar(ch byte) bool { switch ch { @@ -674,26 +680,6 @@ func isValidField(s string) bool { return len(s) > 0 } -// PureReferenceToPath returns a PathNode if s is a pure variable reference, otherwise false. -// This function is similar to dynvar.PureReferenceToPath but returns a *PathNode instead of dyn.Path. -func PureReferenceToPath(s string) (*PathNode, bool) { - ref, ok := dynvar.NewRef(dyn.V(s)) - if !ok { - return nil, false - } - - if !ref.IsPure() { - return nil, false - } - - pattern, err := parse(ref.References()[0], false) - if err != nil { - return nil, false - } - - return (*PathNode)(pattern), true -} - // SkipPrefix returns a new PathNode that skips the first n components of the path. // If n is greater than or equal to the path length, returns nil (root). func (p *PathNode) SkipPrefix(n int) *PathNode { diff --git a/libs/structs/structpath/path_test.go b/libs/structs/structpath/path_test.go index b8623db76d0..8f16fd2ec1d 100644 --- a/libs/structs/structpath/path_test.go +++ b/libs/structs/structpath/path_test.go @@ -714,55 +714,6 @@ func TestLen(t *testing.T) { } } -func TestPureReferenceToPath(t *testing.T) { - tests := []struct { - name string - input string - expected string - ok bool - }{ - { - name: "simple reference", - input: "${resources.jobs.foo.id}", - expected: "resources.jobs.foo.id", - ok: true, - }, - { - name: "simple reference", - input: "${resources.jobs.foo.tasks[1].env.key}", - expected: "resources.jobs.foo.tasks[1].env.key", - ok: true, - }, - { - name: "complex nested reference", - input: "${var.resources.jobs['my_job'].tasks[0]}", - // we use regex from dyn module which only support integers inside brackets: - // expected: "resources.jobs['my_job'].tasks[0]", - }, - { - name: "not a pure reference", - input: "prefix_${var.field}", - }, - { - name: "not a variable reference", - input: "plain_string", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - pathNode, ok := PureReferenceToPath(tt.input) - assert.Equal(t, tt.ok, ok) - if tt.ok { - assert.NotNil(t, pathNode) - assert.Equal(t, tt.expected, pathNode.String()) - } else { - assert.Nil(t, pathNode) - } - }) - } -} - func TestHasPrefix(t *testing.T) { tests := []struct { name string @@ -1274,3 +1225,11 @@ paths: require.Error(t, err) assert.Contains(t, err.Error(), "unexpected end of input") } + +func TestMustParsePaths(t *testing.T) { + paths := MustParsePaths("bundle.terraform", "resources.jobs.foo.tasks[0]") + require.Len(t, paths, 2) + assert.Equal(t, "bundle.terraform", paths[0].String()) + assert.Equal(t, "resources.jobs.foo.tasks[0]", paths[1].String()) + assert.Empty(t, MustParsePaths()) +} From bc78990fb1d61004bf60ca8c45804626c7e9b4f4 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Oct 2026 16:25:42 +0200 Subject: [PATCH 02/12] structpath: build diagnostic paths from keys Co-authored-by: Isaac --- bundle/apps/validate.go | 3 ++- bundle/config/mutator/load_dbalert_files.go | 9 +++++---- .../resourcemutator/secret_scope_fixups.go | 3 ++- bundle/config/validate/files_to_sync.go | 4 ++-- .../config/validate/job_cluster_key_defined.go | 12 +++++++----- bundle/config/validate/required.go | 9 ++++----- .../config/validate/validate_dashboard_etags.go | 4 ++-- .../validate/validate_deployment_fields.go | 16 ++++++++-------- .../validate/validate_genie_space_etags.go | 4 ++-- .../validate_job_run_idempotency_token.go | 4 ++-- bundle/config/validate/validate_sync_patterns.go | 12 ++++++------ bundle/config/validate/validate_volume_path.go | 4 ++-- libs/structs/structpath/path.go | 14 ++++++++++++++ libs/structs/structpath/path_test.go | 12 ++++++++++++ 14 files changed, 70 insertions(+), 40 deletions(-) diff --git a/bundle/apps/validate.go b/bundle/apps/validate.go index f7a2fee7103..b73398f20d8 100644 --- a/bundle/apps/validate.go +++ b/bundle/apps/validate.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/apps" ) @@ -172,7 +173,7 @@ func warnForAppResourcePermissions(b *bundle.Bundle, appKey string, app *resourc ref.permission, appKey, ), - Paths: dyn.ToStructPaths(dyn.MustPathFromString(appPath)), + Paths: structpath.NewStringKeysSlice("resources", "apps", appKey), Locations: b.Config.GetLocations(appPath), }) } diff --git a/bundle/config/mutator/load_dbalert_files.go b/bundle/config/mutator/load_dbalert_files.go index a8571620f44..622fcab7b24 100644 --- a/bundle/config/mutator/load_dbalert_files.go +++ b/bundle/config/mutator/load_dbalert_files.go @@ -13,6 +13,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/sql" ) @@ -85,7 +86,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia Severity: diag.Error, Summary: fmt.Sprintf("field %s is not allowed in the bundle configuration.", k), Detail: "When a .dbalert.json is specified, only the following fields are allowed in the bundle configuration: " + strings.Join(allowedInYAML, ", "), - Paths: dyn.ToStructPaths(dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.%s", alertKey, k))), + Paths: structpath.NewStringKeysSlice("resources", "alerts", alertKey, k), Locations: v.Locations(), }, } @@ -107,7 +108,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia Severity: diag.Error, Summary: fmt.Sprintf("failed to read .dbalert.json file %s: %s", alert.FilePath, err), Detail: "", - Paths: dyn.ToStructPaths(dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.file_path", alertKey))), + Paths: structpath.NewStringKeysSlice("resources", "alerts", alertKey, "file_path"), Locations: alertV.Get("file_path").Locations(), }, } @@ -122,7 +123,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia Severity: diag.Error, Summary: fmt.Sprintf("failed to parse .dbalert.json file %s: %s", alert.FilePath, err), Detail: "", - Paths: dyn.ToStructPaths(dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.file_path", alertKey))), + Paths: structpath.NewStringKeysSlice("resources", "alerts", alertKey, "file_path"), Locations: alertV.Get("file_path").Locations(), }, } @@ -136,7 +137,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia Severity: diag.Error, Summary: fmt.Sprintf(".alert file %s must not contain variable interpolations.", alert.FilePath), Detail: "Please inline the alert configuration in the bundle configuration to use variables", - Paths: dyn.ToStructPaths(dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.file_path", alertKey))), + Paths: structpath.NewStringKeysSlice("resources", "alerts", alertKey, "file_path"), Locations: alertV.Get("file_path").Locations(), }, } diff --git a/bundle/config/mutator/resourcemutator/secret_scope_fixups.go b/bundle/config/mutator/resourcemutator/secret_scope_fixups.go index 66010596dab..e4ca3767f03 100644 --- a/bundle/config/mutator/resourcemutator/secret_scope_fixups.go +++ b/bundle/config/mutator/resourcemutator/secret_scope_fixups.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/iamutil" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/iam" ) @@ -138,7 +139,7 @@ func (m *secretScopeFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Di Severity: diag.Error, Summary: "Failed to collapse permissions for secret scope", Detail: err.Error(), - Paths: dyn.ToStructPaths(dyn.MustPathFromString("resources.secret_scopes." + key)), + Paths: structpath.NewStringKeysSlice("resources", "secret_scopes", key), Locations: []dyn.Location{b.Config.GetLocation("resources.secret_scopes." + key)}, }, } diff --git a/bundle/config/validate/files_to_sync.go b/bundle/config/validate/files_to_sync.go index 7ce1771b5c8..a3104974b9c 100644 --- a/bundle/config/validate/files_to_sync.go +++ b/bundle/config/validate/files_to_sync.go @@ -6,7 +6,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/deploy/files" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/sync" ) @@ -63,7 +63,7 @@ func (v *filesToSync) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost // Show all locations where sync.exclude is defined, since merging // sync.exclude is additive. Locations: b.Config.GetLocations(path), - Paths: dyn.ToStructPaths(dyn.MustPathFromString(path)), + Paths: structpath.NewStringKeysSlice("sync", "exclude"), }) } diff --git a/bundle/config/validate/job_cluster_key_defined.go b/bundle/config/validate/job_cluster_key_defined.go index 1f145867288..0aef74be60f 100644 --- a/bundle/config/validate/job_cluster_key_defined.go +++ b/bundle/config/validate/job_cluster_key_defined.go @@ -6,7 +6,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) func JobClusterKeyDefined() bundle.ReadOnlyMutator { @@ -32,12 +32,14 @@ func (v *jobClusterKeyDefined) Apply(ctx context.Context, b *bundle.Bundle) diag for index, task := range job.Tasks { diags = diags.Extend(checkJobClusterKey(b, jobClusterKeys, task.JobClusterKey, - fmt.Sprintf("resources.jobs.%s.tasks[%d].job_cluster_key", k, index))) + fmt.Sprintf("resources.jobs.%s.tasks[%d].job_cluster_key", k, index), + []*structpath.PathNode{structpath.NewStringKey(structpath.NewIndex(structpath.NewStringKeys(nil, "resources", "jobs", k, "tasks"), index), "job_cluster_key")})) // The Jobs API rejects nested for_each_task, so one level is sufficient. if task.ForEachTask != nil { diags = diags.Extend(checkJobClusterKey(b, jobClusterKeys, task.ForEachTask.Task.JobClusterKey, - fmt.Sprintf("resources.jobs.%s.tasks[%d].for_each_task.task.job_cluster_key", k, index))) + fmt.Sprintf("resources.jobs.%s.tasks[%d].for_each_task.task.job_cluster_key", k, index), + []*structpath.PathNode{structpath.NewStringKeys(structpath.NewIndex(structpath.NewStringKeys(nil, "resources", "jobs", k, "tasks"), index), "for_each_task", "task", "job_cluster_key")})) } } } @@ -46,7 +48,7 @@ func (v *jobClusterKeyDefined) Apply(ctx context.Context, b *bundle.Bundle) diag } // checkJobClusterKey warns if jobClusterKey is set but not defined in the job's job_clusters. -func checkJobClusterKey(b *bundle.Bundle, jobClusterKeys map[string]bool, jobClusterKey, path string) diag.Diagnostics { +func checkJobClusterKey(b *bundle.Bundle, jobClusterKeys map[string]bool, jobClusterKey, path string, paths []*structpath.PathNode) diag.Diagnostics { if jobClusterKey == "" { return nil } @@ -61,6 +63,6 @@ func checkJobClusterKey(b *bundle.Bundle, jobClusterKeys map[string]bool, jobClu // Other associated locations are not relevant since they are // overridden during merging. Locations: b.Config.GetLocations(path), - Paths: dyn.ToStructPaths(dyn.MustPathFromString(path)), + Paths: paths, }} } diff --git a/bundle/config/validate/required.go b/bundle/config/validate/required.go index b82a920e5a3..84c5a734c82 100644 --- a/bundle/config/validate/required.go +++ b/bundle/config/validate/required.go @@ -106,11 +106,11 @@ func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnosti for key, dashboard := range b.Config.Resources.Dashboards { if dashboard.DisplayName == "" { nameLocations = append(nameLocations, b.Config.GetLocations("resources.dashboards."+key)...) - namePaths = append(namePaths, dyn.ToStructPath(dyn.MustPathFromString("resources.dashboards."+key))) + namePaths = append(namePaths, structpath.NewStringKeys(nil, "resources", "dashboards", key)) } if dashboard.WarehouseId == "" { warehouseIdLocations = append(warehouseIdLocations, b.Config.GetLocations("resources.dashboards."+key)...) - warehouseIdPaths = append(warehouseIdPaths, dyn.ToStructPath(dyn.MustPathFromString("resources.dashboards."+key))) + warehouseIdPaths = append(warehouseIdPaths, structpath.NewStringKeys(nil, "resources", "dashboards", key)) } } @@ -140,7 +140,7 @@ func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnosti Severity: diag.Error, Summary: "sql_warehouse name is required", Locations: b.Config.GetLocations(path), - Paths: dyn.ToStructPaths(dyn.MustPathFromString(path)), + Paths: structpath.NewStringKeysSlice("resources", "sql_warehouses", key), }) } } @@ -200,7 +200,6 @@ func errorForInvalidSecretScopePermissions(ctx context.Context, b *bundle.Bundle if perm.UserName != "" || perm.GroupName != "" || perm.ServicePrincipalName != "" { continue } - path := fmt.Sprintf("resources.secret_scopes.%s.permissions[%d]", key, i) // ApplyBundlePermissions rebuilds permissions via convert.FromTyped and drops // per-entry locations, so point at the scope. diags = diags.Append(diag.Diagnostic{ @@ -208,7 +207,7 @@ func errorForInvalidSecretScopePermissions(ctx context.Context, b *bundle.Bundle Summary: "secret scope permission principal is required", Detail: "Set one of user_name, group_name or service_principal_name", Locations: b.Config.GetLocations("resources.secret_scopes." + key), - Paths: dyn.ToStructPaths(dyn.MustPathFromString(path)), + Paths: []*structpath.PathNode{structpath.NewIndex(structpath.NewStringKeys(nil, "resources", "secret_scopes", key, "permissions"), i)}, }) } } diff --git a/bundle/config/validate/validate_dashboard_etags.go b/bundle/config/validate/validate_dashboard_etags.go index ca323c7d388..d47c09a420d 100644 --- a/bundle/config/validate/validate_dashboard_etags.go +++ b/bundle/config/validate/validate_dashboard_etags.go @@ -6,7 +6,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) func ValidateDashboardEtags() bundle.ReadOnlyMutator { @@ -27,7 +27,7 @@ func (v *validateDashboardEtags) Apply(ctx context.Context, b *bundle.Bundle) di { Severity: diag.Error, Summary: fmt.Sprintf("dashboard %q has an etag set. Etags must not be set in bundle configuration", dashboard.DisplayName), - Paths: dyn.ToStructPaths(dyn.MustPathFromString("resources.dashboards." + k)), + Paths: structpath.NewStringKeysSlice("resources", "dashboards", k), Locations: b.Config.GetLocations("resources.dashboards." + k), }, } diff --git a/bundle/config/validate/validate_deployment_fields.go b/bundle/config/validate/validate_deployment_fields.go index cb65f5135ce..71a15991158 100644 --- a/bundle/config/validate/validate_deployment_fields.go +++ b/bundle/config/validate/validate_deployment_fields.go @@ -7,7 +7,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) func ValidateDeploymentFields() bundle.ReadOnlyMutator { @@ -26,29 +26,29 @@ func (v *validateDeploymentFields) Apply(_ context.Context, b *bundle.Bundle) di // deployment_id and version_id identify the bundle deployment and its version // in the Deployment Metadata Service. The CLI sets them on every deploy, so a // value provided by hand would be overwritten; reject it up front. - reject := func(resourcePath, field, value string) { + reject := func(resourceType, name, field, value string) { if value == "" { return } - path := resourcePath + ".deployment." + field + path := "resources." + resourceType + "." + name + ".deployment." + field diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: field + " must not be set in bundle configuration; it is managed by Declarative Automation Bundles", - Paths: dyn.ToStructPaths(dyn.MustPathFromString(path)), + Paths: structpath.NewStringKeysSlice("resources", resourceType, name, "deployment", field), Locations: b.Config.GetLocations(path), }) } for name, job := range b.Config.Resources.Jobs { if d := job.Deployment; d != nil { - reject("resources.jobs."+name, "deployment_id", d.DeploymentId) - reject("resources.jobs."+name, "version_id", d.VersionId) + reject("jobs", name, "deployment_id", d.DeploymentId) + reject("jobs", name, "version_id", d.VersionId) } } for name, pipeline := range b.Config.Resources.Pipelines { if d := pipeline.Deployment; d != nil { - reject("resources.pipelines."+name, "deployment_id", d.DeploymentId) - reject("resources.pipelines."+name, "version_id", d.VersionId) + reject("pipelines", name, "deployment_id", d.DeploymentId) + reject("pipelines", name, "version_id", d.VersionId) } } diff --git a/bundle/config/validate/validate_genie_space_etags.go b/bundle/config/validate/validate_genie_space_etags.go index c7682f08e9e..c0d0c601817 100644 --- a/bundle/config/validate/validate_genie_space_etags.go +++ b/bundle/config/validate/validate_genie_space_etags.go @@ -6,7 +6,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) func ValidateGenieSpaceEtags() bundle.ReadOnlyMutator { @@ -29,7 +29,7 @@ func (v *validateGenieSpaceEtags) Apply(ctx context.Context, b *bundle.Bundle) d { Severity: diag.Error, Summary: fmt.Sprintf("genie space %q has an etag set. Etags must not be set in bundle configuration", genieSpace.Title), - Paths: dyn.ToStructPaths(dyn.MustPathFromString("resources.genie_spaces." + k)), + Paths: structpath.NewStringKeysSlice("resources", "genie_spaces", k), Locations: b.Config.GetLocations("resources.genie_spaces." + k), }, } diff --git a/bundle/config/validate/validate_job_run_idempotency_token.go b/bundle/config/validate/validate_job_run_idempotency_token.go index caf8b6b3cd5..cd5b42a0c25 100644 --- a/bundle/config/validate/validate_job_run_idempotency_token.go +++ b/bundle/config/validate/validate_job_run_idempotency_token.go @@ -7,7 +7,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) func ValidateJobRunIdempotencyToken() bundle.ReadOnlyMutator { @@ -36,7 +36,7 @@ func (v *validateJobRunIdempotencyToken) Apply(_ context.Context, b *bundle.Bund diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: "idempotency_token must not be set in bundle configuration; the CLI sets it on each run-now request", - Paths: dyn.ToStructPaths(dyn.MustPathFromString(path)), + Paths: structpath.NewStringKeysSlice("resources", "job_runs", name, "idempotency_token"), Locations: b.Config.GetLocations(path), }) } diff --git a/bundle/config/validate/validate_sync_patterns.go b/bundle/config/validate/validate_sync_patterns.go index 8810f2e1f24..4ecb3665c84 100644 --- a/bundle/config/validate/validate_sync_patterns.go +++ b/bundle/config/validate/validate_sync_patterns.go @@ -7,9 +7,9 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/fileset" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structpath" "golang.org/x/sync/errgroup" ) @@ -26,16 +26,16 @@ func (v *validateSyncPatterns) Name() string { func (v *validateSyncPatterns) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { s := b.Config.Sync - checkPatterns(ctx, s.Exclude, "sync.exclude", b) + checkPatterns(ctx, s.Exclude, "exclude", b) if logdiag.HasError(ctx) { return nil } - checkPatterns(ctx, s.Include, "sync.include", b) + checkPatterns(ctx, s.Include, "include", b) return nil } -func checkPatterns(ctx context.Context, patterns []string, path string, b *bundle.Bundle) { +func checkPatterns(ctx context.Context, patterns []string, field string, b *bundle.Bundle) { var errs errgroup.Group for index, pattern := range patterns { @@ -57,12 +57,12 @@ func checkPatterns(ctx context.Context, patterns []string, path string, b *bundl } if len(all) == 0 { - path := fmt.Sprintf("%s[%d]", path, index) + path := fmt.Sprintf("sync.%s[%d]", field, index) logdiag.LogDiag(ctx, diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("Pattern %s does not match any files", pattern), Locations: b.Config.GetLocations(path), - Paths: dyn.ToStructPaths(dyn.MustPathFromString(path)), + Paths: []*structpath.PathNode{structpath.NewIndex(structpath.NewStringKeys(nil, "sync", field), index)}, }) } return nil diff --git a/bundle/config/validate/validate_volume_path.go b/bundle/config/validate/validate_volume_path.go index 9a8c1b090cd..4bf6026821a 100644 --- a/bundle/config/validate/validate_volume_path.go +++ b/bundle/config/validate/validate_volume_path.go @@ -7,7 +7,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) type validateVolumePath struct { @@ -39,7 +39,7 @@ func (m *validateVolumePath) Apply(ctx context.Context, b *bundle.Bundle) diag.D Summary: fmt.Sprintf("%s %s starts with /Volumes. /Volumes can only be used with workspace.artifact_path.", check.configName, check.path), Detail: "For more information, see https://docs.databricks.com/aws/en/dev-tools/bundles/settings#workspace", Locations: b.Config.GetLocations(check.configName), - Paths: dyn.ToStructPaths(dyn.MustPathFromString(check.configName)), + Paths: structpath.MustParsePaths(check.configName), }) // Return early for root path validation diff --git a/libs/structs/structpath/path.go b/libs/structs/structpath/path.go index 436a063d865..41062303637 100644 --- a/libs/structs/structpath/path.go +++ b/libs/structs/structpath/path.go @@ -160,6 +160,20 @@ func NewStringKey(prev *PathNode, fieldName string) *PathNode { return NewBracketString(prev, fieldName) } +// NewStringKeys appends the keys to prev, each like [NewStringKey]. +func NewStringKeys(prev *PathNode, keys ...string) *PathNode { + for _, k := range keys { + prev = NewStringKey(prev, k) + } + return prev +} + +// NewStringKeysSlice returns the path of keys from the root as a one-element slice, +// e.g. for diag.Diagnostic.Paths. +func NewStringKeysSlice(keys ...string) []*PathNode { + return []*PathNode{NewStringKeys(nil, keys...)} +} + func NewKeyValue(prev *PathNode, key, value string) *PathNode { return &PathNode{ prev: prev, diff --git a/libs/structs/structpath/path_test.go b/libs/structs/structpath/path_test.go index 8f16fd2ec1d..43d8696cba3 100644 --- a/libs/structs/structpath/path_test.go +++ b/libs/structs/structpath/path_test.go @@ -1233,3 +1233,15 @@ func TestMustParsePaths(t *testing.T) { assert.Equal(t, "resources.jobs.foo.tasks[0]", paths[1].String()) assert.Empty(t, MustParsePaths()) } + +func TestNewStringKeys(t *testing.T) { + assert.Equal(t, "a.b['c d']", NewStringKeys(NewStringKey(nil, "a"), "b", "c d").String()) + assert.Nil(t, NewStringKeys(nil)) + assert.Equal(t, "a", NewStringKeys(NewStringKey(nil, "a")).String()) +} + +func TestNewStringKeysSlice(t *testing.T) { + paths := NewStringKeysSlice("resources", "jobs", "${var.env}_job") + require.Len(t, paths, 1) + assert.Equal(t, "resources.jobs['${var.env}_job']", paths[0].String()) +} From cc729c99acc8523cbcb8a92b14ec5ea64cae0420 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Oct 2026 16:26:25 +0200 Subject: [PATCH 03/12] Changelog link Co-authored-by: Isaac --- .nextchanges/bundles/{6974.md => 6980.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .nextchanges/bundles/{6974.md => 6980.md} (100%) diff --git a/.nextchanges/bundles/6974.md b/.nextchanges/bundles/6980.md similarity index 100% rename from .nextchanges/bundles/6974.md rename to .nextchanges/bundles/6980.md From 0bd2c601adcea52fbe8fe0c8f6a9a88ac9c63630 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Oct 2026 16:38:05 +0200 Subject: [PATCH 04/12] Changelog link to #6980 Co-authored-by: Isaac --- .nextchanges/bundles/6980.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/6980.md b/.nextchanges/bundles/6980.md index bf4b18fc997..7a40caf2cc1 100644 --- a/.nextchanges/bundles/6980.md +++ b/.nextchanges/bundles/6980.md @@ -1 +1 @@ -* Diagnostic paths now quote map keys that are not identifiers, e.g. `resources.jobs['${var.env}_job']`. ([#6974](https://github.com/databricks/cli/pull/6974)) +* Diagnostic paths now quote map keys that are not identifiers, e.g. `resources.jobs['${var.env}_job']`. ([#6980](https://github.com/databricks/cli/pull/6980)) From a5c091542c081e97ed89a6b04a380e918264965b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Oct 2026 17:55:07 +0200 Subject: [PATCH 05/12] Build literal and dyn.NewPath diagnostic paths with NewStringKeysSlice Co-authored-by: Isaac --- bundle/config/loader/process_include.go | 2 +- bundle/config/mutator/python/python_diagnostics_test.go | 2 +- bundle/config/mutator/reject_internal_resources.go | 2 +- bundle/config/mutator/resourcemutator/run_as.go | 2 +- bundle/config/mutator/sync_infer_root.go | 4 ++-- bundle/config/validate/no_interpolation_in_bundle_name.go | 2 +- bundle/config/validate/single_node_cluster_test.go | 2 +- bundle/config/validate/validate_artifact_path.go | 4 ++-- bundle/config/validate/validate_artifact_path_test.go | 4 ++-- bundle/config/validate/validate_engine.go | 2 +- bundle/render/render_text_output_test.go | 2 +- bundle/tests/validate_test.go | 2 +- libs/dyn/convert/normalize_test.go | 7 ++++--- libs/flags/json_flag_test.go | 5 +++-- 14 files changed, 22 insertions(+), 20 deletions(-) diff --git a/bundle/config/loader/process_include.go b/bundle/config/loader/process_include.go index bc82bee1867..ec0d03c0d80 100644 --- a/bundle/config/loader/process_include.go +++ b/bundle/config/loader/process_include.go @@ -169,7 +169,7 @@ func (m *processInclude) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos Detail: `An include section is defined in a file that is not databricks.yml. Only includes defined in databricks.yml are applied.`, Locations: this.GetLocations("include"), - Paths: structpath.MustParsePaths("include"), + Paths: structpath.NewStringKeysSlice("include"), }) } diff --git a/bundle/config/mutator/python/python_diagnostics_test.go b/bundle/config/mutator/python/python_diagnostics_test.go index c00650ea717..289df745283 100644 --- a/bundle/config/mutator/python/python_diagnostics_test.go +++ b/bundle/config/mutator/python/python_diagnostics_test.go @@ -56,7 +56,7 @@ func TestParsePythonDiagnostics(t *testing.T) { { Severity: diag.Error, Summary: "error summary", - Paths: structpath.MustParsePaths("resources.jobs.job0.name"), + Paths: structpath.NewStringKeysSlice("resources", "jobs", "job0", "name"), }, }, }, diff --git a/bundle/config/mutator/reject_internal_resources.go b/bundle/config/mutator/reject_internal_resources.go index 5948f048d93..2eacbbc7f9c 100644 --- a/bundle/config/mutator/reject_internal_resources.go +++ b/bundle/config/mutator/reject_internal_resources.go @@ -26,7 +26,7 @@ func (m *rejectInternalResources) Apply(ctx context.Context, b *bundle.Bundle) d diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "Internal resources cannot be set in bundle configuration", - Paths: structpath.MustParsePaths("resources"), + Paths: structpath.NewStringKeysSlice("resources"), Locations: b.Config.GetLocations("resources"), }) } diff --git a/bundle/config/mutator/resourcemutator/run_as.go b/bundle/config/mutator/resourcemutator/run_as.go index 66f1bd7e7d1..e89f2ee3c4e 100644 --- a/bundle/config/mutator/resourcemutator/run_as.go +++ b/bundle/config/mutator/resourcemutator/run_as.go @@ -239,7 +239,7 @@ func (m *setRunAs) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { { Severity: diag.Warning, Summary: "You are using the legacy mode of run_as. The support for this mode is experimental and might be removed in a future release of the CLI. In order to run the pipelines in your DABs project as the run_as user this mode changes the owners of the pipelines to the run_as identity, which requires the user deploying the bundle to be a workspace admin, and also a Metastore admin if the pipeline target is in UC.", - Paths: structpath.MustParsePaths("experimental.use_legacy_run_as"), + Paths: structpath.NewStringKeysSlice("experimental", "use_legacy_run_as"), Locations: b.Config.GetLocations("experimental.use_legacy_run_as"), }, } diff --git a/bundle/config/mutator/sync_infer_root.go b/bundle/config/mutator/sync_infer_root.go index 9fc58ee992f..07cd74fee26 100644 --- a/bundle/config/mutator/sync_infer_root.go +++ b/bundle/config/mutator/sync_infer_root.go @@ -7,7 +7,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/vfs" ) @@ -94,7 +94,7 @@ func (m *syncInferRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno Severity: diag.Error, Summary: fmt.Sprintf("invalid sync path %q", path), Locations: b.Config.GetLocations(fmt.Sprintf("sync.paths[%d]", i)), - Paths: dyn.ToStructPaths(dyn.NewPath(dyn.Key("sync"), dyn.Key("paths"), dyn.Index(i))), + Paths: []*structpath.PathNode{structpath.NewIndex(structpath.NewStringKeys(nil, "sync", "paths"), i)}, }) } diff --git a/bundle/config/validate/no_interpolation_in_bundle_name.go b/bundle/config/validate/no_interpolation_in_bundle_name.go index 4febeb826cd..1b81655af9c 100644 --- a/bundle/config/validate/no_interpolation_in_bundle_name.go +++ b/bundle/config/validate/no_interpolation_in_bundle_name.go @@ -29,7 +29,7 @@ is a part of the path at which your bundle state is stored at by default. Parame runtime can have unexpected consequences like duplicate deployments or resources not being cleaned up during bundle destroy.`, Locations: b.Config.GetLocations("bundle.name"), - Paths: structpath.MustParsePaths("bundle.name"), + Paths: structpath.NewStringKeysSlice("bundle", "name"), }) } diff --git a/bundle/config/validate/single_node_cluster_test.go b/bundle/config/validate/single_node_cluster_test.go index 46bab0084a1..463991f5539 100644 --- a/bundle/config/validate/single_node_cluster_test.go +++ b/bundle/config/validate/single_node_cluster_test.go @@ -123,7 +123,7 @@ func TestValidateSingleNodeClusterFailForInteractiveClusters(t *testing.T) { Summary: singleNodeWarningSummary, Detail: singleNodeWarningDetail, Locations: []dyn.Location{{File: "a.yml", Line: 1, Column: 1}}, - Paths: dyn.ToStructPaths(dyn.NewPath(dyn.Key("resources"), dyn.Key("clusters"), dyn.Key("foo"))), + Paths: structpath.NewStringKeysSlice("resources", "clusters", "foo"), }, }, diags) }) diff --git a/bundle/config/validate/validate_artifact_path.go b/bundle/config/validate/validate_artifact_path.go index 93f38b272a9..ff6c893ccbd 100644 --- a/bundle/config/validate/validate_artifact_path.go +++ b/bundle/config/validate/validate_artifact_path.go @@ -87,7 +87,7 @@ func (v *validateArtifactPath) Apply(ctx context.Context, b *bundle.Bundle) diag Summary: s, Severity: diag.Error, Locations: b.Config.GetLocations("workspace.artifact_path"), - Paths: structpath.MustParsePaths("workspace.artifact_path"), + Paths: structpath.NewStringKeysSlice("workspace", "artifact_path"), }, } } @@ -119,7 +119,7 @@ this bundle but which has not been deployed yet. Please first deploy the volume using 'bundle deploy' and then switch over to using it in the artifact_path.`, Locations: slices.Concat(b.Config.GetLocations("workspace.artifact_path"), locations), - Paths: append(structpath.MustParsePaths("workspace.artifact_path"), dyn.ToStructPath(path)), + Paths: append(structpath.NewStringKeysSlice("workspace", "artifact_path"), dyn.ToStructPath(path)), }} } diff --git a/bundle/config/validate/validate_artifact_path_test.go b/bundle/config/validate/validate_artifact_path_test.go index 4d85dcd9176..a88e7213953 100644 --- a/bundle/config/validate/validate_artifact_path_test.go +++ b/bundle/config/validate/validate_artifact_path_test.go @@ -84,7 +84,7 @@ func TestValidateArtifactPath(t *testing.T) { Severity: diag.Error, Summary: expected, Locations: []dyn.Location{{File: "file", Line: 1, Column: 1}}, - Paths: structpath.MustParsePaths("workspace.artifact_path"), + Paths: structpath.NewStringKeysSlice("workspace", "artifact_path"), }}, diags) } @@ -171,7 +171,7 @@ func TestValidateArtifactPathWithInvalidPaths(t *testing.T) { Severity: diag.Error, Summary: "expected UC volume path to be in the format /Volumes////..., got " + p, Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}, - Paths: structpath.MustParsePaths("workspace.artifact_path"), + Paths: structpath.NewStringKeysSlice("workspace", "artifact_path"), }}, diags) } } diff --git a/bundle/config/validate/validate_engine.go b/bundle/config/validate/validate_engine.go index 2b9d00971b4..832d1aa330b 100644 --- a/bundle/config/validate/validate_engine.go +++ b/bundle/config/validate/validate_engine.go @@ -31,7 +31,7 @@ func (v *validateEngine) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos Severity: diag.Warning, Summary: "bundle.terraform is deprecated and has no effect: " + engine.TerraformRemovedSummary, Locations: tf.Locations(), - Paths: structpath.MustParsePaths("bundle.terraform"), + Paths: structpath.NewStringKeysSlice("bundle", "terraform"), }) } diff --git a/bundle/render/render_text_output_test.go b/bundle/render/render_text_output_test.go index 0d631ec2ac6..93eb8e84125 100644 --- a/bundle/render/render_text_output_test.go +++ b/bundle/render/render_text_output_test.go @@ -204,7 +204,7 @@ func TestRenderDiagnostics(t *testing.T) { Severity: diag.Error, Detail: "'name' is required", Summary: "failed to load xxx", - Paths: structpath.MustParsePaths("resources.jobs.xxx"), + Paths: structpath.NewStringKeysSlice("resources", "jobs", "xxx"), }, }, expected: "Error: failed to load xxx\n" + diff --git a/bundle/tests/validate_test.go b/bundle/tests/validate_test.go index d2a3a127843..e552a79226e 100644 --- a/bundle/tests/validate_test.go +++ b/bundle/tests/validate_test.go @@ -79,7 +79,7 @@ func TestValidateUniqueResourceIdentifiers(t *testing.T) { {File: "validate/duplicate_resource_name_in_subconfiguration_job_and_job/databricks.yml", Line: 13, Column: 7}, {File: "validate/duplicate_resource_name_in_subconfiguration_job_and_job/resources.yml", Line: 4, Column: 7}, }, - Paths: structpath.MustParsePaths("resources.jobs.foo"), + Paths: structpath.NewStringKeysSlice("resources", "jobs", "foo"), }, }, }, diff --git a/libs/dyn/convert/normalize_test.go b/libs/dyn/convert/normalize_test.go index 08b89d835ba..e7a1d0e7563 100644 --- a/libs/dyn/convert/normalize_test.go +++ b/libs/dyn/convert/normalize_test.go @@ -6,6 +6,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynassert" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" ) @@ -44,7 +45,7 @@ func TestNormalizeStructElementDiagnostic(t *testing.T) { Severity: diag.Warning, Summary: `expected string, found map`, Locations: []dyn.Location{{}}, - Paths: dyn.ToStructPaths(dyn.NewPath(dyn.Key("bar"))), + Paths: structpath.NewStringKeysSlice("bar"), }, diags[0]) // Elements that encounter an error during normalization are dropped. @@ -306,7 +307,7 @@ func TestNormalizeMapElementDiagnostic(t *testing.T) { Severity: diag.Warning, Summary: `expected string, found map`, Locations: []dyn.Location{{}}, - Paths: dyn.ToStructPaths(dyn.NewPath(dyn.Key("bar"))), + Paths: structpath.NewStringKeysSlice("bar"), }, err[0]) // Elements that encounter an error during normalization are dropped. @@ -430,7 +431,7 @@ func TestNormalizeSliceElementDiagnostic(t *testing.T) { Severity: diag.Warning, Summary: `expected string, found map`, Locations: []dyn.Location{{}}, - Paths: dyn.ToStructPaths(dyn.NewPath(dyn.Index(2))), + Paths: []*structpath.PathNode{structpath.NewIndex(nil, 2)}, }, err[0]) // Elements that encounter an error during normalization are dropped. diff --git a/libs/flags/json_flag_test.go b/libs/flags/json_flag_test.go index 99b83a80a28..4b61f619a23 100644 --- a/libs/flags/json_flag_test.go +++ b/libs/flags/json_flag_test.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -259,7 +260,7 @@ func TestJsonUnmarshalWrongTypeReportsCorrectLocation(t *testing.T) { Column: 15, }, }, - Paths: dyn.ToStructPaths(dyn.NewPath(dyn.Key("job_id"))), + Paths: structpath.NewStringKeysSlice("job_id"), }) } @@ -284,7 +285,7 @@ func TestJsonUnmarshalArrayInsteadOfIntReportsCorrectLocation(t *testing.T) { Column: 40, }, }, - Paths: dyn.ToStructPaths(dyn.NewPath(dyn.Key("new_settings"), dyn.Key("timeout_seconds"))), + Paths: structpath.NewStringKeysSlice("new_settings", "timeout_seconds"), }) } From 79d4278824fd92a941d69cdcb4a6987f82bbd754 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Oct 2026 18:02:14 +0200 Subject: [PATCH 06/12] Keep PureReferenceToPath in structpath without dyn; revert unrelated import regrouping Co-authored-by: Isaac --- .../python/apply_python_output_test.go | 4 +- .../config/mutator/python/python_mutator.go | 19 ++++--- .../mutator/python/python_mutator_test.go | 11 ++-- .../expand_pipeline_glob_paths_test.go | 8 ++- .../resourcemutator/validate_target_mode.go | 5 +- bundle/config/mutator/translate_paths.go | 5 +- bundle/libraries/upload.go | 1 + cmd/apps/import.go | 3 +- libs/dyn/dynvar/ref.go | 21 ------- libs/dyn/dynvar/ref_structpath_test.go | 56 ------------------- libs/structs/structpath/path.go | 25 +++++++++ libs/structs/structpath/path_test.go | 49 ++++++++++++++++ libs/structs/structpath/reference_test.go | 30 ++++++++++ 13 files changed, 140 insertions(+), 97 deletions(-) delete mode 100644 libs/dyn/dynvar/ref_structpath_test.go create mode 100644 libs/structs/structpath/reference_test.go diff --git a/bundle/config/mutator/python/apply_python_output_test.go b/bundle/config/mutator/python/apply_python_output_test.go index 32bc409c74c..2d64d6c4bbc 100644 --- a/bundle/config/mutator/python/apply_python_output_test.go +++ b/bundle/config/mutator/python/apply_python_output_test.go @@ -4,8 +4,10 @@ import ( "testing" "github.com/databricks/cli/bundle/config/mutator/resourcemutator" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/merge" + + "github.com/databricks/cli/libs/dyn" "github.com/stretchr/testify/assert" ) diff --git a/bundle/config/mutator/python/python_mutator.go b/bundle/config/mutator/python/python_mutator.go index a3e3d94279a..9112aaa808d 100644 --- a/bundle/config/mutator/python/python_mutator.go +++ b/bundle/config/mutator/python/python_mutator.go @@ -13,20 +13,25 @@ import ( "reflect" "strings" - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator/resourcemutator" - "github.com/databricks/cli/bundle/env" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/databricks-sdk-go/logger" + + "github.com/databricks/cli/libs/python" + + "github.com/databricks/cli/bundle/env" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/dyn/yamlloader" - "github.com/databricks/cli/libs/log" - "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/process" - "github.com/databricks/cli/libs/python" - "github.com/databricks/databricks-sdk-go/logger" ) type phase string diff --git a/bundle/config/mutator/python/python_mutator_test.go b/bundle/config/mutator/python/python_mutator_test.go index b30272dc21f..9b106f727bf 100644 --- a/bundle/config/mutator/python/python_mutator_test.go +++ b/bundle/config/mutator/python/python_mutator_test.go @@ -11,16 +11,19 @@ import ( "slices" "testing" + "github.com/databricks/cli/libs/dyn/convert" + + "github.com/databricks/cli/bundle/env" + "github.com/stretchr/testify/require" + + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/process" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestPythonMutator_Name_loadResources(t *testing.T) { diff --git a/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go b/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go index 7341d747570..56440f7cd1c 100644 --- a/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go +++ b/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go @@ -5,16 +5,18 @@ import ( "path/filepath" "testing" - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/mutator/resourcemutator" + + "github.com/stretchr/testify/assert" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/pipelines" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/bundle/config/mutator/resourcemutator/validate_target_mode.go b/bundle/config/mutator/resourcemutator/validate_target_mode.go index bab38d94700..d100c327fe8 100644 --- a/bundle/config/mutator/resourcemutator/validate_target_mode.go +++ b/bundle/config/mutator/resourcemutator/validate_target_mode.go @@ -5,11 +5,12 @@ import ( "fmt" "strings" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/iamutil" + "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/iamutil" ) type validateTargetMode struct{} diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index 1794da522a4..65a903bf5a3 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -11,11 +11,12 @@ import ( "path/filepath" "strings" + "github.com/databricks/cli/bundle/config/mutator/paths" + "github.com/databricks/cli/bundle/libraries" + "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/mutator/paths" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/notebook" diff --git a/bundle/libraries/upload.go b/bundle/libraries/upload.go index 5cf3cfa19b3..dabd4d8860d 100644 --- a/bundle/libraries/upload.go +++ b/bundle/libraries/upload.go @@ -15,6 +15,7 @@ import ( "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" + "golang.org/x/sync/errgroup" ) diff --git a/cmd/apps/import.go b/cmd/apps/import.go index 1c780729d86..f6089870aec 100644 --- a/cmd/apps/import.go +++ b/cmd/apps/import.go @@ -12,6 +12,8 @@ import ( "slices" "strings" + "go.yaml.in/yaml/v3" + "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/generate" "github.com/databricks/cli/bundle/phases" @@ -34,7 +36,6 @@ import ( "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/workspace" "github.com/spf13/cobra" - "go.yaml.in/yaml/v3" ) func newImportCommand() *cobra.Command { diff --git a/libs/dyn/dynvar/ref.go b/libs/dyn/dynvar/ref.go index 7fd61073f0a..de25f44a212 100644 --- a/libs/dyn/dynvar/ref.go +++ b/libs/dyn/dynvar/ref.go @@ -6,7 +6,6 @@ import ( "strings" "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/structs/structpath" ) // BaseVarDef matches a single dot-separated path segment in ${...} references. @@ -182,23 +181,3 @@ func PureReferenceToPath(s string) (dyn.Path, bool) { return p, true } - -// PureReferenceToStructPath returns a PathNode if s is a pure variable reference, otherwise false. -// This function is similar to PureReferenceToPath but returns a *structpath.PathNode instead of dyn.Path. -func PureReferenceToStructPath(s string) (*structpath.PathNode, bool) { - ref, ok := NewRef(dyn.V(s)) - if !ok { - return nil, false - } - - if !ref.IsPure() { - return nil, false - } - - p, err := structpath.ParsePath(ref.References()[0]) - if err != nil { - return nil, false - } - - return p, true -} diff --git a/libs/dyn/dynvar/ref_structpath_test.go b/libs/dyn/dynvar/ref_structpath_test.go deleted file mode 100644 index 1ca2acf4ceb..00000000000 --- a/libs/dyn/dynvar/ref_structpath_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package dynvar - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestPureReferenceToStructPath(t *testing.T) { - tests := []struct { - name string - input string - expected string - ok bool - }{ - { - name: "simple reference", - input: "${resources.jobs.foo.id}", - expected: "resources.jobs.foo.id", - ok: true, - }, - { - name: "simple reference", - input: "${resources.jobs.foo.tasks[1].env.key}", - expected: "resources.jobs.foo.tasks[1].env.key", - ok: true, - }, - { - name: "complex nested reference", - input: "${var.resources.jobs['my_job'].tasks[0]}", - // we use regex from dyn module which only support integers inside brackets: - // expected: "resources.jobs['my_job'].tasks[0]", - }, - { - name: "not a pure reference", - input: "prefix_${var.field}", - }, - { - name: "not a variable reference", - input: "plain_string", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - pathNode, ok := PureReferenceToStructPath(tt.input) - assert.Equal(t, tt.ok, ok) - if tt.ok { - assert.NotNil(t, pathNode) - assert.Equal(t, tt.expected, pathNode.String()) - } else { - assert.Nil(t, pathNode) - } - }) - } -} diff --git a/libs/structs/structpath/path.go b/libs/structs/structpath/path.go index 41062303637..282a093e964 100644 --- a/libs/structs/structpath/path.go +++ b/libs/structs/structpath/path.go @@ -3,10 +3,19 @@ package structpath import ( "errors" "fmt" + "regexp" "strconv" "strings" ) +// baseVarDef matches a path segment of a ${...} reference. It is the grammar of +// dynvar.BaseVarDef, which structpath cannot import; TestPureReferenceMatchesDynvar +// keeps them in agreement. +const baseVarDef = `_*\p{L}+([-_]*[\p{L}\p{N}]+)*` + +// pureReference matches a string that is a single ${...} reference. +var pureReference = regexp.MustCompile(`^\$\{(` + baseVarDef + `(\.` + baseVarDef + `(\[[0-9]+\])*)*(\[[0-9]+\])*)\}$`) + const ( // Encodes wildcard after a dot: foo.* tagDotStar = -2 @@ -694,6 +703,22 @@ func isValidField(s string) bool { return len(s) > 0 } +// PureReferenceToPath returns a PathNode if s is a pure variable reference, otherwise false. +// This function is similar to dynvar.PureReferenceToPath but returns a *PathNode instead of dyn.Path. +func PureReferenceToPath(s string) (*PathNode, bool) { + m := pureReference.FindStringSubmatch(s) + if m == nil { + return nil, false + } + + pattern, err := parse(m[1], false) + if err != nil { + return nil, false + } + + return (*PathNode)(pattern), true +} + // SkipPrefix returns a new PathNode that skips the first n components of the path. // If n is greater than or equal to the path length, returns nil (root). func (p *PathNode) SkipPrefix(n int) *PathNode { diff --git a/libs/structs/structpath/path_test.go b/libs/structs/structpath/path_test.go index 43d8696cba3..924cd97a1c4 100644 --- a/libs/structs/structpath/path_test.go +++ b/libs/structs/structpath/path_test.go @@ -714,6 +714,55 @@ func TestLen(t *testing.T) { } } +func TestPureReferenceToPath(t *testing.T) { + tests := []struct { + name string + input string + expected string + ok bool + }{ + { + name: "simple reference", + input: "${resources.jobs.foo.id}", + expected: "resources.jobs.foo.id", + ok: true, + }, + { + name: "simple reference", + input: "${resources.jobs.foo.tasks[1].env.key}", + expected: "resources.jobs.foo.tasks[1].env.key", + ok: true, + }, + { + name: "complex nested reference", + input: "${var.resources.jobs['my_job'].tasks[0]}", + // we use regex from dyn module which only support integers inside brackets: + // expected: "resources.jobs['my_job'].tasks[0]", + }, + { + name: "not a pure reference", + input: "prefix_${var.field}", + }, + { + name: "not a variable reference", + input: "plain_string", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pathNode, ok := PureReferenceToPath(tt.input) + assert.Equal(t, tt.ok, ok) + if tt.ok { + assert.NotNil(t, pathNode) + assert.Equal(t, tt.expected, pathNode.String()) + } else { + assert.Nil(t, pathNode) + } + }) + } +} + func TestHasPrefix(t *testing.T) { tests := []struct { name string diff --git a/libs/structs/structpath/reference_test.go b/libs/structs/structpath/reference_test.go new file mode 100644 index 00000000000..485b756c7fb --- /dev/null +++ b/libs/structs/structpath/reference_test.go @@ -0,0 +1,30 @@ +package structpath_test + +import ( + "testing" + + "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/stretchr/testify/assert" +) + +func TestPureReferenceMatchesDynvar(t *testing.T) { + for _, s := range []string{ + "${var.foo}", + "${resources.jobs.foo.tasks[1].env.key}", + "${var.résumé_2}", + "${_private.x}", + "${var.foo[0][1]}", + "${var.jobs['my_job']}", + "${var.-foo}", + "${var.foo-}", + "${var.foo}${var.bar}", + "prefix_${var.field}", + "$${var.escaped}", + "${}", + "plain_string", + } { + _, ok := structpath.PureReferenceToPath(s) + assert.Equal(t, dynvar.IsPureVariableReference(s), ok, s) + } +} From a1a7e1198b9f526027b5400ecc6138bf5e41a04b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Oct 2026 23:04:46 +0200 Subject: [PATCH 07/12] structpath: NewPath and NewPathSlice for mixed key/index paths Replace NewStringKeys/NewStringKeysSlice and nested NewIndex/NewStringKey chains: NewPath(prev, parts...) takes string keys and int indices. Co-authored-by: Isaac --- bundle/apps/validate.go | 2 +- bundle/config/loader/process_include.go | 2 +- bundle/config/mutator/load_dbalert_files.go | 8 +++---- .../mutator/python/python_diagnostics_test.go | 2 +- .../mutator/reject_internal_resources.go | 2 +- .../config/mutator/resourcemutator/run_as.go | 2 +- .../resourcemutator/secret_scope_fixups.go | 2 +- bundle/config/mutator/sync_infer_root.go | 2 +- bundle/config/validate/files_to_sync.go | 2 +- .../validate/job_cluster_key_defined.go | 4 ++-- .../no_interpolation_in_bundle_name.go | 2 +- bundle/config/validate/required.go | 8 +++---- .../validate/single_node_cluster_test.go | 2 +- .../config/validate/validate_artifact_path.go | 4 ++-- .../validate/validate_artifact_path_test.go | 4 ++-- .../validate/validate_dashboard_etags.go | 2 +- .../validate/validate_deployment_fields.go | 2 +- bundle/config/validate/validate_engine.go | 2 +- .../validate/validate_genie_space_etags.go | 2 +- .../validate_job_run_idempotency_token.go | 2 +- .../config/validate/validate_sync_patterns.go | 2 +- bundle/render/render_text_output_test.go | 2 +- bundle/tests/validate_test.go | 2 +- libs/dyn/convert/normalize_test.go | 4 ++-- libs/flags/json_flag_test.go | 4 ++-- libs/structs/structpath/path.go | 24 ++++++++++++------- libs/structs/structpath/path_test.go | 15 +++++++----- 27 files changed, 61 insertions(+), 50 deletions(-) diff --git a/bundle/apps/validate.go b/bundle/apps/validate.go index b73398f20d8..29f6f056a53 100644 --- a/bundle/apps/validate.go +++ b/bundle/apps/validate.go @@ -173,7 +173,7 @@ func warnForAppResourcePermissions(b *bundle.Bundle, appKey string, app *resourc ref.permission, appKey, ), - Paths: structpath.NewStringKeysSlice("resources", "apps", appKey), + Paths: structpath.NewPathSlice("resources", "apps", appKey), Locations: b.Config.GetLocations(appPath), }) } diff --git a/bundle/config/loader/process_include.go b/bundle/config/loader/process_include.go index ec0d03c0d80..dd8932caeed 100644 --- a/bundle/config/loader/process_include.go +++ b/bundle/config/loader/process_include.go @@ -169,7 +169,7 @@ func (m *processInclude) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos Detail: `An include section is defined in a file that is not databricks.yml. Only includes defined in databricks.yml are applied.`, Locations: this.GetLocations("include"), - Paths: structpath.NewStringKeysSlice("include"), + Paths: structpath.NewPathSlice("include"), }) } diff --git a/bundle/config/mutator/load_dbalert_files.go b/bundle/config/mutator/load_dbalert_files.go index 622fcab7b24..a5fa2ff8d62 100644 --- a/bundle/config/mutator/load_dbalert_files.go +++ b/bundle/config/mutator/load_dbalert_files.go @@ -86,7 +86,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia Severity: diag.Error, Summary: fmt.Sprintf("field %s is not allowed in the bundle configuration.", k), Detail: "When a .dbalert.json is specified, only the following fields are allowed in the bundle configuration: " + strings.Join(allowedInYAML, ", "), - Paths: structpath.NewStringKeysSlice("resources", "alerts", alertKey, k), + Paths: structpath.NewPathSlice("resources", "alerts", alertKey, k), Locations: v.Locations(), }, } @@ -108,7 +108,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia Severity: diag.Error, Summary: fmt.Sprintf("failed to read .dbalert.json file %s: %s", alert.FilePath, err), Detail: "", - Paths: structpath.NewStringKeysSlice("resources", "alerts", alertKey, "file_path"), + Paths: structpath.NewPathSlice("resources", "alerts", alertKey, "file_path"), Locations: alertV.Get("file_path").Locations(), }, } @@ -123,7 +123,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia Severity: diag.Error, Summary: fmt.Sprintf("failed to parse .dbalert.json file %s: %s", alert.FilePath, err), Detail: "", - Paths: structpath.NewStringKeysSlice("resources", "alerts", alertKey, "file_path"), + Paths: structpath.NewPathSlice("resources", "alerts", alertKey, "file_path"), Locations: alertV.Get("file_path").Locations(), }, } @@ -137,7 +137,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia Severity: diag.Error, Summary: fmt.Sprintf(".alert file %s must not contain variable interpolations.", alert.FilePath), Detail: "Please inline the alert configuration in the bundle configuration to use variables", - Paths: structpath.NewStringKeysSlice("resources", "alerts", alertKey, "file_path"), + Paths: structpath.NewPathSlice("resources", "alerts", alertKey, "file_path"), Locations: alertV.Get("file_path").Locations(), }, } diff --git a/bundle/config/mutator/python/python_diagnostics_test.go b/bundle/config/mutator/python/python_diagnostics_test.go index 289df745283..43709031ca2 100644 --- a/bundle/config/mutator/python/python_diagnostics_test.go +++ b/bundle/config/mutator/python/python_diagnostics_test.go @@ -56,7 +56,7 @@ func TestParsePythonDiagnostics(t *testing.T) { { Severity: diag.Error, Summary: "error summary", - Paths: structpath.NewStringKeysSlice("resources", "jobs", "job0", "name"), + Paths: structpath.NewPathSlice("resources", "jobs", "job0", "name"), }, }, }, diff --git a/bundle/config/mutator/reject_internal_resources.go b/bundle/config/mutator/reject_internal_resources.go index 2eacbbc7f9c..0e4e28e53b5 100644 --- a/bundle/config/mutator/reject_internal_resources.go +++ b/bundle/config/mutator/reject_internal_resources.go @@ -26,7 +26,7 @@ func (m *rejectInternalResources) Apply(ctx context.Context, b *bundle.Bundle) d diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "Internal resources cannot be set in bundle configuration", - Paths: structpath.NewStringKeysSlice("resources"), + Paths: structpath.NewPathSlice("resources"), Locations: b.Config.GetLocations("resources"), }) } diff --git a/bundle/config/mutator/resourcemutator/run_as.go b/bundle/config/mutator/resourcemutator/run_as.go index e89f2ee3c4e..f93c16b3889 100644 --- a/bundle/config/mutator/resourcemutator/run_as.go +++ b/bundle/config/mutator/resourcemutator/run_as.go @@ -239,7 +239,7 @@ func (m *setRunAs) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { { Severity: diag.Warning, Summary: "You are using the legacy mode of run_as. The support for this mode is experimental and might be removed in a future release of the CLI. In order to run the pipelines in your DABs project as the run_as user this mode changes the owners of the pipelines to the run_as identity, which requires the user deploying the bundle to be a workspace admin, and also a Metastore admin if the pipeline target is in UC.", - Paths: structpath.NewStringKeysSlice("experimental", "use_legacy_run_as"), + Paths: structpath.NewPathSlice("experimental", "use_legacy_run_as"), Locations: b.Config.GetLocations("experimental.use_legacy_run_as"), }, } diff --git a/bundle/config/mutator/resourcemutator/secret_scope_fixups.go b/bundle/config/mutator/resourcemutator/secret_scope_fixups.go index e4ca3767f03..be2a2e0775a 100644 --- a/bundle/config/mutator/resourcemutator/secret_scope_fixups.go +++ b/bundle/config/mutator/resourcemutator/secret_scope_fixups.go @@ -139,7 +139,7 @@ func (m *secretScopeFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Di Severity: diag.Error, Summary: "Failed to collapse permissions for secret scope", Detail: err.Error(), - Paths: structpath.NewStringKeysSlice("resources", "secret_scopes", key), + Paths: structpath.NewPathSlice("resources", "secret_scopes", key), Locations: []dyn.Location{b.Config.GetLocation("resources.secret_scopes." + key)}, }, } diff --git a/bundle/config/mutator/sync_infer_root.go b/bundle/config/mutator/sync_infer_root.go index 07cd74fee26..184ec289f33 100644 --- a/bundle/config/mutator/sync_infer_root.go +++ b/bundle/config/mutator/sync_infer_root.go @@ -94,7 +94,7 @@ func (m *syncInferRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno Severity: diag.Error, Summary: fmt.Sprintf("invalid sync path %q", path), Locations: b.Config.GetLocations(fmt.Sprintf("sync.paths[%d]", i)), - Paths: []*structpath.PathNode{structpath.NewIndex(structpath.NewStringKeys(nil, "sync", "paths"), i)}, + Paths: structpath.NewPathSlice("sync", "paths", i), }) } diff --git a/bundle/config/validate/files_to_sync.go b/bundle/config/validate/files_to_sync.go index a3104974b9c..48b66eb5c59 100644 --- a/bundle/config/validate/files_to_sync.go +++ b/bundle/config/validate/files_to_sync.go @@ -63,7 +63,7 @@ func (v *filesToSync) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost // Show all locations where sync.exclude is defined, since merging // sync.exclude is additive. Locations: b.Config.GetLocations(path), - Paths: structpath.NewStringKeysSlice("sync", "exclude"), + Paths: structpath.NewPathSlice("sync", "exclude"), }) } diff --git a/bundle/config/validate/job_cluster_key_defined.go b/bundle/config/validate/job_cluster_key_defined.go index 0aef74be60f..e29e72c1f32 100644 --- a/bundle/config/validate/job_cluster_key_defined.go +++ b/bundle/config/validate/job_cluster_key_defined.go @@ -33,13 +33,13 @@ func (v *jobClusterKeyDefined) Apply(ctx context.Context, b *bundle.Bundle) diag for index, task := range job.Tasks { diags = diags.Extend(checkJobClusterKey(b, jobClusterKeys, task.JobClusterKey, fmt.Sprintf("resources.jobs.%s.tasks[%d].job_cluster_key", k, index), - []*structpath.PathNode{structpath.NewStringKey(structpath.NewIndex(structpath.NewStringKeys(nil, "resources", "jobs", k, "tasks"), index), "job_cluster_key")})) + structpath.NewPathSlice("resources", "jobs", k, "tasks", index, "job_cluster_key"))) // The Jobs API rejects nested for_each_task, so one level is sufficient. if task.ForEachTask != nil { diags = diags.Extend(checkJobClusterKey(b, jobClusterKeys, task.ForEachTask.Task.JobClusterKey, fmt.Sprintf("resources.jobs.%s.tasks[%d].for_each_task.task.job_cluster_key", k, index), - []*structpath.PathNode{structpath.NewStringKeys(structpath.NewIndex(structpath.NewStringKeys(nil, "resources", "jobs", k, "tasks"), index), "for_each_task", "task", "job_cluster_key")})) + structpath.NewPathSlice("resources", "jobs", k, "tasks", index, "for_each_task", "task", "job_cluster_key"))) } } } diff --git a/bundle/config/validate/no_interpolation_in_bundle_name.go b/bundle/config/validate/no_interpolation_in_bundle_name.go index 1b81655af9c..8fc547b174e 100644 --- a/bundle/config/validate/no_interpolation_in_bundle_name.go +++ b/bundle/config/validate/no_interpolation_in_bundle_name.go @@ -29,7 +29,7 @@ is a part of the path at which your bundle state is stored at by default. Parame runtime can have unexpected consequences like duplicate deployments or resources not being cleaned up during bundle destroy.`, Locations: b.Config.GetLocations("bundle.name"), - Paths: structpath.NewStringKeysSlice("bundle", "name"), + Paths: structpath.NewPathSlice("bundle", "name"), }) } diff --git a/bundle/config/validate/required.go b/bundle/config/validate/required.go index 84c5a734c82..be766fc588c 100644 --- a/bundle/config/validate/required.go +++ b/bundle/config/validate/required.go @@ -106,11 +106,11 @@ func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnosti for key, dashboard := range b.Config.Resources.Dashboards { if dashboard.DisplayName == "" { nameLocations = append(nameLocations, b.Config.GetLocations("resources.dashboards."+key)...) - namePaths = append(namePaths, structpath.NewStringKeys(nil, "resources", "dashboards", key)) + namePaths = append(namePaths, structpath.NewPath(nil, "resources", "dashboards", key)) } if dashboard.WarehouseId == "" { warehouseIdLocations = append(warehouseIdLocations, b.Config.GetLocations("resources.dashboards."+key)...) - warehouseIdPaths = append(warehouseIdPaths, structpath.NewStringKeys(nil, "resources", "dashboards", key)) + warehouseIdPaths = append(warehouseIdPaths, structpath.NewPath(nil, "resources", "dashboards", key)) } } @@ -140,7 +140,7 @@ func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnosti Severity: diag.Error, Summary: "sql_warehouse name is required", Locations: b.Config.GetLocations(path), - Paths: structpath.NewStringKeysSlice("resources", "sql_warehouses", key), + Paths: structpath.NewPathSlice("resources", "sql_warehouses", key), }) } } @@ -207,7 +207,7 @@ func errorForInvalidSecretScopePermissions(ctx context.Context, b *bundle.Bundle Summary: "secret scope permission principal is required", Detail: "Set one of user_name, group_name or service_principal_name", Locations: b.Config.GetLocations("resources.secret_scopes." + key), - Paths: []*structpath.PathNode{structpath.NewIndex(structpath.NewStringKeys(nil, "resources", "secret_scopes", key, "permissions"), i)}, + Paths: structpath.NewPathSlice("resources", "secret_scopes", key, "permissions", i), }) } } diff --git a/bundle/config/validate/single_node_cluster_test.go b/bundle/config/validate/single_node_cluster_test.go index 463991f5539..f4494ae259c 100644 --- a/bundle/config/validate/single_node_cluster_test.go +++ b/bundle/config/validate/single_node_cluster_test.go @@ -123,7 +123,7 @@ func TestValidateSingleNodeClusterFailForInteractiveClusters(t *testing.T) { Summary: singleNodeWarningSummary, Detail: singleNodeWarningDetail, Locations: []dyn.Location{{File: "a.yml", Line: 1, Column: 1}}, - Paths: structpath.NewStringKeysSlice("resources", "clusters", "foo"), + Paths: structpath.NewPathSlice("resources", "clusters", "foo"), }, }, diags) }) diff --git a/bundle/config/validate/validate_artifact_path.go b/bundle/config/validate/validate_artifact_path.go index ff6c893ccbd..a8eac19f3d4 100644 --- a/bundle/config/validate/validate_artifact_path.go +++ b/bundle/config/validate/validate_artifact_path.go @@ -87,7 +87,7 @@ func (v *validateArtifactPath) Apply(ctx context.Context, b *bundle.Bundle) diag Summary: s, Severity: diag.Error, Locations: b.Config.GetLocations("workspace.artifact_path"), - Paths: structpath.NewStringKeysSlice("workspace", "artifact_path"), + Paths: structpath.NewPathSlice("workspace", "artifact_path"), }, } } @@ -119,7 +119,7 @@ this bundle but which has not been deployed yet. Please first deploy the volume using 'bundle deploy' and then switch over to using it in the artifact_path.`, Locations: slices.Concat(b.Config.GetLocations("workspace.artifact_path"), locations), - Paths: append(structpath.NewStringKeysSlice("workspace", "artifact_path"), dyn.ToStructPath(path)), + Paths: append(structpath.NewPathSlice("workspace", "artifact_path"), dyn.ToStructPath(path)), }} } diff --git a/bundle/config/validate/validate_artifact_path_test.go b/bundle/config/validate/validate_artifact_path_test.go index a88e7213953..4f68f645990 100644 --- a/bundle/config/validate/validate_artifact_path_test.go +++ b/bundle/config/validate/validate_artifact_path_test.go @@ -84,7 +84,7 @@ func TestValidateArtifactPath(t *testing.T) { Severity: diag.Error, Summary: expected, Locations: []dyn.Location{{File: "file", Line: 1, Column: 1}}, - Paths: structpath.NewStringKeysSlice("workspace", "artifact_path"), + Paths: structpath.NewPathSlice("workspace", "artifact_path"), }}, diags) } @@ -171,7 +171,7 @@ func TestValidateArtifactPathWithInvalidPaths(t *testing.T) { Severity: diag.Error, Summary: "expected UC volume path to be in the format /Volumes////..., got " + p, Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}, - Paths: structpath.NewStringKeysSlice("workspace", "artifact_path"), + Paths: structpath.NewPathSlice("workspace", "artifact_path"), }}, diags) } } diff --git a/bundle/config/validate/validate_dashboard_etags.go b/bundle/config/validate/validate_dashboard_etags.go index d47c09a420d..3a93a07d553 100644 --- a/bundle/config/validate/validate_dashboard_etags.go +++ b/bundle/config/validate/validate_dashboard_etags.go @@ -27,7 +27,7 @@ func (v *validateDashboardEtags) Apply(ctx context.Context, b *bundle.Bundle) di { Severity: diag.Error, Summary: fmt.Sprintf("dashboard %q has an etag set. Etags must not be set in bundle configuration", dashboard.DisplayName), - Paths: structpath.NewStringKeysSlice("resources", "dashboards", k), + Paths: structpath.NewPathSlice("resources", "dashboards", k), Locations: b.Config.GetLocations("resources.dashboards." + k), }, } diff --git a/bundle/config/validate/validate_deployment_fields.go b/bundle/config/validate/validate_deployment_fields.go index 71a15991158..720efc78512 100644 --- a/bundle/config/validate/validate_deployment_fields.go +++ b/bundle/config/validate/validate_deployment_fields.go @@ -34,7 +34,7 @@ func (v *validateDeploymentFields) Apply(_ context.Context, b *bundle.Bundle) di diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: field + " must not be set in bundle configuration; it is managed by Declarative Automation Bundles", - Paths: structpath.NewStringKeysSlice("resources", resourceType, name, "deployment", field), + Paths: structpath.NewPathSlice("resources", resourceType, name, "deployment", field), Locations: b.Config.GetLocations(path), }) } diff --git a/bundle/config/validate/validate_engine.go b/bundle/config/validate/validate_engine.go index 832d1aa330b..8f6020772ed 100644 --- a/bundle/config/validate/validate_engine.go +++ b/bundle/config/validate/validate_engine.go @@ -31,7 +31,7 @@ func (v *validateEngine) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos Severity: diag.Warning, Summary: "bundle.terraform is deprecated and has no effect: " + engine.TerraformRemovedSummary, Locations: tf.Locations(), - Paths: structpath.NewStringKeysSlice("bundle", "terraform"), + Paths: structpath.NewPathSlice("bundle", "terraform"), }) } diff --git a/bundle/config/validate/validate_genie_space_etags.go b/bundle/config/validate/validate_genie_space_etags.go index c0d0c601817..2c54e762f01 100644 --- a/bundle/config/validate/validate_genie_space_etags.go +++ b/bundle/config/validate/validate_genie_space_etags.go @@ -29,7 +29,7 @@ func (v *validateGenieSpaceEtags) Apply(ctx context.Context, b *bundle.Bundle) d { Severity: diag.Error, Summary: fmt.Sprintf("genie space %q has an etag set. Etags must not be set in bundle configuration", genieSpace.Title), - Paths: structpath.NewStringKeysSlice("resources", "genie_spaces", k), + Paths: structpath.NewPathSlice("resources", "genie_spaces", k), Locations: b.Config.GetLocations("resources.genie_spaces." + k), }, } diff --git a/bundle/config/validate/validate_job_run_idempotency_token.go b/bundle/config/validate/validate_job_run_idempotency_token.go index cd5b42a0c25..1405028c8cf 100644 --- a/bundle/config/validate/validate_job_run_idempotency_token.go +++ b/bundle/config/validate/validate_job_run_idempotency_token.go @@ -36,7 +36,7 @@ func (v *validateJobRunIdempotencyToken) Apply(_ context.Context, b *bundle.Bund diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: "idempotency_token must not be set in bundle configuration; the CLI sets it on each run-now request", - Paths: structpath.NewStringKeysSlice("resources", "job_runs", name, "idempotency_token"), + Paths: structpath.NewPathSlice("resources", "job_runs", name, "idempotency_token"), Locations: b.Config.GetLocations(path), }) } diff --git a/bundle/config/validate/validate_sync_patterns.go b/bundle/config/validate/validate_sync_patterns.go index 4ecb3665c84..f802b996f57 100644 --- a/bundle/config/validate/validate_sync_patterns.go +++ b/bundle/config/validate/validate_sync_patterns.go @@ -62,7 +62,7 @@ func checkPatterns(ctx context.Context, patterns []string, field string, b *bund Severity: diag.Warning, Summary: fmt.Sprintf("Pattern %s does not match any files", pattern), Locations: b.Config.GetLocations(path), - Paths: []*structpath.PathNode{structpath.NewIndex(structpath.NewStringKeys(nil, "sync", field), index)}, + Paths: structpath.NewPathSlice("sync", field, index), }) } return nil diff --git a/bundle/render/render_text_output_test.go b/bundle/render/render_text_output_test.go index 93eb8e84125..3857803a406 100644 --- a/bundle/render/render_text_output_test.go +++ b/bundle/render/render_text_output_test.go @@ -204,7 +204,7 @@ func TestRenderDiagnostics(t *testing.T) { Severity: diag.Error, Detail: "'name' is required", Summary: "failed to load xxx", - Paths: structpath.NewStringKeysSlice("resources", "jobs", "xxx"), + Paths: structpath.NewPathSlice("resources", "jobs", "xxx"), }, }, expected: "Error: failed to load xxx\n" + diff --git a/bundle/tests/validate_test.go b/bundle/tests/validate_test.go index e552a79226e..449f2bd37c0 100644 --- a/bundle/tests/validate_test.go +++ b/bundle/tests/validate_test.go @@ -79,7 +79,7 @@ func TestValidateUniqueResourceIdentifiers(t *testing.T) { {File: "validate/duplicate_resource_name_in_subconfiguration_job_and_job/databricks.yml", Line: 13, Column: 7}, {File: "validate/duplicate_resource_name_in_subconfiguration_job_and_job/resources.yml", Line: 4, Column: 7}, }, - Paths: structpath.NewStringKeysSlice("resources", "jobs", "foo"), + Paths: structpath.NewPathSlice("resources", "jobs", "foo"), }, }, }, diff --git a/libs/dyn/convert/normalize_test.go b/libs/dyn/convert/normalize_test.go index e7a1d0e7563..b2bed72db71 100644 --- a/libs/dyn/convert/normalize_test.go +++ b/libs/dyn/convert/normalize_test.go @@ -45,7 +45,7 @@ func TestNormalizeStructElementDiagnostic(t *testing.T) { Severity: diag.Warning, Summary: `expected string, found map`, Locations: []dyn.Location{{}}, - Paths: structpath.NewStringKeysSlice("bar"), + Paths: structpath.NewPathSlice("bar"), }, diags[0]) // Elements that encounter an error during normalization are dropped. @@ -307,7 +307,7 @@ func TestNormalizeMapElementDiagnostic(t *testing.T) { Severity: diag.Warning, Summary: `expected string, found map`, Locations: []dyn.Location{{}}, - Paths: structpath.NewStringKeysSlice("bar"), + Paths: structpath.NewPathSlice("bar"), }, err[0]) // Elements that encounter an error during normalization are dropped. diff --git a/libs/flags/json_flag_test.go b/libs/flags/json_flag_test.go index 4b61f619a23..1e26d4b3d57 100644 --- a/libs/flags/json_flag_test.go +++ b/libs/flags/json_flag_test.go @@ -260,7 +260,7 @@ func TestJsonUnmarshalWrongTypeReportsCorrectLocation(t *testing.T) { Column: 15, }, }, - Paths: structpath.NewStringKeysSlice("job_id"), + Paths: structpath.NewPathSlice("job_id"), }) } @@ -285,7 +285,7 @@ func TestJsonUnmarshalArrayInsteadOfIntReportsCorrectLocation(t *testing.T) { Column: 40, }, }, - Paths: structpath.NewStringKeysSlice("new_settings", "timeout_seconds"), + Paths: structpath.NewPathSlice("new_settings", "timeout_seconds"), }) } diff --git a/libs/structs/structpath/path.go b/libs/structs/structpath/path.go index 282a093e964..266d9ade1d2 100644 --- a/libs/structs/structpath/path.go +++ b/libs/structs/structpath/path.go @@ -169,18 +169,26 @@ func NewStringKey(prev *PathNode, fieldName string) *PathNode { return NewBracketString(prev, fieldName) } -// NewStringKeys appends the keys to prev, each like [NewStringKey]. -func NewStringKeys(prev *PathNode, keys ...string) *PathNode { - for _, k := range keys { - prev = NewStringKey(prev, k) +// NewPath appends parts to prev: a string is a key (see [NewStringKey]) and an int is +// an index. Other types panic. +func NewPath(prev *PathNode, parts ...any) *PathNode { + for _, part := range parts { + switch v := part.(type) { + case string: + prev = NewStringKey(prev, v) + case int: + prev = NewIndex(prev, v) + default: + panic(fmt.Sprintf("structpath.NewPath: unsupported part %#v", part)) + } } return prev } -// NewStringKeysSlice returns the path of keys from the root as a one-element slice, -// e.g. for diag.Diagnostic.Paths. -func NewStringKeysSlice(keys ...string) []*PathNode { - return []*PathNode{NewStringKeys(nil, keys...)} +// NewPathSlice returns the path of parts from the root (see [NewPath]) as a one-element +// slice, e.g. for diag.Diagnostic.Paths. +func NewPathSlice(parts ...any) []*PathNode { + return []*PathNode{NewPath(nil, parts...)} } func NewKeyValue(prev *PathNode, key, value string) *PathNode { diff --git a/libs/structs/structpath/path_test.go b/libs/structs/structpath/path_test.go index 924cd97a1c4..40fc0282c2b 100644 --- a/libs/structs/structpath/path_test.go +++ b/libs/structs/structpath/path_test.go @@ -1283,14 +1283,17 @@ func TestMustParsePaths(t *testing.T) { assert.Empty(t, MustParsePaths()) } -func TestNewStringKeys(t *testing.T) { - assert.Equal(t, "a.b['c d']", NewStringKeys(NewStringKey(nil, "a"), "b", "c d").String()) - assert.Nil(t, NewStringKeys(nil)) - assert.Equal(t, "a", NewStringKeys(NewStringKey(nil, "a")).String()) +func TestNewPath(t *testing.T) { + assert.Equal(t, "a.b['c d']", NewPath(NewStringKey(nil, "a"), "b", "c d").String()) + assert.Equal(t, "resources.jobs.j.tasks[2].job_cluster_key", NewPath(nil, "resources", "jobs", "j", "tasks", 2, "job_cluster_key").String()) + assert.Nil(t, NewPath(nil)) + assert.Equal(t, "a", NewPath(NewStringKey(nil, "a")).String()) + assert.Panics(t, func() { NewPath(nil, int64(1)) }) } -func TestNewStringKeysSlice(t *testing.T) { - paths := NewStringKeysSlice("resources", "jobs", "${var.env}_job") +func TestNewPathSlice(t *testing.T) { + paths := NewPathSlice("resources", "jobs", "${var.env}_job") require.Len(t, paths, 1) assert.Equal(t, "resources.jobs['${var.env}_job']", paths[0].String()) + assert.Equal(t, "sync.paths[3]", NewPathSlice("sync", "paths", 3)[0].String()) } From 5feed50981374becbfa8cc8b2455103e2538f0c9 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 8 Oct 2026 09:49:52 +0200 Subject: [PATCH 08/12] Build sync pattern paths from parts; drop unneeded clones ToStructPaths copies the walked path, so it no longer needs cloning. Co-authored-by: Isaac --- bundle/config/validate/required.go | 8 +++----- bundle/config/validate/validate_sync_patterns.go | 12 ++++++------ 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/bundle/config/validate/required.go b/bundle/config/validate/required.go index be766fc588c..3032eba8073 100644 --- a/bundle/config/validate/required.go +++ b/bundle/config/validate/required.go @@ -50,8 +50,6 @@ func warnForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnostic return nil } - cloneP := slices.Clone(p) - fields := generated.RequiredFields[pattern.String()] for _, field := range fields { vv := v.Get(field) @@ -60,7 +58,7 @@ func warnForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnostic Severity: diag.Warning, Summary: fmt.Sprintf("required field %q is not set", field), Locations: v.Locations(), - Paths: dyn.ToStructPaths(cloneP), + Paths: dyn.ToStructPaths(p), }) } } @@ -167,7 +165,7 @@ func errorForInvalidGrants(ctx context.Context, b *bundle.Bundle) diag.Diagnosti Severity: diag.Error, Summary: "grant principal is required", Locations: v.Locations(), - Paths: dyn.ToStructPaths(slices.Clone(p)), + Paths: dyn.ToStructPaths(p), }) } if isMissingOrEmptySequence(v.Get("privileges")) { @@ -175,7 +173,7 @@ func errorForInvalidGrants(ctx context.Context, b *bundle.Bundle) diag.Diagnosti Severity: diag.Error, Summary: "grant privileges is required", Locations: v.Locations(), - Paths: dyn.ToStructPaths(slices.Clone(p)), + Paths: dyn.ToStructPaths(p), }) } return v, nil diff --git a/bundle/config/validate/validate_sync_patterns.go b/bundle/config/validate/validate_sync_patterns.go index f802b996f57..44218878424 100644 --- a/bundle/config/validate/validate_sync_patterns.go +++ b/bundle/config/validate/validate_sync_patterns.go @@ -26,16 +26,16 @@ func (v *validateSyncPatterns) Name() string { func (v *validateSyncPatterns) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { s := b.Config.Sync - checkPatterns(ctx, s.Exclude, "exclude", b) + checkPatterns(ctx, s.Exclude, structpath.NewPath(nil, "sync", "exclude"), b) if logdiag.HasError(ctx) { return nil } - checkPatterns(ctx, s.Include, "include", b) + checkPatterns(ctx, s.Include, structpath.NewPath(nil, "sync", "include"), b) return nil } -func checkPatterns(ctx context.Context, patterns []string, field string, b *bundle.Bundle) { +func checkPatterns(ctx context.Context, patterns []string, path *structpath.PathNode, b *bundle.Bundle) { var errs errgroup.Group for index, pattern := range patterns { @@ -57,12 +57,12 @@ func checkPatterns(ctx context.Context, patterns []string, field string, b *bund } if len(all) == 0 { - path := fmt.Sprintf("sync.%s[%d]", field, index) + path := structpath.NewPath(path, index) logdiag.LogDiag(ctx, diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("Pattern %s does not match any files", pattern), - Locations: b.Config.GetLocations(path), - Paths: structpath.NewPathSlice("sync", field, index), + Locations: b.Config.GetLocations(path.String()), + Paths: []*structpath.PathNode{path}, }) } return nil From 6377bff734cd79dd84a6d8dd90e37474557fd663 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 8 Oct 2026 13:24:37 +0200 Subject: [PATCH 09/12] Build source-linked deployment preset paths with structpath Co-authored-by: Isaac --- .../apply_source_linked_deployment_preset.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/bundle/config/mutator/apply_source_linked_deployment_preset.go b/bundle/config/mutator/apply_source_linked_deployment_preset.go index 74293566779..d0eadbedd17 100644 --- a/bundle/config/mutator/apply_source_linked_deployment_preset.go +++ b/bundle/config/mutator/apply_source_linked_deployment_preset.go @@ -8,7 +8,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/dbr" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) type applySourceLinkedDeploymentPreset struct{} @@ -33,13 +33,13 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle if config.IsExplicitlyEnabled((b.Config.Presets.SourceLinkedDeployment)) { if !isDatabricksWorkspace { - path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment")) + path := structpath.NewPath(nil, "targets", target, "presets", "source_linked_deployment") diags = diags.Append( diag.Diagnostic{ Severity: diag.Warning, Summary: "source-linked deployment is available only in the Databricks Workspace", - Paths: dyn.ToStructPaths(path), - Locations: b.Config.GetLocations(path[2:].String()), + Paths: []*structpath.PathNode{path}, + Locations: b.Config.GetLocations(path.SkipPrefix(2).String()), }, ) @@ -51,13 +51,13 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle b.Metrics.AddBoolValue("source_linked_set_for_non_development", b.Config.Bundle.Mode != config.Development) if b.Config.Bundle.Mode != config.Development { - path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment")) + path := structpath.NewPath(nil, "targets", target, "presets", "source_linked_deployment") diags = diags.Append( diag.Diagnostic{ Severity: diag.Warning, Summary: "source-linked deployment in non-development mode is deprecated and will not be supported in a future release", - Paths: dyn.ToStructPaths(path), - Locations: b.Config.GetLocations(path[2:].String()), + Paths: []*structpath.PathNode{path}, + Locations: b.Config.GetLocations(path.SkipPrefix(2).String()), }, ) } @@ -70,13 +70,13 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle // This mutator runs before workspace paths are defaulted so it's safe to check for the user-defined value if b.Config.Workspace.FilePath != "" && config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) { - path := dyn.NewPath(dyn.Key("workspace"), dyn.Key("file_path")) + path := structpath.NewPath(nil, "workspace", "file_path") diags = diags.Append( diag.Diagnostic{ Severity: diag.Warning, Summary: "workspace.file_path setting will be ignored in source-linked deployment mode", Detail: "In source-linked deployment files are not copied to the destination and resources use source files instead", - Paths: dyn.ToStructPaths(path), + Paths: []*structpath.PathNode{path}, Locations: b.Config.GetLocations(path.String()), }, ) From 89328f95eecca78305b79bd9f44f7f542959d28d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 8 Oct 2026 13:28:24 +0200 Subject: [PATCH 10/12] Build dashboard diagnostic paths from parts; drop more unneeded clones Co-authored-by: Isaac --- bundle/config/validate/all_resources_have_values.go | 3 +-- bundle/config/validate/enum.go | 3 +-- bundle/deploy/check_dashboards_modified_remotely.go | 9 +++++---- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/bundle/config/validate/all_resources_have_values.go b/bundle/config/validate/all_resources_have_values.go index ae5a4ba8bca..713ec0f83af 100644 --- a/bundle/config/validate/all_resources_have_values.go +++ b/bundle/config/validate/all_resources_have_values.go @@ -3,7 +3,6 @@ package validate import ( "context" "fmt" - "slices" "strings" "github.com/databricks/cli/bundle" @@ -43,7 +42,7 @@ func (m *allResourcesHaveValues) Apply(ctx context.Context, b *bundle.Bundle) di Severity: diag.Error, Summary: fmt.Sprintf("%s %s is not defined", rType, rName), Locations: v.Locations(), - Paths: dyn.ToStructPaths(slices.Clone(p)), + Paths: dyn.ToStructPaths(p), }) return v, nil diff --git a/bundle/config/validate/enum.go b/bundle/config/validate/enum.go index 5f046ac7c68..2ce99ea1d19 100644 --- a/bundle/config/validate/enum.go +++ b/bundle/config/validate/enum.go @@ -69,13 +69,12 @@ func (f *enum) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { if !validValue { // p is a slice of path components. We need to clone it before using it in diagnostics // since the WalkReadOnly function will mutate it while walking the config tree. - cloneP := slices.Clone(p) diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("invalid value %q for enum field. Valid values are %v", strValue, validValues), Locations: v.Locations(), - Paths: dyn.ToStructPaths(cloneP), + Paths: dyn.ToStructPaths(p), }) } diff --git a/bundle/deploy/check_dashboards_modified_remotely.go b/bundle/deploy/check_dashboards_modified_remotely.go index 34fe393062c..d0ee26b6908 100644 --- a/bundle/deploy/check_dashboards_modified_remotely.go +++ b/bundle/deploy/check_dashboards_modified_remotely.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/libs/agent" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) type dashboardState struct { @@ -72,15 +73,15 @@ func (l *checkDashboardsModifiedRemotely) Apply(ctx context.Context, b *bundle.B continue } - path := dyn.MustPathFromString("resources.dashboards." + dashboard.Name) - loc := b.Config.GetLocation(path.String()) + path := structpath.NewPath(nil, "resources", "dashboards", dashboard.Name) + loc := b.Config.GetLocation("resources.dashboards." + dashboard.Name) actual, err := b.WorkspaceClient(ctx).Lakeview.GetByDashboardId(ctx, dashboard.ID) if err != nil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("failed to get dashboard %q", dashboard.Name), Detail: err.Error(), - Paths: dyn.ToStructPaths(path), + Paths: []*structpath.PathNode{path}, Locations: []dyn.Location{loc}, }) continue @@ -109,7 +110,7 @@ func (l *checkDashboardsModifiedRemotely) Apply(ctx context.Context, b *bundle.B "\n" + "To overwrite the remote changes with your local version, use --force.\n" + "The remote modifications will be lost." + agent.AgentNotice(), - Paths: dyn.ToStructPaths(path), + Paths: []*structpath.PathNode{path}, Locations: []dyn.Location{loc}, }) } From 05f4c0742dd47c500fd243ace34b9324c01172f0 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 8 Oct 2026 13:55:25 +0200 Subject: [PATCH 11/12] config: GetLocationsOf/GetLocationOf for path nodes Co-authored-by: Isaac --- bundle/apps/validate.go | 12 ++++---- .../apply_source_linked_deployment_preset.go | 6 ++-- .../config/mutator/resourcemutator/run_as.go | 2 +- .../resourcemutator/secret_scope_fixups.go | 2 +- bundle/config/mutator/sync_infer_root.go | 5 ++-- bundle/config/root.go | 28 +++++++++++++++++++ bundle/config/validate/files_to_sync.go | 6 ++-- .../validate/job_cluster_key_defined.go | 12 ++++---- .../config/validate/job_task_cluster_spec.go | 6 ++-- bundle/config/validate/required.go | 18 ++++++------ .../config/validate/validate_artifact_path.go | 3 +- .../validate/validate_dashboard_etags.go | 2 +- .../validate/validate_deployment_fields.go | 6 ++-- .../validate/validate_genie_space_etags.go | 2 +- .../validate_job_run_idempotency_token.go | 6 ++-- .../config/validate/validate_sync_patterns.go | 2 +- .../config/validate/validate_volume_path.go | 13 +++++---- .../check_dashboards_modified_remotely.go | 2 +- libs/dyn/structpath.go | 16 +++++++++++ libs/dyn/structpath_test.go | 10 +++++++ 20 files changed, 108 insertions(+), 51 deletions(-) diff --git a/bundle/apps/validate.go b/bundle/apps/validate.go index 29f6f056a53..2ab3384f991 100644 --- a/bundle/apps/validate.go +++ b/bundle/apps/validate.go @@ -28,7 +28,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics Severity: diag.Error, Summary: "Missing app source code path or git source", Detail: fmt.Sprintf("app resource '%s' should have either source_code_path or git_source field", key), - Locations: b.Config.GetLocations("resources.apps." + key), + Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "apps", key)), }) continue } @@ -38,7 +38,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics Severity: diag.Error, Summary: "Both source_code_path and git_source fields are set", Detail: fmt.Sprintf("app resource '%s' should have either source_code_path or git_source field, not both", key), - Locations: b.Config.GetLocations("resources.apps." + key), + Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "apps", key)), }) continue } @@ -48,7 +48,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics Severity: diag.Error, Summary: "Duplicate app source code path", Detail: fmt.Sprintf("app resource '%s' has the same source code path as app resource '%s', this will lead to the app configuration being overridden by each other", key, usedSourceCodePaths[app.SourceCodePath]), - Locations: b.Config.GetLocations(fmt.Sprintf("resources.apps.%s.source_code_path", key)), + Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "apps", key, "source_code_path")), }) } usedSourceCodePaths[app.SourceCodePath] = key @@ -155,7 +155,7 @@ func warnForAppResourcePermissions(b *bundle.Bundle, appKey string, app *resourc continue } - appPath := "resources.apps." + appKey + appPath := structpath.NewPath(nil, "resources", "apps", appKey) diags = append(diags, diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("app %q references %s %q which has permissions set. To prevent permission override after deploying the app, please add the app service principal to the %s permissions", appKey, refType, resourceKey, refType), @@ -173,8 +173,8 @@ func warnForAppResourcePermissions(b *bundle.Bundle, appKey string, app *resourc ref.permission, appKey, ), - Paths: structpath.NewPathSlice("resources", "apps", appKey), - Locations: b.Config.GetLocations(appPath), + Paths: []*structpath.PathNode{appPath}, + Locations: b.Config.GetLocationsOf(appPath), }) } diff --git a/bundle/config/mutator/apply_source_linked_deployment_preset.go b/bundle/config/mutator/apply_source_linked_deployment_preset.go index d0eadbedd17..638f3034140 100644 --- a/bundle/config/mutator/apply_source_linked_deployment_preset.go +++ b/bundle/config/mutator/apply_source_linked_deployment_preset.go @@ -39,7 +39,7 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle Severity: diag.Warning, Summary: "source-linked deployment is available only in the Databricks Workspace", Paths: []*structpath.PathNode{path}, - Locations: b.Config.GetLocations(path.SkipPrefix(2).String()), + Locations: b.Config.GetLocationsOf(path.SkipPrefix(2)), }, ) @@ -57,7 +57,7 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle Severity: diag.Warning, Summary: "source-linked deployment in non-development mode is deprecated and will not be supported in a future release", Paths: []*structpath.PathNode{path}, - Locations: b.Config.GetLocations(path.SkipPrefix(2).String()), + Locations: b.Config.GetLocationsOf(path.SkipPrefix(2)), }, ) } @@ -77,7 +77,7 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle Summary: "workspace.file_path setting will be ignored in source-linked deployment mode", Detail: "In source-linked deployment files are not copied to the destination and resources use source files instead", Paths: []*structpath.PathNode{path}, - Locations: b.Config.GetLocations(path.String()), + Locations: b.Config.GetLocationsOf(path), }, ) } diff --git a/bundle/config/mutator/resourcemutator/run_as.go b/bundle/config/mutator/resourcemutator/run_as.go index f93c16b3889..b36e915b372 100644 --- a/bundle/config/mutator/resourcemutator/run_as.go +++ b/bundle/config/mutator/resourcemutator/run_as.go @@ -109,7 +109,7 @@ func validateRunAs(b *bundle.Bundle) diag.Diagnostics { } diags = diags.Extend(reportRunAsNotSupported( "dashboards with embed_credentials set to true", - b.Config.GetLocation("resources.dashboards."+key), + b.Config.GetLocationOf(structpath.NewPath(nil, "resources", "dashboards", key)), b.Config.Workspace.CurrentUser.UserName, identity, )) diff --git a/bundle/config/mutator/resourcemutator/secret_scope_fixups.go b/bundle/config/mutator/resourcemutator/secret_scope_fixups.go index be2a2e0775a..0d5d541a6ee 100644 --- a/bundle/config/mutator/resourcemutator/secret_scope_fixups.go +++ b/bundle/config/mutator/resourcemutator/secret_scope_fixups.go @@ -140,7 +140,7 @@ func (m *secretScopeFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Di Summary: "Failed to collapse permissions for secret scope", Detail: err.Error(), Paths: structpath.NewPathSlice("resources", "secret_scopes", key), - Locations: []dyn.Location{b.Config.GetLocation("resources.secret_scopes." + key)}, + Locations: []dyn.Location{b.Config.GetLocationOf(structpath.NewPath(nil, "resources", "secret_scopes", key))}, }, } } diff --git a/bundle/config/mutator/sync_infer_root.go b/bundle/config/mutator/sync_infer_root.go index 184ec289f33..bd1a529380a 100644 --- a/bundle/config/mutator/sync_infer_root.go +++ b/bundle/config/mutator/sync_infer_root.go @@ -90,11 +90,12 @@ func (m *syncInferRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno continue } + syncPath := structpath.NewPath(nil, "sync", "paths", i) diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("invalid sync path %q", path), - Locations: b.Config.GetLocations(fmt.Sprintf("sync.paths[%d]", i)), - Paths: structpath.NewPathSlice("sync", "paths", i), + Locations: b.Config.GetLocationsOf(syncPath), + Paths: []*structpath.PathNode{syncPath}, }) } diff --git a/bundle/config/root.go b/bundle/config/root.go index e13bf78bf15..b87a2675692 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -18,6 +18,7 @@ import ( "github.com/databricks/cli/libs/dyn/merge" "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/jobs" ) @@ -605,6 +606,33 @@ func (r Root) GetLocations(path string) []dyn.Location { return v.Locations() } +// GetLocationOf is [Root.GetLocation] for a path node. +func (r Root) GetLocationOf(path *structpath.PathNode) dyn.Location { + v, ok := r.valueOf(path) + if !ok { + return dyn.Location{} + } + return v.Location() +} + +// GetLocationsOf is [Root.GetLocations] for a path node. +func (r Root) GetLocationsOf(path *structpath.PathNode) []dyn.Location { + v, ok := r.valueOf(path) + if !ok { + return nil + } + return v.Locations() +} + +func (r Root) valueOf(path *structpath.PathNode) (dyn.Value, bool) { + p, ok := dyn.FromStructPath(path) + if !ok { + return dyn.InvalidValue, false + } + v, err := dyn.GetByPath(r.value, p) + return v, err == nil +} + // GetNodeAndType and returns parent resource node and type of the resource in direct backend. // Examples: // diff --git a/bundle/config/validate/files_to_sync.go b/bundle/config/validate/files_to_sync.go index 48b66eb5c59..14b032e8da1 100644 --- a/bundle/config/validate/files_to_sync.go +++ b/bundle/config/validate/files_to_sync.go @@ -56,14 +56,14 @@ func (v *filesToSync) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost Summary: "There are no files to sync, please check your .gitignore", }) } else { - path := "sync.exclude" + path := structpath.NewPath(nil, "sync", "exclude") diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: "There are no files to sync, please check your .gitignore and sync.exclude configuration", // Show all locations where sync.exclude is defined, since merging // sync.exclude is additive. - Locations: b.Config.GetLocations(path), - Paths: structpath.NewPathSlice("sync", "exclude"), + Locations: b.Config.GetLocationsOf(path), + Paths: []*structpath.PathNode{path}, }) } diff --git a/bundle/config/validate/job_cluster_key_defined.go b/bundle/config/validate/job_cluster_key_defined.go index e29e72c1f32..eb9bea4df70 100644 --- a/bundle/config/validate/job_cluster_key_defined.go +++ b/bundle/config/validate/job_cluster_key_defined.go @@ -32,14 +32,12 @@ func (v *jobClusterKeyDefined) Apply(ctx context.Context, b *bundle.Bundle) diag for index, task := range job.Tasks { diags = diags.Extend(checkJobClusterKey(b, jobClusterKeys, task.JobClusterKey, - fmt.Sprintf("resources.jobs.%s.tasks[%d].job_cluster_key", k, index), - structpath.NewPathSlice("resources", "jobs", k, "tasks", index, "job_cluster_key"))) + structpath.NewPath(nil, "resources", "jobs", k, "tasks", index, "job_cluster_key"))) // The Jobs API rejects nested for_each_task, so one level is sufficient. if task.ForEachTask != nil { diags = diags.Extend(checkJobClusterKey(b, jobClusterKeys, task.ForEachTask.Task.JobClusterKey, - fmt.Sprintf("resources.jobs.%s.tasks[%d].for_each_task.task.job_cluster_key", k, index), - structpath.NewPathSlice("resources", "jobs", k, "tasks", index, "for_each_task", "task", "job_cluster_key"))) + structpath.NewPath(nil, "resources", "jobs", k, "tasks", index, "for_each_task", "task", "job_cluster_key"))) } } } @@ -48,7 +46,7 @@ func (v *jobClusterKeyDefined) Apply(ctx context.Context, b *bundle.Bundle) diag } // checkJobClusterKey warns if jobClusterKey is set but not defined in the job's job_clusters. -func checkJobClusterKey(b *bundle.Bundle, jobClusterKeys map[string]bool, jobClusterKey, path string, paths []*structpath.PathNode) diag.Diagnostics { +func checkJobClusterKey(b *bundle.Bundle, jobClusterKeys map[string]bool, jobClusterKey string, path *structpath.PathNode) diag.Diagnostics { if jobClusterKey == "" { return nil } @@ -62,7 +60,7 @@ func checkJobClusterKey(b *bundle.Bundle, jobClusterKeys map[string]bool, jobClu // Show only the location where the job_cluster_key is defined. // Other associated locations are not relevant since they are // overridden during merging. - Locations: b.Config.GetLocations(path), - Paths: paths, + Locations: b.Config.GetLocationsOf(path), + Paths: []*structpath.PathNode{path}, }} } diff --git a/bundle/config/validate/job_task_cluster_spec.go b/bundle/config/validate/job_task_cluster_spec.go index 14fa8918bfe..e387d83f6fb 100644 --- a/bundle/config/validate/job_task_cluster_spec.go +++ b/bundle/config/validate/job_task_cluster_spec.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/jobs" ) @@ -88,12 +89,13 @@ func validateJobTask(b *bundle.Bundle, task jobs.Task, taskPath dyn.Path) diag.D strings.Join(unspecified, ", "), ) + taskNode := dyn.ToStructPath(taskPath) diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "Missing required cluster or environment settings", Detail: detail, - Locations: b.Config.GetLocations(taskPath.String()), - Paths: dyn.ToStructPaths(taskPath), + Locations: b.Config.GetLocationsOf(taskNode), + Paths: []*structpath.PathNode{taskNode}, }) } } diff --git a/bundle/config/validate/required.go b/bundle/config/validate/required.go index 3032eba8073..dce5c0686c1 100644 --- a/bundle/config/validate/required.go +++ b/bundle/config/validate/required.go @@ -103,12 +103,14 @@ func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnosti diags := diag.Diagnostics{} for key, dashboard := range b.Config.Resources.Dashboards { if dashboard.DisplayName == "" { - nameLocations = append(nameLocations, b.Config.GetLocations("resources.dashboards."+key)...) - namePaths = append(namePaths, structpath.NewPath(nil, "resources", "dashboards", key)) + path := structpath.NewPath(nil, "resources", "dashboards", key) + nameLocations = append(nameLocations, b.Config.GetLocationsOf(path)...) + namePaths = append(namePaths, path) } if dashboard.WarehouseId == "" { - warehouseIdLocations = append(warehouseIdLocations, b.Config.GetLocations("resources.dashboards."+key)...) - warehouseIdPaths = append(warehouseIdPaths, structpath.NewPath(nil, "resources", "dashboards", key)) + path := structpath.NewPath(nil, "resources", "dashboards", key) + warehouseIdLocations = append(warehouseIdLocations, b.Config.GetLocationsOf(path)...) + warehouseIdPaths = append(warehouseIdPaths, path) } } @@ -133,12 +135,12 @@ func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnosti // by the backend, which rejects whitespace-only names (name.trim.nonEmpty). for key, warehouse := range b.Config.Resources.SqlWarehouses { if strings.TrimSpace(warehouse.Name) == "" { - path := "resources.sql_warehouses." + key + path := structpath.NewPath(nil, "resources", "sql_warehouses", key) diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "sql_warehouse name is required", - Locations: b.Config.GetLocations(path), - Paths: structpath.NewPathSlice("resources", "sql_warehouses", key), + Locations: b.Config.GetLocationsOf(path), + Paths: []*structpath.PathNode{path}, }) } } @@ -204,7 +206,7 @@ func errorForInvalidSecretScopePermissions(ctx context.Context, b *bundle.Bundle Severity: diag.Error, Summary: "secret scope permission principal is required", Detail: "Set one of user_name, group_name or service_principal_name", - Locations: b.Config.GetLocations("resources.secret_scopes." + key), + Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "secret_scopes", key)), Paths: structpath.NewPathSlice("resources", "secret_scopes", key, "permissions", i), }) } diff --git a/bundle/config/validate/validate_artifact_path.go b/bundle/config/validate/validate_artifact_path.go index a8eac19f3d4..33e0a024c7a 100644 --- a/bundle/config/validate/validate_artifact_path.go +++ b/bundle/config/validate/validate_artifact_path.go @@ -69,8 +69,7 @@ func findVolumeInBundle(r config.Root, catalogName, schemaName, volumeName strin if v.SchemaName != schemaName && !isSchemaDefinedInBundle { continue } - pathString := "resources.volumes." + k - return dyn.MustPathFromString(pathString), r.GetLocations(pathString), true + return dyn.Path{dyn.Key("resources"), dyn.Key("volumes"), dyn.Key(k)}, r.GetLocationsOf(structpath.NewPath(nil, "resources", "volumes", k)), true } return nil, nil, false } diff --git a/bundle/config/validate/validate_dashboard_etags.go b/bundle/config/validate/validate_dashboard_etags.go index 3a93a07d553..1715f22cb84 100644 --- a/bundle/config/validate/validate_dashboard_etags.go +++ b/bundle/config/validate/validate_dashboard_etags.go @@ -28,7 +28,7 @@ func (v *validateDashboardEtags) Apply(ctx context.Context, b *bundle.Bundle) di Severity: diag.Error, Summary: fmt.Sprintf("dashboard %q has an etag set. Etags must not be set in bundle configuration", dashboard.DisplayName), Paths: structpath.NewPathSlice("resources", "dashboards", k), - Locations: b.Config.GetLocations("resources.dashboards." + k), + Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "dashboards", k)), }, } } diff --git a/bundle/config/validate/validate_deployment_fields.go b/bundle/config/validate/validate_deployment_fields.go index 720efc78512..d1ca337f4ed 100644 --- a/bundle/config/validate/validate_deployment_fields.go +++ b/bundle/config/validate/validate_deployment_fields.go @@ -30,12 +30,12 @@ func (v *validateDeploymentFields) Apply(_ context.Context, b *bundle.Bundle) di if value == "" { return } - path := "resources." + resourceType + "." + name + ".deployment." + field + path := structpath.NewPath(nil, "resources", resourceType, name, "deployment", field) diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: field + " must not be set in bundle configuration; it is managed by Declarative Automation Bundles", - Paths: structpath.NewPathSlice("resources", resourceType, name, "deployment", field), - Locations: b.Config.GetLocations(path), + Paths: []*structpath.PathNode{path}, + Locations: b.Config.GetLocationsOf(path), }) } diff --git a/bundle/config/validate/validate_genie_space_etags.go b/bundle/config/validate/validate_genie_space_etags.go index 2c54e762f01..963e572346f 100644 --- a/bundle/config/validate/validate_genie_space_etags.go +++ b/bundle/config/validate/validate_genie_space_etags.go @@ -30,7 +30,7 @@ func (v *validateGenieSpaceEtags) Apply(ctx context.Context, b *bundle.Bundle) d Severity: diag.Error, Summary: fmt.Sprintf("genie space %q has an etag set. Etags must not be set in bundle configuration", genieSpace.Title), Paths: structpath.NewPathSlice("resources", "genie_spaces", k), - Locations: b.Config.GetLocations("resources.genie_spaces." + k), + Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "genie_spaces", k)), }, } } diff --git a/bundle/config/validate/validate_job_run_idempotency_token.go b/bundle/config/validate/validate_job_run_idempotency_token.go index 1405028c8cf..61e07e302ca 100644 --- a/bundle/config/validate/validate_job_run_idempotency_token.go +++ b/bundle/config/validate/validate_job_run_idempotency_token.go @@ -32,12 +32,12 @@ func (v *validateJobRunIdempotencyToken) Apply(_ context.Context, b *bundle.Bund } // The CLI mints the token; a configured one would also remain reserved after // the run is deleted and break the next deploy. - path := "resources.job_runs." + name + ".idempotency_token" + path := structpath.NewPath(nil, "resources", "job_runs", name, "idempotency_token") diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: "idempotency_token must not be set in bundle configuration; the CLI sets it on each run-now request", - Paths: structpath.NewPathSlice("resources", "job_runs", name, "idempotency_token"), - Locations: b.Config.GetLocations(path), + Paths: []*structpath.PathNode{path}, + Locations: b.Config.GetLocationsOf(path), }) } diff --git a/bundle/config/validate/validate_sync_patterns.go b/bundle/config/validate/validate_sync_patterns.go index 44218878424..c888a05a7dd 100644 --- a/bundle/config/validate/validate_sync_patterns.go +++ b/bundle/config/validate/validate_sync_patterns.go @@ -61,7 +61,7 @@ func checkPatterns(ctx context.Context, patterns []string, path *structpath.Path logdiag.LogDiag(ctx, diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("Pattern %s does not match any files", pattern), - Locations: b.Config.GetLocations(path.String()), + Locations: b.Config.GetLocationsOf(path), Paths: []*structpath.PathNode{path}, }) } diff --git a/bundle/config/validate/validate_volume_path.go b/bundle/config/validate/validate_volume_path.go index 4bf6026821a..c440dfbf06f 100644 --- a/bundle/config/validate/validate_volume_path.go +++ b/bundle/config/validate/validate_volume_path.go @@ -24,11 +24,12 @@ func (m *validateVolumePath) Apply(ctx context.Context, b *bundle.Bundle) diag.D pathChecks := []struct { path string configName string + configPath *structpath.PathNode }{ - {b.Config.Workspace.RootPath, "workspace.root_path"}, - {b.Config.Workspace.FilePath, "workspace.file_path"}, - {b.Config.Workspace.StatePath, "workspace.state_path"}, - {b.Config.Workspace.ResourcePath, "workspace.resource_path"}, + {b.Config.Workspace.RootPath, "workspace.root_path", structpath.NewPath(nil, "workspace", "root_path")}, + {b.Config.Workspace.FilePath, "workspace.file_path", structpath.NewPath(nil, "workspace", "file_path")}, + {b.Config.Workspace.StatePath, "workspace.state_path", structpath.NewPath(nil, "workspace", "state_path")}, + {b.Config.Workspace.ResourcePath, "workspace.resource_path", structpath.NewPath(nil, "workspace", "resource_path")}, } // Check each path @@ -38,8 +39,8 @@ func (m *validateVolumePath) Apply(ctx context.Context, b *bundle.Bundle) diag.D Severity: diag.Error, Summary: fmt.Sprintf("%s %s starts with /Volumes. /Volumes can only be used with workspace.artifact_path.", check.configName, check.path), Detail: "For more information, see https://docs.databricks.com/aws/en/dev-tools/bundles/settings#workspace", - Locations: b.Config.GetLocations(check.configName), - Paths: structpath.MustParsePaths(check.configName), + Locations: b.Config.GetLocationsOf(check.configPath), + Paths: []*structpath.PathNode{check.configPath}, }) // Return early for root path validation diff --git a/bundle/deploy/check_dashboards_modified_remotely.go b/bundle/deploy/check_dashboards_modified_remotely.go index d0ee26b6908..f362f7967e8 100644 --- a/bundle/deploy/check_dashboards_modified_remotely.go +++ b/bundle/deploy/check_dashboards_modified_remotely.go @@ -74,7 +74,7 @@ func (l *checkDashboardsModifiedRemotely) Apply(ctx context.Context, b *bundle.B } path := structpath.NewPath(nil, "resources", "dashboards", dashboard.Name) - loc := b.Config.GetLocation("resources.dashboards." + dashboard.Name) + loc := b.Config.GetLocationOf(path) actual, err := b.WorkspaceClient(ctx).Lakeview.GetByDashboardId(ctx, dashboard.ID) if err != nil { diags = diags.Append(diag.Diagnostic{ diff --git a/libs/dyn/structpath.go b/libs/dyn/structpath.go index ca9afb5c7b7..1cb740ee0d7 100644 --- a/libs/dyn/structpath.go +++ b/libs/dyn/structpath.go @@ -23,3 +23,19 @@ func ToStructPaths(paths ...Path) []*structpath.PathNode { } return out } + +// FromStructPath converts a [structpath.PathNode] of keys and indices to a [Path]. +// It reports false for nodes a [Path] cannot represent (key-value selectors, wildcards). +func FromStructPath(p *structpath.PathNode) (Path, bool) { + var out Path + for _, n := range p.AsSlice() { + if k, ok := n.StringKey(); ok { + out = append(out, Key(k)) + } else if i, ok := n.Index(); ok { + out = append(out, Index(i)) + } else { + return nil, false + } + } + return out, true +} diff --git a/libs/dyn/structpath_test.go b/libs/dyn/structpath_test.go index 0e24f8ea51a..6eed07a31fa 100644 --- a/libs/dyn/structpath_test.go +++ b/libs/dyn/structpath_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" ) @@ -27,3 +28,12 @@ func TestToStructPaths(t *testing.T) { assert.Equal(t, "c[1]", paths[1].String()) assert.Empty(t, dyn.ToStructPaths()) } + +func TestFromStructPath(t *testing.T) { + p, ok := dyn.FromStructPath(structpath.NewPath(nil, "resources", "jobs", "a.b", "tasks", 2)) + assert.True(t, ok) + assert.Equal(t, dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"), dyn.Key("a.b"), dyn.Key("tasks"), dyn.Index(2)), p) + + _, ok = dyn.FromStructPath(structpath.MustParsePath("tasks[task_key='x']")) + assert.False(t, ok) +} From 6890bb20d69c89fe331073801fd0fb22bf45487f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 8 Oct 2026 18:15:41 +0200 Subject: [PATCH 12/12] Revert unrelated import reordering in annotations_file.go Co-authored-by: Isaac --- bundle/internal/schema/annotations_file.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bundle/internal/schema/annotations_file.go b/bundle/internal/schema/annotations_file.go index bd106b136b2..5c93610036a 100644 --- a/bundle/internal/schema/annotations_file.go +++ b/bundle/internal/schema/annotations_file.go @@ -8,12 +8,13 @@ import ( "slices" "strings" + yaml3 "go.yaml.in/yaml/v3" + "github.com/databricks/cli/bundle/internal/annotation" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/cli/libs/dyn/yamlsaver" - yaml3 "go.yaml.in/yaml/v3" ) // fieldsKey nests a type's block of field nodes inside the node of a field