diag: use structpath for diagnostic paths

.nextchanges/bundles/6980.md

@@ -0,0 +1 @@
+* Diagnostic paths now quote map keys that are not identifiers, e.g. `resources.jobs['${var.env}_job']`. ([#6980](https://github.com/databricks/cli/pull/6980))

acceptance/bundle/variables/variable_in_resource_key/output.txt

@@ -1,13 +1,13 @@
 Warning: unknown field: ${var.env}
-  at targets.dev.resources.jobs.${var.env}_job_2
+  at targets.dev.resources.jobs['${var.env}_job_2']
   in databricks.yml:21:11
 
 Error: resource key "${var.env}_job" must not contain variable references
-  at resources.jobs.${var.env}_job
+  at resources.jobs['${var.env}_job']
   in databricks.yml:12:7
 
 Error: resource key "${var.env}_job_2" must not contain variable references
-  at targets.dev.resources.jobs.${var.env}_job_2
+  at targets.dev.resources.jobs['${var.env}_job_2']
   in databricks.yml:20:11
 
 

bundle/apps/validate.go

@@ -9,6 +9,7 @@ import (
 	"github.com/databricks/cli/bundle/config/resources"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/structs/structpath"
 	"github.com/databricks/databricks-sdk-go/service/apps"
 )
 
@@ -27,7 +28,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
 				Severity:  diag.Error,
 				Summary:   "Missing app source code path or git source",
 				Detail:    fmt.Sprintf("app resource '%s' should have either source_code_path or git_source field", key),
-				Locations: b.Config.GetLocations("resources.apps." + key),
+				Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "apps", key)),
 			})
 			continue
 		}
@@ -37,7 +38,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
 				Severity:  diag.Error,
 				Summary:   "Both source_code_path and git_source fields are set",
 				Detail:    fmt.Sprintf("app resource '%s' should have either source_code_path or git_source field, not both", key),
-				Locations: b.Config.GetLocations("resources.apps." + key),
+				Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "apps", key)),
 			})
 			continue
 		}
@@ -47,7 +48,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
 				Severity:  diag.Error,
 				Summary:   "Duplicate app source code path",
 				Detail:    fmt.Sprintf("app resource '%s' has the same source code path as app resource '%s', this will lead to the app configuration being overridden by each other", key, usedSourceCodePaths[app.SourceCodePath]),
-				Locations: b.Config.GetLocations(fmt.Sprintf("resources.apps.%s.source_code_path", key)),
+				Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "apps", key, "source_code_path")),
 			})
 		}
 		usedSourceCodePaths[app.SourceCodePath] = key
@@ -154,7 +155,7 @@ func warnForAppResourcePermissions(b *bundle.Bundle, appKey string, app *resourc
 			continue
 		}
 
-		appPath := "resources.apps." + appKey
+		appPath := structpath.NewPath(nil, "resources", "apps", appKey)
 		diags = append(diags, diag.Diagnostic{
 			Severity: diag.Warning,
 			Summary:  fmt.Sprintf("app %q references %s %q which has permissions set. To prevent permission override after deploying the app, please add the app service principal to the %s permissions", appKey, refType, resourceKey, refType),
@@ -172,8 +173,8 @@ func warnForAppResourcePermissions(b *bundle.Bundle, appKey string, app *resourc
 				ref.permission,
 				appKey,
 			),
-			Paths:     []dyn.Path{dyn.MustPathFromString(appPath)},
-			Locations: b.Config.GetLocations(appPath),
+			Paths:     []*structpath.PathNode{appPath},
+			Locations: b.Config.GetLocationsOf(appPath),
 		})
 	}
 

bundle/apps/validate_test.go

@@ -213,7 +213,7 @@ func TestAppsValidateResourcePermissionsWarning(t *testing.T) {
 				require.Len(t, warnings, 1)
 				require.Contains(t, warnings[0].Summary, tc.wantSummary)
 				require.Contains(t, warnings[0].Detail, "service_principal_name: ${resources.apps.my_app.service_principal_client_id}")
-				require.Equal(t, dyn.MustPathFromString("resources.apps.my_app"), warnings[0].Paths[0])
+				require.Equal(t, "resources.apps.my_app", warnings[0].Paths[0].String())
 			} else {
 				require.Empty(t, warnings)
 			}

bundle/artifacts/expand_globs.go

@@ -27,7 +27,7 @@ func createGlobError(v dyn.Value, p dyn.Path, message string) diag.Diagnostic {
 		Severity:  diag.Error,
 		Summary:   fmt.Sprintf("%s: %s", source, message),
 		Locations: []dyn.Location{v.Location()},
-		Paths:     []dyn.Path{p},
+		Paths:     dyn.ToStructPaths(p),
 	}
 }
 

bundle/config/loader/process_include.go

@@ -11,6 +11,7 @@ import (
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/structs/structpath"
 )
 
 func validateFileFormat(configRoot dyn.Value, filePath string) diag.Diagnostics {
@@ -108,16 +109,16 @@ func validateSingleResourceDefined(configRoot dyn.Value, ext, typ string) diag.D
 	}
 
 	var locations []dyn.Location
-	var paths []dyn.Path
+	var paths []*structpath.PathNode
 	for _, rr := range resources {
 		locations = append(locations, rr.value.Locations()...)
-		paths = append(paths, rr.path)
+		paths = append(paths, dyn.ToStructPath(rr.path))
 	}
 	// Sort the locations and paths to make the output deterministic.
 	slices.SortFunc(locations, func(a, b dyn.Location) int {
 		return cmp.Compare(a.String(), b.String())
 	})
-	slices.SortFunc(paths, func(a, b dyn.Path) int {
+	slices.SortFunc(paths, func(a, b *structpath.PathNode) int {
 		return cmp.Compare(a.String(), b.String())
 	})
 
@@ -168,7 +169,7 @@ func (m *processInclude) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos
 			Detail: `An include section is defined in a file that is not databricks.yml.
 Only includes defined in databricks.yml are applied.`,
 			Locations: this.GetLocations("include"),
-			Paths:     []dyn.Path{dyn.MustPathFromString("include")},
+			Paths:     structpath.NewPathSlice("include"),
 		})
 	}
 

bundle/config/loader/process_include_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/databricks/cli/bundle/config/loader"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/structs/structpath"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -71,10 +72,10 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) {
 					{File: "testdata/format_not_match/single_job.pipeline.yaml", Line: 11, Column: 11},
 					{File: "testdata/format_not_match/single_job.pipeline.yaml", Line: 4, Column: 7},
 				},
-				Paths: []dyn.Path{
-					dyn.MustPathFromString("resources.jobs.job1"),
-					dyn.MustPathFromString("targets.target1.resources.jobs.job1"),
-				},
+				Paths: structpath.MustParsePaths(
+					"resources.jobs.job1",
+					"targets.target1.resources.jobs.job1",
+				),
 			},
 		},
 		"job_and_pipeline.job.yml": {
@@ -86,10 +87,10 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) {
 					{File: "testdata/format_not_match/job_and_pipeline.job.yml", Line: 11, Column: 11},
 					{File: "testdata/format_not_match/job_and_pipeline.job.yml", Line: 4, Column: 7},
 				},
-				Paths: []dyn.Path{
-					dyn.MustPathFromString("resources.pipelines.pipeline1"),
-					dyn.MustPathFromString("targets.target1.resources.jobs.job1"),
-				},
+				Paths: structpath.MustParsePaths(
+					"resources.pipelines.pipeline1",
+					"targets.target1.resources.jobs.job1",
+				),
 			},
 		},
 		"job_and_pipeline.experiment.yml": {
@@ -101,10 +102,10 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) {
 					{File: "testdata/format_not_match/job_and_pipeline.experiment.yml", Line: 11, Column: 11},
 					{File: "testdata/format_not_match/job_and_pipeline.experiment.yml", Line: 4, Column: 7},
 				},
-				Paths: []dyn.Path{
-					dyn.MustPathFromString("resources.pipelines.pipeline1"),
-					dyn.MustPathFromString("targets.target1.resources.jobs.job1"),
-				},
+				Paths: structpath.MustParsePaths(
+					"resources.pipelines.pipeline1",
+					"targets.target1.resources.jobs.job1",
+				),
 			},
 		},
 		"two_jobs.job.yml": {
@@ -116,10 +117,10 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) {
 					{File: "testdata/format_not_match/two_jobs.job.yml", Line: 4, Column: 7},
 					{File: "testdata/format_not_match/two_jobs.job.yml", Line: 7, Column: 7},
 				},
-				Paths: []dyn.Path{
-					dyn.MustPathFromString("resources.jobs.job1"),
-					dyn.MustPathFromString("resources.jobs.job2"),
-				},
+				Paths: structpath.MustParsePaths(
+					"resources.jobs.job1",
+					"resources.jobs.job2",
+				),
 			},
 		},
 		"second_job_in_target.job.yml": {
@@ -131,10 +132,10 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) {
 					{File: "testdata/format_not_match/second_job_in_target.job.yml", Line: 11, Column: 11},
 					{File: "testdata/format_not_match/second_job_in_target.job.yml", Line: 4, Column: 7},
 				},
-				Paths: []dyn.Path{
-					dyn.MustPathFromString("resources.jobs.job1"),
-					dyn.MustPathFromString("targets.target1.resources.jobs.job2"),
-				},
+				Paths: structpath.MustParsePaths(
+					"resources.jobs.job1",
+					"targets.target1.resources.jobs.job2",
+				),
 			},
 		},
 		"two_jobs_in_target.job.yml": {
@@ -146,10 +147,10 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) {
 					{File: "testdata/format_not_match/two_jobs_in_target.job.yml", Line: 6, Column: 11},
 					{File: "testdata/format_not_match/two_jobs_in_target.job.yml", Line: 8, Column: 11},
 				},
-				Paths: []dyn.Path{
-					dyn.MustPathFromString("targets.target1.resources.jobs.job1"),
-					dyn.MustPathFromString("targets.target1.resources.jobs.job2"),
-				},
+				Paths: structpath.MustParsePaths(
+					"targets.target1.resources.jobs.job1",
+					"targets.target1.resources.jobs.job2",
+				),
 			},
 		},
 		"multiple_resources.model_serving_endpoint.yml": {
@@ -182,19 +183,19 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) {
 					{File: "testdata/format_not_match/multiple_resources.model_serving_endpoint.yml", Line: 4, Column: 7},
 					{File: "testdata/format_not_match/multiple_resources.model_serving_endpoint.yml", Line: 8, Column: 7},
 				},
-				Paths: []dyn.Path{
-					dyn.MustPathFromString("resources.experiments.experiment1"),
-					dyn.MustPathFromString("resources.jobs.job1"),
-					dyn.MustPathFromString("resources.jobs.job2"),
-					dyn.MustPathFromString("resources.model_serving_endpoints.model_serving_endpoint1"),
-					dyn.MustPathFromString("resources.models.model1"),
-					dyn.MustPathFromString("resources.pipelines.pipeline1"),
-					dyn.MustPathFromString("resources.pipelines.pipeline2"),
-					dyn.MustPathFromString("resources.schemas.schema1"),
-					dyn.MustPathFromString("targets.target1.resources.jobs.job3"),
-					dyn.MustPathFromString("targets.target1.resources.quality_monitors.quality_monitor1"),
-					dyn.MustPathFromString("targets.target1.resources.registered_models.registered_model1"),
-				},
+				Paths: structpath.MustParsePaths(
+					"resources.experiments.experiment1",
+					"resources.jobs.job1",
+					"resources.jobs.job2",
+					"resources.model_serving_endpoints.model_serving_endpoint1",
+					"resources.models.model1",
+					"resources.pipelines.pipeline1",
+					"resources.pipelines.pipeline2",
+					"resources.schemas.schema1",
+					"targets.target1.resources.jobs.job3",
+					"targets.target1.resources.quality_monitors.quality_monitor1",
+					"targets.target1.resources.registered_models.registered_model1",
+				),
 			},
 		},
 	} {

bundle/config/mutator/apply_source_linked_deployment_preset.go

@@ -8,7 +8,7 @@ import (
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/libs/dbr"
 	"github.com/databricks/cli/libs/diag"
-	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/structs/structpath"
 )
 
 type applySourceLinkedDeploymentPreset struct{}
@@ -33,15 +33,13 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle
 
 	if config.IsExplicitlyEnabled((b.Config.Presets.SourceLinkedDeployment)) {
 		if !isDatabricksWorkspace {
-			path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment"))
+			path := structpath.NewPath(nil, "targets", target, "presets", "source_linked_deployment")
 			diags = diags.Append(
 				diag.Diagnostic{
-					Severity: diag.Warning,
-					Summary:  "source-linked deployment is available only in the Databricks Workspace",
-					Paths: []dyn.Path{
-						path,
-					},
-					Locations: b.Config.GetLocations(path[2:].String()),
+					Severity:  diag.Warning,
+					Summary:   "source-linked deployment is available only in the Databricks Workspace",
+					Paths:     []*structpath.PathNode{path},
+					Locations: b.Config.GetLocationsOf(path.SkipPrefix(2)),
 				},
 			)
 
@@ -53,15 +51,13 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle
 		b.Metrics.AddBoolValue("source_linked_set_for_non_development", b.Config.Bundle.Mode != config.Development)
 
 		if b.Config.Bundle.Mode != config.Development {
-			path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment"))
+			path := structpath.NewPath(nil, "targets", target, "presets", "source_linked_deployment")
 			diags = diags.Append(
 				diag.Diagnostic{
-					Severity: diag.Warning,
-					Summary:  "source-linked deployment in non-development mode is deprecated and will not be supported in a future release",
-					Paths: []dyn.Path{
-						path,
-					},
-					Locations: b.Config.GetLocations(path[2:].String()),
+					Severity:  diag.Warning,
+					Summary:   "source-linked deployment in non-development mode is deprecated and will not be supported in a future release",
+					Paths:     []*structpath.PathNode{path},
+					Locations: b.Config.GetLocationsOf(path.SkipPrefix(2)),
 				},
 			)
 		}
@@ -74,16 +70,14 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle
 
 	// This mutator runs before workspace paths are defaulted so it's safe to check for the user-defined value
 	if b.Config.Workspace.FilePath != "" && config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) {
-		path := dyn.NewPath(dyn.Key("workspace"), dyn.Key("file_path"))
+		path := structpath.NewPath(nil, "workspace", "file_path")
 		diags = diags.Append(
 			diag.Diagnostic{
-				Severity: diag.Warning,
-				Summary:  "workspace.file_path setting will be ignored in source-linked deployment mode",
-				Detail:   "In source-linked deployment files are not copied to the destination and resources use source files instead",
-				Paths: []dyn.Path{
-					path,
-				},
-				Locations: b.Config.GetLocations(path.String()),
+				Severity:  diag.Warning,
+				Summary:   "workspace.file_path setting will be ignored in source-linked deployment mode",
+				Detail:    "In source-linked deployment files are not copied to the destination and resources use source files instead",
+				Paths:     []*structpath.PathNode{path},
+				Locations: b.Config.GetLocationsOf(path),
 			},
 		)
 	}

bundle/config/mutator/compute_id_compat.go

@@ -55,7 +55,7 @@ func rewriteComputeIdToClusterId(v dyn.Value, p dyn.Path) (dyn.Value, diag.Diagn
 		Severity:  diag.Warning,
 		Summary:   "compute_id is deprecated, please use cluster_id instead",
 		Locations: computeId.Locations(),
-		Paths:     []dyn.Path{computeIdPath},
+		Paths:     dyn.ToStructPaths(computeIdPath),
 	})
 
 	clusterIdPath := p.Append(dyn.Key("cluster_id"))

bundle/config/mutator/load_dbalert_files.go

@@ -13,6 +13,7 @@ import (
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/cli/libs/dyn/dynvar"
+	"github.com/databricks/cli/libs/structs/structpath"
 	"github.com/databricks/databricks-sdk-go/marshal"
 	"github.com/databricks/databricks-sdk-go/service/sql"
 )
@@ -85,7 +86,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia
 					Severity:  diag.Error,
 					Summary:   fmt.Sprintf("field %s is not allowed in the bundle configuration.", k),
 					Detail:    "When a .dbalert.json is specified, only the following fields are allowed in the bundle configuration: " + strings.Join(allowedInYAML, ", "),
-					Paths:     []dyn.Path{dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.%s", alertKey, k))},
+					Paths:     structpath.NewPathSlice("resources", "alerts", alertKey, k),
 					Locations: v.Locations(),
 				},
 			}
@@ -107,7 +108,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia
 					Severity:  diag.Error,
 					Summary:   fmt.Sprintf("failed to read .dbalert.json file %s: %s", alert.FilePath, err),
 					Detail:    "",
-					Paths:     []dyn.Path{dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.file_path", alertKey))},
+					Paths:     structpath.NewPathSlice("resources", "alerts", alertKey, "file_path"),
 					Locations: alertV.Get("file_path").Locations(),
 				},
 			}
@@ -122,7 +123,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia
 					Severity:  diag.Error,
 					Summary:   fmt.Sprintf("failed to parse .dbalert.json file %s: %s", alert.FilePath, err),
 					Detail:    "",
-					Paths:     []dyn.Path{dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.file_path", alertKey))},
+					Paths:     structpath.NewPathSlice("resources", "alerts", alertKey, "file_path"),
 					Locations: alertV.Get("file_path").Locations(),
 				},
 			}
@@ -136,7 +137,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia
 					Severity:  diag.Error,
 					Summary:   fmt.Sprintf(".alert file %s must not contain variable interpolations.", alert.FilePath),
 					Detail:    "Please inline the alert configuration in the bundle configuration to use variables",
-					Paths:     []dyn.Path{dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.file_path", alertKey))},
+					Paths:     structpath.NewPathSlice("resources", "alerts", alertKey, "file_path"),
 					Locations: alertV.Get("file_path").Locations(),
 				},
 			}