diff --git a/.nextchanges/bundles/6980.md b/.nextchanges/bundles/6980.md new file mode 100644 index 00000000000..7a40caf2cc1 --- /dev/null +++ b/.nextchanges/bundles/6980.md @@ -0,0 +1 @@ +* Diagnostic paths now quote map keys that are not identifiers, e.g. `resources.jobs['${var.env}_job']`. ([#6980](https://github.com/databricks/cli/pull/6980)) diff --git a/acceptance/bundle/variables/variable_in_resource_key/output.txt b/acceptance/bundle/variables/variable_in_resource_key/output.txt index 6fc17dc855d..841db2e41c5 100644 --- a/acceptance/bundle/variables/variable_in_resource_key/output.txt +++ b/acceptance/bundle/variables/variable_in_resource_key/output.txt @@ -1,13 +1,13 @@ Warning: unknown field: ${var.env} - at targets.dev.resources.jobs.${var.env}_job_2 + at targets.dev.resources.jobs['${var.env}_job_2'] in databricks.yml:21:11 Error: resource key "${var.env}_job" must not contain variable references - at resources.jobs.${var.env}_job + at resources.jobs['${var.env}_job'] in databricks.yml:12:7 Error: resource key "${var.env}_job_2" must not contain variable references - at targets.dev.resources.jobs.${var.env}_job_2 + at targets.dev.resources.jobs['${var.env}_job_2'] in databricks.yml:20:11 diff --git a/bundle/apps/validate.go b/bundle/apps/validate.go index 45a5b49c672..2ab3384f991 100644 --- a/bundle/apps/validate.go +++ b/bundle/apps/validate.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/apps" ) @@ -27,7 +28,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics Severity: diag.Error, Summary: "Missing app source code path or git source", Detail: fmt.Sprintf("app resource '%s' should have either source_code_path or git_source field", key), - Locations: b.Config.GetLocations("resources.apps." + key), + Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "apps", key)), }) continue } @@ -37,7 +38,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics Severity: diag.Error, Summary: "Both source_code_path and git_source fields are set", Detail: fmt.Sprintf("app resource '%s' should have either source_code_path or git_source field, not both", key), - Locations: b.Config.GetLocations("resources.apps." + key), + Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "apps", key)), }) continue } @@ -47,7 +48,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics Severity: diag.Error, Summary: "Duplicate app source code path", Detail: fmt.Sprintf("app resource '%s' has the same source code path as app resource '%s', this will lead to the app configuration being overridden by each other", key, usedSourceCodePaths[app.SourceCodePath]), - Locations: b.Config.GetLocations(fmt.Sprintf("resources.apps.%s.source_code_path", key)), + Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "apps", key, "source_code_path")), }) } usedSourceCodePaths[app.SourceCodePath] = key @@ -154,7 +155,7 @@ func warnForAppResourcePermissions(b *bundle.Bundle, appKey string, app *resourc continue } - appPath := "resources.apps." + appKey + appPath := structpath.NewPath(nil, "resources", "apps", appKey) diags = append(diags, diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("app %q references %s %q which has permissions set. To prevent permission override after deploying the app, please add the app service principal to the %s permissions", appKey, refType, resourceKey, refType), @@ -172,8 +173,8 @@ func warnForAppResourcePermissions(b *bundle.Bundle, appKey string, app *resourc ref.permission, appKey, ), - Paths: []dyn.Path{dyn.MustPathFromString(appPath)}, - Locations: b.Config.GetLocations(appPath), + Paths: []*structpath.PathNode{appPath}, + Locations: b.Config.GetLocationsOf(appPath), }) } diff --git a/bundle/apps/validate_test.go b/bundle/apps/validate_test.go index 5edae10a884..3285851ce91 100644 --- a/bundle/apps/validate_test.go +++ b/bundle/apps/validate_test.go @@ -213,7 +213,7 @@ func TestAppsValidateResourcePermissionsWarning(t *testing.T) { require.Len(t, warnings, 1) require.Contains(t, warnings[0].Summary, tc.wantSummary) require.Contains(t, warnings[0].Detail, "service_principal_name: ${resources.apps.my_app.service_principal_client_id}") - require.Equal(t, dyn.MustPathFromString("resources.apps.my_app"), warnings[0].Paths[0]) + require.Equal(t, "resources.apps.my_app", warnings[0].Paths[0].String()) } else { require.Empty(t, warnings) } diff --git a/bundle/artifacts/expand_globs.go b/bundle/artifacts/expand_globs.go index af358d3dda9..e7cd5cc1308 100644 --- a/bundle/artifacts/expand_globs.go +++ b/bundle/artifacts/expand_globs.go @@ -27,7 +27,7 @@ func createGlobError(v dyn.Value, p dyn.Path, message string) diag.Diagnostic { Severity: diag.Error, Summary: fmt.Sprintf("%s: %s", source, message), Locations: []dyn.Location{v.Location()}, - Paths: []dyn.Path{p}, + Paths: dyn.ToStructPaths(p), } } diff --git a/bundle/config/loader/process_include.go b/bundle/config/loader/process_include.go index 3a64297814c..dd8932caeed 100644 --- a/bundle/config/loader/process_include.go +++ b/bundle/config/loader/process_include.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) func validateFileFormat(configRoot dyn.Value, filePath string) diag.Diagnostics { @@ -108,16 +109,16 @@ func validateSingleResourceDefined(configRoot dyn.Value, ext, typ string) diag.D } var locations []dyn.Location - var paths []dyn.Path + var paths []*structpath.PathNode for _, rr := range resources { locations = append(locations, rr.value.Locations()...) - paths = append(paths, rr.path) + paths = append(paths, dyn.ToStructPath(rr.path)) } // Sort the locations and paths to make the output deterministic. slices.SortFunc(locations, func(a, b dyn.Location) int { return cmp.Compare(a.String(), b.String()) }) - slices.SortFunc(paths, func(a, b dyn.Path) int { + slices.SortFunc(paths, func(a, b *structpath.PathNode) int { return cmp.Compare(a.String(), b.String()) }) @@ -168,7 +169,7 @@ func (m *processInclude) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos Detail: `An include section is defined in a file that is not databricks.yml. Only includes defined in databricks.yml are applied.`, Locations: this.GetLocations("include"), - Paths: []dyn.Path{dyn.MustPathFromString("include")}, + Paths: structpath.NewPathSlice("include"), }) } diff --git a/bundle/config/loader/process_include_test.go b/bundle/config/loader/process_include_test.go index c768dc57dc9..28c95ac905b 100644 --- a/bundle/config/loader/process_include_test.go +++ b/bundle/config/loader/process_include_test.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/config/loader" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -71,10 +72,10 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) { {File: "testdata/format_not_match/single_job.pipeline.yaml", Line: 11, Column: 11}, {File: "testdata/format_not_match/single_job.pipeline.yaml", Line: 4, Column: 7}, }, - Paths: []dyn.Path{ - dyn.MustPathFromString("resources.jobs.job1"), - dyn.MustPathFromString("targets.target1.resources.jobs.job1"), - }, + Paths: structpath.MustParsePaths( + "resources.jobs.job1", + "targets.target1.resources.jobs.job1", + ), }, }, "job_and_pipeline.job.yml": { @@ -86,10 +87,10 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) { {File: "testdata/format_not_match/job_and_pipeline.job.yml", Line: 11, Column: 11}, {File: "testdata/format_not_match/job_and_pipeline.job.yml", Line: 4, Column: 7}, }, - Paths: []dyn.Path{ - dyn.MustPathFromString("resources.pipelines.pipeline1"), - dyn.MustPathFromString("targets.target1.resources.jobs.job1"), - }, + Paths: structpath.MustParsePaths( + "resources.pipelines.pipeline1", + "targets.target1.resources.jobs.job1", + ), }, }, "job_and_pipeline.experiment.yml": { @@ -101,10 +102,10 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) { {File: "testdata/format_not_match/job_and_pipeline.experiment.yml", Line: 11, Column: 11}, {File: "testdata/format_not_match/job_and_pipeline.experiment.yml", Line: 4, Column: 7}, }, - Paths: []dyn.Path{ - dyn.MustPathFromString("resources.pipelines.pipeline1"), - dyn.MustPathFromString("targets.target1.resources.jobs.job1"), - }, + Paths: structpath.MustParsePaths( + "resources.pipelines.pipeline1", + "targets.target1.resources.jobs.job1", + ), }, }, "two_jobs.job.yml": { @@ -116,10 +117,10 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) { {File: "testdata/format_not_match/two_jobs.job.yml", Line: 4, Column: 7}, {File: "testdata/format_not_match/two_jobs.job.yml", Line: 7, Column: 7}, }, - Paths: []dyn.Path{ - dyn.MustPathFromString("resources.jobs.job1"), - dyn.MustPathFromString("resources.jobs.job2"), - }, + Paths: structpath.MustParsePaths( + "resources.jobs.job1", + "resources.jobs.job2", + ), }, }, "second_job_in_target.job.yml": { @@ -131,10 +132,10 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) { {File: "testdata/format_not_match/second_job_in_target.job.yml", Line: 11, Column: 11}, {File: "testdata/format_not_match/second_job_in_target.job.yml", Line: 4, Column: 7}, }, - Paths: []dyn.Path{ - dyn.MustPathFromString("resources.jobs.job1"), - dyn.MustPathFromString("targets.target1.resources.jobs.job2"), - }, + Paths: structpath.MustParsePaths( + "resources.jobs.job1", + "targets.target1.resources.jobs.job2", + ), }, }, "two_jobs_in_target.job.yml": { @@ -146,10 +147,10 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) { {File: "testdata/format_not_match/two_jobs_in_target.job.yml", Line: 6, Column: 11}, {File: "testdata/format_not_match/two_jobs_in_target.job.yml", Line: 8, Column: 11}, }, - Paths: []dyn.Path{ - dyn.MustPathFromString("targets.target1.resources.jobs.job1"), - dyn.MustPathFromString("targets.target1.resources.jobs.job2"), - }, + Paths: structpath.MustParsePaths( + "targets.target1.resources.jobs.job1", + "targets.target1.resources.jobs.job2", + ), }, }, "multiple_resources.model_serving_endpoint.yml": { @@ -182,19 +183,19 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) { {File: "testdata/format_not_match/multiple_resources.model_serving_endpoint.yml", Line: 4, Column: 7}, {File: "testdata/format_not_match/multiple_resources.model_serving_endpoint.yml", Line: 8, Column: 7}, }, - Paths: []dyn.Path{ - dyn.MustPathFromString("resources.experiments.experiment1"), - dyn.MustPathFromString("resources.jobs.job1"), - dyn.MustPathFromString("resources.jobs.job2"), - dyn.MustPathFromString("resources.model_serving_endpoints.model_serving_endpoint1"), - dyn.MustPathFromString("resources.models.model1"), - dyn.MustPathFromString("resources.pipelines.pipeline1"), - dyn.MustPathFromString("resources.pipelines.pipeline2"), - dyn.MustPathFromString("resources.schemas.schema1"), - dyn.MustPathFromString("targets.target1.resources.jobs.job3"), - dyn.MustPathFromString("targets.target1.resources.quality_monitors.quality_monitor1"), - dyn.MustPathFromString("targets.target1.resources.registered_models.registered_model1"), - }, + Paths: structpath.MustParsePaths( + "resources.experiments.experiment1", + "resources.jobs.job1", + "resources.jobs.job2", + "resources.model_serving_endpoints.model_serving_endpoint1", + "resources.models.model1", + "resources.pipelines.pipeline1", + "resources.pipelines.pipeline2", + "resources.schemas.schema1", + "targets.target1.resources.jobs.job3", + "targets.target1.resources.quality_monitors.quality_monitor1", + "targets.target1.resources.registered_models.registered_model1", + ), }, }, } { diff --git a/bundle/config/mutator/apply_source_linked_deployment_preset.go b/bundle/config/mutator/apply_source_linked_deployment_preset.go index c1e817673d9..638f3034140 100644 --- a/bundle/config/mutator/apply_source_linked_deployment_preset.go +++ b/bundle/config/mutator/apply_source_linked_deployment_preset.go @@ -8,7 +8,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/dbr" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) type applySourceLinkedDeploymentPreset struct{} @@ -33,15 +33,13 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle if config.IsExplicitlyEnabled((b.Config.Presets.SourceLinkedDeployment)) { if !isDatabricksWorkspace { - path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment")) + path := structpath.NewPath(nil, "targets", target, "presets", "source_linked_deployment") diags = diags.Append( diag.Diagnostic{ - Severity: diag.Warning, - Summary: "source-linked deployment is available only in the Databricks Workspace", - Paths: []dyn.Path{ - path, - }, - Locations: b.Config.GetLocations(path[2:].String()), + Severity: diag.Warning, + Summary: "source-linked deployment is available only in the Databricks Workspace", + Paths: []*structpath.PathNode{path}, + Locations: b.Config.GetLocationsOf(path.SkipPrefix(2)), }, ) @@ -53,15 +51,13 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle b.Metrics.AddBoolValue("source_linked_set_for_non_development", b.Config.Bundle.Mode != config.Development) if b.Config.Bundle.Mode != config.Development { - path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment")) + path := structpath.NewPath(nil, "targets", target, "presets", "source_linked_deployment") diags = diags.Append( diag.Diagnostic{ - Severity: diag.Warning, - Summary: "source-linked deployment in non-development mode is deprecated and will not be supported in a future release", - Paths: []dyn.Path{ - path, - }, - Locations: b.Config.GetLocations(path[2:].String()), + Severity: diag.Warning, + Summary: "source-linked deployment in non-development mode is deprecated and will not be supported in a future release", + Paths: []*structpath.PathNode{path}, + Locations: b.Config.GetLocationsOf(path.SkipPrefix(2)), }, ) } @@ -74,16 +70,14 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle // This mutator runs before workspace paths are defaulted so it's safe to check for the user-defined value if b.Config.Workspace.FilePath != "" && config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) { - path := dyn.NewPath(dyn.Key("workspace"), dyn.Key("file_path")) + path := structpath.NewPath(nil, "workspace", "file_path") diags = diags.Append( diag.Diagnostic{ - Severity: diag.Warning, - Summary: "workspace.file_path setting will be ignored in source-linked deployment mode", - Detail: "In source-linked deployment files are not copied to the destination and resources use source files instead", - Paths: []dyn.Path{ - path, - }, - Locations: b.Config.GetLocations(path.String()), + Severity: diag.Warning, + Summary: "workspace.file_path setting will be ignored in source-linked deployment mode", + Detail: "In source-linked deployment files are not copied to the destination and resources use source files instead", + Paths: []*structpath.PathNode{path}, + Locations: b.Config.GetLocationsOf(path), }, ) } diff --git a/bundle/config/mutator/compute_id_compat.go b/bundle/config/mutator/compute_id_compat.go index 8f1ff5868e8..c29b01cbd1c 100644 --- a/bundle/config/mutator/compute_id_compat.go +++ b/bundle/config/mutator/compute_id_compat.go @@ -55,7 +55,7 @@ func rewriteComputeIdToClusterId(v dyn.Value, p dyn.Path) (dyn.Value, diag.Diagn Severity: diag.Warning, Summary: "compute_id is deprecated, please use cluster_id instead", Locations: computeId.Locations(), - Paths: []dyn.Path{computeIdPath}, + Paths: dyn.ToStructPaths(computeIdPath), }) clusterIdPath := p.Append(dyn.Key("cluster_id")) diff --git a/bundle/config/mutator/load_dbalert_files.go b/bundle/config/mutator/load_dbalert_files.go index 5c3e6921b93..a5fa2ff8d62 100644 --- a/bundle/config/mutator/load_dbalert_files.go +++ b/bundle/config/mutator/load_dbalert_files.go @@ -13,6 +13,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/sql" ) @@ -85,7 +86,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia Severity: diag.Error, Summary: fmt.Sprintf("field %s is not allowed in the bundle configuration.", k), Detail: "When a .dbalert.json is specified, only the following fields are allowed in the bundle configuration: " + strings.Join(allowedInYAML, ", "), - Paths: []dyn.Path{dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.%s", alertKey, k))}, + Paths: structpath.NewPathSlice("resources", "alerts", alertKey, k), Locations: v.Locations(), }, } @@ -107,7 +108,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia Severity: diag.Error, Summary: fmt.Sprintf("failed to read .dbalert.json file %s: %s", alert.FilePath, err), Detail: "", - Paths: []dyn.Path{dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.file_path", alertKey))}, + Paths: structpath.NewPathSlice("resources", "alerts", alertKey, "file_path"), Locations: alertV.Get("file_path").Locations(), }, } @@ -122,7 +123,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia Severity: diag.Error, Summary: fmt.Sprintf("failed to parse .dbalert.json file %s: %s", alert.FilePath, err), Detail: "", - Paths: []dyn.Path{dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.file_path", alertKey))}, + Paths: structpath.NewPathSlice("resources", "alerts", alertKey, "file_path"), Locations: alertV.Get("file_path").Locations(), }, } @@ -136,7 +137,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia Severity: diag.Error, Summary: fmt.Sprintf(".alert file %s must not contain variable interpolations.", alert.FilePath), Detail: "Please inline the alert configuration in the bundle configuration to use variables", - Paths: []dyn.Path{dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.file_path", alertKey))}, + Paths: structpath.NewPathSlice("resources", "alerts", alertKey, "file_path"), Locations: alertV.Get("file_path").Locations(), }, } diff --git a/bundle/config/mutator/python/python_diagnostics.go b/bundle/config/mutator/python/python_diagnostics.go index 7a1e13b4e19..b947b3f6ad3 100644 --- a/bundle/config/mutator/python/python_diagnostics.go +++ b/bundle/config/mutator/python/python_diagnostics.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) // pythonDiagnostic is a single entry in diagnostics.json @@ -55,9 +56,9 @@ func parsePythonDiagnostics(input io.Reader) (diag.Diagnostics, error) { if err != nil { return nil, fmt.Errorf("failed to parse path: %s", err) } - var paths []dyn.Path + var paths []*structpath.PathNode if path != nil { - paths = []dyn.Path{path} + paths = dyn.ToStructPaths(path) } var locations []dyn.Location diff --git a/bundle/config/mutator/python/python_diagnostics_test.go b/bundle/config/mutator/python/python_diagnostics_test.go index 6fef9eca9cb..43709031ca2 100644 --- a/bundle/config/mutator/python/python_diagnostics_test.go +++ b/bundle/config/mutator/python/python_diagnostics_test.go @@ -6,6 +6,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" ) @@ -55,7 +56,7 @@ func TestParsePythonDiagnostics(t *testing.T) { { Severity: diag.Error, Summary: "error summary", - Paths: []dyn.Path{dyn.MustPathFromString("resources.jobs.job0.name")}, + Paths: structpath.NewPathSlice("resources", "jobs", "job0", "name"), }, }, }, diff --git a/bundle/config/mutator/reject_internal_resources.go b/bundle/config/mutator/reject_internal_resources.go index 5b1d38f8175..0e4e28e53b5 100644 --- a/bundle/config/mutator/reject_internal_resources.go +++ b/bundle/config/mutator/reject_internal_resources.go @@ -5,7 +5,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) type rejectInternalResources struct{} @@ -26,7 +26,7 @@ func (m *rejectInternalResources) Apply(ctx context.Context, b *bundle.Bundle) d diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "Internal resources cannot be set in bundle configuration", - Paths: []dyn.Path{dyn.MustPathFromString("resources")}, + Paths: structpath.NewPathSlice("resources"), Locations: b.Config.GetLocations("resources"), }) } diff --git a/bundle/config/mutator/resourcemutator/run_as.go b/bundle/config/mutator/resourcemutator/run_as.go index b06acc8b46e..b36e915b372 100644 --- a/bundle/config/mutator/resourcemutator/run_as.go +++ b/bundle/config/mutator/resourcemutator/run_as.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/databricks/databricks-sdk-go/service/sql" @@ -108,7 +109,7 @@ func validateRunAs(b *bundle.Bundle) diag.Diagnostics { } diags = diags.Extend(reportRunAsNotSupported( "dashboards with embed_credentials set to true", - b.Config.GetLocation("resources.dashboards."+key), + b.Config.GetLocationOf(structpath.NewPath(nil, "resources", "dashboards", key)), b.Config.Workspace.CurrentUser.UserName, identity, )) @@ -238,7 +239,7 @@ func (m *setRunAs) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { { Severity: diag.Warning, Summary: "You are using the legacy mode of run_as. The support for this mode is experimental and might be removed in a future release of the CLI. In order to run the pipelines in your DABs project as the run_as user this mode changes the owners of the pipelines to the run_as identity, which requires the user deploying the bundle to be a workspace admin, and also a Metastore admin if the pipeline target is in UC.", - Paths: []dyn.Path{dyn.MustPathFromString("experimental.use_legacy_run_as")}, + Paths: structpath.NewPathSlice("experimental", "use_legacy_run_as"), Locations: b.Config.GetLocations("experimental.use_legacy_run_as"), }, } diff --git a/bundle/config/mutator/resourcemutator/secret_scope_fixups.go b/bundle/config/mutator/resourcemutator/secret_scope_fixups.go index 4b1330d220a..0d5d541a6ee 100644 --- a/bundle/config/mutator/resourcemutator/secret_scope_fixups.go +++ b/bundle/config/mutator/resourcemutator/secret_scope_fixups.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/iamutil" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/iam" ) @@ -138,8 +139,8 @@ func (m *secretScopeFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Di Severity: diag.Error, Summary: "Failed to collapse permissions for secret scope", Detail: err.Error(), - Paths: []dyn.Path{dyn.MustPathFromString("resources.secret_scopes." + key)}, - Locations: []dyn.Location{b.Config.GetLocation("resources.secret_scopes." + key)}, + Paths: structpath.NewPathSlice("resources", "secret_scopes", key), + Locations: []dyn.Location{b.Config.GetLocationOf(structpath.NewPath(nil, "resources", "secret_scopes", key))}, }, } } diff --git a/bundle/config/mutator/rewrite_workspace_prefix.go b/bundle/config/mutator/rewrite_workspace_prefix.go index b67f773f207..30914fd149e 100644 --- a/bundle/config/mutator/rewrite_workspace_prefix.go +++ b/bundle/config/mutator/rewrite_workspace_prefix.go @@ -65,7 +65,7 @@ func (m *rewriteWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) di Summary: fmt.Sprintf("substring %q found in %q. Please update this to %q.", rewrite.pattern, vv, newPath), Detail: "For more information, please refer to: https://docs.databricks.com/en/release-notes/dev-tools/bundles.html#workspace-paths", Locations: v.Locations(), - Paths: []dyn.Path{p}, + Paths: dyn.ToStructPaths(p), }) } diff --git a/bundle/config/mutator/sync_infer_root.go b/bundle/config/mutator/sync_infer_root.go index 373a3701dcc..bd1a529380a 100644 --- a/bundle/config/mutator/sync_infer_root.go +++ b/bundle/config/mutator/sync_infer_root.go @@ -7,7 +7,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/vfs" ) @@ -90,11 +90,12 @@ func (m *syncInferRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno continue } + syncPath := structpath.NewPath(nil, "sync", "paths", i) diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("invalid sync path %q", path), - Locations: b.Config.GetLocations(fmt.Sprintf("sync.paths[%d]", i)), - Paths: []dyn.Path{dyn.NewPath(dyn.Key("sync"), dyn.Key("paths"), dyn.Index(i))}, + Locations: b.Config.GetLocationsOf(syncPath), + Paths: []*structpath.PathNode{syncPath}, }) } diff --git a/bundle/config/mutator/validate_secret_value_is_variable.go b/bundle/config/mutator/validate_secret_value_is_variable.go index 39b7822168a..1f979e15f74 100644 --- a/bundle/config/mutator/validate_secret_value_is_variable.go +++ b/bundle/config/mutator/validate_secret_value_is_variable.go @@ -33,7 +33,7 @@ func (v *validateSecretValueIsVariable) Apply(ctx context.Context, b *bundle.Bun Summary: "Secret value must be a string", Detail: fmt.Sprintf(`The secret value for "%s" must be a string.`, key), Locations: val.Locations(), - Paths: []dyn.Path{p}, + Paths: dyn.ToStructPaths(p), }) continue } @@ -48,7 +48,7 @@ func (v *validateSecretValueIsVariable) Apply(ctx context.Context, b *bundle.Bun Summary: "Secret value must be a string", Detail: fmt.Sprintf(`The secret value for "%s" must be a string.`, key), Locations: val.Locations(), - Paths: []dyn.Path{p}, + Paths: dyn.ToStructPaths(p), }) continue } @@ -62,7 +62,7 @@ func (v *validateSecretValueIsVariable) Apply(ctx context.Context, b *bundle.Bun Plain text secret values are not allowed to prevent leaking secrets in configuration files. Use bundle variables to pass secret values at deployment time.`, key), Locations: val.Locations(), - Paths: []dyn.Path{p}, + Paths: dyn.ToStructPaths(p), }) continue } @@ -106,6 +106,6 @@ func (v *validateSecretValueIsVariable) checkVariableDefault(b *bundle.Bundle, s A default value is stored in plain text in the configuration file, which defeats the purpose of using a variable reference for a secret. Remove the default value and pass the secret value at deployment time using "--var", the BUNDLE_VAR_%s environment variable, or a variable overrides file.`, varName, secretKey, varName), Locations: locations, - Paths: []dyn.Path{p}, + Paths: dyn.ToStructPaths(p), }} } diff --git a/bundle/config/root.go b/bundle/config/root.go index e13bf78bf15..b87a2675692 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -18,6 +18,7 @@ import ( "github.com/databricks/cli/libs/dyn/merge" "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/jobs" ) @@ -605,6 +606,33 @@ func (r Root) GetLocations(path string) []dyn.Location { return v.Locations() } +// GetLocationOf is [Root.GetLocation] for a path node. +func (r Root) GetLocationOf(path *structpath.PathNode) dyn.Location { + v, ok := r.valueOf(path) + if !ok { + return dyn.Location{} + } + return v.Location() +} + +// GetLocationsOf is [Root.GetLocations] for a path node. +func (r Root) GetLocationsOf(path *structpath.PathNode) []dyn.Location { + v, ok := r.valueOf(path) + if !ok { + return nil + } + return v.Locations() +} + +func (r Root) valueOf(path *structpath.PathNode) (dyn.Value, bool) { + p, ok := dyn.FromStructPath(path) + if !ok { + return dyn.InvalidValue, false + } + v, err := dyn.GetByPath(r.value, p) + return v, err == nil +} + // GetNodeAndType and returns parent resource node and type of the resource in direct backend. // Examples: // diff --git a/bundle/config/validate/all_resources_have_values.go b/bundle/config/validate/all_resources_have_values.go index 7f96e529a74..713ec0f83af 100644 --- a/bundle/config/validate/all_resources_have_values.go +++ b/bundle/config/validate/all_resources_have_values.go @@ -3,7 +3,6 @@ package validate import ( "context" "fmt" - "slices" "strings" "github.com/databricks/cli/bundle" @@ -43,7 +42,7 @@ func (m *allResourcesHaveValues) Apply(ctx context.Context, b *bundle.Bundle) di Severity: diag.Error, Summary: fmt.Sprintf("%s %s is not defined", rType, rName), Locations: v.Locations(), - Paths: []dyn.Path{slices.Clone(p)}, + Paths: dyn.ToStructPaths(p), }) return v, nil diff --git a/bundle/config/validate/enum.go b/bundle/config/validate/enum.go index e6266163bc8..2ce99ea1d19 100644 --- a/bundle/config/validate/enum.go +++ b/bundle/config/validate/enum.go @@ -69,13 +69,12 @@ func (f *enum) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { if !validValue { // p is a slice of path components. We need to clone it before using it in diagnostics // since the WalkReadOnly function will mutate it while walking the config tree. - cloneP := slices.Clone(p) diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("invalid value %q for enum field. Valid values are %v", strValue, validValues), Locations: v.Locations(), - Paths: []dyn.Path{cloneP}, + Paths: dyn.ToStructPaths(p), }) } diff --git a/bundle/config/validate/files_to_sync.go b/bundle/config/validate/files_to_sync.go index b3779016d5f..14b032e8da1 100644 --- a/bundle/config/validate/files_to_sync.go +++ b/bundle/config/validate/files_to_sync.go @@ -6,7 +6,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/deploy/files" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/sync" ) @@ -56,14 +56,14 @@ func (v *filesToSync) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost Summary: "There are no files to sync, please check your .gitignore", }) } else { - path := "sync.exclude" + path := structpath.NewPath(nil, "sync", "exclude") diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: "There are no files to sync, please check your .gitignore and sync.exclude configuration", // Show all locations where sync.exclude is defined, since merging // sync.exclude is additive. - Locations: b.Config.GetLocations(path), - Paths: []dyn.Path{dyn.MustPathFromString(path)}, + Locations: b.Config.GetLocationsOf(path), + Paths: []*structpath.PathNode{path}, }) } diff --git a/bundle/config/validate/interpolation_in_auth_config.go b/bundle/config/validate/interpolation_in_auth_config.go index 1598ef49d88..f749f9e0613 100644 --- a/bundle/config/validate/interpolation_in_auth_config.go +++ b/bundle/config/validate/interpolation_in_auth_config.go @@ -78,7 +78,7 @@ func (f *noInterpolationInAuthConfig) Apply(ctx context.Context, b *bundle.Bundl Detail: fmt.Sprintf(`Interpolation is not supported for the field %s. Please set the %s environment variable if you wish to configure this field at runtime.`, p.String(), envVar), Locations: v.Locations(), - Paths: []dyn.Path{p}, + Paths: dyn.ToStructPaths(p), }) } } diff --git a/bundle/config/validate/job_cluster_key_defined.go b/bundle/config/validate/job_cluster_key_defined.go index 019a785658a..eb9bea4df70 100644 --- a/bundle/config/validate/job_cluster_key_defined.go +++ b/bundle/config/validate/job_cluster_key_defined.go @@ -6,7 +6,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) func JobClusterKeyDefined() bundle.ReadOnlyMutator { @@ -32,12 +32,12 @@ func (v *jobClusterKeyDefined) Apply(ctx context.Context, b *bundle.Bundle) diag for index, task := range job.Tasks { diags = diags.Extend(checkJobClusterKey(b, jobClusterKeys, task.JobClusterKey, - fmt.Sprintf("resources.jobs.%s.tasks[%d].job_cluster_key", k, index))) + structpath.NewPath(nil, "resources", "jobs", k, "tasks", index, "job_cluster_key"))) // The Jobs API rejects nested for_each_task, so one level is sufficient. if task.ForEachTask != nil { diags = diags.Extend(checkJobClusterKey(b, jobClusterKeys, task.ForEachTask.Task.JobClusterKey, - fmt.Sprintf("resources.jobs.%s.tasks[%d].for_each_task.task.job_cluster_key", k, index))) + structpath.NewPath(nil, "resources", "jobs", k, "tasks", index, "for_each_task", "task", "job_cluster_key"))) } } } @@ -46,7 +46,7 @@ func (v *jobClusterKeyDefined) Apply(ctx context.Context, b *bundle.Bundle) diag } // checkJobClusterKey warns if jobClusterKey is set but not defined in the job's job_clusters. -func checkJobClusterKey(b *bundle.Bundle, jobClusterKeys map[string]bool, jobClusterKey, path string) diag.Diagnostics { +func checkJobClusterKey(b *bundle.Bundle, jobClusterKeys map[string]bool, jobClusterKey string, path *structpath.PathNode) diag.Diagnostics { if jobClusterKey == "" { return nil } @@ -60,7 +60,7 @@ func checkJobClusterKey(b *bundle.Bundle, jobClusterKeys map[string]bool, jobClu // Show only the location where the job_cluster_key is defined. // Other associated locations are not relevant since they are // overridden during merging. - Locations: b.Config.GetLocations(path), - Paths: []dyn.Path{dyn.MustPathFromString(path)}, + Locations: b.Config.GetLocationsOf(path), + Paths: []*structpath.PathNode{path}, }} } diff --git a/bundle/config/validate/job_task_cluster_spec.go b/bundle/config/validate/job_task_cluster_spec.go index 79672be63e6..e387d83f6fb 100644 --- a/bundle/config/validate/job_task_cluster_spec.go +++ b/bundle/config/validate/job_task_cluster_spec.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/jobs" ) @@ -88,12 +89,13 @@ func validateJobTask(b *bundle.Bundle, task jobs.Task, taskPath dyn.Path) diag.D strings.Join(unspecified, ", "), ) + taskNode := dyn.ToStructPath(taskPath) diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "Missing required cluster or environment settings", Detail: detail, - Locations: b.Config.GetLocations(taskPath.String()), - Paths: []dyn.Path{taskPath}, + Locations: b.Config.GetLocationsOf(taskNode), + Paths: []*structpath.PathNode{taskNode}, }) } } diff --git a/bundle/config/validate/no_interpolation_in_bundle_name.go b/bundle/config/validate/no_interpolation_in_bundle_name.go index c4238ffe25a..8fc547b174e 100644 --- a/bundle/config/validate/no_interpolation_in_bundle_name.go +++ b/bundle/config/validate/no_interpolation_in_bundle_name.go @@ -5,9 +5,9 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structpath" ) type noInterpolationInBundleName struct{} @@ -29,7 +29,7 @@ is a part of the path at which your bundle state is stored at by default. Parame runtime can have unexpected consequences like duplicate deployments or resources not being cleaned up during bundle destroy.`, Locations: b.Config.GetLocations("bundle.name"), - Paths: []dyn.Path{dyn.MustPathFromString("bundle.name")}, + Paths: structpath.NewPathSlice("bundle", "name"), }) } diff --git a/bundle/config/validate/no_variable_reference_in_resource_key.go b/bundle/config/validate/no_variable_reference_in_resource_key.go index 5ad5b58f200..75b8361ce97 100644 --- a/bundle/config/validate/no_variable_reference_in_resource_key.go +++ b/bundle/config/validate/no_variable_reference_in_resource_key.go @@ -41,7 +41,7 @@ func (m *noVariableReferenceInResourceKey) Apply(_ context.Context, b *bundle.Bu Severity: diag.Error, Summary: fmt.Sprintf("resource key %q must not contain variable references", key), Locations: v.Locations(), - Paths: []dyn.Path{p}, + Paths: dyn.ToStructPaths(p), }) } return v, nil diff --git a/bundle/config/validate/pipeline_development_deprecated.go b/bundle/config/validate/pipeline_development_deprecated.go index d726ad3efb1..3eb3c77c5f0 100644 --- a/bundle/config/validate/pipeline_development_deprecated.go +++ b/bundle/config/validate/pipeline_development_deprecated.go @@ -43,7 +43,7 @@ func (v *pipelineDevelopmentDeprecated) Apply(_ context.Context, b *bundle.Bundl Severity: diag.Warning, Summary: pipelineDevelopmentDeprecatedSummary, Locations: v.Locations(), - Paths: []dyn.Path{p}, + Paths: dyn.ToStructPaths(p), }) // The preset overwrites YAML values with true, so a false here was set by a Python @@ -54,7 +54,7 @@ func (v *pipelineDevelopmentDeprecated) Apply(_ context.Context, b *bundle.Bundl Summary: pipelineDevelopmentIgnoredSummary, Detail: pipelineDevelopmentIgnoredDetail, Locations: v.Locations(), - Paths: []dyn.Path{p}, + Paths: dyn.ToStructPaths(p), }) } return v, nil diff --git a/bundle/config/validate/required.go b/bundle/config/validate/required.go index b886c2c1d73..dce5c0686c1 100644 --- a/bundle/config/validate/required.go +++ b/bundle/config/validate/required.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/bundle/internal/validation/generated" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) type required struct{} @@ -49,8 +50,6 @@ func warnForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnostic return nil } - cloneP := slices.Clone(p) - fields := generated.RequiredFields[pattern.String()] for _, field := range fields { vv := v.Get(field) @@ -59,7 +58,7 @@ func warnForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnostic Severity: diag.Warning, Summary: fmt.Sprintf("required field %q is not set", field), Locations: v.Locations(), - Paths: []dyn.Path{cloneP}, + Paths: dyn.ToStructPaths(p), }) } } @@ -97,19 +96,21 @@ func sortDiagnostics(diags diag.Diagnostics) { func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // Dashboards should always have a name and warehouse_id. var nameLocations []dyn.Location - var namePaths []dyn.Path + var namePaths []*structpath.PathNode var warehouseIdLocations []dyn.Location - var warehouseIdPaths []dyn.Path + var warehouseIdPaths []*structpath.PathNode diags := diag.Diagnostics{} for key, dashboard := range b.Config.Resources.Dashboards { if dashboard.DisplayName == "" { - nameLocations = append(nameLocations, b.Config.GetLocations("resources.dashboards."+key)...) - namePaths = append(namePaths, dyn.MustPathFromString("resources.dashboards."+key)) + path := structpath.NewPath(nil, "resources", "dashboards", key) + nameLocations = append(nameLocations, b.Config.GetLocationsOf(path)...) + namePaths = append(namePaths, path) } if dashboard.WarehouseId == "" { - warehouseIdLocations = append(warehouseIdLocations, b.Config.GetLocations("resources.dashboards."+key)...) - warehouseIdPaths = append(warehouseIdPaths, dyn.MustPathFromString("resources.dashboards."+key)) + path := structpath.NewPath(nil, "resources", "dashboards", key) + warehouseIdLocations = append(warehouseIdLocations, b.Config.GetLocationsOf(path)...) + warehouseIdPaths = append(warehouseIdPaths, path) } } @@ -134,12 +135,12 @@ func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnosti // by the backend, which rejects whitespace-only names (name.trim.nonEmpty). for key, warehouse := range b.Config.Resources.SqlWarehouses { if strings.TrimSpace(warehouse.Name) == "" { - path := "resources.sql_warehouses." + key + path := structpath.NewPath(nil, "resources", "sql_warehouses", key) diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "sql_warehouse name is required", - Locations: b.Config.GetLocations(path), - Paths: []dyn.Path{dyn.MustPathFromString(path)}, + Locations: b.Config.GetLocationsOf(path), + Paths: []*structpath.PathNode{path}, }) } } @@ -166,7 +167,7 @@ func errorForInvalidGrants(ctx context.Context, b *bundle.Bundle) diag.Diagnosti Severity: diag.Error, Summary: "grant principal is required", Locations: v.Locations(), - Paths: []dyn.Path{slices.Clone(p)}, + Paths: dyn.ToStructPaths(p), }) } if isMissingOrEmptySequence(v.Get("privileges")) { @@ -174,7 +175,7 @@ func errorForInvalidGrants(ctx context.Context, b *bundle.Bundle) diag.Diagnosti Severity: diag.Error, Summary: "grant privileges is required", Locations: v.Locations(), - Paths: []dyn.Path{slices.Clone(p)}, + Paths: dyn.ToStructPaths(p), }) } return v, nil @@ -199,15 +200,14 @@ func errorForInvalidSecretScopePermissions(ctx context.Context, b *bundle.Bundle if perm.UserName != "" || perm.GroupName != "" || perm.ServicePrincipalName != "" { continue } - path := fmt.Sprintf("resources.secret_scopes.%s.permissions[%d]", key, i) // ApplyBundlePermissions rebuilds permissions via convert.FromTyped and drops // per-entry locations, so point at the scope. diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "secret scope permission principal is required", Detail: "Set one of user_name, group_name or service_principal_name", - Locations: b.Config.GetLocations("resources.secret_scopes." + key), - Paths: []dyn.Path{dyn.MustPathFromString(path)}, + Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "secret_scopes", key)), + Paths: structpath.NewPathSlice("resources", "secret_scopes", key, "permissions", i), }) } } diff --git a/bundle/config/validate/scripts.go b/bundle/config/validate/scripts.go index b0f25ee5368..323db769d23 100644 --- a/bundle/config/validate/scripts.go +++ b/bundle/config/validate/scripts.go @@ -44,7 +44,7 @@ func (f *validateScripts) Apply(ctx context.Context, b *bundle.Bundle) diag.Diag diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("Script %s has no content", k), - Paths: []dyn.Path{contentPath}, + Paths: dyn.ToStructPaths(contentPath), }) continue } @@ -80,7 +80,7 @@ from "content" with $NAME: MY_VAR: ${var.foo} content: echo "$MY_VAR"`, Locations: v.Locations(), - Paths: []dyn.Path{p}, + Paths: dyn.ToStructPaths(p), }} } @@ -105,7 +105,7 @@ func validateScriptEnv(b *bundle.Bundle, key string, env map[string]string) diag Severity: diag.Error, Summary: fmt.Sprintf("${%s} cannot be used in scripts.%s.env.%s; only ${bundle.*}, ${workspace.*}, and ${var.*} are resolved before scripts execute", refPath, key, name), Locations: v.Locations(), - Paths: []dyn.Path{envValuePath}, + Paths: dyn.ToStructPaths(envValuePath), }) } } diff --git a/bundle/config/validate/single_node_cluster.go b/bundle/config/validate/single_node_cluster.go index f4f6f0e5954..88b7536f80b 100644 --- a/bundle/config/validate/single_node_cluster.go +++ b/bundle/config/validate/single_node_cluster.go @@ -128,7 +128,7 @@ func (m *singleNodeCluster) Apply(ctx context.Context, b *bundle.Bundle) diag.Di Summary: singleNodeWarningSummary, Detail: singleNodeWarningDetail, Locations: v.Locations(), - Paths: []dyn.Path{p}, + Paths: dyn.ToStructPaths(p), } if showSingleNodeClusterWarning(ctx, v) { diff --git a/bundle/config/validate/single_node_cluster_test.go b/bundle/config/validate/single_node_cluster_test.go index 1e622a3a0e1..f4494ae259c 100644 --- a/bundle/config/validate/single_node_cluster_test.go +++ b/bundle/config/validate/single_node_cluster_test.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/pipelines" @@ -122,7 +123,7 @@ func TestValidateSingleNodeClusterFailForInteractiveClusters(t *testing.T) { Summary: singleNodeWarningSummary, Detail: singleNodeWarningDetail, Locations: []dyn.Location{{File: "a.yml", Line: 1, Column: 1}}, - Paths: []dyn.Path{dyn.NewPath(dyn.Key("resources"), dyn.Key("clusters"), dyn.Key("foo"))}, + Paths: structpath.NewPathSlice("resources", "clusters", "foo"), }, }, diags) }) @@ -171,7 +172,7 @@ func TestValidateSingleNodeClusterFailForJobClusters(t *testing.T) { Summary: singleNodeWarningSummary, Detail: singleNodeWarningDetail, Locations: []dyn.Location{{File: "b.yml", Line: 1, Column: 1}}, - Paths: []dyn.Path{dyn.MustPathFromString("resources.jobs.foo.job_clusters[0].new_cluster")}, + Paths: structpath.MustParsePaths("resources.jobs.foo.job_clusters[0].new_cluster"), }, }, diags) }) @@ -220,7 +221,7 @@ func TestValidateSingleNodeClusterFailForJobTaskClusters(t *testing.T) { Summary: singleNodeWarningSummary, Detail: singleNodeWarningDetail, Locations: []dyn.Location{{File: "c.yml", Line: 1, Column: 1}}, - Paths: []dyn.Path{dyn.MustPathFromString("resources.jobs.foo.tasks[0].new_cluster")}, + Paths: structpath.MustParsePaths("resources.jobs.foo.tasks[0].new_cluster"), }, }, diags) }) @@ -266,7 +267,7 @@ func TestValidateSingleNodeClusterFailForPipelineClusters(t *testing.T) { Summary: singleNodeWarningSummary, Detail: singleNodeWarningDetail, Locations: []dyn.Location{{File: "d.yml", Line: 1, Column: 1}}, - Paths: []dyn.Path{dyn.MustPathFromString("resources.pipelines.foo.clusters[0]")}, + Paths: structpath.MustParsePaths("resources.pipelines.foo.clusters[0]"), }, }, diags) }) @@ -319,7 +320,7 @@ func TestValidateSingleNodeClusterFailForJobForEachTaskCluster(t *testing.T) { Summary: singleNodeWarningSummary, Detail: singleNodeWarningDetail, Locations: []dyn.Location{{File: "e.yml", Line: 1, Column: 1}}, - Paths: []dyn.Path{dyn.MustPathFromString("resources.jobs.foo.tasks[0].for_each_task.task.new_cluster")}, + Paths: structpath.MustParsePaths("resources.jobs.foo.tasks[0].for_each_task.task.new_cluster"), }, }, diags) }) diff --git a/bundle/config/validate/unique_resource_keys.go b/bundle/config/validate/unique_resource_keys.go index 12c13fd1a86..cff5ef3ccff 100644 --- a/bundle/config/validate/unique_resource_keys.go +++ b/bundle/config/validate/unique_resource_keys.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) // This mutator validates that: @@ -36,7 +37,7 @@ func (m *uniqueResourceKeys) Apply(ctx context.Context, b *bundle.Bundle) diag.D type metadata struct { locations []dyn.Location - paths []dyn.Path + paths []*structpath.PathNode } // Maps of key to the paths and locations the resource / script is defined at. @@ -50,7 +51,7 @@ func (m *uniqueResourceKeys) Apply(ctx context.Context, b *bundle.Bundle) diag.D } } - mv.paths = append(mv.paths, dyn.NewPath(dyn.Key(prefix)).Append(p...)) + mv.paths = append(mv.paths, dyn.ToStructPath(dyn.NewPath(dyn.Key(prefix)).Append(p...))) mv.locations = append(mv.locations, v.Locations()...) resourceAndScriptMetadata[k] = mv @@ -109,7 +110,7 @@ func (m *uniqueResourceKeys) Apply(ctx context.Context, b *bundle.Bundle) diag.D } return cmp.Compare(a.Column, b.Column) }) - slices.SortFunc(v.paths, func(a, b dyn.Path) int { + slices.SortFunc(v.paths, func(a, b *structpath.PathNode) int { return cmp.Compare(a.String(), b.String()) }) diff --git a/bundle/config/validate/validate_artifact_path.go b/bundle/config/validate/validate_artifact_path.go index e7070b20b02..33e0a024c7a 100644 --- a/bundle/config/validate/validate_artifact_path.go +++ b/bundle/config/validate/validate_artifact_path.go @@ -13,6 +13,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/apierr" ) @@ -68,8 +69,7 @@ func findVolumeInBundle(r config.Root, catalogName, schemaName, volumeName strin if v.SchemaName != schemaName && !isSchemaDefinedInBundle { continue } - pathString := "resources.volumes." + k - return dyn.MustPathFromString(pathString), r.GetLocations(pathString), true + return dyn.Path{dyn.Key("resources"), dyn.Key("volumes"), dyn.Key(k)}, r.GetLocationsOf(structpath.NewPath(nil, "resources", "volumes", k)), true } return nil, nil, false } @@ -86,7 +86,7 @@ func (v *validateArtifactPath) Apply(ctx context.Context, b *bundle.Bundle) diag Summary: s, Severity: diag.Error, Locations: b.Config.GetLocations("workspace.artifact_path"), - Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, + Paths: structpath.NewPathSlice("workspace", "artifact_path"), }, } } @@ -118,7 +118,7 @@ this bundle but which has not been deployed yet. Please first deploy the volume using 'bundle deploy' and then switch over to using it in the artifact_path.`, Locations: slices.Concat(b.Config.GetLocations("workspace.artifact_path"), locations), - Paths: append([]dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, path), + Paths: append(structpath.NewPathSlice("workspace", "artifact_path"), dyn.ToStructPath(path)), }} } diff --git a/bundle/config/validate/validate_artifact_path_test.go b/bundle/config/validate/validate_artifact_path_test.go index 08f50e39fb8..4f68f645990 100644 --- a/bundle/config/validate/validate_artifact_path_test.go +++ b/bundle/config/validate/validate_artifact_path_test.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/databricks/databricks-sdk-go/service/catalog" @@ -56,10 +57,10 @@ func TestValidateArtifactPathWithVolumeInBundle(t *testing.T) { {File: "file", Line: 1, Column: 1}, {File: "file", Line: 2, Column: 2}, }, - Paths: []dyn.Path{ - dyn.MustPathFromString("workspace.artifact_path"), - dyn.MustPathFromString("resources.volumes.foo"), - }, + Paths: structpath.MustParsePaths( + "workspace.artifact_path", + "resources.volumes.foo", + ), Detail: `You are using a volume in your artifact_path that is managed by this bundle but which has not been deployed yet. Please first deploy the volume using 'bundle deploy' and then switch over to using it in @@ -83,7 +84,7 @@ func TestValidateArtifactPath(t *testing.T) { Severity: diag.Error, Summary: expected, Locations: []dyn.Location{{File: "file", Line: 1, Column: 1}}, - Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, + Paths: structpath.NewPathSlice("workspace", "artifact_path"), }}, diags) } @@ -170,7 +171,7 @@ func TestValidateArtifactPathWithInvalidPaths(t *testing.T) { Severity: diag.Error, Summary: "expected UC volume path to be in the format /Volumes////..., got " + p, Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}, - Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, + Paths: structpath.NewPathSlice("workspace", "artifact_path"), }}, diags) } } diff --git a/bundle/config/validate/validate_dashboard_etags.go b/bundle/config/validate/validate_dashboard_etags.go index 53c428af1dd..1715f22cb84 100644 --- a/bundle/config/validate/validate_dashboard_etags.go +++ b/bundle/config/validate/validate_dashboard_etags.go @@ -6,7 +6,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) func ValidateDashboardEtags() bundle.ReadOnlyMutator { @@ -27,8 +27,8 @@ func (v *validateDashboardEtags) Apply(ctx context.Context, b *bundle.Bundle) di { Severity: diag.Error, Summary: fmt.Sprintf("dashboard %q has an etag set. Etags must not be set in bundle configuration", dashboard.DisplayName), - Paths: []dyn.Path{dyn.MustPathFromString("resources.dashboards." + k)}, - Locations: b.Config.GetLocations("resources.dashboards." + k), + Paths: structpath.NewPathSlice("resources", "dashboards", k), + Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "dashboards", k)), }, } } diff --git a/bundle/config/validate/validate_deployment_fields.go b/bundle/config/validate/validate_deployment_fields.go index f551c54661b..d1ca337f4ed 100644 --- a/bundle/config/validate/validate_deployment_fields.go +++ b/bundle/config/validate/validate_deployment_fields.go @@ -7,7 +7,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) func ValidateDeploymentFields() bundle.ReadOnlyMutator { @@ -26,29 +26,29 @@ func (v *validateDeploymentFields) Apply(_ context.Context, b *bundle.Bundle) di // deployment_id and version_id identify the bundle deployment and its version // in the Deployment Metadata Service. The CLI sets them on every deploy, so a // value provided by hand would be overwritten; reject it up front. - reject := func(resourcePath, field, value string) { + reject := func(resourceType, name, field, value string) { if value == "" { return } - path := resourcePath + ".deployment." + field + path := structpath.NewPath(nil, "resources", resourceType, name, "deployment", field) diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: field + " must not be set in bundle configuration; it is managed by Declarative Automation Bundles", - Paths: []dyn.Path{dyn.MustPathFromString(path)}, - Locations: b.Config.GetLocations(path), + Paths: []*structpath.PathNode{path}, + Locations: b.Config.GetLocationsOf(path), }) } for name, job := range b.Config.Resources.Jobs { if d := job.Deployment; d != nil { - reject("resources.jobs."+name, "deployment_id", d.DeploymentId) - reject("resources.jobs."+name, "version_id", d.VersionId) + reject("jobs", name, "deployment_id", d.DeploymentId) + reject("jobs", name, "version_id", d.VersionId) } } for name, pipeline := range b.Config.Resources.Pipelines { if d := pipeline.Deployment; d != nil { - reject("resources.pipelines."+name, "deployment_id", d.DeploymentId) - reject("resources.pipelines."+name, "version_id", d.VersionId) + reject("pipelines", name, "deployment_id", d.DeploymentId) + reject("pipelines", name, "version_id", d.VersionId) } } diff --git a/bundle/config/validate/validate_engine.go b/bundle/config/validate/validate_engine.go index 056e0bd50a6..8f6020772ed 100644 --- a/bundle/config/validate/validate_engine.go +++ b/bundle/config/validate/validate_engine.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) type validateEngine struct{ bundle.RO } @@ -30,7 +31,7 @@ func (v *validateEngine) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos Severity: diag.Warning, Summary: "bundle.terraform is deprecated and has no effect: " + engine.TerraformRemovedSummary, Locations: tf.Locations(), - Paths: []dyn.Path{dyn.MustPathFromString("bundle.terraform")}, + Paths: structpath.NewPathSlice("bundle", "terraform"), }) } diff --git a/bundle/config/validate/validate_genie_space_etags.go b/bundle/config/validate/validate_genie_space_etags.go index 84e9626c0f4..963e572346f 100644 --- a/bundle/config/validate/validate_genie_space_etags.go +++ b/bundle/config/validate/validate_genie_space_etags.go @@ -6,7 +6,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) func ValidateGenieSpaceEtags() bundle.ReadOnlyMutator { @@ -29,8 +29,8 @@ func (v *validateGenieSpaceEtags) Apply(ctx context.Context, b *bundle.Bundle) d { Severity: diag.Error, Summary: fmt.Sprintf("genie space %q has an etag set. Etags must not be set in bundle configuration", genieSpace.Title), - Paths: []dyn.Path{dyn.MustPathFromString("resources.genie_spaces." + k)}, - Locations: b.Config.GetLocations("resources.genie_spaces." + k), + Paths: structpath.NewPathSlice("resources", "genie_spaces", k), + Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "genie_spaces", k)), }, } } diff --git a/bundle/config/validate/validate_job_run_idempotency_token.go b/bundle/config/validate/validate_job_run_idempotency_token.go index ea0d608f990..61e07e302ca 100644 --- a/bundle/config/validate/validate_job_run_idempotency_token.go +++ b/bundle/config/validate/validate_job_run_idempotency_token.go @@ -7,7 +7,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) func ValidateJobRunIdempotencyToken() bundle.ReadOnlyMutator { @@ -32,12 +32,12 @@ func (v *validateJobRunIdempotencyToken) Apply(_ context.Context, b *bundle.Bund } // The CLI mints the token; a configured one would also remain reserved after // the run is deleted and break the next deploy. - path := "resources.job_runs." + name + ".idempotency_token" + path := structpath.NewPath(nil, "resources", "job_runs", name, "idempotency_token") diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: "idempotency_token must not be set in bundle configuration; the CLI sets it on each run-now request", - Paths: []dyn.Path{dyn.MustPathFromString(path)}, - Locations: b.Config.GetLocations(path), + Paths: []*structpath.PathNode{path}, + Locations: b.Config.GetLocationsOf(path), }) } diff --git a/bundle/config/validate/validate_sync_patterns.go b/bundle/config/validate/validate_sync_patterns.go index 8488aefebd2..c888a05a7dd 100644 --- a/bundle/config/validate/validate_sync_patterns.go +++ b/bundle/config/validate/validate_sync_patterns.go @@ -7,9 +7,9 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/fileset" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structpath" "golang.org/x/sync/errgroup" ) @@ -26,16 +26,16 @@ func (v *validateSyncPatterns) Name() string { func (v *validateSyncPatterns) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { s := b.Config.Sync - checkPatterns(ctx, s.Exclude, "sync.exclude", b) + checkPatterns(ctx, s.Exclude, structpath.NewPath(nil, "sync", "exclude"), b) if logdiag.HasError(ctx) { return nil } - checkPatterns(ctx, s.Include, "sync.include", b) + checkPatterns(ctx, s.Include, structpath.NewPath(nil, "sync", "include"), b) return nil } -func checkPatterns(ctx context.Context, patterns []string, path string, b *bundle.Bundle) { +func checkPatterns(ctx context.Context, patterns []string, path *structpath.PathNode, b *bundle.Bundle) { var errs errgroup.Group for index, pattern := range patterns { @@ -57,12 +57,12 @@ func checkPatterns(ctx context.Context, patterns []string, path string, b *bundl } if len(all) == 0 { - path := fmt.Sprintf("%s[%d]", path, index) + path := structpath.NewPath(path, index) logdiag.LogDiag(ctx, diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("Pattern %s does not match any files", pattern), - Locations: b.Config.GetLocations(path), - Paths: []dyn.Path{dyn.MustPathFromString(path)}, + Locations: b.Config.GetLocationsOf(path), + Paths: []*structpath.PathNode{path}, }) } return nil diff --git a/bundle/config/validate/validate_volume_path.go b/bundle/config/validate/validate_volume_path.go index 265a0bfd53b..c440dfbf06f 100644 --- a/bundle/config/validate/validate_volume_path.go +++ b/bundle/config/validate/validate_volume_path.go @@ -7,7 +7,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) type validateVolumePath struct { @@ -24,11 +24,12 @@ func (m *validateVolumePath) Apply(ctx context.Context, b *bundle.Bundle) diag.D pathChecks := []struct { path string configName string + configPath *structpath.PathNode }{ - {b.Config.Workspace.RootPath, "workspace.root_path"}, - {b.Config.Workspace.FilePath, "workspace.file_path"}, - {b.Config.Workspace.StatePath, "workspace.state_path"}, - {b.Config.Workspace.ResourcePath, "workspace.resource_path"}, + {b.Config.Workspace.RootPath, "workspace.root_path", structpath.NewPath(nil, "workspace", "root_path")}, + {b.Config.Workspace.FilePath, "workspace.file_path", structpath.NewPath(nil, "workspace", "file_path")}, + {b.Config.Workspace.StatePath, "workspace.state_path", structpath.NewPath(nil, "workspace", "state_path")}, + {b.Config.Workspace.ResourcePath, "workspace.resource_path", structpath.NewPath(nil, "workspace", "resource_path")}, } // Check each path @@ -38,8 +39,8 @@ func (m *validateVolumePath) Apply(ctx context.Context, b *bundle.Bundle) diag.D Severity: diag.Error, Summary: fmt.Sprintf("%s %s starts with /Volumes. /Volumes can only be used with workspace.artifact_path.", check.configName, check.path), Detail: "For more information, see https://docs.databricks.com/aws/en/dev-tools/bundles/settings#workspace", - Locations: b.Config.GetLocations(check.configName), - Paths: []dyn.Path{dyn.MustPathFromString(check.configName)}, + Locations: b.Config.GetLocationsOf(check.configPath), + Paths: []*structpath.PathNode{check.configPath}, }) // Return early for root path validation diff --git a/bundle/deploy/check_dashboards_modified_remotely.go b/bundle/deploy/check_dashboards_modified_remotely.go index 559206b876c..f362f7967e8 100644 --- a/bundle/deploy/check_dashboards_modified_remotely.go +++ b/bundle/deploy/check_dashboards_modified_remotely.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/libs/agent" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) type dashboardState struct { @@ -72,15 +73,15 @@ func (l *checkDashboardsModifiedRemotely) Apply(ctx context.Context, b *bundle.B continue } - path := dyn.MustPathFromString("resources.dashboards." + dashboard.Name) - loc := b.Config.GetLocation(path.String()) + path := structpath.NewPath(nil, "resources", "dashboards", dashboard.Name) + loc := b.Config.GetLocationOf(path) actual, err := b.WorkspaceClient(ctx).Lakeview.GetByDashboardId(ctx, dashboard.ID) if err != nil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("failed to get dashboard %q", dashboard.Name), Detail: err.Error(), - Paths: []dyn.Path{path}, + Paths: []*structpath.PathNode{path}, Locations: []dyn.Location{loc}, }) continue @@ -109,7 +110,7 @@ func (l *checkDashboardsModifiedRemotely) Apply(ctx context.Context, b *bundle.B "\n" + "To overwrite the remote changes with your local version, use --force.\n" + "The remote modifications will be lost." + agent.AgentNotice(), - Paths: []dyn.Path{path}, + Paths: []*structpath.PathNode{path}, Locations: []dyn.Location{loc}, }) } diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go index ab1da3df68d..2bdc81b60a2 100644 --- a/bundle/libraries/expand_glob_references.go +++ b/bundle/libraries/expand_glob_references.go @@ -19,7 +19,7 @@ func matchError(p dyn.Path, l []dyn.Location, message string) diag.Diagnostic { Severity: diag.Error, Summary: message, Locations: l, - Paths: []dyn.Path{p}, + Paths: dyn.ToStructPaths(p), } } diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go index 8fb140d7aa4..1d50c946a27 100644 --- a/bundle/libraries/same_name_libraries.go +++ b/bundle/libraries/same_name_libraries.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) type checkForSameNameLibraries struct{} @@ -26,7 +27,7 @@ var patterns = []dyn.Pattern{ type libData struct { fullPath string locations []dyn.Location - paths []dyn.Path + paths []*structpath.PathNode otherPaths []string } @@ -56,12 +57,12 @@ func (c checkForSameNameLibraries) Apply(ctx context.Context, b *bundle.Bundle) libs[lib] = &libData{ fullPath: libPath, locations: []dyn.Location{libraryValue.Location()}, - paths: []dyn.Path{p}, + paths: dyn.ToStructPaths(p), otherPaths: []string{}, } } else if lp.fullPath != libPath { lp.locations = append(lp.locations, libraryValue.Location()) - lp.paths = append(lp.paths, p) + lp.paths = append(lp.paths, dyn.ToStructPath(p)) lp.otherPaths = append(lp.otherPaths, libPath) } diff --git a/bundle/render/render_text_output_test.go b/bundle/render/render_text_output_test.go index 80c5eb417c5..3857803a406 100644 --- a/bundle/render/render_text_output_test.go +++ b/bundle/render/render_text_output_test.go @@ -13,6 +13,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -203,7 +204,7 @@ func TestRenderDiagnostics(t *testing.T) { Severity: diag.Error, Detail: "'name' is required", Summary: "failed to load xxx", - Paths: []dyn.Path{dyn.MustPathFromString("resources.jobs.xxx")}, + Paths: structpath.NewPathSlice("resources", "jobs", "xxx"), }, }, expected: "Error: failed to load xxx\n" + @@ -218,11 +219,11 @@ func TestRenderDiagnostics(t *testing.T) { Severity: diag.Error, Detail: "'name' is required", Summary: "failed to load xxx", - Paths: []dyn.Path{ - dyn.MustPathFromString("resources.jobs.xxx"), - dyn.MustPathFromString("resources.jobs.yyy"), - dyn.MustPathFromString("resources.jobs.zzz"), - }, + Paths: structpath.MustParsePaths( + "resources.jobs.xxx", + "resources.jobs.yyy", + "resources.jobs.zzz", + ), }, }, expected: "Error: failed to load xxx\n" + @@ -239,10 +240,10 @@ func TestRenderDiagnostics(t *testing.T) { Severity: diag.Recommendation, Summary: "summary", Detail: "detail", - Paths: []dyn.Path{ - dyn.MustPathFromString("resources.jobs.xxx"), - dyn.MustPathFromString("resources.jobs.yyy"), - }, + Paths: structpath.MustParsePaths( + "resources.jobs.xxx", + "resources.jobs.yyy", + ), Locations: []dyn.Location{ {File: "foo.yaml", Line: 1, Column: 2}, {File: "bar.yaml", Line: 3, Column: 4}, diff --git a/bundle/tests/validate_test.go b/bundle/tests/validate_test.go index e10f240aaff..449f2bd37c0 100644 --- a/bundle/tests/validate_test.go +++ b/bundle/tests/validate_test.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -27,10 +28,10 @@ func TestValidateUniqueResourceIdentifiers(t *testing.T) { {File: "validate/duplicate_resource_names_in_root_job_and_pipeline/databricks.yml", Line: 10, Column: 7}, {File: "validate/duplicate_resource_names_in_root_job_and_pipeline/databricks.yml", Line: 13, Column: 7}, }, - Paths: []dyn.Path{ - dyn.MustPathFromString("resources.jobs.foo"), - dyn.MustPathFromString("resources.pipelines.foo"), - }, + Paths: structpath.MustParsePaths( + "resources.jobs.foo", + "resources.pipelines.foo", + ), }, }, }, @@ -44,10 +45,10 @@ func TestValidateUniqueResourceIdentifiers(t *testing.T) { {File: "validate/duplicate_resource_names_in_root_job_and_experiment/databricks.yml", Line: 10, Column: 7}, {File: "validate/duplicate_resource_names_in_root_job_and_experiment/databricks.yml", Line: 18, Column: 7}, }, - Paths: []dyn.Path{ - dyn.MustPathFromString("resources.experiments.foo"), - dyn.MustPathFromString("resources.jobs.foo"), - }, + Paths: structpath.MustParsePaths( + "resources.experiments.foo", + "resources.jobs.foo", + ), }, }, }, @@ -61,10 +62,10 @@ func TestValidateUniqueResourceIdentifiers(t *testing.T) { {File: "validate/duplicate_resource_name_in_subconfiguration/databricks.yml", Line: 13, Column: 7}, {File: "validate/duplicate_resource_name_in_subconfiguration/resources.yml", Line: 4, Column: 7}, }, - Paths: []dyn.Path{ - dyn.MustPathFromString("resources.jobs.foo"), - dyn.MustPathFromString("resources.pipelines.foo"), - }, + Paths: structpath.MustParsePaths( + "resources.jobs.foo", + "resources.pipelines.foo", + ), }, }, }, @@ -78,9 +79,7 @@ func TestValidateUniqueResourceIdentifiers(t *testing.T) { {File: "validate/duplicate_resource_name_in_subconfiguration_job_and_job/databricks.yml", Line: 13, Column: 7}, {File: "validate/duplicate_resource_name_in_subconfiguration_job_and_job/resources.yml", Line: 4, Column: 7}, }, - Paths: []dyn.Path{ - dyn.MustPathFromString("resources.jobs.foo"), - }, + Paths: structpath.NewPathSlice("resources", "jobs", "foo"), }, }, }, @@ -94,10 +93,10 @@ func TestValidateUniqueResourceIdentifiers(t *testing.T) { {File: "validate/duplicate_resource_names_in_different_subconfiguations/resources1.yml", Line: 4, Column: 7}, {File: "validate/duplicate_resource_names_in_different_subconfiguations/resources2.yml", Line: 4, Column: 7}, }, - Paths: []dyn.Path{ - dyn.MustPathFromString("resources.jobs.foo"), - dyn.MustPathFromString("resources.pipelines.foo"), - }, + Paths: structpath.MustParsePaths( + "resources.jobs.foo", + "resources.pipelines.foo", + ), }, }, }, @@ -114,11 +113,11 @@ func TestValidateUniqueResourceIdentifiers(t *testing.T) { {File: "validate/duplicate_resource_name_in_multiple_locations/resources2.yml", Line: 4, Column: 7}, {File: "validate/duplicate_resource_name_in_multiple_locations/resources2.yml", Line: 8, Column: 7}, }, - Paths: []dyn.Path{ - dyn.MustPathFromString("resources.experiments.foo"), - dyn.MustPathFromString("resources.jobs.foo"), - dyn.MustPathFromString("resources.pipelines.foo"), - }, + Paths: structpath.MustParsePaths( + "resources.experiments.foo", + "resources.jobs.foo", + "resources.pipelines.foo", + ), }, }, }, diff --git a/libs/diag/diagnostic.go b/libs/diag/diagnostic.go index fe7090462d4..f189cf0ac6c 100644 --- a/libs/diag/diagnostic.go +++ b/libs/diag/diagnostic.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) type Diagnostic struct { @@ -24,7 +25,7 @@ type Diagnostic struct { // Paths are paths to the values in the configuration tree that the diagnostic is associated with. // It may be nil if there are no associated paths. - Paths []dyn.Path + Paths []*structpath.PathNode // A diagnostic ID. Only used for select diagnostic messages. ID ID diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go index a503fd1afc9..379baf439d1 100644 --- a/libs/dyn/convert/normalize.go +++ b/libs/dyn/convert/normalize.go @@ -85,7 +85,7 @@ func nullWarning(expected dyn.Kind, src dyn.Value, path dyn.Path) diag.Diagnosti Severity: diag.Warning, Summary: fmt.Sprintf("expected a %s value, found null", expected), Locations: []dyn.Location{src.Location()}, - Paths: []dyn.Path{path}, + Paths: dyn.ToStructPaths(path), } } @@ -94,7 +94,7 @@ func typeMismatch(expected dyn.Kind, src dyn.Value, path dyn.Path) diag.Diagnost Severity: diag.Warning, Summary: fmt.Sprintf("expected %s, found %s", expected, src.Kind()), Locations: []dyn.Location{src.Location()}, - Paths: []dyn.Path{path}, + Paths: dyn.ToStructPaths(path), } } @@ -151,7 +151,7 @@ func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen Summary: "Use 'value_from' instead of 'valueFrom'", Detail: "The field 'valueFrom' should be 'value_from' (snake_case). The 'valueFrom' field will be ignored.", Locations: pk.Locations(), - Paths: []dyn.Path{path}, + Paths: dyn.ToStructPaths(path), }) continue } @@ -161,7 +161,7 @@ func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen Severity: diag.Warning, Summary: "unknown field: " + fieldName, Locations: pk.Locations(), - Paths: []dyn.Path{path}, + Paths: dyn.ToStructPaths(path), }) } continue @@ -399,7 +399,7 @@ func (n normalizeOptions) normalizeInt(typ reflect.Type, src dyn.Value, path dyn Severity: diag.Warning, Summary: fmt.Sprintf(`cannot accurately represent "%g" as integer due to precision loss`, src.MustFloat()), Locations: []dyn.Location{src.Location()}, - Paths: []dyn.Path{path}, + Paths: dyn.ToStructPaths(path), }) } return dyn.NewValue(out, src.Locations()), nil @@ -416,7 +416,7 @@ func (n normalizeOptions) normalizeInt(typ reflect.Type, src dyn.Value, path dyn Severity: diag.Warning, Summary: fmt.Sprintf("cannot parse %q as an integer", src.MustString()), Locations: []dyn.Location{src.Location()}, - Paths: []dyn.Path{path}, + Paths: dyn.ToStructPaths(path), }) } return dyn.NewValue(out, src.Locations()), nil @@ -443,7 +443,7 @@ func (n normalizeOptions) normalizeFloat(typ reflect.Type, src dyn.Value, path d Severity: diag.Warning, Summary: fmt.Sprintf(`cannot accurately represent "%d" as floating point number due to precision loss`, src.MustInt()), Locations: []dyn.Location{src.Location()}, - Paths: []dyn.Path{path}, + Paths: dyn.ToStructPaths(path), }) } return dyn.NewValue(out, src.Locations()), nil @@ -460,7 +460,7 @@ func (n normalizeOptions) normalizeFloat(typ reflect.Type, src dyn.Value, path d Severity: diag.Warning, Summary: fmt.Sprintf("cannot parse %q as a floating point number", src.MustString()), Locations: []dyn.Location{src.Location()}, - Paths: []dyn.Path{path}, + Paths: dyn.ToStructPaths(path), }) } return dyn.NewValue(out, src.Locations()), nil diff --git a/libs/dyn/convert/normalize_test.go b/libs/dyn/convert/normalize_test.go index f5f3073aeae..b2bed72db71 100644 --- a/libs/dyn/convert/normalize_test.go +++ b/libs/dyn/convert/normalize_test.go @@ -6,6 +6,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynassert" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" ) @@ -44,7 +45,7 @@ func TestNormalizeStructElementDiagnostic(t *testing.T) { Severity: diag.Warning, Summary: `expected string, found map`, Locations: []dyn.Location{{}}, - Paths: []dyn.Path{dyn.NewPath(dyn.Key("bar"))}, + Paths: structpath.NewPathSlice("bar"), }, diags[0]) // Elements that encounter an error during normalization are dropped. @@ -81,7 +82,7 @@ func TestNormalizeStructUnknownField(t *testing.T) { {File: "hello.yaml", Line: 1, Column: 1}, {File: "world.yaml", Line: 2, Column: 2}, }, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, diags[0]) // The field that can be mapped to the struct field is retained. @@ -115,7 +116,7 @@ func TestNormalizeStructError(t *testing.T) { Severity: diag.Warning, Summary: `expected map, found string`, Locations: []dyn.Location{vin.Get("foo").Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -260,7 +261,7 @@ func TestNormalizeStructRandomStringError(t *testing.T) { Severity: diag.Warning, Summary: `expected map, found string`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -277,7 +278,7 @@ func TestNormalizeStructIntError(t *testing.T) { Severity: diag.Warning, Summary: `expected map, found int`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -306,7 +307,7 @@ func TestNormalizeMapElementDiagnostic(t *testing.T) { Severity: diag.Warning, Summary: `expected string, found map`, Locations: []dyn.Location{{}}, - Paths: []dyn.Path{dyn.NewPath(dyn.Key("bar"))}, + Paths: structpath.NewPathSlice("bar"), }, err[0]) // Elements that encounter an error during normalization are dropped. @@ -332,7 +333,7 @@ func TestNormalizeMapError(t *testing.T) { Severity: diag.Warning, Summary: `expected map, found string`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -387,7 +388,7 @@ func TestNormalizeMapRandomStringError(t *testing.T) { Severity: diag.Warning, Summary: `expected map, found string`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -400,7 +401,7 @@ func TestNormalizeMapIntError(t *testing.T) { Severity: diag.Warning, Summary: `expected map, found int`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -430,7 +431,7 @@ func TestNormalizeSliceElementDiagnostic(t *testing.T) { Severity: diag.Warning, Summary: `expected string, found map`, Locations: []dyn.Location{{}}, - Paths: []dyn.Path{dyn.NewPath(dyn.Index(2))}, + Paths: []*structpath.PathNode{structpath.NewIndex(nil, 2)}, }, err[0]) // Elements that encounter an error during normalization are dropped. @@ -454,7 +455,7 @@ func TestNormalizeSliceError(t *testing.T) { Severity: diag.Warning, Summary: `expected sequence, found string`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -509,7 +510,7 @@ func TestNormalizeSliceRandomStringError(t *testing.T) { Severity: diag.Warning, Summary: `expected sequence, found string`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -522,7 +523,7 @@ func TestNormalizeSliceIntError(t *testing.T) { Severity: diag.Warning, Summary: `expected sequence, found int`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -543,7 +544,7 @@ func TestNormalizeStringNil(t *testing.T) { Severity: diag.Warning, Summary: `expected a string value, found null`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -588,7 +589,7 @@ func TestNormalizeStringError(t *testing.T) { Severity: diag.Warning, Summary: `expected string, found map`, Locations: []dyn.Location{{}}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -609,7 +610,7 @@ func TestNormalizeBoolNil(t *testing.T) { Severity: diag.Warning, Summary: `expected a bool value, found null`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -655,7 +656,7 @@ func TestNormalizeBoolFromStringError(t *testing.T) { Severity: diag.Warning, Summary: `expected bool, found string`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -668,7 +669,7 @@ func TestNormalizeBoolError(t *testing.T) { Severity: diag.Warning, Summary: `expected bool, found map`, Locations: []dyn.Location{{}}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -689,7 +690,7 @@ func TestNormalizeIntNil(t *testing.T) { Severity: diag.Warning, Summary: `expected a int value, found null`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -710,7 +711,7 @@ func TestNormalizeIntFromFloatError(t *testing.T) { Severity: diag.Warning, Summary: `cannot accurately represent "1.5" as integer due to precision loss`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -739,7 +740,7 @@ func TestNormalizeIntFromStringError(t *testing.T) { Severity: diag.Warning, Summary: `cannot parse "abc" as an integer`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -752,7 +753,7 @@ func TestNormalizeIntError(t *testing.T) { Severity: diag.Warning, Summary: `expected int, found map`, Locations: []dyn.Location{{}}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -773,7 +774,7 @@ func TestNormalizeFloatNil(t *testing.T) { Severity: diag.Warning, Summary: `expected a float value, found null`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -798,7 +799,7 @@ func TestNormalizeFloatFromIntError(t *testing.T) { Severity: diag.Warning, Summary: `cannot accurately represent "9007199254740993" as floating point number due to precision loss`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -827,7 +828,7 @@ func TestNormalizeFloatFromStringError(t *testing.T) { Severity: diag.Warning, Summary: `cannot parse "abc" as a floating point number`, Locations: []dyn.Location{vin.Location()}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } @@ -840,7 +841,7 @@ func TestNormalizeFloatError(t *testing.T) { Severity: diag.Warning, Summary: `expected float, found map`, Locations: []dyn.Location{{}}, - Paths: []dyn.Path{dyn.EmptyPath}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }, err[0]) } diff --git a/libs/dyn/structpath.go b/libs/dyn/structpath.go new file mode 100644 index 00000000000..1cb740ee0d7 --- /dev/null +++ b/libs/dyn/structpath.go @@ -0,0 +1,41 @@ +package dyn + +import "github.com/databricks/cli/libs/structs/structpath" + +// ToStructPath converts a [Path] to a [structpath.PathNode]. +func ToStructPath(p Path) *structpath.PathNode { + var result *structpath.PathNode + for _, c := range p { + if c.isKey() { + result = structpath.NewStringKey(result, c.key) + } else { + result = structpath.NewIndex(result, c.index) + } + } + return result +} + +// ToStructPaths converts each of paths with [ToStructPath], e.g. for diag.Diagnostic.Paths. +func ToStructPaths(paths ...Path) []*structpath.PathNode { + out := make([]*structpath.PathNode, len(paths)) + for i, p := range paths { + out[i] = ToStructPath(p) + } + return out +} + +// FromStructPath converts a [structpath.PathNode] of keys and indices to a [Path]. +// It reports false for nodes a [Path] cannot represent (key-value selectors, wildcards). +func FromStructPath(p *structpath.PathNode) (Path, bool) { + var out Path + for _, n := range p.AsSlice() { + if k, ok := n.StringKey(); ok { + out = append(out, Key(k)) + } else if i, ok := n.Index(); ok { + out = append(out, Index(i)) + } else { + return nil, false + } + } + return out, true +} diff --git a/libs/dyn/structpath_test.go b/libs/dyn/structpath_test.go new file mode 100644 index 00000000000..6eed07a31fa --- /dev/null +++ b/libs/dyn/structpath_test.go @@ -0,0 +1,39 @@ +package dyn_test + +import ( + "testing" + + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/stretchr/testify/assert" +) + +func TestToStructPath(t *testing.T) { + assert.Nil(t, dyn.ToStructPath(nil)) + + p := dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"), dyn.Key("a_job"), dyn.Key("tasks"), dyn.Index(1), dyn.Key("task_key")) + assert.Equal(t, "resources.jobs.a_job.tasks[1].task_key", dyn.ToStructPath(p).String()) + + p = dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"), dyn.Key("${var.env}_job")) + assert.Equal(t, "resources.jobs['${var.env}_job']", dyn.ToStructPath(p).String()) +} + +func TestToStructPaths(t *testing.T) { + paths := dyn.ToStructPaths( + dyn.MustPathFromString("a.b"), + dyn.NewPath(dyn.Key("c"), dyn.Index(1)), + ) + assert.Len(t, paths, 2) + assert.Equal(t, "a.b", paths[0].String()) + assert.Equal(t, "c[1]", paths[1].String()) + assert.Empty(t, dyn.ToStructPaths()) +} + +func TestFromStructPath(t *testing.T) { + p, ok := dyn.FromStructPath(structpath.NewPath(nil, "resources", "jobs", "a.b", "tasks", 2)) + assert.True(t, ok) + assert.Equal(t, dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"), dyn.Key("a.b"), dyn.Key("tasks"), dyn.Index(2)), p) + + _, ok = dyn.FromStructPath(structpath.MustParsePath("tasks[task_key='x']")) + assert.False(t, ok) +} diff --git a/libs/flags/json_flag_test.go b/libs/flags/json_flag_test.go index 975976de34b..1e26d4b3d57 100644 --- a/libs/flags/json_flag_test.go +++ b/libs/flags/json_flag_test.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -201,7 +202,7 @@ func TestJsonUnmarshalRequestMismatch(t *testing.T) { Column: 6, }, }, - Paths: []dyn.Path{{}}, + Paths: dyn.ToStructPaths(dyn.EmptyPath), }) } @@ -259,7 +260,7 @@ func TestJsonUnmarshalWrongTypeReportsCorrectLocation(t *testing.T) { Column: 15, }, }, - Paths: []dyn.Path{dyn.NewPath(dyn.Key("job_id"))}, + Paths: structpath.NewPathSlice("job_id"), }) } @@ -284,7 +285,7 @@ func TestJsonUnmarshalArrayInsteadOfIntReportsCorrectLocation(t *testing.T) { Column: 40, }, }, - Paths: []dyn.Path{dyn.NewPath(dyn.Key("new_settings"), dyn.Key("timeout_seconds"))}, + Paths: structpath.NewPathSlice("new_settings", "timeout_seconds"), }) } diff --git a/libs/structs/structpath/path.go b/libs/structs/structpath/path.go index 92715dc09d6..266d9ade1d2 100644 --- a/libs/structs/structpath/path.go +++ b/libs/structs/structpath/path.go @@ -3,13 +3,19 @@ package structpath import ( "errors" "fmt" + "regexp" "strconv" "strings" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" ) +// baseVarDef matches a path segment of a ${...} reference. It is the grammar of +// dynvar.BaseVarDef, which structpath cannot import; TestPureReferenceMatchesDynvar +// keeps them in agreement. +const baseVarDef = `_*\p{L}+([-_]*[\p{L}\p{N}]+)*` + +// pureReference matches a string that is a single ${...} reference. +var pureReference = regexp.MustCompile(`^\$\{(` + baseVarDef + `(\.` + baseVarDef + `(\[[0-9]+\])*)*(\[[0-9]+\])*)\}$`) + const ( // Encodes wildcard after a dot: foo.* tagDotStar = -2 @@ -163,6 +169,28 @@ func NewStringKey(prev *PathNode, fieldName string) *PathNode { return NewBracketString(prev, fieldName) } +// NewPath appends parts to prev: a string is a key (see [NewStringKey]) and an int is +// an index. Other types panic. +func NewPath(prev *PathNode, parts ...any) *PathNode { + for _, part := range parts { + switch v := part.(type) { + case string: + prev = NewStringKey(prev, v) + case int: + prev = NewIndex(prev, v) + default: + panic(fmt.Sprintf("structpath.NewPath: unsupported part %#v", part)) + } + } + return prev +} + +// NewPathSlice returns the path of parts from the root (see [NewPath]) as a one-element +// slice, e.g. for diag.Diagnostic.Paths. +func NewPathSlice(parts ...any) []*PathNode { + return []*PathNode{NewPath(nil, parts...)} +} + func NewKeyValue(prev *PathNode, key, value string) *PathNode { return &PathNode{ prev: prev, @@ -637,6 +665,15 @@ func MustParsePath(s string) *PathNode { return path } +// MustParsePaths parses each of paths like [MustParsePath], e.g. for diag.Diagnostic.Paths. +func MustParsePaths(paths ...string) []*PathNode { + out := make([]*PathNode, len(paths)) + for i, s := range paths { + out[i] = MustParsePath(s) + } + return out +} + // isReservedFieldChar checks if character is reserved and cannot be used in field names func isReservedFieldChar(ch byte) bool { switch ch { @@ -677,16 +714,12 @@ func isValidField(s string) bool { // PureReferenceToPath returns a PathNode if s is a pure variable reference, otherwise false. // This function is similar to dynvar.PureReferenceToPath but returns a *PathNode instead of dyn.Path. func PureReferenceToPath(s string) (*PathNode, bool) { - ref, ok := dynvar.NewRef(dyn.V(s)) - if !ok { - return nil, false - } - - if !ref.IsPure() { + m := pureReference.FindStringSubmatch(s) + if m == nil { return nil, false } - pattern, err := parse(ref.References()[0], false) + pattern, err := parse(m[1], false) if err != nil { return nil, false } diff --git a/libs/structs/structpath/path_test.go b/libs/structs/structpath/path_test.go index b8623db76d0..40fc0282c2b 100644 --- a/libs/structs/structpath/path_test.go +++ b/libs/structs/structpath/path_test.go @@ -1274,3 +1274,26 @@ paths: require.Error(t, err) assert.Contains(t, err.Error(), "unexpected end of input") } + +func TestMustParsePaths(t *testing.T) { + paths := MustParsePaths("bundle.terraform", "resources.jobs.foo.tasks[0]") + require.Len(t, paths, 2) + assert.Equal(t, "bundle.terraform", paths[0].String()) + assert.Equal(t, "resources.jobs.foo.tasks[0]", paths[1].String()) + assert.Empty(t, MustParsePaths()) +} + +func TestNewPath(t *testing.T) { + assert.Equal(t, "a.b['c d']", NewPath(NewStringKey(nil, "a"), "b", "c d").String()) + assert.Equal(t, "resources.jobs.j.tasks[2].job_cluster_key", NewPath(nil, "resources", "jobs", "j", "tasks", 2, "job_cluster_key").String()) + assert.Nil(t, NewPath(nil)) + assert.Equal(t, "a", NewPath(NewStringKey(nil, "a")).String()) + assert.Panics(t, func() { NewPath(nil, int64(1)) }) +} + +func TestNewPathSlice(t *testing.T) { + paths := NewPathSlice("resources", "jobs", "${var.env}_job") + require.Len(t, paths, 1) + assert.Equal(t, "resources.jobs['${var.env}_job']", paths[0].String()) + assert.Equal(t, "sync.paths[3]", NewPathSlice("sync", "paths", 3)[0].String()) +} diff --git a/libs/structs/structpath/reference_test.go b/libs/structs/structpath/reference_test.go new file mode 100644 index 00000000000..485b756c7fb --- /dev/null +++ b/libs/structs/structpath/reference_test.go @@ -0,0 +1,30 @@ +package structpath_test + +import ( + "testing" + + "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/stretchr/testify/assert" +) + +func TestPureReferenceMatchesDynvar(t *testing.T) { + for _, s := range []string{ + "${var.foo}", + "${resources.jobs.foo.tasks[1].env.key}", + "${var.résumé_2}", + "${_private.x}", + "${var.foo[0][1]}", + "${var.jobs['my_job']}", + "${var.-foo}", + "${var.foo-}", + "${var.foo}${var.bar}", + "prefix_${var.field}", + "$${var.escaped}", + "${}", + "plain_string", + } { + _, ok := structpath.PureReferenceToPath(s) + assert.Equal(t, dynvar.IsPureVariableReference(s), ok, s) + } +}