diff --git a/.agents/rules/style-guide-go.md b/.agents/rules/style-guide-go.md index ab2dad3b6e9..bedd86132ca 100644 --- a/.agents/rules/style-guide-go.md +++ b/.agents/rules/style-guide-go.md @@ -113,7 +113,7 @@ return fieldPaths ### Encoding -**RULE: When mutating an API response by round-tripping JSON, decode with `json.Decoder` + `UseNumber()`.** The naive `json.Marshal` → `map[string]any` → `Marshal` path corrupts any int64 larger than 2^53 (it degrades to a float64 mantissa — e.g. a real `spark_context_id`) and alphabetizes object keys. `libs/dyn/jsonloader` preserves key order but also lacks `UseNumber`, so it shares the int64 hazard. +**RULE: When mutating an API response by round-tripping JSON, decode with `json.Decoder` + `UseNumber()`.** The naive `json.Marshal` → `map[string]any` → `Marshal` path corrupts any int64 larger than 2^53 (it degrades to a float64 mantissa — e.g. a real `spark_context_id`) and alphabetizes object keys. **RULE: Be careful with `encoding/csv` `Writer.UseCRLF = true`.** It rewrites both record terminators AND embedded newlines inside quoted fields to `\r\n`, so tests for quoted multiline fields must expect `\r\n`, not just the line endings between rows. @@ -191,7 +191,7 @@ func NewService(deps ServiceDeps) *Service { ... } ### Configuration patterns -- Bundle config uses `dyn.Value` for dynamic typing +- Bundle config is typed structs; `libs/structs/structvar` carries `${...}` references and source locations - Config loading supports includes, variable interpolation, and target overrides - Schema generation is automated from Go struct tags diff --git a/.github/workflows/python_push.yml b/.github/workflows/python_push.yml index 1a391719665..a2db548ddac 100644 --- a/.github/workflows/python_push.yml +++ b/.github/workflows/python_push.yml @@ -39,7 +39,7 @@ jobs: acceptance/bundle/python/ \ go.mod \ go.sum \ - libs/dyn/dynvar/testdata/reference_vectors.json \ + libs/structs/structvar/testdata/reference_vectors.json \ python/ \ tools/task/ \ ) diff --git a/.nextchanges/bundles/6954.md b/.nextchanges/bundles/6954.md new file mode 100644 index 00000000000..73ef9f56fe8 --- /dev/null +++ b/.nextchanges/bundles/6954.md @@ -0,0 +1 @@ +* Speed up bundle commands on large bundles: configuration is kept as typed structs instead of being converted to and from a dynamic tree (`bundle validate` and `bundle plan` with 1000 jobs are about 10x faster and use about 3.5x less memory). ([#6954](https://github.com/databricks/cli/pull/6954)) diff --git a/AGENTS.md b/AGENTS.md index 02b63df8169..c0cf2704b63 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,7 +14,7 @@ This is the Databricks CLI, a command-line interface for interacting with Databr **RULE: Keep each PR focused on one change.** If you notice an unrelated cleanup, bug fix, or refactor while making your primary change, leave it alone or put it in a separate PR. Reviewers consistently ask to split mixed PRs, especially when a dependency bump or schema diff rides along with a feature change. -**RULE: Before adding a new helper, search the codebase for an existing one.** Common homes: `libs/` (shared utilities), `libs/databrickscfg/` (config), `libs/git/`, `libs/filer/`, `libs/cmdio/` (CLI I/O, spinners, prompts), `libs/env/` (env vars), `libs/testserver/`, `libs/structpath/` and `libs/dyn/` (path / dynamic values), `acceptance/bin/` (acceptance test helpers), `internal/mocks/` (generated mocks). A function that duplicates an existing name and signature in the same package is a compile error waiting to happen; grep before you name. +**RULE: Before adding a new helper, search the codebase for an existing one.** Common homes: `libs/` (shared utilities), `libs/databrickscfg/` (config), `libs/git/`, `libs/filer/`, `libs/cmdio/` (CLI I/O, spinners, prompts), `libs/env/` (env vars), `libs/testserver/`, `libs/structs/structpath/` and `libs/structs/structvar/` (paths / typed config values), `acceptance/bin/` (acceptance test helpers), `internal/mocks/` (generated mocks). A function that duplicates an existing name and signature in the same package is a compile error waiting to happen; grep before you name. # Development Commands @@ -73,7 +73,7 @@ GIT_EDITOR=true GIT_SEQUENCE_EDITOR=true VISUAL=true GIT_PAGER=cat git rebase or - `bundle/phases/` - High-level deployment phases **libs/** - Shared libraries and utilities -- `libs/dyn/` - Dynamic configuration value manipulation +- `libs/structs/structvar/` - Typed configuration values with references and locations - `libs/filer/` - File system abstraction (local, DBFS, workspace) - `libs/auth/` - Databricks authentication handling - `libs/sync/` - File synchronization between local and remote diff --git a/acceptance/bundle/run/job-ref-job-id/databricks.yml b/acceptance/bundle/run/job-ref-job-id/databricks.yml new file mode 100644 index 00000000000..3b06625845f --- /dev/null +++ b/acceptance/bundle/run/job-ref-job-id/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: test-bundle + +resources: + jobs: + bar: + name: job bar + foo: + name: job foo + tasks: + - task_key: job_task + run_job_task: + job_id: ${resources.jobs.bar.id} diff --git a/acceptance/bundle/run/job-ref-job-id/out.test.toml b/acceptance/bundle/run/job-ref-job-id/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/run/job-ref-job-id/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/run/job-ref-job-id/output.txt b/acceptance/bundle/run/job-ref-job-id/output.txt new file mode 100644 index 00000000000..570e4b56af2 --- /dev/null +++ b/acceptance/bundle/run/job-ref-job-id/output.txt @@ -0,0 +1,14 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created jobs.bar +Created jobs.foo +Files: 1 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== run resolves the referenced job id into the int job_id field +>>> [CLI] bundle run foo +Run URL: [DATABRICKS_URL]/jobs/[FOO_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] + +[TIMESTAMP] "job foo" RUNNING +[TIMESTAMP] "job foo" TERMINATED SUCCESS diff --git a/acceptance/bundle/run/job-ref-job-id/script b/acceptance/bundle/run/job-ref-job-id/script new file mode 100644 index 00000000000..cc3b902fa05 --- /dev/null +++ b/acceptance/bundle/run/job-ref-job-id/script @@ -0,0 +1,5 @@ +trace $CLI bundle deploy +replace_ids.py + +title "run resolves the referenced job id into the int job_id field" +trace $CLI bundle run foo diff --git a/acceptance/bundle/run/job-ref-job-id/test.toml b/acceptance/bundle/run/job-ref-job-id/test.toml new file mode 100644 index 00000000000..3c887dd20bf --- /dev/null +++ b/acceptance/bundle/run/job-ref-job-id/test.toml @@ -0,0 +1,3 @@ +[[Repls]] +Old = '/runs/\d+' +New = '/runs/[RUN_ID]' diff --git a/acceptance/invariant_test.go b/acceptance/invariant_test.go index ad8e1d14022..99ac4f4707c 100644 --- a/acceptance/invariant_test.go +++ b/acceptance/invariant_test.go @@ -8,10 +8,9 @@ import ( "testing" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" ) const invariantConfigsDir = "bundle/invariant/configs" @@ -88,31 +87,22 @@ func scanInvariantConfigs(t *testing.T) (present, withPermissions, withGrants ma contents, err := os.ReadFile(path) require.NoError(t, err) - v, err := yamlloader.LoadYAML(path, strings.NewReader(string(contents))) - require.NoError(t, err, "failed to parse %s", path) - - resources := v.Get("resources") - if resources.Kind() != dyn.KindMap { - // Some configs (e.g. PyDABs) declare resources outside of YAML. - continue + var doc struct { + Resources map[string]any `yaml:"resources"` } + require.NoError(t, yaml.Unmarshal(contents, &doc), "failed to parse %s", path) - for _, group := range resources.MustMap().Pairs() { - groupName := group.Key.MustString() + // Some configs (e.g. PyDABs) declare resources outside of YAML. + for groupName, group := range doc.Resources { present[groupName] = true - if group.Value.Kind() != dyn.KindMap { - continue - } - for _, resource := range group.Value.MustMap().Pairs() { - cfg := resource.Value - if cfg.Kind() != dyn.KindMap { - continue - } - if cfg.Get("permissions").Kind() != dyn.KindInvalid { + resources, _ := group.(map[string]any) + for _, resource := range resources { + cfg, _ := resource.(map[string]any) + if _, ok := cfg["permissions"]; ok { withPermissions[groupName] = true } - if cfg.Get("grants").Kind() != dyn.KindInvalid { + if _, ok := cfg["grants"]; ok { withGrants[groupName] = true } } diff --git a/bundle/apps/validate.go b/bundle/apps/validate.go index 29f6f056a53..d2b4355c635 100644 --- a/bundle/apps/validate.go +++ b/bundle/apps/validate.go @@ -8,8 +8,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go/service/apps" ) @@ -94,32 +94,26 @@ type appResourceReference struct { // hasPermissions checks if a bundle resource at the given dyn path has a non-empty permissions list. func hasPermissions(b *bundle.Bundle, resourcePath string) bool { - pv, err := dyn.Get(b.Config.Value(), resourcePath+".permissions") + pv, err := valueAt(b, resourcePath+".permissions") if err != nil { return false } - s, ok := pv.AsSequence() - return ok && len(s) > 0 + for range pv.Sequence() { + return true + } + return false } // hasAppSPInPermissions checks if any permission entry for the given resource // references the app's service principal via variable interpolation. func hasAppSPInPermissions(b *bundle.Bundle, resourcePath, appKey string) bool { appSPRef := fmt.Sprintf("${resources.apps.%s.service_principal_client_id}", appKey) - pv, err := dyn.Get(b.Config.Value(), resourcePath+".permissions") + pv, err := valueAt(b, resourcePath+".permissions") if err != nil { return false } - s, ok := pv.AsSequence() - if !ok { - return false - } - for _, entry := range s { - spn, err := dyn.Get(entry, "service_principal_name") - if err != nil { - continue - } - if str, ok := spn.AsString(); ok && str == appSPRef { + for _, entry := range pv.Sequence() { + if str, ok := entry.Get("service_principal_name").AsString(); ok && str == appSPRef { return true } } @@ -188,3 +182,12 @@ func (v *validate) Name() string { func Validate() bundle.Mutator { return &validate{} } + +// valueAt returns the view of the configuration value at path. +func valueAt(b *bundle.Bundle, path string) (structvar.View, error) { + p, err := structpath.ParsePath(path) + if err != nil { + return structvar.View{}, err + } + return b.Config.View().Lookup(p), nil +} diff --git a/bundle/apps/validate_test.go b/bundle/apps/validate_test.go index 3285851ce91..89b4e1d5907 100644 --- a/bundle/apps/validate_test.go +++ b/bundle/apps/validate_test.go @@ -11,7 +11,6 @@ import ( "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/vfs" "github.com/databricks/databricks-sdk-go/service/apps" "github.com/stretchr/testify/require" @@ -48,7 +47,7 @@ func TestAppsValidateSameSourcePath(t *testing.T) { }, } - bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(tmpDir, "databricks.yml")}}) + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(tmpDir, "databricks.yml")}}) diags := bundle.ApplySeq(t.Context(), b, mutator.TranslatePaths(), Validate()) require.Len(t, diags, 1) @@ -205,7 +204,7 @@ func TestAppsValidateResourcePermissionsWarning(t *testing.T) { }, } - bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(tmpDir, "databricks.yml")}}) + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(tmpDir, "databricks.yml")}}) diags := bundle.ApplySeq(t.Context(), b, Validate()) warnings := diags.Filter(diag.Warning) @@ -249,7 +248,7 @@ func TestAppsValidateBothSourceCodePathAndGitSource(t *testing.T) { }, } - bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(tmpDir, "databricks.yml")}}) + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(tmpDir, "databricks.yml")}}) diags := bundle.ApplySeq(t.Context(), b, mutator.TranslatePaths(), Validate()) require.Len(t, diags, 1) diff --git a/bundle/artifacts/expand_globs.go b/bundle/artifacts/expand_globs.go index e7cd5cc1308..6b1c46a8f43 100644 --- a/bundle/artifacts/expand_globs.go +++ b/bundle/artifacts/expand_globs.go @@ -6,16 +6,17 @@ import ( "path/filepath" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/patchwheel" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -func createGlobError(v dyn.Value, p dyn.Path, message string) diag.Diagnostic { +func createGlobError(v structvar.View, p *structpath.PathNode, message string) diag.Diagnostic { // The pattern contained in v is an absolute path. // Make it relative to the value's location to make it more readable. - source := v.MustString() + source, _ := v.AsString() if l := v.Location(); l.File != "" { rel, err := filepath.Rel(filepath.Dir(l.File), source) if err == nil { @@ -26,8 +27,8 @@ func createGlobError(v dyn.Value, p dyn.Path, message string) diag.Diagnostic { return diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("%s: %s", source, message), - Locations: []dyn.Location{v.Location()}, - Paths: dyn.ToStructPaths(p), + Locations: []diag.Location{v.Location()}, + Paths: []*structpath.PathNode{p}, } } @@ -42,82 +43,78 @@ func (e expandGlobs) Name() string { func (e expandGlobs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // Base path for this mutator. // This path is set with the list of expanded globs when done. - base := dyn.NewPath( - dyn.Key("artifacts"), - dyn.Key(e.name), - dyn.Key("files"), - ) + base := structpath.NewPath(nil, "artifacts", e.name, "files") // Pattern to match the source key in the files sequence. - pattern := dyn.NewPatternFromPath(base).Append( - dyn.AnyIndex(), - dyn.Key("source"), - ) + pattern := structpath.NewPatternStringKey(structpath.NewPatternBracketStar(structpath.NewPatternStringKey(structpath.NewPatternStringKey(structpath.NewPatternStringKey(nil, "artifacts"), e.name), "files")), "source") + + artifact := b.Config.Artifacts[e.name] + if artifact == nil { + return nil + } var diags diag.Diagnostics - err := b.Config.Mutate(func(rootv dyn.Value) (dyn.Value, error) { - var output []dyn.Value - _, err := dyn.MapByPattern(rootv, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - if v.Kind() != dyn.KindString { - return v, nil - } - - source := v.MustString() - - // Expand any glob reference in files source path - matches, err := filepath.Glob(source) - if err != nil { - diags = diags.Append(createGlobError(v, p, err.Error())) - - // Drop this value from the list; this does not matter since we've raised an error anyway - return v, nil - } - - // Note, we're applying this for all artifact types, not just "whl". - // Rationale: - // 1. type is optional - // 2. if you have wheels in other artifact type, maybe you still want the filter logic? impossible to say. - matches = patchwheel.FilterLatestWheels(ctx, matches) - - if len(matches) == 1 && matches[0] == source { - // No glob expansion was performed. - // Keep node unchanged. We need to ensure that "patched" field remains and not wiped out by code below. - parent, err := dyn.GetByPath(rootv, p[0:len(p)-1]) - if err != nil { - log.Debugf(ctx, "Failed to get parent of %s", p.String()) - } else { - output = append(output, parent) - } - return v, nil - } - - if len(matches) == 0 { - diags = diags.Append(createGlobError(v, p, "no matching files")) - - // Drop this value from the list; this does not matter since we've raised an error anyway - return v, nil - } - - for _, match := range matches { - output = append(output, dyn.V( - map[string]dyn.Value{ - "source": dyn.NewValue(match, v.Locations()), - }, - )) - } - - return v, nil - }) - - if err != nil || diags.HasError() { - return rootv, err + var output []config.ArtifactFile + var sources [][]int + var locations [][]diag.Location + err := structvar.ForEach(b.Config.View(), pattern, func(np *structpath.PathNode, v structvar.View) error { + if v.Kind() != structvar.KindString { + return nil + } + + index, _ := np.Parent().Index() + source, _ := v.AsString() + + // Expand any glob reference in files source path + matches, err := filepath.Glob(source) + if err != nil { + diags = diags.Append(createGlobError(v, np, err.Error())) + + // Drop this value from the list; this does not matter since we've raised an error anyway + return nil + } + + // Note, we're applying this for all artifact types, not just "whl". + // Rationale: + // 1. type is optional + // 2. if you have wheels in other artifact type, maybe you still want the filter logic? impossible to say. + matches = patchwheel.FilterLatestWheels(ctx, matches) + + if len(matches) == 1 && matches[0] == source { + // No glob expansion was performed. + // Keep node unchanged. We need to ensure that "patched" field remains and not wiped out by code below. + output = append(output, artifact.Files[index]) + sources = append(sources, []int{index}) + locations = append(locations, nil) + return nil } - // Set the expanded globs back into the configuration. - return dyn.SetByPath(rootv, base, dyn.V(output)) + if len(matches) == 0 { + diags = diags.Append(createGlobError(v, np, "no matching files")) + + // Drop this value from the list; this does not matter since we've raised an error anyway + return nil + } + + for _, match := range matches { + output = append(output, config.ArtifactFile{Source: match}) + sources = append(sources, []int{index}) + locations = append(locations, v.Locations()) + } + + return nil }) - if err != nil { - diags = diags.Extend(diag.FromErr(err)) + if err != nil || diags.HasError() { + return diags.Extend(diag.FromErr(err)) + } + + // Set the expanded globs back into the configuration. + artifact.Files = output + b.Config.UpdateSequence(base, sources) + for i, locs := range locations { + if locs != nil { + b.Config.SetLocations(structpath.NewPath(base, i, "source"), locs) + } } return diags diff --git a/bundle/artifacts/prepare.go b/bundle/artifacts/prepare.go index ae93f946c19..d5330208eeb 100644 --- a/bundle/artifacts/prepare.go +++ b/bundle/artifacts/prepare.go @@ -14,7 +14,6 @@ import ( "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/bundle/metrics" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/python" @@ -44,7 +43,7 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics Severity: diag.Error, Summary: "Artifact not properly configured", Detail: "please specify artifact properties", - Locations: []dyn.Location{l}, + Locations: []diag.Location{l}, }) continue } @@ -62,7 +61,7 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics logdiag.LogError(ctx, fmt.Errorf("artifact %q: a tgz artifact needs a `files` entry naming the output path", artifactName)) } - l := b.Config.GetLocation("artifacts." + artifactName) + l := b.Config.DefinitionLocation("artifacts." + artifactName) dirPath := filepath.Dir(l.File) // Check if source paths are absolute, if not, make them absolute diff --git a/bundle/bundle_test.go b/bundle/bundle_test.go index 9bd667afe62..05df7f14228 100644 --- a/bundle/bundle_test.go +++ b/bundle/bundle_test.go @@ -9,7 +9,6 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/logdiag" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -157,9 +156,6 @@ func TestBundleGetResourceConfigJobsPointer(t *testing.T) { }, } - // Initialize the dynamic representation so GetResourceConfig can query it. - require.NoError(t, rootCfg.Mutate(func(v dyn.Value) (dyn.Value, error) { return v, nil })) - b := &Bundle{Config: rootCfg} res, err := b.Config.GetResourceConfig("resources.jobs.my_job") diff --git a/bundle/config/loader/process_include.go b/bundle/config/loader/process_include.go index dd8932caeed..9fc6e13dfc5 100644 --- a/bundle/config/loader/process_include.go +++ b/bundle/config/loader/process_include.go @@ -10,11 +10,11 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -func validateFileFormat(configRoot dyn.Value, filePath string) diag.Diagnostics { +func validateFileFormat(configRoot structvar.View, filePath string) diag.Diagnostics { for _, resourceDescription := range config.SupportedResources() { singularName := resourceDescription.SingularName @@ -29,10 +29,10 @@ func validateFileFormat(configRoot dyn.Value, filePath string) diag.Diagnostics return nil } -func validateSingleResourceDefined(configRoot dyn.Value, ext, typ string) diag.Diagnostics { +func validateSingleResourceDefined(configRoot structvar.View, ext, typ string) diag.Diagnostics { type resource struct { - path dyn.Path - value dyn.Value + path *structpath.PathNode + value structvar.View typ string key string } @@ -41,34 +41,34 @@ func validateSingleResourceDefined(configRoot dyn.Value, ext, typ string) diag.D supportedResources := config.SupportedResources() // Gather all resources defined in the resources block. - _, err := dyn.MapByPattern( + err := structvar.ForEach( configRoot, - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), - func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + structpath.MustParsePattern("resources.*.*"), + func(np *structpath.PathNode, v structvar.View) error { // The key for the resource, e.g. "my_job" for jobs.my_job. - k := p[2].Key() + k := np.KeyAt(2) // The type of the resource, e.g. "job" for jobs.my_job. - typ := supportedResources[p[1].Key()].SingularName + typ := supportedResources[np.KeyAt(1)].SingularName - resources = append(resources, resource{path: p, value: v, typ: typ, key: k}) - return v, nil + resources = append(resources, resource{path: np, value: v, typ: typ, key: k}) + return nil }) if err != nil { return diag.FromErr(err) } // Gather all resources defined in a target block. - _, err = dyn.MapByPattern( + err = structvar.ForEach( configRoot, - dyn.NewPattern(dyn.Key("targets"), dyn.AnyKey(), dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), - func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + structpath.MustParsePattern("targets.*.resources.*.*"), + func(np *structpath.PathNode, v structvar.View) error { // The key for the resource, e.g. "my_job" for jobs.my_job. - k := p[4].Key() + k := np.KeyAt(4) // The type of the resource, e.g. "job" for jobs.my_job. - typ := supportedResources[p[3].Key()].SingularName + typ := supportedResources[np.KeyAt(3)].SingularName - resources = append(resources, resource{path: p, value: v, typ: typ, key: k}) - return v, nil + resources = append(resources, resource{path: np, value: v, typ: typ, key: k}) + return nil }) if err != nil { return diag.FromErr(err) @@ -108,14 +108,14 @@ func validateSingleResourceDefined(configRoot dyn.Value, ext, typ string) diag.D detail.WriteString(l) } - var locations []dyn.Location + var locations []diag.Location var paths []*structpath.PathNode for _, rr := range resources { locations = append(locations, rr.value.Locations()...) - paths = append(paths, dyn.ToStructPath(rr.path)) + paths = append(paths, rr.path) } // Sort the locations and paths to make the output deterministic. - slices.SortFunc(locations, func(a, b dyn.Location) int { + slices.SortFunc(locations, func(a, b diag.Location) int { return cmp.Compare(a.String(), b.String()) }) slices.SortFunc(paths, func(a, b *structpath.PathNode) int { @@ -151,15 +151,29 @@ func (m *processInclude) Name() string { } func (m *processInclude) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { - this, diags := config.Load(m.fullPath) + this, diags := m.load() if diags.HasError() { return diags } + err := b.Config.Merge(this) + if err != nil { + diags = diags.Extend(diag.FromErr(err)) + } + return diags +} + +// load loads and validates the included file. +func (m *processInclude) load() (*config.Root, diag.Diagnostics) { + this, diags := config.Load(m.fullPath) + if diags.HasError() { + return nil, diags + } + // Add any diagnostics associated with the file format. - diags = append(diags, validateFileFormat(this.Value(), m.relPath)...) + diags = append(diags, validateFileFormat(this.View(), m.relPath)...) if diags.HasError() { - return diags + return nil, diags } if len(this.Include) > 0 { @@ -173,9 +187,5 @@ Only includes defined in databricks.yml are applied.`, }) } - err := b.Config.Merge(this) - if err != nil { - diags = diags.Extend(diag.FromErr(err)) - } - return diags + return this, diags } diff --git a/bundle/config/loader/process_include_test.go b/bundle/config/loader/process_include_test.go index 28c95ac905b..16ed455e622 100644 --- a/bundle/config/loader/process_include_test.go +++ b/bundle/config/loader/process_include_test.go @@ -8,7 +8,6 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/loader" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -68,7 +67,7 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) { Severity: diag.Recommendation, Summary: "define a single pipeline in a file with the .pipeline.yaml extension.", Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n", - Locations: []dyn.Location{ + Locations: []diag.Location{ {File: "testdata/format_not_match/single_job.pipeline.yaml", Line: 11, Column: 11}, {File: "testdata/format_not_match/single_job.pipeline.yaml", Line: 4, Column: 7}, }, @@ -83,7 +82,7 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) { Severity: diag.Recommendation, Summary: "define a single job in a file with the .job.yml extension.", Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n - pipeline1 (pipeline)\n", - Locations: []dyn.Location{ + Locations: []diag.Location{ {File: "testdata/format_not_match/job_and_pipeline.job.yml", Line: 11, Column: 11}, {File: "testdata/format_not_match/job_and_pipeline.job.yml", Line: 4, Column: 7}, }, @@ -98,7 +97,7 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) { Severity: diag.Recommendation, Summary: "define a single experiment in a file with the .experiment.yml extension.", Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n - pipeline1 (pipeline)\n", - Locations: []dyn.Location{ + Locations: []diag.Location{ {File: "testdata/format_not_match/job_and_pipeline.experiment.yml", Line: 11, Column: 11}, {File: "testdata/format_not_match/job_and_pipeline.experiment.yml", Line: 4, Column: 7}, }, @@ -113,7 +112,7 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) { Severity: diag.Recommendation, Summary: "define a single job in a file with the .job.yml extension.", Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n - job2 (job)\n", - Locations: []dyn.Location{ + Locations: []diag.Location{ {File: "testdata/format_not_match/two_jobs.job.yml", Line: 4, Column: 7}, {File: "testdata/format_not_match/two_jobs.job.yml", Line: 7, Column: 7}, }, @@ -128,7 +127,7 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) { Severity: diag.Recommendation, Summary: "define a single job in a file with the .job.yml extension.", Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n - job2 (job)\n", - Locations: []dyn.Location{ + Locations: []diag.Location{ {File: "testdata/format_not_match/second_job_in_target.job.yml", Line: 11, Column: 11}, {File: "testdata/format_not_match/second_job_in_target.job.yml", Line: 4, Column: 7}, }, @@ -143,7 +142,7 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) { Severity: diag.Recommendation, Summary: "define a single job in a file with the .job.yml extension.", Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n - job2 (job)\n", - Locations: []dyn.Location{ + Locations: []diag.Location{ {File: "testdata/format_not_match/two_jobs_in_target.job.yml", Line: 6, Column: 11}, {File: "testdata/format_not_match/two_jobs_in_target.job.yml", Line: 8, Column: 11}, }, @@ -170,7 +169,7 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) { - registered_model1 (registered_model) - schema1 (schema) `, - Locations: []dyn.Location{ + Locations: []diag.Location{ {File: "testdata/format_not_match/multiple_resources.model_serving_endpoint.yml", Line: 12, Column: 7}, {File: "testdata/format_not_match/multiple_resources.model_serving_endpoint.yml", Line: 14, Column: 7}, {File: "testdata/format_not_match/multiple_resources.model_serving_endpoint.yml", Line: 18, Column: 7}, diff --git a/bundle/config/loader/process_root_includes.go b/bundle/config/loader/process_root_includes.go index 655ca27162b..36166fa8512 100644 --- a/bundle/config/loader/process_root_includes.go +++ b/bundle/config/loader/process_root_includes.go @@ -10,7 +10,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/logdiag" ) type processRootIncludes struct{} @@ -38,7 +38,7 @@ func hasGlobCharacters(path string) (string, bool) { } func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - var out []bundle.Mutator + var out []*processInclude // Preserve the raw include patterns before they are replaced below with the // expanded list of loaded files, so IsFileIncluded can re-match against them. @@ -127,39 +127,40 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. slices.Sort(includes) files = append(files, includes...) for _, include := range includes { - out = append(out, ProcessInclude(filepath.Join(b.BundleRootPath, include), include)) + out = append(out, &processInclude{fullPath: filepath.Join(b.BundleRootPath, include), relPath: include}) } } - // Swap out the original includes list with the expanded globs. This goes through - // Mutate so the dynamic tree is updated too: the includes below are applied without - // their own mutator scope, so nothing converts the typed field back into the dynamic - // tree afterwards, and the next ToTyped would otherwise restore the raw patterns. - err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - // Include is omitempty in the typed configuration, so an empty list must stay - // absent from the dynamic tree rather than be written as []. - if len(files) == 0 { - return dyn.DropKeys(root, []string{"include"}) - } - - includeValues := make([]dyn.Value, 0, len(files)) - for _, file := range files { - includeValues = append(includeValues, dyn.V(file)) - } - return dyn.Set(root, "include", dyn.NewValue(includeValues, root.Get("include").Locations())) - }) - if err != nil { - return diag.FromErr(err) + // Swap out the original includes list with the expanded globs. + // Include is omitempty in the typed configuration, so an empty list must stay + // absent rather than be written as []. + b.Config.Include = nil + if len(files) > 0 { + b.Config.Include = files } // Track number of bundle YAML (or JSON) files in the configuration. The +1 is there // to account for the root databricks.yaml file. b.Metrics.ConfigurationFileCount = int64(len(files)) + 1 - // ProcessInclude merges into the configuration via [config.Root.Merge], so it does - // not need its own mutator scope. Giving each included file one would re-convert the - // whole accumulated configuration per file, making load quadratic in the number of - // included files (~20 minutes for 6000 files). - bundle.ApplySeqInScopeContext(ctx, b, out...) + // Load all included files first and merge them into the configuration at once. + // Merging them one by one would convert the whole accumulated configuration per + // file, making load quadratic in the number of included files. + var roots []*config.Root + for _, m := range out { + this, diags := m.load() + for _, d := range diags { + logdiag.LogDiag(ctx, d) + } + if diags.HasError() { + return nil + } + roots = append(roots, this) + } + + err := b.Config.Merge(roots...) + if err != nil { + return diag.FromErr(err) + } return nil } diff --git a/bundle/config/loader/process_root_includes_test.go b/bundle/config/loader/process_root_includes_test.go index e7a850a835b..7403c1e01c5 100644 --- a/bundle/config/loader/process_root_includes_test.go +++ b/bundle/config/loader/process_root_includes_test.go @@ -9,7 +9,7 @@ import ( "github.com/databricks/cli/bundle/config/loader" "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structvar" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -119,7 +119,7 @@ func TestProcessRootIncludesUpdatesDynamicValue(t *testing.T) { require.NoError(t, diags.Error()) assert.Equal(t, []string{"a.yml"}, b.Config.Include) - assert.Equal(t, []any{"a.yml"}, b.Config.Value().Get("include").AsAny()) + assert.Equal(t, []any{"a.yml"}, b.Config.View().Get("include").AsAny()) } // An empty include list must stay absent from the dynamic tree: the typed field is @@ -139,7 +139,7 @@ func TestProcessRootIncludesEmptyOmitsDynamicValue(t *testing.T) { diags := bundle.Apply(t.Context(), b, loader.ProcessRootIncludes()) require.NoError(t, diags.Error()) assert.Empty(t, b.Config.Include) - assert.Equal(t, dyn.KindInvalid, b.Config.Value().Get("include").Kind()) + assert.Equal(t, structvar.KindInvalid, b.Config.View().Get("include").Kind()) } func TestProcessRootIncludesNotExists(t *testing.T) { diff --git a/libs/dyn/dynloc/locations.go b/bundle/config/loctable/locations.go similarity index 60% rename from libs/dyn/dynloc/locations.go rename to bundle/config/loctable/locations.go index 5c8e22f0939..e7307e04c1e 100644 --- a/libs/dyn/dynloc/locations.go +++ b/bundle/config/loctable/locations.go @@ -1,4 +1,4 @@ -package dynloc +package loctable import ( "fmt" @@ -6,7 +6,7 @@ import ( "path/filepath" "slices" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/diag" ) const ( @@ -15,7 +15,7 @@ const ( Version = 1 ) -// Locations is a structure that holds location information for (a subset of) a [dyn.Value] value. +// Locations is a structure that holds location information for (a subset of) a configuration. type Locations struct { // Version is the version of the location information. Version int `json:"version"` @@ -23,9 +23,9 @@ type Locations struct { // Files is a list of file paths. Files []string `json:"files"` - // Locations maps the string representation of a [dyn.Path] to a list of 3-tuples that represent the index + // Locations maps the string representation of a path to a list of 3-tuples that represent the index // of the file in the [Files] array, followed by the line and column number. - // A single [dyn.Path] can have multiple locations (e.g. the effective location and original definition). + // A single path can have multiple locations (e.g. the effective location and original definition). Locations map[string][][]int `json:"locations"` // fileToIndex maps file paths to their index in the [Files] array. @@ -37,29 +37,6 @@ type Locations struct { basePath string } -func (l *Locations) gatherLocations(v dyn.Value) (map[string][]dyn.Location, error) { - locs := map[string][]dyn.Location{} - patterns := []dyn.Pattern{ - dyn.NewPattern(dyn.AnyKey()), // Top level fields - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey()), // Resource groups ("resources.jobs") - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), // Resources for all types ("resources.jobs.my_job") - dyn.NewPattern(dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), dyn.Key("tasks")), // Job tasks ("resources.jobs.my_job.tasks") - dyn.NewPattern(dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), dyn.Key("tasks"), dyn.AnyIndex()), // Job task items ("resources.jobs.my_job.tasks[2]") - } - - for _, pattern := range patterns { - _, err := dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - locs[p.String()] = v.Locations() - return v, nil - }) - if err != nil { - return nil, err - } - } - - return locs, nil -} - func (l *Locations) normalizeFilePath(file string) (string, error) { var err error @@ -74,7 +51,7 @@ func (l *Locations) normalizeFilePath(file string) (string, error) { return filepath.ToSlash(file), nil } -func (l *Locations) registerFileNames(locs []dyn.Location) error { +func (l *Locations) registerFileNames(locs []diag.Location) error { cache := map[string]string{} for _, loc := range locs { // Never process the same file path twice. @@ -112,7 +89,7 @@ func (l *Locations) addLocation(path, file string, line, col int) error { // Expect the file to be present in the lookup map. if _, ok := l.fileToIndex[file]; !ok { // This indicates a logic problem below, but we rather not panic. - return fmt.Errorf("dynloc: unknown file %q", file) + return fmt.Errorf("loctable: unknown file %q", file) } // Add the location to the map. @@ -124,8 +101,8 @@ func (l *Locations) addLocation(path, file string, line, col int) error { return nil } -// Build constructs a [Locations] object from a [dyn.Value]. -func Build(v dyn.Value, basePath string) (Locations, error) { +// Build constructs a [Locations] object from locations keyed by the string representation of a path. +func Build(pathToLocations map[string][]diag.Location, basePath string) (Locations, error) { l := Locations{ Version: Version, Files: make([]string, 0), @@ -136,15 +113,9 @@ func Build(v dyn.Value, basePath string) (Locations, error) { basePath: basePath, } - // Traverse the value and collect locations. - pathToLocations, err := l.gatherLocations(v) - if err != nil { - return l, err - } - // Normalize file paths and add locations. // This step adds files to the [Files] array in alphabetical order. - err = l.registerFileNames(slices.Concat(slices.Collect(maps.Values(pathToLocations))...)) + err := l.registerFileNames(slices.Concat(slices.Collect(maps.Values(pathToLocations))...)) if err != nil { return l, err } diff --git a/bundle/config/mutator/apply_source_linked_deployment_preset.go b/bundle/config/mutator/apply_source_linked_deployment_preset.go index d0eadbedd17..c4aa9bb51af 100644 --- a/bundle/config/mutator/apply_source_linked_deployment_preset.go +++ b/bundle/config/mutator/apply_source_linked_deployment_preset.go @@ -13,6 +13,10 @@ import ( type applySourceLinkedDeploymentPreset struct{} +func targetPresetPath(target string) *structpath.PathNode { + return structpath.NewPath(nil, "targets", target, "presets", "source_linked_deployment") +} + // Apply source-linked deployment preset func ApplySourceLinkedDeploymentPreset() *applySourceLinkedDeploymentPreset { return &applySourceLinkedDeploymentPreset{} @@ -33,13 +37,13 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle if config.IsExplicitlyEnabled((b.Config.Presets.SourceLinkedDeployment)) { if !isDatabricksWorkspace { - path := structpath.NewPath(nil, "targets", target, "presets", "source_linked_deployment") + path := targetPresetPath(target) diags = diags.Append( diag.Diagnostic{ Severity: diag.Warning, Summary: "source-linked deployment is available only in the Databricks Workspace", Paths: []*structpath.PathNode{path}, - Locations: b.Config.GetLocations(path.SkipPrefix(2).String()), + Locations: b.Config.GetLocations("presets.source_linked_deployment"), }, ) @@ -51,13 +55,13 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle b.Metrics.AddBoolValue("source_linked_set_for_non_development", b.Config.Bundle.Mode != config.Development) if b.Config.Bundle.Mode != config.Development { - path := structpath.NewPath(nil, "targets", target, "presets", "source_linked_deployment") + path := targetPresetPath(target) diags = diags.Append( diag.Diagnostic{ Severity: diag.Warning, Summary: "source-linked deployment in non-development mode is deprecated and will not be supported in a future release", Paths: []*structpath.PathNode{path}, - Locations: b.Config.GetLocations(path.SkipPrefix(2).String()), + Locations: b.Config.GetLocations("presets.source_linked_deployment"), }, ) } @@ -70,7 +74,7 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle // This mutator runs before workspace paths are defaulted so it's safe to check for the user-defined value if b.Config.Workspace.FilePath != "" && config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) { - path := structpath.NewPath(nil, "workspace", "file_path") + path := structpath.MustParsePath("workspace.file_path") diags = diags.Append( diag.Diagnostic{ Severity: diag.Warning, diff --git a/bundle/config/mutator/apply_source_linked_deployment_preset_test.go b/bundle/config/mutator/apply_source_linked_deployment_preset_test.go index c0371077537..8d50788a3b3 100644 --- a/bundle/config/mutator/apply_source_linked_deployment_preset_test.go +++ b/bundle/config/mutator/apply_source_linked_deployment_preset_test.go @@ -11,7 +11,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/libs/dbr" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/diag" "github.com/stretchr/testify/require" ) @@ -128,8 +128,8 @@ func TestApplyPresetsSourceLinkedDeployment(t *testing.T) { tt.mutateBundle(b) } - bundletest.SetLocation(b, "presets.source_linked_deployment", []dyn.Location{{File: "databricks.yml"}}) - bundletest.SetLocation(b, "workspace.file_path", []dyn.Location{{File: "databricks.yml"}}) + bundletest.SetLocation(b, "presets.source_linked_deployment", []diag.Location{{File: "databricks.yml"}}) + bundletest.SetLocation(b, "workspace.file_path", []diag.Location{{File: "databricks.yml"}}) diags := bundle.Apply(tt.ctx, b, mutator.ApplySourceLinkedDeploymentPreset()) if diags.HasError() && tt.expectedError == "" { diff --git a/bundle/config/mutator/collect_escape_telemetry.go b/bundle/config/mutator/collect_escape_telemetry.go index b24922bb5b3..dad06bb2312 100644 --- a/bundle/config/mutator/collect_escape_telemetry.go +++ b/bundle/config/mutator/collect_escape_telemetry.go @@ -2,11 +2,11 @@ package mutator import ( "context" + "reflect" "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" ) // Temporary: this mutator collects telemetry on escape patterns ($${}, $$, \${}, \$) @@ -24,12 +24,7 @@ func (*collectEscapeTelemetry) Name() string { func (*collectEscapeTelemetry) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { var hasDoubleDollarBrace, hasDoubleDollar, hasBackslashDollarBrace, hasBackslashDollar bool - _, err := dyn.Walk(b.Config.Value(), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - s, ok := v.AsString() - if !ok { - return v, nil - } - + walkStrings(reflect.ValueOf(&b.Config), func(s string) { if !hasDoubleDollarBrace && strings.Contains(s, "$${") { hasDoubleDollarBrace = true } @@ -42,12 +37,7 @@ func (*collectEscapeTelemetry) Apply(ctx context.Context, b *bundle.Bundle) diag if !hasBackslashDollar && containsBackslashDollarWithoutBrace(s) { hasBackslashDollar = true } - - return v, nil }) - if err != nil { - return diag.FromErr(err) - } if hasDoubleDollarBrace { b.Metrics.SetBoolValue("config_has_double_dollar_brace", true) @@ -65,6 +55,40 @@ func (*collectEscapeTelemetry) Apply(ctx context.Context, b *bundle.Bundle) diag return nil } +// walkStrings calls fn for every string value in the configuration value v, +// including strings stored in interface values (e.g. variable defaults). +func walkStrings(v reflect.Value, fn func(string)) { + switch v.Kind() { + case reflect.String: + fn(v.String()) + case reflect.Pointer, reflect.Interface: + if !v.IsNil() { + walkStrings(v.Elem(), fn) + } + case reflect.Struct: + t := v.Type() + for i := range t.NumField() { + f := t.Field(i) + // Like the configuration conversion, skip fields without a JSON name. + name, _, _ := strings.Cut(f.Tag.Get("json"), ",") + if !f.IsExported() || (!f.Anonymous && (name == "" || name == "-")) { + continue + } + walkStrings(v.Field(i), fn) + } + case reflect.Slice: + for i := range v.Len() { + walkStrings(v.Index(i), fn) + } + case reflect.Map: + iter := v.MapRange() + for iter.Next() { + walkStrings(iter.Value(), fn) + } + default: + } +} + // containsDoubleDollarWithoutBrace returns true if s contains "$$" not followed by "{". func containsDoubleDollarWithoutBrace(s string) bool { for i := range len(s) - 1 { diff --git a/bundle/config/mutator/compute_id_compat.go b/bundle/config/mutator/compute_id_compat.go index c29b01cbd1c..6948ba3bf25 100644 --- a/bundle/config/mutator/compute_id_compat.go +++ b/bundle/config/mutator/compute_id_compat.go @@ -2,10 +2,14 @@ package mutator import ( "context" + "maps" + "slices" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type computeIdToClusterId struct{} @@ -21,66 +25,49 @@ func (m *computeIdToClusterId) Name() string { func (m *computeIdToClusterId) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics + // Skip if "compute_id" is not set anywhere. + if b.Config.Bundle.ComputeId == "" && !slices.ContainsFunc(slices.Collect(maps.Values(b.Config.Targets)), func(t *config.Target) bool { + return t != nil && t.ComputeId != "" + }) { + return nil + } + // The "compute_id" key is set; rewrite it to "cluster_id". - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - v, d := rewriteComputeIdToClusterId(v, dyn.NewPath(dyn.Key("bundle"))) - diags = diags.Extend(d) + view := b.Config.View() + bundlePath := structpath.NewStringKey(nil, "bundle") + diags = diags.Extend(rewriteComputeIdToClusterId(b, bundlePath, view.Get("bundle"), bundlePath)) - // Check if the "compute_id" key is set in any target overrides. - return dyn.MapByPattern(v, dyn.NewPattern(dyn.Key("targets"), dyn.AnyKey()), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - v, d := rewriteComputeIdToClusterId(v, dyn.Path{}) - diags = diags.Extend(d) - return v, nil - }) + // Check if the "compute_id" key is set in any target overrides. + err := structvar.ForEach(view, structpath.MustParsePattern("targets.*"), func(p *structpath.PathNode, v structvar.View) error { + diags = diags.Extend(rewriteComputeIdToClusterId(b, p, v, nil)) + return nil }) diags = diags.Extend(diag.FromErr(err)) return diags } -func rewriteComputeIdToClusterId(v dyn.Value, p dyn.Path) (dyn.Value, diag.Diagnostics) { +// rewriteComputeIdToClusterId rewrites the "compute_id" key of the map v at path p to "cluster_id". +// The diagnostic refers to "compute_id" relative to diagPath. +func rewriteComputeIdToClusterId(b *bundle.Bundle, p *structpath.PathNode, v structvar.View, diagPath *structpath.PathNode) diag.Diagnostics { var diags diag.Diagnostics - computeIdPath := p.Append(dyn.Key("compute_id")) - computeId, err := dyn.GetByPath(v, computeIdPath) + computeId := v.Get("compute_id") // If the "compute_id" key is not set, we don't need to do anything. - if err != nil { - return v, nil - } - - if computeId.Kind() == dyn.KindInvalid { - return v, nil + if !computeId.IsValid() { + return nil } diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: "compute_id is deprecated, please use cluster_id instead", Locations: computeId.Locations(), - Paths: dyn.ToStructPaths(computeIdPath), + Paths: []*structpath.PathNode{structpath.NewStringKey(diagPath, "compute_id")}, }) - clusterIdPath := p.Append(dyn.Key("cluster_id")) - nv, err := dyn.SetByPath(v, clusterIdPath, computeId) + err := b.Config.Assign(structpath.NewStringKey(p, "cluster_id"), computeId) if err != nil { - return dyn.InvalidValue, diag.FromErr(err) + return diags.Extend(diag.FromErr(err)) } // Drop the "compute_id" key. - vout, err := dyn.Walk(nv, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - switch len(p) { - case 0: - return v, nil - case 1: - if p[0] == dyn.Key("compute_id") { - return v, dyn.ErrDrop - } - return v, nil - case 2: - if p[1] == dyn.Key("compute_id") { - return v, dyn.ErrDrop - } - } - return v, dyn.ErrSkip - }) - - diags = diags.Extend(diag.FromErr(err)) - return vout, diags + return diags.Extend(diag.FromErr(b.Config.Delete(structpath.NewStringKey(p, "compute_id")))) } diff --git a/bundle/config/mutator/drop_empty_strings.go b/bundle/config/mutator/drop_empty_strings.go index 028351ed943..243a2fb9f58 100644 --- a/bundle/config/mutator/drop_empty_strings.go +++ b/bundle/config/mutator/drop_empty_strings.go @@ -2,12 +2,12 @@ package mutator import ( "context" + "reflect" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type dropEmptyStrings struct{} @@ -17,10 +17,8 @@ type dropEmptyStrings struct{} // either literally (policy_id: "") or via a variable that resolved to "", so // this must run after variable resolution. // -// The direct engine converts the resolved config through convert.ToTyped, which -// force-sends explicitly-set zero values, defeating the omitempty tag. Dropping -// here fixes it and makes the result visible in `bundle validate -o json`, which -// serializes the dynamic value. +// An explicitly-set zero value is force-sent, defeating the omitempty tag. +// Dropping it here fixes it and makes the result visible in `bundle validate -o json`. func DropEmptyStrings() bundle.Mutator { return &dropEmptyStrings{} } @@ -30,31 +28,71 @@ func (m *dropEmptyStrings) Name() string { } func (m *dropEmptyStrings) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - var diags diag.Diagnostics - err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - root, err := dyn.Map(root, "resources", func(_ dyn.Path, resources dyn.Value) (dyn.Value, error) { - // Normalize against the resources type so omitempty is known per field. - // Only DropEmptyStrings is set: existing values are kept as-is otherwise. - out, normDiags := convert.Normalize(config.Resources{}, resources, convert.DropEmptyStrings) - diags = diags.Extend(normDiags) - return out, nil - }) + resourcesPath := structpath.NewStringKey(nil, "resources") + + // Drop the empty strings on all struct fields except the ones that must send their zero value. + var empty []*structpath.PathNode + collectEmptyStrings(b.Config.View().Get("resources"), resourcesPath, &empty) + for _, p := range empty { + if err := b.Config.Delete(p); err != nil { + return diag.FromErr(err) + } + } + + // Keep an empty apps description instead of dropping it: an app update without + // the field leaves the remote description unchanged, so removing description + // from config would never converge. Force-sending "" clears it. + appsPath := structpath.NewStringKey(resourcesPath, "apps") + var apps []string + for name, app := range b.Config.View().Lookup(appsPath).MapItems() { + if !app.Get("description").IsValid() { + apps = append(apps, name) + } + } + for _, name := range apps { + err := b.Config.Set(structpath.NewPath(appsPath, name, "description"), "") if err != nil { - return root, err + return diag.FromErr(err) + } + } + return nil +} + +// collectEmptyStrings appends to out the paths of the struct fields in v that are empty strings, +// unless the field is tagged without omitempty (i.e. its zero value must be sent). +func collectEmptyStrings(v structvar.View, p *structpath.PathNode, out *[]*structpath.PathNode) { + switch v.Kind() { + case structvar.KindSequence: + for i, e := range v.Sequence() { + collectEmptyStrings(e, structpath.NewIndex(p, i), out) + } + case structvar.KindMap: + r := v.Reflect() + for r.Kind() == reflect.Pointer { + r = r.Elem() } - // Keep an empty apps description instead of dropping it: an app update without - // the field leaves the remote description unchanged, so removing description - // from config would never converge. Force-sending "" clears it. - return dyn.MapByPattern(root, dyn.NewPattern(dyn.Key("resources"), dyn.Key("apps"), dyn.AnyKey()), func(_ dyn.Path, app dyn.Value) (dyn.Value, error) { - if _, err := dyn.Get(app, "description"); err != nil { - return dyn.Set(app, "description", dyn.V("")) + var info *structvar.StructInfo + switch { + case r.Kind() == reflect.Struct && !structvar.IsSDKNativeType(r.Type()): + i := structvar.GetStructInfo(r.Type()) + info = &i + case r.Kind() == reflect.Map: + default: + return + } + + for k, c := range v.MapItems() { + if info != nil && !info.ForceEmpty[k] { + if _, ok := info.Fields[k]; ok { + if s, ok := c.AsString(); ok && s == "" { + *out = append(*out, structpath.NewStringKey(p, k)) + continue + } + } } - return app, nil - }) - }) - if err != nil { - diags = diags.Extend(diag.FromErr(err)) + collectEmptyStrings(c, structpath.NewStringKey(p, k), out) + } + default: } - return diags } diff --git a/bundle/config/mutator/environments_compat.go b/bundle/config/mutator/environments_compat.go index fb898edea36..b00a7d824a6 100644 --- a/bundle/config/mutator/environments_compat.go +++ b/bundle/config/mutator/environments_compat.go @@ -2,11 +2,10 @@ package mutator import ( "context" - "fmt" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) type environmentsToTargets struct{} @@ -27,40 +26,26 @@ func (m *environmentsToTargets) Apply(ctx context.Context, b *bundle.Bundle) dia } // The "environments" key is set; validate and rewrite it to "targets". - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - environments := v.Get("environments") - targets := v.Get("targets") - - // Return an error if both "environments" and "targets" are set. - if environments.Kind() != dyn.KindInvalid && targets.Kind() != dyn.KindInvalid { - return dyn.InvalidValue, fmt.Errorf( - "both 'environments' and 'targets' are specified; only 'targets' should be used: %s", - environments.Location().String(), - ) - } + view := b.Config.View() + environments := view.Get("environments") + targets := view.Get("targets") + + // Return an error if both "environments" and "targets" are set. + if environments.IsValid() && targets.IsValid() { + return diag.Errorf( + "both 'environments' and 'targets' are specified; only 'targets' should be used: %s", + environments.Location().String(), + ) + } - // Rewrite "environments" to "targets". - if environments.Kind() != dyn.KindInvalid && targets.Kind() == dyn.KindInvalid { - nv, err := dyn.Set(v, "targets", environments) - if err != nil { - return dyn.InvalidValue, err - } - // Drop the "environments" key. - return dyn.Walk(nv, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - switch len(p) { - case 0: - return v, nil - case 1: - if p[0] == dyn.Key("environments") { - return v, dyn.ErrDrop - } - } - return v, dyn.ErrSkip - }) + // Rewrite "environments" to "targets" and drop the "environments" key. + if environments.IsValid() && !targets.IsValid() { + err := b.Config.Assign(structpath.NewStringKey(nil, "targets"), environments) + if err != nil { + return diag.FromErr(err) } + return diag.FromErr(b.Config.Delete(structpath.NewStringKey(nil, "environments"))) + } - return v, nil - }) - - return diag.FromErr(err) + return nil } diff --git a/bundle/config/mutator/initialize_volume_paths.go b/bundle/config/mutator/initialize_volume_paths.go index 20ea9ae6b38..fa971a92298 100644 --- a/bundle/config/mutator/initialize_volume_paths.go +++ b/bundle/config/mutator/initialize_volume_paths.go @@ -5,11 +5,9 @@ import ( "fmt" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type initializeVolumePaths struct{} @@ -32,26 +30,26 @@ func (m *initializeVolumePaths) Name() string { } func (m *initializeVolumePaths) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { - err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - pattern := dyn.NewPattern(dyn.Key("resources"), dyn.Key("volumes"), dyn.AnyKey()) - return dyn.MapByPattern(root, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - // volume_path is computed and read-only; reject a user-provided value instead of overwriting it. - if existing, ok := v.Get("volume_path").AsString(); ok && existing != "" { - return dyn.InvalidValue, fmt.Errorf("%s.volume_path is computed and read-only; remove it from the configuration", p.String()) - } + // Skip converting the configuration if there are no volumes. + if len(b.Config.Resources.Volumes) == 0 { + return nil + } - var vol resources.Volume - if err := convert.ToTyped(&vol, v); err != nil { - return dyn.InvalidValue, err - } + view := b.Config.View() + pattern := structpath.MustParsePattern("resources.volumes.*") + err := structvar.ForEach(view, pattern, func(p *structpath.PathNode, v structvar.View) error { + // volume_path is computed and read-only; reject a user-provided value instead of overwriting it. + if existing, ok := v.Get("volume_path").AsString(); ok && existing != "" { + return fmt.Errorf("%s.volume_path is computed and read-only; remove it from the configuration", p.String()) + } - // Resolve references to compute the path only; the field values in v are left untouched. - vol.CatalogName = resolveResourceReference(root, vol.CatalogName) - vol.SchemaName = resolveResourceReference(root, vol.SchemaName) - vol.Name = resolveResourceReference(root, vol.Name) + // Resolve references to compute the path only; the field values are left untouched. + vol := *b.Config.Resources.Volumes[p.KeyAt(2)] + vol.CatalogName = resolveResourceReference(view, vol.CatalogName) + vol.SchemaName = resolveResourceReference(view, vol.SchemaName) + vol.Name = resolveResourceReference(view, vol.Name) - return dyn.Set(v, "volume_path", dyn.V(vol.ComputeVolumePath())) - }) + return b.Config.Set(structpath.NewStringKey(p, "volume_path"), vol.ComputeVolumePath()) }) if err != nil { return diag.FromErr(err) @@ -62,16 +60,12 @@ func (m *initializeVolumePaths) Apply(_ context.Context, b *bundle.Bundle) diag. // resolveResourceReference resolves a pure ${resources....} reference by looking it up in root. // Values that are not such a reference, or cannot be resolved, are returned unchanged (still // containing "${"), so the caller embeds the reference verbatim to be resolved later. -func resolveResourceReference(root dyn.Value, s string) string { - p, ok := dynvar.PureReferenceToPath(s) - if !ok || p[0].Key() != "resources" { - return s - } - rv, err := dyn.GetByPath(root, p) - if err != nil { +func resolveResourceReference(root structvar.View, s string) string { + p, ok := structvar.PureReferenceToPath(s) + if !ok || p.KeyAt(0) != "resources" { return s } - rs, ok := rv.AsString() + rs, ok := root.Lookup(p).AsString() if !ok { return s } diff --git a/bundle/config/mutator/load_dbalert_files.go b/bundle/config/mutator/load_dbalert_files.go index a5fa2ff8d62..41bb604597c 100644 --- a/bundle/config/mutator/load_dbalert_files.go +++ b/bundle/config/mutator/load_dbalert_files.go @@ -11,9 +11,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/sql" ) @@ -57,26 +56,21 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia continue } - alertV, err := dyn.GetByPath(b.Config.Value(), dyn.NewPath(dyn.Key("resources"), dyn.Key("alerts"), dyn.Key(alertKey))) - if err != nil { - return diag.FromErr(err) - } + alertPath := structpath.NewPath(nil, "resources", "alerts", alertKey) + alertV := b.Config.View().Lookup(alertPath) // No other fields other than allowedInYAML should be set in the bundle YAML. - m, ok := alertV.AsMap() - if !ok { + if alertV.Kind() != structvar.KindMap { return diag.FromErr(fmt.Errorf("internal error: alert value is not a map, got %s", alertV.Kind())) } - for _, p := range m.Pairs() { - k := p.Key.MustString() - v := p.Value + for k, v := range alertV.MapItems() { if slices.Contains(allowedInYAML, k) { continue } - if v.Kind() == dyn.KindNil || v.Kind() == dyn.KindInvalid { + if v.Kind() == structvar.KindNil || v.Kind() == structvar.KindInvalid { continue } @@ -86,7 +80,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia Severity: diag.Error, Summary: fmt.Sprintf("field %s is not allowed in the bundle configuration.", k), Detail: "When a .dbalert.json is specified, only the following fields are allowed in the bundle configuration: " + strings.Join(allowedInYAML, ", "), - Paths: structpath.NewPathSlice("resources", "alerts", alertKey, k), + Paths: []*structpath.PathNode{structpath.NewStringKey(alertPath, k)}, Locations: v.Locations(), }, } @@ -108,7 +102,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia Severity: diag.Error, Summary: fmt.Sprintf("failed to read .dbalert.json file %s: %s", alert.FilePath, err), Detail: "", - Paths: structpath.NewPathSlice("resources", "alerts", alertKey, "file_path"), + Paths: []*structpath.PathNode{structpath.NewStringKey(alertPath, "file_path")}, Locations: alertV.Get("file_path").Locations(), }, } @@ -123,21 +117,21 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia Severity: diag.Error, Summary: fmt.Sprintf("failed to parse .dbalert.json file %s: %s", alert.FilePath, err), Detail: "", - Paths: structpath.NewPathSlice("resources", "alerts", alertKey, "file_path"), + Paths: []*structpath.PathNode{structpath.NewStringKey(alertPath, "file_path")}, Locations: alertV.Get("file_path").Locations(), }, } } // Check that the file does not have any variable interpolations. - if dynvar.ContainsVariableReference(string(content)) { + if structvar.ContainsVariableReference(string(content)) { return diag.Diagnostics{ { ID: diag.ID(""), Severity: diag.Error, Summary: fmt.Sprintf(".alert file %s must not contain variable interpolations.", alert.FilePath), Detail: "Please inline the alert configuration in the bundle configuration to use variables", - Paths: structpath.NewPathSlice("resources", "alerts", alertKey, "file_path"), + Paths: []*structpath.PathNode{structpath.NewStringKey(alertPath, "file_path")}, Locations: alertV.Get("file_path").Locations(), }, } diff --git a/bundle/config/mutator/load_dbalert_files_test.go b/bundle/config/mutator/load_dbalert_files_test.go index 24d20a28d29..5e8a8d896e6 100644 --- a/bundle/config/mutator/load_dbalert_files_test.go +++ b/bundle/config/mutator/load_dbalert_files_test.go @@ -10,7 +10,7 @@ import ( "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/internal/bundletest" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/diag" "github.com/databricks/databricks-sdk-go/service/sql" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -66,7 +66,7 @@ func TestLoadDBAlertFiles(t *testing.T) { }, } - bundletest.SetLocation(b, "resources.alerts.my_alert", []dyn.Location{{ + bundletest.SetLocation(b, "resources.alerts.my_alert", []diag.Location{{ File: filepath.Join(dir, "databricks.yml"), }}) @@ -109,7 +109,7 @@ func TestLoadDBAlertFilesRelativeToBundleRoot(t *testing.T) { }, } - bundletest.SetLocation(b, "resources.alerts.my_alert", []dyn.Location{{ + bundletest.SetLocation(b, "resources.alerts.my_alert", []diag.Location{{ File: filepath.Join(dir, "databricks.yml"), }}) diff --git a/bundle/config/mutator/normalize_paths.go b/bundle/config/mutator/normalize_paths.go index ecab0e812db..9c44894f48c 100644 --- a/bundle/config/mutator/normalize_paths.go +++ b/bundle/config/mutator/normalize_paths.go @@ -14,7 +14,8 @@ import ( "github.com/databricks/cli/bundle/config/mutator/paths" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type normalizePaths struct{} @@ -43,48 +44,46 @@ func (a normalizePaths) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnost // Do not normalize job task paths if using git source gitSourcePaths := collectGitSourcePaths(b) - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return paths.VisitPaths(v, func(path dyn.Path, kind paths.TranslateMode, v dyn.Value) (dyn.Value, error) { - if slices.ContainsFunc(gitSourcePaths, path.HasPrefix) { - return v, nil - } + err := paths.VisitPaths(b.Config.View(), func(path *structpath.PathNode, kind paths.TranslateMode, v structvar.View) error { + if slices.ContainsFunc(gitSourcePaths, path.HasPrefix) { + return nil + } - value, ok := v.AsString() - if !ok { - return dyn.InvalidValue, fmt.Errorf("value at %s is not a string", path.String()) - } + value, ok := v.AsString() + if !ok { + return fmt.Errorf("value at %s is not a string", path.String()) + } - newValue, err := normalizePath(value, v.Location(), b.BundleRootPath) - if err != nil { - return dyn.InvalidValue, err - } + newValue, err := normalizePath(value, v.Location(), b.BundleRootPath) + if err != nil { + return err + } - return dyn.NewValue(newValue, v.Locations()), nil - }) + return b.Config.Set(path, newValue) }) if err != nil { return diag.FromErr(fmt.Errorf("failed to normalize paths: %w", err)) } - return diag.FromErr(err) + return nil } -func collectGitSourcePaths(b *bundle.Bundle) []dyn.Path { - var jobs []dyn.Path +func collectGitSourcePaths(b *bundle.Bundle) []*structpath.PathNode { + var jobs []*structpath.PathNode for name, job := range b.Config.Resources.Jobs { if job == nil { continue } if job.GitSource != nil { - jobs = append(jobs, dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"), dyn.Key(name))) + jobs = append(jobs, structpath.NewPath(nil, "resources", "jobs", name)) } } return jobs } -func normalizePath(path string, location dyn.Location, bundleRootPath string) (string, error) { +func normalizePath(path string, location diag.Location, bundleRootPath string) (string, error) { // Handle local file paths used inside pip flags for _, flag := range libraries.PipFlagsWithLocalPaths { reqPath, ok := strings.CutPrefix(path, flag+" ") @@ -134,7 +133,7 @@ func normalizePath(path string, location dyn.Location, bundleRootPath string) (s return filepath.ToSlash(filepath.Join(relDir, path)), nil } -func locationDirectory(l dyn.Location) (string, error) { +func locationDirectory(l diag.Location) (string, error) { if l.File == "" { return "", errors.New("no file in location") } diff --git a/bundle/config/mutator/normalize_paths_test.go b/bundle/config/mutator/normalize_paths_test.go index 9ad1dcad128..faefb4b2a63 100644 --- a/bundle/config/mutator/normalize_paths_test.go +++ b/bundle/config/mutator/normalize_paths_test.go @@ -7,7 +7,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -36,21 +37,15 @@ func TestNormalizePaths(t *testing.T) { } // update config as if 'notebook_path' property is defined in resources/job_1.yml - location := dyn.Location{File: filepath.Join(tmpDir, "resources", "job_1.yml")} - path := dyn.MustPathFromString("resources.jobs.job1.tasks[0].notebook_task.notebook_path") - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.MapByPath(v, path, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { - return dyn.NewValue(value.MustString(), []dyn.Location{location}), nil - }) - }) - require.NoError(t, err) + location := diag.Location{File: filepath.Join(tmpDir, "resources", "job_1.yml")} + path := structpath.MustParsePath("resources.jobs.job1.tasks[0].notebook_task.notebook_path") + b.Config.SetLocations(path, []diag.Location{location}) diags := bundle.Apply(t.Context(), b, m) require.NoError(t, diags.Error()) - newValue, err := dyn.GetByPath(b.Config.Value(), path) - require.NoError(t, err) - require.Equal(t, "src/notebook.py", newValue.MustString()) + require.Equal(t, "src/notebook.py", b.Config.Resources.Jobs["job1"].Tasks[0].NotebookTask.NotebookPath) + require.Equal(t, []diag.Location{location}, b.Config.LocationsAt(path)) } func TestNormalizePaths_jobRunOnFileChange(t *testing.T) { @@ -75,38 +70,32 @@ func TestNormalizePaths_jobRunOnFileChange(t *testing.T) { BundleRootPath: tmpDir, } - location := dyn.Location{File: filepath.Join(tmpDir, "resources", "run.yml")} - path := dyn.MustPathFromString("resources.job_runs.run1.lifecycle.triggers[0].on_file_change") - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.MapByPath(v, path, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { - return dyn.NewValue(value.MustString(), []dyn.Location{location}), nil - }) - }) - require.NoError(t, err) + location := diag.Location{File: filepath.Join(tmpDir, "resources", "run.yml")} + path := structpath.MustParsePath("resources.job_runs.run1.lifecycle.triggers[0].on_file_change") + b.Config.SetLocations(path, []diag.Location{location}) diags := bundle.Apply(t.Context(), b, m) require.NoError(t, diags.Error()) - newValue, err := dyn.GetByPath(b.Config.Value(), path) - require.NoError(t, err) - require.Equal(t, "data/*.txt", newValue.MustString()) + require.Equal(t, "data/*.txt", *b.Config.Resources.JobRuns["run1"].Lifecycle.Triggers[0].OnFileChange) + require.Equal(t, []diag.Location{location}, b.Config.LocationsAt(path)) } func TestNormalizePath_absolutePath(t *testing.T) { - value, err := normalizePath("/notebook.py", dyn.Location{}, "/tmp") + value, err := normalizePath("/notebook.py", diag.Location{}, "/tmp") assert.NoError(t, err) assert.Equal(t, "/notebook.py", value) } func TestNormalizePath_url(t *testing.T) { - value, err := normalizePath("s3:///path/to/notebook.py", dyn.Location{}, "/tmp") + value, err := normalizePath("s3:///path/to/notebook.py", diag.Location{}, "/tmp") assert.NoError(t, err) assert.Equal(t, "s3:///path/to/notebook.py", value) } func TestNormalizePath_requirementsFile(t *testing.T) { tmpDir := t.TempDir() - location := dyn.Location{File: filepath.Join(tmpDir, "resources", "job_1.yml")} + location := diag.Location{File: filepath.Join(tmpDir, "resources", "job_1.yml")} value, err := normalizePath("-r ../requirements.txt", location, tmpDir) assert.NoError(t, err) assert.Equal(t, "-r requirements.txt", value) @@ -118,21 +107,21 @@ func TestNormalizePath_requirementsFile(t *testing.T) { func TestNormalizePath_environmentDependency(t *testing.T) { tmpDir := t.TempDir() - location := dyn.Location{File: filepath.Join(tmpDir, "resources", "job_1.yml")} + location := diag.Location{File: filepath.Join(tmpDir, "resources", "job_1.yml")} value, err := normalizePath("-e ../file.py", location, tmpDir) assert.NoError(t, err) assert.Equal(t, "-e file.py", value) } func TestLocationDirectory(t *testing.T) { - loc := dyn.Location{File: "file", Line: 1, Column: 2} + loc := diag.Location{File: "file", Line: 1, Column: 2} dir, err := locationDirectory(loc) assert.NoError(t, err) assert.Equal(t, ".", dir) } func TestLocationDirectoryNoFile(t *testing.T) { - loc := dyn.Location{} + loc := diag.Location{} _, err := locationDirectory(loc) assert.Error(t, err) } diff --git a/bundle/config/mutator/paths/alert_paths_visitor.go b/bundle/config/mutator/paths/alert_paths_visitor.go index 1f1a010c798..91265736264 100644 --- a/bundle/config/mutator/paths/alert_paths_visitor.go +++ b/bundle/config/mutator/paths/alert_paths_visitor.go @@ -1,18 +1,14 @@ package paths import ( - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -func VisitAlertPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { - pattern := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("alerts"), - dyn.AnyKey(), - dyn.Key("file_path"), - ) +func VisitAlertPaths(root structvar.View, fn VisitFunc) error { + pattern := structpath.MustParsePattern("resources.alerts.*.file_path") - return dyn.MapByPattern(value, pattern, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { + return structvar.ForEach(root, pattern, func(path *structpath.PathNode, value structvar.View) error { return fn(path, TranslateModeLocalRelative, value) }) } diff --git a/bundle/config/mutator/paths/app_paths_visitor.go b/bundle/config/mutator/paths/app_paths_visitor.go index 21a7e467ae2..25040263f67 100644 --- a/bundle/config/mutator/paths/app_paths_visitor.go +++ b/bundle/config/mutator/paths/app_paths_visitor.go @@ -1,18 +1,14 @@ package paths import ( - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -func VisitAppPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { - pattern := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("apps"), - dyn.AnyKey(), - dyn.Key("source_code_path"), - ) +func VisitAppPaths(root structvar.View, fn VisitFunc) error { + pattern := structpath.MustParsePattern("resources.apps.*.source_code_path") - return dyn.MapByPattern(value, pattern, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { + return structvar.ForEach(root, pattern, func(path *structpath.PathNode, value structvar.View) error { return fn(path, TranslateModeDirectory, value) }) } diff --git a/bundle/config/mutator/paths/app_paths_visitor_test.go b/bundle/config/mutator/paths/app_paths_visitor_test.go index 9ec01aeaf83..bf08e3d94d2 100644 --- a/bundle/config/mutator/paths/app_paths_visitor_test.go +++ b/bundle/config/mutator/paths/app_paths_visitor_test.go @@ -5,7 +5,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/apps" "github.com/stretchr/testify/assert" ) @@ -24,9 +24,7 @@ func TestAppPathsVisitor(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitAppPaths) - expected := []dyn.Path{ - dyn.MustPathFromString("resources.apps.app0.source_code_path"), - } + expected := structpath.NewPathSlice("resources", "apps", "app0", "source_code_path") assert.ElementsMatch(t, expected, actual) } diff --git a/bundle/config/mutator/paths/artifact_paths_visitor.go b/bundle/config/mutator/paths/artifact_paths_visitor.go index c37ae19fee2..4f934e42ba8 100644 --- a/bundle/config/mutator/paths/artifact_paths_visitor.go +++ b/bundle/config/mutator/paths/artifact_paths_visitor.go @@ -1,42 +1,37 @@ package paths import ( - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type artifactRewritePattern struct { - pattern dyn.Pattern + pattern *structpath.PatternNode mode TranslateMode } func artifactRewritePatterns() []artifactRewritePattern { // Base pattern to match all artifacts. - base := dyn.NewPattern( - dyn.Key("artifacts"), - dyn.AnyKey(), - ) + base := "artifacts.*" // Compile list of configuration paths to rewrite. return []artifactRewritePattern{ { - pattern: base.Append(dyn.Key("path")), + pattern: structpath.MustParsePattern(base + ".path"), mode: TranslateModeLocalAbsoluteDirectory, }, } } -func VisitArtifactPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { - var err error - newValue := value - +func VisitArtifactPaths(root structvar.View, fn VisitFunc) error { for _, rewritePattern := range artifactRewritePatterns() { - newValue, err = dyn.MapByPattern(newValue, rewritePattern.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + err := structvar.ForEach(root, rewritePattern.pattern, func(p *structpath.PathNode, v structvar.View) error { return fn(p, rewritePattern.mode, v) }) if err != nil { - return dyn.InvalidValue, err + return err } } - return newValue, nil + return nil } diff --git a/bundle/config/mutator/paths/artifact_paths_visitor_test.go b/bundle/config/mutator/paths/artifact_paths_visitor_test.go index 5d43036b813..980b284cb11 100644 --- a/bundle/config/mutator/paths/artifact_paths_visitor_test.go +++ b/bundle/config/mutator/paths/artifact_paths_visitor_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" ) @@ -18,9 +18,7 @@ func TestArtifactPathsVisitor(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitArtifactPaths) - expected := []dyn.Path{ - dyn.MustPathFromString("artifacts.artifact0.path"), - } + expected := structpath.NewPathSlice("artifacts", "artifact0", "path") assert.ElementsMatch(t, expected, actual) } diff --git a/bundle/config/mutator/paths/dashboard_paths_visitor.go b/bundle/config/mutator/paths/dashboard_paths_visitor.go index b0a25dfe2ff..40c9de427bd 100644 --- a/bundle/config/mutator/paths/dashboard_paths_visitor.go +++ b/bundle/config/mutator/paths/dashboard_paths_visitor.go @@ -1,18 +1,14 @@ package paths import ( - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -func VisitDashboardPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { - pattern := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("dashboards"), - dyn.AnyKey(), - dyn.Key("file_path"), - ) +func VisitDashboardPaths(root structvar.View, fn VisitFunc) error { + pattern := structpath.MustParsePattern("resources.dashboards.*.file_path") - return dyn.MapByPattern(value, pattern, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { + return structvar.ForEach(root, pattern, func(path *structpath.PathNode, value structvar.View) error { return fn(path, TranslateModeLocalRelative, value) }) } diff --git a/bundle/config/mutator/paths/dashboard_paths_visitor_test.go b/bundle/config/mutator/paths/dashboard_paths_visitor_test.go index 4827e49d3ff..da360776db6 100644 --- a/bundle/config/mutator/paths/dashboard_paths_visitor_test.go +++ b/bundle/config/mutator/paths/dashboard_paths_visitor_test.go @@ -5,7 +5,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" ) @@ -21,9 +21,7 @@ func TestVisitDashboardPaths(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitDashboardPaths) - expected := []dyn.Path{ - dyn.MustPathFromString("resources.dashboards.dashboard0.file_path"), - } + expected := structpath.NewPathSlice("resources", "dashboards", "dashboard0", "file_path") assert.ElementsMatch(t, expected, actual) } diff --git a/bundle/config/mutator/paths/genie_space_paths_visitor.go b/bundle/config/mutator/paths/genie_space_paths_visitor.go index edd6ff2d8df..8cc61bc09c4 100644 --- a/bundle/config/mutator/paths/genie_space_paths_visitor.go +++ b/bundle/config/mutator/paths/genie_space_paths_visitor.go @@ -1,18 +1,14 @@ package paths import ( - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -func VisitGenieSpacePaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { - pattern := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("genie_spaces"), - dyn.AnyKey(), - dyn.Key("file_path"), - ) +func VisitGenieSpacePaths(root structvar.View, fn VisitFunc) error { + pattern := structpath.MustParsePattern("resources.genie_spaces.*.file_path") - return dyn.MapByPattern(value, pattern, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { + return structvar.ForEach(root, pattern, func(path *structpath.PathNode, value structvar.View) error { return fn(path, TranslateModeLocalRelative, value) }) } diff --git a/bundle/config/mutator/paths/job_libraries_paths_visitor.go b/bundle/config/mutator/paths/job_libraries_paths_visitor.go index ad56b1933dc..e96e136f814 100644 --- a/bundle/config/mutator/paths/job_libraries_paths_visitor.go +++ b/bundle/config/mutator/paths/job_libraries_paths_visitor.go @@ -2,18 +2,19 @@ package paths import ( "github.com/databricks/cli/bundle/libraries" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -func jobTaskLibrariesRewritePatterns(base dyn.Pattern) []jobRewritePattern { +func jobTaskLibrariesRewritePatterns(base string) []jobRewritePattern { return []jobRewritePattern{ { - base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("whl")), + structpath.MustParsePattern(base + ".libraries[*].whl"), TranslateModeLocalRelative, noSkipRewrite, }, { - base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("jar")), + structpath.MustParsePattern(base + ".libraries[*].jar"), TranslateModeLocalRelative, noSkipRewrite, }, @@ -22,27 +23,12 @@ func jobTaskLibrariesRewritePatterns(base dyn.Pattern) []jobRewritePattern { func jobLibrariesRewritePatterns() []jobRewritePattern { // Base pattern to match all tasks in all jobs. - base := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("tasks"), - dyn.AnyIndex(), - ) + base := "resources.jobs.*.tasks[*]" // Compile list of patterns and their respective rewrite functions. jobEnvironmentsPatterns := []jobRewritePattern{ { - dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("environments"), - dyn.AnyIndex(), - dyn.Key("spec"), - dyn.Key("dependencies"), - dyn.AnyIndex(), - ), + structpath.MustParsePattern("resources.jobs.*.environments[*].spec.dependencies[*]"), TranslateModeLocalRelativeWithPrefix, func(s string) bool { return !libraries.IsLibraryLocal(s) @@ -52,16 +38,7 @@ func jobLibrariesRewritePatterns() []jobRewritePattern { jobEnvironmentsWithRequirementsPatterns := []jobRewritePattern{ { - dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("environments"), - dyn.AnyIndex(), - dyn.Key("spec"), - dyn.Key("dependencies"), - dyn.AnyIndex(), - ), + structpath.MustParsePattern("resources.jobs.*.environments[*].spec.dependencies[*]"), TranslateModeEnvironmentPipFlag, func(s string) bool { _, _, ok := libraries.IsLocalPathInPipFlag(s) @@ -71,7 +48,7 @@ func jobLibrariesRewritePatterns() []jobRewritePattern { } taskPatterns := jobTaskLibrariesRewritePatterns(base) - forEachPatterns := jobTaskLibrariesRewritePatterns(base.Append(dyn.Key("for_each_task"), dyn.Key("task"))) + forEachPatterns := jobTaskLibrariesRewritePatterns(base + ".for_each_task.task") allPatterns := append(taskPatterns, jobEnvironmentsPatterns...) allPatterns = append(allPatterns, jobEnvironmentsWithRequirementsPatterns...) allPatterns = append(allPatterns, forEachPatterns...) @@ -79,22 +56,13 @@ func jobLibrariesRewritePatterns() []jobRewritePattern { } // VisitJobLibrariesPaths visits all libraries related paths in job resources and applies a function to each path. -func VisitJobLibrariesPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { - var err error - newValue := value - +func VisitJobLibrariesPaths(root structvar.View, fn VisitFunc) error { for _, rewritePattern := range jobLibrariesRewritePatterns() { - newValue, err = dyn.MapByPattern(newValue, rewritePattern.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - if rewritePattern.skipRewrite(v.MustString()) { - return v, nil - } - - return fn(p, rewritePattern.mode, v) - }) + err := visitString(root, rewritePattern.pattern, rewritePattern.mode, rewritePattern.skipRewrite, fn) if err != nil { - return dyn.InvalidValue, err + return err } } - return newValue, nil + return nil } diff --git a/bundle/config/mutator/paths/job_libraries_paths_visitor_test.go b/bundle/config/mutator/paths/job_libraries_paths_visitor_test.go index b35087e579d..b5a7f273e9d 100644 --- a/bundle/config/mutator/paths/job_libraries_paths_visitor_test.go +++ b/bundle/config/mutator/paths/job_libraries_paths_visitor_test.go @@ -5,7 +5,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" @@ -73,10 +73,10 @@ func TestVisitJobLibrariesPaths(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitJobLibrariesPaths) - expected := []dyn.Path{ - dyn.MustPathFromString("resources.jobs.job0.tasks[4].libraries[0].whl"), - dyn.MustPathFromString("resources.jobs.job0.tasks[5].libraries[0].jar"), - } + expected := structpath.MustParsePaths( + "resources.jobs.job0.tasks[4].libraries[0].whl", + "resources.jobs.job0.tasks[5].libraries[0].jar", + ) assert.ElementsMatch(t, expected, actual) } @@ -107,10 +107,10 @@ func TestVisitJobLibrariesPaths_environments(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitJobLibrariesPaths) - expected := []dyn.Path{ - dyn.MustPathFromString("resources.jobs.job0.environments[0].spec.dependencies[0]"), - dyn.MustPathFromString("resources.jobs.job0.environments[0].spec.dependencies[1]"), - } + expected := structpath.MustParsePaths( + "resources.jobs.job0.environments[0].spec.dependencies[0]", + "resources.jobs.job0.environments[0].spec.dependencies[1]", + ) assert.ElementsMatch(t, expected, actual) } diff --git a/bundle/config/mutator/paths/job_paths_visitor.go b/bundle/config/mutator/paths/job_paths_visitor.go index cb400112b7e..41df9fe5984 100644 --- a/bundle/config/mutator/paths/job_paths_visitor.go +++ b/bundle/config/mutator/paths/job_paths_visitor.go @@ -1,11 +1,12 @@ package paths import ( - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type jobRewritePattern struct { - pattern dyn.Pattern + pattern *structpath.PatternNode mode TranslateMode skipRewrite func(string) bool } @@ -14,35 +15,35 @@ func noSkipRewrite(string) bool { return false } -func jobTaskRewritePatterns(base dyn.Pattern) []jobRewritePattern { +func jobTaskRewritePatterns(base string) []jobRewritePattern { return []jobRewritePattern{ { - base.Append(dyn.Key("notebook_task"), dyn.Key("notebook_path")), + structpath.MustParsePattern(base + ".notebook_task.notebook_path"), TranslateModeNotebook, noSkipRewrite, }, { - base.Append(dyn.Key("spark_python_task"), dyn.Key("python_file")), + structpath.MustParsePattern(base + ".spark_python_task.python_file"), TranslateModeFile, noSkipRewrite, }, { - base.Append(dyn.Key("dbt_task"), dyn.Key("project_directory")), + structpath.MustParsePattern(base + ".dbt_task.project_directory"), TranslateModeDirectory, noSkipRewrite, }, { - base.Append(dyn.Key("sql_task"), dyn.Key("file"), dyn.Key("path")), + structpath.MustParsePattern(base + ".sql_task.file.path"), TranslateModeFile, noSkipRewrite, }, { - base.Append(dyn.Key("alert_task"), dyn.Key("workspace_path")), + structpath.MustParsePattern(base + ".alert_task.workspace_path"), TranslateModeFile, noSkipRewrite, }, { - base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("requirements")), + structpath.MustParsePattern(base + ".libraries[*].requirements"), TranslateModeFile, noSkipRewrite, }, @@ -50,7 +51,7 @@ func jobTaskRewritePatterns(base dyn.Pattern) []jobRewritePattern { // The AI Runtime task runs this bash script on each node; the backend // reads it as a workspace file, so translate the local path to its // remote (or immutable-snapshot) location like any other file. - base.Append(dyn.Key("ai_runtime_task"), dyn.Key("deployments"), dyn.AnyIndex(), dyn.Key("command_path")), + structpath.MustParsePattern(base + ".ai_runtime_task.deployments[*].command_path"), TranslateModeFile, noSkipRewrite, }, @@ -59,36 +60,21 @@ func jobTaskRewritePatterns(base dyn.Pattern) []jobRewritePattern { func jobRewritePatterns() []jobRewritePattern { // Base pattern to match all tasks in all jobs. - base := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("tasks"), - dyn.AnyIndex(), - ) + base := "resources.jobs.*.tasks[*]" taskPatterns := jobTaskRewritePatterns(base) - forEachPatterns := jobTaskRewritePatterns(base.Append(dyn.Key("for_each_task"), dyn.Key("task"))) + forEachPatterns := jobTaskRewritePatterns(base + ".for_each_task.task") return append(taskPatterns, forEachPatterns...) } // VisitJobPaths visits all paths in job resources and applies a function to each path. -func VisitJobPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { - var err error - newValue := value - +func VisitJobPaths(root structvar.View, fn VisitFunc) error { for _, rewritePattern := range jobRewritePatterns() { - newValue, err = dyn.MapByPattern(newValue, rewritePattern.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - if rewritePattern.skipRewrite(v.MustString()) { - return v, nil - } - - return fn(p, rewritePattern.mode, v) - }) + err := visitString(root, rewritePattern.pattern, rewritePattern.mode, rewritePattern.skipRewrite, fn) if err != nil { - return dyn.InvalidValue, err + return err } } - return newValue, nil + return nil } diff --git a/bundle/config/mutator/paths/job_paths_visitor_test.go b/bundle/config/mutator/paths/job_paths_visitor_test.go index 7584737b1c0..5b2fc68290c 100644 --- a/bundle/config/mutator/paths/job_paths_visitor_test.go +++ b/bundle/config/mutator/paths/job_paths_visitor_test.go @@ -5,7 +5,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" @@ -87,15 +87,15 @@ func TestVisitJobPaths(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitJobPaths) - expected := []dyn.Path{ - dyn.MustPathFromString("resources.jobs.job0.tasks[0].notebook_task.notebook_path"), - dyn.MustPathFromString("resources.jobs.job0.tasks[1].spark_python_task.python_file"), - dyn.MustPathFromString("resources.jobs.job0.tasks[2].dbt_task.project_directory"), - dyn.MustPathFromString("resources.jobs.job0.tasks[3].sql_task.file.path"), - dyn.MustPathFromString("resources.jobs.job0.tasks[6].libraries[0].requirements"), - dyn.MustPathFromString("resources.jobs.job0.tasks[7].alert_task.workspace_path"), - dyn.MustPathFromString("resources.jobs.job0.tasks[8].ai_runtime_task.deployments[0].command_path"), - } + expected := structpath.MustParsePaths( + "resources.jobs.job0.tasks[0].notebook_task.notebook_path", + "resources.jobs.job0.tasks[1].spark_python_task.python_file", + "resources.jobs.job0.tasks[2].dbt_task.project_directory", + "resources.jobs.job0.tasks[3].sql_task.file.path", + "resources.jobs.job0.tasks[6].libraries[0].requirements", + "resources.jobs.job0.tasks[7].alert_task.workspace_path", + "resources.jobs.job0.tasks[8].ai_runtime_task.deployments[0].command_path", + ) assert.ElementsMatch(t, expected, actual) } @@ -126,7 +126,7 @@ func TestVisitJobPaths_environments(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitJobPaths) - var expected []dyn.Path + var expected []*structpath.PathNode assert.ElementsMatch(t, expected, actual) } @@ -168,10 +168,10 @@ func TestVisitJobPaths_foreach(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitJobPaths) - expected := []dyn.Path{ - dyn.MustPathFromString("resources.jobs.job0.tasks[0].for_each_task.task.notebook_task.notebook_path"), - dyn.MustPathFromString("resources.jobs.job0.tasks[1].for_each_task.task.alert_task.workspace_path"), - } + expected := structpath.MustParsePaths( + "resources.jobs.job0.tasks[0].for_each_task.task.notebook_task.notebook_path", + "resources.jobs.job0.tasks[1].for_each_task.task.alert_task.workspace_path", + ) assert.ElementsMatch(t, expected, actual) } diff --git a/bundle/config/mutator/paths/job_run_paths_visitor.go b/bundle/config/mutator/paths/job_run_paths_visitor.go index 3bc56bacf72..507a5343b44 100644 --- a/bundle/config/mutator/paths/job_run_paths_visitor.go +++ b/bundle/config/mutator/paths/job_run_paths_visitor.go @@ -1,24 +1,17 @@ package paths import ( - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) // VisitJobRunPaths visits local paths on job_runs so NormalizePaths can rewrite // them relative to the bundle root. Not used by TranslatePaths: hashing still // needs a local glob, not a workspace path. -func VisitJobRunPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { - pattern := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("job_runs"), - dyn.AnyKey(), - dyn.Key("lifecycle"), - dyn.Key("triggers"), - dyn.AnyIndex(), - dyn.Key("on_file_change"), - ) +func VisitJobRunPaths(root structvar.View, fn VisitFunc) error { + pattern := structpath.MustParsePattern("resources.job_runs.*.lifecycle.triggers[*].on_file_change") - return dyn.MapByPattern(value, pattern, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { + return structvar.ForEach(root, pattern, func(path *structpath.PathNode, value structvar.View) error { return fn(path, TranslateModeLocalRelative, value) }) } diff --git a/bundle/config/mutator/paths/job_run_paths_visitor_test.go b/bundle/config/mutator/paths/job_run_paths_visitor_test.go index 93df73d3c9e..ac552583a68 100644 --- a/bundle/config/mutator/paths/job_run_paths_visitor_test.go +++ b/bundle/config/mutator/paths/job_run_paths_visitor_test.go @@ -5,7 +5,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" ) @@ -27,9 +27,7 @@ func TestVisitJobRunPaths(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitJobRunPaths) - expected := []dyn.Path{ - dyn.MustPathFromString("resources.job_runs.run0.lifecycle.triggers[0].on_file_change"), - } + expected := structpath.MustParsePaths("resources.job_runs.run0.lifecycle.triggers[0].on_file_change") assert.ElementsMatch(t, expected, actual) } diff --git a/bundle/config/mutator/paths/pipeline_paths_visitor.go b/bundle/config/mutator/paths/pipeline_paths_visitor.go index b5af086dd34..ff50c637db8 100644 --- a/bundle/config/mutator/paths/pipeline_paths_visitor.go +++ b/bundle/config/mutator/paths/pipeline_paths_visitor.go @@ -2,11 +2,12 @@ package paths import ( "github.com/databricks/cli/bundle/libraries" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type pipelineRewritePattern struct { - pattern dyn.Pattern + pattern *structpath.PatternNode mode TranslateMode // If function defined in skipRewrite returns true, we skip rewriting the path. @@ -15,32 +16,28 @@ type pipelineRewritePattern struct { } // Base pattern to match all libraries in all pipelines. -var base = dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("pipelines"), - dyn.AnyKey(), -) +var base = "resources.pipelines.*" func pipelineRewritePatterns() []pipelineRewritePattern { // Compile list of configuration paths to rewrite. allPatterns := []pipelineRewritePattern{ { - pattern: base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("notebook"), dyn.Key("path")), + pattern: structpath.MustParsePattern(base + ".libraries[*].notebook.path"), mode: TranslateModeNotebook, skipRewrite: noSkipRewrite, }, { - pattern: base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("file"), dyn.Key("path")), + pattern: structpath.MustParsePattern(base + ".libraries[*].file.path"), mode: TranslateModeFile, skipRewrite: noSkipRewrite, }, { - pattern: base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("glob"), dyn.Key("include")), + pattern: structpath.MustParsePattern(base + ".libraries[*].glob.include"), mode: TranslateModeGlob, skipRewrite: noSkipRewrite, }, { - pattern: base.Append(dyn.Key("root_path")), + pattern: structpath.MustParsePattern(base + ".root_path"), mode: TranslateModeDirectory, skipRewrite: noSkipRewrite, }, @@ -52,15 +49,8 @@ func pipelineRewritePatterns() []pipelineRewritePattern { func pipelineLibrariesRewritePatterns() []pipelineRewritePattern { pipelineEnvironmentsPatterns := []pipelineRewritePattern{ { - pattern: dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("pipelines"), - dyn.AnyKey(), - dyn.Key("environment"), - dyn.Key("dependencies"), - dyn.AnyIndex(), - ), - mode: TranslateModeLocalRelativeWithPrefix, + pattern: structpath.MustParsePattern("resources.pipelines.*.environment.dependencies[*]"), + mode: TranslateModeLocalRelativeWithPrefix, skipRewrite: func(s string) bool { return !libraries.IsLibraryLocal(s) }, @@ -69,14 +59,7 @@ func pipelineLibrariesRewritePatterns() []pipelineRewritePattern { pipelineEnvironmentsPatternsWithPipFlags := []pipelineRewritePattern{ { - dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("pipelines"), - dyn.AnyKey(), - dyn.Key("environment"), - dyn.Key("dependencies"), - dyn.AnyIndex(), - ), + structpath.MustParsePattern("resources.pipelines.*.environment.dependencies[*]"), TranslateModeEnvironmentPipFlag, func(s string) bool { _, _, ok := libraries.IsLocalPathInPipFlag(s) @@ -88,50 +71,24 @@ func pipelineLibrariesRewritePatterns() []pipelineRewritePattern { return append(pipelineEnvironmentsPatterns, pipelineEnvironmentsPatternsWithPipFlags...) } -func VisitPipelinePaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { - var err error - newValue := value - +func VisitPipelinePaths(root structvar.View, fn VisitFunc) error { for _, rewritePattern := range pipelineRewritePatterns() { - newValue, err = dyn.MapByPattern(newValue, rewritePattern.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - sv, ok := v.AsString() - if !ok { - return v, nil - } - if rewritePattern.skipRewrite(sv) { - return v, nil - } - - return fn(p, rewritePattern.mode, v) - }) + err := visitString(root, rewritePattern.pattern, rewritePattern.mode, rewritePattern.skipRewrite, fn) if err != nil { - return dyn.InvalidValue, err + return err } } - return newValue, nil + return nil } -func VisitPipelineLibrariesPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { - var err error - newValue := value - +func VisitPipelineLibrariesPaths(root structvar.View, fn VisitFunc) error { for _, rewritePattern := range pipelineLibrariesRewritePatterns() { - newValue, err = dyn.MapByPattern(newValue, rewritePattern.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - sv, ok := v.AsString() - if !ok { - return v, nil - } - if rewritePattern.skipRewrite(sv) { - return v, nil - } - - return fn(p, rewritePattern.mode, v) - }) + err := visitString(root, rewritePattern.pattern, rewritePattern.mode, rewritePattern.skipRewrite, fn) if err != nil { - return dyn.InvalidValue, err + return err } } - return newValue, nil + return nil } diff --git a/bundle/config/mutator/paths/pipeline_paths_visitor_test.go b/bundle/config/mutator/paths/pipeline_paths_visitor_test.go index eb9748bba37..6a7b2c6183d 100644 --- a/bundle/config/mutator/paths/pipeline_paths_visitor_test.go +++ b/bundle/config/mutator/paths/pipeline_paths_visitor_test.go @@ -5,7 +5,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/stretchr/testify/assert" ) @@ -47,13 +47,13 @@ func TestVisitPipelinePaths(t *testing.T) { actual := collectVisitedPaths(t, root, VisitPipelinePaths) actual = append(actual, collectVisitedPaths(t, root, VisitPipelineLibrariesPaths)...) - expected := []dyn.Path{ - dyn.MustPathFromString("resources.pipelines.pipeline0.libraries[0].file.path"), - dyn.MustPathFromString("resources.pipelines.pipeline0.libraries[1].notebook.path"), - dyn.MustPathFromString("resources.pipelines.pipeline0.libraries[2].glob.include"), - dyn.MustPathFromString("resources.pipelines.pipeline0.root_path"), - dyn.MustPathFromString("resources.pipelines.pipeline0.environment.dependencies[0]"), - } + expected := structpath.MustParsePaths( + "resources.pipelines.pipeline0.libraries[0].file.path", + "resources.pipelines.pipeline0.libraries[1].notebook.path", + "resources.pipelines.pipeline0.libraries[2].glob.include", + "resources.pipelines.pipeline0.root_path", + "resources.pipelines.pipeline0.environment.dependencies[0]", + ) assert.ElementsMatch(t, expected, actual) } diff --git a/bundle/config/mutator/paths/visitor.go b/bundle/config/mutator/paths/visitor.go index cadc52b75fe..02662c8b43b 100644 --- a/bundle/config/mutator/paths/visitor.go +++ b/bundle/config/mutator/paths/visitor.go @@ -1,14 +1,17 @@ package paths import ( - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -type VisitFunc func(path dyn.Path, mode TranslateMode, value dyn.Value) (dyn.Value, error) +// VisitFunc is called for every matching value. It may change the configuration +// at path (e.g. with [config.Root.Set]). +type VisitFunc func(path *structpath.PathNode, mode TranslateMode, value structvar.View) error // VisitPaths visits all paths in bundle configuration -func VisitPaths(root dyn.Value, fn VisitFunc) (dyn.Value, error) { - visitors := []func(dyn.Value, VisitFunc) (dyn.Value, error){ +func VisitPaths(root structvar.View, fn VisitFunc) error { + visitors := []func(structvar.View, VisitFunc) error{ VisitJobPaths, VisitJobRunPaths, VisitJobLibrariesPaths, @@ -21,14 +24,23 @@ func VisitPaths(root dyn.Value, fn VisitFunc) (dyn.Value, error) { VisitPipelineLibrariesPaths, } - newRoot := root for _, visitor := range visitors { - updatedRoot, err := visitor(newRoot, fn) - if err != nil { - return dyn.InvalidValue, err + if err := visitor(root, fn); err != nil { + return err } - newRoot = updatedRoot } - return newRoot, nil + return nil +} + +// visitString calls fn for every string value matching pattern unless skip reports true for it. +// Values that are not strings are not visited. +func visitString(root structvar.View, pattern *structpath.PatternNode, mode TranslateMode, skip func(string) bool, fn VisitFunc) error { + return structvar.ForEach(root, pattern, func(p *structpath.PathNode, v structvar.View) error { + s, ok := v.AsString() + if !ok || skip(s) { + return nil + } + return fn(p, mode, v) + }) } diff --git a/bundle/config/mutator/paths/visitor_test.go b/bundle/config/mutator/paths/visitor_test.go index 019342404c8..80dc112237d 100644 --- a/bundle/config/mutator/paths/visitor_test.go +++ b/bundle/config/mutator/paths/visitor_test.go @@ -4,18 +4,17 @@ import ( "testing" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/stretchr/testify/require" ) // collectVisitedPaths is a helper function that collects all visited paths for testing -func collectVisitedPaths(t *testing.T, root config.Root, visitFn func(value dyn.Value, fn VisitFunc) (dyn.Value, error)) []dyn.Path { - var actual []dyn.Path - err := root.Mutate(func(value dyn.Value) (dyn.Value, error) { - return visitFn(value, func(p dyn.Path, mode TranslateMode, v dyn.Value) (dyn.Value, error) { - actual = append(actual, p) - return v, nil - }) +func collectVisitedPaths(t *testing.T, root config.Root, visitFn func(value structvar.View, fn VisitFunc) error) []*structpath.PathNode { + var actual []*structpath.PathNode + err := visitFn(root.View(), func(p *structpath.PathNode, mode TranslateMode, v structvar.View) error { + actual = append(actual, p) + return nil }) require.NoError(t, err) return actual diff --git a/bundle/config/mutator/populate_locations.go b/bundle/config/mutator/populate_locations.go index 8f7a1258b1e..a2a217e0c68 100644 --- a/bundle/config/mutator/populate_locations.go +++ b/bundle/config/mutator/populate_locations.go @@ -2,10 +2,14 @@ package mutator import ( "context" + "fmt" + "strings" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/loctable" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn/dynloc" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type populateLocations struct{} @@ -20,9 +24,53 @@ func (m *populateLocations) Name() string { return "PopulateLocations" } +// locationPatterns are the paths for which locations are recorded. +var locationPatterns = []*structpath.PatternNode{ + structpath.MustParsePattern("*"), // Top level fields + structpath.MustParsePattern("resources.*"), // Resource groups ("resources.jobs") + structpath.MustParsePattern("resources.*.*"), // Resources for all types ("resources.jobs.my_job") + structpath.MustParsePattern("resources.jobs.*.tasks"), // Job tasks ("resources.jobs.my_job.tasks") + structpath.MustParsePattern("resources.jobs.*.tasks[*]"), // Job task items ("resources.jobs.my_job.tasks[2]") +} + +// pathString returns the path with keys separated by dots, whatever characters they contain. +func pathString(p *structpath.PathNode) string { + var b strings.Builder + for i, n := range p.AsSlice() { + if k, ok := n.StringKey(); ok { + if i > 0 { + b.WriteByte('.') + } + b.WriteString(k) + } else if idx, ok := n.Index(); ok { + fmt.Fprintf(&b, "[%d]", idx) + } + } + return b.String() +} + +func gatherLocations(v structvar.View) (map[string][]diag.Location, error) { + locs := map[string][]diag.Location{} + for _, pattern := range locationPatterns { + err := structvar.ForEach(v, pattern, func(p *structpath.PathNode, v structvar.View) error { + locs[pathString(p)] = v.Locations() + return nil + }) + if err != nil { + return nil, err + } + } + return locs, nil +} + func (m *populateLocations) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - locs, err := dynloc.Build( - b.Config.Value(), + pathToLocations, err := gatherLocations(b.Config.View()) + if err != nil { + return diag.FromErr(err) + } + + locs, err := loctable.Build( + pathToLocations, // Make all paths relative to the bundle root. b.BundleRootPath, ) diff --git a/bundle/config/mutator/prepend_workspace_prefix.go b/bundle/config/mutator/prepend_workspace_prefix.go index 3124244c857..05892d50f86 100644 --- a/bundle/config/mutator/prepend_workspace_prefix.go +++ b/bundle/config/mutator/prepend_workspace_prefix.go @@ -7,7 +7,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type prependWorkspacePrefix struct{} @@ -27,45 +28,38 @@ var skipPrefixes = []string{ } func (m *prependWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - patterns := []dyn.Pattern{ - dyn.NewPattern(dyn.Key("workspace"), dyn.Key("root_path")), - dyn.NewPattern(dyn.Key("workspace"), dyn.Key("file_path")), - dyn.NewPattern(dyn.Key("workspace"), dyn.Key("artifact_path")), - dyn.NewPattern(dyn.Key("workspace"), dyn.Key("state_path")), - dyn.NewPattern(dyn.Key("workspace"), dyn.Key("resource_path")), + patterns := []*structpath.PatternNode{ + structpath.MustParsePattern("workspace.root_path"), + structpath.MustParsePattern("workspace.file_path"), + structpath.MustParsePattern("workspace.artifact_path"), + structpath.MustParsePattern("workspace.state_path"), + structpath.MustParsePattern("workspace.resource_path"), } - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - var err error - for _, pattern := range patterns { - v, err = dyn.MapByPattern(v, pattern, func(p dyn.Path, pv dyn.Value) (dyn.Value, error) { - path, ok := pv.AsString() - if !ok { - return dyn.InvalidValue, fmt.Errorf("expected string, got %s", pv.Kind()) - } + for _, pattern := range patterns { + err := structvar.ForEach(b.Config.View(), pattern, func(p *structpath.PathNode, pv structvar.View) error { + path, ok := pv.AsString() + if !ok { + return fmt.Errorf("expected string, got %s", pv.Kind()) + } - // Skip prefixing if the path does not start with /, it might be variable reference or smth else. - if !strings.HasPrefix(path, "/") { - return pv, nil - } + // Skip prefixing if the path does not start with /, it might be variable reference or smth else. + if !strings.HasPrefix(path, "/") { + return nil + } - for _, prefix := range skipPrefixes { - if strings.HasPrefix(path, prefix) { - return pv, nil - } + for _, prefix := range skipPrefixes { + if strings.HasPrefix(path, prefix) { + return nil } - - // Use pv's locations, not the root v's, so diagnostics point at the original config line. - return dyn.NewValue("/Workspace"+path, pv.Locations()), nil - }) - if err != nil { - return dyn.InvalidValue, err } + + // Set keeps the locations of the value, so diagnostics point at the original config line. + return b.Config.Set(p, "/Workspace"+path) + }) + if err != nil { + return diag.FromErr(err) } - return v, nil - }) - if err != nil { - return diag.FromErr(err) } return nil diff --git a/bundle/config/mutator/prepend_workspace_prefix_test.go b/bundle/config/mutator/prepend_workspace_prefix_test.go index 93b3acde922..6c439247051 100644 --- a/bundle/config/mutator/prepend_workspace_prefix_test.go +++ b/bundle/config/mutator/prepend_workspace_prefix_test.go @@ -6,7 +6,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/internal/bundletest" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/diag" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/stretchr/testify/require" ) @@ -73,7 +73,7 @@ func TestPrependWorkspacePrefixPreservesLocations(t *testing.T) { }, }, } - locations := []dyn.Location{{File: "databricks.yml", Line: 42, Column: 5}} + locations := []diag.Location{{File: "databricks.yml", Line: 42, Column: 5}} bundletest.SetLocation(b, "workspace.root_path", locations) diags := bundle.Apply(t.Context(), b, PrependWorkspacePrefix()) diff --git a/bundle/config/mutator/python/apply_python_output.go b/bundle/config/mutator/python/apply_python_output.go index fb5452d3b4f..9ed4d7dd56b 100644 --- a/bundle/config/mutator/python/apply_python_output.go +++ b/bundle/config/mutator/python/apply_python_output.go @@ -4,9 +4,8 @@ import ( "fmt" "github.com/databricks/cli/bundle/config/mutator/resourcemutator" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/merge" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) // applyPythonOutputResult contains which resources where added, updated, or deleted by Python mutator. @@ -24,162 +23,170 @@ type applyPythonOutputResult struct { // Return value is equivalent to output except for: // - if property is unchanged in output, it's original location will be preserved // - if empty sequence/mapping is deleted in output, it's original value will be preserved -func applyPythonOutput(root, output dyn.Value) (dyn.Value, applyPythonOutputResult, error) { +func applyPythonOutput(root, output structvar.View) (*structvar.OverridePlan, applyPythonOutputResult, error) { result, visitor := createOverrideVisitor(root, output) - merged, err := merge.Override(root, output, visitor) + plan, err := structvar.PlanOverride(root, output, visitor) if err != nil { - return dyn.InvalidValue, result, err + return nil, result, err } - return merged, result, nil + return plan, result, nil +} + +// addResourceKeys adds the keys of the resources in root that match pattern. +func addResourceKeys(set resourcemutator.ResourceKeySet, pattern *structpath.PatternNode, root structvar.View) error { + return structvar.ForEach(root, pattern, func(np *structpath.PathNode, _ structvar.View) error { + set.AddResourceKey(resourcemutator.ResourceKey{Type: np.KeyAt(1), Name: np.KeyAt(2)}) + return nil + }) } -func createOverrideVisitor(leftRoot, rightRoot dyn.Value) (applyPythonOutputResult, merge.OverrideVisitor) { - resourcesPath := dyn.NewPath(dyn.Key("resources")) +func createOverrideVisitor(leftRoot, rightRoot structvar.View) (applyPythonOutputResult, structvar.OverrideVisitor) { + resourcesPath := structpath.MustParsePath("resources") deleted := resourcemutator.NewResourceKeySet() updated := resourcemutator.NewResourceKeySet() added := resourcemutator.NewResourceKeySet() - visitor := merge.OverrideVisitor{ - VisitDelete: func(valuePath dyn.Path, left dyn.Value) error { + visitor := structvar.OverrideVisitor{ + VisitDelete: func(np *structpath.PathNode, left structvar.View) error { if isOmitemptyDelete(left) { - return merge.ErrOverrideUndoDelete + return structvar.ErrOverrideUndoDelete } - if !valuePath.HasPrefix(resourcesPath) { - return fmt.Errorf("unexpected change at %q (delete)", valuePath.String()) + if !np.HasPrefix(resourcesPath) { + return fmt.Errorf("unexpected change at %q (delete)", np.String()) } // use leftRoot below because it contains deleted resources - if len(valuePath) == 1 { + if np.Len() == 1 { // Example: // // valuePath: "resources" // leftRoot: {"bundle": ..., "resources": ...}, // rightRoot: {"bundle": ...} - return deleted.AddPattern( - dyn.NewPatternFromPath(valuePath).Append(dyn.AnyKey(), dyn.AnyKey()), + return addResourceKeys(deleted, + structpath.MustParsePattern("resources.*.*"), leftRoot, ) - } else if len(valuePath) == 2 { + } else if np.Len() == 2 { // Example: // // valuePath: "resources.jobs" // leftRoot: {"resources": { "jobs": ..., "pipeline": ...}}}, // rightRoot: {"resources": { "jobs": ...}}}, - return deleted.AddPattern( - dyn.NewPatternFromPath(valuePath).Append(dyn.AnyKey()), + return addResourceKeys(deleted, + structpath.NewPatternDotStar(structpath.NewPatternStringKey(structpath.MustParsePattern("resources"), np.KeyAt(1))), leftRoot, ) - } else if len(valuePath) == 3 { + } else if np.Len() == 3 { // Example: "resources.jobs.job_0" - resourceKey := resourcemutator.ResourceKey{Type: valuePath[1].Key(), Name: valuePath[2].Key()} + resourceKey := resourcemutator.ResourceKey{Type: np.KeyAt(1), Name: np.KeyAt(2)} deleted.AddResourceKey(resourceKey) return nil } else { // Example: "resources.jobs.job_0.tags" - resourceKey := resourcemutator.ResourceKey{Type: valuePath[1].Key(), Name: valuePath[2].Key()} + resourceKey := resourcemutator.ResourceKey{Type: np.KeyAt(1), Name: np.KeyAt(2)} updated.AddResourceKey(resourceKey) return nil } }, - VisitInsert: func(valuePath dyn.Path, right dyn.Value) (dyn.Value, error) { - if !valuePath.HasPrefix(resourcesPath) { - return dyn.InvalidValue, fmt.Errorf("unexpected change at %q (insert)", valuePath.String()) + VisitInsert: func(np *structpath.PathNode, right structvar.View) error { + if !np.HasPrefix(resourcesPath) { + return fmt.Errorf("unexpected change at %q (insert)", np.String()) } // use rightRoot below because it contains result - if len(valuePath) == 1 { + if np.Len() == 1 { // Example: // // valuePath: "resources" // leftRoot: {"bundle": ..., } // rightRoot: {"bundle": ..., "resources": {...} } - return right, added.AddPattern( - dyn.NewPatternFromPath(valuePath).Append(dyn.AnyKey(), dyn.AnyKey()), + return addResourceKeys(added, + structpath.MustParsePattern("resources.*.*"), rightRoot, ) - } else if len(valuePath) == 2 { + } else if np.Len() == 2 { // Example: // // valuePath: "resources.jobs" // leftRoot: {"resources": { }} // rightRoot: {"resources": { "jobs": {...} }} - return right, added.AddPattern( - dyn.NewPatternFromPath(valuePath).Append(dyn.AnyKey()), + return addResourceKeys(added, + structpath.NewPatternDotStar(structpath.NewPatternStringKey(structpath.MustParsePattern("resources"), np.KeyAt(1))), rightRoot, ) - } else if len(valuePath) == 3 { + } else if np.Len() == 3 { // Example: // // valuePath: "resources.jobs" // leftRoot: {"resources": { "jobs": { }}} // rightRoot: {"resources": { "jobs": {"job_0": {...} }}} - resourceKey := resourcemutator.ResourceKey{Type: valuePath[1].Key(), Name: valuePath[2].Key()} + resourceKey := resourcemutator.ResourceKey{Type: np.KeyAt(1), Name: np.KeyAt(2)} added.AddResourceKey(resourceKey) - return right, nil + return nil } else { // Example: "resources.jobs.job_0.email_notifications" - resourceKey := resourcemutator.ResourceKey{Type: valuePath[1].Key(), Name: valuePath[2].Key()} + resourceKey := resourcemutator.ResourceKey{Type: np.KeyAt(1), Name: np.KeyAt(2)} updated.AddResourceKey(resourceKey) - return right, nil + return nil } }, - VisitUpdate: func(valuePath dyn.Path, _, right dyn.Value) (dyn.Value, error) { - if !valuePath.HasPrefix(resourcesPath) { - return dyn.InvalidValue, fmt.Errorf("unexpected change at %q (update)", valuePath.String()) + VisitUpdate: func(np *structpath.PathNode, _, right structvar.View) error { + if !np.HasPrefix(resourcesPath) { + return fmt.Errorf("unexpected change at %q (update)", np.String()) } // use rightRoot below because it contains result - if len(valuePath) == 1 { + if np.Len() == 1 { // Example: // // valuePath: "resources" // leftRoot: {"bundle": ..., "resources": null } // rightRoot: {"bundle": ..., "resources": {...} } - return right, added.AddPattern( - dyn.NewPatternFromPath(valuePath).Append(dyn.AnyKey(), dyn.AnyKey()), + return addResourceKeys(added, + structpath.MustParsePattern("resources.*.*"), rightRoot, ) - } else if len(valuePath) == 2 { + } else if np.Len() == 2 { // Example: // // valuePath: "resources.jobs" // leftRoot: {"resources": { "jobs": null }} // rightRoot: {"resources": { "jobs": {...} }} - return right, added.AddPattern( - dyn.NewPatternFromPath(valuePath).Append(dyn.AnyKey()), + return addResourceKeys(added, + structpath.NewPatternDotStar(structpath.NewPatternStringKey(structpath.MustParsePattern("resources"), np.KeyAt(1))), rightRoot, ) - } else if len(valuePath) == 3 { + } else if np.Len() == 3 { // Example: // // valuePath: "resources.jobs.job_0" // leftRoot: {"resources": { "jobs": {"job_0": null }}} // rightRoot: {"resources": { "jobs": {"job_0": {...} }}} - resourceKey := resourcemutator.ResourceKey{Type: valuePath[1].Key(), Name: valuePath[2].Key()} + resourceKey := resourcemutator.ResourceKey{Type: np.KeyAt(1), Name: np.KeyAt(2)} added.AddResourceKey(resourceKey) - return right, nil + return nil } else { // Example: "resources.jobs.job_0.name" - resourceKey := resourcemutator.ResourceKey{Type: valuePath[1].Key(), Name: valuePath[2].Key()} + resourceKey := resourcemutator.ResourceKey{Type: np.KeyAt(1), Name: np.KeyAt(2)} updated.AddResourceKey(resourceKey) - return right, nil + return nil } }, } @@ -191,19 +198,25 @@ func createOverrideVisitor(leftRoot, rightRoot dyn.Value) (applyPythonOutputResu }, visitor } -func isOmitemptyDelete(left dyn.Value) bool { +func isOmitemptyDelete(left structvar.View) bool { // Python output can omit empty sequences/mappings, because we don't track them as optional, // there is no semantic difference between empty and missing, so we keep them as they were before // Python mutator deleted them. switch left.Kind() { - case dyn.KindMap: - return left.MustMap().Len() == 0 + case structvar.KindMap: + for range left.MapItems() { + return false + } + return true - case dyn.KindSequence: - return len(left.MustSequence()) == 0 + case structvar.KindSequence: + for range left.Sequence() { + return false + } + return true - case dyn.KindNil: + case structvar.KindNil: // map/sequence can be nil, for instance, bad YAML like: `foo:` return true diff --git a/bundle/config/mutator/python/apply_python_output_test.go b/bundle/config/mutator/python/apply_python_output_test.go index 2d64d6c4bbc..15d48ecc5bb 100644 --- a/bundle/config/mutator/python/apply_python_output_test.go +++ b/bundle/config/mutator/python/apply_python_output_test.go @@ -1,21 +1,22 @@ package python import ( + "encoding/json" "testing" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator/resourcemutator" - - "github.com/databricks/cli/libs/dyn/merge" - - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type applyPythonOutputTestCase struct { name string - input dyn.Value - output dyn.Value + input any + output any added []resourcemutator.ResourceKey updated []resourcemutator.ResourceKey @@ -23,8 +24,8 @@ type applyPythonOutputTestCase struct { } func TestApplyPythonOutput(t *testing.T) { - job1 := mapOf("name", dyn.V("job 1")) - job2 := mapOf("name", dyn.V("job 2")) + job1 := mapOf("name", "job 1") + job2 := mapOf("name", "job 2") testCases := []applyPythonOutputTestCase{ { @@ -129,10 +130,10 @@ func TestApplyPythonOutput(t *testing.T) { { name: "update job through 'description' insert", input: mapOf("resources", mapOf("jobs", - mapOf("job_1", mapOf("name", dyn.V("name"))), + mapOf("job_1", mapOf("name", "name")), )), output: mapOf("resources", mapOf("jobs", - mapOf("job_1", mapOf2("name", dyn.V("name"), "description", dyn.V("description"))), + mapOf("job_1", mapOf2("name", "name", "description", "description")), )), updated: []resourcemutator.ResourceKey{ { @@ -145,10 +146,13 @@ func TestApplyPythonOutput(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - merged, state, err := applyPythonOutput(tc.input, tc.output) + input := loadRoot(t, tc.input) + output := loadRoot(t, tc.output) + plan, state, err := applyPythonOutput(input.View(), output.View()) + require.NoError(t, err) - assert.NoError(t, err) - assert.Equal(t, tc.output, merged) + require.NoError(t, input.Override(plan)) + assert.Equal(t, output.View().AsAny(), input.View().AsAny()) assert.ElementsMatch(t, tc.added, state.AddedResources.ToArray()) assert.ElementsMatch(t, tc.updated, state.UpdatedResources.ToArray()) @@ -158,36 +162,45 @@ func TestApplyPythonOutput(t *testing.T) { } func TestMergeOutput_disallowDelete(t *testing.T) { - input := mapOf("not_resource", dyn.V("value")) - output := emptyMap() + input := loadRoot(t, mapOf("bundle", mapOf("name", "value"))) + output := loadRoot(t, emptyMap()) - _, _, err := applyPythonOutput(input, output) + _, _, err := applyPythonOutput(input.View(), output.View()) - assert.EqualError(t, err, `unexpected change at "not_resource" (delete)`) + assert.EqualError(t, err, `unexpected change at "bundle" (delete)`) } func TestMergeOutput_disallowInsert(t *testing.T) { - output := mapOf("not_resource", dyn.V("value")) - input := emptyMap() + output := loadRoot(t, mapOf("bundle", mapOf("name", "value"))) + input := loadRoot(t, emptyMap()) - _, _, err := applyPythonOutput(input, output) + _, _, err := applyPythonOutput(input.View(), output.View()) - assert.EqualError(t, err, `unexpected change at "not_resource" (insert)`) + assert.EqualError(t, err, `unexpected change at "bundle" (insert)`) } func TestMergeOutput_disallowUpdate(t *testing.T) { - output := mapOf("not_resource", dyn.V("value")) - input := mapOf("not_resource", dyn.V("new value")) + output := loadRoot(t, mapOf("bundle", mapOf("name", "value"))) + input := loadRoot(t, mapOf("bundle", mapOf("name", "new value"))) - _, _, err := applyPythonOutput(input, output) + _, _, err := applyPythonOutput(input.View(), output.View()) + + assert.EqualError(t, err, `unexpected change at "bundle.name" (update)`) +} - assert.EqualError(t, err, `unexpected change at "not_resource" (update)`) +// loadRoot returns the configuration for the configuration tree v. +func loadRoot(t *testing.T, v any) *config.Root { + raw, err := json.Marshal(v) + require.NoError(t, err) + r, diags := config.LoadFromBytes("output.json", raw) + require.NoError(t, diags.Error()) + return r } type overrideVisitorOmitemptyTestCase struct { name string - path dyn.Path - left dyn.Value + path *structpath.PathNode + left any expectedErr error } @@ -196,76 +209,70 @@ func TestCreateOverrideVisitor_omitempty(t *testing.T) { // there is no semantic difference between empty and missing, so we keep them as they were before // Python code deleted them. - location := dyn.Location{ - File: "databricks.yml", - Line: 10, - Column: 20, - } - testCases := []overrideVisitorOmitemptyTestCase{ { name: "undo delete of empty variables", - path: dyn.MustPathFromString("variables"), - left: dyn.NewValue([]dyn.Value{}, []dyn.Location{location}), - expectedErr: merge.ErrOverrideUndoDelete, + path: structpath.MustParsePath("variables"), + left: &[]string{}, + expectedErr: structvar.ErrOverrideUndoDelete, }, { name: "undo delete of empty job clusters", - path: dyn.MustPathFromString("resources.jobs.job0.job_clusters"), - left: dyn.NewValue([]dyn.Value{}, []dyn.Location{location}), - expectedErr: merge.ErrOverrideUndoDelete, + path: structpath.MustParsePath("resources.jobs.job0.job_clusters"), + left: &[]string{}, + expectedErr: structvar.ErrOverrideUndoDelete, }, { name: "allow delete of non-empty job clusters", - path: dyn.MustPathFromString("resources.jobs.job0.job_clusters"), - left: dyn.NewValue([]dyn.Value{dyn.NewValue("abc", []dyn.Location{location})}, []dyn.Location{location}), + path: structpath.MustParsePath("resources.jobs.job0.job_clusters"), + left: &[]string{"abc"}, expectedErr: nil, }, { name: "undo delete of empty tags", - path: dyn.MustPathFromString("resources.jobs.job0.tags"), - left: dyn.NewValue(map[string]dyn.Value{}, []dyn.Location{location}), - expectedErr: merge.ErrOverrideUndoDelete, + path: structpath.MustParsePath("resources.jobs.job0.tags"), + left: &map[string]string{}, + expectedErr: structvar.ErrOverrideUndoDelete, }, { name: "allow delete of non-empty tags", - path: dyn.MustPathFromString("resources.jobs.job0.tags"), - left: dyn.NewValue(map[string]dyn.Value{"dev": dyn.NewValue("true", []dyn.Location{location})}, []dyn.Location{location}), + path: structpath.MustParsePath("resources.jobs.job0.tags"), + left: &map[string]string{"dev": "true"}, expectedErr: nil, }, { name: "undo delete of nil", - path: dyn.MustPathFromString("resources.jobs.job0.tags"), - left: dyn.NilValue.WithLocations([]dyn.Location{location}), - expectedErr: merge.ErrOverrideUndoDelete, + path: structpath.MustParsePath("resources.jobs.job0.tags"), + left: (*map[string]string)(nil), + expectedErr: structvar.ErrOverrideUndoDelete, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - _, visitor := createOverrideVisitor(dyn.NilValue, dyn.NilValue) + _, visitor := createOverrideVisitor(structvar.View{}, structvar.View{}) - err := visitor.VisitDelete(tc.path, tc.left) + err := visitor.VisitDelete(tc.path, structvar.NewView(tc.left, nil, nil)) assert.Equal(t, tc.expectedErr, err) }) } } -func mapOf(key string, value dyn.Value) dyn.Value { - return dyn.V(map[string]dyn.Value{ +func mapOf(key string, value any) any { + return map[string]any{ key: value, - }) + } } -func mapOf2(key1 string, value1 dyn.Value, key2 string, value2 dyn.Value) dyn.Value { - return dyn.V(map[string]dyn.Value{ +func mapOf2(key1 string, value1 any, key2 string, value2 any) any { + return map[string]any{ key1: value1, key2: value2, - }) + } } -func emptyMap() dyn.Value { - return dyn.V(map[string]dyn.Value{}) +func emptyMap() any { + return map[string]any{} } diff --git a/bundle/config/mutator/python/python_diagnostics.go b/bundle/config/mutator/python/python_diagnostics.go index b947b3f6ad3..0ec85910409 100644 --- a/bundle/config/mutator/python/python_diagnostics.go +++ b/bundle/config/mutator/python/python_diagnostics.go @@ -6,7 +6,6 @@ import ( "io" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" ) @@ -58,12 +57,12 @@ func parsePythonDiagnostics(input io.Reader) (diag.Diagnostics, error) { } var paths []*structpath.PathNode if path != nil { - paths = dyn.ToStructPaths(path) + paths = []*structpath.PathNode{path} } - var locations []dyn.Location + var locations []diag.Location location := convertPythonLocation(parsedLine.Location) - if location != (dyn.Location{}) { + if location != (diag.Location{}) { locations = append(locations, location) } @@ -81,12 +80,12 @@ func parsePythonDiagnostics(input io.Reader) (diag.Diagnostics, error) { return diags, nil } -func convertPythonPath(path string) (dyn.Path, error) { +func convertPythonPath(path string) (*structpath.PathNode, error) { if path == "" { return nil, nil } - return dyn.NewPathFromString(path) + return structpath.ParsePath(path) } func convertPythonSeverity(severity pythonSeverity) (diag.Severity, error) { @@ -100,8 +99,8 @@ func convertPythonSeverity(severity pythonSeverity) (diag.Severity, error) { } } -func convertPythonLocation(location pythonDiagnosticLocation) dyn.Location { - return dyn.Location{ +func convertPythonLocation(location pythonDiagnosticLocation) diag.Location { + return diag.Location{ File: location.File, Line: location.Line, Column: location.Column, diff --git a/bundle/config/mutator/python/python_diagnostics_test.go b/bundle/config/mutator/python/python_diagnostics_test.go index 43709031ca2..faba60e7498 100644 --- a/bundle/config/mutator/python/python_diagnostics_test.go +++ b/bundle/config/mutator/python/python_diagnostics_test.go @@ -5,7 +5,6 @@ import ( "testing" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" ) @@ -17,7 +16,7 @@ func TestConvertPythonLocation(t *testing.T) { Column: 2, }) - assert.Equal(t, dyn.Location{ + assert.Equal(t, diag.Location{ File: "src/examples/file.py", Line: 1, Column: 2, @@ -39,7 +38,7 @@ func TestParsePythonDiagnostics(t *testing.T) { { Severity: diag.Error, Summary: "error summary", - Locations: []dyn.Location{ + Locations: []diag.Location{ { File: "src/examples/file.py", Line: 1, diff --git a/bundle/config/mutator/python/python_locations.go b/bundle/config/mutator/python/python_locations.go index 9cb65c302e5..430f0f716e9 100644 --- a/bundle/config/mutator/python/python_locations.go +++ b/bundle/config/mutator/python/python_locations.go @@ -7,18 +7,20 @@ import ( pathlib "path" "path/filepath" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) // generatedFileName is used as the virtual file name for YAML generated by Python code. // -// mergePythonLocations replaces dyn.Location with generatedFileName with locations loaded +// mergePythonLocations replaces diag.Location with generatedFileName with locations loaded // from locations.json const generatedFileName = "__generated_by_python__.yml" // pythonLocations is data structure for efficient location lookup for a given path // -// Locations form a tree, and we assign locations of the closest ancestor to each dyn.Value based on its path. +// Locations form a tree, and we assign locations of the closest ancestor to each value based on its path. // We implement it as a trie (prefix tree) where keys are components of the path. With that, lookups are O(n) // where n is the number of components in the path. // @@ -42,7 +44,7 @@ type pythonLocations struct { indexes map[int]*pythonLocations // location for the current node if it exists - location dyn.Location + location diag.Location // if true, location is present exists bool @@ -56,15 +58,16 @@ type pythonLocationEntry struct { Column int `json:"column"` } -// mergePythonLocations applies locations from Python mutator into given dyn.Value +// mergePythonLocations returns a location mapper that applies locations from the +// Python mutator. // // The primary use-case is to merge locations.json with output.json, so that any // validation errors will point to Python source code instead of generated YAML. -func mergePythonLocations(value dyn.Value, locations *pythonLocations) (dyn.Value, error) { - return dyn.Walk(value, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { +func mergePythonLocations(locations *pythonLocations) structvar.LocationMapper { + return func(path *structpath.PathNode, locs []diag.Location) []diag.Location { newLocation, ok := findPythonLocation(locations, path) if !ok { - return value, nil + return locs } // The first item in the list is the "last" location used for error reporting @@ -74,17 +77,12 @@ func mergePythonLocations(value dyn.Value, locations *pythonLocations) (dyn.Valu // // We preserve any previous non-virtual locations in case when Python function modified // resource defined in YAML. - newLocations := append( - []dyn.Location{newLocation}, - removeVirtualLocations(value.Locations())..., - ) - - return value.WithLocations(newLocations), nil - }) + return append([]diag.Location{newLocation}, removeVirtualLocations(locs)...) + } } -func removeVirtualLocations(locations []dyn.Location) []dyn.Location { - var newLocations []dyn.Location +func removeVirtualLocations(locations []diag.Location) []diag.Location { + var newLocations []diag.Location for _, location := range locations { if filepath.Base(location.File) == generatedFileName { @@ -112,7 +110,7 @@ func parsePythonLocations(bundleRoot string, input io.Reader) (*pythonLocations, return nil, fmt.Errorf("failed to parse python location: %s", err) } - path, err := dyn.NewPathFromString(entry.Path) + path, err := structpath.ParsePath(entry.Path) if err != nil { return nil, fmt.Errorf("failed to parse python location: %s", err) } @@ -123,7 +121,7 @@ func parsePythonLocations(bundleRoot string, input io.Reader) (*pythonLocations, entry.File = filepath.Join(bundleRoot, entry.File) } - location := dyn.Location{ + location := diag.Location{ File: entry.File, Line: entry.Line, Column: entry.Column, @@ -136,18 +134,18 @@ func parsePythonLocations(bundleRoot string, input io.Reader) (*pythonLocations, } // putPythonLocation puts the location to the trie for the given path -func putPythonLocation(trie *pythonLocations, path dyn.Path, location dyn.Location) { +func putPythonLocation(trie *pythonLocations, path *structpath.PathNode, location diag.Location) { currentNode := trie - for _, component := range path { - if key := component.Key(); key != "" { + for _, component := range path.AsSlice() { + if key, ok := component.StringKey(); ok { if _, ok := currentNode.keys[key]; !ok { currentNode.keys[key] = newPythonLocations() } currentNode = currentNode.keys[key] } else { - index := component.Index() + index, _ := component.Index() if _, ok := currentNode.indexes[index]; !ok { currentNode.indexes[index] = newPythonLocations() } @@ -170,20 +168,20 @@ func newPythonLocations() *pythonLocations { // findPythonLocation finds the location or closest ancestor location in the trie for the given path // if no ancestor or exact location is found, false is returned. -func findPythonLocation(locations *pythonLocations, path dyn.Path) (dyn.Location, bool) { +func findPythonLocation(locations *pythonLocations, path *structpath.PathNode) (diag.Location, bool) { currentNode := locations lastLocation := locations.location exists := locations.exists - for _, component := range path { - if key := component.Key(); key != "" { + for _, component := range path.AsSlice() { + if key, ok := component.StringKey(); ok { if _, ok := currentNode.keys[key]; !ok { break } currentNode = currentNode.keys[key] } else { - index := component.Index() + index, _ := component.Index() if _, ok := currentNode.indexes[index]; !ok { break } diff --git a/bundle/config/mutator/python/python_locations_test.go b/bundle/config/mutator/python/python_locations_test.go index 9c5b33d3b59..f8326c18578 100644 --- a/bundle/config/mutator/python/python_locations_test.go +++ b/bundle/config/mutator/python/python_locations_test.go @@ -6,115 +6,97 @@ import ( "testing" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynassert" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestMergeLocations(t *testing.T) { - pythonLocation := dyn.Location{File: "foo.py", Line: 1, Column: 1} - generatedLocation := dyn.Location{File: generatedFileName, Line: 1, Column: 1} - yamlLocation := dyn.Location{File: "foo.yml", Line: 1, Column: 1} + pythonLocation := diag.Location{File: "foo.py", Line: 1, Column: 1} + generatedLocation := diag.Location{File: generatedFileName, Line: 1, Column: 1} + yamlLocation := diag.Location{File: "foo.yml", Line: 1, Column: 1} locations := newPythonLocations() - putPythonLocation(locations, dyn.MustPathFromString("foo"), pythonLocation) - - input := dyn.NewValue( - map[string]dyn.Value{ - "foo": dyn.V( - map[string]dyn.Value{ - "baz": dyn.NewValue("baz", []dyn.Location{yamlLocation}), - "qux": dyn.NewValue("baz", []dyn.Location{generatedLocation, yamlLocation}), - }, - ), - "bar": dyn.NewValue("baz", []dyn.Location{generatedLocation}), - }, - []dyn.Location{yamlLocation}, - ) - - expected := dyn.NewValue( - map[string]dyn.Value{ - "foo": dyn.NewValue( - map[string]dyn.Value{ - // pythonLocation is appended to the beginning of the list if absent - "baz": dyn.NewValue("baz", []dyn.Location{pythonLocation, yamlLocation}), - // generatedLocation is replaced by pythonLocation - "qux": dyn.NewValue("baz", []dyn.Location{pythonLocation, yamlLocation}), - }, - []dyn.Location{pythonLocation}, - ), - // if location is unknown, we keep it as-is - "bar": dyn.NewValue("baz", []dyn.Location{generatedLocation}), - }, - []dyn.Location{yamlLocation}, - ) - - actual, err := mergePythonLocations(input, locations) + putPythonLocation(locations, structpath.MustParsePath("foo"), pythonLocation) - assert.NoError(t, err) - dynassert.Equal(t, expected, actual) + mapper := mergePythonLocations(locations) + at := func(path string, locs ...diag.Location) []diag.Location { + var p *structpath.PathNode + if path != "" { + p = structpath.MustParsePath(path) + } + return mapper(p, locs) + } + + // pythonLocation is prepended if absent + assert.Equal(t, []diag.Location{pythonLocation, yamlLocation}, at("foo.baz", yamlLocation)) + // generatedLocation is replaced by pythonLocation + assert.Equal(t, []diag.Location{pythonLocation, yamlLocation}, at("foo.qux", generatedLocation, yamlLocation)) + assert.Equal(t, []diag.Location{pythonLocation}, at("foo")) + // if location is unknown, we keep it as-is + assert.Equal(t, []diag.Location{generatedLocation}, at("bar", generatedLocation)) + assert.Equal(t, []diag.Location{yamlLocation}, at("", yamlLocation)) } func TestFindLocation(t *testing.T) { - location0 := dyn.Location{File: "foo.py", Line: 1, Column: 1} - location1 := dyn.Location{File: "foo.py", Line: 2, Column: 1} + location0 := diag.Location{File: "foo.py", Line: 1, Column: 1} + location1 := diag.Location{File: "foo.py", Line: 2, Column: 1} locations := newPythonLocations() - putPythonLocation(locations, dyn.MustPathFromString("foo"), location0) - putPythonLocation(locations, dyn.MustPathFromString("foo.bar"), location1) + putPythonLocation(locations, structpath.MustParsePath("foo"), location0) + putPythonLocation(locations, structpath.MustParsePath("foo.bar"), location1) - actual, exists := findPythonLocation(locations, dyn.MustPathFromString("foo.bar")) + actual, exists := findPythonLocation(locations, structpath.MustParsePath("foo.bar")) assert.True(t, exists) assert.Equal(t, location1, actual) } func TestFindLocation_indexPathComponent(t *testing.T) { - location0 := dyn.Location{File: "foo.py", Line: 1, Column: 1} - location1 := dyn.Location{File: "foo.py", Line: 2, Column: 1} - location2 := dyn.Location{File: "foo.py", Line: 3, Column: 1} + location0 := diag.Location{File: "foo.py", Line: 1, Column: 1} + location1 := diag.Location{File: "foo.py", Line: 2, Column: 1} + location2 := diag.Location{File: "foo.py", Line: 3, Column: 1} locations := newPythonLocations() - putPythonLocation(locations, dyn.MustPathFromString("foo"), location0) - putPythonLocation(locations, dyn.MustPathFromString("foo.bar"), location1) - putPythonLocation(locations, dyn.MustPathFromString("foo.bar[0]"), location2) + putPythonLocation(locations, structpath.MustParsePath("foo"), location0) + putPythonLocation(locations, structpath.MustParsePath("foo.bar"), location1) + putPythonLocation(locations, structpath.MustParsePath("foo.bar[0]"), location2) - actual, exists := findPythonLocation(locations, dyn.MustPathFromString("foo.bar[0]")) + actual, exists := findPythonLocation(locations, structpath.MustParsePath("foo.bar[0]")) assert.True(t, exists) assert.Equal(t, location2, actual) } func TestFindLocation_closestAncestorLocation(t *testing.T) { - location0 := dyn.Location{File: "foo.py", Line: 1, Column: 1} - location1 := dyn.Location{File: "foo.py", Line: 2, Column: 1} + location0 := diag.Location{File: "foo.py", Line: 1, Column: 1} + location1 := diag.Location{File: "foo.py", Line: 2, Column: 1} locations := newPythonLocations() - putPythonLocation(locations, dyn.MustPathFromString("foo"), location0) - putPythonLocation(locations, dyn.MustPathFromString("foo.bar"), location1) + putPythonLocation(locations, structpath.MustParsePath("foo"), location0) + putPythonLocation(locations, structpath.MustParsePath("foo.bar"), location1) - actual, exists := findPythonLocation(locations, dyn.MustPathFromString("foo.bar.baz")) + actual, exists := findPythonLocation(locations, structpath.MustParsePath("foo.bar.baz")) assert.True(t, exists) assert.Equal(t, location1, actual) } func TestFindLocation_unknownLocation(t *testing.T) { - location0 := dyn.Location{File: "foo.py", Line: 1, Column: 1} - location1 := dyn.Location{File: "foo.py", Line: 2, Column: 1} + location0 := diag.Location{File: "foo.py", Line: 1, Column: 1} + location1 := diag.Location{File: "foo.py", Line: 2, Column: 1} locations := newPythonLocations() - putPythonLocation(locations, dyn.MustPathFromString("foo"), location0) - putPythonLocation(locations, dyn.MustPathFromString("foo.bar"), location1) + putPythonLocation(locations, structpath.MustParsePath("foo"), location0) + putPythonLocation(locations, structpath.MustParsePath("foo.bar"), location1) - _, exists := findPythonLocation(locations, dyn.MustPathFromString("bar")) + _, exists := findPythonLocation(locations, structpath.MustParsePath("bar")) assert.False(t, exists) } func TestLoadOutput(t *testing.T) { - location := dyn.Location{File: "my_job.py", Line: 1, Column: 1} + location := diag.Location{File: "my_job.py", Line: 1, Column: 1} bundleRoot := t.TempDir() output := `{ "resources": { @@ -137,26 +119,23 @@ func TestLoadOutput(t *testing.T) { locations := newPythonLocations() putPythonLocation( locations, - dyn.MustPathFromString("resources.jobs.my_job"), + structpath.MustParsePath("resources.jobs.my_job"), location, ) - value, diags := loadOutput( + root, diags := loadOutput( bundleRoot, bytes.NewReader([]byte(output)), locations, ) assert.Equal(t, diag.Diagnostics{}, diags) - - name, err := dyn.Get(value, "resources.jobs.my_job.name") - require.NoError(t, err) - require.Equal(t, []dyn.Location{location}, name.Locations()) + require.Equal(t, []diag.Location{location}, root.LocationsAt(structpath.MustParsePath("resources.jobs.my_job.name"))) } func TestParsePythonLocations_absolutePath(t *testing.T) { // output can contain absolute path that is outside of the bundle root - expected := dyn.Location{File: "/Shared/foo.py", Line: 1, Column: 2} + expected := diag.Location{File: "/Shared/foo.py", Line: 1, Column: 2} input := `{"path": "foo", "file": "/Shared/foo.py", "line": 1, "column": 2}` reader := bytes.NewReader([]byte(input)) @@ -171,7 +150,7 @@ func TestParsePythonLocations_absolutePath(t *testing.T) { func TestParsePythonLocations_relativePath(t *testing.T) { // output can contain relative paths, we expect all locations to be absolute // at this stage of mutator pipeline - expected := dyn.Location{File: filepath.Clean("/tmp/my_project/foo.py"), Line: 1, Column: 2} + expected := diag.Location{File: filepath.Clean("/tmp/my_project/foo.py"), Line: 1, Column: 2} input := `{"path": "foo", "file": "foo.py", "line": 1, "column": 2}` reader := bytes.NewReader([]byte(input)) diff --git a/bundle/config/mutator/python/python_mutator.go b/bundle/config/mutator/python/python_mutator.go index 9112aaa808d..f05fee37d7b 100644 --- a/bundle/config/mutator/python/python_mutator.go +++ b/bundle/config/mutator/python/python_mutator.go @@ -28,10 +28,9 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/cli/libs/process" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type phase string @@ -160,42 +159,25 @@ func getOpts(b *bundle.Bundle, phase phase) (opts, error) { // Later version of Python package should reject 'experimental.python' unless 'python' section // is set to equivalent value, and after that reject 'experimental.python' altogether. func applyBackwardsCompatibilityFixes(b *bundle.Bundle) error { - return b.Config.Mutate(func(value dyn.Value) (dyn.Value, error) { - outValue := value + root := b.Config.View() - pythonValue, _ := dyn.Get(outValue, "python") - if !pythonValue.IsValid() { - // if 'python' section doesn't exist, nothing to do - return value, nil - } - - experimentalPythonValue, _ := dyn.Get(outValue, "experimental.python") - - if experimentalPythonValue.IsValid() { - // if 'experimental.python' section exists, nothing to do - return value, nil - } - - experimentalValue, _ := dyn.Get(outValue, "experimental") - if !experimentalValue.IsValid() { - updated, err := dyn.Set(outValue, "experimental", dyn.NewValue(map[string]dyn.Value{}, nil)) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to create 'experimental' section: %w", err) - } else { - outValue = updated - } - } + // if 'python' section doesn't exist, nothing to do + pythonValue := root.Get("python") + if !pythonValue.IsValid() { + return nil + } - // move 'python' section to 'experimental.python' - updated, err := dyn.Set(outValue, "experimental.python", pythonValue) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to set 'experimental.python' section: %w", err) - } else { - outValue = updated - } + // if 'experimental.python' section exists, nothing to do + experimentalPythonPath := structpath.MustParsePath("experimental.python") + if root.Lookup(experimentalPythonPath).IsValid() { + return nil + } - return outValue, nil - }) + // move 'python' section to 'experimental.python' (creating 'experimental' if needed) + if err := b.Config.Assign(experimentalPythonPath, pythonValue); err != nil { + return fmt.Errorf("failed to set 'experimental.python' section: %w", err) + } + return nil } func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { @@ -227,93 +209,82 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno return diag.FromErr(err) } - // mutateDiags is used because Mutate returns 'error' instead of 'diag.Diagnostics' - var mutateDiags diag.Diagnostics - var result applyPythonOutputResult - mutateDiagsHasError := errors.New("unexpected error") - - err = b.Config.Mutate(func(leftRoot dyn.Value) (dyn.Value, error) { - pythonPath, err := detectExecutable(ctx, opts.venvPath) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to get Python interpreter path: %w", err) - } + result, diags := m.run(ctx, b, opts, authEnv) + // we can precisely track resources that are added/updated, so sum doesn't double-count + b.Metrics.PythonUpdatedResourcesCount += int64(result.UpdatedResources.Size()) + b.Metrics.PythonAddedResourcesCount += int64(result.AddedResources.Size()) - cacheDir, cleanup, err := createCacheDir(ctx) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to create cache dir: %w", err) - } - defer cleanup() - - rightRoot, diags := m.runPythonMutator(ctx, leftRoot, runPythonMutatorOpts{ - cacheDir: cacheDir, - bundleRootPath: b.BundleRootPath, - pythonPath: pythonPath, - loadLocations: opts.loadLocations, - authEnv: authEnv, - }) - mutateDiags = diags - if diags.HasError() { - return dyn.InvalidValue, mutateDiagsHasError - } + if diags.HasError() { + return diags + } - newRoot, result0, err := applyPythonOutput(leftRoot, rightRoot) - result = result0 - if err != nil { - return dyn.InvalidValue, fmt.Errorf("internal error when merging output of Python mutator: %w", err) - } + resourcemutator.NormalizeAndInitializeResources(ctx, b, result.AddedResources) + if logdiag.HasError(ctx) { + return diags + } - for _, resourceKey := range result.AddedResources.ToArray() { - log.Debugf(ctx, "added resource at 'resources.%s.%s'", resourceKey.Type, resourceKey.Name) - } + resourcemutator.NormalizeResources(ctx, b, result.UpdatedResources) + return diags +} - for _, resourceKey := range result.UpdatedResources.ToArray() { - log.Debugf(ctx, "updated resource at 'resources.%s.%s'", resourceKey.Type, resourceKey.Name) - } +// run runs the Python mutator and replaces the configuration with its output. +// The configuration is left unchanged if there are errors. +func (m *pythonMutator) run(ctx context.Context, b *bundle.Bundle, opts opts, authEnv map[string]string) (applyPythonOutputResult, diag.Diagnostics) { + var result applyPythonOutputResult - for _, resourceKey := range result.DeletedResources.ToArray() { - log.Debugf(ctx, "deleted resource at 'resources.%s.%s'", resourceKey.Type, resourceKey.Name) - } + pythonPath, err := detectExecutable(ctx, opts.venvPath) + if err != nil { + return result, diag.FromErr(fmt.Errorf("failed to get Python interpreter path: %w", err)) + } - if !result.DeletedResources.IsEmpty() { - return dyn.InvalidValue, fmt.Errorf("unexpected deleted resources: %s", result.DeletedResources.ToArray()) - } + cacheDir, cleanup, err := createCacheDir(ctx) + if err != nil { + return result, diag.FromErr(fmt.Errorf("failed to create cache dir: %w", err)) + } + defer cleanup() - if !result.AddedResources.IsEmpty() && m.phase == PythonMutatorPhaseApplyMutators { - return dyn.InvalidValue, fmt.Errorf("unexpected added resources: %s", result.AddedResources.ToArray()) - } + leftRoot := b.Config.View() + rightRoot, diags := m.runPythonMutator(ctx, leftRoot, runPythonMutatorOpts{ + cacheDir: cacheDir, + bundleRootPath: b.BundleRootPath, + pythonPath: pythonPath, + loadLocations: opts.loadLocations, + authEnv: authEnv, + }) + if diags.HasError() { + return result, diags + } - if !result.UpdatedResources.IsEmpty() && m.phase == PythonMutatorPhaseLoadResources { - return dyn.InvalidValue, fmt.Errorf("unexpected updated resources: %s", result.UpdatedResources.ToArray()) - } + plan, result, err := applyPythonOutput(leftRoot, rightRoot.View()) + if err != nil { + return result, diags.Extend(diag.FromErr(fmt.Errorf("internal error when merging output of Python mutator: %w", err))) + } - return newRoot, nil - }) + for _, resourceKey := range result.AddedResources.ToArray() { + log.Debugf(ctx, "added resource at 'resources.%s.%s'", resourceKey.Type, resourceKey.Name) + } - // we can precisely track resources that are added/updated, so sum doesn't double-count - b.Metrics.PythonUpdatedResourcesCount += int64(result.UpdatedResources.Size()) - b.Metrics.PythonAddedResourcesCount += int64(result.AddedResources.Size()) + for _, resourceKey := range result.UpdatedResources.ToArray() { + log.Debugf(ctx, "updated resource at 'resources.%s.%s'", resourceKey.Type, resourceKey.Name) + } - if err == mutateDiagsHasError { - if !mutateDiags.HasError() { - panic("mutateDiags has no error, but error is expected") - } + for _, resourceKey := range result.DeletedResources.ToArray() { + log.Debugf(ctx, "deleted resource at 'resources.%s.%s'", resourceKey.Type, resourceKey.Name) + } - return mutateDiags - } else { - mutateDiags = mutateDiags.Extend(diag.FromErr(err)) + if !result.DeletedResources.IsEmpty() { + return result, diags.Extend(diag.Errorf("unexpected deleted resources: %s", result.DeletedResources.ToArray())) } - if mutateDiags.HasError() { - return mutateDiags + if !result.AddedResources.IsEmpty() && m.phase == PythonMutatorPhaseApplyMutators { + return result, diags.Extend(diag.Errorf("unexpected added resources: %s", result.AddedResources.ToArray())) } - resourcemutator.NormalizeAndInitializeResources(ctx, b, result.AddedResources) - if logdiag.HasError(ctx) { - return mutateDiags + if !result.UpdatedResources.IsEmpty() && m.phase == PythonMutatorPhaseLoadResources { + return result, diags.Extend(diag.Errorf("unexpected updated resources: %s", result.UpdatedResources.ToArray())) } - resourcemutator.NormalizeResources(ctx, b, result.UpdatedResources) - return mutateDiags + return result, diags.Extend(diag.FromErr(b.Config.Override(plan))) } // createCacheDir returns the directory for input/output files of the Python subprocess, and a cleanup function. @@ -343,7 +314,7 @@ func createCacheDir(ctx context.Context) (string, func(), error) { return cacheDir, func() { _ = os.RemoveAll(cacheDir) }, nil } -func (m *pythonMutator) runPythonMutator(ctx context.Context, root dyn.Value, opts runPythonMutatorOpts) (dyn.Value, diag.Diagnostics) { +func (m *pythonMutator) runPythonMutator(ctx context.Context, root structvar.View, opts runPythonMutatorOpts) (*config.Root, diag.Diagnostics) { inputPath := filepath.Join(opts.cacheDir, "input.json") outputPath := filepath.Join(opts.cacheDir, "output.json") diagnosticsPath := filepath.Join(opts.cacheDir, "diagnostics.json") @@ -368,7 +339,7 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, root dyn.Value, op } if err := writeInputFile(inputPath, root); err != nil { - return dyn.InvalidValue, diag.Errorf("failed to write input file: %s", err) + return nil, diag.Errorf("failed to write input file: %s", err) } stderrBuf := bytes.Buffer{} @@ -398,7 +369,7 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, root dyn.Value, op // if diagnostics file exists, it gives the most descriptive errors // if there is any error, we treat it as fatal error, and stop processing if pythonDiagnostics.HasError() { - return dyn.InvalidValue, pythonDiagnostics + return nil, pythonDiagnostics } // process can fail without reporting errors in diagnostics file or creating it, for instance, @@ -410,21 +381,24 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, root dyn.Value, op Detail: explainProcessErr(ctx, stderrBuf.String()), } - return dyn.InvalidValue, diag.Diagnostics{diagnostic} + return nil, diag.Diagnostics{diagnostic} } // or we can fail to read diagnostics file, that should always be created if pythonDiagnosticsErr != nil { - return dyn.InvalidValue, diag.Errorf("failed to load diagnostics: %s", pythonDiagnosticsErr) + return nil, diag.Errorf("failed to load diagnostics: %s", pythonDiagnosticsErr) } locations, err := loadLocationsFile(opts.bundleRootPath, locationsPath) if err != nil { - return dyn.InvalidValue, diag.Errorf("failed to load locations: %s", err) + return nil, diag.Errorf("failed to load locations: %s", err) } output, outputDiags := loadOutputFile(opts.bundleRootPath, outputPath, locations) pythonDiagnostics = pythonDiagnostics.Extend(outputDiags) + if pythonDiagnostics.HasError() { + return nil, pythonDiagnostics + } // we pass through pythonDiagnostic because it contains warnings return output, pythonDiagnostics @@ -456,9 +430,9 @@ func explainProcessErr(ctx context.Context, stderr string) string { return stderr } -func writeInputFile(inputPath string, input dyn.Value) error { - // we need to marshal dyn.Value instead of bundle.Config to JSON to support - // non-string fields assigned with bundle variables +func writeInputFile(inputPath string, input structvar.View) error { + // we need to marshal the configuration tree instead of bundle.Config to JSON to + // support non-string fields assigned with bundle variables rootConfigJson, err := json.Marshal(input.AsAny()) if err != nil { return fmt.Errorf("failed to marshal input: %w", err) @@ -481,10 +455,10 @@ func loadLocationsFile(bundleRoot, locationsPath string) (*pythonLocations, erro return parsePythonLocations(bundleRoot, locationsFile) } -func loadOutputFile(rootPath, outputPath string, locations *pythonLocations) (dyn.Value, diag.Diagnostics) { +func loadOutputFile(rootPath, outputPath string, locations *pythonLocations) (*config.Root, diag.Diagnostics) { outputFile, err := os.Open(outputPath) if err != nil { - return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to open output file: %w", err)) + return nil, diag.FromErr(fmt.Errorf("failed to open output file: %w", err)) } defer outputFile.Close() @@ -492,7 +466,7 @@ func loadOutputFile(rootPath, outputPath string, locations *pythonLocations) (dy return loadOutput(rootPath, outputFile, locations) } -func loadOutput(rootPath string, outputFile io.Reader, locations *pythonLocations) (dyn.Value, diag.Diagnostics) { +func loadOutput(rootPath string, outputFile io.Reader, locations *pythonLocations) (*config.Root, diag.Diagnostics) { // we need absolute path because later parts of pipeline assume all paths are absolute // and this file will be used as location to resolve relative paths. // @@ -503,41 +477,27 @@ func loadOutput(rootPath string, outputFile io.Reader, locations *pythonLocation // for that, we pass virtualPath instead of outputPath as file location virtualPath, err := filepath.Abs(filepath.Join(rootPath, generatedFileName)) if err != nil { - return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to get absolute path: %w", err)) + return nil, diag.FromErr(fmt.Errorf("failed to get absolute path: %w", err)) } - generated, err := yamlloader.LoadYAML(virtualPath, outputFile) + // The output has locations as if it comes from the generated YAML file; earlier we + // loaded locations.json with source locations in Python code. + root, diags, err := config.LoadFromReader(virtualPath, outputFile, mergePythonLocations(locations)) if err != nil { - return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to parse output file: %w", err)) + return nil, diag.FromErr(fmt.Errorf("failed to parse output file: %w", err)) } - // generated has dyn.Location as if it comes from generated YAML file - // earlier we loaded locations.json with source locations in Python code - generatedWithLocations, err := mergePythonLocations(generated, locations) - if err != nil { - return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to update locations: %w", err)) - } - - return strictNormalize(config.Root{}, generatedWithLocations) -} - -func strictNormalize(dst any, generated dyn.Value) (dyn.Value, diag.Diagnostics) { - normalized, diags := convert.Normalize(dst, generated) - // warnings shouldn't happen because output should be already normalized // when it happens, it's a bug in the mutator, and should be treated as an error - strictDiags := diag.Diagnostics{} - for _, d := range diags { if d.Severity == diag.Warning { d.Severity = diag.Error } - strictDiags = strictDiags.Append(d) } - return normalized, strictDiags + return root, strictDiags } // loadDiagnosticsFile loads diagnostics from a file. diff --git a/bundle/config/mutator/python/python_mutator_test.go b/bundle/config/mutator/python/python_mutator_test.go index 9b106f727bf..ae6d85c2915 100644 --- a/bundle/config/mutator/python/python_mutator_test.go +++ b/bundle/config/mutator/python/python_mutator_test.go @@ -9,21 +9,19 @@ import ( "path/filepath" "runtime" "slices" + "strings" "testing" - "github.com/databricks/cli/libs/dyn/convert" - - "github.com/databricks/cli/bundle/env" - "github.com/stretchr/testify/require" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/process" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestPythonMutator_Name_loadResources(t *testing.T) { @@ -54,7 +52,7 @@ resources: name: job_0 workspace: { current_user: { userName: test }}`) - // set rootPath so that we can make absolute paths in dyn.Location + // set rootPath so that we can make absolute paths in diag.Location b.BundleRootPath = rootPath ctx := withProcessStub( @@ -119,12 +117,12 @@ workspace: { current_user: { userName: test }}`) assert.Equal(t, "pipeline_0", pipeline0.Name) } - // output of locations.json should be applied to underlying dyn.Value - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + // output of locations.json should be applied to underlying configuration + { // location is databricks.yml, because output contains resource as-is - jobName0, err := dyn.GetByPath(v, dyn.MustPathFromString("resources.jobs.job0.name")) - require.NoError(t, err) - assert.Equal(t, []dyn.Location{ + jobName0 := b.Config.View().Lookup(structpath.MustParsePath("resources.jobs.job0.name")) + require.True(t, jobName0.IsValid()) + assert.Equal(t, []diag.Location{ { File: "databricks.yml", Line: 9, @@ -132,9 +130,9 @@ workspace: { current_user: { userName: test }}`) }, }, jobName0.Locations()) - jobName1, err := dyn.GetByPath(v, dyn.MustPathFromString("resources.jobs.job1.name")) - require.NoError(t, err) - assert.Equal(t, []dyn.Location{ + jobName1 := b.Config.View().Lookup(structpath.MustParsePath("resources.jobs.job1.name")) + require.True(t, jobName1.IsValid()) + assert.Equal(t, []diag.Location{ { File: filepath.Join(rootPath, "src/examples/job1.py"), Line: 5, @@ -142,9 +140,9 @@ workspace: { current_user: { userName: test }}`) }, }, jobName1.Locations()) - pipelineName0, err := dyn.GetByPath(v, dyn.MustPathFromString("resources.pipelines.pipeline0.name")) - require.NoError(t, err) - assert.Equal(t, []dyn.Location{ + pipelineName0 := b.Config.View().Lookup(structpath.MustParsePath("resources.pipelines.pipeline0.name")) + require.True(t, pipelineName0.IsValid()) + assert.Equal(t, []diag.Location{ { File: filepath.Join(rootPath, "src/examples/pipeline0.py"), Line: 7, @@ -152,16 +150,14 @@ workspace: { current_user: { userName: test }}`) }, }, pipelineName0.Locations()) - return v, nil - }) - assert.NoError(t, err) + } assert.Equal(t, int64(2), b.Metrics.PythonAddedResourcesCount) assert.Equal(t, int64(0), b.Metrics.PythonUpdatedResourcesCount) assert.Len(t, diags, 1) assert.Equal(t, "job doesn't have any tasks", diags[0].Summary) - assert.Equal(t, []dyn.Location{ + assert.Equal(t, []diag.Location{ { File: "src/examples/file.py", Line: 10, @@ -218,24 +214,22 @@ resources: assert.Equal(t, "job_0", b.Config.Resources.Jobs["job0"].Name) assert.Equal(t, "my job", b.Config.Resources.Jobs["job0"].Description) - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + { // 'name' wasn't changed, so it keeps its location - name, err := dyn.GetByPath(v, dyn.MustPathFromString("resources.jobs.job0.name")) - require.NoError(t, err) + name := b.Config.View().Lookup(structpath.MustParsePath("resources.jobs.job0.name")) + require.True(t, name.IsValid()) assert.Equal(t, "databricks.yml", name.Location().File) // 'description' was updated by Python code and has location of generated file until // we implement source maps - description, err := dyn.GetByPath(v, dyn.MustPathFromString("resources.jobs.job0.description")) - require.NoError(t, err) + description := b.Config.View().Lookup(structpath.MustParsePath("resources.jobs.job0.description")) + require.True(t, description.IsValid()) expectedVirtualPath, err := filepath.Abs(generatedFileName) require.NoError(t, err) assert.Equal(t, expectedVirtualPath, description.Location().File) - return v, nil - }) - assert.NoError(t, err) + } assert.Equal(t, int64(0), b.Metrics.PythonAddedResourcesCount) assert.Equal(t, int64(1), b.Metrics.PythonUpdatedResourcesCount) @@ -456,20 +450,14 @@ func TestInterpreterPath(t *testing.T) { } } -func TestStrictNormalize(t *testing.T) { - // NB: there is no way to trigger diag.Error, so we don't test it - - type TestStruct struct { - A int `json:"a"` - } - - value := dyn.NewValue(map[string]dyn.Value{"A": dyn.NewValue("abc", nil)}, nil) +func TestLoadOutputStrict(t *testing.T) { + // Warnings when decoding the output are a bug in the Python code and are errors. + output := `{"resources": {"jobs": {"my_job": {"max_concurrent_runs": "abc"}}}}` - _, diags := convert.Normalize(TestStruct{}, value) - _, strictDiags := strictNormalize(TestStruct{}, value) + _, diags := loadOutput(t.TempDir(), strings.NewReader(output), newPythonLocations()) - assert.False(t, diags.HasError()) - assert.True(t, strictDiags.HasError()) + require.True(t, diags.HasError()) + assert.Equal(t, `cannot parse "abc" as an integer`, diags[0].Summary) } func TestCreateCacheDir(t *testing.T) { diff --git a/bundle/config/mutator/resolve_variable_references.go b/bundle/config/mutator/resolve_variable_references.go index 257309c55f8..f1e779ad832 100644 --- a/bundle/config/mutator/resolve_variable_references.go +++ b/bundle/config/mutator/resolve_variable_references.go @@ -4,18 +4,17 @@ import ( "context" "errors" "fmt" + "maps" + "reflect" "slices" "strings" - "github.com/databricks/cli/libs/dyn/merge" - "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/variable" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) /* @@ -45,13 +44,13 @@ var defaultPrefixes = []string{ "variables", } -var artifactPath = dyn.MustPathFromString("artifacts") +var artifactPath = structpath.MustParsePath("artifacts") type resolveVariableReferences struct { prefixes []string - pattern dyn.Pattern - lookupFn func(dyn.Value, dyn.Path, *bundle.Bundle) (dyn.Value, error) - allowPathFn func(dyn.Path) bool + pattern *structpath.PatternNode + lookupFn func(structvar.View, *structpath.PathNode, *bundle.Bundle) (structvar.View, error) + allowPathFn func(*structpath.PathNode) bool extraRounds int // includeResources allows resolving variables in 'resources', otherwise, they are excluded. @@ -76,7 +75,7 @@ func ResolveVariableReferencesOnlyResources(prefixes ...string) bundle.Mutator { prefixes: prefixes, lookupFn: lookup, extraRounds: maxResolutionRounds - 1, - pattern: dyn.NewPattern(dyn.Key("resources")), + pattern: structpath.MustParsePattern("resources"), includeResources: true, } } @@ -95,7 +94,7 @@ func ResolveVariableReferencesWithoutResources(prefixes ...string) bundle.Mutato func ResolveVariableReferencesInLookup() bundle.Mutator { return &resolveVariableReferences{ prefixes: defaultPrefixes, - pattern: dyn.NewPattern(dyn.Key("variables"), dyn.AnyKey(), dyn.Key("lookup")), + pattern: structpath.MustParsePattern("variables.*.lookup"), lookupFn: lookupForVariables, extraRounds: maxResolutionRounds - 1, } @@ -112,36 +111,44 @@ func ResolveVolumePathReferencesOnlyResources() bundle.Mutator { } } -func lookup(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) { +func lookup(v structvar.View, path *structpath.PathNode, b *bundle.Bundle) (structvar.View, error) { if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) { if path.String() == "workspace.file_path" { - return dyn.V(b.SyncRootPath), nil + return structvar.NewView(&b.SyncRootPath, nil, nil), nil } } // Future opportunity: if we lookup this path in both the given root // and the synthesized root, we know if it was explicitly set or implied to be empty. // Then we can emit a warning if it was not explicitly set. - return dyn.GetByPath(v, path) + return lookupValue(v, path) } -func lookupForVariables(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) { - if path[0].Key() != "variables" { - return lookup(v, path, b) - } +// lookupValue returns the value at path. Fields that are declared in the type but not +// set resolve to their zero value. This enables users to interpolate variable references +// to fields that haven't been set, e.g. ${bundle.git.origin_url} resolves to an empty +// string if a bundle isn't located in a Git repository (yet). +func lookupValue(x structvar.View, path *structpath.PathNode) (structvar.View, error) { + return x.LookupWithDefaults(path) +} - varV, err := dyn.GetByPath(v, path[:len(path)-1]) - if err != nil { - return dyn.InvalidValue, err +func lookupForVariables(v structvar.View, path *structpath.PathNode, b *bundle.Bundle) (structvar.View, error) { + if path.KeyAt(0) != "variables" { + return lookup(v, path, b) } - var vv variable.Variable - err = convert.ToTyped(&vv, varV) + varV, err := lookupValue(v, path.Parent()) if err != nil { - return dyn.InvalidValue, err + return structvar.View{}, err } - if vv.Lookup != nil && vv.Lookup.String() != "" { - return dyn.InvalidValue, errors.New("lookup variables cannot contain references to another lookup variables") + if lookupV := varV.Get("lookup"); lookupV.IsValid() { + var vl variable.Lookup + if _, err := (&structvar.StructVar{Value: &vl}).Assign(nil, lookupV); err != nil { + return structvar.View{}, err + } + if vl.String() != "" { + return structvar.View{}, errors.New("lookup variables cannot contain references to another lookup variables") + } } return lookup(v, path, b) @@ -160,14 +167,23 @@ func (m *resolveVariableReferences) Validate(ctx context.Context, b *bundle.Bund } func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - prefixes := make([]dyn.Path, len(m.prefixes)) + prefixes := make([]*structpath.PathNode, len(m.prefixes)) for i, prefix := range m.prefixes { - prefixes[i] = dyn.MustPathFromString(prefix) + prefixes[i] = structpath.MustParsePath(prefix) } // The path ${var.foo} is a shorthand for ${variables.foo.value}. // We rewrite it here to make the resolution logic simpler. - varPath := dyn.NewPath(dyn.Key("var")) + varPath := structpath.MustParsePath("var") + + // Resolution converts the whole configuration on every round; skip it if no + // reference would be resolved. + if !m.hasReferencesToResolve(b, prefixes, varPath) { + if m.artifactsReferenceUsed { + b.Metrics.SetBoolValue("artifacts_reference_used", true) + } + return nil + } var diags diag.Diagnostics maxRounds := 1 + m.extraRounds @@ -185,6 +201,11 @@ func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle) break } + // Another round would only find out that nothing is left to resolve. + if !m.hasReferencesToResolve(b, prefixes, varPath) { + break + } + if round >= maxRounds-1 { diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, @@ -202,86 +223,195 @@ func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle) return diags } -func (m *resolveVariableReferences) resolveOnce(b *bundle.Bundle, prefixes []dyn.Path, varPath dyn.Path) (bool, diag.Diagnostics) { +// hasReferencesToResolve reports whether the configuration in scope of this mutator +// has a reference with one of the prefixes. It is conservative: the scope is all of +// the configuration outside "resources" (or only "resources"), regardless of the pattern. +// Like resolution itself, it records whether "artifacts" is referenced. +func (m *resolveVariableReferences) hasReferencesToResolve(b *bundle.Bundle, prefixes []*structpath.PathNode, varPath *structpath.PathNode) bool { + onlyResources := m.includeResources && m.pattern != nil + inScope := func(path string) bool { + isResources := path == "resources" || strings.HasPrefix(path, "resources.") + return (m.includeResources || !isResources) && (!onlyResources || isResources) + } + + found := false + check := func(s string) { + ref, ok := structvar.NewRef(s) + if !ok { + return + } + for _, r := range ref.References() { + path, err := structpath.ParsePath(r) + if err != nil { + // Let resolution report it. + found = true + return + } + if path.HasPrefix(varPath) { + path = structpath.Join(structpath.NewStringKey(nil, "variables"), path.SkipPrefix(1).AsSlice()...) + } + if path.HasPrefix(artifactPath) { + m.artifactsReferenceUsed = true + } + if slices.ContainsFunc(prefixes, path.HasPrefix) { + found = true + } + } + } + + root := reflect.ValueOf(&b.Config).Elem() + rootType := root.Type() + for i := range rootType.NumField() { + name, _, _ := strings.Cut(rootType.Field(i).Tag.Get("json"), ",") + if name == "" || name == "-" || !inScope(name) { + continue + } + walkStrings(root.Field(i), check) + } + for path, ref := range b.Config.References() { + if inScope(path.String()) { + check(ref) + } + } + return found +} + +func (m *resolveVariableReferences) resolveOnce(b *bundle.Bundle, prefixes []*structpath.PathNode, varPath *structpath.PathNode) (bool, diag.Diagnostics) { var diags diag.Diagnostics hasUpdates := false - err := m.selectivelyMutate(b, func(root dyn.Value) (dyn.Value, error) { - // Synthesize a copy of the root that has all fields that are present in the type - // but not set in the dynamic value set to their corresponding empty value. - // This enables users to interpolate variable references to fields that haven't - // been explicitly set in the dynamic value. - // - // For example: ${bundle.git.origin_url} should resolve to an empty string - // if a bundle isn't located in a Git repository (yet). - // - // This is consistent with the behavior prior to using the dynamic value system. - // - // We can ignore the diagnostics return value because we know that the dynamic value - // has already been normalized when it was first loaded from the configuration file. - // - normalized, _ := convert.Normalize(b.Config, root, convert.IncludeMissingFields) - - // If the pattern is nil, we resolve references in the entire configuration. - root, err := dyn.MapByPattern(root, m.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - // Resolve variable references in all values. - return dynvar.Resolve(v, func(path dyn.Path) (dyn.Value, error) { - // Rewrite the shorthand path ${var.foo} into ${variables.foo.value}. - if path.HasPrefix(varPath) { - newPath := dyn.NewPath( - dyn.Key("variables"), - path[1], - dyn.Key("value"), - ) - - if len(path) > 2 { - newPath = newPath.Append(path[2:]...) - } - - path = newPath - } - - // If the path starts with "artifacts", we need to add a metric to track if this reference is used. - if path.HasPrefix(artifactPath) { - m.artifactsReferenceUsed = true - } - - // Perform resolution only if the path starts with one of the specified prefixes. - if slices.ContainsFunc(prefixes, path.HasPrefix) { - if slices.Contains(m.excludePaths, path.String()) { - return dyn.InvalidValue, dynvar.ErrSkipResolution - } - if m.allowPathFn != nil && !m.allowPathFn(path) { - return dyn.InvalidValue, dynvar.ErrSkipResolution - } - value, err := m.lookupFn(normalized, path, b) - hasUpdates = hasUpdates || (err == nil && value.IsValid()) - return value, err - } - - return dyn.InvalidValue, dynvar.ErrSkipResolution - }) - }) + root := b.Config.View() + + lookupFn := func(sp *structpath.PathNode) (structvar.View, error) { + path := sp + // Rewrite the shorthand path ${var.foo} into ${variables.foo.value}. + if path.HasPrefix(varPath) { + path = structpath.Join(structpath.NewPath(nil, "variables", path.KeyAt(1), "value"), path.SkipPrefix(2).AsSlice()...) + } + + // If the path starts with "artifacts", we need to add a metric to track if this reference is used. + if path.HasPrefix(artifactPath) { + m.artifactsReferenceUsed = true + } + + // Perform resolution only if the path starts with one of the specified prefixes. + if slices.ContainsFunc(prefixes, path.HasPrefix) { + if slices.Contains(m.excludePaths, path.String()) { + return structvar.View{}, structvar.ErrSkipResolution + } + if m.allowPathFn != nil && !m.allowPathFn(path) { + return structvar.View{}, structvar.ErrSkipResolution + } + value, err := m.lookupFn(root, path, b) + hasUpdates = hasUpdates || (err == nil && value.IsValid()) + return value, err + } + + return structvar.View{}, structvar.ErrSkipResolution + } + + // Resolve the references in each value matching the pattern (the whole configuration + // if the pattern is nil). Template keys are relative to that value, like the paths + // in cycle errors. The results are applied after all of them are resolved. + type update struct { + path *structpath.PathNode + value structvar.View + } + var updates []update + resolveIn := func(p *structpath.PathNode, refs []referenceString) error { + templates := map[string]structvar.Template{} + paths := map[string]*structpath.PathNode{} + for _, ref := range refs { + key := ref.path.String() + templates[key] = structvar.Template{Value: ref.value, Locations: ref.locs} + paths[key] = structpath.Join(p, ref.path.AsSlice()...) + } + out, err := structvar.Resolve(templates, lookupFn) if err != nil { - return dyn.InvalidValue, err + return err } + for _, key := range slices.Sorted(maps.Keys(out)) { + updates = append(updates, update{path: paths[key], value: out[key]}) + } + return nil + } - // Normalize the result because variable resolution may have been applied to non-string fields. - // For example, a variable reference may have been resolved to a integer. - root, normaliseDiags := convert.Normalize(b.Config, root) - diags = diags.Extend(normaliseDiags) - return root, nil - }) + var err error + if m.pattern == nil { + err = resolveIn(nil, m.referencesInScope(root)) + } else { + err = structvar.ForEach(root, m.pattern, func(np *structpath.PathNode, v structvar.View) error { + if !m.inScope(np) { + return nil + } + return resolveIn(np, collectReferenceStrings(v)) + }) + } if err != nil { - diags = diags.Extend(resolveErrorDiags(err)) + return hasUpdates, diags.Extend(resolveErrorDiags(err)) + } + + // Store the results in the typed configuration, converting them to the type of the + // field (e.g. a variable reference resolved to an integer). + for _, u := range updates { + d, err := b.Config.Decode(u.path, u.value) + diags = diags.Extend(d).Extend(diag.FromErr(err)) } return hasUpdates, diags } +// inScope reports whether path is in the part of the configuration this mutator resolves. +func (m *resolveVariableReferences) inScope(path *structpath.PathNode) bool { + return m.includeResources || path.KeyAt(0) != "resources" +} + +// referencesInScope returns the reference strings in the whole configuration, +// excluding "resources" unless resources are included. +func (m *resolveVariableReferences) referencesInScope(root structvar.View) []referenceString { + if m.includeResources { + return collectReferenceStrings(root) + } + var out []referenceString + for k, v := range root.MapItems() { + if !m.inScope(structpath.NewStringKey(nil, k)) { + continue + } + p := structpath.NewStringKey(nil, k) + for _, ref := range collectReferenceStrings(v) { + ref.path = structpath.Join(p, ref.path.AsSlice()...) + out = append(out, ref) + } + } + return out +} + +// referenceString is a string with a variable reference at path (relative to the +// value being resolved). +type referenceString struct { + path *structpath.PathNode + value string + locs []diag.Location +} + +// collectReferenceStrings returns the strings in v that contain variable references. +// Pure references in fields that cannot hold a string are strings in the view. +func collectReferenceStrings(v structvar.View) []referenceString { + var out []referenceString + _ = structvar.Walk(v, func(p *structpath.PathNode, v structvar.View) error { + if s, ok := v.AsString(); ok { + if _, ok := structvar.NewRef(s); ok { + out = append(out, referenceString{path: p, value: s, locs: v.Locations()}) + } + } + return nil + }) + return out +} + // resolveErrorDiags renders "did you mean" suggestions as a diagnostic Detail so // libs/diag owns the multi-line formatting. func resolveErrorDiags(err error) diag.Diagnostics { - refErr, ok := errors.AsType[*dynvar.ReferenceError](err) + refErr, ok := errors.AsType[*structvar.ReferenceError](err) if !ok || len(refErr.Suggestions) == 0 { return diag.FromErr(err) } @@ -303,66 +433,11 @@ func resolveErrorDiags(err error) diag.Diagnostics { }} } -// selectivelyMutate applies a function to a subset of the configuration -func (m *resolveVariableReferences) selectivelyMutate(b *bundle.Bundle, fn func(value dyn.Value) (dyn.Value, error)) error { - return b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - allKeys, err := getAllKeys(root) - if err != nil { - return dyn.InvalidValue, err - } - - var included []string - for _, key := range allKeys { - if key == "resources" { - if m.includeResources { - included = append(included, key) - } - } else { - included = append(included, key) - } - } - - includedRoot, err := merge.Select(root, included) - if err != nil { - return dyn.InvalidValue, err - } - - excludedRoot, err := merge.AntiSelect(root, included) - if err != nil { - return dyn.InvalidValue, err - } - - updatedRoot, err := fn(includedRoot) - if err != nil { - return dyn.InvalidValue, err - } - - // merge is recursive, but it doesn't matter because keys are mutually exclusive - return merge.Merge(updatedRoot, excludedRoot) - }) -} - -func getAllKeys(root dyn.Value) ([]string, error) { - var keys []string - - if mapping, ok := root.AsMap(); ok { - for _, key := range mapping.Keys() { - if keyString, ok := key.AsString(); ok { - keys = append(keys, keyString) - } else { - return nil, fmt.Errorf("key is not a string: %v", key) - } - } - } - - return keys, nil -} - -func isVolumePathReferencePath(path dyn.Path) bool { - if len(path) != 4 { +func isVolumePathReferencePath(path *structpath.PathNode) bool { + if path.Len() != 4 { return false } - return path[0].Key() == "resources" && - path[1].Key() == "volumes" && - path[3].Key() == "volume_path" + return path.KeyAt(0) == "resources" && + path.KeyAt(1) == "volumes" && + path.KeyAt(3) == "volume_path" } diff --git a/bundle/config/mutator/resolve_variable_references_test.go b/bundle/config/mutator/resolve_variable_references_test.go index 284b78846b4..4be7f012af2 100644 --- a/bundle/config/mutator/resolve_variable_references_test.go +++ b/bundle/config/mutator/resolve_variable_references_test.go @@ -6,7 +6,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/stretchr/testify/require" ) @@ -106,7 +106,7 @@ func TestResolveVolumePathReferencesOnlyResources_MissingTarget(t *testing.T) { } func TestIsVolumePathReferencePath(t *testing.T) { - require.True(t, isVolumePathReferencePath(dyn.MustPathFromString("resources.volumes.foo.volume_path"))) - require.False(t, isVolumePathReferencePath(dyn.MustPathFromString("resources.volumes.foo.name"))) - require.False(t, isVolumePathReferencePath(dyn.MustPathFromString("resources.jobs.foo.name"))) + require.True(t, isVolumePathReferencePath(structpath.MustParsePath("resources.volumes.foo.volume_path"))) + require.False(t, isVolumePathReferencePath(structpath.MustParsePath("resources.volumes.foo.name"))) + require.False(t, isVolumePathReferencePath(structpath.MustParsePath("resources.jobs.foo.name"))) } diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go index f2a6d857090..33a09135222 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go @@ -3,17 +3,17 @@ package resourcemutator import ( "context" "fmt" + "reflect" "slices" "strings" - "github.com/databricks/cli/bundle/permissions" - "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/permissions" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go/service/iam" ) @@ -108,60 +108,84 @@ func (m *bundlePermissions) Apply(ctx context.Context, b *bundle.Bundle) diag.Di return diag.FromErr(err) } - patterns := make(map[string]dyn.Pattern, 0) + keys := make([]string, 0, len(levelsMap)) for key := range levelsMap { - patterns[key] = dyn.NewPattern( - dyn.Key("resources"), - dyn.Key(key), - dyn.AnyKey(), - ) + keys = append(keys, key) } + slices.Sort(keys) - err = b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - for key, pattern := range patterns { - v, err = dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - var permissions []resources.Permission - pv, err := dyn.Get(v, "permissions") - // If the permissions field is not found, we set to an empty array - if err != nil { - pv = dyn.V([]dyn.Value{}) - } - - err = convert.ToTyped(&permissions, pv) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to convert permissions: %w", err) - } + for _, key := range keys { + pattern := structpath.MustParsePattern("resources." + key + ".*") - permissions = append(permissions, convertPermissions( - ctx, - b.Config.Permissions, - permissions, - key, - levelsMap[key], - )...) + err = structvar.ForEach(b.Config.View(), pattern, func(p *structpath.PathNode, v structvar.View) error { + had := v.Get("permissions").IsValid() + var permissions []resources.Permission + for _, pv := range v.Get("permissions").Sequence() { + level, _ := pv.Get("level").AsString() + userName, _ := pv.Get("user_name").AsString() + groupName, _ := pv.Get("group_name").AsString() + servicePrincipalName, _ := pv.Get("service_principal_name").AsString() + permissions = append(permissions, resources.Permission{ + Level: iam.PermissionLevel(level), + UserName: userName, + GroupName: groupName, + ServicePrincipalName: servicePrincipalName, + }) + } - if len(permissions) == 0 { - permissions = nil + added := convertPermissions( + ctx, + b.Config.Permissions, + permissions, + key, + levelsMap[key], + ) + if len(added) > 0 { + if err := appendPermissions(v, added); err != nil { + return err } + } - pv, err = convert.FromTyped(permissions, dyn.NilValue) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to convert permissions: %w", err) - } + if !had { + return nil + } - return dyn.Set(v, "permissions", pv) - }) - if err != nil { - return dyn.InvalidValue, err + // Empty permissions are dropped. Otherwise they are rebuilt without locations. + permissionsPath := structpath.NewStringKey(p, "permissions") + if len(permissions) == 0 && len(added) == 0 { + return b.Config.Delete(permissionsPath) } + b.Config.SetLocations(permissionsPath, nil) + return nil + }) + if err != nil { + return diag.FromErr(err) } + } - return v, nil - }) - if err != nil { - return diag.FromErr(err) + return nil +} + +// appendPermissions appends permissions to the permissions field of the resource described by v. +// The type of the permissions differs between resources, but all have the same fields. +func appendPermissions(v structvar.View, permissions []resources.Permission) error { + r := v.Reflect() + for r.Kind() == reflect.Pointer || r.Kind() == reflect.Interface { + r = r.Elem() + } + field := r.FieldByName("Permissions") + if !field.IsValid() || !field.CanSet() || field.Kind() != reflect.Slice { + return fmt.Errorf("cannot set permissions of %s", r.Type()) } + for _, p := range permissions { + elem := reflect.New(field.Type().Elem()).Elem() + elem.FieldByName("Level").SetString(string(p.Level)) + elem.FieldByName("UserName").SetString(p.UserName) + elem.FieldByName("GroupName").SetString(p.GroupName) + elem.FieldByName("ServicePrincipalName").SetString(p.ServicePrincipalName) + field.Set(reflect.Append(field, elem)) + } return nil } diff --git a/bundle/config/mutator/resourcemutator/apply_default_task_source.go b/bundle/config/mutator/resourcemutator/apply_default_task_source.go index 30e000b1fb1..a95130597f3 100644 --- a/bundle/config/mutator/resourcemutator/apply_default_task_source.go +++ b/bundle/config/mutator/resourcemutator/apply_default_task_source.go @@ -5,7 +5,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/jobs" ) @@ -37,57 +36,53 @@ func (a *applyDefaultTaskSource) Name() string { return "ApplyDefaultTaskSource" } -// sourceAwareTaskTypes are the task types that support the `source` field. -// https://docs.databricks.com/api/workspace/jobs/create -var sourceAwareTaskTypes = []string{ - "dbt_task", - "gen_ai_compute_task", - "notebook_task", - "spark_python_task", - "sql_task.file", -} - func (a *applyDefaultTaskSource) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - jobsPattern := dyn.NewPattern(dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey()) + for name, job := range b.Config.Resources.Jobs { + // Only git_source jobs need an explicit source; leave the rest untouched. + // A missing key or an explicit `git_source: null` both count as absent, + // matching the typed nil pointer TranslatePaths keys off; otherwise we would + // set source: GIT on a job whose paths get translated to workspace paths. + // A reference is not absent, although the typed field is nil. + if job == nil || (job.GitSource == nil && !b.Config.IsReference("resources.jobs."+name+".git_source")) { + continue + } - err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - return dyn.MapByPattern(root, jobsPattern, func(_ dyn.Path, job dyn.Value) (dyn.Value, error) { - // Only git_source jobs need an explicit source; leave the rest untouched. - // A missing key (KindInvalid) or an explicit `git_source: null` (KindNil) - // both count as absent, matching the typed nil pointer TranslatePaths keys - // off; otherwise we would set source: GIT on a job whose paths get - // translated to workspace paths. - gitSource := job.Get("git_source") - if gitSource.Kind() == dyn.KindInvalid || gitSource.Kind() == dyn.KindNil { - return job, nil + for i := range job.Tasks { + task := &job.Tasks[i] + if task.ForEachTask != nil { + setGitTaskSource(&task.ForEachTask.Task) } - - return dyn.Map(job, "tasks", dyn.Foreach(func(_ dyn.Path, task dyn.Value) (dyn.Value, error) { - task, err := dyn.Map(task, "for_each_task.task", func(_ dyn.Path, foreachTask dyn.Value) (dyn.Value, error) { - return setGitTaskSource(foreachTask) - }) - if err != nil { - return dyn.InvalidValue, err - } - return setGitTaskSource(task) - })) - }) - }) - - return diag.FromErr(err) + setGitTaskSource(task) + } + } + return nil } // setGitTaskSource sets source: GIT on the first task-type block present that // supports the field, unless the user already set it. -func setGitTaskSource(task dyn.Value) (dyn.Value, error) { - for _, taskType := range sourceAwareTaskTypes { - t, err := dyn.Get(task, taskType) - if err != nil { - continue - } - if _, err := dyn.Get(t, "source"); err != nil { - return dyn.Set(task, taskType+".source", dyn.V(string(jobs.SourceGit))) +// The task types are the ones that support the `source` field: +// https://docs.databricks.com/api/workspace/jobs/create +func setGitTaskSource(task *jobs.Task) { + var sources []*jobs.Source + if task.DbtTask != nil { + sources = append(sources, &task.DbtTask.Source) + } + if task.GenAiComputeTask != nil { + sources = append(sources, &task.GenAiComputeTask.Source) + } + if task.NotebookTask != nil { + sources = append(sources, &task.NotebookTask.Source) + } + if task.SparkPythonTask != nil { + sources = append(sources, &task.SparkPythonTask.Source) + } + if task.SqlTask != nil && task.SqlTask.File != nil { + sources = append(sources, &task.SqlTask.File.Source) + } + for _, source := range sources { + if *source == "" { + *source = jobs.SourceGit + return } } - return task, nil } diff --git a/bundle/config/mutator/resourcemutator/apply_presets.go b/bundle/config/mutator/resourcemutator/apply_presets.go index f4e555ed7f3..bb84240d7cc 100644 --- a/bundle/config/mutator/resourcemutator/apply_presets.go +++ b/bundle/config/mutator/resourcemutator/apply_presets.go @@ -11,7 +11,6 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/metrics" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/textutil" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -353,7 +352,7 @@ func validatePauseStatus(b *bundle.Bundle) diag.Diagnostics { return diag.Diagnostics{{ Summary: "Invalid value for trigger_pause_status, should be PAUSED or UNPAUSED", Severity: diag.Error, - Locations: []dyn.Location{b.Config.GetLocation("presets.trigger_pause_status")}, + Locations: []diag.Location{b.Config.GetLocation("presets.trigger_pause_status")}, }} } diff --git a/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go b/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go index 488e215a2e2..d18714e8842 100644 --- a/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go +++ b/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go @@ -8,7 +8,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structvar" ) type captureUCDependencies struct{} @@ -159,7 +159,7 @@ func (m *captureUCDependencies) Apply(ctx context.Context, b *bundle.Bundle) dia } // A name that already contains a reference is left as is: we only rewrite a // fully literal name and do not support a mix of references and literals. - if dynvar.ContainsVariableReference(qm.OutputSchemaName) { + if structvar.ContainsVariableReference(qm.OutputSchemaName) { continue } // OutputSchemaName is a compound "catalog.schema" string. @@ -179,7 +179,7 @@ func (m *captureUCDependencies) Apply(ctx context.Context, b *bundle.Bundle) dia } // A name that already contains a reference is left as is; a mix of // references and literals is not supported. - if dynvar.ContainsVariableReference(idx.Name) { + if structvar.ContainsVariableReference(idx.Name) { continue } // Name is a three-part "catalog.schema.index" UC identifier. @@ -212,7 +212,7 @@ func (m *captureUCDependencies) Apply(ctx context.Context, b *bundle.Bundle) dia } // A parent that already contains a reference is left as is; a mix of // references and literals is not supported. - if dynvar.ContainsVariableReference(ms.Parent) { + if structvar.ContainsVariableReference(ms.Parent) { continue } ms.Parent = resolveAiGatewayParent(b, ms.Parent) diff --git a/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition.go b/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition.go index 1e7a0d6f64b..84dddb367ba 100644 --- a/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition.go +++ b/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition.go @@ -7,7 +7,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) // jsonPolicyFields are the JSON-policy fields normalized from inline YAML to a JSON string. @@ -26,44 +27,43 @@ func (c configureClusterPolicyDefinition) Name() string { func (c configureClusterPolicyDefinition) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics - pattern := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("cluster_policies"), - dyn.AnyKey(), - ) + // Skip converting the configuration if there is nothing to configure. + if len(b.Config.Resources.ClusterPolicies) == 0 { + return nil + } - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - for _, field := range jsonPolicyFields { - def := v.Get(field) + pattern := structpath.MustParsePattern("resources.cluster_policies.*") - // Marshal an inline structured value to a JSON string so both - // config-side and state-side carry the same plain string. Otherwise - // YAML decodes small ints as Go `int` while state JSON round-trip - // decodes them as `float64`, and structdiff reports false drift. - switch def.Kind() { - case dyn.KindInvalid, dyn.KindNil, dyn.KindString: - // KindInvalid means the field is absent; leave it for backend validation. - continue - case dyn.KindMap: - jsonBytes, err := json.Marshal(def.AsAny()) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to marshal inline %s: %w", field, err) - } - v, err = dyn.Set(v, field, dyn.V(string(jsonBytes))) - if err != nil { - return dyn.InvalidValue, err - } - default: - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("%s must be a string or map, got %s", field, def.Kind()), - Locations: def.Locations(), - }) + err := structvar.ForEach(b.Config.View(), pattern, func(p *structpath.PathNode, v structvar.View) error { + for _, field := range jsonPolicyFields { + def := v.Get(field) + + // Marshal an inline structured value to a JSON string so both + // config-side and state-side carry the same plain string. Otherwise + // YAML decodes small ints as Go `int` while state JSON round-trip + // decodes them as `float64`, and structdiff reports false drift. + switch def.Kind() { + case structvar.KindInvalid, structvar.KindNil, structvar.KindString: + // KindInvalid means the field is absent; leave it for backend validation. + continue + case structvar.KindMap: + jsonBytes, err := json.Marshal(def.AsAny()) + if err != nil { + return fmt.Errorf("failed to marshal inline %s: %w", field, err) + } + err = b.Config.Set(structpath.NewStringKey(p, field), string(jsonBytes)) + if err != nil { + return err } + default: + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("%s must be a string or map, got %s", field, def.Kind()), + Locations: def.Locations(), + }) } - return v, nil - }) + } + return nil }) diags = diags.Extend(diag.FromErr(err)) diff --git a/bundle/config/mutator/resourcemutator/configure_dashboards_serialized_dashboard.go b/bundle/config/mutator/resourcemutator/configure_dashboards_serialized_dashboard.go index a17e5c6b07c..5251c92716d 100644 --- a/bundle/config/mutator/resourcemutator/configure_dashboards_serialized_dashboard.go +++ b/bundle/config/mutator/resourcemutator/configure_dashboards_serialized_dashboard.go @@ -7,7 +7,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) const ( @@ -28,62 +29,62 @@ func (c configureDashboardSerializedDashboard) Name() string { func (c configureDashboardSerializedDashboard) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics - pattern := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("dashboards"), - dyn.AnyKey(), - ) + // Skip converting the configuration if there is nothing to configure. + if len(b.Config.Resources.Dashboards) == 0 { + return nil + } - // Configure serialized_dashboard field for all dashboards. - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - // Include "serialized_dashboard" field if "file_path" is set. - // Note: the Terraform resource supports "file_path" natively, but we read the contents of the dashboard here - // to be able to read file contents in Databricks Workspace (reading a dashboard file via file system fails there) - filePath, hasFilePath := v.Get(filePathFieldName).AsString() - sd := v.Get(serializedDashboardFieldName) - - if hasFilePath { - // file_path and serialized_dashboard are two ways to provide the - // same content. Accepting both is ambiguous, so reject it instead - // of silently picking one. - if sd.IsValid() && sd.Kind() != dyn.KindNil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "both file_path and serialized_dashboard are set; specify only one", - Locations: sd.Locations(), - }) - return v, nil - } + pattern := structpath.MustParsePattern("resources.dashboards.*") - contents, err := b.SyncRoot.ReadFile(filePath) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to read serialized dashboard from file_path %s: %w", filePath, err) - } - return dyn.Set(v, serializedDashboardFieldName, dyn.V(string(contents))) - } + // Configure serialized_dashboard field for all dashboards. + err := structvar.ForEach(b.Config.View(), pattern, func(p *structpath.PathNode, v structvar.View) error { + // Include "serialized_dashboard" field if "file_path" is set. + // Note: the Terraform resource supports "file_path" natively, but we read the contents of the dashboard here + // to be able to read file contents in Databricks Workspace (reading a dashboard file via file system fails there) + filePath, hasFilePath := v.Get(filePathFieldName).AsString() + sd := v.Get(serializedDashboardFieldName) + sdPath := structpath.NewStringKey(p, serializedDashboardFieldName) - // Marshal an inline structured serialized_dashboard to a JSON string - switch sd.Kind() { - case dyn.KindInvalid, dyn.KindNil, dyn.KindString: - // KindInvalid means serialized_dashboard is absent (neither it nor - // file_path is set); leave it for backend validation to reject. - return v, nil - case dyn.KindMap: - jsonBytes, err := json.Marshal(sd.AsAny()) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to marshal inline serialized_dashboard: %w", err) - } - return dyn.Set(v, serializedDashboardFieldName, dyn.V(string(jsonBytes))) - default: + if hasFilePath { + // file_path and serialized_dashboard are two ways to provide the + // same content. Accepting both is ambiguous, so reject it instead + // of silently picking one. + if sd.IsValid() && sd.Kind() != structvar.KindNil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("serialized_dashboard must be a string or map, got %s", sd.Kind()), + Summary: "both file_path and serialized_dashboard are set; specify only one", Locations: sd.Locations(), }) - return v, nil + return nil + } + + contents, err := b.SyncRoot.ReadFile(filePath) + if err != nil { + return fmt.Errorf("failed to read serialized dashboard from file_path %s: %w", filePath, err) + } + return b.Config.Set(sdPath, string(contents)) + } + + // Marshal an inline structured serialized_dashboard to a JSON string + switch sd.Kind() { + case structvar.KindInvalid, structvar.KindNil, structvar.KindString: + // KindInvalid means serialized_dashboard is absent (neither it nor + // file_path is set); leave it for backend validation to reject. + return nil + case structvar.KindMap: + jsonBytes, err := json.Marshal(sd.AsAny()) + if err != nil { + return fmt.Errorf("failed to marshal inline serialized_dashboard: %w", err) } - }) + return b.Config.Set(sdPath, string(jsonBytes)) + default: + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("serialized_dashboard must be a string or map, got %s", sd.Kind()), + Locations: sd.Locations(), + }) + return nil + } }) diags = diags.Extend(diag.FromErr(err)) diff --git a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go index e16bd57625b..0eecb3b2bb9 100644 --- a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go +++ b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go @@ -7,7 +7,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) const serializedSpaceFieldName = "serialized_space" @@ -25,64 +26,64 @@ func (c configureGenieSpaceSerializedSpace) Name() string { func (c configureGenieSpaceSerializedSpace) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics - pattern := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("genie_spaces"), - dyn.AnyKey(), - ) + // Skip converting the configuration if there is nothing to configure. + if len(b.Config.Resources.GenieSpaces) == 0 { + return nil + } - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - filePath, hasFilePath := v.Get(filePathFieldName).AsString() - ss := v.Get(serializedSpaceFieldName) + pattern := structpath.MustParsePattern("resources.genie_spaces.*") - if hasFilePath { - // file_path and serialized_space are two ways to provide the same - // content. Accepting both is ambiguous, so reject it instead of - // silently picking one. - if ss.IsValid() && ss.Kind() != dyn.KindNil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "both file_path and serialized_space are set; specify only one", - Locations: ss.Locations(), - }) - return v, nil - } - contents, err := b.SyncRoot.ReadFile(filePath) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to read serialized genie space from file_path %s: %w", filePath, err) - } - return dyn.Set(v, serializedSpaceFieldName, dyn.V(string(contents))) - } + err := structvar.ForEach(b.Config.View(), pattern, func(p *structpath.PathNode, v structvar.View) error { + filePath, hasFilePath := v.Get(filePathFieldName).AsString() + ss := v.Get(serializedSpaceFieldName) + ssPath := structpath.NewStringKey(p, serializedSpaceFieldName) - // Marshal an inline structured serialized_space to a JSON string so - // both config-side and state-side carry the same plain string. - // Otherwise YAML decodes small ints as Go `int` while state JSON - // round-trip decodes them as `float64`, and structdiff reports - // false drift on every plan. - switch ss.Kind() { - case dyn.KindInvalid, dyn.KindNil, dyn.KindString: - // KindInvalid means serialized_space is absent (neither it nor - // file_path is set); leave it for backend validation to reject. - return v, nil - case dyn.KindMap: - // A top-level sequence would be valid JSON but is meaningless for a - // genie space, so KindSequence is not accepted here and falls through - // to the default rejection below. - jsonBytes, err := json.Marshal(ss.AsAny()) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to marshal inline serialized_space: %w", err) - } - return dyn.Set(v, serializedSpaceFieldName, dyn.V(string(jsonBytes))) - default: + if hasFilePath { + // file_path and serialized_space are two ways to provide the same + // content. Accepting both is ambiguous, so reject it instead of + // silently picking one. + if ss.IsValid() && ss.Kind() != structvar.KindNil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("serialized_space must be a string or map, got %s", ss.Kind()), + Summary: "both file_path and serialized_space are set; specify only one", Locations: ss.Locations(), }) - return v, nil + return nil + } + contents, err := b.SyncRoot.ReadFile(filePath) + if err != nil { + return fmt.Errorf("failed to read serialized genie space from file_path %s: %w", filePath, err) + } + return b.Config.Set(ssPath, string(contents)) + } + + // Marshal an inline structured serialized_space to a JSON string so + // both config-side and state-side carry the same plain string. + // Otherwise YAML decodes small ints as Go `int` while state JSON + // round-trip decodes them as `float64`, and structdiff reports + // false drift on every plan. + switch ss.Kind() { + case structvar.KindInvalid, structvar.KindNil, structvar.KindString: + // KindInvalid means serialized_space is absent (neither it nor + // file_path is set); leave it for backend validation to reject. + return nil + case structvar.KindMap: + // A top-level sequence would be valid JSON but is meaningless for a + // genie space, so KindSequence is not accepted here and falls through + // to the default rejection below. + jsonBytes, err := json.Marshal(ss.AsAny()) + if err != nil { + return fmt.Errorf("failed to marshal inline serialized_space: %w", err) } - }) + return b.Config.Set(ssPath, string(jsonBytes)) + default: + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("serialized_space must be a string or map, got %s", ss.Kind()), + Locations: ss.Locations(), + }) + return nil + } }) diags = diags.Extend(diag.FromErr(err)) diff --git a/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths.go b/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths.go index 38f835793ea..1f9f2427851 100644 --- a/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths.go +++ b/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths.go @@ -8,8 +8,10 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/patchwheel" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" + "github.com/databricks/databricks-sdk-go/service/pipelines" ) type expandPipelineGlobPaths struct{} @@ -18,92 +20,92 @@ func ExpandPipelineGlobPaths() bundle.Mutator { return &expandPipelineGlobPaths{} } -func (m *expandPipelineGlobPaths) expandLibrary(ctx context.Context, dir string, v dyn.Value) ([]dyn.Value, error) { +// expandLibrary returns the libraries lib is replaced with. +func (m *expandPipelineGlobPaths) expandLibrary(ctx context.Context, dir string, lib pipelines.PipelineLibrary) ([]pipelines.PipelineLibrary, error) { // Probe for the path field in the library. - for _, p := range []dyn.Path{ - dyn.NewPath(dyn.Key("notebook"), dyn.Key("path")), - dyn.NewPath(dyn.Key("file"), dyn.Key("path")), - } { - pv, err := dyn.GetByPath(v, p) - if dyn.IsNoSuchKeyError(err) { - continue - } - if err != nil { - return nil, err - } + var path string + switch { + case lib.Notebook != nil && lib.Notebook.Path != "": + path = lib.Notebook.Path + case lib.File != nil && lib.File.Path != "": + path = lib.File.Path + default: + // Neither of the library paths were found (or the path is empty). This is likely an invalid node, + // but it isn't this mutator's job to enforce that. Return the original value. + return []pipelines.PipelineLibrary{lib}, nil + } - // If the path is empty or not a local path, return the original value. - path := pv.MustString() - if path == "" || !libraries.IsLocalPath(path) { - return []dyn.Value{v}, nil - } + // If the path is not a local path, return the original value. + if !libraries.IsLocalPath(path) { + return []pipelines.PipelineLibrary{lib}, nil + } - matches, err := filepath.Glob(filepath.Join(dir, path)) - if err != nil { - return nil, err - } + matches, err := filepath.Glob(filepath.Join(dir, path)) + if err != nil { + return nil, err + } - // If there are no matches, return the original value. - if len(matches) == 0 { - return []dyn.Value{v}, nil - } + // If there are no matches, return the original value. + if len(matches) == 0 { + return []pipelines.PipelineLibrary{lib}, nil + } - matches = patchwheel.FilterLatestWheels(ctx, matches) + matches = patchwheel.FilterLatestWheels(ctx, matches) - // Emit a new value for each match. - var ev []dyn.Value - for _, match := range matches { - m, err := filepath.Rel(dir, match) - if err != nil { - return nil, err - } - nv, err := dyn.SetByPath(v, p, dyn.NewValue(filepath.ToSlash(m), pv.Locations())) - if err != nil { - return nil, err - } - ev = append(ev, nv) + // Emit a new value for each match. + var expanded []pipelines.PipelineLibrary + for _, match := range matches { + m, err := filepath.Rel(dir, match) + if err != nil { + return nil, err } - - return ev, nil + nl := lib + if lib.Notebook != nil { + notebook := *lib.Notebook + notebook.Path = filepath.ToSlash(m) + nl.Notebook = ¬ebook + } else { + file := *lib.File + file.Path = filepath.ToSlash(m) + nl.File = &file + } + expanded = append(expanded, nl) } - // Neither of the library paths were found. This is likely an invalid node, - // but it isn't this mutator's job to enforce that. Return the original value. - return []dyn.Value{v}, nil + return expanded, nil } -func (m *expandPipelineGlobPaths) expandSequence(ctx context.Context, dir string, p dyn.Path, v dyn.Value) (dyn.Value, error) { - s, ok := v.AsSequence() - if !ok { - return dyn.InvalidValue, fmt.Errorf("expected sequence, got %s", v.Kind()) - } +func (m *expandPipelineGlobPaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + p := structpath.MustParsePattern("resources.pipelines.*.libraries") - var vs []dyn.Value - for _, sv := range s { - v, err := m.expandLibrary(ctx, dir, sv) - if err != nil { - return dyn.InvalidValue, err + // Visit each pipeline's "libraries" field and expand any glob patterns. + err := structvar.ForEach(b.Config.View(), p, func(path *structpath.PathNode, value structvar.View) error { + if value.Kind() != structvar.KindSequence { + return fmt.Errorf("expected sequence, got %s", value.Kind()) } - vs = append(vs, v...) - } + pipeline := b.Config.Resources.Pipelines[path.KeyAt(2)] + if len(pipeline.Libraries) == 0 { + return nil + } - return dyn.NewValue(vs, v.Locations()), nil -} + var expanded []pipelines.PipelineLibrary + var sources [][]int + for i, lib := range pipeline.Libraries { + libs, err := m.expandLibrary(ctx, b.BundleRootPath, lib) + if err != nil { + return err + } -func (m *expandPipelineGlobPaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - p := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("pipelines"), - dyn.AnyKey(), - dyn.Key("libraries"), - ) - - // Visit each pipeline's "libraries" field and expand any glob patterns. - return dyn.MapByPattern(v, p, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { - return m.expandSequence(ctx, b.BundleRootPath, path, value) - }) + expanded = append(expanded, libs...) + for range libs { + sources = append(sources, []int{i}) + } + } + + pipeline.Libraries = expanded + b.Config.UpdateSequence(path, sources) + return nil }) return diag.FromErr(err) diff --git a/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go b/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go index 56440f7cd1c..d064e47aa28 100644 --- a/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go +++ b/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go @@ -5,18 +5,16 @@ import ( "path/filepath" "testing" - "github.com/databricks/cli/bundle/config/mutator" - "github.com/databricks/cli/bundle/config/mutator/resourcemutator" - - "github.com/stretchr/testify/assert" - "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/mutator/resourcemutator" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/internal/bundletest" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/diag" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -110,8 +108,8 @@ func TestExpandGlobPathsInPipelines(t *testing.T) { }, } - bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) - bundletest.SetLocation(b, "resources.pipelines.pipeline.libraries[3]", []dyn.Location{{File: filepath.Join(dir, "relative", "resource.yml")}}) + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "resource.yml")}}) + bundletest.SetLocation(b, "resources.pipelines.pipeline.libraries[3]", []diag.Location{{File: filepath.Join(dir, "relative", "resource.yml")}}) diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), resourcemutator.ExpandPipelineGlobPaths()) require.NoError(t, diags.Error()) diff --git a/bundle/config/mutator/resourcemutator/fix_permissions.go b/bundle/config/mutator/resourcemutator/fix_permissions.go index 9e6c785b0d9..6f631fe6dd9 100644 --- a/bundle/config/mutator/resourcemutator/fix_permissions.go +++ b/bundle/config/mutator/resourcemutator/fix_permissions.go @@ -2,13 +2,16 @@ package resourcemutator import ( "context" + "fmt" + "reflect" "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/iamutil" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) const ( @@ -52,165 +55,132 @@ func (m *fixPermissions) Name() string { return "FixPermissions" } -func processPermissions(currentUser string) dyn.MapFunc { - return func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - // Extract resource type from path: resources...permissions - if len(p) != 4 || p[0].Key() != "resources" || p[3].Key() != "permissions" { - return v, nil - } - - resourceType := p[1].Key() - if ignoredResources[resourceType] { - return v, nil - } - - v, err := ensureCurrentUserMgmtPermissions(v, currentUser, resourceType) - if err != nil { - return v, err - } - - return useMaximumLevel(v, resourceType) - } +// permission is a permission entry: a level and the principal it is granted to. +type permission struct { + level string + userName string + servicePrincipalName string + groupName string } -func readUser(v dyn.Value) string { - userName, _ := dyn.GetValue(v, "user_name").AsString() - if userName != "" { - return userName +func (p permission) user() string { + if p.userName != "" { + return p.userName } - servicePrincipalName, _ := dyn.GetValue(v, "service_principal_name").AsString() - return servicePrincipalName + return p.servicePrincipalName } -func readPrincipal(v dyn.Value) string { - value, _ := dyn.GetValue(v, "user_name").AsString() - if value != "" { - return "user_name:" + value - } - value, _ = dyn.GetValue(v, "service_principal_name").AsString() - if value != "" { - return "service_principal_name:" + value - } - value, _ = dyn.GetValue(v, "group_name").AsString() - if value != "" { - return "group_name:" + value +// principal returns the principal in the form ":", or "" if there is none. +func (p permission) principal() string { + switch { + case p.userName != "": + return "user_name:" + p.userName + case p.servicePrincipalName != "": + return "service_principal_name:" + p.servicePrincipalName + case p.groupName != "": + return "group_name:" + p.groupName } return "" } -func ensureCurrentUserMgmtPermissions(permissions dyn.Value, currentUser, resourceType string) (dyn.Value, error) { +func processPermissions(currentUser, resourceType string, permissions []permission) []permission { + return useMaximumLevel(ensureCurrentUserMgmtPermissions(permissions, currentUser, resourceType)) +} + +func ensureCurrentUserMgmtPermissions(permissions []permission, currentUser, resourceType string) []permission { currentUserHasIsOwner := false currentUserIndCanManage := -1 canAddIsOwner := hasIsOwner[resourceType] - permissionArray, ok := permissions.AsSequence() - if !ok { - return permissions, nil - } - - for ind, permission := range permissionArray { - level, ok := dyn.GetValue(permission, "level").AsString() - if !ok { + for ind, p := range permissions { + if p.level == "" { continue } - user := readUser(permission) - if level == isOwner { + user := p.user() + if p.level == isOwner { canAddIsOwner = false if user == currentUser { currentUserHasIsOwner = true } } - if user == currentUser && level == canManage { + if user == currentUser && p.level == canManage { currentUserIndCanManage = ind } } if currentUserHasIsOwner { - return dyn.V(permissionArray), nil + return permissions } if canAddIsOwner { if currentUserIndCanManage >= 0 { // Upgrade current user's CAN_MANAGE to IS_OWNER. We do this because terraform will add IS_OWNER if it does not see one // and that may confuse backend. We can stop doing it when removed terraform. - v, _ := dyn.Set(permissionArray[currentUserIndCanManage], "level", dyn.V(isOwner)) - permissionArray[currentUserIndCanManage] = v + permissions[currentUserIndCanManage].level = isOwner } else { - permissionArray = append(permissionArray, createPermission(currentUser, isOwner)) + permissions = append(permissions, createPermission(currentUser, isOwner)) } - return dyn.V(permissionArray), nil + return permissions } if currentUserIndCanManage < 0 { - permissionArray = append(permissionArray, createPermission(currentUser, canManage)) + permissions = append(permissions, createPermission(currentUser, canManage)) } - return dyn.V(permissionArray), nil + return permissions } -func useMaximumLevel(permissions dyn.Value, resourceType string) (dyn.Value, error) { - permissionArray, ok := permissions.AsSequence() - if !ok { - return permissions, nil - } - +func useMaximumLevel(permissions []permission) []permission { levelPerPrincipal := make(map[string]string) - principalIndex := make(map[string]int) + seen := make(map[string]bool) var principals []string - for _, permission := range permissionArray { - level, _ := dyn.GetValue(permission, "level").AsString() - if level == "" { + for _, p := range permissions { + if p.level == "" { continue } - principal := readPrincipal(permission) + principal := p.principal() if principal == "" { continue } - _, ok = principalIndex[principal] - if !ok { - ind := len(principalIndex) - principalIndex[principal] = ind + if !seen[principal] { + seen[principal] = true principals = append(principals, principal) } - levelPerPrincipal[principal] = resources.GetMaxLevel(levelPerPrincipal[principal], level) + levelPerPrincipal[principal] = resources.GetMaxLevel(levelPerPrincipal[principal], p.level) } - var newPermissions []dyn.Value - + var newPermissions []permission for _, principal := range principals { newPermissions = append(newPermissions, createPermissionFromPrincipal(principal, levelPerPrincipal[principal])) } - return dyn.V(newPermissions), nil + return newPermissions } -func createPermission(user, level string) dyn.Value { - permission := map[string]dyn.Value{ - "level": dyn.V(level), - } - +func createPermission(user, level string) permission { // Determine if currentUser is a service principal or user if iamutil.IsServicePrincipalName(user) { - permission["service_principal_name"] = dyn.V(user) - } else { - permission["user_name"] = dyn.V(user) + return permission{level: level, servicePrincipalName: user} } - - return dyn.V(permission) + return permission{level: level, userName: user} } -func createPermissionFromPrincipal(principal, level string) dyn.Value { - permission := map[string]dyn.Value{ - "level": dyn.V(level), - } - +func createPermissionFromPrincipal(principal, level string) permission { items := strings.SplitN(principal, ":", 2) field := items[0] value := items[1] - permission[field] = dyn.V(value) - return dyn.V(permission) + p := permission{level: level} + switch field { + case "user_name": + p.userName = value + case "service_principal_name": + p.servicePrincipalName = value + case "group_name": + p.groupName = value + } + return p } func (m *fixPermissions) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { @@ -222,15 +192,55 @@ func (m *fixPermissions) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn } currentUser := b.Config.Workspace.CurrentUser.UserName - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - // Use MapByPattern to directly process permissions arrays - return dyn.MapByPattern(v, dyn.NewPattern( - dyn.Key("resources"), - dyn.AnyKey(), - dyn.AnyKey(), - dyn.Key("permissions"), - ), processPermissions(currentUser)) + err := structvar.ForEach(b.Config.View(), structpath.MustParsePattern("resources.*.*.permissions"), func(p *structpath.PathNode, v structvar.View) error { + // Extract resource type from path: resources...permissions + resourceType := p.KeyAt(1) + if ignoredResources[resourceType] || v.Kind() != structvar.KindSequence { + return nil + } + + var permissions []permission + for _, pv := range v.Sequence() { + var perm permission + perm.level, _ = pv.Get("level").AsString() + perm.userName, _ = pv.Get("user_name").AsString() + perm.servicePrincipalName, _ = pv.Get("service_principal_name").AsString() + perm.groupName, _ = pv.Get("group_name").AsString() + permissions = append(permissions, perm) + } + + err := setPermissions(v, processPermissions(currentUser, resourceType, permissions)) + if err != nil { + return err + } + + // The permissions are rebuilt without locations. + b.Config.SetLocations(p, nil) + return nil }) return diag.FromErr(err) } + +// setPermissions replaces the permissions described by v. The type of the permissions +// differs between resources, but all have the same fields. +func setPermissions(v structvar.View, permissions []permission) error { + field := v.Reflect() + for field.Kind() == reflect.Pointer || field.Kind() == reflect.Interface { + field = field.Elem() + } + if !field.CanSet() || field.Kind() != reflect.Slice { + return fmt.Errorf("cannot set permissions of type %s", field.Type()) + } + + out := reflect.MakeSlice(field.Type(), len(permissions), len(permissions)) + for i, p := range permissions { + elem := out.Index(i) + elem.FieldByName("Level").SetString(p.level) + elem.FieldByName("UserName").SetString(p.userName) + elem.FieldByName("ServicePrincipalName").SetString(p.servicePrincipalName) + elem.FieldByName("GroupName").SetString(p.groupName) + } + field.Set(out) + return nil +} diff --git a/bundle/config/mutator/resourcemutator/keyed_merge.go b/bundle/config/mutator/resourcemutator/keyed_merge.go new file mode 100644 index 00000000000..a263f3acbf2 --- /dev/null +++ b/bundle/config/mutator/resourcemutator/keyed_merge.go @@ -0,0 +1,80 @@ +package resourcemutator + +import ( + "slices" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" +) + +// mergeByKey merges, for every resource in resources (visited in sorted order), the sequence +// at resources... by key, skipping sequences for which the merge +// is a no-op (see [keyedMergeIsNoop]). The key of an element is noopKey(element), which must +// agree with keyFn applied to the key field of the element. +func mergeByKey[R, E any]( + b *bundle.Bundle, + resourceType string, + resources map[string]*R, + field, keyField string, + elems func(*R) []E, + noopKey func(E) string, + keyFn func(structvar.View) string, + sortKeys bool, +) error { + names := make([]string, 0, len(resources)) + for name := range resources { + names = append(names, name) + } + slices.Sort(names) + for _, name := range names { + r := resources[name] + if r == nil { + continue + } + if keyedMergeIsNoop(b, "resources."+resourceType+"."+name+"."+field, elems(r), noopKey, sortKeys) { + continue + } + path := structpath.NewPath(nil, "resources", resourceType, name, field) + if err := b.Config.MergeElementsByKey(path, keyField, keyFn, sortKeys); err != nil { + return err + } + } + return nil +} + +// stringKey returns a key function for sequence elements whose key is a string +// (an absent key is the empty string). +func stringKey(what string) func(structvar.View) string { + return func(v structvar.View) string { + switch v.Kind() { + case structvar.KindInvalid, structvar.KindNil: + return "" + case structvar.KindString: + s, _ := v.AsString() + return s + default: + panic(what + " must be a string") + } + } +} + +// keyedMergeIsNoop reports whether merging the elements of the sequence at path by key +// leaves it unchanged: the sequence is not a reference, and every key is non-empty and +// unique (and sorted, if the merge sorts by key). The merge operates on the dynamic +// configuration, which is built from the whole typed configuration, so mutators skip +// it when it would have no effect. +func keyedMergeIsNoop[T any](b *bundle.Bundle, path string, elems []T, key func(T) string, sorted bool) bool { + if b.Config.IsReference(path) { + return false + } + keys := make([]string, 0, len(elems)) + for _, e := range elems { + k := key(e) + if k == "" || slices.Contains(keys, k) { + return false + } + keys = append(keys, k) + } + return !sorted || slices.IsSorted(keys) +} diff --git a/bundle/config/mutator/resourcemutator/merge_apps.go b/bundle/config/mutator/resourcemutator/merge_apps.go index edbdf7712ce..d0646c2db5c 100644 --- a/bundle/config/mutator/resourcemutator/merge_apps.go +++ b/bundle/config/mutator/resourcemutator/merge_apps.go @@ -2,11 +2,12 @@ package resourcemutator import ( "context" + "slices" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/merge" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/databricks-sdk-go/service/apps" ) type mergeApps struct{} @@ -19,27 +20,52 @@ func (m *mergeApps) Name() string { return "MergeApps" } -func (m *mergeApps) resourceName(v dyn.Value) string { - switch v.Kind() { - case dyn.KindInvalid, dyn.KindNil: - return "" - case dyn.KindString: - return v.MustString() - default: - panic("app name must be a string") +func (m *mergeApps) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + names := make([]string, 0, len(b.Config.Resources.Apps)) + for name := range b.Config.Resources.Apps { + names = append(names, name) } -} + slices.Sort(names) -func (m *mergeApps) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - if v.Kind() == dyn.KindNil { - return v, nil + for _, name := range names { + app := b.Config.Resources.Apps[name] + if app == nil { + continue + } + path := structpath.NewPath(nil, "resources", "apps", name, "resources") + if keyedMergeIsNoop(b, "resources.apps."+name+".resources", app.Resources, func(r apps.AppResource) string { return r.Name }, false) { + continue } - return dyn.Map(v, "resources.apps", dyn.Foreach(func(_ dyn.Path, app dyn.Value) (dyn.Value, error) { - return dyn.Map(app, "resources", merge.ElementsByKeyWithOverride("name", m.resourceName)) - })) - }) + // Elements with the same name are overridden by the later ones. The result + // is ordered by the first appearance of the name. + var order []string + last := map[string]int{} + for i, r := range app.Resources { + if _, ok := last[r.Name]; !ok { + order = append(order, r.Name) + } + last[r.Name] = i + } - return diag.FromErr(err) + merged := make([]apps.AppResource, 0, len(order)) + sources := make([][]int, 0, len(order)) + for _, key := range order { + merged = append(merged, app.Resources[last[key]]) + sources = append(sources, []int{last[key]}) + } + app.Resources = merged + b.Config.UpdateSequence(path, sources) + + // The merge sets the key of every element, even if it is empty. + for i, r := range merged { + if r.Name != "" { + continue + } + if err := b.Config.Set(structpath.NewPath(path, i, "name"), ""); err != nil { + return diag.FromErr(err) + } + } + } + return nil } diff --git a/bundle/config/mutator/resourcemutator/merge_grants.go b/bundle/config/mutator/resourcemutator/merge_grants.go index 3966190a5ee..ed3c7f7fe4b 100644 --- a/bundle/config/mutator/resourcemutator/merge_grants.go +++ b/bundle/config/mutator/resourcemutator/merge_grants.go @@ -2,24 +2,16 @@ package resourcemutator import ( "context" + "slices" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/merge" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" + "github.com/databricks/databricks-sdk-go/service/catalog" ) -// Resource types that support grants. -var grantResourceTypes = []string{ - "catalogs", - "schemas", - "external_locations", - "secrets", - "volumes", - "registered_models", - "vector_search_indexes", -} - type mergeGrants struct{} // MergeGrants returns a mutator that deduplicates grant entries. @@ -33,58 +25,83 @@ func (m *mergeGrants) Name() string { } func (m *mergeGrants) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - if v.Kind() == dyn.KindNil { - return v, nil - } - - for _, resourceType := range grantResourceTypes { - var mapErr error - v, mapErr = dyn.Map(v, "resources."+resourceType, dyn.Foreach(func(_ dyn.Path, resource dyn.Value) (dyn.Value, error) { - // Merge grant entries by principal. This concatenates privileges - // for entries with the same principal via the standard merge semantics. - resource, err := dyn.Map(resource, "grants", merge.ElementsByKey("principal", func(v dyn.Value) string { - s, _ := v.AsString() - return s - })) - if err != nil { - return resource, err - } - - // Deduplicate privileges within each grant entry. - return dyn.Map(resource, "grants", dyn.Foreach(func(_ dyn.Path, grant dyn.Value) (dyn.Value, error) { - return dyn.Map(grant, "privileges", deduplicateSequence) - })) - })) - if mapErr != nil { - return v, mapErr - } - } - - return v, nil - }) + r := &b.Config.Resources + // Resource types that support grants. + err := mergeGrantsOf(b, "catalogs", r.Catalogs, func(x *resources.Catalog) []catalog.PrivilegeAssignment { return x.Grants }) + if err == nil { + err = mergeGrantsOf(b, "schemas", r.Schemas, func(x *resources.Schema) []catalog.PrivilegeAssignment { return x.Grants }) + } + if err == nil { + err = mergeGrantsOf(b, "external_locations", r.ExternalLocations, func(x *resources.ExternalLocation) []catalog.PrivilegeAssignment { return x.Grants }) + } + if err == nil { + err = mergeGrantsOf(b, "secrets", r.Secrets, func(x *resources.Secret) []catalog.PrivilegeAssignment { return x.Grants }) + } + if err == nil { + err = mergeGrantsOf(b, "volumes", r.Volumes, func(x *resources.Volume) []catalog.PrivilegeAssignment { return x.Grants }) + } + if err == nil { + err = mergeGrantsOf(b, "registered_models", r.RegisteredModels, func(x *resources.RegisteredModel) []catalog.PrivilegeAssignment { return x.Grants }) + } + if err == nil { + err = mergeGrantsOf(b, "vector_search_indexes", r.VectorSearchIndexes, func(x *resources.VectorSearchIndex) []catalog.PrivilegeAssignment { return x.Grants }) + } return diag.FromErr(err) } -// deduplicateSequence removes duplicate values from a dyn sequence, -// preserving the order of first appearance. -func deduplicateSequence(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - elements, ok := v.AsSequence() - if !ok { - return v, nil +// mergeGrantsOf merges the grants of every resource of the given type (visited in sorted +// order) by principal, and deduplicates the privileges of each grant. +func mergeGrantsOf[R any](b *bundle.Bundle, resourceType string, rs map[string]*R, grants func(*R) []catalog.PrivilegeAssignment) error { + names := make([]string, 0, len(rs)) + for name := range rs { + names = append(names, name) } + slices.Sort(names) - seen := make(map[string]bool, len(elements)) - out := make([]dyn.Value, 0, len(elements)) - for _, elem := range elements { - key, _ := elem.AsString() - if seen[key] { + for _, name := range names { + if rs[name] == nil || len(grants(rs[name])) == 0 { continue } - seen[key] = true - out = append(out, elem) + + // Merge grant entries by principal. This concatenates privileges + // for entries with the same principal via the standard merge semantics. + path := structpath.NewPath(nil, "resources", resourceType, name, "grants") + err := b.Config.MergeElementsByKey(path, "principal", func(v structvar.View) string { + s, _ := v.AsString() + return s + }, false) + if err != nil { + return err + } + + // Deduplicate privileges within each grant entry. + gs := grants(rs[name]) + for i := range gs { + privileges, sources := deduplicatePrivileges(gs[i].Privileges) + if len(privileges) == len(gs[i].Privileges) { + continue + } + gs[i].Privileges = privileges + b.Config.UpdateSequence(structpath.NewPath(path, i, "privileges"), sources) + } } + return nil +} - return dyn.NewValue(out, v.Locations()), nil +// deduplicatePrivileges removes duplicate privileges, preserving the order of first +// appearance. It also returns, for every remaining privilege, its index in the input. +func deduplicatePrivileges(in []catalog.Privilege) ([]catalog.Privilege, [][]int) { + seen := make(map[catalog.Privilege]bool, len(in)) + out := make([]catalog.Privilege, 0, len(in)) + sources := make([][]int, 0, len(in)) + for i, p := range in { + if seen[p] { + continue + } + seen[p] = true + out = append(out, p) + sources = append(sources, []int{i}) + } + return out, sources } diff --git a/bundle/config/mutator/resourcemutator/merge_job_clusters.go b/bundle/config/mutator/resourcemutator/merge_job_clusters.go index c7c46a3fdef..9d63460dc50 100644 --- a/bundle/config/mutator/resourcemutator/merge_job_clusters.go +++ b/bundle/config/mutator/resourcemutator/merge_job_clusters.go @@ -4,9 +4,9 @@ import ( "context" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/merge" + "github.com/databricks/databricks-sdk-go/service/jobs" ) type mergeJobClusters struct{} @@ -19,27 +19,10 @@ func (m *mergeJobClusters) Name() string { return "MergeJobClusters" } -func (m *mergeJobClusters) jobClusterKey(v dyn.Value) string { - switch v.Kind() { - case dyn.KindInvalid, dyn.KindNil: - return "" - case dyn.KindString: - return v.MustString() - default: - panic("job cluster key must be a string") - } -} - func (m *mergeJobClusters) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - if v.Kind() == dyn.KindNil { - return v, nil - } - - return dyn.Map(v, "resources.jobs", dyn.Foreach(func(_ dyn.Path, job dyn.Value) (dyn.Value, error) { - return dyn.Map(job, "job_clusters", merge.ElementsByKey("job_cluster_key", m.jobClusterKey)) - })) - }) - + err := mergeByKey(b, "jobs", b.Config.Resources.Jobs, "job_clusters", "job_cluster_key", + func(r *resources.Job) []jobs.JobCluster { return r.JobClusters }, + func(c jobs.JobCluster) string { return c.JobClusterKey }, + stringKey("job cluster key"), false) return diag.FromErr(err) } diff --git a/bundle/config/mutator/resourcemutator/merge_job_parameters.go b/bundle/config/mutator/resourcemutator/merge_job_parameters.go index 79458b36a60..8b1413e4bf6 100644 --- a/bundle/config/mutator/resourcemutator/merge_job_parameters.go +++ b/bundle/config/mutator/resourcemutator/merge_job_parameters.go @@ -4,9 +4,9 @@ import ( "context" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/merge" + "github.com/databricks/databricks-sdk-go/service/jobs" ) type mergeJobParameters struct{} @@ -19,27 +19,10 @@ func (m *mergeJobParameters) Name() string { return "MergeJobParameters" } -func (m *mergeJobParameters) parameterNameString(v dyn.Value) string { - switch v.Kind() { - case dyn.KindInvalid, dyn.KindNil: - return "" - case dyn.KindString: - return v.MustString() - default: - panic("task key must be a string") - } -} - func (m *mergeJobParameters) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - if v.Kind() == dyn.KindNil { - return v, nil - } - - return dyn.Map(v, "resources.jobs", dyn.Foreach(func(_ dyn.Path, job dyn.Value) (dyn.Value, error) { - return dyn.Map(job, "parameters", merge.ElementsByKey("name", m.parameterNameString)) - })) - }) - + err := mergeByKey(b, "jobs", b.Config.Resources.Jobs, "parameters", "name", + func(r *resources.Job) []jobs.JobParameterDefinition { return r.Parameters }, + func(p jobs.JobParameterDefinition) string { return p.Name }, + stringKey("parameter name"), false) return diag.FromErr(err) } diff --git a/bundle/config/mutator/resourcemutator/merge_job_tasks.go b/bundle/config/mutator/resourcemutator/merge_job_tasks.go index b85a863d80f..562157a46a6 100644 --- a/bundle/config/mutator/resourcemutator/merge_job_tasks.go +++ b/bundle/config/mutator/resourcemutator/merge_job_tasks.go @@ -4,9 +4,9 @@ import ( "context" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/merge" + "github.com/databricks/databricks-sdk-go/service/jobs" ) type mergeJobTasks struct{} @@ -19,31 +19,14 @@ func (m *mergeJobTasks) Name() string { return "MergeJobTasks" } -func (m *mergeJobTasks) taskKeyString(v dyn.Value) string { - switch v.Kind() { - case dyn.KindInvalid, dyn.KindNil: - return "" - case dyn.KindString: - return v.MustString() - default: - panic("task key must be a string") - } -} - func (m *mergeJobTasks) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - if v.Kind() == dyn.KindNil { - return v, nil - } - - return dyn.Map(v, "resources.jobs", dyn.Foreach(func(_ dyn.Path, job dyn.Value) (dyn.Value, error) { - // Sorting keys here since it'll be sorted by TF anyway - // https://github.com/databricks/terraform-provider-databricks/blob/0a932c2/jobs/resource_job.go#L343 - // However, if we don't sort we have a difference between direct and TF and between configs in - // "bundle validate" and configs sent to backend. - return dyn.Map(job, "tasks", merge.ElementsBySortedKey("task_key", m.taskKeyString)) - })) - }) - + // Sorting keys here since it'll be sorted by TF anyway + // https://github.com/databricks/terraform-provider-databricks/blob/0a932c2/jobs/resource_job.go#L343 + // However, if we don't sort we have a difference between direct and TF and between configs in + // "bundle validate" and configs sent to backend. + err := mergeByKey(b, "jobs", b.Config.Resources.Jobs, "tasks", "task_key", + func(r *resources.Job) []jobs.Task { return r.Tasks }, + func(t jobs.Task) string { return t.TaskKey }, + stringKey("task key"), true) return diag.FromErr(err) } diff --git a/bundle/config/mutator/resourcemutator/merge_pipeline_clusters.go b/bundle/config/mutator/resourcemutator/merge_pipeline_clusters.go index 6496a00d90e..622f15bfec4 100644 --- a/bundle/config/mutator/resourcemutator/merge_pipeline_clusters.go +++ b/bundle/config/mutator/resourcemutator/merge_pipeline_clusters.go @@ -5,9 +5,10 @@ import ( "strings" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/merge" + "github.com/databricks/cli/libs/structs/structvar" + "github.com/databricks/databricks-sdk-go/service/pipelines" ) type mergePipelineClusters struct{} @@ -20,29 +21,31 @@ func (m *mergePipelineClusters) Name() string { return "MergePipelineClusters" } -func (m *mergePipelineClusters) clusterLabel(v dyn.Value) string { +func (m *mergePipelineClusters) clusterLabel(v structvar.View) string { switch v.Kind() { - case dyn.KindInvalid, dyn.KindNil: + case structvar.KindInvalid, structvar.KindNil: // Note: the cluster label is optional and defaults to 'default'. // We therefore ALSO merge all clusters without a label. return "default" - case dyn.KindString: - return strings.ToLower(v.MustString()) + case structvar.KindString: + s, _ := v.AsString() + return strings.ToLower(s) default: panic("task key must be a string") } } func (m *mergePipelineClusters) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - if v.Kind() == dyn.KindNil { - return v, nil - } - - return dyn.Map(v, "resources.pipelines", dyn.Foreach(func(_ dyn.Path, pipeline dyn.Value) (dyn.Value, error) { - return dyn.Map(pipeline, "clusters", merge.ElementsByKey("label", m.clusterLabel)) - })) - }) - + // The merge also lowercases labels and sets the "default" label, so only + // lowercase labels are left unchanged. + err := mergeByKey(b, "pipelines", b.Config.Resources.Pipelines, "clusters", "label", + func(r *resources.Pipeline) []pipelines.PipelineCluster { return r.Clusters }, + func(c pipelines.PipelineCluster) string { + if c.Label != strings.ToLower(c.Label) { + return "" + } + return c.Label + }, + m.clusterLabel, false) return diag.FromErr(err) } diff --git a/bundle/config/mutator/resourcemutator/model_serving_endpoint_fixups.go b/bundle/config/mutator/resourcemutator/model_serving_endpoint_fixups.go index 87962a85e86..6c25146e6bd 100644 --- a/bundle/config/mutator/resourcemutator/model_serving_endpoint_fixups.go +++ b/bundle/config/mutator/resourcemutator/model_serving_endpoint_fixups.go @@ -5,7 +5,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go/service/serving" ) @@ -56,7 +55,7 @@ func (m *modelServingEndpointFixups) Apply(ctx context.Context, b *bundle.Bundle Severity: diag.Error, Summary: "Cannot use both served_models and served_entities", Detail: "Model serving endpoint cannot specify both served_models and served_entities at the same time.", - Locations: []dyn.Location{ + Locations: []diag.Location{ b.Config.GetLocation("resources.model_serving_endpoints." + key), }, }) @@ -71,7 +70,7 @@ func (m *modelServingEndpointFixups) Apply(ctx context.Context, b *bundle.Bundle Severity: diag.Warning, Summary: "Using served_models is deprecated", Detail: "The served_models field is deprecated. Please use served_entities instead.", - Locations: []dyn.Location{ + Locations: []diag.Location{ b.Config.GetLocation("resources.model_serving_endpoints." + key + ".config.served_models"), }, }) diff --git a/bundle/config/mutator/resourcemutator/process_static_resources.go b/bundle/config/mutator/resourcemutator/process_static_resources.go index b1f689808f6..7998517328b 100644 --- a/bundle/config/mutator/resourcemutator/process_static_resources.go +++ b/bundle/config/mutator/resourcemutator/process_static_resources.go @@ -2,12 +2,10 @@ package resourcemutator import ( "context" - "fmt" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/logdiag" ) @@ -64,18 +62,10 @@ func (p processStaticResources) Apply(ctx context.Context, b *bundle.Bundle) dia func getAllResources(b *bundle.Bundle) (ResourceKeySet, error) { set := NewResourceKeySet() - err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - pattern := dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()) - err := set.AddPattern(pattern, root) - if err != nil { - return dyn.InvalidValue, err + for _, group := range b.Config.Resources.AllResources() { + for name := range group.Resources { + set.AddResourceKey(ResourceKey{Type: group.Description.PluralName, Name: name}) } - - return root, nil - }) - if err != nil { - return nil, fmt.Errorf("failed to collect resources: %s", err) } - return set, nil } diff --git a/bundle/config/mutator/resourcemutator/resource_key.go b/bundle/config/mutator/resourcemutator/resource_key.go index 699867b4f40..f5fe19deddb 100644 --- a/bundle/config/mutator/resourcemutator/resource_key.go +++ b/bundle/config/mutator/resourcemutator/resource_key.go @@ -3,7 +3,8 @@ package resourcemutator import ( "errors" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) // ResourceKey uniquely identifies a resource in configuration. @@ -48,23 +49,21 @@ func (r ResourceKeySet) Size() int { } // AddPattern adds all resource keys that match the pattern. -func (r ResourceKeySet) AddPattern(pattern dyn.Pattern, root dyn.Value) error { - if len(pattern) != 3 { +func (r ResourceKeySet) AddPattern(pattern *structpath.PatternNode, root structvar.View) error { + if pattern.Len() != 3 { return errors.New("pattern must have 3 keys") } - _, err := dyn.MapByPattern(root, pattern, func(path dyn.Path, v dyn.Value) (dyn.Value, error) { + return structvar.ForEach(root, pattern, func(path *structpath.PathNode, v structvar.View) error { parsed, err := getResourceKey(path) if err != nil { - return dyn.InvalidValue, err + return err } r.AddResourceKey(parsed) - return v, nil + return nil }) - - return err } // Types returns the types of all resources in the set. @@ -105,17 +104,13 @@ func (r ResourceKeySet) ToArray() []ResourceKey { return result } -func getResourceKey(path dyn.Path) (ResourceKey, error) { - if len(path) < 3 { - return ResourceKey{}, errors.New("can't parse resource key") - } - - if path[0].Key() != "resources" { +func getResourceKey(path *structpath.PathNode) (ResourceKey, error) { + if path.KeyAt(0) != "resources" { return ResourceKey{}, errors.New("can't parse resource key") } - resourceType := path[1].Key() - resourceName := path[2].Key() + resourceType := path.KeyAt(1) + resourceName := path.KeyAt(2) if resourceType == "" || resourceName == "" { return ResourceKey{}, errors.New("can't parse resource key") diff --git a/bundle/config/mutator/resourcemutator/resource_key_test.go b/bundle/config/mutator/resourcemutator/resource_key_test.go index 1af72a9ba2d..9722359a236 100644 --- a/bundle/config/mutator/resourcemutator/resource_key_test.go +++ b/bundle/config/mutator/resourcemutator/resource_key_test.go @@ -3,9 +3,12 @@ package resourcemutator import ( "testing" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" + "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" - - "github.com/databricks/cli/libs/dyn" ) type getResourceKeyTestCase struct { @@ -38,7 +41,7 @@ func TestGetResourceKey(t *testing.T) { for _, tc := range testCases { t.Run(tc.path, func(t *testing.T) { - key, err := getResourceKey(dyn.MustPathFromString(tc.path)) + key, err := getResourceKey(structpath.MustParsePath(tc.path)) if tc.err { assert.Error(t, err) } else { @@ -51,29 +54,23 @@ func TestGetResourceKey(t *testing.T) { type resourceKeySetAddTestCase struct { name string - pattern dyn.Pattern - root dyn.Value + pattern *structpath.PatternNode + root structvar.View expected []ResourceKey } func TestResourceKeySet_AddPattern(t *testing.T) { - root := dyn.V(map[string]dyn.Value{ - "resources": dyn.V(map[string]dyn.Value{ - "jobs": dyn.V(map[string]dyn.Value{ - "job_1": dyn.V(map[string]dyn.Value{ - "name": dyn.V("job_1"), - }), - "job_2": dyn.V(map[string]dyn.Value{ - "name": dyn.V("job_2"), - }), - }), - }), - }) + var cfg config.Root + cfg.Resources.Jobs = map[string]*resources.Job{ + "job_1": {JobSettings: jobs.JobSettings{Name: "job_1"}}, + "job_2": {JobSettings: jobs.JobSettings{Name: "job_2"}}, + } + root := cfg.View() testCases := []resourceKeySetAddTestCase{ { name: "one job pattern", - pattern: dyn.NewPattern(dyn.Key("resources"), dyn.Key("jobs"), dyn.Key("job_1")), + pattern: structpath.MustParsePattern("resources.jobs.job_1"), root: root, expected: []ResourceKey{ { @@ -84,7 +81,7 @@ func TestResourceKeySet_AddPattern(t *testing.T) { }, { name: "all resources pattern", - pattern: dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), + pattern: structpath.MustParsePattern("resources.*.*"), root: root, expected: []ResourceKey{ { diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index 4d0b3bd47ab..5f424eb82fd 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -2,17 +2,16 @@ package resourcemutator import ( "context" - "errors" "fmt" + "reflect" + "strings" + "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/validate" - - "github.com/databricks/cli/libs/dyn/merge" "github.com/databricks/cli/libs/logdiag" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/databricks-sdk-go/service/jobs" ) // When a new resource is added to configuration, we apply bundle @@ -46,68 +45,61 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { return } - defaults := []struct { - pattern string - value any - }{ - {"resources.dashboards.*.parent_path", b.Config.Workspace.ResourcePath}, - {"resources.dashboards.*.embed_credentials", false}, - {"resources.genie_spaces.*.parent_path", b.Config.Workspace.ResourcePath}, - {"resources.volumes.*.volume_type", "MANAGED"}, + defaults := []bundle.Default{ + {Pattern: "resources.dashboards.*.parent_path", Value: b.Config.Workspace.ResourcePath}, + {Pattern: "resources.dashboards.*.embed_credentials", Value: false}, + {Pattern: "resources.genie_spaces.*.parent_path", Value: b.Config.Workspace.ResourcePath}, + {Pattern: "resources.volumes.*.volume_type", Value: "MANAGED"}, - {"resources.alerts.*.parent_path", b.Config.Workspace.ResourcePath}, + {Pattern: "resources.alerts.*.parent_path", Value: b.Config.Workspace.ResourcePath}, // Jobs: // The defaults are the same as for terraform provider latest version (v1.75.0) // https://github.com/databricks/terraform-provider-databricks/blob/v1.75.0/jobs/resource_job.go#L532 - {"resources.jobs.*.name", "Untitled"}, - {"resources.jobs.*.max_concurrent_runs", 1}, - {"resources.jobs.*.schedule.pause_status", "UNPAUSED"}, - {"resources.jobs.*.trigger.pause_status", "UNPAUSED"}, - {"resources.jobs.*.continuous.pause_status", "UNPAUSED"}, + {Pattern: "resources.jobs.*.name", Value: "Untitled"}, + {Pattern: "resources.jobs.*.max_concurrent_runs", Value: 1}, + {Pattern: "resources.jobs.*.schedule.pause_status", Value: "UNPAUSED"}, + {Pattern: "resources.jobs.*.trigger.pause_status", Value: "UNPAUSED"}, + {Pattern: "resources.jobs.*.continuous.pause_status", Value: "UNPAUSED"}, // Enable queueing for jobs by default, following the behavior from API 2.2+. // As of 2024-04, we're still using API 2.1 which has queueing disabled by default. - {"resources.jobs.*.queue", map[string]dyn.Value{ - "enabled": dyn.V(true), - }}, + {Pattern: "resources.jobs.*.queue", Value: jobs.QueueSettings{Enabled: true}}, // This is converted from single-task to multi-task - {"resources.jobs.*.task[*].dbt_task.schema", "default"}, - {"resources.jobs.*.task[*].for_each_task.task.dbt_task.schema", "default"}, + {Pattern: "resources.jobs.*.task[*].dbt_task.schema", Value: "default"}, + {Pattern: "resources.jobs.*.task[*].for_each_task.task.dbt_task.schema", Value: "default"}, // https://github.com/databricks/terraform-provider-databricks/blob/v1.75.0/clusters/resource_cluster.go - {"resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.notebooks", true}, - {"resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.jobs", true}, + {Pattern: "resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.notebooks", Value: true}, + {Pattern: "resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.jobs", Value: true}, // Pipelines (same as terraform) // https://github.com/databricks/terraform-provider-databricks/blob/v1.75.0/pipelines/resource_pipeline.go#L253 - {"resources.pipelines.*.edition", "ADVANCED"}, - {"resources.pipelines.*.channel", "CURRENT"}, + {Pattern: "resources.pipelines.*.edition", Value: "ADVANCED"}, + {Pattern: "resources.pipelines.*.channel", Value: "CURRENT"}, // SqlWarehouses (same as terraform) // https://github.com/databricks/terraform-provider-databricks/blob/v1.75.0/sql/resource_sql_endpoint.go#L59 - {"resources.sql_warehouses.*.auto_stop_mins", 120}, - {"resources.sql_warehouses.*.enable_photon", true}, - {"resources.sql_warehouses.*.max_num_clusters", 1}, - {"resources.sql_warehouses.*.spot_instance_policy", "COST_OPTIMIZED"}, + {Pattern: "resources.sql_warehouses.*.auto_stop_mins", Value: 120}, + {Pattern: "resources.sql_warehouses.*.enable_photon", Value: true}, + {Pattern: "resources.sql_warehouses.*.max_num_clusters", Value: 1}, + {Pattern: "resources.sql_warehouses.*.spot_instance_policy", Value: "COST_OPTIMIZED"}, // Apps: - {"resources.apps.*.description", ""}, + {Pattern: "resources.apps.*.description", Value: ""}, // Clusters (same as terraform) // https://github.com/databricks/terraform-provider-databricks/blob/v1.75.0/clusters/resource_cluster.go#L315 - {"resources.clusters.*.autotermination_minutes", 60}, - {"resources.clusters.*.workload_type.clients.notebooks", true}, - {"resources.clusters.*.workload_type.clients.jobs", true}, + {Pattern: "resources.clusters.*.autotermination_minutes", Value: 60}, + {Pattern: "resources.clusters.*.workload_type.clients.notebooks", Value: true}, + {Pattern: "resources.clusters.*.workload_type.clients.jobs", Value: true}, } - for _, defaultDef := range defaults { - bundle.SetDefault(ctx, b, defaultDef.pattern, defaultDef.value) - if logdiag.HasError(ctx) { - return - } + bundle.SetDefaults(ctx, b, defaults) + if logdiag.HasError(ctx) { + return } bundle.ApplySeqContext(ctx, b, @@ -224,17 +216,7 @@ func NormalizeAndInitializeResources( return } - var snapshot dyn.Value - - err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - snapshot = root - - return selectResources(root, addedResources) - }) - if err != nil { - logdiag.LogError(ctx, fmt.Errorf("failed to select resources: %s", err)) - return - } + restore := selectResources(b, addedResources) applyNormalizeMutators(ctx, b) if logdiag.HasError(ctx) { @@ -246,12 +228,10 @@ func NormalizeAndInitializeResources( return } - // after mutators, we merge updated resources back to snapshot to preserve non-selected resources - err = b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - return mergeResources(root, snapshot) - }) + // after mutators, we merge updated resources back to the snapshot to preserve non-selected resources + err := restore() if err != nil { - logdiag.LogError(ctx, fmt.Errorf("failed to merge resources: %s", err)) + logdiag.LogError(ctx, fmt.Errorf("failed to merge resources: %w", err)) } } @@ -266,17 +246,7 @@ func NormalizeResources( return } - var snapshot dyn.Value - - err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - snapshot = root - - return selectResources(root, updatedResources) - }) - if err != nil { - logdiag.LogError(ctx, fmt.Errorf("failed to select resources: %s", err)) - return - } + restore := selectResources(b, updatedResources) applyNormalizeMutators(ctx, b) if logdiag.HasError(ctx) { @@ -294,129 +264,82 @@ func NormalizeResources( return } - // after mutators, we merge updated resources back to snapshot to preserve non-selected resources - err = b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - return mergeResources(root, snapshot) - }) + // after mutators, we merge updated resources back to the snapshot to preserve non-selected resources + err := restore() if err != nil { - logdiag.LogError(ctx, fmt.Errorf("failed to merge resources: %s", err)) + logdiag.LogError(ctx, fmt.Errorf("failed to merge resources: %w", err)) } } -// selectResources returns bundle configuration with resources only present in resourcePaths. -func selectResources(root dyn.Value, resourcePaths ResourceKeySet) (dyn.Value, error) { - resourcesKeyString := "resources" - resourcesPath := dyn.NewPath(dyn.Key(resourcesKeyString)) - - newRoot := root - var err error - - // remove resource types that are not in resourcePaths - newRoot, err = dyn.MapByPath( - newRoot, - resourcesPath, - func(p dyn.Path, resources dyn.Value) (dyn.Value, error) { - return merge.Select(resources, resourcePaths.Types()) - }, - ) - if err != nil { - return dyn.InvalidValue, err - } - - // for each resource type, remove resources by name - for _, resourceType := range resourcePaths.Types() { - resourceTypePath := resourcesPath.Append(dyn.Key(resourceType)) - - newRoot, err = dyn.MapByPath( - newRoot, - resourceTypePath, - func(p dyn.Path, resource dyn.Value) (dyn.Value, error) { - return merge.Select(resource, resourcePaths.Names(resourceType)) - }, - ) - if err != nil { - return dyn.InvalidValue, err +// resourceTypeFields returns the fields of resources that hold the resources of each type, +// by the name of the type (e.g. "jobs"). +func resourceTypeFields(resources reflect.Value) map[string]reflect.Value { + fields := map[string]reflect.Value{} + t := resources.Type() + for i := range t.NumField() { + name, _, _ := strings.Cut(t.Field(i).Tag.Get("json"), ",") + if name == "" || name == "-" || resources.Field(i).Kind() != reflect.Map { + continue } + fields[name] = resources.Field(i) } - - return newRoot, err + return fields } -// mergeResources returns bundle configuration by merging all resources from src into dst, -// overriding existing resources if they exist. -func mergeResources(src, dst dyn.Value) (dyn.Value, error) { - resourcesKey := dyn.Key("resources") - - newDst := dst - - // merge 'resources..' - _, err := dyn.MapByPattern( - src, - dyn.NewPattern(resourcesKey, dyn.AnyKey(), dyn.AnyKey()), - func(path dyn.Path, v dyn.Value) (dyn.Value, error) { - // if parent 'resources.' doesn't exist, handle it on the next pass - updated, _ := dyn.SetByPath(newDst, path, v) - if !updated.IsValid() { - return v, nil - } else { - newDst = updated - } - - return v, nil - }, - ) - if err != nil { - return newDst, err - } - - // merge 'resources.' - _, err = dyn.MapByPattern( - src, - dyn.NewPattern(resourcesKey, dyn.AnyKey()), - func(path dyn.Path, v dyn.Value) (dyn.Value, error) { - // if already exists, we already handled it in the previous pass - existing, _ := dyn.GetByPath(newDst, path) - if existing.IsValid() { - return v, nil - } +// selectResources removes all resources except the ones in resourceKeys from the bundle +// configuration. It returns a function that restores the removed resources: it puts the +// configuration back to what it was before, except for the selected resources, which take +// the values they have at that point. Any other changes to the configuration are discarded. +func selectResources(b *bundle.Bundle, resourceKeys ResourceKeySet) func() error { + // A shallow copy of the configuration: the resources maps are replaced below, not + // modified, so the copy keeps all resources. + snapshot := b.Config + + for resourceType, field := range resourceTypeFields(reflect.ValueOf(&b.Config.Resources).Elem()) { + if _, ok := resourceKeys[resourceType]; !ok { + field.SetZero() + continue + } - // if parent 'resources' doesn't exist, handle it on the next pass - updated, _ := dyn.SetByPath(newDst, path, v) - if !updated.IsValid() { - return v, nil - } else { - newDst = updated - return v, nil + selected := reflect.MakeMap(field.Type()) + for _, name := range resourceKeys.Names(resourceType) { + v := field.MapIndex(reflect.ValueOf(name)) + if v.IsValid() { + selected.SetMapIndex(reflect.ValueOf(name), v) } - }, - ) - if err != nil { - return newDst, err + } + field.Set(selected) } - // merge 'resources' - _, err = dyn.MapByPattern( - src, - dyn.NewPattern(resourcesKey), - func(path dyn.Path, v dyn.Value) (dyn.Value, error) { - // if already exists, we already handled it in the previous pass - existing, _ := dyn.GetByPath(newDst, path) - if existing.IsValid() { - return v, nil - } - - updated, _ := dyn.SetByPath(newDst, path, v) - if !updated.IsValid() { - return v, errors.New("failed to update resources") - } else { - newDst = updated - return v, nil + return func() error { + updated := b.Config + updatedView := updated.View() + updatedFields := resourceTypeFields(reflect.ValueOf(&updated.Resources).Elem()) + + b.Config = snapshot + fields := resourceTypeFields(reflect.ValueOf(&b.Config.Resources).Elem()) + + for resourceType, names := range resourceKeys { + for name := range names { + v := updatedFields[resourceType].MapIndex(reflect.ValueOf(name)) + if !v.IsValid() { + continue + } + + path := structpath.NewPath(nil, "resources", resourceType, name) + err := b.Config.Assign(path, updatedView.Lookup(path)) + if err != nil { + return err + } + + // Keep the resource that was updated, it may have more than the configuration describes. + field := fields[resourceType] + if field.IsNil() { + field.Set(reflect.MakeMap(field.Type())) + } + field.SetMapIndex(reflect.ValueOf(name), v) } - }, - ) - if err != nil { - return newDst, err + } + return nil } - - return newDst, nil } diff --git a/bundle/config/mutator/resourcemutator/resource_mutator_test.go b/bundle/config/mutator/resourcemutator/resource_mutator_test.go index 42aee5bf883..572b688946b 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator_test.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator_test.go @@ -3,114 +3,77 @@ package resourcemutator import ( "testing" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynassert" + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -type mergeResourcesTestCase struct { - name string - src dyn.Value - dst dyn.Value - expected dyn.Value +func newJob(name string) *resources.Job { + return &resources.Job{JobSettings: jobs.JobSettings{Name: name}} } -func TestMergeResources(t *testing.T) { - job1 := dyn.V("job_1") - job2 := dyn.V("job_2") - job3 := dyn.V("job_3") - - testCases := []mergeResourcesTestCase{ - { - name: "add resources to empty bundle (1)", - src: mapOf("resources", mapOf("jobs", mapOf("job_1", job1))), - dst: mapOf("resources", emptyMap()), - expected: mapOf("resources", mapOf("jobs", mapOf("job_1", job1))), - }, - { - name: "add resources to empty bundle (2)", - src: mapOf("resources", mapOf("jobs", mapOf("job_1", job1))), - dst: emptyMap(), - expected: mapOf("resources", mapOf("jobs", mapOf("job_1", job1))), - }, - { - name: "add new resource", - src: mapOf("resources", mapOf("jobs", mapOf("job_1", job1))), - dst: mapOf("resources", mapOf("jobs", mapOf("job_2", job2))), - expected: mapOf("resources", mapOf("jobs", mapOf2("job_1", job1, "job_2", job2))), - }, - { - name: "override resource", - src: mapOf("resources", mapOf("jobs", mapOf("job_1", job3))), - dst: mapOf("resources", mapOf("jobs", mapOf2("job_1", job1, "job_2", job2))), - expected: mapOf("resources", mapOf("jobs", mapOf2("job_1", job3, "job_2", job2))), +func TestSelectResources(t *testing.T) { + job1 := newJob("job_1") + job2 := newJob("job_2") + + b := &bundle.Bundle{Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{"job_1": job1, "job_2": job2}, + Pipelines: map[string]*resources.Pipeline{"pipeline_1": {}}, }, - } + }} - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - actual, err := mergeResources(tc.src, tc.dst) + keys := NewResourceKeySet() + keys.AddResourceKey(ResourceKey{Type: "jobs", Name: "job_1"}) - assert.NoError(t, err) - dynassert.Equal(t, tc.expected, actual) - }) - } -} + restore := selectResources(b, keys) + assert.Equal(t, map[string]*resources.Job{"job_1": job1}, b.Config.Resources.Jobs) + assert.Nil(t, b.Config.Resources.Pipelines) -type selectResourcesTestCase struct { - name string - config dyn.Value - resourcePaths []ResourceKey - expected dyn.Value + require.NoError(t, restore()) + assert.Equal(t, map[string]*resources.Job{"job_1": job1, "job_2": job2}, b.Config.Resources.Jobs) + assert.Len(t, b.Config.Resources.Pipelines, 1) } -func TestSelectResources(t *testing.T) { - job1 := dyn.V("job_1") - job2 := dyn.V("job_2") - - testCases := []selectResourcesTestCase{ - { - name: "extract resources", - config: mapOf("resources", mapOf("jobs", mapOf2("job_1", job1, "job_2", job2))), - resourcePaths: []ResourceKey{ - { - Type: "jobs", - Name: "job_1", - }, - }, - expected: mapOf("resources", mapOf("jobs", mapOf("job_1", job1))), - }, - { - name: "extract no resources", - config: mapOf("resources", mapOf("jobs", mapOf2("job_1", job1, "job_2", job2))), - resourcePaths: []ResourceKey{}, - expected: mapOf("resources", emptyMap()), +func TestSelectResourcesRestoreMergesUpdates(t *testing.T) { + b := &bundle.Bundle{Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{"job_1": newJob("job_1"), "job_2": newJob("job_2")}, }, - } + }} + b.Config.Bundle.Name = "before" - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - resourceKeySet := NewResourceKeySet() - for _, key := range tc.resourcePaths { - resourceKeySet.AddResourceKey(key) - } + keys := NewResourceKeySet() + keys.AddResourceKey(ResourceKey{Type: "jobs", Name: "job_1"}) - actual, err := selectResources(tc.config, resourceKeySet) + restore := selectResources(b, keys) - assert.NoError(t, err) - assert.Equal(t, tc.expected, actual) - }) - } -} + // Changes to selected resources are kept, other changes are discarded. + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.jobs.job_1.name"), "updated")) + b.Config.Bundle.Name = "after" -func mapOf(key string, value dyn.Value) dyn.Value { - return dyn.V(map[string]dyn.Value{key: value}) + require.NoError(t, restore()) + assert.Equal(t, "updated", b.Config.Resources.Jobs["job_1"].Name) + assert.Equal(t, "job_2", b.Config.Resources.Jobs["job_2"].Name) + assert.Equal(t, "before", b.Config.Bundle.Name) } -func mapOf2(k0 string, v0 dyn.Value, k1 string, v1 dyn.Value) dyn.Value { - return dyn.V(map[string]dyn.Value{k0: v0, k1: v1}) -} +func TestSelectResourcesRestoreKeepsResourceType(t *testing.T) { + b := &bundle.Bundle{Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{"job_1": newJob("job_1")}, + }, + }} + + keys := NewResourceKeySet() + keys.AddResourceKey(ResourceKey{Type: "jobs", Name: "job_1"}) -func emptyMap() dyn.Value { - return dyn.V(map[string]dyn.Value{}) + restore := selectResources(b, keys) + require.NoError(t, restore()) + assert.Equal(t, "job_1", b.Config.Resources.Jobs["job_1"].Name) + assert.Nil(t, b.Config.Resources.Pipelines) } diff --git a/bundle/config/mutator/resourcemutator/run_as.go b/bundle/config/mutator/resourcemutator/run_as.go index f93c16b3889..80fa2ab4802 100644 --- a/bundle/config/mutator/resourcemutator/run_as.go +++ b/bundle/config/mutator/resourcemutator/run_as.go @@ -8,7 +8,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/pipelines" @@ -32,12 +31,12 @@ func (m *setRunAs) Name() string { return "SetRunAs" } -func reportRunAsNotSupported(resourceType string, location dyn.Location, currentUser, runAsUser string) diag.Diagnostics { +func reportRunAsNotSupported(resourceType string, location diag.Location, currentUser, runAsUser string) diag.Diagnostics { return diag.Diagnostics{{ Summary: fmt.Sprintf("%s do not support a setting a run_as user that is different from the owner.\n"+ "Current identity: %s. Run as identity: %s.\n"+ "See https://docs.databricks.com/dev-tools/bundles/run-as.html to learn more about the run_as property.", resourceType, currentUser, runAsUser), - Locations: []dyn.Location{location}, + Locations: []diag.Location{location}, Severity: diag.Error, }} } @@ -57,7 +56,7 @@ func validateRunAs(b *bundle.Bundle) diag.Diagnostics { if identityCount != 1 { return diag.Diagnostics{{ Summary: "run_as section must specify exactly one non-empty identity: user_name, service_principal_name, or group_name", - Locations: []dyn.Location{b.Config.GetLocation("run_as")}, + Locations: []diag.Location{b.Config.GetLocation("run_as")}, Severity: diag.Error, }} } @@ -220,17 +219,17 @@ func (m *setRunAs) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { b.Metrics.AddBoolValue("experimental.use_legacy_run_as", b.Config.Experimental != nil && b.Config.Experimental.UseLegacyRunAs) // Track whether top level run_as is set. - b.Metrics.AddBoolValue("run_as_set", b.Config.Value().Get("run_as").Kind() != dyn.KindInvalid) + b.Metrics.AddBoolValue("run_as_set", b.Config.RunAs != nil) // Mutator is a no-op if run_as is not specified in the bundle - if b.Config.Value().Get("run_as").Kind() == dyn.KindInvalid { + if b.Config.RunAs == nil { return nil } // User has opted to use the legacy behavior of run_as with the // experimental.use_legacy_run_as flag. if b.Config.Experimental != nil && b.Config.Experimental.UseLegacyRunAs { - if b.Config.Value().Get("run_as").Get("group_name").Kind() != dyn.KindInvalid { + if b.Config.RunAs.GroupName != "" || slices.Contains(b.Config.RunAs.ForceSendFields, "GroupName") { return diag.Errorf("run_as.group_name is not supported with experimental.use_legacy_run_as; disable experimental.use_legacy_run_as to use a group identity") } setPipelineOwnersToRunAsIdentity(b) diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 10ad8fd9dbb..1be8d37957c 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -10,8 +10,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/diag" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/pipelines" @@ -22,18 +21,15 @@ import ( func allResourceTypes(t *testing.T) []string { // Compute supported resource types based on the `Resources{}` struct. - r := &config.Resources{} - rv, err := convert.FromTyped(r, dyn.NilValue) - require.NoError(t, err) - normalized, _ := convert.Normalize(r, rv, convert.IncludeMissingFields) var resourceTypes []string - for _, k := range normalized.MustMap().Keys() { - resourceTypes = append(resourceTypes, k.MustString()) + for f := range reflect.TypeFor[config.Resources]().Fields() { + name, _, _ := strings.Cut(f.Tag.Get("json"), ",") + resourceTypes = append(resourceTypes, name) } slices.Sort(resourceTypes) // Assert the total list of resource supported, as a sanity check that using - // the dyn library gives us the correct list of all resources supported. Please + // the struct gives us the correct list of all resources supported. Please // also update this check when adding a new resource require.Equal( t, []string{ @@ -297,14 +293,6 @@ func TestRunAsErrorForUnsupportedResources(t *testing.T) { }, } - v, err := convert.FromTyped(base, dyn.NilValue) - require.NoError(t, err) - - // Define top level resources key in the bundle configuration. - // This is not part of the typed configuration, so we need to add it manually. - v, err = dyn.Set(v, "resources", dyn.V(map[string]dyn.Value{})) - require.NoError(t, err) - for _, rt := range allResourceTypes(t) { // Skip allowed resources if slices.Contains(allowList, rt) { @@ -313,23 +301,16 @@ func TestRunAsErrorForUnsupportedResources(t *testing.T) { // Add an instance of the resource type that is not on the allow list to // the bundle configuration. - nv, err := dyn.SetByPath(v, dyn.NewPath(dyn.Key("resources"), dyn.Key(rt)), dyn.V(map[string]dyn.Value{ - "foo": dyn.V(map[string]dyn.Value{ - "path": dyn.V("bar"), - }), - })) - require.NoError(t, err) - - // Get back typed configuration from the newly created invalid bundle configuration. - r := &config.Root{} - err = convert.ToTyped(r, nv) - require.NoError(t, err) + r, diags := config.LoadFromBytes("databricks.yml", fmt.Appendf(nil, "resources:\n %s:\n foo:\n path: bar\n", rt)) + require.NoError(t, diags.Error()) + r.Workspace = base.Workspace + r.RunAs = base.RunAs // Assert this invalid bundle configuration fails validation. b := &bundle.Bundle{ Config: *r, } - diags := bundle.Apply(t.Context(), b, SetRunAs()) + diags = bundle.Apply(t.Context(), b, SetRunAs()) require.Error(t, diags.Error()) assert.Contains(t, diags.Error().Error(), "do not support a setting a run_as user that is different from the owner.\n"+ "Current identity: alice. Run as identity: bob.\n"+ @@ -351,14 +332,6 @@ func TestRunAsNoErrorForSupportedResources(t *testing.T) { }, } - v, err := convert.FromTyped(base, dyn.NilValue) - require.NoError(t, err) - - // Define top level resources key in the bundle configuration. - // This is not part of the typed configuration, so we need to add it manually. - v, err = dyn.Set(v, "resources", dyn.V(map[string]dyn.Value{})) - require.NoError(t, err) - for _, rt := range allResourceTypes(t) { // Skip unsupported resources if !slices.Contains(allowList, rt) { @@ -367,23 +340,16 @@ func TestRunAsNoErrorForSupportedResources(t *testing.T) { // Add an instance of the resource type that is not on the allow list to // the bundle configuration. - nv, err := dyn.SetByPath(v, dyn.NewPath(dyn.Key("resources"), dyn.Key(rt)), dyn.V(map[string]dyn.Value{ - "foo": dyn.V(map[string]dyn.Value{ - "name": dyn.V("bar"), - }), - })) - require.NoError(t, err) - - // Get back typed configuration from the newly created invalid bundle configuration. - r := &config.Root{} - err = convert.ToTyped(r, nv) - require.NoError(t, err) + r, diags := config.LoadFromBytes("databricks.yml", fmt.Appendf(nil, "resources:\n %s:\n foo:\n name: bar\n", rt)) + require.NoError(t, diags.Error()) + r.Workspace = base.Workspace + r.RunAs = base.RunAs // Assert this configuration passes validation. b := &bundle.Bundle{ Config: *r, } - diags := bundle.Apply(t.Context(), b, SetRunAs()) + diags = bundle.Apply(t.Context(), b, SetRunAs()) require.NoError(t, diags.Error()) } } @@ -393,7 +359,8 @@ func TestRunAsIdentities(t *testing.T) { runAs string wantError bool }{ - {`null`, true}, + // A null run_as is the same as not specifying it. + {`null`, false}, {`{}`, true}, {`{user_name: ""}`, true}, {`{service_principal_name: ""}`, true}, @@ -416,7 +383,7 @@ func TestRunAsIdentities(t *testing.T) { diags = bundle.Apply(t.Context(), b, SetRunAs()) if tc.wantError { require.ErrorContains(t, diags.Error(), "run_as section must specify exactly one non-empty identity: user_name, service_principal_name, or group_name") - assert.Equal(t, []dyn.Location{r.GetLocation("run_as")}, diags[0].Locations) + assert.Equal(t, []diag.Location{r.GetLocation("run_as")}, diags[0].Locations) } else { require.NoError(t, diags.Error()) } @@ -452,14 +419,14 @@ func TestRunAsGroupResources(t *testing.T) { r, diags := config.LoadFromBytes("databricks.yml", []byte(yaml)) require.NoError(t, diags.Error()) b := &bundle.Bundle{Config: *r} - before := b.Config.Value().Get("resources") + before := b.Config.View().Get("resources").AsAny() diags = bundle.Apply(t.Context(), b, SetRunAs()) if tc.wantError != "" { require.Error(t, diags.Error()) assert.Contains(t, diags.Error().Error(), tc.wantError) } else { require.NoError(t, diags.Error()) - assert.Equal(t, before, b.Config.Value().Get("resources")) + assert.Equal(t, before, b.Config.View().Get("resources").AsAny()) } }) } diff --git a/bundle/config/mutator/resourcemutator/secret_scope_fixups.go b/bundle/config/mutator/resourcemutator/secret_scope_fixups.go index be2a2e0775a..468d324905d 100644 --- a/bundle/config/mutator/resourcemutator/secret_scope_fixups.go +++ b/bundle/config/mutator/resourcemutator/secret_scope_fixups.go @@ -9,7 +9,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/iamutil" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/iam" @@ -132,6 +131,7 @@ func (m *secretScopeFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Di currentUser := b.Config.Workspace.CurrentUser.User addManageForCurrentUser(scope, currentUser) + path := structpath.NewPath(nil, "resources", "secret_scopes", key) err := collapsePermissions(scope) if err != nil { return diag.Diagnostics{ @@ -139,8 +139,8 @@ func (m *secretScopeFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Di Severity: diag.Error, Summary: "Failed to collapse permissions for secret scope", Detail: err.Error(), - Paths: structpath.NewPathSlice("resources", "secret_scopes", key), - Locations: []dyn.Location{b.Config.GetLocation("resources.secret_scopes." + key)}, + Paths: []*structpath.PathNode{path}, + Locations: []diag.Location{b.Config.GetLocation(path.String())}, }, } } diff --git a/bundle/config/mutator/resourcemutator/validate_target_mode.go b/bundle/config/mutator/resourcemutator/validate_target_mode.go index d100c327fe8..1eb11006c4b 100644 --- a/bundle/config/mutator/resourcemutator/validate_target_mode.go +++ b/bundle/config/mutator/resourcemutator/validate_target_mode.go @@ -5,12 +5,10 @@ import ( "fmt" "strings" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/iamutil" - "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/iamutil" ) type validateTargetMode struct{} @@ -55,7 +53,7 @@ func validateDevelopmentMode(b *bundle.Bundle) diag.Diagnostics { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "target with 'mode: development' cannot set trigger pause status to UNPAUSED by default", - Locations: []dyn.Location{b.Config.GetLocation("presets.trigger_pause_status")}, + Locations: []diag.Location{b.Config.GetLocation("presets.trigger_pause_status")}, }) } @@ -77,7 +75,7 @@ func validateDevelopmentMode(b *bundle.Bundle) diag.Diagnostics { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "prefix should contain the current username or ${workspace.current_user.short_name} to ensure uniqueness when using 'mode: development'", - Locations: []dyn.Location{b.Config.GetLocation("presets.name_prefix")}, + Locations: []diag.Location{b.Config.GetLocation("presets.name_prefix")}, }) } return diags diff --git a/bundle/config/mutator/rewrite_sync_paths.go b/bundle/config/mutator/rewrite_sync_paths.go index 228c5484def..c4eefebae2e 100644 --- a/bundle/config/mutator/rewrite_sync_paths.go +++ b/bundle/config/mutator/rewrite_sync_paths.go @@ -8,7 +8,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type rewriteSyncPaths struct{} @@ -21,9 +22,8 @@ func (m *rewriteSyncPaths) Name() string { return "RewriteSyncPaths" } -// makeRelativeTo returns a dyn.MapFunc that joins the relative path -// of the file it was defined in w.r.t. the bundle root path, with -// the contents of the string node. +// makeRelativeTo joins the relative path of the file the string node was +// defined in w.r.t. the bundle root path, with the contents of the string node. // // For example: // - The bundle root is /foo @@ -31,63 +31,46 @@ func (m *rewriteSyncPaths) Name() string { // - The string node contains "somefile.*" // // Then the resulting value will be "bar/somefile.*". -func (m *rewriteSyncPaths) makeRelativeTo(root string) dyn.MapFunc { - return func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - dir := filepath.Dir(v.Location().File) - rel, err := filepath.Rel(root, dir) - if err != nil { - return dyn.InvalidValue, err - } +func (m *rewriteSyncPaths) makeRelativeTo(root string, v structvar.View) (string, error) { + dir := filepath.Dir(v.Location().File) + rel, err := filepath.Rel(root, dir) + if err != nil { + return "", err + } - return dyn.NewValue(filepath.Join(rel, v.MustString()), v.Locations()), nil + s, ok := v.AsString() + if !ok { + return "", fmt.Errorf("expected string value but got %s", v.Kind()) } + + return filepath.Join(rel, s), nil } func (m *rewriteSyncPaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.Map(v, "sync", func(_ dyn.Path, v dyn.Value) (nv dyn.Value, err error) { - v, err = dyn.Map(v, "paths", dyn.Foreach(m.makeRelativeTo(b.BundleRootPath))) - if err != nil { - return dyn.InvalidValue, err - } - - makeRelativeFn := m.makeRelativeTo(b.BundleRootPath) - - // Makes include and exclude paths relative to the bundle root first. - // Then converts them to use Unix-style slashes. - // This is required for the ignore.GitIgnore we use in libs/fileset to work correctly. - v, err = dyn.Map(v, "include", dyn.Foreach(func(p dyn.Path, val dyn.Value) (dyn.Value, error) { - relPath, err := makeRelativeFn(p, val) - if err != nil { - return dyn.InvalidValue, err - } - str, ok := relPath.AsString() - if !ok { - return dyn.InvalidValue, fmt.Errorf("expected string value but got %s", relPath.Kind()) - } - return dyn.NewValue(filepath.ToSlash(str), relPath.Locations()), nil - })) + rewrite := func(field string, toSlash bool) error { + pattern := structpath.MustParsePattern("sync." + field + "[*]") + return structvar.ForEach(b.Config.View(), pattern, func(p *structpath.PathNode, v structvar.View) error { + path, err := m.makeRelativeTo(b.BundleRootPath, v) if err != nil { - return dyn.InvalidValue, err + return err } - - v, err = dyn.Map(v, "exclude", dyn.Foreach(func(p dyn.Path, val dyn.Value) (dyn.Value, error) { - relPath, err := makeRelativeFn(p, val) - if err != nil { - return dyn.InvalidValue, err - } - str, ok := relPath.AsString() - if !ok { - return dyn.InvalidValue, fmt.Errorf("expected string value but got %s", relPath.Kind()) - } - return dyn.NewValue(filepath.ToSlash(str), relPath.Locations()), nil - })) - if err != nil { - return dyn.InvalidValue, err + if toSlash { + path = filepath.ToSlash(path) } - return v, nil + return b.Config.Set(p, path) }) - }) + } + + // Makes include and exclude paths relative to the bundle root first. + // Then converts them to use Unix-style slashes. + // This is required for the ignore.GitIgnore we use in libs/fileset to work correctly. + err := rewrite("paths", false) + if err == nil { + err = rewrite("include", true) + } + if err == nil { + err = rewrite("exclude", true) + } return diag.FromErr(err) } diff --git a/bundle/config/mutator/rewrite_sync_paths_test.go b/bundle/config/mutator/rewrite_sync_paths_test.go index 2b9d291ddd8..e01b5cc8eee 100644 --- a/bundle/config/mutator/rewrite_sync_paths_test.go +++ b/bundle/config/mutator/rewrite_sync_paths_test.go @@ -8,7 +8,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/internal/bundletest" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/diag" "github.com/stretchr/testify/assert" ) @@ -33,12 +33,12 @@ func TestRewriteSyncPathsRelative(t *testing.T) { }, } - bundletest.SetLocation(b, "sync.paths[0]", []dyn.Location{{File: "./databricks.yml"}}) - bundletest.SetLocation(b, "sync.paths[1]", []dyn.Location{{File: "./databricks.yml"}}) - bundletest.SetLocation(b, "sync.include[0]", []dyn.Location{{File: "./file.yml"}}) - bundletest.SetLocation(b, "sync.include[1]", []dyn.Location{{File: "./a/file.yml"}}) - bundletest.SetLocation(b, "sync.exclude[0]", []dyn.Location{{File: "./a/b/file.yml"}}) - bundletest.SetLocation(b, "sync.exclude[1]", []dyn.Location{{File: "./a/b/c/file.yml"}}) + bundletest.SetLocation(b, "sync.paths[0]", []diag.Location{{File: "./databricks.yml"}}) + bundletest.SetLocation(b, "sync.paths[1]", []diag.Location{{File: "./databricks.yml"}}) + bundletest.SetLocation(b, "sync.include[0]", []diag.Location{{File: "./file.yml"}}) + bundletest.SetLocation(b, "sync.include[1]", []diag.Location{{File: "./a/file.yml"}}) + bundletest.SetLocation(b, "sync.exclude[0]", []diag.Location{{File: "./a/b/file.yml"}}) + bundletest.SetLocation(b, "sync.exclude[1]", []diag.Location{{File: "./a/b/c/file.yml"}}) diags := bundle.Apply(t.Context(), b, mutator.RewriteSyncPaths()) assert.NoError(t, diags.Error()) @@ -72,12 +72,12 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) { }, } - bundletest.SetLocation(b, "sync.paths[0]", []dyn.Location{{File: "/tmp/dir/databricks.yml"}}) - bundletest.SetLocation(b, "sync.paths[1]", []dyn.Location{{File: "/tmp/dir/databricks.yml"}}) - bundletest.SetLocation(b, "sync.include[0]", []dyn.Location{{File: "/tmp/dir/file.yml"}}) - bundletest.SetLocation(b, "sync.include[1]", []dyn.Location{{File: "/tmp/dir/a/file.yml"}}) - bundletest.SetLocation(b, "sync.exclude[0]", []dyn.Location{{File: "/tmp/dir/a/b/file.yml"}}) - bundletest.SetLocation(b, "sync.exclude[1]", []dyn.Location{{File: "/tmp/dir/a/b/c/file.yml"}}) + bundletest.SetLocation(b, "sync.paths[0]", []diag.Location{{File: "/tmp/dir/databricks.yml"}}) + bundletest.SetLocation(b, "sync.paths[1]", []diag.Location{{File: "/tmp/dir/databricks.yml"}}) + bundletest.SetLocation(b, "sync.include[0]", []diag.Location{{File: "/tmp/dir/file.yml"}}) + bundletest.SetLocation(b, "sync.include[1]", []diag.Location{{File: "/tmp/dir/a/file.yml"}}) + bundletest.SetLocation(b, "sync.exclude[0]", []diag.Location{{File: "/tmp/dir/a/b/file.yml"}}) + bundletest.SetLocation(b, "sync.exclude[1]", []diag.Location{{File: "/tmp/dir/a/b/c/file.yml"}}) diags := bundle.Apply(t.Context(), b, mutator.RewriteSyncPaths()) assert.NoError(t, diags.Error()) diff --git a/bundle/config/mutator/rewrite_workspace_prefix.go b/bundle/config/mutator/rewrite_workspace_prefix.go index 30914fd149e..390a13bb490 100644 --- a/bundle/config/mutator/rewrite_workspace_prefix.go +++ b/bundle/config/mutator/rewrite_workspace_prefix.go @@ -7,7 +7,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type rewriteWorkspacePrefix struct{} @@ -44,38 +45,36 @@ func (m *rewriteWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) di {"/Workspace${workspace.resource_path}", "${workspace.resource_path}"}, } - err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - // Walk through the bundle configuration, check all the string leafs and - // see if any of the prefixes are used in the remote path. - return dyn.Walk(root, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - vv, ok := v.AsString() - if !ok { - return v, nil - } - - newPath := vv - for _, rewrite := range paths { - if !strings.Contains(newPath, rewrite.pattern) { - continue - } + // Walk through the bundle configuration, check all the string leafs and + // see if any of the prefixes are used in the remote path. + err := structvar.Walk(b.Config.View(), func(p *structpath.PathNode, v structvar.View) error { + vv, ok := v.AsString() + if !ok { + return nil + } - newPath = strings.ReplaceAll(newPath, rewrite.pattern, rewrite.replacement) - diags = append(diags, diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("substring %q found in %q. Please update this to %q.", rewrite.pattern, vv, newPath), - Detail: "For more information, please refer to: https://docs.databricks.com/en/release-notes/dev-tools/bundles.html#workspace-paths", - Locations: v.Locations(), - Paths: dyn.ToStructPaths(p), - }) + newPath := vv + for _, rewrite := range paths { + if !strings.Contains(newPath, rewrite.pattern) { + continue } - if newPath == vv { - return v, nil - } + newPath = strings.ReplaceAll(newPath, rewrite.pattern, rewrite.replacement) + diags = append(diags, diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("substring %q found in %q. Please update this to %q.", rewrite.pattern, vv, newPath), + Detail: "For more information, please refer to: https://docs.databricks.com/en/release-notes/dev-tools/bundles.html#workspace-paths", + Locations: v.Locations(), + Paths: []*structpath.PathNode{p}, + }) + } + + if newPath == vv { + return nil + } - // Remove the workspace prefix from the string. - return dyn.NewValue(newPath, v.Locations()), nil - }) + // Remove the workspace prefix from the string. + return b.Config.Set(p, newPath) }) if err != nil { return diag.FromErr(err) diff --git a/bundle/config/mutator/set_variables.go b/bundle/config/mutator/set_variables.go index 3038b05acaf..c12df9f6d3c 100644 --- a/bundle/config/mutator/set_variables.go +++ b/bundle/config/mutator/set_variables.go @@ -7,11 +7,12 @@ import ( "path/filepath" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/variable" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/jsonloader" "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) const bundleVarPrefix = "BUNDLE_VAR_" @@ -30,90 +31,99 @@ func getDefaultVariableFilePath(target string) string { return ".databricks/bundle/" + target + "/variable-overrides.json" } -func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable, name string, fileDefault dyn.Value) (dyn.Value, error) { +func setVariable(ctx context.Context, cfg *config.Root, variable *variable.Variable, name string, fileDefault structvar.View) error { // case: variable already has value initialized, so skip if variable.HasValue() { - return v, nil + return nil } + variablePath := structpath.NewPath(nil, "variables", name) + valuePath := structpath.NewStringKey(variablePath, "value") + // case: read and set variable value from process environment envVarName := bundleVarPrefix + name if val, ok := env.Lookup(ctx, envVarName); ok { if variable.IsComplex() { - return dyn.InvalidValue, fmt.Errorf(`setting via environment variables (%s) is not supported for complex variable %s`, envVarName, name) + return fmt.Errorf(`setting via environment variables (%s) is not supported for complex variable %s`, envVarName, name) } - v, err := dyn.Set(v, "value", dyn.V(val)) + err := cfg.Set(valuePath, val) if err != nil { - return dyn.InvalidValue, fmt.Errorf(`failed to assign value "%s" to variable %s from environment variable %s with error: %v`, val, name, envVarName, err) + return fmt.Errorf(`failed to assign value "%s" to variable %s from environment variable %s with error: %v`, val, name, envVarName, err) } - return v, nil + return nil } // case: Defined a variable for named lookup for a resource // It will be resolved later in ResolveResourceReferences mutator if variable.Lookup != nil { - return v, nil + return nil } // case: Set the variable to the default value from the variable file - if fileDefault.Kind() != dyn.KindInvalid && fileDefault.Kind() != dyn.KindNil { + if fileDefault.Kind() != structvar.KindInvalid && fileDefault.Kind() != structvar.KindNil { hasComplexType := variable.IsComplex() - hasComplexValue := fileDefault.Kind() == dyn.KindMap || fileDefault.Kind() == dyn.KindSequence + hasComplexValue := fileDefault.Kind() == structvar.KindMap || fileDefault.Kind() == structvar.KindSequence if hasComplexType && !hasComplexValue { - return dyn.InvalidValue, fmt.Errorf(`variable %s is of type complex, but the value in the variable file is not a complex type`, name) + return fmt.Errorf(`variable %s is of type complex, but the value in the variable file is not a complex type`, name) } if !hasComplexType && hasComplexValue { - return dyn.InvalidValue, fmt.Errorf(`variable %s is not of type complex, but the value in the variable file is a complex type`, name) + return fmt.Errorf(`variable %s is not of type complex, but the value in the variable file is a complex type`, name) } - v, err := dyn.Set(v, "value", fileDefault) + err := cfg.Assign(valuePath, fileDefault) if err != nil { - return dyn.InvalidValue, fmt.Errorf(`failed to assign default value from variable file to variable %s with error: %v`, name, err) + return fmt.Errorf(`failed to assign default value from variable file to variable %s with error: %v`, name, err) } - return v, nil + return nil } // case: Set the variable to its default value if variable.HasDefault() { - vDefault, err := dyn.Get(v, "default") - if err != nil { - return dyn.InvalidValue, fmt.Errorf(`failed to get default value from config "%s" for variable %s with error: %v`, variable.Default, name, err) + vDefault := cfg.View().Lookup(structpath.NewStringKey(variablePath, "default")) + if !vDefault.IsValid() { + return fmt.Errorf(`failed to get default value from config "%s" for variable %s with error: no such key: default`, variable.Default, name) } - v, err := dyn.Set(v, "value", vDefault) + err := cfg.Assign(valuePath, vDefault) if err != nil { - return dyn.InvalidValue, fmt.Errorf(`failed to assign default value from config "%s" to variable %s with error: %v`, variable.Default, name, err) + return fmt.Errorf(`failed to assign default value from config "%s" to variable %s with error: %v`, variable.Default, name, err) } - return v, nil + return nil } // We should have had a value to set for the variable at this point. - return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Variables are usually assigned in databricks.yml, and they can be overridden using "--var", the %s environment variable, or %s`, name, bundleVarPrefix+name, getDefaultVariableFilePath("")) + return fmt.Errorf(`no value assigned to required variable %s. Variables are usually assigned in databricks.yml, and they can be overridden using "--var", the %s environment variable, or %s`, name, bundleVarPrefix+name, getDefaultVariableFilePath("")) } -func readVariablesFromFile(b *bundle.Bundle) (dyn.Value, diag.Diagnostics) { +func readVariablesFromFile(b *bundle.Bundle) (structvar.View, diag.Diagnostics) { var diags diag.Diagnostics filePath := filepath.Join(b.BundleRootPath, getDefaultVariableFilePath(b.Config.Bundle.Target)) if _, err := os.Stat(filePath); err != nil { - return dyn.InvalidValue, nil + return structvar.View{}, nil } f, err := os.ReadFile(filePath) if err != nil { - return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to read variables file: %w", err)) + return structvar.View{}, diag.FromErr(fmt.Errorf("failed to read variables file: %w", err)) } - val, err := jsonloader.LoadJSON(f, filePath) + node, err := structvar.ParseJSON(filePath, f) if err != nil { - return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to parse variables file %s: %w", filePath, err)) + return structvar.View{}, diag.FromErr(fmt.Errorf("failed to parse variables file %s: %w", filePath, err)) + } + var v any + decoded, _, err := structvar.DecodeYAMLNode(filePath, node, &v, nil) + if err != nil { + return structvar.View{}, diag.FromErr(fmt.Errorf("failed to parse variables file %s: %w", filePath, err)) } - if val.Kind() != dyn.KindMap { - return dyn.InvalidValue, diags.Append(diag.Diagnostic{ + val := decoded.View() + if val.Kind() != structvar.KindMap { + return structvar.View{}, diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("failed to parse variables file %s: invalid format", filePath), Detail: "Variables file must be a JSON object with the following format:\n{\"var1\": \"value1\", \"var2\": \"value2\"}", @@ -128,18 +138,18 @@ func (m *setVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos if diags.HasError() { return diags } - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.Map(v, "variables", dyn.Foreach(func(p dyn.Path, variable dyn.Value) (dyn.Value, error) { - name := p[1].Key() - v, ok := b.Config.Variables[name] - if !ok { - return dyn.InvalidValue, fmt.Errorf(`variable "%s" is not defined`, name) - } - - fileDefault, _ := dyn.Get(defaults, name) - return setVariable(ctx, variable, v, name, fileDefault) - })) - }) - - return diags.Extend(diag.FromErr(err)) + for name := range b.Config.View().Get("variables").MapItems() { + v, ok := b.Config.Variables[name] + if !ok { + return diags.Extend(diag.Errorf(`variable "%s" is not defined`, name)) + } + + fileDefault := defaults.Get(name) + err := setVariable(ctx, &b.Config, v, name, fileDefault) + if err != nil { + return diags.Extend(diag.FromErr(err)) + } + } + + return diags } diff --git a/bundle/config/mutator/set_variables_test.go b/bundle/config/mutator/set_variables_test.go index ecc67548017..977e7e94529 100644 --- a/bundle/config/mutator/set_variables_test.go +++ b/bundle/config/mutator/set_variables_test.go @@ -6,12 +6,16 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/variable" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/structs/structvar" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +// configWithVariable returns a configuration that has the variable with name "foo". +func configWithVariable(v *variable.Variable) *config.Root { + return &config.Root{Variables: map[string]*variable.Variable{"foo": v}} +} + func TestSetVariableFromProcessEnvVar(t *testing.T) { defaultVal := "default" variable := variable.Variable{ @@ -21,13 +25,9 @@ func TestSetVariableFromProcessEnvVar(t *testing.T) { // set value for variable as an environment variable t.Setenv("BUNDLE_VAR_foo", "process-env") - v, err := convert.FromTyped(variable, dyn.NilValue) - require.NoError(t, err) - - v, err = setVariable(t.Context(), v, &variable, "foo", dyn.NilValue) - require.NoError(t, err) + cfg := configWithVariable(&variable) - err = convert.ToTyped(&variable, v) + err := setVariable(t.Context(), cfg, &variable, "foo", structvar.View{}) require.NoError(t, err) assert.Equal(t, "process-env", variable.Value) } @@ -39,13 +39,9 @@ func TestSetVariableUsingDefaultValue(t *testing.T) { Default: defaultVal, } - v, err := convert.FromTyped(variable, dyn.NilValue) - require.NoError(t, err) - - v, err = setVariable(t.Context(), v, &variable, "foo", dyn.NilValue) - require.NoError(t, err) + cfg := configWithVariable(&variable) - err = convert.ToTyped(&variable, v) + err := setVariable(t.Context(), cfg, &variable, "foo", structvar.View{}) require.NoError(t, err) assert.Equal(t, "default", variable.Value) } @@ -61,13 +57,9 @@ func TestSetVariableWhenAlreadyAValueIsAssigned(t *testing.T) { // since a value is already assigned to the variable, it would not be overridden // by the default value - v, err := convert.FromTyped(variable, dyn.NilValue) - require.NoError(t, err) - - v, err = setVariable(t.Context(), v, &variable, "foo", dyn.NilValue) - require.NoError(t, err) + cfg := configWithVariable(&variable) - err = convert.ToTyped(&variable, v) + err := setVariable(t.Context(), cfg, &variable, "foo", structvar.View{}) require.NoError(t, err) assert.Equal(t, "assigned-value", variable.Value) } @@ -86,13 +78,9 @@ func TestSetVariableEnvVarValueDoesNotOverridePresetValue(t *testing.T) { // since a value is already assigned to the variable, it would not be overridden // by the value from environment - v, err := convert.FromTyped(variable, dyn.NilValue) - require.NoError(t, err) + cfg := configWithVariable(&variable) - v, err = setVariable(t.Context(), v, &variable, "foo", dyn.NilValue) - require.NoError(t, err) - - err = convert.ToTyped(&variable, v) + err := setVariable(t.Context(), cfg, &variable, "foo", structvar.View{}) require.NoError(t, err) assert.Equal(t, "assigned-value", variable.Value) } @@ -103,10 +91,9 @@ func TestSetVariablesErrorsIfAValueCouldNotBeResolved(t *testing.T) { } // fails because we could not resolve a value for the variable - v, err := convert.FromTyped(variable, dyn.NilValue) - require.NoError(t, err) + cfg := configWithVariable(&variable) - _, err = setVariable(t.Context(), v, &variable, "foo", dyn.NilValue) + err := setVariable(t.Context(), cfg, &variable, "foo", structvar.View{}) assert.ErrorContains(t, err, "no value assigned to required variable foo. Variables are usually assigned in databricks.yml, and they can be overridden using \"--var\", the BUNDLE_VAR_foo environment variable, or .databricks/bundle//variable-overrides.json") } @@ -153,9 +140,8 @@ func TestSetComplexVariablesViaEnvVariablesIsNotAllowed(t *testing.T) { // set value for variable as an environment variable t.Setenv("BUNDLE_VAR_foo", "process-env") - v, err := convert.FromTyped(variable, dyn.NilValue) - require.NoError(t, err) + cfg := configWithVariable(&variable) - _, err = setVariable(t.Context(), v, &variable, "foo", dyn.NilValue) + err := setVariable(t.Context(), cfg, &variable, "foo", structvar.View{}) assert.ErrorContains(t, err, "setting via environment variables (BUNDLE_VAR_foo) is not supported for complex variable foo") } diff --git a/bundle/config/mutator/sync_default_path.go b/bundle/config/mutator/sync_default_path.go index 16d4a4d6179..586d864e98c 100644 --- a/bundle/config/mutator/sync_default_path.go +++ b/bundle/config/mutator/sync_default_path.go @@ -5,7 +5,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" ) type syncDefaultPath struct{} @@ -20,24 +19,9 @@ func (m *syncDefaultPath) Name() string { } func (m *syncDefaultPath) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - isset := false - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - pv, _ := dyn.Get(v, "sync.paths") - - // If the sync paths field is already set, do nothing. - // We know it is set if its value is either a nil or a sequence (empty or not). - if pv.Kind() == dyn.KindNil || pv.Kind() == dyn.KindSequence { - isset = true - } - - return v, nil - }) - if err != nil { - return diag.FromErr(err) - } - // If the sync paths field is already set, do nothing. - if isset { + // We know it is set if it is a sequence (empty or not) or a reference. + if b.Config.Sync.Paths != nil || b.Config.IsReference("sync.paths") { return nil } diff --git a/bundle/config/mutator/sync_default_path_test.go b/bundle/config/mutator/sync_default_path_test.go index c4bd6a65af7..a85821b6cc2 100644 --- a/bundle/config/mutator/sync_default_path_test.go +++ b/bundle/config/mutator/sync_default_path_test.go @@ -7,8 +7,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -28,22 +28,22 @@ func TestSyncDefaultPath_DefaultIfUnset(t *testing.T) { func TestSyncDefaultPath_SkipIfSet(t *testing.T) { tcases := []struct { name string - paths dyn.Value + paths any expect []string }{ { name: "nil", - paths: dyn.V(nil), + paths: nil, expect: []string{"."}, }, { name: "empty sequence", - paths: dyn.V([]dyn.Value{}), + paths: []string{}, expect: []string{}, }, { name: "non-empty sequence", - paths: dyn.V([]dyn.Value{dyn.V("something")}), + paths: []string{"something"}, expect: []string{"something"}, }, } @@ -58,17 +58,7 @@ func TestSyncDefaultPath_SkipIfSet(t *testing.T) { ctx := logdiag.InitContext(t.Context()) bundle.ApplyFuncContext(ctx, b, func(ctx context.Context, b *bundle.Bundle) { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - v, err := dyn.Set(v, "sync", dyn.V(dyn.NewMapping())) - if err != nil { - return dyn.InvalidValue, err - } - v, err = dyn.Set(v, "sync.paths", tcase.paths) - if err != nil { - return dyn.InvalidValue, err - } - return v, nil - }) + err := b.Config.Set(structpath.MustParsePath("sync.paths"), tcase.paths) require.NoError(t, err) }) require.False(t, logdiag.HasError(ctx)) diff --git a/bundle/config/mutator/sync_infer_root.go b/bundle/config/mutator/sync_infer_root.go index 184ec289f33..75223780081 100644 --- a/bundle/config/mutator/sync_infer_root.go +++ b/bundle/config/mutator/sync_infer_root.go @@ -94,7 +94,7 @@ func (m *syncInferRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno Severity: diag.Error, Summary: fmt.Sprintf("invalid sync path %q", path), Locations: b.Config.GetLocations(fmt.Sprintf("sync.paths[%d]", i)), - Paths: structpath.NewPathSlice("sync", "paths", i), + Paths: []*structpath.PathNode{structpath.NewIndex(structpath.MustParsePath("sync.paths"), i)}, }) } diff --git a/bundle/config/mutator/sync_infer_root_test.go b/bundle/config/mutator/sync_infer_root_test.go index c8f4a09c472..e34b4210bd6 100644 --- a/bundle/config/mutator/sync_infer_root_test.go +++ b/bundle/config/mutator/sync_infer_root_test.go @@ -8,7 +8,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/internal/bundletest" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/diag" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -184,7 +184,7 @@ func TestSyncInferRoot_Error(t *testing.T) { }, } - bundletest.SetLocation(b, "sync.paths", []dyn.Location{{File: "databricks.yml"}}) + bundletest.SetLocation(b, "sync.paths", []diag.Location{{File: "databricks.yml"}}) ctx := t.Context() diags := bundle.Apply(ctx, b, mutator.SyncInferRoot()) diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index 65a903bf5a3..df6c575fb20 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -11,15 +11,15 @@ import ( "path/filepath" "strings" - "github.com/databricks/cli/bundle/config/mutator/paths" - "github.com/databricks/cli/bundle/libraries" - "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator/paths" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/notebook" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) // translateOptions control path translation behavior. @@ -300,37 +300,51 @@ func (t *translateContext) translateLocalRelativeWithPrefixPath(ctx context.Cont return localRelPath, nil } -func (t *translateContext) rewriteValue(ctx context.Context, p dyn.Path, v dyn.Value, dir string, opts translateOptions) (dyn.Value, error) { - out, err := t.rewritePath(ctx, dir, v.MustString(), opts) +// rewriteValue rewrites the path input found at p. It returns an empty string if the path was not rewritten. +func (t *translateContext) rewriteValue(ctx context.Context, p *structpath.PathNode, input, dir string, opts translateOptions) (string, error) { + out, err := t.rewritePath(ctx, dir, input, opts) if err != nil { if target, ok := errors.AsType[ErrIsNotebook](err); ok { - return dyn.InvalidValue, fmt.Errorf(`expected a file for "%s" but got a notebook: %w`, p, target) + return "", fmt.Errorf(`expected a file for "%s" but got a notebook: %w`, p, target) } if target, ok := errors.AsType[ErrIsNotNotebook](err); ok { - return dyn.InvalidValue, fmt.Errorf(`expected a notebook for "%s" but got a file: %w`, p, target) + return "", fmt.Errorf(`expected a notebook for "%s" but got a file: %w`, p, target) } - return dyn.InvalidValue, err + return "", err } - // If the path was not rewritten, return the original value. + return out, nil +} + +// setRewritten stores the rewritten path at p (keeping its locations) unless the path was not rewritten. +func (t *translateContext) setRewritten(p *structpath.PathNode, out string) error { if out == "" { - return v, nil + return nil } + return t.b.Config.Set(p, out) +} - return dyn.NewValue(out, v.Locations()), nil +// rewriteAt rewrites the string value v found at p relative to the bundle root and stores the result. +func (t *translateContext) rewriteAt(ctx context.Context, p *structpath.PathNode, v structvar.View, opts translateOptions) error { + input, _ := v.AsString() + out, err := t.rewriteValue(ctx, p, input, t.b.BundleRootPath, opts) + if err != nil { + return err + } + return t.setRewritten(p, out) } -func applyTranslations(ctx context.Context, b *bundle.Bundle, t *translateContext, translations []func(context.Context, dyn.Value) (dyn.Value, error)) diag.Diagnostics { +func applyTranslations(ctx context.Context, b *bundle.Bundle, t *translateContext, translations []translateFunc) diag.Diagnostics { switch { case b.IsImmutableFolder(): // Reject an explicit workspace.file_path: immutable bundles set it // automatically to the content-addressed snapshot location. A user-supplied // value would be silently discarded during path translation, so we error early. - if loc := b.Config.GetLocation("workspace.file_path"); loc.File != "" { + if loc := b.Config.DefinitionLocation("workspace.file_path"); loc.File != "" { return diag.Diagnostics{{ Severity: diag.Error, Summary: "workspace.file_path cannot be configured when experimental.immutable_folder is enabled", - Locations: []dyn.Location{loc}, + Locations: []diag.Location{loc}, }} } t.remoteRoot = resources.SnapshotFullPathRef + "/files" @@ -340,18 +354,13 @@ func applyTranslations(ctx context.Context, b *bundle.Bundle, t *translateContex t.remoteRoot = t.b.Config.Workspace.FilePath } - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - var err error - for _, fn := range translations { - v, err = fn(ctx, v) - if err != nil { - return dyn.InvalidValue, err - } + for _, fn := range translations { + if err := fn(ctx, b.Config.View()); err != nil { + return diag.FromErr(err) } - return v, nil - }) + } - return diag.FromErr(err) + return nil } func (m *translatePaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { @@ -361,7 +370,7 @@ func (m *translatePaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn skipLocalFileValidation: b.SkipLocalFileValidation, } - return applyTranslations(ctx, b, t, []func(context.Context, dyn.Value) (dyn.Value, error){ + return applyTranslations(ctx, b, t, []translateFunc{ t.applyJobTranslations(paths.VisitJobPaths, false), t.applyJobTranslations(paths.VisitJobLibrariesPaths, true), t.applyPipelineTranslations(paths.VisitPipelinePaths, false), @@ -378,7 +387,7 @@ func (m *translatePathsDashboards) Apply(ctx context.Context, b *bundle.Bundle) skipLocalFileValidation: b.SkipLocalFileValidation, } - return applyTranslations(ctx, b, t, []func(context.Context, dyn.Value) (dyn.Value, error){ + return applyTranslations(ctx, b, t, []translateFunc{ t.applyDashboardTranslations, t.applyGenieSpaceTranslations, }) @@ -386,27 +395,27 @@ func (m *translatePathsDashboards) Apply(ctx context.Context, b *bundle.Bundle) // gatherFallbackPaths collects the fallback paths for relative paths in the configuration. // Read more about the motivation for this functionality in the "fallback" path translation tests. -func gatherFallbackPaths(v dyn.Value, typ string) (map[string]string, error) { +func gatherFallbackPaths(v structvar.View, typ string) (map[string]string, error) { fallback := make(map[string]string) - pattern := dyn.NewPattern(dyn.Key("resources"), dyn.Key(typ), dyn.AnyKey()) + pattern := structpath.NewPatternDotStar(structpath.NewPatternStringKey(structpath.MustParsePattern("resources"), typ)) // Previous behavior was to use a resource's location as the base path to resolve - // relative paths in its definition. With the introduction of [dyn.Value] throughout, - // we can use the location of the [dyn.Value] of the relative path itself. + // relative paths in its definition. With the introduction of [structvar.View] throughout, + // we can use the location of the [structvar.View] of the relative path itself. // // This is more flexible, as resources may have overrides that are not // located in the same directory as the resource configuration file. // // To maintain backwards compatibility, we allow relative paths to be resolved using - // the original approach as fallback if the [dyn.Value] location cannot be resolved. - _, err := dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - key := p[2].Key() + // the original approach as fallback if the [structvar.View] location cannot be resolved. + err := structvar.ForEach(v, pattern, func(p *structpath.PathNode, v structvar.View) error { + key := p.KeyAt(2) dir, err := locationDirectory(v.Location()) if err != nil { - return dyn.InvalidValue, fmt.Errorf("unable to determine directory for %s: %w", p, err) + return fmt.Errorf("unable to determine directory for %s: %w", p, err) } fallback[key] = dir - return v, nil + return nil }) if err != nil { return nil, err diff --git a/bundle/config/mutator/translate_paths_apps.go b/bundle/config/mutator/translate_paths_apps.go index 8daef7e4857..8c8c688b23c 100644 --- a/bundle/config/mutator/translate_paths_apps.go +++ b/bundle/config/mutator/translate_paths_apps.go @@ -5,18 +5,19 @@ import ( "github.com/databricks/cli/bundle/config/mutator/paths" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -func (t *translateContext) applyAppsTranslations(ctx context.Context, v dyn.Value) (dyn.Value, error) { +func (t *translateContext) applyAppsTranslations(ctx context.Context, v structvar.View) error { // Convert the `source_code_path` field to a remote absolute path. // We use this path for app deployment to point to the source code. - return paths.VisitAppPaths(v, func(p dyn.Path, mode paths.TranslateMode, v dyn.Value) (dyn.Value, error) { + return paths.VisitAppPaths(v, func(p *structpath.PathNode, mode paths.TranslateMode, v structvar.View) error { opts := translateOptions{ Mode: mode, } - return t.rewriteValue(ctx, p, v, t.b.BundleRootPath, opts) + return t.rewriteAt(ctx, p, v, opts) }) } diff --git a/bundle/config/mutator/translate_paths_apps_test.go b/bundle/config/mutator/translate_paths_apps_test.go index a69999b2b2a..05c902235b1 100644 --- a/bundle/config/mutator/translate_paths_apps_test.go +++ b/bundle/config/mutator/translate_paths_apps_test.go @@ -9,7 +9,7 @@ import ( "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/internal/bundletest" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/vfs" "github.com/databricks/databricks-sdk-go/service/apps" "github.com/stretchr/testify/assert" @@ -41,7 +41,7 @@ func TestTranslatePathsApps_FilePathRelativeSubDirectory(t *testing.T) { }, } - bundletest.SetLocation(b, "resources.apps", []dyn.Location{{ + bundletest.SetLocation(b, "resources.apps", []diag.Location{{ File: filepath.Join(dir, "resources/app.yml"), }}) diff --git a/bundle/config/mutator/translate_paths_artifacts.go b/bundle/config/mutator/translate_paths_artifacts.go index 51ae0d8f8f5..17112c43e66 100644 --- a/bundle/config/mutator/translate_paths_artifacts.go +++ b/bundle/config/mutator/translate_paths_artifacts.go @@ -5,11 +5,12 @@ import ( "github.com/databricks/cli/bundle/config/mutator/paths" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -func (t *translateContext) applyArtifactTranslations(ctx context.Context, v dyn.Value) (dyn.Value, error) { - return paths.VisitArtifactPaths(v, func(p dyn.Path, mode paths.TranslateMode, v dyn.Value) (dyn.Value, error) { +func (t *translateContext) applyArtifactTranslations(ctx context.Context, v structvar.View) error { + return paths.VisitArtifactPaths(v, func(p *structpath.PathNode, mode paths.TranslateMode, v structvar.View) error { opts := translateOptions{ Mode: mode, @@ -18,6 +19,6 @@ func (t *translateContext) applyArtifactTranslations(ctx context.Context, v dyn. AllowPathOutsideSyncRoot: true, } - return t.rewriteValue(ctx, p, v, t.b.BundleRootPath, opts) + return t.rewriteAt(ctx, p, v, opts) }) } diff --git a/bundle/config/mutator/translate_paths_artifacts_test.go b/bundle/config/mutator/translate_paths_artifacts_test.go index 31f9a0ea4ad..cf55d3d70dd 100644 --- a/bundle/config/mutator/translate_paths_artifacts_test.go +++ b/bundle/config/mutator/translate_paths_artifacts_test.go @@ -9,7 +9,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/internal/bundletest" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/vfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -38,7 +38,7 @@ func TestTranslatePathsArtifacts_InsideSyncRoot(t *testing.T) { }, } - bundletest.SetLocation(b, "artifacts", []dyn.Location{{ + bundletest.SetLocation(b, "artifacts", []diag.Location{{ File: filepath.Join(dir, "config/artifacts.yml"), }}) @@ -72,7 +72,7 @@ func TestTranslatePathsArtifacts_OutsideSyncRoot(t *testing.T) { }, } - bundletest.SetLocation(b, "artifacts", []dyn.Location{{ + bundletest.SetLocation(b, "artifacts", []diag.Location{{ File: filepath.Join(dir, "config/artifacts.yml"), }}) diff --git a/bundle/config/mutator/translate_paths_dashboards.go b/bundle/config/mutator/translate_paths_dashboards.go index 2e93ed3998f..85d4c809022 100644 --- a/bundle/config/mutator/translate_paths_dashboards.go +++ b/bundle/config/mutator/translate_paths_dashboards.go @@ -5,18 +5,19 @@ import ( "github.com/databricks/cli/bundle/config/mutator/paths" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -func (t *translateContext) applyDashboardTranslations(ctx context.Context, v dyn.Value) (dyn.Value, error) { +func (t *translateContext) applyDashboardTranslations(ctx context.Context, v structvar.View) error { // Convert the `file_path` field to a local absolute path. // We load the file at this path and use its contents for the dashboard contents. - return paths.VisitDashboardPaths(v, func(p dyn.Path, mode paths.TranslateMode, v dyn.Value) (dyn.Value, error) { + return paths.VisitDashboardPaths(v, func(p *structpath.PathNode, mode paths.TranslateMode, v structvar.View) error { opts := translateOptions{ Mode: mode, } - return t.rewriteValue(ctx, p, v, t.b.BundleRootPath, opts) + return t.rewriteAt(ctx, p, v, opts) }) } diff --git a/bundle/config/mutator/translate_paths_dashboards_test.go b/bundle/config/mutator/translate_paths_dashboards_test.go index 89625798805..84c36bc3706 100644 --- a/bundle/config/mutator/translate_paths_dashboards_test.go +++ b/bundle/config/mutator/translate_paths_dashboards_test.go @@ -9,7 +9,7 @@ import ( "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/internal/bundletest" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/vfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -37,7 +37,7 @@ func TestTranslatePathsDashboards_FilePathRelativeSubDirectory(t *testing.T) { }, } - bundletest.SetLocation(b, "resources.dashboards", []dyn.Location{{ + bundletest.SetLocation(b, "resources.dashboards", []diag.Location{{ File: filepath.Join(dir, "resources/dashboard.yml"), }}) diff --git a/bundle/config/mutator/translate_paths_genie_spaces.go b/bundle/config/mutator/translate_paths_genie_spaces.go index 4e6d41f1cec..d8f46498492 100644 --- a/bundle/config/mutator/translate_paths_genie_spaces.go +++ b/bundle/config/mutator/translate_paths_genie_spaces.go @@ -4,18 +4,19 @@ import ( "context" "github.com/databricks/cli/bundle/config/mutator/paths" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -func (t *translateContext) applyGenieSpaceTranslations(ctx context.Context, v dyn.Value) (dyn.Value, error) { +func (t *translateContext) applyGenieSpaceTranslations(ctx context.Context, v structvar.View) error { // Rewrite the `file_path` field to a path relative to the bundle sync root. // We load the file at this path and use its contents for the genie space contents. - return paths.VisitGenieSpacePaths(v, func(p dyn.Path, mode paths.TranslateMode, v dyn.Value) (dyn.Value, error) { + return paths.VisitGenieSpacePaths(v, func(p *structpath.PathNode, mode paths.TranslateMode, v structvar.View) error { opts := translateOptions{ Mode: mode, } - return t.rewriteValue(ctx, p, v, t.b.BundleRootPath, opts) + return t.rewriteAt(ctx, p, v, opts) }) } diff --git a/bundle/config/mutator/translate_paths_genie_spaces_test.go b/bundle/config/mutator/translate_paths_genie_spaces_test.go index a1ac0b160b1..ea053335a75 100644 --- a/bundle/config/mutator/translate_paths_genie_spaces_test.go +++ b/bundle/config/mutator/translate_paths_genie_spaces_test.go @@ -9,7 +9,7 @@ import ( "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/internal/bundletest" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/vfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -37,7 +37,7 @@ func TestTranslatePathsGenieSpaces_FilePathRelativeSubDirectory(t *testing.T) { }, } - bundletest.SetLocation(b, "resources.genie_spaces", []dyn.Location{{ + bundletest.SetLocation(b, "resources.genie_spaces", []diag.Location{{ File: filepath.Join(dir, "resources", "genie_space.yml"), }}) diff --git a/bundle/config/mutator/translate_paths_jobs.go b/bundle/config/mutator/translate_paths_jobs.go index 4ffab62c89b..ea8c9f92b60 100644 --- a/bundle/config/mutator/translate_paths_jobs.go +++ b/bundle/config/mutator/translate_paths_jobs.go @@ -7,24 +7,22 @@ import ( "slices" "github.com/databricks/cli/bundle/config/mutator/paths" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type ( - visitFunc func(dyn.Value, paths.VisitFunc) (dyn.Value, error) - translateFunc func(ctx context.Context, v dyn.Value) (dyn.Value, error) + visitFunc func(structvar.View, paths.VisitFunc) error + translateFunc func(ctx context.Context, v structvar.View) error ) func (t *translateContext) applyJobTranslations(visitor visitFunc, allowOutsideSyncRoot bool) translateFunc { - return func(ctx context.Context, v dyn.Value) (dyn.Value, error) { - var err error - + return func(ctx context.Context, v structvar.View) error { fallback, err := gatherFallbackPaths(v, "jobs") if err != nil { - return dyn.InvalidValue, err + return err } // Do not translate job task paths if using Git source @@ -35,12 +33,13 @@ func (t *translateContext) applyJobTranslations(visitor visitFunc, allowOutsideS } } - return visitor(v, func(p dyn.Path, mode paths.TranslateMode, v dyn.Value) (dyn.Value, error) { - key := p[2].Key() + return visitor(v, func(p *structpath.PathNode, mode paths.TranslateMode, v structvar.View) error { + key := p.KeyAt(2) + input, _ := v.AsString() // Skip path translation if the job is using git source. if slices.Contains(ignore, key) { - return v, nil + return nil } opts := translateOptions{ @@ -49,9 +48,9 @@ func (t *translateContext) applyJobTranslations(visitor visitFunc, allowOutsideS } // Handle path as if it's relative to the bundle root - nv, err := t.rewriteValue(ctx, p, v, t.b.BundleRootPath, opts) + nv, err := t.rewriteValue(ctx, p, input, t.b.BundleRootPath, opts) if err == nil { - return nv, nil + return t.setRewritten(p, nv) } // If we failed to rewrite the path, try to rewrite it relative to the fallback directory. @@ -59,32 +58,34 @@ func (t *translateContext) applyJobTranslations(visitor visitFunc, allowOutsideS if fallback[key] != "" { dir, nerr := locationDirectory(v.Location()) if nerr != nil { - return dyn.InvalidValue, nerr + return nerr } dirRel, nerr := filepath.Rel(t.b.BundleRootPath, dir) if nerr != nil { - return dyn.InvalidValue, nerr + return nerr } - originalPath, nerr := filepath.Rel(dirRel, v.MustString()) + originalPath, nerr := filepath.Rel(dirRel, input) if nerr != nil { - return dyn.InvalidValue, nerr + return nerr } - originalValue := dyn.NewValue(originalPath, v.Locations()) - nv, nerr := t.rewriteValue(ctx, p, originalValue, fallback[key], opts) + nv, nerr := t.rewriteValue(ctx, p, originalPath, fallback[key], opts) if nerr == nil { logdiag.LogDiag(ctx, diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("path %s is defined relative to the %s directory (%s). Please update the path to be relative to the file where it is defined or use earlier version of CLI (0.261.0 or earlier).", originalPath, fallback[key], v.Location()), Locations: v.Locations(), }) - return nv, nil + if nv == "" { + nv = originalPath + } + return t.b.Config.Set(p, nv) } } - return dyn.InvalidValue, err + return err }) } } diff --git a/bundle/config/mutator/translate_paths_pipelines.go b/bundle/config/mutator/translate_paths_pipelines.go index 53848461bd2..20ec5f9a351 100644 --- a/bundle/config/mutator/translate_paths_pipelines.go +++ b/bundle/config/mutator/translate_paths_pipelines.go @@ -6,64 +6,65 @@ import ( "path/filepath" "github.com/databricks/cli/bundle/config/mutator/paths" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) func (t *translateContext) applyPipelineTranslations(visitor visitFunc, allowOutsideSyncRoot bool) translateFunc { - return func(ctx context.Context, v dyn.Value) (dyn.Value, error) { - var err error - + return func(ctx context.Context, v structvar.View) error { fallback, err := gatherFallbackPaths(v, "pipelines") if err != nil { - return dyn.InvalidValue, err + return err } - return visitor(v, func(p dyn.Path, mode paths.TranslateMode, v dyn.Value) (dyn.Value, error) { - key := p[2].Key() + return visitor(v, func(p *structpath.PathNode, mode paths.TranslateMode, v structvar.View) error { + key := p.KeyAt(2) + input, _ := v.AsString() opts := translateOptions{ Mode: mode, AllowPathOutsideSyncRoot: allowOutsideSyncRoot, } // Handle path as if it's relative to the bundle root - nv, err := t.rewriteValue(ctx, p, v, t.b.BundleRootPath, opts) + nv, err := t.rewriteValue(ctx, p, input, t.b.BundleRootPath, opts) if err == nil { - return nv, nil + return t.setRewritten(p, nv) } // If we failed to rewrite the path, it uses an old path format which relied on fallback. if fallback[key] != "" { dir, nerr := locationDirectory(v.Location()) if nerr != nil { - return dyn.InvalidValue, nerr + return nerr } dirRel, nerr := filepath.Rel(t.b.BundleRootPath, dir) if nerr != nil { - return dyn.InvalidValue, nerr + return nerr } - originalPath, nerr := filepath.Rel(dirRel, v.MustString()) + originalPath, nerr := filepath.Rel(dirRel, input) if nerr != nil { - return dyn.InvalidValue, nerr + return nerr } - originalValue := dyn.NewValue(originalPath, v.Locations()) - nv, nerr := t.rewriteValue(ctx, p, originalValue, fallback[key], opts) + nv, nerr := t.rewriteValue(ctx, p, originalPath, fallback[key], opts) if nerr == nil { logdiag.LogDiag(ctx, diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("path %s is defined relative to the %s directory (%s). Please update the path to be relative to the file where it is defined or use earlier version of CLI (0.261.0 or earlier).", originalPath, fallback[key], v.Location()), Locations: v.Locations(), }) - return nv, nil + if nv == "" { + nv = originalPath + } + return t.b.Config.Set(p, nv) } } - return dyn.InvalidValue, err + return err }) } } diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index 226a848b723..63569c0f747 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -14,7 +14,8 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/config/variable" "github.com/databricks/cli/bundle/internal/bundletest" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/vfs" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -93,7 +94,7 @@ func TestTranslatePathsSkippedWithGitSource(t *testing.T) { }, } - bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "resource.yml")}}) diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) require.NoError(t, diags.Error()) @@ -222,7 +223,7 @@ func TestTranslatePaths(t *testing.T) { }, } - bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "resource.yml")}}) diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) require.NoError(t, diags.Error()) @@ -365,8 +366,8 @@ func TestTranslatePathsInSubdirectories(t *testing.T) { }, } - bundletest.SetLocation(b, "resources.jobs", []dyn.Location{{File: filepath.Join(dir, "job/resource.yml")}}) - bundletest.SetLocation(b, "resources.pipelines", []dyn.Location{{File: filepath.Join(dir, "pipeline/resource.yml")}}) + bundletest.SetLocation(b, "resources.jobs", []diag.Location{{File: filepath.Join(dir, "job/resource.yml")}}) + bundletest.SetLocation(b, "resources.pipelines", []diag.Location{{File: filepath.Join(dir, "pipeline/resource.yml")}}) diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) require.NoError(t, diags.Error()) @@ -433,7 +434,7 @@ func TestTranslatePathsOutsideSyncRoot(t *testing.T) { }, } - bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "../resource.yml")}}) + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "../resource.yml")}}) diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) assert.ErrorContains(t, diags.Error(), "is not contained in sync root path") @@ -465,7 +466,7 @@ func TestJobNotebookDoesNotExistError(t *testing.T) { }, } - bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "fake.yml")}}) + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "fake.yml")}}) diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) assert.EqualError(t, diags.Error(), "notebook doesnt_exist.py not found") @@ -497,7 +498,7 @@ func TestJobFileDoesNotExistError(t *testing.T) { }, } - bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "fake.yml")}}) + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "fake.yml")}}) diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) assert.EqualError(t, diags.Error(), "file doesnt_exist.py not found") @@ -529,7 +530,7 @@ func TestPipelineNotebookDoesNotExistError(t *testing.T) { }, } - bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "fake.yml")}}) + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "fake.yml")}}) diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) assert.EqualError(t, diags.Error(), "notebook doesnt_exist.py not found") @@ -574,7 +575,7 @@ func TestPipelineNotebookDoesNotExistErrorWithoutExtension(t *testing.T) { }, } - bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "fake.yml")}}) + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "fake.yml")}}) diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) if ext == "" { @@ -615,7 +616,7 @@ func TestPipelineFileDoesNotExistError(t *testing.T) { }, } - bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "fake.yml")}}) + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "fake.yml")}}) diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) assert.EqualError(t, diags.Error(), "file doesnt_exist.py not found") @@ -651,7 +652,7 @@ func TestJobSparkPythonTaskWithNotebookSourceError(t *testing.T) { }, } - bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "resource.yml")}}) diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) assert.ErrorContains(t, diags.Error(), `expected a file for "resources.jobs.job.tasks[0].spark_python_task.python_file" but got a notebook`) @@ -687,7 +688,7 @@ func TestJobNotebookTaskWithFileSourceError(t *testing.T) { }, } - bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "resource.yml")}}) diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) assert.ErrorContains(t, diags.Error(), `expected a notebook for "resources.jobs.job.tasks[0].notebook_task.notebook_path" but got a file`) @@ -723,7 +724,7 @@ func TestPipelineNotebookLibraryWithFileSourceError(t *testing.T) { }, } - bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "resource.yml")}}) diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) assert.ErrorContains(t, diags.Error(), `expected a notebook for "resources.pipelines.pipeline.libraries[0].notebook.path" but got a file`) @@ -759,7 +760,7 @@ func TestPipelineFileLibraryWithNotebookSourceError(t *testing.T) { }, } - bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "resource.yml")}}) diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) assert.ErrorContains(t, diags.Error(), `expected a file for "resources.pipelines.pipeline.libraries[0].file.path" but got a notebook`) @@ -801,7 +802,7 @@ func TestTranslatePathJobEnvironments(t *testing.T) { }, } - bundletest.SetLocation(b, "resources.jobs", []dyn.Location{{File: filepath.Join(dir, "job/resource.yml")}}) + bundletest.SetLocation(b, "resources.jobs", []diag.Location{{File: filepath.Join(dir, "job/resource.yml")}}) diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) require.NoError(t, diags.Error()) @@ -848,16 +849,14 @@ func TestTranslatePathWithComplexVariables(t *testing.T) { }, } - bundletest.SetLocation(b, "variables", []dyn.Location{{File: filepath.Join(dir, "variables/variables.yml")}}) - bundletest.SetLocation(b, "resources.jobs", []dyn.Location{{File: filepath.Join(dir, "job/resource.yml")}}) + bundletest.SetLocation(b, "variables", []diag.Location{{File: filepath.Join(dir, "variables/variables.yml")}}) + bundletest.SetLocation(b, "resources.jobs", []diag.Location{{File: filepath.Join(dir, "job/resource.yml")}}) ctx := t.Context() // Assign the variables to the dynamic configuration. bundle.ApplyFuncContext(ctx, b, func(ctx context.Context, b *bundle.Bundle) { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - p := dyn.MustPathFromString("resources.jobs.job.tasks[0]") - return dyn.SetByPath(v, p.Append(dyn.Key("libraries")), dyn.V("${var.cluster_libraries}")) - }) + p := structpath.MustParsePath("resources.jobs.job.tasks[0]") + err := b.Config.SetReference(structpath.NewStringKey(p, "libraries"), "${var.cluster_libraries}") require.NoError(t, err) }) @@ -977,7 +976,7 @@ func TestTranslatePathsWithSourceLinkedDeployment(t *testing.T) { }, } - bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "resource.yml")}}) diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) require.NoError(t, diags.Error()) @@ -1085,7 +1084,7 @@ func TestTranslatePathsWithSkipLocalFileValidation(t *testing.T) { }, } - bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "databricks.yml")}}) + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "databricks.yml")}}) diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) require.NoError(t, diags.Error()) @@ -1125,7 +1124,7 @@ func TestTranslatePathsWithSkipLocalFileValidationDirectory(t *testing.T) { }, } - bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "databricks.yml")}}) + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "databricks.yml")}}) diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) require.NoError(t, diags.Error()) @@ -1174,7 +1173,7 @@ func TestTranslatePathsDesignerNotebook(t *testing.T) { }, } - bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "databricks.yml")}}) + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "databricks.yml")}}) diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) require.NoError(t, diags.Error()) @@ -1228,7 +1227,7 @@ func TestTranslatePathsDesignerNotebookSkipLocalFileValidation(t *testing.T) { }, } - bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "databricks.yml")}}) + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "databricks.yml")}}) diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) require.NoError(t, diags.Error()) diff --git a/bundle/config/mutator/validate_secret_value_is_variable.go b/bundle/config/mutator/validate_secret_value_is_variable.go index 1f979e15f74..467f8a6d2ae 100644 --- a/bundle/config/mutator/validate_secret_value_is_variable.go +++ b/bundle/config/mutator/validate_secret_value_is_variable.go @@ -6,8 +6,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type validateSecretValueIsVariable struct{} @@ -25,21 +25,18 @@ func (v *validateSecretValueIsVariable) Apply(ctx context.Context, b *bundle.Bun // Iterate over all secrets in the bundle for key := range b.Config.Resources.Secrets { - p := dyn.NewPath(dyn.Key("resources"), dyn.Key("secrets"), dyn.Key(key), dyn.Key("value")) - val, err := dyn.GetByPath(b.Config.Value(), p) - if dyn.IsNoSuchKeyError(err) { + p := structpath.NewPath(nil, "resources", "secrets", key, "value") + val := b.Config.View().Lookup(p) + if !val.IsValid() { diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: "Secret value must be a string", Detail: fmt.Sprintf(`The secret value for "%s" must be a string.`, key), Locations: val.Locations(), - Paths: dyn.ToStructPaths(p), + Paths: []*structpath.PathNode{p}, }) continue } - if err != nil { - return diag.FromErr(err) - } valueStr, ok := val.AsString() if !ok { @@ -48,13 +45,13 @@ func (v *validateSecretValueIsVariable) Apply(ctx context.Context, b *bundle.Bun Summary: "Secret value must be a string", Detail: fmt.Sprintf(`The secret value for "%s" must be a string.`, key), Locations: val.Locations(), - Paths: dyn.ToStructPaths(p), + Paths: []*structpath.PathNode{p}, }) continue } // Value must be a variable reference to prevent leaking secrets in config files - if !dynvar.IsPureVariableReference(valueStr) { + if !structvar.IsPureVariableReference(valueStr) { diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: "Secret value must be a variable reference", @@ -62,7 +59,7 @@ func (v *validateSecretValueIsVariable) Apply(ctx context.Context, b *bundle.Bun Plain text secret values are not allowed to prevent leaking secrets in configuration files. Use bundle variables to pass secret values at deployment time.`, key), Locations: val.Locations(), - Paths: dyn.ToStructPaths(p), + Paths: []*structpath.PathNode{p}, }) continue } @@ -79,25 +76,21 @@ Use bundle variables to pass secret values at deployment time.`, key), // checkVariableDefault emits an error if valueStr is a pure ${var.} reference // and the referenced variable has a default value set. -func (v *validateSecretValueIsVariable) checkVariableDefault(b *bundle.Bundle, secretKey, valueStr string, p dyn.Path, val dyn.Value) diag.Diagnostics { - refPath, ok := dynvar.PureReferenceToPath(valueStr) - if !ok || len(refPath) < 2 || refPath[0].Key() != "var" { +func (v *validateSecretValueIsVariable) checkVariableDefault(b *bundle.Bundle, secretKey, valueStr string, p *structpath.PathNode, val structvar.View) diag.Diagnostics { + refPath, ok := structvar.PureReferenceToPath(valueStr) + if !ok || refPath.Len() < 2 || refPath.KeyAt(0) != "var" { return nil } - varName := refPath[1].Key() + varName := refPath.KeyAt(1) variable, exists := b.Config.Variables[varName] if !exists || variable == nil || !variable.HasDefault() { return nil } // The default path in the dynamic config for the variable's default field. - defaultPath := dyn.NewPath(dyn.Key("variables"), dyn.Key(varName), dyn.Key("default")) - defaultVal, err := dyn.GetByPath(b.Config.Value(), defaultPath) - locations := val.Locations() - if err == nil { - locations = append(defaultVal.Locations(), locations...) - } + defaultPath := structpath.NewPath(nil, "variables", varName, "default") + locations := append(b.Config.LocationsAt(defaultPath), val.Locations()...) return diag.Diagnostics{{ Severity: diag.Error, @@ -106,6 +99,6 @@ func (v *validateSecretValueIsVariable) checkVariableDefault(b *bundle.Bundle, s A default value is stored in plain text in the configuration file, which defeats the purpose of using a variable reference for a secret. Remove the default value and pass the secret value at deployment time using "--var", the BUNDLE_VAR_%s environment variable, or a variable overrides file.`, varName, secretKey, varName), Locations: locations, - Paths: dyn.ToStructPaths(p), + Paths: []*structpath.PathNode{p}, }} } diff --git a/bundle/config/resources_types_test.go b/bundle/config/resources_types_test.go index 7a698604612..af53fe0ef04 100644 --- a/bundle/config/resources_types_test.go +++ b/bundle/config/resources_types_test.go @@ -3,20 +3,19 @@ package config import ( "encoding/json" "fmt" + "maps" "reflect" "slices" "strings" "testing" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/structs/structtag" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/cli/libs/structs/structwalk" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestResourcesTypesMap(t *testing.T) { @@ -32,14 +31,14 @@ func TestResourcesTypesMap(t *testing.T) { } // TestResourceTypesZeroValueFieldsSerialize guards against the ForceSendFields -// routing bug fixed in libs/dyn/convert: a field declared in a struct embedded +// routing bug: a field declared in a struct embedded // more than one level deep (e.g. PostgresProject -> PostgresProjectConfig -> // ProjectSpec) had its zero value recorded in the wrong struct's ForceSendFields, // which the SDK marshaler rejects with "field X cannot be found in struct Y". // The direct engine hits this path when it serializes planned state to JSON. // // For every registered resource type it sets every omitempty scalar field (at any -// depth) to its zero value, converts via ToTyped, and marshals - the same round +// depth) to its zero value, decodes it into the typed value, and marshals - the same round // trip the direct engine performs. Any newly added resource whose wrapper embeds // an SDK spec is covered automatically. func TestResourceTypesZeroValueFieldsSerialize(t *testing.T) { @@ -53,41 +52,38 @@ func TestResourceTypesZeroValueFieldsSerialize(t *testing.T) { t.Run(name, func(t *testing.T) { typ := ResourcesTypes[name] zeros := zeroValueScalars(typ, 0, map[reflect.Type]bool{}) - if zeros.Kind() != dyn.KindMap { + if zeros == nil { return } ptr := reflect.New(typ) - require.NoError(t, convert.ToTyped(ptr.Interface(), zeros)) + _, err := (&structvar.StructVar{Value: ptr.Interface()}).Assign(nil, structvar.NewView(&zeros, nil, nil)) + require.NoError(t, err) - _, err := json.Marshal(ptr.Interface()) + _, err = json.Marshal(ptr.Interface()) require.NoError(t, err) }) } } -// zeroValueScalars builds a [dyn.Value] map that sets every omitempty scalar field +// zeroValueScalars builds a map that sets every omitempty scalar field // reachable through embedded anonymous structs to its zero value. Those are exactly // the fields the convert layer records in ForceSendFields, so they exercise the // routing logic. depth and seen bound recursion against deep or recursive types. -func zeroValueScalars(t reflect.Type, depth int, seen map[reflect.Type]bool) dyn.Value { +func zeroValueScalars(t reflect.Type, depth int, seen map[reflect.Type]bool) map[string]any { for t.Kind() == reflect.Pointer { t = t.Elem() } if t.Kind() != reflect.Struct || depth > 6 || seen[t] { - return dyn.NilValue + return nil } seen[t] = true defer delete(seen, t) - m := dyn.NewMapping() + m := map[string]any{} for f := range t.Fields() { if f.Anonymous { - if sub := zeroValueScalars(f.Type, depth+1, seen); sub.Kind() == dyn.KindMap { - for _, p := range sub.MustMap().Pairs() { - m.SetLoc(p.Key.MustString(), nil, p.Value) - } - } + maps.Copy(m, zeroValueScalars(f.Type, depth+1, seen)) continue } @@ -99,18 +95,18 @@ func zeroValueScalars(t reflect.Type, depth int, seen map[reflect.Type]bool) dyn switch f.Type.Kind() { case reflect.Bool: - m.SetLoc(name, nil, dyn.V(false)) + m[name] = false case reflect.String: - m.SetLoc(name, nil, dyn.V("")) + m[name] = "" case reflect.Int, reflect.Int32, reflect.Int64: - m.SetLoc(name, nil, dyn.V(int64(0))) + m[name] = int64(0) case reflect.Float32, reflect.Float64: - m.SetLoc(name, nil, dyn.V(float64(0))) + m[name] = float64(0) default: // Only basic types are eligible for ForceSendFields; skip the rest. } } - return dyn.V(m) + return m } // TestNoSameDepthJSONShadows uses structwalk.WalkType — the dumb walker that diff --git a/bundle/config/root.go b/bundle/config/root.go index e13bf78bf15..289112610e1 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -2,23 +2,24 @@ package config import ( "bytes" - "context" "errors" "fmt" + "io" + "iter" + "maps" "os" "reflect" + "slices" "strings" + "github.com/databricks/cli/bundle/config/loctable" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/config/variable" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/cli/libs/dyn/dynloc" - "github.com/databricks/cli/libs/dyn/merge" - "github.com/databricks/cli/libs/dyn/yamlloader" - "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go/service/jobs" + "go.yaml.in/yaml/v3" ) type Script struct { @@ -34,8 +35,12 @@ type Script struct { } type Root struct { //nolint:recvcheck // value receivers for read-only accessors, pointer for mutators - value dyn.Value - depth int + // refs holds the pure variable references in fields that cannot hold a string + // (e.g. `max_retries: ${var.n}`, where the typed field is zero), keyed by path. + refs map[string]string + + // locations holds the source locations of the values. + locations *structvar.Locations // Contains user defined variables Variables map[string]*variable.Variable `json:"variables,omitempty"` @@ -87,7 +92,7 @@ type Root struct { //nolint:recvcheck // value receivers for read-only accessors // Locations is an output-only field that holds configuration location // information for every path in the configuration tree. - Locations *dynloc.Locations `json:"__locations,omitempty" bundle:"internal"` + Locations *loctable.Locations `json:"__locations,omitempty" bundle:"internal"` Scripts map[string]Script `json:"scripts,omitempty"` @@ -109,161 +114,55 @@ func Load(path string) (*Root, diag.Diagnostics) { } func LoadFromBytes(path string, raw []byte) (*Root, diag.Diagnostics) { - r := Root{} - - // Load configuration tree from YAML. - v, err := yamlloader.LoadYAML(path, bytes.NewBuffer(raw)) + node, err := structvar.ParseYAML(bytes.NewReader(raw)) if err != nil { - if le, ok := errors.AsType[*yamlloader.LocationError](err); ok { - return nil, diag.Diagnostics{{ - Severity: diag.Error, - Summary: le.Summary, - Locations: []dyn.Location{le.Loc}, - }} - } return nil, diag.Errorf("failed to load %s: %v", path, err) } // Rewrite configuration tree where necessary. - v, err = rewriteShorthands(v) - if err != nil { - return nil, diag.Errorf("failed to rewrite %s: %v", path, err) - } - - // Normalize dynamic configuration tree according to configuration type. - v, diags := convert.Normalize(r, v) + rewriteShorthands(node) - // Convert normalized configuration tree to typed configuration. - err = r.updateWithDynamicValue(v) + var r Root + sv, diags, err := structvar.DecodeYAMLNode(path, node, &r, nil) if err != nil { - diags = diags.Extend(diag.Errorf("failed to load %s: %v", path, err)) - return nil, diags + if le, ok := errors.AsType[*structvar.LocationError](err); ok { + return nil, diag.Diagnostics{{ + Severity: diag.Error, + Summary: le.Summary, + Locations: []diag.Location{le.Loc}, + }} + } + return nil, diags.Extend(diag.Errorf("failed to load %s: %v", path, err)) } + r.store(sv) return &r, diags } -func (r *Root) initializeDynamicValue() error { - // Many test cases initialize a config as a Go struct literal. - // The value will be invalid and we need to populate it from the typed configuration. - if r.value.IsValid() { - return nil - } - - nv, err := convert.FromTyped(r, dyn.NilValue) +// LoadFromReader decodes the configuration in r, recording the locations mapLocations +// returns (if not nil) instead of the locations in the file. +func LoadFromReader(path string, r io.Reader, mapLocations structvar.LocationMapper) (*Root, diag.Diagnostics, error) { + node, err := structvar.ParseYAML(r) if err != nil { - return err + return nil, nil, err } - - r.value = nv - return nil -} - -func (r *Root) updateWithDynamicValue(nv dyn.Value) error { - // Hack: restore state; it may be cleared by [ToTyped] if - // the configuration equals nil (happens in tests). - depth := r.depth - - defer func() { - r.depth = depth - }() - - // Convert normalized configuration tree to typed configuration. - err := convert.ToTyped(r, nv) + var root Root + sv, diags, err := structvar.DecodeYAMLNode(path, node, &root, mapLocations) if err != nil { - return err + return nil, diags, err } - - // Assign the normalized configuration tree. - r.value = nv - return nil + root.store(sv) + return &root, diags, nil } -// Mutate applies a transformation to the dynamic configuration value of a Root object. -// -// Parameters: -// - fn: A function that mutates a dyn.Value object -// -// Example usage, setting bundle.deployment.lock.enabled to false: -// -// err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { -// return dyn.Map(v, "bundle.deployment.lock", func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { -// return dyn.Set(v, "enabled", dyn.V(false)) -// }) -// }) -func (r *Root) Mutate(fn func(dyn.Value) (dyn.Value, error)) error { - err := r.initializeDynamicValue() - if err != nil { - return err - } - nv, err := fn(r.value) - if err != nil { - return err - } - err = r.updateWithDynamicValue(nv) - if err != nil { - return err - } - return nil +// vars returns the configuration as a [structvar.StructVar]; use [Root.store] to +// keep changes made to its references and locations. +func (r *Root) vars() *structvar.StructVar { + return &structvar.StructVar{Value: r, Refs: r.refs, Locations: r.locations} } -func (r *Root) MarkMutatorEntry(ctx context.Context) error { - err := r.initializeDynamicValue() - if err != nil { - return err - } - - r.depth++ - - // If we are entering a mutator at depth 1, we need to convert - // the dynamic configuration tree to typed configuration. - if r.depth == 1 { - // Always run ToTyped upon entering a mutator. - // Convert normalized configuration tree to typed configuration. - err := r.updateWithDynamicValue(r.value) - if err != nil { - log.Warnf(ctx, "unable to convert dynamic configuration to typed configuration: %v", err) - return err - } - - } else { - nv, err := convert.FromTyped(r, r.value) - if err != nil { - log.Warnf(ctx, "unable to convert typed configuration to dynamic configuration: %v", err) - return err - } - - // Re-run ToTyped to ensure that no state is piggybacked - err = r.updateWithDynamicValue(nv) - if err != nil { - log.Warnf(ctx, "unable to convert dynamic configuration to typed configuration: %v", err) - return err - } - } - - return nil -} - -func (r *Root) MarkMutatorExit(ctx context.Context) error { - r.depth-- - - // If we are exiting a mutator at depth 0, we need to convert - // the typed configuration to a dynamic configuration tree. - if r.depth == 0 { - nv, err := convert.FromTyped(r, r.value) - if err != nil { - log.Warnf(ctx, "unable to convert typed configuration to dynamic configuration: %v", err) - return err - } - - // Re-run ToTyped to ensure that no state is piggybacked - err = r.updateWithDynamicValue(nv) - if err != nil { - log.Warnf(ctx, "unable to convert dynamic configuration to typed configuration: %v", err) - return err - } - } - - return nil +func (r *Root) store(sv *structvar.StructVar) { + r.refs = sv.Refs + r.locations = sv.Locations } // Initializes variables using values passed from the command line flag @@ -294,52 +193,34 @@ func (r *Root) InitializeVariables(vars []string) error { return nil } -func (r *Root) Merge(other *Root) error { - // Merge dynamic configuration values. - return r.Mutate(func(root dyn.Value) (dyn.Value, error) { - return merge.Merge(root, other.value) - }) -} - -func mergeField(rv, ov dyn.Value, name string) (dyn.Value, error) { - path := dyn.NewPath(dyn.Key(name)) - reference, _ := dyn.GetByPath(rv, path) - override, _ := dyn.GetByPath(ov, path) - - // Merge the override into the reference. - var out dyn.Value - var err error - if reference.IsValid() && override.IsValid() { - out, err = merge.Merge(reference, override) - if err != nil { - return dyn.InvalidValue, err +// Merge merges the other configurations into this one, in order. +func (r *Root) Merge(others ...*Root) error { + for _, other := range others { + if err := r.MergeAt(nil, other.View()); err != nil { + return err } - } else if reference.IsValid() { - out = reference - } else if override.IsValid() { - out = override - } else { - return rv, nil } - - return dyn.SetByPath(rv, path, out) + return nil } +var bundleGitPath = structpath.MustParsePath("bundle.git") + func (r *Root) MergeTargetOverrides(name string) error { - root := r.value - target, err := dyn.GetByPath(root, dyn.NewPath(dyn.Key("targets"), dyn.Key(name))) - if err != nil { - return err + targetPath := structpath.NewPath(nil, "targets", name) + target := r.View().Lookup(targetPath) + if !target.IsValid() { + return fmt.Errorf("target %s not found", name) } // Confirm validity of variable overrides. - err = validateVariableOverrides(root, target) + err := validateVariableOverrides(r.Variables, r.Targets[name]) if err != nil { return err } - // Merge fields that can be merged 1:1. - for _, f := range []string{ + // Merge fields that can be merged 1:1. Check all of them first so that a failed + // merge leaves the configuration unchanged. + fields := []string{ "bundle", "workspace", "artifacts", @@ -347,106 +228,73 @@ func (r *Root) MergeTargetOverrides(name string) error { "sync", "permissions", "presets", - } { - if root, err = mergeField(root, target, f); err != nil { + } + for _, f := range fields { + if err := structvar.CheckMerge(r.View().Get(f), target.Get(f)); err != nil { + return fmt.Errorf("failed to merge target=%s field=%s: %w", name, f, err) + } + } + if err := structvar.CheckMerge(r.View().Lookup(bundleGitPath), target.Get("git")); err != nil { + return err + } + for _, f := range fields { + if err := r.MergeAt(structpath.NewStringKey(nil, f), target.Get(f)); err != nil { return fmt.Errorf("failed to merge target=%s field=%s: %w", name, f, err) } } // Merge `variables`. This field must be overwritten if set, not merged. - if v := target.Get("variables"); v.Kind() != dyn.KindInvalid { - _, err = dyn.Map(v, ".", dyn.Foreach(func(p dyn.Path, variable dyn.Value) (dyn.Value, error) { - varPath := dyn.MustPathFromString("variables").Append(p...) - - vDefault := variable.Get("default") - if vDefault.Kind() != dyn.KindInvalid { - defaultPath := varPath.Append(dyn.Key("default")) - root, err = dyn.SetByPath(root, defaultPath, vDefault) - if err != nil { - return root, err - } + for varName, variable := range target.Get("variables").MapItems() { + varPath := structpath.NewPath(nil, "variables", varName) - // If the target explicitly sets a default value, drop any lookup from the - // root variable definition so SetVariables can assign this default. - lookupPath := varPath.Append(dyn.Key("lookup")) - root, err = dyn.SetByPath(root, lookupPath, dyn.NilValue) + if vDefault := variable.Get("default"); vDefault.IsValid() { + if err := r.Assign(structpath.NewStringKey(varPath, "default"), vDefault); err != nil { + return err } - vLookup := variable.Get("lookup") - if vLookup.Kind() != dyn.KindInvalid { - lookupPath := varPath.Append(dyn.Key("lookup")) - root, err = dyn.SetByPath(root, lookupPath, vLookup) - if err != nil { - return root, err - } + // If the target explicitly sets a default value, drop any lookup from the + // root variable definition so SetVariables can assign this default. + if err := r.Delete(structpath.NewStringKey(varPath, "lookup")); err != nil { + return err + } + } - // If the target explicitly sets a lookup, drop any default value from the - // root variable definition so lookup resolution remains authoritative. - defaultPath := varPath.Append(dyn.Key("default")) - root, err = dyn.SetByPath(root, defaultPath, dyn.NilValue) + if vLookup := variable.Get("lookup"); vLookup.IsValid() { + if err := r.Assign(structpath.NewStringKey(varPath, "lookup"), vLookup); err != nil { + return err } - return root, err - })) - if err != nil { - return err + // If the target explicitly sets a lookup, drop any default value from the + // root variable definition so lookup resolution remains authoritative. + if err := r.Delete(structpath.NewStringKey(varPath, "default")); err != nil { + return err + } } } // Merge `run_as`. This field must be overwritten if set, not merged. - if v := target.Get("run_as"); v.Kind() != dyn.KindInvalid { - root, err = dyn.Set(root, "run_as", v) - if err != nil { - return err - } - } - - // Below, we're setting fields on the bundle key, so make sure it exists. - if root.Get("bundle").Kind() == dyn.KindInvalid { - root, err = dyn.Set(root, "bundle", dyn.V(map[string]dyn.Value{})) - if err != nil { + if v := target.Get("run_as"); v.IsValid() { + if err := r.Assign(structpath.NewStringKey(nil, "run_as"), v); err != nil { return err } } // Merge `mode`. This field must be overwritten if set, not merged. - if v := target.Get("mode"); v.Kind() != dyn.KindInvalid { - root, err = dyn.SetByPath(root, dyn.NewPath(dyn.Key("bundle"), dyn.Key("mode")), v) - if err != nil { + if v := target.Get("mode"); v.IsValid() { + if err := r.Assign(structpath.MustParsePath("bundle.mode"), v); err != nil { return err } } // Merge `cluster_id`. This field must be overwritten if set, not merged. - if v := target.Get("cluster_id"); v.Kind() != dyn.KindInvalid { - root, err = dyn.SetByPath(root, dyn.NewPath(dyn.Key("bundle"), dyn.Key("cluster_id")), v) - if err != nil { + if v := target.Get("cluster_id"); v.IsValid() { + if err := r.Assign(structpath.MustParsePath("bundle.cluster_id"), v); err != nil { return err } } // Merge `git`. - if v := target.Get("git"); v.Kind() != dyn.KindInvalid { - ref, err := dyn.GetByPath(root, dyn.NewPath(dyn.Key("bundle"), dyn.Key("git"))) - if err != nil { - ref = dyn.V(map[string]dyn.Value{}) - } - - // Merge the override into the reference. - out, err := merge.Merge(ref, v) - if err != nil { - return err - } - - // Set the merged value. - root, err = dyn.SetByPath(root, dyn.NewPath(dyn.Key("bundle"), dyn.Key("git")), out) - if err != nil { - return err - } - } - - // Convert normalized configuration tree to typed configuration. - return r.updateWithDynamicValue(root) + return r.MergeAt(bundleGitPath, target.Get("git")) } var allowedVariableDefinitions = []([]string){ @@ -458,30 +306,25 @@ var allowedVariableDefinitions = []([]string){ {"lookup"}, } -// isFullVariableOverrideDef checks if the given value is a full syntax variable override. -// A full syntax variable override is a map with either 1 of 2 keys. +// isFullVariableOverrideDef checks if a mapping with the given keys is a full syntax +// variable override. A full syntax variable override is a map with either 1 of 2 keys. // If it's 2 keys, the keys should be "default" and "type". // If it's 1 key, the key should be one of the following keys: "default", "lookup". -func isFullVariableOverrideDef(v dyn.Value) bool { - mv, ok := v.AsMap() - if !ok { - return false - } - +func isFullVariableOverrideDef(keys []string) bool { // If the map has more than 3 keys, it is not a full variable override. - if mv.Len() > 3 { + if len(keys) > 3 { return false } - for _, keys := range allowedVariableDefinitions { - if len(keys) != mv.Len() { + for _, allowed := range allowedVariableDefinitions { + if len(allowed) != len(keys) { continue } // Check if the keys are the same. match := true - for _, key := range keys { - if _, ok := mv.GetByString(key); !ok { + for _, key := range allowed { + if !slices.Contains(keys, key) { match = false break } @@ -495,114 +338,248 @@ func isFullVariableOverrideDef(v dyn.Value) bool { return false } +// dealias returns the node an alias refers to, or node itself. +func dealias(node *yaml.Node) *yaml.Node { + for node != nil && node.Kind == yaml.AliasNode { + node = node.Alias + } + return node +} + +// mappingValue returns the value of key in the YAML mapping node (following aliases). +func mappingValue(node *yaml.Node, key string) *yaml.Node { + node = dealias(node) + if node == nil || node.Kind != yaml.MappingNode { + return nil + } + for i := 0; i+1 < len(node.Content); i += 2 { + if node.Content[i].Value == key { + return dealias(node.Content[i+1]) + } + } + return nil +} + +// expandMergeKeys returns a copy of the mapping node with "<<" merge keys replaced +// by the pairs they merge in (keys set explicitly take precedence), so the pairs +// can be rewritten individually. +func expandMergeKeys(node *yaml.Node) *yaml.Node { + var merged, explicit []*yaml.Node + for i := 0; i+1 < len(node.Content); i += 2 { + if node.Content[i].Value != "<<" || node.Content[i].ShortTag() != "!!merge" { + explicit = append(explicit, node.Content[i], node.Content[i+1]) + continue + } + sources := []*yaml.Node{dealias(node.Content[i+1])} + if sources[0].Kind == yaml.SequenceNode { + sources = sources[0].Content + } + for _, src := range sources { + if src = dealias(src); src.Kind == yaml.MappingNode { + merged = append(merged, expandMergeKeys(src).Content...) + } + } + } + if merged == nil { + return node + } + out := *node + out.Content = explicit + for i := 0; i+1 < len(merged); i += 2 { + if mappingValue(&out, merged[i].Value) == nil { + out.Content = append(out.Content, merged[i], merged[i+1]) + } + } + return &out +} + +func mappingKeys(node *yaml.Node) []string { + var keys []string + for i := 0; i+1 < len(node.Content); i += 2 { + keys = append(keys, node.Content[i].Value) + } + return keys +} + +// mappingNode returns a mapping node with the given keys and values, at the location of at. +func mappingNode(at *yaml.Node, kvs ...any) *yaml.Node { + out := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map", Line: at.Line, Column: at.Column} + for i := 0; i < len(kvs); i += 2 { + out.Content = append(out.Content, &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: kvs[i].(string)}, kvs[i+1].(*yaml.Node)) + } + return out +} + // rewriteShorthands performs lightweight rewriting of the configuration // tree where we allow users to write a shorthand and must rewrite to the full form. -func rewriteShorthands(v dyn.Value) (dyn.Value, error) { - if v.Kind() != dyn.KindMap { - return v, nil +func rewriteShorthands(root *yaml.Node) { + targets := mappingValue(root, "targets") + if targets == nil || targets.Kind != yaml.MappingNode { + return } // For each target, rewrite the variables block. - return dyn.Map(v, "targets", dyn.Foreach(func(_ dyn.Path, target dyn.Value) (dyn.Value, error) { - // Confirm it has a variables block. - if target.Get("variables").Kind() == dyn.KindInvalid { - return target, nil + for i := 1; i < len(targets.Content); i += 2 { + variables := mappingValue(targets.Content[i], "variables") + if variables == nil || variables.Kind != yaml.MappingNode { + continue } + *variables = *expandMergeKeys(variables) - // For each variable, normalize its contents if it is a single string. - return dyn.Map(target, "variables", dyn.Foreach(func(p dyn.Path, variable dyn.Value) (dyn.Value, error) { - switch variable.Kind() { - - case dyn.KindString, dyn.KindBool, dyn.KindFloat, dyn.KindInt: + for j := 0; j+1 < len(variables.Content); j += 2 { + name := variables.Content[j].Value + variable := dealias(variables.Content[j+1]) + switch { + case variable.Kind == yaml.ScalarNode && variable.ShortTag() != "!!null": // Rewrite the variable to a map with a single key called "default". // This conforms to the variable type. Normalization back to the typed // configuration will convert this to a string if necessary. - return dyn.NewValue(map[string]dyn.Value{ - "default": variable, - }, variable.Locations()), nil + variables.Content[j+1] = mappingNode(variable, "default", variable) - case dyn.KindMap, dyn.KindSequence: + case variable.Kind == yaml.MappingNode || variable.Kind == yaml.SequenceNode: // If it's a full variable definition, leave it as is. - if isFullVariableOverrideDef(variable) { - return variable, nil + if variable.Kind == yaml.MappingNode && isFullVariableOverrideDef(mappingKeys(variable)) { + continue } // Check if the original definition of variable has a type field. // If it has a type field, it means the shorthand is a value of a complex type. // Type might not be found if the variable overridden in a separate file // and configuration is not merged yet. - typeV, err := dyn.GetByPath(v, p.Append(dyn.Key("type"))) - if err == nil && typeV.MustString() == "complex" { - return dyn.NewValue(map[string]dyn.Value{ - "type": typeV, - "default": variable, - }, variable.Locations()), nil + typeV := mappingValue(mappingValue(mappingValue(root, "variables"), name), "type") + if typeV != nil && typeV.Value == "complex" { + variables.Content[j+1] = mappingNode(variable, "type", typeV, "default", variable) + continue } // If it's a shorthand, rewrite it to a full variable definition. - return dyn.NewValue(map[string]dyn.Value{ - "default": variable, - }, variable.Locations()), nil - + variables.Content[j+1] = mappingNode(variable, "default", variable) default: - return variable, nil } - })) - })) + } + } } // validateVariableOverrides checks that all variables specified // in the target override are also defined in the root. -func validateVariableOverrides(root, target dyn.Value) (err error) { - var rv map[string]variable.Variable - var tv map[string]variable.Variable - - // Collect variables from the root. - if v := root.Get("variables"); v.Kind() != dyn.KindInvalid { - err = convert.ToTyped(&rv, v) - if err != nil { - return fmt.Errorf("unable to collect variables from root: %w", err) - } - } - - // Collect variables from the target. - if v := target.Get("variables"); v.Kind() != dyn.KindInvalid { - err = convert.ToTyped(&tv, v) - if err != nil { - return fmt.Errorf("unable to collect variables from target: %w", err) - } +func validateVariableOverrides(root map[string]*variable.Variable, target *Target) error { + if target == nil { + return nil } - - // Check that all variables in the target exist in the root. - for k := range tv { - if _, ok := rv[k]; !ok { + for k := range target.Variables { + if _, ok := root[k]; !ok { return fmt.Errorf("variable %s is not defined but is assigned a value", k) } } - return nil } +// Set sets value at path (see [structvar.StructVar.Set]). +func (r *Root) Set(path *structpath.PathNode, value any) error { + sv := r.vars() + defer r.store(sv) + return sv.Set(path, value) +} + +// SetReference records the pure reference ref at path, a field that cannot hold a string. +func (r *Root) SetReference(path *structpath.PathNode, ref string) error { + sv := r.vars() + defer r.store(sv) + return sv.SetReference(path, ref) +} + +// Assign sets the value described by v at path, with its locations and references. +func (r *Root) Assign(path *structpath.PathNode, v structvar.View) error { + _, err := r.Decode(path, v) + return err +} + +// Decode sets the value described by v at path, converting it to the type at path; +// the diagnostics explain values that could not be converted and were dropped. +func (r *Root) Decode(path *structpath.PathNode, v structvar.View) (diag.Diagnostics, error) { + sv := r.vars() + defer r.store(sv) + return sv.Assign(path, v) +} + +// Delete removes the value at path (see [structvar.StructVar.Delete]). +func (r *Root) Delete(path *structpath.PathNode) error { + sv := r.vars() + defer r.store(sv) + return sv.Delete(path) +} + +// MergeAt merges the value described by v into the value at path (see [structvar.StructVar.Merge]). +func (r *Root) MergeAt(path *structpath.PathNode, v structvar.View) error { + sv := r.vars() + defer r.store(sv) + return sv.Merge(path, v) +} + +// MergeElementsByKey merges the elements of the sequence at path that have the same +// key (see [structvar.StructVar.MergeElementsByKey]). +func (r *Root) MergeElementsByKey(path *structpath.PathNode, keyField string, keyFn func(structvar.View) string, sortKeys bool) error { + sv := r.vars() + defer r.store(sv) + return sv.MergeElementsByKey(path, keyField, keyFn, sortKeys) +} + +// SetLocations sets the locations of the value at path and all values below it. +func (r *Root) SetLocations(path *structpath.PathNode, locs []diag.Location) { + sv := r.vars() + sv.SetLocations(path, locs) + r.store(sv) +} + +// UpdateSequence records that the elements of the sequence at path were rebuilt from +// the old ones (see [structvar.StructVar.UpdateSequence]). +func (r *Root) UpdateSequence(path *structpath.PathNode, sources [][]int) { + sv := r.vars() + sv.UpdateSequence(path, sources) + r.store(sv) +} + +// LocationsAt returns all locations of the configuration value at the specified path. +func (r Root) LocationsAt(path *structpath.PathNode) []diag.Location { + return r.locations.At(path) +} + // Best effort to get the location of configuration value at the specified path. // This function is useful to annotate error messages with the location, because // we don't want to fail with a different error message if we cannot retrieve the location. -func (r Root) GetLocation(path string) dyn.Location { - v, err := dyn.Get(r.value, path) - if err != nil { - return dyn.Location{} +func (r Root) GetLocation(path string) diag.Location { + locs := r.GetLocations(path) + if len(locs) == 0 { + return diag.Location{} } - return v.Location() + return locs[0] } // Get all locations of the configuration value at the specified path. We need both // this function and it's singular version (GetLocation) because some diagnostics just need // the primary location and some need all locations associated with a configuration value. -func (r Root) GetLocations(path string) []dyn.Location { - v, err := dyn.Get(r.value, path) +// A value without locations (e.g. set by a mutator) gets those of its closest ancestor +// that has some; use [Root.DefinitionLocation] to find where a value is defined. +func (r Root) GetLocations(path string) []diag.Location { + p, err := structpath.ParsePath(path) if err != nil { return nil } - return v.Locations() + return r.locations.Nearest(p) +} + +// DefinitionLocation returns the primary location the value at path is defined at, +// or an empty location if it has none (e.g. it was set by a mutator). +func (r Root) DefinitionLocation(path string) diag.Location { + p, err := structpath.ParsePath(path) + if err != nil { + return diag.Location{} + } + locs := r.locations.At(p) + if len(locs) == 0 { + return diag.Location{} + } + return locs[0] } // GetNodeAndType and returns parent resource node and type of the resource in direct backend. @@ -610,33 +587,33 @@ func (r Root) GetLocations(path string) []dyn.Location { // // "resources.jobs.foo.name" -> ("resources.jobs.foo", "jobs") // "resources.jobs.foo.permissions[0].level -> ("resources.jobs.foo.permissions", "jobs.permissions") -func GetNodeAndType(path dyn.Path) (dyn.Path, string) { - if len(path) < 3 { +func GetNodeAndType(path *structpath.PathNode) (*structpath.PathNode, string) { + if path.Len() < 3 { return nil, "" } - if path[0].Key() != "resources" { + if path.KeyAt(0) != "resources" { return nil, "" } - if len(path) >= 4 { - if path[3].Key() == "permissions" || path[3].Key() == "grants" { - return path[:4], path[1].Key() + "." + path[3].Key() + if path.Len() >= 4 { + if k := path.KeyAt(3); k == "permissions" || k == "grants" { + return path.Prefix(4), path.KeyAt(1) + "." + k } } - return path[:3], path[1].Key() + return path.Prefix(3), path.KeyAt(1) } // GetResourceTypeFromKey extracts the resource group from a resource path. // For example, "resources.jobs.foo" returns "jobs". // Returns empty string if the path is not in the expected format. func GetResourceTypeFromKey(path string) string { - dp, err := dyn.NewPathFromString(path) + p, err := structpath.ParsePath(path) if err != nil { return "" } - _, rType := GetNodeAndType(dp) + _, rType := GetNodeAndType(p) return rType } @@ -645,13 +622,13 @@ func GetResourceTypeFromKey(path string) string { // The returned value is a pointer to the concrete struct that represents that resource type. // When the path is invalid or resource is not found, the second return value is false. func (r *Root) GetResourceConfig(path string) (any, error) { - dynPath, err := dyn.NewPathFromString(path) + p, err := structpath.ParsePath(path) if err != nil { return nil, err } // Extract and validate the resource group from the path - node, resourceType := GetNodeAndType(dynPath) + node, resourceType := GetNodeAndType(p) if resourceType == "" { return nil, fmt.Errorf("path does not correspond to resource: %q", path) } @@ -662,18 +639,15 @@ func (r *Root) GetResourceConfig(path string) (any, error) { return nil, fmt.Errorf("no such resource type in the config: %q", resourceType) } - // Fetch the raw value from the dynamic representation of the bundle config. - v, err := dyn.GetByPath(r.Value(), node) - if err != nil { - if dyn.IsNoSuchKeyError(err) { - return nil, nil - } - return nil, fmt.Errorf("cannot fetch config for %s: %w", node, err) + // Copy the value, so that the caller can't change the configuration through it. + v := r.View().Lookup(node) + if !v.IsValid() { + return nil, nil } typedConfigPtr := reflect.New(typ) - err = convert.ToTyped(typedConfigPtr.Interface(), v) + _, err = (&structvar.StructVar{Value: typedConfigPtr.Interface()}).Assign(nil, v) if err != nil { return nil, fmt.Errorf("cannot convert config to %s: %w", typ.String(), err) } @@ -681,8 +655,38 @@ func (r *Root) GetResourceConfig(path string) (any, error) { return typedConfigPtr.Interface(), nil } -// Value returns the dynamic configuration value of the root object. This value -// is the source of truth and is kept in sync with values in the typed configuration. -func (r Root) Value() dyn.Value { - return r.value +// IsReference reports whether the value at path is a pure variable reference in a +// field that cannot hold a string (its typed value is the zero value). +func (r Root) IsReference(path string) bool { + p, err := structpath.ParsePath(path) + if err != nil { + return false + } + _, ok := r.refs[p.String()] + return ok +} + +// References returns all pure variable references in fields that cannot hold a string, +// keyed by path. References in string fields are part of their value. +func (r Root) References() iter.Seq2[*structpath.PathNode, string] { + return func(yield func(*structpath.PathNode, string) bool) { + for _, k := range slices.Sorted(maps.Keys(r.refs)) { + p, err := structpath.ParsePath(k) + if err == nil && !yield(p, r.refs[k]) { + return + } + } + } +} + +// View returns the read-only view of the configuration tree. +func (r *Root) View() structvar.View { + return structvar.NewView(r, r.refs, r.locations) +} + +// Override replaces the configuration with the result of plan (see [structvar.PlanOverride]). +func (r *Root) Override(plan *structvar.OverridePlan) error { + sv := r.vars() + defer r.store(sv) + return sv.Override(plan) } diff --git a/bundle/config/root_test.go b/bundle/config/root_test.go index 42fae49d98c..e9f7fae65f9 100644 --- a/bundle/config/root_test.go +++ b/bundle/config/root_test.go @@ -2,11 +2,13 @@ package config import ( "encoding/json" + "maps" "reflect" + "slices" "testing" "github.com/databricks/cli/bundle/config/variable" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -100,7 +102,6 @@ func TestRootMergeTargetOverridesWithMode(t *testing.T) { }, }, } - require.NoError(t, root.initializeDynamicValue()) require.NoError(t, root.MergeTargetOverrides("development")) assert.Equal(t, Development, root.Bundle.Mode) } @@ -156,7 +157,6 @@ func TestRootMergeTargetOverridesWithVariables(t *testing.T) { }, }, } - require.NoError(t, root.initializeDynamicValue()) require.NoError(t, root.MergeTargetOverrides("development")) assert.Equal(t, "bar", root.Variables["foo"].Default) assert.Equal(t, "foo var", root.Variables["foo"].Description) @@ -172,83 +172,138 @@ func TestRootMergeTargetOverridesWithVariables(t *testing.T) { func TestIsFullVariableOverrideDef(t *testing.T) { testCases := []struct { - value dyn.Value + value map[string]any expected bool }{ { - value: dyn.V(map[string]dyn.Value{ - "type": dyn.V("string"), - "default": dyn.V("foo"), - "description": dyn.V("foo var"), - }), + value: map[string]any{ + "type": "string", + "default": "foo", + "description": "foo var", + }, expected: true, }, { - value: dyn.V(map[string]dyn.Value{ - "type": dyn.V("string"), - "lookup": dyn.V("foo"), - "description": dyn.V("foo var"), - }), + value: map[string]any{ + "type": "string", + "lookup": "foo", + "description": "foo var", + }, expected: false, }, { - value: dyn.V(map[string]dyn.Value{ - "type": dyn.V("string"), - "default": dyn.V("foo"), - }), + value: map[string]any{ + "type": "string", + "default": "foo", + }, expected: true, }, { - value: dyn.V(map[string]dyn.Value{ - "type": dyn.V("string"), - "lookup": dyn.V("foo"), - }), + value: map[string]any{ + "type": "string", + "lookup": "foo", + }, expected: false, }, { - value: dyn.V(map[string]dyn.Value{ - "description": dyn.V("string"), - "default": dyn.V("foo"), - }), + value: map[string]any{ + "description": "string", + "default": "foo", + }, expected: true, }, { - value: dyn.V(map[string]dyn.Value{ - "description": dyn.V("string"), - "lookup": dyn.V("foo"), - }), + value: map[string]any{ + "description": "string", + "lookup": "foo", + }, expected: true, }, { - value: dyn.V(map[string]dyn.Value{ - "default": dyn.V("foo"), - }), + value: map[string]any{ + "default": "foo", + }, expected: true, }, { - value: dyn.V(map[string]dyn.Value{ - "lookup": dyn.V("foo"), - }), + value: map[string]any{ + "lookup": "foo", + }, expected: true, }, { - value: dyn.V(map[string]dyn.Value{ - "type": dyn.V("string"), - }), + value: map[string]any{ + "type": "string", + }, expected: false, }, { - value: dyn.V(map[string]dyn.Value{ - "type": dyn.V("string"), - "default": dyn.V("foo"), - "description": dyn.V("foo var"), - "lookup": dyn.V("foo"), - }), + value: map[string]any{ + "type": "string", + "default": "foo", + "description": "foo var", + "lookup": "foo", + }, expected: false, }, } for i, tc := range testCases { - assert.Equal(t, tc.expected, isFullVariableOverrideDef(tc.value), "test case %d", i) + keys := slices.Collect(maps.Keys(tc.value)) + assert.Equal(t, tc.expected, isFullVariableOverrideDef(keys), "test case %d", i) + } +} + +func TestLoadFromBytesNotAMap(t *testing.T) { + for _, content := range []string{"hello\n", "- a\n- b\n"} { + r, diags := LoadFromBytes("databricks.yml", []byte(content)) + assert.Nil(t, r) + assert.ErrorContains(t, diags.Error(), "failed to load databricks.yml") } } + +func TestMergeComplexVariableSequenceDefaults(t *testing.T) { + a, diags := LoadFromBytes("a.yml", []byte("variables:\n v:\n type: complex\n default: [1, 2]\n")) + require.NoError(t, diags.Error()) + b, diags := LoadFromBytes("b.yml", []byte("variables:\n v:\n type: complex\n default: [3]\n")) + require.NoError(t, diags.Error()) + require.NoError(t, a.Merge(b)) + assert.Equal(t, []any{1, 2, 3}, a.Variables["v"].Default) +} + +func TestTargetVariableShorthandThroughAliases(t *testing.T) { + r, diags := LoadFromBytes("a.yml", []byte(` +x: &dev dev_value +y: &common {b: from_anchor, c: from_anchor} +variables: + a: {default: d} + b: {default: d} + c: {default: d} +targets: + dev: + variables: + <<: *common + a: *dev + c: explicit +`)) + require.NoError(t, diags.Error()) + require.NoError(t, r.MergeTargetOverrides("dev")) + assert.Equal(t, "dev_value", r.Variables["a"].Default) + assert.Equal(t, "from_anchor", r.Variables["b"].Default) + assert.Equal(t, "explicit", r.Variables["c"].Default) +} + +func TestNestedYAMLAnchorCycle(t *testing.T) { + _, diags := LoadFromBytes("a.yml", []byte("variables:\n v:\n default: &x {a: [*x]}\n")) + assert.ErrorContains(t, diags.Error(), `cyclic reference to anchor "x"`) +} + +func TestSetReferenceAtPointerField(t *testing.T) { + r, diags := LoadFromBytes("a.yml", []byte("resources:\n jobs:\n j:\n name: j\n")) + require.NoError(t, diags.Error()) + path := structpath.MustParsePath("resources.jobs.j.trigger") + require.NoError(t, r.SetReference(path, "${var.t}")) + s, ok := r.View().Lookup(path).AsString() + require.True(t, ok) + assert.Equal(t, "${var.t}", s) +} diff --git a/bundle/config/validate/all_resources_have_values.go b/bundle/config/validate/all_resources_have_values.go index 713ec0f83af..9e3aecb9743 100644 --- a/bundle/config/validate/all_resources_have_values.go +++ b/bundle/config/validate/all_resources_have_values.go @@ -3,11 +3,14 @@ package validate import ( "context" "fmt" + "maps" + "reflect" + "slices" "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) func AllResourcesHaveValues() bundle.Mutator { @@ -23,33 +26,25 @@ func (m *allResourcesHaveValues) Name() string { func (m *allResourcesHaveValues) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { diags := diag.Diagnostics{} - _, err := dyn.MapByPattern( - b.Config.Value(), - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), - func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - if v.Kind() != dyn.KindNil { - return v, nil + for _, group := range b.Config.Resources.AllResources() { + for _, rName := range slices.Sorted(maps.Keys(group.Resources)) { + // A resource declared without a body is a nil pointer. + if !reflect.ValueOf(group.Resources[rName]).IsNil() { + continue } // Type of the resource, stripped of the trailing 's' to make it // singular. - rType := strings.TrimSuffix(p[1].Key(), "s") - - // Name of the resource. Eg: "foo" in "jobs.foo". - rName := p[2].Key() + rType := strings.TrimSuffix(group.Description.PluralName, "s") + p := structpath.NewPath(nil, "resources", group.Description.PluralName, rName) diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("%s %s is not defined", rType, rName), - Locations: v.Locations(), - Paths: dyn.ToStructPaths(p), + Locations: b.Config.LocationsAt(p), + Paths: []*structpath.PathNode{p}, }) - - return v, nil - }, - ) - if err != nil { - diags = append(diags, diag.FromErr(err)...) + } } return diags diff --git a/bundle/config/validate/enum.go b/bundle/config/validate/enum.go index 2ce99ea1d19..392b431c50c 100644 --- a/bundle/config/validate/enum.go +++ b/bundle/config/validate/enum.go @@ -4,13 +4,14 @@ import ( "cmp" "context" "fmt" + "maps" "slices" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/internal/validation/generated" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type enum struct{} @@ -27,23 +28,15 @@ func (f *enum) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { diags := diag.Diagnostics{} // Generate prefix tree for all enum fields. - trie := &dyn.TrieNode{} - for k := range generated.EnumFields { - pattern, err := dyn.NewPatternFromString(k) - if err != nil { - return diag.FromErr(fmt.Errorf("invalid pattern %q for enum field validation: %w", k, err)) - } - - err = trie.Insert(pattern) - if err != nil { - return diag.FromErr(fmt.Errorf("failed to insert pattern %q into trie: %w", k, err)) - } + patterns, err := newPatternSet(slices.Collect(maps.Keys(generated.EnumFields))) + if err != nil { + return diag.FromErr(fmt.Errorf("enum field validation: %w", err)) } - err := dyn.WalkReadOnly(b.Config.Value(), func(p dyn.Path, v dyn.Value) error { - // If the path is not found in the prefix tree, we do not need to validate any enum + err = structvar.Walk(b.Config.View(), func(np *structpath.PathNode, v structvar.View) error { + // If the path matches no pattern, we do not need to validate any enum // fields in it. - pattern, ok := trie.SearchPath(p) + pattern, ok := patterns.find(np) if !ok { return nil } @@ -56,25 +49,22 @@ func (f *enum) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // Skip validation for values containing variable references (e.g. // ${resources.jobs.my_job.id}) since they are not yet resolved. - if dynvar.ContainsVariableReference(strValue) { + if structvar.ContainsVariableReference(strValue) { return nil } // Get valid values for this pattern - validValues := generated.EnumFields[pattern.String()] + validValues := generated.EnumFields[pattern] // Check if the value is in the list of valid enum values validValue := slices.Contains(validValues, strValue) if !validValue { - // p is a slice of path components. We need to clone it before using it in diagnostics - // since the WalkReadOnly function will mutate it while walking the config tree. - diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("invalid value %q for enum field. Valid values are %v", strValue, validValues), Locations: v.Locations(), - Paths: dyn.ToStructPaths(p), + Paths: []*structpath.PathNode{np}, }) } diff --git a/bundle/config/validate/files_to_sync.go b/bundle/config/validate/files_to_sync.go index 48b66eb5c59..d1b811e17a7 100644 --- a/bundle/config/validate/files_to_sync.go +++ b/bundle/config/validate/files_to_sync.go @@ -63,7 +63,7 @@ func (v *filesToSync) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost // Show all locations where sync.exclude is defined, since merging // sync.exclude is additive. Locations: b.Config.GetLocations(path), - Paths: structpath.NewPathSlice("sync", "exclude"), + Paths: structpath.MustParsePaths(path), }) } diff --git a/bundle/config/validate/interpolation_in_auth_config.go b/bundle/config/validate/interpolation_in_auth_config.go index f749f9e0613..a282b72790e 100644 --- a/bundle/config/validate/interpolation_in_auth_config.go +++ b/bundle/config/validate/interpolation_in_auth_config.go @@ -7,8 +7,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type noInterpolationInAuthConfig struct{} @@ -51,14 +51,11 @@ func (f *noInterpolationInAuthConfig) Apply(ctx context.Context, b *bundle.Bundl diags := diag.Diagnostics{} for _, fieldName := range authFields { - p := dyn.NewPath(dyn.Key("workspace"), dyn.Key(fieldName)) - v, err := dyn.GetByPath(b.Config.Value(), p) - if dyn.IsNoSuchKeyError(err) { + p := structpath.NewPath(nil, "workspace", fieldName) + v := b.Config.View().Lookup(p) + if !v.IsValid() { continue } - if err != nil { - return diag.FromErr(err) - } vv, ok := v.AsString() if !ok { @@ -66,7 +63,7 @@ func (f *noInterpolationInAuthConfig) Apply(ctx context.Context, b *bundle.Bundl } // Check if the field contains interpolation. - if dynvar.ContainsVariableReference(vv) { + if structvar.ContainsVariableReference(vv) { envVar, ok := auth.GetEnvFor(fieldName) if !ok { continue @@ -78,7 +75,7 @@ func (f *noInterpolationInAuthConfig) Apply(ctx context.Context, b *bundle.Bundl Detail: fmt.Sprintf(`Interpolation is not supported for the field %s. Please set the %s environment variable if you wish to configure this field at runtime.`, p.String(), envVar), Locations: v.Locations(), - Paths: dyn.ToStructPaths(p), + Paths: []*structpath.PathNode{p}, }) } } diff --git a/bundle/config/validate/job_cluster_key_defined.go b/bundle/config/validate/job_cluster_key_defined.go index e29e72c1f32..23f7f4161c0 100644 --- a/bundle/config/validate/job_cluster_key_defined.go +++ b/bundle/config/validate/job_cluster_key_defined.go @@ -31,15 +31,14 @@ func (v *jobClusterKeyDefined) Apply(ctx context.Context, b *bundle.Bundle) diag } for index, task := range job.Tasks { + taskPath := structpath.NewPath(nil, "resources", "jobs", k, "tasks", index) diags = diags.Extend(checkJobClusterKey(b, jobClusterKeys, task.JobClusterKey, - fmt.Sprintf("resources.jobs.%s.tasks[%d].job_cluster_key", k, index), - structpath.NewPathSlice("resources", "jobs", k, "tasks", index, "job_cluster_key"))) + structpath.NewPath(taskPath, "job_cluster_key"))) // The Jobs API rejects nested for_each_task, so one level is sufficient. if task.ForEachTask != nil { diags = diags.Extend(checkJobClusterKey(b, jobClusterKeys, task.ForEachTask.Task.JobClusterKey, - fmt.Sprintf("resources.jobs.%s.tasks[%d].for_each_task.task.job_cluster_key", k, index), - structpath.NewPathSlice("resources", "jobs", k, "tasks", index, "for_each_task", "task", "job_cluster_key"))) + structpath.NewPath(taskPath, "for_each_task", "task", "job_cluster_key"))) } } } @@ -48,7 +47,7 @@ func (v *jobClusterKeyDefined) Apply(ctx context.Context, b *bundle.Bundle) diag } // checkJobClusterKey warns if jobClusterKey is set but not defined in the job's job_clusters. -func checkJobClusterKey(b *bundle.Bundle, jobClusterKeys map[string]bool, jobClusterKey, path string, paths []*structpath.PathNode) diag.Diagnostics { +func checkJobClusterKey(b *bundle.Bundle, jobClusterKeys map[string]bool, jobClusterKey string, path *structpath.PathNode) diag.Diagnostics { if jobClusterKey == "" { return nil } @@ -62,7 +61,7 @@ func checkJobClusterKey(b *bundle.Bundle, jobClusterKeys map[string]bool, jobClu // Show only the location where the job_cluster_key is defined. // Other associated locations are not relevant since they are // overridden during merging. - Locations: b.Config.GetLocations(path), - Paths: paths, + Locations: b.Config.GetLocations(path.String()), + Paths: []*structpath.PathNode{path}, }} } diff --git a/bundle/config/validate/job_task_cluster_spec.go b/bundle/config/validate/job_task_cluster_spec.go index 14fa8918bfe..56191d114b7 100644 --- a/bundle/config/validate/job_task_cluster_spec.go +++ b/bundle/config/validate/job_task_cluster_spec.go @@ -7,7 +7,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/jobs" ) @@ -26,13 +26,13 @@ func (v *jobTaskClusterSpec) Name() string { func (v *jobTaskClusterSpec) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { diags := diag.Diagnostics{} - jobsPath := dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs")) + jobsPath := structpath.NewPath(nil, "resources", "jobs") for resourceName, job := range b.Config.Resources.Jobs { - resourcePath := jobsPath.Append(dyn.Key(resourceName)) + resourcePath := structpath.NewPath(jobsPath, resourceName) for taskIndex, task := range job.Tasks { - taskPath := resourcePath.Append(dyn.Key("tasks"), dyn.Index(taskIndex)) + taskPath := structpath.NewPath(resourcePath, "tasks", taskIndex) diags = diags.Extend(validateJobTask(b, task, taskPath)) } @@ -41,7 +41,7 @@ func (v *jobTaskClusterSpec) Apply(ctx context.Context, b *bundle.Bundle) diag.D return diags } -func validateJobTask(b *bundle.Bundle, task jobs.Task, taskPath dyn.Path) diag.Diagnostics { +func validateJobTask(b *bundle.Bundle, task jobs.Task, taskPath *structpath.PathNode) diag.Diagnostics { diags := diag.Diagnostics{} var specified []string @@ -72,7 +72,7 @@ func validateJobTask(b *bundle.Bundle, task jobs.Task, taskPath dyn.Path) diag.D } if task.ForEachTask != nil { - forEachTaskPath := taskPath.Append(dyn.Key("for_each_task"), dyn.Key("task")) + forEachTaskPath := structpath.NewPath(taskPath, "for_each_task", "task") diags = diags.Extend(validateJobTask(b, task.ForEachTask.Task, forEachTaskPath)) } @@ -93,7 +93,7 @@ func validateJobTask(b *bundle.Bundle, task jobs.Task, taskPath dyn.Path) diag.D Summary: "Missing required cluster or environment settings", Detail: detail, Locations: b.Config.GetLocations(taskPath.String()), - Paths: dyn.ToStructPaths(taskPath), + Paths: []*structpath.PathNode{taskPath}, }) } } diff --git a/bundle/config/validate/no_interpolation_in_bundle_name.go b/bundle/config/validate/no_interpolation_in_bundle_name.go index 8fc547b174e..0850f27ac15 100644 --- a/bundle/config/validate/no_interpolation_in_bundle_name.go +++ b/bundle/config/validate/no_interpolation_in_bundle_name.go @@ -5,9 +5,9 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type noInterpolationInBundleName struct{} @@ -21,7 +21,7 @@ func (m *noInterpolationInBundleName) Name() string { } func (m *noInterpolationInBundleName) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - if dynvar.ContainsVariableReference(b.Config.Bundle.Name) { + if structvar.ContainsVariableReference(b.Config.Bundle.Name) { logdiag.LogDiag(ctx, diag.Diagnostic{ Severity: diag.Warning, Summary: `Please do not use variable interpolation in the name of your bundle. The name of your bundle diff --git a/bundle/config/validate/no_variable_reference_in_resource_key.go b/bundle/config/validate/no_variable_reference_in_resource_key.go index 75b8361ce97..a2db01ffa14 100644 --- a/bundle/config/validate/no_variable_reference_in_resource_key.go +++ b/bundle/config/validate/no_variable_reference_in_resource_key.go @@ -3,11 +3,14 @@ package validate import ( "context" "fmt" + "maps" + "slices" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type noVariableReferenceInResourceKey struct{} @@ -25,30 +28,27 @@ func (m *noVariableReferenceInResourceKey) Name() string { func (m *noVariableReferenceInResourceKey) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics - patterns := []dyn.Pattern{ - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), - dyn.NewPattern(dyn.Key("targets"), dyn.AnyKey(), dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), + check := func(prefix *structpath.PathNode, r *config.Resources) { + for _, group := range r.AllResources() { + for _, key := range slices.Sorted(maps.Keys(group.Resources)) { + if !structvar.ContainsVariableReference(key) { + continue + } + p := structpath.NewPath(prefix, group.Description.PluralName, key) + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("resource key %q must not contain variable references", key), + Locations: b.Config.LocationsAt(p), + Paths: []*structpath.PathNode{p}, + }) + } + } } - for _, pattern := range patterns { - _, err := dyn.MapByPattern( - b.Config.Value(), - pattern, - func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - key := p[len(p)-1].Key() - if dynvar.ContainsVariableReference(key) { - diags = append(diags, diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("resource key %q must not contain variable references", key), - Locations: v.Locations(), - Paths: dyn.ToStructPaths(p), - }) - } - return v, nil - }, - ) - if err != nil { - diags = append(diags, diag.FromErr(err)...) + check(structpath.NewPath(nil, "resources"), &b.Config.Resources) + for _, name := range slices.Sorted(maps.Keys(b.Config.Targets)) { + if t := b.Config.Targets[name]; t != nil && t.Resources != nil { + check(structpath.NewPath(nil, "targets", name, "resources"), t.Resources) } } diff --git a/bundle/config/validate/pattern_set.go b/bundle/config/validate/pattern_set.go new file mode 100644 index 00000000000..07d68f88212 --- /dev/null +++ b/bundle/config/validate/pattern_set.go @@ -0,0 +1,37 @@ +package validate + +import ( + "fmt" + + "github.com/databricks/cli/libs/structs/structpath" +) + +type patternEntry struct { + key string + pattern *structpath.PatternNode +} + +// patternSet finds the pattern that matches a concrete path exactly, where wildcards match a single component. +type patternSet map[int][]patternEntry + +func newPatternSet(keys []string) (patternSet, error) { + s := patternSet{} + for _, k := range keys { + pattern, err := structpath.ParsePattern(k) + if err != nil { + return nil, fmt.Errorf("invalid pattern %q: %w", k, err) + } + s[pattern.Len()] = append(s[pattern.Len()], patternEntry{k, pattern}) + } + return s, nil +} + +// find returns the key of the pattern matching p. +func (s patternSet) find(p *structpath.PathNode) (string, bool) { + for _, e := range s[p.Len()] { + if p.HasPatternPrefix(e.pattern) { + return e.key, true + } + } + return "", false +} diff --git a/bundle/config/validate/pipeline_development_deprecated.go b/bundle/config/validate/pipeline_development_deprecated.go index 3eb3c77c5f0..f6672a4646f 100644 --- a/bundle/config/validate/pipeline_development_deprecated.go +++ b/bundle/config/validate/pipeline_development_deprecated.go @@ -2,11 +2,13 @@ package validate import ( "context" + "maps" + "slices" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) const ( @@ -32,35 +34,40 @@ func (v *pipelineDevelopmentDeprecated) Apply(_ context.Context, b *bundle.Bundl presetEnabled := config.IsExplicitlyEnabled(b.Config.Presets.PipelinesDevelopment) - pattern := dyn.NewPattern(dyn.Key("resources"), dyn.Key("pipelines"), dyn.AnyKey(), dyn.Key("development")) - _, err := dyn.MapByPattern(b.Config.Value(), pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + pipelines := b.Config.Resources.Pipelines + for _, key := range slices.Sorted(maps.Keys(pipelines)) { + pipeline := pipelines[key] + //nolint:staticcheck // SA1019: pipeline development is deprecated in the SDK but remains a supported bundle config field + if pipeline == nil || (!pipeline.Development && !slices.Contains(pipeline.ForceSendFields, "Development")) { + continue + } + + p := structpath.NewPath(nil, "resources", "pipelines", key, "development") + // Only user-written values have a location; the value set by "mode: development" does not. - if len(v.Locations()) == 0 { - return v, nil + locs := b.Config.LocationsAt(p) + if len(locs) == 0 { + continue } diags = append(diags, diag.Diagnostic{ Severity: diag.Warning, Summary: pipelineDevelopmentDeprecatedSummary, - Locations: v.Locations(), - Paths: dyn.ToStructPaths(p), + Locations: locs, + Paths: []*structpath.PathNode{p}, }) // The preset overwrites YAML values with true, so a false here was set by a Python // mutator after the preset ran. "bundle run" still sends development: true. - if development, ok := v.AsBool(); ok && presetEnabled && !development { + if presetEnabled && !pipeline.Development { //nolint:staticcheck // SA1019: see above diags = append(diags, diag.Diagnostic{ Severity: diag.Warning, Summary: pipelineDevelopmentIgnoredSummary, Detail: pipelineDevelopmentIgnoredDetail, - Locations: v.Locations(), - Paths: dyn.ToStructPaths(p), + Locations: locs, + Paths: []*structpath.PathNode{p}, }) } - return v, nil - }) - if err != nil { - return diag.FromErr(err) } return diags diff --git a/bundle/config/validate/required.go b/bundle/config/validate/required.go index 3032eba8073..3de90d4c335 100644 --- a/bundle/config/validate/required.go +++ b/bundle/config/validate/required.go @@ -4,14 +4,15 @@ import ( "cmp" "context" "fmt" + "maps" "slices" "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/internal/validation/generated" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type required struct{} @@ -29,36 +30,28 @@ func warnForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnostic diags := diag.Diagnostics{} // Generate prefix tree for all required fields. - trie := &dyn.TrieNode{} - for k := range generated.RequiredFields { - pattern, err := dyn.NewPatternFromString(k) - if err != nil { - return diag.FromErr(fmt.Errorf("invalid pattern %q for required field validation: %w", k, err)) - } - - err = trie.Insert(pattern) - if err != nil { - return diag.FromErr(fmt.Errorf("failed to insert pattern %q into trie: %w", k, err)) - } + patterns, err := newPatternSet(slices.Collect(maps.Keys(generated.RequiredFields))) + if err != nil { + return diag.FromErr(fmt.Errorf("required field validation: %w", err)) } - err := dyn.WalkReadOnly(b.Config.Value(), func(p dyn.Path, v dyn.Value) error { - // If the path is not found in the prefix tree, we do not need to validate any required + err = structvar.Walk(b.Config.View(), func(np *structpath.PathNode, v structvar.View) error { + // If the path matches no pattern, we do not need to validate any required // fields in it. - pattern, ok := trie.SearchPath(p) + pattern, ok := patterns.find(np) if !ok { return nil } - fields := generated.RequiredFields[pattern.String()] + fields := generated.RequiredFields[pattern] for _, field := range fields { vv := v.Get(field) - if vv.Kind() == dyn.KindInvalid || vv.Kind() == dyn.KindNil { + if vv.Kind() == structvar.KindInvalid || vv.Kind() == structvar.KindNil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("required field %q is not set", field), Locations: v.Locations(), - Paths: dyn.ToStructPaths(p), + Paths: []*structpath.PathNode{np}, }) } } @@ -95,9 +88,9 @@ func sortDiagnostics(diags diag.Diagnostics) { // Bespoke code to error for fields that are not marked as required in the Go SDK / OpenAPI spec. func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // Dashboards should always have a name and warehouse_id. - var nameLocations []dyn.Location + var nameLocations []diag.Location var namePaths []*structpath.PathNode - var warehouseIdLocations []dyn.Location + var warehouseIdLocations []diag.Location var warehouseIdPaths []*structpath.PathNode diags := diag.Diagnostics{} @@ -133,12 +126,12 @@ func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnosti // by the backend, which rejects whitespace-only names (name.trim.nonEmpty). for key, warehouse := range b.Config.Resources.SqlWarehouses { if strings.TrimSpace(warehouse.Name) == "" { - path := "resources.sql_warehouses." + key + path := structpath.NewPath(nil, "resources", "sql_warehouses", key) diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "sql_warehouse name is required", - Locations: b.Config.GetLocations(path), - Paths: structpath.NewPathSlice("resources", "sql_warehouses", key), + Locations: b.Config.GetLocations(path.String()), + Paths: []*structpath.PathNode{path}, }) } } @@ -156,16 +149,16 @@ func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnosti func errorForInvalidGrants(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { diags := diag.Diagnostics{} - _, err := dyn.MapByPattern( - b.Config.Value(), - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("grants"), dyn.AnyIndex()), - func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + err := structvar.ForEach( + b.Config.View(), + structpath.MustParsePattern("resources.*.*.grants[*]"), + func(np *structpath.PathNode, v structvar.View) error { if isMissingOrEmptyString(v.Get("principal")) { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "grant principal is required", Locations: v.Locations(), - Paths: dyn.ToStructPaths(p), + Paths: []*structpath.PathNode{np}, }) } if isMissingOrEmptySequence(v.Get("privileges")) { @@ -173,10 +166,10 @@ func errorForInvalidGrants(ctx context.Context, b *bundle.Bundle) diag.Diagnosti Severity: diag.Error, Summary: "grant privileges is required", Locations: v.Locations(), - Paths: dyn.ToStructPaths(p), + Paths: []*structpath.PathNode{np}, }) } - return v, nil + return nil }, ) if err != nil { @@ -198,14 +191,16 @@ func errorForInvalidSecretScopePermissions(ctx context.Context, b *bundle.Bundle if perm.UserName != "" || perm.GroupName != "" || perm.ServicePrincipalName != "" { continue } + scopePath := structpath.NewPath(nil, "resources", "secret_scopes", key) + path := structpath.NewPath(scopePath, "permissions", i) // ApplyBundlePermissions rebuilds permissions via convert.FromTyped and drops // per-entry locations, so point at the scope. diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "secret scope permission principal is required", Detail: "Set one of user_name, group_name or service_principal_name", - Locations: b.Config.GetLocations("resources.secret_scopes." + key), - Paths: structpath.NewPathSlice("resources", "secret_scopes", key, "permissions", i), + Locations: b.Config.GetLocations(scopePath.String()), + Paths: []*structpath.PathNode{path}, }) } } @@ -216,24 +211,28 @@ func errorForInvalidSecretScopePermissions(ctx context.Context, b *bundle.Bundle } // isMissingOrEmptyString reports whether v is unset, null, or an empty string. -func isMissingOrEmptyString(v dyn.Value) bool { +func isMissingOrEmptyString(v structvar.View) bool { switch v.Kind() { - case dyn.KindInvalid, dyn.KindNil: + case structvar.KindInvalid, structvar.KindNil: return true - case dyn.KindString: - return v.MustString() == "" + case structvar.KindString: + s, _ := v.AsString() + return s == "" default: return false } } // isMissingOrEmptySequence reports whether v is unset, null, or an empty sequence. -func isMissingOrEmptySequence(v dyn.Value) bool { +func isMissingOrEmptySequence(v structvar.View) bool { switch v.Kind() { - case dyn.KindInvalid, dyn.KindNil: + case structvar.KindInvalid, structvar.KindNil: + return true + case structvar.KindSequence: + for range v.Sequence() { + return false + } return true - case dyn.KindSequence: - return len(v.MustSequence()) == 0 default: return false } diff --git a/bundle/config/validate/scripts.go b/bundle/config/validate/scripts.go index 323db769d23..328ba2404b6 100644 --- a/bundle/config/validate/scripts.go +++ b/bundle/config/validate/scripts.go @@ -9,8 +9,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type validateScripts struct{} @@ -38,13 +38,13 @@ func (f *validateScripts) Apply(ctx context.Context, b *bundle.Bundle) diag.Diag for _, k := range scriptKeys { script := b.Config.Scripts[k] - contentPath := dyn.NewPath(dyn.Key("scripts"), dyn.Key(k), dyn.Key("content")) + contentPath := structpath.NewPath(nil, "scripts", k, "content") if script.Content == "" { diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("Script %s has no content", k), - Paths: dyn.ToStructPaths(contentPath), + Paths: []*structpath.PathNode{contentPath}, }) continue } @@ -59,14 +59,13 @@ func (f *validateScripts) Apply(ctx context.Context, b *bundle.Bundle) diag.Diag // validateScriptContent rejects any ${...} reference in a script's content. // Content is passed to the shell as-is, so ${...} would be ambiguous with a // bundle reference; reference a declared env entry with $NAME instead. -func validateScriptContent(b *bundle.Bundle, key, content string, p dyn.Path) diag.Diagnostics { - ref, ok := dynvar.NewRef(dyn.V(content)) +func validateScriptContent(b *bundle.Bundle, key, content string, p *structpath.PathNode) diag.Diagnostics { + ref, ok := structvar.NewRef(content) if !ok { return nil } first := ref.Matches[0][0] - v, _ := dyn.GetByPath(b.Config.Value(), p) return diag.Diagnostics{{ Severity: diag.Error, Summary: fmt.Sprintf("Found %s in script %s.content. Interpolation syntax ${...} is not supported in script content", first, key), @@ -79,8 +78,8 @@ from "content" with $NAME: env: MY_VAR: ${var.foo} content: echo "$MY_VAR"`, - Locations: v.Locations(), - Paths: dyn.ToStructPaths(p), + Locations: b.Config.LocationsAt(p), + Paths: []*structpath.PathNode{p}, }} } @@ -88,13 +87,12 @@ func validateScriptEnv(b *bundle.Bundle, key string, env map[string]string) diag var diags diag.Diagnostics for _, name := range slices.Sorted(maps.Keys(env)) { - ref, ok := dynvar.NewRef(dyn.V(env[name])) + ref, ok := structvar.NewRef(env[name]) if !ok { continue } - envValuePath := dyn.NewPath(dyn.Key("scripts"), dyn.Key(key), dyn.Key("env"), dyn.Key(name)) - v, _ := dyn.GetByPath(b.Config.Value(), envValuePath) + envValuePath := structpath.NewPath(nil, "scripts", key, "env", name) for _, refPath := range ref.References() { prefix, _, _ := strings.Cut(refPath, ".") @@ -104,8 +102,8 @@ func validateScriptEnv(b *bundle.Bundle, key string, env map[string]string) diag diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("${%s} cannot be used in scripts.%s.env.%s; only ${bundle.*}, ${workspace.*}, and ${var.*} are resolved before scripts execute", refPath, key, name), - Locations: v.Locations(), - Paths: dyn.ToStructPaths(envValuePath), + Locations: b.Config.LocationsAt(envValuePath), + Paths: []*structpath.PathNode{envValuePath}, }) } } diff --git a/bundle/config/validate/single_node_cluster.go b/bundle/config/validate/single_node_cluster.go index 88b7536f80b..b5a1355030b 100644 --- a/bundle/config/validate/single_node_cluster.go +++ b/bundle/config/validate/single_node_cluster.go @@ -6,9 +6,9 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) // Validates that any single node clusters defined in the bundle are correctly configured. @@ -36,7 +36,27 @@ are correctly set in the cluster specification: const singleNodeWarningSummary = `Single node cluster is not correctly configured` -func showSingleNodeClusterWarning(ctx context.Context, v dyn.Value) bool { +// stringMap reads a view of a map of strings. It returns false if v is not a map. +func stringMap(v structvar.View) (map[string]string, bool) { + switch v.Kind() { + case structvar.KindInvalid, structvar.KindNil: + return nil, true + case structvar.KindMap: + default: + return nil, false + } + out := map[string]string{} + for k, c := range v.MapItems() { + s, ok := c.AsString() + if !ok { + return nil, false + } + out[k] = s + } + return out, true +} + +func showSingleNodeClusterWarning(ctx context.Context, v structvar.View) bool { // Check if the user has explicitly set the num_workers to 0. Skip the warning // if that's not the case. numWorkers, ok := v.Get("num_workers").AsInt() @@ -51,28 +71,26 @@ func showSingleNodeClusterWarning(ctx context.Context, v dyn.Value) bool { return false } - // Convenient type that contains the common fields from compute.ClusterSpec and - // pipelines.PipelineCluster that we are interested in. - type ClusterConf struct { - SparkConf map[string]string `json:"spark_conf"` - CustomTags map[string]string `json:"custom_tags"` - PolicyId string `json:"policy_id"` + // Read the common fields from compute.ClusterSpec and pipelines.PipelineCluster + // that we are interested in. + sparkConf, ok := stringMap(v.Get("spark_conf")) + if !ok { + return false } - - conf := &ClusterConf{} - err := convert.ToTyped(conf, v) - if err != nil { + customTags, ok := stringMap(v.Get("custom_tags")) + if !ok { return false } + policyId, _ := v.Get("policy_id").AsString() // If the policy id is set, we don't want to show the warning. This is because // the user might have configured `spark_conf` and `custom_tags` correctly // in their cluster policy. - if conf.PolicyId != "" { + if policyId != "" { return false } - profile, ok := conf.SparkConf["spark.databricks.cluster.profile"] + profile, ok := sparkConf["spark.databricks.cluster.profile"] if !ok { log.Debugf(ctx, "spark_conf spark.databricks.cluster.profile not found in single-node cluster spec") return true @@ -82,7 +100,7 @@ func showSingleNodeClusterWarning(ctx context.Context, v dyn.Value) bool { return true } - master, ok := conf.SparkConf["spark.master"] + master, ok := sparkConf["spark.master"] if !ok { log.Debugf(ctx, "spark_conf spark.master not found in single-node cluster spec") return true @@ -92,7 +110,7 @@ func showSingleNodeClusterWarning(ctx context.Context, v dyn.Value) bool { return true } - resourceClass, ok := conf.CustomTags["ResourceClass"] + resourceClass, ok := customTags["ResourceClass"] if !ok { log.Debugf(ctx, "custom_tag ResourceClass not found in single-node cluster spec") return true @@ -108,33 +126,34 @@ func showSingleNodeClusterWarning(ctx context.Context, v dyn.Value) bool { func (m *singleNodeCluster) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { diags := diag.Diagnostics{} - patterns := []dyn.Pattern{ + patterns := []*structpath.PatternNode{ // Interactive clusters - dyn.NewPattern(dyn.Key("resources"), dyn.Key("clusters"), dyn.AnyKey()), + structpath.MustParsePattern("resources.clusters.*"), // Job clusters - dyn.NewPattern(dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), dyn.Key("job_clusters"), dyn.AnyIndex(), dyn.Key("new_cluster")), + structpath.MustParsePattern("resources.jobs.*.job_clusters[*].new_cluster"), // Job task clusters - dyn.NewPattern(dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), dyn.Key("tasks"), dyn.AnyIndex(), dyn.Key("new_cluster")), + structpath.MustParsePattern("resources.jobs.*.tasks[*].new_cluster"), // Job for each task clusters - dyn.NewPattern(dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), dyn.Key("tasks"), dyn.AnyIndex(), dyn.Key("for_each_task"), dyn.Key("task"), dyn.Key("new_cluster")), + structpath.MustParsePattern("resources.jobs.*.tasks[*].for_each_task.task.new_cluster"), // Pipeline clusters - dyn.NewPattern(dyn.Key("resources"), dyn.Key("pipelines"), dyn.AnyKey(), dyn.Key("clusters"), dyn.AnyIndex()), + structpath.MustParsePattern("resources.pipelines.*.clusters[*]"), } + root := b.Config.View() for _, p := range patterns { - _, err := dyn.MapByPattern(b.Config.Value(), p, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + err := structvar.ForEach(root, p, func(np *structpath.PathNode, v structvar.View) error { warning := diag.Diagnostic{ Severity: diag.Warning, Summary: singleNodeWarningSummary, Detail: singleNodeWarningDetail, Locations: v.Locations(), - Paths: dyn.ToStructPaths(p), + Paths: []*structpath.PathNode{np}, } if showSingleNodeClusterWarning(ctx, v) { diags = append(diags, warning) } - return v, nil + return nil }) if err != nil { log.Debugf(ctx, "Error while applying single node cluster validation: %s", err) diff --git a/bundle/config/validate/single_node_cluster_test.go b/bundle/config/validate/single_node_cluster_test.go index f4494ae259c..1c8d79af4f2 100644 --- a/bundle/config/validate/single_node_cluster_test.go +++ b/bundle/config/validate/single_node_cluster_test.go @@ -8,12 +8,12 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func failCases() []struct { @@ -109,20 +109,18 @@ func TestValidateSingleNodeClusterFailForInteractiveClusters(t *testing.T) { }, } - bundletest.SetLocation(b, "resources.clusters.foo", []dyn.Location{{File: "a.yml", Line: 1, Column: 1}}) + bundletest.SetLocation(b, "resources.clusters.foo", []diag.Location{{File: "a.yml", Line: 1, Column: 1}}) // We can't set num_workers to 0 explicitly in the typed configuration. - // Do it on the dyn.Value directly. - bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources.clusters.foo.num_workers", dyn.V(0)) - }) + // Set it explicitly, since zero values are otherwise treated as unset. + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.clusters.foo.num_workers"), 0)) diags := SingleNodeCluster().Apply(ctx, b) assert.Equal(t, diag.Diagnostics{ { Severity: diag.Warning, Summary: singleNodeWarningSummary, Detail: singleNodeWarningDetail, - Locations: []dyn.Location{{File: "a.yml", Line: 1, Column: 1}}, + Locations: []diag.Location{{File: "a.yml", Line: 1, Column: 1}}, Paths: structpath.NewPathSlice("resources", "clusters", "foo"), }, }, diags) @@ -157,13 +155,11 @@ func TestValidateSingleNodeClusterFailForJobClusters(t *testing.T) { }, } - bundletest.SetLocation(b, "resources.jobs.foo.job_clusters[0].new_cluster", []dyn.Location{{File: "b.yml", Line: 1, Column: 1}}) + bundletest.SetLocation(b, "resources.jobs.foo.job_clusters[0].new_cluster", []diag.Location{{File: "b.yml", Line: 1, Column: 1}}) // We can't set num_workers to 0 explicitly in the typed configuration. - // Do it on the dyn.Value directly. - bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources.jobs.foo.job_clusters[0].new_cluster.num_workers", dyn.V(0)) - }) + // Set it explicitly, since zero values are otherwise treated as unset. + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.jobs.foo.job_clusters[0].new_cluster.num_workers"), 0)) diags := SingleNodeCluster().Apply(ctx, b) assert.Equal(t, diag.Diagnostics{ @@ -171,7 +167,7 @@ func TestValidateSingleNodeClusterFailForJobClusters(t *testing.T) { Severity: diag.Warning, Summary: singleNodeWarningSummary, Detail: singleNodeWarningDetail, - Locations: []dyn.Location{{File: "b.yml", Line: 1, Column: 1}}, + Locations: []diag.Location{{File: "b.yml", Line: 1, Column: 1}}, Paths: structpath.MustParsePaths("resources.jobs.foo.job_clusters[0].new_cluster"), }, }, diags) @@ -206,13 +202,11 @@ func TestValidateSingleNodeClusterFailForJobTaskClusters(t *testing.T) { }, } - bundletest.SetLocation(b, "resources.jobs.foo.tasks[0].new_cluster", []dyn.Location{{File: "c.yml", Line: 1, Column: 1}}) + bundletest.SetLocation(b, "resources.jobs.foo.tasks[0].new_cluster", []diag.Location{{File: "c.yml", Line: 1, Column: 1}}) // We can't set num_workers to 0 explicitly in the typed configuration. - // Do it on the dyn.Value directly. - bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources.jobs.foo.tasks[0].new_cluster.num_workers", dyn.V(0)) - }) + // Set it explicitly, since zero values are otherwise treated as unset. + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.jobs.foo.tasks[0].new_cluster.num_workers"), 0)) diags := bundle.Apply(ctx, b, SingleNodeCluster()) assert.Equal(t, diag.Diagnostics{ @@ -220,7 +214,7 @@ func TestValidateSingleNodeClusterFailForJobTaskClusters(t *testing.T) { Severity: diag.Warning, Summary: singleNodeWarningSummary, Detail: singleNodeWarningDetail, - Locations: []dyn.Location{{File: "c.yml", Line: 1, Column: 1}}, + Locations: []diag.Location{{File: "c.yml", Line: 1, Column: 1}}, Paths: structpath.MustParsePaths("resources.jobs.foo.tasks[0].new_cluster"), }, }, diags) @@ -252,13 +246,11 @@ func TestValidateSingleNodeClusterFailForPipelineClusters(t *testing.T) { }, } - bundletest.SetLocation(b, "resources.pipelines.foo.clusters[0]", []dyn.Location{{File: "d.yml", Line: 1, Column: 1}}) + bundletest.SetLocation(b, "resources.pipelines.foo.clusters[0]", []diag.Location{{File: "d.yml", Line: 1, Column: 1}}) // We can't set num_workers to 0 explicitly in the typed configuration. - // Do it on the dyn.Value directly. - bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources.pipelines.foo.clusters[0].num_workers", dyn.V(0)) - }) + // Set it explicitly, since zero values are otherwise treated as unset. + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.pipelines.foo.clusters[0].num_workers"), 0)) diags := bundle.Apply(ctx, b, SingleNodeCluster()) assert.Equal(t, diag.Diagnostics{ @@ -266,7 +258,7 @@ func TestValidateSingleNodeClusterFailForPipelineClusters(t *testing.T) { Severity: diag.Warning, Summary: singleNodeWarningSummary, Detail: singleNodeWarningDetail, - Locations: []dyn.Location{{File: "d.yml", Line: 1, Column: 1}}, + Locations: []diag.Location{{File: "d.yml", Line: 1, Column: 1}}, Paths: structpath.MustParsePaths("resources.pipelines.foo.clusters[0]"), }, }, diags) @@ -305,13 +297,11 @@ func TestValidateSingleNodeClusterFailForJobForEachTaskCluster(t *testing.T) { }, } - bundletest.SetLocation(b, "resources.jobs.foo.tasks[0].for_each_task.task.new_cluster", []dyn.Location{{File: "e.yml", Line: 1, Column: 1}}) + bundletest.SetLocation(b, "resources.jobs.foo.tasks[0].for_each_task.task.new_cluster", []diag.Location{{File: "e.yml", Line: 1, Column: 1}}) // We can't set num_workers to 0 explicitly in the typed configuration. - // Do it on the dyn.Value directly. - bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources.jobs.foo.tasks[0].for_each_task.task.new_cluster.num_workers", dyn.V(0)) - }) + // Set it explicitly, since zero values are otherwise treated as unset. + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.jobs.foo.tasks[0].for_each_task.task.new_cluster.num_workers"), 0)) diags := bundle.Apply(ctx, b, SingleNodeCluster()) assert.Equal(t, diag.Diagnostics{ @@ -319,7 +309,7 @@ func TestValidateSingleNodeClusterFailForJobForEachTaskCluster(t *testing.T) { Severity: diag.Warning, Summary: singleNodeWarningSummary, Detail: singleNodeWarningDetail, - Locations: []dyn.Location{{File: "e.yml", Line: 1, Column: 1}}, + Locations: []diag.Location{{File: "e.yml", Line: 1, Column: 1}}, Paths: structpath.MustParsePaths("resources.jobs.foo.tasks[0].for_each_task.task.new_cluster"), }, }, diags) @@ -392,9 +382,7 @@ func TestValidateSingleNodeClusterPassInteractiveClusters(t *testing.T) { } if tc.numWorkers != nil { - bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources.clusters.foo.num_workers", dyn.V(*tc.numWorkers)) - }) + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.clusters.foo.num_workers"), *tc.numWorkers)) } diags := bundle.Apply(ctx, b, SingleNodeCluster()) @@ -432,9 +420,7 @@ func TestValidateSingleNodeClusterPassJobClusters(t *testing.T) { } if tc.numWorkers != nil { - bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources.jobs.foo.job_clusters[0].new_cluster.num_workers", dyn.V(*tc.numWorkers)) - }) + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.jobs.foo.job_clusters[0].new_cluster.num_workers"), *tc.numWorkers)) } diags := bundle.Apply(ctx, b, SingleNodeCluster()) @@ -472,9 +458,7 @@ func TestValidateSingleNodeClusterPassJobTaskClusters(t *testing.T) { } if tc.numWorkers != nil { - bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources.jobs.foo.tasks[0].new_cluster.num_workers", dyn.V(*tc.numWorkers)) - }) + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.jobs.foo.tasks[0].new_cluster.num_workers"), *tc.numWorkers)) } diags := bundle.Apply(ctx, b, SingleNodeCluster()) @@ -509,9 +493,7 @@ func TestValidateSingleNodeClusterPassPipelineClusters(t *testing.T) { } if tc.numWorkers != nil { - bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources.pipelines.foo.clusters[0].num_workers", dyn.V(*tc.numWorkers)) - }) + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.pipelines.foo.clusters[0].num_workers"), *tc.numWorkers)) } diags := bundle.Apply(ctx, b, SingleNodeCluster()) @@ -553,9 +535,7 @@ func TestValidateSingleNodeClusterPassJobForEachTaskCluster(t *testing.T) { } if tc.numWorkers != nil { - bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources.jobs.foo.tasks[0].for_each_task.task.new_cluster.num_workers", dyn.V(*tc.numWorkers)) - }) + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.jobs.foo.tasks[0].for_each_task.task.new_cluster.num_workers"), *tc.numWorkers)) } diags := bundle.Apply(ctx, b, SingleNodeCluster()) @@ -590,13 +570,8 @@ func TestValidateSingleNodeClusterWithIsSingleNode(t *testing.T) { } // Set num_workers to 0 and is_single_node to true - bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { - v, err := dyn.Set(v, "resources.jobs.foo.job_clusters[0].new_cluster.num_workers", dyn.V(0)) - if err != nil { - return v, err - } - return dyn.Set(v, "resources.jobs.foo.job_clusters[0].new_cluster.is_single_node", dyn.V(true)) - }) + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.jobs.foo.job_clusters[0].new_cluster.num_workers"), 0)) + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.jobs.foo.job_clusters[0].new_cluster.is_single_node"), true)) diags := bundle.Apply(ctx, b, SingleNodeCluster()) assert.Empty(t, diags) diff --git a/bundle/config/validate/tf_only_references.go b/bundle/config/validate/tf_only_references.go index 60eb87e4249..ecbc3d42047 100644 --- a/bundle/config/validate/tf_only_references.go +++ b/bundle/config/validate/tf_only_references.go @@ -8,9 +8,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/terraform_dabs_map" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type tfOnlyReferences struct{} @@ -30,8 +29,12 @@ func (m *tfOnlyReferences) Apply(_ context.Context, b *bundle.Bundle) diag.Diagn var diags diag.Diagnostics // Walk the entire config looking for ${resources.*} references. - _ = dyn.WalkReadOnly(b.Config.Value(), func(_ dyn.Path, v dyn.Value) error { - ref, ok := dynvar.NewRef(v) + _ = structvar.Walk(b.Config.View(), func(_ *structpath.PathNode, v structvar.View) error { + s, ok := v.AsString() + if !ok { + return nil + } + ref, ok := structvar.NewRef(s) if !ok { return nil } @@ -56,32 +59,27 @@ func (m *tfOnlyReferences) Apply(_ context.Context, b *bundle.Bundle) diag.Diagn // checkTFOnlyReference checks a single reference string like // "resources.jobs.src.always_running" and returns a diagnostic when it refers // to a TF-only field, or nil otherwise. -func checkTFOnlyReference(ref string, loc dyn.Location) *diag.Diagnostic { - p, err := dyn.NewPathFromString(ref) +func checkTFOnlyReference(ref string, loc diag.Location) *diag.Diagnostic { + p, err := structpath.ParsePath(ref) // Need at least resources... - if err != nil || len(p) < 4 || p[0].Key() != "resources" { + if err != nil || p.Len() < 4 || p.KeyAt(0) != "resources" { return nil } - group := p[1].Key() + group := p.KeyAt(1) tfOnlyFields, ok := terraform_dabs_map.TerraformOnlyFields[group] if !ok || len(tfOnlyFields) == 0 { return nil } // Field path is everything after resources... - fieldNode, err := structpath.ParsePath(p[3:].String()) - if err != nil { - return nil - } - - if !tfOnlyFields.Contains(fieldNode) { + if !tfOnlyFields.Contains(p.SkipPrefix(3)) { return nil } return &diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("%q: Terraform-only field; cross-resource references to Terraform-only fields are not supported by the direct engine", ref), - Locations: []dyn.Location{loc}, + Locations: []diag.Location{loc}, } } diff --git a/bundle/config/validate/tf_only_references_test.go b/bundle/config/validate/tf_only_references_test.go index 7b994cf864e..a4f5850f8c8 100644 --- a/bundle/config/validate/tf_only_references_test.go +++ b/bundle/config/validate/tf_only_references_test.go @@ -6,9 +6,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -31,9 +30,7 @@ func makeBundle(t *testing.T) *bundle.Bundle { func TestTFOnlyReferences_Error(t *testing.T) { b := makeBundle(t) - bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources.jobs.dst.name", dyn.V("${resources.jobs.src.always_running}")) - }) + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.jobs.dst.name"), "${resources.jobs.src.always_running}")) diags := TFOnlyReferences().Apply(t.Context(), b) require.Len(t, diags, 1) @@ -44,10 +41,8 @@ func TestTFOnlyReferences_Error(t *testing.T) { func TestTFOnlyReferences_NormalReference(t *testing.T) { b := makeBundle(t) - bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { - // "name" is not a TF-only field; no diagnostic expected. - return dyn.Set(v, "resources.jobs.dst.name", dyn.V("${resources.jobs.src.name}")) - }) + // "name" is not a TF-only field; no diagnostic expected. + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.jobs.dst.name"), "${resources.jobs.src.name}")) diags := TFOnlyReferences().Apply(t.Context(), b) assert.Empty(t, diags) @@ -55,10 +50,8 @@ func TestTFOnlyReferences_NormalReference(t *testing.T) { func TestTFOnlyReferences_RenamedField(t *testing.T) { b := makeBundle(t) - bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { - // "git_source[0].branch" is a TF rename (not TF-only), should not error. - return dyn.Set(v, "resources.jobs.dst.name", dyn.V("${resources.jobs.src.git_source[0].branch}")) - }) + // "git_source[0].branch" is a TF rename (not TF-only), should not error. + require.NoError(t, b.Config.Set(structpath.MustParsePath("resources.jobs.dst.name"), "${resources.jobs.src.git_source[0].branch}")) diags := TFOnlyReferences().Apply(t.Context(), b) assert.Empty(t, diags) diff --git a/bundle/config/validate/unique_resource_keys.go b/bundle/config/validate/unique_resource_keys.go index cff5ef3ccff..bced70c6ffc 100644 --- a/bundle/config/validate/unique_resource_keys.go +++ b/bundle/config/validate/unique_resource_keys.go @@ -7,7 +7,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" ) @@ -36,13 +35,13 @@ func (m *uniqueResourceKeys) Apply(ctx context.Context, b *bundle.Bundle) diag.D diags := diag.Diagnostics{} type metadata struct { - locations []dyn.Location + locations []diag.Location paths []*structpath.PathNode } // Maps of key to the paths and locations the resource / script is defined at. resourceAndScriptMetadata := map[string]*metadata{} - addLocationToMetadata := func(k, prefix string, p dyn.Path, v dyn.Value) { + addLocationToMetadata := func(k string, fullPath *structpath.PathNode) { mv, ok := resourceAndScriptMetadata[k] if !ok { mv = &metadata{ @@ -51,46 +50,22 @@ func (m *uniqueResourceKeys) Apply(ctx context.Context, b *bundle.Bundle) diag.D } } - mv.paths = append(mv.paths, dyn.ToStructPath(dyn.NewPath(dyn.Key(prefix)).Append(p...))) - mv.locations = append(mv.locations, v.Locations()...) + mv.paths = append(mv.paths, fullPath) + mv.locations = append(mv.locations, b.Config.LocationsAt(fullPath)...) resourceAndScriptMetadata[k] = mv } // Gather the paths and locations of all resources - rv := b.Config.Value().Get("resources") - if rv.Kind() != dyn.KindInvalid && rv.Kind() != dyn.KindNil { - _, err := dyn.MapByPattern( - rv, - dyn.NewPattern(dyn.AnyKey(), dyn.AnyKey()), - func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - // The key for the resource. Eg: "my_job" for jobs.my_job. - k := p[1].Key() - addLocationToMetadata(k, "resources", p, v) - return v, nil - }, - ) - if err != nil { - return diag.FromErr(err) + for _, group := range b.Config.Resources.AllResources() { + for k := range group.Resources { + addLocationToMetadata(k, structpath.NewPath(nil, "resources", group.Description.PluralName, k)) } } // track locations for all scripts. - sv := b.Config.Value().Get("scripts") - if sv.Kind() != dyn.KindInvalid && sv.Kind() != dyn.KindNil { - _, err := dyn.MapByPattern( - sv, - dyn.NewPattern(dyn.AnyKey()), - func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - // The key for the script. Eg: "my_script" for scripts.my_script. - k := p[0].Key() - addLocationToMetadata(k, "scripts", p, v) - return v, nil - }, - ) - if err != nil { - return diag.FromErr(err) - } + for k := range b.Config.Scripts { + addLocationToMetadata(k, structpath.NewPath(nil, "scripts", k)) } // If duplicate keys are found, report an error. @@ -101,7 +76,7 @@ func (m *uniqueResourceKeys) Apply(ctx context.Context, b *bundle.Bundle) diag.D // Sort the locations and paths for consistent error messages. This helps // with unit testing. - slices.SortFunc(v.locations, func(a, b dyn.Location) int { + slices.SortFunc(v.locations, func(a, b diag.Location) int { if n := cmp.Compare(a.File, b.File); n != 0 { return n } diff --git a/bundle/config/validate/validate_artifact_path.go b/bundle/config/validate/validate_artifact_path.go index a8eac19f3d4..1eaa41518cc 100644 --- a/bundle/config/validate/validate_artifact_path.go +++ b/bundle/config/validate/validate_artifact_path.go @@ -11,9 +11,8 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go/apierr" ) @@ -52,7 +51,7 @@ func extractVolumeFromPath(artifactPath string) (string, string, string, error) return catalogName, schemaName, volumeName, nil } -func findVolumeInBundle(r config.Root, catalogName, schemaName, volumeName string) (dyn.Path, []dyn.Location, bool) { +func findVolumeInBundle(r config.Root, catalogName, schemaName, volumeName string) (*structpath.PathNode, []diag.Location, bool) { volumes := r.Resources.Volumes for k, v := range volumes { if v.CatalogName != catalogName || v.Name != volumeName { @@ -64,13 +63,13 @@ func findVolumeInBundle(r config.Root, catalogName, schemaName, volumeName strin // schema name is interpolated. // We only have to check for ${resources.schemas...} references because any // other valid reference (like ${var.foo}) would have been interpolated by this point. - p, ok := dynvar.PureReferenceToPath(v.SchemaName) - isSchemaDefinedInBundle := ok && p.HasPrefix(dyn.Path{dyn.Key("resources"), dyn.Key("schemas")}) + p, ok := structvar.PureReferenceToPath(v.SchemaName) + isSchemaDefinedInBundle := ok && p.HasPrefix(structpath.MustParsePath("resources.schemas")) if v.SchemaName != schemaName && !isSchemaDefinedInBundle { continue } pathString := "resources.volumes." + k - return dyn.MustPathFromString(pathString), r.GetLocations(pathString), true + return structpath.MustParsePath(pathString), r.GetLocations(pathString), true } return nil, nil, false } @@ -119,7 +118,7 @@ this bundle but which has not been deployed yet. Please first deploy the volume using 'bundle deploy' and then switch over to using it in the artifact_path.`, Locations: slices.Concat(b.Config.GetLocations("workspace.artifact_path"), locations), - Paths: append(structpath.NewPathSlice("workspace", "artifact_path"), dyn.ToStructPath(path)), + Paths: []*structpath.PathNode{structpath.MustParsePath("workspace.artifact_path"), path}, }} } diff --git a/bundle/config/validate/validate_artifact_path_test.go b/bundle/config/validate/validate_artifact_path_test.go index 4f68f645990..778326430e0 100644 --- a/bundle/config/validate/validate_artifact_path_test.go +++ b/bundle/config/validate/validate_artifact_path_test.go @@ -8,7 +8,6 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/experimental/mocks" @@ -38,8 +37,8 @@ func TestValidateArtifactPathWithVolumeInBundle(t *testing.T) { }, } - bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "file", Line: 1, Column: 1}}) - bundletest.SetLocation(b, "resources.volumes.foo", []dyn.Location{{File: "file", Line: 2, Column: 2}}) + bundletest.SetLocation(b, "workspace.artifact_path", []diag.Location{{File: "file", Line: 1, Column: 1}}) + bundletest.SetLocation(b, "resources.volumes.foo", []diag.Location{{File: "file", Line: 2, Column: 2}}) ctx := t.Context() m := mocks.NewMockWorkspaceClient(t) @@ -53,7 +52,7 @@ func TestValidateArtifactPathWithVolumeInBundle(t *testing.T) { assert.Equal(t, diag.Diagnostics{{ Severity: diag.Error, Summary: "volume catalogN.schemaN.volumeN does not exist", - Locations: []dyn.Location{ + Locations: []diag.Location{ {File: "file", Line: 1, Column: 1}, {File: "file", Line: 2, Column: 2}, }, @@ -77,13 +76,13 @@ func TestValidateArtifactPath(t *testing.T) { }, } - bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "file", Line: 1, Column: 1}}) + bundletest.SetLocation(b, "workspace.artifact_path", []diag.Location{{File: "file", Line: 1, Column: 1}}) assertDiags := func(t *testing.T, diags diag.Diagnostics, expected string) { assert.Len(t, diags, 1) assert.Equal(t, diag.Diagnostics{{ Severity: diag.Error, Summary: expected, - Locations: []dyn.Location{{File: "file", Line: 1, Column: 1}}, + Locations: []diag.Location{{File: "file", Line: 1, Column: 1}}, Paths: structpath.NewPathSlice("workspace", "artifact_path"), }}, diags) } @@ -164,13 +163,13 @@ func TestValidateArtifactPathWithInvalidPaths(t *testing.T) { }, } - bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}) + bundletest.SetLocation(b, "workspace.artifact_path", []diag.Location{{File: "config.yml", Line: 1, Column: 2}}) diags := ValidateArtifactPath().Apply(t.Context(), b) require.Equal(t, diag.Diagnostics{{ Severity: diag.Error, Summary: "expected UC volume path to be in the format /Volumes////..., got " + p, - Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}, + Locations: []diag.Location{{File: "config.yml", Line: 1, Column: 2}}, Paths: structpath.NewPathSlice("workspace", "artifact_path"), }}, diags) } @@ -193,7 +192,7 @@ func TestFindVolumeInBundle(t *testing.T) { }, } - bundletest.SetLocation(b, "resources.volumes.foo", []dyn.Location{ + bundletest.SetLocation(b, "resources.volumes.foo", []diag.Location{ { File: "volume.yml", Line: 1, @@ -204,12 +203,12 @@ func TestFindVolumeInBundle(t *testing.T) { // volume is in DAB. path, locations, ok := findVolumeInBundle(b.Config, "main", "my_schema", "my_volume") assert.True(t, ok) - assert.Equal(t, []dyn.Location{{ + assert.Equal(t, []diag.Location{{ File: "volume.yml", Line: 1, Column: 2, }}, locations) - assert.Equal(t, dyn.MustPathFromString("resources.volumes.foo"), path) + assert.Equal(t, structpath.MustParsePath("resources.volumes.foo"), path) // wrong volume name _, _, ok = findVolumeInBundle(b.Config, "main", "my_schema", "doesnotexist") @@ -233,10 +232,10 @@ func TestFindVolumeInBundle(t *testing.T) { b.Config.Resources.Volumes["foo"].SchemaName = "${resources.schemas.my_schema.name}" path, locations, ok = findVolumeInBundle(b.Config, "main", "valuedoesnotmatter", "my_volume") assert.True(t, ok) - assert.Equal(t, []dyn.Location{{ + assert.Equal(t, []diag.Location{{ File: "volume.yml", Line: 1, Column: 2, }}, locations) - assert.Equal(t, dyn.MustPathFromString("resources.volumes.foo"), path) + assert.Equal(t, structpath.MustParsePath("resources.volumes.foo"), path) } diff --git a/bundle/config/validate/validate_deployment_fields.go b/bundle/config/validate/validate_deployment_fields.go index 720efc78512..10d95644d13 100644 --- a/bundle/config/validate/validate_deployment_fields.go +++ b/bundle/config/validate/validate_deployment_fields.go @@ -26,29 +26,29 @@ func (v *validateDeploymentFields) Apply(_ context.Context, b *bundle.Bundle) di // deployment_id and version_id identify the bundle deployment and its version // in the Deployment Metadata Service. The CLI sets them on every deploy, so a // value provided by hand would be overwritten; reject it up front. - reject := func(resourceType, name, field, value string) { + reject := func(resource *structpath.PathNode, field, value string) { if value == "" { return } - path := "resources." + resourceType + "." + name + ".deployment." + field + path := structpath.NewPath(resource, "deployment", field) diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: field + " must not be set in bundle configuration; it is managed by Declarative Automation Bundles", - Paths: structpath.NewPathSlice("resources", resourceType, name, "deployment", field), - Locations: b.Config.GetLocations(path), + Paths: []*structpath.PathNode{path}, + Locations: b.Config.GetLocations(path.String()), }) } for name, job := range b.Config.Resources.Jobs { if d := job.Deployment; d != nil { - reject("jobs", name, "deployment_id", d.DeploymentId) - reject("jobs", name, "version_id", d.VersionId) + reject(structpath.NewPath(nil, "resources", "jobs", name), "deployment_id", d.DeploymentId) + reject(structpath.NewPath(nil, "resources", "jobs", name), "version_id", d.VersionId) } } for name, pipeline := range b.Config.Resources.Pipelines { if d := pipeline.Deployment; d != nil { - reject("pipelines", name, "deployment_id", d.DeploymentId) - reject("pipelines", name, "version_id", d.VersionId) + reject(structpath.NewPath(nil, "resources", "pipelines", name), "deployment_id", d.DeploymentId) + reject(structpath.NewPath(nil, "resources", "pipelines", name), "version_id", d.VersionId) } } diff --git a/bundle/config/validate/validate_engine.go b/bundle/config/validate/validate_engine.go index 8f6020772ed..11f7b106b87 100644 --- a/bundle/config/validate/validate_engine.go +++ b/bundle/config/validate/validate_engine.go @@ -7,7 +7,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" ) @@ -26,7 +25,8 @@ func (v *validateEngine) Name() string { func (v *validateEngine) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics - if tf := dyn.GetValue(b.Config.Value(), "bundle.terraform"); tf.IsValid() { + tf := b.Config.View().Lookup(structpath.MustParsePath("bundle.terraform")) + if tf.IsValid() { diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: "bundle.terraform is deprecated and has no effect: " + engine.TerraformRemovedSummary, @@ -40,14 +40,14 @@ func (v *validateEngine) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos return diags } - loc := dyn.GetValue(b.Config.Value(), "bundle.engine").Location() + loc := b.Config.GetLocation("bundle.engine") parsed, ok := engine.Parse(string(configEngine)) if !ok { return diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("invalid value %q for bundle.engine (expected %q)", configEngine, engine.EngineDirect), - Locations: []dyn.Location{loc}, + Locations: []diag.Location{loc}, }) } @@ -60,7 +60,7 @@ func (v *validateEngine) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos Severity: severity, Summary: engine.TerraformRemovedSummary, Detail: engine.TerraformRemovedConfigDetail, - Locations: []dyn.Location{loc}, + Locations: []diag.Location{loc}, }) } diff --git a/bundle/config/validate/validate_engine_test.go b/bundle/config/validate/validate_engine_test.go index f74cb002650..da1bd7bf12d 100644 --- a/bundle/config/validate/validate_engine_test.go +++ b/bundle/config/validate/validate_engine_test.go @@ -8,7 +8,6 @@ import ( "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/stretchr/testify/assert" ) @@ -20,7 +19,7 @@ func TestValidateEngineDirect(t *testing.T) { }, }, } - bundletest.SetLocation(b, "bundle.engine", []dyn.Location{{File: "databricks.yml", Line: 5, Column: 3}}) + bundletest.SetLocation(b, "bundle.engine", []diag.Location{{File: "databricks.yml", Line: 5, Column: 3}}) diags := ValidateEngine().Apply(t.Context(), b) assert.Empty(t, diags) } @@ -33,13 +32,13 @@ func TestValidateEngineTerraformRemoved(t *testing.T) { }, }, } - loc := dyn.Location{File: "databricks.yml", Line: 5, Column: 3} - bundletest.SetLocation(b, "bundle.engine", []dyn.Location{loc}) + loc := diag.Location{File: "databricks.yml", Line: 5, Column: 3} + bundletest.SetLocation(b, "bundle.engine", []diag.Location{loc}) diags := ValidateEngine().Apply(t.Context(), b) assert.Len(t, diags, 1) assert.Equal(t, diag.Error, diags[0].Severity) assert.Contains(t, diags[0].Summary, "has been removed") - assert.Equal(t, []dyn.Location{loc}, diags[0].Locations) + assert.Equal(t, []diag.Location{loc}, diags[0].Locations) } func TestValidateEngineNotSet(t *testing.T) { @@ -58,7 +57,7 @@ func TestValidateEngineInvalid(t *testing.T) { }, }, } - bundletest.SetLocation(b, "bundle.engine", []dyn.Location{{File: "databricks.yml", Line: 5, Column: 3}}) + bundletest.SetLocation(b, "bundle.engine", []diag.Location{{File: "databricks.yml", Line: 5, Column: 3}}) diags := ValidateEngine().Apply(t.Context(), b) assert.Len(t, diags, 1) assert.Equal(t, diag.Error, diags[0].Severity) @@ -73,13 +72,13 @@ func TestValidateEngineTerraformConfigDeprecated(t *testing.T) { }, }, } - loc := dyn.Location{File: "databricks.yml", Line: 3, Column: 5} - bundletest.SetLocation(b, "bundle.terraform", []dyn.Location{loc}) + loc := diag.Location{File: "databricks.yml", Line: 3, Column: 5} + bundletest.SetLocation(b, "bundle.terraform", []diag.Location{loc}) diags := ValidateEngine().Apply(t.Context(), b) assert.Len(t, diags, 1) assert.Equal(t, diag.Warning, diags[0].Severity) assert.Contains(t, diags[0].Summary, "bundle.terraform is deprecated") - assert.Equal(t, []dyn.Location{loc}, diags[0].Locations) + assert.Equal(t, []diag.Location{loc}, diags[0].Locations) } func TestValidateEngineTerraformAllowed(t *testing.T) { @@ -91,7 +90,7 @@ func TestValidateEngineTerraformAllowed(t *testing.T) { }, AllowTerraformEngineConfig: true, } - bundletest.SetLocation(b, "bundle.engine", []dyn.Location{{File: "databricks.yml", Line: 5, Column: 3}}) + bundletest.SetLocation(b, "bundle.engine", []diag.Location{{File: "databricks.yml", Line: 5, Column: 3}}) diags := ValidateEngine().Apply(t.Context(), b) assert.Len(t, diags, 1) assert.Equal(t, diag.Warning, diags[0].Severity) diff --git a/bundle/config/validate/validate_job_run_idempotency_token.go b/bundle/config/validate/validate_job_run_idempotency_token.go index 1405028c8cf..4ed6cc75694 100644 --- a/bundle/config/validate/validate_job_run_idempotency_token.go +++ b/bundle/config/validate/validate_job_run_idempotency_token.go @@ -32,12 +32,12 @@ func (v *validateJobRunIdempotencyToken) Apply(_ context.Context, b *bundle.Bund } // The CLI mints the token; a configured one would also remain reserved after // the run is deleted and break the next deploy. - path := "resources.job_runs." + name + ".idempotency_token" + path := structpath.NewPath(nil, "resources", "job_runs", name, "idempotency_token") diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: "idempotency_token must not be set in bundle configuration; the CLI sets it on each run-now request", - Paths: structpath.NewPathSlice("resources", "job_runs", name, "idempotency_token"), - Locations: b.Config.GetLocations(path), + Paths: []*structpath.PathNode{path}, + Locations: b.Config.GetLocations(path.String()), }) } diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index e42f0a1d83a..aeb593afe50 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -8,10 +8,9 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structdiff" + "github.com/databricks/cli/libs/structs/structvar" ) type OperationType string @@ -39,13 +38,8 @@ type ResourceChanges map[string]*ConfigChangeDesc type Changes map[string]ResourceChanges -func normalizeValue(v any) (any, error) { - dynValue, err := convert.FromTyped(v, dyn.NilValue) - if err != nil { - return nil, fmt.Errorf("failed to convert value of type %T: %w", v, err) - } - - return dynValue.AsAny(), nil +func normalizeValue(v any) any { + return structvar.NewView(v, nil, nil).AsAny() } func filterEntityDefaults(basePath string, value any) any { @@ -92,10 +86,7 @@ func convertChangeDesc(path string, cd *deployplan.ChangeDesc) (*ConfigChangeDes // cd.Old in this check would classify the change as Replace and fail later in // resolveSelectors because the old key no longer exists in the YAML. hasConfigValue := cd.New != nil - normalizedValue, err := normalizeValue(cd.Remote) - if err != nil { - return nil, fmt.Errorf("failed to normalize remote value: %w", err) - } + normalizedValue := normalizeValue(cd.Remote) if shouldSkipField(path, normalizedValue, hasConfigValue) { return &ConfigChangeDesc{ @@ -129,11 +120,7 @@ func convertChangeDesc(path string, cd *deployplan.ChangeDesc) (*ConfigChangeDes // which keys on the path component (index 3), so a resource literally named // "permissions" ("resources.jobs.permissions") is not misclassified. func isPermissionsOrGrantsSubResource(resourceKey string) bool { - path, err := dyn.NewPathFromString(resourceKey) - if err != nil { - return false - } - _, nodeType := config.GetNodeAndType(path) + nodeType := config.GetResourceTypeFromKey(resourceKey) return strings.HasSuffix(nodeType, ".permissions") || strings.HasSuffix(nodeType, ".grants") } diff --git a/bundle/configsync/resolve.go b/bundle/configsync/resolve.go index cdad5182336..abd74295216 100644 --- a/bundle/configsync/resolve.go +++ b/bundle/configsync/resolve.go @@ -12,10 +12,10 @@ import ( "strings" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/notebook" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type FieldChange struct { @@ -42,7 +42,7 @@ type resolvedChange struct { path *structpath.PatternNode steps []sequenceStep // leaf is the merged value the change addresses, invalid for a new field. - leaf dyn.Value + leaf structvar.View // operation decides how many destinations the change needs: a removal has to // reach every definition, an edit only the one that wins the merge. operation OperationType @@ -62,28 +62,23 @@ func resolveSelectors(pathStr string, b *bundle.Bundle, operation OperationType) nodes := node.AsSlice() var result *structpath.PatternNode var steps []sequenceStep - var currentPath dyn.Path - currentValue := b.Config.Value() + var currentPath *structpath.PathNode + currentValue := b.Config.View() for component, n := range nodes { if key, ok := n.StringKey(); ok { result = structpath.NewPatternStringKey(result, key) - currentPath = append(currentPath, dyn.Key(key)) - if currentValue.IsValid() { - currentValue, _ = dyn.GetByPath(currentValue, dyn.Path{dyn.Key(key)}) - } + currentPath = structpath.NewStringKey(currentPath, key) + currentValue = currentValue.Get(key) continue } if idx, ok := n.Index(); ok { - sequencePath := slices.Clone(currentPath) + sequencePath := currentPath result = structpath.NewPatternIndex(result, idx) - currentPath = append(currentPath, dyn.Index(idx)) - var element dyn.Value - if currentValue.IsValid() { - element, _ = dyn.GetByPath(currentValue, dyn.Path{dyn.Index(idx)}) - currentValue = element - } + currentPath = structpath.NewIndex(currentPath, idx) + element := currentValue.Index(idx) + currentValue = element steps = append(steps, sequenceStep{ component: component, sequencePath: sequencePath, @@ -98,26 +93,22 @@ func resolveSelectors(pathStr string, b *bundle.Bundle, operation OperationType) // Check for key-value selector: [key='value'] if key, value, ok := n.KeyValue(); ok { - if !currentValue.IsValid() || currentValue.Kind() != dyn.KindSequence { + if !currentValue.IsValid() || currentValue.Kind() != structvar.KindSequence { return resolvedChange{}, fmt.Errorf("cannot apply [%s='%s'] selector to non-array value in path %s", key, value, pathStr) } - seq, _ := currentValue.AsSequence() foundIndex := -1 + var foundElement structvar.View - for i, elem := range seq { - keyValue, err := dyn.GetByPath(elem, dyn.Path{dyn.Key(key)}) - if err != nil { - continue - } - - if keyValue.Kind() == dyn.KindString && keyValue.MustString() == value { + for i, elem := range currentValue.Sequence() { + if keyValue, ok := elem.Get(key).AsString(); ok && keyValue == value { foundIndex = i + foundElement = elem break } } - sequencePath := slices.Clone(currentPath) + sequencePath := currentPath if foundIndex == -1 { if operation == OperationAdd { @@ -128,20 +119,20 @@ func resolveSelectors(pathStr string, b *bundle.Bundle, operation OperationType) newElement: true, }) // Can't navigate further into non-existent element - currentValue = dyn.Value{} + currentValue = structvar.View{} continue } return resolvedChange{}, fmt.Errorf("no array element found with %s='%s' in path %s", key, value, pathStr) } result = structpath.NewPatternIndex(result, foundIndex) - currentPath = append(currentPath, dyn.Index(foundIndex)) + currentPath = structpath.NewIndex(currentPath, foundIndex) steps = append(steps, sequenceStep{ component: component, sequencePath: sequencePath, - element: seq[foundIndex], + element: foundElement, }) - currentValue = seq[foundIndex] + currentValue = foundElement continue } } @@ -200,7 +191,7 @@ func adjustArrayIndex(path *structpath.PatternNode, scope string, operations map // disappears looks identical to one that was written. preResolved is the merged // config with ${...} references still literal, used to detect variable-reference // parents. -func ResolveChanges(ctx context.Context, b *bundle.Bundle, configChanges Changes, preResolved dyn.Value) ([]FieldChange, int, error) { +func ResolveChanges(ctx context.Context, b *bundle.Bundle, configChanges Changes, preResolved structvar.View) ([]FieldChange, int, error) { var result []FieldChange skipped := 0 targetName := b.Config.Bundle.Target @@ -360,7 +351,7 @@ func ResolveChanges(ctx context.Context, b *bundle.Bundle, configChanges Changes // on it belongs in that same block. filePath = block.file } else { - resourceLocation := b.Config.GetLocation(resourceKey) + resourceLocation := b.Config.DefinitionLocation(resourceKey) filePath = resourceLocation.File } if filePath == "" { diff --git a/bundle/configsync/sequences.go b/bundle/configsync/sequences.go index d0b6f599c72..3a9bd84dd1e 100644 --- a/bundle/configsync/sequences.go +++ b/bundle/configsync/sequences.go @@ -11,9 +11,10 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) // Write-back has to turn a position in the merged configuration into a position in a @@ -55,15 +56,15 @@ func (b sourceBlock) scopeKey() string { } // blockResolver answers which physical block a merged value came from, by location: -// merging accumulates them (libs/dyn/merge), so a value assembled from two blocks +// merging accumulates them, so a value assembled from two blocks // reports a location in each. Selecting a target folds its overrides into resources and // drops the targets subtree, so the blocks are recovered by parsing the files again. type blockResolver struct { // blocks holds one parsed file per contributing file, keyed by block, so a // path relative to a block can be looked up inside it. - blocks map[sourceBlock]dyn.Value + blocks map[sourceBlock]structvar.View target string - byLocation map[dyn.Location]sourceBlock + byLocation map[diag.Location]sourceBlock } // newBlockResolver builds the location -> block mapping for the bundle's resources, @@ -73,15 +74,12 @@ type blockResolver struct { // The contributing files are parsed directly rather than reloaded through the mutator // pipeline, which would run the bundle's preinit script a second time. func newBlockResolver(ctx context.Context, b *bundle.Bundle) *blockResolver { - root := b.Config.Value() - if !root.IsValid() { - return nil - } + root := b.Config.View() r := &blockResolver{ - blocks: make(map[sourceBlock]dyn.Value), + blocks: make(map[sourceBlock]structvar.View), target: b.Config.Bundle.Target, - byLocation: make(map[dyn.Location]sourceBlock), + byLocation: make(map[diag.Location]sourceBlock), } sourceFiles := slices.Sorted(maps.Keys(referencedFiles(root))) @@ -91,14 +89,15 @@ func newBlockResolver(ctx context.Context, b *bundle.Bundle) *blockResolver { log.Debugf(ctx, "config-remote-sync: cannot read %s, treating its sequences as unsplit: %v", file, err) continue } - parsed, diags := config.LoadFromBytes(file, contents) + loaded, diags := config.LoadFromBytes(file, contents) if diags.HasError() { log.Debugf(ctx, "config-remote-sync: cannot parse %s, treating its sequences as unsplit: %v", file, diags.Error()) continue } - r.registerBlock(parsed.Value(), sourceBlock{file: file}) + parsed := loaded.View() + r.registerBlock(parsed, sourceBlock{file: file}) if r.target != "" { - r.registerBlock(parsed.Value(), sourceBlock{override: true, file: file}) + r.registerBlock(parsed, sourceBlock{override: true, file: file}) } } @@ -108,9 +107,9 @@ func newBlockResolver(ctx context.Context, b *bundle.Bundle) *blockResolver { return r } -func referencedFiles(root dyn.Value) map[string]struct{} { +func referencedFiles(root structvar.View) map[string]struct{} { files := map[string]struct{}{} - _ = dyn.WalkReadOnly(root, func(_ dyn.Path, v dyn.Value) error { + _ = structvar.Walk(root, func(_ *structpath.PathNode, v structvar.View) error { for _, location := range v.Locations() { if location.File != "" { files[location.File] = struct{}{} @@ -125,16 +124,16 @@ func referencedFiles(root dyn.Value) map[string]struct{} { // resources subtree back to it, so a merged value can later be traced to the // region it was written in. Does nothing when the file has no such region, which // is why r.blocks doubles as the set of blocks that exist. -func (r *blockResolver) registerBlock(parsed dyn.Value, block sourceBlock) { - subtree, err := dyn.GetByPath(parsed, r.regionPath(block, dyn.NewPath(dyn.Key("resources")))) - if err != nil { +func (r *blockResolver) registerBlock(parsed structvar.View, block sourceBlock) { + subtree := parsed.Lookup(r.regionPath(block, structpath.NewStringKey(nil, "resources"))) + if !subtree.IsValid() { return } // Keep the parsed file: resolving a path inside this block needs the tree it // came from, and the entry also marks the block as present for sortedBlocks. r.blocks[block] = parsed - _ = dyn.WalkReadOnly(subtree, func(_ dyn.Path, v dyn.Value) error { + _ = structvar.Walk(subtree, func(_ *structpath.PathNode, v structvar.View) error { for _, location := range v.Locations() { // A file only carries locations of its own, so a location seen here // belongs to this block. First writer wins: an outer node accumulates @@ -152,11 +151,11 @@ func (r *blockResolver) registerBlock(parsed dyn.Value, block sourceBlock) { } // regionPath prefixes a resources-relative path with the region it belongs to. -func (r *blockResolver) regionPath(block sourceBlock, path dyn.Path) dyn.Path { +func (r *blockResolver) regionPath(block sourceBlock, path *structpath.PathNode) *structpath.PathNode { if !block.override { return path } - return append(dyn.NewPath(dyn.Key("targets"), dyn.Key(r.target)), path...) + return structpath.Join(structpath.NewPath(nil, "targets", r.target), path.AsSlice()...) } // candidatePath renders a resolved path the way the patch layer addresses it, @@ -210,7 +209,7 @@ func compareBlocks(a, b sourceBlock) int { // blocksOf returns the distinct blocks that contributed to value, sorted with the // top-level block first. More than one result means the value is assembled from // several regions and has no single source location. -func (r *blockResolver) blocksOf(value dyn.Value) []sourceBlock { +func (r *blockResolver) blocksOf(value structvar.View) []sourceBlock { var blocks []sourceBlock for _, location := range value.Locations() { block, ok := r.byLocation[location] @@ -232,7 +231,7 @@ func (r *blockResolver) blocksOf(value dyn.Value) []sourceBlock { // Locations accumulate in merge order, so the first one that maps to a block is the // winner and writing any other copy would leave the effective value unchanged. Load // order is the only thing that distinguishes two blocks in the same scope. -func (r *blockResolver) winningBlock(value dyn.Value) (sourceBlock, bool) { +func (r *blockResolver) winningBlock(value structvar.View) (sourceBlock, bool) { for _, location := range value.Locations() { if block, ok := r.byLocation[location]; ok { return block, true @@ -244,26 +243,22 @@ func (r *blockResolver) winningBlock(value dyn.Value) (sourceBlock, bool) { // indexWithinBlock returns the position of element inside block, where sequencePath is // relative to the block. A block is one parsed file, so the sequence read here holds // only that file's entries and a plain index into it is block-local. -func (r *blockResolver) indexWithinBlock(block sourceBlock, sequencePath dyn.Path, element dyn.Value) (int, bool) { +func (r *blockResolver) indexWithinBlock(block sourceBlock, sequencePath *structpath.PathNode, element structvar.View) (int, bool) { parsed, ok := r.blocks[block] if !ok { return 0, false } - sequence, err := dyn.GetByPath(parsed, r.regionPath(block, sequencePath)) - if err != nil { - return 0, false - } - entries, ok := sequence.AsSequence() - if !ok { + sequence := parsed.Lookup(r.regionPath(block, sequencePath)) + if sequence.Kind() != structvar.KindSequence { return 0, false } - locations := make(map[dyn.Location]struct{}, len(element.Locations())) + locations := make(map[diag.Location]struct{}, len(element.Locations())) for _, location := range element.Locations() { locations[location] = struct{}{} } - for local, entry := range entries { + for local, entry := range sequence.Sequence() { for _, location := range entry.Locations() { if _, ok := locations[location]; ok { return local, true @@ -322,7 +317,7 @@ func (d routeDestination) scopeKey() string { func (r *blockResolver) routeDestinations(change resolvedChange) ([]routeDestination, error) { // The value whose definitions have to be reached: the element itself when the // change addresses one, otherwise the field being removed. - var target dyn.Value + var target structvar.View if addressesWholeElement(change) { target = change.steps[len(change.steps)-1].element } else if change.operation == OperationRemove { @@ -488,18 +483,18 @@ func (r *blockResolver) blockForNewElement(change resolvedChange) (sourceBlock, // blocksDefiningSequence returns the blocks that write the sequence at sequencePath. // This is for a value that does not exist yet, where only the receiving sequence is // known; an existing value is traced through its own locations instead. -func (r *blockResolver) blocksDefiningSequence(change resolvedChange, sequencePath dyn.Path) []sourceBlock { +func (r *blockResolver) blocksDefiningSequence(change resolvedChange, sequencePath *structpath.PathNode) []sourceBlock { var blocks []sourceBlock for _, block := range r.sortedBlocks() { blockPath, ok := r.sequencePathWithinBlock(block, change, sequencePath) if !ok { continue } - sequence, err := dyn.GetByPath(r.blocks[block], r.regionPath(block, blockPath)) - if err != nil { + sequence := r.blocks[block].Lookup(r.regionPath(block, blockPath)) + if !sequence.IsValid() { continue } - if slices.ContainsFunc(sequence.Locations(), func(l dyn.Location) bool { return l.File == block.file }) { + if slices.ContainsFunc(sequence.Locations(), func(l diag.Location) bool { return l.File == block.file }) { blocks = append(blocks, block) } } @@ -512,22 +507,22 @@ func (r *blockResolver) blocksDefiningSequence(change resolvedChange, sequencePa // // change.steps is ordered outermost first, so each translated index is already known // by the time a deeper step needs it. -func (r *blockResolver) sequencePathWithinBlock(block sourceBlock, change resolvedChange, sequencePath dyn.Path) (dyn.Path, bool) { - result := slices.Clone(sequencePath) +func (r *blockResolver) sequencePathWithinBlock(block sourceBlock, change resolvedChange, sequencePath *structpath.PathNode) (*structpath.PathNode, bool) { + result := sequencePath.AsSlice() for _, step := range change.steps { - at := len(step.sequencePath) - if step.newElement || at >= len(sequencePath) { + at := step.sequencePath.Len() + if step.newElement || at >= len(result) { continue } // The step's own path is a prefix of sequencePath, so the indices translated // so far already apply to it. - index, ok := r.indexWithinBlock(block, result[:at], step.element) + index, ok := r.indexWithinBlock(block, structpath.Join(nil, result[:at]...), step.element) if !ok { return nil, false } - result[at] = dyn.Index(index) + result[at] = structpath.NewIndex(nil, index) } - return result, true + return structpath.Join(nil, result...), true } // pathWithinBlock rewrites a change's path so every sequence index addresses the @@ -581,8 +576,8 @@ type sequenceStep struct { // component is how many path components precede this sequence's index. component int // sequencePath is the sequence itself, e.g. resources.jobs.j.tasks. - sequencePath dyn.Path - element dyn.Value + sequencePath *structpath.PathNode + element structvar.View // newElement marks an Add whose key is not in the merged sequence yet. newElement bool } diff --git a/bundle/configsync/telemetry_test.go b/bundle/configsync/telemetry_test.go index d2a3da537f8..001ecb2f9d2 100644 --- a/bundle/configsync/telemetry_test.go +++ b/bundle/configsync/telemetry_test.go @@ -3,7 +3,6 @@ package configsync import ( "testing" - "github.com/databricks/cli/libs/dyn" "github.com/stretchr/testify/assert" ) @@ -63,37 +62,30 @@ func TestResourceTypeFromKey(t *testing.T) { } func TestRestoreStatsCounters(t *testing.T) { - resolved := dyn.V(map[string]dyn.Value{ - "variables": dyn.V(map[string]dyn.Value{ - "region": dyn.V(map[string]dyn.Value{"value": dyn.V("us-east-1")}), - "other": dyn.V(map[string]dyn.Value{"value": dyn.V("eu-west-1")}), - }), - }) + resolved := variablesConfig(t, map[string]any{"region": "us-east-1", "other": "eu-west-1"}) // Original pure ref still matching: restored but not counted (safe path). var kept RestoreStats - result := restoreOriginalRefs("us-east-1", dyn.V("${var.region}"), resolved, &kept) + result := restoreOriginalRefs("us-east-1", scalarView(t, "${var.region}"), resolved, &kept) assert.Equal(t, "${var.region}", result) assert.Equal(t, RestoreStats{}, kept) // Pure ref whose value changed to another variable's value: re-targeted. var retargeted RestoreStats - result = restoreOriginalRefs("eu-west-1", dyn.V("${var.region}"), resolved, &retargeted) + result = restoreOriginalRefs("eu-west-1", scalarView(t, "${var.region}"), resolved, &retargeted) assert.Equal(t, "${var.other}", result) assert.Equal(t, RestoreStats{Retargeted: 1}, retargeted) // New sequence element leaf restored from a sibling reference. - siblings := []dyn.Value{ - dyn.V(map[string]dyn.Value{"region": dyn.V("${var.region}")}), - } + siblings := tasksView(t, `task_key: "${var.region}"`) var fromSiblings RestoreStats - resultMap := restoreFromSiblings(map[string]any{"region": "us-east-1"}, siblings, resolved, &fromSiblings).(map[string]any) - assert.Equal(t, "${var.region}", resultMap["region"]) + resultMap := restoreFromSiblings(map[string]any{"task_key": "us-east-1"}, siblings, resolved, &fromSiblings).(map[string]any) + assert.Equal(t, "${var.region}", resultMap["task_key"]) assert.Equal(t, RestoreStats{FromSiblings: 1}, fromSiblings) // Hardcoded value: nothing restored, nothing counted. var none RestoreStats - result = restoreOriginalRefs("hardcoded", dyn.V("hardcoded"), resolved, &none) + result = restoreOriginalRefs("hardcoded", scalarView(t, "hardcoded"), resolved, &none) assert.Equal(t, "hardcoded", result) assert.Equal(t, RestoreStats{}, none) } diff --git a/bundle/configsync/variables.go b/bundle/configsync/variables.go index 442224158f5..1030a912719 100644 --- a/bundle/configsync/variables.go +++ b/bundle/configsync/variables.go @@ -9,20 +9,19 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/mutator/resourcemutator" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -// varPrefix is the dyn.Path prefix for the ${var.X} shorthand. -var varPrefix = dyn.NewPath(dyn.Key("var")) +// varPrefix is the path prefix for the ${var.X} shorthand. +var varPrefix = structpath.NewStringKey(nil, "var") // RestoreVariableReferences replaces hardcoded change values with variable // references (${var.foo}, ${bundle.target}, ${resources.X.Y.id}) when the // value can be traced back to a reference in the original YAML. Resource IDs // are injected from state since they aren't materialized into the resolved -// config's dyn.Value tree. +// config's tree. // // For Replace operations, restoration consults the pre-resolved YAML at the // exact field position and tries three steps in order: @@ -52,11 +51,11 @@ var varPrefix = dyn.NewPath(dyn.Key("var")) // Restoration counts by mechanism are accumulated into stats (used for // telemetry); pass nil when counters are not needed (the counter methods are // nil-safe). -func RestoreVariableReferences(ctx context.Context, b *bundle.Bundle, fieldChanges []FieldChange, preResolved dyn.Value, stats *RestoreStats) error { +func RestoreVariableReferences(ctx context.Context, b *bundle.Bundle, fieldChanges []FieldChange, preResolved structvar.View, stats *RestoreStats) error { if !preResolved.IsValid() { return errors.New("pre-resolved config unavailable; variable-backed fields will be hardcoded") } - resolved := b.Config.Value() + resolved := resolvedConfig{view: b.Config.View(), overrides: map[string]any{}} // Mirror mutator.lookup's source-linked deployment override: when enabled, // ${workspace.file_path} resolves to b.SyncRootPath rather than the typed @@ -65,21 +64,18 @@ func RestoreVariableReferences(ctx context.Context, b *bundle.Bundle, fieldChang // actual deployed path and variables are lost on Replace. Keep this in // sync with mutator.lookup if new overrides are added there. if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) { - fpPath := dyn.NewPath(dyn.Key("workspace"), dyn.Key("file_path")) - if updated, err := dyn.SetByPath(resolved, fpPath, dyn.V(b.SyncRootPath)); err == nil { - resolved = updated - } + resolved.overrides["workspace.file_path"] = b.SyncRootPath } // Augment resolved with resource IDs from state — only when the config // actually uses ${resources.X.Y.id} references. The IDs aren't materialized - // into b.Config.Value() (they live in the StateDB), so we inject them here + // into b.Config (they live in the StateDB), so we inject them here // to enable sibling-based restoration. Skipped entirely for bundles with // no resource refs to avoid opening state DB files unnecessarily. resourceRefs := collectResourceIDRefs(preResolved) if len(resourceRefs) > 0 { if lookup := resourceIDLookup(b); lookup != nil { - resolved = injectResourceIDs(ctx, resolved, resourceRefs, lookup) + injectResourceIDs(ctx, resolved, resourceRefs, lookup) } else { log.Debugf(ctx, "variable restoration: state DB unavailable, skipping resource ID injection for %d refs", len(resourceRefs)) } @@ -116,10 +112,10 @@ func RestoreVariableReferences(ctx context.Context, b *bundle.Bundle, fieldChang // LoadPreResolvedConfig loads the bundle's configuration through the standard // loader mutators (entry point, includes, target overrides) but without -// variable resolution. The resulting dyn.Value is fully merged across files +// variable resolution. The resulting configuration is fully merged across files // and targets, yet retains ${...} references as literal strings. Returns -// InvalidValue if loading fails (restoration is then skipped). -func LoadPreResolvedConfig(ctx context.Context, b *bundle.Bundle) dyn.Value { +// an invalid value if loading fails (restoration is then skipped). +func LoadPreResolvedConfig(ctx context.Context, b *bundle.Bundle) structvar.View { fresh := &bundle.Bundle{ BundleRootPath: b.BundleRootPath, BundleRoot: b.BundleRoot, @@ -141,7 +137,26 @@ func LoadPreResolvedConfig(ctx context.Context, b *bundle.Bundle) dyn.Value { resourcemutator.MergePipelineClusters(), resourcemutator.MergeApps(), ) - return fresh.Config.Value() + return fresh.Config.View() +} + +// resolvedConfig is the bundle's resolved configuration plus values that are known +// only outside of it (the source-linked file path, resource IDs from state). +type resolvedConfig struct { + view structvar.View + overrides map[string]any // by path +} + +// lookup returns the value at path as a Go value, and whether it exists. +func (c resolvedConfig) lookup(path *structpath.PathNode) (any, bool) { + if v, ok := c.overrides[path.String()]; ok { + return v, true + } + v := c.view.Lookup(path) + if !v.IsValid() { + return nil, false + } + return v.AsAny(), true } // resourceIDLookup returns a function that resolves resource keys to their @@ -158,20 +173,17 @@ func resourceIDLookup(b *bundle.Bundle) func(string) string { // ${resources...id} references. Returns the unique set of paths // so the caller can inject IDs at those positions; returns nil if no such // references exist. -func collectResourceIDRefs(preResolved dyn.Value) []dyn.Path { +func collectResourceIDRefs(preResolved structvar.View) []*structpath.PathNode { seen := map[string]bool{} - var paths []dyn.Path - _ = dyn.WalkReadOnly(preResolved, func(_ dyn.Path, v dyn.Value) error { + var paths []*structpath.PathNode + _ = structvar.Walk(preResolved, func(_ *structpath.PathNode, v structvar.View) error { s, ok := v.AsString() - if !ok || !dynvar.IsPureVariableReference(s) || seen[s] { + if !ok || !structvar.IsPureVariableReference(s) || seen[s] { return nil } seen[s] = true - p, ok := dynvar.PureReferenceToPath(s) - if !ok { - return nil - } - if len(p) != 4 || p[0].Key() != "resources" || p[3].Key() != "id" { + p, ok := structvar.PureReferenceToPath(s) + if !ok || p.Len() != 4 || p.KeyAt(0) != "resources" || p.KeyAt(3) != "id" { return nil } paths = append(paths, p) @@ -180,47 +192,33 @@ func collectResourceIDRefs(preResolved dyn.Value) []dyn.Path { return paths } -// injectResourceIDs populates the resolved dyn.Value with IDs from state for -// the given resource reference paths. Skips references whose IDs aren't in -// state or that can't be written back into the dyn.Value tree. -func injectResourceIDs(ctx context.Context, resolved dyn.Value, paths []dyn.Path, lookupID func(string) string) dyn.Value { +// injectResourceIDs records IDs from state for the given resource reference paths in +// resolved. Skips references whose IDs aren't in state. +func injectResourceIDs(ctx context.Context, resolved resolvedConfig, paths []*structpath.PathNode, lookupID func(string) string) { for _, p := range paths { - resourceKey := p[:3].String() + resourceKey := p.Prefix(3).String() id := lookupID(resourceKey) if id == "" { log.Debugf(ctx, "variable restoration: no state entry for resource %q", resourceKey) continue } - updated, err := dyn.SetByPath(resolved, p, dyn.V(id)) - if err != nil { - log.Debugf(ctx, "variable restoration: SetByPath failed for %s: %v", p, err) - continue - } - resolved = updated + resolved.overrides[p.String()] = id } - return resolved } -// resolveReferencePath converts a variable reference string to the dyn.Path +// resolveReferencePath converts a variable reference string to the path // where its resolved value can be found in the bundle config. It applies the // same ${var.X} → variables.X.value shorthand rewriting as the variable // resolution mutator. -func resolveReferencePath(refStr string) (dyn.Path, bool) { - p, ok := dynvar.PureReferenceToPath(refStr) +func resolveReferencePath(refStr string) (*structpath.PathNode, bool) { + p, ok := structvar.PureReferenceToPath(refStr) if !ok { return nil, false } - if p.HasPrefix(varPrefix) && len(p) >= 2 { - newPath := dyn.NewPath( - dyn.Key("variables"), - p[1], - dyn.Key("value"), - ) - if len(p) > 2 { - newPath = newPath.Append(p[2:]...) - } - return newPath, true + if p.HasPrefix(varPrefix) && p.Len() >= 2 { + newPath := structpath.NewPath(nil, "variables", p.KeyAt(1), "value") + return structpath.Join(newPath, p.AsSlice()[2:]...), true } return p, true @@ -234,7 +232,7 @@ func resolveReferencePath(refStr string) (dyn.Path, bool) { // new value, falls back to a global lookup: if the new value uniquely matches // a different variable, that variable is used instead. The field's prior use // of a variable is the false-positive guard. -func restoreOriginalRefs(value any, preResolved, resolved dyn.Value, stats *RestoreStats) any { +func restoreOriginalRefs(value any, preResolved structvar.View, resolved resolvedConfig, stats *RestoreStats) any { switch v := value.(type) { case string, bool, int64: if ref, ok := matchOriginalRef(value, preResolved, resolved); ok { @@ -254,26 +252,14 @@ func restoreOriginalRefs(value any, preResolved, resolved dyn.Value, stats *Rest return value case map[string]any: - preMap, _ := preResolved.AsMap() for key, val := range v { - var childPre dyn.Value - if preMap.Len() > 0 { - if p, ok := preMap.GetPairByString(key); ok { - childPre = p.Value - } - } - v[key] = restoreOriginalRefs(val, childPre, resolved, stats) + v[key] = restoreOriginalRefs(val, preResolved.Get(key), resolved, stats) } return v case []any: - preSeq, _ := preResolved.AsSequence() for i, val := range v { - var childPre dyn.Value - if i < len(preSeq) { - childPre = preSeq[i] - } - v[i] = restoreOriginalRefs(val, childPre, resolved, stats) + v[i] = restoreOriginalRefs(val, preResolved.Index(i), resolved, stats) } return v @@ -287,37 +273,34 @@ func restoreOriginalRefs(value any, preResolved, resolved dyn.Value, stats *Rest // relative path: if exactly one unique pure variable reference across siblings // resolves to the leaf value, that reference is substituted. Multiple // different matching references are treated as ambiguous and skipped. -func restoreFromSiblings(value any, siblings []dyn.Value, resolved dyn.Value, stats *RestoreStats) any { - return restoreFromSiblingsAt(value, siblings, resolved, dyn.EmptyPath, stats) +func restoreFromSiblings(value any, siblings []structvar.View, resolved resolvedConfig, stats *RestoreStats) any { + return restoreFromSiblingsAt(value, siblings, resolved, nil, stats) } -func restoreFromSiblingsAt(value any, siblings []dyn.Value, resolved dyn.Value, relPath dyn.Path, stats *RestoreStats) any { +func restoreFromSiblingsAt(value any, siblings []structvar.View, resolved resolvedConfig, relPath *structpath.PathNode, stats *RestoreStats) any { switch v := value.(type) { case string, bool, int64: refs := map[string]struct{}{} strVal, isStr := value.(string) for _, sib := range siblings { - sv, err := dyn.GetByPath(sib, relPath) - if err != nil { - continue - } + sv := sib.Lookup(relPath) s, ok := sv.AsString() if !ok { continue } - if dynvar.IsPureVariableReference(s) { + if structvar.IsPureVariableReference(s) { rp, ok := resolveReferencePath(s) if !ok { continue } - rv, getErr := dyn.GetByPath(resolved, rp) - if getErr != nil { + rv, found := resolved.lookup(rp) + if !found { continue } - if rv.AsAny() == value { + if rv == value { refs[s] = struct{}{} } - } else if isStr && dynvar.ContainsVariableReference(s) { + } else if isStr && structvar.ContainsVariableReference(s) { // Compound interpolation in sibling: try to align the new // value against the sibling's template. If all variables // match at their positions, the template (possibly with @@ -337,13 +320,13 @@ func restoreFromSiblingsAt(value any, siblings []dyn.Value, resolved dyn.Value, case map[string]any: for key, val := range v { - v[key] = restoreFromSiblingsAt(val, siblings, resolved, relPath.Append(dyn.Key(key)), stats) + v[key] = restoreFromSiblingsAt(val, siblings, resolved, structpath.NewStringKey(relPath, key), stats) } return v case []any: for i, val := range v { - v[i] = restoreFromSiblingsAt(val, siblings, resolved, relPath.Append(dyn.Index(i)), stats) + v[i] = restoreFromSiblingsAt(val, siblings, resolved, structpath.NewIndex(relPath, i), stats) } return v @@ -355,51 +338,33 @@ func restoreFromSiblingsAt(value any, siblings []dyn.Value, resolved dyn.Value, // isPureVarRef reports whether the pre-resolved value at the field is a pure // ${var.X} reference. Used to gate the fallback substitution: only fields that // already used a variable can be re-targeted to a different variable. -func isPureVarRef(preResolved dyn.Value) bool { +func isPureVarRef(preResolved structvar.View) bool { if !preResolved.IsValid() { return false } s, ok := preResolved.AsString() - if !ok || !dynvar.IsPureVariableReference(s) { + if !ok || !structvar.IsPureVariableReference(s) { return false } - p, ok := dynvar.PureReferenceToPath(s) - if !ok { - return false - } - return p.HasPrefix(varPrefix) + p, ok := structvar.PureReferenceToPath(s) + return ok && p.HasPrefix(varPrefix) } // matchAnyVariable searches all bundle variables for a unique scalar value that // equals remoteValue. Returns the ${var.X} reference on a unique match, "" // otherwise. Multiple matches are treated as ambiguous and skipped. -func matchAnyVariable(remoteValue any, resolved dyn.Value) (string, bool) { - variables, err := dyn.GetByPath(resolved, dyn.NewPath(dyn.Key("variables"))) - if err != nil { - return "", false - } - vmap, ok := variables.AsMap() - if !ok { - return "", false - } +func matchAnyVariable(remoteValue any, resolved resolvedConfig) (string, bool) { var match string count := 0 - for _, pair := range vmap.Pairs() { - name, ok := pair.Key.AsString() - if !ok { - continue - } - v, getErr := dyn.GetByPath(pair.Value, dyn.NewPath(dyn.Key("value"))) - if getErr != nil { - continue - } + for name, variable := range resolved.view.Get("variables").MapItems() { + v := variable.Get("value") switch v.Kind() { - case dyn.KindString, dyn.KindInt, dyn.KindBool: + case structvar.KindString, structvar.KindInt, structvar.KindBool: if v.AsAny() == remoteValue { - match = pathToRef(varPrefix.Append(dyn.Key(name))) + match = pathToRef(structpath.NewStringKey(varPrefix, name)) count++ } - case dyn.KindInvalid, dyn.KindMap, dyn.KindSequence, dyn.KindFloat, dyn.KindTime, dyn.KindNil: + case structvar.KindInvalid, structvar.KindMap, structvar.KindSequence, structvar.KindFloat, structvar.KindTime, structvar.KindNil: // Skip non-scalar / unsupported variable values. } } @@ -409,19 +374,19 @@ func matchAnyVariable(remoteValue any, resolved dyn.Value) (string, bool) { return "", false } -// pathToRef formats a dyn.Path as a "${...}" interpolation reference. -func pathToRef(p dyn.Path) string { +// pathToRef formats a path as a "${...}" interpolation reference. +func pathToRef(p *structpath.PathNode) string { return "${" + p.String() + "}" } // matchOriginalRef checks if the pre-resolved config value at this position // was a pure variable reference whose resolved value equals remoteValue. -func matchOriginalRef(remoteValue any, preResolved, resolved dyn.Value) (string, bool) { +func matchOriginalRef(remoteValue any, preResolved structvar.View, resolved resolvedConfig) (string, bool) { if !preResolved.IsValid() { return "", false } s, ok := preResolved.AsString() - if !ok || !dynvar.IsPureVariableReference(s) { + if !ok || !structvar.IsPureVariableReference(s) { return "", false } @@ -430,12 +395,12 @@ func matchOriginalRef(remoteValue any, preResolved, resolved dyn.Value) (string, return "", false } - resolvedV, err := dyn.GetByPath(resolved, resolvedPath) - if err != nil { + resolvedV, found := resolved.lookup(resolvedPath) + if !found { return "", false } - if resolvedV.AsAny() == remoteValue { + if resolvedV == remoteValue { return s, true } return "", false @@ -455,12 +420,12 @@ func matchOriginalRef(remoteValue any, preResolved, resolved dyn.Value) (string, // new value contains "in" inside an unrelated word, that occurrence is still // rewritten to ${var.X}. Variables in the template are processed in order of // appearance, which is usually what the user expects. -func restoreCompoundInterpolation(remoteValue string, preResolved, resolved dyn.Value) (string, bool) { +func restoreCompoundInterpolation(remoteValue string, preResolved structvar.View, resolved resolvedConfig) (string, bool) { if !preResolved.IsValid() { return "", false } template, ok := preResolved.AsString() - if !ok || !dynvar.ContainsVariableReference(template) || dynvar.IsPureVariableReference(template) { + if !ok || !structvar.ContainsVariableReference(template) || structvar.IsPureVariableReference(template) { return "", false } @@ -481,7 +446,7 @@ func restoreCompoundInterpolation(remoteValue string, preResolved, resolved dyn. result = result[:idx] + seg.raw + result[idx+len(seg.resolvedValue):] } - if !dynvar.ContainsVariableReference(result) { + if !structvar.ContainsVariableReference(result) { return "", false } return result, true @@ -498,8 +463,8 @@ type templateSegment struct { // parseTemplateSegments splits a template string like "/mnt/${var.X}/raw" // into alternating literal and variable segments, resolving each variable. // Returns nil if any variable can't be resolved. -func parseTemplateSegments(template string, resolved dyn.Value) []templateSegment { - ref, ok := dynvar.NewRef(dyn.V(template)) +func parseTemplateSegments(template string, resolved resolvedConfig) []templateSegment { + ref, ok := structvar.NewRef(template) if !ok { return nil } @@ -526,12 +491,12 @@ func parseTemplateSegments(template string, resolved dyn.Value) []templateSegmen return nil } - resolvedV, err := dyn.GetByPath(resolved, resolvedPath) - if err != nil { + resolvedV, found := resolved.lookup(resolvedPath) + if !found { return nil } - resolvedStr, ok := resolvedV.AsString() + resolvedStr, ok := resolvedV.(string) if !ok { return nil } @@ -554,18 +519,15 @@ func parseTemplateSegments(template string, resolved dyn.Value) []templateSegmen return segments } -// preResolvedValueAt returns the pre-resolved dyn.Value at the field path, +// preResolvedValueAt returns the pre-resolved value at the field path, // if the field exists in the merged pre-resolved config. -func preResolvedValueAt(preResolved dyn.Value, fieldPath string) (dyn.Value, bool) { - p, err := dyn.NewPathFromString(fieldPath) +func preResolvedValueAt(preResolved structvar.View, fieldPath string) (structvar.View, bool) { + p, err := structpath.ParsePath(fieldPath) if err != nil { - return dyn.InvalidValue, false + return structvar.View{}, false } - v, err := dyn.GetByPath(preResolved, p) - if err != nil { - return dyn.InvalidValue, false - } - return v, true + v := preResolved.Lookup(p) + return v, v.IsValid() } // parentIsVariableReference reports whether the parent of fieldPath resolves to a @@ -573,7 +535,7 @@ func preResolvedValueAt(preResolved dyn.Value, fieldPath string) (dyn.Value, boo // ${var.spark_conf}). A nested key or index cannot be written into such a scalar, // so config-remote-sync skips the change. fieldPath is a merged-index path, the // same space as FieldChange.originalPath. -func parentIsVariableReference(preResolved dyn.Value, fieldPath string) bool { +func parentIsVariableReference(preResolved structvar.View, fieldPath string) bool { node, err := structpath.ParsePattern(fieldPath) if err != nil { return false @@ -587,7 +549,7 @@ func parentIsVariableReference(preResolved dyn.Value, fieldPath string) bool { return false } s, ok := v.AsString() - return ok && dynvar.ContainsVariableReference(s) + return ok && structvar.ContainsVariableReference(s) } // sequenceSiblings returns the sibling elements of the parent sequence when @@ -595,7 +557,7 @@ func parentIsVariableReference(preResolved dyn.Value, fieldPath string) bool { // last component must be an index ([*] or [N]) and the parent must resolve // to a sequence in the pre-resolved config. Returns false for non-sequence // Adds (e.g., new map fields). -func sequenceSiblings(preResolved dyn.Value, fieldPath string) ([]dyn.Value, bool) { +func sequenceSiblings(preResolved structvar.View, fieldPath string) ([]structvar.View, bool) { node, err := structpath.ParsePattern(fieldPath) if err != nil { return nil, false @@ -604,17 +566,17 @@ func sequenceSiblings(preResolved dyn.Value, fieldPath string) ([]dyn.Value, boo if !hasIndex && !node.BracketStar() { return nil, false } - p, err := dyn.NewPathFromString(node.Parent().String()) + p, err := structpath.ParsePath(node.Parent().String()) if err != nil { return nil, false } - parentValue, err := dyn.GetByPath(preResolved, p) - if err != nil { + parentValue := preResolved.Lookup(p) + if parentValue.Kind() != structvar.KindSequence { return nil, false } - seq, ok := parentValue.AsSequence() - if !ok { - return nil, false + var seq []structvar.View + for _, elem := range parentValue.Sequence() { + seq = append(seq, elem) } return seq, true } diff --git a/bundle/configsync/variables_test.go b/bundle/configsync/variables_test.go index 15f11d5eb3c..e25917c2f9e 100644 --- a/bundle/configsync/variables_test.go +++ b/bundle/configsync/variables_test.go @@ -1,22 +1,64 @@ package configsync import ( + "fmt" + "strings" "testing" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func loadTestConfig(t *testing.T, yaml string) *config.Root { + t.Helper() + root, diags := config.LoadFromBytes("test.yml", []byte(yaml)) + require.NoError(t, diags.Error()) + return root +} + +// scalarView returns a view of the string s. +func scalarView(t *testing.T, s string) structvar.View { + t.Helper() + root := loadTestConfig(t, fmt.Sprintf("bundle:\n name: %q\n", s)) + return root.View().Lookup(structpath.MustParsePath("bundle.name")) +} + +// tasksView returns views of job tasks, each given as the YAML of its fields in flow style. +func tasksView(t *testing.T, tasks ...string) []structvar.View { + t.Helper() + var sb strings.Builder + sb.WriteString("resources:\n jobs:\n j:\n tasks:\n") + for _, task := range tasks { + fmt.Fprintf(&sb, " - {%s}\n", task) + } + root := loadTestConfig(t, sb.String()) + var out []structvar.View + for _, elem := range root.View().Lookup(structpath.MustParsePath("resources.jobs.j.tasks")).Sequence() { + out = append(out, elem) + } + return out +} + +// variablesConfig returns a resolved configuration with the given variable values. +func variablesConfig(t *testing.T, values map[string]any) resolvedConfig { + t.Helper() + var sb strings.Builder + sb.WriteString("variables:\n") + for name, value := range values { + fmt.Fprintf(&sb, " %s:\n value: %#v\n", name, value) + } + return resolvedConfig{view: loadTestConfig(t, sb.String()).View(), overrides: map[string]any{}} +} + // TestRestoreOriginalRefs_HardcodedFieldNotRewritten fences the Replace safety // invariant: a hardcoded leaf must never be rewritten to a variable reference // just because the remote value coincidentally matches a variable elsewhere. func TestRestoreOriginalRefs_HardcodedFieldNotRewritten(t *testing.T) { - preResolved := dyn.V("us-east-1") - resolved := dyn.V(map[string]dyn.Value{ - "variables": dyn.V(map[string]dyn.Value{ - "region": dyn.V(map[string]dyn.Value{"value": dyn.V("main")}), - }), - }) + preResolved := scalarView(t, "us-east-1") + resolved := variablesConfig(t, map[string]any{"region": "main"}) // Even though "main" matches ${var.region}, restoreOriginalRefs must NOT // rewrite it — the original was hardcoded. result := restoreOriginalRefs("main", preResolved, resolved, &RestoreStats{}) @@ -29,17 +71,8 @@ func TestRestoreOriginalRefs_HardcodedFieldNotRewritten(t *testing.T) { func TestRestoreFromSiblings_ValueMatchesVariableButDifferentPath(t *testing.T) { // Sibling uses ${var.retry_count}=5 at .max_retries. New element has // .min_retry_interval=5 — coincidental match at a DIFFERENT relative path. - siblings := []dyn.Value{ - dyn.V(map[string]dyn.Value{ - "task_key": dyn.V("main"), - "max_retries": dyn.V("${var.retry_count}"), - }), - } - resolved := dyn.V(map[string]dyn.Value{ - "variables": dyn.V(map[string]dyn.Value{ - "retry_count": dyn.V(map[string]dyn.Value{"value": dyn.V(int64(5))}), - }), - }) + siblings := tasksView(t, `task_key: main, max_retries: "${var.retry_count}"`) + resolved := variablesConfig(t, map[string]any{"retry_count": 5}) value := map[string]any{ "task_key": "other", "min_retry_interval": int64(5), @@ -53,32 +86,22 @@ func TestRestoreFromSiblings_ValueMatchesVariableButDifferentPath(t *testing.T) // same-value rule: when two siblings use different variables at the same // relative path that both resolve to the same value, restoration is skipped. func TestRestoreFromSiblings_AmbiguousAcrossSiblings(t *testing.T) { - siblings := []dyn.Value{ - dyn.V(map[string]dyn.Value{"default": dyn.V("${var.landing_schema}")}), - dyn.V(map[string]dyn.Value{"default": dyn.V("${var.curated_schema}")}), - } - resolved := dyn.V(map[string]dyn.Value{ - "variables": dyn.V(map[string]dyn.Value{ - "landing_schema": dyn.V(map[string]dyn.Value{"value": dyn.V("raw_data")}), - "curated_schema": dyn.V(map[string]dyn.Value{"value": dyn.V("raw_data")}), - }), - }) - value := map[string]any{"default": "raw_data"} + siblings := tasksView(t, `task_key: "${var.landing_schema}"`, `task_key: "${var.curated_schema}"`) + resolved := variablesConfig(t, map[string]any{"landing_schema": "raw_data", "curated_schema": "raw_data"}) + value := map[string]any{"task_key": "raw_data"} result := restoreFromSiblings(value, siblings, resolved, &RestoreStats{}).(map[string]any) - assert.Equal(t, "raw_data", result["default"]) + assert.Equal(t, "raw_data", result["task_key"]) } // TestRestoreCompoundInterpolation covers the template alignment algorithm. // End-to-end coverage (pure ref match, sibling match, non-sequence skip, etc.) // lives in acceptance/bundle/config-remote-sync/resolve_variables. func TestRestoreCompoundInterpolation(t *testing.T) { - resolved := dyn.V(map[string]dyn.Value{ - "variables": dyn.V(map[string]dyn.Value{ - "host": dyn.V(map[string]dyn.Value{"value": dyn.V("dev-sql.example.com")}), - "port": dyn.V(map[string]dyn.Value{"value": dyn.V("1433")}), - "db": dyn.V(map[string]dyn.Value{"value": dyn.V("analytics_dev")}), - "acct": dyn.V(map[string]dyn.Value{"value": dyn.V("acct")}), - }), + resolved := variablesConfig(t, map[string]any{ + "host": "dev-sql.example.com", + "port": "1433", + "db": "analytics_dev", + "acct": "acct", }) tests := []struct { @@ -108,7 +131,7 @@ func TestRestoreCompoundInterpolation(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := restoreOriginalRefs(tt.remote, dyn.V(tt.template), resolved, &RestoreStats{}) + result := restoreOriginalRefs(tt.remote, scalarView(t, tt.template), resolved, &RestoreStats{}) assert.Equal(t, tt.want, result) }) } diff --git a/bundle/deploy/check_dashboards_modified_remotely.go b/bundle/deploy/check_dashboards_modified_remotely.go index d0ee26b6908..06e3e29f3a1 100644 --- a/bundle/deploy/check_dashboards_modified_remotely.go +++ b/bundle/deploy/check_dashboards_modified_remotely.go @@ -8,7 +8,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/agent" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" ) @@ -74,7 +73,7 @@ func (l *checkDashboardsModifiedRemotely) Apply(ctx context.Context, b *bundle.B } path := structpath.NewPath(nil, "resources", "dashboards", dashboard.Name) - loc := b.Config.GetLocation("resources.dashboards." + dashboard.Name) + loc := b.Config.GetLocation(path.String()) actual, err := b.WorkspaceClient(ctx).Lakeview.GetByDashboardId(ctx, dashboard.ID) if err != nil { diags = diags.Append(diag.Diagnostic{ @@ -82,7 +81,7 @@ func (l *checkDashboardsModifiedRemotely) Apply(ctx context.Context, b *bundle.B Summary: fmt.Sprintf("failed to get dashboard %q", dashboard.Name), Detail: err.Error(), Paths: []*structpath.PathNode{path}, - Locations: []dyn.Location{loc}, + Locations: []diag.Location{loc}, }) continue } @@ -111,7 +110,7 @@ func (l *checkDashboardsModifiedRemotely) Apply(ctx context.Context, b *bundle.B "To overwrite the remote changes with your local version, use --force.\n" + "The remote modifications will be lost." + agent.AgentNotice(), Paths: []*structpath.PathNode{path}, - Locations: []dyn.Location{loc}, + Locations: []diag.Location{loc}, }) } diff --git a/bundle/deploy/metadata/compute.go b/bundle/deploy/metadata/compute.go index 1fa4c3bf2d9..c8775748151 100644 --- a/bundle/deploy/metadata/compute.go +++ b/bundle/deploy/metadata/compute.go @@ -46,7 +46,7 @@ func (m *compute) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics for name, job := range b.Config.Resources.Jobs { // Compute config file path the job is defined in, relative to the bundle // root - l := b.Config.GetLocation("resources.jobs." + name) + l := b.Config.DefinitionLocation("resources.jobs." + name) if l.File == "" { // Skip resources that exist only in the deployment state: statemgmt.Load, // which runs before this mutator, injects them into the config without a @@ -73,7 +73,7 @@ func (m *compute) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics for name, pipeline := range b.Config.Resources.Pipelines { // Compute config file path the pipeline is defined in, relative to the bundle // root - l := b.Config.GetLocation("resources.pipelines." + name) + l := b.Config.DefinitionLocation("resources.pipelines." + name) if l.File == "" { // Skip resources that exist only in the deployment state: statemgmt.Load, // which runs before this mutator, injects them into the config without a @@ -98,7 +98,7 @@ func (m *compute) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics for name, dashboard := range b.Config.Resources.Dashboards { // Compute config file path the dashboard is defined in, relative to the bundle // root - l := b.Config.GetLocation("resources.dashboards." + name) + l := b.Config.DefinitionLocation("resources.dashboards." + name) if l.File == "" { // Skip resources that exist only in the deployment state: statemgmt.Load, // which runs before this mutator, injects them into the config without a diff --git a/bundle/deploy/metadata/compute_test.go b/bundle/deploy/metadata/compute_test.go index 58b8b390d35..cb163914bdb 100644 --- a/bundle/deploy/metadata/compute_test.go +++ b/bundle/deploy/metadata/compute_test.go @@ -9,7 +9,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/bundle/metadata" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/vfs" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/pipelines" @@ -81,12 +81,12 @@ func TestComputeMetadataMutator(t *testing.T) { }, } - bundletest.SetLocation(b, "resources.jobs.my-job-1", []dyn.Location{{File: "a/b/c"}}) - bundletest.SetLocation(b, "resources.jobs.my-job-2", []dyn.Location{{File: "d/e/f"}}) - bundletest.SetLocation(b, "resources.pipelines.my-pipeline-1", []dyn.Location{{File: "x/y/z"}}) - bundletest.SetLocation(b, "resources.pipelines.my-pipeline-2", []dyn.Location{{File: "u/v/w"}}) - bundletest.SetLocation(b, "resources.dashboards.my-dashboard-1", []dyn.Location{{File: "g/h/i"}}) - bundletest.SetLocation(b, "resources.dashboards.my-dashboard-2", []dyn.Location{{File: "j/k/l"}}) + bundletest.SetLocation(b, "resources.jobs.my-job-1", []diag.Location{{File: "a/b/c"}}) + bundletest.SetLocation(b, "resources.jobs.my-job-2", []diag.Location{{File: "d/e/f"}}) + bundletest.SetLocation(b, "resources.pipelines.my-pipeline-1", []diag.Location{{File: "x/y/z"}}) + bundletest.SetLocation(b, "resources.pipelines.my-pipeline-2", []diag.Location{{File: "u/v/w"}}) + bundletest.SetLocation(b, "resources.dashboards.my-dashboard-1", []diag.Location{{File: "g/h/i"}}) + bundletest.SetLocation(b, "resources.dashboards.my-dashboard-2", []diag.Location{{File: "j/k/l"}}) expectedMetadata := metadata.Metadata{ Version: metadata.Version, diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index d403ca53f76..6cf6c1bb261 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -15,8 +15,6 @@ import ( "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/terraform_dabs_map" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/structs/structaccess" @@ -884,7 +882,7 @@ func isEmpty(rv reflect.Value) bool { // Empty slices and maps cannot be represented in proto and because of that they cannot be represented // by SDK's JSON encoder. However, they can be provided by users in the config and can be represented in - // Bundle struct (currently libs/structs and libs/dyn use ForceSendFields for maps and slices, unlike SDK). + // Bundle struct (currently libs/structs uses ForceSendFields for maps and slices, unlike SDK). // Thus we get permanent drift because we see that new config is [] but in the state it is omitted. if rv.Kind() == reflect.Slice { @@ -1085,7 +1083,7 @@ func (b *DeploymentBundle) resolveReferences(ctx context.Context, resourceKey st var resolved bool for fieldPathStr, refString := range sv.Refs { - refs, ok := dynvar.NewRef(dyn.V(refString)) + refs, ok := structvar.NewRef(refString) if !ok { logdiag.LogError(ctx, fmt.Errorf("%s: cannot parse %q", errorPrefix, refString)) return false @@ -1159,31 +1157,36 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root existingKeys := maps.Clone(db.State) - patterns := []dyn.Pattern{ - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("permissions")), - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("grants")), + patterns := []*structpath.PatternNode{ + structpath.MustParsePattern("resources.*.*"), + structpath.MustParsePattern("resources.*.*.permissions"), + structpath.MustParsePattern("resources.*.*.grants"), + } + + var rootView structvar.View + if configRoot != nil { + rootView = configRoot.View() } // Walk? if configRoot != nil { for _, pat := range patterns { - _, err := dyn.MapByPattern( - configRoot.Value(), + err := structvar.ForEach( + rootView, pat, - func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + func(p *structpath.PathNode, _ structvar.View) error { s := p.String() resourceType := config.GetResourceTypeFromKey(s) if resourceType == "" { - return v, fmt.Errorf("cannot parse resource key: %q", s) + return fmt.Errorf("cannot parse resource key: %q", s) } _, ok := dresources.SupportedResources[resourceType] if !ok { - return v, fmt.Errorf("unsupported resource type: %s", resourceType) + return fmt.Errorf("unsupported resource type: %s", resourceType) } nodes = append(nodes, s) - return dyn.InvalidValue, nil + return nil }, ) if err != nil { @@ -1243,7 +1246,7 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root // This means input and state must be compatible: input can have more fields, but existing fields should not be moved // This means one cannot refer to fields not present in state (e.g. ${resources.jobs.foo.permissions}) - refs, err := extractReferences(configRoot.Value(), node, adapter.StateType()) + refs, err := extractReferences(rootView, node, adapter.StateType()) if err != nil { return nil, fmt.Errorf("failed to read references from config for %s: %w", node, err) } @@ -1252,13 +1255,13 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root var dependsOn []deployplan.DependsOnEntry for _, reference := range refs { - ref, ok := dynvar.NewRef(dyn.V(reference)) + ref, ok := structvar.NewRef(reference) if !ok { continue } for _, targetPath := range ref.References() { - targetPathParsed, err := dyn.NewPathFromString(targetPath) + targetPathParsed, err := structpath.ParsePath(targetPath) if err != nil { return nil, fmt.Errorf("parsing %q: %w", targetPath, err) } @@ -1347,26 +1350,26 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root // keeping only those whose field path exists in stateType (references in input-only or // bundle:"readonly" fields, such as volumes' computed volume_path, are skipped). // Returns a map from structpath string (field path within the resource) to template string. -func ExtractReferences(root dyn.Value, node string, stateType reflect.Type) (map[string]string, error) { +func ExtractReferences(root structvar.View, node string, stateType reflect.Type) (map[string]string, error) { return extractReferences(root, node, stateType) } -func extractReferences(root dyn.Value, node string, stateType reflect.Type) (map[string]string, error) { +func extractReferences(root structvar.View, node string, stateType reflect.Type) (map[string]string, error) { nodeType := config.GetResourceTypeFromKey(node) refs := make(map[string]string) - path, err := dyn.NewPathFromString(node) + path, err := structpath.ParsePath(node) if err != nil { return nil, fmt.Errorf("internal error: bad node key: %q: %w", node, err) } - val, err := dyn.GetByPath(root, path) - if err != nil { - return nil, err + val := root.Lookup(path) + if !val.IsValid() { + return nil, fmt.Errorf("no value at %s", node) } - err = dyn.WalkReadOnly(val, func(p dyn.Path, v dyn.Value) error { - fullPath := append(path, p...) + err = structvar.Walk(val, func(p *structpath.PathNode, v structvar.View) error { + fullPath := structpath.Join(path, p.AsSlice()...) targetType := config.GetResourceTypeFromKey(fullPath.String()) if targetType != nodeType { // Make sure these are associated with different nodes: @@ -1375,15 +1378,19 @@ func extractReferences(root dyn.Value, node string, stateType reflect.Type) (map // resources.jobs.foo.grants... return nil } - ref, ok := dynvar.NewRef(v) + str, ok := v.AsString() + if !ok { + return nil + } + ref, ok := structvar.NewRef(str) if !ok { return nil } // ValidatePath and the refs keys both operate on structpath (keys are // re-parsed and applied to the typed state in structvar.ResolveRef). // structpath's bracket notation (['key.with.dots']) also round-trips - // keys with dots, which dyn.Path.String()'s dot notation cannot. - fieldPath := dynPathToStructPath(p) + // keys with dots. + fieldPath := p // References resolve against the state type, not the input config (see PlanResources // and dresources.TestInputSubset). A field in input but not in state — e.g. a @@ -1402,19 +1409,6 @@ func extractReferences(root dyn.Value, node string, stateType reflect.Type) (map return refs, nil } -// dynPathToStructPath converts a dyn.Path to a structpath.PathNode. -func dynPathToStructPath(p dyn.Path) *structpath.PathNode { - var node *structpath.PathNode - for _, c := range p { - if key := c.Key(); key != "" { - node = structpath.NewStringKey(node, key) - } else { - node = structpath.NewIndex(node, c.Index()) - } - } - return node -} - func (b *DeploymentBundle) getAdapterForKey(resourceKey string) (*dresources.Adapter, error) { group := config.GetResourceTypeFromKey(resourceKey) if group == "" { diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index 025ef91df6e..4eb3a6883c8 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -1,15 +1,13 @@ package direct import ( - "bytes" "slices" "testing" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dresources" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/cli/libs/structs/structdiff" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/structs/structvar" @@ -20,35 +18,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestDynPathToStructPath(t *testing.T) { - tests := []struct { - path dyn.Path - expected string - }{ - { - path: dyn.NewPath(dyn.Key("foo"), dyn.Key("bar")), - expected: "foo.bar", - }, - { - path: dyn.NewPath(dyn.Key("foo"), dyn.Index(1), dyn.Key("bar")), - expected: "foo[1].bar", - }, - { - path: dyn.NewPath(dyn.Key("configuration"), dyn.Key("europris.swipe.egress_streaming_schema")), - expected: "configuration['europris.swipe.egress_streaming_schema']", - }, - { - path: dyn.NewPath(dyn.Key("tags"), dyn.Key("it's.here")), - expected: "tags['it''s.here']", - }, - } - - for _, tc := range tests { - node := dynPathToStructPath(tc.path) - assert.Equal(t, tc.expected, node.String()) - } -} - // extractReferences gates references on the state type: a reference in an input-only field // (e.g. a bundle:"readonly" field like volumes' volume_path) must not become a dependency, // while references in state fields (e.g. comment) are still extracted. @@ -69,10 +38,10 @@ resources: comment: "${resources.schemas.kept.name}" volume_path: "/Volumes/main/${resources.schemas.dropped.name}/myvol" ` - root, err := yamlloader.LoadYAML("test", bytes.NewBufferString(yml)) - require.NoError(t, err) + root, diags := config.LoadFromBytes("test", []byte(yml)) + require.NoError(t, diags.Error()) - refs, err := extractReferences(root, "resources.volumes.v", stateType) + refs, err := extractReferences(root.View(), "resources.volumes.v", stateType) require.NoError(t, err) assert.Equal(t, map[string]string{ diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index 80ece24b2be..3f2da957804 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -512,7 +512,7 @@ func (a *Adapter) FieldIsStableOutput(path *structpath.PathNode) bool { // references needed to complete it. Resources without PrepareInputConfig pass their config through. func (a *Adapter) PrepareInputConfig(inputConfig any, resourceKey string) (*structvar.StructVar, error) { if a.prepareInputConfig == nil { - return &structvar.StructVar{Value: inputConfig, Refs: nil}, nil + return &structvar.StructVar{Value: inputConfig, Refs: nil, Locations: nil}, nil } outs, err := a.prepareInputConfig.Call(inputConfig, resourceKey) diff --git a/bundle/direct/dresources/grants.go b/bundle/direct/dresources/grants.go index 3aa48c58279..3fac2e0800b 100644 --- a/bundle/direct/dresources/grants.go +++ b/bundle/direct/dresources/grants.go @@ -75,6 +75,7 @@ func (r *ResourceGrants) PrepareInputConfig(inputConfig *[]catalog.PrivilegeAssi Refs: map[string]string{ "full_name": "${" + baseNode + ".id}", }, + Locations: nil, }, nil } diff --git a/bundle/direct/dresources/permissions.go b/bundle/direct/dresources/permissions.go index c2aebb4d0ac..b09e9cfe2ff 100644 --- a/bundle/direct/dresources/permissions.go +++ b/bundle/direct/dresources/permissions.go @@ -116,6 +116,7 @@ func (r *ResourcePermissions) PrepareInputConfig(inputConfig any, resourceKey st Refs: map[string]string{ "object_id": r.objectType + "${" + baseNode + "." + r.idField + "}", }, + Locations: nil, }, nil } diff --git a/bundle/direct/dresources/secret_scope_acls.go b/bundle/direct/dresources/secret_scope_acls.go index 1de301bde2c..61269ce4097 100644 --- a/bundle/direct/dresources/secret_scope_acls.go +++ b/bundle/direct/dresources/secret_scope_acls.go @@ -60,6 +60,7 @@ func (*ResourceSecretScopeAcls) PrepareInputConfig(inputConfig *[]resources.Secr Refs: map[string]string{ "scope_name": "${" + baseNode + ".name}", }, + Locations: nil, }, nil } diff --git a/bundle/direct/dresources/serialize_test.go b/bundle/direct/dresources/serialize_test.go index 76078460193..e1bd30579c6 100644 --- a/bundle/direct/dresources/serialize_test.go +++ b/bundle/direct/dresources/serialize_test.go @@ -130,7 +130,7 @@ func TestRoundtripAllFieldsRemoteType(t *testing.T) { // TestRoundtripAllFieldsInputConfigType verifies InputConfigType, the typed // bundle config a resource is loaded into, survives a JSON round-trip with every -// field populated. Bundle config is normally read and written through libs/dyn, +// field populated. Bundle config is normally read and written through libs/structs/structvar, // which walks the struct itself and never calls these marshalers, so this is a // latent trap rather than live corruption. It is guarded anyway because it is the // same trap as StateType and RemoteType: a resource that embeds a member with its diff --git a/bundle/direct/resolve_config.go b/bundle/direct/resolve_config.go index 8ce28d710e1..45ab83ef1fb 100644 --- a/bundle/direct/resolve_config.go +++ b/bundle/direct/resolve_config.go @@ -1,14 +1,15 @@ package direct import ( + "fmt" + "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/cli/libs/dyn/dynvar" - "github.com/databricks/cli/libs/dyn/jsonloader" + "github.com/databricks/cli/libs/structs/structaccess" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -var resourcesPrefix = dyn.MustPathFromString("resources") +var resourcesPrefix = structpath.MustParsePath("resources") // ResolveConfigAgainstState resolves ${resources.*} references within the resource at // target so its runner ("bundle run") sees concrete values rather than references. For a @@ -24,48 +25,97 @@ var resourcesPrefix = dyn.MustPathFromString("resources") // // The state DB holds fields like the immutable snapshot's full_path that never reach the // config, so it must be open. -func (b *DeploymentBundle) ResolveConfigAgainstState(cfg *config.Root, target dyn.Path) error { - return cfg.Mutate(func(root dyn.Value) (dyn.Value, error) { - // Fall back to the fully-normalized config so references to fields that are - // implied (not explicitly set) still resolve. - normalized, _ := convert.Normalize(cfg, root, convert.IncludeMissingFields) +func (b *DeploymentBundle) ResolveConfigAgainstState(cfg *config.Root, target *structpath.PathNode) error { + view := cfg.View() + resource := view.Lookup(target) + if !resource.IsValid() { + return fmt.Errorf("resource %s not found in configuration", target) + } + + // Each string is resolved on its own (references only point at other paths), then + // written back; collect first because writing changes the view. + type update struct { + path *structpath.PathNode + value any + } + var updates []update + + lookup := func(path *structpath.PathNode) (structvar.View, error) { + if !path.HasPrefix(resourcesPrefix) { + return structvar.View{}, structvar.ErrSkipResolution + } + if v, ok := b.lookupStateField(path); ok { + return v, nil + } + // Fall back to the config, including fields that are implied (not explicitly set). + if v := view.Lookup(path); v.IsValid() { + return v, nil + } + got, err := structaccess.Get(cfg, path) + if err != nil { + return structvar.View{}, structvar.ErrSkipResolution + } + return structvar.NewView(&got, nil, nil), nil + } - return dyn.MapByPath(root, target, func(_ dyn.Path, resource dyn.Value) (dyn.Value, error) { - return dynvar.Resolve(resource, func(path dyn.Path) (dyn.Value, error) { - if !path.HasPrefix(resourcesPrefix) { - return dyn.InvalidValue, dynvar.ErrSkipResolution - } - if v, ok := b.lookupStateField(path); ok { - return v, nil - } - v, err := dyn.GetByPath(normalized, path) - if err != nil { - return dyn.InvalidValue, dynvar.ErrSkipResolution - } - return v, nil - }) - }) + err := structvar.Walk(resource, func(p *structpath.PathNode, v structvar.View) error { + s, ok := v.AsString() + if !ok { + return nil + } + if _, ok := structvar.NewRef(s); !ok { + return nil + } + key := p.String() + out, err := structvar.Resolve(map[string]structvar.Template{key: {Value: s}}, lookup) + if err != nil { + return err + } + if resolved, ok := out[key]; ok { + updates = append(updates, update{structpath.Join(target, p.AsSlice()...), resolved.AsAny()}) + } + return nil }) + if err != nil { + return err + } + + for _, u := range updates { + // Decode converts like loading did, e.g. a string id into an int job_id. + diags, err := cfg.Decode(u.path, structvar.NewView(&u.value, nil, nil)) + if err != nil { + return err + } + if diags.HasError() { + return diags.Error() + } + } + return nil } // lookupStateField returns the value at resources... from the // resource's persisted state, if that resource is in state and holds the field. -func (b *DeploymentBundle) lookupStateField(path dyn.Path) (dyn.Value, bool) { - if len(path) < 4 || path[0].Key() != "resources" { - return dyn.InvalidValue, false +func (b *DeploymentBundle) lookupStateField(path *structpath.PathNode) (structvar.View, bool) { + if path.Len() < 4 || path.KeyAt(0) != "resources" { + return structvar.View{}, false } - resourceKey := "resources." + path[1].Key() + "." + path[2].Key() + resourceKey := "resources." + path.KeyAt(1) + "." + path.KeyAt(2) entry, ok := b.StateDB.GetResourceEntry(resourceKey) if !ok || len(entry.State) == 0 { - return dyn.InvalidValue, false + return structvar.View{}, false } - stateVal, err := jsonloader.LoadJSON(entry.State, resourceKey) + // ParseJSON keeps large ids exact. + node, err := structvar.ParseJSON(resourceKey, entry.State) if err != nil { - return dyn.InvalidValue, false + return structvar.View{}, false + } + var state any + if _, _, err := structvar.DecodeYAMLNode(resourceKey, node, &state, nil); err != nil { + return structvar.View{}, false } - fieldVal, err := dyn.GetByPath(stateVal, path[3:]) - if err != nil || !fieldVal.IsValid() { - return dyn.InvalidValue, false + v := structvar.NewView(&state, nil, nil).Lookup(path.SkipPrefix(3)) + if !v.IsValid() { + return structvar.View{}, false } - return fieldVal, true + return v, true } diff --git a/bundle/direct/unescape.go b/bundle/direct/unescape.go index 31f9925e098..40ec5d02bed 100644 --- a/bundle/direct/unescape.go +++ b/bundle/direct/unescape.go @@ -5,9 +5,9 @@ import ( "reflect" "strings" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/cli/libs/structs/structwalk" ) @@ -24,7 +24,7 @@ func unescapeRefs(state any) error { return } paths = append(paths, path) - values = append(values, dynvar.Unescape(s)) + values = append(values, structvar.Unescape(s)) }) if err != nil { return err diff --git a/bundle/generate/alert.go b/bundle/generate/alert.go index 3f2428b72e3..d3765205d17 100644 --- a/bundle/generate/alert.go +++ b/bundle/generate/alert.go @@ -1,18 +1,16 @@ package generate import ( - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/databricks/databricks-sdk-go/service/sql" ) -func ConvertAlertToValue(alert *sql.AlertV2, filePath string) (dyn.Value, error) { +func ConvertAlertToValue(alert *sql.AlertV2, filePath string) (structyaml.Map, error) { // The majority of fields of the alert struct are present in .dbalert.json file. // We copy the relevant fields manually. - dv := map[string]dyn.Value{ - "display_name": dyn.NewValue(alert.DisplayName, []dyn.Location{{Line: 1}}), - "warehouse_id": dyn.NewValue(alert.WarehouseId, []dyn.Location{{Line: 2}}), - "file_path": dyn.NewValue(filePath, []dyn.Location{{Line: 3}}), - } - - return dyn.V(dv), nil + return structyaml.M( + "display_name", alert.DisplayName, + "warehouse_id", alert.WarehouseId, + "file_path", filePath, + ), nil } diff --git a/bundle/generate/app.go b/bundle/generate/app.go index 234967ce072..a0f40a5a4ef 100644 --- a/bundle/generate/app.go +++ b/bundle/generate/app.go @@ -1,53 +1,47 @@ package generate import ( - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/databricks/databricks-sdk-go/service/apps" ) -func ConvertAppToValue(app *apps.App, sourceCodePath string) (dyn.Value, error) { - ar, err := convert.FromTyped(app.Resources, dyn.NilValue) - if err != nil { - return dyn.NilValue, err - } - +func ConvertAppToValue(app *apps.App, sourceCodePath string) (structyaml.Map, error) { // The majority of fields of the app struct are read-only. // We copy the relevant fields manually. - dv := map[string]dyn.Value{ - "name": dyn.NewValue(app.Name, []dyn.Location{{Line: 1}}), - "description": dyn.NewValue(app.Description, []dyn.Location{{Line: 2}}), - } + dv := structyaml.M( + "name", app.Name, + "description", app.Description, + ) // For a git-backed app, emit git_repository + git_source instead of a // workspace source_code_path. Otherwise the generated bundle would silently // down-convert the app to workspace source and point source_code_path at a // local directory that has nothing downloaded into it. if app.GitRepository != nil { - dv["git_repository"] = gitRepositoryValue(app.GitRepository) - if gs := gitSourceValue(app); gs.Kind() != dyn.KindNil { - dv["git_source"] = gs + dv.Add("git_repository", gitRepositoryValue(app.GitRepository)) + if gs := gitSourceValue(app); len(gs) > 0 { + dv.Add("git_source", gs) } } else { - dv["source_code_path"] = dyn.NewValue(sourceCodePath, []dyn.Location{{Line: 4}}) + dv.Add("source_code_path", sourceCodePath) } - if ar.Kind() != dyn.KindNil { - dv["resources"] = ar.WithLocations([]dyn.Location{{Line: 5}}) + if ar := structyaml.Value(app.Resources); ar != nil { + dv.Add("resources", ar) } - return dyn.V(dv), nil + return dv, nil } -func gitRepositoryValue(r *apps.GitRepository) dyn.Value { - m := map[string]dyn.Value{ - "url": dyn.NewValue(r.Url, []dyn.Location{{Line: 1}}), - "provider": dyn.NewValue(r.Provider, []dyn.Location{{Line: 2}}), - } +func gitRepositoryValue(r *apps.GitRepository) structyaml.Map { + m := structyaml.M( + "url", r.Url, + "provider", r.Provider, + ) if r.AutoDeploy { - m["auto_deploy"] = dyn.NewValue(r.AutoDeploy, []dyn.Location{{Line: 3}}) + m.Add("auto_deploy", r.AutoDeploy) } - return dyn.NewValue(m, []dyn.Location{{Line: 3}}) + return m } // gitSourceValue returns the reference the app deploys from (branch, tag, or @@ -55,29 +49,26 @@ func gitRepositoryValue(r *apps.GitRepository) dyn.Value { // configured git_source and falls back to the default source of the app's most // recent deployment. System-populated fields (resolved_commit and the nested // git_repository) are intentionally omitted. -func gitSourceValue(app *apps.App) dyn.Value { +func gitSourceValue(app *apps.App) structyaml.Map { src := app.GitSource if src == nil { src = app.DefaultGitSource } if src == nil { - return dyn.NilValue + return nil } - m := map[string]dyn.Value{} + var m structyaml.Map switch { case src.Branch != "": - m["branch"] = dyn.NewValue(src.Branch, []dyn.Location{{Line: 1}}) + m.Add("branch", src.Branch) case src.Tag != "": - m["tag"] = dyn.NewValue(src.Tag, []dyn.Location{{Line: 1}}) + m.Add("tag", src.Tag) case src.Commit != "": - m["commit"] = dyn.NewValue(src.Commit, []dyn.Location{{Line: 1}}) + m.Add("commit", src.Commit) } if src.SourceCodePath != "" { - m["source_code_path"] = dyn.NewValue(src.SourceCodePath, []dyn.Location{{Line: 2}}) - } - if len(m) == 0 { - return dyn.NilValue + m.Add("source_code_path", src.SourceCodePath) } - return dyn.NewValue(m, []dyn.Location{{Line: 4}}) + return m } diff --git a/bundle/generate/dashboard.go b/bundle/generate/dashboard.go index 46014080693..a12e8b57f5d 100644 --- a/bundle/generate/dashboard.go +++ b/bundle/generate/dashboard.go @@ -1,18 +1,16 @@ package generate import ( - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/databricks/databricks-sdk-go/service/dashboards" ) -func ConvertDashboardToValue(dashboard *dashboards.Dashboard, filePath string) (dyn.Value, error) { +func ConvertDashboardToValue(dashboard *dashboards.Dashboard, filePath string) (structyaml.Map, error) { // The majority of fields of the dashboard struct are read-only. // We copy the relevant fields manually. - dv := map[string]dyn.Value{ - "display_name": dyn.NewValue(dashboard.DisplayName, []dyn.Location{{Line: 1}}), - "warehouse_id": dyn.NewValue(dashboard.WarehouseId, []dyn.Location{{Line: 2}}), - "file_path": dyn.NewValue(filePath, []dyn.Location{{Line: 3}}), - } - - return dyn.V(dv), nil + return structyaml.M( + "display_name", dashboard.DisplayName, + "warehouse_id", dashboard.WarehouseId, + "file_path", filePath, + ), nil } diff --git a/bundle/generate/genie_space.go b/bundle/generate/genie_space.go index 613ccf67ef0..88cbe8b05d9 100644 --- a/bundle/generate/genie_space.go +++ b/bundle/generate/genie_space.go @@ -1,28 +1,28 @@ package generate import ( - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/databricks/databricks-sdk-go/service/dashboards" ) -func ConvertGenieSpaceToValue(genieSpace *dashboards.GenieSpace, filePath string) (dyn.Value, error) { +func ConvertGenieSpaceToValue(genieSpace *dashboards.GenieSpace, filePath string) (structyaml.Map, error) { // Emit only the fields a user authors in a bundle. serialized_space is // written to a separate file and referenced via file_path, and output-only // fields (e.g. space_id, etag) must not appear in the generated config, so // we build the value field by field rather than marshaling the struct. - dv := map[string]dyn.Value{ - "title": dyn.NewValue(genieSpace.Title, []dyn.Location{{Line: 1}}), - "warehouse_id": dyn.NewValue(genieSpace.WarehouseId, []dyn.Location{{Line: 2}}), - "file_path": dyn.NewValue(filePath, []dyn.Location{{Line: 3}}), - } + dv := structyaml.M( + "title", genieSpace.Title, + "warehouse_id", genieSpace.WarehouseId, + "file_path", filePath, + ) if genieSpace.Description != "" { - dv["description"] = dyn.NewValue(genieSpace.Description, []dyn.Location{{Line: 4}}) + dv.Add("description", genieSpace.Description) } if genieSpace.ParentPath != "" { - dv["parent_path"] = dyn.NewValue(ensureWorkspacePrefix(genieSpace.ParentPath), []dyn.Location{{Line: 5}}) + dv.Add("parent_path", ensureWorkspacePrefix(genieSpace.ParentPath)) } - return dyn.V(dv), nil + return dv, nil } diff --git a/bundle/generate/job.go b/bundle/generate/job.go index 825caaf4ce6..b1784258e1c 100644 --- a/bundle/generate/job.go +++ b/bundle/generate/job.go @@ -1,49 +1,51 @@ package generate import ( - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlsaver" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/databricks/databricks-sdk-go/service/jobs" ) var ( - jobOrder = yamlsaver.NewOrder([]string{"name", "job_clusters", "compute", "tasks"}) - taskOrder = yamlsaver.NewOrder([]string{"task_key", "depends_on", "existing_cluster_id", "new_cluster", "job_cluster_key"}) + jobOrder = []string{"name", "job_clusters", "compute", "tasks", "parameters"} + taskOrder = []string{"task_key", "depends_on", "existing_cluster_id", "new_cluster", "job_cluster_key"} ) -func ConvertJobToValue(job *jobs.Job) (dyn.Value, error) { - value := make(map[string]dyn.Value) +func ConvertJobToValue(job *jobs.Job) (structyaml.Map, error) { + // Tasks and parameters are processed separately. + value, err := structyaml.Struct(job.Settings, "format", "new_cluster", "existing_cluster_id", "tasks", "parameters") + if err != nil { + return nil, err + } + if job.Settings.Tasks != nil { - var tasks []dyn.Value + var tasks []any for _, task := range job.Settings.Tasks { - v, err := convertTaskToValue(task, taskOrder) + v, err := convertTaskToValue(task) if err != nil { - return dyn.InvalidValue, err + return nil, err } tasks = append(tasks, v) } - // We're using location lines to define the order of keys in exported YAML. - value["tasks"] = dyn.NewValue(tasks, []dyn.Location{{Line: jobOrder.Get("tasks")}}) + value.Add("tasks", tasks) } // We're processing job.Settings.Parameters separately to retain empty default values. if len(job.Settings.Parameters) > 0 { - var params []dyn.Value + var params []any for _, parameter := range job.Settings.Parameters { - p := map[string]dyn.Value{ - "name": dyn.NewValue(parameter.Name, []dyn.Location{{Line: 0}}), // We use Line: 0 to ensure that the name goes first. - "default": dyn.NewValue(parameter.Default, []dyn.Location{{Line: 1}}), - } - params = append(params, dyn.V(p)) + params = append(params, structyaml.M("name", parameter.Name, "default", parameter.Default)) } - value["parameters"] = dyn.NewValue(params, []dyn.Location{{Line: jobOrder.Get("parameters")}}) + value.Add("parameters", params) } - return yamlsaver.ConvertToMapValue(job.Settings, jobOrder, []string{"format", "new_cluster", "existing_cluster_id"}, value) + return value.Order(jobOrder...), nil } -func convertTaskToValue(task jobs.Task, order *yamlsaver.Order) (dyn.Value, error) { - dst := make(map[string]dyn.Value) - return yamlsaver.ConvertToMapValue(task, order, []string{"format"}, dst) +func convertTaskToValue(task jobs.Task) (structyaml.Map, error) { + value, err := structyaml.Struct(task, "format") + if err != nil { + return nil, err + } + return value.Order(taskOrder...), nil } diff --git a/bundle/generate/pipeline.go b/bundle/generate/pipeline.go index 115330ea0f6..beb8bf2e5ad 100644 --- a/bundle/generate/pipeline.go +++ b/bundle/generate/pipeline.go @@ -3,14 +3,13 @@ package generate import ( "strings" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlsaver" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/databricks/databricks-sdk-go/service/pipelines" ) -var pipelineOrder = yamlsaver.NewOrder([]string{"name", "clusters", "configuration", "libraries"}) +var pipelineOrder = []string{"name", "clusters", "configuration", "libraries"} -func ConvertPipelineToValue(pipeline *pipelines.PipelineSpec, rootPath, remoteRootPath string) (dyn.Value, error) { +func ConvertPipelineToValue(pipeline *pipelines.PipelineSpec, rootPath, remoteRootPath string) (structyaml.Map, error) { if pipeline.RootPath != "" { pipeline.RootPath = rootPath } @@ -24,11 +23,14 @@ func ConvertPipelineToValue(pipeline *pipelines.PipelineSpec, rootPath, remoteRo } } - value := make(map[string]dyn.Value) // We ignore the following fields: // - id: this is a read-only field // - storage: changes to this field are rare because changing the storage recreates pipeline-related resources // - edition: this field is rarely changed // - development: this field is specific to the mode where it's used and does not need to be saved to the bundle configuration. - return yamlsaver.ConvertToMapValue(pipeline, pipelineOrder, []string{"id", "storage", "edition", "development"}, value) + value, err := structyaml.Struct(pipeline, "id", "storage", "edition", "development") + if err != nil { + return nil, err + } + return value.Order(pipelineOrder...), nil } diff --git a/bundle/internal/bundletest/benchmark.go b/bundle/internal/bundletest/benchmark.go index 59647d814fd..2cb4b738652 100644 --- a/bundle/internal/bundletest/benchmark.go +++ b/bundle/internal/bundletest/benchmark.go @@ -9,7 +9,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/require" ) @@ -210,32 +209,6 @@ const jobExample = ` } }` -func BundleV(b *testing.B, numJobs int) dyn.Value { - allJobs := map[string]*resources.Job{} - for i := range numJobs { - job := jobs.JobSettings{} - err := json.Unmarshal([]byte(jobExample), &job) - require.NoError(b, err) - - allJobs[strconv.Itoa(i)] = &resources.Job{ - JobSettings: job, - } - } - - myBundle := bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Jobs: allJobs, - }, - }, - } - - // Apply noop mutator to initialize the bundle value. - bundle.ApplyFuncContext(b.Context(), &myBundle, func(ctx context.Context, b *bundle.Bundle) {}) - - return myBundle.Config.Value() -} - func Bundle(b *testing.B, numJobs int) *bundle.Bundle { allJobs := map[string]*resources.Job{} for i := range numJobs { diff --git a/bundle/internal/bundletest/benchmark_test.go b/bundle/internal/bundletest/benchmark_test.go index 75b121e095b..df8d3659a4b 100644 --- a/bundle/internal/bundletest/benchmark_test.go +++ b/bundle/internal/bundletest/benchmark_test.go @@ -3,61 +3,18 @@ package bundletest import ( "testing" - "github.com/databricks/cli/bundle/internal/validation/generated" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/stretchr/testify/assert" ) -// This took 40ms to run on 18th June 2025. func BenchmarkWalkReadOnly(b *testing.B) { - input := BundleV(b, 10000) + input := Bundle(b, 10000).Config.View() for b.Loop() { - err := dyn.WalkReadOnly(input, func(p dyn.Path, v dyn.Value) error { + err := structvar.Walk(input, func(p *structpath.PathNode, v structvar.View) error { return nil }) assert.NoError(b, err) } } - -// This took 160ms to run on 18th June 2025. -func BenchmarkWalk(b *testing.B) { - input := BundleV(b, 10000) - - for b.Loop() { - _, err := dyn.Walk(input, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - return v, nil - }) - assert.NoError(b, err) - } -} - -// This took 49 microseconds to run on 6th Aug 2025. -func BenchmarkEnumPrefixTree(b *testing.B) { - for b.Loop() { - // Generate prefix tree for all enum fields. - trie := &dyn.TrieNode{} - for k := range generated.EnumFields { - pattern, err := dyn.NewPatternFromString(k) - assert.NoError(b, err) - - err = trie.Insert(pattern) - assert.NoError(b, err) - } - } -} - -// This took 15 microseconds to run on 6th Aug 2025. -func BenchmarkRequiredPrefixTree(b *testing.B) { - for b.Loop() { - // Generate prefix tree for all required fields. - trie := &dyn.TrieNode{} - for k := range generated.RequiredFields { - pattern, err := dyn.NewPatternFromString(k) - assert.NoError(b, err) - - err = trie.Insert(pattern) - assert.NoError(b, err) - } - } -} diff --git a/bundle/internal/bundletest/location.go b/bundle/internal/bundletest/location.go index 5dcd9d78f4d..7290a8ae2a9 100644 --- a/bundle/internal/bundletest/location.go +++ b/bundle/internal/bundletest/location.go @@ -2,32 +2,18 @@ package bundletest import ( "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structpath" ) // SetLocation sets the location of all values in the bundle to the given path. // This is useful for testing where we need to associate configuration // with the path it is loaded from. -func SetLocation(b *bundle.Bundle, prefix string, locations []dyn.Location) { - start := dyn.MustPathFromString(prefix) - err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - return dyn.Walk(root, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - // If the path has the given prefix, set the location. - if p.HasPrefix(start) { - return v.WithLocations(locations), nil - } - - // The path is not nested under the given prefix. - // If the path is a prefix of the prefix, keep traversing and return the node verbatim. - if start.HasPrefix(p) { - return v, nil - } - - // Return verbatim, but skip traversal. - return v, dyn.ErrSkip - }) - }) - if err != nil { - panic("Mutate() failed: " + err.Error()) +func SetLocation(b *bundle.Bundle, prefix string, locations []diag.Location) { + // "." is the root of the configuration. + var path *structpath.PathNode + if prefix != "." { + path = structpath.MustParsePath(prefix) } + b.Config.SetLocations(path, locations) } diff --git a/bundle/internal/bundletest/mutate.go b/bundle/internal/bundletest/mutate.go deleted file mode 100644 index 00adc7d29e0..00000000000 --- a/bundle/internal/bundletest/mutate.go +++ /dev/null @@ -1,17 +0,0 @@ -package bundletest - -import ( - "context" - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/require" -) - -func Mutate(t *testing.T, b *bundle.Bundle, f func(v dyn.Value) (dyn.Value, error)) { - bundle.ApplyFuncContext(t.Context(), b, func(ctx context.Context, b *bundle.Bundle) { - err := b.Config.Mutate(f) - require.NoError(t, err) - }) -} diff --git a/bundle/internal/bundletest/mutator_benchmark_test.go b/bundle/internal/bundletest/mutator_benchmark_test.go index 03d63dcd072..5e60f0cfab0 100644 --- a/bundle/internal/bundletest/mutator_benchmark_test.go +++ b/bundle/internal/bundletest/mutator_benchmark_test.go @@ -7,7 +7,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/validate" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/stretchr/testify/assert" ) @@ -35,9 +36,9 @@ func benchmarkWalkReadOnlyBaseline(b *testing.B, numJobs int) { myBundle := Bundle(b, numJobs) for b.Loop() { - var paths []dyn.Path + var paths []*structpath.PathNode bundle.ApplyFuncContext(b.Context(), myBundle, func(ctx context.Context, b *bundle.Bundle) { - _ = dyn.WalkReadOnly(b.Config.Value(), func(p dyn.Path, v dyn.Value) error { + _ = structvar.Walk(b.Config.View(), func(p *structpath.PathNode, v structvar.View) error { paths = append(paths, p) return nil }) diff --git a/bundle/internal/schema/annotations.go b/bundle/internal/schema/annotations.go index 548cf5ae95f..7cc5776ff4c 100644 --- a/bundle/internal/schema/annotations.go +++ b/bundle/internal/schema/annotations.go @@ -8,9 +8,6 @@ import ( "github.com/databricks/cli/bundle/internal/annotation" "github.com/databricks/cli/internal/clijson" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/cli/libs/dyn/merge" "github.com/databricks/cli/libs/jsonschema" ) @@ -27,12 +24,8 @@ type annotationHandler struct { // Adds annotations to the JSON schema reading from the annotation files. // More details https://json-schema.org/understanding-json-schema/reference/annotations func newAnnotationHandler(extracted, fromFile annotation.File) (*annotationHandler, error) { - merged, err := mergeAnnotationFiles(extracted, fromFile) - if err != nil { - return nil, err - } return &annotationHandler{ - parsedAnnotations: merged, + parsedAnnotations: mergeAnnotationFiles(extracted, fromFile), fileAnnotations: fromFile, missingAnnotations: annotation.File{}, }, nil @@ -79,25 +72,42 @@ func isEmptyDescriptor(d annotation.Descriptor) bool { // mergeAnnotationFiles merges later layers over earlier ones with the same // semantics the on-disk annotation files used to be merged with: maps merge // recursively, scalars take the later value, sequences concatenate. -func mergeAnnotationFiles(files ...annotation.File) (annotation.File, error) { - prev := dyn.NilValue +func mergeAnnotationFiles(files ...annotation.File) annotation.File { + merged := annotation.File{} for _, f := range files { - v, err := convert.FromTyped(f, dyn.NilValue) - if err != nil { - return nil, err - } - prev, err = merge.Merge(prev, v) - if err != nil { - return nil, err + for typeKey, ta := range f { + merged.SetSelf(typeKey, mergeDescriptor(merged[typeKey].Self, ta.Self)) + for name, d := range ta.Fields { + merged.SetField(typeKey, name, mergeDescriptor(merged[typeKey].Fields[name], d)) + } } } + return merged +} - var data annotation.File - err := convert.ToTyped(&data, prev) - if err != nil { - return nil, err +// mergeDescriptor merges the set fields of b over a: maps merge by key, sequences +// concatenate and other values are replaced. +func mergeDescriptor(a, b annotation.Descriptor) annotation.Descriptor { + dst, src := reflect.ValueOf(&a).Elem(), reflect.ValueOf(b) + for i := range dst.NumField() { + f, g := dst.Field(i), src.Field(i) + switch { + case g.IsZero(): + case f.Kind() == reflect.Slice: + f.Set(reflect.AppendSlice(reflect.AppendSlice(reflect.Zero(f.Type()), f), g)) + case f.Kind() == reflect.Map: + m := reflect.MakeMap(f.Type()) + for _, v := range []reflect.Value{f, g} { + for iter := v.MapRange(); iter.Next(); { + m.SetMapIndex(iter.Key(), iter.Value()) + } + } + f.Set(m) + default: + f.Set(g) + } } - return data, nil + return a } func (d *annotationHandler) addAnnotations(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { @@ -125,10 +135,7 @@ func (d *annotationHandler) addAnnotations(typ reflect.Type, s jsonschema.Schema // descriptions for fields that have no documentation anywhere. Entries for // fields that no longer exist in the config are dropped with a warning. func (d *annotationHandler) syncWithMissingAnnotations(outputPath string, g *typeGraph) error { - updated, err := mergeAnnotationFiles(d.fileAnnotations, d.missingAnnotations) - if err != nil { - return err - } + updated := mergeAnnotationFiles(d.fileAnnotations, d.missingAnnotations) detached, err := saveAnnotationsFile(outputPath, updated, g) if err != nil { diff --git a/bundle/internal/schema/annotations_file.go b/bundle/internal/schema/annotations_file.go index bd106b136b2..5d9f96b55bc 100644 --- a/bundle/internal/schema/annotations_file.go +++ b/bundle/internal/schema/annotations_file.go @@ -2,18 +2,19 @@ package main import ( "bytes" + "cmp" + "errors" "fmt" "os" + "path/filepath" "reflect" "slices" + "strconv" "strings" "github.com/databricks/cli/bundle/internal/annotation" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/cli/libs/dyn/yamlloader" - "github.com/databricks/cli/libs/dyn/yamlsaver" - yaml3 "go.yaml.in/yaml/v3" + "github.com/databricks/cli/libs/structs/structvar" + "go.yaml.in/yaml/v3" ) // fieldsKey nests a type's block of field nodes inside the node of a field @@ -32,14 +33,6 @@ const ( typeDocKey = "$type" ) -// lineTypeDoc and lineFields sort the "$type" and "$fields" keys after a -// node's inline descriptor keys, which the saver orders with small line -// numbers (see descriptorKeyOrder). -const ( - lineTypeDoc = 9999 - lineFields = 10000 -) - const annotationsFileHeader = `# This file contains the documentation the CLI owns for the bundle # configuration JSON schema: docs for fields that do not exist in the upstream # API spec (.codegen/cli.json), and overrides of upstream docs. Documentation @@ -80,26 +73,71 @@ var descriptorKeys = func() map[string]bool { // follow alphabetically. var descriptorKeyOrder = []string{"description", "markdown_description", "title", "default", "enum"} -// descriptorToMap serializes d into dst with its keys ordered. It returns the -// nil value (writing nothing) when d carries no content. -// -// The empty check relies on the dyn.NilValue reference: with a nil reference, -// FromTyped omits zero-valued fields, so an all-zero descriptor collapses to -// KindNil rather than an empty map. Do not pass a map reference here, or empty -// descriptors would start serializing as "{}". -func descriptorToMap(d annotation.Descriptor, dst map[string]dyn.Value) (dyn.Value, error) { - v, err := convert.FromTyped(d, dyn.NilValue) - if err != nil || v.Kind() == dyn.KindNil { - return dyn.NilValue, err - } - return yamlsaver.ConvertToMapValue(d, yamlsaver.NewOrder(descriptorKeyOrder), []string{}, dst) +// descriptorNodes serializes d into the key and value nodes of a mapping, with its +// keys ordered. It returns nothing when d carries no content. +func descriptorNodes(d annotation.Descriptor, style yaml.Style) []*yaml.Node { + var pairs [][2]*yaml.Node + for key, v := range structvar.NewView(d, nil, nil).MapItems() { + pairs = append(pairs, [2]*yaml.Node{{Kind: yaml.ScalarNode, Value: key, Style: style}, valueNode(v, style)}) + } + // Keys in descriptorKeyOrder come first, the others follow alphabetically. + rank := func(key string) int { + if i := slices.Index(descriptorKeyOrder, key); i >= 0 { + return i + } + return len(descriptorKeyOrder) + } + slices.SortFunc(pairs, func(a, b [2]*yaml.Node) int { + return cmp.Or(cmp.Compare(rank(a[0].Value), rank(b[0].Value)), cmp.Compare(a[0].Value, b[0].Value)) + }) + var out []*yaml.Node + for _, p := range pairs { + out = append(out, p[:]...) + } + return out +} + +// valueNode converts a value of a descriptor to a YAML node. +func valueNode(x structvar.View, style yaml.Style) *yaml.Node { + switch x.Kind() { + case structvar.KindMap: + n := &yaml.Node{Kind: yaml.MappingNode, Style: style} + for k, c := range x.MapItems() { + n.Content = append(n.Content, &yaml.Node{Kind: yaml.ScalarNode, Value: k, Style: style}, valueNode(c, style)) + } + return n + case structvar.KindSequence: + n := &yaml.Node{Kind: yaml.SequenceNode, Style: style} + for _, c := range x.Sequence() { + n.Content = append(n.Content, valueNode(c, style)) + } + return n + case structvar.KindString: + s, _ := x.AsString() + // A string that reads as another scalar (bool, number) is quoted to stay a string. + if isScalarValueInString(s) { + style = yaml.DoubleQuotedStyle + } + return &yaml.Node{Kind: yaml.ScalarNode, Value: s, Style: style} + default: + return &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprint(x.AsAny()), Style: style} + } +} + +func isScalarValueInString(s string) bool { + if s == "true" || s == "false" || s == "" { + return true + } + if _, err := strconv.ParseInt(s, 0, 64); err == nil { + return true + } + _, err := strconv.ParseFloat(s, 64) + return err == nil } -// descriptorEmpty reports whether d carries no documentation. See -// descriptorToMap for why the nil reference is what makes this work. +// descriptorEmpty reports whether d carries no documentation. func descriptorEmpty(d annotation.Descriptor) bool { - v, err := convert.FromTyped(d, dyn.NilValue) - return err == nil && v.Kind() == dyn.KindNil + return structvar.NewView(d, nil, nil).Kind() == structvar.KindNil } // loadAnnotationsFile reads the tree-format annotations file and flattens it @@ -111,7 +149,7 @@ func loadAnnotationsFile(path string, g *typeGraph) (annotation.File, []string, if err != nil { return nil, nil, err } - v, err := yamlloader.LoadYAML(path, bytes.NewBuffer(b)) + v, err := structvar.ParseYAML(bytes.NewReader(b)) if err != nil { return nil, nil, err } @@ -131,17 +169,14 @@ type fileLoader struct { } // block loads one type's block of field nodes. -func (l *fileLoader) block(v dyn.Value, typeKey, where string) error { - if v.Kind() == dyn.KindNil { - return nil - } - m, ok := v.AsMap() - if !ok { - return fmt.Errorf("%s: expected a mapping, got %s", where, v.Kind()) +func (l *fileLoader) block(v *yaml.Node, typeKey, where string) error { + pairs, err := mappingPairs(v, where) + if err != nil { + return err } - for _, pair := range m.Pairs() { - key := pair.Key.MustString() + for i := 0; i < len(pairs); i += 2 { + key := pairs[i].Value child := where + "." + key if where == "" { child = key @@ -152,7 +187,7 @@ func (l *fileLoader) block(v dyn.Value, typeKey, where string) error { l.unknown = append(l.unknown, child) continue } - err := l.node(pair.Value, typeKey, edge, child) + err := l.node(pairs[i+1], typeKey, edge, child) if err != nil { return err } @@ -163,26 +198,23 @@ func (l *fileLoader) block(v dyn.Value, typeKey, where string) error { // node loads one field's node: the inline descriptor for the field, the // "$type" docs for the type it resolves to, and the "$fields" block of that // type's fields. -func (l *fileLoader) node(v dyn.Value, typeKey string, edge fieldEdge, where string) error { - if v.Kind() == dyn.KindNil { - return nil - } - m, ok := v.AsMap() - if !ok { - return fmt.Errorf("%s: expected a mapping, got %s", where, v.Kind()) +func (l *fileLoader) node(v *yaml.Node, typeKey string, edge fieldEdge, where string) error { + pairs, err := mappingPairs(v, where) + if err != nil { + return err } - desc := dyn.NewMapping() - for _, pair := range m.Pairs() { - key := pair.Key.MustString() + var desc []*yaml.Node + for i := 0; i < len(pairs); i += 2 { + key, value := pairs[i].Value, pairs[i+1] switch { case key == fieldsKey && edge.typ != "": - err := l.block(pair.Value, edge.typ, where+"."+fieldsKey) + err := l.block(value, edge.typ, where+"."+fieldsKey) if err != nil { return err } case key == typeDocKey && edge.typ != "": - d, ok, err := l.descriptor(pair.Value, where+"."+typeDocKey) + d, ok, err := l.descriptor(value, where+"."+typeDocKey) if err != nil { return err } @@ -190,13 +222,13 @@ func (l *fileLoader) node(v dyn.Value, typeKey string, edge fieldEdge, where str l.data.SetSelf(edge.typ, d) } case descriptorKeys[key]: - desc.SetLoc(key, nil, pair.Value) + desc = append(desc, pairs[i], value) default: l.unknown = append(l.unknown, where+"."+key) } } - if desc.Len() > 0 { + if len(desc) > 0 { d, err := toDescriptor(desc, where) if err != nil { return err @@ -209,21 +241,20 @@ func (l *fileLoader) node(v dyn.Value, typeKey string, edge fieldEdge, where str // descriptor parses a mapping of descriptor keys (the value of a "$type" key). // Non-descriptor keys are flagged as unknown. The second return is false when // the mapping carries no descriptor keys. -func (l *fileLoader) descriptor(v dyn.Value, where string) (annotation.Descriptor, bool, error) { - m, ok := v.AsMap() - if !ok { - return annotation.Descriptor{}, false, fmt.Errorf("%s: expected a mapping, got %s", where, v.Kind()) - } - desc := dyn.NewMapping() - for _, pair := range m.Pairs() { - key := pair.Key.MustString() +func (l *fileLoader) descriptor(v *yaml.Node, where string) (annotation.Descriptor, bool, error) { + if v.Kind != yaml.MappingNode { + return annotation.Descriptor{}, false, fmt.Errorf("%s: expected a mapping, got %s", where, nodeKind(v)) + } + var desc []*yaml.Node + for i := 0; i < len(v.Content); i += 2 { + key := v.Content[i].Value if descriptorKeys[key] { - desc.SetLoc(key, nil, pair.Value) + desc = append(desc, v.Content[i], v.Content[i+1]) } else { l.unknown = append(l.unknown, where+"."+key) } } - if desc.Len() == 0 { + if len(desc) == 0 { return annotation.Descriptor{}, false, nil } d, err := toDescriptor(desc, where) @@ -231,15 +262,40 @@ func (l *fileLoader) descriptor(v dyn.Value, where string) (annotation.Descripto } // toDescriptor converts a mapping of descriptor keys to a typed descriptor. -func toDescriptor(desc dyn.Mapping, where string) (annotation.Descriptor, error) { +func toDescriptor(desc []*yaml.Node, where string) (annotation.Descriptor, error) { var d annotation.Descriptor - err := convert.ToTyped(&d, dyn.V(desc)) + _, diags, err := structvar.DecodeYAMLNode(where, &yaml.Node{Kind: yaml.MappingNode, Content: desc}, &d, nil) + if err == nil { + err = diags.Error() + } if err != nil { return annotation.Descriptor{}, fmt.Errorf("%s: %w", where, err) } return d, nil } +// mappingPairs returns the alternating key and value nodes of a mapping. A null node has none. +func mappingPairs(v *yaml.Node, where string) ([]*yaml.Node, error) { + switch { + case v.ShortTag() == "!!null": + return nil, nil + case v.Kind != yaml.MappingNode: + return nil, fmt.Errorf("%s: expected a mapping, got %s", where, nodeKind(v)) + } + return v.Content, nil +} + +func nodeKind(v *yaml.Node) string { + switch v.Kind { + case yaml.MappingNode: + return "map" + case yaml.SequenceNode: + return "sequence" + default: + return strings.TrimPrefix(v.ShortTag(), "!!") + } +} + // saveAnnotationsFile writes data to path in the canonical tree layout: a // depth-first walk over the config type graph in struct declaration order // expands every type at its first occurrence; keys are emitted alphabetically. @@ -256,18 +312,10 @@ func saveAnnotationsFile(path string, data annotation.File, g *typeGraph) ([]str } s.assignCanonical(g.root) - root, err := s.block(g.root) - if err != nil { - return nil, err - } - - // Style every top-level key so all nested scalars render in literal block - // style, matching the formatting of the previous annotation files. - style := map[string]yaml3.Style{} - for k := range root { - style[k] = yaml3.LiteralStyle - } - err = yamlsaver.NewSaverWithStyle(style).SaveAsYAML(root, path, true) + // Everything below the top-level keys renders in literal block style, matching + // the formatting of the previous annotation files. + root := &yaml.Node{Kind: yaml.MappingNode, Content: s.block(g.root, 0)} + err := writeYAML(path, root) if err != nil { return nil, err } @@ -278,6 +326,20 @@ func saveAnnotationsFile(path string, data annotation.File, g *typeGraph) ([]str return s.detached(), nil } +func writeYAML(path string, node *yaml.Node) error { + err := os.MkdirAll(filepath.Dir(path), 0o755) + if err != nil { + return err + } + f, err := os.Create(path) + if err != nil { + return err + } + enc := yaml.NewEncoder(f) + enc.SetIndent(2) + return errors.Join(enc.Encode(node), f.Close()) +} + type edgeKey struct { typ string name string @@ -305,61 +367,52 @@ func (s *fileSaver) assignCanonical(typeKey string) { } } -// block renders one type's block of field nodes, emitted alphabetically. -// Lines in the value locations encode the output order for the YAML saver. -func (s *fileSaver) block(typeKey string) (map[string]dyn.Value, error) { - out := map[string]dyn.Value{} - line := 0 +// block renders one type's block of field nodes as the content of a mapping, emitted +// alphabetically, with the keys in keyStyle. +func (s *fileSaver) block(typeKey string, keyStyle yaml.Style) []*yaml.Node { + var out []*yaml.Node edges := slices.Clone(s.graph.fields[typeKey]) slices.SortFunc(edges, func(a, b fieldEdge) int { return strings.Compare(a.name, b.name) }) for _, edge := range edges { - node, err := s.node(typeKey, edge) - if err != nil { - return nil, err - } + node := s.node(typeKey, edge) if len(node) > 0 { - out[edge.name] = dyn.NewValue(node, []dyn.Location{{Line: line}}) - line++ + out = append(out, &yaml.Node{Kind: yaml.ScalarNode, Value: edge.name, Style: keyStyle}, mapping(node)) } } - return out, nil + return out +} + +func mapping(content []*yaml.Node) *yaml.Node { + return &yaml.Node{Kind: yaml.MappingNode, Content: content, Style: yaml.LiteralStyle} +} + +func literalKey(name string) *yaml.Node { + return &yaml.Node{Kind: yaml.ScalarNode, Value: name, Style: yaml.LiteralStyle} } // node renders one field's node: the inline field descriptor plus, at the // field's canonical position, the resolved type's "$type" docs and the // "fields" block of its fields. -func (s *fileSaver) node(typeKey string, edge fieldEdge) (map[string]dyn.Value, error) { - out := map[string]dyn.Value{} +func (s *fileSaver) node(typeKey string, edge fieldEdge) []*yaml.Node { + var out []*yaml.Node - // The inline descriptor keys are written directly into the node, sharing - // it with the "$type" and "$fields" keys added below. if d, ok := s.takeField(typeKey, edge.name); ok { - if _, err := descriptorToMap(d, out); err != nil { - return nil, err - } + out = descriptorNodes(d, yaml.LiteralStyle) } if s.expandAt[edgeKey{typeKey, edge.name}] { - v, err := descriptorToMap(s.takeSelf(edge.typ), map[string]dyn.Value{}) - if err != nil { - return nil, err - } - if v.Kind() != dyn.KindNil { - out[typeDocKey] = v.WithLocations([]dyn.Location{{Line: lineTypeDoc}}) + if doc := descriptorNodes(s.takeSelf(edge.typ), yaml.LiteralStyle); len(doc) > 0 { + out = append(out, literalKey(typeDocKey), mapping(doc)) } - child, err := s.block(edge.typ) - if err != nil { - return nil, err - } - if len(child) > 0 { - out[fieldsKey] = dyn.NewValue(child, []dyn.Location{{Line: lineFields}}) + if child := s.block(edge.typ, yaml.LiteralStyle); len(child) > 0 { + out = append(out, literalKey(fieldsKey), mapping(child)) } } - return out, nil + return out } // takeField returns the descriptor for a field and marks it consumed for the diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go index 01a743933b2..75d36dd4824 100644 --- a/bundle/internal/schema/main.go +++ b/bundle/internal/schema/main.go @@ -13,17 +13,17 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/config/variable" "github.com/databricks/cli/internal/clijson" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/jsonschema" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/pipelines" ) // interpolationPattern builds a JSON Schema regex for ${prefix.path...} references. -// Path segments use [dynvar.BaseVarDef]; unlike the runtime matcher in ref.go, this +// Path segments use [structvar.BaseVarDef]; unlike the runtime matcher in ref.go, this // requires a fixed prefix (var, resources, ...) and at least one ".segment" after it. func interpolationPattern(s string) string { - return fmt.Sprintf(`\$\{(%s(\.%s(\[[0-9]+\])*)+)\}`, s, dynvar.BaseVarDef) + return fmt.Sprintf(`\$\{(%s(\.%s(\[[0-9]+\])*)+)\}`, s, structvar.BaseVarDef) } func addInterpolationPatterns(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { diff --git a/bundle/internal/schema/main_test.go b/bundle/internal/schema/main_test.go index 92184a2d48f..4f6a4c741d8 100644 --- a/bundle/internal/schema/main_test.go +++ b/bundle/internal/schema/main_test.go @@ -1,21 +1,20 @@ package main import ( - "bytes" "io" + "maps" "os" "path" "reflect" + "slices" "strings" "testing" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/merge" - "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/cli/libs/jsonschema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" ) const cliJSONPath = "../../../.codegen/cli.json" @@ -73,36 +72,45 @@ func TestRequiredAnnotationsForNewFields(t *testing.T) { require.NoError(t, err) currentFile, err := os.ReadFile(annotationsPath) require.NoError(t, err) - original, err := yamlloader.LoadYAML("", bytes.NewBuffer(originalFile)) - require.NoError(t, err) - current, err := yamlloader.LoadYAML("", bytes.NewBuffer(currentFile)) - require.NoError(t, err) + var original, current any + require.NoError(t, yaml.Unmarshal(originalFile, &original)) + require.NoError(t, yaml.Unmarshal(currentFile, ¤t)) // Regenerating from the committed file must be a no-op: no new placeholders // (a new undocumented config field) and no deletes/updates (stale - // placeholders not yet pruned). VisitDelete/VisitUpdate must be set or - // Override panics on any change. + // placeholders not yet pruned). var addedFieldPaths []string var changedFieldPaths []string - _, err = merge.Override(original, current, merge.OverrideVisitor{ - VisitInsert: func(basePath dyn.Path, right dyn.Value) (dyn.Value, error) { - addedFieldPaths = append(addedFieldPaths, basePath.String()) - return right, nil - }, - VisitDelete: func(basePath dyn.Path, left dyn.Value) error { - changedFieldPaths = append(changedFieldPaths, basePath.String()) - return nil - }, - VisitUpdate: func(basePath dyn.Path, left, right dyn.Value) (dyn.Value, error) { - changedFieldPaths = append(changedFieldPaths, basePath.String()) - return right, nil - }, - }) - assert.NoError(t, err) + diffYAML("", original, current, &addedFieldPaths, &changedFieldPaths) assert.Empty(t, addedFieldPaths, "Missing JSON-schema descriptions for new config fields in bundle/internal/schema/annotations.yml:\n%s", strings.Join(addedFieldPaths, "\n")) assert.Empty(t, changedFieldPaths, "annotations.yml is out of sync; run `./task generate-schema` and commit the result:\n%s", strings.Join(changedFieldPaths, "\n")) } +// diffYAML records the paths of the values added to, and removed from or changed in, left to get right. +func diffYAML(path string, left, right any, added, changed *[]string) { + l, lok := left.(map[string]any) + r, rok := right.(map[string]any) + if !lok || !rok { + if !reflect.DeepEqual(left, right) { + *changed = append(*changed, path) + } + return + } + for _, k := range slices.Sorted(maps.Keys(l)) { + if _, ok := r[k]; !ok { + *changed = append(*changed, strings.TrimPrefix(path+"."+k, ".")) + } + } + for _, k := range slices.Sorted(maps.Keys(r)) { + p := strings.TrimPrefix(path+"."+k, ".") + if lv, ok := l[k]; ok { + diffYAML(p, lv, r[k], added, changed) + } else { + *added = append(*added, p) + } + } +} + // Checks that the annotations file only contains entries that match the // current bundle configuration structure. func TestNoDetachedAnnotations(t *testing.T) { diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go index 2bdc81b60a2..9277c349100 100644 --- a/bundle/libraries/expand_glob_references.go +++ b/bundle/libraries/expand_glob_references.go @@ -4,35 +4,35 @@ import ( "context" "fmt" "path/filepath" + "reflect" "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/patchwheel" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" + "github.com/databricks/databricks-sdk-go/service/compute" ) type expand struct{} -func matchError(p dyn.Path, l []dyn.Location, message string) diag.Diagnostic { +func matchError(p *structpath.PathNode, l []diag.Location, message string) diag.Diagnostic { return diag.Diagnostic{ Severity: diag.Error, Summary: message, Locations: l, - Paths: dyn.ToStructPaths(p), + Paths: []*structpath.PathNode{p}, } } -func getLibDetails(v dyn.Value) (string, string, bool) { - m := v.MustMap() - whl, ok := m.GetByString("whl") - if ok { - return whl.MustString(), "whl", true +func getLibDetails(lib compute.Library) (string, string, bool) { + if lib.Whl != "" { + return lib.Whl, "whl", true } - jar, ok := m.GetByString("jar") - if ok { - return jar.MustString(), "jar", true + if lib.Jar != "" { + return lib.Jar, "jar", true } return "", "", false @@ -72,48 +72,92 @@ func isGlobPattern(path string) bool { return strings.ContainsAny(path, "*?[") } -func expandLibraries(ctx context.Context, b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostics, []dyn.Value) { - var output []dyn.Value +// expandFunc expands the item at path p into the items it is replaced with. +// If relocate is not empty, it names the field of each new item that takes the locations of the item. +type expandFunc[T any] func(ctx context.Context, b *bundle.Bundle, p *structpath.PathNode, item T) (output []T, relocate string, diags diag.Diagnostics) + +// expandSequence replaces the items of the sequence at path p with the items expandFunc returns for them. +// New items keep the locations of the item they were expanded from. +func expandSequence[T any](ctx context.Context, b *bundle.Bundle, p *structpath.PathNode, lv structvar.View, fn expandFunc[T]) diag.Diagnostics { + items, ok := sequencePointer[T](lv) + if !ok { + return nil + } + var diags diag.Diagnostics + output := make([]T, 0, len(*items)) + var sources [][]int + var relocates []string + var locations [][]diag.Location + for i, item := range *items { + ip := structpath.NewIndex(p, i) + expanded, relocate, d := fn(ctx, b, ip, item) + diags = diags.Extend(d) + locs := b.Config.LocationsAt(ip) + for _, e := range expanded { + output = append(output, e) + sources = append(sources, []int{i}) + relocates = append(relocates, relocate) + locations = append(locations, locs) + } + } + + *items = output + b.Config.UpdateSequence(p, sources) + for i, relocate := range relocates { + if relocate != "" { + b.Config.SetLocations(structpath.NewPath(p, i, relocate), locations[i]) + } + } + + return diags +} - libs := v.MustSequence() - for i, lib := range libs { - lp := p.Append(dyn.Index(i)) +// sequencePointer returns a pointer to the typed sequence the view is based on. +func sequencePointer[T any](lv structvar.View) (*[]T, bool) { + if lv.Kind() != structvar.KindSequence { + return nil, false + } + + v := lv.Reflect() + for v.Kind() == reflect.Pointer || v.Kind() == reflect.Interface { + v = v.Elem() + } + if !v.CanAddr() { + return nil, false + } + + return reflect.TypeAssert[*[]T](v.Addr()) +} + +func expandLibraries(ctx context.Context, lv structvar.View, p *structpath.PathNode, b *bundle.Bundle) diag.Diagnostics { + return expandSequence(ctx, b, p, lv, func(ctx context.Context, b *bundle.Bundle, ip *structpath.PathNode, lib compute.Library) ([]compute.Library, string, diag.Diagnostics) { path, libType, supported := getLibDetails(lib) if !supported || !IsLibraryLocal(path) { - output = append(output, lib) - continue + return []compute.Library{lib}, "", nil } - lp = lp.Append(dyn.Key(libType)) - matches, err := findMatches(ctx, b, path) if err != nil { - diags = diags.Append(matchError(lp, lib.Locations(), err.Error())) - continue + return nil, "", diag.Diagnostics{matchError(structpath.NewStringKey(ip, libType), b.Config.LocationsAt(ip), err.Error())} } + var output []compute.Library for _, match := range matches { - output = append(output, dyn.NewValue(map[string]dyn.Value{ - libType: dyn.NewValue(match, lib.Locations()), - }, lib.Locations())) + if libType == "whl" { + output = append(output, compute.Library{Whl: match}) + } else { + output = append(output, compute.Library{Jar: match}) + } } - } - - return diags, output + return output, libType, nil + }) } -func expandEnvironmentDeps(ctx context.Context, b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostics, []dyn.Value) { - var output []dyn.Value - var diags diag.Diagnostics - - deps := v.MustSequence() - for i, dep := range deps { - lp := p.Append(dyn.Index(i)) - path := dep.MustString() +func expandEnvironmentDeps(ctx context.Context, lv structvar.View, p *structpath.PathNode, b *bundle.Bundle) diag.Diagnostics { + return expandSequence(ctx, b, p, lv, func(ctx context.Context, b *bundle.Bundle, ip *structpath.PathNode, path string) ([]string, string, diag.Diagnostics) { if !IsLibraryLocal(path) { - output = append(output, dep) - continue + return []string{path}, "", nil } // Strip extras before globbing so "[...]" isn't read as a glob class, then re-append. @@ -121,88 +165,30 @@ func expandEnvironmentDeps(ctx context.Context, b *bundle.Bundle, p dyn.Path, v matches, err := findMatches(ctx, b, path) if err != nil { - diags = diags.Append(matchError(lp, dep.Locations(), err.Error())) - continue + return nil, "", diag.Diagnostics{matchError(ip, b.Config.LocationsAt(ip), err.Error())} } + var output []string for _, match := range matches { - output = append(output, dyn.NewValue(match+extras, dep.Locations())) + output = append(output, match+extras) } - } - - return diags, output + return output, "", nil + }) } type expandPattern struct { - pattern dyn.Pattern - fn func(ctx context.Context, b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostics, []dyn.Value) + pattern *structpath.PatternNode + fn func(ctx context.Context, lv structvar.View, p *structpath.PathNode, b *bundle.Bundle) diag.Diagnostics } -var taskLibrariesPattern = dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("tasks"), - dyn.AnyIndex(), - dyn.Key("libraries"), -) - -var forEachTaskLibrariesPattern = dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("tasks"), - dyn.AnyIndex(), - dyn.Key("for_each_task"), - dyn.Key("task"), - dyn.Key("libraries"), -) - -var aiRuntimeCodeSourcePattern = dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("tasks"), - dyn.AnyIndex(), - dyn.Key("ai_runtime_task"), - dyn.Key("code_source_path"), -) - -var forEachAiRuntimeCodeSourcePattern = dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("tasks"), - dyn.AnyIndex(), - dyn.Key("for_each_task"), - dyn.Key("task"), - dyn.Key("ai_runtime_task"), - dyn.Key("code_source_path"), -) - -var envDepsPattern = dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("environments"), - dyn.AnyIndex(), - dyn.Key("spec"), - dyn.Key("dependencies"), -) - -var pipelineEnvDepsPattern = dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("pipelines"), - dyn.AnyKey(), - dyn.Key("environment"), - dyn.Key("dependencies"), -) - -var clusterLibrariesPattern = dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("clusters"), - dyn.AnyKey(), - dyn.Key("libraries"), +var ( + taskLibrariesPattern = structpath.MustParsePattern("resources.jobs.*.tasks[*].libraries") + forEachTaskLibrariesPattern = structpath.MustParsePattern("resources.jobs.*.tasks[*].for_each_task.task.libraries") + aiRuntimeCodeSourcePattern = structpath.MustParsePattern("resources.jobs.*.tasks[*].ai_runtime_task.code_source_path") + forEachAiRuntimeCodeSourcePattern = structpath.MustParsePattern("resources.jobs.*.tasks[*].for_each_task.task.ai_runtime_task.code_source_path") + envDepsPattern = structpath.MustParsePattern("resources.jobs.*.environments[*].spec.dependencies") + pipelineEnvDepsPattern = structpath.MustParsePattern("resources.pipelines.*.environment.dependencies") + clusterLibrariesPattern = structpath.MustParsePattern("resources.clusters.*.libraries") ) func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { @@ -231,23 +217,15 @@ func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - var err error - for _, expander := range expanders { - v, err = dyn.MapByPattern(v, expander.pattern, func(p dyn.Path, lv dyn.Value) (dyn.Value, error) { - d, output := expander.fn(ctx, b, p, lv) - diags = diags.Extend(d) - return dyn.NewValue(output, lv.Locations()), nil - }) - if err != nil { - return dyn.InvalidValue, err - } + for _, expander := range expanders { + err := structvar.ForEach(b.Config.View(), expander.pattern, func(p *structpath.PathNode, lv structvar.View) error { + diags = diags.Extend(expander.fn(ctx, lv, p, b)) + return nil + }) + if err != nil { + diags = diags.Extend(diag.FromErr(err)) + break } - - return v, nil - }) - if err != nil { - diags = diags.Extend(diag.FromErr(err)) } return diags diff --git a/bundle/libraries/expand_glob_references_test.go b/bundle/libraries/expand_glob_references_test.go index 44ffe97a6eb..5bb4a49fff8 100644 --- a/bundle/libraries/expand_glob_references_test.go +++ b/bundle/libraries/expand_glob_references_test.go @@ -9,7 +9,8 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/internal/testutil" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/pipelines" @@ -63,7 +64,7 @@ func TestGlobReferencesExpandedForTaskLibraries(t *testing.T) { }, } - bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "resource.yml")}}) diags := bundle.Apply(t.Context(), b, ExpandGlobReferences()) require.Empty(t, diags) @@ -148,7 +149,7 @@ func TestGlobReferencesExpandedForForeachTaskLibraries(t *testing.T) { }, } - bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "resource.yml")}}) diags := bundle.Apply(t.Context(), b, ExpandGlobReferences()) require.Empty(t, diags) @@ -223,7 +224,7 @@ func TestGlobReferencesExpandedForEnvironmentsDeps(t *testing.T) { }, } - bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "resource.yml")}}) diags := bundle.Apply(t.Context(), b, ExpandGlobReferences()) require.Empty(t, diags) @@ -276,17 +277,25 @@ func TestExpandGlobReferencesPreservesLocations(t *testing.T) { }, } - loc := dyn.Location{File: filepath.Join(dir, "resource.yml"), Line: 10, Column: 5} - bundletest.SetLocation(b, ".", []dyn.Location{loc}) + loc := diag.Location{File: filepath.Join(dir, "resource.yml"), Line: 10, Column: 5} + bundletest.SetLocation(b, ".", []diag.Location{loc}) diags := bundle.Apply(t.Context(), b, ExpandGlobReferences()) require.Empty(t, diags) - libs, err := dyn.GetByPath(b.Config.Value(), dyn.MustPathFromString("resources.jobs.job.tasks[0].libraries")) - require.NoError(t, err) - assert.Equal(t, loc.File, libs.Location().File) + libs := b.Config.LocationsAt(structpath.MustParsePath("resources.jobs.job.tasks[0].libraries")) + require.NotEmpty(t, libs) + assert.Equal(t, loc.File, libs[0].File) - deps, err := dyn.GetByPath(b.Config.Value(), dyn.MustPathFromString("resources.pipelines.pipeline.environment.dependencies")) - require.NoError(t, err) - assert.Equal(t, loc.File, deps.Location().File) + lib := b.Config.LocationsAt(structpath.MustParsePath("resources.jobs.job.tasks[0].libraries[0].whl")) + require.NotEmpty(t, lib) + assert.Equal(t, loc.File, lib[0].File) + + deps := b.Config.LocationsAt(structpath.MustParsePath("resources.pipelines.pipeline.environment.dependencies")) + require.NotEmpty(t, deps) + assert.Equal(t, loc.File, deps[0].File) + + dep := b.Config.LocationsAt(structpath.MustParsePath("resources.pipelines.pipeline.environment.dependencies[0]")) + require.NotEmpty(t, dep) + assert.Equal(t, loc.File, dep[0].File) } diff --git a/bundle/libraries/remote_path.go b/bundle/libraries/remote_path.go index d45e8ff4c7e..d28e21c10c1 100644 --- a/bundle/libraries/remote_path.go +++ b/bundle/libraries/remote_path.go @@ -11,8 +11,9 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/patchwheel" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) // LocalLibraryPaths returns the local file paths of all libraries in the bundle @@ -101,25 +102,20 @@ func ReplaceWithRemotePath(ctx context.Context, b *bundle.Bundle) (map[string][] sources := slices.Sorted(maps.Keys(libs)) // Update all the config paths to point to the uploaded location - err = b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - for _, source := range sources { - locations := libs[source] - remotePath := path.Join(uploadPath, filepath.Base(source)) - - for _, location := range locations { - // Re-append the extras suffix that was stripped before upload. - remotePathWithExtras := remotePath + location.extras - v, err = dyn.SetByPath(v, location.configPath, dyn.NewValue(remotePathWithExtras, []dyn.Location{location.location})) - if err != nil { - return v, fmt.Errorf("internal error: failed to update path %#v to %#v: %w", source, remotePathWithExtras, err) - } + for _, source := range sources { + locations := libs[source] + remotePath := path.Join(uploadPath, filepath.Base(source)) + + for _, location := range locations { + // Re-append the extras suffix that was stripped before upload. + remotePathWithExtras := remotePath + location.extras + err = b.Config.Set(location.configPath, remotePathWithExtras) + if err != nil { + diags = diags.Extend(diag.FromErr(fmt.Errorf("internal error: failed to update path %#v to %#v: %w", source, remotePathWithExtras, err))) + return libs, diags } + b.Config.SetLocations(location.configPath, []diag.Location{location.location}) } - - return v, nil - }) - if err != nil { - diags = diags.Extend(diag.FromErr(err)) } return libs, diags @@ -134,15 +130,15 @@ func ReplaceWithRemotePath(ctx context.Context, b *bundle.Bundle) (map[string][] func collectLocalLibraries(b *bundle.Bundle) (map[string][]LocationToUpdate, error) { libs := make(map[string]([]LocationToUpdate)) - patterns := []dyn.Pattern{ - taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), - taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), - forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), - forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), - clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), - clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), - envDepsPattern.Append(dyn.AnyIndex()), - pipelineEnvDepsPattern.Append(dyn.AnyIndex()), + patterns := []*structpath.PatternNode{ + structpath.MustParsePattern(taskLibrariesPattern.String() + "[*].whl"), + structpath.MustParsePattern(taskLibrariesPattern.String() + "[*].jar"), + structpath.MustParsePattern(forEachTaskLibrariesPattern.String() + "[*].whl"), + structpath.MustParsePattern(forEachTaskLibrariesPattern.String() + "[*].jar"), + structpath.MustParsePattern(clusterLibrariesPattern.String() + "[*].whl"), + structpath.MustParsePattern(clusterLibrariesPattern.String() + "[*].jar"), + structpath.MustParsePattern(envDepsPattern.String() + "[*]"), + structpath.MustParsePattern(pipelineEnvDepsPattern.String() + "[*]"), // The AI Runtime task's code_source_path is a local archive (typically an // artifact-built .tar.gz) that must be uploaded and referenced by its remote // path, exactly like a wheel or jar library. @@ -150,75 +146,63 @@ func collectLocalLibraries(b *bundle.Bundle) (map[string][]LocationToUpdate, err forEachAiRuntimeCodeSourcePattern, } + root := b.Config.View() for _, pattern := range patterns { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - source, ok := v.AsString() - if !ok { - return v, fmt.Errorf("expected string, got %s", v.Kind()) - } - - if !IsLibraryLocal(source) { - return v, nil - } + err := structvar.ForEach(root, pattern, func(p *structpath.PathNode, v structvar.View) error { + source, ok := v.AsString() + if !ok { + return fmt.Errorf("expected string, got %s", v.Kind()) + } - // Split off any pip extras suffix so the upload targets the real - // file; the suffix is re-appended to the remote path afterwards. - source, extras := patchwheel.SplitWheelExtras(source) + if !IsLibraryLocal(source) { + return nil + } - source = filepath.Join(b.SyncRootPath, source) - libs[source] = append(libs[source], LocationToUpdate{ - configPath: p, - location: v.Location(), - extras: extras, - }) + // Split off any pip extras suffix so the upload targets the real + // file; the suffix is re-appended to the remote path afterwards. + source, extras := patchwheel.SplitWheelExtras(source) - return v, nil + source = filepath.Join(b.SyncRootPath, source) + libs[source] = append(libs[source], LocationToUpdate{ + configPath: p, + location: v.Location(), + extras: extras, }) + + return nil }) if err != nil { return nil, err } } - artifactPattern := dyn.NewPattern( - dyn.Key("artifacts"), - dyn.AnyKey(), - dyn.Key("files"), - dyn.AnyIndex(), - ) - - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.MapByPattern(v, artifactPattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - file, ok := v.AsMap() - if !ok { - return v, fmt.Errorf("expected map, got %s", v.Kind()) - } + artifactPattern := structpath.MustParsePattern("artifacts.*.files[*]") - sv, ok := file.GetByString("source") - if !ok { - return v, nil - } + err := structvar.ForEach(root, artifactPattern, func(p *structpath.PathNode, v structvar.View) error { + if v.Kind() != structvar.KindMap { + return fmt.Errorf("expected map, got %s", v.Kind()) + } - source, ok := sv.AsString() - if !ok { - return v, fmt.Errorf("expected string, got %s", v.Kind()) - } + sv := v.Get("source") + if !sv.IsValid() { + return nil + } - if sv, ok = file.GetByString("patched"); ok { - patched, ok := sv.AsString() - if ok && patched != "" { - source = patched - } - } + source, ok := sv.AsString() + if !ok { + return fmt.Errorf("expected string, got %s", v.Kind()) + } - libs[source] = append(libs[source], LocationToUpdate{ - configPath: p.Append(dyn.Key("remote_path")), - location: v.Location(), - }) + if patched, ok := v.Get("patched").AsString(); ok && patched != "" { + source = patched + } - return v, nil + libs[source] = append(libs[source], LocationToUpdate{ + configPath: structpath.NewStringKey(p, "remote_path"), + location: v.Location(), }) + + return nil }) if err != nil { return nil, err diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go index 1d50c946a27..a2ba9591a4f 100644 --- a/bundle/libraries/same_name_libraries.go +++ b/bundle/libraries/same_name_libraries.go @@ -7,26 +7,26 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type checkForSameNameLibraries struct{} -var patterns = []dyn.Pattern{ - taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), - taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), - forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), - forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), - clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), - clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), - envDepsPattern.Append(dyn.AnyIndex()), - pipelineEnvDepsPattern.Append(dyn.AnyIndex()), +var patterns = []*structpath.PatternNode{ + structpath.MustParsePattern(taskLibrariesPattern.String() + "[*].whl"), + structpath.MustParsePattern(taskLibrariesPattern.String() + "[*].jar"), + structpath.MustParsePattern(forEachTaskLibrariesPattern.String() + "[*].whl"), + structpath.MustParsePattern(forEachTaskLibrariesPattern.String() + "[*].jar"), + structpath.MustParsePattern(clusterLibrariesPattern.String() + "[*].whl"), + structpath.MustParsePattern(clusterLibrariesPattern.String() + "[*].jar"), + structpath.MustParsePattern(envDepsPattern.String() + "[*]"), + structpath.MustParsePattern(pipelineEnvDepsPattern.String() + "[*]"), } type libData struct { fullPath string - locations []dyn.Location + locations []diag.Location paths []*structpath.PathNode otherPaths []string } @@ -35,50 +35,43 @@ func (c checkForSameNameLibraries) Apply(ctx context.Context, b *bundle.Bundle) var diags diag.Diagnostics libs := make(map[string]*libData) - err := b.Config.Mutate(func(rootConfig dyn.Value) (dyn.Value, error) { - var err error - for _, pattern := range patterns { - rootConfig, err = dyn.MapByPattern(rootConfig, pattern, func(p dyn.Path, libraryValue dyn.Value) (dyn.Value, error) { - libPath, ok := libraryValue.AsString() - if !ok { - return libraryValue, nil - } + root := b.Config.View() + var err error + for _, pattern := range patterns { + err = structvar.ForEach(root, pattern, func(p *structpath.PathNode, libraryValue structvar.View) error { + libPath, ok := libraryValue.AsString() + if !ok { + return nil + } - // If not local library, skip the check - if !IsLibraryLocal(libPath) { - return libraryValue, nil - } + // If not local library, skip the check + if !IsLibraryLocal(libPath) { + return nil + } - lib := filepath.Base(libPath) - // If the same basename was seen already but full path is different - // then it's a duplicate. Add the location to the location list. - lp, ok := libs[lib] - if !ok { - libs[lib] = &libData{ - fullPath: libPath, - locations: []dyn.Location{libraryValue.Location()}, - paths: dyn.ToStructPaths(p), - otherPaths: []string{}, - } - } else if lp.fullPath != libPath { - lp.locations = append(lp.locations, libraryValue.Location()) - lp.paths = append(lp.paths, dyn.ToStructPath(p)) - lp.otherPaths = append(lp.otherPaths, libPath) + lib := filepath.Base(libPath) + // If the same basename was seen already but full path is different + // then it's a duplicate. Add the location to the location list. + lp, ok := libs[lib] + if !ok { + libs[lib] = &libData{ + fullPath: libPath, + locations: []diag.Location{libraryValue.Location()}, + paths: []*structpath.PathNode{p}, + otherPaths: []string{}, } - - return libraryValue, nil - }) - if err != nil { - return dyn.InvalidValue, err + } else if lp.fullPath != libPath { + lp.locations = append(lp.locations, libraryValue.Location()) + lp.paths = append(lp.paths, p) + lp.otherPaths = append(lp.otherPaths, libPath) } - } + return nil + }) if err != nil { - return dyn.InvalidValue, err + break } - - return rootConfig, nil - }) + } // Iterate over all the libraries and check if there are any duplicates. // Duplicates will have more than one location. diff --git a/bundle/libraries/upload.go b/bundle/libraries/upload.go index dabd4d8860d..09e81ab01b4 100644 --- a/bundle/libraries/upload.go +++ b/bundle/libraries/upload.go @@ -12,10 +12,9 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" - + "github.com/databricks/cli/libs/structs/structpath" "golang.org/x/sync/errgroup" ) @@ -36,8 +35,8 @@ type upload struct { } type LocationToUpdate struct { - configPath dyn.Path - location dyn.Location + configPath *structpath.PathNode + location diag.Location // extras is the pip extras suffix (e.g. "[train]") to re-append to the // rewritten remote path. Empty for libraries that carry no extras. extras string diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index 0f0776152eb..4bc28889fc5 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -13,8 +13,6 @@ import ( "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/cli/libs/structs/structpath" @@ -58,18 +56,20 @@ func BuildStateFromTF( warningsSeen := false // Collect all resource nodes (same patterns as makePlan). var nodes []string - patterns := []dyn.Pattern{ - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("permissions")), - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("grants")), + rootView := configRoot.View() + + patterns := []*structpath.PatternNode{ + structpath.MustParsePattern("resources.*.*"), + structpath.MustParsePattern("resources.*.*.permissions"), + structpath.MustParsePattern("resources.*.*.grants"), } for _, pat := range patterns { - _, err := dyn.MapByPattern( - configRoot.Value(), + err := structvar.ForEach( + rootView, pat, - func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + func(p *structpath.PathNode, _ structvar.View) error { nodes = append(nodes, p.String()) - return dyn.InvalidValue, nil + return nil }, ) if err != nil { @@ -112,7 +112,7 @@ func BuildStateFromTF( return warningsSeen, fmt.Errorf("%s: PrepareState: %w", node, err) } - refs, err := direct.ExtractReferences(configRoot.Value(), node, adapter.StateType()) + refs, err := direct.ExtractReferences(rootView, node, adapter.StateType()) if err != nil { return warningsSeen, fmt.Errorf("%s: extracting references: %w", node, err) } @@ -125,12 +125,12 @@ func BuildStateFromTF( // Same logic as makePlan in bundle/direct/bundle_plan.go. var dependsOn []deployplan.DependsOnEntry //nolint:prealloc for _, refTemplate := range refs { - ref, ok := dynvar.NewRef(dyn.V(refTemplate)) + ref, ok := structvar.NewRef(refTemplate) if !ok { continue } for _, targetPath := range ref.References() { - targetPathParsed, err := dyn.NewPathFromString(targetPath) + targetPathParsed, err := structpath.ParsePath(targetPath) if err != nil { continue } diff --git a/bundle/migrate/build_state_test.go b/bundle/migrate/build_state_test.go index a495436008c..c73ac2131b9 100644 --- a/bundle/migrate/build_state_test.go +++ b/bundle/migrate/build_state_test.go @@ -1,7 +1,6 @@ package migrate_test import ( - "bytes" "encoding/json" "os" "path/filepath" @@ -12,24 +11,18 @@ import ( "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/migrate" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // rootFromYAML builds a config.Root from a YAML snippet. // Template strings like "${resources.jobs.src.name}" are preserved in the -// internal dyn.Value so BuildStateFromTF can find them via ExtractReferences. +// configuration so BuildStateFromTF can find them via ExtractReferences. func rootFromYAML(t *testing.T, yaml string) config.Root { t.Helper() - v, err := yamlloader.LoadYAML("test", bytes.NewBufferString(yaml)) - require.NoError(t, err) - var root config.Root - require.NoError(t, convert.ToTyped(&root, v)) - require.NoError(t, root.Mutate(func(_ dyn.Value) (dyn.Value, error) { return v, nil })) - return root + root, diags := config.LoadFromBytes("test", []byte(yaml)) + require.NoError(t, diags.Error()) + return *root } func runBuildStateFromTF( diff --git a/bundle/migrate/resolve.go b/bundle/migrate/resolve.go index 23d0180e3c6..41e09176e47 100644 --- a/bundle/migrate/resolve.go +++ b/bundle/migrate/resolve.go @@ -5,16 +5,15 @@ import ( "fmt" "strings" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) // evaluateTemplate evaluates a template string like "${resources.pipelines.bar.cluster[0].label}" // by looking up each ${...} reference from TF state. func evaluateTemplate(state TFStateAttrs, template string) (string, error) { - ref, ok := dynvar.NewRef(dyn.V(template)) + ref, ok := structvar.NewRef(template) if !ok { return template, nil } diff --git a/bundle/mutator.go b/bundle/mutator.go index 41ef8fda48c..ed9bd5e56e5 100644 --- a/bundle/mutator.go +++ b/bundle/mutator.go @@ -75,19 +75,6 @@ func ApplyContext(ctx context.Context, b *Bundle, m Mutator) { log.Debugf(ctx, "Apply") - err := b.Config.MarkMutatorEntry(ctx) - if err != nil { - logdiag.LogError(ctx, fmt.Errorf("entry error: %w", err)) - return - } - - defer func() { - err := b.Config.MarkMutatorExit(ctx) - if err != nil { - logdiag.LogError(ctx, fmt.Errorf("exit error: %w", err)) - } - }() - diags := m.Apply(ctx, b) for _, d := range diags { @@ -104,34 +91,6 @@ func ApplySeqContext(ctx context.Context, b *Bundle, mutators ...Mutator) { } } -// ApplySeqInScopeContext applies mutators without opening a mutator scope per mutator, -// reusing the caller's scope instead. -// -// [ApplyContext] converts the whole configuration tree between its typed and dynamic -// representations on entry and exit (see [config.Root.MarkMutatorEntry]). That cost is -// proportional to the size of the accumulated configuration, so applying N mutators this -// way is quadratic in N. For a bundle with thousands of included files that dominates -// load time, hence this variant. -// -// Only use it for mutators that modify the configuration through [config.Root.Mutate] -// (which keeps both representations in sync). A mutator that assigns to a typed field -// directly relies on the scope entry to carry that value into the dynamic tree, and -// would lose it here. -func ApplySeqInScopeContext(ctx context.Context, b *Bundle, mutators ...Mutator) { - for _, m := range mutators { - mctx := log.NewContext(ctx, log.GetLogger(ctx).With("mutator", m.Name())) - log.Debugf(mctx, "Apply") - - for _, d := range m.Apply(mctx, b) { - logdiag.LogDiag(mctx, d) - } - - if logdiag.HasError(ctx) { - break - } - } -} - type funcMutator struct { fn func(context.Context, *Bundle) } diff --git a/bundle/mutator_test.go b/bundle/mutator_test.go index 45473634119..c8bddb24eed 100644 --- a/bundle/mutator_test.go +++ b/bundle/mutator_test.go @@ -5,10 +5,7 @@ import ( "testing" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/logdiag" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) type testMutator struct { @@ -52,79 +49,6 @@ func TestMutator(t *testing.T) { assert.Equal(t, 1, nested[1].applyCalled) } -// ApplySeqInScopeContext applies each mutator once, in order. -func TestApplySeqInScopeContext(t *testing.T) { - var order []string - makeMutator := func(name string) *testMutator { - return &testMutator{fn: func(ctx context.Context, b *Bundle) diag.Diagnostics { - order = append(order, name) - return nil - }} - } - first := makeMutator("first") - second := makeMutator("second") - - b := &Bundle{} - ctx := logdiag.InitContext(t.Context()) - logdiag.SetCollect(ctx, true) - ApplySeqInScopeContext(ctx, b, first, second) - - assert.Equal(t, 1, first.applyCalled) - assert.Equal(t, 1, second.applyCalled) - assert.Equal(t, []string{"first", "second"}, order) - assert.Empty(t, logdiag.FlushCollected(ctx)) -} - -// ApplySeqInScopeContext collects the diagnostics returned by each mutator and -// stops at the first one that logs an error, without applying later mutators. -func TestApplySeqInScopeContextStopsOnError(t *testing.T) { - failing := &testMutator{fn: func(ctx context.Context, b *Bundle) diag.Diagnostics { - return diag.Diagnostics{diag.Diagnostic{Severity: diag.Error, Summary: "boom"}} - }} - later := &testMutator{} - - b := &Bundle{} - ctx := logdiag.InitContext(t.Context()) - logdiag.SetCollect(ctx, true) - ApplySeqInScopeContext(ctx, b, failing, later) - - assert.Equal(t, 1, failing.applyCalled) - assert.Equal(t, 0, later.applyCalled, "mutator after an error must not run") - - diags := logdiag.FlushCollected(ctx) - require.Len(t, diags, 1) - assert.Equal(t, "boom", diags[0].Summary) -} - -// Unlike ApplySeqContext, ApplySeqInScopeContext does not open a mutator scope per -// mutator, so it must be called from within one. Changes made through Root.Mutate -// keep the typed and dynamic configuration in sync and survive without a per-mutator -// scope. This is the property ProcessRootIncludes relies on. -func TestApplySeqInScopeContextPreservesMutateChanges(t *testing.T) { - setHost := &testMutator{fn: func(ctx context.Context, b *Bundle) diag.Diagnostics { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "bundle", dyn.V(map[string]dyn.Value{"name": dyn.V("set-in-scope")})) - }) - require.NoError(t, err) - return nil - }} - - // Enclosing mutator that applies setHost in its own scope, mirroring how - // ProcessRootIncludes applies the per-file includes. - outer := &testMutator{fn: func(ctx context.Context, b *Bundle) diag.Diagnostics { - ApplySeqInScopeContext(ctx, b, setHost) - return nil - }} - - b := &Bundle{} - diags := Apply(t.Context(), b, outer) - require.NoError(t, diags.Error()) - - // Visible in both representations once the enclosing scope exits. - assert.Equal(t, "set-in-scope", b.Config.Bundle.Name) - assert.Equal(t, "set-in-scope", b.Config.Value().Get("bundle").Get("name").MustString()) -} - func TestSafeMutatorName(t *testing.T) { tests := []struct { name string @@ -137,9 +61,9 @@ func TestSafeMutatorName(t *testing.T) { expected: "bundle.(funcMutator)", }, { - name: "setDefault mutator", - mutator: SetDefaultMutator(dyn.NewPattern(dyn.Key("test")), "key", "value"), - expected: "bundle.(setDefault)", + name: "setDefaults mutator", + mutator: &setDefaults{}, + expected: "bundle.(setDefaults)", }, { name: "funcMutator as pointer", diff --git a/bundle/permissions/permission_diagnostics.go b/bundle/permissions/permission_diagnostics.go index e25ccd5e724..0e01f72a0f4 100644 --- a/bundle/permissions/permission_diagnostics.go +++ b/bundle/permissions/permission_diagnostics.go @@ -8,7 +8,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/iamutil" "github.com/databricks/cli/libs/set" ) @@ -59,7 +58,7 @@ func (m *permissionDiagnostics) Apply(ctx context.Context, b *bundle.Bundle) dia identityType, b.Config.Workspace.CurrentUser.UserName, ), - Locations: []dyn.Location{b.Config.GetLocation("permissions")}, + Locations: []diag.Location{b.Config.GetLocation("permissions")}, ID: diag.PermissionNotIncluded, }} } diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index 4be81a102d2..0bd7fa5e432 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -13,7 +13,7 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/statemgmt" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) // PreDeployChecks is common set of mutators between "bundle plan" and "bundle deploy". @@ -52,22 +52,22 @@ func pipelineDeletionCascades(b *bundle.Bundle, action deployplan.Action) (bool, // checkForPreventDestroy checks if the resource has lifecycle.prevent_destroy set, but the plan calls for this resource to be recreated or destroyed. // If it does, it returns an error. func checkForPreventDestroy(b *bundle.Bundle, actions []deployplan.Action) error { - root := b.Config.Value() + root := b.Config.View() var errs []error for _, action := range actions { if action.ActionType != deployplan.Recreate && action.ActionType != deployplan.Delete { continue } - path, err := dyn.NewPathFromString(action.ResourceKey) + path, err := structpath.ParsePath(action.ResourceKey) if err != nil { return fmt.Errorf("failed to parse %q", action.ResourceKey) } - path = append(path, dyn.Key("lifecycle"), dyn.Key("prevent_destroy")) + path = structpath.NewPath(path, "lifecycle", "prevent_destroy") - preventDestroyV, err := dyn.GetByPath(root, path) - if err != nil { + preventDestroyV := root.Lookup(path) + if !preventDestroyV.IsValid() { continue } diff --git a/bundle/phases/plan_test.go b/bundle/phases/plan_test.go index 772f7401f9b..6063cc6a0bd 100644 --- a/bundle/phases/plan_test.go +++ b/bundle/phases/plan_test.go @@ -7,9 +7,10 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dstate" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -77,20 +78,14 @@ func TestPipelineDeletionCascades(t *testing.T) { func TestCheckPreventDestroyForAllResources(t *testing.T) { for resourceType := range config.SupportedResources() { + // Snapshots are internal and their lifecycle is not part of the configuration (json:"-"). + if resourceType == "internal_immutable_snapshots" { + continue + } t.Run(resourceType, func(t *testing.T) { b := &bundle.Bundle{} - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources", dyn.NewValue(map[string]dyn.Value{ - resourceType: dyn.NewValue(map[string]dyn.Value{ - "test_resource": dyn.NewValue(map[string]dyn.Value{ - "lifecycle": dyn.NewValue(map[string]dyn.Value{ - "prevent_destroy": dyn.NewValue(true, nil), - }, nil), - }, nil), - }, nil), - }, nil)) - }) + err := b.Config.Set(structpath.MustParsePath("resources."+resourceType+".test_resource.lifecycle.prevent_destroy"), true) require.NoError(t, err) actions := []deployplan.Action{ @@ -111,17 +106,7 @@ func TestCheckPreventDestroyForAllResources(t *testing.T) { func TestCheckPreventDestroyForJob(t *testing.T) { b := &bundle.Bundle{} - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources", dyn.NewValue(map[string]dyn.Value{ - "jobs": dyn.NewValue(map[string]dyn.Value{ - "test_resource": dyn.NewValue(map[string]dyn.Value{ - "lifecycle": dyn.NewValue(map[string]dyn.Value{ - "prevent_destroy": dyn.NewValue(true, nil), - }, nil), - }, nil), - }, nil), - }, nil)) - }) + err := b.Config.Set(structpath.MustParsePath("resources.jobs.test_resource.lifecycle.prevent_destroy"), true) require.NoError(t, err) actions := []deployplan.Action{ @@ -140,17 +125,7 @@ func TestCheckPreventDestroyForJob(t *testing.T) { func TestCheckPreventDestroyForApp(t *testing.T) { b := &bundle.Bundle{} - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources", dyn.NewValue(map[string]dyn.Value{ - "apps": dyn.NewValue(map[string]dyn.Value{ - "test_resource": dyn.NewValue(map[string]dyn.Value{ - "lifecycle": dyn.NewValue(map[string]dyn.Value{ - "prevent_destroy": dyn.NewValue(true, nil), - }, nil), - }, nil), - }, nil), - }, nil)) - }) + err := b.Config.Set(structpath.MustParsePath("resources.apps.test_resource.lifecycle.prevent_destroy"), true) require.NoError(t, err) actions := []deployplan.Action{ @@ -167,14 +142,7 @@ func TestCheckPreventDestroyForApp(t *testing.T) { func TestCheckPreventDestroyNoError(t *testing.T) { b := &bundle.Bundle{} - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources", dyn.NewValue(map[string]dyn.Value{ - "jobs": dyn.NewValue(map[string]dyn.Value{ - "test_resource": dyn.NewValue(map[string]dyn.Value{}, nil), - }, nil), - }, nil)) - }) - require.NoError(t, err) + b.Config.Resources.Jobs = map[string]*resources.Job{"test_resource": {}} actions := []deployplan.Action{ { @@ -183,26 +151,14 @@ func TestCheckPreventDestroyNoError(t *testing.T) { }, } - err = checkForPreventDestroy(b, actions) + err := checkForPreventDestroy(b, actions) require.NoError(t, err) } func TestCheckForPreventDestroyWhenFirstHasNoPreventDestroy(t *testing.T) { b := &bundle.Bundle{} - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, "resources", dyn.NewValue(map[string]dyn.Value{ - "jobs": dyn.NewValue(map[string]dyn.Value{ - "test_job": dyn.NewValue(map[string]dyn.Value{}, nil), - }, nil), - "apps": dyn.NewValue(map[string]dyn.Value{ - "test_app": dyn.NewValue(map[string]dyn.Value{ - "lifecycle": dyn.NewValue(map[string]dyn.Value{ - "prevent_destroy": dyn.NewValue(true, nil), - }, nil), - }, nil), - }, nil), - }, nil)) - }) + b.Config.Resources.Jobs = map[string]*resources.Job{"test_job": {}} + err := b.Config.Set(structpath.MustParsePath("resources.apps.test_app.lifecycle.prevent_destroy"), true) require.NoError(t, err) actions := []deployplan.Action{ diff --git a/bundle/phases/telemetry.go b/bundle/phases/telemetry.go index 0806e13ae66..45274ca58fd 100644 --- a/bundle/phases/telemetry.go +++ b/bundle/phases/telemetry.go @@ -14,8 +14,9 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/bundle/metrics" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -201,9 +202,9 @@ func LogDeployTelemetry(ctx context.Context, b *bundle.Bundle, errMsg string) { b.Metrics.SetBoolValue(metrics.DeploymentHistoryEnabled, b.DeploymentBundle.StateDB.IsDeploymentMetadataService()) resourcesCount := int64(0) - _, err := dyn.MapByPattern(b.Config.Value(), dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + err := structvar.ForEach(b.Config.View(), structpath.MustParsePattern("resources.*.*"), func(p *structpath.PathNode, v structvar.View) error { resourcesCount++ - return v, nil + return nil }) if err != nil { log.Debugf(ctx, "failed to count resources: %s", err) diff --git a/bundle/render/render_text_output_test.go b/bundle/render/render_text_output_test.go index 3857803a406..4c87fbb4eb9 100644 --- a/bundle/render/render_text_output_test.go +++ b/bundle/render/render_text_output_test.go @@ -11,7 +11,6 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/catalog" @@ -160,7 +159,7 @@ func TestRenderDiagnostics(t *testing.T) { Severity: diag.Error, Summary: "failed to load xxx", Detail: "'name' is required", - Locations: []dyn.Location{{ + Locations: []diag.Location{{ File: "foo.yaml", Line: 1, Column: 2, @@ -178,7 +177,7 @@ func TestRenderDiagnostics(t *testing.T) { Severity: diag.Error, Summary: "failed to load xxx", Detail: "'name' is required", - Locations: []dyn.Location{ + Locations: []diag.Location{ { File: "foo.yaml", Line: 1, @@ -244,7 +243,7 @@ func TestRenderDiagnostics(t *testing.T) { "resources.jobs.xxx", "resources.jobs.yyy", ), - Locations: []dyn.Location{ + Locations: []diag.Location{ {File: "foo.yaml", Line: 1, Column: 2}, {File: "bar.yaml", Line: 3, Column: 4}, }, diff --git a/bundle/run/app_test.go b/bundle/run/app_test.go index abdb7dd4f73..d43e8a92b16 100644 --- a/bundle/run/app_test.go +++ b/bundle/run/app_test.go @@ -15,7 +15,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/vfs" "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/databricks/databricks-sdk-go/service/apps" @@ -68,7 +68,7 @@ func setupBundle(t *testing.T) (context.Context, *bundle.Bundle, *mocks.MockWork mwc := mocks.NewMockWorkspaceClient(t) b.SetWorkpaceClient(mwc.WorkspaceClient) - bundletest.SetLocation(b, "resources.apps.my_app", []dyn.Location{{File: filepath.Join(root, "./databricks.yml")}}) + bundletest.SetLocation(b, "resources.apps.my_app", []diag.Location{{File: filepath.Join(root, "./databricks.yml")}}) ctx := cmdio.MockDiscard(t.Context()) diff --git a/bundle/set_default.go b/bundle/set_default.go index 3100f67d57c..5c4543eb1f6 100644 --- a/bundle/set_default.go +++ b/bundle/set_default.go @@ -5,60 +5,66 @@ import ( "fmt" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -type setDefault struct { - pattern dyn.Pattern - key dyn.Path +// Default is a value to set at every path matching Pattern where it is not set yet. +type Default struct { + Pattern string + Value any +} + +type defaultValue struct { + pattern *structpath.PatternNode + key string value any } -func SetDefaultMutator(pattern dyn.Pattern, key string, value any) Mutator { - return &setDefault{ - pattern: pattern, - key: dyn.NewPath(dyn.Key(key)), - value: value, - } +type setDefaults struct { + defaults []defaultValue } -func (m *setDefault) Name() string { - return fmt.Sprintf("SetDefaultMutator(%v, %v, %v)", m.pattern, m.key, m.value) +func (m *setDefaults) Name() string { + return "SetDefaults" } -func (m *setDefault) Apply(ctx context.Context, b *Bundle) diag.Diagnostics { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.MapByPattern(v, m.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - _, err := dyn.GetByPath(v, m.key) - switch { - case dyn.IsNoSuchKeyError(err): - return dyn.SetByPath(v, m.key, dyn.V(m.value)) - default: - return v, err +func (m *setDefaults) Apply(ctx context.Context, b *Bundle) diag.Diagnostics { + for _, d := range m.defaults { + err := structvar.ForEach(b.Config.View(), d.pattern, func(p *structpath.PathNode, v structvar.View) error { + if v.Get(d.key).IsValid() { + return nil } + return b.Config.Set(structpath.NewStringKey(p, d.key), d.value) }) - }) - if err != nil { - return diag.FromErr(err) + if err != nil { + return diag.FromErr(err) + } } return nil } -func SetDefault(ctx context.Context, b *Bundle, pattern string, value any) { - pat, err := dyn.NewPatternFromString(pattern) - if err != nil { - logdiag.LogError(ctx, fmt.Errorf("internal error: invalid pattern: %s: %w", pattern, err)) - return - } +// SetDefaults sets each default value at every path matching its pattern where no value is set. +// Defaults are applied in order. +func SetDefaults(ctx context.Context, b *Bundle, defaults []Default) { + m := &setDefaults{} + for _, d := range defaults { + pat, err := structpath.ParsePattern(d.Pattern) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("internal error: invalid pattern: %s: %w", d.Pattern, err)) + return + } + + key, ok := pat.StringKey() + if !ok || key == "" { + logdiag.LogError(ctx, fmt.Errorf("internal error: invalid pattern: %s", d.Pattern)) + return + } - pat, key := pat.SplitKey() - if pat == nil || key == "" { - logdiag.LogError(ctx, fmt.Errorf("internal error: invalid pattern: %s", pattern)) - return + m.defaults = append(m.defaults, defaultValue{pattern: pat.Parent(), key: key, value: d.Value}) } - m := SetDefaultMutator(pat, key, value) ApplyContext(ctx, b, m) } diff --git a/bundle/statemgmt/direct_migration.go b/bundle/statemgmt/direct_migration.go index f3d632f138f..7ad17b780ab 100644 --- a/bundle/statemgmt/direct_migration.go +++ b/bundle/statemgmt/direct_migration.go @@ -21,7 +21,6 @@ import ( "github.com/databricks/cli/bundle/metrics" "github.com/databricks/cli/bundle/migrate" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" @@ -269,19 +268,11 @@ func convertTFStateToDirect(ctx context.Context, b *bundle.Bundle, tfState *migr // The config may use terraform-style references (${databricks_pipeline.x.id}). // BuildStateFromTF expects ${resources.*} references, so rewrite them first. - uninterpolatedRoot, err := reverseInterpolate(b.Config.Value()) + uninterpolatedConfig, err := reverseInterpolateConfig(&b.Config) if err != nil { return tempStatePath, false, nil, fmt.Errorf("failed to reverse interpolation: %w", err) } - var uninterpolatedConfig config.Root - err = uninterpolatedConfig.Mutate(func(_ dyn.Value) (dyn.Value, error) { - return uninterpolatedRoot, nil - }) - if err != nil { - return tempStatePath, false, nil, fmt.Errorf("failed to create uninterpolated config: %w", err) - } - adapters, err := dresources.InitAll(nil) if err != nil { return tempStatePath, false, nil, err @@ -296,7 +287,7 @@ func convertTFStateToDirect(ctx context.Context, b *bundle.Bundle, tfState *migr } // warnPrefix labels the conversion's warnings as coming from the background dry run. - hasWarnings, err := migrate.BuildStateFromTF(ctx, &uninterpolatedConfig, adapters, &stateDB, tfState.Attrs, tfState.IDs, warnPrefix) + hasWarnings, err := migrate.BuildStateFromTF(ctx, uninterpolatedConfig, adapters, &stateDB, tfState.Attrs, tfState.IDs, warnPrefix) if err != nil { return tempStatePath, hasWarnings, nil, err } @@ -310,5 +301,5 @@ func convertTFStateToDirect(ctx context.Context, b *bundle.Bundle, tfState *migr return tempStatePath, hasWarnings, nil, errors.New("state conversion failed") } - return tempStatePath, hasWarnings, &uninterpolatedConfig, nil + return tempStatePath, hasWarnings, uninterpolatedConfig, nil } diff --git a/bundle/statemgmt/reverse_interpolate.go b/bundle/statemgmt/reverse_interpolate.go index d133c0e0f7d..b498f85eafb 100644 --- a/bundle/statemgmt/reverse_interpolate.go +++ b/bundle/statemgmt/reverse_interpolate.go @@ -3,34 +3,85 @@ package statemgmt import ( "fmt" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/deploy/terraform" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) -// reverseInterpolate converts terraform-style resource references to bundle-style. +// reverseInterpolate converts terraform-style resource references in s to bundle-style. // Example: ${databricks_pipeline.my_etl.id} → ${resources.pipelines.my_etl.id} -func reverseInterpolate(root dyn.Value) (dyn.Value, error) { - return dynvar.Resolve(root, func(path dyn.Path) (dyn.Value, error) { +func reverseInterpolate(s string) (string, error) { + out, err := structvar.Resolve(map[string]structvar.Template{"": {Value: s}}, func(sp *structpath.PathNode) (structvar.View, error) { // Need at least 2 components: resource_type.resource_name - if len(path) < 2 { - return dyn.InvalidValue, dynvar.ErrSkipResolution + if sp.Len() < 2 { + return structvar.View{}, structvar.ErrSkipResolution } - resourceType := path[0].Key() + resourceType := sp.KeyAt(0) isAlreadyBundleFormat := resourceType == "resources" if isAlreadyBundleFormat { - return dyn.InvalidValue, dynvar.ErrSkipResolution + return structvar.View{}, structvar.ErrSkipResolution } bundleGroup, ok := terraform.TerraformToGroupName[resourceType] if !ok { - return dyn.InvalidValue, dynvar.ErrSkipResolution + return structvar.View{}, structvar.ErrSkipResolution } // Reconstruct path in bundle format: // databricks_pipeline.my_pipeline.id → resources.pipelines.my_pipeline.id - bundlePath := dyn.NewPath(dyn.Key("resources"), dyn.Key(bundleGroup)).Append(path[1:]...) - return dyn.V(fmt.Sprintf("${%s}", bundlePath.String())), nil + ref := fmt.Sprintf("${resources.%s.%s}", bundleGroup, sp.SkipPrefix(1)) + return structvar.NewView(&ref, nil, nil), nil }) + if err != nil { + return "", err + } + if v, ok := out[""]; ok { + rs, _ := v.AsString() + return rs, nil + } + return s, nil +} + +// reverseInterpolateConfig returns a copy of cfg in which terraform-style references +// are rewritten to bundle-style ones (see [reverseInterpolate]). cfg is not changed. +func reverseInterpolateConfig(cfg *config.Root) (*config.Root, error) { + out := &config.Root{} + if err := out.Assign(nil, cfg.View()); err != nil { + return nil, err + } + + type update struct { + path *structpath.PathNode + s string + } + var updates []update + err := structvar.Walk(cfg.View(), func(p *structpath.PathNode, v structvar.View) error { + s, ok := v.AsString() + if !ok { + return nil + } + if _, ok := structvar.NewRef(s); !ok { + return nil + } + rs, err := reverseInterpolate(s) + if err != nil { + return err + } + if rs != s { + updates = append(updates, update{p, rs}) + } + return nil + }) + if err != nil { + return nil, err + } + + for _, u := range updates { + if err := out.SetReference(u.path, u.s); err != nil { + return nil, err + } + } + return out, nil } diff --git a/bundle/statemgmt/reverse_interpolate_test.go b/bundle/statemgmt/reverse_interpolate_test.go index fe1779cd65b..ef6a40ee5da 100644 --- a/bundle/statemgmt/reverse_interpolate_test.go +++ b/bundle/statemgmt/reverse_interpolate_test.go @@ -6,129 +6,86 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestReverseInterpolatePreservesBConfigValue(t *testing.T) { - // This test verifies that our approach of getting b.Config.Value(), - // reverse interpolating it, and wrapping in a new config.Root - // does NOT mutate b.Config + // This test verifies that reverse interpolating a config returns a new + // config.Root and does NOT mutate the original. - b := &bundle.Bundle{ - Config: config.Root{ - Bundle: config.Bundle{ - Name: "test", - }, - }, - } + root, diags := config.LoadFromBytes("test.yml", []byte(` +bundle: + name: test +resources: + jobs: + my_job: + name: My Job + description: ${databricks_pipeline.my_pipeline.id} + max_concurrent_runs: ${databricks_job.other.id} +`)) + require.NoError(t, diags.Error()) + b := &bundle.Bundle{Config: *root} - err := b.Config.Mutate(func(_ dyn.Value) (dyn.Value, error) { - return dyn.V(map[string]dyn.Value{ - "bundle": dyn.V(map[string]dyn.Value{ - "name": dyn.V("test"), - }), - "resources": dyn.V(map[string]dyn.Value{ - "jobs": dyn.V(map[string]dyn.Value{ - "my_job": dyn.V(map[string]dyn.Value{ - "name": dyn.V("My Job"), - "depends_on": dyn.V("${databricks_pipeline.my_pipeline.id}"), - }), - }), - }), - }), nil - }) + originalJSON, err := json.Marshal(b.Config.View().AsAny()) require.NoError(t, err) - originalValue := b.Config.Value() - originalJSON, err := json.Marshal(originalValue.AsAny()) + uninterpolatedConfig, err := reverseInterpolateConfig(&b.Config) require.NoError(t, err) - interpolatedRoot := b.Config.Value() - - uninterpolatedRoot, err := reverseInterpolate(interpolatedRoot) - require.NoError(t, err) + uninterpolated := uninterpolatedConfig.View() + description, ok := uninterpolated.Lookup(structpath.MustParsePath("resources.jobs.my_job.description")).AsString() + require.True(t, ok) + assert.Equal(t, "${resources.pipelines.my_pipeline.id}", description, "should be bundle-style after reverse interpolation") - dependsOn, err := dyn.GetByPath(uninterpolatedRoot, dyn.MustPathFromString("resources.jobs.my_job.depends_on")) - require.NoError(t, err) - dependsOnStr, ok := dependsOn.AsString() + // References in fields that cannot hold a string are rewritten too. + maxRuns, ok := uninterpolated.Lookup(structpath.MustParsePath("resources.jobs.my_job.max_concurrent_runs")).AsString() require.True(t, ok) - assert.Equal(t, "${resources.pipelines.my_pipeline.id}", dependsOnStr, "should be bundle-style after reverse interpolation") + assert.Equal(t, "${resources.jobs.other.id}", maxRuns) - var uninterpolatedConfig config.Root - err = uninterpolatedConfig.Mutate(func(_ dyn.Value) (dyn.Value, error) { - return uninterpolatedRoot, nil - }) - require.NoError(t, err) + // Locations are kept. + assert.Equal(t, b.Config.GetLocations("resources.jobs.my_job.description"), uninterpolatedConfig.GetLocations("resources.jobs.my_job.description")) - afterValue := b.Config.Value() - afterJSON, err := json.Marshal(afterValue.AsAny()) + afterJSON, err := json.Marshal(b.Config.View().AsAny()) require.NoError(t, err) + assert.Equal(t, string(originalJSON), string(afterJSON), "b.Config should not change") - assert.Equal(t, string(originalJSON), string(afterJSON), "b.Config.Value() should not change") - - originalDependsOn, err := dyn.GetByPath(afterValue, dyn.MustPathFromString("resources.jobs.my_job.depends_on")) - require.NoError(t, err) - originalDependsOnStr, ok := originalDependsOn.AsString() + originalDescription, ok := b.Config.View().Lookup(structpath.MustParsePath("resources.jobs.my_job.description")).AsString() require.True(t, ok) - assert.Equal(t, "${databricks_pipeline.my_pipeline.id}", originalDependsOnStr, "terraform-style reference should be preserved in b.Config") + assert.Equal(t, "${databricks_pipeline.my_pipeline.id}", originalDescription, "terraform-style reference should be preserved in b.Config") } func TestReverseInterpolate(t *testing.T) { tests := []struct { name string - input dyn.Value - expected dyn.Value + input string + expected string }{ { - name: "converts terraform-style job reference to bundle-style", - input: dyn.V(map[string]dyn.Value{ - "job_id": dyn.V("${databricks_job.my_job.id}"), - }), - expected: dyn.V(map[string]dyn.Value{ - "job_id": dyn.V("${resources.jobs.my_job.id}"), - }), + name: "converts terraform-style job reference to bundle-style", + input: "${databricks_job.my_job.id}", + expected: "${resources.jobs.my_job.id}", }, { - name: "leaves bundle-style references unchanged", - input: dyn.V(map[string]dyn.Value{ - "pipeline_id": dyn.V("${resources.pipelines.my_pipeline.id}"), - }), - expected: dyn.V(map[string]dyn.Value{ - "pipeline_id": dyn.V("${resources.pipelines.my_pipeline.id}"), - }), + name: "leaves bundle-style references unchanged", + input: "${resources.pipelines.my_pipeline.id}", + expected: "${resources.pipelines.my_pipeline.id}", }, { - name: "handles nested paths", - input: dyn.V(map[string]dyn.Value{ - "config": dyn.V(map[string]dyn.Value{ - "source": dyn.V("${databricks_pipeline.my_pipeline.url}"), - }), - }), - expected: dyn.V(map[string]dyn.Value{ - "config": dyn.V(map[string]dyn.Value{ - "source": dyn.V("${resources.pipelines.my_pipeline.url}"), - }), - }), + name: "handles nested paths", + input: "${databricks_pipeline.my_pipeline.url}", + expected: "${resources.pipelines.my_pipeline.url}", }, { - name: "skips unknown terraform resource types", - input: dyn.V(map[string]dyn.Value{ - "unknown": dyn.V("${unknown_resource.my_resource.id}"), - }), - expected: dyn.V(map[string]dyn.Value{ - "unknown": dyn.V("${unknown_resource.my_resource.id}"), - }), + name: "skips unknown terraform resource types", + input: "${unknown_resource.my_resource.id}", + expected: "${unknown_resource.my_resource.id}", }, { - name: "handles multiple references in one value", - input: dyn.V(map[string]dyn.Value{ - "combined": dyn.V("${databricks_job.job1.id}/${databricks_pipeline.pipeline1.id}"), - }), - expected: dyn.V(map[string]dyn.Value{ - "combined": dyn.V("${resources.jobs.job1.id}/${resources.pipelines.pipeline1.id}"), - }), + name: "handles multiple references in one value", + input: "${databricks_job.job1.id}/${databricks_pipeline.pipeline1.id}", + expected: "${resources.jobs.job1.id}/${resources.pipelines.pipeline1.id}", }, } @@ -136,7 +93,7 @@ func TestReverseInterpolate(t *testing.T) { t.Run(tt.name, func(t *testing.T) { result, err := reverseInterpolate(tt.input) require.NoError(t, err) - assert.Equal(t, tt.expected.AsAny(), result.AsAny()) + assert.Equal(t, tt.expected, result) }) } } diff --git a/bundle/statemgmt/state_load.go b/bundle/statemgmt/state_load.go index abe260d4d4a..2e3849c8bf9 100644 --- a/bundle/statemgmt/state_load.go +++ b/bundle/statemgmt/state_load.go @@ -3,7 +3,6 @@ package statemgmt import ( "context" "errors" - "fmt" "slices" "strings" @@ -12,7 +11,8 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/statemgmt/resourcestate" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" ) type ( @@ -93,80 +93,64 @@ func applyState(ctx context.Context, b *bundle.Bundle, state ExportedResourcesMa return nil } -func ensureMap(v dyn.Value, path dyn.Path) (dyn.Value, error) { - item, _ := dyn.GetByPath(v, path) - if !item.IsValid() { - var err error - v, err = dyn.SetByPath(v, path, dyn.V(dyn.NewMapping())) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("internal error: failed to create %s: %s", path, err) +func StateToBundle(ctx context.Context, state ExportedResourcesMap, cfg *config.Root) error { + resourcesPath := structpath.NewStringKey(nil, "resources") + if !cfg.View().Lookup(resourcesPath).IsValid() { + if err := cfg.Set(resourcesPath, config.Resources{}); err != nil { + return err } } - return v, nil -} -func StateToBundle(ctx context.Context, state ExportedResourcesMap, config *config.Root) error { - return config.Mutate(func(v dyn.Value) (dyn.Value, error) { - var err error - v, err = ensureMap(v, dyn.Path{dyn.Key("resources")}) - if err != nil { - return v, err + for resourceKey, attrs := range state { + // Parse resource key like "resources.jobs.foo" or "resources.jobs.foo.permissions" + parts := strings.Split(resourceKey, ".") + if len(parts) < 3 || parts[0] != "resources" { + continue // Skip invalid resource keys } - for resourceKey, attrs := range state { - // Parse resource key like "resources.jobs.foo" or "resources.jobs.foo.permissions" - parts := strings.Split(resourceKey, ".") - if len(parts) < 3 || parts[0] != "resources" { - continue // Skip invalid resource keys - } + groupName := parts[1] + resourceName := parts[2] - groupName := parts[1] - resourceName := parts[2] - - // Skip permissions for now as they are sub-resources - if len(parts) > 3 { - continue - } + // Skip permissions for now as they are sub-resources + if len(parts) > 3 { + continue + } - var err error - v, err = ensureMap(v, dyn.Path{dyn.Key("resources"), dyn.Key(groupName)}) - if err != nil { - return v, err - } + if !hasID(groupName) { + continue + } - path := dyn.Path{dyn.Key("resources"), dyn.Key(groupName), dyn.Key(resourceName)} - resource, err := dyn.GetByPath(v, path) - if !resource.IsValid() { - m := dyn.NewMapping() - m.SetLoc("id", nil, dyn.V(attrs.ID)) - m.SetLoc("modified_status", nil, dyn.V(resources.ModifiedStatusDeleted)) - v, err = dyn.SetByPath(v, path, dyn.V(m)) - if err != nil { - return dyn.InvalidValue, err - } - } else if err != nil { - return dyn.InvalidValue, err - } else { - v, err = dyn.SetByPath(v, dyn.Path{dyn.Key("resources"), dyn.Key(groupName), dyn.Key(resourceName), dyn.Key("id")}, dyn.V(attrs.ID)) - if err != nil { - return dyn.InvalidValue, err - } + path := structpath.NewPath(resourcesPath, groupName, resourceName) + if !cfg.View().Lookup(path).IsValid() { + if err := cfg.Set(structpath.NewStringKey(path, "modified_status"), resources.ModifiedStatusDeleted); err != nil { + return err } } + if err := cfg.Set(structpath.NewStringKey(path, "id"), attrs.ID); err != nil { + return err + } + } - return dyn.MapByPattern(v, dyn.Pattern{dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()}, func(p dyn.Path, inner dyn.Value) (dyn.Value, error) { - idPath := dyn.Path{dyn.Key("id")} - statusPath := dyn.Path{dyn.Key("modified_status")} - id, _ := dyn.GetByPath(inner, idPath) - status, _ := dyn.GetByPath(inner, statusPath) - if !id.IsValid() && !status.IsValid() { - return dyn.SetByPath(inner, statusPath, dyn.V(resources.ModifiedStatusCreated)) - } - return inner, nil - }) + return structvar.ForEach(cfg.View(), structpath.MustParsePattern("resources.*.*"), func(p *structpath.PathNode, inner structvar.View) error { + group, _ := p.Parent().StringKey() + if !hasID(group) || inner.Get("id").IsValid() || inner.Get("modified_status").IsValid() { + return nil + } + return cfg.Set(structpath.NewStringKey(p, "modified_status"), resources.ModifiedStatusCreated) }) } +// hasID reports whether resources of group can hold the deployed id and status. Groups +// that are not part of the configuration, and internal snapshots, cannot. +func hasID(group string) bool { + typ, ok := config.ResourcesTypes[group] + if !ok { + return false + } + _, ok = typ.FieldByName("ID") + return ok +} + func validateLoadedState(state ExportedResourcesMap, modes []LoadMode) error { if len(state) == 0 && slices.Contains(modes, ErrorOnEmptyState) { return errors.New("resource not found or not yet deployed. Did you forget to run 'databricks bundle deploy'?") diff --git a/bundle/tests/validate_test.go b/bundle/tests/validate_test.go index 449f2bd37c0..065dbf0652c 100644 --- a/bundle/tests/validate_test.go +++ b/bundle/tests/validate_test.go @@ -6,7 +6,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" @@ -24,7 +23,7 @@ func TestValidateUniqueResourceIdentifiers(t *testing.T) { { Severity: diag.Error, Summary: "multiple resources or scripts have been defined with the same key: foo", - Locations: []dyn.Location{ + Locations: []diag.Location{ {File: "validate/duplicate_resource_names_in_root_job_and_pipeline/databricks.yml", Line: 10, Column: 7}, {File: "validate/duplicate_resource_names_in_root_job_and_pipeline/databricks.yml", Line: 13, Column: 7}, }, @@ -41,7 +40,7 @@ func TestValidateUniqueResourceIdentifiers(t *testing.T) { { Severity: diag.Error, Summary: "multiple resources or scripts have been defined with the same key: foo", - Locations: []dyn.Location{ + Locations: []diag.Location{ {File: "validate/duplicate_resource_names_in_root_job_and_experiment/databricks.yml", Line: 10, Column: 7}, {File: "validate/duplicate_resource_names_in_root_job_and_experiment/databricks.yml", Line: 18, Column: 7}, }, @@ -58,7 +57,7 @@ func TestValidateUniqueResourceIdentifiers(t *testing.T) { { Severity: diag.Error, Summary: "multiple resources or scripts have been defined with the same key: foo", - Locations: []dyn.Location{ + Locations: []diag.Location{ {File: "validate/duplicate_resource_name_in_subconfiguration/databricks.yml", Line: 13, Column: 7}, {File: "validate/duplicate_resource_name_in_subconfiguration/resources.yml", Line: 4, Column: 7}, }, @@ -75,7 +74,7 @@ func TestValidateUniqueResourceIdentifiers(t *testing.T) { { Severity: diag.Error, Summary: "multiple resources or scripts have been defined with the same key: foo", - Locations: []dyn.Location{ + Locations: []diag.Location{ {File: "validate/duplicate_resource_name_in_subconfiguration_job_and_job/databricks.yml", Line: 13, Column: 7}, {File: "validate/duplicate_resource_name_in_subconfiguration_job_and_job/resources.yml", Line: 4, Column: 7}, }, @@ -89,7 +88,7 @@ func TestValidateUniqueResourceIdentifiers(t *testing.T) { { Severity: diag.Error, Summary: "multiple resources or scripts have been defined with the same key: foo", - Locations: []dyn.Location{ + Locations: []diag.Location{ {File: "validate/duplicate_resource_names_in_different_subconfiguations/resources1.yml", Line: 4, Column: 7}, {File: "validate/duplicate_resource_names_in_different_subconfiguations/resources2.yml", Line: 4, Column: 7}, }, @@ -106,7 +105,7 @@ func TestValidateUniqueResourceIdentifiers(t *testing.T) { { Severity: diag.Error, Summary: "multiple resources or scripts have been defined with the same key: foo", - Locations: []dyn.Location{ + Locations: []diag.Location{ {File: "validate/duplicate_resource_name_in_multiple_locations/databricks.yml", Line: 13, Column: 7}, {File: "validate/duplicate_resource_name_in_multiple_locations/resources1.yml", Line: 4, Column: 7}, {File: "validate/duplicate_resource_name_in_multiple_locations/resources1.yml", Line: 8, Column: 7}, diff --git a/bundle/trampoline/python_dbr_warning.go b/bundle/trampoline/python_dbr_warning.go index 4cc7a67dc8c..bfcd885d1b8 100644 --- a/bundle/trampoline/python_dbr_warning.go +++ b/bundle/trampoline/python_dbr_warning.go @@ -10,8 +10,8 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go" "golang.org/x/mod/semver" ) @@ -82,18 +82,18 @@ func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) diag.Diagn // So we can get the version from the cluster definition. // It's defined in a form of resources.clusters..id if strings.HasPrefix(task.ExistingClusterId, "${") { - p, ok := dynvar.PureReferenceToPath(task.ExistingClusterId) - if !ok || len(p) < 3 { + p, ok := structvar.PureReferenceToPath(task.ExistingClusterId) + if !ok || p.Len() < 3 { log.Warnf(ctx, "unable to parse cluster key from %s", task.ExistingClusterId) continue } - if p[0].Key() != "resources" || p[1].Key() != "clusters" { + if p.KeyAt(0) != "resources" || p.KeyAt(1) != "clusters" { log.Warnf(ctx, "incorrect variable reference for cluster id %s", task.ExistingClusterId) continue } - clusterKey := p[2].Key() + clusterKey := p.KeyAt(2) cluster, ok := b.Config.Resources.Clusters[clusterKey] if !ok { log.Warnf(ctx, "unable to find cluster with key %s", clusterKey) diff --git a/cmd/air/convert_to_dabs.go b/cmd/air/convert_to_dabs.go index 2a2410accf2..910835a036a 100644 --- a/cmd/air/convert_to_dabs.go +++ b/cmd/air/convert_to_dabs.go @@ -13,8 +13,7 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlsaver" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/spf13/cobra" ) @@ -111,10 +110,10 @@ does not contact the workspace.`, // convertToDabs builds the DABs bundle value and the loose launch artifacts for a // run config. It reads only what the run path's buildArtifacts reads, so the // mapping is unit-testable in isolation. Returns the bundle root as a -// map[string]dyn.Value (ready for yamlsaver) and the loose artifacts (command.sh + +// structyaml.Map (ready for structyaml.Save) and the loose artifacts (command.sh + // env/secret/param sidecars) to write under generated_artifacts/. It does not touch the // code_source; the emitted `tgz` artifact packages it at deploy. -func convertToDabs(ctx context.Context, cfg *runConfig, configPath, bundleDir string) (map[string]dyn.Value, []uploadItem, error) { +func convertToDabs(ctx context.Context, cfg *runConfig, configPath, bundleDir string) (structyaml.Map, []uploadItem, error) { // idempotency_token is intentionally not mapped: it dedups a single runs/submit // call, which has no analogue for a persistent, repeatedly-runnable bundle job. // @@ -258,14 +257,6 @@ func bundleCodeSourcePath(ctx context.Context, cfg *runConfig, configPath, bundl return localBundlePath(filepath.ToSlash(rel)), nil } -// nv builds a dyn.Value at "position" n: yamlsaver orders a map's keys by their -// Location line, so assigning ascending n values fixes the emitted key order. -// It routes through dyn.V so nested Go maps/slices are converted recursively, -// then stamps the ordering location. -func nv(v any, n int) dyn.Value { - return dyn.V(v).WithLocations([]dyn.Location{{Line: n}}) -} - // localBundlePath renders a bundle-relative path with a leading "./" so bundle // deploy classifies it as a local artifact to upload (see IsLibraryLocal). It is // built from path.Join (forward slashes) so the emitted YAML is identical across @@ -274,10 +265,10 @@ func localBundlePath(p string) string { return "./" + p } -// buildBundleValue assembles the bundle root as an ordered map[string]dyn.Value. +// buildBundleValue assembles the bundle root as an ordered structyaml.Map. // codeSourcePath is the "./"-prefixed code_source dir relative to the bundle (empty // when the config has no code_source); command.sh is a bundle-local artifact. -func buildBundleValue(ctx context.Context, cfg *runConfig, configPath, codeSourcePath string, art *codeArtifact) map[string]dyn.Value { +func buildBundleValue(ctx context.Context, cfg *runConfig, configPath, codeSourcePath string, art *codeArtifact) structyaml.Map { name := cfg.ExperimentName // ai_runtime_task: experiment + one deployment (command_path + compute) + @@ -287,67 +278,57 @@ func buildBundleValue(ctx context.Context, cfg *runConfig, configPath, codeSourc // command_path is "./"-prefixed so bundle deploy treats it as LOCAL and uploads // it: libraries.IsLibraryLocal classifies a bare, extensionless path as a PyPI // package name, which would deploy a path the backend can't resolve. - compute := map[string]dyn.Value{ - "accelerator_type": nv(cfg.Compute.AcceleratorType, 1), - "accelerator_count": nv(cfg.Compute.NumAccelerators, 2), - } + compute := structyaml.M( + "accelerator_type", cfg.Compute.AcceleratorType, + "accelerator_count", cfg.Compute.NumAccelerators, + ) if cfg.Compute.PoolID != nil { - compute["provisioned_capacity_id"] = nv(*cfg.Compute.PoolID, 3) - } - deployment := map[string]dyn.Value{ - "command_path": nv(localBundlePath(path.Join(generatedArtifactsDir, commandScriptName)), 1), - "compute": nv(compute, 2), + compute.Add("provisioned_capacity_id", *cfg.Compute.PoolID) } + deployment := structyaml.M( + "command_path", localBundlePath(path.Join(generatedArtifactsDir, commandScriptName)), + "compute", compute, + ) - aiRuntimeTask := map[string]dyn.Value{ - "experiment": nv(name, 1), - "deployments": nv([]dyn.Value{dyn.V(deployment)}, 2), - } - line := 3 + aiRuntimeTask := structyaml.M( + "experiment", name, + "deployments", []any{deployment}, + ) if codeSourcePath != "" { // Points at the `tgz` artifact's built tarball; deploy uploads it and rewrites // this to the uploaded workspace path. - aiRuntimeTask["code_source_path"] = nv(codeSourcePath, line) - line++ + aiRuntimeTask.Add("code_source_path", codeSourcePath) } if cfg.MLflowRunName != nil { - aiRuntimeTask["mlflow_run"] = nv(*cfg.MLflowRunName, line) - line++ + aiRuntimeTask.Add("mlflow_run", *cfg.MLflowRunName) } if cfg.MLflowExperimentDirectory != nil { - aiRuntimeTask["mlflow_experiment_directory"] = nv(*cfg.MLflowExperimentDirectory, line) - line++ + aiRuntimeTask.Add("mlflow_experiment_directory", *cfg.MLflowExperimentDirectory) } if cfg.MLflowArtifactLocation != nil { - aiRuntimeTask["mlflow_artifact_location"] = nv(*cfg.MLflowArtifactLocation, line) - line++ + aiRuntimeTask.Add("mlflow_artifact_location", *cfg.MLflowArtifactLocation) } if cfg.Compute.PriorityClass != nil { - aiRuntimeTask["priority_class"] = nv(*cfg.Compute.PriorityClass, line) - line++ + aiRuntimeTask.Add("priority_class", *cfg.Compute.PriorityClass) } if cfg.Environment != nil && cfg.Environment.UnityCatalogImage != "" { - aiRuntimeTask["unity_catalog_image_path"] = nv(cfg.Environment.UnityCatalogImage, line) + aiRuntimeTask.Add("unity_catalog_image_path", cfg.Environment.UnityCatalogImage) } // Task wrapper: task_key + framework fields (retries/timeout) + env key + // the ai_runtime_task. Framework fields live here per the schema, not inside // ai_runtime_task. - task := map[string]dyn.Value{ - "task_key": nv(name, 1), - "environment_key": nv(aiRuntimeEnvironmentKey, 2), - } - taskLine := 3 maxRetries := cfg.maxRetries() - task["max_retries"] = nv(maxRetries, taskLine) - taskLine++ - task["retry_on_timeout"] = nv(maxRetries > 0, taskLine) - taskLine++ + task := structyaml.M( + "task_key", name, + "environment_key", aiRuntimeEnvironmentKey, + "max_retries", maxRetries, + "retry_on_timeout", maxRetries > 0, + ) if cfg.TimeoutMinutes != nil { - task["timeout_seconds"] = nv(cfg.timeoutSeconds(), taskLine) - taskLine++ + task.Add("timeout_seconds", cfg.timeoutSeconds()) } - task["ai_runtime_task"] = nv(aiRuntimeTask, taskLine) + task.Add("ai_runtime_task", aiRuntimeTask) // environments[]: version + the dependency set. The runtime installs deps from this // spec directly, so the full dependency set (whether authored inline or in a @@ -357,65 +338,57 @@ func buildBundleValue(ctx context.Context, cfg *runConfig, configPath, codeSourc // default channel) so a config without an explicit version still pins the version // the workload would have run with — not an empty spec. envVersion, deps := bundleEnvironmentDeps(ctx, cfg) - envSpec := map[string]dyn.Value{} + var envSpec structyaml.Map if strings.HasPrefix(envVersion, databricksAIPrefix) { - envSpec["base_environment"] = nv("workspace-base-environments/"+envVersion, 1) + envSpec.Add("base_environment", "workspace-base-environments/"+envVersion) } else { - envSpec["environment_version"] = nv(envVersion, 1) + envSpec.Add("environment_version", envVersion) } if len(deps) > 0 { - depVals := make([]dyn.Value, len(deps)) - for i, d := range deps { - depVals[i] = dyn.V(d) - } - envSpec["dependencies"] = nv(depVals, 2) - } - environment := map[string]dyn.Value{ - "environment_key": nv(aiRuntimeEnvironmentKey, 1), - "spec": nv(envSpec, 2), + envSpec.Add("dependencies", deps) } + environment := structyaml.M( + "environment_key", aiRuntimeEnvironmentKey, + "spec", envSpec, + ) - job := map[string]dyn.Value{ - "name": nv(name, 1), - "tasks": nv([]dyn.Value{dyn.V(task)}, 2), - "environments": nv([]dyn.Value{dyn.V(environment)}, 3), - } + job := structyaml.M( + "name", name, + "tasks", []any{task}, + "environments", []any{environment}, + ) // usage_policy_id is an already-resolved budget policy id, so it maps directly // to the job's budget_policy_id. (usage_policy_name needs server-side resolution // and is rejected in convertToDabs.) if cfg.UsagePolicyID != nil { - job["budget_policy_id"] = nv(*cfg.UsagePolicyID, 4) + job.Add("budget_policy_id", *cfg.UsagePolicyID) } - if perms := buildPermissionsValue(cfg.Permissions); perms.Kind() != dyn.KindInvalid { - job["permissions"] = nv(perms.MustSequence(), 5) + if perms := buildPermissionsValue(cfg.Permissions); perms != nil { + job.Add("permissions", perms) } - rootValue := map[string]dyn.Value{ - "bundle": nv(map[string]dyn.Value{ - "name": nv(name, 1), - }, 1), + rootValue := structyaml.M( + "bundle", structyaml.M("name", name), // sync.paths replaces the default of syncing the whole bundle root. The code // directory is omitted deliberately: the `tgz` artifact packages it, so syncing // it too would upload the tree twice. - "sync": nv(map[string]dyn.Value{ - "paths": nv([]dyn.Value{nv(generatedArtifactsDir, 1)}, 1), - }, 2), - "targets": nv(map[string]dyn.Value{ - dabsTargetName: nv(map[string]dyn.Value{ - "mode": nv("development", 1), - "default": nv(true, 2), - }, 1), - }, 4), - "resources": nv(map[string]dyn.Value{ - "jobs": nv(map[string]dyn.Value{ - bundleResourceKey(name): nv(job, 1), - }, 1), - }, 5), - } + "sync", structyaml.M("paths", []any{generatedArtifactsDir}), + ) // The `tgz` artifact that packages the code_source (nil only when there is none). if art != nil { - rootValue["artifacts"] = nv(buildArtifactsValue(art), 3) - } + rootValue.Add("artifacts", buildArtifactsValue(art)) + } + rootValue.Add("targets", structyaml.M( + dabsTargetName, structyaml.M( + "mode", "development", + "default", true, + ), + )) + rootValue.Add("resources", structyaml.M( + "jobs", structyaml.M( + bundleResourceKey(name), job, + ), + )) return rootValue } @@ -423,34 +396,23 @@ func buildBundleValue(ctx context.Context, cfg *runConfig, configPath, codeSourc // single `tgz` artifact whose `path` is the code-source root, carrying the git ref // and/or include subpaths, and whose `files` output is the tarball code_source_path // points at. -func buildArtifactsValue(art *codeArtifact) map[string]dyn.Value { - a := map[string]dyn.Value{ - "type": nv("tgz", 1), - "path": nv(art.path, 2), - } - fileLine := 3 +func buildArtifactsValue(art *codeArtifact) structyaml.Map { + a := structyaml.M( + "type", "tgz", + "path", art.path, + ) if art.gitCommit != nil || art.gitBranch != nil { - g := map[string]dyn.Value{} // commit wins over branch, matching the artifact builder. if art.gitCommit != nil { - g["commit"] = nv(*art.gitCommit, 1) + a.Add("git", structyaml.M("commit", *art.gitCommit)) } else { - g["branch"] = nv(*art.gitBranch, 1) + a.Add("git", structyaml.M("branch", *art.gitBranch)) } - a["git"] = nv(g, fileLine) - fileLine++ } // include is always set: the basename (whole dir) or basename-prefixed subpaths. - vals := make([]dyn.Value, len(art.include)) - for i, p := range art.include { - vals[i] = dyn.V(p) - } - a["include"] = nv(vals, fileLine) - fileLine++ - a["files"] = nv([]dyn.Value{ - dyn.V(map[string]dyn.Value{"source": nv(art.tgzPath, 1)}), - }, fileLine) - return map[string]dyn.Value{codeSourceArtifactKey: nv(a, 1)} + a.Add("include", art.include) + a.Add("files", []any{structyaml.M("source", art.tgzPath)}) + return structyaml.M(codeSourceArtifactKey, a) } // bundleEnvironmentDeps resolves the runtime version and the inline dependency @@ -475,7 +437,7 @@ func bundleResourceKey(name string) string { } // quoteJobKey rewrites the emitted job resource key as a quoted YAML key when the -// name would otherwise load as a non-string scalar. yamlsaver emits map keys +// name would otherwise load as a non-string scalar. structyaml emits map keys // unquoted, and the bundle loader rejects a key that types as something other than // a string ("12345" -> !!int, "true" -> !!bool) with "invalid key tag". Only the // job key needs this: every other key convert emits is a fixed schema field name. @@ -515,25 +477,22 @@ func yamlKeyNeedsQuoting(key string) bool { } // buildPermissionsValue maps run-config permissions to DABs job permissions -// (level → principal). Returns an invalid value when there are none. -func buildPermissionsValue(perms []permission) dyn.Value { - if len(perms) == 0 { - return dyn.InvalidValue - } - out := make([]dyn.Value, 0, len(perms)) +// (level → principal). Returns nil when there are none. +func buildPermissionsValue(perms []permission) []any { + var out []any for _, p := range perms { - m := map[string]dyn.Value{"level": nv(p.Level, 1)} + m := structyaml.M("level", p.Level) switch { case p.UserName != nil: - m["user_name"] = nv(*p.UserName, 2) + m.Add("user_name", *p.UserName) case p.GroupName != nil: - m["group_name"] = nv(*p.GroupName, 2) + m.Add("group_name", *p.GroupName) case p.ServicePrincipalName != nil: - m["service_principal_name"] = nv(*p.ServicePrincipalName, 2) + m.Add("service_principal_name", *p.ServicePrincipalName) } - out = append(out, dyn.V(m)) + out = append(out, m) } - return dyn.V(out) + return out } // writeBundle writes the bundle into dir: databricks.yml plus the loose launch @@ -579,9 +538,9 @@ func writeBundle(ctx context.Context, cfg *runConfig, configPath, dir string, fo return nil, fmt.Errorf("databricks.yml already exists in %s; pass --force to overwrite or remove it", dir) } } - // SaveAsYAML's force arg is passed true unconditionally: the collision check + // Save's force arg is passed true unconditionally: the collision check // above already decided whether overwriting is allowed. - if err := yamlsaver.NewSaver().SaveAsYAML(root, bundlePath, true); err != nil { + if err := structyaml.Save(bundlePath, root, true, nil); err != nil { return nil, err } if err := quoteJobKey(bundlePath, bundleResourceKey(cfg.ExperimentName)); err != nil { diff --git a/cmd/air/convert_to_dabs_test.go b/cmd/air/convert_to_dabs_test.go index cbffa6b224c..ac07cac17d5 100644 --- a/cmd/air/convert_to_dabs_test.go +++ b/cmd/air/convert_to_dabs_test.go @@ -3,10 +3,11 @@ package aircmd import ( "os" "path/filepath" + "strconv" "strings" "testing" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.yaml.in/yaml/v3" @@ -86,17 +87,71 @@ func TestConvertToDabsCommandShape(t *testing.T) { assert.Error(t, cmd.Args(cmd, []string{"a", "b"})) } +// val is a value of the emitted bundle root. +type val struct{ v any } + +func (v val) MustString() string { return v.v.(string) } +func (v val) MustInt() int64 { return int64(v.v.(int)) } +func (v val) MustBool() bool { return v.v.(bool) } + +func (v val) MustSequence() []val { + if ss, ok := v.v.([]string); ok { + out := make([]val, len(ss)) + for i, s := range ss { + out[i] = val{s} + } + return out + } + var out []val + for _, e := range v.v.([]any) { + out = append(out, val{e}) + } + return out +} + +func (v val) Get(key string) val { + for _, p := range v.v.(structyaml.Map) { + if p.Key == key { + return val{p.Value} + } + } + return val{} +} + +// lookup reads a dotted path, with optional [index] suffixes, out of the emitted bundle root. +func lookup(root structyaml.Map, path string) (val, bool) { + v := val{root} + for part := range strings.SplitSeq(path, ".") { + key, idx, hasIdx := strings.Cut(strings.TrimSuffix(part, "]"), "[") + if _, ok := v.v.(structyaml.Map); !ok { + return val{}, false + } + if v = v.Get(key); v.v == nil { + return val{}, false + } + if hasIdx { + i, err := strconv.Atoi(idx) + seq, ok := v.v.([]any) + if err != nil || !ok || i >= len(seq) { + return val{}, false + } + v = val{seq[i]} + } + } + return v, true +} + // get is a small helper: read a dotted path out of the emitted bundle root. -func get(t *testing.T, root map[string]dyn.Value, path string) dyn.Value { +func get(t *testing.T, root structyaml.Map, path string) val { t.Helper() - v, err := dyn.GetByPath(dyn.V(root), dyn.MustPathFromString(path)) - require.NoError(t, err, "path %q should exist", path) + v, ok := lookup(root, path) + require.True(t, ok, "path %q should exist", path) return v } -func has(root map[string]dyn.Value, path string) bool { - _, err := dyn.GetByPath(dyn.V(root), dyn.MustPathFromString(path)) - return err == nil +func has(root structyaml.Map, path string) bool { + _, ok := lookup(root, path) + return ok } // A full config maps onto a schema-shaped bundle: bundle name, job/task keys, the @@ -543,11 +598,7 @@ func TestConvertToDabsSafeJobKey(t *testing.T) { root, _, err := convertToDabs(t.Context(), loaded, path, dir) require.NoError(t, err) // The key is the raw name; name/experiment keep the same value. - jobs, err := dyn.GetByPath(dyn.V(root), dyn.MustPathFromString("resources.jobs")) - require.NoError(t, err) - job := jobs.Get(name) - require.True(t, job.IsValid(), "job must be keyed by %q", name) - assert.Equal(t, name, job.Get("name").MustString()) + assert.Equal(t, name, get(t, root, "resources.jobs."+name+".name").MustString()) // The emitted YAML must load back with the key still a string. _, err = writeBundle(t.Context(), loaded, path, dir, true) diff --git a/cmd/apps/import.go b/cmd/apps/import.go index f6089870aec..4aefe33f057 100644 --- a/cmd/apps/import.go +++ b/cmd/apps/import.go @@ -12,8 +12,6 @@ import ( "slices" "strings" - "go.yaml.in/yaml/v3" - "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/generate" "github.com/databricks/cli/bundle/phases" @@ -25,17 +23,16 @@ import ( "github.com/databricks/cli/libs/apps/prompt" "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/cli/libs/dyn/yamlsaver" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/databricks/cli/libs/textutil" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/apps" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/workspace" "github.com/spf13/cobra" + "go.yaml.in/yaml/v3" ) func newImportCommand() *cobra.Command { @@ -455,22 +452,13 @@ func generateAppBundle(ctx context.Context, w *databricks.WorkspaceClient, app * } } - // Create the bundle configuration with explicit line numbers to control ordering // Use the app name for the bundle name bundleName := textutil.NormalizeString(app.Name) - bundleConfig := map[string]dyn.Value{ - "bundle": dyn.NewValue(map[string]dyn.Value{ - "name": dyn.NewValue(bundleName, []dyn.Location{{Line: 1}}), - }, []dyn.Location{{Line: 1}}), - "workspace": dyn.NewValue(map[string]dyn.Value{ - "host": dyn.NewValue(w.Config.Host, []dyn.Location{{Line: 2}}), - }, []dyn.Location{{Line: 10}}), - "resources": dyn.NewValue(map[string]dyn.Value{ - "apps": dyn.V(map[string]dyn.Value{ - appKey: v, - }), - }, []dyn.Location{{Line: 20}}), - } + bundleConfig := structyaml.M( + "bundle", structyaml.M("name", bundleName), + "workspace", structyaml.M("host", w.Config.Host), + "resources", structyaml.M("apps", structyaml.M(appKey, v)), + ) // Download the app source files err = downloader.FlushToDisk(ctx, false) @@ -480,8 +468,7 @@ func generateAppBundle(ctx context.Context, w *databricks.WorkspaceClient, app * // Save databricks.yml databricksYml := filepath.Join(".", "databricks.yml") - saver := yamlsaver.NewSaver() - err = saver.SaveAsYAML(bundleConfig, databricksYml, false) + err = structyaml.Save(databricksYml, bundleConfig, false, nil) if err != nil { return "", fmt.Errorf("failed to save databricks.yml: %w", err) } @@ -544,7 +531,7 @@ func addBlankLinesBetweenTopLevelKeys(filename string) error { } // inlineAppConfigFile reads app.yml or app.yaml, inlines it into the app value, and returns the filename -func inlineAppConfigFile(appValue *dyn.Value) (string, error) { +func inlineAppConfigFile(appValue *structyaml.Map) (string, error) { // Check for app.yml first, then app.yaml var appConfigFile string var appConfigData []byte @@ -573,62 +560,33 @@ func inlineAppConfigFile(appValue *dyn.Value) (string, error) { return "", fmt.Errorf("failed to parse %s: %w", appConfigFile, err) } - // Get the current app value as a map - appMap, ok := appValue.AsMap() - if !ok { - return "", errors.New("app value is not a map") - } - - // Build the new app map with the config section - newPairs := make([]dyn.Pair, 0, len(appMap.Pairs())+2) - - // Copy existing pairs - newPairs = append(newPairs, appMap.Pairs()...) - // Create config section - configMap := make(map[string]dyn.Value) + var config structyaml.Map // Add command if present if cmd, ok := appConfig["command"]; ok { - cmdValue, err := convert.FromTyped(cmd, dyn.NilValue) - if err != nil { - return "", fmt.Errorf("failed to convert command: %w", err) - } - configMap["command"] = cmdValue + config.Add("command", structyaml.Value(cmd)) } // Add env if present if env, ok := appConfig["env"]; ok { - envValue, err := convert.FromTyped(env, dyn.NilValue) - if err != nil { - return "", fmt.Errorf("failed to convert env: %w", err) - } - configMap["env"] = envValue + config.Add("env", structyaml.Value(env)) } + // The config section and the top-level resources go before the existing fields. + var added structyaml.Map + // Add the config section if we have any items - if len(configMap) > 0 { - newPairs = append(newPairs, dyn.Pair{ - Key: dyn.V("config"), - Value: dyn.V(configMap), - }) + if len(config) > 0 { + added.Add("config", config) } // Add resources at top level if present if resources, ok := appConfig["resources"]; ok { - resourcesValue, err := convert.FromTyped(resources, dyn.NilValue) - if err != nil { - return "", fmt.Errorf("failed to convert resources: %w", err) - } - newPairs = append(newPairs, dyn.Pair{ - Key: dyn.V("resources"), - Value: resourcesValue, - }) + added.Add("resources", structyaml.Value(resources)) } - // Create the new app value with the config section - newMapping := dyn.NewMappingFromPairs(newPairs) - *appValue = dyn.NewValue(newMapping, appValue.Locations()) + *appValue = append(added, *appValue...) return appConfigFile, nil } diff --git a/cmd/apps/import_test.go b/cmd/apps/import_test.go index b50ec937991..30e9c8d239f 100644 --- a/cmd/apps/import_test.go +++ b/cmd/apps/import_test.go @@ -7,7 +7,7 @@ import ( "strings" "testing" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -16,17 +16,15 @@ func TestInlineAppConfigFile(t *testing.T) { tests := []struct { name string setupFiles map[string]string - inputValue dyn.Value + inputValue structyaml.Map expectedFile string expectedConfig map[string]any expectError bool }{ { - name: "no app config file", - setupFiles: map[string]string{}, - inputValue: dyn.V(map[string]dyn.Value{ - "name": dyn.V("test-app"), - }), + name: "no app config file", + setupFiles: map[string]string{}, + inputValue: structyaml.M("name", "test-app"), expectedFile: "", expectedConfig: map[string]any{"name": "test-app"}, }, @@ -38,9 +36,7 @@ env: - name: FOO value: bar`, }, - inputValue: dyn.V(map[string]dyn.Value{ - "name": dyn.V("test-app"), - }), + inputValue: structyaml.M("name", "test-app"), expectedFile: "app.yml", expectedConfig: nil, // Will check manually }, @@ -53,9 +49,7 @@ env: - name: TEST value: value`, }, - inputValue: dyn.V(map[string]dyn.Value{ - "name": dyn.V("test-app"), - }), + inputValue: structyaml.M("name", "test-app"), expectedFile: "app.yml", expectedConfig: nil, // Will check manually }, @@ -68,9 +62,7 @@ resources: serving_endpoint: name: my-endpoint`, }, - inputValue: dyn.V(map[string]dyn.Value{ - "name": dyn.V("test-app"), - }), + inputValue: structyaml.M("name", "test-app"), expectedFile: "app.yml", expectedConfig: nil, // Will check manually }, @@ -80,9 +72,7 @@ resources: "app.yml": `resources: - name: SERVING_ENDPOINT`, }, - inputValue: dyn.V(map[string]dyn.Value{ - "name": dyn.V("test-app"), - }), + inputValue: structyaml.M("name", "test-app"), expectedFile: "app.yml", expectedConfig: nil, // Will check manually }, @@ -92,9 +82,7 @@ resources: "app.yml": `command: ["python", "app.py"] env: []`, }, - inputValue: dyn.V(map[string]dyn.Value{ - "name": dyn.V("test-app"), - }), + inputValue: structyaml.M("name", "test-app"), expectedFile: "app.yml", expectedConfig: nil, // Will check manually }, @@ -125,20 +113,17 @@ env: []`, // Verify the structure if expectedConfig is set if tt.expectedConfig != nil { - appMap := appValue.MustMap() result := make(map[string]any) - for _, pair := range appMap.Pairs() { - key := pair.Key.MustString() - result[key] = pair.Value.AsAny() + for _, pair := range appValue { + result[pair.Key] = pair.Value } assert.Equal(t, tt.expectedConfig, result) } else if tt.expectedFile != "" { // Just verify that config or resources were added - appMap := appValue.MustMap() var hasConfigOrResources bool - for _, pair := range appMap.Pairs() { - key := pair.Key.MustString() + for _, pair := range appValue { + key := pair.Key if key == "config" || key == "resources" { hasConfigOrResources = true break @@ -158,24 +143,12 @@ func TestInlineAppConfigFileErrors(t *testing.T) { err := os.WriteFile("app.yml", []byte("invalid: yaml: content:\n - broken"), 0o644) require.NoError(t, err) - appValue := dyn.V(map[string]dyn.Value{"name": dyn.V("test")}) + appValue := structyaml.M("name", "test") _, err = inlineAppConfigFile(&appValue) assert.Error(t, err) assert.Contains(t, err.Error(), "failed to parse") }) - t.Run("app value not a map", func(t *testing.T) { - t.Chdir(t.TempDir()) - - err := os.WriteFile("app.yml", []byte("command: [\"test\"]"), 0o644) - require.NoError(t, err) - - appValue := dyn.V("not a map") - _, err = inlineAppConfigFile(&appValue) - assert.Error(t, err) - assert.Contains(t, err.Error(), "app value is not a map") - }) - t.Run("unreadable app.yml", func(t *testing.T) { t.Chdir(t.TempDir()) @@ -202,7 +175,7 @@ func TestInlineAppConfigFileErrors(t *testing.T) { require.NoError(t, err) } - appValue := dyn.V(map[string]dyn.Value{"name": dyn.V("test")}) + appValue := structyaml.M("name", "test") _, err = inlineAppConfigFile(&appValue) assert.Error(t, err) assert.Contains(t, err.Error(), "failed to read") @@ -223,21 +196,19 @@ resources: name: test`), 0o644) require.NoError(t, err) - appValue := dyn.V(map[string]dyn.Value{ - "name": dyn.V("test-app"), - "description": dyn.V("existing description"), - }) + appValue := structyaml.M( + "name", "test-app", + "description", "existing description", + ) filename, err := inlineAppConfigFile(&appValue) require.NoError(t, err) assert.Equal(t, "app.yml", filename) // Verify structure - appMap := appValue.MustMap() result := make(map[string]any) - for _, pair := range appMap.Pairs() { - key := pair.Key.MustString() - result[key] = pair.Value.AsAny() + for _, pair := range appValue { + result[pair.Key] = pair.Value } // Should have original fields plus config and resources diff --git a/cmd/bundle/generate/alert.go b/cmd/bundle/generate/alert.go index cb78b24de1e..2e4c5150370 100644 --- a/cmd/bundle/generate/alert.go +++ b/cmd/bundle/generate/alert.go @@ -12,9 +12,8 @@ import ( "github.com/databricks/cli/bundle/generate" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlsaver" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/databricks/cli/libs/textutil" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/sql" @@ -141,13 +140,7 @@ After generation, you can deploy this alert to other targets using: return err } - result := map[string]dyn.Value{ - "resources": dyn.V(map[string]dyn.Value{ - "alerts": dyn.V(map[string]dyn.Value{ - alertKey: v, - }), - }), - } + result := structyaml.M("resources", structyaml.M("alerts", structyaml.M(alertKey, v))) // Create config directory if needed if err := os.MkdirAll(configDir, 0o755); err != nil { @@ -156,11 +149,11 @@ After generation, you can deploy this alert to other targets using: // Save configuration file configPath := filepath.Join(configDir, alertKey+".alert.yml") - saver := yamlsaver.NewSaverWithStyle(map[string]yaml.Style{ + styles := map[string]yaml.Style{ "display_name": yaml.DoubleQuotedStyle, - }) + } - err = saver.SaveAsYAML(result, configPath, force) + err = structyaml.Save(configPath, result, force, styles) if err != nil { return err } diff --git a/cmd/bundle/generate/app.go b/cmd/bundle/generate/app.go index bec005408cd..0705dc3f0c2 100644 --- a/cmd/bundle/generate/app.go +++ b/cmd/bundle/generate/app.go @@ -8,9 +8,8 @@ import ( "github.com/databricks/cli/cmd/bundle/deployment" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlsaver" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/databricks/cli/libs/textutil" "github.com/databricks/databricks-sdk-go/service/apps" "github.com/spf13/cobra" @@ -107,13 +106,7 @@ per target environment.`, appKey = textutil.NormalizeString(app.Name) } - result := map[string]dyn.Value{ - "resources": dyn.V(map[string]dyn.Value{ - "apps": dyn.V(map[string]dyn.Value{ - appKey: v, - }), - }), - } + result := structyaml.M("resources", structyaml.M("apps", structyaml.M(appKey, v))) err = downloader.FlushToDisk(ctx, force) if err != nil { @@ -122,8 +115,7 @@ per target environment.`, filename := filepath.Join(configDir, appKey+".app.yml") - saver := yamlsaver.NewSaver() - err = saver.SaveAsYAML(result, filename, force) + err = structyaml.Save(filename, result, force, nil) if err != nil { return err } diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index 9c4be81b391..d5058e60a5e 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -22,9 +22,8 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlsaver" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/databricks/cli/libs/textutil" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" @@ -215,13 +214,7 @@ func (d *dashboard) saveConfiguration(ctx context.Context, b *bundle.Bundle, das return err } - result := map[string]dyn.Value{ - "resources": dyn.V(map[string]dyn.Value{ - "dashboards": dyn.V(map[string]dyn.Value{ - key: v, - }), - }), - } + result := structyaml.M("resources", structyaml.M("dashboards", structyaml.M(key, v))) // Make sure the output directory exists. if err := os.MkdirAll(d.resourceDir, 0o755); err != nil { @@ -230,9 +223,9 @@ func (d *dashboard) saveConfiguration(ctx context.Context, b *bundle.Bundle, das // Save the configuration to the resource directory. resourcePath := filepath.Join(d.resourceDir, key+".dashboard.yml") - saver := yamlsaver.NewSaverWithStyle(map[string]yaml.Style{ + styles := map[string]yaml.Style{ "display_name": yaml.DoubleQuotedStyle, - }) + } // Attempt to make the path relative to the bundle root. rel, err := filepath.Rel(b.BundleRootPath, resourcePath) @@ -241,7 +234,7 @@ func (d *dashboard) saveConfiguration(ctx context.Context, b *bundle.Bundle, das } cmdio.LogString(ctx, "Writing configuration to "+filepath.ToSlash(rel)) - err = saver.SaveAsYAML(result, resourcePath, d.force) + err = structyaml.Save(resourcePath, result, d.force, styles) if err != nil { return err } diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index 7c520abc320..2107d62ffd8 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -18,9 +18,8 @@ import ( "github.com/databricks/cli/cmd/bundle/deployment" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlsaver" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/databricks/cli/libs/textutil" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/dashboards" @@ -130,13 +129,7 @@ func (g *genieSpace) saveConfiguration(ctx context.Context, b *bundle.Bundle, ge return err } - result := map[string]dyn.Value{ - "resources": dyn.V(map[string]dyn.Value{ - "genie_spaces": dyn.V(map[string]dyn.Value{ - key: v, - }), - }), - } + result := structyaml.M("resources", structyaml.M("genie_spaces", structyaml.M(key, v))) // Make sure the output directory exists. if err := os.MkdirAll(g.resourceDir, 0o755); err != nil { @@ -145,9 +138,9 @@ func (g *genieSpace) saveConfiguration(ctx context.Context, b *bundle.Bundle, ge // Save the configuration to the resource directory. resourcePath := filepath.Join(g.resourceDir, key+".genie_space.yml") - saver := yamlsaver.NewSaverWithStyle(map[string]yaml.Style{ + styles := map[string]yaml.Style{ "title": yaml.DoubleQuotedStyle, - }) + } // Attempt to make the path relative to the bundle root. rel, err := filepath.Rel(b.BundleRootPath, resourcePath) @@ -156,7 +149,7 @@ func (g *genieSpace) saveConfiguration(ctx context.Context, b *bundle.Bundle, ge } cmdio.LogString(ctx, "Writing configuration to "+filepath.ToSlash(rel)) - err = saver.SaveAsYAML(result, resourcePath, g.force) + err = structyaml.Save(resourcePath, result, g.force, styles) if err != nil { return err } diff --git a/cmd/bundle/generate/job.go b/cmd/bundle/generate/job.go index db6839ade98..07066c0ec8a 100644 --- a/cmd/bundle/generate/job.go +++ b/cmd/bundle/generate/job.go @@ -12,9 +12,8 @@ import ( "github.com/databricks/cli/cmd/bundle/deployment" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlsaver" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/databricks/cli/libs/textutil" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/spf13/cobra" @@ -115,13 +114,7 @@ After generation, you can deploy this job to other targets using: jobKey = textutil.NormalizeString(job.Settings.Name) } - result := map[string]dyn.Value{ - "resources": dyn.V(map[string]dyn.Value{ - "jobs": dyn.V(map[string]dyn.Value{ - jobKey: v, - }), - }), - } + result := structyaml.M("resources", structyaml.M("jobs", structyaml.M(jobKey, v))) err = downloader.FlushToDisk(ctx, force) if err != nil { @@ -141,13 +134,13 @@ After generation, you can deploy this job to other targets using: return fmt.Errorf("failed to rename file %s. DABs uses the resource type as a sub-extension for generated content, please rename it to %s, err: %w", oldFilename, filename, err) } - saver := yamlsaver.NewSaverWithStyle(map[string]yaml.Style{ + styles := map[string]yaml.Style{ // Including all JobSettings and nested fields which are map[string]string type "spark_conf": yaml.DoubleQuotedStyle, "custom_tags": yaml.DoubleQuotedStyle, "tags": yaml.DoubleQuotedStyle, - }) - err = saver.SaveAsYAML(result, filename, force) + } + err = structyaml.Save(filename, result, force, styles) if err != nil { return err } diff --git a/cmd/bundle/generate/pipeline.go b/cmd/bundle/generate/pipeline.go index cdde5984d40..d6de9ab9cc4 100644 --- a/cmd/bundle/generate/pipeline.go +++ b/cmd/bundle/generate/pipeline.go @@ -11,9 +11,8 @@ import ( "github.com/databricks/cli/cmd/bundle/deployment" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlsaver" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/databricks/cli/libs/textutil" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/spf13/cobra" @@ -112,13 +111,7 @@ like catalogs, schemas, and compute configurations per target.`, pipelineKey = textutil.NormalizeString(pipeline.Name) } - result := map[string]dyn.Value{ - "resources": dyn.V(map[string]dyn.Value{ - "pipelines": dyn.V(map[string]dyn.Value{ - pipelineKey: v, - }), - }), - } + result := structyaml.M("resources", structyaml.M("pipelines", structyaml.M(pipelineKey, v))) err = downloader.FlushToDisk(ctx, force) if err != nil { @@ -136,15 +129,13 @@ like catalogs, schemas, and compute configurations per target.`, return fmt.Errorf("failed to rename file %s. DABs uses the resource type as a sub-extension for generated content, please rename it to %s, err: %w", oldFilename, filename, err) } - saver := yamlsaver.NewSaverWithStyle( - // Including all CreatePipeline and nested fields which are map[string]string type - map[string]yaml.Style{ - "spark_conf": yaml.DoubleQuotedStyle, - "custom_tags": yaml.DoubleQuotedStyle, - "configuration": yaml.DoubleQuotedStyle, - }, - ) - err = saver.SaveAsYAML(result, filename, force) + // Including all CreatePipeline and nested fields which are map[string]string type + styles := map[string]yaml.Style{ + "spark_conf": yaml.DoubleQuotedStyle, + "custom_tags": yaml.DoubleQuotedStyle, + "configuration": yaml.DoubleQuotedStyle, + } + err = structyaml.Save(filename, result, force, styles) if err != nil { return err } diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index 17199a6cabf..4ca32fd92a1 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -21,10 +21,10 @@ import ( "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/execv" "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structpath" "github.com/spf13/cobra" ) @@ -189,7 +189,7 @@ Example usage: // runner sees concrete values (e.g. an app's env vars referencing another // resource, or its source_code_path pointing at the immutable snapshot's // full_path, which lives only in the deployed state). - target := dyn.NewPath(dyn.Key("resources"), dyn.Key(ref.Description.PluralName), dyn.Key(ref.Key)) + target := structpath.NewPath(nil, "resources", ref.Description.PluralName, ref.Key) if err := b.DeploymentBundle.ResolveConfigAgainstState(&b.Config, target); err != nil { return err } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 7e60716ad6c..ae12cb00ef2 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -28,7 +28,6 @@ import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dms" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/sync" @@ -710,8 +709,7 @@ func isNewerVersion(stateVersion, currentVersion string) bool { func rejectDefinitions(ctx context.Context, b *bundle.Bundle) { if b.Config.Definitions != nil { - v := dyn.GetValue(b.Config.Value(), "definitions") - loc := v.Locations() + loc := b.Config.GetLocations("definitions") filename := "input yaml" if len(loc) > 0 { filename = filepath.ToSlash(loc[0].File) diff --git a/cmd/bundle/validate.go b/cmd/bundle/validate.go index 9f8257731a5..52c17d99c09 100644 --- a/cmd/bundle/validate.go +++ b/cmd/bundle/validate.go @@ -9,7 +9,6 @@ import ( "github.com/databricks/cli/bundle/render" "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" - "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/logdiag" "github.com/spf13/cobra" @@ -23,11 +22,7 @@ func renderJsonOutput(cmd *cobra.Command, b *bundle.Bundle) error { if err != nil { return err } - converted, err := convert.FromTyped(redactedRoot, b.Config.Value()) - if err != nil { - return err - } - buf, err := json.MarshalIndent(converted.AsAny(), "", " ") + buf, err := json.MarshalIndent(redactedRoot.View().AsAny(), "", " ") if err != nil { return err } diff --git a/cmd/pipelines/generate.go b/cmd/pipelines/generate.go index e1e5666b429..cf362bd89f1 100644 --- a/cmd/pipelines/generate.go +++ b/cmd/pipelines/generate.go @@ -10,11 +10,9 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/cli/libs/dyn/yamlloader" - "github.com/databricks/cli/libs/dyn/yamlsaver" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structvar" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/spf13/cobra" ) @@ -82,8 +80,7 @@ Use --existing-pipeline-dir to generate pipeline configuration from spark-pipeli return fmt.Errorf("failed to construct .pipeline.yml: %w", err) } - saver := yamlsaver.NewSaver() - err = saver.SaveAsYAML(resourcesMap, outputFile, force) + err = structyaml.Save(outputFile, resourcesMap, force, nil) if err != nil { return err } @@ -207,31 +204,21 @@ func parseSparkPipelineYAML(ctx context.Context, filePath string) (*sdpPipeline, } defer file.Close() - dv, err := yamlloader.LoadYAML(filePath, file) + out := sdpPipeline{} + _, diags, err := structvar.DecodeYAML(filePath, file, &out) if err != nil { return nil, fmt.Errorf("failed to load %s: %w", filePath, err) } - out := sdpPipeline{} - normalized, diags := convert.Normalize(&out, dv) - if diags.HasError() { - return nil, fmt.Errorf("failed to parse %s: %w", filePath, diags.Error()) - } - for _, diag := range diags { logdiag.LogDiag(ctx, diag) } - err = convert.ToTyped(&out, normalized) - if err != nil { - return nil, fmt.Errorf("failed to parse %s: %w", filePath, diags.Error()) - } - return &out, nil } // convertToResources converts a spark-pipeline.yml spec to DABs YAML format with "resources" property -func convertToResources(spec *sdpPipeline, resourceName, srcFolder string) (map[string]dyn.Value, error) { +func convertToResources(spec *sdpPipeline, resourceName, srcFolder string) (structyaml.Map, error) { // YAML paths are relative to directory containing YAML file, in this case: // DABs YAML is in "./resources/.pipeline.yml" // SDP YAML is in ".//spark-pipeline.yml" @@ -255,55 +242,29 @@ func convertToResources(spec *sdpPipeline, resourceName, srcFolder string) (map[ }, } - environmentDyn, err := convert.FromTyped(environment, dyn.NilValue) - if err != nil { - return nil, fmt.Errorf("failed to convert environments into dyn.Value: %w", err) - } - - librariesDyn, err := convertLibraries(relativePath, spec.Libraries) - if err != nil { - return nil, fmt.Errorf("failed to convert libraries into dyn.Value: %w", err) - } - - // maps are unordered, and saver is sorting keys by dyn.Location - // this is helper function to monotonically assign locations as keys are created - var line int - nextLocation := func() []dyn.Location { - line += 1 - return []dyn.Location{{Line: line}} - } - - pipelineMap := map[string]dyn.Value{ - "name": dyn.V(spec.Name).WithLocations(nextLocation()), - "catalog": dyn.V(catalog).WithLocations(nextLocation()), - "schema": dyn.V(schema).WithLocations(nextLocation()), - "root_path": dyn.V(relativePath).WithLocations(nextLocation()), - "serverless": dyn.V(true).WithLocations(nextLocation()), - "libraries": librariesDyn.WithLocations(nextLocation()), - } + // Keys are written in the order they are added. + pipelineMap := structyaml.M( + "name", spec.Name, + "catalog", catalog, + "schema", schema, + "root_path", relativePath, + "serverless", true, + "libraries", convertLibraries(relativePath, spec.Libraries), + ) // configuration is optional field, skip if empty if spec.Configuration != nil { - dv, err := convert.FromTyped(spec.Configuration, dyn.NilValue) - if err != nil { - return nil, fmt.Errorf("failed to convert configuration into dyn.Value: %w", err) - } - - // NB: golang maps are unordered, and currently we don't preserve the order - pipelineMap["configuration"] = dv.WithLocations(nextLocation()) + pipelineMap.Add("configuration", structyaml.Value(spec.Configuration)) } - pipelineMap["environment"] = environmentDyn.WithLocations(nextLocation()) + pipelineMap.Add("environment", structyaml.Value(environment)) - resourcesMap := map[string]dyn.Value{ - "resources": dyn.V(map[string]dyn.Value{ - "pipelines": dyn.V(map[string]dyn.Value{ - resourceName: dyn.V(pipelineMap), - }), - }), - } + resourcesMap := structyaml.M("resources", structyaml.M("pipelines", structyaml.M(resourceName, pipelineMap))) - _, diag := convert.Normalize(&config.Root{}, dyn.V(resourcesMap)) + _, diag, err := structvar.DecodeYAMLNode("", structyaml.Node(resourcesMap, nil), &config.Root{}, nil) + if err != nil { + return nil, err + } if len(diag) > 0 { return nil, fmt.Errorf("generated output doesn't match expected schema: %v", diag) } @@ -315,7 +276,7 @@ func convertToResources(spec *sdpPipeline, resourceName, srcFolder string) (map[ // // relativePath contains a path to append into SDP libraries path to make // them relative to generated DABs YAML -func convertLibraries(relativePath string, specLibraries []sdpPipelineLibrary) (dyn.Value, error) { +func convertLibraries(relativePath string, specLibraries []sdpPipelineLibrary) any { var libraries []pipelines.PipelineLibrary for _, lib := range specLibraries { @@ -328,16 +289,11 @@ func convertLibraries(relativePath string, specLibraries []sdpPipelineLibrary) ( } } - librariesDyn, err := convert.FromTyped(libraries, dyn.NilValue) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to convert libraries into dyn.Value: %w", err) - } - - // FromTyped returns NilValue if libraries is an empty array - if librariesDyn.Kind() == dyn.KindNil { - // we always want to leave empty array as a placeholder in generated YAML - return dyn.V([]dyn.Value{}), nil + // Value returns nil if libraries is an empty array + if v := structyaml.Value(libraries); v != nil { + return v } - return librariesDyn, nil + // we always want to leave empty array as a placeholder in generated YAML + return []any{} } diff --git a/cmd/pipelines/generate_test.go b/cmd/pipelines/generate_test.go index d9396932dfa..c795d463164 100644 --- a/cmd/pipelines/generate_test.go +++ b/cmd/pipelines/generate_test.go @@ -5,8 +5,7 @@ import ( "path/filepath" "testing" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynassert" + "github.com/databricks/cli/libs/structs/structyaml" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -173,38 +172,27 @@ func TestConvertToResources(t *testing.T) { }, } - expected := map[string]dyn.Value{ - "resources": dyn.V(map[string]dyn.Value{ - "pipelines": dyn.V(map[string]dyn.Value{ - "my_pipeline": dyn.V(map[string]dyn.Value{ - "name": dyn.V("My Pipeline").WithLocations([]dyn.Location{{Line: 1}}), - "catalog": dyn.V("${var.catalog}").WithLocations([]dyn.Location{{Line: 2}}), - "schema": dyn.V("${var.schema}").WithLocations([]dyn.Location{{Line: 3}}), - "root_path": dyn.V("../src/my_pipeline").WithLocations([]dyn.Location{{Line: 4}}), - "serverless": dyn.V(true).WithLocations([]dyn.Location{{Line: 5}}), - "libraries": dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "glob": dyn.V(map[string]dyn.Value{ - "include": dyn.V("../src/my_pipeline/transformations/**"), - }), - }), - }).WithLocations([]dyn.Location{{Line: 6}}), - "configuration": dyn.V(map[string]dyn.Value{ - "key0": dyn.V("value0"), - "key1": dyn.V("value1"), - }).WithLocations([]dyn.Location{{Line: 7}}), - "environment": dyn.V(map[string]dyn.Value{ - "dependencies": dyn.V([]dyn.Value{ - dyn.V("--editable ${workspace.file_path}"), - }), - }).WithLocations([]dyn.Location{{Line: 8}}), - }), - }), - }), - } + expected := structyaml.M( + "resources", structyaml.M( + "pipelines", structyaml.M( + "my_pipeline", structyaml.M( + "name", "My Pipeline", + "catalog", "${var.catalog}", + "schema", "${var.schema}", + "root_path", "../src/my_pipeline", + "serverless", true, + "libraries", []any{ + structyaml.M("glob", structyaml.M("include", "../src/my_pipeline/transformations/**")), + }, + "configuration", structyaml.M("key0", "value0", "key1", "value1"), + "environment", structyaml.M("dependencies", []any{"--editable ${workspace.file_path}"}), + ), + ), + ), + ) actual, err := convertToResources(&input, "my_pipeline", "src/my_pipeline") require.NoError(t, err) - dynassert.Equal(t, dyn.V(expected), dyn.V(actual)) + assert.Equal(t, expected, actual) } diff --git a/integration/assumptions/dashboard_assumptions_test.go b/integration/assumptions/dashboard_assumptions_test.go index 111482a6f81..be991dd081d 100644 --- a/integration/assumptions/dashboard_assumptions_test.go +++ b/integration/assumptions/dashboard_assumptions_test.go @@ -6,13 +6,11 @@ import ( "github.com/databricks/cli/integration/internal/acc" "github.com/databricks/cli/internal/testutil" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/cli/libs/dyn/merge" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/dashboards" "github.com/databricks/databricks-sdk-go/service/workspace" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -86,27 +84,21 @@ func TestDashboardAssumptions_WorkspaceImport(t *testing.T) { }) require.NoError(t, err) - // Convert the dashboard object to a [dyn.Value] to make comparison easier. - previous, err := convert.FromTyped(previousDashboard, dyn.NilValue) - require.NoError(t, err) - current, err := convert.FromTyped(currentDashboard, dyn.NilValue) - require.NoError(t, err) - // Collect updated and deleted paths. var updatedFieldPaths []string var deletedFieldPaths []string - _, err = merge.Override(previous, current, merge.OverrideVisitor{ - VisitDelete: func(basePath dyn.Path, left dyn.Value) error { - deletedFieldPaths = append(deletedFieldPaths, basePath.String()) + _, err = structvar.PlanOverride(structvar.NewView(previousDashboard, nil, nil), structvar.NewView(currentDashboard, nil, nil), structvar.OverrideVisitor{ + VisitDelete: func(path *structpath.PathNode, left structvar.View) error { + deletedFieldPaths = append(deletedFieldPaths, path.String()) return nil }, - VisitInsert: func(basePath dyn.Path, right dyn.Value) (dyn.Value, error) { + VisitInsert: func(path *structpath.PathNode, right structvar.View) error { assert.Fail(t, "unexpected insert operation") - return right, nil + return nil }, - VisitUpdate: func(basePath dyn.Path, left, right dyn.Value) (dyn.Value, error) { - updatedFieldPaths = append(updatedFieldPaths, basePath.String()) - return right, nil + VisitUpdate: func(path *structpath.PathNode, left, right structvar.View) error { + updatedFieldPaths = append(updatedFieldPaths, path.String()) + return nil }, }) require.NoError(t, err) diff --git a/libs/diag/diagnostic.go b/libs/diag/diagnostic.go index f189cf0ac6c..9356cc9ddd6 100644 --- a/libs/diag/diagnostic.go +++ b/libs/diag/diagnostic.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" ) @@ -21,7 +20,7 @@ type Diagnostic struct { // Locations are the source code locations associated with the diagnostic message. // It may be empty if there are no associated locations. - Locations []dyn.Location + Locations []Location // Paths are paths to the values in the configuration tree that the diagnostic is associated with. // It may be nil if there are no associated paths. diff --git a/libs/dyn/location.go b/libs/diag/location.go similarity index 93% rename from libs/dyn/location.go rename to libs/diag/location.go index b8ec818fcde..3b4a799e348 100644 --- a/libs/dyn/location.go +++ b/libs/diag/location.go @@ -1,4 +1,4 @@ -package dyn +package diag import ( "fmt" diff --git a/libs/dyn/location_test.go b/libs/diag/location_test.go similarity index 55% rename from libs/dyn/location_test.go rename to libs/diag/location_test.go index 29226d73d03..005254009e9 100644 --- a/libs/dyn/location_test.go +++ b/libs/diag/location_test.go @@ -1,13 +1,13 @@ -package dyn_test +package diag_test import ( "testing" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/diag" "github.com/stretchr/testify/assert" ) func TestLocation(t *testing.T) { - loc := dyn.Location{File: "file", Line: 1, Column: 2} + loc := diag.Location{File: "file", Line: 1, Column: 2} assert.Equal(t, "file:1:2", loc.String()) } diff --git a/libs/dyn/convert/end_to_end_test.go b/libs/dyn/convert/end_to_end_test.go deleted file mode 100644 index e3ea273cb04..00000000000 --- a/libs/dyn/convert/end_to_end_test.go +++ /dev/null @@ -1,171 +0,0 @@ -package convert - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func assertFromTypedToTypedEqual[T any](t *testing.T, src T) { - nv, err := FromTyped(src, dyn.NilValue) - require.NoError(t, err) - - var dst T - err = ToTyped(&dst, nv) - require.NoError(t, err) - assert.Equal(t, src, dst) -} - -func TestAdditional(t *testing.T) { - type StructType struct { - Str string `json:"str"` - } - - type Tmp struct { - MapToPointer map[string]*string `json:"map_to_pointer"` - SliceOfPointer []*string `json:"slice_of_pointer"` - NestedStruct StructType `json:"nested_struct"` - } - - t.Run("nil", func(t *testing.T) { - assertFromTypedToTypedEqual(t, Tmp{}) - }) - - t.Run("empty map", func(t *testing.T) { - assertFromTypedToTypedEqual(t, Tmp{ - MapToPointer: map[string]*string{}, - }) - }) - - t.Run("map with empty string value", func(t *testing.T) { - s := "" - assertFromTypedToTypedEqual(t, Tmp{ - MapToPointer: map[string]*string{ - "key": &s, - }, - }) - }) - - t.Run("map with nil value", func(t *testing.T) { - assertFromTypedToTypedEqual(t, Tmp{ - MapToPointer: map[string]*string{ - "key": nil, - }, - }) - }) - - t.Run("empty slice", func(t *testing.T) { - assertFromTypedToTypedEqual(t, Tmp{ - SliceOfPointer: []*string{}, - }) - }) - - t.Run("slice with nil value", func(t *testing.T) { - assertFromTypedToTypedEqual(t, Tmp{ - SliceOfPointer: []*string{nil}, - }) - }) - - t.Run("pointer to a empty string", func(t *testing.T) { - s := "" - assertFromTypedToTypedEqual(t, &s) - }) - - t.Run("nil pointer", func(t *testing.T) { - var s *string - assertFromTypedToTypedEqual(t, s) - }) - - t.Run("pointer to struct with scalar values", func(t *testing.T) { - s := "" - type foo struct { - A string `json:"a"` - B int `json:"b"` - C bool `json:"c"` - D *string `json:"d"` - } - assertFromTypedToTypedEqual(t, &foo{ - A: "a", - B: 1, - C: true, - D: &s, - }) - assertFromTypedToTypedEqual(t, &foo{ - A: "", - B: 0, - C: false, - D: nil, - }) - }) - - t.Run("map with scalar values", func(t *testing.T) { - assertFromTypedToTypedEqual(t, map[string]string{ - "a": "a", - "b": "b", - "c": "", - }) - assertFromTypedToTypedEqual(t, map[string]int{ - "a": 1, - "b": 0, - "c": 2, - }) - }) -} - -func TestEndToEndForceSendFields(t *testing.T) { - type Inner struct { - InnerField string `json:"inner_field"` - ForceSendFields []string `json:"-"` - } - type Outer struct { - OuterField string `json:"outer_field"` - Inner - } - - // Test with zero value in embedded struct - src := Outer{ - OuterField: "outer_value", - Inner: Inner{ - InnerField: "", // Zero value - ForceSendFields: []string{"InnerField"}, // Should be preserved - }, - } - - assertFromTypedToTypedEqual(t, src) -} - -func TestEndToEndPointerForceSendFields(t *testing.T) { - type NewCluster struct { - NumWorkers int `json:"num_workers"` - SparkVersion string `json:"spark_version"` - ForceSendFields []string `json:"-"` - } - type JobCluster struct { - JobClusterKey string `json:"job_cluster_key"` - NewCluster *NewCluster `json:"new_cluster"` - } - type JobSettings struct { - JobClusters []JobCluster `json:"job_clusters"` - Name string `json:"name"` - ForceSendFields []string `json:"-"` - } - - // Test with zero value in pointer embedded struct (like acceptance test) - src := JobSettings{ - Name: "test-job", - JobClusters: []JobCluster{ - { - JobClusterKey: "key", - NewCluster: &NewCluster{ - NumWorkers: 0, // Zero value - SparkVersion: "13.3.x-scala2.12", - ForceSendFields: []string{"NumWorkers"}, // Should be preserved - }, - }, - }, - } - - assertFromTypedToTypedEqual(t, src) -} diff --git a/libs/dyn/convert/error.go b/libs/dyn/convert/error.go deleted file mode 100644 index d3770d82d74..00000000000 --- a/libs/dyn/convert/error.go +++ /dev/null @@ -1,16 +0,0 @@ -package convert - -import ( - "fmt" - - "github.com/databricks/cli/libs/dyn" -) - -type TypeError struct { - value dyn.Value - msg string -} - -func (e TypeError) Error() string { - return fmt.Sprintf("%s: %s", e.value.Location(), e.msg) -} diff --git a/libs/dyn/convert/from_typed.go b/libs/dyn/convert/from_typed.go deleted file mode 100644 index 66451124293..00000000000 --- a/libs/dyn/convert/from_typed.go +++ /dev/null @@ -1,362 +0,0 @@ -package convert - -import ( - "fmt" - "reflect" - "slices" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" -) - -type fromTypedOptions int - -const ( - // If this flag is set, zero values in the typed representation are resolved to - // the equivalent zero value in the dynamic representation. - // If it is not set, zero values resolve to [dyn.NilValue]. - // - // This flag exists to reconcile default values in Go being zero values with values - // being intentionally set to their zero value. We capture zero values in the dynamic - // configuration if they are 1) behind a pointer, 2) a map value, 3) a slice element, - // in the typed configuration. - includeZeroValues fromTypedOptions = 1 << iota -) - -// FromTyped converts changes made in the typed structure w.r.t. the configuration value -// back to the configuration value, retaining existing location information where possible. -// -// It uses the reference value both for location information and to determine if the typed -// value was changed or not. For example, if a struct-by-value field is nil in the reference -// it will be zero-valued in the typed configuration. If it remains zero-valued, this -// function will still emit a nil value in the dynamic representation. -func FromTyped(src any, ref dyn.Value) (dyn.Value, error) { - return fromTyped(src, ref) -} - -// Private implementation of FromTyped that allows for additional options not exposed -// in the public API. -func fromTyped(src any, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, error) { - srcv := reflect.ValueOf(src) - - // Dereference pointer if necessary - for srcv.Kind() == reflect.Pointer { - if srcv.IsNil() { - return dyn.NilValue.WithLocations(ref.Locations()), nil - } - srcv = srcv.Elem() - - // If a pointer to a type points to a zero value, we should include - // that zero value in the dynamic representation. - // This is because by default a pointer is nil in Go, and it not being nil - // indicates its value was intentionally set to zero. - if !slices.Contains(options, includeZeroValues) { - options = append(options, includeZeroValues) - } - } - - var v dyn.Value - var err error - switch srcv.Kind() { - case reflect.Struct: - // Handle SDK native types using JSON marshaling. - if slices.Contains(sdkNativeTypes, srcv.Type()) { - v, err = fromTypedSDKNative(srcv, ref, options...) - } else { - v, err = fromTypedStruct(srcv, ref, options...) - } - case reflect.Map: - v, err = fromTypedMap(srcv, ref) - case reflect.Slice: - v, err = fromTypedSlice(srcv, ref) - case reflect.String: - v, err = fromTypedString(srcv, ref, options...) - case reflect.Bool: - v, err = fromTypedBool(srcv, ref, options...) - case reflect.Int, reflect.Int32, reflect.Int64: - v, err = fromTypedInt(srcv, ref, options...) - case reflect.Float32, reflect.Float64: - v, err = fromTypedFloat(srcv, ref, options...) - case reflect.Invalid: - // If the value is untyped and not set (e.g. any type with nil value), we return nil. - v, err = dyn.NilValue, nil - default: - return dyn.InvalidValue, fmt.Errorf("unsupported type: %s", srcv.Kind()) - } - - // Ensure the location metadata is retained. - if err != nil { - return dyn.InvalidValue, err - } - return v.WithLocations(ref.Locations()), err -} - -func fromTypedStruct(src reflect.Value, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, error) { - // Check that the reference value is compatible or nil. - switch ref.Kind() { - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(ref.MustString()) { - return ref, nil - } - case dyn.KindMap, dyn.KindNil: - default: - return dyn.InvalidValue, fmt.Errorf("cannot convert struct field to dynamic type %#v: src=%#v ref=%#v", ref.Kind().String(), src, ref.AsAny()) - } - - refm, _ := ref.AsMap() - out := dyn.NewMapping() - info := getStructInfo(src.Type()) - - for _, fieldval := range info.FieldValues(src) { - k := fieldval.Key - v := fieldval.Value - isForced := fieldval.IsForced - pair, ok := refm.GetPairByString(k) - refloc := pair.Key.Locations() - refv := pair.Value - - // Use nil reference if there is no reference for this key - if !ok { - refloc = nil - refv = dyn.NilValue - } - - var options []fromTypedOptions - if v.Kind() == reflect.Interface { - options = append(options, includeZeroValues) - } - - // Convert the field taking into account the reference value (may be equal to config.NilValue). - nv, err := fromTyped(v.Interface(), refv, options...) - if err != nil { - return dyn.InvalidValue, err - } - - // Either if the key was set in the reference, the field is not zero-valued, OR it's forced - if ok || nv.Kind() != dyn.KindNil || isForced { - // If v isZero, it could be because it's a variable reference; so we check that nv is zero as well - // BUT: always include if it's forced - if v.Kind() != reflect.Struct && v.IsZero() && nv.IsZero() && !info.ForceEmpty[k] && !isForced { - continue - } - - // If the field is forced but nv is nil, convert it to the appropriate zero value - if isForced && nv.Kind() == dyn.KindNil { - // Convert the zero value using proper recursive conversion instead of dyn.V() directly - // This prevents "not handled" panics for complex types like structs, slices, maps, etc. - // Use refv to preserve location information from the original reference - var err error - nv, err = fromTyped(v.Interface(), refv, includeZeroValues) - if err != nil { - return dyn.InvalidValue, err - } - } - - out.SetLoc(k, refloc, nv) - } - } - - // Return the new mapping if: - // 1. The mapping has entries (i.e. the struct was not empty). - // 2. The reference is a map (i.e. the struct was and still is empty). - // 3. The "includeZeroValues" option is set (i.e. the struct is a non-nil pointer). - if out.Len() > 0 || ref.Kind() == dyn.KindMap || slices.Contains(options, includeZeroValues) { - return dyn.V(out), nil - } - - // Otherwise, return nil. - return dyn.NilValue, nil -} - -func fromTypedMap(src reflect.Value, ref dyn.Value) (dyn.Value, error) { - // Check that the reference value is compatible or nil. - switch ref.Kind() { - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(ref.MustString()) { - return ref, nil - } - case dyn.KindMap, dyn.KindNil: - default: - return dyn.InvalidValue, fmt.Errorf("cannot convert map field to dynamic type %#v: src=%#v ref=%#v", ref.Kind().String(), src, ref.AsAny()) - } - - // Return nil if the map is nil. - if src.IsNil() { - return dyn.NilValue, nil - } - - refm, _ := ref.AsMap() - out := dyn.NewMapping() - iter := src.MapRange() - for iter.Next() { - k := iter.Key().String() - v := iter.Value() - pair, ok := refm.GetPairByString(k) - refloc := pair.Key.Locations() - refv := pair.Value - - // Use nil reference if there is no reference for this key - if !ok { - refloc = nil - refv = dyn.NilValue - } - - // Convert entry taking into account the reference value (may be equal to dyn.NilValue). - nv, err := fromTyped(v.Interface(), refv, includeZeroValues) - if err != nil { - return dyn.InvalidValue, err - } - - // Every entry is represented, even if it is a nil. - // Otherwise, a map with zero-valued structs would yield a nil as well. - out.SetLoc(k, refloc, nv) - } - - return dyn.V(out), nil -} - -func fromTypedSlice(src reflect.Value, ref dyn.Value) (dyn.Value, error) { - // Check that the reference value is compatible or nil. - switch ref.Kind() { - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(ref.MustString()) { - return ref, nil - } - case dyn.KindSequence, dyn.KindNil: - default: - return dyn.InvalidValue, fmt.Errorf("cannot convert slice field to dynamic type %#v: src=%#v ref=%#v", ref.Kind().String(), src, ref.AsAny()) - } - - // Return nil if the slice is nil. - if src.IsNil() { - return dyn.NilValue, nil - } - - out := make([]dyn.Value, src.Len()) - for i := range src.Len() { - v := src.Index(i) - refv := ref.Index(i) - - // Use nil reference if there is no reference for this index. - if refv.Kind() == dyn.KindInvalid { - refv = dyn.NilValue - } - - // Convert entry taking into account the reference value (may be equal to dyn.NilValue). - nv, err := fromTyped(v.Interface(), refv, includeZeroValues) - if err != nil { - return dyn.InvalidValue, err - } - - out[i] = nv - } - - return dyn.V(out), nil -} - -func fromTypedString(src reflect.Value, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, error) { - switch ref.Kind() { - case dyn.KindString: - value := src.String() - if value == ref.MustString() { - return ref, nil - } - - return dyn.V(value), nil - case dyn.KindNil: - // This field is not set in the reference. We set it to nil if it's zero - // valued in the typed representation and the includeZeroValues option is not set. - if src.IsZero() && !slices.Contains(options, includeZeroValues) { - return dyn.NilValue, nil - } - return dyn.V(src.String()), nil - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, fmt.Errorf("cannot convert string field to dynamic type %#v: src=%#v ref=%#v", ref.Kind().String(), src, ref.AsAny()) -} - -func fromTypedBool(src reflect.Value, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, error) { - switch ref.Kind() { - case dyn.KindBool: - value := src.Bool() - if value == ref.MustBool() { - return ref, nil - } - return dyn.V(value), nil - case dyn.KindNil: - // This field is not set in the reference. We set it to nil if it's zero - // valued in the typed representation and the includeZeroValues option is not set. - if src.IsZero() && !slices.Contains(options, includeZeroValues) { - return dyn.NilValue, nil - } - return dyn.V(src.Bool()), nil - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(ref.MustString()) { - return ref, nil - } - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, fmt.Errorf("cannot convert bool field to dynamic type %#v: src=%#v ref=%#v", ref.Kind().String(), src, ref.AsAny()) -} - -func fromTypedInt(src reflect.Value, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, error) { - switch ref.Kind() { - case dyn.KindInt: - value := src.Int() - if value == ref.MustInt() { - return ref, nil - } - return dyn.V(value), nil - case dyn.KindNil: - // This field is not set in the reference. We set it to nil if it's zero - // valued in the typed representation and the includeZeroValues option is not set. - if src.IsZero() && !slices.Contains(options, includeZeroValues) { - return dyn.NilValue, nil - } - return dyn.V(src.Int()), nil - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(ref.MustString()) { - return ref, nil - } - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, fmt.Errorf("cannot convert int field to dynamic type %#v: src=%#v ref=%#v", ref.Kind().String(), src, ref.AsAny()) -} - -func fromTypedFloat(src reflect.Value, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, error) { - switch ref.Kind() { - case dyn.KindFloat: - value := src.Float() - if value == ref.MustFloat() { - return ref, nil - } - return dyn.V(value), nil - case dyn.KindNil: - // This field is not set in the reference. We set it to nil if it's zero - // valued in the typed representation and the includeZeroValues option is not set. - if src.IsZero() && !slices.Contains(options, includeZeroValues) { - return dyn.NilValue, nil - } - return dyn.V(src.Float()), nil - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(ref.MustString()) { - return ref, nil - } - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, fmt.Errorf("cannot convert float field to dynamic type %#v: src=%#v ref=%#v", ref.Kind().String(), src, ref.AsAny()) -} diff --git a/libs/dyn/convert/from_typed_test.go b/libs/dyn/convert/from_typed_test.go deleted file mode 100644 index 0c8cf902bb4..00000000000 --- a/libs/dyn/convert/from_typed_test.go +++ /dev/null @@ -1,928 +0,0 @@ -package convert - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynassert" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestFromTypedStructZeroFields(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - src := Tmp{} - - // For an empty struct with a nil reference we expect a nil. - nv, err := FromTyped(src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) - - // For an empty struct with a non-nil reference we expect an empty map. - nv, err = FromTyped(src, dyn.V(map[string]dyn.Value{})) - require.NoError(t, err) - assert.Equal(t, dyn.V(map[string]dyn.Value{}), nv) -} - -func TestFromTypedStructPointerZeroFields(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - var src *Tmp - var nv dyn.Value - var err error - - // For a nil pointer with a nil reference we expect a nil. - src = nil - nv, err = FromTyped(src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) - - // For a nil pointer with a non-nil reference we expect a nil. - src = nil - nv, err = FromTyped(src, dyn.V(map[string]dyn.Value{})) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) - - // For an initialized pointer with a nil reference we expect an empty map. - src = &Tmp{} - nv, err = FromTyped(src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, dyn.V(map[string]dyn.Value{}), nv) - - // For an initialized pointer with a non-nil reference we expect an empty map. - src = &Tmp{} - nv, err = FromTyped(src, dyn.V(map[string]dyn.Value{})) - require.NoError(t, err) - assert.Equal(t, dyn.V(map[string]dyn.Value{}), nv) -} - -func TestFromTypedStructNilFields(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - // For a zero value struct with a reference containing nil fields we expect the nils to be retained. - src := Tmp{} - ref := dyn.V(map[string]dyn.Value{ - "foo": dyn.NilValue, - "bar": dyn.NilValue, - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.NilValue, - "bar": dyn.NilValue, - }), nv) -} - -func TestFromTypedStructSetFields(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - src := Tmp{ - Foo: "foo", - Bar: "bar", - } - - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V("foo"), - "bar": dyn.V("bar"), - }), nv) -} - -func TestFromTypedStructSetFieldsIntoExistingFields(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - src := Tmp{ - Foo: "foo", - Bar: "bar", - } - - ref := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("foo_old"), - "bar": dyn.V("bar_old"), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V("foo"), - "bar": dyn.V("bar"), - }), nv) -} - -func TestFromTypedStructZeroFieldsIntoExistingFields(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar,omitempty"` - BazBugz string `json:"baz_bugs,omitempty"` - ForceSendFields []string `json:"-"` - } - - src := Tmp{ForceSendFields: []string{"BazBugz"}} - - ref := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("foo_old"), - "bar": dyn.V("bar_old"), - "baz_bugs": dyn.V("baz_old"), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(""), - "baz_bugs": dyn.V(""), - }), nv) -} - -func TestFromTypedStructZeroFieldsIntoExistingFieldsFloat64(t *testing.T) { - type Tmp struct { - Foo float64 `json:"foo"` - Bar float64 `json:"bar,omitempty"` - BazBugz float64 `json:"baz_bugs,omitempty"` - ForceSendFields []string `json:"-"` - } - - src := Tmp{ForceSendFields: []string{"BazBugz"}} - - ref := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(1.2), - "bar": dyn.V(1.2), - "baz_bugs": dyn.V(1.2), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(0.0), - "baz_bugs": dyn.V(0.0), - }), nv) -} - -func TestFromTypedStructSetFieldsRetainLocation(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - src := Tmp{ - Foo: "bar", - Bar: "qux", - } - - ref := dyn.V(map[string]dyn.Value{ - "foo": dyn.NewValue("bar", []dyn.Location{{File: "foo"}}), - "bar": dyn.NewValue("baz", []dyn.Location{{File: "bar"}}), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - - // Assert foo and bar have retained their location. - assert.Equal(t, dyn.NewValue("bar", []dyn.Location{{File: "foo"}}), nv.Get("foo")) - assert.Equal(t, dyn.NewValue("qux", []dyn.Location{{File: "bar"}}), nv.Get("bar")) -} - -func TestFromTypedStringMapWithZeroValue(t *testing.T) { - ref := dyn.NilValue - src := map[string]string{ - "foo": "", - "bar": "fuzz", - } - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(""), - "bar": dyn.V("fuzz"), - }), nv) -} - -func TestFromTypedStringSliceWithZeroValue(t *testing.T) { - ref := dyn.NilValue - src := []string{"a", "", "c"} - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V([]dyn.Value{ - dyn.V("a"), dyn.V(""), dyn.V("c"), - }), nv) -} - -func TestFromTypedStringStructWithZeroValue(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - ref := dyn.NilValue - src := Tmp{ - Foo: "foo", - Bar: "", - } - - // Note, the zero value is not included in the output. - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V("foo"), - }), nv) -} - -func TestFromTypedBoolMapWithZeroValue(t *testing.T) { - ref := dyn.NilValue - src := map[string]bool{ - "foo": false, - "bar": true, - } - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(false), - "bar": dyn.V(true), - }), nv) -} - -func TestFromTypedBoolSliceWithZeroValue(t *testing.T) { - ref := dyn.NilValue - src := []bool{true, false, true} - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V([]dyn.Value{ - dyn.V(true), dyn.V(false), dyn.V(true), - }), nv) -} - -func TestFromTypedBoolStructWithZeroValue(t *testing.T) { - type Tmp struct { - Foo bool `json:"foo"` - Bar bool `json:"bar"` - } - - ref := dyn.NilValue - src := Tmp{ - Foo: true, - Bar: false, - } - - // Note, the zero value is not included in the output. - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(true), - }), nv) -} - -func TestFromTypedIntMapWithZeroValue(t *testing.T) { - ref := dyn.NilValue - src := map[string]int{ - "foo": 0, - "bar": 1, - } - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(int64(0)), - "bar": dyn.V(int64(1)), - }), nv) -} - -func TestFromTypedIntSliceWithZeroValue(t *testing.T) { - ref := dyn.NilValue - src := []int{1, 0, 2} - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V([]dyn.Value{ - dyn.V(int64(1)), dyn.V(int64(0)), dyn.V(int64(2)), - }), nv) -} - -func TestFromTypedIntStructWithZeroValue(t *testing.T) { - type Tmp struct { - Foo int `json:"foo"` - Bar int `json:"bar"` - } - - ref := dyn.NilValue - src := Tmp{ - Foo: 1, - Bar: 0, - } - - // Note, the zero value is not included in the output. - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(int64(1)), - }), nv) -} - -func TestFromTypedFloatMapWithZeroValue(t *testing.T) { - ref := dyn.NilValue - src := map[string]float64{ - "foo": 0.0, - "bar": 1.0, - } - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(0.0), - "bar": dyn.V(1.0), - }), nv) -} - -func TestFromTypedFloatSliceWithZeroValue(t *testing.T) { - ref := dyn.NilValue - src := []float64{1.0, 0.0, 2.0} - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V([]dyn.Value{ - dyn.V(1.0), dyn.V(0.0), dyn.V(2.0), - }), nv) -} - -func TestFromTypedFloatStructWithZeroValue(t *testing.T) { - type Tmp struct { - Foo float64 `json:"foo"` - Bar float64 `json:"bar"` - } - - ref := dyn.NilValue - src := Tmp{ - Foo: 1.0, - Bar: 0.0, - } - - // Note, the zero value is not included in the output. - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(1.0), - }), nv) -} - -func TestFromTypedMapNil(t *testing.T) { - var src map[string]string = nil - - ref := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) -} - -func TestFromTypedMapEmpty(t *testing.T) { - src := map[string]string{} - - ref := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(map[string]dyn.Value{}), nv) -} - -func TestFromTypedMapNonEmpty(t *testing.T) { - src := map[string]string{ - "foo": "foo", - "bar": "bar", - } - - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V("foo"), - "bar": dyn.V("bar"), - }), nv) -} - -func TestFromTypedMapNonEmptyRetainLocation(t *testing.T) { - src := map[string]string{ - "foo": "bar", - "bar": "qux", - } - - ref := dyn.V(map[string]dyn.Value{ - "foo": dyn.NewValue("bar", []dyn.Location{{File: "foo"}}), - "bar": dyn.NewValue("baz", []dyn.Location{{File: "bar"}}), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - - // Assert foo and bar have retained their locations. - assert.Equal(t, dyn.NewValue("bar", []dyn.Location{{File: "foo"}}), nv.Get("foo")) - assert.Equal(t, dyn.NewValue("qux", []dyn.Location{{File: "bar"}}), nv.Get("bar")) -} - -func TestFromTypedMapFieldWithZeroValue(t *testing.T) { - src := map[string]string{ - "foo": "", - } - - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(""), - }), nv) -} - -func TestFromTypedSliceNil(t *testing.T) { - var src []string = nil - - ref := dyn.V([]dyn.Value{ - dyn.V("bar"), - dyn.V("baz"), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) -} - -func TestFromTypedSliceEmpty(t *testing.T) { - //nolint:gocritic - src := []string{} - - ref := dyn.V([]dyn.Value{ - dyn.V("bar"), - dyn.V("baz"), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V([]dyn.Value{}), nv) -} - -func TestFromTypedSliceNonEmpty(t *testing.T) { - src := []string{ - "foo", - "bar", - } - - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V([]dyn.Value{ - dyn.V("foo"), - dyn.V("bar"), - }), nv) -} - -func TestFromTypedSliceNonEmptyRetainLocation(t *testing.T) { - src := []string{ - "foo", - "bar", - } - - ref := dyn.V([]dyn.Value{ - dyn.NewValue("foo", []dyn.Location{{File: "foo"}}), - dyn.NewValue("bar", []dyn.Location{{File: "bar"}}), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - - // Assert foo and bar have retained their locations. - assert.Equal(t, dyn.NewValue("foo", []dyn.Location{{File: "foo"}}), nv.Index(0)) - assert.Equal(t, dyn.NewValue("bar", []dyn.Location{{File: "bar"}}), nv.Index(1)) -} - -func TestFromTypedStringEmpty(t *testing.T) { - var src string - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) -} - -func TestFromTypedStringEmptyOverwrite(t *testing.T) { - var src string - ref := dyn.V("old") - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(""), nv) -} - -func TestFromTypedStringNonEmpty(t *testing.T) { - src := "new" - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V("new"), nv) -} - -func TestFromTypedStringNonEmptyOverwrite(t *testing.T) { - src := "new" - ref := dyn.V("old") - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V("new"), nv) -} - -func TestFromTypedStringRetainsLocations(t *testing.T) { - ref := dyn.NewValue("foo", []dyn.Location{{File: "foo"}}) - - // case: value has not been changed - src := "foo" - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue("foo", []dyn.Location{{File: "foo"}}), nv) - - // case: value has been changed - src = "bar" - nv, err = FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue("bar", []dyn.Location{{File: "foo"}}), nv) -} - -func TestFromTypedStringTypeError(t *testing.T) { - src := "foo" - ref := dyn.V(1234) - _, err := FromTyped(src, ref) - require.Error(t, err) -} - -func TestFromTypedBoolEmpty(t *testing.T) { - var src bool - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) -} - -func TestFromTypedBoolEmptyOverwrite(t *testing.T) { - var src bool - ref := dyn.V(true) - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(false), nv) -} - -func TestFromTypedBoolNonEmpty(t *testing.T) { - src := true - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(true), nv) -} - -func TestFromTypedBoolNonEmptyOverwrite(t *testing.T) { - src := true - ref := dyn.V(false) - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(true), nv) -} - -func TestFromTypedBoolRetainsLocations(t *testing.T) { - ref := dyn.NewValue(true, []dyn.Location{{File: "foo"}}) - - // case: value has not been changed - src := true - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue(true, []dyn.Location{{File: "foo"}}), nv) - - // case: value has been changed - src = false - nv, err = FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue(false, []dyn.Location{{File: "foo"}}), nv) -} - -func TestFromTypedBoolVariableReference(t *testing.T) { - src := true - ref := dyn.V("${var.foo}") - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V("${var.foo}"), nv) -} - -func TestFromTypedBoolTypeError(t *testing.T) { - src := true - ref := dyn.V("string") - _, err := FromTyped(src, ref) - require.Error(t, err) -} - -func TestFromTypedIntEmpty(t *testing.T) { - var src int - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) -} - -func TestFromTypedIntEmptyOverwrite(t *testing.T) { - var src int - ref := dyn.V(1234) - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(int64(0)), nv) -} - -func TestFromTypedIntNonEmpty(t *testing.T) { - src := 1234 - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(int64(1234)), nv) -} - -func TestFromTypedIntNonEmptyOverwrite(t *testing.T) { - src := 1234 - ref := dyn.V(1233) - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(int64(1234)), nv) -} - -func TestFromTypedIntRetainsLocations(t *testing.T) { - ref := dyn.NewValue(1234, []dyn.Location{{File: "foo"}}) - - // case: value has not been changed - src := 1234 - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue(1234, []dyn.Location{{File: "foo"}}), nv) - - // case: value has been changed - src = 1235 - nv, err = FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue(int64(1235), []dyn.Location{{File: "foo"}}), nv) -} - -func TestFromTypedIntVariableReference(t *testing.T) { - src := 1234 - ref := dyn.V("${var.foo}") - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V("${var.foo}"), nv) -} - -func TestFromTypedIntTypeError(t *testing.T) { - src := 1234 - ref := dyn.V("string") - _, err := FromTyped(src, ref) - require.Error(t, err) -} - -func TestFromTypedFloatEmpty(t *testing.T) { - var src float64 - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) -} - -func TestFromTypedFloatEmptyOverwrite(t *testing.T) { - var src float64 - ref := dyn.V(1.23) - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(0.0), nv) -} - -func TestFromTypedFloatNonEmpty(t *testing.T) { - src := 1.23 - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(1.23), nv) -} - -func TestFromTypedFloatNonEmptyOverwrite(t *testing.T) { - src := 1.23 - ref := dyn.V(1.24) - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(1.23), nv) -} - -func TestFromTypedFloatRetainsLocations(t *testing.T) { - var src float64 - ref := dyn.NewValue(1.23, []dyn.Location{{File: "foo"}}) - - // case: value has not been changed - src = 1.23 - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue(1.23, []dyn.Location{{File: "foo"}}), nv) - - // case: value has been changed - src = 1.24 - nv, err = FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue(1.24, []dyn.Location{{File: "foo"}}), nv) -} - -func TestFromTypedFloatVariableReference(t *testing.T) { - src := 1.23 - ref := dyn.V("${var.foo}") - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V("${var.foo}"), nv) -} - -func TestFromTypedFloatTypeError(t *testing.T) { - src := 1.23 - ref := dyn.V("string") - _, err := FromTyped(src, ref) - require.Error(t, err) -} - -func TestFromTypedAny(t *testing.T) { - type Tmp struct { - Foo any `json:"foo"` - Bar any `json:"bar"` - Foz any `json:"foz"` - Baz any `json:"baz"` - } - - src := Tmp{ - Foo: "foo", - Bar: false, - Foz: 0, - Baz: map[string]any{ - "foo": "foo", - "bar": 1234, - "qux": 0, - "nil": nil, - }, - } - - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V("foo"), - "bar": dyn.V(false), - "foz": dyn.V(int64(0)), - "baz": dyn.V(map[string]dyn.Value{ - "foo": dyn.V("foo"), - "bar": dyn.V(int64(1234)), - "qux": dyn.V(int64(0)), - "nil": dyn.V(nil), - }), - }), nv) -} - -func TestFromTypedAnyNil(t *testing.T) { - var src any = nil - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) -} - -func TestFromTypedNilPointerRetainsLocations(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - var src *Tmp - ref := dyn.NewValue(nil, []dyn.Location{{File: "foobar"}}) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue(nil, []dyn.Location{{File: "foobar"}}), nv) -} - -func TestFromTypedNilMapRetainsLocation(t *testing.T) { - var src map[string]string - ref := dyn.NewValue(nil, []dyn.Location{{File: "foobar"}}) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue(nil, []dyn.Location{{File: "foobar"}}), nv) -} - -func TestFromTypedNilSliceRetainsLocation(t *testing.T) { - var src []string - ref := dyn.NewValue(nil, []dyn.Location{{File: "foobar"}}) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue(nil, []dyn.Location{{File: "foobar"}}), nv) -} - -func TestFromTypedForceSendFieldsComplexTypes(t *testing.T) { - type Inner struct { - Value string `json:"value"` - } - - tests := []struct { - name string - src any - }{ - { - name: "struct_pointer", - src: struct { - Field *Inner `json:"field"` - ForceSendFields []string `json:"-"` - }{Field: nil, ForceSendFields: []string{"Field"}}, - }, - { - name: "struct_value", - src: struct { - Field Inner `json:"field"` - ForceSendFields []string `json:"-"` - }{Field: Inner{}, ForceSendFields: []string{"Field"}}, - }, - { - name: "slice", - src: struct { - Field []string `json:"field"` - ForceSendFields []string `json:"-"` - }{Field: nil, ForceSendFields: []string{"Field"}}, - }, - { - name: "map", - src: struct { - Field map[string]string `json:"field"` - ForceSendFields []string `json:"-"` - }{Field: nil, ForceSendFields: []string{"Field"}}, - }, - { - name: "interface", - src: struct { - Field any `json:"field"` - ForceSendFields []string `json:"-"` - }{Field: (*string)(nil), ForceSendFields: []string{"Field"}}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - nv, err := FromTyped(tt.src, dyn.NilValue) - require.NoError(t, err) - - // All should include the field because it's in ForceSendFields - field := nv.Get("field") - assert.True(t, field.IsValid(), "field should be present due to ForceSendFields") - }) - } -} - -// Test embedded structs with ForceSendFields (separate test due to different structure) -func TestFromTypedForceSendFieldsEmbedded(t *testing.T) { - type Inner struct { - Field *string `json:"field"` - ForceSendFields []string `json:"-"` - } - type Outer struct { - Inner - Other string `json:"other"` - } - - src := Outer{ - Inner: Inner{ - Field: nil, - ForceSendFields: []string{"Field"}, - }, - Other: "value", - } - - nv, err := FromTyped(src, dyn.NilValue) - require.NoError(t, err) - - // Both fields should be present - field := nv.Get("field") - other := nv.Get("other") - assert.True(t, field.IsValid(), "embedded field should be present due to ForceSendFields") - assert.Equal(t, dyn.KindNil, field.Kind(), "embedded field should be present due to ForceSendFields") - assert.Equal(t, dyn.V("value"), other) -} diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go deleted file mode 100644 index 379baf439d1..00000000000 --- a/libs/dyn/convert/normalize.go +++ /dev/null @@ -1,507 +0,0 @@ -package convert - -import ( - "fmt" - "reflect" - "slices" - "strconv" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" -) - -// NormalizeOption is the type for options that can be passed to Normalize. -type NormalizeOption int - -const ( - // IncludeMissingFields causes the normalization to include fields that defined on the given - // type but are missing in the source value. They are included with their zero values. - IncludeMissingFields NormalizeOption = iota - - // DropEmptyStrings drops struct fields whose value is an empty string, unless - // the field is tagged without omitempty (i.e. its zero value must be sent). - // This mirrors JSON serialization, where an omitempty string field set to "" - // is omitted from the request. Without this, an explicitly-set "" reaches - // ToTyped, which force-sends it, and the backend rejects it (e.g. - // "'' is not a valid cluster policy ID"). - DropEmptyStrings -) - -type normalizeOptions struct { - includeMissingFields bool - dropEmptyStrings bool -} - -func Normalize(dst any, src dyn.Value, opts ...NormalizeOption) (dyn.Value, diag.Diagnostics) { - var n normalizeOptions - for _, opt := range opts { - switch opt { - case IncludeMissingFields: - n.includeMissingFields = true - case DropEmptyStrings: - n.dropEmptyStrings = true - } - } - - return n.normalizeType(reflect.TypeOf(dst), src, []reflect.Type{}, dyn.EmptyPath) -} - -func (n normalizeOptions) normalizeType(typ reflect.Type, src dyn.Value, seen []reflect.Type, path dyn.Path) (dyn.Value, diag.Diagnostics) { - for typ.Kind() == reflect.Pointer { - typ = typ.Elem() - } - - switch typ.Kind() { - case reflect.Struct: - // Handle SDK native types as strings since they use custom JSON marshaling. - if slices.Contains(sdkNativeTypes, typ) { - return n.normalizeString(reflect.TypeFor[string](), src, path) - } - return n.normalizeStruct(typ, src, append(seen, typ), path) - case reflect.Map: - return n.normalizeMap(typ, src, append(seen, typ), path) - case reflect.Slice: - return n.normalizeSlice(typ, src, append(seen, typ), path) - case reflect.String: - return n.normalizeString(typ, src, path) - case reflect.Bool: - return n.normalizeBool(typ, src, path) - case reflect.Int, reflect.Int32, reflect.Int64: - return n.normalizeInt(typ, src, path) - case reflect.Float32, reflect.Float64: - return n.normalizeFloat(typ, src, path) - case reflect.Interface: - return n.normalizeInterface(typ, src, path) - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, diag.Errorf("unsupported type: %s", typ.Kind()) -} - -func nullWarning(expected dyn.Kind, src dyn.Value, path dyn.Path) diag.Diagnostic { - return diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("expected a %s value, found null", expected), - Locations: []dyn.Location{src.Location()}, - Paths: dyn.ToStructPaths(path), - } -} - -func typeMismatch(expected dyn.Kind, src dyn.Value, path dyn.Path) diag.Diagnostic { - return diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("expected %s, found %s", expected, src.Kind()), - Locations: []dyn.Location{src.Location()}, - Paths: dyn.ToStructPaths(path), - } -} - -// isAnchorContainer reports whether v is a YAML anchor or a non-empty -// sequence/map composed entirely of anchor containers. Anchors define reusable -// blocks and must not trigger "unknown field" warnings, including when nested -// inside a container. -func isAnchorContainer(v dyn.Value) bool { - if v.IsAnchor() { - return true - } - - var elements []dyn.Value - switch v.Kind() { - case dyn.KindSequence: - elements = v.MustSequence() - case dyn.KindMap: - elements = v.MustMap().Values() - default: - return false - } - - if len(elements) == 0 { - return false - } - for _, e := range elements { - if !isAnchorContainer(e) { - return false - } - } - return true -} - -func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen []reflect.Type, path dyn.Path) (dyn.Value, diag.Diagnostics) { - var diags diag.Diagnostics - - switch src.Kind() { - case dyn.KindMap: - out := dyn.NewMapping() - info := getStructInfo(typ) - for _, pair := range src.MustMap().Pairs() { - pk := pair.Key - pv := pair.Value - - fieldName := pk.MustString() - index, ok := info.Fields[fieldName] - if !ok { - if !isAnchorContainer(pv) { - // Special case: provide a more helpful message for "valueFrom" vs "value_from" - if fieldName == "valueFrom" { - if _, hasValueFrom := info.Fields["value_from"]; hasValueFrom { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: "Use 'value_from' instead of 'valueFrom'", - Detail: "The field 'valueFrom' should be 'value_from' (snake_case). The 'valueFrom' field will be ignored.", - Locations: pk.Locations(), - Paths: dyn.ToStructPaths(path), - }) - continue - } - } - - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: "unknown field: " + fieldName, - Locations: pk.Locations(), - Paths: dyn.ToStructPaths(path), - }) - } - continue - } - - // Normalize the value according to the field type. - nv, err := n.normalizeType(typ.FieldByIndex(index).Type, pv, seen, path.Append(dyn.Key(pk.MustString()))) - if err != nil { - diags = diags.Extend(err) - // Skip the element if it cannot be normalized. - if !nv.IsValid() { - continue - } - } - - // Drop an empty string on an omitempty field so it is not force-sent - // to the backend (see DropEmptyStrings). ForceEmpty marks fields whose - // zero value must be kept, so those are left in place. - if n.dropEmptyStrings && !info.ForceEmpty[fieldName] { - if s, ok := nv.AsString(); ok && s == "" { - continue - } - } - - out.SetLoc(pk.MustString(), pk.Locations(), nv) - } - - // Return the normalized value if missing fields are not included. - if !n.includeMissingFields { - return dyn.NewValue(out, src.Locations()), diags - } - - // Populate missing fields with their zero values. - for k, index := range info.Fields { - if _, ok := out.GetByString(k); ok { - continue - } - - // Optionally dereference pointers to get the underlying field type. - ftyp := typ.FieldByIndex(index).Type - for ftyp.Kind() == reflect.Pointer { - ftyp = ftyp.Elem() - } - - // Skip field if we have already seen its type to avoid infinite recursion - // when filling in the zero value of a recursive type. - if slices.Contains(seen, ftyp) { - continue - } - - var v dyn.Value - switch ftyp.Kind() { - case reflect.Struct, reflect.Map: - v, _ = n.normalizeType(ftyp, dyn.V(map[string]dyn.Value{}), seen, path.Append(dyn.Key(k))) - case reflect.Slice: - v, _ = n.normalizeType(ftyp, dyn.V([]dyn.Value{}), seen, path.Append(dyn.Key(k))) - case reflect.String: - v, _ = n.normalizeType(ftyp, dyn.V(""), seen, path.Append(dyn.Key(k))) - case reflect.Bool: - v, _ = n.normalizeType(ftyp, dyn.V(false), seen, path.Append(dyn.Key(k))) - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - v, _ = n.normalizeType(ftyp, dyn.V(int64(0)), seen, path.Append(dyn.Key(k))) - case reflect.Float32, reflect.Float64: - v, _ = n.normalizeType(ftyp, dyn.V(float64(0)), seen, path.Append(dyn.Key(k))) - default: - // Skip fields for which we do not have a natural [dyn.Value] equivalent. - // For example, we don't handle reflect.Complex* and reflect.Uint* types. - continue - } - if v.IsValid() { - out.SetLoc(k, nil, v) - } - } - - return dyn.NewValue(out, src.Locations()), diags - case dyn.KindNil: - return src, diags - - case dyn.KindString: - // Return verbatim if it's a pure variable reference. - if dynvar.IsPureVariableReference(src.MustString()) { - return src, nil - } - default: - // Fall through to the error case. - } - - // Cannot interpret as a struct. - return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindMap, src, path)) -} - -func (n normalizeOptions) normalizeMap(typ reflect.Type, src dyn.Value, seen []reflect.Type, path dyn.Path) (dyn.Value, diag.Diagnostics) { - var diags diag.Diagnostics - - switch src.Kind() { - case dyn.KindMap: - out := dyn.NewMapping() - for _, pair := range src.MustMap().Pairs() { - pk := pair.Key - pv := pair.Value - - // Normalize the value according to the map element type. - nv, err := n.normalizeType(typ.Elem(), pv, seen, path.Append(dyn.Key(pk.MustString()))) - if err != nil { - diags = diags.Extend(err) - // Skip the element if it cannot be normalized. - if !nv.IsValid() { - continue - } - } - - out.SetLoc(pk.MustString(), pk.Locations(), nv) - } - - return dyn.NewValue(out, src.Locations()), diags - case dyn.KindNil: - return src, diags - - case dyn.KindString: - // Return verbatim if it's a pure variable reference. - if dynvar.IsPureVariableReference(src.MustString()) { - return src, nil - } - default: - // Fall through to the error case. - } - - // Cannot interpret as a map. - return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindMap, src, path)) -} - -func (n normalizeOptions) normalizeSlice(typ reflect.Type, src dyn.Value, seen []reflect.Type, path dyn.Path) (dyn.Value, diag.Diagnostics) { - var diags diag.Diagnostics - - switch src.Kind() { - case dyn.KindSequence: - out := make([]dyn.Value, 0, len(src.MustSequence())) - for _, v := range src.MustSequence() { - // Normalize the value according to the slice element type. - v, err := n.normalizeType(typ.Elem(), v, seen, path.Append(dyn.Index(len(out)))) - if err != nil { - diags = diags.Extend(err) - // Skip the element if it cannot be normalized. - if !v.IsValid() { - continue - } - } - - out = append(out, v) - } - - return dyn.NewValue(out, src.Locations()), diags - case dyn.KindNil: - return src, diags - - case dyn.KindString: - // Return verbatim if it's a pure variable reference. - if dynvar.IsPureVariableReference(src.MustString()) { - return src, nil - } - default: - // Fall through to the error case. - } - - // Cannot interpret as a slice. - return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindSequence, src, path)) -} - -func (n normalizeOptions) normalizeString(typ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) { - var diags diag.Diagnostics - - switch src.Kind() { - case dyn.KindString: - return dyn.NewValue(src.MustString(), src.Locations()), nil - case dyn.KindBool: - return dyn.NewValue(strconv.FormatBool(src.MustBool()), src.Locations()), nil - case dyn.KindInt: - return dyn.NewValue(strconv.FormatInt(src.MustInt(), 10), src.Locations()), nil - case dyn.KindFloat: - return dyn.NewValue(strconv.FormatFloat(src.MustFloat(), 'f', -1, 64), src.Locations()), nil - case dyn.KindTime: - return dyn.NewValue(src.MustTime().String(), src.Locations()), nil - case dyn.KindNil: - // Return a warning if the field is present but has a null value. - return dyn.InvalidValue, diags.Append(nullWarning(dyn.KindString, src, path)) - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindString, src, path)) -} - -func (n normalizeOptions) normalizeBool(typ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) { - var diags diag.Diagnostics - - switch src.Kind() { - case dyn.KindBool: - return dyn.NewValue(src.MustBool(), src.Locations()), nil - case dyn.KindString: - // See https://yaml.org/type/bool.html. - switch src.MustString() { - case "true", "True", "TRUE", "y", "Y", "yes", "Yes", "YES", "on", "On", "ON": - return dyn.NewValue(true, src.Locations()), nil - case "false", "False", "FALSE", "n", "N", "no", "No", "NO", "off", "Off", "OFF": - return dyn.NewValue(false, src.Locations()), nil - default: - // Return verbatim if it's a pure variable reference. - if dynvar.IsPureVariableReference(src.MustString()) { - return src, nil - } - - // Cannot interpret as a boolean. - return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindBool, src, path)) - } - case dyn.KindNil: - // Return a warning if the field is present but has a null value. - return dyn.InvalidValue, diags.Append(nullWarning(dyn.KindBool, src, path)) - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindBool, src, path)) -} - -func (n normalizeOptions) normalizeInt(typ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) { - var diags diag.Diagnostics - - switch src.Kind() { - case dyn.KindInt: - return dyn.NewValue(src.MustInt(), src.Locations()), nil - case dyn.KindFloat: - out := int64(src.MustFloat()) - if src.MustFloat() != float64(out) { - return dyn.InvalidValue, diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf(`cannot accurately represent "%g" as integer due to precision loss`, src.MustFloat()), - Locations: []dyn.Location{src.Location()}, - Paths: dyn.ToStructPaths(path), - }) - } - return dyn.NewValue(out, src.Locations()), nil - case dyn.KindString: - var err error - out, err := strconv.ParseInt(src.MustString(), 10, 64) - if err != nil { - // Return verbatim if it's a pure variable reference. - if dynvar.IsPureVariableReference(src.MustString()) { - return src, nil - } - - return dyn.InvalidValue, diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("cannot parse %q as an integer", src.MustString()), - Locations: []dyn.Location{src.Location()}, - Paths: dyn.ToStructPaths(path), - }) - } - return dyn.NewValue(out, src.Locations()), nil - case dyn.KindNil: - // Return a warning if the field is present but has a null value. - return dyn.InvalidValue, diags.Append(nullWarning(dyn.KindInt, src, path)) - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindInt, src, path)) -} - -func (n normalizeOptions) normalizeFloat(typ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) { - var diags diag.Diagnostics - - switch src.Kind() { - case dyn.KindFloat: - return dyn.NewValue(src.MustFloat(), src.Locations()), nil - case dyn.KindInt: - out := float64(src.MustInt()) - if src.MustInt() != int64(out) { - return dyn.InvalidValue, diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf(`cannot accurately represent "%d" as floating point number due to precision loss`, src.MustInt()), - Locations: []dyn.Location{src.Location()}, - Paths: dyn.ToStructPaths(path), - }) - } - return dyn.NewValue(out, src.Locations()), nil - case dyn.KindString: - var err error - out, err := strconv.ParseFloat(src.MustString(), 64) - if err != nil { - // Return verbatim if it's a pure variable reference. - if dynvar.IsPureVariableReference(src.MustString()) { - return src, nil - } - - return dyn.InvalidValue, diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("cannot parse %q as a floating point number", src.MustString()), - Locations: []dyn.Location{src.Location()}, - Paths: dyn.ToStructPaths(path), - }) - } - return dyn.NewValue(out, src.Locations()), nil - case dyn.KindNil: - // Return a warning if the field is present but has a null value. - return dyn.InvalidValue, diags.Append(nullWarning(dyn.KindFloat, src, path)) - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindFloat, src, path)) -} - -func (n normalizeOptions) normalizeInterface(_ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) { - // Deal with every [dyn.Kind] here to ensure completeness. - switch src.Kind() { - case dyn.KindMap: - // Fall through - case dyn.KindSequence: - // Fall through - case dyn.KindString: - // Fall through - case dyn.KindBool: - // Fall through - case dyn.KindInt: - // Fall through - case dyn.KindFloat: - // Fall through - case dyn.KindTime: - // Conversion of a time value to an interface{}. - // The [dyn.Value.AsAny] equivalent for this kind is the [time.Time] struct. - // If we convert to a typed representation and back again, we cannot distinguish - // a [time.Time] struct from any other struct. - // - // Therefore, we normalize the time value to a string. - return dyn.NewValue(src.MustTime().String(), src.Locations()), nil - case dyn.KindNil: - // Fall through - default: - return dyn.InvalidValue, diag.Errorf("unsupported kind: %s", src.Kind()) - } - - return src, nil -} diff --git a/libs/dyn/convert/normalize_test.go b/libs/dyn/convert/normalize_test.go deleted file mode 100644 index b2bed72db71..00000000000 --- a/libs/dyn/convert/normalize_test.go +++ /dev/null @@ -1,1018 +0,0 @@ -package convert - -import ( - "testing" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynassert" - "github.com/databricks/cli/libs/structs/structpath" - "github.com/stretchr/testify/assert" -) - -func TestNormalizeStruct(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - var typ Tmp - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - - vout, diags := Normalize(typ, vin) - assert.Empty(t, diags) - assert.Equal(t, vin, vout) -} - -func TestNormalizeStructElementDiagnostic(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - var typ Tmp - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V(map[string]dyn.Value{"an": dyn.V("error")}), - }) - - vout, diags := Normalize(typ, vin) - assert.Len(t, diags, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected string, found map`, - Locations: []dyn.Location{{}}, - Paths: structpath.NewPathSlice("bar"), - }, diags[0]) - - // Elements that encounter an error during normalization are dropped. - assert.Equal(t, map[string]any{ - "foo": "bar", - }, vout.AsAny()) -} - -func TestNormalizeStructUnknownField(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - var typ Tmp - - m := dyn.NewMapping() - m.SetLoc("foo", nil, dyn.V("val-foo")) - - // Set the unknown field, with location information. - m.SetLoc("bar", []dyn.Location{ - {File: "hello.yaml", Line: 1, Column: 1}, - {File: "world.yaml", Line: 2, Column: 2}, - }, dyn.V("var-bar")) - - vin := dyn.V(m) - - vout, diags := Normalize(typ, vin) - assert.Len(t, diags, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `unknown field: bar`, - // Assert location of the unknown field is included in the diagnostic. - Locations: []dyn.Location{ - {File: "hello.yaml", Line: 1, Column: 1}, - {File: "world.yaml", Line: 2, Column: 2}, - }, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, diags[0]) - - // The field that can be mapped to the struct field is retained. - assert.Equal(t, map[string]any{ - "foo": "val-foo", - }, vout.AsAny()) -} - -func TestNormalizeStructNil(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - var typ Tmp - vin := dyn.NilValue - vout, err := Normalize(typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeStructError(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - var typ Tmp - vin := dyn.V("string") - _, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found string`, - Locations: []dyn.Location{vin.Get("foo").Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeStructNestedError(t *testing.T) { - type Nested struct { - F1 int `json:"f1"` - F2 int `json:"f2"` - } - type Tmp struct { - Foo Nested `json:"foo"` - Bar Nested `json:"bar"` - } - - var typ Tmp - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(map[string]dyn.Value{ - "f1": dyn.V("error"), - "f2": dyn.V(1), - }), - "bar": dyn.V(map[string]dyn.Value{ - "f1": dyn.V(1), - "f2": dyn.V("error"), - }), - }) - vout, err := Normalize(typ, vin) - assert.Len(t, err, 2) - - // Verify that valid fields are retained. - dynassert.Equal(t, - dyn.V(map[string]dyn.Value{ - "foo": dyn.V(map[string]dyn.Value{ - "f2": dyn.V(int64(1)), - }), - "bar": dyn.V(map[string]dyn.Value{ - "f1": dyn.V(int64(1)), - }), - }), - vout, - ) -} - -func TestNormalizeStructIncludeMissingFields(t *testing.T) { - type Nested struct { - String string `json:"string"` - } - - type Tmp struct { - // Verify that fields that are already set in the dynamic value are not overridden. - Existing string `json:"existing"` - - // Verify that structs are recursively normalized if not set. - Nested Nested `json:"nested"` - Ptr *Nested `json:"ptr"` - - // Verify that containers are also zero-initialized if not set. - Map map[string]string `json:"map"` - Slice []string `json:"slice"` - - // Verify that primitive types are zero-initialized if not set. - String string `json:"string"` - Bool bool `json:"bool"` - Int int `json:"int"` - Float float64 `json:"float"` - } - - var typ Tmp - vin := dyn.V(map[string]dyn.Value{ - "existing": dyn.V("already set"), - }) - vout, err := Normalize(typ, vin, IncludeMissingFields) - assert.Empty(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "existing": dyn.V("already set"), - "nested": dyn.V(map[string]dyn.Value{ - "string": dyn.V(""), - }), - "ptr": dyn.V(map[string]dyn.Value{ - "string": dyn.V(""), - }), - "map": dyn.V(map[string]dyn.Value{}), - "slice": dyn.V([]dyn.Value{}), - "string": dyn.V(""), - "bool": dyn.V(false), - "int": dyn.V(int64(0)), - "float": dyn.V(float64(0)), - }), vout) -} - -func TestNormalizeStructIncludeMissingFieldsOnRecursiveType(t *testing.T) { - type Tmp struct { - // Verify that structs are recursively normalized if not set. - Ptr *Tmp `json:"ptr"` - - // Verify that primitive types are zero-initialized if not set. - String string `json:"string"` - } - - var typ Tmp - vin := dyn.V(map[string]dyn.Value{ - "ptr": dyn.V(map[string]dyn.Value{ - "ptr": dyn.V(map[string]dyn.Value{ - "string": dyn.V("already set"), - }), - }), - }) - vout, err := Normalize(typ, vin, IncludeMissingFields) - assert.Empty(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "ptr": dyn.V(map[string]dyn.Value{ - "ptr": dyn.V(map[string]dyn.Value{ - // Note: the ptr field is not zero-initialized because that would recurse. - "string": dyn.V("already set"), - }), - "string": dyn.V(""), - }), - "string": dyn.V(""), - }), vout) -} - -func TestNormalizeStructVariableReference(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - var typ Tmp - vin := dyn.NewValue("${var.foo}", []dyn.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeStructRandomStringError(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - var typ Tmp - vin := dyn.NewValue("var foo", []dyn.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found string`, - Locations: []dyn.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeStructIntError(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - var typ Tmp - vin := dyn.NewValue(1, []dyn.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found int`, - Locations: []dyn.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeMap(t *testing.T) { - var typ map[string]string - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - - vout, err := Normalize(typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeMapElementDiagnostic(t *testing.T) { - var typ map[string]string - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V(map[string]dyn.Value{"an": dyn.V("error")}), - }) - - vout, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected string, found map`, - Locations: []dyn.Location{{}}, - Paths: structpath.NewPathSlice("bar"), - }, err[0]) - - // Elements that encounter an error during normalization are dropped. - assert.Equal(t, map[string]any{ - "foo": "bar", - }, vout.AsAny()) -} - -func TestNormalizeMapNil(t *testing.T) { - var typ map[string]string - vin := dyn.NilValue - vout, err := Normalize(typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeMapError(t *testing.T) { - var typ map[string]string - vin := dyn.V("string") - _, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found string`, - Locations: []dyn.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeMapNestedError(t *testing.T) { - type Nested struct { - F1 int `json:"f1"` - F2 int `json:"f2"` - } - - var typ map[string]Nested - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(map[string]dyn.Value{ - "f1": dyn.V("error"), - "f2": dyn.V(1), - }), - "bar": dyn.V(map[string]dyn.Value{ - "f1": dyn.V(1), - "f2": dyn.V("error"), - }), - }) - vout, err := Normalize(typ, vin) - assert.Len(t, err, 2) - - // Verify that valid fields are retained. - dynassert.Equal(t, - dyn.V(map[string]dyn.Value{ - "foo": dyn.V(map[string]dyn.Value{ - "f2": dyn.V(int64(1)), - }), - "bar": dyn.V(map[string]dyn.Value{ - "f1": dyn.V(int64(1)), - }), - }), - vout, - ) -} - -func TestNormalizeMapVariableReference(t *testing.T) { - var typ map[string]string - vin := dyn.NewValue("${var.foo}", []dyn.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeMapRandomStringError(t *testing.T) { - var typ map[string]string - vin := dyn.NewValue("var foo", []dyn.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found string`, - Locations: []dyn.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeMapIntError(t *testing.T) { - var typ map[string]string - vin := dyn.NewValue(1, []dyn.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found int`, - Locations: []dyn.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeSlice(t *testing.T) { - var typ []string - vin := dyn.V([]dyn.Value{ - dyn.V("foo"), - dyn.V("bar"), - }) - - vout, err := Normalize(typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeSliceElementDiagnostic(t *testing.T) { - var typ []string - vin := dyn.V([]dyn.Value{ - dyn.V("foo"), - dyn.V("bar"), - dyn.V(map[string]dyn.Value{"an": dyn.V("error")}), - }) - - vout, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected string, found map`, - Locations: []dyn.Location{{}}, - Paths: []*structpath.PathNode{structpath.NewIndex(nil, 2)}, - }, err[0]) - - // Elements that encounter an error during normalization are dropped. - assert.Equal(t, []any{"foo", "bar"}, vout.AsAny()) -} - -func TestNormalizeSliceNil(t *testing.T) { - var typ []string - vin := dyn.NilValue - vout, err := Normalize(typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeSliceError(t *testing.T) { - var typ []string - vin := dyn.V("string") - _, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected sequence, found string`, - Locations: []dyn.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeSliceNestedError(t *testing.T) { - type Nested struct { - F1 int `json:"f1"` - F2 int `json:"f2"` - } - - var typ []Nested - vin := dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "f1": dyn.V("error"), - "f2": dyn.V(1), - }), - dyn.V(map[string]dyn.Value{ - "f1": dyn.V(1), - "f2": dyn.V("error"), - }), - }) - vout, err := Normalize(typ, vin) - assert.Len(t, err, 2) - - // Verify that valid fields are retained. - dynassert.Equal(t, - dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "f2": dyn.V(int64(1)), - }), - dyn.V(map[string]dyn.Value{ - "f1": dyn.V(int64(1)), - }), - }), - vout, - ) -} - -func TestNormalizeSliceVariableReference(t *testing.T) { - var typ []string - vin := dyn.NewValue("${var.foo}", []dyn.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeSliceRandomStringError(t *testing.T) { - var typ []string - vin := dyn.NewValue("var foo", []dyn.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected sequence, found string`, - Locations: []dyn.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeSliceIntError(t *testing.T) { - var typ []string - vin := dyn.NewValue(1, []dyn.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected sequence, found int`, - Locations: []dyn.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeString(t *testing.T) { - var typ string - vin := dyn.V("string") - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeStringNil(t *testing.T) { - var typ string - vin := dyn.NewValue(nil, []dyn.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected a string value, found null`, - Locations: []dyn.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeStringFromBool(t *testing.T) { - var typ string - vin := dyn.NewValue(true, []dyn.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.NewValue("true", vin.Locations()), vout) -} - -func TestNormalizeStringFromInt(t *testing.T) { - var typ string - vin := dyn.NewValue(123, []dyn.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.NewValue("123", vin.Locations()), vout) -} - -func TestNormalizeStringFromFloat(t *testing.T) { - var typ string - vin := dyn.NewValue(1.20, []dyn.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.NewValue("1.2", vin.Locations()), vout) -} - -func TestNormalizeStringFromTime(t *testing.T) { - var typ string - vin := dyn.NewValue(dyn.MustTime("2024-08-29"), []dyn.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.NewValue("2024-08-29", vin.Locations()), vout) -} - -func TestNormalizeStringError(t *testing.T) { - var typ string - vin := dyn.V(map[string]dyn.Value{"an": dyn.V("error")}) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected string, found map`, - Locations: []dyn.Location{{}}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeBool(t *testing.T) { - var typ bool - vin := dyn.V(true) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.V(true), vout) -} - -func TestNormalizeBoolNil(t *testing.T) { - var typ bool - vin := dyn.NewValue(nil, []dyn.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected a bool value, found null`, - Locations: []dyn.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeBoolFromString(t *testing.T) { - var typ bool - - for _, c := range []struct { - Input string - Output bool - }{ - {"true", true}, - {"false", false}, - {"True", true}, - {"False", false}, - {"TRUE", true}, - {"FALSE", false}, - {"Y", true}, - {"N", false}, - {"on", true}, - {"off", false}, - } { - vin := dyn.V(c.Input) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.V(c.Output), vout) - } -} - -func TestNormalizeBoolFromStringVariableReference(t *testing.T) { - var typ bool - vin := dyn.V("${var.foo}") - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeBoolFromStringError(t *testing.T) { - var typ bool - vin := dyn.V("abc") - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected bool, found string`, - Locations: []dyn.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeBoolError(t *testing.T) { - var typ bool - vin := dyn.V(map[string]dyn.Value{"an": dyn.V("error")}) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected bool, found map`, - Locations: []dyn.Location{{}}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeInt(t *testing.T) { - var typ int - vin := dyn.V(123) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.V(int64(123)), vout) -} - -func TestNormalizeIntNil(t *testing.T) { - var typ int - vin := dyn.NewValue(nil, []dyn.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected a int value, found null`, - Locations: []dyn.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeIntFromFloat(t *testing.T) { - var typ int - vin := dyn.V(float64(1.0)) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.V(int64(1)), vout) -} - -func TestNormalizeIntFromFloatError(t *testing.T) { - var typ int - vin := dyn.V(1.5) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `cannot accurately represent "1.5" as integer due to precision loss`, - Locations: []dyn.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeIntFromString(t *testing.T) { - var typ int - vin := dyn.V("123") - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.V(int64(123)), vout) -} - -func TestNormalizeIntFromStringVariableReference(t *testing.T) { - var typ int - vin := dyn.V("${var.foo}") - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeIntFromStringError(t *testing.T) { - var typ int - vin := dyn.V("abc") - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `cannot parse "abc" as an integer`, - Locations: []dyn.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeIntError(t *testing.T) { - var typ int - vin := dyn.V(map[string]dyn.Value{"an": dyn.V("error")}) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected int, found map`, - Locations: []dyn.Location{{}}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeFloat(t *testing.T) { - var typ float64 - vin := dyn.V(1.2) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.V(1.2), vout) -} - -func TestNormalizeFloatNil(t *testing.T) { - var typ float64 - vin := dyn.NewValue(nil, []dyn.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected a float value, found null`, - Locations: []dyn.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeFloatFromInt(t *testing.T) { - var typ float64 - - // Maximum safe integer that can be accurately represented as a float. - vin := dyn.V(int64(9007199254740992)) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.V(float64(9007199254740992)), vout) -} - -func TestNormalizeFloatFromIntError(t *testing.T) { - var typ float64 - - // Minimum integer that cannot be accurately represented as a float. - vin := dyn.V(9007199254740992 + 1) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `cannot accurately represent "9007199254740993" as floating point number due to precision loss`, - Locations: []dyn.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeFloatFromString(t *testing.T) { - var typ float64 - vin := dyn.V("1.2") - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.V(1.2), vout) -} - -func TestNormalizeFloatFromStringVariableReference(t *testing.T) { - var typ float64 - vin := dyn.V("${var.foo}") - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeFloatFromStringError(t *testing.T) { - var typ float64 - vin := dyn.V("abc") - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `cannot parse "abc" as a floating point number`, - Locations: []dyn.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeFloatError(t *testing.T) { - var typ float64 - vin := dyn.V(map[string]dyn.Value{"an": dyn.V("error")}) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected float, found map`, - Locations: []dyn.Location{{}}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeAnchors(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - var typ Tmp - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "anchor": dyn.V("anchor").MarkAnchor(), - }) - - vout, err := Normalize(typ, vin) - assert.Empty(t, err) - - // The field that can be mapped to the struct field is retained. - assert.Equal(t, map[string]any{ - "foo": "bar", - }, vout.AsAny()) -} - -func TestNormalizeAnchorContainers(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - anchor := func() dyn.Value { - return dyn.V(map[string]dyn.Value{"name": dyn.V("x")}).MarkAnchor() - } - - tcases := []struct { - name string - value dyn.Value - wantWarn bool - }{ - { - name: "list of anchors", - value: dyn.V([]dyn.Value{anchor(), anchor()}), - wantWarn: false, - }, - { - name: "map of anchors", - value: dyn.V(map[string]dyn.Value{ - "a": anchor(), - "b": anchor(), - }), - wantWarn: false, - }, - { - name: "nested list of anchors", - value: dyn.V([]dyn.Value{dyn.V([]dyn.Value{anchor()})}), - wantWarn: false, - }, - { - name: "list with a non-anchor element", - value: dyn.V([]dyn.Value{anchor(), dyn.V(map[string]dyn.Value{"name": dyn.V("x")})}), - wantWarn: true, - }, - { - name: "empty list", - value: dyn.V([]dyn.Value{}), - wantWarn: true, - }, - { - name: "empty map", - value: dyn.V(map[string]dyn.Value{}), - wantWarn: true, - }, - } - - for _, tc := range tcases { - t.Run(tc.name, func(t *testing.T) { - var typ Tmp - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "thing": tc.value, - }) - - vout, diags := Normalize(typ, vin) - if tc.wantWarn { - assert.Len(t, diags, 1) - assert.Equal(t, "unknown field: thing", diags[0].Summary) - } else { - assert.Empty(t, diags) - } - - // The unknown field is never retained regardless of the warning. - assert.Equal(t, map[string]any{ - "foo": "bar", - }, vout.AsAny()) - }) - } -} - -func TestNormalizeAnyFromSlice(t *testing.T) { - var typ any - v1 := dyn.NewValue(1, []dyn.Location{{File: "file", Line: 1, Column: 1}}) - v2 := dyn.NewValue(2, []dyn.Location{{File: "file", Line: 1, Column: 1}}) - vin := dyn.NewValue([]dyn.Value{v1, v2}, []dyn.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.NewValue([]dyn.Value{v1, v2}, []dyn.Location{{File: "file", Line: 1, Column: 1}}), vout) -} - -func TestNormalizeAnyFromString(t *testing.T) { - var typ any - vin := dyn.NewValue("string", []dyn.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.NewValue("string", []dyn.Location{{File: "file", Line: 1, Column: 1}}), vout) -} - -func TestNormalizeAnyFromBool(t *testing.T) { - var typ any - vin := dyn.NewValue(false, []dyn.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.NewValue(false, []dyn.Location{{File: "file", Line: 1, Column: 1}}), vout) -} - -func TestNormalizeAnyFromInt(t *testing.T) { - var typ any - vin := dyn.NewValue(10, []dyn.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.NewValue(10, []dyn.Location{{File: "file", Line: 1, Column: 1}}), vout) -} - -func TestNormalizeAnyFromTime(t *testing.T) { - var typ any - vin := dyn.NewValue(dyn.MustTime("2024-08-29"), []dyn.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.NewValue("2024-08-29", vin.Locations()), vout) -} - -func TestNormalizeStructDropEmptyStrings(t *testing.T) { - type Tmp struct { - // Optional: dropped when empty. - Opt string `json:"opt,omitempty"` - // Required (no omitempty): kept even when empty. - Req string `json:"req"` - } - - vin := dyn.V(map[string]dyn.Value{ - "opt": dyn.V(""), - "req": dyn.V(""), - }) - - vout, diags := Normalize(Tmp{}, vin, DropEmptyStrings) - assert.Empty(t, diags) - - _, hasOpt := vout.MustMap().Get(dyn.V("opt")) - assert.False(t, hasOpt, "empty omitempty string should be dropped") - req, hasReq := vout.MustMap().Get(dyn.V("req")) - assert.True(t, hasReq, "empty non-omitempty string should be kept") - assert.Empty(t, req.MustString()) -} - -func TestNormalizeStructDropEmptyStringsKeepsNonEmpty(t *testing.T) { - type Tmp struct { - Opt string `json:"opt,omitempty"` - } - - vin := dyn.V(map[string]dyn.Value{"opt": dyn.V("value")}) - - vout, diags := Normalize(Tmp{}, vin, DropEmptyStrings) - assert.Empty(t, diags) - opt, ok := vout.MustMap().Get(dyn.V("opt")) - assert.True(t, ok) - assert.Equal(t, "value", opt.MustString()) -} diff --git a/libs/dyn/convert/sdk_native_types.go b/libs/dyn/convert/sdk_native_types.go deleted file mode 100644 index b67d9b2b1e3..00000000000 --- a/libs/dyn/convert/sdk_native_types.go +++ /dev/null @@ -1,97 +0,0 @@ -package convert - -import ( - "encoding/json" - "fmt" - "reflect" - "slices" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" - sdkduration "github.com/databricks/databricks-sdk-go/common/types/duration" - sdkfieldmask "github.com/databricks/databricks-sdk-go/common/types/fieldmask" - sdktime "github.com/databricks/databricks-sdk-go/common/types/time" -) - -// sdkNativeTypes is a list of SDK native types that use custom JSON marshaling -// and should be treated as strings in dyn.Value. These types all implement -// json.Marshaler and json.Unmarshaler interfaces. -var sdkNativeTypes = []reflect.Type{ - reflect.TypeFor[sdkduration.Duration](), // Protobuf duration format (e.g., "300s") - reflect.TypeFor[sdktime.Time](), // RFC3339 timestamp format (e.g., "2023-12-25T10:30:00Z") - reflect.TypeFor[sdkfieldmask.FieldMask](), // Comma-separated paths (e.g., "name,age,email") -} - -// fromTypedSDKNative converts SDK native types to dyn.Value. -// SDK native types (duration.Duration, time.Time, fieldmask.FieldMask) use -// custom JSON marshaling with string representations. -func fromTypedSDKNative(src reflect.Value, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, error) { - // Check that the reference value is compatible or nil. - switch ref.Kind() { - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(ref.MustString()) { - return ref, nil - } - case dyn.KindNil: - // Allow nil reference. - default: - return dyn.InvalidValue, fmt.Errorf("cannot convert SDK native type to dynamic type %#v", ref.Kind().String()) - } - - // Check for zero value first. - if src.IsZero() && !slices.Contains(options, includeZeroValues) { - return dyn.NilValue, nil - } - - // Use JSON marshaling since SDK native types implement json.Marshaler. - jsonBytes, err := json.Marshal(src.Interface()) - if err != nil { - return dyn.InvalidValue, err - } - - // All SDK native types marshal to JSON strings. Unmarshal to get the raw string value. - // For example: duration.Duration(300s) -> JSON "300s" -> string "300s" - var str string - if err := json.Unmarshal(jsonBytes, &str); err != nil { - return dyn.InvalidValue, err - } - - // Handle empty string as zero value. - if str == "" && !slices.Contains(options, includeZeroValues) { - return dyn.NilValue, nil - } - - return dyn.V(str), nil -} - -// toTypedSDKNative converts a dyn.Value to an SDK native type. -// SDK native types (duration.Duration, time.Time, fieldmask.FieldMask) use -// custom JSON marshaling with string representations. -func toTypedSDKNative(dst reflect.Value, src dyn.Value) error { - switch src.Kind() { - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(src.MustString()) { - dst.SetZero() - return nil - } - // Use JSON unmarshaling since SDK native types implement json.Unmarshaler. - // Marshal the string to create a valid JSON string literal for unmarshaling. - jsonBytes, err := json.Marshal(src.MustString()) - if err != nil { - return err - } - return json.Unmarshal(jsonBytes, dst.Addr().Interface()) - case dyn.KindNil: - dst.SetZero() - return nil - default: - // Fall through to the error case. - } - - return TypeError{ - value: src, - msg: fmt.Sprintf("expected a string, found a %s", src.Kind()), - } -} diff --git a/libs/dyn/convert/sdk_native_types_test.go b/libs/dyn/convert/sdk_native_types_test.go deleted file mode 100644 index cd3d14fa129..00000000000 --- a/libs/dyn/convert/sdk_native_types_test.go +++ /dev/null @@ -1,417 +0,0 @@ -package convert - -import ( - "reflect" - "testing" - "time" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - sdkduration "github.com/databricks/databricks-sdk-go/common/types/duration" - sdkfieldmask "github.com/databricks/databricks-sdk-go/common/types/fieldmask" - sdktime "github.com/databricks/databricks-sdk-go/common/types/time" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// Roundtrip tests - verify SDK native types convert to dyn.Value and back for both value and pointer types - -func TestDurationRoundtrip(t *testing.T) { - tests := []struct { - name string - duration time.Duration - expectedString string - }{ - {"5min", 5 * time.Minute, "300s"}, - {"7days", 7 * 24 * time.Hour, "604800s"}, - {"1hour", 1 * time.Hour, "3600s"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Test value type - t.Run("value", func(t *testing.T) { - src := *sdkduration.New(tt.duration) - dynValue, err := FromTyped(src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, tt.expectedString, dynValue.MustString()) - - var out sdkduration.Duration - err = ToTyped(&out, dynValue) - require.NoError(t, err) - assert.Equal(t, src.AsDuration(), out.AsDuration()) - }) - - // Test pointer type - t.Run("pointer", func(t *testing.T) { - src := *sdkduration.New(tt.duration) - dynValue, err := FromTyped(&src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, tt.expectedString, dynValue.MustString()) - - var out *sdkduration.Duration - err = ToTyped(&out, dynValue) - require.NoError(t, err) - require.NotNil(t, out) - assert.Equal(t, src.AsDuration(), out.AsDuration()) - }) - }) - } -} - -func TestTimeRoundtrip(t *testing.T) { - tests := []struct { - name string - time time.Time - expectedString string - }{ - { - "no_nanos", - time.Date(2023, 12, 25, 10, 30, 0, 0, time.UTC), - "2023-12-25T10:30:00Z", - }, - { - "with_nanos", - time.Date(2023, 12, 25, 10, 30, 0, 123456789, time.UTC), - "2023-12-25T10:30:00.123456789Z", - }, - { - "epoch", - time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC), - "1970-01-01T00:00:00Z", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Test value type - t.Run("value", func(t *testing.T) { - src := *sdktime.New(tt.time) - dynValue, err := FromTyped(src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, tt.expectedString, dynValue.MustString()) - - var out sdktime.Time - err = ToTyped(&out, dynValue) - require.NoError(t, err) - assert.Equal(t, src.AsTime(), out.AsTime()) - }) - - // Test pointer type - t.Run("pointer", func(t *testing.T) { - src := *sdktime.New(tt.time) - dynValue, err := FromTyped(&src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, tt.expectedString, dynValue.MustString()) - - var out *sdktime.Time - err = ToTyped(&out, dynValue) - require.NoError(t, err) - require.NotNil(t, out) - assert.Equal(t, src.AsTime(), out.AsTime()) - }) - }) - } -} - -func TestFieldMaskRoundtrip(t *testing.T) { - tests := []struct { - name string - paths []string - expectedString string - }{ - {"single", []string{"name"}, "name"}, - {"multiple", []string{"name", "age", "email"}, "name,age,email"}, - {"nested", []string{"user.name", "user.email"}, "user.name,user.email"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Test value type - t.Run("value", func(t *testing.T) { - src := *sdkfieldmask.New(tt.paths) - dynValue, err := FromTyped(src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, tt.expectedString, dynValue.MustString()) - - var out sdkfieldmask.FieldMask - err = ToTyped(&out, dynValue) - require.NoError(t, err) - assert.Equal(t, src.Paths, out.Paths) - }) - - // Test pointer type - t.Run("pointer", func(t *testing.T) { - src := *sdkfieldmask.New(tt.paths) - dynValue, err := FromTyped(&src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, tt.expectedString, dynValue.MustString()) - - var out *sdkfieldmask.FieldMask - err = ToTyped(&out, dynValue) - require.NoError(t, err) - require.NotNil(t, out) - assert.Equal(t, src.Paths, out.Paths) - }) - }) - } -} - -// Edge case tests - -func TestNilValues(t *testing.T) { - tests := []struct { - name string - typ any - }{ - {"duration", (*sdkduration.Duration)(nil)}, - {"time", (*sdktime.Time)(nil)}, - {"fieldmask", (*sdkfieldmask.FieldMask)(nil)}, - } - - t.Run("from_typed", func(t *testing.T) { - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - nv, err := FromTyped(tt.typ, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) - }) - } - }) - - t.Run("normalize", func(t *testing.T) { - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - vout, diags := Normalize(tt.typ, dyn.NilValue) - assert.Len(t, diags, 1) - assert.Equal(t, diag.Warning, diags[0].Severity) - assert.Equal(t, `expected a string value, found null`, diags[0].Summary) - assert.Equal(t, dyn.InvalidValue, vout) - }) - } - }) -} - -func TestToTypedErrors(t *testing.T) { - wrongTypeInput := dyn.V(map[string]dyn.Value{"foo": dyn.V("bar")}) - - tests := []struct { - name string - destValue any - input dyn.Value - expectedError string - }{ - { - name: "duration_wrong_type", - destValue: new(sdkduration.Duration), - input: wrongTypeInput, - expectedError: "expected a string", - }, - { - name: "duration_invalid_format", - destValue: new(sdkduration.Duration), - input: dyn.V("7d"), - expectedError: "invalid google.protobuf.Duration value", - }, - { - name: "time_wrong_type", - destValue: new(sdktime.Time), - input: wrongTypeInput, - expectedError: "expected a string", - }, - { - name: "time_invalid_format", - destValue: new(sdktime.Time), - input: dyn.V("not-a-time"), - expectedError: "invalid google.protobuf.Timestamp value", - }, - { - name: "fieldmask_wrong_type", - destValue: new(sdkfieldmask.FieldMask), - input: wrongTypeInput, - expectedError: "expected a string", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := ToTyped(tt.destValue, tt.input) - require.Error(t, err) - assert.Contains(t, err.Error(), tt.expectedError) - }) - } -} - -func TestFromTypedVariableReferences(t *testing.T) { - tests := []struct { - name string - src any - }{ - { - name: "duration", - src: *sdkduration.New(5 * time.Minute), - }, - { - name: "time", - src: *sdktime.New(time.Date(2023, 12, 25, 10, 30, 0, 0, time.UTC)), - }, - { - name: "fieldmask", - src: *sdkfieldmask.New([]string{"name", "age"}), - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ref := dyn.V("${var.foo}") - nv, err := FromTyped(tt.src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V("${var.foo}"), nv) - }) - } -} - -func TestToTypedVariableReferences(t *testing.T) { - tests := []struct { - name string - dst any - zero any - }{ - { - name: "duration", - dst: sdkduration.New(5 * time.Minute), - zero: sdkduration.Duration{}, - }, - { - name: "time", - dst: sdktime.New(time.Date(2023, 12, 25, 10, 30, 0, 0, time.UTC)), - zero: sdktime.Time{}, - }, - { - name: "fieldmask", - dst: sdkfieldmask.New([]string{"name", "age"}), - zero: sdkfieldmask.FieldMask{}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := ToTyped(tt.dst, dyn.V("${var.foo}")) - require.NoError(t, err) - assert.Equal(t, tt.zero, reflect.ValueOf(tt.dst).Elem().Interface()) - }) - } -} - -func TestSpecialCases(t *testing.T) { - t.Run("duration_zero", func(t *testing.T) { - var src sdkduration.Duration - nv, err := FromTyped(src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) - }) - - t.Run("fieldmask_empty_fromtyped", func(t *testing.T) { - src := sdkfieldmask.New([]string{}) - nv, err := FromTyped(src, dyn.NilValue) - require.NoError(t, err) - // Empty field mask marshals to empty string - assert.Equal(t, dyn.V(""), nv) - }) - - t.Run("fieldmask_empty_totyped", func(t *testing.T) { - var out sdkfieldmask.FieldMask - err := ToTyped(&out, dyn.V("")) - require.NoError(t, err) - assert.Empty(t, out.Paths) - }) -} - -// End-to-end tests with structs containing SDK native types - -func TestSDKTypesRoundTripWithPostgresBranchSpec(t *testing.T) { - type BranchSpec struct { - ExpireTime *sdktime.Time `json:"expire_time,omitempty"` - SourceBranchTime *sdktime.Time `json:"source_branch_time,omitempty"` - Ttl *sdkduration.Duration `json:"ttl,omitempty"` - IsProtected bool `json:"is_protected,omitempty"` - } - - original := BranchSpec{ - ExpireTime: sdktime.New(time.Date(2024, 12, 31, 23, 59, 59, 0, time.UTC)), - SourceBranchTime: sdktime.New(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)), - Ttl: sdkduration.New(7 * 24 * time.Hour), - IsProtected: true, - } - - dynValue, err := FromTyped(original, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, "2024-12-31T23:59:59Z", dynValue.Get("expire_time").MustString()) - assert.Equal(t, "2024-01-01T00:00:00Z", dynValue.Get("source_branch_time").MustString()) - assert.Equal(t, "604800s", dynValue.Get("ttl").MustString()) - assert.True(t, dynValue.Get("is_protected").MustBool()) - - var roundtrip BranchSpec - err = ToTyped(&roundtrip, dynValue) - require.NoError(t, err) - require.NotNil(t, roundtrip.ExpireTime) - require.NotNil(t, roundtrip.SourceBranchTime) - require.NotNil(t, roundtrip.Ttl) - assert.Equal(t, original.ExpireTime.AsTime(), roundtrip.ExpireTime.AsTime()) - assert.Equal(t, original.SourceBranchTime.AsTime(), roundtrip.SourceBranchTime.AsTime()) - assert.Equal(t, original.Ttl.AsDuration(), roundtrip.Ttl.AsDuration()) - assert.Equal(t, original.IsProtected, roundtrip.IsProtected) -} - -func TestSDKTypesRoundTripWithUpdateRequest(t *testing.T) { - type UpdateRequest struct { - Name string `json:"name"` - UpdateMask sdkfieldmask.FieldMask `json:"update_mask"` - } - - original := UpdateRequest{ - Name: "projects/123/branches/456", - UpdateMask: *sdkfieldmask.New([]string{"spec.ttl", "spec.is_protected"}), - } - - dynValue, err := FromTyped(original, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, "projects/123/branches/456", dynValue.Get("name").MustString()) - assert.Equal(t, "spec.ttl,spec.is_protected", dynValue.Get("update_mask").MustString()) - - var roundtrip UpdateRequest - err = ToTyped(&roundtrip, dynValue) - require.NoError(t, err) - assert.Equal(t, original.Name, roundtrip.Name) - assert.Equal(t, []string{"spec.ttl", "spec.is_protected"}, roundtrip.UpdateMask.Paths) -} - -func TestSDKTypesNormalizeWithStruct(t *testing.T) { - type BranchSpec struct { - ExpireTime *sdktime.Time `json:"expire_time,omitempty"` - Ttl *sdkduration.Duration `json:"ttl,omitempty"` - IsProtected bool `json:"is_protected,omitempty"` - } - - var typ BranchSpec - vin := dyn.V(map[string]dyn.Value{ - "expire_time": dyn.V("2024-12-31T23:59:59Z"), - "ttl": dyn.V("604800s"), - "is_protected": dyn.V(true), - }) - - vout, diags := Normalize(typ, vin) - assert.Empty(t, diags) - assert.Equal(t, "2024-12-31T23:59:59Z", vout.Get("expire_time").MustString()) - assert.Equal(t, "604800s", vout.Get("ttl").MustString()) - assert.True(t, vout.Get("is_protected").MustBool()) - - var out BranchSpec - err := ToTyped(&out, vout) - require.NoError(t, err) - require.NotNil(t, out.ExpireTime) - require.NotNil(t, out.Ttl) - assert.Equal(t, time.Date(2024, 12, 31, 23, 59, 59, 0, time.UTC), out.ExpireTime.AsTime()) - assert.Equal(t, 7*24*time.Hour, out.Ttl.AsDuration()) - assert.True(t, out.IsProtected) -} diff --git a/libs/dyn/convert/struct_info_test.go b/libs/dyn/convert/struct_info_test.go deleted file mode 100644 index f921523c391..00000000000 --- a/libs/dyn/convert/struct_info_test.go +++ /dev/null @@ -1,228 +0,0 @@ -package convert - -import ( - "reflect" - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestStructInfoPlain(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar,omitempty"` - - // Baz must be skipped. - Baz string `json:""` - - // Qux must be skipped. - Qux string `json:"-"` - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - assert.Len(t, si.Fields, 2) - assert.Equal(t, []int{0}, si.Fields["foo"]) - assert.Equal(t, []int{1}, si.Fields["bar"]) -} - -func TestStructInfoAnonymousByValue(t *testing.T) { - type Bar struct { - Bar string `json:"bar"` - } - - type Foo struct { - Foo string `json:"foo"` - Bar - } - - type Tmp struct { - Foo - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - assert.Len(t, si.Fields, 2) - assert.Equal(t, []int{0, 0}, si.Fields["foo"]) - assert.Equal(t, []int{0, 1, 0}, si.Fields["bar"]) -} - -func TestStructInfoAnonymousByValuePrecedence(t *testing.T) { - type Bar struct { - Bar string `json:"bar"` - } - - type Foo struct { - Foo string `json:"foo"` - Bar - } - - type Tmp struct { - // "foo" comes from [Foo]. - Foo - // "bar" comes from [Bar] directly, not through [Foo]. - Bar - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - assert.Len(t, si.Fields, 2) - assert.Equal(t, []int{0, 0}, si.Fields["foo"]) - assert.Equal(t, []int{1, 0}, si.Fields["bar"]) -} - -func TestStructInfoAnonymousByPointer(t *testing.T) { - type Bar struct { - Bar string `json:"bar"` - } - - type Foo struct { - Foo string `json:"foo"` - *Bar - } - - type Tmp struct { - *Foo - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - assert.Len(t, si.Fields, 2) - assert.Equal(t, []int{0, 0}, si.Fields["foo"]) - assert.Equal(t, []int{0, 1, 0}, si.Fields["bar"]) -} - -func TestStructInfoFieldValues(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - src := Tmp{ - Foo: "foo", - Bar: "bar", - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - fv := si.FieldValues(reflect.ValueOf(src)) - assert.Len(t, fv, 2) - assert.Equal(t, "foo", fv[0].Key) - assert.True(t, reflect.ValueOf("foo").Equal(fv[0].Value)) - assert.Equal(t, "bar", fv[1].Key) - assert.True(t, reflect.ValueOf("bar").Equal(fv[1].Value)) -} - -func TestStructInfoFieldValuesAnonymousByValue(t *testing.T) { - type Bar struct { - Bar string `json:"bar"` - } - - type Foo struct { - Foo string `json:"foo"` - Bar - } - - type Tmp struct { - Foo - } - - src := Tmp{ - Foo: Foo{ - Foo: "foo", - Bar: Bar{ - Bar: "bar", - }, - }, - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - fv := si.FieldValues(reflect.ValueOf(src)) - assert.Len(t, fv, 2) - assert.Equal(t, "foo", fv[0].Key) - assert.Equal(t, "bar", fv[1].Key) -} - -func TestStructInfoFieldValuesAnonymousByPointer(t *testing.T) { - type Bar struct { - Bar string `json:"bar"` - } - - type Foo struct { - Foo string `json:"foo"` - *Bar - } - - type Tmp struct { - *Foo - } - - // Test that the embedded fields are dereferenced properly. - t.Run("all are set", func(t *testing.T) { - src := Tmp{ - Foo: &Foo{ - Foo: "foo", - Bar: &Bar{ - Bar: "bar", - }, - }, - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - fv := si.FieldValues(reflect.ValueOf(src)) - assert.Len(t, fv, 2) - assert.Equal(t, "foo", fv[0].Key) - assert.Equal(t, "bar", fv[1].Key) - }) - - // Test that fields of embedded types are skipped if the embedded type is nil. - t.Run("top level is set", func(t *testing.T) { - src := Tmp{ - Foo: &Foo{ - Foo: "foo", - Bar: nil, - }, - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - fv := si.FieldValues(reflect.ValueOf(src)) - assert.Len(t, fv, 1) - assert.Equal(t, "foo", fv[0].Key) - }) - - // Test that fields of embedded types are skipped if the embedded type is nil. - t.Run("none are set", func(t *testing.T) { - src := Tmp{ - Foo: nil, - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - fv := si.FieldValues(reflect.ValueOf(src)) - assert.Empty(t, fv) - }) -} - -func TestStructInfoValueFieldAbsent(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - assert.Nil(t, si.ValueField) -} - -func TestStructInfoValueFieldPresent(t *testing.T) { - type Tmp struct { - Foo dyn.Value - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - assert.NotNil(t, si.ValueField) -} - -func TestStructInfoValueFieldMultiple(t *testing.T) { - type Tmp struct { - Foo dyn.Value - Bar dyn.Value - } - - assert.Panics(t, func() { - getStructInfo(reflect.TypeFor[Tmp]()) - }) -} diff --git a/libs/dyn/convert/to_typed.go b/libs/dyn/convert/to_typed.go deleted file mode 100644 index 87b0fe7d8ca..00000000000 --- a/libs/dyn/convert/to_typed.go +++ /dev/null @@ -1,350 +0,0 @@ -package convert - -import ( - "fmt" - "reflect" - "slices" - "strconv" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" -) - -// Populate a destination typed value from a source dynamic value. -// -// At any point while walking the destination type tree using -// reflection, if this function sees an exported field with type dyn.Value it -// will populate that field with the appropriate source dynamic value. -// see PR: https://github.com/databricks/cli/pull/1010 -func ToTyped(dst any, src dyn.Value) error { - dstv := reflect.ValueOf(dst) - - // Dereference pointer if necessary - for dstv.Kind() == reflect.Pointer { - // If the source value is nil and the destination is a settable pointer, - // set the destination to nil. Also see `end_to_end_test.go`. - if dstv.CanSet() && src.Kind() == dyn.KindNil { - dstv.SetZero() - return nil - } - if dstv.IsNil() { - dstv.Set(reflect.New(dstv.Type().Elem())) - } - dstv = dstv.Elem() - } - - // Verify that vv is settable. - if !dstv.CanSet() { - panic("cannot set destination value") - } - - switch dstv.Kind() { - case reflect.Struct: - // Handle SDK native types using JSON unmarshaling. - if slices.Contains(sdkNativeTypes, dstv.Type()) { - return toTypedSDKNative(dstv, src) - } - return toTypedStruct(dstv, src) - case reflect.Map: - return toTypedMap(dstv, src) - case reflect.Slice: - return toTypedSlice(dstv, src) - case reflect.String: - return toTypedString(dstv, src) - case reflect.Bool: - return toTypedBool(dstv, src) - case reflect.Int, reflect.Int32, reflect.Int64: - return toTypedInt(dstv, src) - case reflect.Float32, reflect.Float64: - return toTypedFloat(dstv, src) - case reflect.Interface: - return toTypedInterface(dstv, src) - default: - // Fall through to the error case. - } - - return fmt.Errorf("unsupported type: %s", dstv.Kind()) -} - -func toTypedStruct(dst reflect.Value, src dyn.Value) error { - switch src.Kind() { - case dyn.KindMap: - // Zero the destination struct such that fields - // that aren't present in [src] are cleared. - dst.SetZero() - - info := getStructInfo(dst.Type()) - - for _, pair := range src.MustMap().Pairs() { - pk := pair.Key - pv := pair.Value - jsonKey := pk.MustString() - - index, ok := info.Fields[jsonKey] - if !ok { - // Ignore unknown fields. - // A warning will be printed later. See PR #904. - continue - } - - f := getOrNewFieldByIndex(dst, index) - - err := ToTyped(f.Addr().Interface(), pv) - if err != nil { - return err - } - - if pv.IsZero() { - // The field's zero value must still serialize, so append its Go name - // to the ForceSendFields of the struct that declares it. That struct - // shares a prefix with the field's index path, so it is already - // allocated by the walk above. - if fsfIndex, ok := info.ForceSendFieldsIndex[jsonKey]; ok { - fsf := getOrNewFieldByIndex(dst, fsfIndex) - fsf.Set(reflect.Append(fsf, reflect.ValueOf(info.GolangNames[jsonKey]))) - } - } - } - - // Populate field(s) for [dyn.Value], if any. - if info.ValueField != nil { - vv := dst.FieldByIndex(info.ValueField) - vv.Set(reflect.ValueOf(src)) - } - - return nil - case dyn.KindNil: - dst.SetZero() - return nil - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(src.MustString()) { - dst.SetZero() - return nil - } - default: - // Fall through to the error case. - } - - return TypeError{ - value: src, - msg: fmt.Sprintf("expected a map, found a %s", src.Kind()), - } -} - -// getOrNewFieldByIndex resolves the value at the given index path within an addressable -// struct, allocating intermediate structs embedded as pointer types along the way. -// Code inspired by [reflect.FieldByIndex] implementation. -func getOrNewFieldByIndex(v reflect.Value, index []int) reflect.Value { - for i, x := range index { - if i > 0 { - if v.Kind() == reflect.Pointer { - if v.IsNil() { - v.Set(reflect.New(v.Type().Elem())) - } - v = v.Elem() - } - } - v = v.Field(x) - } - return v -} - -func toTypedMap(dst reflect.Value, src dyn.Value) error { - switch src.Kind() { - case dyn.KindMap: - m := src.MustMap() - - // Always overwrite. - dst.Set(reflect.MakeMapWithSize(dst.Type(), m.Len())) - for _, pair := range m.Pairs() { - pk := pair.Key - pv := pair.Value - kv := reflect.ValueOf(pk.MustString()) - kt := dst.Type().Key() - vv := reflect.New(dst.Type().Elem()) - err := ToTyped(vv.Interface(), pv) - if err != nil { - return err - } - dst.SetMapIndex(kv.Convert(kt), vv.Elem()) - } - return nil - case dyn.KindNil: - dst.SetZero() - return nil - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(src.MustString()) { - dst.SetZero() - return nil - } - default: - // Fall through to the error case. - } - - return TypeError{ - value: src, - msg: fmt.Sprintf("expected a map, found a %s", src.Kind()), - } -} - -func toTypedSlice(dst reflect.Value, src dyn.Value) error { - switch src.Kind() { - case dyn.KindSequence: - seq := src.MustSequence() - - // Always overwrite. - dst.Set(reflect.MakeSlice(dst.Type(), len(seq), len(seq))) - for i := range seq { - err := ToTyped(dst.Index(i).Addr().Interface(), seq[i]) - if err != nil { - return err - } - } - return nil - case dyn.KindNil: - dst.SetZero() - return nil - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(src.MustString()) { - dst.SetZero() - return nil - } - default: - // Fall through to the error case. - } - - return TypeError{ - value: src, - msg: fmt.Sprintf("expected a sequence, found a %s", src.Kind()), - } -} - -func toTypedString(dst reflect.Value, src dyn.Value) error { - switch src.Kind() { - case dyn.KindString: - dst.SetString(src.MustString()) - return nil - case dyn.KindBool: - dst.SetString(strconv.FormatBool(src.MustBool())) - return nil - case dyn.KindInt: - dst.SetString(strconv.FormatInt(src.MustInt(), 10)) - return nil - case dyn.KindFloat: - dst.SetString(strconv.FormatFloat(src.MustFloat(), 'f', -1, 64)) - return nil - default: - // Fall through to the error case. - } - - return TypeError{ - value: src, - msg: fmt.Sprintf("expected a string, found a %s", src.Kind()), - } -} - -func toTypedBool(dst reflect.Value, src dyn.Value) error { - switch src.Kind() { - case dyn.KindBool: - dst.SetBool(src.MustBool()) - return nil - case dyn.KindString: - // See https://yaml.org/type/bool.html. - switch src.MustString() { - case "y", "Y", "yes", "Yes", "YES", "on", "On", "ON", "true", "True", "TRUE": - dst.SetBool(true) - return nil - case "n", "N", "no", "No", "NO", "off", "Off", "OFF", "false", "False", "FALSE": - dst.SetBool(false) - return nil - } - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(src.MustString()) { - dst.SetZero() - return nil - } - default: - // Fall through to the error case. - } - - return TypeError{ - value: src, - msg: fmt.Sprintf("expected a boolean, found a %s", src.Kind()), - } -} - -func toTypedInt(dst reflect.Value, src dyn.Value) error { - switch src.Kind() { - case dyn.KindInt: - dst.SetInt(src.MustInt()) - return nil - case dyn.KindFloat: - v := src.MustFloat() - if v == float64(int64(v)) { - // If the destination is smaller than int64, but the value to set is bigger - // then destination overflows and is set to -1 - dst.SetInt(int64(src.MustFloat())) - return nil - } - - return TypeError{ - value: src, - msg: fmt.Sprintf("expected an int, found a %s", src.Kind()), - } - case dyn.KindString: - if i64, err := strconv.ParseInt(src.MustString(), 10, 64); err == nil { - dst.SetInt(i64) - return nil - } - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(src.MustString()) { - dst.SetZero() - return nil - } - default: - // Fall through to the error case. - } - - return TypeError{ - value: src, - msg: fmt.Sprintf("expected an int, found a %s", src.Kind()), - } -} - -func toTypedFloat(dst reflect.Value, src dyn.Value) error { - switch src.Kind() { - case dyn.KindFloat: - dst.SetFloat(src.MustFloat()) - return nil - case dyn.KindString: - if f64, err := strconv.ParseFloat(src.MustString(), 64); err == nil { - dst.SetFloat(f64) - return nil - } - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(src.MustString()) { - dst.SetZero() - return nil - } - default: - // Fall through to the error case. - } - - return TypeError{ - value: src, - msg: fmt.Sprintf("expected a float, found a %s", src.Kind()), - } -} - -func toTypedInterface(dst reflect.Value, src dyn.Value) error { - if src.Kind() == dyn.KindNil { - dst.Set(reflect.Zero(dst.Type())) - return nil - } - - dst.Set(reflect.ValueOf(src.AsAny())) - return nil -} diff --git a/libs/dyn/convert/to_typed_test.go b/libs/dyn/convert/to_typed_test.go deleted file mode 100644 index a838dfab33f..00000000000 --- a/libs/dyn/convert/to_typed_test.go +++ /dev/null @@ -1,790 +0,0 @@ -package convert - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestToTypedStruct(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar,omitempty"` - - // Baz must be skipped. - Baz string `json:""` - - // Qux must be skipped. - Qux string `json:"-"` - } - - var out Tmp - v := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, "bar", out.Foo) - assert.Equal(t, "baz", out.Bar) -} - -func TestToTypedStructOverwrite(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar,omitempty"` - - // Baz must be skipped. - Baz string `json:""` - - // Qux must be skipped. - Qux string `json:"-"` - } - - out := Tmp{ - Foo: "baz", - Bar: "qux", - } - v := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, "bar", out.Foo) - assert.Equal(t, "baz", out.Bar) -} - -func TestToTypedStructClearFields(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar,omitempty"` - } - - // Struct value with non-empty fields. - out := Tmp{ - Foo: "baz", - Bar: "qux", - } - - // Value is an empty map. - v := dyn.V(map[string]dyn.Value{}) - - // The previously set fields should be cleared. - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, Tmp{}, out) -} - -func TestToTypedStructClearFieldsForceSend(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar,omitempty"` - ForceSendFields []string `json:"-"` - } - - // Struct value with non-empty fields. - out := Tmp{ - Foo: "baz", - Bar: "qux", - } - - // Value is an empty map. - v := dyn.V(map[string]dyn.Value{}) - - // The previously set fields should be cleared. - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, Tmp{}, out) -} - -func TestToTypedStructZeroFields(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar,omitempty"` - } - - // Struct value with non-empty fields. - out := Tmp{ - Foo: "baz", - Bar: "qux", - } - - // Value is an empty map. - v := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(""), - "bar": dyn.V(""), - }) - - // The previously set fields should be cleared. - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, Tmp{}, out) -} - -func TestToTypedStructZeroFieldsForceSend(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar,omitempty"` - NumWorkers int `json:"num_workers,omitempty"` - ForceSendFields []string `json:"-"` - } - - // Struct value with non-empty fields. - out := Tmp{ - Foo: "baz", - Bar: "qux", - } - - // Value is an empty map. - m := dyn.Mapping{} - m.SetLoc("foo", nil, dyn.V("")) - m.SetLoc("bar", nil, dyn.V("")) - m.SetLoc("num_workers", nil, dyn.V(int64(0))) - v := dyn.V(m) - - // The previously set fields should be cleared. - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, Tmp{ForceSendFields: []string{"Foo", "Bar", "NumWorkers"}}, out) -} - -func TestToTypedStructAnonymousByValue(t *testing.T) { - type Bar struct { - Bar string `json:"bar"` - } - - type Foo struct { - Foo string `json:"foo"` - Bar - } - - type Tmp struct { - Foo - } - - var out Tmp - v := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, "bar", out.Foo.Foo) - assert.Equal(t, "baz", out.Bar.Bar) -} - -func TestToTypedStructAnonymousByPointer(t *testing.T) { - type Bar struct { - Bar string `json:"bar"` - } - - type Foo struct { - Foo string `json:"foo"` - *Bar - } - - type Tmp struct { - *Foo - } - - var out Tmp - v := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, "bar", out.Foo.Foo) - assert.Equal(t, "baz", out.Bar.Bar) -} - -func TestToTypedStructNil(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - out := Tmp{} - err := ToTyped(&out, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, Tmp{}, out) -} - -func TestToTypedStructNilOverwrite(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - out := Tmp{"bar"} - err := ToTyped(&out, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, Tmp{}, out) -} - -func TestToTypedStructWithValueField(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - - ConfigValue dyn.Value - } - - var out Tmp - v := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, "bar", out.Foo) - assert.Equal(t, v, out.ConfigValue) -} - -func TestToTypedMap(t *testing.T) { - out := map[string]string{} - - v := dyn.V(map[string]dyn.Value{ - "key": dyn.V("value"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Len(t, out, 1) - assert.Equal(t, "value", out["key"]) -} - -func TestToTypedMapOverwrite(t *testing.T) { - out := map[string]string{ - "foo": "bar", - } - - v := dyn.V(map[string]dyn.Value{ - "bar": dyn.V("qux"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Len(t, out, 1) - assert.Equal(t, "qux", out["bar"]) -} - -func TestToTypedMapWithPointerElement(t *testing.T) { - var out map[string]*string - - v := dyn.V(map[string]dyn.Value{ - "key": dyn.V("value"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Len(t, out, 1) - assert.Equal(t, "value", *out["key"]) -} - -func TestToTypedMapNil(t *testing.T) { - out := map[string]string{} - err := ToTyped(&out, dyn.NilValue) - require.NoError(t, err) - assert.Nil(t, out) -} - -func TestToTypedMapNilOverwrite(t *testing.T) { - out := map[string]string{ - "foo": "bar", - } - err := ToTyped(&out, dyn.NilValue) - require.NoError(t, err) - assert.Nil(t, out) -} - -func TestToTypedSlice(t *testing.T) { - var out []string - - v := dyn.V([]dyn.Value{ - dyn.V("foo"), - dyn.V("bar"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Len(t, out, 2) - assert.Equal(t, "foo", out[0]) - assert.Equal(t, "bar", out[1]) -} - -func TestToTypedSliceOverwrite(t *testing.T) { - out := []string{"qux"} - - v := dyn.V([]dyn.Value{ - dyn.V("foo"), - dyn.V("bar"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Len(t, out, 2) - assert.Equal(t, "foo", out[0]) - assert.Equal(t, "bar", out[1]) -} - -func TestToTypedSliceWithPointerElement(t *testing.T) { - var out []*string - - v := dyn.V([]dyn.Value{ - dyn.V("foo"), - dyn.V("bar"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Len(t, out, 2) - assert.Equal(t, "foo", *out[0]) - assert.Equal(t, "bar", *out[1]) -} - -func TestToTypedSliceNil(t *testing.T) { - var out []string - err := ToTyped(&out, dyn.NilValue) - require.NoError(t, err) - assert.Nil(t, out) -} - -func TestToTypedSliceNilOverwrite(t *testing.T) { - out := []string{"foo"} - err := ToTyped(&out, dyn.NilValue) - require.NoError(t, err) - assert.Nil(t, out) -} - -func TestToTypedString(t *testing.T) { - var out string - err := ToTyped(&out, dyn.V("foo")) - require.NoError(t, err) - assert.Equal(t, "foo", out) -} - -func TestToTypedStringOverwrite(t *testing.T) { - out := "bar" - err := ToTyped(&out, dyn.V("foo")) - require.NoError(t, err) - assert.Equal(t, "foo", out) -} - -func TestToTypedStringFromBool(t *testing.T) { - var out string - err := ToTyped(&out, dyn.V(true)) - require.NoError(t, err) - assert.Equal(t, "true", out) -} - -func TestToTypedStringFromInt(t *testing.T) { - var out string - err := ToTyped(&out, dyn.V(123)) - require.NoError(t, err) - assert.Equal(t, "123", out) -} - -func TestToTypedStringFromFloat(t *testing.T) { - var out string - err := ToTyped(&out, dyn.V(1.2)) - require.NoError(t, err) - assert.Equal(t, "1.2", out) -} - -func TestToTypedBool(t *testing.T) { - var out bool - err := ToTyped(&out, dyn.V(true)) - require.NoError(t, err) - assert.True(t, out) -} - -func TestToTypedBoolOverwrite(t *testing.T) { - out := true - err := ToTyped(&out, dyn.V(false)) - require.NoError(t, err) - assert.False(t, out) -} - -func TestToTypedBoolFromString(t *testing.T) { - var out bool - - // True-ish - for _, v := range []string{"y", "yes", "on", "true", "True", "TRUE"} { - err := ToTyped(&out, dyn.V(v)) - require.NoError(t, err) - assert.True(t, out) - } - - // False-ish - for _, v := range []string{"n", "no", "off", "false", "False", "FALSE"} { - err := ToTyped(&out, dyn.V(v)) - require.NoError(t, err) - assert.False(t, out) - } - - // Other - err := ToTyped(&out, dyn.V("some other string")) - require.Error(t, err) -} - -func TestToTypedBoolFromStringVariableReference(t *testing.T) { - out := true - err := ToTyped(&out, dyn.V("${var.foo}")) - require.NoError(t, err) - assert.False(t, out) -} - -func TestToTypedInt(t *testing.T) { - var out int - err := ToTyped(&out, dyn.V(1234)) - require.NoError(t, err) - assert.Equal(t, int(1234), out) -} - -func TestToTypedInt32(t *testing.T) { - var out32 int32 - err := ToTyped(&out32, dyn.V(1235)) - require.NoError(t, err) - assert.Equal(t, int32(1235), out32) -} - -func TestToTypedInt64(t *testing.T) { - var out64 int64 - err := ToTyped(&out64, dyn.V(1236)) - require.NoError(t, err) - assert.Equal(t, int64(1236), out64) -} - -func TestToTypedIntOverwrite(t *testing.T) { - out := 123 - err := ToTyped(&out, dyn.V(1234)) - require.NoError(t, err) - assert.Equal(t, int(1234), out) -} - -func TestToTypedInt32Overwrite(t *testing.T) { - var out32 int32 = 123 - err := ToTyped(&out32, dyn.V(1234)) - require.NoError(t, err) - assert.Equal(t, int32(1234), out32) -} - -func TestToTypedInt64Overwrite(t *testing.T) { - var out64 int64 = 123 - err := ToTyped(&out64, dyn.V(1234)) - require.NoError(t, err) - assert.Equal(t, int64(1234), out64) -} - -func TestToTypedIntFromStringError(t *testing.T) { - var out int - err := ToTyped(&out, dyn.V("abc")) - require.Error(t, err) -} - -func TestToTypedIntFromStringInt(t *testing.T) { - var out int - err := ToTyped(&out, dyn.V("123")) - require.NoError(t, err) - assert.Equal(t, int(123), out) -} - -func TestToTypedIntFromStringVariableReference(t *testing.T) { - out := 123 - err := ToTyped(&out, dyn.V("${var.foo}")) - require.NoError(t, err) - assert.Equal(t, int(0), out) -} - -func TestToTypedIntFromFloat(t *testing.T) { - var out int - err := ToTyped(&out, dyn.V(1.0)) - require.NoError(t, err) - assert.Equal(t, int(1), out) -} - -func TestToTypedIntFromFloatError(t *testing.T) { - var out int - err := ToTyped(&out, dyn.V(1.2)) - require.ErrorContains(t, err, "expected an int, found a float") -} - -func TestToTypedFloat32(t *testing.T) { - var out float32 - err := ToTyped(&out, dyn.V(float32(1.0))) - require.NoError(t, err) - assert.Zero(t, 1.0-out) -} - -func TestToTypedFloat64(t *testing.T) { - var out float64 - err := ToTyped(&out, dyn.V(float64(1.0))) - require.NoError(t, err) - assert.Zero(t, 1.0-out) -} - -func TestToTypedFloat32Overwrite(t *testing.T) { - var out float32 = 1.0 - err := ToTyped(&out, dyn.V(float32(2.0))) - require.NoError(t, err) - assert.Zero(t, 2.0-out) -} - -func TestToTypedFloat64Overwrite(t *testing.T) { - out := 1.0 - err := ToTyped(&out, dyn.V(float64(2.0))) - require.NoError(t, err) - assert.Zero(t, 2.0-out) -} - -func TestToTypedFloat32FromStringError(t *testing.T) { - var out float32 - err := ToTyped(&out, dyn.V("abc")) - require.Error(t, err) -} - -func TestToTypedFloat64FromStringError(t *testing.T) { - var out float64 - err := ToTyped(&out, dyn.V("abc")) - require.Error(t, err) -} - -func TestToTypedFloat32FromString(t *testing.T) { - var out float32 - err := ToTyped(&out, dyn.V("1.2")) - require.NoError(t, err) - assert.Zero(t, 1.2-out) -} - -func TestToTypedFloat64FromString(t *testing.T) { - var out float64 - err := ToTyped(&out, dyn.V("1.2")) - require.NoError(t, err) - assert.Zero(t, 1.2-out) -} - -func TestToTypedFloat32FromStringVariableReference(t *testing.T) { - var out float32 = 1.0 - err := ToTyped(&out, dyn.V("${var.foo}")) - require.NoError(t, err) - assert.Zero(t, out) -} - -func TestToTypedFloat64FromStringVariableReference(t *testing.T) { - out := 1.0 - err := ToTyped(&out, dyn.V("${var.foo}")) - require.NoError(t, err) - assert.Zero(t, out) -} - -func TestToTypedWithAliasKeyType(t *testing.T) { - type custom string - - var out map[custom]string - v := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Len(t, out, 2) - assert.Equal(t, "bar", out["foo"]) - assert.Equal(t, "baz", out["bar"]) -} - -func TestToTypedAnyWithBool(t *testing.T) { - var out any - err := ToTyped(&out, dyn.V(false)) - require.NoError(t, err) - assert.Equal(t, false, out) - - err = ToTyped(&out, dyn.V(true)) - require.NoError(t, err) - assert.Equal(t, true, out) -} - -func TestToTypedAnyWithMap(t *testing.T) { - var out any - v := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, map[string]any{"foo": "bar", "bar": "baz"}, out) -} - -func TestToTypedAnyWithNil(t *testing.T) { - var out any - err := ToTyped(&out, dyn.NilValue) - require.NoError(t, err) - assert.Nil(t, out) -} - -func TestToTypedEmbeddedStructForceSendFields(t *testing.T) { - type Inner struct { - InnerField string `json:"inner_field"` - ForceSendFields []string `json:"-"` - } - - type Outer struct { - OuterField string `json:"outer_field"` - Inner - } - - var out Outer - m := dyn.Mapping{} - m.SetLoc("outer_field", nil, dyn.V("")) - m.SetLoc("inner_field", nil, dyn.V("")) - v := dyn.V(m) - - err := ToTyped(&out, v) - require.NoError(t, err) - - // Bug: ForceSendFields contains "OuterField" which belongs to outer struct - // Expected: ForceSendFields should only contain "InnerField" - assert.Equal(t, []string{"InnerField"}, out.ForceSendFields) -} - -func TestToTypedMultipleEmbeddedStructsForceSendFields(t *testing.T) { - type First struct { - FirstField string `json:"first_field"` - ForceSendFields []string `json:"-"` - } - - type Second struct { - SecondField string `json:"second_field"` - ForceSendFields []string `json:"-"` - } - - type Outer struct { - OuterField string `json:"outer_field"` - First - Second - } - - var out Outer - m := dyn.Mapping{} - m.SetLoc("outer_field", nil, dyn.V("")) - m.SetLoc("first_field", nil, dyn.V("")) - m.SetLoc("second_field", nil, dyn.V("")) - v := dyn.V(m) - - err := ToTyped(&out, v) - require.NoError(t, err) - - // Each embedded struct should only get its own fields in ForceSendFields - assert.Equal(t, []string{"FirstField"}, out.First.ForceSendFields) - assert.Equal(t, []string{"SecondField"}, out.Second.ForceSendFields) -} - -func TestToTypedMixedForceSendFields(t *testing.T) { - type First struct { - FirstField string `json:"first_field"` - // No ForceSendFields - } - - type Second struct { - SecondField string `json:"second_field"` - ForceSendFields []string `json:"-"` - } - - type Outer struct { - OuterField string `json:"outer_field"` - ForceSendFields []string `json:"-"` - First - Second - } - - var out Outer - m := dyn.Mapping{} - m.SetLoc("outer_field", nil, dyn.V("")) - m.SetLoc("first_field", nil, dyn.V("")) - m.SetLoc("second_field", nil, dyn.V("")) - v := dyn.V(m) - - err := ToTyped(&out, v) - require.NoError(t, err) - - // Outer should get its own direct fields - assert.Equal(t, []string{"OuterField"}, out.ForceSendFields) - // First has no ForceSendFields field, so nothing to check - // Second should get its own field - assert.Equal(t, []string{"SecondField"}, out.Second.ForceSendFields) -} - -func TestToTypedFieldByNameBugRegressionTest(t *testing.T) { - // This test reproduces the EXACT bug: only direct field is zero, embedded fields are non-zero - // The bug occurred because dst.FieldByName("ForceSendFields") finds embedded ForceSendFields - // instead of direct ForceSendFields (which doesn't exist) - type BaseResource struct { - ID string `json:"id,omitempty"` - } - - type JobSettings struct { - Name string `json:"name"` - ForceSendFields []string `json:"-"` - } - - type Job struct { - BaseResource // embedded at [0] - JobSettings // embedded at [1], has ForceSendFields - Permissions []string `json:"permissions,omitempty"` // direct at [2] - } - - var out Job - m := dyn.Mapping{} - // Non-zero JobSettings fields - these should NOT go to ForceSendFields - m.SetLoc("name", nil, dyn.V("test-job")) - // Zero direct field - this SHOULD trigger the bug with old code - m.SetLoc("permissions", nil, dyn.V([]dyn.Value{})) - v := dyn.V(m) - - err := ToTyped(&out, v) - require.NoError(t, err) - - // "Permissions" should be dropped, JobSettings.ForceSendFields stays empty - assert.Empty(t, out.ForceSendFields) - assert.Equal(t, "test-job", out.Name) - assert.Empty(t, out.Permissions) -} - -func TestToTypedDeeplyEmbeddedStructForceSendFields(t *testing.T) { - // Mirrors resources.PostgresProject -> PostgresProjectConfig -> ProjectSpec: - // Spec is embedded two levels down and Wrapper shadows ForceSendFields to keep - // its own direct field out of Spec's ForceSendFields. A zero-value spec field - // must route to Spec.ForceSendFields, not Wrapper's, otherwise the SDK marshaler - // fails with "field ... cannot be found in struct". - type Spec struct { - SpecField bool `json:"spec_field,omitempty"` - ForceSendFields []string `json:"-"` - } - - type Wrapper struct { - Spec - WrapperField string `json:"wrapper_field,omitempty"` - ForceSendFields []string `json:"-"` - } - - type Outer struct { - Wrapper - } - - var out Outer - m := dyn.Mapping{} - m.SetLoc("spec_field", nil, dyn.V(false)) - m.SetLoc("wrapper_field", nil, dyn.V("")) - v := dyn.V(m) - - err := ToTyped(&out, v) - require.NoError(t, err) - - assert.Equal(t, []string{"SpecField"}, out.Spec.ForceSendFields) - assert.Equal(t, []string{"WrapperField"}, out.ForceSendFields) -} diff --git a/libs/dyn/drop_keys.go b/libs/dyn/drop_keys.go deleted file mode 100644 index 494f9b9cd88..00000000000 --- a/libs/dyn/drop_keys.go +++ /dev/null @@ -1,27 +0,0 @@ -package dyn - -func DropKeys(v Value, drop []string) (Value, error) { - var err error - nv, err := Walk(v, func(p Path, v Value) (Value, error) { - if len(p) == 0 { - return v, nil - } - - // Check if this key should be dropped. - for _, key := range drop { - if p[0].Key() != key { - continue - } - - return InvalidValue, ErrDrop - } - - // Pass through all other values. - return v, ErrSkip - }) - if err != nil { - return InvalidValue, err - } - - return nv, nil -} diff --git a/libs/dyn/drop_keys_test.go b/libs/dyn/drop_keys_test.go deleted file mode 100644 index 83a9744ca65..00000000000 --- a/libs/dyn/drop_keys_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package dyn - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestDropKeysTest(t *testing.T) { - v := V(map[string]Value{ - "key1": V("value1"), - "key2": V("value2"), - "key3": V("value3"), - }) - - vout, err := DropKeys(v, []string{"key1", "key3"}) - require.NoError(t, err) - - mv := vout.MustMap() - require.Equal(t, 1, mv.Len()) - v, ok := mv.GetByString("key2") - require.True(t, ok) - require.Equal(t, "value2", v.MustString()) -} diff --git a/libs/dyn/dynassert/assert.go b/libs/dyn/dynassert/assert.go deleted file mode 100644 index 9e02eeff39c..00000000000 --- a/libs/dyn/dynassert/assert.go +++ /dev/null @@ -1,36 +0,0 @@ -package dynassert - -import ( - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -// Equal is a variant of assert.Equal that first converts dyn.Value to any. The practical consequence is that maps are compared -// without order. Another difference here is that locations are compared via Location() function, which drops all -// but first and get rids of nils. -func Equal(t assert.TestingT, expected, actual any, msgAndArgs ...any) bool { - ev, eok := expected.(dyn.Value) - av, aok := actual.(dyn.Value) - if eok && aok && ev.IsValid() && av.IsValid() { - if !assert.Equal(t, ev.AsAny(), av.AsAny(), msgAndArgs...) { - return false - } - - // The values are equal on contents. Now compare the locations. - if !assert.Equal(t, ev.Location(), av.Location(), msgAndArgs...) { - return false - } - - // Walk ev and av and compare the locations of each element. - _, err := dyn.Walk(ev, func(p dyn.Path, evv dyn.Value) (dyn.Value, error) { - avv, err := dyn.GetByPath(av, p) - if assert.NoError(t, err, "unable to get value from actual value at path %v", p.String()) { - assert.Equal(t, evv.Location(), avv.Location()) - } - return evv, nil - }) - return assert.NoError(t, err) - } - - return assert.Equal(t, expected, actual, msgAndArgs...) -} diff --git a/libs/dyn/dynvar/lookup.go b/libs/dyn/dynvar/lookup.go deleted file mode 100644 index 2bc08f47d47..00000000000 --- a/libs/dyn/dynvar/lookup.go +++ /dev/null @@ -1,21 +0,0 @@ -package dynvar - -import ( - "errors" - - "github.com/databricks/cli/libs/dyn" -) - -// Lookup is the type of lookup functions that can be used with [Resolve]. -type Lookup func(path dyn.Path) (dyn.Value, error) - -// ErrSkipResolution is returned by a lookup function to indicate that the -// resolution of a variable reference should be skipped. -var ErrSkipResolution = errors.New("skip resolution") - -// DefaultLookup is the default lookup function used by [Resolve]. -func DefaultLookup(in dyn.Value) Lookup { - return func(path dyn.Path) (dyn.Value, error) { - return dyn.GetByPath(in, path) - } -} diff --git a/libs/dyn/dynvar/lookup_test.go b/libs/dyn/dynvar/lookup_test.go deleted file mode 100644 index 2341d72084b..00000000000 --- a/libs/dyn/dynvar/lookup_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package dynvar_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" - "github.com/stretchr/testify/assert" -) - -func TestDefaultLookup(t *testing.T) { - lookup := dynvar.DefaultLookup(dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("b"), - })) - - v1, err := lookup(dyn.NewPath(dyn.Key("a"))) - assert.NoError(t, err) - assert.Equal(t, dyn.V("a"), v1) - - v2, err := lookup(dyn.NewPath(dyn.Key("b"))) - assert.NoError(t, err) - assert.Equal(t, dyn.V("b"), v2) - - _, err = lookup(dyn.NewPath(dyn.Key("c"))) - assert.True(t, dyn.IsNoSuchKeyError(err)) -} diff --git a/libs/dyn/dynvar/resolve.go b/libs/dyn/dynvar/resolve.go deleted file mode 100644 index 99981a2e51b..00000000000 --- a/libs/dyn/dynvar/resolve.go +++ /dev/null @@ -1,255 +0,0 @@ -package dynvar - -import ( - "errors" - "fmt" - "maps" - "slices" - "strings" - - "github.com/databricks/cli/libs/dyn" -) - -// Resolve resolves variable references in the given input value using the provided lookup function. -// It returns the resolved output value and any error encountered during the resolution process. -// -// For example, given the input value: -// -// { -// "a": "a", -// "b": "${a}", -// "c": "${b}${b}", -// } -// -// The output value will be: -// -// { -// "a": "a", -// "b": "a", -// "c": "aa", -// } -// -// If the input value contains a variable reference that cannot be resolved, an error is returned. -// If a cycle is detected in the variable references, an error is returned. -// If for some path the resolution function returns [ErrSkipResolution], the variable reference is left in place. -// This is useful when some variable references are not yet ready to be interpolated. -func Resolve(in dyn.Value, fn Lookup) (out dyn.Value, err error) { - return resolver{in: in, fn: fn}.run() -} - -// ReferenceError is returned for an unresolved variable reference. Suggestions -// are carried as data so callers (which can import libs/diag) format them. -type ReferenceError struct { - Reference string // original reference text, e.g. "var.hst" - Suggestions []string // corrected references, e.g. ["var.host", "var.hosts"] -} - -func (e *ReferenceError) Error() string { - return fmt.Sprintf("reference does not exist: ${%s}", e.Reference) -} - -type lookupResult struct { - v dyn.Value - err error -} - -type resolver struct { //nolint:recvcheck // value receiver for run(), pointer for mutation methods - in dyn.Value - fn Lookup - - refs map[string]Ref - resolved map[string]dyn.Value - - // Memoization for lookups. - lookups map[string]lookupResult -} - -func (r resolver) run() (out dyn.Value, err error) { - err = r.collectVariableReferences() - if err != nil { - return dyn.InvalidValue, err - } - - err = r.resolveVariableReferences() - if err != nil { - return dyn.InvalidValue, err - } - - out, err = r.replaceVariableReferences() - if err != nil { - return dyn.InvalidValue, err - } - - return out, nil -} - -func (r *resolver) collectVariableReferences() (err error) { - r.refs = make(map[string]Ref) - - // First walk the input to gather all values with a variable reference. - _, err = dyn.Walk(r.in, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - ref, ok := NewRef(v) - if !ok { - // Skip values without variable references. - return v, nil - } - - r.refs[p.String()] = ref - return v, nil - }) - - return err -} - -func (r *resolver) resolveVariableReferences() (err error) { - // Initialize cache for lookups. - r.lookups = make(map[string]lookupResult) - - // Initialize cache for resolved variable references. - r.resolved = make(map[string]dyn.Value) - - // Resolve each variable reference (in order). - // We sort the keys here to ensure that we always resolve the same variable reference first. - // This is done such that the cycle detection error is deterministic. If we did not do this, - // we could enter the cycle at any point in the cycle and return varying errors. - keys := slices.Sorted(maps.Keys(r.refs)) - for _, key := range keys { - v, err := r.resolveRef(r.refs[key], []string{key}) - if err != nil { - return err - } - r.resolved[key] = v - } - - return nil -} - -func (r *resolver) resolveRef(ref Ref, seen []string) (dyn.Value, error) { - // This is an unresolved variable reference. - deps := ref.References() - - // Resolve each of the dependencies, then interpolate them in the ref. - resolved := make([]dyn.Value, len(deps)) - complete := true - - for j, dep := range deps { - // Cycle detection. - if slices.Contains(seen, dep) { - return dyn.InvalidValue, fmt.Errorf( - "cycle detected in field resolution: %s", - strings.Join(append(seen, dep), " -> "), - ) - } - - v, err := r.resolveKey(dep, append(seen, dep)) - - // If we should skip resolution of this key, index j will hold an invalid [dyn.Value]. - if errors.Is(err, ErrSkipResolution) { - complete = false - continue - } else if err != nil { - // Otherwise, propagate the error. - return dyn.InvalidValue, err - } - - resolved[j] = v - } - - // Interpolate the resolved values. - if ref.IsPure() && complete { - // If the variable reference is pure, we can substitute it. - // This is useful for interpolating values of non-string types. - // - // Note: we use the location of the variable reference to preserve the information - // of where it is used. This also means that relative path resolution is done - // relative to where a variable is used, not where it is defined. - // - return dyn.NewValue(resolved[0].Value(), ref.Value.Locations()), nil - } - - // Not pure; perform string interpolation. - // - // Substitute by byte offset rather than by searching for the match text: the same - // reference may also appear escaped ("$${foo} ${foo}"), and a search would replace - // that occurrence instead, corrupting the literal and leaving the real one unresolved. - var sb strings.Builder - consumed := 0 - for j := range ref.Matches { - // The value is invalid if resolution returned [ErrSkipResolution]. - // We must skip those and leave the original variable reference in place, - // which happens naturally by not advancing past its span. - if !resolved[j].IsValid() { - continue - } - - // Try to turn the resolved value into a string. - s, ok := resolved[j].AsString() - if !ok { - // Only allow primitive types to be converted to string. - switch resolved[j].Kind() { - case dyn.KindString, dyn.KindBool, dyn.KindInt, dyn.KindFloat, dyn.KindTime, dyn.KindNil: - s = fmt.Sprint(resolved[j].AsAny()) - default: - return dyn.InvalidValue, fmt.Errorf("cannot interpolate non-primitive value of type %s into string", resolved[j].Kind()) - } - } - - start, end := ref.Spans[j][0], ref.Spans[j][1] - sb.WriteString(ref.Str[consumed:start]) - sb.WriteString(s) - consumed = end - } - sb.WriteString(ref.Str[consumed:]) - - return dyn.NewValue(sb.String(), ref.Value.Locations()), nil -} - -func (r *resolver) resolveKey(key string, seen []string) (dyn.Value, error) { - // Check if we have already looked up this key. - if v, ok := r.lookups[key]; ok { - return v.v, v.err - } - - // Parse the key into a path. - p, err := dyn.NewPathFromString(key) - if err != nil { - return dyn.InvalidValue, err - } - - // Look up the value for the given key. - v, err := r.fn(p) - if err != nil { - if dyn.IsNoSuchKeyError(err) { - // Carry suggestions as data; the caller formats them via libs/diag. - err = &ReferenceError{Reference: key, Suggestions: dyn.SuggestedReferences(err, key)} - } - - // Cache the return value and return to the caller. - r.lookups[key] = lookupResult{v: dyn.InvalidValue, err: err} - return dyn.InvalidValue, err - } - - // If the returned value is a valid variable reference, resolve it. - ref, ok := NewRef(v) - if ok { - v, err = r.resolveRef(ref, seen) - } - - // Cache the return value and return to the caller. - r.lookups[key] = lookupResult{v: v, err: err} - return v, err -} - -func (r *resolver) replaceVariableReferences() (dyn.Value, error) { - // Walk the input and replace all variable references. - return dyn.Walk(r.in, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - nv, ok := r.resolved[p.String()] - if !ok { - // No variable reference; return the original value. - return v, nil - } - - // We have a variable reference; return the resolved value. - return nv, nil - }) -} diff --git a/libs/dyn/dynvar/resolve_test.go b/libs/dyn/dynvar/resolve_test.go deleted file mode 100644 index c4b28e88d3f..00000000000 --- a/libs/dyn/dynvar/resolve_test.go +++ /dev/null @@ -1,430 +0,0 @@ -package dynvar_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func getByPath(t *testing.T, v dyn.Value, path string) dyn.Value { - v, err := dyn.Get(v, path) - require.NoError(t, err) - return v -} - -func TestResolve(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("${a}"), - "c": dyn.V("${a}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "a", getByPath(t, out, "b").MustString()) - assert.Equal(t, "a", getByPath(t, out, "c").MustString()) -} - -func TestResolveNotFound(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "b": dyn.V("${a}"), - }) - - _, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.ErrorContains(t, err, `reference does not exist: ${a}`) -} - -func TestResolveNotFoundSuggestsCloseKey(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "host": dyn.V("example.com"), - "b": dyn.V("${hst}"), - }) - - _, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.ErrorContains(t, err, "reference does not exist: ${hst}") - - var refErr *dynvar.ReferenceError - require.ErrorAs(t, err, &refErr) - assert.Equal(t, []string{"host"}, refErr.Suggestions) -} - -func TestResolveWithNesting(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("${f.a}"), - "f": dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("${f.a}"), - }), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "a", getByPath(t, out, "f.a").MustString()) - assert.Equal(t, "a", getByPath(t, out, "f.b").MustString()) -} - -func TestResolveWithRecursion(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("${a}"), - "c": dyn.V("${b}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "a", getByPath(t, out, "b").MustString()) - assert.Equal(t, "a", getByPath(t, out, "c").MustString()) -} - -func TestResolveWithRecursionLoop(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("${c}"), - "c": dyn.V("${d}"), - "d": dyn.V("${b}"), - }) - - _, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - assert.ErrorContains(t, err, "cycle detected in field resolution: b -> c -> d -> b") -} - -func TestResolveWithRecursionLoopSelf(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("${a}"), - }) - - _, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - assert.ErrorContains(t, err, "cycle detected in field resolution: a -> a") -} - -func TestResolveWithStringConcatenation(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("b"), - "c": dyn.V("${a}${b}${a}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "b", getByPath(t, out, "b").MustString()) - assert.Equal(t, "aba", getByPath(t, out, "c").MustString()) -} - -func TestResolveWithEscapedReference(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("A"), - "escaped_only": dyn.V("$${a}"), - "escaped_first": dyn.V("$${a} ${a}"), - "real_first": dyn.V("${a} $${a}"), - "surrounded": dyn.V("$${a} ${a} $${a}"), - "other_name": dyn.V("$${zz} ${a}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - // The escape is preserved here; it is removed when the API payload is built. - assert.Equal(t, "$${a}", getByPath(t, out, "escaped_only").MustString()) - assert.Equal(t, "$${a} A", getByPath(t, out, "escaped_first").MustString()) - assert.Equal(t, "A $${a}", getByPath(t, out, "real_first").MustString()) - assert.Equal(t, "$${a} A $${a}", getByPath(t, out, "surrounded").MustString()) - assert.Equal(t, "$${zz} A", getByPath(t, out, "other_name").MustString()) -} - -func TestResolveWithTypeInterpolation(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V(1), - "b": dyn.V(2), - "c": dyn.V("${a} ${b}"), - "float_val": dyn.V(3.14), - "bool_true": dyn.V(true), - "bool_false": dyn.V(false), - "time_val": dyn.V(dyn.MustTime("2024-01-01")), - "nil_val": dyn.NilValue, - // Test interpolation of different types in string templates - "float_interp": dyn.V("Value: ${float_val}"), - "bool_true_interp": dyn.V("Enabled: ${bool_true}"), - "bool_false_interp": dyn.V("Disabled: ${bool_false}"), - "time_interp": dyn.V("Date: ${time_val}"), - "nil_interp": dyn.V("Null: ${nil_val}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - // Integer interpolation - assert.Equal(t, "1 2", getByPath(t, out, "c").MustString()) - - // Float interpolation - assert.Equal(t, "Value: 3.14", getByPath(t, out, "float_interp").MustString()) - - // Bool interpolation - assert.Equal(t, "Enabled: true", getByPath(t, out, "bool_true_interp").MustString()) - assert.Equal(t, "Disabled: false", getByPath(t, out, "bool_false_interp").MustString()) - - // Time interpolation should convert to string representation of time.Time - // Note: time.Time string representation includes timezone info - timeResult := getByPath(t, out, "time_interp").MustString() - assert.Contains(t, timeResult, "Date: 2024-01-01") - assert.Contains(t, timeResult, "00:00:00") - - // Nil interpolation - assert.Equal(t, "Null: ", getByPath(t, out, "nil_interp").MustString()) -} - -func TestResolveWithTypeRetentionFailure(t *testing.T) { - // Test that mapping interpolation fails with an error - mappingTest := dyn.V(map[string]dyn.Value{ - "mapping": dyn.V(map[string]dyn.Value{"key": dyn.V("value")}), - "interp": dyn.V("Config: ${mapping}"), - }) - _, err := dynvar.Resolve(mappingTest, dynvar.DefaultLookup(mappingTest)) - require.Error(t, err) - assert.Equal(t, "cannot interpolate non-primitive value of type map into string", err.Error()) -} - -func TestResolveWithTypeRetention(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "int": dyn.V(1), - "int_var": dyn.V("${int}"), - "bool_true": dyn.V(true), - "bool_true_var": dyn.V("${bool_true}"), - "bool_false": dyn.V(false), - "bool_false_var": dyn.V("${bool_false}"), - "float": dyn.V(1.0), - "float_var": dyn.V("${float}"), - "string": dyn.V("a"), - "string_var": dyn.V("${string}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.EqualValues(t, 1, getByPath(t, out, "int").MustInt()) - assert.EqualValues(t, 1, getByPath(t, out, "int_var").MustInt()) - - assert.True(t, getByPath(t, out, "bool_true").MustBool()) - assert.True(t, getByPath(t, out, "bool_true_var").MustBool()) - - assert.False(t, getByPath(t, out, "bool_false").MustBool()) - assert.False(t, getByPath(t, out, "bool_false_var").MustBool()) - - assert.Zero(t, 1.0-getByPath(t, out, "float").MustFloat()) - assert.Zero(t, 1.0-getByPath(t, out, "float_var").MustFloat()) - - assert.Equal(t, "a", getByPath(t, out, "string").MustString()) - assert.Equal(t, "a", getByPath(t, out, "string_var").MustString()) -} - -func TestResolveWithSkip(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("b"), - "c": dyn.V("${a}"), - "d": dyn.V("${b}"), - "e": dyn.V("${a} ${b}"), - "f": dyn.V("${b} ${a} ${a} ${b}"), - }) - - fallback := dynvar.DefaultLookup(in) - ignore := func(path dyn.Path) (dyn.Value, error) { - // If the variable reference to look up starts with "b", skip it. - if path.HasPrefix(dyn.NewPath(dyn.Key("b"))) { - return dyn.InvalidValue, dynvar.ErrSkipResolution - } - return fallback(path) - } - - out, err := dynvar.Resolve(in, ignore) - require.NoError(t, err) - - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "b", getByPath(t, out, "b").MustString()) - assert.Equal(t, "a", getByPath(t, out, "c").MustString()) - - // Check that the skipped variable references are not interpolated. - assert.Equal(t, "${b}", getByPath(t, out, "d").MustString()) - assert.Equal(t, "a ${b}", getByPath(t, out, "e").MustString()) - assert.Equal(t, "${b} a a ${b}", getByPath(t, out, "f").MustString()) //nolint:dupword -} - -func TestResolveWithSkipEverything(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("b"), - "c": dyn.V("${a}"), - "d": dyn.V("${b}"), - "e": dyn.V("${a} ${b}"), - "f": dyn.V("${b} ${a} ${a} ${b}"), - "g": dyn.V("${d} ${c} ${c} ${d}"), - }) - - // The call must not replace anything if the lookup function returns ErrSkipResolution. - out, err := dynvar.Resolve(in, func(path dyn.Path) (dyn.Value, error) { - return dyn.InvalidValue, dynvar.ErrSkipResolution - }) - require.NoError(t, err) - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "b", getByPath(t, out, "b").MustString()) - assert.Equal(t, "${a}", getByPath(t, out, "c").MustString()) - assert.Equal(t, "${b}", getByPath(t, out, "d").MustString()) - assert.Equal(t, "${a} ${b}", getByPath(t, out, "e").MustString()) - assert.Equal(t, "${b} ${a} ${a} ${b}", getByPath(t, out, "f").MustString()) - assert.Equal(t, "${d} ${c} ${c} ${d}", getByPath(t, out, "g").MustString()) -} - -func TestResolveWithInterpolateNewRef(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("${a}"), - }) - - // The call replaces ${a} with ${foobar} and skips everything else. - out, err := dynvar.Resolve(in, func(path dyn.Path) (dyn.Value, error) { - if path.String() == "a" { - return dyn.V("${foobar}"), nil - } - return dyn.InvalidValue, dynvar.ErrSkipResolution - }) - - require.NoError(t, err) - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "${foobar}", getByPath(t, out, "b").MustString()) -} - -func TestResolveWithInterpolateAliasedRef(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("${a}"), - "c": dyn.V("${x}"), - }) - - // The call replaces ${x} with ${b} and skips everything else. - out, err := dynvar.Resolve(in, func(path dyn.Path) (dyn.Value, error) { - if path.String() == "x" { - return dyn.V("${b}"), nil - } - return dyn.GetByPath(in, path) - }) - - require.NoError(t, err) - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "a", getByPath(t, out, "b").MustString()) - assert.Equal(t, "a", getByPath(t, out, "c").MustString()) -} - -func TestResolveIndexedRefs(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "slice": dyn.V([]dyn.Value{dyn.V("a"), dyn.V("b")}), - "a": dyn.V("a: ${slice[0]}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a: a", getByPath(t, out, "a").MustString()) -} - -func TestResolveIndexedRefsFromMap(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "map": dyn.V( - map[string]dyn.Value{ - "slice": dyn.V([]dyn.Value{dyn.V("a")}), - }), - "a": dyn.V("a: ${map.slice[0]}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a: a", getByPath(t, out, "a").MustString()) -} - -func TestResolveMapFieldFromIndexedRefs(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "map": dyn.V( - map[string]dyn.Value{ - "slice": dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "value": dyn.V("a"), - }), - }), - }), - "a": dyn.V("a: ${map.slice[0].value}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a: a", getByPath(t, out, "a").MustString()) -} - -func TestResolveNestedIndexedRefs(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "slice": dyn.V([]dyn.Value{ - dyn.V([]dyn.Value{dyn.V("a")}), - }), - "a": dyn.V("a: ${slice[0][0]}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a: a", getByPath(t, out, "a").MustString()) -} - -func TestResolveMapVariable(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "map": dyn.V(map[string]dyn.Value{ - "key1": dyn.V("value1"), - "key2": dyn.V("value2"), - }), - "var": dyn.V("${map}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - // Verify that the map variable was interpolated correctly - mapVal := getByPath(t, out, "var") - _, ok := mapVal.AsMap() - require.True(t, ok, "expected map value") - - // Verify the map contents - assert.Equal(t, "value1", getByPath(t, mapVal, "key1").MustString()) - assert.Equal(t, "value2", getByPath(t, mapVal, "key2").MustString()) -} - -func TestResolveSequenceVariable(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "seq": dyn.V([]dyn.Value{ - dyn.V("value1"), - dyn.V("value2"), - }), - "var": dyn.V("${seq}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - // Verify that the sequence variable was interpolated correctly - seqVal := getByPath(t, out, "var") - seq, ok := seqVal.AsSequence() - require.True(t, ok, "expected sequence value") - require.Len(t, seq, 2) - - // Verify the sequence contents - assert.Equal(t, "value1", seq[0].MustString()) - assert.Equal(t, "value2", seq[1].MustString()) -} diff --git a/libs/dyn/jsonloader/json.go b/libs/dyn/jsonloader/json.go deleted file mode 100644 index 3161ef637b1..00000000000 --- a/libs/dyn/jsonloader/json.go +++ /dev/null @@ -1,133 +0,0 @@ -package jsonloader - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "io" - - "github.com/databricks/cli/libs/dyn" -) - -func LoadJSON(data []byte, source string) (dyn.Value, error) { - offset := BuildLineOffsets(data) - offset.SetSource(source) - - reader := bytes.NewReader(data) - decoder := json.NewDecoder(reader) - - // Use json.Number to avoid losing precision on int64 values above 2^53 (e.g. job and run IDs). - decoder.UseNumber() - - // Start decoding from the top-level value - value, err := decodeValue(decoder, &offset) - if err != nil { - if err == io.EOF { - err = errors.New("unexpected end of JSON input") - } - // Get the current offset for error reporting - errorOffset := decoder.InputOffset() - errorLocation := offset.GetPosition(errorOffset) - return dyn.InvalidValue, fmt.Errorf("error decoding JSON at %s: %v", errorLocation, err) - } - - // Check if there are any remaining tokens (should not be valid JSON) - if decoder.More() { - errorOffset := decoder.InputOffset() - errorLocation := offset.GetPosition(errorOffset) - return dyn.InvalidValue, fmt.Errorf("error decoding JSON at %s: unexpected additional content", errorLocation) - } - - return value, nil -} - -func decodeValue(decoder *json.Decoder, o *Offset) (dyn.Value, error) { - // Read the next JSON token - token, err := decoder.Token() - if err != nil { - return dyn.InvalidValue, err - } - - // Get the current byte offset and the location. - // We will later use this location to store the location of the value in the file - // For objects and arrays, we will store the location of the opening '{' or '[' - // For primitive types, we will store the location of the value itself (end of the value) - // We can't reliably calculate the beginning of the value for primitive types because - // the decoder doesn't provide the offset of the beginning of the value and the value might or might not be quoted. - offset := decoder.InputOffset() - location := o.GetPosition(offset) - - switch tok := token.(type) { - case json.Delim: - switch tok { - case '{': - location = o.GetPosition(offset - 1) - // Decode JSON object - obj := dyn.NewMapping() - for decoder.More() { - // Decode the key - keyToken, err := decoder.Token() - if err != nil { - return invalidValueWithLocation(decoder, o), err - } - key, ok := keyToken.(string) - if !ok { - return invalidValueWithLocation(decoder, o), errors.New("expected string for object key") - } - - // Get the offset of the key by subtracting the length of the key and the '"' character - keyOffset := decoder.InputOffset() - int64(len(key)+1) - loc := []dyn.Location{o.GetPosition(keyOffset)} - - // Decode the value recursively - val, err := decodeValue(decoder, o) - if err != nil { - return invalidValueWithLocation(decoder, o), err - } - - obj.SetLoc(key, loc, val) - } - // Consume the closing '}' - if _, err := decoder.Token(); err != nil { - return invalidValueWithLocation(decoder, o), err - } - return dyn.NewValue(obj, []dyn.Location{location}), nil - case '[': - location = o.GetPosition(offset - 1) - // Decode JSON array - var arr []dyn.Value - for decoder.More() { - val, err := decodeValue(decoder, o) - if err != nil { - return invalidValueWithLocation(decoder, o), err - } - arr = append(arr, val) - } - // Consume the closing ']' - if _, err := decoder.Token(); err != nil { - return invalidValueWithLocation(decoder, o), err - } - return dyn.NewValue(arr, []dyn.Location{location}), nil - } - case json.Number: - // Integers that overflow int64 fall back to float64, matching the decoder's behavior without UseNumber. - if i64, err := tok.Int64(); err == nil { - return dyn.NewValue(i64, []dyn.Location{location}), nil - } - f64, err := tok.Float64() - if err != nil { - return invalidValueWithLocation(decoder, o), fmt.Errorf("invalid number %q: %w", tok.String(), err) - } - return dyn.NewValue(f64, []dyn.Location{location}), nil - default: - return dyn.NewValue(tok, []dyn.Location{location}), nil - } - - return invalidValueWithLocation(decoder, o), fmt.Errorf("unexpected token: %v", token) -} - -func invalidValueWithLocation(decoder *json.Decoder, o *Offset) dyn.Value { - location := o.GetPosition(decoder.InputOffset()) - return dyn.InvalidValue.WithLocations([]dyn.Location{location}) -} diff --git a/libs/dyn/jsonloader/json_test.go b/libs/dyn/jsonloader/json_test.go deleted file mode 100644 index 1de43c933f1..00000000000 --- a/libs/dyn/jsonloader/json_test.go +++ /dev/null @@ -1,161 +0,0 @@ -package jsonloader - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/databricks-sdk-go/service/jobs" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -const jsonData = ` -{ - "job_id": 123, - "new_settings": { - "name": "xxx", - "email_notifications": { - "on_start": [], - "on_success": [], - "on_failure": [] - }, - "webhook_notifications": { - "on_start": [], - "on_failure": [] - }, - "notification_settings": { - "no_alert_for_skipped_runs": true, - "no_alert_for_canceled_runs": true - }, - "timeout_seconds": 0, - "max_concurrent_runs": 1, - "tasks": [ - { - "task_key": "xxx", - "email_notifications": {}, - "notification_settings": {}, - "timeout_seconds": 0, - "max_retries": 0, - "min_retry_interval_millis": 0, - "retry_on_timeout": "true" - } - ] - } -} -` - -func TestJsonLoader(t *testing.T) { - v, err := LoadJSON([]byte(jsonData), "(inline)") - assert.NoError(t, err) - - var r jobs.ResetJob - err = convert.ToTyped(&r, v) - assert.NoError(t, err) -} - -const malformedMap = ` -{ - "job_id": 123, - "new_settings": { - "name": "xxx", - "wrong", - } -} -` - -func TestJsonLoaderMalformedMap(t *testing.T) { - _, err := LoadJSON([]byte(malformedMap), "(inline)") - assert.ErrorContains(t, err, "error decoding JSON at (inline):6:16: invalid character ',' after object key") -} - -const malformedArray = ` -{ - "job_id": 123, - "new_settings": { - "name": "xxx", - "tasks": [1, "asd",] - } -}` - -func TestJsonLoaderMalformedArray(t *testing.T) { - _, err := LoadJSON([]byte(malformedArray), "path/to/file.json") - assert.ErrorContains(t, err, "error decoding JSON at path/to/file.json:6:28: invalid character ']' looking for beginning of value") -} - -const eofData = ` -{ - "job_id": 123, - "new_settings": { - "name": "xxx",` - -func TestJsonLoaderEOF(t *testing.T) { - _, err := LoadJSON([]byte(eofData), "path/to/file.json") - assert.ErrorContains(t, err, "unexpected end of JSON input") -} - -const mapWithNoBraces = ` -"job_id": 123, -"new_settings": { - "name": "xxx", - "wrong": "xxx", -} -` - -func TestJsonMapWithoutBraces(t *testing.T) { - _, err := LoadJSON([]byte(mapWithNoBraces), "path/to/file.json") - assert.ErrorContains(t, err, "error decoding JSON at") -} - -const validInline = `["job_id", 123]` - -func TestJsonValidInline(t *testing.T) { - _, err := LoadJSON([]byte(validInline), "path/to/file.json") - assert.NoError(t, err) -} - -func TestJsonLoaderNumbers(t *testing.T) { - for _, tc := range []struct { - input string - expected any - }{ - {`123`, int64(123)}, - {`-1`, int64(-1)}, - {`123456789012345678`, int64(123456789012345678)}, - {`-123456789012345678`, int64(-123456789012345678)}, - {`9223372036854775807`, int64(9223372036854775807)}, - {`2.0`, 2.0}, - {`2.5`, 2.5}, - {`1e3`, 1000.0}, - {`18446744073709551615`, 1.8446744073709552e+19}, - } { - v, err := LoadJSON([]byte(tc.input), "(inline)") - assert.NoError(t, err, tc.input) - assert.Equal(t, tc.expected, v.AsAny(), tc.input) - } -} - -func TestJsonLoaderNumberOutOfRange(t *testing.T) { - _, err := LoadJSON([]byte(`1e400`), "(inline)") - assert.ErrorContains(t, err, "value out of range") -} - -const mixedNumbersData = ` -{ - "job_id": 123456789012345678, - "new_settings": { - "name": "xxx", - "timeout_seconds": 100 - } -} -` - -func TestJsonLoaderMixedNumbersToTyped(t *testing.T) { - v, err := LoadJSON([]byte(mixedNumbersData), "(inline)") - require.NoError(t, err) - - var r jobs.ResetJob - err = convert.ToTyped(&r, v) - require.NoError(t, err) - assert.Equal(t, int64(123456789012345678), r.JobId) - assert.Equal(t, 100, r.NewSettings.TimeoutSeconds) -} diff --git a/libs/dyn/jsonloader/locations.go b/libs/dyn/jsonloader/locations.go deleted file mode 100644 index d69c3c5c861..00000000000 --- a/libs/dyn/jsonloader/locations.go +++ /dev/null @@ -1,49 +0,0 @@ -package jsonloader - -import ( - "sort" - - "github.com/databricks/cli/libs/dyn" -) - -type LineOffset struct { - Line int - Start int64 -} - -type Offset struct { //nolint:recvcheck // value receiver for read-only GetPosition, pointer for SetSource - offsets []LineOffset - source string -} - -// buildLineOffsets scans the input data and records the starting byte offset of each line. -func BuildLineOffsets(data []byte) Offset { - offsets := []LineOffset{{Line: 1, Start: 0}} - line := 1 - for i, b := range data { - if b == '\n' { - line++ - offsets = append(offsets, LineOffset{Line: line, Start: int64(i + 1)}) - } - } - return Offset{offsets: offsets} -} - -// GetPosition maps a byte offset to its corresponding line and column numbers. -func (o Offset) GetPosition(offset int64) dyn.Location { - // Binary search to find the line - idx := max(sort.Search(len(o.offsets), func(i int) bool { - return o.offsets[i].Start > offset - })-1, 0) - - lineOffset := o.offsets[idx] - return dyn.Location{ - File: o.source, - Line: lineOffset.Line, - Column: int(offset-lineOffset.Start) + 1, - } -} - -func (o *Offset) SetSource(source string) { - o.source = source -} diff --git a/libs/dyn/kind_test.go b/libs/dyn/kind_test.go deleted file mode 100644 index 84c90713fb8..00000000000 --- a/libs/dyn/kind_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package dyn_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestKindZeroValue(t *testing.T) { - // Assert that the zero value of [dyn.Kind] is the invalid kind. - var k dyn.Kind - assert.Equal(t, dyn.KindInvalid, k) -} - -func TestKindToString(t *testing.T) { - for _, tt := range []struct { - k dyn.Kind - s string - }{ - {dyn.KindInvalid, "invalid"}, - {dyn.KindMap, "map"}, - {dyn.KindSequence, "sequence"}, - {dyn.KindString, "string"}, - {dyn.KindBool, "bool"}, - {dyn.KindInt, "int"}, - {dyn.KindFloat, "float"}, - {dyn.KindTime, "time"}, - {dyn.KindNil, "nil"}, - } { - assert.Equal(t, tt.s, tt.k.String()) - } - - // Panic on unknown kind. - assert.PanicsWithValue(t, "invalid kind value: 100", func() { - _ = dyn.Kind(100).String() - }) -} diff --git a/libs/dyn/mapping.go b/libs/dyn/mapping.go deleted file mode 100644 index e7e1bbbc670..00000000000 --- a/libs/dyn/mapping.go +++ /dev/null @@ -1,156 +0,0 @@ -package dyn - -import ( - "maps" - "slices" -) - -// Pair represents a single key-value pair in a Mapping. -type Pair struct { - Key Value - Value Value -} - -// Mapping represents a key-value map of dynamic values. -// It exists because plain Go maps cannot use dynamic values for keys. -// We need to use dynamic values for keys because it lets us associate metadata -// with keys (i.e. their definition location). Keys must be strings. -type Mapping struct { //nolint:recvcheck // value receivers for read-only accessors, pointer for mutators - pairs []Pair - index map[string]int -} - -// NewMapping creates a new empty Mapping. -func NewMapping() Mapping { - return Mapping{ - pairs: make([]Pair, 0), - index: make(map[string]int), - } -} - -// NewMappingFromPairs computes a [Mapping] from a list of [Pair]s. The index -// map does not need to be provided since that will be computed from the -// key-value pairs provided. -func NewMappingFromPairs(pairs []Pair) Mapping { - index := make(map[string]int) - for i, p := range pairs { - index[p.Key.MustString()] = i - } - - return Mapping{ - pairs: pairs, - index: index, - } -} - -// newMappingWithSize creates a new Mapping preallocated to the specified size. -func newMappingWithSize(size int) Mapping { - return Mapping{ - pairs: make([]Pair, 0, size), - index: make(map[string]int, size), - } -} - -// newMappingFromGoMap creates a new Mapping from a Go map of string keys and dynamic values. -func newMappingFromGoMap(vin map[string]Value) Mapping { - m := newMappingWithSize(len(vin)) - for k, v := range vin { - m.SetLoc(k, nil, v) - } - return m -} - -// Pairs returns all the key-value pairs in the Mapping. The pairs are sorted by -// their key in lexicographic order. -func (m Mapping) Pairs() []Pair { - return m.pairs -} - -// Len returns the number of key-value pairs in the Mapping. -func (m Mapping) Len() int { - return len(m.pairs) -} - -// GetPair returns the key-value pair with the specified key. -// It also returns a boolean indicating whether the pair was found. -func (m Mapping) GetPair(key Value) (Pair, bool) { - skey, ok := key.AsString() - if !ok { - return Pair{}, false - } - return m.GetPairByString(skey) -} - -// GetPairByString returns the key-value pair with the specified string key. -// It also returns a boolean indicating whether the pair was found. -func (m Mapping) GetPairByString(skey string) (Pair, bool) { - if i, ok := m.index[skey]; ok { - return m.pairs[i], true - } - return Pair{}, false -} - -// Get returns the value associated with the specified key. -// It also returns a boolean indicating whether the value was found. -func (m Mapping) Get(key Value) (Value, bool) { - p, ok := m.GetPair(key) - return p.Value, ok -} - -// GetByString returns the value associated with the specified string key. -// It also returns a boolean indicating whether the value was found. -func (m *Mapping) GetByString(skey string) (Value, bool) { - p, ok := m.GetPairByString(skey) - return p.Value, ok -} - -// Set sets the value for the given key in the mapping. -// If the key already exists, the value is updated. The location loc is ignored. -// If the key does not exist, a new key-value pair is added. -func (m *Mapping) SetLoc(skey string, loc []Location, value Value) { - // If the key already exists, update the value. - if i, ok := m.index[skey]; ok { - m.pairs[i].Value = value - return - } - - // Otherwise, add a new pair. - m.pairs = append(m.pairs, Pair{NewValue(skey, loc), value}) - if m.index == nil { - m.index = make(map[string]int) - } - m.index[skey] = len(m.pairs) - 1 -} - -// Keys returns all the keys in the Mapping. -func (m Mapping) Keys() []Value { - keys := make([]Value, 0, len(m.pairs)) - for _, p := range m.pairs { - keys = append(keys, p.Key) - } - return keys -} - -// Values returns all the values in the Mapping. -func (m Mapping) Values() []Value { - values := make([]Value, 0, len(m.pairs)) - for _, p := range m.pairs { - values = append(values, p.Value) - } - return values -} - -// Clone creates a shallow copy of the Mapping. -func (m Mapping) Clone() Mapping { - return Mapping{ - pairs: slices.Clone(m.pairs), - index: maps.Clone(m.index), - } -} - -// Merge merges the key-value pairs from another Mapping into the current Mapping. -func (m *Mapping) Merge(n Mapping) { - for _, p := range n.pairs { - m.SetLoc(p.Key.MustString(), p.Key.Locations(), p.Value) - } -} diff --git a/libs/dyn/mapping_test.go b/libs/dyn/mapping_test.go deleted file mode 100644 index f646ab7c1d1..00000000000 --- a/libs/dyn/mapping_test.go +++ /dev/null @@ -1,182 +0,0 @@ -package dyn_test - -import ( - "strconv" - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestNewMapping(t *testing.T) { - m := dyn.NewMapping() - assert.Equal(t, 0, m.Len()) -} - -func TestMappingZeroValue(t *testing.T) { - var m dyn.Mapping - assert.Equal(t, 0, m.Len()) - - value, ok := m.Get(dyn.V("key")) - assert.Equal(t, dyn.InvalidValue, value) - assert.False(t, ok) - assert.Empty(t, m.Keys()) - assert.Empty(t, m.Values()) -} - -func TestMappingGet(t *testing.T) { - var m dyn.Mapping - m.SetLoc("key", nil, dyn.V("value")) - assert.Equal(t, 1, m.Len()) - - // Call GetPair - p, ok := m.GetPair(dyn.V("key")) - assert.True(t, ok) - assert.Equal(t, dyn.V("key"), p.Key) - assert.Equal(t, dyn.V("value"), p.Value) - - // Modify the value to make sure we're not getting a reference - p.Value = dyn.V("newvalue") - - // Call GetPair with invalid key - p, ok = m.GetPair(dyn.V(1234)) - assert.False(t, ok) - assert.Equal(t, dyn.InvalidValue, p.Key) - assert.Equal(t, dyn.InvalidValue, p.Value) - - // Call GetPair with non-existent key - p, ok = m.GetPair(dyn.V("enoexist")) - assert.False(t, ok) - assert.Equal(t, dyn.InvalidValue, p.Key) - assert.Equal(t, dyn.InvalidValue, p.Value) - - // Call GetPairByString - p, ok = m.GetPairByString("key") - assert.True(t, ok) - assert.Equal(t, dyn.V("key"), p.Key) - assert.Equal(t, dyn.V("value"), p.Value) - - // Modify the value to make sure we're not getting a reference - p.Value = dyn.V("newvalue") - - // Call GetPairByString with non-existent key - p, ok = m.GetPairByString("enoexist") - assert.False(t, ok) - assert.Equal(t, dyn.InvalidValue, p.Key) - assert.Equal(t, dyn.InvalidValue, p.Value) - - // Call Get - value, ok := m.Get(dyn.V("key")) - assert.True(t, ok) - assert.Equal(t, dyn.V("value"), value) - - // Call Get with invalid key - value, ok = m.Get(dyn.V(1234)) - assert.False(t, ok) - assert.Equal(t, dyn.InvalidValue, value) - - // Call Get with non-existent key - value, ok = m.Get(dyn.V("enoexist")) - assert.False(t, ok) - assert.Equal(t, dyn.InvalidValue, value) - - // Call GetByString - value, ok = m.GetByString("key") - assert.True(t, ok) - assert.Equal(t, dyn.V("value"), value) - - // Call GetByString with non-existent key - value, ok = m.GetByString("enoexist") - assert.False(t, ok) - assert.Equal(t, dyn.InvalidValue, value) -} - -func TestMappingSetLoc(t *testing.T) { - var m dyn.Mapping - - // Set a value - m.SetLoc("key1", nil, dyn.V("foo")) - assert.Equal(t, 1, m.Len()) - - // Confirm the value - value, ok := m.GetByString("key1") - assert.True(t, ok) - assert.Equal(t, dyn.V("foo"), value) - - // Set another value - m.SetLoc("key2", nil, dyn.V("bar")) - assert.Equal(t, 2, m.Len()) - - // Confirm the value - value, ok = m.Get(dyn.V("key2")) - assert.True(t, ok) - assert.Equal(t, dyn.V("bar"), value) - - // Overwrite first value - m.SetLoc("key1", nil, dyn.V("qux")) - assert.Equal(t, 2, m.Len()) - - // Confirm the value - value, ok = m.Get(dyn.V("key1")) - assert.True(t, ok) - assert.Equal(t, dyn.V("qux"), value) -} - -func TestMappingKeysValues(t *testing.T) { - // Configure mapping - var m dyn.Mapping - m.SetLoc("key1", nil, dyn.V("foo")) - m.SetLoc("key2", nil, dyn.V("bar")) - - // Confirm keys - keys := m.Keys() - assert.Len(t, keys, 2) - assert.Contains(t, keys, dyn.V("key1")) - assert.Contains(t, keys, dyn.V("key2")) - - // Confirm values - values := m.Values() - assert.Len(t, values, 2) - assert.Contains(t, values, dyn.V("foo")) - assert.Contains(t, values, dyn.V("bar")) -} - -func TestMappingClone(t *testing.T) { - // Configure mapping - var m1 dyn.Mapping - m1.SetLoc("key1", nil, dyn.V("foo")) - m1.SetLoc("key2", nil, dyn.V("bar")) - - // Clone mapping - m2 := m1.Clone() - assert.Equal(t, m1.Len(), m2.Len()) - - // Modify original mapping - m1.SetLoc("key1", nil, dyn.V("qux")) - - // Confirm values - value, ok := m1.Get(dyn.V("key1")) - assert.True(t, ok) - assert.Equal(t, dyn.V("qux"), value) - value, ok = m2.Get(dyn.V("key1")) - assert.True(t, ok) - assert.Equal(t, dyn.V("foo"), value) -} - -func TestMappingMerge(t *testing.T) { - var m1 dyn.Mapping - for i := range 10 { - m1.SetLoc(strconv.Itoa(i), nil, dyn.V(i)) - } - - var m2 dyn.Mapping - for i := 5; i < 15; i++ { - m2.SetLoc(strconv.Itoa(i), nil, dyn.V(i)) - } - - var out dyn.Mapping - out.Merge(m1) - assert.Equal(t, 10, out.Len()) - out.Merge(m2) - assert.Equal(t, 15, out.Len()) -} diff --git a/libs/dyn/merge/elements_by_key.go b/libs/dyn/merge/elements_by_key.go deleted file mode 100644 index 6bf71d41ae5..00000000000 --- a/libs/dyn/merge/elements_by_key.go +++ /dev/null @@ -1,107 +0,0 @@ -package merge - -import ( - "slices" - - "github.com/databricks/cli/libs/dyn" -) - -type elementsByKey struct { - key string - keyFunc func(dyn.Value) string - sortKeys bool -} - -func (e elementsByKey) doMap(_ dyn.Path, v dyn.Value, mergeFunc func(a, b dyn.Value) (dyn.Value, error)) (dyn.Value, error) { - // We know the type of this value is a sequence. - // For additional defence, return self if it is not. - elements, ok := v.AsSequence() - if !ok { - return v, nil - } - - seen := make(map[string]dyn.Value, len(elements)) - keys := make([]string, 0, len(elements)) - - // Iterate in natural order. For a given key, we first see the - // base definition and merge instances that come after it. - for i := range elements { - kv := elements[i].Get(e.key) - key := e.keyFunc(kv) - - // Register element with key if not yet seen before. - ref, ok := seen[key] - if !ok { - keys = append(keys, key) - seen[key] = elements[i] - continue - } - - // Merge this instance into the reference. - nv, err := mergeFunc(ref, elements[i]) - if err != nil { - return v, err - } - - // Overwrite reference. - seen[key] = nv - } - - if e.sortKeys { - slices.Sort(keys) - } - - // Gather resulting elements in natural order. - out := make([]dyn.Value, 0, len(keys)) - for _, key := range keys { - // Preserve the location from the original key value so that - // downstream code can tell this field was defined in the YAML. - keyLoc := seen[key].Get(e.key).Locations() - nv, err := dyn.Set(seen[key], e.key, dyn.NewValue(key, keyLoc)) - if err != nil { - return dyn.InvalidValue, err - } - out = append(out, nv) - } - - return dyn.NewValue(out, v.Locations()), nil -} - -func (e elementsByKey) Map(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return e.doMap(nil, v, Merge) -} - -func (e elementsByKey) MapWithOverride(p dyn.Path, v dyn.Value) (dyn.Value, error) { - return e.doMap(nil, v, func(a, b dyn.Value) (dyn.Value, error) { - return Override(a, b, OverrideVisitor{ - VisitInsert: func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return v, nil - }, - VisitDelete: func(valuePath dyn.Path, left dyn.Value) error { - return nil - }, - VisitUpdate: func(_ dyn.Path, a, b dyn.Value) (dyn.Value, error) { - return b, nil - }, - }) - }) -} - -// ElementsByKey returns a [dyn.MapFunc] that operates on a sequence -// where each element is a map. It groups elements by a key and merges -// elements with the same key. -// -// The function that extracts the key from an element is provided as -// a parameter. The resulting elements get their key field overwritten -// with the value as returned by the key function. -func ElementsByKey(key string, keyFunc func(dyn.Value) string) dyn.MapFunc { - return elementsByKey{key, keyFunc, false}.Map -} - -func ElementsBySortedKey(key string, keyFunc func(dyn.Value) string) dyn.MapFunc { - return elementsByKey{key, keyFunc, true}.Map -} - -func ElementsByKeyWithOverride(key string, keyFunc func(dyn.Value) string) dyn.MapFunc { - return elementsByKey{key, keyFunc, false}.MapWithOverride -} diff --git a/libs/dyn/merge/elements_by_key_test.go b/libs/dyn/merge/elements_by_key_test.go deleted file mode 100644 index 1cc873636f9..00000000000 --- a/libs/dyn/merge/elements_by_key_test.go +++ /dev/null @@ -1,108 +0,0 @@ -package merge - -import ( - "strings" - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestElementByKey(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "key": dyn.V("foo"), - "value": dyn.V(42), - }), - dyn.V(map[string]dyn.Value{ - "key": dyn.V("bar"), - "value": dyn.V(43), - }), - dyn.V(map[string]dyn.Value{ - // Use upper case key to test that the resulting element has its - // key field assigned to the output of the key function. - // The key function in this test returns the lower case version of the key. - "key": dyn.V("FOO"), - "value": dyn.V(44), - }), - }) - - keyFunc := func(v dyn.Value) string { - return strings.ToLower(v.MustString()) - } - - vout, err := dyn.MapByPath(vin, dyn.EmptyPath, ElementsByKey("key", keyFunc)) - require.NoError(t, err) - assert.Len(t, vout.MustSequence(), 2) - assert.Equal(t, - map[string]any{ - "key": "foo", - "value": 44, - }, vout.Index(0).AsAny(), - ) - assert.Equal(t, - map[string]any{ - "key": "bar", - "value": 43, - }, vout.Index(1).AsAny(), - ) -} - -func TestElementByKeyPreservesLocations(t *testing.T) { - loc := dyn.Location{File: "config.yml", Line: 10, Column: 5} - vin := dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "key": dyn.NewValue("foo", []dyn.Location{loc}), - "value": dyn.V(42), - }), - }) - - keyFunc := func(v dyn.Value) string { - return v.MustString() - } - - vout, err := dyn.MapByPath(vin, dyn.EmptyPath, ElementsByKey("key", keyFunc)) - require.NoError(t, err) - - // Verify the key field retains its original location. - keyValue := vout.Index(0).Get("key") - assert.Equal(t, loc, keyValue.Location()) -} - -func TestElementByKeyWithOverride(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "key": dyn.V("foo"), - "value": dyn.V(42), - }), - dyn.V(map[string]dyn.Value{ - "key": dyn.V("bar"), - "value": dyn.V(43), - }), - dyn.V(map[string]dyn.Value{ - "key": dyn.V("foo"), - "othervalue": dyn.V(44), - }), - }) - - keyFunc := func(v dyn.Value) string { - return strings.ToLower(v.MustString()) - } - - vout, err := dyn.MapByPath(vin, dyn.EmptyPath, ElementsByKeyWithOverride("key", keyFunc)) - require.NoError(t, err) - assert.Len(t, vout.MustSequence(), 2) - assert.Equal(t, - map[string]any{ - "key": "foo", - "othervalue": 44, - }, vout.Index(0).AsAny(), - ) - assert.Equal(t, - map[string]any{ - "key": "bar", - "value": 43, - }, vout.Index(1).AsAny(), - ) -} diff --git a/libs/dyn/merge/merge.go b/libs/dyn/merge/merge.go deleted file mode 100644 index 374450ef240..00000000000 --- a/libs/dyn/merge/merge.go +++ /dev/null @@ -1,119 +0,0 @@ -package merge - -import ( - "fmt" - - "github.com/databricks/cli/libs/dyn" -) - -// Merge recursively merges the specified values. -// -// Semantics are as follows: -// * Merging x with nil or nil with x always yields x. -// * Merging maps a and b means entries from map b take precedence. -// * Merging sequences a and b means concatenating them. -// -// Merging retains and accumulates the locations metadata associated with the values. -// This allows users of the module to track the provenance of values across merging of -// configuration trees, which is useful for reporting errors and warnings. -// -// Semantics for location metadata in the merged value are similar to the semantics -// for the values themselves: -// -// - When merging x with nil or nil with x, the location of x is retained. -// -// - When merging maps or sequences, the combined value retains the location of a and -// accumulates the location of b. The individual elements of the map or sequence retain -// their original locations, i.e., whether they were originally defined in a or b. -// -// The rationale for retaining location of a is that we would like to return -// the first location a bit of configuration showed up when reporting errors and warnings. -// -// - Merging primitive values means using the incoming value `b`. The location of the -// incoming value is retained and the location of the existing value `a` is accumulated. -// This is because the incoming value overwrites the existing value. -func Merge(a, b dyn.Value) (dyn.Value, error) { - return merge(a, b) -} - -func merge(a, b dyn.Value) (dyn.Value, error) { - ak := a.Kind() - bk := b.Kind() - - // If a is nil, return b. - if ak == dyn.KindNil { - return b.AppendLocationsFromValue(a), nil - } - - // If b is nil, return a. - if bk == dyn.KindNil { - return a.AppendLocationsFromValue(b), nil - } - - // Call the appropriate merge function based on the kind of a and b. - switch ak { - case dyn.KindMap: - if bk != dyn.KindMap { - return dyn.InvalidValue, fmt.Errorf("cannot merge map with %s", bk) - } - return mergeMap(a, b) - case dyn.KindSequence: - if bk != dyn.KindSequence { - return dyn.InvalidValue, fmt.Errorf("cannot merge sequence with %s", bk) - } - return mergeSequence(a, b) - default: - if ak != bk { - return dyn.InvalidValue, fmt.Errorf("cannot merge %s with %s", ak, bk) - } - return mergePrimitive(a, b) - } -} - -func mergeMap(a, b dyn.Value) (dyn.Value, error) { - out := dyn.NewMapping() - am := a.MustMap() - bm := b.MustMap() - - // Add the values from a into the output map. - out.Merge(am) - - // Merge the values from b into the output map. - for _, pair := range bm.Pairs() { - pk := pair.Key - key := pk.MustString() - pv := pair.Value - if ov, ok := out.Get(pk); ok { - // If the key already exists, merge the values. - merged, err := merge(ov, pv) - if err != nil { - return dyn.InvalidValue, err - } - out.SetLoc(key, pair.Key.Locations(), merged) - } else { - // Otherwise, just set the value. - out.SetLoc(key, pair.Key.Locations(), pv) - } - } - - // Preserve the location of the first value. Accumulate the locations of the second value. - return dyn.NewValue(out, a.Locations()).AppendLocationsFromValue(b), nil -} - -func mergeSequence(a, b dyn.Value) (dyn.Value, error) { - as := a.MustSequence() - bs := b.MustSequence() - - // Merging sequences means concatenating them. - out := make([]dyn.Value, len(as)+len(bs)) - copy(out[:], as) - copy(out[len(as):], bs) - - // Preserve the location of the first value. Accumulate the locations of the second value. - return dyn.NewValue(out, a.Locations()).AppendLocationsFromValue(b), nil -} - -func mergePrimitive(a, b dyn.Value) (dyn.Value, error) { - // Merging primitive values means using the incoming value. - return b.AppendLocationsFromValue(a), nil -} diff --git a/libs/dyn/merge/merge_test.go b/libs/dyn/merge/merge_test.go deleted file mode 100644 index 43e0e08c182..00000000000 --- a/libs/dyn/merge/merge_test.go +++ /dev/null @@ -1,300 +0,0 @@ -package merge - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestMergeMaps(t *testing.T) { - l1 := dyn.Location{File: "file1", Line: 1, Column: 2} - v1 := dyn.NewValue(map[string]dyn.Value{ - "foo": dyn.NewValue("bar", []dyn.Location{l1}), - "bar": dyn.NewValue("baz", []dyn.Location{l1}), - }, []dyn.Location{l1}) - - l2 := dyn.Location{File: "file2", Line: 3, Column: 4} - v2 := dyn.NewValue(map[string]dyn.Value{ - "bar": dyn.NewValue("qux", []dyn.Location{l2}), - "qux": dyn.NewValue("foo", []dyn.Location{l2}), - }, []dyn.Location{l2}) - - // Merge v2 into v1. - { - out, err := Merge(v1, v2) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": "bar", - "bar": "qux", - "qux": "foo", - }, out.AsAny()) - - // Locations of both values should be preserved. - assert.Equal(t, []dyn.Location{l1, l2}, out.Locations()) - assert.Equal(t, []dyn.Location{l2, l1}, out.Get("bar").Locations()) - assert.Equal(t, []dyn.Location{l1}, out.Get("foo").Locations()) - assert.Equal(t, []dyn.Location{l2}, out.Get("qux").Locations()) - - // Location of the merged value should be the location of v1. - assert.Equal(t, l1, out.Location()) - - // Value of bar is "qux" which comes from v2. This .Location() should - // return the location of v2. - assert.Equal(t, l2, out.Get("bar").Location()) - - // Original locations of keys that were not overwritten should be preserved. - assert.Equal(t, l1, out.Get("foo").Location()) - assert.Equal(t, l2, out.Get("qux").Location()) - } - - // Merge v1 into v2. - { - out, err := Merge(v2, v1) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": "bar", - "bar": "baz", - "qux": "foo", - }, out.AsAny()) - - // Locations of both values should be preserved. - assert.Equal(t, []dyn.Location{l2, l1}, out.Locations()) - assert.Equal(t, []dyn.Location{l1, l2}, out.Get("bar").Locations()) - assert.Equal(t, []dyn.Location{l1}, out.Get("foo").Locations()) - assert.Equal(t, []dyn.Location{l2}, out.Get("qux").Locations()) - - // Location of the merged value should be the location of v2. - assert.Equal(t, l2, out.Location()) - - // Value of bar is "baz" which comes from v1. This .Location() should - // return the location of v1. - assert.Equal(t, l1, out.Get("bar").Location()) - - // Original locations of keys that were not overwritten should be preserved. - assert.Equal(t, l1, out.Get("foo").Location()) - assert.Equal(t, l2, out.Get("qux").Location()) - } -} - -func TestMergeMapsNil(t *testing.T) { - l := dyn.Location{File: "file", Line: 1, Column: 2} - v := dyn.NewValue(map[string]dyn.Value{ - "foo": dyn.V("bar"), - }, []dyn.Location{l}) - - nilL := dyn.Location{File: "file", Line: 3, Column: 4} - nilV := dyn.NewValue(nil, []dyn.Location{nilL}) - - // Merge nil into v. - { - out, err := Merge(v, nilV) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": "bar", - }, out.AsAny()) - - // Locations of both values should be preserved. - assert.Equal(t, []dyn.Location{l, nilL}, out.Locations()) - - // Location of the non-nil value should be returned by .Location(). - assert.Equal(t, l, out.Location()) - } - - // Merge v into nil. - { - out, err := Merge(nilV, v) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": "bar", - }, out.AsAny()) - - // Locations of both values should be preserved. - assert.Equal(t, []dyn.Location{l, nilL}, out.Locations()) - - // Location of the non-nil value should be returned by .Location(). - assert.Equal(t, l, out.Location()) - } -} - -func TestMergeMapsError(t *testing.T) { - v := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - }) - - other := dyn.V("string") - - // Merge a string into v. - { - out, err := Merge(v, other) - assert.EqualError(t, err, "cannot merge map with string") - assert.Equal(t, dyn.InvalidValue, out) - } -} - -func TestMergeSequences(t *testing.T) { - l1 := dyn.Location{File: "file1", Line: 1, Column: 2} - v1 := dyn.NewValue([]dyn.Value{ - dyn.NewValue("bar", []dyn.Location{l1}), - dyn.NewValue("baz", []dyn.Location{l1}), - }, []dyn.Location{l1}) - - l2 := dyn.Location{File: "file2", Line: 3, Column: 4} - l3 := dyn.Location{File: "file3", Line: 5, Column: 6} - v2 := dyn.NewValue([]dyn.Value{ - dyn.NewValue("qux", []dyn.Location{l2}), - dyn.NewValue("foo", []dyn.Location{l3}), - }, []dyn.Location{l2, l3}) - - // Merge v2 into v1. - { - out, err := Merge(v1, v2) - assert.NoError(t, err) - assert.Equal(t, []any{ - "bar", - "baz", - "qux", - "foo", - }, out.AsAny()) - - // Locations of both values should be preserved. - assert.Equal(t, []dyn.Location{l1, l2, l3}, out.Locations()) - - // Location of the merged value should be the location of v1. - assert.Equal(t, l1, out.Location()) - - // Location of the individual values should be preserved. - assert.Equal(t, l1, out.Index(0).Location()) // "bar" - assert.Equal(t, l1, out.Index(1).Location()) // "baz" - assert.Equal(t, l2, out.Index(2).Location()) // "qux" - assert.Equal(t, l3, out.Index(3).Location()) // "foo" - } - - // Merge v1 into v2. - { - out, err := Merge(v2, v1) - assert.NoError(t, err) - assert.Equal(t, []any{ - "qux", - "foo", - "bar", - "baz", - }, out.AsAny()) - - // Locations of both values should be preserved. - assert.Equal(t, []dyn.Location{l2, l3, l1}, out.Locations()) - - // Location of the merged value should be the location of v2. - assert.Equal(t, l2, out.Location()) - - // Location of the individual values should be preserved. - assert.Equal(t, l2, out.Index(0).Location()) // "qux" - assert.Equal(t, l3, out.Index(1).Location()) // "foo" - assert.Equal(t, l1, out.Index(2).Location()) // "bar" - assert.Equal(t, l1, out.Index(3).Location()) // "baz" - } -} - -func TestMergeSequencesNil(t *testing.T) { - v := dyn.V([]dyn.Value{ - dyn.V("bar"), - }) - - // Merge nil into v. - { - out, err := Merge(v, dyn.NilValue) - assert.NoError(t, err) - assert.Equal(t, []any{ - "bar", - }, out.AsAny()) - } - - // Merge v into nil. - { - out, err := Merge(dyn.NilValue, v) - assert.NoError(t, err) - assert.Equal(t, []any{ - "bar", - }, out.AsAny()) - } -} - -func TestMergeSequencesError(t *testing.T) { - v := dyn.V([]dyn.Value{ - dyn.V("bar"), - }) - - other := dyn.V("string") - - // Merge a string into v. - { - out, err := Merge(v, other) - assert.EqualError(t, err, "cannot merge sequence with string") - assert.Equal(t, dyn.InvalidValue, out) - } -} - -func TestMergePrimitives(t *testing.T) { - l1 := dyn.Location{File: "file1", Line: 1, Column: 2} - l2 := dyn.Location{File: "file2", Line: 3, Column: 4} - v1 := dyn.NewValue("bar", []dyn.Location{l1}) - v2 := dyn.NewValue("baz", []dyn.Location{l2}) - - // Merge v2 into v1. - { - out, err := Merge(v1, v2) - assert.NoError(t, err) - assert.Equal(t, "baz", out.AsAny()) - - // Locations of both values should be preserved. - assert.Equal(t, []dyn.Location{l2, l1}, out.Locations()) - - // Location of the merged value should be the location of v2, the second value. - assert.Equal(t, l2, out.Location()) - } - - // Merge v1 into v2. - { - out, err := Merge(v2, v1) - assert.NoError(t, err) - assert.Equal(t, "bar", out.AsAny()) - - // Locations of both values should be preserved. - assert.Equal(t, []dyn.Location{l1, l2}, out.Locations()) - - // Location of the merged value should be the location of v1, the second value. - assert.Equal(t, l1, out.Location()) - } -} - -func TestMergePrimitivesNil(t *testing.T) { - v := dyn.V("bar") - - // Merge nil into v. - { - out, err := Merge(v, dyn.NilValue) - assert.NoError(t, err) - assert.Equal(t, "bar", out.AsAny()) - } - - // Merge v into nil. - { - out, err := Merge(dyn.NilValue, v) - assert.NoError(t, err) - assert.Equal(t, "bar", out.AsAny()) - } -} - -func TestMergePrimitivesError(t *testing.T) { - v := dyn.V("bar") - other := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - }) - - // Merge a map into v. - { - out, err := Merge(v, other) - assert.EqualError(t, err, "cannot merge string with map") - assert.Equal(t, dyn.InvalidValue, out) - } -} diff --git a/libs/dyn/merge/override.go b/libs/dyn/merge/override.go deleted file mode 100644 index 873c38eead1..00000000000 --- a/libs/dyn/merge/override.go +++ /dev/null @@ -1,200 +0,0 @@ -package merge - -import ( - "errors" - "fmt" - - "github.com/databricks/cli/libs/dyn" -) - -// OverrideVisitor is visiting the changes during the override process -// and allows to control what changes are allowed, or update the effective -// value. -// -// For instance, it can disallow changes outside the specific path(s), or update -// the location of the effective value. -// -// Values returned by 'VisitInsert' and 'VisitUpdate' are used as the final value -// of the node. 'VisitDelete' can return ErrOverrideUndoDelete to undo delete. -// -// 'VisitDelete' is called when a value is removed from mapping or sequence -// 'VisitInsert' is called when a new value is added to mapping or sequence -// 'VisitUpdate' is called when a leaf value is updated -type OverrideVisitor struct { - VisitDelete func(valuePath dyn.Path, left dyn.Value) error - VisitInsert func(valuePath dyn.Path, right dyn.Value) (dyn.Value, error) - VisitUpdate func(valuePath dyn.Path, left, right dyn.Value) (dyn.Value, error) -} - -var ErrOverrideUndoDelete = errors.New("undo delete operation") - -// Override overrides value 'leftRoot' with 'rightRoot', keeping 'location' if values -// haven't changed. Preserving 'location' is important to preserve the original source of the value -// for error reporting. -func Override(leftRoot, rightRoot dyn.Value, visitor OverrideVisitor) (dyn.Value, error) { - return override(dyn.EmptyPath, leftRoot, rightRoot, visitor) -} - -func override(basePath dyn.Path, left, right dyn.Value, visitor OverrideVisitor) (dyn.Value, error) { - if left.Kind() != right.Kind() { - return visitor.VisitUpdate(basePath, left, right) - } - - // NB: we only call 'VisitUpdate' on leaf values, and for sequences and mappings - // we don't know if value was updated or not - - switch left.Kind() { - case dyn.KindMap: - merged, err := overrideMapping(basePath, left.MustMap(), right.MustMap(), visitor) - if err != nil { - return dyn.InvalidValue, err - } - - return dyn.NewValue(merged, left.Locations()), nil - - case dyn.KindSequence: - // some sequences are keyed, and we can detect which elements are added/removed/updated, - // but we don't have this information - merged, err := overrideSequence(basePath, left.MustSequence(), right.MustSequence(), visitor) - if err != nil { - return dyn.InvalidValue, err - } - - return dyn.NewValue(merged, left.Locations()), nil - - case dyn.KindString: - if left.MustString() == right.MustString() { - return left, nil - } else { - return visitor.VisitUpdate(basePath, left, right) - } - - case dyn.KindFloat: - // TODO consider comparison with epsilon if normalization doesn't help, where do we use floats? - - if left.MustFloat() == right.MustFloat() { - return left, nil - } else { - return visitor.VisitUpdate(basePath, left, right) - } - - case dyn.KindBool: - if left.MustBool() == right.MustBool() { - return left, nil - } else { - return visitor.VisitUpdate(basePath, left, right) - } - - case dyn.KindTime: - if left.MustTime() == right.MustTime() { - return left, nil - } else { - return visitor.VisitUpdate(basePath, left, right) - } - - case dyn.KindInt: - if left.MustInt() == right.MustInt() { - return left, nil - } else { - return visitor.VisitUpdate(basePath, left, right) - } - case dyn.KindNil: - return left, nil - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, fmt.Errorf("unexpected kind %s at %s", left.Kind(), basePath.String()) -} - -func overrideMapping(basePath dyn.Path, leftMapping, rightMapping dyn.Mapping, visitor OverrideVisitor) (dyn.Mapping, error) { - out := dyn.NewMapping() - - for _, leftPair := range leftMapping.Pairs() { - // detect if key was removed - if _, ok := rightMapping.GetPair(leftPair.Key); !ok { - key := leftPair.Key.MustString() - keyLoc := leftPair.Key.Locations() - path := basePath.Append(dyn.Key(key)) - - err := visitor.VisitDelete(path, leftPair.Value) - - // if 'delete' was undone, add it back - if errors.Is(err, ErrOverrideUndoDelete) { - out.SetLoc(key, keyLoc, leftPair.Value) - } else if err != nil { - return dyn.NewMapping(), err - } - } - } - - // iterating only right mapping will remove keys not present anymore - // and insert new keys - - for _, rightPair := range rightMapping.Pairs() { - key := rightPair.Key.MustString() - keyLoc := rightPair.Key.Locations() - if leftPair, ok := leftMapping.GetPair(rightPair.Key); ok { - path := basePath.Append(dyn.Key(key)) - newValue, err := override(path, leftPair.Value, rightPair.Value, visitor) - if err != nil { - return dyn.NewMapping(), err - } - - // key was there before, so keep its location - out.SetLoc(key, keyLoc, newValue) - } else { - path := basePath.Append(dyn.Key(rightPair.Key.MustString())) - - newValue, err := visitor.VisitInsert(path, rightPair.Value) - if err != nil { - return dyn.NewMapping(), err - } - - out.SetLoc(key, keyLoc, newValue) - } - } - - return out, nil -} - -func overrideSequence(basePath dyn.Path, left, right []dyn.Value, visitor OverrideVisitor) ([]dyn.Value, error) { - minLen := min(len(left), len(right)) - var values []dyn.Value - - for i := range minLen { - path := basePath.Append(dyn.Index(i)) - merged, err := override(path, left[i], right[i], visitor) - if err != nil { - return nil, err - } - - values = append(values, merged) - } - - if len(right) > len(left) { - for i := minLen; i < len(right); i++ { - path := basePath.Append(dyn.Index(i)) - newValue, err := visitor.VisitInsert(path, right[i]) - if err != nil { - return nil, err - } - - values = append(values, newValue) - } - } else if len(left) > len(right) { - for i := minLen; i < len(left); i++ { - path := basePath.Append(dyn.Index(i)) - err := visitor.VisitDelete(path, left[i]) - - // if 'delete' was undone, add it back - if errors.Is(err, ErrOverrideUndoDelete) { - values = append(values, left[i]) - } else if err != nil { - return nil, err - } - } - } - - return values, nil -} diff --git a/libs/dyn/merge/override_test.go b/libs/dyn/merge/override_test.go deleted file mode 100644 index adc565fec05..00000000000 --- a/libs/dyn/merge/override_test.go +++ /dev/null @@ -1,519 +0,0 @@ -package merge - -import ( - "errors" - "testing" - "time" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynassert" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -type overrideTestCase struct { - name string - left dyn.Value - right dyn.Value - state visitorState - expected dyn.Value -} - -func TestOverride_Primitive(t *testing.T) { - leftLocation := dyn.Location{File: "left.yml", Line: 1, Column: 1} - rightLocation := dyn.Location{File: "right.yml", Line: 1, Column: 1} - - modifiedTestCases := []overrideTestCase{ - { - name: "string (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue("a", []dyn.Location{leftLocation}), - right: dyn.NewValue("b", []dyn.Location{rightLocation}), - expected: dyn.NewValue("b", []dyn.Location{rightLocation}), - }, - { - name: "string (not updated)", - state: visitorState{}, - left: dyn.NewValue("a", []dyn.Location{leftLocation}), - right: dyn.NewValue("a", []dyn.Location{rightLocation}), - expected: dyn.NewValue("a", []dyn.Location{leftLocation}), - }, - { - name: "bool (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(true, []dyn.Location{leftLocation}), - right: dyn.NewValue(false, []dyn.Location{rightLocation}), - expected: dyn.NewValue(false, []dyn.Location{rightLocation}), - }, - { - name: "bool (not updated)", - state: visitorState{}, - left: dyn.NewValue(true, []dyn.Location{leftLocation}), - right: dyn.NewValue(true, []dyn.Location{rightLocation}), - expected: dyn.NewValue(true, []dyn.Location{leftLocation}), - }, - { - name: "int (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(1, []dyn.Location{leftLocation}), - right: dyn.NewValue(2, []dyn.Location{rightLocation}), - expected: dyn.NewValue(2, []dyn.Location{rightLocation}), - }, - { - name: "int (not updated)", - state: visitorState{}, - left: dyn.NewValue(int32(1), []dyn.Location{leftLocation}), - right: dyn.NewValue(int64(1), []dyn.Location{rightLocation}), - expected: dyn.NewValue(int32(1), []dyn.Location{leftLocation}), - }, - { - name: "float (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(1.0, []dyn.Location{leftLocation}), - right: dyn.NewValue(2.0, []dyn.Location{rightLocation}), - expected: dyn.NewValue(2.0, []dyn.Location{rightLocation}), - }, - { - name: "float (not updated)", - state: visitorState{}, - left: dyn.NewValue(float32(1.0), []dyn.Location{leftLocation}), - right: dyn.NewValue(float64(1.0), []dyn.Location{rightLocation}), - expected: dyn.NewValue(float32(1.0), []dyn.Location{leftLocation}), - }, - { - name: "time (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(dyn.FromTime(time.UnixMilli(10000)), []dyn.Location{leftLocation}), - right: dyn.NewValue(dyn.FromTime(time.UnixMilli(10001)), []dyn.Location{rightLocation}), - expected: dyn.NewValue(dyn.FromTime(time.UnixMilli(10001)), []dyn.Location{rightLocation}), - }, - { - name: "time (not updated)", - state: visitorState{}, - left: dyn.NewValue(dyn.FromTime(time.UnixMilli(10000)), []dyn.Location{leftLocation}), - right: dyn.NewValue(dyn.FromTime(time.UnixMilli(10000)), []dyn.Location{rightLocation}), - expected: dyn.NewValue(dyn.FromTime(time.UnixMilli(10000)), []dyn.Location{leftLocation}), - }, - { - name: "different types (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue("a", []dyn.Location{leftLocation}), - right: dyn.NewValue(42, []dyn.Location{rightLocation}), - expected: dyn.NewValue(42, []dyn.Location{rightLocation}), - }, - { - name: "map - remove 'a', update 'b'", - state: visitorState{ - removed: []string{"root.a"}, - updated: []string{"root.b"}, - }, - left: dyn.NewValue( - map[string]dyn.Value{ - "a": dyn.NewValue(42, []dyn.Location{leftLocation}), - "b": dyn.NewValue(10, []dyn.Location{leftLocation}), - }, - []dyn.Location{leftLocation}), - - right: dyn.NewValue( - map[string]dyn.Value{ - "b": dyn.NewValue(20, []dyn.Location{rightLocation}), - }, - []dyn.Location{rightLocation}), - - expected: dyn.NewValue( - map[string]dyn.Value{ - "b": dyn.NewValue(20, []dyn.Location{rightLocation}), - }, - []dyn.Location{leftLocation}), - }, - { - name: "map - add 'a'", - state: visitorState{ - added: []string{"root.a"}, - }, - left: dyn.NewValue( - map[string]dyn.Value{ - "b": dyn.NewValue(10, []dyn.Location{leftLocation}), - }, - []dyn.Location{leftLocation}, - ), - - right: dyn.NewValue( - map[string]dyn.Value{ - "a": dyn.NewValue(42, []dyn.Location{rightLocation}), - "b": dyn.NewValue(10, []dyn.Location{rightLocation}), - }, - []dyn.Location{leftLocation}, - ), - - expected: dyn.NewValue( - map[string]dyn.Value{ - "a": dyn.NewValue(42, []dyn.Location{rightLocation}), - // location hasn't changed because value hasn't changed - "b": dyn.NewValue(10, []dyn.Location{leftLocation}), - }, - []dyn.Location{leftLocation}, - ), - }, - { - name: "map - remove 'a'", - state: visitorState{ - removed: []string{"root.a"}, - }, - left: dyn.NewValue( - map[string]dyn.Value{ - "a": dyn.NewValue(42, []dyn.Location{leftLocation}), - "b": dyn.NewValue(10, []dyn.Location{leftLocation}), - }, - []dyn.Location{leftLocation}, - ), - - right: dyn.NewValue( - map[string]dyn.Value{ - "b": dyn.NewValue(10, []dyn.Location{rightLocation}), - }, - []dyn.Location{leftLocation}, - ), - - expected: dyn.NewValue( - map[string]dyn.Value{ - // location hasn't changed because value hasn't changed - "b": dyn.NewValue(10, []dyn.Location{leftLocation}), - }, - []dyn.Location{leftLocation}, - ), - }, - { - name: "map - add 'jobs.job_1'", - state: visitorState{ - added: []string{"root.jobs.job_1"}, - }, - left: dyn.NewValue( - map[string]dyn.Value{ - "jobs": dyn.NewValue( - map[string]dyn.Value{ - "job_0": dyn.NewValue(42, []dyn.Location{leftLocation}), - }, - []dyn.Location{leftLocation}, - ), - }, - []dyn.Location{leftLocation}, - ), - - right: dyn.NewValue( - map[string]dyn.Value{ - "jobs": dyn.NewValue( - map[string]dyn.Value{ - "job_0": dyn.NewValue(42, []dyn.Location{rightLocation}), - "job_1": dyn.NewValue(1337, []dyn.Location{rightLocation}), - }, - []dyn.Location{rightLocation}, - ), - }, - []dyn.Location{rightLocation}, - ), - - expected: dyn.NewValue( - map[string]dyn.Value{ - "jobs": dyn.NewValue( - map[string]dyn.Value{ - "job_0": dyn.NewValue(42, []dyn.Location{leftLocation}), - "job_1": dyn.NewValue(1337, []dyn.Location{rightLocation}), - }, - []dyn.Location{leftLocation}, - ), - }, - []dyn.Location{leftLocation}, - ), - }, - { - name: "map - remove nested key", - state: visitorState{removed: []string{"root.jobs.job_1"}}, - left: dyn.NewValue( - map[string]dyn.Value{ - "jobs": dyn.NewValue( - map[string]dyn.Value{ - "job_0": dyn.NewValue(42, []dyn.Location{leftLocation}), - "job_1": dyn.NewValue(1337, []dyn.Location{rightLocation}), - }, - []dyn.Location{leftLocation}, - ), - }, - []dyn.Location{leftLocation}, - ), - right: dyn.NewValue( - map[string]dyn.Value{ - "jobs": dyn.NewValue( - map[string]dyn.Value{ - "job_0": dyn.NewValue(42, []dyn.Location{rightLocation}), - }, - []dyn.Location{rightLocation}, - ), - }, - []dyn.Location{rightLocation}, - ), - expected: dyn.NewValue( - map[string]dyn.Value{ - "jobs": dyn.NewValue( - map[string]dyn.Value{ - "job_0": dyn.NewValue(42, []dyn.Location{leftLocation}), - }, - []dyn.Location{leftLocation}, - ), - }, - []dyn.Location{leftLocation}, - ), - }, - { - name: "sequence - add", - state: visitorState{added: []string{"root[1]"}}, - left: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []dyn.Location{leftLocation}), - }, - []dyn.Location{leftLocation}, - ), - right: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []dyn.Location{rightLocation}), - dyn.NewValue(10, []dyn.Location{rightLocation}), - }, - []dyn.Location{rightLocation}, - ), - expected: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []dyn.Location{leftLocation}), - dyn.NewValue(10, []dyn.Location{rightLocation}), - }, - []dyn.Location{leftLocation}, - ), - }, - { - name: "sequence - remove", - state: visitorState{removed: []string{"root[1]"}}, - left: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []dyn.Location{leftLocation}), - dyn.NewValue(10, []dyn.Location{leftLocation}), - }, - []dyn.Location{leftLocation}, - ), - right: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []dyn.Location{rightLocation}), - }, - []dyn.Location{rightLocation}, - ), - expected: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []dyn.Location{leftLocation}), - }, - []dyn.Location{leftLocation}, - ), - // location hasn't changed because value hasn't changed - }, - { - name: "sequence (not updated)", - state: visitorState{}, - left: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []dyn.Location{leftLocation}), - }, - []dyn.Location{leftLocation}, - ), - right: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []dyn.Location{rightLocation}), - }, - []dyn.Location{rightLocation}, - ), - expected: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []dyn.Location{leftLocation}), - }, - []dyn.Location{leftLocation}, - ), - }, - { - name: "nil (not updated)", - state: visitorState{}, - left: dyn.NilValue.WithLocations([]dyn.Location{leftLocation}), - right: dyn.NilValue.WithLocations([]dyn.Location{rightLocation}), - expected: dyn.NilValue.WithLocations([]dyn.Location{leftLocation}), - }, - { - name: "nil (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NilValue, - right: dyn.NewValue(42, []dyn.Location{rightLocation}), - expected: dyn.NewValue(42, []dyn.Location{rightLocation}), - }, - { - name: "change kind (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(42.0, []dyn.Location{leftLocation}), - right: dyn.NewValue(42, []dyn.Location{rightLocation}), - expected: dyn.NewValue(42, []dyn.Location{rightLocation}), - }, - } - - for _, tc := range modifiedTestCases { - t.Run(tc.name, func(t *testing.T) { - s, visitor := createVisitor(visitorOpts{}) - out, err := override(dyn.NewPath(dyn.Key("root")), tc.left, tc.right, visitor) - - assert.NoError(t, err) - dynassert.Equal(t, tc.state, *s) - dynassert.Equal(t, tc.expected, out) - }) - - modified := len(tc.state.removed)+len(tc.state.added)+len(tc.state.updated) > 0 - - // visitor is not used unless there is a change - - if modified { - t.Run(tc.name+" - visitor has error", func(t *testing.T) { - _, visitor := createVisitor(visitorOpts{error: errors.New("unexpected change in test")}) - _, err := override(dyn.EmptyPath, tc.left, tc.right, visitor) - - assert.EqualError(t, err, "unexpected change in test") - }) - - t.Run(tc.name+" - visitor overrides value", func(t *testing.T) { - expected := dyn.V("return value") - s, visitor := createVisitor(visitorOpts{returnValue: &expected}) - out, err := override(dyn.EmptyPath, tc.left, tc.right, visitor) - - assert.NoError(t, err) - - for _, added := range s.added { - actual, err := dyn.GetByPath(out, dyn.MustPathFromString(added)) - - assert.NoError(t, err) - dynassert.Equal(t, expected, actual) - } - - for _, updated := range s.updated { - actual, err := dyn.GetByPath(out, dyn.MustPathFromString(updated)) - - assert.NoError(t, err) - dynassert.Equal(t, expected, actual) - } - }) - - if len(tc.state.removed) > 0 { - t.Run(tc.name+" - visitor can undo delete", func(t *testing.T) { - s, visitor := createVisitor(visitorOpts{deleteError: ErrOverrideUndoDelete}) - out, err := override(dyn.EmptyPath, tc.left, tc.right, visitor) - require.NoError(t, err) - - for _, removed := range s.removed { - expected, err := dyn.GetByPath(tc.left, dyn.MustPathFromString(removed)) - require.NoError(t, err) - - actual, err := dyn.GetByPath(out, dyn.MustPathFromString(removed)) - - assert.NoError(t, err) - dynassert.Equal(t, expected, actual) - } - }) - } - } - } -} - -func TestOverride_PreserveMappingKeys(t *testing.T) { - leftLocation := dyn.Location{File: "left.yml", Line: 1, Column: 1} - leftKeyLocation := dyn.Location{File: "left.yml", Line: 2, Column: 1} - leftValueLocation := dyn.Location{File: "left.yml", Line: 3, Column: 1} - - rightLocation := dyn.Location{File: "right.yml", Line: 1, Column: 1} - rightKeyLocation := dyn.Location{File: "right.yml", Line: 2, Column: 1} - rightValueLocation := dyn.Location{File: "right.yml", Line: 3, Column: 1} - - left := dyn.NewMapping() - left.SetLoc("a", []dyn.Location{leftKeyLocation}, dyn.NewValue(42, []dyn.Location{leftValueLocation})) - - right := dyn.NewMapping() - right.SetLoc("a", []dyn.Location{rightKeyLocation}, dyn.NewValue(7, []dyn.Location{rightValueLocation})) - - state, visitor := createVisitor(visitorOpts{}) - - out, err := override( - dyn.EmptyPath, - dyn.NewValue(left, []dyn.Location{leftLocation}), - dyn.NewValue(right, []dyn.Location{rightLocation}), - visitor, - ) - - assert.NoError(t, err) - - if err != nil { - outPairs := out.MustMap().Pairs() - - assert.Equal(t, visitorState{updated: []string{"a"}}, state) - assert.Len(t, outPairs, 1) - - // mapping was first defined in left, so it should keep its location - assert.Equal(t, leftLocation, out.Location()) - - // if there is a validation error for key value, it should point - // to where it was initially defined - assert.Equal(t, leftKeyLocation, outPairs[0].Key.Location()) - - // the value should have updated location, because it has changed - assert.Equal(t, rightValueLocation, outPairs[0].Value.Location()) - } -} - -type visitorState struct { - added []string - removed []string - updated []string -} - -type visitorOpts struct { - error error - deleteError error - returnValue *dyn.Value -} - -func createVisitor(opts visitorOpts) (*visitorState, OverrideVisitor) { - s := visitorState{} - - return &s, OverrideVisitor{ - VisitUpdate: func(valuePath dyn.Path, left, right dyn.Value) (dyn.Value, error) { - s.updated = append(s.updated, valuePath.String()) - - if opts.error != nil { - return dyn.NilValue, opts.error - } else if opts.returnValue != nil { - return *opts.returnValue, nil - } else { - return right, nil - } - }, - VisitDelete: func(valuePath dyn.Path, left dyn.Value) error { - s.removed = append(s.removed, valuePath.String()) - - if opts.error != nil { - return opts.error - } else if opts.deleteError != nil { - return opts.deleteError - } else { - return nil - } - }, - VisitInsert: func(valuePath dyn.Path, right dyn.Value) (dyn.Value, error) { - s.added = append(s.added, valuePath.String()) - - if opts.error != nil { - return dyn.NilValue, opts.error - } else if opts.returnValue != nil { - return *opts.returnValue, nil - } else { - return right, nil - } - }, - } -} diff --git a/libs/dyn/merge/select.go b/libs/dyn/merge/select.go deleted file mode 100644 index 9b09bfa2162..00000000000 --- a/libs/dyn/merge/select.go +++ /dev/null @@ -1,53 +0,0 @@ -package merge - -import ( - "fmt" - - "github.com/databricks/cli/libs/dyn" -) - -// Select returns a new map that contains only the keys specified in the included list. -func Select(value dyn.Value, included []string) (dyn.Value, error) { - mapping, ok := value.AsMap() - if !ok { - return dyn.InvalidValue, fmt.Errorf("expected a map, but found %s", value.Kind()) - } - - newMapping := dyn.NewMapping() - for _, key := range included { - pair, ok := mapping.GetPairByString(key) - - if ok { - newMapping.SetLoc(key, pair.Key.Locations(), pair.Value) - } - } - - return dyn.NewValue(newMapping, value.Locations()), nil -} - -// AntiSelect returns a new map with all keys from the input map except for the ones in the excluded list. -func AntiSelect(value dyn.Value, excluded []string) (dyn.Value, error) { - mapping, ok := value.AsMap() - if !ok { - return dyn.InvalidValue, fmt.Errorf("expected a map, but found %s", value.Kind()) - } - - excludedSet := make(map[string]struct{}) - for _, key := range excluded { - excludedSet[key] = struct{}{} - } - - included := make([]string, 0, len(mapping.Pairs())) - for _, pair := range mapping.Pairs() { - key, ok := pair.Key.AsString() - if !ok { - return dyn.InvalidValue, fmt.Errorf("expected a string key, but found %s", pair.Key.Kind()) - } - - if _, ok := excludedSet[key]; !ok { - included = append(included, pair.Key.MustString()) - } - } - - return Select(value, included) -} diff --git a/libs/dyn/merge/select_test.go b/libs/dyn/merge/select_test.go deleted file mode 100644 index f9f010fb9db..00000000000 --- a/libs/dyn/merge/select_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package merge - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestSelect(t *testing.T) { - locations := []dyn.Location{{File: "foo.yml", Line: 1, Column: 1}} - included := []string{"foo"} - input := dyn.NewValue( - map[string]dyn.Value{ - "foo": dyn.V("bar"), - "baz": dyn.V("qux"), - }, - locations, - ) - expected := dyn.NewValue( - map[string]dyn.Value{ - "foo": dyn.V("bar"), - }, - locations, - ) - - actual, err := Select(input, included) - - assert.NoError(t, err) - assert.Equal(t, expected, actual) -} - -func TestAntiSelect(t *testing.T) { - locations := []dyn.Location{{File: "foo.yml", Line: 1, Column: 1}} - excluded := []string{"foo"} - input := dyn.NewValue( - map[string]dyn.Value{ - "foo": dyn.V("bar"), - "baz": dyn.V("qux"), - }, - locations, - ) - expected := dyn.NewValue( - map[string]dyn.Value{ - "baz": dyn.V("qux"), - }, - locations, - ) - - actual, err := AntiSelect(input, excluded) - - assert.NoError(t, err) - assert.Equal(t, expected, actual) -} diff --git a/libs/dyn/path.go b/libs/dyn/path.go deleted file mode 100644 index 60579103ca8..00000000000 --- a/libs/dyn/path.go +++ /dev/null @@ -1,146 +0,0 @@ -package dyn - -import ( - "bytes" - "fmt" -) - -type pathComponent struct { - key string - index int -} - -func (c pathComponent) Key() string { - return c.key -} - -func (c pathComponent) Index() int { - return c.index -} - -func (c pathComponent) isKey() bool { - return c.key != "" -} - -func (c pathComponent) isIndex() bool { - return c.key == "" -} - -// Path represents a path to a value in a [Value] configuration tree. -type Path []pathComponent - -// EmptyPath is the empty path. -// It is defined for convenience and clarity. -var EmptyPath = Path{} - -// Key returns a path component for a key. -func Key(k string) pathComponent { - return pathComponent{key: k} -} - -// Index returns a path component for an index. -func Index(i int) pathComponent { - return pathComponent{index: i} -} - -// NewPath returns a new path from the given components. -// The individual components may be created with [Key] or [Index]. -func NewPath(cs ...pathComponent) Path { - return cs -} - -// Append appends the given components to the path. -// Mutations to the returned path do not affect the original path. -func (p Path) Append(cs ...pathComponent) Path { - out := make(Path, len(p)+len(cs)) - copy(out, p) - copy(out[len(p):], cs) - return out -} - -// Equal returns true if the paths are equal. -func (p Path) Equal(q Path) bool { - pl := len(p) - ql := len(q) - if pl != ql { - return false - } - for i := range pl { - if p[i] != q[i] { - return false - } - } - return true -} - -// HasPrefix returns true if the path has the specified prefix. -// The empty path is a prefix of all paths. -func (p Path) HasPrefix(q Path) bool { - pl := len(p) - ql := len(q) - if pl < ql { - return false - } - for i := range ql { - if p[i] != q[i] { - return false - } - } - return true -} - -// HasSuffix returns true if the path has the specified suffix. -// The empty path is a suffix of all paths. -func (p Path) HasSuffix(q Path) bool { - pl := len(p) - ql := len(q) - if pl < ql { - return false - } - for i := range ql { - if p[pl-ql+i] != q[i] { - return false - } - } - return true -} - -// CutPrefix returns the path with the specified prefix removed. -// If the path does not have the specified prefix, the original path is returned. -// The second return value is true if the prefix was removed. -// Logically equivalent to [strings.CutPrefix]. -func (p Path) CutPrefix(q Path) (Path, bool) { - if !p.HasPrefix(q) { - return p, false - } - return p[len(q):], true -} - -// CutSuffix returns the path with the specified suffix removed. -// If the path does not have the specified suffix, the original path is returned. -// The second return value is true if the suffix was removed. -// Logically equivalent to [strings.CutSuffix]. -func (p Path) CutSuffix(q Path) (Path, bool) { - if !p.HasSuffix(q) { - return p, false - } - return p[:len(p)-len(q)], true -} - -// String returns a string representation of the path. -func (p Path) String() string { - var buf bytes.Buffer - - for i, c := range p { - if i > 0 && c.key != "" { - buf.WriteRune('.') - } - if c.key != "" { - buf.WriteString(c.key) - } else { - fmt.Fprintf(&buf, "[%d]", c.index) - } - } - - return buf.String() -} diff --git a/libs/dyn/path_string.go b/libs/dyn/path_string.go deleted file mode 100644 index 2f17f9279c2..00000000000 --- a/libs/dyn/path_string.go +++ /dev/null @@ -1,89 +0,0 @@ -package dyn - -import ( - "fmt" - "strconv" - "strings" -) - -// MustPathFromString is like NewPathFromString but panics on error. -func MustPathFromString(input string) Path { - p, err := NewPathFromString(input) - if err != nil { - panic(err) - } - return p -} - -// NewPathFromString parses a path from a string. -// -// The string must be a sequence of keys and indices separated by dots. -// Indices must be enclosed in square brackets. -// The string may include a leading dot. -// -// Examples: -// - foo.bar -// - foo[1].bar -// - foo.bar[1] -// - foo.bar[1][2] -// - . -func NewPathFromString(input string) (Path, error) { - var path Path - - p := input - - // Trim leading dot. - if p != "" && p[0] == '.' { - p = p[1:] - } - - for p != "" { - // Every component may have a leading dot. - if p[0] == '.' { - p = p[1:] - } - - if p == "" { - return nil, fmt.Errorf("invalid path: %s", input) - } - - if p[0] == '[' { - // Find next ] - i := strings.Index(p, "]") - if i < 0 { - return nil, fmt.Errorf("invalid path: %s", input) - } - - // Parse index - j, err := strconv.Atoi(p[1:i]) - if err != nil { - return nil, fmt.Errorf("invalid path: %s", input) - } - - // Append index - path = append(path, Index(j)) - p = p[i+1:] - - // The next character must be a . or [ - if p != "" && strings.IndexAny(p, ".[") != 0 { - return nil, fmt.Errorf("invalid path: %s", input) - } - } else { - // Find next . or [ - i := strings.IndexAny(p, ".[") - if i < 0 { - i = len(p) - } - - if i == 0 { - return nil, fmt.Errorf("invalid path: %s", input) - } - - // Append key - path = append(path, Key(p[:i])) - p = p[i:] - } - } - - return path, nil -} diff --git a/libs/dyn/path_string_test.go b/libs/dyn/path_string_test.go deleted file mode 100644 index d3bfbd7f493..00000000000 --- a/libs/dyn/path_string_test.go +++ /dev/null @@ -1,115 +0,0 @@ -package dyn_test - -import ( - "errors" - "testing" - - . "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestNewPathFromString(t *testing.T) { - for _, tc := range []struct { - input string - output Path - err error - }{ - { - input: "", - output: NewPath(), - }, - { - input: ".", - output: NewPath(), - }, - { - input: "foo.bar", - output: NewPath(Key("foo"), Key("bar")), - }, - { - input: "[1]", - output: NewPath(Index(1)), - }, - { - input: "foo[1].bar", - output: NewPath(Key("foo"), Index(1), Key("bar")), - }, - { - input: "foo.bar[1]", - output: NewPath(Key("foo"), Key("bar"), Index(1)), - }, - { - input: "foo.bar[1][2]", - output: NewPath(Key("foo"), Key("bar"), Index(1), Index(2)), - }, - { - input: "foo.bar[1][2][3]", - output: NewPath(Key("foo"), Key("bar"), Index(1), Index(2), Index(3)), - }, - { - input: "foo[1234]", - output: NewPath(Key("foo"), Index(1234)), - }, - { - input: "foo[123", - err: errors.New("invalid path: foo[123"), - }, - { - input: "foo[123]]", - err: errors.New("invalid path: foo[123]]"), - }, - { - input: "foo[[123]", - err: errors.New("invalid path: foo[[123]"), - }, - { - input: "foo[[123]]", - err: errors.New("invalid path: foo[[123]]"), - }, - { - input: "foo[foo]", - err: errors.New("invalid path: foo[foo]"), - }, - { - input: "foo..bar", - err: errors.New("invalid path: foo..bar"), - }, - { - input: "foo.bar.", - err: errors.New("invalid path: foo.bar."), - }, - { - // Every component may have a leading dot. - input: ".foo.[1].bar", - output: NewPath(Key("foo"), Index(1), Key("bar")), - }, - { - // But after an index there must be a dot. - input: "foo[1]bar", - err: errors.New("invalid path: foo[1]bar"), - }, - { - // * is parsed as regular string in NewPathFromString - input: "foo.*", - output: NewPath(Key("foo"), Key("*")), - }, - { - // * is parsed as regular string in NewPathFromString - input: "foo.*.bar", - output: NewPath(Key("foo"), Key("*"), Key("bar")), - }, - { - // This is an invalid path (but would be valid for patterns) - input: "foo[*].bar", - err: errors.New("invalid path: foo[*].bar"), - }, - } { - p, err := NewPathFromString(tc.input) - if tc.err != nil { - assert.EqualError(t, err, tc.err.Error(), tc.input) - } else { - assert.NoError(t, err) - assert.Equal(t, tc.output, p) - } - } -} diff --git a/libs/dyn/path_test.go b/libs/dyn/path_test.go deleted file mode 100644 index 49114587175..00000000000 --- a/libs/dyn/path_test.go +++ /dev/null @@ -1,157 +0,0 @@ -package dyn_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestPathAppend(t *testing.T) { - p := dyn.NewPath(dyn.Key("foo")) - - // Single arg. - p1 := p.Append(dyn.Key("bar")) - assert.True(t, p1.Equal(dyn.NewPath(dyn.Key("foo"), dyn.Key("bar")))) - - // Multiple args. - p2 := p.Append(dyn.Key("bar"), dyn.Index(1)) - assert.True(t, p2.Equal(dyn.NewPath(dyn.Key("foo"), dyn.Key("bar"), dyn.Index(1)))) -} - -func TestPathAppendAlwaysNew(t *testing.T) { - p := make(dyn.Path, 0, 2) - p = append(p, dyn.Key("foo")) - - // There is room for a second element in the slice. - p1 := p.Append(dyn.Index(1)) - p2 := p.Append(dyn.Index(2)) - assert.NotEqual(t, p1, p2) -} - -func TestPathEqualEmpty(t *testing.T) { - assert.True(t, dyn.EmptyPath.Equal(dyn.EmptyPath)) -} - -func TestPathEqual(t *testing.T) { - p1 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1)) - p2 := dyn.NewPath(dyn.Key("bar"), dyn.Index(2)) - assert.False(t, p1.Equal(p2), "expected %q to not equal %q", p1, p2) - - p3 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1)) - assert.True(t, p1.Equal(p3), "expected %q to equal %q", p1, p3) - - p4 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1), dyn.Key("bar"), dyn.Index(2)) - assert.False(t, p1.Equal(p4), "expected %q to not equal %q", p1, p4) -} - -func TestPathHasPrefixEmpty(t *testing.T) { - empty := dyn.EmptyPath - nonEmpty := dyn.NewPath(dyn.Key("foo")) - assert.True(t, empty.HasPrefix(empty)) - assert.True(t, nonEmpty.HasPrefix(empty)) - assert.False(t, empty.HasPrefix(nonEmpty)) -} - -func TestPathHasPrefix(t *testing.T) { - p1 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1)) - p2 := dyn.NewPath(dyn.Key("bar"), dyn.Index(2)) - assert.False(t, p1.HasPrefix(p2), "expected %q to not have prefix %q", p1, p2) - - p3 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1), dyn.Key("bar")) - assert.False(t, p1.HasPrefix(p3), "expected %q to not have prefix %q", p1, p3) - - p4 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1)) - assert.True(t, p1.HasPrefix(p4), "expected %q to have prefix %q", p1, p4) - - p5 := dyn.NewPath(dyn.Key("foo")) - assert.True(t, p1.HasPrefix(p5), "expected %q to have prefix %q", p1, p5) -} - -func TestPathHasSuffixEmpty(t *testing.T) { - empty := dyn.EmptyPath - nonEmpty := dyn.NewPath(dyn.Key("foo")) - assert.True(t, empty.HasSuffix(empty)) - assert.True(t, nonEmpty.HasSuffix(empty)) - assert.False(t, empty.HasSuffix(nonEmpty)) -} - -func TestPathHasSuffix(t *testing.T) { - p1 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1)) - p2 := dyn.NewPath(dyn.Key("bar"), dyn.Index(2)) - assert.False(t, p1.HasSuffix(p2), "expected %q to not have suffix %q", p1, p2) - - p3 := dyn.NewPath(dyn.Index(1)) - assert.True(t, p1.HasSuffix(p3), "expected %q to have suffix %q", p1, p3) - - p4 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1)) - assert.True(t, p1.HasSuffix(p4), "expected %q to have suffix %q", p1, p4) - - p5 := dyn.NewPath(dyn.Key("bar"), dyn.Index(2), dyn.Key("baz")) - assert.False(t, p1.HasSuffix(p5), "expected %q to not have suffix %q", p1, p5) -} - -func TestPathCutPrefix(t *testing.T) { - p1 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1), dyn.Key("bar")) - prefix := dyn.NewPath(dyn.Key("foo"), dyn.Index(1)) - - // Cut a valid prefix. - rest, ok := p1.CutPrefix(prefix) - assert.True(t, ok, "expected %q to have prefix %q", p1, prefix) - assert.True(t, rest.Equal(dyn.NewPath(dyn.Key("bar"))), "expected rest to be %q, got %q", dyn.NewPath(dyn.Key("bar")), rest) - - // Try to cut an invalid prefix. - invalidPrefix := dyn.NewPath(dyn.Key("bar")) - rest, ok = p1.CutPrefix(invalidPrefix) - assert.False(t, ok, "expected %q to not have prefix %q", p1, invalidPrefix) - assert.True(t, rest.Equal(p1), "expected rest to be %q, got %q", p1, rest) - - // Cut an empty prefix. - emptyPrefix := dyn.EmptyPath - rest, ok = p1.CutPrefix(emptyPrefix) - assert.True(t, ok, "expected %q to have prefix %q", p1, emptyPrefix) - assert.True(t, rest.Equal(p1), "expected rest to be %q, got %q", p1, rest) - - // Cut a prefix equal to the path. - rest, ok = p1.CutPrefix(p1) - assert.True(t, ok, "expected %q to have prefix %q", p1, p1) - assert.True(t, rest.Equal(dyn.EmptyPath), "expected rest to be %q, got %q", dyn.EmptyPath, rest) -} - -func TestPathCutSuffix(t *testing.T) { - p1 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1), dyn.Key("bar")) - suffix := dyn.NewPath(dyn.Index(1), dyn.Key("bar")) - - // Cut a valid suffix. - rest, ok := p1.CutSuffix(suffix) - assert.True(t, ok, "expected %q to have suffix %q", p1, suffix) - assert.True(t, rest.Equal(dyn.NewPath(dyn.Key("foo"))), "expected rest to be %q, got %q", dyn.NewPath(dyn.Key("foo")), rest) - - // Try to cut an invalid suffix. - invalidSuffix := dyn.NewPath(dyn.Key("foo")) - rest, ok = p1.CutSuffix(invalidSuffix) - assert.False(t, ok, "expected %q to not have suffix %q", p1, invalidSuffix) - assert.True(t, rest.Equal(p1), "expected rest to be %q, got %q", p1, rest) - - // Cut an empty suffix. - emptySuffix := dyn.EmptyPath - rest, ok = p1.CutSuffix(emptySuffix) - assert.True(t, ok, "expected %q to have suffix %q", p1, emptySuffix) - assert.True(t, rest.Equal(p1), "expected rest to be %q, got %q", p1, rest) - - // Cut a suffix equal to the path. - rest, ok = p1.CutSuffix(p1) - assert.True(t, ok, "expected %q to have suffix %q", p1, p1) - assert.True(t, rest.Equal(dyn.EmptyPath), "expected rest to be %q, got %q", dyn.EmptyPath, rest) -} - -func TestPathString(t *testing.T) { - p1 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1)) - assert.Equal(t, "foo[1]", p1.String()) - - p2 := dyn.NewPath(dyn.Key("bar"), dyn.Index(2), dyn.Key("baz")) - assert.Equal(t, "bar[2].baz", p2.String()) - - p3 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1), dyn.Key("bar"), dyn.Index(2), dyn.Key("baz")) - assert.Equal(t, "foo[1].bar[2].baz", p3.String()) -} diff --git a/libs/dyn/pattern.go b/libs/dyn/pattern.go deleted file mode 100644 index cc53a2b0480..00000000000 --- a/libs/dyn/pattern.go +++ /dev/null @@ -1,191 +0,0 @@ -package dyn - -import ( - "errors" - "fmt" - "slices" - "strings" -) - -// Pattern represents a matcher for paths in a [Value] configuration tree. -// It is used by [MapByPattern] to apply a function to the values whose paths match the pattern. -// Every [Path] is a valid [Pattern] that matches a single unique path. -// The reverse is not true; not every [Pattern] is a valid [Path], as patterns may contain wildcards. -type Pattern []patternComponent - -func (p Pattern) String() string { - buf := strings.Builder{} - first := true - - for _, c := range p { - switch c := c.(type) { - case anyKeyComponent: - if !first { - buf.WriteString(".") - } - buf.WriteString("*") - case anyIndexComponent: - buf.WriteString("[*]") - case pathComponent: - if c.isKey() { - if !first { - buf.WriteString(".") - } - buf.WriteString(c.Key()) - } else { - fmt.Fprintf(&buf, "[%d]", c.Index()) - } - default: - buf.WriteString("???") - } - - first = false - } - return buf.String() -} - -// A pattern component can visit a [Value] and recursively call into [visit] for matching elements. -// Fixed components can match a single key or index, while wildcards can match any key or index. -type patternComponent interface { - visit(v Value, prefix Path, suffix Pattern, opts visitOptions) (Value, error) -} - -// NewPattern returns a new pattern from the given components. -// The individual components may be created with [Key], [Index], or [Any]. -func NewPattern(cs ...patternComponent) Pattern { - return cs -} - -// NewPatternFromPath returns a new pattern from the given path. -func NewPatternFromPath(p Path) Pattern { - cs := make(Pattern, len(p)) - for i, c := range p { - cs[i] = c - } - return cs -} - -// Split . into and -// The last component must be dyn.Key() and there must be at least two components. -func (p Pattern) SplitKey() (Pattern, string) { - if len(p) <= 1 { - return nil, "" - } - parent := p[:len(p)-1] - leaf := p[len(p)-1] - pc, ok := leaf.(pathComponent) - if !ok { - return nil, "" - } - key := pc.Key() - if key == "" { - return nil, "" - } - return parent, key -} - -// Append appends the given components to the pattern. -func (p Pattern) Append(cs ...patternComponent) Pattern { - out := make(Pattern, len(p)+len(cs)) - copy(out, p) - copy(out[len(p):], cs) - return out -} - -type anyKeyComponent struct{} - -// AnyKey returns a pattern component that matches any key. -func AnyKey() patternComponent { - return anyKeyComponent{} -} - -type expectedMapError struct { - p Path - v Value -} - -func (e expectedMapError) Error() string { - return fmt.Sprintf("expected a map at %q, found %s", e.p, e.v.Kind()) -} - -type expectedSequenceError struct { - p Path - v Value -} - -func (e expectedSequenceError) Error() string { - return fmt.Sprintf("expected a sequence at %q, found %s", e.p, e.v.Kind()) -} - -// isNoMatchError reports whether err means the pattern suffix didn't match the -// visited value. Wildcard components skip such elements (e.g. a job with an -// empty "tasks:" block) instead of failing the visit for all valid siblings. -func isNoMatchError(err error) bool { - if IsNoSuchKeyError(err) || IsIndexOutOfBoundsError(err) || IsCannotTraverseNilError(err) { - return true - } - if _, ok := errors.AsType[expectedMapError](err); ok { - return true - } - _, ok := errors.AsType[expectedSequenceError](err) - return ok -} - -// This function implements the patternComponent interface. -func (c anyKeyComponent) visit(v Value, prefix Path, suffix Pattern, opts visitOptions) (Value, error) { - m, ok := v.AsMap() - if !ok { - return InvalidValue, expectedMapError{p: prefix, v: v} - } - - m = m.Clone() - for _, pair := range m.Pairs() { - pk := pair.Key - pv := pair.Value - - var err error - nv, err := visit(pv, append(prefix, Key(pk.MustString())), suffix, opts) - if err != nil { - // Leave the value intact if the suffix pattern didn't match any value. - if isNoMatchError(err) { - continue - } - return InvalidValue, err - } - - m.SetLoc(pk.MustString(), pk.Locations(), nv) - } - - return NewValue(m, v.Locations()), nil -} - -type anyIndexComponent struct{} - -// AnyIndex returns a pattern component that matches any index. -func AnyIndex() patternComponent { - return anyIndexComponent{} -} - -// This function implements the patternComponent interface. -func (c anyIndexComponent) visit(v Value, prefix Path, suffix Pattern, opts visitOptions) (Value, error) { - s, ok := v.AsSequence() - if !ok { - return InvalidValue, expectedSequenceError{p: prefix, v: v} - } - - s = slices.Clone(s) - for i, value := range s { - var err error - nv, err := visit(value, append(prefix, Index(i)), suffix, opts) - if err != nil { - // Leave the value intact if the suffix pattern didn't match any value. - if isNoMatchError(err) { - continue - } - return InvalidValue, err - } - s[i] = nv - } - - return NewValue(s, v.Locations()), nil -} diff --git a/libs/dyn/pattern_string.go b/libs/dyn/pattern_string.go deleted file mode 100644 index 6d770faeb92..00000000000 --- a/libs/dyn/pattern_string.go +++ /dev/null @@ -1,102 +0,0 @@ -package dyn - -import ( - "fmt" - "strconv" - "strings" -) - -// MustPatternFromString is like NewPatternFromString but panics on error. -func MustPatternFromString(input string) Pattern { - p, err := NewPatternFromString(input) - if err != nil { - panic(err) - } - return p -} - -// NewPatternFromString parses a pattern from a string. -// -// The string must be a sequence of keys and indices separated by dots. -// Indices must be enclosed in square brackets. -// The string may include a leading dot. -// The wildcard character '*' can be used to match any key or index. -// -// Examples: -// - foo.bar -// - foo[1].bar -// - foo.*.bar -// - foo[*].bar -// - . -func NewPatternFromString(input string) (Pattern, error) { - var pattern Pattern - - p := input - - // Trim leading dot. - if p != "" && p[0] == '.' { - p = p[1:] - } - - for p != "" { - // Every component may have a leading dot. - if p[0] == '.' { - p = p[1:] - } - - if p == "" { - return nil, fmt.Errorf("invalid pattern: %s", input) - } - - if p[0] == '[' { - // Find next ] - i := strings.Index(p, "]") - if i < 0 { - return nil, fmt.Errorf("invalid pattern: %s", input) - } - - // Check for wildcard - if p[1:i] == "*" { - pattern = append(pattern, AnyIndex()) - } else { - // Parse index - j, err := strconv.Atoi(p[1:i]) - if err != nil { - return nil, fmt.Errorf("invalid pattern: %s", input) - } - - // Append index - pattern = append(pattern, Index(j)) - } - - p = p[i+1:] - - // The next character must be a . or [ - if p != "" && strings.IndexAny(p, ".[") != 0 { - return nil, fmt.Errorf("invalid pattern: %s", input) - } - } else { - // Find next . or [ - i := strings.IndexAny(p, ".[") - if i < 0 { - i = len(p) - } - - if i == 0 { - return nil, fmt.Errorf("invalid pattern: %s", input) - } - - // Check for wildcard - if p[:i] == "*" { - pattern = append(pattern, AnyKey()) - } else { - // Append key - pattern = append(pattern, Key(p[:i])) - } - - p = p[i:] - } - } - - return pattern, nil -} diff --git a/libs/dyn/pattern_string_test.go b/libs/dyn/pattern_string_test.go deleted file mode 100644 index 85127c12ed1..00000000000 --- a/libs/dyn/pattern_string_test.go +++ /dev/null @@ -1,147 +0,0 @@ -package dyn_test - -import ( - "errors" - "fmt" - "testing" - - . "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestNewPatternFromString(t *testing.T) { - for ind, tc := range []struct { - input string - output Pattern - err error - }{ - { - input: "", - output: NewPattern(), - }, - { - input: ".", - output: NewPattern(), - }, - { - input: "foo.bar", - output: NewPattern(Key("foo"), Key("bar")), - }, - { - input: "[1]", - output: NewPattern(Index(1)), - }, - { - input: "foo[1].bar", - output: NewPattern(Key("foo"), Index(1), Key("bar")), - }, - { - input: "foo.bar[1]", - output: NewPattern(Key("foo"), Key("bar"), Index(1)), - }, - { - input: "foo.bar[1][2]", - output: NewPattern(Key("foo"), Key("bar"), Index(1), Index(2)), - }, - { - input: "foo.bar[1][2][3]", - output: NewPattern(Key("foo"), Key("bar"), Index(1), Index(2), Index(3)), - }, - { - input: "foo[1234]", - output: NewPattern(Key("foo"), Index(1234)), - }, - { - input: "foo[123", - err: errors.New("invalid pattern: foo[123"), - }, - { - input: "foo[123]]", - err: errors.New("invalid pattern: foo[123]]"), - }, - { - input: "foo[[123]", - err: errors.New("invalid pattern: foo[[123]"), - }, - { - input: "foo[[123]]", - err: errors.New("invalid pattern: foo[[123]]"), - }, - { - input: "foo[foo]", - err: errors.New("invalid pattern: foo[foo]"), - }, - { - input: "foo..bar", - err: errors.New("invalid pattern: foo..bar"), - }, - { - input: "foo.bar.", - err: errors.New("invalid pattern: foo.bar."), - }, - { - // Every component may have a leading dot. - input: ".foo.[1].bar", - output: NewPattern(Key("foo"), Index(1), Key("bar")), - }, - { - // But after an index there must be a dot. - input: "foo[1]bar", - err: errors.New("invalid pattern: foo[1]bar"), - }, - // Wildcard tests - { - input: "foo.*", - output: NewPattern(Key("foo"), AnyKey()), - }, - { - input: "foo.*.bar", - output: NewPattern(Key("foo"), AnyKey(), Key("bar")), - }, - { - input: "foo[*]", - output: NewPattern(Key("foo"), AnyIndex()), - }, - { - input: "foo[*].bar", - output: NewPattern(Key("foo"), AnyIndex(), Key("bar")), - }, - { - input: "*[1]", - output: NewPattern(AnyKey(), Index(1)), - }, - { - input: "*.*", - output: NewPattern(AnyKey(), AnyKey()), - }, - { - input: "*[*]", - output: NewPattern(AnyKey(), AnyIndex()), - }, - } { - t.Run(fmt.Sprintf("%d %s", ind, tc.input), func(t *testing.T) { - p, err := NewPatternFromString(tc.input) - if tc.err != nil { - assert.EqualError(t, err, tc.err.Error(), tc.input) - } else { - assert.NoError(t, err) - assert.Equal(t, tc.output, p) - } - }) - } -} - -func TestMustPatternFromString(t *testing.T) { - // Test valid pattern - p := MustPatternFromString("foo.bar") - assert.Equal(t, NewPattern(Key("foo"), Key("bar")), p) - - // Test with wildcards - p = MustPatternFromString("foo.*.bar[*]") - assert.Equal(t, NewPattern(Key("foo"), AnyKey(), Key("bar"), AnyIndex()), p) - - // Test that invalid pattern panics - assert.Panics(t, func() { - MustPatternFromString("foo[") - }) -} diff --git a/libs/dyn/pattern_test.go b/libs/dyn/pattern_test.go deleted file mode 100644 index be78b6df61a..00000000000 --- a/libs/dyn/pattern_test.go +++ /dev/null @@ -1,110 +0,0 @@ -package dyn_test - -import ( - "fmt" - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestPatternString(t *testing.T) { - patterns := []string{ - "foo.bar", - "foo.bar.baz", - "foo[1]", - "foo[*]", - "*[1].bar", - "foo.*.bar", - "foo[*].bar", - "", - "foo", - "[1]", - "[*]", - "*", - } - - for _, p := range patterns { - pp := dyn.MustPatternFromString(p) - assert.Equal(t, p, pp.String()) - } -} - -func TestNewPattern(t *testing.T) { - pat := dyn.NewPattern( - dyn.Key("foo"), - dyn.Index(1), - ) - - assert.Len(t, pat, 2) -} - -func TestNewPatternFromPath(t *testing.T) { - path := dyn.NewPath( - dyn.Key("foo"), - dyn.Index(1), - ) - - pat1 := dyn.NewPattern(dyn.Key("foo"), dyn.Index(1)) - pat2 := dyn.NewPatternFromPath(path) - assert.Equal(t, pat1, pat2) -} - -func TestPatternAppend(t *testing.T) { - p := dyn.NewPattern(dyn.Key("foo")) - - // Single arg. - p1 := p.Append(dyn.Key("bar")) - assert.Equal(t, dyn.NewPattern(dyn.Key("foo"), dyn.Key("bar")), p1) - - // Multiple args. - p2 := p.Append(dyn.Key("bar"), dyn.Index(1)) - assert.Equal(t, dyn.NewPattern(dyn.Key("foo"), dyn.Key("bar"), dyn.Index(1)), p2) -} - -func TestPatternAppendAlwaysNew(t *testing.T) { - p := make(dyn.Pattern, 0, 2) - p = append(p, dyn.Key("foo")) - - // There is room for a second element in the slice. - p1 := p.Append(dyn.Index(1)) - p2 := p.Append(dyn.Index(2)) - assert.NotEqual(t, p1, p2) -} - -func TestPatternSplitKey(t *testing.T) { - p := dyn.NewPattern( - dyn.Key("foo"), - dyn.Key("bar"), - ) - - pat, key := p.SplitKey() - assert.Equal(t, "bar", key) - assert.Equal(t, dyn.NewPattern(dyn.Key("foo")), pat) -} - -func TestPatternSplitKeyError(t *testing.T) { - patterns := []dyn.Pattern{ - dyn.NewPattern( - dyn.Key("foo"), - dyn.AnyKey(), - ), - dyn.NewPattern( - dyn.Key("foo"), - dyn.AnyIndex(), - ), - dyn.NewPattern( - dyn.Key("foo"), - dyn.Index(1), - ), - dyn.NewPattern(), - } - - for ind, p := range patterns { - t.Run(fmt.Sprintf("%d %#v", ind, p), func(t *testing.T) { - pat, key := p.SplitKey() - assert.Empty(t, key) - assert.Empty(t, pat) - }) - } -} diff --git a/libs/dyn/pattern_trie.go b/libs/dyn/pattern_trie.go deleted file mode 100644 index a4239c3cb53..00000000000 --- a/libs/dyn/pattern_trie.go +++ /dev/null @@ -1,172 +0,0 @@ -package dyn - -import ( - "fmt" -) - -// TrieNode is a trie data structure for storing and querying patterns. -// It supports both exact matches and wildcard matches. You can insert [Pattern]s -// into the trie and then query it to see if a given [Path] matches any of the -// patterns. -// -// TrieNode represents a node in the pattern trie. -// Each node in the array represents one or more of: -// 1. An [AnyKey] component. This is the "*" wildcard which matches any map key. -// 2. An [AnyIndex] component. This is the "[*]" wildcard which matches any array index. -// 3. Multiple [Key] components. These are multiple static path keys for this node would match. -// -// Note: It's valid for both anyKey and pathKey to be set at the same time. -// For example, adding both "foo.*.bar" and "foo.bar" to a trie is valid. -// -// Note: Setting both key (one of pathKey or anyKey) and index (anyIndex) -// is not supported by the [PatternTrie.SearchPath] method. We don't perform validation for this -// case because it's not expected to arise in practice where a field is either a map or an array, -// but not both. -type TrieNode struct { - // If set this indicates the trie node is an AnyKey node. - // Maps to the [AnyKey] component. - AnyKey *TrieNode - - // Indicates the trie node is an AnyIndex node. - // Maps to the [AnyIndex] component. - AnyIndex *TrieNode - - // Set of strings which this trie node matches. - // Maps to the [Key] component. - PathKey map[string]*TrieNode - - // Indicates if this node is the end of a pattern. Encountering a node - // with IsEnd set to true in a trie means the pattern from the root to this - // node is a complete pattern. - IsEnd bool -} - -// Insert adds a pattern to the trie. -func (t *TrieNode) Insert(pattern Pattern) error { - // Empty pattern represents the root. - if len(pattern) == 0 { - t.IsEnd = true - return nil - } - - current := t - for i, component := range pattern { - // Create next node based on component type - var next *TrieNode - switch c := component.(type) { - case anyKeyComponent: - if current.AnyKey == nil { - current.AnyKey = &TrieNode{} - } - next = current.AnyKey - - case anyIndexComponent: - if current.AnyIndex == nil { - current.AnyIndex = &TrieNode{} - } - next = current.AnyIndex - - case pathComponent: - if key := c.Key(); key != "" { - if current.PathKey == nil { - current.PathKey = make(map[string]*TrieNode) - } - if _, exists := current.PathKey[key]; !exists { - current.PathKey[key] = &TrieNode{} - } - next = current.PathKey[key] - } else { - return fmt.Errorf("fixed index patterns are not supported: %#v", pattern) - } - } - - if next == nil { - return fmt.Errorf("invalid component type: %T", component) - } - - // Mark as end of pattern if this is the last component. - if i == len(pattern)-1 { - next.IsEnd = true - } - - // Move to next node - current = next - } - - return nil -} - -// SearchPath checks if the given path matches any pattern in the trie. -// A path matches if it exactly matches a pattern or if it matches a pattern with wildcards. -func (t *TrieNode) SearchPath(path Path) (Pattern, bool) { - // We pre-allocate the prefix array that is used to track the current - // prefix accumulated while walking the prefix tree. Pre-allocating - // ensures that we do not allocate memory on every recursive call. - prefix := make(Pattern, len(path)) - pattern, ok := t.searchPathRecursive(t, path, prefix, 0) - return pattern, ok -} - -// searchPathRecursive is a helper function that recursively checks if a path matches any pattern. -// Arguments: -// - node: the current node in the trie. -// - path: the path to check. -// - prefix: the prefix accumulated while walking the prefix tree. -// - index: the current index in the path / prefix -// -// Note we always expect the path and prefix to be the same length because wildcards like * and [*] -// only match a single path component. -func (t *TrieNode) searchPathRecursive(node *TrieNode, path Path, prefix Pattern, index int) (Pattern, bool) { - if node == nil { - return nil, false - } - - // Zero case, when the query path is the root node. We return nil here to match - // the semantics of [MustPatternFromString] which returns nil for the empty string. - // - // We cannot return a Pattern{} object here because then MustPatternFromString(""), which - // returns nil will not be equal to the Pattern{} object returned by this function. An equality - // is useful because users of this function can use it to check whether the root / empty pattern - // had been inserted into the trie. - if len(path) == 0 { - return nil, node.IsEnd - } - - // If we've reached the end of the path, check if this node is a valid end of a pattern. - isLast := index == len(path) - if isLast { - return prefix, node.IsEnd - } - - currentComponent := path[index] - - // First check if the key wildcard is set for the current index. - if currentComponent.isKey() && node.AnyKey != nil { - prefix[index] = AnyKey() - pattern, ok := t.searchPathRecursive(node.AnyKey, path, prefix, index+1) - if ok { - return pattern, true - } - } - - // If no key wildcard is set, check if the key is an exact match. - if currentComponent.isKey() { - child, exists := node.PathKey[currentComponent.Key()] - if !exists { - return nil, false - } - prefix[index] = currentComponent - return t.searchPathRecursive(child, path, prefix, index+1) - } - - if currentComponent.isIndex() && node.AnyIndex != nil { - prefix[index] = AnyIndex() - pattern, ok := t.searchPathRecursive(node.AnyIndex, path, prefix, index+1) - if ok { - return pattern, true - } - } - - // If we've reached this point, the path does not match any patterns in the trie. - return nil, false -} diff --git a/libs/dyn/pattern_trie_test.go b/libs/dyn/pattern_trie_test.go deleted file mode 100644 index 13bcad457e1..00000000000 --- a/libs/dyn/pattern_trie_test.go +++ /dev/null @@ -1,195 +0,0 @@ -package dyn_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestPatternTrie_SearchPath(t *testing.T) { - tests := []struct { - name string - pattern string - mustMatch []string - mustNotMatch []string - }{ - { - name: "empty pattern", - pattern: "", - mustMatch: []string{""}, - mustNotMatch: []string{"foo"}, - }, - { - name: "simple key pattern", - pattern: "foo", - mustMatch: []string{"foo"}, - mustNotMatch: []string{"foo.bar", "foo[0]", "bar"}, - }, - - { - name: "nested key pattern", - pattern: "foo.bar", - mustMatch: []string{"foo.bar"}, - mustNotMatch: []string{"foo", "foo[0]", "bar.foo", "foo.baz"}, - }, - { - name: "root wildcard", - pattern: "*", - mustMatch: []string{"foo", "bar"}, - mustNotMatch: []string{"", "bar.foo", "foo.baz"}, - }, - { - name: "wildcard * after foo", - pattern: "foo.*", - mustMatch: []string{"foo.bar", "foo.baz"}, - mustNotMatch: []string{"foo", "bar", "foo.bar.baz"}, - }, - { - name: "wildcard [*] after foo", - pattern: "foo[*]", - mustMatch: []string{"foo[0]", "foo[1]", "foo[2025]"}, - mustNotMatch: []string{"foo", "bar", "foo[0].bar"}, - }, - { - name: "key after * wildcard", - pattern: "foo.*.bar", - mustMatch: []string{"foo.abc.bar", "foo.def.bar"}, - mustNotMatch: []string{"foo", "bar", "foo.bar.baz"}, - }, - { - name: "key after [*] wildcard", - pattern: "foo[*].bar", - mustMatch: []string{"foo[0].bar", "foo[1].bar", "foo[2025].bar"}, - mustNotMatch: []string{"foo", "bar", "foo[0].baz"}, - }, - { - name: "multiple * wildcards", - pattern: "*.*.*", - mustMatch: []string{"foo.bar.baz", "foo.bar.qux"}, - mustNotMatch: []string{"foo", "bar", "foo.bar", "foo.bar.baz.qux"}, - }, - { - name: "multiple [*] wildcards", - pattern: "foo[*][*]", - mustMatch: []string{"foo[0][0]", "foo[1][1]", "foo[2025][2025]"}, - mustNotMatch: []string{"foo", "bar", "foo[0][0][0]"}, - }, - { - name: "[*] after * wildcard", - pattern: "*[*]", - mustMatch: []string{"foo[0]", "foo[1]", "foo[2025]"}, - mustNotMatch: []string{"foo", "bar", "foo[0].bar", "[0].foo"}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - trie := &dyn.TrieNode{} - pattern := dyn.MustPatternFromString(tt.pattern) - - // None of the expected paths should match yet. - for _, path := range tt.mustMatch { - _, ok := trie.SearchPath(dyn.MustPathFromString(path)) - assert.False(t, ok) - } - for _, path := range tt.mustNotMatch { - _, ok := trie.SearchPath(dyn.MustPathFromString(path)) - assert.False(t, ok) - } - - err := trie.Insert(pattern) - assert.NoError(t, err) - - // Now all the expected paths should match. - for _, path := range tt.mustMatch { - pattern, ok := trie.SearchPath(dyn.MustPathFromString(path)) - assert.True(t, ok) - assert.Equal(t, dyn.MustPatternFromString(tt.pattern), pattern) - } - for _, path := range tt.mustNotMatch { - _, ok := trie.SearchPath(dyn.MustPathFromString(path)) - assert.False(t, ok) - } - }) - } -} - -func TestPatternTrie_MultiplePatterns(t *testing.T) { - trie := &dyn.TrieNode{} - - patterns := []string{ - "foo.bar", - "foo.*.baz", - "def[*]", - } - - mustMatch := map[string]string{ - "foo.bar": "foo.bar", - "foo.abc.baz": "foo.*.baz", - "foo.def.baz": "foo.*.baz", - "def[0]": "def[*]", - "def[1]": "def[*]", - } - - mustNotMatch := []string{ - "foo", - "abc[0]", - "abc[1]", - "def[2].x", - "foo.y", - "foo.bar.baz.qux", - } - - for _, pattern := range patterns { - err := trie.Insert(dyn.MustPatternFromString(pattern)) - assert.NoError(t, err) - } - - for path, expectedPattern := range mustMatch { - pattern, ok := trie.SearchPath(dyn.MustPathFromString(path)) - assert.True(t, ok) - assert.Equal(t, dyn.MustPatternFromString(expectedPattern), pattern) - } - - for _, path := range mustNotMatch { - _, ok := trie.SearchPath(dyn.MustPathFromString(path)) - assert.False(t, ok) - } -} - -func TestPatternTrie_OverlappingPatterns(t *testing.T) { - trie := &dyn.TrieNode{} - - // Insert overlapping patterns - patterns := []string{ - "foo.bar", - "foo.*", - "*.bar", - "*.*", - } - - for _, pattern := range patterns { - err := trie.Insert(dyn.MustPatternFromString(pattern)) - assert.NoError(t, err) - } - - for _, path := range []string{ - "foo.bar", - "foo.baz", - "baz.bar", - "baz.qux", - } { - _, ok := trie.SearchPath(dyn.MustPathFromString(path)) - assert.True(t, ok) - } -} - -func TestPatternTrie_FixedIndexPatterns(t *testing.T) { - trie := &dyn.TrieNode{} - - err := trie.Insert(dyn.MustPatternFromString("foo[0]")) - assert.EqualError(t, err, "fixed index patterns are not supported: dyn.Pattern{dyn.pathComponent{key:\"foo\", index:0}, dyn.pathComponent{key:\"\", index:0}}") - - err = trie.Insert(dyn.MustPatternFromString("foo[2]")) - assert.EqualError(t, err, "fixed index patterns are not supported: dyn.Pattern{dyn.pathComponent{key:\"foo\", index:0}, dyn.pathComponent{key:\"\", index:2}}") -} diff --git a/libs/dyn/structpath.go b/libs/dyn/structpath.go deleted file mode 100644 index ca9afb5c7b7..00000000000 --- a/libs/dyn/structpath.go +++ /dev/null @@ -1,25 +0,0 @@ -package dyn - -import "github.com/databricks/cli/libs/structs/structpath" - -// ToStructPath converts a [Path] to a [structpath.PathNode]. -func ToStructPath(p Path) *structpath.PathNode { - var result *structpath.PathNode - for _, c := range p { - if c.isKey() { - result = structpath.NewStringKey(result, c.key) - } else { - result = structpath.NewIndex(result, c.index) - } - } - return result -} - -// ToStructPaths converts each of paths with [ToStructPath], e.g. for diag.Diagnostic.Paths. -func ToStructPaths(paths ...Path) []*structpath.PathNode { - out := make([]*structpath.PathNode, len(paths)) - for i, p := range paths { - out[i] = ToStructPath(p) - } - return out -} diff --git a/libs/dyn/structpath_test.go b/libs/dyn/structpath_test.go deleted file mode 100644 index 0e24f8ea51a..00000000000 --- a/libs/dyn/structpath_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package dyn_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestToStructPath(t *testing.T) { - assert.Nil(t, dyn.ToStructPath(nil)) - - p := dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"), dyn.Key("a_job"), dyn.Key("tasks"), dyn.Index(1), dyn.Key("task_key")) - assert.Equal(t, "resources.jobs.a_job.tasks[1].task_key", dyn.ToStructPath(p).String()) - - p = dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"), dyn.Key("${var.env}_job")) - assert.Equal(t, "resources.jobs['${var.env}_job']", dyn.ToStructPath(p).String()) -} - -func TestToStructPaths(t *testing.T) { - paths := dyn.ToStructPaths( - dyn.MustPathFromString("a.b"), - dyn.NewPath(dyn.Key("c"), dyn.Index(1)), - ) - assert.Len(t, paths, 2) - assert.Equal(t, "a.b", paths[0].String()) - assert.Equal(t, "c[1]", paths[1].String()) - assert.Empty(t, dyn.ToStructPaths()) -} diff --git a/libs/dyn/suggest_test.go b/libs/dyn/suggest_test.go deleted file mode 100644 index b0aa4a87168..00000000000 --- a/libs/dyn/suggest_test.go +++ /dev/null @@ -1,164 +0,0 @@ -package dyn - -import ( - "errors" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestLevenshteinDistance(t *testing.T) { - tests := []struct { - a string - b string - want int - }{ - {"", "", 0}, - {"", "abc", 3}, - {"abc", "", 3}, - {"abc", "abc", 0}, - {"abc", "abd", 1}, - {"kitten", "sitting", 3}, - {"host", "hosts", 1}, - } - for _, tt := range tests { - assert.Equal(t, tt.want, levenshteinDistance(tt.a, tt.b), "levenshteinDistance(%q, %q)", tt.a, tt.b) - } -} - -func newSuggestMapping(keys ...string) Mapping { - var m Mapping - for _, k := range keys { - m.SetLoc(k, nil, V(k)) - } - return m -} - -func TestSuggestKeys(t *testing.T) { - tests := []struct { - name string - keys []string - typo string - want []string - }{ - { - // Keys within distance 2 are returned ordered by increasing - // distance; ties keep the map's insertion order. - name: "ordered by distance", - keys: []string{"host", "hosts", "token", "auth_type"}, - typo: "host", - want: []string{"host", "hosts"}, - }, - { - name: "no key close enough", - keys: []string{"host", "hosts", "token", "auth_type"}, - typo: "completely_different", - want: []string{}, - }, - { - // Distance-2 substitutions and insertions are both included. - name: "distance two included", - keys: []string{"profile", "prfile", "prof"}, - typo: "prfil", - want: []string{"prfile", "profile"}, - }, - { - name: "empty map", - keys: nil, - typo: "anything", - want: []string{}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, suggestKeys(newSuggestMapping(tt.keys...), tt.typo)) - }) - } -} - -func TestDidYouMean(t *testing.T) { - tests := []struct { - name string - suggestions []string - want string - }{ - {"nil", nil, ""}, - {"empty", []string{}, ""}, - {"single", []string{"host"}, `, did you mean "host"?`}, - {"multiple", []string{"host", "hosts"}, `, did you mean one of: "host", "hosts"?`}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, didYouMean(tt.suggestions)) - }) - } -} - -func TestSuggestedReferences(t *testing.T) { - tests := []struct { - name string - err error - reference string - want []string - }{ - { - name: "single suggestion", - err: noSuchKeyError{p: NewPath(Key("variables"), Key("hst")), suggestions: []string{"host"}}, - reference: "var.hst", - want: []string{"var.host"}, - }, - { - name: "multiple suggestions", - err: noSuchKeyError{p: NewPath(Key("variables"), Key("hst")), suggestions: []string{"host", "hosts"}}, - reference: "var.hst", - want: []string{"var.host", "var.hosts"}, - }, - { - name: "nested outer-key typo keeps suffix", - err: noSuchKeyError{p: NewPath(Key("variables"), Key("clustr")), suggestions: []string{"cluster"}}, - reference: "var.clustr.spark_version", - want: []string{"var.cluster.spark_version"}, - }, - { - name: "deep leaf typo keeps prefix", - err: noSuchKeyError{p: NewPath(Key("variables"), Key("cluster"), Key("value"), Key("spark_versio")), suggestions: []string{"spark_version"}}, - reference: "var.cluster.spark_versio", - want: []string{"var.cluster.spark_version"}, - }, - { - name: "index component preserved", - err: noSuchKeyError{p: NewPath(Key("variables"), Key("librariez")), suggestions: []string{"libraries"}}, - reference: "var.librariez[0].jar", - want: []string{"var.libraries[0].jar"}, - }, - { - name: "non-var prefix", - err: noSuchKeyError{p: NewPath(Key("workspace"), Key("stot_path")), suggestions: []string{"root_path", "state_path"}}, - reference: "workspace.stot_path", - want: []string{"workspace.root_path", "workspace.state_path"}, - }, - { - name: "no suggestions", - err: noSuchKeyError{p: NewPath(Key("xyz"))}, - reference: "var.xyz", - want: nil, - }, - { - name: "other error type", - err: errors.New("some other error"), - reference: "var.xyz", - want: nil, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, SuggestedReferences(tt.err, tt.reference)) - }) - } -} - -func TestReplaceKeyFallsBackWhenKeyAbsent(t *testing.T) { - // The failed key is not present in the reference, so only the replacement - // itself is returned rather than a spliced reference. - assert.Equal(t, "host", replaceKey("var.foo", "missing", "host")) -} diff --git a/libs/dyn/time.go b/libs/dyn/time.go deleted file mode 100644 index c7f043f06ca..00000000000 --- a/libs/dyn/time.go +++ /dev/null @@ -1,66 +0,0 @@ -package dyn - -import ( - "fmt" - "time" -) - -// Time represents a time-like primitive value. -// -// It represents a timestamp and includes the original string value -// that was parsed to create the timestamp. This makes it possible -// to coalesce a value that YAML interprets as a timestamp back into -// a string without losing information. -type Time struct { - t time.Time - s string -} - -// NewTime creates a new Time from the given string. -func NewTime(str string) (Time, error) { - // Try a couple of layouts - for _, layout := range []string{ - "2006-1-2T15:4:5.999999999Z07:00", // RCF3339Nano with short date fields. - "2006-1-2t15:4:5.999999999Z07:00", // RFC3339Nano with short date fields and lower-case "t". - "2006-1-2 15:4:5.999999999", // space separated with no time zone - "2006-1-2", // date only - } { - t, terr := time.Parse(layout, str) - if terr == nil { - return Time{t: t, s: str}, nil - } - } - - return Time{}, fmt.Errorf("invalid time value: %q", str) -} - -// MustTime creates a new Time from the given string. -// It panics if the string cannot be parsed. -func MustTime(str string) Time { - t, err := NewTime(str) - if err != nil { - panic(err) - } - return t -} - -// FromTime creates a new Time from the given time.Time. -// It uses the RFC3339Nano format for its string representation. -// This guarantees that it can roundtrip into a string without losing information. -func FromTime(t time.Time) Time { - return Time{t: t, s: t.Format(time.RFC3339Nano)} -} - -// Time returns the time.Time value. -func (t Time) Time() time.Time { - return t.t -} - -// String returns the original string value that was parsed to create the timestamp. -func (t Time) String() string { - return t.s -} - -func (t Time) IsZero() bool { - return t.s == "" && t.t.IsZero() -} diff --git a/libs/dyn/time_test.go b/libs/dyn/time_test.go deleted file mode 100644 index 61cc4fef93d..00000000000 --- a/libs/dyn/time_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package dyn_test - -import ( - "testing" - "time" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestTimeValid(t *testing.T) { - for _, tc := range []string{ - "2024-08-29", - "2024-01-15T12:34:56.789012345Z", - } { - tm, err := dyn.NewTime(tc) - if assert.NoError(t, err) { - assert.NotEqual(t, time.Time{}, tm.Time()) - assert.Equal(t, tc, tm.String()) - } - } -} - -func TestTimeInvalid(t *testing.T) { - tm, err := dyn.NewTime("invalid") - assert.Error(t, err) - assert.Equal(t, dyn.Time{}, tm) -} - -func TestTimeFromTime(t *testing.T) { - tref := time.Now() - t1 := dyn.FromTime(tref) - - // Verify that the underlying value is the same. - assert.Equal(t, tref, t1.Time()) - - // Verify that the string representation can be used to construct the same. - t2, err := dyn.NewTime(t1.String()) - assert.NoError(t, err) - assert.True(t, t1.Time().Equal(t2.Time())) -} diff --git a/libs/dyn/value.go b/libs/dyn/value.go deleted file mode 100644 index 72803511b80..00000000000 --- a/libs/dyn/value.go +++ /dev/null @@ -1,243 +0,0 @@ -package dyn - -import ( - "fmt" - "reflect" - "slices" -) - -type Value struct { - v any - - k Kind - - // List of locations this value is defined at. The first location in the slice - // is the location returned by the `.Location()` method and is typically used - // for reporting errors and warnings associated with the value. - l []Location - - // Whether or not this value is an anchor. - // If this node doesn't map to a type, we don't need to warn about it. - anchor bool -} - -// InvalidValue is equal to the zero-value of Value. -var InvalidValue = Value{ - k: KindInvalid, -} - -// NilValue is a convenient constant for a nil value. -var NilValue = Value{ - k: KindNil, -} - -// V constructs a new Value with the given value. -func V(v any) Value { - return NewValue(v, nil) -} - -// NewValue constructs a new Value with the given value and location. -func NewValue(v any, loc []Location) Value { - switch vin := v.(type) { - case map[string]Value: - v = newMappingFromGoMap(vin) - } - - return Value{ - v: v, - k: kindOf(v), - - // create a copy of the locations, so that mutations to the original slice - // don't affect new value. - l: slices.Clone(loc), - } -} - -// WithLocations returns a new Value with its location set to the given value. -func (v Value) WithLocations(loc []Location) Value { - return Value{ - v: v.v, - k: v.k, - - // create a copy of the locations, so that mutations to the original slice - // don't affect new value. - l: slices.Clone(loc), - } -} - -func (v Value) AppendLocationsFromValue(w Value) Value { - return Value{ - v: v.v, - k: v.k, - l: append(v.l, w.l...), - } -} - -func (v Value) Kind() Kind { - return v.k -} - -func (v Value) Value() any { - return v.v -} - -func (v Value) Locations() []Location { - return v.l -} - -func (v Value) Location() Location { - if len(v.l) == 0 { - return Location{} - } - - return v.l[0] -} - -func (v Value) IsValid() bool { - return v.k != KindInvalid -} - -func (v Value) AsAny() any { - switch v.k { - case KindInvalid: - panic("invoked AsAny on invalid value") - case KindMap: - m := v.v.(Mapping) - out := make(map[string]any, m.Len()) - for _, pair := range m.pairs { - pk := pair.Key - pv := pair.Value - out[pk.MustString()] = pv.AsAny() - } - return out - case KindSequence: - vv := v.v.([]Value) - a := make([]any, len(vv)) - for i, v := range vv { - a[i] = v.AsAny() - } - return a - case KindNil: - return v.v - case KindString: - return v.v - case KindBool: - return v.v - case KindInt: - return v.v - case KindFloat: - return v.v - case KindTime: - t := v.v.(Time) - return t.Time() - default: - // Panic because we only want to deal with known types. - panic(fmt.Sprintf("invalid kind: %d", v.k)) - } -} - -func (v Value) IsZero() bool { - if v.v == nil { - return true - } - - switch x := v.v.(type) { - case Mapping: - return x.Len() == 0 - case []Value: - return len(x) == 0 - default: - return reflect.ValueOf(x).IsZero() - } -} - -func (v Value) Get(key string) Value { - m, ok := v.AsMap() - if !ok { - return InvalidValue - } - - vv, ok := m.GetByString(key) - if !ok { - return InvalidValue - } - - return vv -} - -func (v Value) Index(i int) Value { - s, ok := v.v.([]Value) - if !ok { - return InvalidValue - } - - if i < 0 || i >= len(s) { - return InvalidValue - } - - return s[i] -} - -func (v Value) MarkAnchor() Value { - return Value{ - v: v.v, - k: v.k, - l: v.l, - - anchor: true, - } -} - -func (v Value) IsAnchor() bool { - return v.anchor -} - -// eq is an internal only method that compares two values. -// It is used to determine if a value has changed during a visit. -// We need a custom implementation because maps and slices -// cannot be compared with the regular == operator. -func (v Value) eq(w Value) bool { - if v.k != w.k { - return false - } - if !slices.Equal(v.l, w.l) { - return false - } - - switch v.k { - case KindMap: - // Comparing &v.v to &w.v is always false (value receivers), so compare the Mapping's backing storage. - vm := v.v.(Mapping) - wm := w.v.(Mapping) - lv := vm.Len() - lw := wm.Len() - if lv == 0 && lw == 0 { - return true - } - if lv != lw { - return false - } - // Compare pointers to the underlying pairs slice. - // This is safe because we don't allow maps to be mutated. - return &vm.pairs[0] == &wm.pairs[0] - case KindSequence: - vs := v.v.([]Value) - ws := w.v.([]Value) - lv := len(vs) - lw := len(ws) - // If both slices are empty, they are equal. - if lv == 0 && lw == 0 { - return true - } - // If they have different lengths, they are not equal. - if lv != lw { - return false - } - // They are both non-empty and have the same length. - // Compare pointers to the underlying slice. - // This is safe because we don't allow slices to be mutated. - return &vs[0] == &ws[0] - default: - return v.v == w.v - } -} diff --git a/libs/dyn/value_eq_test.go b/libs/dyn/value_eq_test.go deleted file mode 100644 index 83c255edd54..00000000000 --- a/libs/dyn/value_eq_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package dyn - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestValueEqMap(t *testing.T) { - loc := []Location{{File: "file", Line: 1, Column: 2}} - v := NewValue(map[string]Value{"key": V("value")}, loc) - - tests := []struct { - name string - a, b Value - want bool - }{ - { - name: "same underlying mapping", - a: v, - b: v, - want: true, - }, - { - name: "cloned mapping with equal contents", - a: v, - b: Value{v: v.v.(Mapping).Clone(), k: KindMap, l: v.l}, - want: false, - }, - { - name: "different lengths", - a: v, - b: NewValue(map[string]Value{"key": V("value"), "other": V("value")}, loc), - want: false, - }, - { - name: "different locations", - a: v, - b: v.WithLocations([]Location{{File: "other", Line: 1, Column: 2}}), - want: false, - }, - { - name: "empty mappings", - a: NewValue(map[string]Value{}, loc), - b: NewValue(map[string]Value{}, loc), - want: true, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - assert.Equal(t, tc.want, tc.a.eq(tc.b)) - }) - } -} - -func TestValueEqMapVisitDoesNotRebuildAncestors(t *testing.T) { - vin := V(map[string]Value{ - "a": V(map[string]Value{ - "b": V("value"), - }), - }) - - vout, err := Map(vin, "a.b", func(_ Path, v Value) (Value, error) { - return v, nil - }) - require.NoError(t, err) - - // The identity transform must return the original value without - // cloning the ancestor maps. - vm := vin.v.(Mapping) - wm := vout.v.(Mapping) - require.Equal(t, vm.Len(), wm.Len()) - assert.Same(t, &vm.pairs[0], &wm.pairs[0]) -} diff --git a/libs/dyn/value_test.go b/libs/dyn/value_test.go deleted file mode 100644 index 8717aae620e..00000000000 --- a/libs/dyn/value_test.go +++ /dev/null @@ -1,91 +0,0 @@ -package dyn_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestInvalidValue(t *testing.T) { - // Assert that the zero value of [dyn.Value] is the invalid value. - var zero dyn.Value - assert.Equal(t, zero, dyn.InvalidValue) -} - -func TestValueIsAnchor(t *testing.T) { - var zero dyn.Value - assert.False(t, zero.IsAnchor()) - mark := zero.MarkAnchor() - assert.True(t, mark.IsAnchor()) -} - -func TestValueAsMap(t *testing.T) { - var zeroValue dyn.Value - _, ok := zeroValue.AsMap() - assert.False(t, ok) - - intValue := dyn.V(1) - _, ok = intValue.AsMap() - assert.False(t, ok) - - mapValue := dyn.NewValue( - map[string]dyn.Value{ - "key": dyn.NewValue( - "value", - []dyn.Location{{File: "file", Line: 1, Column: 2}}), - }, - []dyn.Location{{File: "file", Line: 1, Column: 2}}, - ) - - m, ok := mapValue.AsMap() - assert.True(t, ok) - assert.Equal(t, 1, m.Len()) -} - -func TestValueIsValid(t *testing.T) { - var zeroValue dyn.Value - assert.False(t, zeroValue.IsValid()) - intValue := dyn.V(1) - assert.True(t, intValue.IsValid()) -} - -func TestIsZero(t *testing.T) { - assert.True(t, dyn.V(0).IsZero(), "int") - assert.True(t, dyn.V(int(0)).IsZero(), "int") - assert.False(t, dyn.V(int(1)).IsZero(), "int") - assert.True(t, dyn.V(uint(0)).IsZero(), "uint") - - assert.True(t, dyn.V(int8(0)).IsZero(), "int8") - assert.True(t, dyn.V(uint8(0)).IsZero(), "uint8") - - assert.True(t, dyn.V(int16(0)).IsZero(), "int16") - assert.True(t, dyn.V(uint16(0)).IsZero(), "uint16") - - assert.True(t, dyn.V(int32(0)).IsZero(), "int32") - assert.False(t, dyn.V(int32(1)).IsZero(), "int32") - assert.True(t, dyn.V(uint32(0)).IsZero(), "uint32") - - assert.True(t, dyn.V(int64(0)).IsZero(), "int64") - assert.False(t, dyn.V(int64(-1)).IsZero(), "int64") - - assert.True(t, dyn.V(uint64(0)).IsZero(), "uint64") - assert.False(t, dyn.V(uint64(2)).IsZero(), "uint64") - - assert.True(t, dyn.V("").IsZero(), "string") - assert.False(t, dyn.V("x").IsZero(), "string") - - assert.True(t, dyn.V(false).IsZero(), "bool") - assert.False(t, dyn.V(true).IsZero(), "bool") - - assert.True(t, dyn.V(float32(0.0)).IsZero(), "float32") - assert.False(t, dyn.V(float32(0.01)).IsZero(), "float32") - - assert.True(t, dyn.V(float64(0.0)).IsZero(), "float64") - assert.False(t, dyn.V(float64(0.01)).IsZero(), "float64") - - assert.True(t, dyn.V(dyn.Time{}).IsZero(), "time") - assert.True(t, dyn.V(dyn.Mapping{}).IsZero(), "Mapping") - assert.True(t, dyn.V([]dyn.Value{}).IsZero(), "Sequence") - assert.False(t, dyn.V([]dyn.Value{dyn.V(0)}).IsZero(), "Sequence") -} diff --git a/libs/dyn/value_underlying.go b/libs/dyn/value_underlying.go deleted file mode 100644 index a33ecd38ed8..00000000000 --- a/libs/dyn/value_underlying.go +++ /dev/null @@ -1,138 +0,0 @@ -package dyn - -import ( - "fmt" -) - -// AsMap returns the underlying mapping if this value is a map, -// the zero value and false otherwise. -func (v Value) AsMap() (Mapping, bool) { - vv, ok := v.v.(Mapping) - return vv, ok -} - -// MustMap returns the underlying mapping if this value is a map, -// panics otherwise. -func (v Value) MustMap() Mapping { - vv, ok := v.AsMap() - if !ok || v.k != KindMap { - panic(fmt.Sprintf("expected kind %s, got %s", KindMap, v.k)) - } - return vv -} - -// AsSequence returns the underlying sequence if this value is a sequence, -// the zero value and false otherwise. -func (v Value) AsSequence() ([]Value, bool) { - vv, ok := v.v.([]Value) - return vv, ok -} - -// MustSequence returns the underlying sequence if this value is a sequence, -// panics otherwise. -func (v Value) MustSequence() []Value { - vv, ok := v.AsSequence() - if !ok || v.k != KindSequence { - panic(fmt.Sprintf("expected kind %s, got %s", KindSequence, v.k)) - } - return vv -} - -// AsString returns the underlying string if this value is a string, -// the zero value and false otherwise. -func (v Value) AsString() (string, bool) { - vv, ok := v.v.(string) - return vv, ok -} - -// MustString returns the underlying string if this value is a string, -// panics otherwise. -func (v Value) MustString() string { - vv, ok := v.AsString() - if !ok || v.k != KindString { - panic(fmt.Sprintf("expected kind %s, got %s", KindString, v.k)) - } - return vv -} - -// AsBool returns the underlying bool if this value is a bool, -// the zero value and false otherwise. -func (v Value) AsBool() (bool, bool) { - vv, ok := v.v.(bool) - return vv, ok -} - -// MustBool returns the underlying bool if this value is a bool, -// panics otherwise. -func (v Value) MustBool() bool { - vv, ok := v.AsBool() - if !ok || v.k != KindBool { - panic(fmt.Sprintf("expected kind %s, got %s", KindBool, v.k)) - } - return vv -} - -// AsInt returns the underlying int if this value is an int, -// the zero value and false otherwise. -func (v Value) AsInt() (int64, bool) { - switch vv := v.v.(type) { - case int: - return int64(vv), true - case int32: - return int64(vv), true - case int64: - return vv, true - default: - return 0, false - } -} - -// MustInt returns the underlying int if this value is an int, -// panics otherwise. -func (v Value) MustInt() int64 { - vv, ok := v.AsInt() - if !ok || v.k != KindInt { - panic(fmt.Sprintf("expected kind %s, got %s", KindInt, v.k)) - } - return vv -} - -// AsFloat returns the underlying float if this value is a float, -// the zero value and false otherwise. -func (v Value) AsFloat() (float64, bool) { - switch vv := v.v.(type) { - case float32: - return float64(vv), true - case float64: - return float64(vv), true - default: - return 0, false - } -} - -// MustFloat returns the underlying float if this value is a float, -// panics otherwise. -func (v Value) MustFloat() float64 { - vv, ok := v.AsFloat() - if !ok || v.k != KindFloat { - panic(fmt.Sprintf("expected kind %s, got %s", KindFloat, v.k)) - } - return vv -} - -// AsTime returns the underlying time if this value is a time, -// the zero value and false otherwise. -func (v Value) AsTime() (Time, bool) { - vv, ok := v.v.(Time) - return vv, ok -} - -// MustTime returns the underlying time if this value is a time, -// panics otherwise. -func (v Value) MustTime() Time { - vv, ok := v.AsTime() - if !ok || v.k != KindTime { - panic(fmt.Sprintf("expected kind %s, got %s", KindTime, v.k)) - } - return vv -} diff --git a/libs/dyn/value_underlying_test.go b/libs/dyn/value_underlying_test.go deleted file mode 100644 index 7f9f9184b92..00000000000 --- a/libs/dyn/value_underlying_test.go +++ /dev/null @@ -1,161 +0,0 @@ -package dyn_test - -import ( - "testing" - "time" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestValueUnderlyingMap(t *testing.T) { - v := dyn.V( - map[string]dyn.Value{ - "key": dyn.NewValue("value", []dyn.Location{{File: "file", Line: 1, Column: 2}}), - }, - ) - - vv1, ok := v.AsMap() - assert.True(t, ok) - - _, ok = dyn.InvalidValue.AsMap() - assert.False(t, ok) - - vv2 := v.MustMap() - assert.Equal(t, vv1, vv2) - - // Test panic. - assert.PanicsWithValue(t, "expected kind map, got invalid", func() { - dyn.InvalidValue.MustMap() - }) -} - -func TestValueUnderlyingSequence(t *testing.T) { - v := dyn.V( - []dyn.Value{ - dyn.NewValue("value", []dyn.Location{{File: "file", Line: 1, Column: 2}}), - }, - ) - - vv1, ok := v.AsSequence() - assert.True(t, ok) - - _, ok = dyn.InvalidValue.AsSequence() - assert.False(t, ok) - - vv2 := v.MustSequence() - assert.Equal(t, vv1, vv2) - - // Test panic. - assert.PanicsWithValue(t, "expected kind sequence, got invalid", func() { - dyn.InvalidValue.MustSequence() - }) -} - -func TestValueUnderlyingString(t *testing.T) { - v := dyn.V("value") - - vv1, ok := v.AsString() - assert.True(t, ok) - - _, ok = dyn.InvalidValue.AsString() - assert.False(t, ok) - - vv2 := v.MustString() - assert.Equal(t, vv1, vv2) - - // Test panic. - assert.PanicsWithValue(t, "expected kind string, got invalid", func() { - dyn.InvalidValue.MustString() - }) -} - -func TestValueUnderlyingBool(t *testing.T) { - v := dyn.V(true) - - vv1, ok := v.AsBool() - assert.True(t, ok) - - _, ok = dyn.InvalidValue.AsBool() - assert.False(t, ok) - - vv2 := v.MustBool() - assert.Equal(t, vv1, vv2) - - // Test panic. - assert.PanicsWithValue(t, "expected kind bool, got invalid", func() { - dyn.InvalidValue.MustBool() - }) -} - -func TestValueUnderlyingInt(t *testing.T) { - v := dyn.V(int(1)) - - vv1, ok := v.AsInt() - assert.True(t, ok) - - _, ok = dyn.InvalidValue.AsInt() - assert.False(t, ok) - - vv2 := v.MustInt() - assert.Equal(t, vv1, vv2) - - // Test panic. - assert.PanicsWithValue(t, "expected kind int, got invalid", func() { - dyn.InvalidValue.MustInt() - }) - - // Test int32 type specifically. - v = dyn.V(int32(1)) - vv1, ok = v.AsInt() - assert.True(t, ok) - assert.Equal(t, int64(1), vv1) - - // Test int64 type specifically. - v = dyn.V(int64(1)) - vv1, ok = v.AsInt() - assert.True(t, ok) - assert.Equal(t, int64(1), vv1) -} - -func TestValueUnderlyingFloat(t *testing.T) { - v := dyn.V(float32(1.0)) - - vv1, ok := v.AsFloat() - assert.True(t, ok) - - _, ok = dyn.InvalidValue.AsFloat() - assert.False(t, ok) - - vv2 := v.MustFloat() - assert.Zero(t, vv1-vv2) - - // Test panic. - assert.PanicsWithValue(t, "expected kind float, got invalid", func() { - dyn.InvalidValue.MustFloat() - }) - - // Test float64 type specifically. - v = dyn.V(float64(1.0)) - vv1, ok = v.AsFloat() - assert.True(t, ok) - assert.Zero(t, 1.0-vv1) -} - -func TestValueUnderlyingTime(t *testing.T) { - v := dyn.V(dyn.FromTime(time.Now())) - - vv1, ok := v.AsTime() - assert.True(t, ok) - - _, ok = dyn.InvalidValue.AsTime() - assert.False(t, ok) - - vv2 := v.MustTime() - assert.Equal(t, vv1, vv2) - - // Test panic. - assert.PanicsWithValue(t, "expected kind time, got invalid", func() { - dyn.InvalidValue.MustTime() - }) -} diff --git a/libs/dyn/visit.go b/libs/dyn/visit.go deleted file mode 100644 index 58223e792b9..00000000000 --- a/libs/dyn/visit.go +++ /dev/null @@ -1,210 +0,0 @@ -package dyn - -import ( - "errors" - "fmt" - "slices" -) - -// This error is returned if the path indicates that a map or sequence is expected, but the value is nil. -type cannotTraverseNilError struct { - p Path -} - -func (e cannotTraverseNilError) Error() string { - component := e.p[len(e.p)-1] - switch { - case component.isKey(): - return fmt.Sprintf("expected a map to index %q, found nil", e.p) - case component.isIndex(): - return fmt.Sprintf("expected a sequence to index %q, found nil", e.p) - default: - panic("invalid component") - } -} - -func IsCannotTraverseNilError(err error) bool { - _, ok := errors.AsType[cannotTraverseNilError](err) - return ok -} - -type noSuchKeyError struct { - p Path - suggestions []string -} - -func (e noSuchKeyError) Error() string { - return fmt.Sprintf("key not found at %q%s", e.p, didYouMean(e.suggestions)) -} - -func IsNoSuchKeyError(err error) bool { - _, ok := errors.AsType[noSuchKeyError](err) - return ok -} - -// SuggestedReferences returns drop-in replacement references for a noSuchKeyError -// (nil otherwise), rebuilt by swapping the failed segment of reference for each -// suggestion (e.g. "var.hst" -> ["var.host"]). -func SuggestedReferences(err error, reference string) []string { - e, ok := errors.AsType[noSuchKeyError](err) - if !ok || len(e.suggestions) == 0 { - return nil - } - // Last component of e.p is the failed key (same in original and rewritten space). - failedKey := e.p[len(e.p)-1].Key() - refs := make([]string, len(e.suggestions)) - for i, s := range e.suggestions { - refs[i] = replaceKey(reference, failedKey, s) - } - return refs -} - -type indexOutOfBoundsError struct { - p Path -} - -func (e indexOutOfBoundsError) Error() string { - return fmt.Sprintf("index out of bounds at %q", e.p) -} - -func IsIndexOutOfBoundsError(err error) bool { - _, ok := errors.AsType[indexOutOfBoundsError](err) - return ok -} - -type expectedMapToIndexError struct { - p Path - v Value -} - -func (e expectedMapToIndexError) Error() string { - return fmt.Sprintf("expected a map to index %q, found %s", e.p, e.v.Kind()) -} - -type expectedSequenceToIndexError struct { - p Path - v Value -} - -func (e expectedSequenceToIndexError) Error() string { - return fmt.Sprintf("expected a sequence to index %q, found %s", e.p, e.v.Kind()) -} - -type visitOptions struct { - // The function to apply to the value once found. - // - // If this function returns the same value as it receives as argument, - // the original visit function call returns the original value unmodified. - // - // If this function returns a new value, the original visit function call - // returns a value with all the intermediate values updated. - // - // If this function returns an error, the original visit function call - // returns this error and the value is left unmodified. - fn func(Path, Value) (Value, error) -} - -func visit(v Value, prefix Path, suffix Pattern, opts visitOptions) (Value, error) { - if len(suffix) == 0 { - return opts.fn(slices.Clone(prefix), v) - } - - // Initialize prefix if it is empty. - // It is pre-allocated to its maximum size to avoid additional allocations. - if len(prefix) == 0 { - prefix = make(Path, 0, len(suffix)) - } - - component := suffix[0] - suffix = suffix[1:] - - // Visit the value with the current component. - return component.visit(v, prefix, suffix, opts) -} - -func (c pathComponent) visit(v Value, prefix Path, suffix Pattern, opts visitOptions) (Value, error) { - path := append(prefix, c) - - switch { - case c.isKey(): - // Expect a map to be set if this is a key. - switch v.Kind() { - case KindMap: - // OK - case KindNil: - return InvalidValue, cannotTraverseNilError{path} - default: - return InvalidValue, expectedMapToIndexError{p: path, v: v} - } - - m := v.MustMap() - - // Lookup current value in the map. - ev, ok := m.GetByString(c.key) - if !ok { - return InvalidValue, noSuchKeyError{p: path, suggestions: suggestKeys(m, c.key)} - } - - // Recursively transform the value. - nv, err := visit(ev, path, suffix, opts) - if err != nil { - return InvalidValue, err - } - - // Return the original value if the value hasn't changed. - if nv.eq(ev) { - return v, nil - } - - // Return an updated map value. - m = m.Clone() - m.SetLoc(c.key, nil, nv) - return Value{ - v: m, - k: KindMap, - l: v.l, - }, nil - - case c.isIndex(): - // Expect a sequence to be set if this is an index. - switch v.Kind() { - case KindSequence: - // OK - case KindNil: - return InvalidValue, cannotTraverseNilError{path} - default: - return InvalidValue, expectedSequenceToIndexError{p: path, v: v} - } - - s := v.MustSequence() - - // Lookup current value in the sequence. - if c.index < 0 || c.index >= len(s) { - return InvalidValue, indexOutOfBoundsError{path} - } - - // Recursively transform the value. - ev := s[c.index] - nv, err := visit(ev, path, suffix, opts) - if err != nil { - return InvalidValue, err - } - - // Return the original value if the value hasn't changed. - if nv.eq(ev) { - return v, nil - } - - // Return an updated sequence value. - s = slices.Clone(s) - s[c.index] = nv - return Value{ - v: s, - k: KindSequence, - l: v.l, - }, nil - - default: - panic("invalid component") - } -} diff --git a/libs/dyn/visit_get.go b/libs/dyn/visit_get.go deleted file mode 100644 index d23c1f97427..00000000000 --- a/libs/dyn/visit_get.go +++ /dev/null @@ -1,31 +0,0 @@ -package dyn - -// Get returns the value inside the specified value at the specified path. -// It is identical to [GetByPath], except that it takes a string path instead of a [Path]. -func Get(v Value, path string) (Value, error) { - p, err := NewPathFromString(path) - if err != nil { - return InvalidValue, err - } - return GetByPath(v, p) -} - -// GetValue returns the value without the error (InvalidValue indicates that error did happen) -func GetValue(v Value, path string) Value { - val, _ := Get(v, path) - return val -} - -// GetByPath returns the value inside the specified value at the specified path. -// If the path doesn't exist, it returns InvalidValue and an error. -func GetByPath(v Value, p Path) (Value, error) { - out := InvalidValue - _, err := visit(v, EmptyPath, NewPatternFromPath(p), visitOptions{ - fn: func(_ Path, ev Value) (Value, error) { - // Capture the value argument to return it. - out = ev - return ev, nil - }, - }) - return out, err -} diff --git a/libs/dyn/visit_get_test.go b/libs/dyn/visit_get_test.go deleted file mode 100644 index 524b4ed5cd9..00000000000 --- a/libs/dyn/visit_get_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package dyn_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestGetWithEmptyPath(t *testing.T) { - // An empty path means to return the value itself. - vin := dyn.V(42) - vout, err := dyn.GetByPath(vin, dyn.NewPath()) - assert.NoError(t, err) - assert.Equal(t, vin, vout) -} - -func TestGetOnNilValue(t *testing.T) { - var err error - _, err = dyn.GetByPath(dyn.NilValue, dyn.NewPath(dyn.Key("foo"))) - assert.ErrorContains(t, err, `expected a map to index "foo", found nil`) - _, err = dyn.GetByPath(dyn.NilValue, dyn.NewPath(dyn.Index(42))) - assert.ErrorContains(t, err, `expected a sequence to index "[42]", found nil`) -} - -func TestGetOnMap(t *testing.T) { - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(42), - "bar": dyn.V(43), - }) - - var err error - - _, err = dyn.GetByPath(vin, dyn.NewPath(dyn.Index(42))) - assert.ErrorContains(t, err, `expected a sequence to index "[42]", found map`) - - _, err = dyn.GetByPath(vin, dyn.NewPath(dyn.Key("baz"))) - assert.True(t, dyn.IsNoSuchKeyError(err)) - // "baz" is one edit away from "bar", so the error suggests it. - assert.ErrorContains(t, err, `key not found at "baz", did you mean "bar"?`) - - // A key that is close to no existing key gets no suggestion. - _, err = dyn.GetByPath(vin, dyn.NewPath(dyn.Key("completely_different"))) - assert.True(t, dyn.IsNoSuchKeyError(err)) - assert.ErrorContains(t, err, `key not found at "completely_different"`) - assert.NotContains(t, err.Error(), "did you mean") - - vfoo, err := dyn.GetByPath(vin, dyn.NewPath(dyn.Key("foo"))) - assert.NoError(t, err) - assert.Equal(t, dyn.V(42), vfoo) - - vbar, err := dyn.GetByPath(vin, dyn.NewPath(dyn.Key("bar"))) - assert.NoError(t, err) - assert.Equal(t, dyn.V(43), vbar) -} - -func TestGetOnSequence(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V(42), - dyn.V(43), - }) - - var err error - - _, err = dyn.GetByPath(vin, dyn.NewPath(dyn.Key("foo"))) - assert.ErrorContains(t, err, `expected a map to index "foo", found sequence`) - - _, err = dyn.GetByPath(vin, dyn.NewPath(dyn.Index(-1))) - assert.True(t, dyn.IsIndexOutOfBoundsError(err)) - assert.ErrorContains(t, err, `index out of bounds at "[-1]"`) - - _, err = dyn.GetByPath(vin, dyn.NewPath(dyn.Index(2))) - assert.True(t, dyn.IsIndexOutOfBoundsError(err)) - assert.ErrorContains(t, err, `index out of bounds at "[2]"`) - - v0, err := dyn.GetByPath(vin, dyn.NewPath(dyn.Index(0))) - assert.NoError(t, err) - assert.Equal(t, dyn.V(42), v0) - - v1, err := dyn.GetByPath(vin, dyn.NewPath(dyn.Index(1))) - assert.NoError(t, err) - assert.Equal(t, dyn.V(43), v1) -} diff --git a/libs/dyn/visit_map.go b/libs/dyn/visit_map.go deleted file mode 100644 index a20655e5c21..00000000000 --- a/libs/dyn/visit_map.go +++ /dev/null @@ -1,89 +0,0 @@ -package dyn - -import ( - "fmt" - "slices" -) - -// MapFunc is a function that maps a value to another value. -type MapFunc func(Path, Value) (Value, error) - -// Foreach returns a [MapFunc] that applies the specified [MapFunc] to each -// value in a map or sequence and returns the new map or sequence. -// If the input is nil, it returns nil. -func Foreach(fn MapFunc) MapFunc { - return func(p Path, v Value) (Value, error) { - switch v.Kind() { - case KindNil: - return v, nil - case KindMap: - m := v.MustMap().Clone() - for _, pair := range m.Pairs() { - pk := pair.Key - pv := pair.Value - nv, err := fn(p.Append(Key(pk.MustString())), pv) - if err != nil { - return InvalidValue, err - } - m.SetLoc(pk.MustString(), pk.Locations(), nv) - } - return NewValue(m, v.Locations()), nil - case KindSequence: - s := slices.Clone(v.MustSequence()) - for i, value := range s { - var err error - s[i], err = fn(p.Append(Index(i)), value) - if err != nil { - return InvalidValue, err - } - } - return NewValue(s, v.Locations()), nil - default: - return InvalidValue, fmt.Errorf("expected a map or sequence, found %s", v.Kind()) - } - } -} - -// Map applies a function to the value at the given path in the given value. -// It is identical to [MapByPath], except that it takes a string path instead of a [Path]. -func Map(v Value, path string, fn MapFunc) (Value, error) { - p, err := NewPathFromString(path) - if err != nil { - return InvalidValue, err - } - return MapByPath(v, p, fn) -} - -// MapByPath applies a function to the value at the given path in the given value. -// It is identical to [MapByPattern], except that it takes a [Path] instead of a [Pattern]. -// This means it only matches a single value, not a pattern of values. -func MapByPath(v Value, p Path, fn MapFunc) (Value, error) { - return MapByPattern(v, NewPatternFromPath(p), fn) -} - -// MapByPattern applies a function to the values whose paths match the given pattern in the given value. -// If successful, it returns the new value with all intermediate values copied and updated. -// -// If the pattern contains a key that doesn't exist, or an index that is out of bounds, -// it returns the original value and no error. -// -// If the pattern is invalid for the given value, it returns InvalidValue and an error. -func MapByPattern(v Value, p Pattern, fn MapFunc) (Value, error) { - nv, err := visit(v, EmptyPath, p, visitOptions{ - fn: fn, - }) - - // Check for success. - if err == nil { - return nv, nil - } - - // Return original value if: - // - any map or sequence is a nil, or - // - a key or index is missing - if IsCannotTraverseNilError(err) || IsNoSuchKeyError(err) || IsIndexOutOfBoundsError(err) { - return v, nil - } - - return nv, err -} diff --git a/libs/dyn/visit_map_test.go b/libs/dyn/visit_map_test.go deleted file mode 100644 index bff4b0bd3f9..00000000000 --- a/libs/dyn/visit_map_test.go +++ /dev/null @@ -1,477 +0,0 @@ -package dyn_test - -import ( - "errors" - "fmt" - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestMapWithEmptyPath(t *testing.T) { - // An empty path means to return the value itself. - vin := dyn.V(42) - vout, err := dyn.MapByPath(dyn.InvalidValue, dyn.EmptyPath, func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return vin, nil - }) - assert.NoError(t, err) - assert.Equal(t, vin, vout) -} - -func TestMapOnNilValue(t *testing.T) { - var nv dyn.Value - var err error - nv, err = dyn.MapByPath(dyn.NilValue, dyn.NewPath(dyn.Key("foo")), nil) - assert.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) - nv, err = dyn.MapByPath(dyn.NilValue, dyn.NewPath(dyn.Index(42)), nil) - assert.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) -} - -func TestMapFuncOnMap(t *testing.T) { - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(42), - "bar": dyn.V(43), - }) - - var err error - - _, err = dyn.MapByPath(vin, dyn.NewPath(dyn.Index(42)), nil) - assert.ErrorContains(t, err, `expected a sequence to index "[42]", found map`) - - // A key that does not exist is not an error. - vout, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Key("baz")), nil) - assert.NoError(t, err) - assert.Equal(t, vin, vout) - - // Note: in the test cases below we implicitly test that the original - // value is not modified as we repeatedly set values on it. - vfoo, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Key("foo")), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - assert.Equal(t, dyn.NewPath(dyn.Key("foo")), p) - assert.Equal(t, dyn.V(42), v) - return dyn.V(44), nil - }) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": 44, - "bar": 43, - }, vfoo.AsAny()) - - vbar, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Key("bar")), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - assert.Equal(t, dyn.NewPath(dyn.Key("bar")), p) - assert.Equal(t, dyn.V(43), v) - return dyn.V(45), nil - }) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": 42, - "bar": 45, - }, vbar.AsAny()) - - // Return error from map function. - ref := errors.New("error") - verr, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Key("foo")), func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return dyn.InvalidValue, ref - }) - assert.Equal(t, dyn.InvalidValue, verr) - assert.ErrorIs(t, err, ref) -} - -func TestMapFuncOnMapWithEmptySequence(t *testing.T) { - variants := []dyn.Value{ - // empty sequence - dyn.V([]dyn.Value{}), - // non-empty sequence - dyn.V([]dyn.Value{dyn.V(42)}), - } - - for i := range variants { - vin := dyn.V(map[string]dyn.Value{ - "key": variants[i], - }) - - for j := range variants { - vout, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Key("key")), func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return variants[j], nil - }) - assert.NoError(t, err) - assert.Equal(t, variants[j], vout.Get("key")) - } - } -} - -func TestMapFuncOnSequence(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V(42), - dyn.V(43), - }) - - var err error - - _, err = dyn.MapByPath(vin, dyn.NewPath(dyn.Key("foo")), nil) - assert.ErrorContains(t, err, `expected a map to index "foo", found sequence`) - - // An index that does not exist is not an error. - vout, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Index(2)), nil) - assert.NoError(t, err) - assert.Equal(t, vin, vout) - - // Note: in the test cases below we implicitly test that the original - // value is not modified as we repeatedly set values on it. - v0, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Index(0)), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - assert.Equal(t, dyn.NewPath(dyn.Index(0)), p) - assert.Equal(t, dyn.V(42), v) - return dyn.V(44), nil - }) - assert.NoError(t, err) - assert.Equal(t, []any{44, 43}, v0.AsAny()) - - v1, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Index(1)), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - assert.Equal(t, dyn.NewPath(dyn.Index(1)), p) - assert.Equal(t, dyn.V(43), v) - return dyn.V(45), nil - }) - assert.NoError(t, err) - assert.Equal(t, []any{42, 45}, v1.AsAny()) - - // Return error from map function. - ref := errors.New("error") - verr, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Index(0)), func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return dyn.InvalidValue, ref - }) - assert.Equal(t, dyn.InvalidValue, verr) - assert.ErrorIs(t, err, ref) -} - -func TestMapFuncOnSequenceWithEmptySequence(t *testing.T) { - variants := []dyn.Value{ - // empty sequence - dyn.V([]dyn.Value{}), - // non-empty sequence - dyn.V([]dyn.Value{dyn.V(42)}), - } - - for i := range variants { - vin := dyn.V([]dyn.Value{ - variants[i], - }) - - for j := range variants { - vout, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Index(0)), func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return variants[j], nil - }) - assert.NoError(t, err) - assert.Equal(t, variants[j], vout.Index(0)) - } - } -} - -func TestMapForeachOnMap(t *testing.T) { - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(42), - "bar": dyn.V(43), - }) - - var err error - - // Run foreach, adding 1 to each of the elements. - vout, err := dyn.Map(vin, ".", dyn.Foreach(func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - i, ok := v.AsInt() - require.True(t, ok, "expected an integer") - switch p[0].Key() { - case "foo": - assert.EqualValues(t, 42, i) - return dyn.V(43), nil - case "bar": - assert.EqualValues(t, 43, i) - return dyn.V(44), nil - default: - return dyn.InvalidValue, fmt.Errorf("unexpected key %q", p[0].Key()) - } - })) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": 43, - "bar": 44, - }, vout.AsAny()) - - // Check that the original has not been modified. - assert.Equal(t, map[string]any{ - "foo": 42, - "bar": 43, - }, vin.AsAny()) -} - -func TestMapForeachOnMapError(t *testing.T) { - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(42), - "bar": dyn.V(43), - }) - - // Check that an error from the map function propagates. - ref := errors.New("error") - _, err := dyn.Map(vin, ".", dyn.Foreach(func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return dyn.InvalidValue, ref - })) - assert.ErrorIs(t, err, ref) -} - -func TestMapForeachOnSequence(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V(42), - dyn.V(43), - }) - - var err error - - // Run foreach, adding 1 to each of the elements. - vout, err := dyn.Map(vin, ".", dyn.Foreach(func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - i, ok := v.AsInt() - require.True(t, ok, "expected an integer") - switch p[0].Index() { - case 0: - assert.EqualValues(t, 42, i) - return dyn.V(43), nil - case 1: - assert.EqualValues(t, 43, i) - return dyn.V(44), nil - default: - return dyn.InvalidValue, fmt.Errorf("unexpected index %d", p[0].Index()) - } - })) - assert.NoError(t, err) - assert.Equal(t, []any{43, 44}, vout.AsAny()) - - // Check that the original has not been modified. - assert.Equal(t, []any{42, 43}, vin.AsAny()) -} - -func TestMapForeachOnSequenceError(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V(42), - dyn.V(43), - }) - - // Check that an error from the map function propagates. - ref := errors.New("error") - _, err := dyn.Map(vin, ".", dyn.Foreach(func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return dyn.InvalidValue, ref - })) - assert.ErrorIs(t, err, ref) -} - -func TestMapForeachOnOtherError(t *testing.T) { - vin := dyn.V(42) - - // Check that if foreach is applied to something other than a map or a sequence, it returns an error. - _, err := dyn.Map(vin, ".", dyn.Foreach(func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return dyn.InvalidValue, nil - })) - assert.ErrorContains(t, err, "expected a map or sequence, found int") -} - -func TestMapForeachOnNil(t *testing.T) { - vin := dyn.NilValue - - // Check that if foreach is applied to nil, it returns nil. - vout, err := dyn.Map(vin, ".", dyn.Foreach(func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return dyn.InvalidValue, nil - })) - assert.NoError(t, err) - assert.Equal(t, dyn.NilValue, vout) -} - -func TestMapByPatternOnNilValue(t *testing.T) { - var err error - _, err = dyn.MapByPattern(dyn.NilValue, dyn.NewPattern(dyn.AnyKey()), nil) - assert.ErrorContains(t, err, `expected a map at "", found nil`) - _, err = dyn.MapByPattern(dyn.NilValue, dyn.NewPattern(dyn.AnyIndex()), nil) - assert.ErrorContains(t, err, `expected a sequence at "", found nil`) -} - -func TestMapByPatternOnMap(t *testing.T) { - vin := dyn.V(map[string]dyn.Value{ - "a": dyn.V(map[string]dyn.Value{ - "b": dyn.V(42), - }), - "b": dyn.V(map[string]dyn.Value{ - "c": dyn.V(43), - }), - }) - - var err error - - // Expect an error if the pattern structure doesn't match the value structure. - _, err = dyn.MapByPattern(vin, dyn.NewPattern(dyn.AnyKey(), dyn.Index(0)), nil) - assert.ErrorContains(t, err, `expected a sequence to index`) - - // Apply function to pattern "*.b". - vout, err := dyn.MapByPattern(vin, dyn.NewPattern(dyn.AnyKey(), dyn.Key("b")), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - assert.Equal(t, dyn.NewPath(dyn.Key("a"), dyn.Key("b")), p) - assert.Equal(t, dyn.V(42), v) - return dyn.V(44), nil - }) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "a": map[string]any{ - "b": 44, - }, - "b": map[string]any{ - "c": 43, - }, - }, vout.AsAny()) -} - -func TestMapByPatternOnMapWithoutMatch(t *testing.T) { - vin := dyn.V(map[string]dyn.Value{ - "a": dyn.V(map[string]dyn.Value{ - "b": dyn.V(42), - }), - "b": dyn.V(map[string]dyn.Value{ - "c": dyn.V(43), - }), - }) - - // Apply function to pattern "*.zzz". - vout, err := dyn.MapByPattern(vin, dyn.NewPattern(dyn.AnyKey(), dyn.Key("zzz")), nil) - assert.NoError(t, err) - assert.Equal(t, vin, vout) -} - -func TestMapByPatternOnSequence(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V([]dyn.Value{ - dyn.V(42), - }), - dyn.V([]dyn.Value{ - dyn.V(43), - dyn.V(44), - }), - }) - - var err error - - // Expect an error if the pattern structure doesn't match the value structure. - _, err = dyn.MapByPattern(vin, dyn.NewPattern(dyn.AnyIndex(), dyn.Key("a")), nil) - assert.ErrorContains(t, err, `expected a map to index`) - - // Apply function to pattern "*.c". - vout, err := dyn.MapByPattern(vin, dyn.NewPattern(dyn.AnyIndex(), dyn.Index(1)), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - assert.Equal(t, dyn.NewPath(dyn.Index(1), dyn.Index(1)), p) - assert.Equal(t, dyn.V(44), v) - return dyn.V(45), nil - }) - assert.NoError(t, err) - assert.Equal(t, []any{ - []any{42}, - []any{43, 45}, - }, vout.AsAny()) -} - -func TestMapByPatternWildcardSkipsNonMatchingSiblings(t *testing.T) { - tests := []struct { - name string - vin dyn.Value - pattern dyn.Pattern - want any - visited []string - }{ - { - name: "nil sibling under any key", - vin: dyn.V(map[string]dyn.Value{ - "a": dyn.NilValue, - "b": dyn.V(map[string]dyn.Value{"x": dyn.V(1)}), - }), - pattern: dyn.NewPattern(dyn.AnyKey(), dyn.Key("x")), - want: map[string]any{ - "a": nil, - "b": map[string]any{"x": 42}, - }, - visited: []string{"b.x"}, - }, - { - name: "nil sequence sibling under any key", - vin: dyn.V(map[string]dyn.Value{ - "a": dyn.V(map[string]dyn.Value{"tasks": dyn.NilValue}), - "b": dyn.V(map[string]dyn.Value{"tasks": dyn.V([]dyn.Value{dyn.V(1)})}), - }), - pattern: dyn.NewPattern(dyn.AnyKey(), dyn.Key("tasks"), dyn.AnyIndex()), - want: map[string]any{ - "a": map[string]any{"tasks": nil}, - "b": map[string]any{"tasks": []any{42}}, - }, - visited: []string{"b.tasks[0]"}, - }, - { - name: "wrong kind sibling under any key", - vin: dyn.V(map[string]dyn.Value{ - "a": dyn.V(map[string]dyn.Value{"tasks": dyn.V("oops")}), - "b": dyn.V(map[string]dyn.Value{"tasks": dyn.V([]dyn.Value{dyn.V(1)})}), - }), - pattern: dyn.NewPattern(dyn.AnyKey(), dyn.Key("tasks"), dyn.AnyIndex()), - want: map[string]any{ - "a": map[string]any{"tasks": "oops"}, - "b": map[string]any{"tasks": []any{42}}, - }, - visited: []string{"b.tasks[0]"}, - }, - { - name: "nil element under any index", - vin: dyn.V([]dyn.Value{ - dyn.NilValue, - dyn.V(map[string]dyn.Value{"x": dyn.V(1)}), - }), - pattern: dyn.NewPattern(dyn.AnyIndex(), dyn.Key("x")), - want: []any{ - nil, - map[string]any{"x": 42}, - }, - visited: []string{"[1].x"}, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - var visited []string - vout, err := dyn.MapByPattern(tc.vin, tc.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - visited = append(visited, p.String()) - return dyn.V(42), nil - }) - assert.NoError(t, err) - assert.Equal(t, tc.want, vout.AsAny()) - assert.Equal(t, tc.visited, visited) - }) - } -} - -func TestMapByPatternWildcardPropagatesMapFuncError(t *testing.T) { - vin := dyn.V(map[string]dyn.Value{ - "a": dyn.V(map[string]dyn.Value{"x": dyn.V(1)}), - }) - - ref := errors.New("error") - _, err := dyn.MapByPattern(vin, dyn.NewPattern(dyn.AnyKey(), dyn.Key("x")), func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return dyn.InvalidValue, ref - }) - assert.ErrorIs(t, err, ref) -} - -func TestMapByPatternOnSequenceWithoutMatch(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V([]dyn.Value{ - dyn.V(42), - }), - dyn.V([]dyn.Value{ - dyn.V(43), - dyn.V(44), - }), - }) - - // Apply function to pattern "*.zzz". - vout, err := dyn.MapByPattern(vin, dyn.NewPattern(dyn.AnyIndex(), dyn.Index(42)), nil) - assert.NoError(t, err) - assert.Equal(t, vin, vout) -} diff --git a/libs/dyn/visit_set.go b/libs/dyn/visit_set.go deleted file mode 100644 index 1b3c7f1b661..00000000000 --- a/libs/dyn/visit_set.go +++ /dev/null @@ -1,77 +0,0 @@ -package dyn - -import ( - "fmt" - "slices" -) - -// Set assigns a new value at the specified path in the specified value. -// It is identical to [SetByPath], except that it takes a string path instead of a [Path]. -func Set(v Value, path string, nv Value) (Value, error) { - p, err := NewPathFromString(path) - if err != nil { - return InvalidValue, err - } - return SetByPath(v, p, nv) -} - -// SetByPath assigns the given value at the specified path in the specified value. -// If successful, it returns the new value with all intermediate values copied and updated. -// If the path doesn't exist, it returns InvalidValue and an error. -func SetByPath(v Value, p Path, nv Value) (Value, error) { - lp := len(p) - if lp == 0 { - return nv, nil - } - - component := p[lp-1] - p = p[:lp-1] - - return visit(v, EmptyPath, NewPatternFromPath(p), visitOptions{ - fn: func(prefix Path, v Value) (Value, error) { - path := append(prefix, component) - - switch { - case component.isKey(): - // Expect a map to be set if this is a key. - m, ok := v.AsMap() - if !ok { - return InvalidValue, fmt.Errorf("expected a map to index %q, found %s", path, v.Kind()) - } - - // Return an updated map value. - m = m.Clone() - m.SetLoc(component.key, nil, nv) - return Value{ - v: m, - k: KindMap, - l: v.l, - }, nil - - case component.isIndex(): - // Expect a sequence to be set if this is an index. - s, ok := v.AsSequence() - if !ok { - return InvalidValue, fmt.Errorf("expected a sequence to index %q, found %s", path, v.Kind()) - } - - // Lookup current value in the sequence. - if component.index < 0 || component.index >= len(s) { - return InvalidValue, indexOutOfBoundsError{prefix} - } - - // Return an updated sequence value. - s = slices.Clone(s) - s[component.index] = nv - return Value{ - v: s, - k: KindSequence, - l: v.l, - }, nil - - default: - panic("invalid component") - } - }, - }) -} diff --git a/libs/dyn/visit_set_test.go b/libs/dyn/visit_set_test.go deleted file mode 100644 index b3847158757..00000000000 --- a/libs/dyn/visit_set_test.go +++ /dev/null @@ -1,90 +0,0 @@ -package dyn_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestSetWithEmptyPath(t *testing.T) { - // An empty path means to return the value itself. - vin := dyn.V(42) - vout, err := dyn.SetByPath(dyn.InvalidValue, dyn.EmptyPath, vin) - assert.NoError(t, err) - assert.Equal(t, vin, vout) -} - -func TestSetOnNilValue(t *testing.T) { - var err error - _, err = dyn.SetByPath(dyn.NilValue, dyn.NewPath(dyn.Key("foo")), dyn.V(42)) - assert.ErrorContains(t, err, `expected a map to index "foo", found nil`) - _, err = dyn.SetByPath(dyn.NilValue, dyn.NewPath(dyn.Index(42)), dyn.V(42)) - assert.ErrorContains(t, err, `expected a sequence to index "[42]", found nil`) -} - -func TestSetOnMap(t *testing.T) { - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(42), - "bar": dyn.V(43), - }) - - var err error - - _, err = dyn.SetByPath(vin, dyn.NewPath(dyn.Index(42)), dyn.V(42)) - assert.ErrorContains(t, err, `expected a sequence to index "[42]", found map`) - - // Note: in the test cases below we implicitly test that the original - // value is not modified as we repeatedly set values on it. - - vfoo, err := dyn.SetByPath(vin, dyn.NewPath(dyn.Key("foo")), dyn.V(44)) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": 44, - "bar": 43, - }, vfoo.AsAny()) - - vbar, err := dyn.SetByPath(vin, dyn.NewPath(dyn.Key("bar")), dyn.V(45)) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": 42, - "bar": 45, - }, vbar.AsAny()) - - vbaz, err := dyn.SetByPath(vin, dyn.NewPath(dyn.Key("baz")), dyn.V(46)) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": 42, - "bar": 43, - "baz": 46, - }, vbaz.AsAny()) -} - -func TestSetOnSequence(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V(42), - dyn.V(43), - }) - - var err error - - _, err = dyn.SetByPath(vin, dyn.NewPath(dyn.Key("foo")), dyn.V(42)) - assert.ErrorContains(t, err, `expected a map to index "foo", found sequence`) - - // It is not allowed to set a value at an index that is out of bounds. - _, err = dyn.SetByPath(vin, dyn.NewPath(dyn.Index(-1)), dyn.V(42)) - assert.True(t, dyn.IsIndexOutOfBoundsError(err)) - _, err = dyn.SetByPath(vin, dyn.NewPath(dyn.Index(2)), dyn.V(42)) - assert.True(t, dyn.IsIndexOutOfBoundsError(err)) - - // Note: in the test cases below we implicitly test that the original - // value is not modified as we repeatedly set values on it. - - v0, err := dyn.SetByPath(vin, dyn.NewPath(dyn.Index(0)), dyn.V(44)) - assert.NoError(t, err) - assert.Equal(t, []any{44, 43}, v0.AsAny()) - - v1, err := dyn.SetByPath(vin, dyn.NewPath(dyn.Index(1)), dyn.V(45)) - assert.NoError(t, err) - assert.Equal(t, []any{42, 45}, v1.AsAny()) -} diff --git a/libs/dyn/visit_test.go b/libs/dyn/visit_test.go deleted file mode 100644 index b54e6ad37c7..00000000000 --- a/libs/dyn/visit_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package dyn_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestVisitCallbackPathCopy(t *testing.T) { - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(42), - "bar": dyn.V(43), - }) - - var paths []dyn.Path - - // The callback should receive a copy of the path. - // If the same underlying value is used, all collected paths will be the same. - // This test uses `MapByPattern` to collect all paths in the map. - // Visit itself doesn't have public functions and we exclusively use black-box testing for this package. - _, _ = dyn.MapByPattern(vin, dyn.NewPattern(dyn.AnyKey()), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - paths = append(paths, p) - return v, nil - }) - - // Verify that the paths retained their original values. - var strings []string - for _, p := range paths { - strings = append(strings, p.String()) - } - assert.ElementsMatch(t, strings, []string{ - "foo", - "bar", - }) -} diff --git a/libs/dyn/walk.go b/libs/dyn/walk.go deleted file mode 100644 index af03ef27743..00000000000 --- a/libs/dyn/walk.go +++ /dev/null @@ -1,95 +0,0 @@ -package dyn - -import "errors" - -// WalkValueFunc is the type of the function called by Walk to traverse the configuration tree. -type WalkValueFunc func(p Path, v Value) (Value, error) - -// ErrDrop may be returned by WalkValueFunc to remove a value from the subtree. -var ErrDrop = errors.New("drop value from subtree") - -// ErrSkip may be returned by WalkValueFunc to skip traversal of a subtree. -var ErrSkip = errors.New("skip traversal of subtree") - -// Walk walks the configuration tree and calls the given function on each node. -// The callback may return ErrDrop to remove a value from the subtree. -// The callback may return ErrSkip to skip traversal of a subtree. -// If the callback returns another error, the walk is aborted, and the error is returned. -func Walk(v Value, fn func(p Path, v Value) (Value, error)) (Value, error) { - return walk(v, EmptyPath, fn) -} - -// Unexported counterpart to Walk. -// It carries the path leading up to the current node, -// such that it can be passed to the WalkValueFunc. -func walk(v Value, p Path, fn func(p Path, v Value) (Value, error)) (Value, error) { - v, err := fn(p, v) - if err != nil { - if err == ErrSkip { - return v, nil - } - return InvalidValue, err - } - - switch v.Kind() { - case KindMap: - m := v.MustMap() - out := newMappingWithSize(m.Len()) - for _, pair := range m.Pairs() { - pk := pair.Key - pv := pair.Value - nv, err := walk(pv, append(p, Key(pk.MustString())), fn) - if err == ErrDrop { - continue - } - if err != nil { - return InvalidValue, err - } - out.SetLoc(pk.MustString(), pk.Locations(), nv) - } - v.v = out - case KindSequence: - s := v.MustSequence() - out := make([]Value, 0, len(s)) - for i := range s { - nv, err := walk(s[i], append(p, Index(i)), fn) - if err == ErrDrop { - continue - } - if err != nil { - return InvalidValue, err - } - out = append(out, nv) - } - v.v = out - default: - // Other kinds are not walkable. - // This was a leaf node. - } - - return v, nil -} - -// CollectLeafPaths traverses the value and returns all paths (as dot notation strings) to leaf nodes (non-map, non-sequence). -// The return value is not ordered. -func CollectLeafPaths(v Value) []string { - var paths []string - - Walk(v, func(p Path, v Value) (Value, error) { //nolint:errcheck - if len(p) == 0 { - return v, nil - } - - switch v.Kind() { - case KindMap, KindSequence: - // Ignore internal nodes. - default: - // This was a leaf node. - paths = append(paths, p.String()) - } - - return v, nil - }) - - return paths -} diff --git a/libs/dyn/walk_read_only.go b/libs/dyn/walk_read_only.go deleted file mode 100644 index caead36289e..00000000000 --- a/libs/dyn/walk_read_only.go +++ /dev/null @@ -1,44 +0,0 @@ -package dyn - -// WalkReadOnly walks the configuration tree in readonly mode and calls the given function on each node. -// The callback may return ErrSkip to skip traversal of a subtree. -// If the callback returns another error, the walk is aborted, and the error is returned. -func WalkReadOnly(v Value, fn func(p Path, v Value) error) error { - return walkReadOnly(v, EmptyPath, fn) -} - -// Unexported counterpart to WalkReadOnly. -// It carries the path leading up to the current node, -// such that it can be passed to the callback function. -func walkReadOnly(v Value, p Path, fn func(p Path, v Value) error) error { - if err := fn(p, v); err != nil { - if err == ErrSkip { - return nil - } - return err - } - - switch v.Kind() { - case KindMap: - m := v.MustMap() - for _, pair := range m.Pairs() { - pk := pair.Key - pv := pair.Value - if err := walkReadOnly(pv, append(p, Key(pk.MustString())), fn); err != nil { - return err - } - } - case KindSequence: - s := v.MustSequence() - for i := range s { - if err := walkReadOnly(s[i], append(p, Index(i)), fn); err != nil { - return err - } - } - default: - // Other kinds are not walkable. - // This was a leaf node. - } - - return nil -} diff --git a/libs/dyn/walk_read_only_test.go b/libs/dyn/walk_read_only_test.go deleted file mode 100644 index f6df5344d8c..00000000000 --- a/libs/dyn/walk_read_only_test.go +++ /dev/null @@ -1,245 +0,0 @@ -package dyn_test - -import ( - "errors" - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestWalkReadOnly(t *testing.T) { - tests := []struct { - name string - input dyn.Value - expectedPaths []dyn.Path - expectedValues []dyn.Value - }{ - { - name: "simple map", - input: dyn.V(dyn.NewMappingFromPairs( - []dyn.Pair{ - {Key: dyn.V("a"), Value: dyn.V("1")}, - {Key: dyn.V("b"), Value: dyn.V("2")}, - }, - )), - expectedPaths: []dyn.Path{ - dyn.EmptyPath, - {dyn.Key("a")}, - {dyn.Key("b")}, - }, - expectedValues: []dyn.Value{ - dyn.V(dyn.NewMappingFromPairs( - []dyn.Pair{ - {Key: dyn.V("a"), Value: dyn.V("1")}, - {Key: dyn.V("b"), Value: dyn.V("2")}, - }, - )), - dyn.V("1"), - dyn.V("2"), - }, - }, - { - name: "nested map", - input: dyn.V(map[string]dyn.Value{ - "a": dyn.V(map[string]dyn.Value{ - "b": dyn.V("1"), - }), - }), - expectedPaths: []dyn.Path{ - dyn.EmptyPath, - {dyn.Key("a")}, - {dyn.Key("a"), dyn.Key("b")}, - }, - expectedValues: []dyn.Value{ - dyn.V(map[string]dyn.Value{ - "a": dyn.V(map[string]dyn.Value{ - "b": dyn.V("1"), - }), - }), - dyn.V(map[string]dyn.Value{ - "b": dyn.V("1"), - }), - dyn.V("1"), - }, - }, - { - name: "sequence", - input: dyn.V([]dyn.Value{ - dyn.V("1"), - dyn.V("2"), - }), - expectedPaths: []dyn.Path{ - dyn.EmptyPath, - {dyn.Index(0)}, - {dyn.Index(1)}, - }, - expectedValues: []dyn.Value{ - dyn.V([]dyn.Value{ - dyn.V("1"), - dyn.V("2"), - }), - dyn.V("1"), - dyn.V("2"), - }, - }, - { - name: "nested sequence", - input: dyn.V([]dyn.Value{ - dyn.V([]dyn.Value{ - dyn.V("1"), - }), - }), - expectedPaths: []dyn.Path{ - dyn.EmptyPath, - {dyn.Index(0)}, - {dyn.Index(0), dyn.Index(0)}, - }, - expectedValues: []dyn.Value{ - dyn.V([]dyn.Value{ - dyn.V([]dyn.Value{ - dyn.V("1"), - }), - }), - dyn.V([]dyn.Value{ - dyn.V("1"), - }), - dyn.V("1"), - }, - }, - { - name: "complex structure", - input: dyn.V(map[string]dyn.Value{ - "a": dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "b": dyn.V("1"), - }), - }), - }), - expectedPaths: []dyn.Path{ - dyn.EmptyPath, - {dyn.Key("a")}, - {dyn.Key("a"), dyn.Index(0)}, - {dyn.Key("a"), dyn.Index(0), dyn.Key("b")}, - }, - expectedValues: []dyn.Value{ - dyn.V(map[string]dyn.Value{ - "a": dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "b": dyn.V("1"), - }), - }), - }), - dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "b": dyn.V("1"), - }), - }), - dyn.V(map[string]dyn.Value{ - "b": dyn.V("1"), - }), - dyn.V("1"), - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - visitedPaths := make([]dyn.Path, 0, len(tt.expectedPaths)) - visitedValues := make([]dyn.Value, 0, len(tt.expectedValues)) - - err := dyn.WalkReadOnly(tt.input, func(p dyn.Path, v dyn.Value) error { - visitedPaths = append(visitedPaths, p) - visitedValues = append(visitedValues, v) - return nil - }) - assert.NoError(t, err) - assert.Equal(t, tt.expectedPaths, visitedPaths) - assert.Equal(t, tt.expectedValues, visitedValues) - }) - } -} - -func TestWalkReadOnly_Error(t *testing.T) { - testErr := errors.New("test error") - input := dyn.V(map[string]dyn.Value{ - "a": dyn.V("1"), - }) - - err := dyn.WalkReadOnly(input, func(p dyn.Path, v dyn.Value) error { - if p.Equal(dyn.Path{dyn.Key("a")}) { - return testErr - } - return nil - }) - - assert.Equal(t, err, testErr) -} - -func TestWalkReadOnly_SkipPaths(t *testing.T) { - va := dyn.V(dyn.NewMappingFromPairs( - []dyn.Pair{ - {Key: dyn.V("b"), Value: dyn.V("1")}, - {Key: dyn.V("c"), Value: dyn.V("2")}, - }, - )) - - vd := dyn.V(dyn.NewMappingFromPairs( - []dyn.Pair{ - {Key: dyn.V("e"), Value: dyn.V("3")}, - }, - )) - - input := dyn.V(dyn.NewMappingFromPairs( - []dyn.Pair{ - { - Key: dyn.V("a"), - Value: va, - }, - { - Key: dyn.V("d"), - Value: vd, - }, - { - Key: dyn.V("f"), - Value: dyn.V("4"), - }, - }, - )) - - skipPaths := map[string]bool{ - "a.b": true, - "d": true, - } - - var visitedPaths []dyn.Path - var visitedValues []dyn.Value - - err := dyn.WalkReadOnly(input, func(p dyn.Path, v dyn.Value) error { - _, ok := skipPaths[p.String()] - if ok { - return dyn.ErrSkip - } - - visitedPaths = append(visitedPaths, p) - visitedValues = append(visitedValues, v) - return nil - }) - assert.NoError(t, err) - - expectedPaths := []dyn.Path{ - dyn.EmptyPath, - {dyn.Key("a")}, - {dyn.Key("a"), dyn.Key("c")}, - {dyn.Key("f")}, - } - expectedValues := []dyn.Value{ - input, - va, - dyn.V("2"), - dyn.V("4"), - } - - assert.Equal(t, expectedPaths, visitedPaths) - assert.Equal(t, expectedValues, visitedValues) -} diff --git a/libs/dyn/walk_test.go b/libs/dyn/walk_test.go deleted file mode 100644 index f59683f0381..00000000000 --- a/libs/dyn/walk_test.go +++ /dev/null @@ -1,269 +0,0 @@ -package dyn_test - -import ( - "errors" - "testing" - - . "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// Return values for specific paths. -type walkReturn struct { - path Path - - // Return values. - fn func(Value) Value - err error -} - -// Track the calls to the callback. -type walkCall struct { - path Path - value Value -} - -// Track the calls to the callback. -type walkCallTracker struct { - returns []walkReturn - calls []walkCall -} - -func (w *walkCallTracker) on(path string, fn func(Value) Value, err error) { - w.returns = append(w.returns, walkReturn{MustPathFromString(path), fn, err}) -} - -func (w *walkCallTracker) returnSkip(path string) { - w.on(path, func(v Value) Value { return v }, ErrSkip) -} - -func (w *walkCallTracker) returnDrop(path string) { - w.on(path, func(v Value) Value { return InvalidValue }, ErrDrop) -} - -func (w *walkCallTracker) track(p Path, v Value) (Value, error) { - w.calls = append(w.calls, walkCall{p, v}) - - // Look for matching return. - for _, r := range w.returns { - if p.Equal(r.path) { - return r.fn(v), r.err - } - } - - return v, nil -} - -func TestWalkEmpty(t *testing.T) { - var tracker walkCallTracker - - value := V(nil) - out, err := Walk(value, tracker.track) - require.NoError(t, err) - assert.Equal(t, value, out) - - // The callback should have been called once. - assert.Len(t, tracker.calls, 1) - - // The call should have been made with the empty path. - assert.Equal(t, EmptyPath, tracker.calls[0].path) - - // The value should be the same as the input. - assert.Equal(t, value, tracker.calls[0].value) -} - -func TestWalkMapSkip(t *testing.T) { - var tracker walkCallTracker - - // Skip traversal of the root value. - tracker.returnSkip(".") - - value := V(map[string]Value{ - "key": V("value"), - }) - out, err := Walk(value, tracker.track) - require.NoError(t, err) - assert.Equal( - t, - V(map[string]Value{ - "key": V("value"), - }), - out, - ) - - // The callback should have been called once. - assert.Len(t, tracker.calls, 1) - - // The call should have been made with the empty path. - assert.Equal(t, EmptyPath, tracker.calls[0].path) - - // The value should be the same as the input. - assert.Equal(t, value, tracker.calls[0].value) -} - -func TestWalkMapDrop(t *testing.T) { - var tracker walkCallTracker - - // Drop the value at key "foo". - tracker.returnDrop(".foo") - - value := V(map[string]Value{ - "foo": V("bar"), - "bar": V("baz"), - }) - out, err := Walk(value, tracker.track) - require.NoError(t, err) - assert.Equal( - t, - V(map[string]Value{ - "bar": V("baz"), - }), - out, - ) - - // The callback should have been called for the root and every key in the map. - assert.Len(t, tracker.calls, 3) - - // Calls 2 and 3 have been made for the keys in the map. - assert.ElementsMatch(t, - []Path{ - tracker.calls[1].path, - tracker.calls[2].path, - }, []Path{ - MustPathFromString(".foo"), - MustPathFromString(".bar"), - }) -} - -func TestWalkMapError(t *testing.T) { - var tracker walkCallTracker - - // Return an error from the callback for key "foo". - cerr := errors.New("error!") - tracker.on(".foo", func(v Value) Value { return v }, cerr) - - value := V(map[string]Value{ - "foo": V("bar"), - }) - out, err := Walk(value, tracker.track) - assert.Equal(t, cerr, err) - assert.Equal(t, InvalidValue, out) - - // The callback should have been called twice. - assert.Len(t, tracker.calls, 2) - - // The second call was for the value at key "foo". - assert.Equal(t, MustPathFromString(".foo"), tracker.calls[1].path) -} - -func TestWalkSequenceSkip(t *testing.T) { - var tracker walkCallTracker - - // Skip traversal of the root value. - tracker.returnSkip(".") - - value := V([]Value{ - V("foo"), - V("bar"), - }) - out, err := Walk(value, tracker.track) - require.NoError(t, err) - assert.Equal( - t, - V([]Value{ - V("foo"), - V("bar"), - }), - out, - ) - - // The callback should have been called once. - assert.Len(t, tracker.calls, 1) - - // The call should have been made with the empty path. - assert.Equal(t, EmptyPath, tracker.calls[0].path) - - // The value should be the same as the input. - assert.Equal(t, value, tracker.calls[0].value) -} - -func TestWalkSequenceDrop(t *testing.T) { - var tracker walkCallTracker - - // Drop the value at index 1. - tracker.returnDrop(".[1]") - - value := V([]Value{ - V("foo"), - V("bar"), - V("baz"), - }) - out, err := Walk(value, tracker.track) - require.NoError(t, err) - assert.Equal( - t, - V([]Value{ - V("foo"), - V("baz"), - }), - out, - ) - - // The callback should have been called for the root and every value in the sequence. - assert.Len(t, tracker.calls, 4) - - // The second call was for the value at index 0. - assert.Equal(t, MustPathFromString(".[0]"), tracker.calls[1].path) - assert.Equal(t, V("foo"), tracker.calls[1].value) - - // The third call was for the value at index 1. - assert.Equal(t, MustPathFromString(".[1]"), tracker.calls[2].path) - assert.Equal(t, V("bar"), tracker.calls[2].value) - - // The fourth call was for the value at index 2. - assert.Equal(t, MustPathFromString(".[2]"), tracker.calls[3].path) - assert.Equal(t, V("baz"), tracker.calls[3].value) -} - -func TestWalkSequenceError(t *testing.T) { - var tracker walkCallTracker - - // Return an error from the callback for index 1. - cerr := errors.New("error!") - tracker.on(".[1]", func(v Value) Value { return v }, cerr) - - value := V([]Value{ - V("foo"), - V("bar"), - }) - out, err := Walk(value, tracker.track) - assert.Equal(t, cerr, err) - assert.Equal(t, InvalidValue, out) - - // The callback should have been called three times. - assert.Len(t, tracker.calls, 3) - - // The second call was for the value at index 0. - assert.Equal(t, MustPathFromString(".[0]"), tracker.calls[1].path) - assert.Equal(t, V("foo"), tracker.calls[1].value) - - // The third call was for the value at index 1. - assert.Equal(t, MustPathFromString(".[1]"), tracker.calls[2].path) - assert.Equal(t, V("bar"), tracker.calls[2].value) -} - -func TestCollectLeafPaths(t *testing.T) { - v := V(map[string]Value{ - "a": V(1), - "b": V(map[string]Value{ - "c": V(2), - "d": V(map[string]Value{ - "e": V(3), - }), - }), - "f": V([]Value{V(4), V(5)}), - }) - paths := CollectLeafPaths(v) - assert.ElementsMatch(t, []string{"a", "b.c", "b.d.e", "f[0]", "f[1]"}, paths) -} diff --git a/libs/dyn/yamlloader/loader.go b/libs/dyn/yamlloader/loader.go deleted file mode 100644 index 0f69f0c638b..00000000000 --- a/libs/dyn/yamlloader/loader.go +++ /dev/null @@ -1,290 +0,0 @@ -package yamlloader - -import ( - "fmt" - "math" - "strconv" - "strings" - - "github.com/databricks/cli/libs/dyn" - "go.yaml.in/yaml/v3" -) - -// LocationError is an error with a YAML source location that can be displayed -// to the user with a file path, line, and column number. -type LocationError struct { - Loc dyn.Location - Summary string -} - -func (e *LocationError) Error() string { - return fmt.Sprintf("yaml (%s): %s", e.Loc, e.Summary) -} - -type loader struct { - path string - - // Aliases currently being expanded. A self-referential anchor (e.g. - // "a: &x {b: *x}") makes the yaml.v3 node graph cyclic; without this - // bookkeeping, expansion would recurse until stack overflow. - activeAliases map[*yaml.Node]bool -} - -func errorf(loc dyn.Location, format string, args ...any) error { - return fmt.Errorf("yaml (%s): %s", loc, fmt.Sprintf(format, args...)) -} - -func newLoader(path string) *loader { - return &loader{ - path: path, - activeAliases: make(map[*yaml.Node]bool), - } -} - -func (d *loader) location(node *yaml.Node) dyn.Location { - return dyn.Location{ - File: d.path, - Line: node.Line, - Column: node.Column, - } -} - -func (d *loader) load(node *yaml.Node) (dyn.Value, error) { - loc := dyn.Location{ - File: d.path, - Line: node.Line, - Column: node.Column, - } - - var value dyn.Value - var err error - - switch node.Kind { - case yaml.DocumentNode: - value, err = d.loadDocument(node, loc) - case yaml.SequenceNode: - value, err = d.loadSequence(node, loc) - case yaml.MappingNode: - value, err = d.loadMapping(node, loc) - case yaml.ScalarNode: - value, err = d.loadScalar(node, loc) - case yaml.AliasNode: - value, err = d.loadAlias(node, loc) - default: - return dyn.InvalidValue, errorf(loc, "unknown node kind: %v", node.Kind) - } - - if err != nil { - return value, err - } - - // Mark value as anchor if needed. - // If this node doesn't map to a type, we don't need to warn about it. - if node.Anchor != "" { - value = value.MarkAnchor() - } - - return value, nil -} - -func (d *loader) loadDocument(node *yaml.Node, loc dyn.Location) (dyn.Value, error) { - return d.load(node.Content[0]) -} - -func (d *loader) loadSequence(node *yaml.Node, loc dyn.Location) (dyn.Value, error) { - acc := make([]dyn.Value, len(node.Content)) - for i, n := range node.Content { - v, err := d.load(n) - if err != nil { - return dyn.InvalidValue, err - } - - acc[i] = v - } - - return dyn.NewValue(acc, []dyn.Location{loc}), nil -} - -func (d *loader) loadMapping(node *yaml.Node, loc dyn.Location) (dyn.Value, error) { - var merge *yaml.Node - - acc := dyn.NewMapping() - for i := 0; i < len(node.Content); i += 2 { - key := node.Content[i] - val := node.Content[i+1] - - // Assert that keys are strings - if key.Kind != yaml.ScalarNode { - return dyn.InvalidValue, errorf(loc, "key is not a scalar") - } - - st := key.ShortTag() - switch st { - case "!!str": - // OK - case "!!null": - // A literal unquoted "null" is treated as a null value by the YAML parser. - // However, when used as a key, it is treated as the string "null". - case "!!merge": - if merge != nil { - // The YAML merge key spec allows a single '<<' key per mapping. - // To merge multiple maps, use a sequence: '<<: [*anchor1, *anchor2]'. - return dyn.InvalidValue, &LocationError{ - Loc: d.location(key), - Summary: "duplicate YAML merge key ('<<') is not allowed; to merge multiple maps, use a sequence: '<<: [*anchor1, *anchor2]'", - } - } - merge = val - continue - default: - return dyn.InvalidValue, errorf(loc, "invalid key tag: %v", st) - } - - loc := []dyn.Location{{ - File: d.path, - Line: key.Line, - Column: key.Column, - }} - - v, err := d.load(val) - if err != nil { - return dyn.InvalidValue, err - } - - acc.SetLoc(key.Value, loc, v) - } - - if merge == nil { - return dyn.NewValue(acc, []dyn.Location{loc}), nil - } - - // Build location for the merge node. - mloc := d.location(merge) - merr := errorf(mloc, "map merge requires map or sequence of maps as the value") - - // Flatten the merge node into a slice of nodes. - // It can be either a single node or a sequence of nodes. - var mnodes []*yaml.Node - switch merge.Kind { - case yaml.SequenceNode: - mnodes = merge.Content - case yaml.AliasNode: - mnodes = []*yaml.Node{merge} - default: - return dyn.NilValue, merr - } - - // Build a sequence of values to merge. - // The entries that we already accumulated have precedence. - var seq []dyn.Mapping - for _, n := range mnodes { - v, err := d.load(n) - if err != nil { - return dyn.InvalidValue, err - } - m, ok := v.AsMap() - if !ok { - return dyn.NilValue, merr - } - seq = append(seq, m) - } - - // Append the accumulated entries to the sequence. - seq = append(seq, acc) - out := dyn.NewMapping() - for _, m := range seq { - out.Merge(m) - } - - return dyn.NewValue(out, []dyn.Location{loc}), nil -} - -func newIntValue(i64 int64, loc dyn.Location) dyn.Value { - // Use regular int type instead of int64 if possible. - if i64 >= math.MinInt32 && i64 <= math.MaxInt32 { - return dyn.NewValue(int(i64), []dyn.Location{loc}) - } - return dyn.NewValue(i64, []dyn.Location{loc}) -} - -func (d *loader) loadScalar(node *yaml.Node, loc dyn.Location) (dyn.Value, error) { - st := node.ShortTag() - switch st { - case "!!str": - return dyn.NewValue(node.Value, []dyn.Location{loc}), nil - case "!!bool": - switch strings.ToLower(node.Value) { - case "true": - return dyn.NewValue(true, []dyn.Location{loc}), nil - case "false": - return dyn.NewValue(false, []dyn.Location{loc}), nil - default: - return dyn.InvalidValue, errorf(loc, "invalid bool value: %v", node.Value) - } - case "!!int": - // Try to parse the an integer value in base 10. - // We trim leading zeros to avoid octal parsing of the "0" prefix. - // See "testdata/spec_example_2.19.yml" for background. - i64, err := strconv.ParseInt(strings.TrimLeft(node.Value, "0"), 10, 64) - if err == nil { - return newIntValue(i64, loc), nil - } - // Let the [ParseInt] function figure out the base. - i64, err = strconv.ParseInt(node.Value, 0, 64) - if err == nil { - return newIntValue(i64, loc), nil - } - return dyn.InvalidValue, errorf(loc, "invalid int value: %v", node.Value) - case "!!float": - f64, err := strconv.ParseFloat(node.Value, 64) - if err != nil { - // Deal with infinity prefixes. - v := strings.ToLower(node.Value) - switch { - case strings.HasPrefix(v, "+"): - v = strings.TrimPrefix(v, "+") - f64 = math.Inf(1) - case strings.HasPrefix(v, "-"): - v = strings.TrimPrefix(v, "-") - f64 = math.Inf(-1) - default: - // No prefix. - f64 = math.Inf(1) - } - - // Deal with infinity and NaN values. - switch v { - case ".inf": - return dyn.NewValue(f64, []dyn.Location{loc}), nil - case ".nan": - return dyn.NewValue(math.NaN(), []dyn.Location{loc}), nil - } - - return dyn.InvalidValue, errorf(loc, "invalid float value: %v", node.Value) - } - return dyn.NewValue(f64, []dyn.Location{loc}), nil - case "!!null": - return dyn.NewValue(nil, []dyn.Location{loc}), nil - case "!!timestamp": - t, err := dyn.NewTime(node.Value) - if err == nil { - return dyn.NewValue(t, []dyn.Location{loc}), nil - } - return dyn.InvalidValue, errorf(loc, "invalid timestamp value: %v", node.Value) - default: - return dyn.InvalidValue, errorf(loc, "unknown tag: %v", st) - } -} - -func (d *loader) loadAlias(node *yaml.Node, loc dyn.Location) (dyn.Value, error) { - if d.activeAliases[node] { - return dyn.InvalidValue, errorf(loc, "cyclic reference to anchor %q", node.Value) - } - - // The same alias node may be reached again through another alias to an - // enclosing anchor, which is not a cycle. - d.activeAliases[node] = true - defer delete(d.activeAliases, node) - - return d.load(node.Alias) -} diff --git a/libs/dyn/yamlloader/testdata/anchor_01.yml b/libs/dyn/yamlloader/testdata/anchor_01.yml deleted file mode 100644 index b677d60b808..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_01.yml +++ /dev/null @@ -1,12 +0,0 @@ -# 1. Basic Anchor and Alias -defaults: &DEFAULTS - color: red - size: large - -shirt1: - <<: *DEFAULTS - pattern: striped - -shirt2: - <<: *DEFAULTS - pattern: plain diff --git a/libs/dyn/yamlloader/testdata/anchor_02.yml b/libs/dyn/yamlloader/testdata/anchor_02.yml deleted file mode 100644 index 86650987f3d..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_02.yml +++ /dev/null @@ -1,13 +0,0 @@ -# 2. Merging Anchors -# Here, multiple anchors can be merged into a single item. -defaults: &DEFAULTS - color: red - size: large - -materials: &MATERIALS - primary: cotton - secondary: polyester - -shirt: - <<: [*DEFAULTS, *MATERIALS] - pattern: striped diff --git a/libs/dyn/yamlloader/testdata/anchor_03.yml b/libs/dyn/yamlloader/testdata/anchor_03.yml deleted file mode 100644 index f06b7999ef0..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_03.yml +++ /dev/null @@ -1,10 +0,0 @@ -# 3. Overriding Merged Anchor Values -# You can override values when merging. -defaults: &DEFAULTS - color: red - size: large - pattern: plain - -shirt: - <<: *DEFAULTS - color: blue diff --git a/libs/dyn/yamlloader/testdata/anchor_04.yml b/libs/dyn/yamlloader/testdata/anchor_04.yml deleted file mode 100644 index bbc4bc02707..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_04.yml +++ /dev/null @@ -1,16 +0,0 @@ -# 4. Nested Anchors -# This demonstrates the reuse of nested content. -address: &ADDRESS - city: San Francisco - state: CA - country: USA - -person1: - name: Alice - address: *ADDRESS - -person2: - name: Bob - address: - <<: *ADDRESS - city: Los Angeles diff --git a/libs/dyn/yamlloader/testdata/anchor_05.yml b/libs/dyn/yamlloader/testdata/anchor_05.yml deleted file mode 100644 index 9a4c961b7c8..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_05.yml +++ /dev/null @@ -1,15 +0,0 @@ -# 5. Using Anchors for List Items -# You can use anchors for list items too. -features: &FEATURES - - wifi - - bluetooth - -phone1: - name: PhoneA - features: *FEATURES - -phone2: - name: PhoneB - features: - - camera - - *FEATURES diff --git a/libs/dyn/yamlloader/testdata/anchor_06.yml b/libs/dyn/yamlloader/testdata/anchor_06.yml deleted file mode 100644 index f27650fad47..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_06.yml +++ /dev/null @@ -1,5 +0,0 @@ -# 6. String Anchors -commonMessage: &msg "Hello, World!" - -greeting1: *msg -greeting2: *msg diff --git a/libs/dyn/yamlloader/testdata/anchor_07.yml b/libs/dyn/yamlloader/testdata/anchor_07.yml deleted file mode 100644 index b8f32214568..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_07.yml +++ /dev/null @@ -1,10 +0,0 @@ -# 7. Number Anchors -defaultAge: &age 25 - -person1: - name: Alice - age: *age - -person2: - name: Bob - age: *age diff --git a/libs/dyn/yamlloader/testdata/anchor_08.yml b/libs/dyn/yamlloader/testdata/anchor_08.yml deleted file mode 100644 index 8b01e21f14b..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_08.yml +++ /dev/null @@ -1,10 +0,0 @@ -# 8. Boolean Anchors -isActive: &active true - -user1: - username: user1 - active: *active - -user2: - username: user2 - active: *active diff --git a/libs/dyn/yamlloader/testdata/anchor_09.yml b/libs/dyn/yamlloader/testdata/anchor_09.yml deleted file mode 100644 index 8005bf445ef..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_09.yml +++ /dev/null @@ -1,2 +0,0 @@ -anchortest: &x - child: *x diff --git a/libs/dyn/yamlloader/testdata/anchor_10.yml b/libs/dyn/yamlloader/testdata/anchor_10.yml deleted file mode 100644 index 013832ab3ac..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_10.yml +++ /dev/null @@ -1,5 +0,0 @@ -leaf: &leaf 1 -shared: &shared - value: *leaf -use1: *shared -use2: *shared diff --git a/libs/dyn/yamlloader/testdata/empty.yml b/libs/dyn/yamlloader/testdata/empty.yml deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/libs/dyn/yamlloader/testdata/error_01.yml b/libs/dyn/yamlloader/testdata/error_01.yml deleted file mode 100644 index 11b96486e32..00000000000 --- a/libs/dyn/yamlloader/testdata/error_01.yml +++ /dev/null @@ -1,6 +0,0 @@ -# Use string anchor to extend a mapping. -str: &str "Hello world!" - -map: - <<: *str - key: value diff --git a/libs/dyn/yamlloader/testdata/error_02.yml b/libs/dyn/yamlloader/testdata/error_02.yml deleted file mode 100644 index 5e9c6276d09..00000000000 --- a/libs/dyn/yamlloader/testdata/error_02.yml +++ /dev/null @@ -1,6 +0,0 @@ -# Use string anchor inside sequence to extend a mapping. -str: &str "Hello world!" - -map: - <<: [*str] - key: value diff --git a/libs/dyn/yamlloader/testdata/error_03.yml b/libs/dyn/yamlloader/testdata/error_03.yml deleted file mode 100644 index c4d3c397dd9..00000000000 --- a/libs/dyn/yamlloader/testdata/error_03.yml +++ /dev/null @@ -1,4 +0,0 @@ -# Extend a mapping with a literal string. -map: - <<: value - key: value diff --git a/libs/dyn/yamlloader/testdata/mix_01.yml b/libs/dyn/yamlloader/testdata/mix_01.yml deleted file mode 100644 index 98706bf6f01..00000000000 --- a/libs/dyn/yamlloader/testdata/mix_01.yml +++ /dev/null @@ -1,90 +0,0 @@ -# Scalars - -# Strings can be unquoted -name: John Doe - -# Strings can be single quoted -single_quoted_string: 'This is a single quoted string' - -# Strings can be double quoted (allows for escape sequences) -double_quoted_string: "This is a double quoted string with an escaped newline \n and tab \t." - -# Multiline string with folded style (newlines become spaces) -folded_style: > - This is a very - long string that - spans several lines. - -# Multiline string with literal block style (newlines are preserved) -literal_style: | - Line 1 - Line 2 - Line 3 - -# Integers -age: 30 - -# Floating point -price: 19.99 - -# Boolean values -is_student: true -is_employed: false - -# Null value -middle_name: ~ # null can also be represented with 'null' - -# Timestamp -timestamp: 2023-01-01T12:00:00Z - -# Sequences (lists) - -# Inline list -fruits: [apple, banana, cherry] - -# Block style list -books: - - 'Moby Dick' - - '1984' - - 'Pride and Prejudice' - -# Mappings (dictionaries) - -# Inline dictionary -address: {street: '123 Main St', city: 'Anytown', zip: '12345'} - -# Block style dictionary -employee: - first_name: Jane - last_name: Smith - age: 40 - -# Edge cases and advanced features - -# Strings that look like other types must be quoted -looks_like_number: '12345' -looks_like_boolean: 'yes' -looks_like_null: 'null' -looks_like_timestamp: '2023-01-01T12:00:00Z' - -# Using anchors and aliases to reuse properties -base_address: &base - street: '456 Elm St' - city: 'Sometown' - -office_address: - <<: *base # Merge the base address into this mapping - suite: 500 - -# Nested structures -users: - - name: Alice - age: 28 - interests: - - reading - - cycling - - name: Bob - age: 35 - interests: - - cooking - - hiking diff --git a/libs/dyn/yamlloader/testdata/mix_02.yml b/libs/dyn/yamlloader/testdata/mix_02.yml deleted file mode 100644 index 757d626dbfe..00000000000 --- a/libs/dyn/yamlloader/testdata/mix_02.yml +++ /dev/null @@ -1,49 +0,0 @@ -# Scalars with special characters - -# Commas and square brackets in strings should be enclosed in quotes -special_chars: "[This, string] has, special chars." - -# Strings starting with reserved indicators must be quoted -reserved_string: "@not_a_directive" -colon_string: "this: looks like a mapping, but it's not" - -# Explicit data type declaration -explicit_string: !!str 12345 - -# Sequences with nested mappings and lists - -teams: - - - name: Team A - members: - - Alice - - Bob - tasks: - - task1: - status: incomplete - due_date: 2023-01-15 - - task2: - status: complete - -# Complex mapping keys -? | - Multi-line key - which is unusual but valid -: multi-line key's value - -"complex key: with colon": complex key's value - -# Set (a mapping with null values) -set_example: - item1: ~ - item2: ~ - -# Merge multiple mappings (with override) -base_colors: &colors - red: "#FF0000" - blue: "#0000FF" - green: "#00FF00" - -theme: - <<: *colors - blue: "#001122" # Overriding the blue color from base_colors diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.1.yml b/libs/dyn/yamlloader/testdata/spec_example_2.1.yml deleted file mode 100644 index c9e26274c09..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.1.yml +++ /dev/null @@ -1,5 +0,0 @@ -# Example 2.1 Sequence of Scalars (ball players) - -- Mark McGwire -- Sammy Sosa -- Ken Griffey diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.10.yml b/libs/dyn/yamlloader/testdata/spec_example_2.10.yml deleted file mode 100644 index a3459dedfc8..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.10.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Example 2.10 Node for “Sammy Sosa” appears twice in this document - ---- -hr: -- Mark McGwire -# Following node labeled SS -- &SS Sammy Sosa -rbi: -- *SS # Subsequent occurrence -- Ken Griffey diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.11.yml b/libs/dyn/yamlloader/testdata/spec_example_2.11.yml deleted file mode 100644 index e3c5c115e02..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.11.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Example 2.11 Mapping between Sequences - -? - Detroit Tigers - - Chicago cubs -: - 2001-07-23 - -? [ New York Yankees, - Atlanta Braves ] -: [ 2001-07-02, 2001-08-12, - 2001-08-14 ] diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.12.yml b/libs/dyn/yamlloader/testdata/spec_example_2.12.yml deleted file mode 100644 index eb4a526f65b..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.12.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Example 2.12 Compact Nested Mapping - ---- -# Products purchased -- item : Super Hoop - quantity: 1 -- item : Basketball - quantity: 4 -- item : Big Shoes - quantity: 1 diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.13.yml b/libs/dyn/yamlloader/testdata/spec_example_2.13.yml deleted file mode 100644 index e55abff107e..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.13.yml +++ /dev/null @@ -1,6 +0,0 @@ -# Example 2.13 In literals, newlines are preserved - -# ASCII Art ---- | - \//||\/|| - // || ||__ diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.14.yml b/libs/dyn/yamlloader/testdata/spec_example_2.14.yml deleted file mode 100644 index 439fca30cb1..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.14.yml +++ /dev/null @@ -1,6 +0,0 @@ -# Example 2.14 In the folded scalars, newlines become spaces - ---- > - Mark McGwire's - year was crippled - by a knee injury. diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.15.yml b/libs/dyn/yamlloader/testdata/spec_example_2.15.yml deleted file mode 100644 index 266e7ce4711..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.15.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Example 2.15 Folded newlines are preserved for “more indented” and blank lines - ---- > - Sammy Sosa completed another - fine season with great stats. - - 63 Home Runs - 0.288 Batting Average - - What a year! diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.16.yml b/libs/dyn/yamlloader/testdata/spec_example_2.16.yml deleted file mode 100644 index 6db6b0878f8..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.16.yml +++ /dev/null @@ -1,9 +0,0 @@ -# Example 2.16 Indentation determines scope - -name: Mark McGwire -accomplishment: > - Mark set a major league - home run record in 1998. -stats: | - 65 Home Runs - 0.278 Batting Average diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.17.yml b/libs/dyn/yamlloader/testdata/spec_example_2.17.yml deleted file mode 100644 index af0777abab5..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.17.yml +++ /dev/null @@ -1,9 +0,0 @@ -# Example 2.17 Quoted Scalars - -unicode: "Sosa did fine.\u263A" -control: "\b1998\t1999\t2000\n" -hex esc: "\x0d\x0a is \r\n" - -single: '"Howdy!" he cried.' -quoted: ' # Not a ''comment''.' -tie-fighter: '|\-*-/|' diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.18.yml b/libs/dyn/yamlloader/testdata/spec_example_2.18.yml deleted file mode 100644 index 741bcd8c0c0..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.18.yml +++ /dev/null @@ -1,8 +0,0 @@ -# Example 2.18 Multi-line Flow Scalars - -plain: - This unquoted scalar - spans many lines. - -quoted: "So does this - quoted scalar.\n" diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.19.yml b/libs/dyn/yamlloader/testdata/spec_example_2.19.yml deleted file mode 100644 index 6ed95e092aa..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.19.yml +++ /dev/null @@ -1,15 +0,0 @@ -# Example 2.19 Integers - -canonical: 12345 -decimal: +12345 -octal: 0o14 -hexadecimal: 0xC - -# Note: this example is not part of the spec but added for completeness. -# -# Octal numbers: -# - YAML 1.1: prefix is "0" -# - YAML 1.2: prefix is "0o" -# The "gopkg.in/yaml.v3" package accepts both for backwards compat. -# We accept only the YAML 1.2 prefix "0o". -octal11: 012345 diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.2.yml b/libs/dyn/yamlloader/testdata/spec_example_2.2.yml deleted file mode 100644 index 29c16105e3b..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.2.yml +++ /dev/null @@ -1,5 +0,0 @@ -# Example 2.2 Mapping Scalars to Scalars (player statistics) - -hr: 65 # Home runs -avg: 0.278 # Batting average -rbi: 147 # Runs Batted In diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.20.yml b/libs/dyn/yamlloader/testdata/spec_example_2.20.yml deleted file mode 100644 index 77a79a0c3e0..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.20.yml +++ /dev/null @@ -1,7 +0,0 @@ -# Example 2.20 Floating Point - -canonical: 1.23015e+3 -exponential: 12.3015e+02 -fixed: 1230.15 -negative infinity: -.inf -not a number: .nan diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.21.yml b/libs/dyn/yamlloader/testdata/spec_example_2.21.yml deleted file mode 100644 index cdb423c566e..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.21.yml +++ /dev/null @@ -1,5 +0,0 @@ -# Example 2.21 Miscellaneous - -null: -booleans: [ true, false ] -string: '012345' diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.22.yml b/libs/dyn/yamlloader/testdata/spec_example_2.22.yml deleted file mode 100644 index bef2addfa07..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.22.yml +++ /dev/null @@ -1,6 +0,0 @@ -# Example 2.22 Timestamps - -canonical: 2001-12-15T02:59:43.1Z -iso8601: 2001-12-14t21:59:43.10-05:00 -spaced: 2001-12-14 21:59:43.10 -5 -date: 2002-12-14 diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.23.yml b/libs/dyn/yamlloader/testdata/spec_example_2.23.yml deleted file mode 100644 index 56e9898e21b..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.23.yml +++ /dev/null @@ -1,15 +0,0 @@ -# Example 2.23 Various Explicit Tags - ---- -not-date: !!str 2002-04-28 - -picture: !!binary | - R0lGODlhDAAMAIQAAP//9/X - 17unp5WZmZgAAAOfn515eXv - Pz7Y6OjuDg4J+fn5OTk6enp - 56enmleECcgggoBADs= - -application specific tag: !something | - The semantics of the tag - above may be different for - different documents. diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.24.yml b/libs/dyn/yamlloader/testdata/spec_example_2.24.yml deleted file mode 100644 index f7c11d0fb11..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.24.yml +++ /dev/null @@ -1,16 +0,0 @@ -# Example 2.24 Global Tags - -%TAG ! tag:clarkevans.com,2002: ---- !shape - # Use the ! handle for presenting - # tag:clarkevans.com,2002:circle -- !circle - center: &ORIGIN {x: 73, y: 129} - radius: 7 -- !line - start: *ORIGIN - finish: { x: 89, y: 102 } -- !label - start: *ORIGIN - color: 0xFFEEBB - text: Pretty vector drawing. diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.25.yml b/libs/dyn/yamlloader/testdata/spec_example_2.25.yml deleted file mode 100644 index 73bac862755..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.25.yml +++ /dev/null @@ -1,9 +0,0 @@ -# Example 2.25 Unordered Sets - -# Sets are represented as a -# Mapping where each key is -# associated with a null value ---- !!set -? Mark McGwire -? Sammy Sosa -? Ken Griffey diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.26.yml b/libs/dyn/yamlloader/testdata/spec_example_2.26.yml deleted file mode 100644 index 00863a6b854..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.26.yml +++ /dev/null @@ -1,9 +0,0 @@ -# Example 2.26 Ordered Mappings - -# Ordered maps are represented as -# A sequence of mappings, with -# each mapping having one key ---- !!omap -- Mark McGwire: 65 -- Sammy Sosa: 63 -- Ken Griffey: 58 diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.27.yml b/libs/dyn/yamlloader/testdata/spec_example_2.27.yml deleted file mode 100644 index fc9b460c299..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.27.yml +++ /dev/null @@ -1,31 +0,0 @@ -# Example 2.27 Invoice - ---- ! -invoice: 34843 -date : 2001-01-23 -bill-to: &id001 - given : Chris - family : Dumars - address: - lines: | - 458 Walkman Dr. - Suite #292 - city : Royal Oak - state : MI - postal : 48046 -ship-to: *id001 -product: -- sku : BL394D - quantity : 4 - description : Basketball - price : 450.00 -- sku : BL4438H - quantity : 1 - description : Super Hoop - price : 2392.00 -tax : 251.42 -total: 4443.52 -comments: - Late afternoon is best. - Backup contact is Nancy - Billsmer @ 338-4338. diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.28.yml b/libs/dyn/yamlloader/testdata/spec_example_2.28.yml deleted file mode 100644 index 35369472967..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.28.yml +++ /dev/null @@ -1,28 +0,0 @@ -# Example 2.28 Log File - ---- -Time: 2001-11-23 15:01:42 -5 -User: ed -Warning: - This is an error message - for the log file ---- -Time: 2001-11-23 15:02:31 -5 -User: ed -Warning: - A slightly different error - message. ---- -Date: 2001-11-23 15:03:17 -5 -User: ed -Fatal: - Unknown variable "bar" -Stack: -- file: TopClass.py - line: 23 - code: | - x = MoreObject("345\n") -- file: MoreClass.py - line: 58 - code: |- - foo = bar diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.3.yml b/libs/dyn/yamlloader/testdata/spec_example_2.3.yml deleted file mode 100644 index 70cbe07d391..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.3.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Example 2.3 Mapping Scalars to Sequences (ball clubs in each league) - -american: -- Boston Red Sox -- Detroit Tigers -- New York Yankees -national: -- New York Mets -- Chicago Cubs -- Atlanta Braves diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.4.yml b/libs/dyn/yamlloader/testdata/spec_example_2.4.yml deleted file mode 100644 index cce28625f56..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.4.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Example 2.4 Sequence of Mappings (players’ statistics) - -- - name: Mark McGwire - hr: 65 - avg: 0.278 -- - name: Sammy Sosa - hr: 63 - avg: 0.288 diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.5.yml b/libs/dyn/yamlloader/testdata/spec_example_2.5.yml deleted file mode 100644 index a585faee790..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.5.yml +++ /dev/null @@ -1,5 +0,0 @@ -# Example 2.5 Sequence of Sequences - -- [name , hr, avg ] -- [Mark McGwire, 65, 0.278] -- [Sammy Sosa , 63, 0.288] diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.6.yml b/libs/dyn/yamlloader/testdata/spec_example_2.6.yml deleted file mode 100644 index cc137e5d56d..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.6.yml +++ /dev/null @@ -1,7 +0,0 @@ -# Example 2.6 Mapping of Mappings - -Mark McGwire: {hr: 65, avg: 0.278} -Sammy Sosa: { - hr: 63, - avg: 0.288, - } diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.7.yml b/libs/dyn/yamlloader/testdata/spec_example_2.7.yml deleted file mode 100644 index 35c2541d6cc..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.7.yml +++ /dev/null @@ -1,12 +0,0 @@ -# Example 2.7 Two Documents in a Stream (each with a leading comment) - -# Ranking of 1998 home runs ---- -- Mark McGwire -- Sammy Sosa -- Ken Griffey - -# Team ranking ---- -- Chicago Cubs -- St Louis Cardinals diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.8.yml b/libs/dyn/yamlloader/testdata/spec_example_2.8.yml deleted file mode 100644 index ae6e8bf279f..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.8.yml +++ /dev/null @@ -1,12 +0,0 @@ -# Example 2.8 Play by Play Feed from a Game - ---- -time: 20:03:20 -player: Sammy Sosa -action: strike (miss) -... ---- -time: 20:03:47 -player: Sammy Sosa -action: grand slam -... diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.9.yml b/libs/dyn/yamlloader/testdata/spec_example_2.9.yml deleted file mode 100644 index 75217b25ec9..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.9.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Example 2.9 Single Document with Two Comments - ---- -hr: # 1998 hr ranking -- Mark McGwire -- Sammy Sosa -# 1998 rbi ranking -rbi: -- Sammy Sosa -- Ken Griffey diff --git a/libs/dyn/yamlloader/yaml.go b/libs/dyn/yamlloader/yaml.go deleted file mode 100644 index 1225fcc7e2f..00000000000 --- a/libs/dyn/yamlloader/yaml.go +++ /dev/null @@ -1,22 +0,0 @@ -package yamlloader - -import ( - "io" - - "github.com/databricks/cli/libs/dyn" - "go.yaml.in/yaml/v3" -) - -func LoadYAML(path string, r io.Reader) (dyn.Value, error) { - var node yaml.Node - dec := yaml.NewDecoder(r) - err := dec.Decode(&node) - if err != nil { - if err == io.EOF { - return dyn.NilValue, nil - } - return dyn.InvalidValue, err - } - - return newLoader(path).load(&node) -} diff --git a/libs/dyn/yamlloader/yaml_anchor_test.go b/libs/dyn/yamlloader/yaml_anchor_test.go deleted file mode 100644 index 53cd6945f93..00000000000 --- a/libs/dyn/yamlloader/yaml_anchor_test.go +++ /dev/null @@ -1,139 +0,0 @@ -package yamlloader_test - -import ( - "bytes" - "os" - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlloader" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestYAMLAnchor01(t *testing.T) { - file := "testdata/anchor_01.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - assert.True(t, self.Get("defaults").IsAnchor()) - assert.False(t, self.Get("shirt1").IsAnchor()) - assert.False(t, self.Get("shirt2").IsAnchor()) - - pattern := self.Get("shirt1").Get("pattern") - assert.Equal(t, "striped", pattern.AsAny()) - assert.Equal(t, dyn.Location{File: file, Line: 8, Column: 12}, pattern.Location()) -} - -func TestYAMLAnchor02(t *testing.T) { - file := "testdata/anchor_02.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - color := self.Get("shirt").Get("color") - assert.Equal(t, "red", color.AsAny()) - assert.Equal(t, dyn.Location{File: file, Line: 4, Column: 10}, color.Location()) - - primary := self.Get("shirt").Get("primary") - assert.Equal(t, "cotton", primary.AsAny()) - assert.Equal(t, dyn.Location{File: file, Line: 8, Column: 12}, primary.Location()) - - pattern := self.Get("shirt").Get("pattern") - assert.Equal(t, "striped", pattern.AsAny()) - assert.Equal(t, dyn.Location{File: file, Line: 13, Column: 12}, pattern.Location()) -} - -func TestYAMLAnchor03(t *testing.T) { - file := "testdata/anchor_03.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - // Assert the override took place. - blue := self.Get("shirt").Get("color") - assert.Equal(t, "blue", blue.AsAny()) - assert.Equal(t, file, blue.Location().File) - assert.Equal(t, 10, blue.Location().Line) - assert.Equal(t, 10, blue.Location().Column) -} - -func TestYAMLAnchor04(t *testing.T) { - file := "testdata/anchor_04.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - p1 := self.Get("person1").Get("address").Get("city") - assert.Equal(t, "San Francisco", p1.AsAny()) - assert.Equal(t, dyn.Location{File: file, Line: 4, Column: 9}, p1.Location()) - - p2 := self.Get("person2").Get("address").Get("city") - assert.Equal(t, "Los Angeles", p2.AsAny()) - assert.Equal(t, dyn.Location{File: file, Line: 16, Column: 11}, p2.Location()) -} - -func TestYAMLAnchor05(t *testing.T) { - file := "testdata/anchor_05.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - features := self.Get("phone1").Get("features") - assert.Equal(t, "wifi", features.Index(0).AsAny()) - assert.Equal(t, dyn.Location{File: file, Line: 4, Column: 5}, features.Index(0).Location()) - assert.Equal(t, "bluetooth", features.Index(1).AsAny()) - assert.Equal(t, dyn.Location{File: file, Line: 5, Column: 5}, features.Index(1).Location()) -} - -func TestYAMLAnchor06(t *testing.T) { - file := "testdata/anchor_06.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - greeting := self.Get("greeting1") - assert.Equal(t, "Hello, World!", greeting.AsAny()) - assert.Equal(t, dyn.Location{File: file, Line: 2, Column: 16}, greeting.Location()) -} - -func TestYAMLAnchor07(t *testing.T) { - file := "testdata/anchor_07.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - name := self.Get("person1").Get("name") - assert.Equal(t, "Alice", name.AsAny()) - assert.Equal(t, dyn.Location{File: file, Line: 5, Column: 9}, name.Location()) - - age := self.Get("person1").Get("age") - assert.Equal(t, 25, age.AsAny()) - assert.Equal(t, dyn.Location{File: file, Line: 2, Column: 13}, age.Location()) -} - -func TestYAMLAnchor08(t *testing.T) { - file := "testdata/anchor_08.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - username := self.Get("user1").Get("username") - assert.Equal(t, "user1", username.AsAny()) - assert.Equal(t, dyn.Location{File: file, Line: 5, Column: 13}, username.Location()) - - active := self.Get("user1").Get("active") - assert.Equal(t, true, active.AsAny()) - assert.Equal(t, dyn.Location{File: file, Line: 2, Column: 11}, active.Location()) -} - -func TestYAMLAnchor09(t *testing.T) { - file := "testdata/anchor_09.yml" - input, err := os.ReadFile(file) - require.NoError(t, err) - - _, err = yamlloader.LoadYAML(file, bytes.NewBuffer(input)) - assert.ErrorContains(t, err, `cyclic reference to anchor "x"`) -} - -func TestYAMLAnchor10(t *testing.T) { - file := "testdata/anchor_10.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - assert.Equal(t, 1, self.Get("use1").Get("value").AsAny()) - assert.Equal(t, 1, self.Get("use2").Get("value").AsAny()) -} diff --git a/libs/dyn/yamlloader/yaml_error_test.go b/libs/dyn/yamlloader/yaml_error_test.go deleted file mode 100644 index a18f9599b05..00000000000 --- a/libs/dyn/yamlloader/yaml_error_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package yamlloader_test - -import ( - "bytes" - "os" - "testing" - - "github.com/databricks/cli/libs/dyn/yamlloader" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.yaml.in/yaml/v3" -) - -func TestYAMLErrorMapMerge(t *testing.T) { - for _, file := range []string{ - "testdata/error_01.yml", - "testdata/error_02.yml", - "testdata/error_03.yml", - } { - input, err := os.ReadFile(file) - require.NoError(t, err) - - t.Run(file, func(t *testing.T) { - t.Run("reference", func(t *testing.T) { - var ref any - err = yaml.Unmarshal(input, &ref) - assert.ErrorContains(t, err, "map merge requires map or sequence of maps as the value") - }) - - t.Run("self", func(t *testing.T) { - _, err := yamlloader.LoadYAML(file, bytes.NewBuffer(input)) - assert.ErrorContains(t, err, "map merge requires map or sequence of maps as the value") - }) - }) - } -} diff --git a/libs/dyn/yamlloader/yaml_mix_test.go b/libs/dyn/yamlloader/yaml_mix_test.go deleted file mode 100644 index 307b93dbf3a..00000000000 --- a/libs/dyn/yamlloader/yaml_mix_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package yamlloader_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestYAMLMix01(t *testing.T) { - file := "testdata/mix_01.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - assert.True(t, self.Get("base_address").IsAnchor()) - assert.False(t, self.Get("office_address").IsAnchor()) -} - -func TestYAMLMix02(t *testing.T) { - file := "testdata/mix_02.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - assert.True(t, self.Get("base_colors").IsAnchor()) - assert.False(t, self.Get("theme").IsAnchor()) -} diff --git a/libs/dyn/yamlloader/yaml_spec_test.go b/libs/dyn/yamlloader/yaml_spec_test.go deleted file mode 100644 index 9501409c92f..00000000000 --- a/libs/dyn/yamlloader/yaml_spec_test.go +++ /dev/null @@ -1,823 +0,0 @@ -package yamlloader_test - -import ( - "bytes" - "math" - "os" - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynassert" - "github.com/databricks/cli/libs/dyn/yamlloader" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -const NL = "\n" - -func loadExample(t *testing.T, file string) dyn.Value { - input, err := os.ReadFile(file) - require.NoError(t, err) - self, err := yamlloader.LoadYAML(file, bytes.NewBuffer(input)) - require.NoError(t, err) - return self -} - -func TestYAMLSpecExample_2_1(t *testing.T) { - file := "testdata/spec_example_2.1.yml" - self := loadExample(t, file) - - assert.Equal(t, dyn.NewValue( - []dyn.Value{ - dyn.NewValue("Mark McGwire", []dyn.Location{{File: file, Line: 3, Column: 3}}), - dyn.NewValue("Sammy Sosa", []dyn.Location{{File: file, Line: 4, Column: 3}}), - dyn.NewValue("Ken Griffey", []dyn.Location{{File: file, Line: 5, Column: 3}}), - }, - []dyn.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_2(t *testing.T) { - file := "testdata/spec_example_2.2.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "hr": dyn.NewValue(65, []dyn.Location{{File: file, Line: 3, Column: 6}}), - "avg": dyn.NewValue(0.278, []dyn.Location{{File: file, Line: 4, Column: 6}}), - "rbi": dyn.NewValue(147, []dyn.Location{{File: file, Line: 5, Column: 6}}), - }, - []dyn.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_3(t *testing.T) { - file := "testdata/spec_example_2.3.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "american": dyn.NewValue( - []dyn.Value{ - dyn.NewValue("Boston Red Sox", []dyn.Location{{File: file, Line: 4, Column: 3}}), - dyn.NewValue("Detroit Tigers", []dyn.Location{{File: file, Line: 5, Column: 3}}), - dyn.NewValue("New York Yankees", []dyn.Location{{File: file, Line: 6, Column: 3}}), - }, - []dyn.Location{{File: file, Line: 4, Column: 1}}, - ), - "national": dyn.NewValue( - []dyn.Value{ - dyn.NewValue("New York Mets", []dyn.Location{{File: file, Line: 8, Column: 3}}), - dyn.NewValue("Chicago Cubs", []dyn.Location{{File: file, Line: 9, Column: 3}}), - dyn.NewValue("Atlanta Braves", []dyn.Location{{File: file, Line: 10, Column: 3}}), - }, - []dyn.Location{{File: file, Line: 8, Column: 1}}, - ), - }, - []dyn.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_4(t *testing.T) { - file := "testdata/spec_example_2.4.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - []dyn.Value{ - dyn.NewValue( - map[string]dyn.Value{ - "name": dyn.NewValue("Mark McGwire", []dyn.Location{{File: file, Line: 4, Column: 9}}), - "hr": dyn.NewValue(65, []dyn.Location{{File: file, Line: 5, Column: 9}}), - "avg": dyn.NewValue(0.278, []dyn.Location{{File: file, Line: 6, Column: 9}}), - }, - []dyn.Location{{File: file, Line: 4, Column: 3}}, - ), - dyn.NewValue( - map[string]dyn.Value{ - "name": dyn.NewValue("Sammy Sosa", []dyn.Location{{File: file, Line: 8, Column: 9}}), - "hr": dyn.NewValue(63, []dyn.Location{{File: file, Line: 9, Column: 9}}), - "avg": dyn.NewValue(0.288, []dyn.Location{{File: file, Line: 10, Column: 9}}), - }, - []dyn.Location{{File: file, Line: 8, Column: 3}}, - ), - }, - []dyn.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_5(t *testing.T) { - file := "testdata/spec_example_2.5.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - []dyn.Value{ - dyn.NewValue( - []dyn.Value{ - dyn.NewValue("name", []dyn.Location{{File: file, Line: 3, Column: 4}}), - dyn.NewValue("hr", []dyn.Location{{File: file, Line: 3, Column: 18}}), - dyn.NewValue("avg", []dyn.Location{{File: file, Line: 3, Column: 22}}), - }, - []dyn.Location{{File: file, Line: 3, Column: 3}}, - ), - dyn.NewValue( - []dyn.Value{ - dyn.NewValue("Mark McGwire", []dyn.Location{{File: file, Line: 4, Column: 4}}), - dyn.NewValue(65, []dyn.Location{{File: file, Line: 4, Column: 18}}), - dyn.NewValue(0.278, []dyn.Location{{File: file, Line: 4, Column: 22}}), - }, - []dyn.Location{{File: file, Line: 4, Column: 3}}, - ), - dyn.NewValue( - []dyn.Value{ - dyn.NewValue("Sammy Sosa", []dyn.Location{{File: file, Line: 5, Column: 4}}), - dyn.NewValue(63, []dyn.Location{{File: file, Line: 5, Column: 18}}), - dyn.NewValue(0.288, []dyn.Location{{File: file, Line: 5, Column: 22}}), - }, - []dyn.Location{{File: file, Line: 5, Column: 3}}, - ), - }, - []dyn.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_6(t *testing.T) { - file := "testdata/spec_example_2.6.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "Mark McGwire": dyn.NewValue( - map[string]dyn.Value{ - "hr": dyn.NewValue(65, []dyn.Location{{File: file, Line: 3, Column: 20}}), - "avg": dyn.NewValue(0.278, []dyn.Location{{File: file, Line: 3, Column: 29}}), - }, - []dyn.Location{{File: file, Line: 3, Column: 15}}, - ), - "Sammy Sosa": dyn.NewValue( - map[string]dyn.Value{ - "hr": dyn.NewValue(63, []dyn.Location{{File: file, Line: 5, Column: 9}}), - "avg": dyn.NewValue(0.288, []dyn.Location{{File: file, Line: 6, Column: 10}}), - }, - []dyn.Location{{File: file, Line: 4, Column: 13}}, - ), - }, - []dyn.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_7(t *testing.T) { - file := "testdata/spec_example_2.7.yml" - self := loadExample(t, file) - - // Note: we do not support multiple documents in a single YAML file. - - dynassert.Equal(t, dyn.NewValue( - []dyn.Value{ - dyn.NewValue( - "Mark McGwire", - []dyn.Location{{File: file, Line: 5, Column: 3}}, - ), - dyn.NewValue( - "Sammy Sosa", - []dyn.Location{{File: file, Line: 6, Column: 3}}, - ), - dyn.NewValue( - "Ken Griffey", - []dyn.Location{{File: file, Line: 7, Column: 3}}, - ), - }, - []dyn.Location{{File: file, Line: 5, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_8(t *testing.T) { - file := "testdata/spec_example_2.8.yml" - self := loadExample(t, file) - - // Note: we do not support multiple documents in a single YAML file. - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "time": dyn.NewValue("20:03:20", []dyn.Location{{File: file, Line: 4, Column: 7}}), - "player": dyn.NewValue("Sammy Sosa", []dyn.Location{{File: file, Line: 5, Column: 9}}), - "action": dyn.NewValue("strike (miss)", []dyn.Location{{File: file, Line: 6, Column: 9}}), - }, - []dyn.Location{{File: file, Line: 4, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_9(t *testing.T) { - file := "testdata/spec_example_2.9.yml" - self := loadExample(t, file) - - // Note: we do not support multiple documents in a single YAML file. - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "hr": dyn.NewValue( - []dyn.Value{ - dyn.NewValue("Mark McGwire", []dyn.Location{{File: file, Line: 5, Column: 3}}), - dyn.NewValue("Sammy Sosa", []dyn.Location{{File: file, Line: 6, Column: 3}}), - }, - []dyn.Location{{File: file, Line: 5, Column: 1}}, - ), - "rbi": dyn.NewValue( - []dyn.Value{ - dyn.NewValue("Sammy Sosa", []dyn.Location{{File: file, Line: 9, Column: 3}}), - dyn.NewValue("Ken Griffey", []dyn.Location{{File: file, Line: 10, Column: 3}}), - }, - []dyn.Location{{File: file, Line: 9, Column: 1}}, - ), - }, - []dyn.Location{{File: file, Line: 4, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_10(t *testing.T) { - file := "testdata/spec_example_2.10.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "hr": dyn.NewValue( - []dyn.Value{ - dyn.NewValue("Mark McGwire", []dyn.Location{{File: file, Line: 5, Column: 3}}), - dyn.NewValue("Sammy Sosa", []dyn.Location{{File: file, Line: 7, Column: 3}}), - }, - []dyn.Location{{File: file, Line: 5, Column: 1}}, - ), - "rbi": dyn.NewValue( - []dyn.Value{ - // The location for an anchored value refers to the anchor, not the reference. - // This is the same location as the anchor that appears in the "hr" mapping. - dyn.NewValue("Sammy Sosa", []dyn.Location{{File: file, Line: 7, Column: 3}}), - dyn.NewValue("Ken Griffey", []dyn.Location{{File: file, Line: 10, Column: 3}}), - }, - []dyn.Location{{File: file, Line: 9, Column: 1}}, - ), - }, - []dyn.Location{{File: file, Line: 4, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_11(t *testing.T) { - file := "testdata/spec_example_2.11.yml" - input, err := os.ReadFile(file) - require.NoError(t, err) - - // Note: non-string mapping keys are not supported by "go.yaml.in/yaml/v3". - _, err = yamlloader.LoadYAML(file, bytes.NewBuffer(input)) - assert.ErrorContains(t, err, `: key is not a scalar`) -} - -func TestYAMLSpecExample_2_12(t *testing.T) { - file := "testdata/spec_example_2.12.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - []dyn.Value{ - dyn.NewValue( - map[string]dyn.Value{ - "item": dyn.NewValue("Super Hoop", []dyn.Location{{File: file, Line: 5, Column: 13}}), - "quantity": dyn.NewValue(1, []dyn.Location{{File: file, Line: 6, Column: 13}}), - }, - []dyn.Location{{File: file, Line: 5, Column: 3}}, - ), - dyn.NewValue( - map[string]dyn.Value{ - "item": dyn.NewValue("Basketball", []dyn.Location{{File: file, Line: 7, Column: 13}}), - "quantity": dyn.NewValue(4, []dyn.Location{{File: file, Line: 8, Column: 13}}), - }, - []dyn.Location{{File: file, Line: 7, Column: 3}}, - ), - dyn.NewValue( - map[string]dyn.Value{ - "item": dyn.NewValue("Big Shoes", []dyn.Location{{File: file, Line: 9, Column: 13}}), - "quantity": dyn.NewValue(1, []dyn.Location{{File: file, Line: 10, Column: 13}}), - }, - []dyn.Location{{File: file, Line: 9, Column: 3}}, - ), - }, - []dyn.Location{{File: file, Line: 5, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_13(t *testing.T) { - file := "testdata/spec_example_2.13.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - ``+ - `\//||\/||`+NL+ - "// || ||__"+NL, - []dyn.Location{{File: file, Line: 4, Column: 5}}, - ), self) -} - -func TestYAMLSpecExample_2_14(t *testing.T) { - file := "testdata/spec_example_2.14.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - `Mark McGwire's year was crippled by a knee injury.`+NL, - []dyn.Location{{File: file, Line: 3, Column: 5}}, - ), self) -} - -func TestYAMLSpecExample_2_15(t *testing.T) { - file := "testdata/spec_example_2.15.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - ``+ - `Sammy Sosa completed another fine season with great stats.`+NL+ - NL+ - ` 63 Home Runs`+NL+ - ` 0.288 Batting Average`+NL+ - NL+ - `What a year!`+NL, - []dyn.Location{{File: file, Line: 3, Column: 5}}, - ), self) -} - -func TestYAMLSpecExample_2_16(t *testing.T) { - file := "testdata/spec_example_2.16.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "name": dyn.NewValue( - "Mark McGwire", - []dyn.Location{{File: file, Line: 3, Column: 7}}, - ), - "accomplishment": dyn.NewValue( - `Mark set a major league home run record in 1998.`+NL, - []dyn.Location{{File: file, Line: 4, Column: 17}}, - ), - "stats": dyn.NewValue( - ``+ - `65 Home Runs`+NL+ - `0.278 Batting Average`+NL, - []dyn.Location{{File: file, Line: 7, Column: 8}}, - ), - }, - []dyn.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_17(t *testing.T) { - file := "testdata/spec_example_2.17.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "unicode": dyn.NewValue( - `Sosa did fine.`+"\u263A", - []dyn.Location{{File: file, Line: 3, Column: 10}}, - ), - "control": dyn.NewValue( - "\b1998\t1999\t2000\n", - []dyn.Location{{File: file, Line: 4, Column: 10}}, - ), - "hex esc": dyn.NewValue( - "\x0d\x0a is \r\n", - []dyn.Location{{File: file, Line: 5, Column: 10}}, - ), - "single": dyn.NewValue( - `"Howdy!" he cried.`, - []dyn.Location{{File: file, Line: 7, Column: 9}}, - ), - "quoted": dyn.NewValue( - ` # Not a 'comment'.`, - []dyn.Location{{File: file, Line: 8, Column: 9}}, - ), - "tie-fighter": dyn.NewValue( - `|\-*-/|`, - []dyn.Location{{File: file, Line: 9, Column: 14}}, - ), - }, - []dyn.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_18(t *testing.T) { - file := "testdata/spec_example_2.18.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "plain": dyn.NewValue( - `This unquoted scalar spans many lines.`, - []dyn.Location{{File: file, Line: 4, Column: 3}}, - ), - "quoted": dyn.NewValue( - `So does this quoted scalar.`+NL, - []dyn.Location{{File: file, Line: 7, Column: 9}}, - ), - }, - []dyn.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_19(t *testing.T) { - file := "testdata/spec_example_2.19.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "canonical": dyn.NewValue( - 12345, - []dyn.Location{{File: file, Line: 3, Column: 12}}, - ), - "decimal": dyn.NewValue( - 12345, - []dyn.Location{{File: file, Line: 4, Column: 10}}, - ), - "octal": dyn.NewValue( - 12, - []dyn.Location{{File: file, Line: 5, Column: 8}}, - ), - "hexadecimal": dyn.NewValue( - 12, - []dyn.Location{{File: file, Line: 6, Column: 14}}, - ), - "octal11": dyn.NewValue( - 12345, - []dyn.Location{{File: file, Line: 15, Column: 10}}, - ), - }, - []dyn.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_20(t *testing.T) { - file := "testdata/spec_example_2.20.yml" - self := loadExample(t, file) - - // Equality assertion doesn't work with NaNs. - // See https://github.com/stretchr/testify/issues/624. - // - // Remove the NaN entry. - self, _ = dyn.Walk(self, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - if f, ok := v.AsFloat(); ok && math.IsNaN(f) { - return dyn.InvalidValue, dyn.ErrDrop - } - return v, nil - }) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "canonical": dyn.NewValue( - 1230.15, - []dyn.Location{{File: file, Line: 3, Column: 12}}, - ), - "exponential": dyn.NewValue( - 1230.15, - []dyn.Location{{File: file, Line: 4, Column: 14}}, - ), - "fixed": dyn.NewValue( - 1230.15, - []dyn.Location{{File: file, Line: 5, Column: 8}}, - ), - "negative infinity": dyn.NewValue( - math.Inf(-1), - []dyn.Location{{File: file, Line: 6, Column: 20}}, - ), - }, - []dyn.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_21(t *testing.T) { - file := "testdata/spec_example_2.21.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "null": dyn.NewValue( - nil, - []dyn.Location{{File: file, Line: 3, Column: 6}}, - ), - "booleans": dyn.NewValue( - []dyn.Value{ - dyn.NewValue(true, []dyn.Location{{File: file, Line: 4, Column: 13}}), - dyn.NewValue(false, []dyn.Location{{File: file, Line: 4, Column: 19}}), - }, - []dyn.Location{{File: file, Line: 4, Column: 11}}, - ), - "string": dyn.NewValue( - "012345", - []dyn.Location{{File: file, Line: 5, Column: 9}}, - ), - }, - []dyn.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_22(t *testing.T) { - file := "testdata/spec_example_2.22.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "canonical": dyn.NewValue( - dyn.MustTime("2001-12-15T02:59:43.1Z"), - []dyn.Location{{File: file, Line: 3, Column: 12}}, - ), - "iso8601": dyn.NewValue( - dyn.MustTime("2001-12-14t21:59:43.10-05:00"), - []dyn.Location{{File: file, Line: 4, Column: 10}}, - ), - "spaced": dyn.NewValue( - // This is parsed as a string, not a timestamp, - // both by "go.yaml.in/yaml/v3" and by our implementation. - "2001-12-14 21:59:43.10 -5", - []dyn.Location{{File: file, Line: 5, Column: 9}}, - ), - "date": dyn.NewValue( - dyn.MustTime("2002-12-14"), - []dyn.Location{{File: file, Line: 6, Column: 7}}, - ), - }, - []dyn.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_23(t *testing.T) { - file := "testdata/spec_example_2.23.yml" - input, err := os.ReadFile(file) - require.NoError(t, err) - - // Note: the !!binary tag is not supported by us. - - _, err = yamlloader.LoadYAML(file, bytes.NewBuffer(input)) - assert.ErrorContains(t, err, `: unknown tag: !!binary`) -} - -func TestYAMLSpecExample_2_24(t *testing.T) { - file := "testdata/spec_example_2.24.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - []dyn.Value{ - dyn.NewValue( - map[string]dyn.Value{ - "center": dyn.NewValue( - map[string]dyn.Value{ - "x": dyn.NewValue(73, []dyn.Location{{File: file, Line: 8, Column: 23}}), - "y": dyn.NewValue(129, []dyn.Location{{File: file, Line: 8, Column: 30}}), - }, - []dyn.Location{{File: file, Line: 8, Column: 11}}, - ), - "radius": dyn.NewValue(7, []dyn.Location{{File: file, Line: 9, Column: 11}}), - }, - []dyn.Location{{File: file, Line: 7, Column: 3}}, - ), - dyn.NewValue( - map[string]dyn.Value{ - "start": dyn.NewValue( - map[string]dyn.Value{ - "x": dyn.NewValue(73, []dyn.Location{{File: file, Line: 8, Column: 23}}), - "y": dyn.NewValue(129, []dyn.Location{{File: file, Line: 8, Column: 30}}), - }, - []dyn.Location{{File: file, Line: 8, Column: 11}}, - ), - "finish": dyn.NewValue( - map[string]dyn.Value{ - "x": dyn.NewValue(89, []dyn.Location{{File: file, Line: 12, Column: 16}}), - "y": dyn.NewValue(102, []dyn.Location{{File: file, Line: 12, Column: 23}}), - }, - []dyn.Location{{File: file, Line: 12, Column: 11}}, - ), - }, - []dyn.Location{{File: file, Line: 10, Column: 3}}, - ), - dyn.NewValue( - map[string]dyn.Value{ - "start": dyn.NewValue( - map[string]dyn.Value{ - "x": dyn.NewValue(73, []dyn.Location{{File: file, Line: 8, Column: 23}}), - "y": dyn.NewValue(129, []dyn.Location{{File: file, Line: 8, Column: 30}}), - }, - []dyn.Location{{File: file, Line: 8, Column: 11}}, - ), - "color": dyn.NewValue(16772795, []dyn.Location{{File: file, Line: 15, Column: 10}}), - "text": dyn.NewValue("Pretty vector drawing.", []dyn.Location{{File: file, Line: 16, Column: 9}}), - }, - []dyn.Location{{File: file, Line: 13, Column: 3}}, - ), - }, - []dyn.Location{{File: file, Line: 4, Column: 5}}, - ), self) -} - -func TestYAMLSpecExample_2_25(t *testing.T) { - file := "testdata/spec_example_2.25.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "Mark McGwire": dyn.NewValue(nil, []dyn.Location{{File: file, Line: 8, Column: 1}}), - "Sammy Sosa": dyn.NewValue(nil, []dyn.Location{{File: file, Line: 9, Column: 1}}), - "Ken Griffey": dyn.NewValue(nil, []dyn.Location{{File: file, Line: 10, Column: 1}}), - }, - []dyn.Location{{File: file, Line: 6, Column: 5}}, - ), self) -} - -func TestYAMLSpecExample_2_26(t *testing.T) { - file := "testdata/spec_example_2.26.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - []dyn.Value{ - dyn.NewValue( - map[string]dyn.Value{ - "Mark McGwire": dyn.NewValue(65, []dyn.Location{{File: file, Line: 7, Column: 17}}), - }, - []dyn.Location{{File: file, Line: 7, Column: 3}}, - ), - dyn.NewValue( - map[string]dyn.Value{ - "Sammy Sosa": dyn.NewValue(63, []dyn.Location{{File: file, Line: 8, Column: 15}}), - }, - []dyn.Location{{File: file, Line: 8, Column: 3}}, - ), - dyn.NewValue( - map[string]dyn.Value{ - "Ken Griffey": dyn.NewValue(58, []dyn.Location{{File: file, Line: 9, Column: 16}}), - }, - []dyn.Location{{File: file, Line: 9, Column: 3}}, - ), - }, - []dyn.Location{{File: file, Line: 6, Column: 5}}, - ), self) -} - -func TestYAMLSpecExample_2_27(t *testing.T) { - file := "testdata/spec_example_2.27.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "invoice": dyn.NewValue( - 34843, - []dyn.Location{{File: file, Line: 4, Column: 10}}, - ), - "date": dyn.NewValue( - dyn.MustTime("2001-01-23"), - []dyn.Location{{File: file, Line: 5, Column: 10}}, - ), - "bill-to": dyn.NewValue( - map[string]dyn.Value{ - "given": dyn.NewValue( - "Chris", - []dyn.Location{{File: file, Line: 7, Column: 12}}, - ), - "family": dyn.NewValue( - "Dumars", - []dyn.Location{{File: file, Line: 8, Column: 12}}, - ), - "address": dyn.NewValue( - map[string]dyn.Value{ - "lines": dyn.NewValue( - "458 Walkman Dr.\nSuite #292\n", - []dyn.Location{{File: file, Line: 10, Column: 12}}, - ), - "city": dyn.NewValue( - "Royal Oak", - []dyn.Location{{File: file, Line: 13, Column: 15}}, - ), - "state": dyn.NewValue( - "MI", - []dyn.Location{{File: file, Line: 14, Column: 15}}, - ), - "postal": dyn.NewValue( - 48046, - []dyn.Location{{File: file, Line: 15, Column: 15}}, - ), - }, - []dyn.Location{{File: file, Line: 10, Column: 5}}, - ), - }, - []dyn.Location{{File: file, Line: 6, Column: 10}}, - ), - "ship-to": dyn.NewValue( - map[string]dyn.Value{ - "given": dyn.NewValue( - "Chris", - []dyn.Location{{File: file, Line: 7, Column: 12}}, - ), - "family": dyn.NewValue( - "Dumars", - []dyn.Location{{File: file, Line: 8, Column: 12}}, - ), - "address": dyn.NewValue( - map[string]dyn.Value{ - "lines": dyn.NewValue( - "458 Walkman Dr.\nSuite #292\n", - []dyn.Location{{File: file, Line: 10, Column: 12}}, - ), - "city": dyn.NewValue( - "Royal Oak", - []dyn.Location{{File: file, Line: 13, Column: 15}}, - ), - "state": dyn.NewValue( - "MI", - []dyn.Location{{File: file, Line: 14, Column: 15}}, - ), - "postal": dyn.NewValue( - 48046, - []dyn.Location{{File: file, Line: 15, Column: 15}}, - ), - }, - []dyn.Location{{File: file, Line: 10, Column: 5}}, - ), - }, - []dyn.Location{{File: file, Line: 6, Column: 10}}, - ), - "product": dyn.NewValue( - []dyn.Value{ - dyn.NewValue( - map[string]dyn.Value{ - "sku": dyn.NewValue( - "BL394D", - []dyn.Location{{File: file, Line: 18, Column: 17}}, - ), - "quantity": dyn.NewValue( - 4, - []dyn.Location{{File: file, Line: 19, Column: 17}}, - ), - "description": dyn.NewValue( - "Basketball", - []dyn.Location{{File: file, Line: 20, Column: 17}}, - ), - "price": dyn.NewValue( - 450.0, - []dyn.Location{{File: file, Line: 21, Column: 17}}, - ), - }, - []dyn.Location{{File: file, Line: 18, Column: 3}}, - ), dyn.NewValue( - map[string]dyn.Value{ - "sku": dyn.NewValue( - "BL4438H", - []dyn.Location{{File: file, Line: 22, Column: 17}}, - ), - "quantity": dyn.NewValue( - 1, - []dyn.Location{{File: file, Line: 23, Column: 17}}, - ), - "description": dyn.NewValue( - "Super Hoop", - []dyn.Location{{File: file, Line: 24, Column: 17}}, - ), - "price": dyn.NewValue( - 2392.0, - []dyn.Location{{File: file, Line: 25, Column: 17}}, - ), - }, - []dyn.Location{{File: file, Line: 22, Column: 3}}, - ), - }, - []dyn.Location{{File: file, Line: 18, Column: 1}}, - ), - "tax": dyn.NewValue( - 251.42, - []dyn.Location{{File: file, Line: 26, Column: 8}}, - ), - "total": dyn.NewValue( - 4443.52, - []dyn.Location{{File: file, Line: 27, Column: 8}}, - ), - "comments": dyn.NewValue( - "Late afternoon is best. Backup contact is Nancy Billsmer @ 338-4338.", - []dyn.Location{{File: file, Line: 29, Column: 3}}, - ), - }, - []dyn.Location{{File: file, Line: 3, Column: 5}}, - ), self) -} - -func TestYAMLSpecExample_2_28(t *testing.T) { - file := "testdata/spec_example_2.28.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "Time": dyn.NewValue( - "2001-11-23 15:01:42 -5", - []dyn.Location{{File: file, Line: 4, Column: 7}}, - ), - "User": dyn.NewValue( - "ed", - []dyn.Location{{File: file, Line: 5, Column: 7}}, - ), - "Warning": dyn.NewValue( - "This is an error message for the log file", - []dyn.Location{{File: file, Line: 7, Column: 3}}, - ), - }, - []dyn.Location{{File: file, Line: 4, Column: 1}}, - ), self) -} diff --git a/libs/dyn/yamlloader/yaml_test.go b/libs/dyn/yamlloader/yaml_test.go deleted file mode 100644 index e21065e92c0..00000000000 --- a/libs/dyn/yamlloader/yaml_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package yamlloader_test - -import ( - "bytes" - "os" - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlloader" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.yaml.in/yaml/v3" -) - -func loadYAML(t *testing.T, path string) dyn.Value { - input, err := os.ReadFile(path) - require.NoError(t, err) - - var ref any - err = yaml.Unmarshal(input, &ref) - require.NoError(t, err) - - self, err := yamlloader.LoadYAML(path, bytes.NewBuffer(input)) - require.NoError(t, err) - assert.NotNil(t, self) - - // Deep-equal the two values to ensure that the loader is producing - assert.EqualValues(t, ref, self.AsAny()) - return self -} - -func TestYAMLEmpty(t *testing.T) { - self := loadYAML(t, "testdata/empty.yml") - assert.Equal(t, dyn.NilValue, self) -} diff --git a/libs/dyn/yamlsaver/order.go b/libs/dyn/yamlsaver/order.go deleted file mode 100644 index 439be8474a6..00000000000 --- a/libs/dyn/yamlsaver/order.go +++ /dev/null @@ -1,33 +0,0 @@ -package yamlsaver - -import "slices" - -// This struct is used to generate indexes for ordering of map keys. -// The ordering defined based on any predefined Order in `Order` field -// or running Order based on `index` -type Order struct { - index int - order []string -} - -func NewOrder(o []string) *Order { - return &Order{index: 0, order: o} -} - -// Returns an integer which represents the order of map key in resulting -// The lower the index, the earlier in the list the key is. -// If the order is not predefined, it uses running order and any subsequential call to -// order.Get returns an increasing index. -func (o *Order) Get(key string) int { - index := slices.Index(o.order, key) - // If the key is found in predefined order list - // We return a negative index which put the value at the top of the order compared to other - // not predefined keys. The earlier value in predefined list, the lower negative index value - if index != -1 { - return index - len(o.order) - } - - // Otherwise we just increase the order index - o.index += 1 - return o.index -} diff --git a/libs/dyn/yamlsaver/order_test.go b/libs/dyn/yamlsaver/order_test.go deleted file mode 100644 index ed2877f6c8c..00000000000 --- a/libs/dyn/yamlsaver/order_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package yamlsaver - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestOrderReturnsIncreasingIndex(t *testing.T) { - o := NewOrder([]string{}) - assert.Equal(t, 1, o.Get("a")) - assert.Equal(t, 2, o.Get("b")) - assert.Equal(t, 3, o.Get("c")) -} - -func TestOrderReturnsNegativeIndexForPredefinedKeys(t *testing.T) { - o := NewOrder([]string{"a", "b", "c"}) - assert.Equal(t, -3, o.Get("a")) - assert.Equal(t, -2, o.Get("b")) - assert.Equal(t, -1, o.Get("c")) - assert.Equal(t, 1, o.Get("d")) - assert.Equal(t, 2, o.Get("e")) - assert.Equal(t, 3, o.Get("f")) -} diff --git a/libs/dyn/yamlsaver/saver.go b/libs/dyn/yamlsaver/saver.go deleted file mode 100644 index 4c302e26f0d..00000000000 --- a/libs/dyn/yamlsaver/saver.go +++ /dev/null @@ -1,172 +0,0 @@ -package yamlsaver - -import ( - "cmp" - "fmt" - "io" - "os" - "path/filepath" - "slices" - "strconv" - - "github.com/databricks/cli/libs/dyn" - "go.yaml.in/yaml/v3" -) - -type saver struct { - nodesWithStyle map[string]yaml.Style -} - -func NewSaver() *saver { - return &saver{} -} - -func NewSaverWithStyle(nodesWithStyle map[string]yaml.Style) *saver { - return &saver{ - nodesWithStyle: nodesWithStyle, - } -} - -func (s *saver) SaveAsYAML(data any, filename string, force bool) error { - err := os.MkdirAll(filepath.Dir(filename), 0o755) - if err != nil { - return err - } - - // check that file exists - info, err := os.Stat(filename) - if err == nil { - if info.IsDir() { - return fmt.Errorf("%s is a directory", filename) - } - if !force { - return fmt.Errorf("%s already exists. Use --force to overwrite", filename) - } - } - - file, err := os.Create(filename) - if err != nil { - return err - } - defer file.Close() - - err = s.encode(data, file) - if err != nil { - return err - } - return nil -} - -func (s *saver) encode(data any, w io.Writer) error { - yamlNode, err := s.toYamlNode(dyn.V(data)) - if err != nil { - return err - } - enc := yaml.NewEncoder(w) - enc.SetIndent(2) - return enc.Encode(yamlNode) -} - -func (s *saver) toYamlNode(v dyn.Value) (*yaml.Node, error) { - return s.toYamlNodeWithStyle(v, yaml.Style(0)) -} - -func (s *saver) toYamlNodeWithStyle(v dyn.Value, style yaml.Style) (*yaml.Node, error) { - switch v.Kind() { - case dyn.KindMap: - m, _ := v.AsMap() - - // We're using location lines to define the order of keys in YAML. - // The location is set when we convert API response struct to config.Value representation - // See convert.convertMap for details - pairs := m.Pairs() - slices.SortStableFunc(pairs, func(a, b dyn.Pair) int { - return cmp.Compare(a.Value.Location().Line, b.Value.Location().Line) - }) - - var content []*yaml.Node - for _, pair := range pairs { - pk := pair.Key - pv := pair.Value - node := yaml.Node{Kind: yaml.ScalarNode, Value: pk.MustString(), Style: style} - var nestedNodeStyle yaml.Style - if customStyle, ok := s.hasStyle(pk.MustString()); ok { - nestedNodeStyle = customStyle - } else { - nestedNodeStyle = style - } - c, err := s.toYamlNodeWithStyle(pv, nestedNodeStyle) - if err != nil { - return nil, err - } - content = append(content, &node) - content = append(content, c) - } - - return &yaml.Node{Kind: yaml.MappingNode, Content: content, Style: style}, nil - case dyn.KindSequence: - seq, _ := v.AsSequence() - var content []*yaml.Node - for _, item := range seq { - node, err := s.toYamlNodeWithStyle(item, style) - if err != nil { - return nil, err - } - content = append(content, node) - } - return &yaml.Node{Kind: yaml.SequenceNode, Content: content, Style: style}, nil - case dyn.KindNil: - return &yaml.Node{Kind: yaml.ScalarNode, Value: "null", Style: style}, nil - case dyn.KindString: - // If the string is a scalar value (bool, int, float and etc.), we want to quote it. - if isScalarValueInString(v) { - return &yaml.Node{Kind: yaml.ScalarNode, Value: v.MustString(), Style: yaml.DoubleQuotedStyle}, nil - } - return &yaml.Node{Kind: yaml.ScalarNode, Value: v.MustString(), Style: style}, nil - case dyn.KindBool: - return &yaml.Node{Kind: yaml.ScalarNode, Value: strconv.FormatBool(v.MustBool()), Style: style}, nil - case dyn.KindInt: - return &yaml.Node{Kind: yaml.ScalarNode, Value: strconv.FormatInt(v.MustInt(), 10), Style: style}, nil - case dyn.KindFloat: - return &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprint(v.MustFloat()), Style: style}, nil - case dyn.KindTime: - return &yaml.Node{Kind: yaml.ScalarNode, Value: v.MustTime().String(), Style: style}, nil - default: - // Panic because we only want to deal with known types. - panic(fmt.Sprintf("invalid kind: %d", v.Kind())) - } -} - -func (s *saver) hasStyle(key string) (yaml.Style, bool) { - style, ok := s.nodesWithStyle[key] - return style, ok -} - -func isScalarValueInString(v dyn.Value) bool { - if v.Kind() != dyn.KindString { - return false - } - - // Parse value of the string and check if it's a scalar value. - // If it's a scalar value, we want to quote it. - switch v.MustString() { - case "true", "false": - return true - case "": - return true - default: - _, err := parseNumber(v.MustString()) - return err == nil - } -} - -func parseNumber(s string) (any, error) { - if i, err := strconv.ParseInt(s, 0, 64); err == nil { - return i, nil - } - - if f, err := strconv.ParseFloat(s, 64); err == nil { - return f, nil - } - return nil, fmt.Errorf("invalid number: %s", s) -} diff --git a/libs/dyn/yamlsaver/saver_test.go b/libs/dyn/yamlsaver/saver_test.go deleted file mode 100644 index 5752fb4aaa4..00000000000 --- a/libs/dyn/yamlsaver/saver_test.go +++ /dev/null @@ -1,274 +0,0 @@ -package yamlsaver - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.yaml.in/yaml/v3" -) - -func TestMarshalNilValue(t *testing.T) { - s := NewSaver() - nilValue := dyn.NilValue - v, err := s.toYamlNode(nilValue) - assert.NoError(t, err) - assert.Equal(t, "null", v.Value) -} - -func TestMarshalIntValue(t *testing.T) { - s := NewSaver() - intValue := dyn.V(1) - v, err := s.toYamlNode(intValue) - assert.NoError(t, err) - assert.Equal(t, "1", v.Value) - assert.Equal(t, yaml.ScalarNode, v.Kind) -} - -func TestMarshalFloatValue(t *testing.T) { - s := NewSaver() - floatValue := dyn.V(1.0) - v, err := s.toYamlNode(floatValue) - assert.NoError(t, err) - assert.Equal(t, "1", v.Value) - assert.Equal(t, yaml.ScalarNode, v.Kind) -} - -func TestMarshalBoolValue(t *testing.T) { - s := NewSaver() - boolValue := dyn.V(true) - v, err := s.toYamlNode(boolValue) - assert.NoError(t, err) - assert.Equal(t, "true", v.Value) - assert.Equal(t, yaml.ScalarNode, v.Kind) -} - -func TestMarshalTimeValue(t *testing.T) { - tm, err := dyn.NewTime("1970-01-01") - require.NoError(t, err) - - s := NewSaver() - timeValue := dyn.V(tm) - v, err := s.toYamlNode(timeValue) - assert.NoError(t, err) - assert.Equal(t, "1970-01-01", v.Value) - assert.Equal(t, yaml.ScalarNode, v.Kind) -} - -func TestMarshalSequenceValue(t *testing.T) { - s := NewSaver() - sequenceValue := dyn.NewValue( - []dyn.Value{ - dyn.NewValue("value1", []dyn.Location{{File: "file", Line: 1, Column: 2}}), - dyn.NewValue("value2", []dyn.Location{{File: "file", Line: 2, Column: 2}}), - }, - []dyn.Location{{File: "file", Line: 1, Column: 2}}, - ) - v, err := s.toYamlNode(sequenceValue) - assert.NoError(t, err) - assert.Equal(t, yaml.SequenceNode, v.Kind) - assert.Equal(t, "value1", v.Content[0].Value) - assert.Equal(t, "value2", v.Content[1].Value) -} - -func TestMarshalStringValue(t *testing.T) { - s := NewSaver() - stringValue := dyn.V("value") - v, err := s.toYamlNode(stringValue) - assert.NoError(t, err) - assert.Equal(t, "value", v.Value) - assert.Equal(t, yaml.ScalarNode, v.Kind) -} - -func TestMarshalMapValue(t *testing.T) { - s := NewSaver() - mapValue := dyn.NewValue( - map[string]dyn.Value{ - "key3": dyn.NewValue("value3", []dyn.Location{{File: "file", Line: 3, Column: 2}}), - "key2": dyn.NewValue("value2", []dyn.Location{{File: "file", Line: 2, Column: 2}}), - "key1": dyn.NewValue("value1", []dyn.Location{{File: "file", Line: 1, Column: 2}}), - }, - []dyn.Location{{File: "file", Line: 1, Column: 2}}, - ) - - v, err := s.toYamlNode(mapValue) - assert.NoError(t, err) - assert.Equal(t, yaml.MappingNode, v.Kind) - assert.Equal(t, "key1", v.Content[0].Value) - assert.Equal(t, "value1", v.Content[1].Value) - - assert.Equal(t, "key2", v.Content[2].Value) - assert.Equal(t, "value2", v.Content[3].Value) - - assert.Equal(t, "key3", v.Content[4].Value) - assert.Equal(t, "value3", v.Content[5].Value) -} - -func TestMarshalNestedValues(t *testing.T) { - s := NewSaver() - mapValue := dyn.NewValue( - map[string]dyn.Value{ - "key1": dyn.NewValue( - map[string]dyn.Value{ - "key2": dyn.NewValue("value", []dyn.Location{{File: "file", Line: 1, Column: 2}}), - }, - []dyn.Location{{File: "file", Line: 1, Column: 2}}, - ), - }, - []dyn.Location{{File: "file", Line: 1, Column: 2}}, - ) - v, err := s.toYamlNode(mapValue) - assert.NoError(t, err) - assert.Equal(t, yaml.MappingNode, v.Kind) - assert.Equal(t, "key1", v.Content[0].Value) - assert.Equal(t, yaml.MappingNode, v.Content[1].Kind) - assert.Equal(t, "key2", v.Content[1].Content[0].Value) - assert.Equal(t, "value", v.Content[1].Content[1].Value) -} - -func TestMarshalHexadecimalValueIsQuoted(t *testing.T) { - s := NewSaver() - hexValue := dyn.V(0x123) - v, err := s.toYamlNode(hexValue) - assert.NoError(t, err) - assert.Equal(t, "291", v.Value) - assert.Equal(t, yaml.Style(0), v.Style) - assert.Equal(t, yaml.ScalarNode, v.Kind) - - stringValue := dyn.V("0x123") - v, err = s.toYamlNode(stringValue) - assert.NoError(t, err) - assert.Equal(t, "0x123", v.Value) - assert.Equal(t, yaml.DoubleQuotedStyle, v.Style) - assert.Equal(t, yaml.ScalarNode, v.Kind) -} - -func TestMarshalBinaryValueIsQuoted(t *testing.T) { - s := NewSaver() - binaryValue := dyn.V(0b101) - v, err := s.toYamlNode(binaryValue) - assert.NoError(t, err) - assert.Equal(t, "5", v.Value) - assert.Equal(t, yaml.Style(0), v.Style) - assert.Equal(t, yaml.ScalarNode, v.Kind) - - stringValue := dyn.V("0b101") - v, err = s.toYamlNode(stringValue) - assert.NoError(t, err) - assert.Equal(t, "0b101", v.Value) - assert.Equal(t, yaml.DoubleQuotedStyle, v.Style) - assert.Equal(t, yaml.ScalarNode, v.Kind) -} - -func TestMarshalOctalValueIsQuoted(t *testing.T) { - s := NewSaver() - octalValue := dyn.V(0o123) - v, err := s.toYamlNode(octalValue) - assert.NoError(t, err) - assert.Equal(t, "83", v.Value) - assert.Equal(t, yaml.Style(0), v.Style) - assert.Equal(t, yaml.ScalarNode, v.Kind) - - stringValue := dyn.V("0123") - v, err = s.toYamlNode(stringValue) - assert.NoError(t, err) - assert.Equal(t, "0123", v.Value) - assert.Equal(t, yaml.DoubleQuotedStyle, v.Style) - assert.Equal(t, yaml.ScalarNode, v.Kind) -} - -func TestMarshalFloatValueIsQuoted(t *testing.T) { - s := NewSaver() - floatValue := dyn.V(1.0) - v, err := s.toYamlNode(floatValue) - assert.NoError(t, err) - assert.Equal(t, "1", v.Value) - assert.Equal(t, yaml.Style(0), v.Style) - assert.Equal(t, yaml.ScalarNode, v.Kind) - - stringValue := dyn.V("1.0") - v, err = s.toYamlNode(stringValue) - assert.NoError(t, err) - assert.Equal(t, "1.0", v.Value) - assert.Equal(t, yaml.DoubleQuotedStyle, v.Style) - assert.Equal(t, yaml.ScalarNode, v.Kind) -} - -func TestMarshalBoolValueIsQuoted(t *testing.T) { - s := NewSaver() - boolValue := dyn.V(true) - v, err := s.toYamlNode(boolValue) - assert.NoError(t, err) - assert.Equal(t, "true", v.Value) - assert.Equal(t, yaml.Style(0), v.Style) - assert.Equal(t, yaml.ScalarNode, v.Kind) - - stringValue := dyn.V("true") - v, err = s.toYamlNode(stringValue) - assert.NoError(t, err) - assert.Equal(t, "true", v.Value) - assert.Equal(t, yaml.DoubleQuotedStyle, v.Style) - assert.Equal(t, yaml.ScalarNode, v.Kind) -} - -func TestCustomStylingWithNestedMap(t *testing.T) { - s := NewSaverWithStyle(map[string]yaml.Style{ - "styled": yaml.DoubleQuotedStyle, - }) - - styledMap := dyn.NewValue( - map[string]dyn.Value{ - "key1": dyn.NewValue("value1", []dyn.Location{{File: "file", Line: 1, Column: 2}}), - "key2": dyn.NewValue("value2", []dyn.Location{{File: "file", Line: 2, Column: 2}}), - }, - []dyn.Location{{File: "file", Line: -2, Column: 2}}, - ) - - unstyledMap := dyn.NewValue( - map[string]dyn.Value{ - "key3": dyn.NewValue("value3", []dyn.Location{{File: "file", Line: 1, Column: 2}}), - "key4": dyn.NewValue("value4", []dyn.Location{{File: "file", Line: 2, Column: 2}}), - }, - []dyn.Location{{File: "file", Line: -1, Column: 2}}, - ) - - val := dyn.NewValue( - map[string]dyn.Value{ - "styled": styledMap, - "unstyled": unstyledMap, - }, - []dyn.Location{{File: "file", Line: 1, Column: 2}}, - ) - - mv, err := s.toYamlNode(val) - assert.NoError(t, err) - - // Check that the styled map is quoted - v := mv.Content[1] - - assert.Equal(t, yaml.MappingNode, v.Kind) - assert.Equal(t, "key1", v.Content[0].Value) - assert.Equal(t, "value1", v.Content[1].Value) - assert.Equal(t, yaml.DoubleQuotedStyle, v.Content[0].Style) - assert.Equal(t, yaml.DoubleQuotedStyle, v.Content[1].Style) - - assert.Equal(t, "key2", v.Content[2].Value) - assert.Equal(t, "value2", v.Content[3].Value) - assert.Equal(t, yaml.DoubleQuotedStyle, v.Content[2].Style) - assert.Equal(t, yaml.DoubleQuotedStyle, v.Content[3].Style) - - // Check that the unstyled map is not quoted - v = mv.Content[3] - - assert.Equal(t, yaml.MappingNode, v.Kind) - assert.Equal(t, "key3", v.Content[0].Value) - assert.Equal(t, "value3", v.Content[1].Value) - assert.Equal(t, yaml.Style(0), v.Content[0].Style) - assert.Equal(t, yaml.Style(0), v.Content[1].Style) - - assert.Equal(t, "key4", v.Content[2].Value) - assert.Equal(t, "value4", v.Content[3].Value) - assert.Equal(t, yaml.Style(0), v.Content[2].Style) - assert.Equal(t, yaml.Style(0), v.Content[3].Style) -} diff --git a/libs/dyn/yamlsaver/utils.go b/libs/dyn/yamlsaver/utils.go deleted file mode 100644 index aa1e925e559..00000000000 --- a/libs/dyn/yamlsaver/utils.go +++ /dev/null @@ -1,93 +0,0 @@ -package yamlsaver - -import ( - "fmt" - "slices" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" -) - -// Converts a struct to map. Skips any nil fields. -// It uses `skipFields` to skip unnecessary fields. -// Uses `order` to define the order of keys in resulting outout -func ConvertToMapValue(strct any, order *Order, skipFields []string, dst map[string]dyn.Value) (dyn.Value, error) { - ref := dyn.NilValue - mv, err := convert.FromTyped(strct, ref) - if err != nil { - return dyn.InvalidValue, err - } - - if mv.Kind() != dyn.KindMap { - return dyn.InvalidValue, fmt.Errorf("expected map, got %s", mv.Kind()) - } - - mv, err = sortMapAlphabetically(mv) - if err != nil { - return dyn.InvalidValue, err - } - - return skipAndOrder(mv, order, skipFields, dst) -} - -// Sort the map alphabetically by keys. This is used to produce stable output for generated YAML files. -func sortMapAlphabetically(mv dyn.Value) (dyn.Value, error) { - sortedMap := dyn.NewMapping() - mapV := mv.MustMap() - keys := mapV.Keys() - slices.SortStableFunc(keys, func(i, j dyn.Value) int { - iKey := i.MustString() - jKey := j.MustString() - if iKey < jKey { - return -1 - } - - if iKey > jKey { - return 1 - } - return 0 - }) - - for _, key := range keys { - value, _ := mapV.Get(key) - var err error - if value.Kind() == dyn.KindMap { - value, err = sortMapAlphabetically(value) - if err != nil { - return dyn.InvalidValue, err - } - } - sortedMap.SetLoc(key.MustString(), key.Locations(), value) - } - - return dyn.V(sortedMap), nil -} - -func skipAndOrder(mv dyn.Value, order *Order, skipFields []string, dst map[string]dyn.Value) (dyn.Value, error) { - for _, pair := range mv.MustMap().Pairs() { - k := pair.Key.MustString() - v := pair.Value - if v.Kind() == dyn.KindNil { - continue - } - - if slices.Contains(skipFields, k) { - continue - } - - // If the value is already defined in destination, it means it was - // manually set due to custom ordering or other customisation required - // So we're skipping processing it again - if _, ok := dst[k]; ok { - continue - } - - if order == nil { - dst[k] = v - } else { - dst[k] = dyn.NewValue(v.Value(), []dyn.Location{{Line: order.Get(k)}}) - } - } - - return dyn.V(dst), nil -} diff --git a/libs/dyn/yamlsaver/utils_test.go b/libs/dyn/yamlsaver/utils_test.go deleted file mode 100644 index d9f81239f7e..00000000000 --- a/libs/dyn/yamlsaver/utils_test.go +++ /dev/null @@ -1,103 +0,0 @@ -package yamlsaver - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynassert" - "github.com/stretchr/testify/assert" -) - -func TestConvertToMap(t *testing.T) { - type test struct { - Name string `json:"name"` - Map map[string]string `json:"map"` - List []string `json:"list"` - LongNameField string `json:"long_name_field"` - ForceSendFields []string `json:"-"` - Format string `json:"format"` - } - - v := &test{ - Name: "test", - Map: map[string]string{ - "key2": "value2", - "key1": "value1", - }, - List: []string{"a", "b", "c"}, - ForceSendFields: []string{ - "Name", - }, - LongNameField: "long name goes here", - } - result, err := ConvertToMapValue(v, nil, []string{"format"}, map[string]dyn.Value{}) - assert.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "list": dyn.V( - []dyn.Value{ - dyn.V("a"), - dyn.V("b"), - dyn.V("c"), - }, - ), - "long_name_field": dyn.V("long name goes here"), - "map": dyn.V( - map[string]dyn.Value{ - "key1": dyn.V("value1"), - "key2": dyn.V("value2"), - }, - ), - "name": dyn.V( - "test", - ), - }), result) -} - -func TestConvertToMapValueWithOrder(t *testing.T) { - type test struct { - Name string `json:"name"` - Map map[string]string `json:"map"` - List []string `json:"list"` - LongNameField string `json:"long_name_field"` - ForceSendFields []string `json:"-"` - Format string `json:"format"` - } - - v := &test{ - Name: "test", - Map: map[string]string{ - "key1": "value1", - "key2": "value2", - }, - List: []string{"a", "b", "c"}, - ForceSendFields: []string{ - "Name", - }, - LongNameField: "long name goes here", - } - result, err := ConvertToMapValue(v, NewOrder([]string{"list", "name", "map"}), []string{"format"}, map[string]dyn.Value{}) - assert.NoError(t, err) - - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "list": dyn.NewValue( - []dyn.Value{ - dyn.V("a"), - dyn.V("b"), - dyn.V("c"), - }, - []dyn.Location{{Line: -3}}, - ), - "name": dyn.NewValue( - "test", - []dyn.Location{{Line: -2}}, - ), - "map": dyn.NewValue( - map[string]dyn.Value{ - "key1": dyn.V("value1"), - "key2": dyn.V("value2"), - }, - []dyn.Location{{Line: -1}}, - ), - "long_name_field": dyn.NewValue("long name goes here", []dyn.Location{{Line: 1}}), - }), result) -} diff --git a/libs/flags/json_flag.go b/libs/flags/json_flag.go index 3ea4834a5f2..775ea346c70 100644 --- a/libs/flags/json_flag.go +++ b/libs/flags/json_flag.go @@ -8,9 +8,9 @@ import ( "reflect" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/cli/libs/dyn/jsonloader" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go/marshal" + "go.yaml.in/yaml/v3" ) type JsonFlag struct { @@ -48,27 +48,18 @@ func (j *JsonFlag) Unmarshal(v any) diag.Diagnostics { return nil } - dv, err := jsonloader.LoadJSON(j.raw, j.source) + node, err := structvar.ParseJSON(j.source, j.raw) if err != nil { return diag.FromErr(err) } - // First normalize the input data. - // It will convert all the values to the correct types. + // Convert the input to the types of the target, in a fresh value: fields set by + // other flags are already in v and must be kept. // For example string literals for booleans and integers will be converted to the correct types. - nv, diags := convert.Normalize(v, dv) - if diags.HasError() { - return diags - } - - if !nv.IsValid() { - kind := reflect.TypeOf(v).Kind() - if kind == reflect.Pointer { - kind = reflect.TypeOf(v).Elem().Kind() - } - + sv, diags, err := structvar.DecodeYAMLNode(j.source, node, reflect.New(reflect.TypeOf(v).Elem()).Interface(), nil) + if err != nil { var expectedJsonType string - switch kind { + switch reflect.TypeOf(v).Elem().Kind() { case reflect.Struct, reflect.Map: expectedJsonType = "object" case reflect.Slice: @@ -80,37 +71,28 @@ func (j *JsonFlag) Unmarshal(v any) diag.Diagnostics { return diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "Invalid command input", - Detail: fmt.Sprintf("expected JSON %s, received %s", expectedJsonType, dv.Kind()), + Detail: fmt.Sprintf("expected JSON %s, received %s", expectedJsonType, nodeKind(node)), }) } // Then marshal the normalized data to the output. // It will serialize all set data with the correct types. - data, err := json.Marshal(nv.AsAny()) + data, err := json.Marshal(sv.View().AsAny()) if err != nil { return diags.Extend(diag.FromErr(err)) } - kind := reflect.ValueOf(v).Kind() - if kind == reflect.Pointer { - kind = reflect.ValueOf(v).Elem().Kind() - } - - if kind == reflect.Struct { + if reflect.TypeOf(v).Elem().Kind() == reflect.Struct { // Finally unmarshal the normalized data to the output. // It will fill in the ForceSendFields field if the struct contains it. err = marshal.Unmarshal(data, v) - if err != nil { - return diags.Extend(diag.FromErr(err)) - } } else { // If the output is not a struct, just unmarshal the data to the output. err = json.Unmarshal(data, v) - if err != nil { - return diags.Extend(diag.FromErr(err)) - } } - + if err != nil { + return diags.Extend(diag.FromErr(err)) + } return diags } @@ -133,7 +115,7 @@ func (j *JsonFlag) Validate() error { if j.raw == nil { return nil } - _, err := jsonloader.LoadJSON(j.raw, j.source) + _, err := structvar.ParseJSON(j.source, j.raw) return err } @@ -170,3 +152,15 @@ func (j *JsonFlag) RejectWrappedJSON(outerKey, example string) error { } return fmt.Errorf("%s", msg) } + +// nodeKind returns the kind of the JSON value in node, as a user sees it. +func nodeKind(node *yaml.Node) string { + switch node.Kind { + case yaml.MappingNode: + return "map" + case yaml.SequenceNode: + return "sequence" + default: + return map[string]string{"!!str": "string", "!!bool": "bool", "!!int": "int", "!!float": "float", "!!null": "nil"}[node.Tag] + } +} diff --git a/libs/flags/json_flag_test.go b/libs/flags/json_flag_test.go index 1e26d4b3d57..63c3a9fda9f 100644 --- a/libs/flags/json_flag_test.go +++ b/libs/flags/json_flag_test.go @@ -6,7 +6,6 @@ import ( "testing" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" @@ -195,14 +194,14 @@ func TestJsonUnmarshalRequestMismatch(t *testing.T) { assert.Contains(t, diags, diag.Diagnostic{ Severity: diag.Warning, Summary: "unknown field: settings", - Locations: []dyn.Location{ + Locations: []diag.Location{ { File: "(inline)", Line: 3, Column: 6, }, }, - Paths: dyn.ToStructPaths(dyn.EmptyPath), + Paths: []*structpath.PathNode{nil}, }) } @@ -253,7 +252,7 @@ func TestJsonUnmarshalWrongTypeReportsCorrectLocation(t *testing.T) { assert.Contains(t, diags, diag.Diagnostic{ Severity: diag.Warning, Summary: "expected int, found sequence", - Locations: []dyn.Location{ + Locations: []diag.Location{ { File: "(inline)", Line: 2, @@ -278,7 +277,7 @@ func TestJsonUnmarshalArrayInsteadOfIntReportsCorrectLocation(t *testing.T) { assert.Contains(t, diags, diag.Diagnostic{ Severity: diag.Warning, Summary: "cannot parse \"wrong_type\" as an integer", - Locations: []dyn.Location{ + Locations: []diag.Location{ { File: "(inline)", Line: 14, diff --git a/libs/structs/structaccess/bundle_test.go b/libs/structs/structaccess/bundle_test.go index 4de21ede392..12c06830dbc 100644 --- a/libs/structs/structaccess/bundle_test.go +++ b/libs/structs/structaccess/bundle_test.go @@ -1,4 +1,4 @@ -package structaccess +package structaccess_test import ( "reflect" @@ -6,6 +6,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/databricks-sdk-go/service/apps" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/require" @@ -45,37 +46,37 @@ func TestGet_ConfigRoot_JobTagsAccess(t *testing.T) { } // Access a value inside the tags map - v, err := GetByString(root, "resources.jobs.my_job.tags.env") + v, err := structaccess.GetByString(root, "resources.jobs.my_job.tags.env") require.NoError(t, err) require.Equal(t, "dev", v) - require.NoError(t, ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tags.env")) - require.NoError(t, ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tags.anything")) - require.Error(t, ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tags.env.inner")) - require.Error(t, ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tags1")) + require.NoError(t, structaccess.ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tags.env")) + require.NoError(t, structaccess.ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tags.anything")) + require.Error(t, structaccess.ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tags.env.inner")) + require.Error(t, structaccess.ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tags1")) // Array indexing test (1) - v, err = GetByString(root, "resources.jobs.my_job.tasks[0].task_key") + v, err = structaccess.GetByString(root, "resources.jobs.my_job.tasks[0].task_key") require.NoError(t, err) require.Equal(t, "t1", v) - require.NoError(t, ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tasks[0].task_key")) - require.Error(t, ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tasks[0].task_key.inner")) - require.Error(t, ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tasks[0].task_key1")) + require.NoError(t, structaccess.ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tasks[0].task_key")) + require.Error(t, structaccess.ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tasks[0].task_key.inner")) + require.Error(t, structaccess.ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tasks[0].task_key1")) // Array indexing test (2) - v, err = GetByString(root, "resources.jobs.my_job.tasks[0].notebook_task.notebook_path") + v, err = structaccess.GetByString(root, "resources.jobs.my_job.tasks[0].notebook_task.notebook_path") require.NoError(t, err) require.Equal(t, "/Workspace/Users/user@example.com/nb", v) - require.NoError(t, ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tasks[0].notebook_task.notebook_path")) - require.Error(t, ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tasks[0].notebook_task.notebook_path.inner")) - require.Error(t, ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tasks[0].notebook_task.notebook_path1")) + require.NoError(t, structaccess.ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tasks[0].notebook_task.notebook_path")) + require.Error(t, structaccess.ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tasks[0].notebook_task.notebook_path.inner")) + require.Error(t, structaccess.ValidateByString(reflect.TypeFor[config.Root](), "resources.jobs.my_job.tasks[0].notebook_task.notebook_path1")) // Test ambiguous field access: outer is ignored because it has bundle tag - v, err = GetByString(root, "resources.apps.my_app.url") + v, err = structaccess.GetByString(root, "resources.apps.my_app.url") require.NoError(t, err) require.Equal(t, "app_inner_url", v) - require.NoError(t, ValidateByString(reflect.TypeFor[config.Root](), "resources.apps.my_app.url")) - require.Error(t, ValidateByString(reflect.TypeFor[config.Root](), "resources.apps.my_app.url.inner")) - require.Error(t, ValidateByString(reflect.TypeFor[config.Root](), "resources.apps.my_app.url1")) + require.NoError(t, structaccess.ValidateByString(reflect.TypeFor[config.Root](), "resources.apps.my_app.url")) + require.Error(t, structaccess.ValidateByString(reflect.TypeFor[config.Root](), "resources.apps.my_app.url.inner")) + require.Error(t, structaccess.ValidateByString(reflect.TypeFor[config.Root](), "resources.apps.my_app.url1")) } // A bundle resource embeds a config struct that embeds the SDK request struct, so its @@ -86,29 +87,29 @@ func TestGetSet_DoublyEmbeddedField(t *testing.T) { project := &resources.PostgresProject{} //exhaustruct:ignore project.ProjectId = "p" - require.NoError(t, ValidateByString(reflect.TypeOf(project), "budget_policy_id")) + require.NoError(t, structaccess.ValidateByString(reflect.TypeOf(project), "budget_policy_id")) - require.NoError(t, SetByString(project, "budget_policy_id", "abc")) + require.NoError(t, structaccess.SetByString(project, "budget_policy_id", "abc")) require.Equal(t, "abc", project.BudgetPolicyId) - value, err := GetByString(project, "budget_policy_id") + value, err := structaccess.GetByString(project, "budget_policy_id") require.NoError(t, err) require.Equal(t, "abc", value) // An explicit empty value is recorded on ProjectSpec, which declares the field. - require.NoError(t, SetByString(project, "budget_policy_id", "")) + require.NoError(t, structaccess.SetByString(project, "budget_policy_id", "")) require.Contains(t, project.ProjectSpec.ForceSendFields, "BudgetPolicyId") require.NotContains(t, project.ForceSendFields, "BudgetPolicyId") - value, err = GetByString(project, "budget_policy_id") + value, err = structaccess.GetByString(project, "budget_policy_id") require.NoError(t, err) // The empty string, not nil: that is what separates an explicit "" from an absent field. require.Equal(t, any(""), value) // And dropping it again leaves the field absent. - require.NoError(t, SetByString(project, "budget_policy_id", nil)) + require.NoError(t, structaccess.SetByString(project, "budget_policy_id", nil)) require.NotContains(t, project.ProjectSpec.ForceSendFields, "BudgetPolicyId") - value, err = GetByString(project, "budget_policy_id") + value, err = structaccess.GetByString(project, "budget_policy_id") require.NoError(t, err) require.Nil(t, value) } diff --git a/libs/structs/structpath/path.go b/libs/structs/structpath/path.go index 266d9ade1d2..bd0c33e4858 100644 --- a/libs/structs/structpath/path.go +++ b/libs/structs/structpath/path.go @@ -9,7 +9,7 @@ import ( ) // baseVarDef matches a path segment of a ${...} reference. It is the grammar of -// dynvar.BaseVarDef, which structpath cannot import; TestPureReferenceMatchesDynvar +// structvar.BaseVarDef, which structpath cannot import; TestPureReferenceMatchesStructvar // keeps them in agreement. const baseVarDef = `_*\p{L}+([-_]*[\p{L}\p{N}]+)*` @@ -191,6 +191,15 @@ func NewPathSlice(parts ...any) []*PathNode { return []*PathNode{NewPath(nil, parts...)} } +// Join appends the components of nodes (each node's own key, index or key-value, not +// its parents) to prefix. +func Join(prefix *PathNode, nodes ...*PathNode) *PathNode { + for _, n := range nodes { + prefix = &PathNode{prev: prefix, key: n.key, index: n.index, value: n.value} + } + return prefix +} + func NewKeyValue(prev *PathNode, key, value string) *PathNode { return &PathNode{ prev: prev, @@ -665,6 +674,15 @@ func MustParsePath(s string) *PathNode { return path } +// MustParsePattern parses a pattern string and panics on error. Wildcards are allowed. +func MustParsePattern(s string) *PatternNode { + pattern, err := ParsePattern(s) + if err != nil { + panic(err) + } + return pattern +} + // MustParsePaths parses each of paths like [MustParsePath], e.g. for diag.Diagnostic.Paths. func MustParsePaths(paths ...string) []*PathNode { out := make([]*PathNode, len(paths)) @@ -712,7 +730,6 @@ func isValidField(s string) bool { } // PureReferenceToPath returns a PathNode if s is a pure variable reference, otherwise false. -// This function is similar to dynvar.PureReferenceToPath but returns a *PathNode instead of dyn.Path. func PureReferenceToPath(s string) (*PathNode, bool) { m := pureReference.FindStringSubmatch(s) if m == nil { @@ -986,3 +1003,13 @@ func (p *PatternNode) UnmarshalYAML(unmarshal func(any) error) error { *p = *parsed return nil } + +// KeyAt returns the string key of the i-th component of the path, or "" if it is not a key. +func (p *PathNode) KeyAt(i int) string { + nodes := p.AsSlice() + if i < 0 || i >= len(nodes) { + return "" + } + k, _ := nodes[i].StringKey() + return k +} diff --git a/libs/structs/structpath/path_test.go b/libs/structs/structpath/path_test.go index 40fc0282c2b..4e9901e6a46 100644 --- a/libs/structs/structpath/path_test.go +++ b/libs/structs/structpath/path_test.go @@ -1275,14 +1275,6 @@ paths: assert.Contains(t, err.Error(), "unexpected end of input") } -func TestMustParsePaths(t *testing.T) { - paths := MustParsePaths("bundle.terraform", "resources.jobs.foo.tasks[0]") - require.Len(t, paths, 2) - assert.Equal(t, "bundle.terraform", paths[0].String()) - assert.Equal(t, "resources.jobs.foo.tasks[0]", paths[1].String()) - assert.Empty(t, MustParsePaths()) -} - func TestNewPath(t *testing.T) { assert.Equal(t, "a.b['c d']", NewPath(NewStringKey(nil, "a"), "b", "c d").String()) assert.Equal(t, "resources.jobs.j.tasks[2].job_cluster_key", NewPath(nil, "resources", "jobs", "j", "tasks", 2, "job_cluster_key").String()) @@ -1291,6 +1283,21 @@ func TestNewPath(t *testing.T) { assert.Panics(t, func() { NewPath(nil, int64(1)) }) } +func TestJoin(t *testing.T) { + p := MustParsePath("tasks[task_key='x'].libraries[0]['a.b']") + assert.Equal(t, "resources.jobs.j.tasks[task_key='x'].libraries[0]['a.b']", Join(MustParsePath("resources.jobs.j"), p.AsSlice()...).String()) + assert.Equal(t, p.String(), Join(nil, p.AsSlice()...).String()) + assert.Equal(t, "a", Join(MustParsePath("a")).String()) +} + +func TestMustParsePaths(t *testing.T) { + paths := MustParsePaths("bundle.terraform", "resources.jobs.foo.tasks[0]") + require.Len(t, paths, 2) + assert.Equal(t, "bundle.terraform", paths[0].String()) + assert.Equal(t, "resources.jobs.foo.tasks[0]", paths[1].String()) + assert.Empty(t, MustParsePaths()) +} + func TestNewPathSlice(t *testing.T) { paths := NewPathSlice("resources", "jobs", "${var.env}_job") require.Len(t, paths, 1) diff --git a/libs/structs/structpath/reference_test.go b/libs/structs/structpath/reference_test.go index 485b756c7fb..45360c24b4b 100644 --- a/libs/structs/structpath/reference_test.go +++ b/libs/structs/structpath/reference_test.go @@ -3,12 +3,12 @@ package structpath_test import ( "testing" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/stretchr/testify/assert" ) -func TestPureReferenceMatchesDynvar(t *testing.T) { +func TestPureReferenceMatchesStructvar(t *testing.T) { for _, s := range []string{ "${var.foo}", "${resources.jobs.foo.tasks[1].env.key}", @@ -25,6 +25,6 @@ func TestPureReferenceMatchesDynvar(t *testing.T) { "plain_string", } { _, ok := structpath.PureReferenceToPath(s) - assert.Equal(t, dynvar.IsPureVariableReference(s), ok, s) + assert.Equal(t, structvar.IsPureVariableReference(s), ok, s) } } diff --git a/libs/structs/structvar/decode.go b/libs/structs/structvar/decode.go new file mode 100644 index 00000000000..2ea9d008e64 --- /dev/null +++ b/libs/structs/structvar/decode.go @@ -0,0 +1,746 @@ +package structvar + +import ( + "encoding/json" + "fmt" + "io" + "maps" + "reflect" + "strconv" + + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structpath" + "go.yaml.in/yaml/v3" +) + +// source is a configuration tree to decode from (a parsed YAML document or a view). +type source interface { + kind() Kind + locations() []diag.Location + // anchor reports whether the value is a YAML anchor (or an alias of one). + anchor() bool + // scalar returns the value of a string, bool, int (int or int64), float or time (string). + scalar() any + pairs() ([]sourcePair, error) + elems() ([]source, error) +} + +type sourcePair struct { + key string + keyLocs []diag.Location + value source +} + +// DecodeYAML decodes the YAML document in r (read from file) into dst, a pointer to a +// typed configuration value, and returns its references and locations. +// +// Values are converted to the field types the way configuration is normalized: scalars +// are converted where possible, pure variable references are recorded in the returned +// references for fields that cannot hold a string, and values that cannot be converted +// or fields that do not exist are dropped with a warning. An error is returned for +// invalid YAML. +func DecodeYAML(file string, r io.Reader, dst any) (*StructVar, diag.Diagnostics, error) { + node, err := ParseYAML(r) + if err != nil { + return nil, nil, err + } + return DecodeYAMLNode(file, node, dst, nil) +} + +// LocationMapper returns the locations to record for the value at path, given the +// locations it has in the YAML document. +type LocationMapper func(path *structpath.PathNode, locs []diag.Location) []diag.Location + +// DecodeYAMLNode is [DecodeYAML] for a parsed YAML document (nil if it is empty). +// If mapLocations is not nil, it is applied to the location of every value. +func DecodeYAMLNode(file string, node *yaml.Node, dst any, mapLocations LocationMapper) (*StructVar, diag.Diagnostics, error) { + sv := &StructVar{Value: dst} + v := reflect.ValueOf(dst).Elem() + if node == nil { + // An empty document is null. + v.SetZero() + return sv, nil, nil + } + src, err := newYAMLSource(file, node, nil) + if err != nil { + return nil, nil, err + } + // Report YAML errors anywhere in the document, also in values that are dropped. + if err := validate(src); err != nil { + return nil, nil, err + } + src.mapLocations = mapLocations + var d decoder + n, ok, err := d.decode(v, src, nil) + if err == nil && !ok { + // The value could not be converted to dst at all (e.g. a list for a struct). + err = fmt.Errorf("expected a %s, found a %s", kindOf(v.Type()), src.kind()) + } + sv.Refs = n.refs(nil, nil) + sv.Locations = n.locations() + return sv, d.diags, err +} + +// Assign sets the value described by src at path, replacing what was there, with the +// references and locations of src. The value is converted to the type at path like +// [DecodeYAML] converts values; the diagnostics explain values that were dropped. +func (sv *StructVar) Assign(path *structpath.PathNode, src View) (diag.Diagnostics, error) { + var d decoder + var n *dnode + err := update(reflect.ValueOf(sv.Value), components(path), func(dst reflect.Value) (bool, error) { + var ok bool + var err error + n, ok, err = d.decode(dst, viewSource{src}, path) + if !ok { + dst.SetZero() + return false, err + } + return n.isZeroValue(dst), err + }) + if err != nil { + return d.diags, err + } + sv.Refs = n.refs(path, withoutRefs(sv.Refs, path)) + sv.Locations = sv.Locations.With(path, n.locations()) + return d.diags, nil +} + +func validate(s source) error { + switch s.kind() { + case KindMap: + pairs, err := s.pairs() + if err != nil { + return err + } + for _, p := range pairs { + if err := validate(p.value); err != nil { + return err + } + } + case KindSequence: + elems, err := s.elems() + if err != nil { + return err + } + for _, e := range elems { + if err := validate(e); err != nil { + return err + } + } + default: + } + return nil +} + +type decoder struct { + diags diag.Diagnostics +} + +// dnode is the decoded shape of a value: what the typed value does not hold. +type dnode struct { + kind Kind + locs []diag.Location + ref string + keys map[string]*dnode + elems []*dnode +} + +// locations returns the locations of the decoded value. +func (n *dnode) locations() *Locations { + if n == nil { + return nil + } + out := &Locations{locs: n.locs} + for k, c := range n.keys { + if out.keys == nil { + out.keys = make(map[string]*Locations, len(n.keys)) + } + out.keys[k] = c.locations() + } + for _, c := range n.elems { + out.elems = append(out.elems, c.locations()) + } + return out +} + +// refs returns refs (copied) plus the references of the decoded value at path. +func (n *dnode) refs(path *structpath.PathNode, refs map[string]string) map[string]string { + var out map[string]string + n.collectRefs(path, func(p *structpath.PathNode, ref string) { + if out == nil { + out = maps.Clone(refs) + if out == nil { + out = map[string]string{} + } + } + out[p.String()] = ref + }) + if out == nil { + return refs + } + return out +} + +func (n *dnode) collectRefs(path *structpath.PathNode, fn func(*structpath.PathNode, string)) { + if n == nil { + return + } + if n.ref != "" { + fn(path, n.ref) + } + for k, c := range n.keys { + c.collectRefs(structpath.NewStringKey(path, k), fn) + } + for i, c := range n.elems { + c.collectRefs(structpath.NewIndex(path, i), fn) + } +} + +func (d *decoder) warn(summary string, src source, path *structpath.PathNode) { + d.diags = d.diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: summary, + Locations: []diag.Location{firstLocation(src)}, + Paths: []*structpath.PathNode{path}, + }) +} + +func firstLocation(src source) diag.Location { + locs := src.locations() + if len(locs) == 0 { + return diag.Location{} + } + return locs[0] +} + +func (d *decoder) typeMismatch(expected Kind, src source, path *structpath.PathNode) { + d.warn(fmt.Sprintf("expected %s, found %s", expected, src.kind()), src, path) +} + +func (d *decoder) nullWarning(expected Kind, src source, path *structpath.PathNode) { + d.warn(fmt.Sprintf("expected a %s value, found null", expected), src, path) +} + +func pureRefOf(src source) string { + if src.kind() != KindString { + return "" + } + s, _ := src.scalar().(string) + if IsPureVariableReference(s) { + return s + } + return "" +} + +// decode decodes src into dst (settable) and returns its meta. It reports false if the +// value cannot be decoded and is dropped (the diagnostics say why). +func (d *decoder) decode(dst reflect.Value, src source, path *structpath.PathNode) (*dnode, bool, error) { + typ := dst.Type() + if typ.Kind() == reflect.Pointer { + if src.kind() == KindNil { + n, ok, err := d.decode(reflect.New(typ.Elem()).Elem(), src, path) + if ok { + dst.SetZero() + } + return n, ok, err + } + p := reflect.New(typ.Elem()) + n, ok, err := d.decode(p.Elem(), src, path) + if ok { + dst.Set(p) + } + return n, ok, err + } + + n := &dnode{kind: src.kind(), locs: src.locations()} + ref := pureRefOf(src) + + switch typ.Kind() { + case reflect.Struct: + if IsSDKNativeType(typ) { + s, ok := d.decodeString(src, path) + if !ok { + return nil, false, nil + } + n.kind = KindString + if ref != "" { + n.ref = ref + dst.SetZero() + return n, true, nil + } + buf, err := json.Marshal(s) + if err != nil { + return nil, false, err + } + return n, true, json.Unmarshal(buf, dst.Addr().Interface()) + } + return d.decodeStruct(dst, src, path, n, ref) + case reflect.Map: + switch src.kind() { + case KindMap: + pairs, err := src.pairs() + if err != nil { + return nil, false, err + } + out := reflect.MakeMapWithSize(typ, len(pairs)) + n.keys = make(map[string]*dnode, len(pairs)) + for _, p := range pairs { + e := reflect.New(typ.Elem()).Elem() + en, ok, err := d.decode(e, p.value, structpath.NewStringKey(path, p.key)) + if err != nil { + return nil, false, err + } + if !ok { + continue + } + out.SetMapIndex(reflect.ValueOf(p.key).Convert(typ.Key()), e) + n.keys[p.key] = en + } + dst.Set(out) + return n, true, nil + case KindNil: + dst.SetZero() + return n, true, nil + default: + } + if ref != "" { + n.ref = ref + dst.SetZero() + return n, true, nil + } + d.typeMismatch(KindMap, src, path) + return nil, false, nil + case reflect.Slice: + switch src.kind() { + case KindSequence: + elems, err := src.elems() + if err != nil { + return nil, false, err + } + out := reflect.MakeSlice(typ, 0, len(elems)) + for _, ev := range elems { + e := reflect.New(typ.Elem()).Elem() + en, ok, err := d.decode(e, ev, structpath.NewIndex(path, out.Len())) + if err != nil { + return nil, false, err + } + if !ok { + continue + } + out = reflect.Append(out, e) + n.elems = append(n.elems, en) + } + dst.Set(out) + return n, true, nil + case KindNil: + dst.SetZero() + return n, true, nil + default: + } + if ref != "" { + n.ref = ref + dst.SetZero() + return n, true, nil + } + d.typeMismatch(KindSequence, src, path) + return nil, false, nil + case reflect.String: + s, ok := d.decodeString(src, path) + if !ok { + return nil, false, nil + } + n.kind = KindString + dst.SetString(s) + return n, true, nil + case reflect.Bool: + return d.decodeBool(dst, src, path, n, ref) + case reflect.Int, reflect.Int32, reflect.Int64: + return d.decodeInt(dst, src, path, n, ref) + case reflect.Float32, reflect.Float64: + return d.decodeFloat(dst, src, path, n, ref) + case reflect.Interface: + v, err := d.decodeAny(src, n) + if err != nil { + return nil, false, err + } + if v == nil { + dst.SetZero() + } else { + dst.Set(reflect.ValueOf(v)) + } + return n, true, nil + default: + return nil, false, fmt.Errorf("unsupported type: %s", typ.Kind()) + } +} + +// isAnchorContainer reports whether v is a YAML anchor or a non-empty sequence or map +// composed entirely of anchor containers. Anchors define reusable blocks and must not +// trigger "unknown field" warnings, including when nested inside a container. +func isAnchorContainer(v source) bool { + if v.anchor() { + return true + } + var elements []source + switch v.kind() { + case KindSequence: + elements, _ = v.elems() + case KindMap: + pairs, _ := v.pairs() + for _, p := range pairs { + elements = append(elements, p.value) + } + default: + return false + } + if len(elements) == 0 { + return false + } + for _, e := range elements { + if !isAnchorContainer(e) { + return false + } + } + return true +} + +func (d *decoder) decodeStruct(dst reflect.Value, src source, path *structpath.PathNode, n *dnode, ref string) (*dnode, bool, error) { + switch src.kind() { + case KindMap: + case KindNil: + dst.SetZero() + return n, true, nil + default: + if ref != "" { + n.ref = ref + dst.SetZero() + return n, true, nil + } + d.typeMismatch(KindMap, src, path) + return nil, false, nil + } + + pairs, err := src.pairs() + if err != nil { + return nil, false, err + } + + dst.SetZero() + info := GetStructInfo(dst.Type()) + n.keys = make(map[string]*dnode, len(pairs)) + for _, p := range pairs { + index, ok := info.Fields[p.key] + if !ok { + if isAnchorContainer(p.value) { + continue + } + // Special case: provide a more helpful message for "valueFrom" vs "value_from". + if _, hasValueFrom := info.Fields["value_from"]; p.key == "valueFrom" && hasValueFrom { + d.diags = d.diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: "Use 'value_from' instead of 'valueFrom'", + Detail: "The field 'valueFrom' should be 'value_from' (snake_case). The 'valueFrom' field will be ignored.", + Locations: p.keyLocs, + Paths: []*structpath.PathNode{path}, + }) + continue + } + d.diags = d.diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: "unknown field: " + p.key, + Locations: p.keyLocs, + Paths: []*structpath.PathNode{path}, + }) + continue + } + + // Decode into a fresh value so a dropped value leaves the field (and any + // embedded struct pointers on the way to it) untouched. + ft := dst.Type().FieldByIndex(index).Type + fv := reflect.New(ft).Elem() + fn, ok, err := d.decode(fv, p.value, structpath.NewStringKey(path, p.key)) + if err != nil { + return nil, false, err + } + if !ok { + continue + } + GetOrNewFieldByIndex(dst, index).Set(fv) + n.keys[p.key] = fn + + // An explicitly set zero value must still serialize: add it to ForceSendFields. + if fn.isZeroValue(fv) { + addForceSend(dst, &info, p.key) + } + } + return n, true, nil +} + +// isZeroValue mirrors dyn.Value.IsZero for the decoded value: nil, an empty map or +// sequence, or a zero scalar (a pure reference is not zero). +func (n *dnode) isZeroValue(v reflect.Value) bool { + switch n.kind { + case KindNil: + return true + case KindMap: + return len(n.keys) == 0 + case KindSequence: + return len(n.elems) == 0 + default: + } + if n.ref != "" { + return false + } + for v.Kind() == reflect.Pointer || v.Kind() == reflect.Interface { + if v.IsNil() { + return true + } + v = v.Elem() + } + if v.Kind() == reflect.Struct { + // SDK native types are strings in the configuration tree. + s, ok := sdkNativeString(v, includeZero) + return !ok || s == "" + } + return v.IsZero() +} + +func (d *decoder) decodeString(src source, path *structpath.PathNode) (string, bool) { + switch src.kind() { + case KindString, KindTime: + return src.scalar().(string), true + case KindBool: + return strconv.FormatBool(src.scalar().(bool)), true + case KindInt: + return strconv.FormatInt(toInt64(src.scalar()), 10), true + case KindFloat: + return strconv.FormatFloat(src.scalar().(float64), 'f', -1, 64), true + case KindNil: + d.nullWarning(KindString, src, path) + return "", false + default: + d.typeMismatch(KindString, src, path) + return "", false + } +} + +func toInt64(v any) int64 { + switch v := v.(type) { + case int: + return int64(v) + case int64: + return v + default: + panic(fmt.Sprintf("unexpected int type %T", v)) + } +} + +func (d *decoder) decodeBool(dst reflect.Value, src source, path *structpath.PathNode, n *dnode, ref string) (*dnode, bool, error) { + switch src.kind() { + case KindBool: + dst.SetBool(src.scalar().(bool)) + return n, true, nil + case KindString: + // See https://yaml.org/type/bool.html. + switch src.scalar().(string) { + case "true", "True", "TRUE", "y", "Y", "yes", "Yes", "YES", "on", "On", "ON": + dst.SetBool(true) + case "false", "False", "FALSE", "n", "N", "no", "No", "NO", "off", "Off", "OFF": + dst.SetBool(false) + default: + if ref != "" { + n.ref = ref + dst.SetZero() + return n, true, nil + } + d.typeMismatch(KindBool, src, path) + return nil, false, nil + } + n.kind = KindBool + return n, true, nil + case KindNil: + d.nullWarning(KindBool, src, path) + return nil, false, nil + default: + d.typeMismatch(KindBool, src, path) + return nil, false, nil + } +} + +func (d *decoder) decodeInt(dst reflect.Value, src source, path *structpath.PathNode, n *dnode, ref string) (*dnode, bool, error) { + switch src.kind() { + case KindInt: + dst.SetInt(toInt64(src.scalar())) + return n, true, nil + case KindFloat: + f := src.scalar().(float64) + out := int64(f) + if f != float64(out) { + d.warn(fmt.Sprintf(`cannot accurately represent "%g" as integer due to precision loss`, f), src, path) + return nil, false, nil + } + dst.SetInt(out) + n.kind = KindInt + return n, true, nil + case KindString: + s := src.scalar().(string) + out, err := strconv.ParseInt(s, 10, 64) + if err != nil { + if ref != "" { + n.ref = ref + dst.SetZero() + return n, true, nil + } + d.warn(fmt.Sprintf("cannot parse %q as an integer", s), src, path) + return nil, false, nil + } + dst.SetInt(out) + n.kind = KindInt + return n, true, nil + case KindNil: + d.nullWarning(KindInt, src, path) + return nil, false, nil + default: + d.typeMismatch(KindInt, src, path) + return nil, false, nil + } +} + +func (d *decoder) decodeFloat(dst reflect.Value, src source, path *structpath.PathNode, n *dnode, ref string) (*dnode, bool, error) { + switch src.kind() { + case KindFloat: + dst.SetFloat(src.scalar().(float64)) + return n, true, nil + case KindInt: + i := toInt64(src.scalar()) + out := float64(i) + if i != int64(out) { + d.warn(fmt.Sprintf(`cannot accurately represent "%d" as floating point number due to precision loss`, i), src, path) + return nil, false, nil + } + dst.SetFloat(out) + n.kind = KindFloat + return n, true, nil + case KindString: + s := src.scalar().(string) + out, err := strconv.ParseFloat(s, 64) + if err != nil { + if ref != "" { + n.ref = ref + dst.SetZero() + return n, true, nil + } + d.warn(fmt.Sprintf("cannot parse %q as a floating point number", s), src, path) + return nil, false, nil + } + dst.SetFloat(out) + n.kind = KindFloat + return n, true, nil + case KindNil: + d.nullWarning(KindFloat, src, path) + return nil, false, nil + default: + d.typeMismatch(KindFloat, src, path) + return nil, false, nil + } +} + +// decodeAny returns the generic Go value for src (maps, slices, scalars) and fills in +// the meta of everything below n. Timestamps become the string they were written as. +func (d *decoder) decodeAny(src source, n *dnode) (any, error) { + switch src.kind() { + case KindMap: + pairs, err := src.pairs() + if err != nil { + return nil, err + } + out := make(map[string]any, len(pairs)) + n.keys = make(map[string]*dnode, len(pairs)) + for _, p := range pairs { + cn := &dnode{kind: p.value.kind(), locs: p.value.locations()} + v, err := d.decodeAny(p.value, cn) + if err != nil { + return nil, err + } + out[p.key] = v + n.keys[p.key] = cn + } + return out, nil + case KindSequence: + elems, err := src.elems() + if err != nil { + return nil, err + } + out := make([]any, len(elems)) + for i, e := range elems { + cn := &dnode{kind: e.kind(), locs: e.locations()} + v, err := d.decodeAny(e, cn) + if err != nil { + return nil, err + } + out[i] = v + n.elems = append(n.elems, cn) + } + return out, nil + case KindTime: + n.kind = KindString + return src.scalar(), nil + case KindNil: + return nil, nil + default: + return src.scalar(), nil + } +} + +// viewSource is a [source] backed by a view. +type viewSource struct { + v View +} + +func (s viewSource) kind() Kind { return s.v.Kind() } +func (s viewSource) locations() []diag.Location { return s.v.Locations() } +func (s viewSource) anchor() bool { return false } + +func (s viewSource) scalar() any { + if str, ok := s.v.AsString(); ok { + return str + } + return s.v.AsAny() +} + +func (s viewSource) pairs() ([]sourcePair, error) { + var out []sourcePair + for k, c := range s.v.MapItems() { + out = append(out, sourcePair{key: k, value: viewSource{c}}) + } + return out, nil +} + +func (s viewSource) elems() ([]source, error) { + var out []source + for _, c := range s.v.Sequence() { + out = append(out, viewSource{c}) + } + return out, nil +} + +// kindOf returns the kind of configuration tree values of type t. +func kindOf(t reflect.Type) Kind { + for t.Kind() == reflect.Pointer { + t = t.Elem() + } + switch t.Kind() { + case reflect.Struct, reflect.Map: + return KindMap + case reflect.Slice: + return KindSequence + case reflect.Bool: + return KindBool + case reflect.Int, reflect.Int32, reflect.Int64: + return KindInt + case reflect.Float32, reflect.Float64: + return KindFloat + default: + return KindString + } +} diff --git a/libs/structs/structvar/decode_json.go b/libs/structs/structvar/decode_json.go new file mode 100644 index 00000000000..c0b91b3699b --- /dev/null +++ b/libs/structs/structvar/decode_json.go @@ -0,0 +1,126 @@ +package structvar + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "slices" + "strconv" + + "github.com/databricks/cli/libs/diag" + "go.yaml.in/yaml/v3" +) + +// ParseJSON parses data into a YAML node tree that carries the positions of the values +// in data: the start of an object or array, the first character of a key and the end of +// a scalar (the decoder does not report where a scalar starts). +func ParseJSON(source string, data []byte) (*yaml.Node, error) { + decoder := json.NewDecoder(bytes.NewReader(data)) + // Use json.Number to avoid losing precision on int64 values above 2^53 (e.g. job and run IDs). + decoder.UseNumber() + + lines := newLineIndex(data) + node, err := decodeJSONValue(decoder, lines) + if err == nil && decoder.More() { + err = errors.New("unexpected additional content") + } + if err != nil { + if errors.Is(err, io.EOF) { + err = errors.New("unexpected end of JSON input") + } + line, column := lines.position(decoder.InputOffset()) + return nil, fmt.Errorf("error decoding JSON at %s: %v", diag.Location{File: source, Line: line, Column: column}, err) + } + return node, nil +} + +// lineIndex holds the offsets at which the lines of the input start. +type lineIndex []int64 + +func newLineIndex(data []byte) lineIndex { + lines := lineIndex{0} + for i, b := range data { + if b == '\n' { + lines = append(lines, int64(i+1)) + } + } + return lines +} + +// position returns the 1-based line and column of offset. +func (l lineIndex) position(offset int64) (line, column int) { + // The line is the last one starting at or before offset. + i, found := slices.BinarySearch(l, offset) + if !found { + i-- + } + return i + 1, int(offset-l[i]) + 1 +} + +func newNode(lines lineIndex, offset int64, kind yaml.Kind, tag, value string) *yaml.Node { + line, column := lines.position(offset) + return &yaml.Node{Kind: kind, Tag: tag, Value: value, Line: line, Column: column} +} + +func decodeJSONValue(decoder *json.Decoder, lines lineIndex) (*yaml.Node, error) { + token, err := decoder.Token() + if err != nil { + return nil, err + } + offset := decoder.InputOffset() + + switch tok := token.(type) { + case json.Delim: + if tok == '{' { + node := newNode(lines, offset-1, yaml.MappingNode, "!!map", "") + for decoder.More() { + keyToken, err := decoder.Token() + if err != nil { + return nil, err + } + key, ok := keyToken.(string) + if !ok { + return nil, errors.New("expected string for object key") + } + // The decoder reports the end of the key: step back over the key and the closing quote. + keyNode := newNode(lines, decoder.InputOffset()-int64(len(key)+1), yaml.ScalarNode, "!!str", key) + value, err := decodeJSONValue(decoder, lines) + if err != nil { + return nil, err + } + node.Content = append(node.Content, keyNode, value) + } + // Consume the closing '}'. + _, err := decoder.Token() + return node, err + } + node := newNode(lines, offset-1, yaml.SequenceNode, "!!seq", "") + for decoder.More() { + value, err := decodeJSONValue(decoder, lines) + if err != nil { + return nil, err + } + node.Content = append(node.Content, value) + } + // Consume the closing ']'. + _, err := decoder.Token() + return node, err + case json.Number: + if _, err := tok.Int64(); err == nil { + return newNode(lines, offset, yaml.ScalarNode, "!!int", tok.String()), nil + } + // Integers that overflow int64 fall back to float64. + if _, err := tok.Float64(); err != nil { + return nil, fmt.Errorf("invalid number %q: %w", tok.String(), err) + } + return newNode(lines, offset, yaml.ScalarNode, "!!float", tok.String()), nil + case string: + return newNode(lines, offset, yaml.ScalarNode, "!!str", tok), nil + case bool: + return newNode(lines, offset, yaml.ScalarNode, "!!bool", strconv.FormatBool(tok)), nil + default: + return newNode(lines, offset, yaml.ScalarNode, "!!null", "null"), nil + } +} diff --git a/libs/structs/structvar/decode_yaml.go b/libs/structs/structvar/decode_yaml.go new file mode 100644 index 00000000000..f4b540ce44a --- /dev/null +++ b/libs/structs/structvar/decode_yaml.go @@ -0,0 +1,347 @@ +package structvar + +import ( + "fmt" + "io" + "math" + "strconv" + "strings" + "time" + + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structpath" + "go.yaml.in/yaml/v3" +) + +// LocationError is a YAML syntax error with the location it occurred at. +type LocationError struct { + Loc diag.Location + Summary string +} + +func (e *LocationError) Error() string { + return fmt.Sprintf("yaml (%s): %s", e.Loc, e.Summary) +} + +// yamlSource is a [source] backed by a YAML node. Aliases are followed, merge keys +// are applied, and scalars are resolved by their tag. +type yamlSource struct { + file string + node *yaml.Node + + // The node the value comes from, after following aliases. + target *yaml.Node + + // Aliases followed on the way from the document root to this value, to detect + // cyclic anchors. + aliases *aliasChain + + // If set, mapLocations is applied to the locations of the value at path. + mapLocations LocationMapper + path *structpath.PathNode + + // Resolved scalar. + k Kind + v any +} + +// ParseYAML parses the YAML document in r. It returns nil for an empty document. +func ParseYAML(r io.Reader) (*yaml.Node, error) { + var doc yaml.Node + err := yaml.NewDecoder(r).Decode(&doc) + if err == io.EOF { + return nil, nil + } + if err != nil { + return nil, err + } + return doc.Content[0], nil +} + +// aliasChain is an immutable list of alias nodes. +type aliasChain struct { + node *yaml.Node + prev *aliasChain +} + +func (c *aliasChain) contains(n *yaml.Node) bool { + for ; c != nil; c = c.prev { + if c.node == n { + return true + } + } + return false +} + +func newYAMLSource(file string, node *yaml.Node, aliases *aliasChain) (*yamlSource, error) { + s := &yamlSource{file: file, node: node, target: node, aliases: aliases} + loc := s.loc(node) + for s.target.Kind == yaml.AliasNode { + // The same alias may be reached again through another path, which is not a + // cycle; only an alias inside its own expansion is. + if s.aliases.contains(s.target) { + return nil, yamlErrorf(loc, "cyclic reference to anchor %q", s.target.Value) + } + s.aliases = &aliasChain{node: s.target, prev: s.aliases} + s.target = s.target.Alias + } + + switch s.target.Kind { + case yaml.MappingNode: + s.k = KindMap + case yaml.SequenceNode: + s.k = KindSequence + case yaml.ScalarNode: + if err := s.resolveScalar(loc); err != nil { + return nil, err + } + default: + return nil, yamlErrorf(loc, "unknown node kind: %v", s.target.Kind) + } + return s, nil +} + +func yamlErrorf(loc diag.Location, format string, args ...any) error { + return fmt.Errorf("yaml (%s): %s", loc, fmt.Sprintf(format, args...)) +} + +func (s *yamlSource) loc(n *yaml.Node) diag.Location { + return diag.Location{File: s.file, Line: n.Line, Column: n.Column} +} + +func intValue(i64 int64) any { + // Use regular int type instead of int64 if possible. + if i64 >= math.MinInt32 && i64 <= math.MaxInt32 { + return int(i64) + } + return i64 +} + +func (s *yamlSource) resolveScalar(loc diag.Location) error { + n := s.target + switch st := n.ShortTag(); st { + case "!!str": + s.k, s.v = KindString, n.Value + case "!!bool": + switch strings.ToLower(n.Value) { + case "true": + s.k, s.v = KindBool, true + case "false": + s.k, s.v = KindBool, false + default: + return yamlErrorf(loc, "invalid bool value: %v", n.Value) + } + case "!!int": + // Try to parse the integer value in base 10. Trim leading zeros to avoid + // octal parsing of the "0" prefix (YAML 1.2 spec example 2.19). + i64, err := strconv.ParseInt(strings.TrimLeft(n.Value, "0"), 10, 64) + if err != nil { + // Let ParseInt figure out the base. + i64, err = strconv.ParseInt(n.Value, 0, 64) + } + if err != nil { + return yamlErrorf(loc, "invalid int value: %v", n.Value) + } + s.k, s.v = KindInt, intValue(i64) + case "!!float": + f64, err := strconv.ParseFloat(n.Value, 64) + if err == nil { + s.k, s.v = KindFloat, f64 + break + } + // Deal with infinity prefixes. + v := strings.ToLower(n.Value) + switch { + case strings.HasPrefix(v, "+"): + v = strings.TrimPrefix(v, "+") + f64 = math.Inf(1) + case strings.HasPrefix(v, "-"): + v = strings.TrimPrefix(v, "-") + f64 = math.Inf(-1) + default: + f64 = math.Inf(1) + } + switch v { + case ".inf": + s.k, s.v = KindFloat, f64 + case ".nan": + s.k, s.v = KindFloat, math.NaN() + default: + return yamlErrorf(loc, "invalid float value: %v", n.Value) + } + case "!!null": + s.k = KindNil + case "!!timestamp": + if !isTimestamp(n.Value) { + return yamlErrorf(loc, "invalid timestamp value: %v", n.Value) + } + // Timestamps are kept as the string they were written as. + s.k, s.v = KindTime, n.Value + default: + return yamlErrorf(loc, "unknown tag: %v", st) + } + return nil +} + +func isTimestamp(s string) bool { + for _, layout := range []string{ + "2006-1-2T15:4:5.999999999Z07:00", // RCF3339Nano with short date fields. + "2006-1-2t15:4:5.999999999Z07:00", // RFC3339Nano with short date fields and lower-case "t". + "2006-1-2 15:4:5.999999999", // space separated with no time zone + "2006-1-2", // date only + } { + if _, err := time.Parse(layout, s); err == nil { + return true + } + } + return false +} + +func (s *yamlSource) kind() Kind { + return s.k +} + +func (s *yamlSource) locations() []diag.Location { + // An alias has the location of the anchored value. + locs := []diag.Location{s.loc(s.target)} + if s.mapLocations != nil { + return s.mapLocations(s.path, locs) + } + return locs +} + +func (s *yamlSource) anchor() bool { + return s.node.Anchor != "" || s.target.Anchor != "" +} + +func (s *yamlSource) scalar() any { + return s.v +} + +// child returns the source of n, the value at path below this one (computed by path, +// only when locations are mapped). +func (s *yamlSource) child(n *yaml.Node, path func() *structpath.PathNode) (*yamlSource, error) { + c, err := newYAMLSource(s.file, n, s.aliases) + if err != nil { + return nil, err + } + if s.mapLocations != nil { + c.mapLocations = s.mapLocations + c.path = path() + } + return c, nil +} + +func (s *yamlSource) pairs() ([]sourcePair, error) { + node := s.target + loc := s.loc(node) + var merge *yaml.Node + var acc pairSet + for i := 0; i < len(node.Content); i += 2 { + key := node.Content[i] + val := node.Content[i+1] + + if key.Kind != yaml.ScalarNode { + return nil, yamlErrorf(loc, "key is not a scalar") + } + + switch st := key.ShortTag(); st { + case "!!str": + case "!!null": + // A literal unquoted "null" is treated as a null value by the YAML parser. + // However, when used as a key, it is treated as the string "null". + case "!!merge": + if merge != nil { + // The YAML merge key spec allows a single '<<' key per mapping. + return nil, &LocationError{ + Loc: s.loc(key), + Summary: "duplicate YAML merge key ('<<') is not allowed; to merge multiple maps, use a sequence: '<<: [*anchor1, *anchor2]'", + } + } + merge = val + continue + default: + return nil, yamlErrorf(loc, "invalid key tag: %v", st) + } + + v, err := s.child(val, func() *structpath.PathNode { return structpath.NewStringKey(s.path, key.Value) }) + if err != nil { + return nil, err + } + acc.set(sourcePair{key: key.Value, keyLocs: []diag.Location{s.loc(key)}, value: v}) + } + + if merge == nil { + return acc.pairs, nil + } + + mloc := s.loc(merge) + merr := yamlErrorf(mloc, "map merge requires map or sequence of maps as the value") + + var mnodes []*yaml.Node + switch merge.Kind { + case yaml.SequenceNode: + mnodes = merge.Content + case yaml.AliasNode: + mnodes = []*yaml.Node{merge} + default: + return nil, merr + } + + // Merged maps come first; the entries of this mapping take precedence. + var out pairSet + for _, n := range mnodes { + // The merged mapping's values are at the paths of this mapping's. + ms, err := s.child(n, func() *structpath.PathNode { return s.path }) + if err != nil { + return nil, err + } + if ms.kind() != KindMap { + return nil, merr + } + mps, err := ms.pairs() + if err != nil { + return nil, err + } + for _, p := range mps { + out.set(p) + } + } + for _, p := range acc.pairs { + out.set(p) + } + return out.pairs, nil +} + +// setPair sets the pair in pairs: an existing key keeps its position and key location. +// pairSet is a list of pairs with unique keys, in the order the keys were first set. +type pairSet struct { + pairs []sourcePair + index map[string]int +} + +// set adds p, or replaces the value of the pair with the same key (keeping its key +// location and position). +func (s *pairSet) set(p sourcePair) { + if i, ok := s.index[p.key]; ok { + s.pairs[i].value = p.value + return + } + if s.index == nil { + s.index = map[string]int{} + } + s.index[p.key] = len(s.pairs) + s.pairs = append(s.pairs, p) +} + +func (s *yamlSource) elems() ([]source, error) { + var out []source + for i, n := range s.target.Content { + v, err := s.child(n, func() *structpath.PathNode { return structpath.NewIndex(s.path, i) }) + if err != nil { + return nil, err + } + out = append(out, v) + } + return out, nil +} diff --git a/libs/dyn/kind.go b/libs/structs/structvar/kind.go similarity index 54% rename from libs/dyn/kind.go rename to libs/structs/structvar/kind.go index 1890e9e2ccd..a0775f98afe 100644 --- a/libs/dyn/kind.go +++ b/libs/structs/structvar/kind.go @@ -1,9 +1,8 @@ -package dyn +package structvar -import ( - "fmt" -) +import "fmt" +// Kind is the kind of a value in the configuration tree. type Kind int const ( @@ -19,31 +18,6 @@ const ( KindNil ) -func kindOf(v any) Kind { - switch v.(type) { - case Mapping: - return KindMap - case []Value: - return KindSequence - case string: - return KindString - case bool: - return KindBool - case int, int8, int16, int32, int64: - return KindInt - case uint, uint8, uint16, uint32, uint64: - return KindInt - case float32, float64: - return KindFloat - case Time: - return KindTime - case nil: - return KindNil - default: - panic(fmt.Sprintf("not handled: %T", v)) - } -} - func (k Kind) String() string { switch k { case KindInvalid: diff --git a/libs/structs/structvar/locations.go b/libs/structs/structvar/locations.go new file mode 100644 index 00000000000..733c9b34df6 --- /dev/null +++ b/libs/structs/structvar/locations.go @@ -0,0 +1,262 @@ +package structvar + +import ( + "maps" + "slices" + + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structpath" +) + +// Locations records the source locations of the values of a typed value, as a tree +// that follows the value: one node per map key or sequence index. +// +// Locations is immutable: operations that change it return a new tree that shares +// unchanged subtrees, so copies of a struct holding one can share it safely. +// A nil *Locations is an empty tree. +type Locations struct { + locs []diag.Location + keys map[string]*Locations + elems []*Locations +} + +// pathComponent is a key or an index of a path. +type pathComponent struct { + key string + index int + isKey bool +} + +// components returns the keys and indices of p. +func components(p *structpath.PathNode) []pathComponent { + var out []pathComponent + for _, n := range p.AsSlice() { + if k, ok := n.StringKey(); ok { + out = append(out, pathComponent{key: k, isKey: true}) + } else if i, ok := n.Index(); ok { + out = append(out, pathComponent{index: i}) + } + } + return out +} + +func (l *Locations) child(c pathComponent) *Locations { + if l == nil { + return nil + } + if c.isKey { + return l.keys[c.key] + } + if c.index < 0 || c.index >= len(l.elems) { + return nil + } + return l.elems[c.index] +} + +// Key returns the locations of map key or struct field k. +func (l *Locations) Key(k string) *Locations { + return l.child(pathComponent{key: k, isKey: true}) +} + +// Index returns the locations of sequence element i. +func (l *Locations) Index(i int) *Locations { + return l.child(pathComponent{index: i}) +} + +// Get returns the locations recorded for the value itself. +func (l *Locations) Get() []diag.Location { + if l == nil { + return nil + } + return slices.Clone(l.locs) +} + +// At returns the locations of the value at path. Values without recorded locations +// (e.g. set by code) have none. +func (l *Locations) At(path *structpath.PathNode) []diag.Location { + return l.lookup(components(path)).Get() +} + +// Nearest returns the locations of the value at path or, if it has none, of the +// closest ancestor that has some. Use it to point diagnostics at the most specific +// place there is, not to decide where a value was defined. +func (l *Locations) Nearest(path *structpath.PathNode) []diag.Location { + var nearest []diag.Location + n := l + for _, c := range components(path) { + if n == nil { + break + } + if len(n.locs) > 0 { + nearest = n.locs + } + n = n.child(c) + } + if n != nil && len(n.locs) > 0 { + nearest = n.locs + } + return slices.Clone(nearest) +} + +func (l *Locations) lookup(path []pathComponent) *Locations { + n := l + for _, c := range path { + n = n.child(c) + if n == nil { + return nil + } + } + return n +} + +// Sub returns the locations of the value at path and below. +func (l *Locations) Sub(path *structpath.PathNode) *Locations { + return l.lookup(components(path)) +} + +// With returns a copy of l where the locations of the value at path and below are sub. +func (l *Locations) With(path *structpath.PathNode, sub *Locations) *Locations { + return l.with(components(path), sub) +} + +func (l *Locations) with(path []pathComponent, sub *Locations) *Locations { + if len(path) == 0 { + return sub + } + child := l.child(path[0]).with(path[1:], sub) + return l.withChild(path[0], child) +} + +func (l *Locations) clone() *Locations { + if l == nil { + return &Locations{} + } + c := *l + c.keys = maps.Clone(l.keys) + c.elems = slices.Clone(l.elems) + return &c +} + +func (l *Locations) withChild(c pathComponent, child *Locations) *Locations { + out := l.clone() + out.setChild(c, child) + return out +} + +// setChild sets the child c of l in place. Only use it on a node the caller owns, +// e.g. one just returned by [Locations.WithLocations], which builds a fresh node. +func (l *Locations) setChild(c pathComponent, child *Locations) { + if c.isKey { + if child == nil { + delete(l.keys, c.key) + return + } + if l.keys == nil { + l.keys = map[string]*Locations{} + } + l.keys[c.key] = child + return + } + if c.index < 0 { + return + } + for len(l.elems) <= c.index { + l.elems = append(l.elems, nil) + } + l.elems[c.index] = child +} + +// WithLocations returns a copy of l where the value itself has the locations locs. +func (l *Locations) WithLocations(locs []diag.Location) *Locations { + c := l.clone() + c.locs = slices.Clone(locs) + return c +} + +// WithSequence returns a copy of l where the elements of the sequence at path are +// rebuilt from the old ones: new element i has the locations of old elements sources[i] +// merged (see [MergeLocations]). Use it after reordering, merging, splitting or +// dropping sequence elements. +func (l *Locations) WithSequence(path *structpath.PathNode, sources [][]int) *Locations { + p := components(path) + n := l.lookup(p) + if n == nil { + return l + } + c := n.clone() + c.elems = make([]*Locations, len(sources)) + for i, src := range sources { + var merged *Locations + for j, k := range src { + if j == 0 { + merged = n.Index(k) + continue + } + merged = MergeLocations(merged, n.Index(k)) + } + c.elems[i] = merged + } + return l.with(p, c) +} + +// MergeLocations returns the locations of the value obtained by merging the value +// described by b into the one described by a, following merge.Merge: maps and +// sequences keep a's locations and accumulate b's, other values take b's and +// accumulate a's. +func MergeLocations(a, b *Locations) *Locations { + switch { + case a == nil: + return b + case b == nil: + return a + } + isComposite := len(a.keys) > 0 || len(a.elems) > 0 || len(b.keys) > 0 || len(b.elems) > 0 + if !isComposite { + return b.WithLocations(concatLocations(b.locs, a.locs)) + } + c := a.clone() + c.locs = concatLocations(a.locs, b.locs) + for k, bv := range b.keys { + if c.keys == nil { + c.keys = map[string]*Locations{} + } + c.keys[k] = MergeLocations(a.keys[k], bv) + } + c.elems = append(slices.Clone(a.elems), b.elems...) + return c +} + +func concatLocations(a, b []diag.Location) []diag.Location { + if len(b) == 0 { + return a + } + return append(slices.Clone(a), b...) +} + +// Map returns a copy of l where the locations of every value are replaced by fn(path, locations). +func (l *Locations) Map(fn func(path *structpath.PathNode, locs []diag.Location) []diag.Location) *Locations { + return l.mapLocations(nil, fn) +} + +func (l *Locations) mapLocations(path *structpath.PathNode, fn func(*structpath.PathNode, []diag.Location) []diag.Location) *Locations { + if l == nil { + return nil + } + c := l.clone() + c.locs = fn(path, slices.Clone(l.locs)) + for k, v := range c.keys { + c.keys[k] = v.mapLocations(structpath.NewStringKey(path, k), fn) + } + for i, v := range c.elems { + c.elems[i] = v.mapLocations(structpath.NewIndex(path, i), fn) + } + return c +} + +// child returns the path of this component below prefix. +func (c pathComponent) child(prefix *structpath.PathNode) *structpath.PathNode { + if c.isKey { + return structpath.NewStringKey(prefix, c.key) + } + return structpath.NewIndex(prefix, c.index) +} diff --git a/libs/structs/structvar/lookup.go b/libs/structs/structvar/lookup.go new file mode 100644 index 00000000000..b4f3e3f1465 --- /dev/null +++ b/libs/structs/structvar/lookup.go @@ -0,0 +1,157 @@ +package structvar + +import ( + "fmt" + "reflect" + "slices" + + "github.com/databricks/cli/libs/structs/structpath" +) + +// LookupError is returned by [View.LookupWithDefaults] when the path cannot be followed. +type LookupError struct { + msg string +} + +func (e *LookupError) Error() string { + return e.msg +} + +// KeyNotFoundError is returned by [View.LookupWithDefaults] for a missing key. +type KeyNotFoundError struct { + Path *structpath.PathNode + Suggestions []string +} + +func (e *KeyNotFoundError) Error() string { + return fmt.Sprintf("key not found at %q%s", e.Path, DidYouMean(e.Suggestions)) +} + +func indexedKind(c pathComponent) string { + if c.isKey { + return "map" + } + return "sequence" +} + +// LookupWithDefaults returns the value at path. Fields that are declared in the type +// but not set resolve to their zero value: e.g. ${bundle.git.origin_url} resolves to an +// empty string if a bundle isn't located in a Git repository (yet). +func (x View) LookupWithDefaults(path *structpath.PathNode) (View, error) { + var p *structpath.PathNode + for _, c := range components(path) { + p = c.child(p) + kind := x.Kind() + if c.isKey { + switch kind { + case KindMap: + case KindNil: + return View{}, &LookupError{fmt.Sprintf("expected a %s to index %q, found nil", indexedKind(c), p)} + default: + return View{}, &LookupError{fmt.Sprintf("expected a map to index %q, found %s", p, kind)} + } + child := x.Get(c.key) + if !child.IsValid() { + zero, ok := zeroField(x, c.key) + if !ok { + return View{}, &KeyNotFoundError{Path: p, Suggestions: SuggestKeys(lookupKeys(x), c.key)} + } + child = zero + } + x = child + continue + } + + switch kind { + case KindSequence: + case KindNil: + return View{}, &LookupError{fmt.Sprintf("expected a %s to index %q, found nil", indexedKind(c), p)} + default: + return View{}, &LookupError{fmt.Sprintf("expected a sequence to index %q, found %s", p, kind)} + } + child := x.Index(c.index) + if !child.IsValid() { + return View{}, &LookupError{fmt.Sprintf("index out of bounds at %q", p)} + } + x = child + } + return x, nil +} + +// structType returns the struct type of the value, if it is a struct. +func structType(x View) (reflect.Type, bool) { + t := x.Reflect().Type() + for t.Kind() == reflect.Pointer || t.Kind() == reflect.Interface { + if t.Kind() == reflect.Interface { + v := x.Reflect() + for v.Kind() == reflect.Pointer || v.Kind() == reflect.Interface { + if v.IsNil() { + return nil, false + } + v = v.Elem() + } + t = v.Type() + continue + } + t = t.Elem() + } + return t, t.Kind() == reflect.Struct && !IsSDKNativeType(t) +} + +// fillableFieldType returns the type of the struct field with JSON name k, if a field of +// that type is given a zero value when it is not set (see convert.IncludeMissingFields). +func fillableFieldType(t reflect.Type, k string) (reflect.Type, bool) { + info := GetStructInfo(t) + index, ok := info.Fields[k] + if !ok { + return nil, false + } + ft := t.FieldByIndex(index).Type + for ft.Kind() == reflect.Pointer { + ft = ft.Elem() + } + switch ft.Kind() { + case reflect.Struct, reflect.Map, reflect.Slice, reflect.String, reflect.Bool, + reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Float32, reflect.Float64: + return ft, true + default: + return nil, false + } +} + +// zeroField returns the zero value of the struct field k of x that is not set. +func zeroField(x View, k string) (View, bool) { + t, ok := structType(x) + if !ok { + return View{}, false + } + ft, ok := fillableFieldType(t, k) + if !ok { + return View{}, false + } + zero := reflect.New(ft) + switch ft.Kind() { + case reflect.Map: + zero.Elem().Set(reflect.MakeMap(ft)) + case reflect.Slice: + zero.Elem().Set(reflect.MakeSlice(ft, 0, 0)) + default: + } + return NewView(zero.Interface(), nil, nil), true +} + +// lookupKeys returns the keys a lookup in x can find, for "did you mean" suggestions. +func lookupKeys(x View) []string { + var keys []string + for k := range x.MapItems() { + keys = append(keys, k) + } + if t, ok := structType(x); ok { + for _, k := range GetStructInfo(t).FieldNames { + if _, ok := fillableFieldType(t, k); ok && !slices.Contains(keys, k) { + keys = append(keys, k) + } + } + } + return keys +} diff --git a/libs/structs/structvar/merge.go b/libs/structs/structvar/merge.go new file mode 100644 index 00000000000..8aab3370149 --- /dev/null +++ b/libs/structs/structvar/merge.go @@ -0,0 +1,213 @@ +package structvar + +import ( + "fmt" + "maps" + "reflect" + "slices" + + "github.com/databricks/cli/libs/structs/structpath" +) + +// Merge merges the value described by src into the value at path, with the semantics +// of merge.Merge: maps are merged recursively, sequences are concatenated, primitive +// values are replaced, and a nil value on either side yields the other one. +// Locations accumulate like merge.Merge accumulates them. A failed merge changes nothing. +func (sv *StructVar) Merge(path *structpath.PathNode, src View) error { + dstView := sv.View().Lookup(path) + if err := CheckMerge(dstView, src); err != nil { + return err + } + + m := merger{refs: maps.Clone(sv.Refs)} + var locs *Locations + err := update(reflect.ValueOf(sv.Value), components(path), func(dst reflect.Value) (bool, error) { + dstView.v = dst + var err error + locs, err = m.merge(dst, dstView, src, path) + return isZeroScalar(src), err + }) + if err != nil { + return err + } + sv.Refs = m.refs + sv.Locations = sv.Locations.With(path, locs) + return nil +} + +// isZeroScalar reports whether src is a present zero scalar, which must be kept as an +// explicitly set value. +func isZeroScalar(src View) bool { + k := src.Kind() + return k != KindNil && k != KindMap && k != KindSequence && src.isZero() +} + +// CheckMerge returns the error [StructVar.Merge] would return for merging src into dst. +func CheckMerge(dst, src View) error { + ak, bk := dst.Kind(), src.Kind() + if ak == KindInvalid || ak == KindNil || bk == KindInvalid || bk == KindNil { + return nil + } + switch ak { + case KindMap: + if bk != KindMap { + return fmt.Errorf("cannot merge map with %s", bk) + } + for k, sc := range src.MapItems() { + if err := CheckMerge(dst.Get(k), sc); err != nil { + return err + } + } + return nil + case KindSequence: + if bk != KindSequence { + return fmt.Errorf("cannot merge sequence with %s", bk) + } + return nil + default: + if ak != bk { + return fmt.Errorf("cannot merge %s with %s", ak, bk) + } + return nil + } +} + +type merger struct { + refs map[string]string +} + +// assign sets dst to src (see [StructVar.Assign]) and returns its locations. +func (m *merger) assign(dst reflect.Value, src View, path *structpath.PathNode) (*Locations, error) { + var d decoder + n, ok, err := d.decode(dst, viewSource{src}, path) + if err != nil { + return nil, err + } + if !ok { + dst.SetZero() + } + m.refs = n.refs(path, withoutRefs(m.refs, path)) + return n.locations(), nil +} + +// merge merges src into dst (settable; dstView is its view, invalid if absent). +func (m *merger) merge(dst reflect.Value, dstView, src View, path *structpath.PathNode) (*Locations, error) { + ak, bk := dstView.Kind(), src.Kind() + switch { + case bk == KindInvalid: + return dstView.loc, nil + case ak == KindInvalid: + return m.assign(dst, src, path) + case ak == KindNil: + locs, err := m.assign(dst, src, path) + return locs.WithLocations(concatLocations(src.loc.Get(), dstView.loc.Get())), err + case bk == KindNil: + return dstView.loc.WithLocations(concatLocations(dstView.loc.Get(), src.loc.Get())), nil + } + + switch ak { + case KindMap: + out := dstView.loc.WithLocations(concatLocations(dstView.loc.Get(), src.loc.Get())) + v := derefAlloc(dst) + for k, sc := range src.MapItems() { + c := pathComponent{key: k, isKey: true} + cp := structpath.NewStringKey(path, k) + var locs *Locations + err := updateChild(v, c, func(child reflect.Value) (bool, error) { + childView := dstView.Get(k) + childView.v = child + var err error + locs, err = m.merge(child, childView, sc, cp) + return isZeroScalar(sc), err + }) + if err != nil { + return nil, err + } + out.setChild(c, locs) + } + return out, nil + case KindSequence: + out := dstView.loc.WithLocations(concatLocations(dstView.loc.Get(), src.loc.Get())) + v := derefAlloc(dst) + holder := v + if v.Kind() == reflect.Interface { + // A sequence held in an interface (e.g. a complex variable default) is + // appended to as a copy and stored back. + v = reflect.New(holder.Elem().Type()).Elem() + v.Set(holder.Elem()) + } + for _, sc := range src.Sequence() { + i := v.Len() + e := reflect.New(v.Type().Elem()).Elem() + locs, err := m.assign(e, sc, structpath.NewIndex(path, i)) + if err != nil { + return nil, err + } + v.Set(reflect.Append(v, e)) + out.setChild(pathComponent{index: i}, locs) + } + if holder.Kind() == reflect.Interface { + holder.Set(v) + } + return out, nil + default: + locs, err := m.assign(dst, src, path) + return locs.WithLocations(concatLocations(src.loc.Get(), dstView.loc.Get())), err + } +} + +// MergeElementsByKey merges the elements of the sequence at path that have the same +// key, like merge.ElementsByKey: elements are merged into the first one with the same +// key (in order), the key field of every element is set to its key, and the keys are +// sorted if sortKeys is set. +func (sv *StructVar) MergeElementsByKey(path *structpath.PathNode, keyField string, keyFn func(View) string, sortKeys bool) error { + seq := sv.View().Lookup(path) + if seq.Kind() != KindSequence { + return nil + } + + var keys []string + groups := map[string][]int{} + for i, e := range seq.Sequence() { + k := keyFn(e.Get(keyField)) + if _, ok := groups[k]; !ok { + keys = append(keys, k) + } + groups[k] = append(groups[k], i) + } + if len(keys) == 0 { + return nil + } + if sortKeys { + slices.Sort(keys) + } + + // Build the merged sequence separately: the first element of each group is + // assigned, the others are merged into it. + sliceType := seq.Reflect().Type() + for sliceType.Kind() == reflect.Pointer { + sliceType = sliceType.Elem() + } + tmp := &StructVar{Value: reflect.New(sliceType).Interface()} + for i, k := range keys { + elem := structpath.NewIndex(nil, i) + for j, idx := range groups[k] { + var err error + if j == 0 { + _, err = tmp.Assign(elem, seq.Index(idx)) + } else { + err = tmp.Merge(elem, seq.Index(idx)) + } + if err != nil { + return err + } + } + // The key field holds the key (e.g. a normalized one), keeping its location. + if err := tmp.Set(structpath.NewStringKey(elem, keyField), k); err != nil { + return err + } + } + tmp.Locations = tmp.Locations.WithLocations(seq.loc.Get()) + _, err := sv.Assign(path, tmp.View()) + return err +} diff --git a/libs/structs/structvar/override.go b/libs/structs/structvar/override.go new file mode 100644 index 00000000000..f44b5e08f5c --- /dev/null +++ b/libs/structs/structvar/override.go @@ -0,0 +1,163 @@ +package structvar + +import ( + "errors" + "fmt" + "reflect" + + "github.com/databricks/cli/libs/structs/structpath" +) + +// ErrOverrideUndoDelete can be returned by [OverrideVisitor.VisitDelete] to keep the +// deleted value. +var ErrOverrideUndoDelete = errors.New("undo delete operation") + +// OverrideVisitor is notified of the differences found by [PlanOverride]. +// Any error aborts the override. +type OverrideVisitor struct { + VisitDelete func(path *structpath.PathNode, left View) error + VisitInsert func(path *structpath.PathNode, right View) error + VisitUpdate func(path *structpath.PathNode, left, right View) error +} + +// OverridePlan is the result of [PlanOverride]; apply it with [StructVar.Override]. +type OverridePlan struct { + src View + locs *Locations + undos []undo +} + +type undo struct { + path *structpath.PathNode + value *StructVar +} + +// PlanOverride compares dst with src like merge.Override and reports the differences +// to the visitor. Applying the plan replaces dst with src, except that values unchanged +// in src keep their locations in dst, and values whose deletion the visitor undid are kept. +func PlanOverride(dst, src View, visitor OverrideVisitor) (*OverridePlan, error) { + plan := &OverridePlan{src: src} + locs, err := plan.override(nil, dst, src, visitor) + if err != nil { + return nil, err + } + plan.locs = locs + return plan, nil +} + +func (p *OverridePlan) override(path *structpath.PathNode, left, right View, visitor OverrideVisitor) (*Locations, error) { + lk, rk := left.Kind(), right.Kind() + if lk != rk { + return right.loc, visitor.VisitUpdate(path, left, right) + } + + switch lk { + case KindMap: + out := (*Locations)(nil).WithLocations(left.loc.Get()) + for k, lv := range left.MapItems() { + if right.Get(k).IsValid() { + continue + } + kp := structpath.NewStringKey(path, k) + err := visitor.VisitDelete(kp, lv) + if errors.Is(err, ErrOverrideUndoDelete) { + p.keep(kp, lv) + out.setChild(pathComponent{key: k, isKey: true}, lv.loc) + } else if err != nil { + return nil, err + } + } + for k, rv := range right.MapItems() { + kp := structpath.NewStringKey(path, k) + locs := rv.loc + if lv := left.Get(k); lv.IsValid() { + var err error + locs, err = p.override(kp, lv, rv, visitor) + if err != nil { + return nil, err + } + } else if err := visitor.VisitInsert(kp, rv); err != nil { + return nil, err + } + out.setChild(pathComponent{key: k, isKey: true}, locs) + } + return out, nil + case KindSequence: + out := (*Locations)(nil).WithLocations(left.loc.Get()) + var ls, rs []View + for _, v := range left.Sequence() { + ls = append(ls, v) + } + for _, v := range right.Sequence() { + rs = append(rs, v) + } + n := 0 + for i := range min(len(ls), len(rs)) { + locs, err := p.override(structpath.NewIndex(path, i), ls[i], rs[i], visitor) + if err != nil { + return nil, err + } + out.setChild(pathComponent{index: n}, locs) + n++ + } + for i := len(ls); i < len(rs); i++ { + if err := visitor.VisitInsert(structpath.NewIndex(path, i), rs[i]); err != nil { + return nil, err + } + out.setChild(pathComponent{index: n}, rs[i].loc) + n++ + } + for i := len(rs); i < len(ls); i++ { + err := visitor.VisitDelete(structpath.NewIndex(path, i), ls[i]) + if errors.Is(err, ErrOverrideUndoDelete) { + p.keep(structpath.NewIndex(path, n), ls[i]) + out.setChild(pathComponent{index: n}, ls[i].loc) + n++ + } else if err != nil { + return nil, err + } + } + return out, nil + case KindNil: + return left.loc, nil + case KindString, KindBool, KindInt, KindFloat: + if left.AsAny() == right.AsAny() || sameNumber(left, right) { + return left.loc, nil + } + return right.loc, visitor.VisitUpdate(path, left, right) + default: + return nil, fmt.Errorf("unexpected kind %s at %s", lk, path) + } +} + +func sameNumber(a, b View) bool { + if ai, ok := a.AsInt(); ok { + bi, ok := b.AsInt() + return ok && ai == bi + } + return false +} + +// keep records that the value at path (in the result) is the kept left value. It is +// copied now, because applying the plan overwrites the left side. +func (p *OverridePlan) keep(path *structpath.PathNode, v View) { + cp := &StructVar{Value: reflect.New(v.v.Type()).Interface()} + _, _ = cp.Assign(nil, v) + p.undos = append(p.undos, undo{path: path, value: cp}) +} + +// Override replaces sv.Value with the result of plan (see [PlanOverride]). +func (sv *StructVar) Override(plan *OverridePlan) error { + sv.Refs = nil + sv.Locations = nil + if _, err := sv.Assign(nil, plan.src); err != nil { + return err + } + for _, u := range plan.undos { + if _, err := sv.Assign(u.path, u.value.View()); err != nil { + return err + } + } + sv.Locations = plan.locs + return nil +} diff --git a/libs/structs/structvar/parity_walk_test.go b/libs/structs/structvar/parity_walk_test.go new file mode 100644 index 00000000000..f3111c64342 --- /dev/null +++ b/libs/structs/structvar/parity_walk_test.go @@ -0,0 +1,71 @@ +package structvar_test + +import ( + "testing" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWalkVisitsParentsBeforeChildren(t *testing.T) { + root := config.Root{ + Bundle: config.Bundle{Name: "x"}, + Sync: config.Sync{Include: []string{"a", "b"}}, + } + + var visited []string + err := structvar.Walk(root.View(), func(p *structpath.PathNode, v structvar.View) error { + if s, ok := v.AsString(); ok { + visited = append(visited, p.String()+"="+s) + } + return nil + }) + require.NoError(t, err) + assert.Equal(t, []string{"bundle.name=x", "sync.include[0]=a", "sync.include[1]=b"}, visited) +} + +func TestWalkVisitsAllValues(t *testing.T) { + root, diags := config.LoadFromBytes("test.yml", []byte(` +resources: + jobs: + j: + name: n + max_concurrent_runs: ${var.x} + tags: + a: b +`)) + require.NoError(t, diags.Error()) + + var visited []string + err := structvar.Walk(root.View().Lookup(structpath.MustParsePath("resources.jobs.j")), func(p *structpath.PathNode, _ structvar.View) error { + visited = append(visited, p.String()) + return nil + }) + require.NoError(t, err) + assert.Equal(t, []string{"", "max_concurrent_runs", "name", "tags", "tags.a"}, visited) +} + +func TestSetReference(t *testing.T) { + root, diags := config.LoadFromBytes("test.yml", []byte(` +resources: + jobs: + j: + name: n + max_concurrent_runs: ${var.x} +`)) + require.NoError(t, diags.Error()) + + path := structpath.MustParsePath("resources.jobs.j.max_concurrent_runs") + require.NoError(t, root.SetReference(path, "${var.y}")) + s, ok := root.View().Lookup(path).AsString() + require.True(t, ok) + assert.Equal(t, "${var.y}", s) + + // A string field holds the reference itself. + name := structpath.MustParsePath("resources.jobs.j.name") + require.NoError(t, root.SetReference(name, "${var.y}")) + assert.Equal(t, "${var.y}", root.Resources.Jobs["j"].Name) +} diff --git a/libs/structs/structvar/pattern.go b/libs/structs/structvar/pattern.go new file mode 100644 index 00000000000..63cbdd0658e --- /dev/null +++ b/libs/structs/structvar/pattern.go @@ -0,0 +1,110 @@ +package structvar + +import ( + "errors" + "fmt" + + "github.com/databricks/cli/libs/structs/structpath" +) + +// ForEach calls fn for every value in x that matches pattern, like dyn.MapByPattern +// visits them: map keys in order, wildcards skip values the rest of the pattern does +// not match, and a missing key or index outside a wildcard visits nothing. +// +// fn may change the configuration at or below the visited path (e.g. with [Set]), +// but must not add or remove the keys and elements being iterated. +func ForEach(x View, pattern *structpath.PatternNode, fn func(*structpath.PathNode, View) error) error { + err := forEach(x, nil, pattern.AsSlice(), fn) + if errors.Is(err, errNoMatch) { + return nil + } + return err +} + +// errNoMatch means the pattern did not match: a key or index is missing or a value is nil. +var errNoMatch = errors.New("no match") + +// errUnexpectedKind means a wildcard was applied to a value of the wrong kind. +type errUnexpectedKind struct { + msg string +} + +func (e errUnexpectedKind) Error() string { + return e.msg +} + +func forEach(x View, prefix *structpath.PathNode, pattern []*structpath.PatternNode, fn func(*structpath.PathNode, View) error) error { + if len(pattern) == 0 { + return fn(prefix, x) + } + + c := pattern[0] + rest := pattern[1:] + kind := x.Kind() + + if c.DotStar() { + if kind != KindMap { + return errUnexpectedKind{fmt.Sprintf("expected a map at %q, found %s", prefix, kind)} + } + for k, child := range x.MapItems() { + if err := forEach(child, structpath.NewStringKey(prefix, k), rest, fn); err != nil && !isNoMatch(err) { + return err + } + } + return nil + } + + if c.BracketStar() { + if kind != KindSequence { + return errUnexpectedKind{fmt.Sprintf("expected a sequence at %q, found %s", prefix, kind)} + } + for i, child := range x.Sequence() { + if err := forEach(child, structpath.NewIndex(prefix, i), rest, fn); err != nil && !isNoMatch(err) { + return err + } + } + return nil + } + + if key, ok := c.StringKey(); ok { + path := structpath.NewStringKey(prefix, key) + switch kind { + case KindMap: + case KindNil: + return errNoMatch + default: + return fmt.Errorf("expected a map to index %q, found %s", path, kind) + } + child := x.Get(key) + if !child.IsValid() { + return errNoMatch + } + return forEach(child, path, rest, fn) + } + + index, ok := c.Index() + if !ok { + return fmt.Errorf("unsupported pattern component %q", c) + } + path := structpath.NewIndex(prefix, index) + switch kind { + case KindSequence: + case KindNil: + return errNoMatch + default: + return fmt.Errorf("expected a sequence to index %q, found %s", path, kind) + } + child := x.Index(index) + if !child.IsValid() { + return errNoMatch + } + return forEach(child, path, rest, fn) +} + +func isNoMatch(err error) bool { + if errors.Is(err, errNoMatch) { + return true + } + _, ok := errors.AsType[errUnexpectedKind](err) + return ok +} diff --git a/libs/dyn/dynvar/ref.go b/libs/structs/structvar/ref.go similarity index 82% rename from libs/dyn/dynvar/ref.go rename to libs/structs/structvar/ref.go index de25f44a212..b619ae096d2 100644 --- a/libs/dyn/dynvar/ref.go +++ b/libs/structs/structvar/ref.go @@ -1,11 +1,11 @@ -package dynvar +package structvar import ( "fmt" "regexp" "strings" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) // BaseVarDef matches a single dot-separated path segment in ${...} references. @@ -17,14 +17,9 @@ const BaseVarDef = `_*\p{L}+([-_]*[\p{L}\p{N}]+)*` var re = regexp.MustCompile(fmt.Sprintf(`\$\{(%s(\.%s(\[[0-9]+\])*)*(\[[0-9]+\])*)\}`, BaseVarDef, BaseVarDef)) -// Ref represents a variable reference. -// It is a string [dyn.Value] contained in a larger [dyn.Value]. -// Its path within the containing [dyn.Value] is also stored. +// Ref represents the variable references in a string. type Ref struct { - // Original value. - Value dyn.Value - - // String value in the original [dyn.Value]. + // The string. Str string // Matches of the variable reference in the string. @@ -36,9 +31,7 @@ type Ref struct { Spans [][2]int } -// NewRef returns a new Ref if the given [dyn.Value] contains a string -// with one or more variable references. It returns false if the given -// [dyn.Value] does not contain variable references. +// NewRef returns a new Ref if s contains one or more variable references. // // A reference preceded by "$" is treated as an escaped literal and is not // included in the returned Ref. For example, "$${foo}" contains no references @@ -49,12 +42,7 @@ type Ref struct { // - "${a.b.c}" // - "${a.b[0].c}" // - "${a} ${b} ${c}" -func NewRef(v dyn.Value) (Ref, bool) { - s, ok := v.AsString() - if !ok { - return Ref{}, false - } - +func NewRef(s string) (Ref, bool) { // Find all matches with their byte positions so we can detect escaped references. indices := re.FindAllStringSubmatchIndex(s, -1) if len(indices) == 0 { @@ -84,7 +72,6 @@ func NewRef(v dyn.Value) (Ref, bool) { } return Ref{ - Value: v, Str: s, Matches: m, Spans: spans, @@ -162,19 +149,14 @@ func ContainsVariableReference(s string) bool { return false } -// If s is a pure variable reference, this function returns the corresponding -// dyn.Path. Otherwise, it returns false. -func PureReferenceToPath(s string) (dyn.Path, bool) { - ref, ok := NewRef(dyn.V(s)) - if !ok { - return nil, false - } - - if !ref.IsPure() { +// PureReferenceToPath returns the path s refers to if s is a pure variable reference. +func PureReferenceToPath(s string) (*structpath.PathNode, bool) { + ref, ok := NewRef(s) + if !ok || !ref.IsPure() { return nil, false } - p, err := dyn.NewPathFromString(ref.References()[0]) + p, err := structpath.ParsePath(ref.References()[0]) if err != nil { return nil, false } diff --git a/libs/dyn/dynvar/ref_test.go b/libs/structs/structvar/ref_test.go similarity index 56% rename from libs/dyn/dynvar/ref_test.go rename to libs/structs/structvar/ref_test.go index 7b93ac633b9..107c612f0b3 100644 --- a/libs/dyn/dynvar/ref_test.go +++ b/libs/structs/structvar/ref_test.go @@ -1,22 +1,16 @@ -package dynvar +package structvar import ( "testing" - "github.com/databricks/cli/libs/dyn" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestNewRefNoString(t *testing.T) { - _, ok := NewRef(dyn.V(1)) - require.False(t, ok, "should not match non-string") -} - func TestNewRefSpans(t *testing.T) { // Spans must point at the unescaped occurrence, not the escaped one that // appears earlier with identical text. - ref, ok := NewRef(dyn.V("$${a.b} ${a.b}")) + ref, ok := NewRef("$${a.b} ${a.b}") require.True(t, ok) require.Len(t, ref.Matches, 1) require.Len(t, ref.Spans, 1) @@ -63,3 +57,52 @@ func TestReplaceRef(t *testing.T) { assert.Equal(t, tt.want, ReplaceRef(tt.in, "${a.b}", "V"), "input %q", tt.in) } } + +func TestPureReferenceToPath(t *testing.T) { + tests := []struct { + name string + input string + expected string + ok bool + }{ + { + name: "simple reference", + input: "${resources.jobs.foo.id}", + expected: "resources.jobs.foo.id", + ok: true, + }, + { + name: "simple reference", + input: "${resources.jobs.foo.tasks[1].env.key}", + expected: "resources.jobs.foo.tasks[1].env.key", + ok: true, + }, + { + name: "complex nested reference", + input: "${var.resources.jobs['my_job'].tasks[0]}", + // we use regex from dyn module which only support integers inside brackets: + // expected: "resources.jobs['my_job'].tasks[0]", + }, + { + name: "not a pure reference", + input: "prefix_${var.field}", + }, + { + name: "not a variable reference", + input: "plain_string", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pathNode, ok := PureReferenceToPath(tt.input) + assert.Equal(t, tt.ok, ok) + if tt.ok { + assert.NotNil(t, pathNode) + assert.Equal(t, tt.expected, pathNode.String()) + } else { + assert.Nil(t, pathNode) + } + }) + } +} diff --git a/libs/dyn/dynvar/ref_vectors_test.go b/libs/structs/structvar/ref_vectors_test.go similarity index 91% rename from libs/dyn/dynvar/ref_vectors_test.go rename to libs/structs/structvar/ref_vectors_test.go index d0084a6ae5c..ea58cc03fba 100644 --- a/libs/dyn/dynvar/ref_vectors_test.go +++ b/libs/structs/structvar/ref_vectors_test.go @@ -1,4 +1,4 @@ -package dynvar +package structvar import ( "encoding/json" @@ -6,7 +6,6 @@ import ( "path/filepath" "testing" - "github.com/databricks/cli/libs/dyn" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -41,7 +40,7 @@ func loadReferenceVectors(t *testing.T) []referenceVector { func TestReferenceVectors(t *testing.T) { for _, v := range loadReferenceVectors(t) { t.Run(v.ID, func(t *testing.T) { - ref, ok := NewRef(dyn.V(v.Input)) + ref, ok := NewRef(v.Input) assert.Equal(t, v.Match, ok, "NewRef match") if v.Match { @@ -59,7 +58,7 @@ func TestReferenceVectors(t *testing.T) { case v.Path != nil: path, ok := PureReferenceToPath(v.Input) require.True(t, ok, "PureReferenceToPath") - assert.Equal(t, dyn.MustPathFromString(*v.Path), path) + assert.Equal(t, *v.Path, path.String()) case v.PathOK != nil: _, ok := PureReferenceToPath(v.Input) assert.Equal(t, *v.PathOK, ok, "PureReferenceToPath") diff --git a/libs/structs/structvar/resolve.go b/libs/structs/structvar/resolve.go new file mode 100644 index 00000000000..3cb8ec44683 --- /dev/null +++ b/libs/structs/structvar/resolve.go @@ -0,0 +1,200 @@ +package structvar + +import ( + "errors" + "fmt" + "maps" + "slices" + "strings" + + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structpath" +) + +// ErrSkipResolution is returned by a [Lookup] to leave a reference in place. +var ErrSkipResolution = errors.New("skip resolution") + +// ReferenceError is returned for a reference to a path that does not exist. +// Suggestions are carried as data so callers (which can import libs/diag) format them. +type ReferenceError struct { + Reference string // original reference text, e.g. "var.hst" + Suggestions []string // corrected references, e.g. ["var.host", "var.hosts"] +} + +func (e *ReferenceError) Error() string { + return fmt.Sprintf("reference does not exist: ${%s}", e.Reference) +} + +// Template is a string with variable references and the locations it was written at. +type Template struct { + Value string + Locations []diag.Location +} + +// Lookup returns the value a reference path refers to. It returns +// ErrSkipResolution to leave the reference in place. +type Lookup func(path *structpath.PathNode) (View, error) + +// Resolve resolves the references in templates (keyed by path; the keys appear in +// cycle errors) using lookup, with the semantics of dynvar.Resolve: a referenced value +// that is a reference itself is resolved first, cycles are an error, a pure reference +// is replaced by the referenced value (with the location of the reference), and other +// references are interpolated into the string. It returns the views of the resolved +// values; templates that resolve to themselves are omitted. +func Resolve(templates map[string]Template, lookup Lookup) (map[string]View, error) { + r := resolver{lookup: lookup, lookups: map[string]lookupResult{}} + out := map[string]View{} + + // Resolve in key order, so the cycle detected is deterministic. + for _, key := range slices.Sorted(maps.Keys(templates)) { + t := templates[key] + ref, ok := NewRef(t.Value) + if !ok { + continue + } + v, err := r.resolveRef(ref, t.Locations, []string{key}) + if err != nil { + return nil, err + } + if s, ok := v.AsString(); ok && s == t.Value { + continue + } + out[key] = v + } + return out, nil +} + +type lookupResult struct { + v View + err error +} + +type resolver struct { + lookup Lookup + lookups map[string]lookupResult +} + +// stringView returns a view of the string s with locations locs. +func stringView(s string, locs []diag.Location) View { + return NewView(&s, nil, (*Locations)(nil).WithLocations(locs)) +} + +func (r *resolver) resolveRef(ref Ref, locs []diag.Location, seen []string) (View, error) { + deps := ref.References() + + // Resolve each of the dependencies, then interpolate them in the ref. + resolved := make([]View, len(deps)) + complete := true + + for j, dep := range deps { + // Cycle detection. + if slices.Contains(seen, dep) { + return View{}, fmt.Errorf( + "cycle detected in field resolution: %s", + strings.Join(append(seen, dep), " -> "), + ) + } + + v, err := r.resolveKey(dep, append(seen, dep)) + + // If we should skip resolution of this key, index j holds an invalid view. + if errors.Is(err, ErrSkipResolution) { + complete = false + continue + } else if err != nil { + return View{}, err + } + + resolved[j] = v + } + + // A pure reference is replaced by the value, which keeps its type. It takes the + // location of the reference, so relative paths resolve relative to where a + // variable is used, not where it is defined. + if ref.IsPure() && complete { + return resolved[0].WithLocations(locs), nil + } + + // Not pure; perform string interpolation. Substitute by byte offset: the same + // reference may also appear escaped ("$${foo} ${foo}"). + var sb strings.Builder + consumed := 0 + for j := range ref.Matches { + // Leave references that were skipped in place. + if !resolved[j].IsValid() { + continue + } + + s, ok := resolved[j].AsString() + if !ok { + // Only allow primitive types to be converted to string. + switch kind := resolved[j].Kind(); kind { + case KindBool, KindInt, KindFloat, KindNil: + s = fmt.Sprint(resolved[j].AsAny()) + default: + return View{}, fmt.Errorf("cannot interpolate non-primitive value of type %s into string", kind) + } + } + + start, end := ref.Spans[j][0], ref.Spans[j][1] + sb.WriteString(ref.Str[consumed:start]) + sb.WriteString(s) + consumed = end + } + sb.WriteString(ref.Str[consumed:]) + + return stringView(sb.String(), locs), nil +} + +func (r *resolver) resolveKey(key string, seen []string) (View, error) { + if v, ok := r.lookups[key]; ok { + return v.v, v.err + } + + p, err := structpath.ParsePath(key) + if err != nil { + return View{}, err + } + + v, err := r.lookup(p) + if err != nil { + if knf, ok := errors.AsType[*KeyNotFoundError](err); ok { + // Carry suggestions as data; the caller formats them. + err = &ReferenceError{Reference: key, Suggestions: suggestedReferences(knf, key)} + } + r.lookups[key] = lookupResult{err: err} + return View{}, err + } + + // If the value is a reference itself, resolve it. + if s, ok := v.AsString(); ok { + if ref, ok := NewRef(s); ok { + v, err = r.resolveRef(ref, v.Locations(), seen) + } + } + + r.lookups[key] = lookupResult{v: v, err: err} + return v, err +} + +// WithLocations returns the view with its own locations replaced by locs; the values +// below it keep theirs. +func (x View) WithLocations(locs []diag.Location) View { + x.loc = x.loc.WithLocations(locs) + return x +} + +// suggestedReferences returns drop-in replacement references for reference, rebuilt +// by swapping the key that was not found for each suggestion. +func suggestedReferences(err *KeyNotFoundError, reference string) []string { + if len(err.Suggestions) == 0 { + return nil + } + nodes := err.Path.AsSlice() + failedKey, _ := nodes[len(nodes)-1].StringKey() + refs := make([]string, len(err.Suggestions)) + for i, s := range err.Suggestions { + refs[i] = ReplaceKey(reference, failedKey, s) + } + return refs +} diff --git a/libs/dyn/convert/struct_info.go b/libs/structs/structvar/struct_info.go similarity index 65% rename from libs/dyn/convert/struct_info.go rename to libs/structs/structvar/struct_info.go index 69e8f868ed5..46ca6cde995 100644 --- a/libs/dyn/convert/struct_info.go +++ b/libs/structs/structvar/struct_info.go @@ -1,27 +1,25 @@ -package convert +package structvar import ( "reflect" "slices" "sync" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structtag" + sdkduration "github.com/databricks/databricks-sdk-go/common/types/duration" + sdkfieldmask "github.com/databricks/databricks-sdk-go/common/types/fieldmask" + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" ) -// structInfo holds the type information we need to efficiently -// convert data from a [dyn.Value] to a Go struct. -type structInfo struct { +// StructInfo holds the type information of a struct for converting between +// the configuration tree and the typed value: JSON names, omitempty, ForceSendFields. +type StructInfo struct { // FieldNames is ordered list of fields FieldNames []string // Fields maps the JSON-name of the field to the field's index for use with [FieldByIndex]. Fields map[string][]int - // ValueField maps to the field with a [dyn.Value]. - // The underlying type is expected to only have one of these. - ValueField []int - // Tracks which fields do not have omitempty annotation ForceEmpty map[string]bool @@ -39,31 +37,31 @@ type structInfo struct { ForceSendFieldsIndex map[string][]int } -// structInfoCache caches type information. -var structInfoCache = make(map[reflect.Type]structInfo) +// StructInfoCache caches type information. +var StructInfoCache = make(map[reflect.Type]StructInfo) -// structInfoCacheLock guards concurrent access to structInfoCache. -var structInfoCacheLock sync.Mutex +// StructInfoCacheLock guards concurrent access to StructInfoCache. +var StructInfoCacheLock sync.Mutex -// getStructInfo returns the [structInfo] for the given type. +// getStructInfo returns the [StructInfo] for the given type. // It lazily populates a cache, so the first call for a given // type is slower than subsequent calls for that same type. -func getStructInfo(typ reflect.Type) structInfo { - structInfoCacheLock.Lock() - defer structInfoCacheLock.Unlock() +func GetStructInfo(typ reflect.Type) StructInfo { + StructInfoCacheLock.Lock() + defer StructInfoCacheLock.Unlock() - si, ok := structInfoCache[typ] + si, ok := StructInfoCache[typ] if !ok { si = buildStructInfo(typ) - structInfoCache[typ] = si + StructInfoCache[typ] = si } return si } -// buildStructInfo populates a new [structInfo] for the given type. -func buildStructInfo(typ reflect.Type) structInfo { - out := structInfo{ +// buildStructInfo populates a new [StructInfo] for the given type. +func buildStructInfo(typ reflect.Type) StructInfo { + out := StructInfo{ Fields: make(map[string][]int), ForceEmpty: make(map[string]bool), GolangNames: make(map[string]string), @@ -106,15 +104,6 @@ func buildStructInfo(typ reflect.Type) structInfo { continue } - // If this field has type [dyn.Value], we populate it with the source [dyn.Value] from [ToTyped]. - if sf.IsExported() && sf.Type == configValueType { - if out.ValueField != nil { - panic("multiple dyn.Value fields") - } - out.ValueField = append(prefix, sf.Index...) - continue - } - jtag := structtag.JSONTag(sf.Tag.Get("json")) name := jtag.Name() if name == "" || name == "-" { @@ -151,18 +140,18 @@ type FieldValue struct { IsForced bool } -func (s *structInfo) FieldValues(v reflect.Value) []FieldValue { +func (s *StructInfo) FieldValues(v reflect.Value) []FieldValue { out := make([]FieldValue, 0, len(s.Fields)) for _, k := range s.FieldNames { - fv := fieldByIndex(v, s.Fields[k]) + fv := FieldByIndex(v, s.Fields[k]) if fv.IsValid() { isForced := true // TODO: we should use isEmptyForOmitEmpty instead of IsZero() if fv.IsZero() { - isForced = s.isForceSend(v, k) + isForced = s.IsForceSend(v, k) } out = append(out, FieldValue{ @@ -177,13 +166,13 @@ func (s *structInfo) FieldValues(v reflect.Value) []FieldValue { } // isForceSend reports whether the field named k is listed in the ForceSendFields -// that governs it (see structInfo.ForceSendFieldsIndex). -func (s *structInfo) isForceSend(v reflect.Value, k string) bool { +// that governs it (see StructInfo.ForceSendFieldsIndex). +func (s *StructInfo) IsForceSend(v reflect.Value, k string) bool { index, ok := s.ForceSendFieldsIndex[k] if !ok { return false } - fsf := fieldByIndex(v, index) + fsf := FieldByIndex(v, index) if !fsf.IsValid() { return false } @@ -192,7 +181,7 @@ func (s *structInfo) isForceSend(v reflect.Value, k string) bool { // fieldByIndex resolves the value at the given index path, dereferencing embedded // pointer structs on the way. It returns an invalid value if a nil pointer is met. -func fieldByIndex(v reflect.Value, index []int) reflect.Value { +func FieldByIndex(v reflect.Value, index []int) reflect.Value { for i, x := range index { if i > 0 { if v.Kind() == reflect.Pointer && v.Type().Elem().Kind() == reflect.Struct { @@ -207,5 +196,32 @@ func fieldByIndex(v reflect.Value, index []int) reflect.Value { return v } -// Type of [dyn.Value]. -var configValueType = reflect.TypeFor[dyn.Value]() +// GetOrNewFieldByIndex resolves the value at the given index path within an addressable +// struct, allocating intermediate structs embedded as pointer types along the way. +func GetOrNewFieldByIndex(v reflect.Value, index []int) reflect.Value { + for i, x := range index { + if i > 0 { + if v.Kind() == reflect.Pointer { + if v.IsNil() { + v.Set(reflect.New(v.Type().Elem())) + } + v = v.Elem() + } + } + v = v.Field(x) + } + return v +} + +// sdkNativeTypes are SDK types with custom JSON marshaling to and from a string. +// The configuration represents them as strings. +var sdkNativeTypes = []reflect.Type{ + reflect.TypeFor[sdkduration.Duration](), // Protobuf duration format (e.g., "300s") + reflect.TypeFor[sdktime.Time](), // RFC3339 timestamp format (e.g., "2023-12-25T10:30:00Z") + reflect.TypeFor[sdkfieldmask.FieldMask](), // Comma-separated paths (e.g., "name,age,email") +} + +// IsSDKNativeType reports whether typ is an SDK type represented as a string. +func IsSDKNativeType(typ reflect.Type) bool { + return slices.Contains(sdkNativeTypes, typ) +} diff --git a/libs/structs/structvar/structvar.go b/libs/structs/structvar/structvar.go index cc894aced7f..2f5bde6642e 100644 --- a/libs/structs/structvar/structvar.go +++ b/libs/structs/structvar/structvar.go @@ -8,7 +8,6 @@ import ( "strings" "sync" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/cli/libs/structs/structpath" ) @@ -20,6 +19,9 @@ type StructVar struct { // Refs holds unresolved references. Key is serialized PathNode pointing inside a struct (e.g. "name") // and value is either pure or multiple references string: "${resources.foo.jobs.id}" or "${a} ${b}" Refs map[string]string `json:"vars,omitempty"` + + // Locations holds the source locations of the values, if known (configuration loaded from files). + Locations *Locations `json:"-"` } // StructVarJSON is the serialized form of StructVar for persisting in plan files. @@ -103,18 +105,18 @@ func (sv *StructVar) ResolveRef(reference string, value any) error { return fmt.Errorf("cannot set %s to %T (%#v): %w", pathNode.String(), value, value, err) } - newValue := dynvar.ReplaceRef(refValue, reference, valueStr) + newValue := ReplaceRef(refValue, reference, valueStr) // The struct gets the unescaped form ("$${x}" -> "${x}"), since that is what // is sent to the API. sv.Refs keeps the escaped form so the still-pending // check below can tell a literal apart from a real reference. - err = structaccess.Set(sv.Value, pathNode, dynvar.Unescape(newValue)) + err = structaccess.Set(sv.Value, pathNode, Unescape(newValue)) if err != nil { return fmt.Errorf("cannot update %s to string: %w", pathNode.String(), err) } // Check if fully resolved (no unescaped ${} patterns left) - if !dynvar.ContainsVariableReference(newValue) { + if !ContainsVariableReference(newValue) { delete(sv.Refs, pathKey) } else { sv.Refs[pathKey] = newValue diff --git a/libs/dyn/suggest.go b/libs/structs/structvar/suggest.go similarity index 60% rename from libs/dyn/suggest.go rename to libs/structs/structvar/suggest.go index ef8008aae6f..2d2f4bccef7 100644 --- a/libs/dyn/suggest.go +++ b/libs/structs/structvar/suggest.go @@ -1,15 +1,17 @@ -package dyn +package structvar import ( "fmt" "slices" "strings" + + "github.com/databricks/cli/libs/structs/structpath" ) const maxSuggestionDistance = 2 -// levenshteinDistance computes the edit distance between two strings. -func levenshteinDistance(a, b string) int { +// LevenshteinDistance computes the edit distance between two strings. +func LevenshteinDistance(a, b string) int { if len(a) == 0 { return len(b) } @@ -43,19 +45,18 @@ func levenshteinDistance(a, b string) int { return prev[len(b)] } -// suggestKeys returns the keys in m whose edit distance from name is at most +// SuggestKeys returns the keys whose edit distance from name is at most // maxSuggestionDistance, ordered by increasing distance. It is used to build -// "did you mean" hints for a key that was not found in the map. -func suggestKeys(m Mapping, name string) []string { +// "did you mean" hints for a key that was not found in a map. +func SuggestKeys(keys []string, name string) []string { type candidate struct { key string dist int } var candidates []candidate - for _, kv := range m.Keys() { - key := kv.MustString() - d := levenshteinDistance(name, key) + for _, key := range keys { + d := LevenshteinDistance(name, key) if d <= maxSuggestionDistance { candidates = append(candidates, candidate{key, d}) } @@ -72,10 +73,10 @@ func suggestKeys(m Mapping, name string) []string { return suggestions } -// didYouMean formats a suggestion clause like `, did you mean "x"?` (or, for +// DidYouMean formats a suggestion clause like `, did you mean "x"?` (or, for // multiple candidates, `, did you mean one of: "x", "y"?`). It returns an empty // string when there are no suggestions. -func didYouMean(suggestions []string) string { +func DidYouMean(suggestions []string) string { switch len(suggestions) { case 0: return "" @@ -90,19 +91,28 @@ func didYouMean(suggestions []string) string { } } -// replaceKey returns reference with the component matching failedKey swapped for +// ReplaceKey returns reference with the component matching failedKey swapped for // replacement, or just replacement if reference can't be parsed or has no match. -func replaceKey(reference, failedKey, replacement string) string { - p, err := NewPathFromString(reference) +func ReplaceKey(reference, failedKey, replacement string) string { + p, err := structpath.ParsePath(reference) if err != nil { return replacement } - for i, c := range p { - if c.Key() == failedKey { - out := p.Append() - out[i] = Key(replacement) - return out.String() + var out *structpath.PathNode + replaced := false + for _, n := range p.AsSlice() { + if k, ok := n.StringKey(); ok { + if k == failedKey && !replaced { + k = replacement + replaced = true + } + out = structpath.NewStringKey(out, k) + } else if i, ok := n.Index(); ok { + out = structpath.NewIndex(out, i) } } - return replacement + if !replaced { + return replacement + } + return out.String() } diff --git a/libs/dyn/dynvar/testdata/reference_vectors.json b/libs/structs/structvar/testdata/reference_vectors.json similarity index 99% rename from libs/dyn/dynvar/testdata/reference_vectors.json rename to libs/structs/structvar/testdata/reference_vectors.json index 2c47b135450..c9c380ad7d0 100644 --- a/libs/dyn/dynvar/testdata/reference_vectors.json +++ b/libs/structs/structvar/testdata/reference_vectors.json @@ -1,5 +1,5 @@ { - "comment": "Shared contract for ${...} reference parsing. Consumed by libs/dyn/dynvar and python/databricks/bundles/core/_transform.py tests.", + "comment": "Shared contract for ${...} reference parsing. Consumed by libs/structs/structvar and python/databricks/bundles/core/_transform.py tests.", "vectors": [ { "id": "ascii-simple", diff --git a/libs/structs/structvar/view.go b/libs/structs/structvar/view.go new file mode 100644 index 00000000000..ecdc0c73825 --- /dev/null +++ b/libs/structs/structvar/view.go @@ -0,0 +1,504 @@ +package structvar + +import ( + "encoding/json" + "iter" + "reflect" + "slices" + + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structpath" +) + +type option uint8 + +const ( + // includeZero keeps zero values: the value is behind a pointer, a map value, + // a slice element, or held in an interface. + includeZero option = 1 << iota + + // fromInterface is set for values held in an interface (e.g. variable defaults). + fromInterface +) + +// View is a read-only view of a typed value with its references and locations, as the +// configuration tree they describe. It is what [convert.FromTyped] would return for the typed value given +// the source tree as reference, without building that tree: it decides presence the +// same way (zero values are absent unless forced, present in the source, behind a +// pointer, or a map value or slice element) and keeps pure references. +// +// The zero View is invalid and represents an absent value. +type View struct { + v reflect.Value + + // Locations of the value. + loc *Locations + + // Path of the value and the references of the whole value the view was created for. + // The path is only tracked if there are references. + path *structpath.PathNode + refs map[string]string + + opts option + valid bool +} + +// NewView returns the view of the typed value v (typically a pointer to a struct) +// with references refs (keyed by path string) and locations. +func NewView(v any, refs map[string]string, locs *Locations) View { + return View{v: reflect.ValueOf(v), loc: locs, refs: refs, valid: true} +} + +// View returns the view of sv. +func (sv *StructVar) View() View { + return NewView(sv.Value, sv.Refs, sv.Locations) +} + +// ref returns the pure reference held by the value, if its type cannot hold a string. +func (x View) ref() string { + if len(x.refs) == 0 { + return "" + } + return x.refs[x.path.String()] +} + +// child returns a view of the child value v at key k (or index i). +func (x View) childKey(v reflect.Value, k string, opts option) View { + c := View{v: v, loc: x.loc.Key(k), refs: x.refs, opts: opts, valid: true} + if len(x.refs) > 0 { + c.path = structpath.NewStringKey(x.path, k) + } + return c +} + +func (x View) childIndex(v reflect.Value, i int, opts option) View { + c := View{v: v, loc: x.loc.Index(i), refs: x.refs, opts: opts, valid: true} + if len(x.refs) > 0 { + c.path = structpath.NewIndex(x.path, i) + } + return c +} + +// IsValid reports whether the value is present. +func (x View) IsValid() bool { + return x.valid +} + +// Locations returns the source locations of the value. +func (x View) Locations() []diag.Location { + if !x.valid { + return nil + } + return x.loc.Get() +} + +// Location returns the first source location of the value. +func (x View) Location() diag.Location { + locs := x.loc.Get() + if !x.valid || len(locs) == 0 { + return diag.Location{} + } + return locs[0] +} + +// Reflect returns the typed value the view is based on (it may be a pointer or interface). +// It is settable if it was reached through pointers or is the root pointer's element. +func (x View) Reflect() reflect.Value { + return x.v +} + +// Locs returns the locations of this value and the values below it. +func (x View) Locs() *Locations { + return x.loc +} + +// elemOptions returns the options for a map value or a slice element: they keep +// zero values. +func elemOptions(v reflect.Value) option { + if v.Kind() == reflect.Interface { + return includeZero | fromInterface + } + return includeZero +} + +// deref follows pointers and interfaces; it returns an invalid value for nil. +func (x View) deref() (reflect.Value, option) { + v, opts := x.v, x.opts + for { + switch v.Kind() { + case reflect.Pointer: + if v.IsNil() { + return reflect.Value{}, opts + } + v = v.Elem() + // A pointer to a zero value was intentionally set. + opts |= includeZero + case reflect.Interface: + if v.IsNil() { + return reflect.Value{}, opts + } + v = v.Elem() + default: + return v, opts + } + } +} + +// Kind returns the kind of the value in the configuration tree. +func (x View) Kind() Kind { + if !x.valid { + return KindInvalid + } + v, opts := x.deref() + if !v.IsValid() { + return KindNil + } + ref := x.ref() + switch v.Kind() { + case reflect.Struct: + if ref != "" { + return KindString + } + if IsSDKNativeType(v.Type()) { + if _, ok := sdkNativeString(v, opts); ok { + return KindString + } + return KindNil + } + // A struct is present if it was present in the source, even if empty. + if opts&includeZero != 0 || x.loc != nil || x.hasFields(v) { + return KindMap + } + return KindNil + case reflect.Map: + if ref != "" { + return KindString + } + if v.IsNil() { + return KindNil + } + return KindMap + case reflect.Slice: + if ref != "" { + return KindString + } + if v.IsNil() { + return KindNil + } + return KindSequence + case reflect.String, reflect.Bool, reflect.Int, reflect.Int32, reflect.Int64, reflect.Float32, reflect.Float64: + if ref != "" { + return KindString + } + // A zero value is absent unless it was present in the source or is kept. + if v.IsZero() && opts&includeZero == 0 && x.loc == nil { + return KindNil + } + switch v.Kind() { + case reflect.String: + return KindString + case reflect.Bool: + return KindBool + case reflect.Float32, reflect.Float64: + return KindFloat + default: + return KindInt + } + default: + return KindInvalid + } +} + +func sdkNativeString(v reflect.Value, opts option) (string, bool) { + if v.IsZero() && opts&includeZero == 0 { + return "", false + } + // SDK native types implement json.Marshaler and marshal to a JSON string. + buf, err := json.Marshal(v.Interface()) + if err != nil { + return "", false + } + var s string + if json.Unmarshal(buf, &s) != nil { + return "", false + } + if s == "" && opts&includeZero == 0 { + return "", false + } + return s, true +} + +// isZero mirrors [dyn.Value.IsZero] for the value. +func (x View) isZero() bool { + switch x.Kind() { + case KindInvalid, KindNil: + return true + case KindMap: + for range x.MapItems() { + return false + } + return true + case KindSequence: + v, _ := x.deref() + return v.Len() == 0 + default: + if x.ref() != "" { + return false + } + s, ok := x.AsString() + if ok { + return s == "" + } + v, _ := x.deref() + return v.IsZero() + } +} + +// field returns the view of the struct field k, and whether it is represented. +func (x View) field(info *StructInfo, sv reflect.Value, k string, fv reflect.Value, isForced bool) (View, bool) { + var opts option + if fv.Kind() == reflect.Interface { + opts = includeZero | fromInterface + } + child := x.childKey(fv, k, opts) + kind := child.Kind() + + // The field was present in the source (it has a node in the locations tree), it is + // not zero-valued, or it's forced. + if child.loc == nil && kind == KindNil && !isForced { + return View{}, false + } + + // A zero field is absent unless forced (it may still be a reference: check the view too). + if fv.Kind() != reflect.Struct && fv.IsZero() && child.isZero() && !info.ForceEmpty[k] && !isForced { + return View{}, false + } + + // A forced field with a nil value is represented by its zero value. + if isForced && kind == KindNil { + child.opts |= includeZero + } + return child, true +} + +func (x View) hasFields(v reflect.Value) bool { + for range x.structFields(v) { + return true + } + return false +} + +func (x View) structFields(v reflect.Value) iter.Seq2[string, View] { + return func(yield func(string, View) bool) { + info := GetStructInfo(v.Type()) + for _, fv := range info.FieldValues(v) { + child, ok := x.field(&info, v, fv.Key, fv.Value, fv.IsForced) + if ok && !yield(fv.Key, child) { + return + } + } + } +} + +// MapItems returns the entries of a map value: struct fields in declaration order, +// map entries in key order. +func (x View) MapItems() iter.Seq2[string, View] { + return func(yield func(string, View) bool) { + if x.ref() != "" { + return + } + v, _ := x.deref() + switch { + case !v.IsValid(): + case v.Kind() == reflect.Struct && !IsSDKNativeType(v.Type()): + for k, c := range x.structFields(v) { + if !yield(k, c) { + return + } + } + case v.Kind() == reflect.Map: + keys := v.MapKeys() + slices.SortFunc(keys, func(a, b reflect.Value) int { + switch { + case a.String() < b.String(): + return -1 + case a.String() > b.String(): + return 1 + } + return 0 + }) + for _, key := range keys { + ev := v.MapIndex(key) + child := x.childKey(ev, key.String(), elemOptions(ev)) + if !yield(key.String(), child) { + return + } + } + } + } +} + +// Sequence returns the elements of a sequence value. +func (x View) Sequence() iter.Seq2[int, View] { + return func(yield func(int, View) bool) { + if x.Kind() != KindSequence { + return + } + v, _ := x.deref() + for i := range v.Len() { + ev := v.Index(i) + child := x.childIndex(ev, i, elemOptions(ev)) + if !yield(i, child) { + return + } + } + } +} + +// Get returns the value of key k of a map value, or an invalid value. +func (x View) Get(k string) View { + if x.Kind() != KindMap { + return View{} + } + v, _ := x.deref() + switch v.Kind() { + case reflect.Struct: + info := GetStructInfo(v.Type()) + index, ok := info.Fields[k] + if !ok { + return View{} + } + fv := FieldByIndex(v, index) + if !fv.IsValid() { + return View{} + } + isForced := !fv.IsZero() || info.IsForceSend(v, k) + child, ok := x.field(&info, v, k, fv, isForced) + if !ok { + return View{} + } + return child + case reflect.Map: + ev := v.MapIndex(reflect.ValueOf(k).Convert(v.Type().Key())) + if !ev.IsValid() { + return View{} + } + child := x.childKey(ev, k, elemOptions(ev)) + return child + default: + return View{} + } +} + +// Index returns element i of a sequence value, or an invalid value. +func (x View) Index(i int) View { + if x.Kind() != KindSequence { + return View{} + } + v, _ := x.deref() + if i < 0 || i >= v.Len() { + return View{} + } + ev := v.Index(i) + child := x.childIndex(ev, i, elemOptions(ev)) + return child +} + +// Lookup returns the value at path, or an invalid value if there is none. +func (x View) Lookup(path *structpath.PathNode) View { + return x.lookupComponents(components(path)) +} + +func (x View) lookupComponents(path []pathComponent) View { + for _, c := range path { + if c.isKey { + x = x.Get(c.key) + } else { + x = x.Index(c.index) + } + if !x.valid { + return x + } + } + return x +} + +// AsString returns the string value (a string, a pure reference, or an SDK native value). +func (x View) AsString() (string, bool) { + if x.Kind() != KindString { + return "", false + } + if ref := x.ref(); ref != "" { + return ref, true + } + v, opts := x.deref() + if v.Kind() == reflect.Struct { + return sdkNativeString(v, opts) + } + return v.String(), true +} + +// AsBool returns the bool value. +func (x View) AsBool() (bool, bool) { + if x.Kind() != KindBool { + return false, false + } + v, _ := x.deref() + return v.Bool(), true +} + +// AsInt returns the int value. +func (x View) AsInt() (int64, bool) { + if x.Kind() != KindInt { + return 0, false + } + v, _ := x.deref() + return v.Int(), true +} + +// AsFloat returns the float value. +func (x View) AsFloat() (float64, bool) { + if x.Kind() != KindFloat { + return 0, false + } + v, _ := x.deref() + return v.Float(), true +} + +// AsAny returns the value as a generic Go value (map[string]any, []any, string, +// bool, int64 or int, float64, or nil), like [dyn.Value.AsAny]. +func (x View) AsAny() any { + switch x.Kind() { + case KindMap: + out := map[string]any{} + for k, c := range x.MapItems() { + out[k] = c.AsAny() + } + return out + case KindSequence: + out := []any{} //nolint:gocritic // an empty sequence renders as [], not null + for _, c := range x.Sequence() { + out = append(out, c.AsAny()) + } + return out + case KindString: + s, _ := x.AsString() + return s + case KindBool: + b, _ := x.AsBool() + return b + case KindInt: + v, opts := x.deref() + // Values held in an interface keep the type the YAML loader produced (int), + // typed fields hold the normalized int64. + if opts&fromInterface != 0 && v.Type() == reflect.TypeFor[int]() { + return int(v.Int()) + } + return v.Int() + case KindFloat: + f, _ := x.AsFloat() + return f + default: + return nil + } +} diff --git a/libs/structs/structvar/walk.go b/libs/structs/structvar/walk.go new file mode 100644 index 00000000000..b7a11a91499 --- /dev/null +++ b/libs/structs/structvar/walk.go @@ -0,0 +1,47 @@ +package structvar + +import ( + "errors" + + "github.com/databricks/cli/libs/structs/structpath" +) + +// Walk calls fn for x and every value below it, parents before children, like +// [dyn.WalkReadOnly] visits a tree. Maps are visited in [View.MapItems] order. +// fn may return [ErrSkip] to skip the subtree of the visited value. +// +// fn may change the configuration at or below the visited path (e.g. with [Set]), +// but must not add or remove the keys and elements being iterated. +func Walk(x View, fn func(*structpath.PathNode, View) error) error { + return walk(x, nil, fn) +} + +func walk(x View, p *structpath.PathNode, fn func(*structpath.PathNode, View) error) error { + if err := fn(p, x); err != nil { + if errors.Is(err, ErrSkip) { + return nil + } + return err + } + + switch x.Kind() { + case KindMap: + for k, c := range x.MapItems() { + if err := walk(c, structpath.NewStringKey(p, k), fn); err != nil { + return err + } + } + case KindSequence: + for i, c := range x.Sequence() { + if err := walk(c, structpath.NewIndex(p, i), fn); err != nil { + return err + } + } + default: + } + return nil +} + +// ErrSkip can be returned by the function passed to [Walk] to skip the subtree of the +// visited value. +var ErrSkip = errors.New("skip traversal of subtree") diff --git a/libs/structs/structvar/write.go b/libs/structs/structvar/write.go new file mode 100644 index 00000000000..42c3c6f464d --- /dev/null +++ b/libs/structs/structvar/write.go @@ -0,0 +1,352 @@ +package structvar + +import ( + "errors" + "fmt" + "reflect" + "slices" + "strings" + + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structpath" +) + +// The methods in this file change sv.Value (a pointer) and keep sv.Refs and +// sv.Locations in sync. Values along the path are allocated as needed (nil pointers, +// nil maps), and map values are written back, so maps of structs can be changed too. +// Refs is replaced rather than modified, so copies of a StructVar can share it. + +// Set sets value at path, like setting it in the configuration tree: a zero value +// is kept as explicitly set (it is added to the governing ForceSendFields). The +// locations recorded for path are kept; references at and below path are dropped. +func (sv *StructVar) Set(path *structpath.PathNode, value any) error { + err := update(reflect.ValueOf(sv.Value), components(path), func(dst reflect.Value) (bool, error) { + rv := reflect.ValueOf(value) + if !rv.IsValid() { + dst.SetZero() + return false, nil + } + if err := setGoValue(dst, rv); err != nil { + return false, fmt.Errorf("cannot set %s: %w", path, err) + } + return rv.IsZero(), nil + }) + if err != nil { + return err + } + sv.Refs = withoutRefs(sv.Refs, path) + // The node marks the value as present even if it is zero (e.g. an empty struct). + var locs *Locations + if value != nil { + locs = (*Locations)(nil).WithLocations(sv.Locations.At(path)) + } + sv.Locations = sv.Locations.With(path, locs) + return nil +} + +func setGoValue(dst, src reflect.Value) error { + for src.Kind() == reflect.Interface || (src.Kind() == reflect.Pointer && dst.Kind() != reflect.Pointer) { + if src.IsNil() { + dst.SetZero() + return nil + } + src = src.Elem() + } + switch { + case src.Type().AssignableTo(dst.Type()): + dst.Set(src) + case dst.Kind() == reflect.Pointer && src.Type().AssignableTo(dst.Type().Elem()): + p := reflect.New(dst.Type().Elem()) + p.Elem().Set(src) + dst.Set(p) + case isScalarKind(src.Kind()) && isScalarKind(dst.Kind()) && src.Type().ConvertibleTo(dst.Type()) && (src.Kind() == reflect.String) == (dst.Kind() == reflect.String): + dst.Set(src.Convert(dst.Type())) + default: + return fmt.Errorf("cannot assign %s to %s", src.Type(), dst.Type()) + } + return nil +} + +func isScalarKind(k reflect.Kind) bool { + switch k { + case reflect.String, reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Float32, reflect.Float64: + return true + default: + return false + } +} + +// Delete removes the value at path: a map entry is deleted, a struct field is zeroed. +func (sv *StructVar) Delete(path *structpath.PathNode) error { + p := components(path) + if len(p) == 0 { + return errors.New("cannot delete the root") + } + parent, last := p[:len(p)-1], p[len(p)-1] + err := walkTo(reflect.ValueOf(sv.Value), parent, func(v reflect.Value) error { + v = derefAlloc(v) + switch { + case v.Kind() == reflect.Map && last.isKey: + if !v.IsNil() { + v.SetMapIndex(reflect.ValueOf(last.key).Convert(v.Type().Key()), reflect.Value{}) + } + case v.Kind() == reflect.Struct && last.isKey: + info := GetStructInfo(v.Type()) + index, ok := info.Fields[last.key] + if !ok { + return fmt.Errorf("no field %q at %s", last.key, path.Parent()) + } + if f := FieldByIndex(v, index); f.IsValid() { + f.SetZero() + } + removeForceSend(v, &info, last.key) + default: + return fmt.Errorf("cannot delete %s", path) + } + return nil + }) + if err != nil { + return err + } + sv.Refs = withoutRefs(sv.Refs, path) + sv.Locations = sv.Locations.With(path, nil) + return nil +} + +// SetLocations sets the locations of the value at path and of every value below it, +// including values that have no locations yet (e.g. set in Go code). +func (sv *StructVar) SetLocations(path *structpath.PathNode, locs []diag.Location) { + x := sv.View().Lookup(path) + if !x.IsValid() { + return + } + sv.Locations = sv.Locations.With(path, locationsOf(x, locs)) +} + +func locationsOf(x View, locs []diag.Location) *Locations { + out := (*Locations)(nil).WithLocations(locs) + switch x.Kind() { + case KindMap: + for k, c := range x.MapItems() { + out.setChild(pathComponent{key: k, isKey: true}, locationsOf(c, locs)) + } + case KindSequence: + for i, c := range x.Sequence() { + out.setChild(pathComponent{index: i}, locationsOf(c, locs)) + } + default: + } + return out +} + +// UpdateSequence records that the elements of the sequence at path were rebuilt from +// the old ones: new element i comes from old elements sources[i] (see +// [Locations.WithSequence]). References of the old elements move along. +func (sv *StructVar) UpdateSequence(path *structpath.PathNode, sources [][]int) { + sv.Locations = sv.Locations.WithSequence(path, sources) + if len(sv.Refs) == 0 { + return + } + elems := path.String() + "[" + refs := make(map[string]string, len(sv.Refs)) + for k, v := range sv.Refs { + if !strings.HasPrefix(k, elems) { + refs[k] = v + } + } + for i, src := range sources { + to := structpath.NewIndex(path, i).String() + for _, old := range src { + from := structpath.NewIndex(path, old).String() + for k, v := range sv.Refs { + if isUnder(k, from) { + refs[to+strings.TrimPrefix(k, from)] = v + } + } + } + } + sv.Refs = refs +} + +// isUnder reports whether the path string key is prefix or a path below it. +func isUnder(key, prefix string) bool { + if prefix == "" || key == prefix { + return true + } + if !strings.HasPrefix(key, prefix) { + return false + } + c := key[len(prefix)] + return c == '.' || c == '[' +} + +// withoutRefs returns a copy of refs without the references at and below path. +func withoutRefs(refs map[string]string, path *structpath.PathNode) map[string]string { + if len(refs) == 0 { + return refs + } + prefix := path.String() + out := make(map[string]string, len(refs)) + for k, v := range refs { + if !isUnder(k, prefix) { + out[k] = v + } + } + return out +} + +// SetReference sets the value at path to the reference ref. A string (or interface) +// field holds the reference itself; for other fields the typed value is set to zero, +// like decoding does, and the reference is recorded in Refs. +func (sv *StructVar) SetReference(path *structpath.PathNode, ref string) error { + isText := false + err := walkTo(reflect.ValueOf(sv.Value), components(path), func(dst reflect.Value) error { + switch dst.Kind() { + case reflect.String, reflect.Interface: + isText = true + case reflect.Pointer: + // The reference is recorded on an allocated zero value, so it is present. + dst.Set(reflect.New(dst.Type().Elem())) + default: + dst.SetZero() + } + return nil + }) + if err != nil { + return err + } + if isText { + return sv.Set(path, ref) + } + locs := sv.Locations.At(path) + // withoutRefs returns a fresh map unless there is nothing to filter. + refs := withoutRefs(sv.Refs, path) + if len(refs) == 0 { + refs = map[string]string{} + } + refs[path.String()] = ref + sv.Refs = refs + sv.Locations = sv.Locations.With(path, (*Locations)(nil).WithLocations(locs)) + return nil +} + +func addForceSend(v reflect.Value, info *StructInfo, k string) { + index, ok := info.ForceSendFieldsIndex[k] + if !ok { + return + } + fsf := GetOrNewFieldByIndex(v, index) + name := info.GolangNames[k] + if slices.Contains(fsf.Interface().([]string), name) { + return + } + fsf.Set(reflect.Append(fsf, reflect.ValueOf(name))) +} + +func removeForceSend(v reflect.Value, info *StructInfo, k string) { + index, ok := info.ForceSendFieldsIndex[k] + if !ok { + return + } + fsf := FieldByIndex(v, index) + if !fsf.IsValid() { + return + } + name := info.GolangNames[k] + names := fsf.Interface().([]string) + if i := slices.Index(names, name); i >= 0 { + fsf.Set(reflect.ValueOf(slices.Delete(slices.Clone(names), i, i+1))) + } +} + +// derefAlloc follows pointers, allocating nil ones, and returns the value they point to. +func derefAlloc(v reflect.Value) reflect.Value { + for v.Kind() == reflect.Pointer { + if v.IsNil() { + v.Set(reflect.New(v.Type().Elem())) + } + v = v.Elem() + } + return v +} + +// walkTo calls fn with the settable value at path. +func walkTo(v reflect.Value, path []pathComponent, fn func(reflect.Value) error) error { + if len(path) == 0 { + return fn(v) + } + return updateChild(derefAlloc(v), path[0], func(child reflect.Value) (bool, error) { + return false, walkTo(child, path[1:], fn) + }) +} + +// update calls fn with the settable value at path. If fn reports that it set a zero +// value into a struct field, the field is added to the governing ForceSendFields. +func update(v reflect.Value, path []pathComponent, fn func(reflect.Value) (bool, error)) error { + if len(path) == 0 { + if v.Kind() == reflect.Pointer && !v.CanSet() { + v = v.Elem() + } + _, err := fn(v) + return err + } + return walkTo(v, path[:len(path)-1], func(parent reflect.Value) error { + return updateChild(derefAlloc(parent), path[len(path)-1], fn) + }) +} + +// updateChild calls fn with the settable child c of v. Interfaces and map values are +// copied and written back; an index one past the end of a slice appends an element. +func updateChild(v reflect.Value, c pathComponent, fn func(reflect.Value) (bool, error)) error { + if v.Kind() == reflect.Interface { + if v.IsNil() { + return errors.New("cannot index nil value") + } + cp := reflect.New(v.Elem().Type()).Elem() + cp.Set(v.Elem()) + err := updateChild(derefAlloc(cp), c, fn) + v.Set(cp) + return err + } + + switch { + case c.isKey && v.Kind() == reflect.Struct: + info := GetStructInfo(v.Type()) + index, ok := info.Fields[c.key] + if !ok { + return fmt.Errorf("no field %q", c.key) + } + zero, err := fn(GetOrNewFieldByIndex(v, index)) + if err != nil { + return err + } + if zero { + addForceSend(v, &info, c.key) + } + return nil + case c.isKey && v.Kind() == reflect.Map: + if v.IsNil() { + v.Set(reflect.MakeMap(v.Type())) + } + key := reflect.ValueOf(c.key).Convert(v.Type().Key()) + e := reflect.New(v.Type().Elem()).Elem() + if old := v.MapIndex(key); old.IsValid() { + e.Set(old) + } + _, err := fn(e) + v.SetMapIndex(key, e) + return err + case !c.isKey && v.Kind() == reflect.Slice: + if c.index == v.Len() { + v.Set(reflect.Append(v, reflect.New(v.Type().Elem()).Elem())) + } + if c.index < 0 || c.index >= v.Len() { + return fmt.Errorf("index %d out of range", c.index) + } + _, err := fn(v.Index(c.index)) + return err + default: + return fmt.Errorf("cannot index %s", v.Kind()) + } +} diff --git a/libs/structs/structyaml/structyaml.go b/libs/structs/structyaml/structyaml.go new file mode 100644 index 00000000000..9c88f6fb21c --- /dev/null +++ b/libs/structs/structyaml/structyaml.go @@ -0,0 +1,187 @@ +// Package structyaml writes typed values as YAML with a controlled key order. +package structyaml + +import ( + "cmp" + "fmt" + "os" + "path/filepath" + "slices" + "strconv" + + "github.com/databricks/cli/libs/structs/structvar" + "go.yaml.in/yaml/v3" +) + +// Pair is an entry of a [Map]. +type Pair struct { + Key string + Value any +} + +// Map is a YAML mapping that keeps the order of its keys. Values are maps (Map or +// map[string]any, keys sorted), sequences ([]any or []string), strings, bools, ints, floats or nil. +type Map []Pair + +// M returns a Map of the given alternating keys (strings) and values. +func M(kv ...any) Map { + m := make(Map, 0, len(kv)/2) + for i := 0; i < len(kv); i += 2 { + m = append(m, Pair{Key: kv[i].(string), Value: kv[i+1]}) + } + return m +} + +// Add appends an entry. +func (m *Map) Add(key string, value any) { + *m = append(*m, Pair{Key: key, Value: value}) +} + +// Order moves the given keys to the front, in the given order. The other keys keep +// their relative order. +func (m *Map) Order(keys ...string) Map { + index := func(p Pair) int { + if i := slices.Index(keys, p.Key); i >= 0 { + return i + } + return len(keys) + } + slices.SortStableFunc(*m, func(a, b Pair) int { return cmp.Compare(index(a), index(b)) }) + return *m +} + +// Struct returns the fields of the struct (or map) v that are set, except skip, as a +// Map with the keys sorted alphabetically, and so are the keys of maps and structs +// nested in maps. Sequence elements keep the field order of their type. +func Struct(v any, skip ...string) (Map, error) { + x := structvar.NewView(v, nil, nil) + if x.Kind() != structvar.KindMap { + return nil, fmt.Errorf("expected map, got %s", x.Kind()) + } + m := fromView(x, true).(Map) + return slices.DeleteFunc(m, func(p Pair) bool { return slices.Contains(skip, p.Key) }), nil +} + +// Value returns the typed value v as a value for a Map, in field order. Unset +// fields are dropped. It returns nil if v is unset. +func Value(v any) any { + return fromView(structvar.NewView(v, nil, nil), false) +} + +func fromView(x structvar.View, sorted bool) any { + switch x.Kind() { + case structvar.KindMap: + var m Map + for k, c := range x.MapItems() { + m.Add(k, fromView(c, sorted)) + } + if sorted { + slices.SortStableFunc(m, func(a, b Pair) int { return cmp.Compare(a.Key, b.Key) }) + } + return m + case structvar.KindSequence: + s := []any{} //nolint:gocritic // an empty sequence is written as [], not null + for _, c := range x.Sequence() { + s = append(s, fromView(c, false)) + } + return s + default: + return x.AsAny() + } +} + +// Node returns v as a YAML node. Values of keys in styles (and everything below +// them) get that style. +func Node(v any, styles map[string]yaml.Style) *yaml.Node { + return toNode(v, 0, styles) +} + +func toNode(v any, style yaml.Style, styles map[string]yaml.Style) *yaml.Node { + switch v := v.(type) { + case Map: + n := &yaml.Node{Kind: yaml.MappingNode, Style: style} + for _, p := range v { + s, ok := styles[p.Key] + if !ok { + s = style + } + n.Content = append(n.Content, &yaml.Node{Kind: yaml.ScalarNode, Value: p.Key, Style: style}, toNode(p.Value, s, styles)) + } + return n + case map[string]any: + m := make(Map, 0, len(v)) + for k, e := range v { + m.Add(k, e) + } + slices.SortFunc(m, func(a, b Pair) int { return cmp.Compare(a.Key, b.Key) }) + return toNode(m, style, styles) + case []any: + n := &yaml.Node{Kind: yaml.SequenceNode, Style: style} + for _, e := range v { + n.Content = append(n.Content, toNode(e, style, styles)) + } + return n + case []string: + n := &yaml.Node{Kind: yaml.SequenceNode, Style: style} + for _, e := range v { + n.Content = append(n.Content, toNode(e, style, styles)) + } + return n + case nil: + return &yaml.Node{Kind: yaml.ScalarNode, Value: "null", Style: style} + case string: + // A string that reads as another scalar (bool, number or empty) is quoted. + if v == "" || v == "true" || v == "false" || isNumber(v) { + style = yaml.DoubleQuotedStyle + } + return &yaml.Node{Kind: yaml.ScalarNode, Value: v, Style: style} + case bool: + return &yaml.Node{Kind: yaml.ScalarNode, Value: strconv.FormatBool(v), Style: style} + case int: + return &yaml.Node{Kind: yaml.ScalarNode, Value: strconv.Itoa(v), Style: style} + case int64: + return &yaml.Node{Kind: yaml.ScalarNode, Value: strconv.FormatInt(v, 10), Style: style} + case float64: + return &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprint(v), Style: style} + default: + // Panic because we only want to deal with known types. + panic(fmt.Sprintf("invalid type: %T", v)) + } +} + +func isNumber(s string) bool { + if _, err := strconv.ParseInt(s, 0, 64); err == nil { + return true + } + _, err := strconv.ParseFloat(s, 64) + return err == nil +} + +// Save writes v as YAML to path, creating parent directories. It fails if path exists +// and force is not set. See [Node] for styles. +func Save(path string, v any, force bool, styles map[string]yaml.Style) error { + err := os.MkdirAll(filepath.Dir(path), 0o755) + if err != nil { + return err + } + + info, err := os.Stat(path) + if err == nil { + if info.IsDir() { + return fmt.Errorf("%s is a directory", path) + } + if !force { + return fmt.Errorf("%s already exists. Use --force to overwrite", path) + } + } + + file, err := os.Create(path) + if err != nil { + return err + } + defer file.Close() + + enc := yaml.NewEncoder(file) + enc.SetIndent(2) + return enc.Encode(Node(v, styles)) +} diff --git a/libs/structs/structyaml/structyaml_test.go b/libs/structs/structyaml/structyaml_test.go new file mode 100644 index 00000000000..6c62fcfcac2 --- /dev/null +++ b/libs/structs/structyaml/structyaml_test.go @@ -0,0 +1,163 @@ +package structyaml + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" +) + +type testStruct struct { + Name string `json:"name"` + Map map[string]string `json:"map"` + List []string `json:"list"` + LongNameField string `json:"long_name_field"` + ForceSendFields []string `json:"-"` + Format string `json:"format"` +} + +func newTestStruct() *testStruct { + return &testStruct{ + Name: "test", + Map: map[string]string{"key2": "value2", "key1": "value1"}, + List: []string{"a", "b", "c"}, + ForceSendFields: []string{"Name"}, + LongNameField: "long name goes here", + } +} + +func TestStruct(t *testing.T) { + result, err := Struct(newTestStruct(), "format") + require.NoError(t, err) + assert.Equal(t, M( + "list", []any{"a", "b", "c"}, + "long_name_field", "long name goes here", + "map", M("key1", "value1", "key2", "value2"), + "name", "test", + ), result) +} + +func TestStructWithOrder(t *testing.T) { + result, err := Struct(newTestStruct(), "format") + require.NoError(t, err) + assert.Equal(t, M( + "list", []any{"a", "b", "c"}, + "name", "test", + "map", M("key1", "value1", "key2", "value2"), + "long_name_field", "long name goes here", + ), result.Order("list", "name", "map")) +} + +func TestStructNotAMap(t *testing.T) { + _, err := Struct([]string{"a"}) + assert.ErrorContains(t, err, "expected map") +} + +func TestOrderUnknownKeysKeepOrder(t *testing.T) { + m := M("d", 1, "c", 2, "b", 3, "a", 4) + assert.Equal(t, M("a", 4, "b", 3, "d", 1, "c", 2), m.Order("a", "x", "b")) +} + +func TestValueKeepsFieldOrder(t *testing.T) { + type inner struct { + Zeta string `json:"zeta"` + Alpha string `json:"alpha"` + } + assert.Equal(t, []any{M("zeta", "z", "alpha", "a")}, Value([]inner{{Zeta: "z", Alpha: "a"}})) + assert.Nil(t, Value([]inner(nil))) +} + +func TestNodeScalars(t *testing.T) { + tests := []struct { + value any + want string + style yaml.Style + }{ + {nil, "null", 0}, + {1, "1", 0}, + {int64(291), "291", 0}, + {1.0, "1", 0}, + {true, "true", 0}, + {"value", "value", 0}, + {"0x123", "0x123", yaml.DoubleQuotedStyle}, + {"0b101", "0b101", yaml.DoubleQuotedStyle}, + {"0123", "0123", yaml.DoubleQuotedStyle}, + {"1.0", "1.0", yaml.DoubleQuotedStyle}, + {"true", "true", yaml.DoubleQuotedStyle}, + {"", "", yaml.DoubleQuotedStyle}, + } + for _, tt := range tests { + n := Node(tt.value, nil) + assert.Equal(t, yaml.ScalarNode, n.Kind) + assert.Equal(t, tt.want, n.Value) + assert.Equal(t, tt.style, n.Style, "%v", tt.value) + } +} + +func TestNodeSequence(t *testing.T) { + for _, v := range []any{[]any{"value1", "value2"}, []string{"value1", "value2"}} { + n := Node(v, nil) + assert.Equal(t, yaml.SequenceNode, n.Kind) + assert.Equal(t, "value1", n.Content[0].Value) + assert.Equal(t, "value2", n.Content[1].Value) + } +} + +func TestNodeMapKeepsOrder(t *testing.T) { + n := Node(M("key3", "value3", "key1", "value1"), nil) + assert.Equal(t, yaml.MappingNode, n.Kind) + assert.Equal(t, "key3", n.Content[0].Value) + assert.Equal(t, "value3", n.Content[1].Value) + assert.Equal(t, "key1", n.Content[2].Value) + assert.Equal(t, "value1", n.Content[3].Value) +} + +func TestNodeGoMapSortsKeys(t *testing.T) { + n := Node(map[string]any{"b": 1, "a": M("y", 1, "x", 2)}, nil) + assert.Equal(t, "a", n.Content[0].Value) + assert.Equal(t, "y", n.Content[1].Content[0].Value) + assert.Equal(t, "b", n.Content[2].Value) +} + +func TestNodeStyle(t *testing.T) { + styles := map[string]yaml.Style{"styled": yaml.DoubleQuotedStyle} + n := Node(M( + "styled", M("key1", "value1", "key2", "value2"), + "unstyled", M("key3", "value3"), + ), styles) + + styled := n.Content[1] + assert.Equal(t, yaml.MappingNode, styled.Kind) + for _, c := range styled.Content { + assert.Equal(t, yaml.DoubleQuotedStyle, c.Style) + } + + unstyled := n.Content[3] + assert.Equal(t, yaml.MappingNode, unstyled.Kind) + for _, c := range unstyled.Content { + assert.Equal(t, yaml.Style(0), c.Style) + } +} + +func TestSave(t *testing.T) { + path := filepath.Join(t.TempDir(), "dir", "out.yml") + require.NoError(t, Save(path, M("b", "1", "a", M("c", []any{1, true})), false, nil)) + + data, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, "b: \"1\"\na:\n c:\n - 1\n - true\n", string(data)) + + err = Save(path, M("a", 1), false, nil) + assert.ErrorContains(t, err, "already exists. Use --force to overwrite") + + require.NoError(t, Save(path, M("a", 1), true, nil)) + data, err = os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, "a: 1\n", string(data)) + + err = Save(filepath.Dir(path), M("a", 1), true, nil) + assert.ErrorContains(t, err, "is a directory") +} diff --git a/python/Taskfile.yml b/python/Taskfile.yml index 137b567050a..68c452e228f 100644 --- a/python/Taskfile.yml +++ b/python/Taskfile.yml @@ -16,7 +16,7 @@ tasks: desc: Run pydabs tests sources: - "**/*.py" - - ../libs/dyn/dynvar/testdata/reference_vectors.json + - ../libs/structs/structvar/testdata/reference_vectors.json - pyproject.toml - uv.lock - README.md diff --git a/python/codegen/codegen/packages.py b/python/codegen/codegen/packages.py index 46674518dca..69d4944fa9a 100644 --- a/python/codegen/codegen/packages.py +++ b/python/codegen/codegen/packages.py @@ -63,7 +63,7 @@ def _load_resource_namespace() -> dict[str, str]: # pass through untyped. The Go config mutators validate the value. "interface": "Any", # time.Time is a scalar serialized as an RFC3339 string; the Go side models - # it as a string too (see libs/dyn/convert/sdk_native_types.go). + # it as a string too (see libs/structs/structvar/struct_info.go). "time.Time": "str", # duration.Duration is a scalar serialized as a seconds string (e.g. "3600s"). "duration.Duration": "str", diff --git a/python/databricks/bundles/core/_transform.py b/python/databricks/bundles/core/_transform.py index 8e3b4a6ec12..c39b3555869 100644 --- a/python/databricks/bundles/core/_transform.py +++ b/python/databricks/bundles/core/_transform.py @@ -19,7 +19,7 @@ _T = TypeVar("_T") -# Same strings as normalizeBool in libs/dyn/convert/normalize.go. +# Same strings as the bool decoding in libs/structs/structvar/decode.go. # See https://yaml.org/type/bool.html. _TRUE_STRINGS = [ "true", @@ -311,8 +311,8 @@ def _unwrap_variable(tpe: type) -> Optional[type]: # Regex for string corresponding to variables. # -# The source of truth is regex in libs/dyn/dynvar/ref.go. -# Behavioral parity is enforced by libs/dyn/dynvar/testdata/reference_vectors.json. +# The source of truth is regex in libs/structs/structvar/ref.go. +# Behavioral parity is enforced by libs/structs/structvar/testdata/reference_vectors.json. # # The (?