From 4f8c09f5b66e4b07f8c127727db8fa089445d558 Mon Sep 17 00:00:00 2001 From: Yiwei Dai <125326326+yiweidai-db@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:49:53 +0000 Subject: [PATCH 1/4] Prepare bundle job project-environment path translation --- .../config/mutator/paths/job_paths_visitor.go | 10 +- .../mutator/paths/job_paths_visitor_test.go | 41 +++++++ bundle/config/mutator/translate_paths_test.go | 100 ++++++++++++++++++ 3 files changed, 150 insertions(+), 1 deletion(-) diff --git a/bundle/config/mutator/paths/job_paths_visitor.go b/bundle/config/mutator/paths/job_paths_visitor.go index cb400112b7e..22da4475a5c 100644 --- a/bundle/config/mutator/paths/job_paths_visitor.go +++ b/bundle/config/mutator/paths/job_paths_visitor.go @@ -69,7 +69,15 @@ func jobRewritePatterns() []jobRewritePattern { taskPatterns := jobTaskRewritePatterns(base) forEachPatterns := jobTaskRewritePatterns(base.Append(dyn.Key("for_each_task"), dyn.Key("task"))) - return append(taskPatterns, forEachPatterns...) + patterns := append(taskPatterns, forEachPatterns...) + return append(patterns, jobRewritePattern{ + dyn.NewPattern( + dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), + dyn.Key("environments"), dyn.AnyIndex(), dyn.Key("spec"), dyn.Key("project_environment"), + ), + TranslateModeFile, + noSkipRewrite, + }) } // VisitJobPaths visits all paths in job resources and applies a function to each path. diff --git a/bundle/config/mutator/paths/job_paths_visitor_test.go b/bundle/config/mutator/paths/job_paths_visitor_test.go index 7584737b1c0..f6f20a7b94f 100644 --- a/bundle/config/mutator/paths/job_paths_visitor_test.go +++ b/bundle/config/mutator/paths/job_paths_visitor_test.go @@ -1,14 +1,17 @@ package paths import ( + "strings" "testing" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestVisitJobPaths(t *testing.T) { @@ -131,6 +134,44 @@ func TestVisitJobPaths_environments(t *testing.T) { assert.ElementsMatch(t, expected, actual) } +func TestVisitJobPaths_projectEnvironments(t *testing.T) { + value, err := yamlloader.LoadYAML("resources/jobs.yml", strings.NewReader(` +resources: + jobs: + first: + environments: + - environment_key: project + spec: + project_environment: ../pyproject.toml + - environment_key: standard + spec: + environment_version: "4" + dependencies: [requests] + - environment_key: requirements + spec: + project_environment: ../requirements.txt + second: + environments: + - environment_key: project + spec: + project_environment: /Workspace/shared/pyproject.toml +`)) + require.NoError(t, err) + + var actual []string + _, err = VisitJobPaths(value, func(p dyn.Path, mode TranslateMode, v dyn.Value) (dyn.Value, error) { + assert.Equal(t, TranslateModeFile, mode) + actual = append(actual, p.String()) + return v, nil + }) + require.NoError(t, err) + assert.ElementsMatch(t, []string{ + "resources.jobs.first.environments[0].spec.project_environment", + "resources.jobs.first.environments[2].spec.project_environment", + "resources.jobs.second.environments[0].spec.project_environment", + }, actual) +} + func TestVisitJobPaths_foreach(t *testing.T) { task0 := jobs.Task{ ForEachTask: &jobs.ForEachTask{ diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index 226a848b723..94fae442859 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -48,6 +48,106 @@ func touchEmptyFile(t *testing.T, path string) { f.Close() } +func TestTranslatePathsProjectEnvironment(t *testing.T) { + for _, tc := range []struct { + name string + path string + localFile string + gitSource bool + sourceLinked bool + want string + wantError string + }{ + { + name: "relative to included YAML", + path: "../pyproject.toml", + localFile: "pyproject.toml", + want: "/Workspace/bundle/files/pyproject.toml", + }, + { + name: "requirements file with spaces", + path: "./env files/requirements.txt", + localFile: "resources/env files/requirements.txt", + want: "/Workspace/bundle/files/resources/env files/requirements.txt", + }, + { + name: "absolute workspace path", + path: "/Workspace/shared/pyproject.toml", + want: "/Workspace/shared/pyproject.toml", + }, + { + name: "source-linked deployment", + path: "../pyproject.toml", + localFile: "pyproject.toml", + sourceLinked: true, + want: "pyproject.toml", + }, + { + name: "git source is unchanged", + path: "../pyproject.toml", + gitSource: true, + want: "../pyproject.toml", + }, + { + name: "missing file", + path: "../missing.toml", + wantError: "file missing.toml not found", + }, + { + name: "outside sync root", + path: "../../pyproject.toml", + wantError: "is not contained in sync root path", + }, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + if tc.localFile != "" { + touchEmptyFile(t, filepath.Join(dir, tc.localFile)) + } + job := &resources.Job{ + JobSettings: jobs.JobSettings{ + Environments: []jobs.JobEnvironment{{EnvironmentKey: "project", Spec: &compute.Environment{}}}, + }, + } + if tc.gitSource { + job.GitSource = &jobs.GitSource{GitUrl: "https://example.test/repo.git"} + } + b := &bundle.Bundle{ + BundleRootPath: dir, + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), + Config: config.Root{ + Workspace: config.Workspace{FilePath: "/Workspace/bundle/files"}, + Resources: config.Resources{Jobs: map[string]*resources.Job{"job": job}}, + Presets: config.Presets{SourceLinkedDeployment: &tc.sourceLinked}, + }, + } + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resources", "job.yml")}}) + const field = "resources.jobs.job.environments[0].spec.project_environment" + // Until the SDK exposes this field, invoke the mutators directly so + // bundle.Apply does not discard it when converting from SDK types. + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.Set(v, field, dyn.NewValue(tc.path, []dyn.Location{{File: filepath.Join(dir, "resources", "job.yml")}})) + }) + require.NoError(t, err) + require.NoError(t, mutator.NormalizePaths().Apply(t.Context(), b).Error()) + diags := mutator.TranslatePaths().Apply(t.Context(), b) + if tc.wantError != "" { + require.ErrorContains(t, diags.Error(), tc.wantError) + return + } + require.NoError(t, diags.Error()) + actual, err := dyn.Get(b.Config.Value(), field) + require.NoError(t, err) + want := tc.want + if tc.sourceLinked { + want = filepath.ToSlash(filepath.Join(dir, want)) + } + assert.Equal(t, want, actual.MustString()) + }) + } +} + func TestTranslatePathsSkippedWithGitSource(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ From d6eeebb1c16ebab8fa6eda7b4d7db15748e7aaad Mon Sep 17 00:00:00 2001 From: Yiwei Dai <125326326+yiweidai-db@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:53:47 +0000 Subject: [PATCH 2/4] Preserve project-environment paths relative to notebooks --- .../config/mutator/paths/job_paths_visitor.go | 12 +-- .../mutator/paths/job_paths_visitor_test.go | 29 +++--- bundle/config/mutator/translate_paths_test.go | 90 +++++++------------ 3 files changed, 52 insertions(+), 79 deletions(-) diff --git a/bundle/config/mutator/paths/job_paths_visitor.go b/bundle/config/mutator/paths/job_paths_visitor.go index 22da4475a5c..c5cfd5826d4 100644 --- a/bundle/config/mutator/paths/job_paths_visitor.go +++ b/bundle/config/mutator/paths/job_paths_visitor.go @@ -69,15 +69,9 @@ func jobRewritePatterns() []jobRewritePattern { taskPatterns := jobTaskRewritePatterns(base) forEachPatterns := jobTaskRewritePatterns(base.Append(dyn.Key("for_each_task"), dyn.Key("task"))) - patterns := append(taskPatterns, forEachPatterns...) - return append(patterns, jobRewritePattern{ - dyn.NewPattern( - dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), - dyn.Key("environments"), dyn.AnyIndex(), dyn.Key("spec"), dyn.Key("project_environment"), - ), - TranslateModeFile, - noSkipRewrite, - }) + // project_environment is relative to the task's notebook, not the defining + // YAML file, so it must not participate in bundle path translation. + return append(taskPatterns, forEachPatterns...) } // VisitJobPaths visits all paths in job resources and applies a function to each path. diff --git a/bundle/config/mutator/paths/job_paths_visitor_test.go b/bundle/config/mutator/paths/job_paths_visitor_test.go index f6f20a7b94f..9ab27e5e16d 100644 --- a/bundle/config/mutator/paths/job_paths_visitor_test.go +++ b/bundle/config/mutator/paths/job_paths_visitor_test.go @@ -134,11 +134,15 @@ func TestVisitJobPaths_environments(t *testing.T) { assert.ElementsMatch(t, expected, actual) } -func TestVisitJobPaths_projectEnvironments(t *testing.T) { +func TestVisitJobPaths_preservesProjectEnvironments(t *testing.T) { value, err := yamlloader.LoadYAML("resources/jobs.yml", strings.NewReader(` resources: jobs: first: + tasks: + - task_key: notebook + notebook_task: + notebook_path: notebook.py environments: - environment_key: project spec: @@ -158,18 +162,21 @@ resources: `)) require.NoError(t, err) - var actual []string - _, err = VisitJobPaths(value, func(p dyn.Path, mode TranslateMode, v dyn.Value) (dyn.Value, error) { - assert.Equal(t, TranslateModeFile, mode) - actual = append(actual, p.String()) - return v, nil + updated, err := VisitJobPaths(value, func(_ dyn.Path, mode TranslateMode, v dyn.Value) (dyn.Value, error) { + assert.Equal(t, TranslateModeNotebook, mode) + return dyn.NewValue("/Workspace/bundle/files/"+v.MustString(), v.Locations()), nil }) require.NoError(t, err) - assert.ElementsMatch(t, []string{ - "resources.jobs.first.environments[0].spec.project_environment", - "resources.jobs.first.environments[2].spec.project_environment", - "resources.jobs.second.environments[0].spec.project_environment", - }, actual) + for path, want := range map[string]string{ + "resources.jobs.first.tasks[0].notebook_task.notebook_path": "/Workspace/bundle/files/notebook.py", + "resources.jobs.first.environments[0].spec.project_environment": "../pyproject.toml", + "resources.jobs.first.environments[2].spec.project_environment": "../requirements.txt", + "resources.jobs.second.environments[0].spec.project_environment": "/Workspace/shared/pyproject.toml", + } { + actual, err := dyn.Get(updated, path) + require.NoError(t, err) + assert.Equal(t, want, actual.MustString(), path) + } } func TestVisitJobPaths_foreach(t *testing.T) { diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index 94fae442859..7b3593d0b65 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -48,64 +48,35 @@ func touchEmptyFile(t *testing.T, path string) { f.Close() } -func TestTranslatePathsProjectEnvironment(t *testing.T) { +func TestTranslatePathsPreservesNotebookRelativeProjectEnvironment(t *testing.T) { for _, tc := range []struct { name string path string - localFile string gitSource bool sourceLinked bool - want string - wantError string }{ - { - name: "relative to included YAML", - path: "../pyproject.toml", - localFile: "pyproject.toml", - want: "/Workspace/bundle/files/pyproject.toml", - }, - { - name: "requirements file with spaces", - path: "./env files/requirements.txt", - localFile: "resources/env files/requirements.txt", - want: "/Workspace/bundle/files/resources/env files/requirements.txt", - }, - { - name: "absolute workspace path", - path: "/Workspace/shared/pyproject.toml", - want: "/Workspace/shared/pyproject.toml", - }, - { - name: "source-linked deployment", - path: "../pyproject.toml", - localFile: "pyproject.toml", - sourceLinked: true, - want: "pyproject.toml", - }, - { - name: "git source is unchanged", - path: "../pyproject.toml", - gitSource: true, - want: "../pyproject.toml", - }, - { - name: "missing file", - path: "../missing.toml", - wantError: "file missing.toml not found", - }, - { - name: "outside sync root", - path: "../../pyproject.toml", - wantError: "is not contained in sync root path", - }, + {name: "notebook directory", path: "./pyproject.toml"}, + {name: "notebook parent", path: "../pyproject.toml"}, + {name: "multiple notebook ancestors", path: "../../pyproject.toml"}, + {name: "requirements file with spaces", path: "../env files/requirements.txt"}, + {name: "absolute workspace path", path: "/Workspace/shared/pyproject.toml"}, + {name: "source-linked deployment", path: "../pyproject.toml", sourceLinked: true}, + {name: "git source", path: "../pyproject.toml", gitSource: true}, } { t.Run(tc.name, func(t *testing.T) { dir := t.TempDir() - if tc.localFile != "" { - touchEmptyFile(t, filepath.Join(dir, tc.localFile)) - } + notebookPath := filepath.Join(dir, "src", "notebooks", "notebook.py") + require.NoError(t, os.MkdirAll(filepath.Dir(notebookPath), 0o700)) + touchNotebookFile(t, notebookPath) job := &resources.Job{ JobSettings: jobs.JobSettings{ + Tasks: []jobs.Task{{ + TaskKey: "notebook", + NotebookTask: &jobs.NotebookTask{ + NotebookPath: "../src/notebooks/notebook.py", + }, + EnvironmentKey: "project", + }}, Environments: []jobs.JobEnvironment{{EnvironmentKey: "project", Spec: &compute.Environment{}}}, }, } @@ -122,28 +93,29 @@ func TestTranslatePathsProjectEnvironment(t *testing.T) { Presets: config.Presets{SourceLinkedDeployment: &tc.sourceLinked}, }, } - bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resources", "job.yml")}}) + locations := []dyn.Location{{File: filepath.Join(dir, "resources", "job.yml")}} + bundletest.SetLocation(b, ".", locations) const field = "resources.jobs.job.environments[0].spec.project_environment" // Until the SDK exposes this field, invoke the mutators directly so // bundle.Apply does not discard it when converting from SDK types. err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, field, dyn.NewValue(tc.path, []dyn.Location{{File: filepath.Join(dir, "resources", "job.yml")}})) + return dyn.Set(v, field, dyn.NewValue(tc.path, locations)) }) require.NoError(t, err) require.NoError(t, mutator.NormalizePaths().Apply(t.Context(), b).Error()) - diags := mutator.TranslatePaths().Apply(t.Context(), b) - if tc.wantError != "" { - require.ErrorContains(t, diags.Error(), tc.wantError) - return - } - require.NoError(t, diags.Error()) + require.NoError(t, mutator.TranslatePaths().Apply(t.Context(), b).Error()) actual, err := dyn.Get(b.Config.Value(), field) require.NoError(t, err) - want := tc.want - if tc.sourceLinked { - want = filepath.ToSlash(filepath.Join(dir, want)) + assert.Equal(t, tc.path, actual.MustString()) + + wantNotebook := "/Workspace/bundle/files/src/notebooks/notebook" + switch { + case tc.gitSource: + wantNotebook = "../src/notebooks/notebook.py" + case tc.sourceLinked: + wantNotebook = filepath.ToSlash(filepath.Join(dir, "src", "notebooks", "notebook")) } - assert.Equal(t, want, actual.MustString()) + assert.Equal(t, wantNotebook, b.Config.Resources.Jobs["job"].Tasks[0].NotebookTask.NotebookPath) }) } } From d0b8ec57cac50c2c655bfbce355fa2170b865137 Mon Sep 17 00:00:00 2001 From: Yiwei Dai <125326326+yiweidai-db@users.noreply.github.com> Date: Fri, 2 Oct 2026 00:01:31 +0000 Subject: [PATCH 3/4] Resolve project-environment paths relative to defining YAML --- .../config/mutator/paths/job_paths_visitor.go | 12 +++- .../mutator/paths/job_paths_visitor_test.go | 23 ++++--- bundle/config/mutator/translate_paths_test.go | 64 ++++++++++++++++--- 3 files changed, 78 insertions(+), 21 deletions(-) diff --git a/bundle/config/mutator/paths/job_paths_visitor.go b/bundle/config/mutator/paths/job_paths_visitor.go index c5cfd5826d4..22da4475a5c 100644 --- a/bundle/config/mutator/paths/job_paths_visitor.go +++ b/bundle/config/mutator/paths/job_paths_visitor.go @@ -69,9 +69,15 @@ func jobRewritePatterns() []jobRewritePattern { taskPatterns := jobTaskRewritePatterns(base) forEachPatterns := jobTaskRewritePatterns(base.Append(dyn.Key("for_each_task"), dyn.Key("task"))) - // project_environment is relative to the task's notebook, not the defining - // YAML file, so it must not participate in bundle path translation. - return append(taskPatterns, forEachPatterns...) + patterns := append(taskPatterns, forEachPatterns...) + return append(patterns, jobRewritePattern{ + dyn.NewPattern( + dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), + dyn.Key("environments"), dyn.AnyIndex(), dyn.Key("spec"), dyn.Key("project_environment"), + ), + TranslateModeFile, + noSkipRewrite, + }) } // VisitJobPaths visits all paths in job resources and applies a function to each path. diff --git a/bundle/config/mutator/paths/job_paths_visitor_test.go b/bundle/config/mutator/paths/job_paths_visitor_test.go index 9ab27e5e16d..a66e35ea12d 100644 --- a/bundle/config/mutator/paths/job_paths_visitor_test.go +++ b/bundle/config/mutator/paths/job_paths_visitor_test.go @@ -134,7 +134,7 @@ func TestVisitJobPaths_environments(t *testing.T) { assert.ElementsMatch(t, expected, actual) } -func TestVisitJobPaths_preservesProjectEnvironments(t *testing.T) { +func TestVisitJobPaths_projectEnvironments(t *testing.T) { value, err := yamlloader.LoadYAML("resources/jobs.yml", strings.NewReader(` resources: jobs: @@ -162,16 +162,23 @@ resources: `)) require.NoError(t, err) - updated, err := VisitJobPaths(value, func(_ dyn.Path, mode TranslateMode, v dyn.Value) (dyn.Value, error) { - assert.Equal(t, TranslateModeNotebook, mode) - return dyn.NewValue("/Workspace/bundle/files/"+v.MustString(), v.Locations()), nil + visited := map[string]TranslateMode{} + updated, err := VisitJobPaths(value, func(p dyn.Path, mode TranslateMode, v dyn.Value) (dyn.Value, error) { + visited[p.String()] = mode + return dyn.NewValue("rewritten:"+v.MustString(), v.Locations()), nil }) require.NoError(t, err) + assert.Equal(t, map[string]TranslateMode{ + "resources.jobs.first.tasks[0].notebook_task.notebook_path": TranslateModeNotebook, + "resources.jobs.first.environments[0].spec.project_environment": TranslateModeFile, + "resources.jobs.first.environments[2].spec.project_environment": TranslateModeFile, + "resources.jobs.second.environments[0].spec.project_environment": TranslateModeFile, + }, visited) for path, want := range map[string]string{ - "resources.jobs.first.tasks[0].notebook_task.notebook_path": "/Workspace/bundle/files/notebook.py", - "resources.jobs.first.environments[0].spec.project_environment": "../pyproject.toml", - "resources.jobs.first.environments[2].spec.project_environment": "../requirements.txt", - "resources.jobs.second.environments[0].spec.project_environment": "/Workspace/shared/pyproject.toml", + "resources.jobs.first.tasks[0].notebook_task.notebook_path": "rewritten:notebook.py", + "resources.jobs.first.environments[0].spec.project_environment": "rewritten:../pyproject.toml", + "resources.jobs.first.environments[2].spec.project_environment": "rewritten:../requirements.txt", + "resources.jobs.second.environments[0].spec.project_environment": "rewritten:/Workspace/shared/pyproject.toml", } { actual, err := dyn.Get(updated, path) require.NoError(t, err) diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index 7b3593d0b65..dffb151f19a 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -48,23 +48,58 @@ func touchEmptyFile(t *testing.T, path string) { f.Close() } -func TestTranslatePathsPreservesNotebookRelativeProjectEnvironment(t *testing.T) { +func TestTranslatePathsProjectEnvironment(t *testing.T) { for _, tc := range []struct { name string path string + localFile string + want string + wantErr string gitSource bool sourceLinked bool }{ - {name: "notebook directory", path: "./pyproject.toml"}, - {name: "notebook parent", path: "../pyproject.toml"}, - {name: "multiple notebook ancestors", path: "../../pyproject.toml"}, - {name: "requirements file with spaces", path: "../env files/requirements.txt"}, - {name: "absolute workspace path", path: "/Workspace/shared/pyproject.toml"}, - {name: "source-linked deployment", path: "../pyproject.toml", sourceLinked: true}, - {name: "git source", path: "../pyproject.toml", gitSource: true}, + { + name: "YAML directory", path: "./pyproject.toml", localFile: "resources/pyproject.toml", + want: "/Workspace/bundle/files/resources/pyproject.toml", + }, + { + name: "YAML parent", path: "../pyproject.toml", localFile: "pyproject.toml", + want: "/Workspace/bundle/files/pyproject.toml", + }, + { + name: "bare filename", path: "pyproject.toml", localFile: "resources/pyproject.toml", + want: "/Workspace/bundle/files/resources/pyproject.toml", + }, + { + name: "requirements file with spaces", path: "../env files/requirements.txt", localFile: "env files/requirements.txt", + want: "/Workspace/bundle/files/env files/requirements.txt", + }, + { + name: "absolute workspace path", path: "/Workspace/shared/pyproject.toml", + want: "/Workspace/shared/pyproject.toml", + }, + { + name: "source-linked deployment", path: "../pyproject.toml", localFile: "pyproject.toml", + want: "pyproject.toml", sourceLinked: true, + }, + { + name: "git source", path: "../pyproject.toml", + want: "../pyproject.toml", gitSource: true, + }, + { + name: "missing file", path: "../pyproject.toml", + wantErr: "file pyproject.toml not found", + }, + { + name: "outside sync root", path: "../../pyproject.toml", + wantErr: "is not contained in sync root path", + }, } { t.Run(tc.name, func(t *testing.T) { dir := t.TempDir() + if tc.localFile != "" { + touchEmptyFile(t, filepath.Join(dir, tc.localFile)) + } notebookPath := filepath.Join(dir, "src", "notebooks", "notebook.py") require.NoError(t, os.MkdirAll(filepath.Dir(notebookPath), 0o700)) touchNotebookFile(t, notebookPath) @@ -103,10 +138,19 @@ func TestTranslatePathsPreservesNotebookRelativeProjectEnvironment(t *testing.T) }) require.NoError(t, err) require.NoError(t, mutator.NormalizePaths().Apply(t.Context(), b).Error()) - require.NoError(t, mutator.TranslatePaths().Apply(t.Context(), b).Error()) + err = mutator.TranslatePaths().Apply(t.Context(), b).Error() + if tc.wantErr != "" { + require.ErrorContains(t, err, tc.wantErr) + return + } + require.NoError(t, err) actual, err := dyn.Get(b.Config.Value(), field) require.NoError(t, err) - assert.Equal(t, tc.path, actual.MustString()) + want := tc.want + if tc.sourceLinked { + want = filepath.ToSlash(filepath.Join(dir, want)) + } + assert.Equal(t, want, actual.MustString()) wantNotebook := "/Workspace/bundle/files/src/notebooks/notebook" switch { From ac61e27e4d6addbbb21785abfb00f7f29bea4f1e Mon Sep 17 00:00:00 2001 From: Yiwei Dai <125326326+yiweidai-db@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:49:37 +0000 Subject: [PATCH 4/4] Simplify project-environment paths and verify job deployment --- .../bundles/project-environment-paths.md | 1 + .../project_environment/databricks.yml | 5 ++ .../project_environment/out.test.toml | 3 + .../project_environment/output.txt | 77 +++++++++++++++++++ .../project_environment/pyproject.toml | 6 ++ .../project_environment/resources/job.yml | 13 ++++ .../resources/pyproject.toml | 6 ++ .../environments/project_environment/script | 8 ++ .../project_environment/src/notebook.py | 2 + .../project_environment/test.toml | 3 + .../config/mutator/paths/job_paths_visitor.go | 20 ++--- .../mutator/paths/job_paths_visitor_test.go | 62 ++------------- bundle/config/mutator/translate_paths_test.go | 39 ++-------- 13 files changed, 142 insertions(+), 103 deletions(-) create mode 100644 .nextchanges/bundles/project-environment-paths.md create mode 100644 acceptance/bundle/environments/project_environment/databricks.yml create mode 100644 acceptance/bundle/environments/project_environment/out.test.toml create mode 100644 acceptance/bundle/environments/project_environment/output.txt create mode 100644 acceptance/bundle/environments/project_environment/pyproject.toml create mode 100644 acceptance/bundle/environments/project_environment/resources/job.yml create mode 100644 acceptance/bundle/environments/project_environment/resources/pyproject.toml create mode 100644 acceptance/bundle/environments/project_environment/script create mode 100644 acceptance/bundle/environments/project_environment/src/notebook.py create mode 100644 acceptance/bundle/environments/project_environment/test.toml diff --git a/.nextchanges/bundles/project-environment-paths.md b/.nextchanges/bundles/project-environment-paths.md new file mode 100644 index 00000000000..85c2d0108aa --- /dev/null +++ b/.nextchanges/bundles/project-environment-paths.md @@ -0,0 +1 @@ +* Resolve job `project_environment` paths relative to the YAML file that defines them. ([#6909](https://github.com/databricks/cli/pull/6909)) diff --git a/acceptance/bundle/environments/project_environment/databricks.yml b/acceptance/bundle/environments/project_environment/databricks.yml new file mode 100644 index 00000000000..03565b8a553 --- /dev/null +++ b/acceptance/bundle/environments/project_environment/databricks.yml @@ -0,0 +1,5 @@ +bundle: + name: project_environment + +include: + - resources/*.yml diff --git a/acceptance/bundle/environments/project_environment/out.test.toml b/acceptance/bundle/environments/project_environment/out.test.toml new file mode 100644 index 00000000000..59b56a2037c --- /dev/null +++ b/acceptance/bundle/environments/project_environment/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/environments/project_environment/output.txt b/acceptance/bundle/environments/project_environment/output.txt new file mode 100644 index 00000000000..6d3e465e0ed --- /dev/null +++ b/acceptance/bundle/environments/project_environment/output.txt @@ -0,0 +1,77 @@ + +>>> [CLI] bundle validate -o json +{ + "tasks": [ + { + "environment_key": "project", + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/src/notebook" + }, + "task_key": "notebook" + } + ], + "environments": [ + { + "environment_key": "project", + "spec": { + "project_environment": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/pyproject.toml" + } + } + ] +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/project_environment/default/files... +Created jobs.notebook +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.2/jobs/create +{ + "tasks": [ + { + "environment_key": "project", + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/src/notebook" + }, + "task_key": "notebook" + } + ], + "environments": [ + { + "environment_key": "project", + "spec": { + "project_environment": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/pyproject.toml" + } + } + ] +} + +>>> update_file.py resources/job.yml ../pyproject.toml ./pyproject.toml + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/project_environment/default/files... +Updated jobs.notebook +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.2/jobs/reset +{ + "tasks": [ + { + "environment_key": "project", + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/src/notebook" + }, + "task_key": "notebook" + } + ], + "environments": [ + { + "environment_key": "project", + "spec": { + "project_environment": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/resources/pyproject.toml" + } + } + ] +} diff --git a/acceptance/bundle/environments/project_environment/pyproject.toml b/acceptance/bundle/environments/project_environment/pyproject.toml new file mode 100644 index 00000000000..b22de93e858 --- /dev/null +++ b/acceptance/bundle/environments/project_environment/pyproject.toml @@ -0,0 +1,6 @@ +[project] +name = "project-environment-test" +version = "0.1.0" + +[tool.databricks.environment] +environment_version = "4" diff --git a/acceptance/bundle/environments/project_environment/resources/job.yml b/acceptance/bundle/environments/project_environment/resources/job.yml new file mode 100644 index 00000000000..bce7abe7c65 --- /dev/null +++ b/acceptance/bundle/environments/project_environment/resources/job.yml @@ -0,0 +1,13 @@ +resources: + jobs: + notebook: + name: project_environment + tasks: + - task_key: notebook + notebook_task: + notebook_path: ../src/notebook.py + environment_key: project + environments: + - environment_key: project + spec: + project_environment: ../pyproject.toml diff --git a/acceptance/bundle/environments/project_environment/resources/pyproject.toml b/acceptance/bundle/environments/project_environment/resources/pyproject.toml new file mode 100644 index 00000000000..d2854958830 --- /dev/null +++ b/acceptance/bundle/environments/project_environment/resources/pyproject.toml @@ -0,0 +1,6 @@ +[project] +name = "updated-project-environment-test" +version = "0.1.0" + +[tool.databricks.environment] +environment_version = "4" diff --git a/acceptance/bundle/environments/project_environment/script b/acceptance/bundle/environments/project_environment/script new file mode 100644 index 00000000000..440ab89e91e --- /dev/null +++ b/acceptance/bundle/environments/project_environment/script @@ -0,0 +1,8 @@ +trace $CLI bundle validate -o json | jq '.resources.jobs.notebook | {tasks, environments}' + +trace $CLI bundle deploy +trace print_requests.py //api/2.2/jobs/create | jq '.body | {tasks, environments}' + +trace update_file.py resources/job.yml ../pyproject.toml ./pyproject.toml +trace $CLI bundle deploy +trace print_requests.py //api/2.2/jobs/reset | jq '.body.new_settings | {tasks, environments}' diff --git a/acceptance/bundle/environments/project_environment/src/notebook.py b/acceptance/bundle/environments/project_environment/src/notebook.py new file mode 100644 index 00000000000..5bee9c9eae0 --- /dev/null +++ b/acceptance/bundle/environments/project_environment/src/notebook.py @@ -0,0 +1,2 @@ +# Databricks notebook source +print("Project environment") diff --git a/acceptance/bundle/environments/project_environment/test.toml b/acceptance/bundle/environments/project_environment/test.toml new file mode 100644 index 00000000000..1670d61e0f6 --- /dev/null +++ b/acceptance/bundle/environments/project_environment/test.toml @@ -0,0 +1,3 @@ +# The pinned Terraform provider does not support project_environment. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +RecordRequests = true diff --git a/bundle/config/mutator/paths/job_paths_visitor.go b/bundle/config/mutator/paths/job_paths_visitor.go index 22da4475a5c..e37e34849bd 100644 --- a/bundle/config/mutator/paths/job_paths_visitor.go +++ b/bundle/config/mutator/paths/job_paths_visitor.go @@ -58,23 +58,13 @@ func jobTaskRewritePatterns(base dyn.Pattern) []jobRewritePattern { } func jobRewritePatterns() []jobRewritePattern { - // Base pattern to match all tasks in all jobs. - base := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("tasks"), - dyn.AnyIndex(), - ) + job := dyn.NewPattern(dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey()) + task := job.Append(dyn.Key("tasks"), dyn.AnyIndex()) - taskPatterns := jobTaskRewritePatterns(base) - forEachPatterns := jobTaskRewritePatterns(base.Append(dyn.Key("for_each_task"), dyn.Key("task"))) - patterns := append(taskPatterns, forEachPatterns...) + patterns := jobTaskRewritePatterns(task) + patterns = append(patterns, jobTaskRewritePatterns(task.Append(dyn.Key("for_each_task"), dyn.Key("task")))...) return append(patterns, jobRewritePattern{ - dyn.NewPattern( - dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), - dyn.Key("environments"), dyn.AnyIndex(), dyn.Key("spec"), dyn.Key("project_environment"), - ), + job.Append(dyn.Key("environments"), dyn.AnyIndex(), dyn.Key("spec"), dyn.Key("project_environment")), TranslateModeFile, noSkipRewrite, }) diff --git a/bundle/config/mutator/paths/job_paths_visitor_test.go b/bundle/config/mutator/paths/job_paths_visitor_test.go index a66e35ea12d..d7bd3674133 100644 --- a/bundle/config/mutator/paths/job_paths_visitor_test.go +++ b/bundle/config/mutator/paths/job_paths_visitor_test.go @@ -1,17 +1,14 @@ package paths import ( - "strings" "testing" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestVisitJobPaths(t *testing.T) { @@ -116,6 +113,8 @@ func TestVisitJobPaths_environments(t *testing.T) { JobSettings: jobs.JobSettings{ Environments: []jobs.JobEnvironment{ environment0, + {Spec: &compute.Environment{ProjectEnvironment: "../pyproject.toml"}}, + {Spec: &compute.Environment{ProjectEnvironment: "/Workspace/shared/pyproject.toml"}}, }, }, } @@ -129,63 +128,14 @@ func TestVisitJobPaths_environments(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitJobPaths) - var expected []dyn.Path + expected := []dyn.Path{ + dyn.MustPathFromString("resources.jobs.job0.environments[1].spec.project_environment"), + dyn.MustPathFromString("resources.jobs.job0.environments[2].spec.project_environment"), + } assert.ElementsMatch(t, expected, actual) } -func TestVisitJobPaths_projectEnvironments(t *testing.T) { - value, err := yamlloader.LoadYAML("resources/jobs.yml", strings.NewReader(` -resources: - jobs: - first: - tasks: - - task_key: notebook - notebook_task: - notebook_path: notebook.py - environments: - - environment_key: project - spec: - project_environment: ../pyproject.toml - - environment_key: standard - spec: - environment_version: "4" - dependencies: [requests] - - environment_key: requirements - spec: - project_environment: ../requirements.txt - second: - environments: - - environment_key: project - spec: - project_environment: /Workspace/shared/pyproject.toml -`)) - require.NoError(t, err) - - visited := map[string]TranslateMode{} - updated, err := VisitJobPaths(value, func(p dyn.Path, mode TranslateMode, v dyn.Value) (dyn.Value, error) { - visited[p.String()] = mode - return dyn.NewValue("rewritten:"+v.MustString(), v.Locations()), nil - }) - require.NoError(t, err) - assert.Equal(t, map[string]TranslateMode{ - "resources.jobs.first.tasks[0].notebook_task.notebook_path": TranslateModeNotebook, - "resources.jobs.first.environments[0].spec.project_environment": TranslateModeFile, - "resources.jobs.first.environments[2].spec.project_environment": TranslateModeFile, - "resources.jobs.second.environments[0].spec.project_environment": TranslateModeFile, - }, visited) - for path, want := range map[string]string{ - "resources.jobs.first.tasks[0].notebook_task.notebook_path": "rewritten:notebook.py", - "resources.jobs.first.environments[0].spec.project_environment": "rewritten:../pyproject.toml", - "resources.jobs.first.environments[2].spec.project_environment": "rewritten:../requirements.txt", - "resources.jobs.second.environments[0].spec.project_environment": "rewritten:/Workspace/shared/pyproject.toml", - } { - actual, err := dyn.Get(updated, path) - require.NoError(t, err) - assert.Equal(t, want, actual.MustString(), path) - } -} - func TestVisitJobPaths_foreach(t *testing.T) { task0 := jobs.Task{ ForEachTask: &jobs.ForEachTask{ diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index dffb151f19a..e237be3be81 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -71,8 +71,8 @@ func TestTranslatePathsProjectEnvironment(t *testing.T) { want: "/Workspace/bundle/files/resources/pyproject.toml", }, { - name: "requirements file with spaces", path: "../env files/requirements.txt", localFile: "env files/requirements.txt", - want: "/Workspace/bundle/files/env files/requirements.txt", + name: "path with spaces", path: "../env files/pyproject.toml", localFile: "env files/pyproject.toml", + want: "/Workspace/bundle/files/env files/pyproject.toml", }, { name: "absolute workspace path", path: "/Workspace/shared/pyproject.toml", @@ -100,19 +100,12 @@ func TestTranslatePathsProjectEnvironment(t *testing.T) { if tc.localFile != "" { touchEmptyFile(t, filepath.Join(dir, tc.localFile)) } - notebookPath := filepath.Join(dir, "src", "notebooks", "notebook.py") - require.NoError(t, os.MkdirAll(filepath.Dir(notebookPath), 0o700)) - touchNotebookFile(t, notebookPath) job := &resources.Job{ JobSettings: jobs.JobSettings{ - Tasks: []jobs.Task{{ - TaskKey: "notebook", - NotebookTask: &jobs.NotebookTask{ - NotebookPath: "../src/notebooks/notebook.py", - }, + Environments: []jobs.JobEnvironment{{ EnvironmentKey: "project", + Spec: &compute.Environment{ProjectEnvironment: tc.path}, }}, - Environments: []jobs.JobEnvironment{{EnvironmentKey: "project", Spec: &compute.Environment{}}}, }, } if tc.gitSource { @@ -130,36 +123,18 @@ func TestTranslatePathsProjectEnvironment(t *testing.T) { } locations := []dyn.Location{{File: filepath.Join(dir, "resources", "job.yml")}} bundletest.SetLocation(b, ".", locations) - const field = "resources.jobs.job.environments[0].spec.project_environment" - // Until the SDK exposes this field, invoke the mutators directly so - // bundle.Apply does not discard it when converting from SDK types. - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - return dyn.Set(v, field, dyn.NewValue(tc.path, locations)) - }) - require.NoError(t, err) - require.NoError(t, mutator.NormalizePaths().Apply(t.Context(), b).Error()) - err = mutator.TranslatePaths().Apply(t.Context(), b).Error() + diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) + err := diags.Error() if tc.wantErr != "" { require.ErrorContains(t, err, tc.wantErr) return } require.NoError(t, err) - actual, err := dyn.Get(b.Config.Value(), field) - require.NoError(t, err) want := tc.want if tc.sourceLinked { want = filepath.ToSlash(filepath.Join(dir, want)) } - assert.Equal(t, want, actual.MustString()) - - wantNotebook := "/Workspace/bundle/files/src/notebooks/notebook" - switch { - case tc.gitSource: - wantNotebook = "../src/notebooks/notebook.py" - case tc.sourceLinked: - wantNotebook = filepath.ToSlash(filepath.Join(dir, "src", "notebooks", "notebook")) - } - assert.Equal(t, wantNotebook, b.Config.Resources.Jobs["job"].Tasks[0].NotebookTask.NotebookPath) + assert.Equal(t, want, b.Config.Resources.Jobs["job"].Environments[0].Spec.ProjectEnvironment) }) } }