From 463b271c4cad0797ac892fbb875efbeac7bcfdc0 Mon Sep 17 00:00:00 2001 From: Yiwei Dai <125326326+yiweidai-db@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:21:45 +0000 Subject: [PATCH 01/10] Resolve job project-environment paths relative to bundle YAML --- .../bundles/project-environment-paths.md | 1 + .../project_environment/databricks.yml | 5 + .../project_environment/out.test.toml | 3 + .../project_environment/output.txt | 77 ++++++++++++++++ .../project_environment/pyproject.toml | 6 ++ .../project_environment/resources/job.yml | 13 +++ .../resources/pyproject.toml | 6 ++ .../environments/project_environment/script | 8 ++ .../project_environment/src/notebook.py | 2 + .../project_environment/test.toml | 2 + .../config/mutator/paths/job_paths_visitor.go | 20 ++-- .../mutator/paths/job_paths_visitor_test.go | 7 +- bundle/config/mutator/translate_paths_test.go | 91 +++++++++++++++++++ 13 files changed, 234 insertions(+), 7 deletions(-) create mode 100644 .nextchanges/bundles/project-environment-paths.md create mode 100644 acceptance/bundle/environments/project_environment/databricks.yml create mode 100644 acceptance/bundle/environments/project_environment/out.test.toml create mode 100644 acceptance/bundle/environments/project_environment/output.txt create mode 100644 acceptance/bundle/environments/project_environment/pyproject.toml create mode 100644 acceptance/bundle/environments/project_environment/resources/job.yml create mode 100644 acceptance/bundle/environments/project_environment/resources/pyproject.toml create mode 100644 acceptance/bundle/environments/project_environment/script create mode 100644 acceptance/bundle/environments/project_environment/src/notebook.py create mode 100644 acceptance/bundle/environments/project_environment/test.toml diff --git a/.nextchanges/bundles/project-environment-paths.md b/.nextchanges/bundles/project-environment-paths.md new file mode 100644 index 00000000000..85c2d0108aa --- /dev/null +++ b/.nextchanges/bundles/project-environment-paths.md @@ -0,0 +1 @@ +* Resolve job `project_environment` paths relative to the YAML file that defines them. ([#6909](https://github.com/databricks/cli/pull/6909)) diff --git a/acceptance/bundle/environments/project_environment/databricks.yml b/acceptance/bundle/environments/project_environment/databricks.yml new file mode 100644 index 00000000000..03565b8a553 --- /dev/null +++ b/acceptance/bundle/environments/project_environment/databricks.yml @@ -0,0 +1,5 @@ +bundle: + name: project_environment + +include: + - resources/*.yml diff --git a/acceptance/bundle/environments/project_environment/out.test.toml b/acceptance/bundle/environments/project_environment/out.test.toml new file mode 100644 index 00000000000..59b56a2037c --- /dev/null +++ b/acceptance/bundle/environments/project_environment/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/environments/project_environment/output.txt b/acceptance/bundle/environments/project_environment/output.txt new file mode 100644 index 00000000000..6d3e465e0ed --- /dev/null +++ b/acceptance/bundle/environments/project_environment/output.txt @@ -0,0 +1,77 @@ + +>>> [CLI] bundle validate -o json +{ + "tasks": [ + { + "environment_key": "project", + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/src/notebook" + }, + "task_key": "notebook" + } + ], + "environments": [ + { + "environment_key": "project", + "spec": { + "project_environment": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/pyproject.toml" + } + } + ] +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/project_environment/default/files... +Created jobs.notebook +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.2/jobs/create +{ + "tasks": [ + { + "environment_key": "project", + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/src/notebook" + }, + "task_key": "notebook" + } + ], + "environments": [ + { + "environment_key": "project", + "spec": { + "project_environment": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/pyproject.toml" + } + } + ] +} + +>>> update_file.py resources/job.yml ../pyproject.toml ./pyproject.toml + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/project_environment/default/files... +Updated jobs.notebook +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.2/jobs/reset +{ + "tasks": [ + { + "environment_key": "project", + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/src/notebook" + }, + "task_key": "notebook" + } + ], + "environments": [ + { + "environment_key": "project", + "spec": { + "project_environment": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/resources/pyproject.toml" + } + } + ] +} diff --git a/acceptance/bundle/environments/project_environment/pyproject.toml b/acceptance/bundle/environments/project_environment/pyproject.toml new file mode 100644 index 00000000000..b22de93e858 --- /dev/null +++ b/acceptance/bundle/environments/project_environment/pyproject.toml @@ -0,0 +1,6 @@ +[project] +name = "project-environment-test" +version = "0.1.0" + +[tool.databricks.environment] +environment_version = "4" diff --git a/acceptance/bundle/environments/project_environment/resources/job.yml b/acceptance/bundle/environments/project_environment/resources/job.yml new file mode 100644 index 00000000000..bce7abe7c65 --- /dev/null +++ b/acceptance/bundle/environments/project_environment/resources/job.yml @@ -0,0 +1,13 @@ +resources: + jobs: + notebook: + name: project_environment + tasks: + - task_key: notebook + notebook_task: + notebook_path: ../src/notebook.py + environment_key: project + environments: + - environment_key: project + spec: + project_environment: ../pyproject.toml diff --git a/acceptance/bundle/environments/project_environment/resources/pyproject.toml b/acceptance/bundle/environments/project_environment/resources/pyproject.toml new file mode 100644 index 00000000000..d2854958830 --- /dev/null +++ b/acceptance/bundle/environments/project_environment/resources/pyproject.toml @@ -0,0 +1,6 @@ +[project] +name = "updated-project-environment-test" +version = "0.1.0" + +[tool.databricks.environment] +environment_version = "4" diff --git a/acceptance/bundle/environments/project_environment/script b/acceptance/bundle/environments/project_environment/script new file mode 100644 index 00000000000..440ab89e91e --- /dev/null +++ b/acceptance/bundle/environments/project_environment/script @@ -0,0 +1,8 @@ +trace $CLI bundle validate -o json | jq '.resources.jobs.notebook | {tasks, environments}' + +trace $CLI bundle deploy +trace print_requests.py //api/2.2/jobs/create | jq '.body | {tasks, environments}' + +trace update_file.py resources/job.yml ../pyproject.toml ./pyproject.toml +trace $CLI bundle deploy +trace print_requests.py //api/2.2/jobs/reset | jq '.body.new_settings | {tasks, environments}' diff --git a/acceptance/bundle/environments/project_environment/src/notebook.py b/acceptance/bundle/environments/project_environment/src/notebook.py new file mode 100644 index 00000000000..5bee9c9eae0 --- /dev/null +++ b/acceptance/bundle/environments/project_environment/src/notebook.py @@ -0,0 +1,2 @@ +# Databricks notebook source +print("Project environment") diff --git a/acceptance/bundle/environments/project_environment/test.toml b/acceptance/bundle/environments/project_environment/test.toml new file mode 100644 index 00000000000..ba0016a753f --- /dev/null +++ b/acceptance/bundle/environments/project_environment/test.toml @@ -0,0 +1,2 @@ +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +RecordRequests = true diff --git a/bundle/config/mutator/paths/job_paths_visitor.go b/bundle/config/mutator/paths/job_paths_visitor.go index 6e82e544f02..6e3c1166b8d 100644 --- a/bundle/config/mutator/paths/job_paths_visitor.go +++ b/bundle/config/mutator/paths/job_paths_visitor.go @@ -58,19 +58,27 @@ func jobTaskRewritePatterns(base dyn.Pattern) []jobRewritePattern { } func jobRewritePatterns() []jobRewritePattern { - // Base pattern to match all tasks in all jobs. + // Base pattern to match all jobs. base := dyn.NewPattern( dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), - dyn.Key("tasks"), - dyn.AnyIndex(), ) + task := base.Append(dyn.Key("tasks"), dyn.AnyIndex()) - taskPatterns := jobTaskRewritePatterns(base) - forEachPatterns := jobTaskRewritePatterns(base.Append(dyn.Key("for_each_task"), dyn.Key("task"))) - patterns := append(taskPatterns, forEachPatterns...) + patterns := jobTaskRewritePatterns(task) + patterns = append(patterns, jobTaskRewritePatterns(task.Append(dyn.Key("for_each_task"), dyn.Key("task")))...) return append(patterns, + jobRewritePattern{ + base.Append( + dyn.Key("environments"), + dyn.AnyIndex(), + dyn.Key("spec"), + dyn.Key("project_environment"), + ), + TranslateModeFile, + noSkipRewrite, + }, jobRewritePattern{ dyn.NewPattern( dyn.Key("resources"), diff --git a/bundle/config/mutator/paths/job_paths_visitor_test.go b/bundle/config/mutator/paths/job_paths_visitor_test.go index 23fb6408267..e27cdc185eb 100644 --- a/bundle/config/mutator/paths/job_paths_visitor_test.go +++ b/bundle/config/mutator/paths/job_paths_visitor_test.go @@ -113,6 +113,8 @@ func TestVisitJobPaths_environments(t *testing.T) { JobSettings: jobs.JobSettings{ Environments: []jobs.JobEnvironment{ environment0, + {Spec: &compute.Environment{ProjectEnvironment: "../pyproject.toml"}}, + {Spec: &compute.Environment{ProjectEnvironment: "/Workspace/shared/pyproject.toml"}}, }, }, } @@ -126,7 +128,10 @@ func TestVisitJobPaths_environments(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitJobPaths) - var expected []dyn.Path + expected := []dyn.Path{ + dyn.MustPathFromString("resources.jobs.job0.environments[1].spec.project_environment"), + dyn.MustPathFromString("resources.jobs.job0.environments[2].spec.project_environment"), + } assert.ElementsMatch(t, expected, actual) } diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index 226a848b723..e237be3be81 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -48,6 +48,97 @@ func touchEmptyFile(t *testing.T, path string) { f.Close() } +func TestTranslatePathsProjectEnvironment(t *testing.T) { + for _, tc := range []struct { + name string + path string + localFile string + want string + wantErr string + gitSource bool + sourceLinked bool + }{ + { + name: "YAML directory", path: "./pyproject.toml", localFile: "resources/pyproject.toml", + want: "/Workspace/bundle/files/resources/pyproject.toml", + }, + { + name: "YAML parent", path: "../pyproject.toml", localFile: "pyproject.toml", + want: "/Workspace/bundle/files/pyproject.toml", + }, + { + name: "bare filename", path: "pyproject.toml", localFile: "resources/pyproject.toml", + want: "/Workspace/bundle/files/resources/pyproject.toml", + }, + { + name: "path with spaces", path: "../env files/pyproject.toml", localFile: "env files/pyproject.toml", + want: "/Workspace/bundle/files/env files/pyproject.toml", + }, + { + name: "absolute workspace path", path: "/Workspace/shared/pyproject.toml", + want: "/Workspace/shared/pyproject.toml", + }, + { + name: "source-linked deployment", path: "../pyproject.toml", localFile: "pyproject.toml", + want: "pyproject.toml", sourceLinked: true, + }, + { + name: "git source", path: "../pyproject.toml", + want: "../pyproject.toml", gitSource: true, + }, + { + name: "missing file", path: "../pyproject.toml", + wantErr: "file pyproject.toml not found", + }, + { + name: "outside sync root", path: "../../pyproject.toml", + wantErr: "is not contained in sync root path", + }, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + if tc.localFile != "" { + touchEmptyFile(t, filepath.Join(dir, tc.localFile)) + } + job := &resources.Job{ + JobSettings: jobs.JobSettings{ + Environments: []jobs.JobEnvironment{{ + EnvironmentKey: "project", + Spec: &compute.Environment{ProjectEnvironment: tc.path}, + }}, + }, + } + if tc.gitSource { + job.GitSource = &jobs.GitSource{GitUrl: "https://example.test/repo.git"} + } + b := &bundle.Bundle{ + BundleRootPath: dir, + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), + Config: config.Root{ + Workspace: config.Workspace{FilePath: "/Workspace/bundle/files"}, + Resources: config.Resources{Jobs: map[string]*resources.Job{"job": job}}, + Presets: config.Presets{SourceLinkedDeployment: &tc.sourceLinked}, + }, + } + locations := []dyn.Location{{File: filepath.Join(dir, "resources", "job.yml")}} + bundletest.SetLocation(b, ".", locations) + diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) + err := diags.Error() + if tc.wantErr != "" { + require.ErrorContains(t, err, tc.wantErr) + return + } + require.NoError(t, err) + want := tc.want + if tc.sourceLinked { + want = filepath.ToSlash(filepath.Join(dir, want)) + } + assert.Equal(t, want, b.Config.Resources.Jobs["job"].Environments[0].Spec.ProjectEnvironment) + }) + } +} + func TestTranslatePathsSkippedWithGitSource(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ From 99aeb092fca55b3a6d2591a7fae778d8622627ea Mon Sep 17 00:00:00 2001 From: Yiwei Dai <125326326+yiweidai-db@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:23:52 +0000 Subject: [PATCH 02/10] Link project-environment release note to upstream PR --- .nextchanges/bundles/project-environment-paths.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/project-environment-paths.md b/.nextchanges/bundles/project-environment-paths.md index 85c2d0108aa..2f1c406fee0 100644 --- a/.nextchanges/bundles/project-environment-paths.md +++ b/.nextchanges/bundles/project-environment-paths.md @@ -1 +1 @@ -* Resolve job `project_environment` paths relative to the YAML file that defines them. ([#6909](https://github.com/databricks/cli/pull/6909)) +* Resolve job `project_environment` paths relative to the YAML file that defines them. ([#6983](https://github.com/databricks/cli/pull/6983)) From 76b97048fee7a261d73a665256f49a803e7ff5c5 Mon Sep 17 00:00:00 2001 From: Yiwei Dai <125326326+yiweidai-db@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:55:43 +0000 Subject: [PATCH 03/10] Use a workspace-style path in the source-linked test fixture --- bundle/config/mutator/translate_paths_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index e237be3be81..518838d9c91 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -96,7 +96,8 @@ func TestTranslatePathsProjectEnvironment(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - dir := t.TempDir() + // Source-linked deployments reuse the sync root as a POSIX workspace path. + dir := filepath.ToSlash(t.TempDir()) if tc.localFile != "" { touchEmptyFile(t, filepath.Join(dir, tc.localFile)) } From c2be77c65e0dd9a1c1e9438346ae96c93e9968c8 Mon Sep 17 00:00:00 2001 From: Yiwei Dai <125326326+yiweidai-db@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:08:43 +0000 Subject: [PATCH 04/10] Retrigger CI after GitHub missed the branch update From dbe16d46301cfde278dc0b225de20c445b903066 Mon Sep 17 00:00:00 2001 From: Yiwei Dai <125326326+yiweidai-db@users.noreply.github.com> Date: Wed, 7 Oct 2026 19:46:03 +0000 Subject: [PATCH 05/10] Keep task path base unchanged for project environments --- .../config/mutator/paths/job_paths_visitor.go | 63 ++++++++++--------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/bundle/config/mutator/paths/job_paths_visitor.go b/bundle/config/mutator/paths/job_paths_visitor.go index 6e3c1166b8d..f8a90e772dd 100644 --- a/bundle/config/mutator/paths/job_paths_visitor.go +++ b/bundle/config/mutator/paths/job_paths_visitor.go @@ -58,42 +58,45 @@ func jobTaskRewritePatterns(base dyn.Pattern) []jobRewritePattern { } func jobRewritePatterns() []jobRewritePattern { - // Base pattern to match all jobs. + // Base pattern to match all tasks in all jobs. base := dyn.NewPattern( dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), + dyn.Key("tasks"), + dyn.AnyIndex(), ) - task := base.Append(dyn.Key("tasks"), dyn.AnyIndex()) - patterns := jobTaskRewritePatterns(task) - patterns = append(patterns, jobTaskRewritePatterns(task.Append(dyn.Key("for_each_task"), dyn.Key("task")))...) - return append(patterns, - jobRewritePattern{ - base.Append( - dyn.Key("environments"), - dyn.AnyIndex(), - dyn.Key("spec"), - dyn.Key("project_environment"), - ), - TranslateModeFile, - noSkipRewrite, - }, - jobRewritePattern{ - dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("environment_variables"), - dyn.AnyIndex(), - dyn.Key("spec"), - dyn.Key("files"), - dyn.AnyIndex(), - ), - TranslateModeFile, - noSkipRewrite, - }, - ) + taskPatterns := jobTaskRewritePatterns(base) + forEachPatterns := jobTaskRewritePatterns(base.Append(dyn.Key("for_each_task"), dyn.Key("task"))) + patterns := append(taskPatterns, forEachPatterns...) + patterns = append(patterns, jobRewritePattern{ + dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("jobs"), + dyn.AnyKey(), + dyn.Key("environment_variables"), + dyn.AnyIndex(), + dyn.Key("spec"), + dyn.Key("files"), + dyn.AnyIndex(), + ), + TranslateModeFile, + noSkipRewrite, + }) + return append(patterns, jobRewritePattern{ + dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("jobs"), + dyn.AnyKey(), + dyn.Key("environments"), + dyn.AnyIndex(), + dyn.Key("spec"), + dyn.Key("project_environment"), + ), + TranslateModeFile, + noSkipRewrite, + }) } // VisitJobPaths visits all paths in job resources and applies a function to each path. From 5734aefe6b4acb8e0938f65577d904256c8cfee4 Mon Sep 17 00:00:00 2001 From: Yiwei Dai <125326326+yiweidai-db@users.noreply.github.com> Date: Wed, 7 Oct 2026 20:17:15 +0000 Subject: [PATCH 06/10] Extract job environment rewrite pattern helper --- .../config/mutator/paths/job_paths_visitor.go | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/bundle/config/mutator/paths/job_paths_visitor.go b/bundle/config/mutator/paths/job_paths_visitor.go index f8a90e772dd..c6684cf3e29 100644 --- a/bundle/config/mutator/paths/job_paths_visitor.go +++ b/bundle/config/mutator/paths/job_paths_visitor.go @@ -57,6 +57,16 @@ func jobTaskRewritePatterns(base dyn.Pattern) []jobRewritePattern { } } +func jobEnvironmentRewritePatterns(base dyn.Pattern) []jobRewritePattern { + return []jobRewritePattern{ + { + base.Append(dyn.Key("spec"), dyn.Key("project_environment")), + TranslateModeFile, + noSkipRewrite, + }, + } +} + func jobRewritePatterns() []jobRewritePattern { // Base pattern to match all tasks in all jobs. base := dyn.NewPattern( @@ -69,6 +79,16 @@ func jobRewritePatterns() []jobRewritePattern { taskPatterns := jobTaskRewritePatterns(base) forEachPatterns := jobTaskRewritePatterns(base.Append(dyn.Key("for_each_task"), dyn.Key("task"))) + + environmentBase := dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("jobs"), + dyn.AnyKey(), + dyn.Key("environments"), + dyn.AnyIndex(), + ) + environmentPatterns := jobEnvironmentRewritePatterns(environmentBase) + patterns := append(taskPatterns, forEachPatterns...) patterns = append(patterns, jobRewritePattern{ dyn.NewPattern( @@ -84,19 +104,7 @@ func jobRewritePatterns() []jobRewritePattern { TranslateModeFile, noSkipRewrite, }) - return append(patterns, jobRewritePattern{ - dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("environments"), - dyn.AnyIndex(), - dyn.Key("spec"), - dyn.Key("project_environment"), - ), - TranslateModeFile, - noSkipRewrite, - }) + return append(patterns, environmentPatterns...) } // VisitJobPaths visits all paths in job resources and applies a function to each path. From d2941b88feeaf3007f710ae3dd254ba1789eeecb Mon Sep 17 00:00:00 2001 From: Yiwei Dai <125326326+yiweidai-db@users.noreply.github.com> Date: Wed, 7 Oct 2026 20:29:35 +0000 Subject: [PATCH 07/10] Clarify task and environment path pattern groups --- .../config/mutator/paths/job_paths_visitor.go | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/bundle/config/mutator/paths/job_paths_visitor.go b/bundle/config/mutator/paths/job_paths_visitor.go index c6684cf3e29..490793c3cd5 100644 --- a/bundle/config/mutator/paths/job_paths_visitor.go +++ b/bundle/config/mutator/paths/job_paths_visitor.go @@ -1,6 +1,8 @@ package paths import ( + "slices" + "github.com/databricks/cli/libs/dyn" ) @@ -14,35 +16,35 @@ func noSkipRewrite(string) bool { return false } -func jobTaskRewritePatterns(base dyn.Pattern) []jobRewritePattern { +func jobTaskRewritePatterns(taskBase dyn.Pattern) []jobRewritePattern { return []jobRewritePattern{ { - base.Append(dyn.Key("notebook_task"), dyn.Key("notebook_path")), + taskBase.Append(dyn.Key("notebook_task"), dyn.Key("notebook_path")), TranslateModeNotebook, noSkipRewrite, }, { - base.Append(dyn.Key("spark_python_task"), dyn.Key("python_file")), + taskBase.Append(dyn.Key("spark_python_task"), dyn.Key("python_file")), TranslateModeFile, noSkipRewrite, }, { - base.Append(dyn.Key("dbt_task"), dyn.Key("project_directory")), + taskBase.Append(dyn.Key("dbt_task"), dyn.Key("project_directory")), TranslateModeDirectory, noSkipRewrite, }, { - base.Append(dyn.Key("sql_task"), dyn.Key("file"), dyn.Key("path")), + taskBase.Append(dyn.Key("sql_task"), dyn.Key("file"), dyn.Key("path")), TranslateModeFile, noSkipRewrite, }, { - base.Append(dyn.Key("alert_task"), dyn.Key("workspace_path")), + taskBase.Append(dyn.Key("alert_task"), dyn.Key("workspace_path")), TranslateModeFile, noSkipRewrite, }, { - base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("requirements")), + taskBase.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("requirements")), TranslateModeFile, noSkipRewrite, }, @@ -50,17 +52,17 @@ func jobTaskRewritePatterns(base dyn.Pattern) []jobRewritePattern { // The AI Runtime task runs this bash script on each node; the backend // reads it as a workspace file, so translate the local path to its // remote (or immutable-snapshot) location like any other file. - base.Append(dyn.Key("ai_runtime_task"), dyn.Key("deployments"), dyn.AnyIndex(), dyn.Key("command_path")), + taskBase.Append(dyn.Key("ai_runtime_task"), dyn.Key("deployments"), dyn.AnyIndex(), dyn.Key("command_path")), TranslateModeFile, noSkipRewrite, }, } } -func jobEnvironmentRewritePatterns(base dyn.Pattern) []jobRewritePattern { +func jobEnvironmentRewritePatterns(environmentBase dyn.Pattern) []jobRewritePattern { return []jobRewritePattern{ { - base.Append(dyn.Key("spec"), dyn.Key("project_environment")), + environmentBase.Append(dyn.Key("spec"), dyn.Key("project_environment")), TranslateModeFile, noSkipRewrite, }, @@ -69,7 +71,7 @@ func jobEnvironmentRewritePatterns(base dyn.Pattern) []jobRewritePattern { func jobRewritePatterns() []jobRewritePattern { // Base pattern to match all tasks in all jobs. - base := dyn.NewPattern( + taskBase := dyn.NewPattern( dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), @@ -77,8 +79,10 @@ func jobRewritePatterns() []jobRewritePattern { dyn.AnyIndex(), ) - taskPatterns := jobTaskRewritePatterns(base) - forEachPatterns := jobTaskRewritePatterns(base.Append(dyn.Key("for_each_task"), dyn.Key("task"))) + taskPatterns := slices.Concat( + jobTaskRewritePatterns(taskBase), + jobTaskRewritePatterns(taskBase.Append(dyn.Key("for_each_task"), dyn.Key("task"))), + ) environmentBase := dyn.NewPattern( dyn.Key("resources"), @@ -89,8 +93,7 @@ func jobRewritePatterns() []jobRewritePattern { ) environmentPatterns := jobEnvironmentRewritePatterns(environmentBase) - patterns := append(taskPatterns, forEachPatterns...) - patterns = append(patterns, jobRewritePattern{ + taskPatterns = append(taskPatterns, jobRewritePattern{ dyn.NewPattern( dyn.Key("resources"), dyn.Key("jobs"), @@ -104,7 +107,7 @@ func jobRewritePatterns() []jobRewritePattern { TranslateModeFile, noSkipRewrite, }) - return append(patterns, environmentPatterns...) + return slices.Concat(taskPatterns, environmentPatterns) } // VisitJobPaths visits all paths in job resources and applies a function to each path. From 7b9d466d0acfa704cd43291dc78fce0df7f683b9 Mon Sep 17 00:00:00 2001 From: Yiwei Dai <125326326+yiweidai-db@users.noreply.github.com> Date: Wed, 7 Oct 2026 20:46:56 +0000 Subject: [PATCH 08/10] Keep task base naming and clarify pattern scopes --- .../config/mutator/paths/job_paths_visitor.go | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/bundle/config/mutator/paths/job_paths_visitor.go b/bundle/config/mutator/paths/job_paths_visitor.go index 490793c3cd5..a1b9a634ada 100644 --- a/bundle/config/mutator/paths/job_paths_visitor.go +++ b/bundle/config/mutator/paths/job_paths_visitor.go @@ -16,35 +16,35 @@ func noSkipRewrite(string) bool { return false } -func jobTaskRewritePatterns(taskBase dyn.Pattern) []jobRewritePattern { +func jobTaskRewritePatterns(base dyn.Pattern) []jobRewritePattern { return []jobRewritePattern{ { - taskBase.Append(dyn.Key("notebook_task"), dyn.Key("notebook_path")), + base.Append(dyn.Key("notebook_task"), dyn.Key("notebook_path")), TranslateModeNotebook, noSkipRewrite, }, { - taskBase.Append(dyn.Key("spark_python_task"), dyn.Key("python_file")), + base.Append(dyn.Key("spark_python_task"), dyn.Key("python_file")), TranslateModeFile, noSkipRewrite, }, { - taskBase.Append(dyn.Key("dbt_task"), dyn.Key("project_directory")), + base.Append(dyn.Key("dbt_task"), dyn.Key("project_directory")), TranslateModeDirectory, noSkipRewrite, }, { - taskBase.Append(dyn.Key("sql_task"), dyn.Key("file"), dyn.Key("path")), + base.Append(dyn.Key("sql_task"), dyn.Key("file"), dyn.Key("path")), TranslateModeFile, noSkipRewrite, }, { - taskBase.Append(dyn.Key("alert_task"), dyn.Key("workspace_path")), + base.Append(dyn.Key("alert_task"), dyn.Key("workspace_path")), TranslateModeFile, noSkipRewrite, }, { - taskBase.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("requirements")), + base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("requirements")), TranslateModeFile, noSkipRewrite, }, @@ -52,7 +52,7 @@ func jobTaskRewritePatterns(taskBase dyn.Pattern) []jobRewritePattern { // The AI Runtime task runs this bash script on each node; the backend // reads it as a workspace file, so translate the local path to its // remote (or immutable-snapshot) location like any other file. - taskBase.Append(dyn.Key("ai_runtime_task"), dyn.Key("deployments"), dyn.AnyIndex(), dyn.Key("command_path")), + base.Append(dyn.Key("ai_runtime_task"), dyn.Key("deployments"), dyn.AnyIndex(), dyn.Key("command_path")), TranslateModeFile, noSkipRewrite, }, @@ -70,8 +70,8 @@ func jobEnvironmentRewritePatterns(environmentBase dyn.Pattern) []jobRewritePatt } func jobRewritePatterns() []jobRewritePattern { - // Base pattern to match all tasks in all jobs. - taskBase := dyn.NewPattern( + // Task-level base pattern for all jobs. + base := dyn.NewPattern( dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), @@ -80,10 +80,11 @@ func jobRewritePatterns() []jobRewritePattern { ) taskPatterns := slices.Concat( - jobTaskRewritePatterns(taskBase), - jobTaskRewritePatterns(taskBase.Append(dyn.Key("for_each_task"), dyn.Key("task"))), + jobTaskRewritePatterns(base), + jobTaskRewritePatterns(base.Append(dyn.Key("for_each_task"), dyn.Key("task"))), ) + // Environment-level base pattern for all jobs. environmentBase := dyn.NewPattern( dyn.Key("resources"), dyn.Key("jobs"), From 4df3f0e70afddd3c86a740b3386b3c858c00f995 Mon Sep 17 00:00:00 2001 From: Yiwei Dai <125326326+yiweidai-db@users.noreply.github.com> Date: Wed, 7 Oct 2026 22:19:14 +0000 Subject: [PATCH 09/10] Use append for job path pattern groups --- bundle/config/mutator/paths/job_paths_visitor.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/bundle/config/mutator/paths/job_paths_visitor.go b/bundle/config/mutator/paths/job_paths_visitor.go index a1b9a634ada..aa2940d5c04 100644 --- a/bundle/config/mutator/paths/job_paths_visitor.go +++ b/bundle/config/mutator/paths/job_paths_visitor.go @@ -1,8 +1,6 @@ package paths import ( - "slices" - "github.com/databricks/cli/libs/dyn" ) @@ -79,9 +77,9 @@ func jobRewritePatterns() []jobRewritePattern { dyn.AnyIndex(), ) - taskPatterns := slices.Concat( + taskPatterns := append( jobTaskRewritePatterns(base), - jobTaskRewritePatterns(base.Append(dyn.Key("for_each_task"), dyn.Key("task"))), + jobTaskRewritePatterns(base.Append(dyn.Key("for_each_task"), dyn.Key("task")))..., ) // Environment-level base pattern for all jobs. @@ -108,7 +106,7 @@ func jobRewritePatterns() []jobRewritePattern { TranslateModeFile, noSkipRewrite, }) - return slices.Concat(taskPatterns, environmentPatterns) + return append(taskPatterns, environmentPatterns...) } // VisitJobPaths visits all paths in job resources and applies a function to each path. From 3d4c1f9b691ff3480a767bd0c0bab87a890506b9 Mon Sep 17 00:00:00 2001 From: Yiwei Dai <125326326+yiweidai-db@users.noreply.github.com> Date: Thu, 8 Oct 2026 21:42:08 +0000 Subject: [PATCH 10/10] Address project environment path review feedback --- .../project_environment/output.txt | 57 ++---------- .../environments/project_environment/script | 13 ++- .../config/mutator/paths/job_paths_visitor.go | 71 +++++++------- bundle/config/mutator/translate_paths_test.go | 92 ------------------- 4 files changed, 48 insertions(+), 185 deletions(-) diff --git a/acceptance/bundle/environments/project_environment/output.txt b/acceptance/bundle/environments/project_environment/output.txt index 6d3e465e0ed..a0090b9d8c3 100644 --- a/acceptance/bundle/environments/project_environment/output.txt +++ b/acceptance/bundle/environments/project_environment/output.txt @@ -1,23 +1,8 @@ >>> [CLI] bundle validate -o json { - "tasks": [ - { - "environment_key": "project", - "notebook_task": { - "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/src/notebook" - }, - "task_key": "notebook" - } - ], - "environments": [ - { - "environment_key": "project", - "spec": { - "project_environment": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/pyproject.toml" - } - } - ] + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/src/notebook", + "project_environment": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/pyproject.toml" } >>> [CLI] bundle deploy @@ -28,23 +13,8 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py //api/2.2/jobs/create { - "tasks": [ - { - "environment_key": "project", - "notebook_task": { - "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/src/notebook" - }, - "task_key": "notebook" - } - ], - "environments": [ - { - "environment_key": "project", - "spec": { - "project_environment": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/pyproject.toml" - } - } - ] + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/src/notebook", + "project_environment": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/pyproject.toml" } >>> update_file.py resources/job.yml ../pyproject.toml ./pyproject.toml @@ -57,21 +27,6 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged >>> print_requests.py //api/2.2/jobs/reset { - "tasks": [ - { - "environment_key": "project", - "notebook_task": { - "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/src/notebook" - }, - "task_key": "notebook" - } - ], - "environments": [ - { - "environment_key": "project", - "spec": { - "project_environment": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/resources/pyproject.toml" - } - } - ] + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/src/notebook", + "project_environment": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/resources/pyproject.toml" } diff --git a/acceptance/bundle/environments/project_environment/script b/acceptance/bundle/environments/project_environment/script index 440ab89e91e..9b5a185900c 100644 --- a/acceptance/bundle/environments/project_environment/script +++ b/acceptance/bundle/environments/project_environment/script @@ -1,8 +1,15 @@ -trace $CLI bundle validate -o json | jq '.resources.jobs.notebook | {tasks, environments}' +job_paths() { + jq '{ + notebook_path: .tasks[].notebook_task.notebook_path, + project_environment: .environments[].spec.project_environment + }' +} + +trace $CLI bundle validate -o json | jq '.resources.jobs.notebook' | job_paths trace $CLI bundle deploy -trace print_requests.py //api/2.2/jobs/create | jq '.body | {tasks, environments}' +trace print_requests.py //api/2.2/jobs/create | jq '.body' | job_paths trace update_file.py resources/job.yml ../pyproject.toml ./pyproject.toml trace $CLI bundle deploy -trace print_requests.py //api/2.2/jobs/reset | jq '.body.new_settings | {tasks, environments}' +trace print_requests.py //api/2.2/jobs/reset | jq '.body.new_settings' | job_paths diff --git a/bundle/config/mutator/paths/job_paths_visitor.go b/bundle/config/mutator/paths/job_paths_visitor.go index aa2940d5c04..a3abd0090a4 100644 --- a/bundle/config/mutator/paths/job_paths_visitor.go +++ b/bundle/config/mutator/paths/job_paths_visitor.go @@ -57,16 +57,6 @@ func jobTaskRewritePatterns(base dyn.Pattern) []jobRewritePattern { } } -func jobEnvironmentRewritePatterns(environmentBase dyn.Pattern) []jobRewritePattern { - return []jobRewritePattern{ - { - environmentBase.Append(dyn.Key("spec"), dyn.Key("project_environment")), - TranslateModeFile, - noSkipRewrite, - }, - } -} - func jobRewritePatterns() []jobRewritePattern { // Task-level base pattern for all jobs. base := dyn.NewPattern( @@ -77,36 +67,39 @@ func jobRewritePatterns() []jobRewritePattern { dyn.AnyIndex(), ) - taskPatterns := append( - jobTaskRewritePatterns(base), - jobTaskRewritePatterns(base.Append(dyn.Key("for_each_task"), dyn.Key("task")))..., - ) - - // Environment-level base pattern for all jobs. - environmentBase := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("environments"), - dyn.AnyIndex(), + taskPatterns := jobTaskRewritePatterns(base) + forEachPatterns := jobTaskRewritePatterns(base.Append(dyn.Key("for_each_task"), dyn.Key("task"))) + patterns := append(taskPatterns, forEachPatterns...) + return append(patterns, + jobRewritePattern{ + dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("jobs"), + dyn.AnyKey(), + dyn.Key("environment_variables"), + dyn.AnyIndex(), + dyn.Key("spec"), + dyn.Key("files"), + dyn.AnyIndex(), + ), + TranslateModeFile, + noSkipRewrite, + }, + // Environment-level pattern for all jobs. + jobRewritePattern{ + dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("jobs"), + dyn.AnyKey(), + dyn.Key("environments"), + dyn.AnyIndex(), + dyn.Key("spec"), + dyn.Key("project_environment"), + ), + TranslateModeFile, + noSkipRewrite, + }, ) - environmentPatterns := jobEnvironmentRewritePatterns(environmentBase) - - taskPatterns = append(taskPatterns, jobRewritePattern{ - dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("environment_variables"), - dyn.AnyIndex(), - dyn.Key("spec"), - dyn.Key("files"), - dyn.AnyIndex(), - ), - TranslateModeFile, - noSkipRewrite, - }) - return append(taskPatterns, environmentPatterns...) } // VisitJobPaths visits all paths in job resources and applies a function to each path. diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index 518838d9c91..226a848b723 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -48,98 +48,6 @@ func touchEmptyFile(t *testing.T, path string) { f.Close() } -func TestTranslatePathsProjectEnvironment(t *testing.T) { - for _, tc := range []struct { - name string - path string - localFile string - want string - wantErr string - gitSource bool - sourceLinked bool - }{ - { - name: "YAML directory", path: "./pyproject.toml", localFile: "resources/pyproject.toml", - want: "/Workspace/bundle/files/resources/pyproject.toml", - }, - { - name: "YAML parent", path: "../pyproject.toml", localFile: "pyproject.toml", - want: "/Workspace/bundle/files/pyproject.toml", - }, - { - name: "bare filename", path: "pyproject.toml", localFile: "resources/pyproject.toml", - want: "/Workspace/bundle/files/resources/pyproject.toml", - }, - { - name: "path with spaces", path: "../env files/pyproject.toml", localFile: "env files/pyproject.toml", - want: "/Workspace/bundle/files/env files/pyproject.toml", - }, - { - name: "absolute workspace path", path: "/Workspace/shared/pyproject.toml", - want: "/Workspace/shared/pyproject.toml", - }, - { - name: "source-linked deployment", path: "../pyproject.toml", localFile: "pyproject.toml", - want: "pyproject.toml", sourceLinked: true, - }, - { - name: "git source", path: "../pyproject.toml", - want: "../pyproject.toml", gitSource: true, - }, - { - name: "missing file", path: "../pyproject.toml", - wantErr: "file pyproject.toml not found", - }, - { - name: "outside sync root", path: "../../pyproject.toml", - wantErr: "is not contained in sync root path", - }, - } { - t.Run(tc.name, func(t *testing.T) { - // Source-linked deployments reuse the sync root as a POSIX workspace path. - dir := filepath.ToSlash(t.TempDir()) - if tc.localFile != "" { - touchEmptyFile(t, filepath.Join(dir, tc.localFile)) - } - job := &resources.Job{ - JobSettings: jobs.JobSettings{ - Environments: []jobs.JobEnvironment{{ - EnvironmentKey: "project", - Spec: &compute.Environment{ProjectEnvironment: tc.path}, - }}, - }, - } - if tc.gitSource { - job.GitSource = &jobs.GitSource{GitUrl: "https://example.test/repo.git"} - } - b := &bundle.Bundle{ - BundleRootPath: dir, - SyncRootPath: dir, - SyncRoot: vfs.MustNew(dir), - Config: config.Root{ - Workspace: config.Workspace{FilePath: "/Workspace/bundle/files"}, - Resources: config.Resources{Jobs: map[string]*resources.Job{"job": job}}, - Presets: config.Presets{SourceLinkedDeployment: &tc.sourceLinked}, - }, - } - locations := []dyn.Location{{File: filepath.Join(dir, "resources", "job.yml")}} - bundletest.SetLocation(b, ".", locations) - diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) - err := diags.Error() - if tc.wantErr != "" { - require.ErrorContains(t, err, tc.wantErr) - return - } - require.NoError(t, err) - want := tc.want - if tc.sourceLinked { - want = filepath.ToSlash(filepath.Join(dir, want)) - } - assert.Equal(t, want, b.Config.Resources.Jobs["job"].Environments[0].Spec.ProjectEnvironment) - }) - } -} - func TestTranslatePathsSkippedWithGitSource(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{