Resolve job project-environment paths relative to bundle YAML

.nextchanges/bundles/project-environment-paths.md

@@ -0,0 +1 @@
+* Resolve job `project_environment` paths relative to the YAML file that defines them. ([#6909](https://github.com/databricks/cli/pull/6909))

acceptance/bundle/environments/project_environment/databricks.yml

@@ -0,0 +1,5 @@
+bundle:
+  name: project_environment
+
+include:
+  - resources/*.yml

acceptance/bundle/environments/project_environment/out.test.toml

@@ -0,0 +1,3 @@
+Cloud = false
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/environments/project_environment/output.txt

@@ -0,0 +1,77 @@
+
+>>> [CLI] bundle validate -o json
+{
+  "tasks": [
+    {
+      "environment_key": "project",
+      "notebook_task": {
+        "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/src/notebook"
+      },
+      "task_key": "notebook"
+    }
+  ],
+  "environments": [
+    {
+      "environment_key": "project",
+      "spec": {
+        "project_environment": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/pyproject.toml"
+      }
+    }
+  ]
+}
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/project_environment/default/files...
+Created jobs.notebook
+Files: 5 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+
+>>> print_requests.py //api/2.2/jobs/create
+{
+  "tasks": [
+    {
+      "environment_key": "project",
+      "notebook_task": {
+        "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/src/notebook"
+      },
+      "task_key": "notebook"
+    }
+  ],
+  "environments": [
+    {
+      "environment_key": "project",
+      "spec": {
+        "project_environment": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/pyproject.toml"
+      }
+    }
+  ]
+}
+
+>>> update_file.py resources/job.yml ../pyproject.toml ./pyproject.toml
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/project_environment/default/files...
+Updated jobs.notebook
+Files: 1 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
+
+>>> print_requests.py //api/2.2/jobs/reset
+{
+  "tasks": [
+    {
+      "environment_key": "project",
+      "notebook_task": {
+        "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/src/notebook"
+      },
+      "task_key": "notebook"
+    }
+  ],
+  "environments": [
+    {
+      "environment_key": "project",
+      "spec": {
+        "project_environment": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/resources/pyproject.toml"
+      }
+    }
+  ]
+}

acceptance/bundle/environments/project_environment/pyproject.toml

@@ -0,0 +1,6 @@
+[project]
+name = "project-environment-test"
+version = "0.1.0"
+
+[tool.databricks.environment]
+environment_version = "4"

acceptance/bundle/environments/project_environment/resources/job.yml

@@ -0,0 +1,13 @@
+resources:
+  jobs:
+    notebook:
+      name: project_environment
+      tasks:
+        - task_key: notebook
+          notebook_task:
+            notebook_path: ../src/notebook.py
+          environment_key: project
+      environments:
+        - environment_key: project
+          spec:
+            project_environment: ../pyproject.toml

acceptance/bundle/environments/project_environment/resources/pyproject.toml

@@ -0,0 +1,6 @@
+[project]
+name = "updated-project-environment-test"
+version = "0.1.0"
+
+[tool.databricks.environment]
+environment_version = "4"

acceptance/bundle/environments/project_environment/script

@@ -0,0 +1,8 @@
+trace $CLI bundle validate -o json | jq '.resources.jobs.notebook | {tasks, environments}'
+
+trace $CLI bundle deploy
+trace print_requests.py //api/2.2/jobs/create | jq '.body | {tasks, environments}'
+
+trace update_file.py resources/job.yml ../pyproject.toml ./pyproject.toml
+trace $CLI bundle deploy
+trace print_requests.py //api/2.2/jobs/reset | jq '.body.new_settings | {tasks, environments}'

acceptance/bundle/environments/project_environment/src/notebook.py

@@ -0,0 +1,2 @@
+# Databricks notebook source
+print("Project environment")

acceptance/bundle/environments/project_environment/test.toml

@@ -0,0 +1,3 @@
+# The pinned Terraform provider does not support project_environment.
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+RecordRequests = true

bundle/config/mutator/paths/job_paths_visitor.go

@@ -58,18 +58,16 @@ func jobTaskRewritePatterns(base dyn.Pattern) []jobRewritePattern {
 }
 
 func jobRewritePatterns() []jobRewritePattern {
-	// Base pattern to match all tasks in all jobs.
-	base := dyn.NewPattern(
-		dyn.Key("resources"),
-		dyn.Key("jobs"),
-		dyn.AnyKey(),
-		dyn.Key("tasks"),
-		dyn.AnyIndex(),
-	)
+	job := dyn.NewPattern(dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey())
+	task := job.Append(dyn.Key("tasks"), dyn.AnyIndex())
 
-	taskPatterns := jobTaskRewritePatterns(base)
-	forEachPatterns := jobTaskRewritePatterns(base.Append(dyn.Key("for_each_task"), dyn.Key("task")))
-	return append(taskPatterns, forEachPatterns...)
+	patterns := jobTaskRewritePatterns(task)
+	patterns = append(patterns, jobTaskRewritePatterns(task.Append(dyn.Key("for_each_task"), dyn.Key("task")))...)
+	return append(patterns, jobRewritePattern{
+		job.Append(dyn.Key("environments"), dyn.AnyIndex(), dyn.Key("spec"), dyn.Key("project_environment")),
+		TranslateModeFile,
+		noSkipRewrite,
+	})
 }
 
 // VisitJobPaths visits all paths in job resources and applies a function to each path.

bundle/config/mutator/paths/job_paths_visitor_test.go

@@ -113,6 +113,8 @@ func TestVisitJobPaths_environments(t *testing.T) {
 		JobSettings: jobs.JobSettings{
 			Environments: []jobs.JobEnvironment{
 				environment0,
+				{Spec: &compute.Environment{ProjectEnvironment: "../pyproject.toml"}},
+				{Spec: &compute.Environment{ProjectEnvironment: "/Workspace/shared/pyproject.toml"}},
 			},
 		},
 	}
@@ -126,7 +128,10 @@ func TestVisitJobPaths_environments(t *testing.T) {
 	}
 
 	actual := collectVisitedPaths(t, root, VisitJobPaths)
-	var expected []dyn.Path
+	expected := []dyn.Path{
+		dyn.MustPathFromString("resources.jobs.job0.environments[1].spec.project_environment"),
+		dyn.MustPathFromString("resources.jobs.job0.environments[2].spec.project_environment"),
+	}
 
 	assert.ElementsMatch(t, expected, actual)
 }

bundle/config/mutator/translate_paths_test.go

@@ -48,6 +48,97 @@ func touchEmptyFile(t *testing.T, path string) {
 	f.Close()
 }
 
+func TestTranslatePathsProjectEnvironment(t *testing.T) {
+	for _, tc := range []struct {
+		name         string
+		path         string
+		localFile    string
+		want         string
+		wantErr      string
+		gitSource    bool
+		sourceLinked bool
+	}{
+		{
+			name: "YAML directory", path: "./pyproject.toml", localFile: "resources/pyproject.toml",
+			want: "/Workspace/bundle/files/resources/pyproject.toml",
+		},
+		{
+			name: "YAML parent", path: "../pyproject.toml", localFile: "pyproject.toml",
+			want: "/Workspace/bundle/files/pyproject.toml",
+		},
+		{
+			name: "bare filename", path: "pyproject.toml", localFile: "resources/pyproject.toml",
+			want: "/Workspace/bundle/files/resources/pyproject.toml",
+		},
+		{
+			name: "path with spaces", path: "../env files/pyproject.toml", localFile: "env files/pyproject.toml",
+			want: "/Workspace/bundle/files/env files/pyproject.toml",
+		},
+		{
+			name: "absolute workspace path", path: "/Workspace/shared/pyproject.toml",
+			want: "/Workspace/shared/pyproject.toml",
+		},
+		{
+			name: "source-linked deployment", path: "../pyproject.toml", localFile: "pyproject.toml",
+			want: "pyproject.toml", sourceLinked: true,
+		},
+		{
+			name: "git source", path: "../pyproject.toml",
+			want: "../pyproject.toml", gitSource: true,
+		},
+		{
+			name: "missing file", path: "../pyproject.toml",
+			wantErr: "file pyproject.toml not found",
+		},
+		{
+			name: "outside sync root", path: "../../pyproject.toml",
+			wantErr: "is not contained in sync root path",
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			dir := t.TempDir()
+			if tc.localFile != "" {
+				touchEmptyFile(t, filepath.Join(dir, tc.localFile))
+			}
+			job := &resources.Job{
+				JobSettings: jobs.JobSettings{
+					Environments: []jobs.JobEnvironment{{
+						EnvironmentKey: "project",
+						Spec:           &compute.Environment{ProjectEnvironment: tc.path},
+					}},
+				},
+			}
+			if tc.gitSource {
+				job.GitSource = &jobs.GitSource{GitUrl: "https://example.test/repo.git"}
+			}
+			b := &bundle.Bundle{
+				BundleRootPath: dir,
+				SyncRootPath:   dir,
+				SyncRoot:       vfs.MustNew(dir),
+				Config: config.Root{
+					Workspace: config.Workspace{FilePath: "/Workspace/bundle/files"},
+					Resources: config.Resources{Jobs: map[string]*resources.Job{"job": job}},
+					Presets:   config.Presets{SourceLinkedDeployment: &tc.sourceLinked},
+				},
+			}
+			locations := []dyn.Location{{File: filepath.Join(dir, "resources", "job.yml")}}
+			bundletest.SetLocation(b, ".", locations)
+			diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths())
+			err := diags.Error()
+			if tc.wantErr != "" {
+				require.ErrorContains(t, err, tc.wantErr)
+				return
+			}
+			require.NoError(t, err)
+			want := tc.want
+			if tc.sourceLinked {
+				want = filepath.ToSlash(filepath.Join(dir, want))
+			}
+			assert.Equal(t, want, b.Config.Resources.Jobs["job"].Environments[0].Spec.ProjectEnvironment)
+		})
+	}
+}
+
 func TestTranslatePathsSkippedWithGitSource(t *testing.T) {
 	dir := t.TempDir()
 	b := &bundle.Bundle{