Resolve job project-environment paths relative to bundle YAML

.nextchanges/bundles/project-environment-paths.md

@@ -0,0 +1 @@
+* Resolve job `project_environment` paths relative to the YAML file that defines them. ([#6983](https://github.com/databricks/cli/pull/6983))

acceptance/bundle/environments/project_environment/databricks.yml

@@ -0,0 +1,5 @@
+bundle:
+  name: project_environment
+
+include:
+  - resources/*.yml

acceptance/bundle/environments/project_environment/out.test.toml

@@ -0,0 +1,3 @@
+Cloud = false
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/environments/project_environment/output.txt

@@ -0,0 +1,32 @@
+
+>>> [CLI] bundle validate -o json
+{
+  "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/src/notebook",
+  "project_environment": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/pyproject.toml"
+}
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/project_environment/default/files...
+Created jobs.notebook
+Files: 5 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+
+>>> print_requests.py //api/2.2/jobs/create
+{
+  "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/src/notebook",
+  "project_environment": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/pyproject.toml"
+}
+
+>>> update_file.py resources/job.yml ../pyproject.toml ./pyproject.toml
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/project_environment/default/files...
+Updated jobs.notebook
+Files: 1 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
+
+>>> print_requests.py //api/2.2/jobs/reset
+{
+  "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/src/notebook",
+  "project_environment": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/resources/pyproject.toml"
+}

acceptance/bundle/environments/project_environment/pyproject.toml

@@ -0,0 +1,6 @@
+[project]
+name = "project-environment-test"
+version = "0.1.0"
+
+[tool.databricks.environment]
+environment_version = "4"

acceptance/bundle/environments/project_environment/resources/job.yml

@@ -0,0 +1,13 @@
+resources:
+  jobs:
+    notebook:
+      name: project_environment
+      tasks:
+        - task_key: notebook
+          notebook_task:
+            notebook_path: ../src/notebook.py
+          environment_key: project
+      environments:
+        - environment_key: project
+          spec:
+            project_environment: ../pyproject.toml

acceptance/bundle/environments/project_environment/resources/pyproject.toml

@@ -0,0 +1,6 @@
+[project]
+name = "updated-project-environment-test"
+version = "0.1.0"
+
+[tool.databricks.environment]
+environment_version = "4"

acceptance/bundle/environments/project_environment/script

@@ -0,0 +1,15 @@
+job_paths() {
+    jq '{
+        notebook_path: .tasks[].notebook_task.notebook_path,
+        project_environment: .environments[].spec.project_environment
+    }'
+}
+
+trace $CLI bundle validate -o json | jq '.resources.jobs.notebook' | job_paths
+
+trace $CLI bundle deploy
+trace print_requests.py //api/2.2/jobs/create | jq '.body' | job_paths
+
+trace update_file.py resources/job.yml ../pyproject.toml ./pyproject.toml
+trace $CLI bundle deploy
+trace print_requests.py //api/2.2/jobs/reset | jq '.body.new_settings' | job_paths

acceptance/bundle/environments/project_environment/src/notebook.py

@@ -0,0 +1,2 @@
+# Databricks notebook source
+print("Project environment")

acceptance/bundle/environments/project_environment/test.toml

@@ -0,0 +1,2 @@
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+RecordRequests = true

bundle/config/mutator/paths/job_paths_visitor.go

@@ -58,7 +58,7 @@ func jobTaskRewritePatterns(base dyn.Pattern) []jobRewritePattern {
 }
 
 func jobRewritePatterns() []jobRewritePattern {
-	// Base pattern to match all tasks in all jobs.
+	// Task-level base pattern for all jobs.
 	base := dyn.NewPattern(
 		dyn.Key("resources"),
 		dyn.Key("jobs"),
@@ -85,6 +85,20 @@ func jobRewritePatterns() []jobRewritePattern {
 			TranslateModeFile,
 			noSkipRewrite,
 		},
+		// Environment-level pattern for all jobs.
+		jobRewritePattern{
+			dyn.NewPattern(
+				dyn.Key("resources"),
+				dyn.Key("jobs"),
+				dyn.AnyKey(),
+				dyn.Key("environments"),
+				dyn.AnyIndex(),
+				dyn.Key("spec"),
+				dyn.Key("project_environment"),
+			),
+			TranslateModeFile,
+			noSkipRewrite,
+		},
 	)
 }
 

bundle/config/mutator/paths/job_paths_visitor_test.go

@@ -113,6 +113,8 @@ func TestVisitJobPaths_environments(t *testing.T) {
 		JobSettings: jobs.JobSettings{
 			Environments: []jobs.JobEnvironment{
 				environment0,
+				{Spec: &compute.Environment{ProjectEnvironment: "../pyproject.toml"}},
+				{Spec: &compute.Environment{ProjectEnvironment: "/Workspace/shared/pyproject.toml"}},
 			},
 		},
 	}
@@ -126,7 +128,10 @@ func TestVisitJobPaths_environments(t *testing.T) {
 	}
 
 	actual := collectVisitedPaths(t, root, VisitJobPaths)
-	var expected []dyn.Path
+	expected := []dyn.Path{
+		dyn.MustPathFromString("resources.jobs.job0.environments[1].spec.project_environment"),
+		dyn.MustPathFromString("resources.jobs.job0.environments[2].spec.project_environment"),
+	}
 
 	assert.ElementsMatch(t, expected, actual)
 }