diff --git a/.nextchanges/bundles/project-environment-paths.md b/.nextchanges/bundles/project-environment-paths.md new file mode 100644 index 00000000000..2f1c406fee0 --- /dev/null +++ b/.nextchanges/bundles/project-environment-paths.md @@ -0,0 +1 @@ +* Resolve job `project_environment` paths relative to the YAML file that defines them. ([#6983](https://github.com/databricks/cli/pull/6983)) diff --git a/acceptance/bundle/environments/project_environment/databricks.yml b/acceptance/bundle/environments/project_environment/databricks.yml new file mode 100644 index 00000000000..03565b8a553 --- /dev/null +++ b/acceptance/bundle/environments/project_environment/databricks.yml @@ -0,0 +1,5 @@ +bundle: + name: project_environment + +include: + - resources/*.yml diff --git a/acceptance/bundle/environments/project_environment/out.test.toml b/acceptance/bundle/environments/project_environment/out.test.toml new file mode 100644 index 00000000000..59b56a2037c --- /dev/null +++ b/acceptance/bundle/environments/project_environment/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/environments/project_environment/output.txt b/acceptance/bundle/environments/project_environment/output.txt new file mode 100644 index 00000000000..a0090b9d8c3 --- /dev/null +++ b/acceptance/bundle/environments/project_environment/output.txt @@ -0,0 +1,32 @@ + +>>> [CLI] bundle validate -o json +{ + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/src/notebook", + "project_environment": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/pyproject.toml" +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/project_environment/default/files... +Created jobs.notebook +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.2/jobs/create +{ + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/src/notebook", + "project_environment": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/pyproject.toml" +} + +>>> update_file.py resources/job.yml ../pyproject.toml ./pyproject.toml + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/project_environment/default/files... +Updated jobs.notebook +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.2/jobs/reset +{ + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/src/notebook", + "project_environment": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/resources/pyproject.toml" +} diff --git a/acceptance/bundle/environments/project_environment/pyproject.toml b/acceptance/bundle/environments/project_environment/pyproject.toml new file mode 100644 index 00000000000..b22de93e858 --- /dev/null +++ b/acceptance/bundle/environments/project_environment/pyproject.toml @@ -0,0 +1,6 @@ +[project] +name = "project-environment-test" +version = "0.1.0" + +[tool.databricks.environment] +environment_version = "4" diff --git a/acceptance/bundle/environments/project_environment/resources/job.yml b/acceptance/bundle/environments/project_environment/resources/job.yml new file mode 100644 index 00000000000..bce7abe7c65 --- /dev/null +++ b/acceptance/bundle/environments/project_environment/resources/job.yml @@ -0,0 +1,13 @@ +resources: + jobs: + notebook: + name: project_environment + tasks: + - task_key: notebook + notebook_task: + notebook_path: ../src/notebook.py + environment_key: project + environments: + - environment_key: project + spec: + project_environment: ../pyproject.toml diff --git a/acceptance/bundle/environments/project_environment/resources/pyproject.toml b/acceptance/bundle/environments/project_environment/resources/pyproject.toml new file mode 100644 index 00000000000..d2854958830 --- /dev/null +++ b/acceptance/bundle/environments/project_environment/resources/pyproject.toml @@ -0,0 +1,6 @@ +[project] +name = "updated-project-environment-test" +version = "0.1.0" + +[tool.databricks.environment] +environment_version = "4" diff --git a/acceptance/bundle/environments/project_environment/script b/acceptance/bundle/environments/project_environment/script new file mode 100644 index 00000000000..9b5a185900c --- /dev/null +++ b/acceptance/bundle/environments/project_environment/script @@ -0,0 +1,15 @@ +job_paths() { + jq '{ + notebook_path: .tasks[].notebook_task.notebook_path, + project_environment: .environments[].spec.project_environment + }' +} + +trace $CLI bundle validate -o json | jq '.resources.jobs.notebook' | job_paths + +trace $CLI bundle deploy +trace print_requests.py //api/2.2/jobs/create | jq '.body' | job_paths + +trace update_file.py resources/job.yml ../pyproject.toml ./pyproject.toml +trace $CLI bundle deploy +trace print_requests.py //api/2.2/jobs/reset | jq '.body.new_settings' | job_paths diff --git a/acceptance/bundle/environments/project_environment/src/notebook.py b/acceptance/bundle/environments/project_environment/src/notebook.py new file mode 100644 index 00000000000..5bee9c9eae0 --- /dev/null +++ b/acceptance/bundle/environments/project_environment/src/notebook.py @@ -0,0 +1,2 @@ +# Databricks notebook source +print("Project environment") diff --git a/acceptance/bundle/environments/project_environment/test.toml b/acceptance/bundle/environments/project_environment/test.toml new file mode 100644 index 00000000000..ba0016a753f --- /dev/null +++ b/acceptance/bundle/environments/project_environment/test.toml @@ -0,0 +1,2 @@ +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +RecordRequests = true diff --git a/bundle/config/mutator/paths/job_paths_visitor.go b/bundle/config/mutator/paths/job_paths_visitor.go index 6e82e544f02..a3abd0090a4 100644 --- a/bundle/config/mutator/paths/job_paths_visitor.go +++ b/bundle/config/mutator/paths/job_paths_visitor.go @@ -58,7 +58,7 @@ func jobTaskRewritePatterns(base dyn.Pattern) []jobRewritePattern { } func jobRewritePatterns() []jobRewritePattern { - // Base pattern to match all tasks in all jobs. + // Task-level base pattern for all jobs. base := dyn.NewPattern( dyn.Key("resources"), dyn.Key("jobs"), @@ -85,6 +85,20 @@ func jobRewritePatterns() []jobRewritePattern { TranslateModeFile, noSkipRewrite, }, + // Environment-level pattern for all jobs. + jobRewritePattern{ + dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("jobs"), + dyn.AnyKey(), + dyn.Key("environments"), + dyn.AnyIndex(), + dyn.Key("spec"), + dyn.Key("project_environment"), + ), + TranslateModeFile, + noSkipRewrite, + }, ) } diff --git a/bundle/config/mutator/paths/job_paths_visitor_test.go b/bundle/config/mutator/paths/job_paths_visitor_test.go index 23fb6408267..e27cdc185eb 100644 --- a/bundle/config/mutator/paths/job_paths_visitor_test.go +++ b/bundle/config/mutator/paths/job_paths_visitor_test.go @@ -113,6 +113,8 @@ func TestVisitJobPaths_environments(t *testing.T) { JobSettings: jobs.JobSettings{ Environments: []jobs.JobEnvironment{ environment0, + {Spec: &compute.Environment{ProjectEnvironment: "../pyproject.toml"}}, + {Spec: &compute.Environment{ProjectEnvironment: "/Workspace/shared/pyproject.toml"}}, }, }, } @@ -126,7 +128,10 @@ func TestVisitJobPaths_environments(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitJobPaths) - var expected []dyn.Path + expected := []dyn.Path{ + dyn.MustPathFromString("resources.jobs.job0.environments[1].spec.project_environment"), + dyn.MustPathFromString("resources.jobs.job0.environments[2].spec.project_environment"), + } assert.ElementsMatch(t, expected, actual) }