diff --git a/.nextchanges/bundles/project-environment-paths.md b/.nextchanges/bundles/project-environment-paths.md new file mode 100644 index 00000000000..85c2d0108aa --- /dev/null +++ b/.nextchanges/bundles/project-environment-paths.md @@ -0,0 +1 @@ +* Resolve job `project_environment` paths relative to the YAML file that defines them. ([#6909](https://github.com/databricks/cli/pull/6909)) diff --git a/acceptance/bundle/environments/project_environment/databricks.yml b/acceptance/bundle/environments/project_environment/databricks.yml new file mode 100644 index 00000000000..03565b8a553 --- /dev/null +++ b/acceptance/bundle/environments/project_environment/databricks.yml @@ -0,0 +1,5 @@ +bundle: + name: project_environment + +include: + - resources/*.yml diff --git a/acceptance/bundle/environments/project_environment/out.test.toml b/acceptance/bundle/environments/project_environment/out.test.toml new file mode 100644 index 00000000000..59b56a2037c --- /dev/null +++ b/acceptance/bundle/environments/project_environment/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/environments/project_environment/output.txt b/acceptance/bundle/environments/project_environment/output.txt new file mode 100644 index 00000000000..6d3e465e0ed --- /dev/null +++ b/acceptance/bundle/environments/project_environment/output.txt @@ -0,0 +1,77 @@ + +>>> [CLI] bundle validate -o json +{ + "tasks": [ + { + "environment_key": "project", + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/src/notebook" + }, + "task_key": "notebook" + } + ], + "environments": [ + { + "environment_key": "project", + "spec": { + "project_environment": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/pyproject.toml" + } + } + ] +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/project_environment/default/files... +Created jobs.notebook +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.2/jobs/create +{ + "tasks": [ + { + "environment_key": "project", + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/src/notebook" + }, + "task_key": "notebook" + } + ], + "environments": [ + { + "environment_key": "project", + "spec": { + "project_environment": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/pyproject.toml" + } + } + ] +} + +>>> update_file.py resources/job.yml ../pyproject.toml ./pyproject.toml + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/project_environment/default/files... +Updated jobs.notebook +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.2/jobs/reset +{ + "tasks": [ + { + "environment_key": "project", + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/src/notebook" + }, + "task_key": "notebook" + } + ], + "environments": [ + { + "environment_key": "project", + "spec": { + "project_environment": "/Workspace/Users/[USERNAME]/.bundle/project_environment/default/files/resources/pyproject.toml" + } + } + ] +} diff --git a/acceptance/bundle/environments/project_environment/pyproject.toml b/acceptance/bundle/environments/project_environment/pyproject.toml new file mode 100644 index 00000000000..b22de93e858 --- /dev/null +++ b/acceptance/bundle/environments/project_environment/pyproject.toml @@ -0,0 +1,6 @@ +[project] +name = "project-environment-test" +version = "0.1.0" + +[tool.databricks.environment] +environment_version = "4" diff --git a/acceptance/bundle/environments/project_environment/resources/job.yml b/acceptance/bundle/environments/project_environment/resources/job.yml new file mode 100644 index 00000000000..bce7abe7c65 --- /dev/null +++ b/acceptance/bundle/environments/project_environment/resources/job.yml @@ -0,0 +1,13 @@ +resources: + jobs: + notebook: + name: project_environment + tasks: + - task_key: notebook + notebook_task: + notebook_path: ../src/notebook.py + environment_key: project + environments: + - environment_key: project + spec: + project_environment: ../pyproject.toml diff --git a/acceptance/bundle/environments/project_environment/resources/pyproject.toml b/acceptance/bundle/environments/project_environment/resources/pyproject.toml new file mode 100644 index 00000000000..d2854958830 --- /dev/null +++ b/acceptance/bundle/environments/project_environment/resources/pyproject.toml @@ -0,0 +1,6 @@ +[project] +name = "updated-project-environment-test" +version = "0.1.0" + +[tool.databricks.environment] +environment_version = "4" diff --git a/acceptance/bundle/environments/project_environment/script b/acceptance/bundle/environments/project_environment/script new file mode 100644 index 00000000000..440ab89e91e --- /dev/null +++ b/acceptance/bundle/environments/project_environment/script @@ -0,0 +1,8 @@ +trace $CLI bundle validate -o json | jq '.resources.jobs.notebook | {tasks, environments}' + +trace $CLI bundle deploy +trace print_requests.py //api/2.2/jobs/create | jq '.body | {tasks, environments}' + +trace update_file.py resources/job.yml ../pyproject.toml ./pyproject.toml +trace $CLI bundle deploy +trace print_requests.py //api/2.2/jobs/reset | jq '.body.new_settings | {tasks, environments}' diff --git a/acceptance/bundle/environments/project_environment/src/notebook.py b/acceptance/bundle/environments/project_environment/src/notebook.py new file mode 100644 index 00000000000..5bee9c9eae0 --- /dev/null +++ b/acceptance/bundle/environments/project_environment/src/notebook.py @@ -0,0 +1,2 @@ +# Databricks notebook source +print("Project environment") diff --git a/acceptance/bundle/environments/project_environment/test.toml b/acceptance/bundle/environments/project_environment/test.toml new file mode 100644 index 00000000000..1670d61e0f6 --- /dev/null +++ b/acceptance/bundle/environments/project_environment/test.toml @@ -0,0 +1,3 @@ +# The pinned Terraform provider does not support project_environment. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +RecordRequests = true diff --git a/bundle/config/mutator/paths/job_paths_visitor.go b/bundle/config/mutator/paths/job_paths_visitor.go index cb400112b7e..e37e34849bd 100644 --- a/bundle/config/mutator/paths/job_paths_visitor.go +++ b/bundle/config/mutator/paths/job_paths_visitor.go @@ -58,18 +58,16 @@ func jobTaskRewritePatterns(base dyn.Pattern) []jobRewritePattern { } func jobRewritePatterns() []jobRewritePattern { - // Base pattern to match all tasks in all jobs. - base := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("tasks"), - dyn.AnyIndex(), - ) + job := dyn.NewPattern(dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey()) + task := job.Append(dyn.Key("tasks"), dyn.AnyIndex()) - taskPatterns := jobTaskRewritePatterns(base) - forEachPatterns := jobTaskRewritePatterns(base.Append(dyn.Key("for_each_task"), dyn.Key("task"))) - return append(taskPatterns, forEachPatterns...) + patterns := jobTaskRewritePatterns(task) + patterns = append(patterns, jobTaskRewritePatterns(task.Append(dyn.Key("for_each_task"), dyn.Key("task")))...) + return append(patterns, jobRewritePattern{ + job.Append(dyn.Key("environments"), dyn.AnyIndex(), dyn.Key("spec"), dyn.Key("project_environment")), + TranslateModeFile, + noSkipRewrite, + }) } // VisitJobPaths visits all paths in job resources and applies a function to each path. diff --git a/bundle/config/mutator/paths/job_paths_visitor_test.go b/bundle/config/mutator/paths/job_paths_visitor_test.go index 7584737b1c0..d7bd3674133 100644 --- a/bundle/config/mutator/paths/job_paths_visitor_test.go +++ b/bundle/config/mutator/paths/job_paths_visitor_test.go @@ -113,6 +113,8 @@ func TestVisitJobPaths_environments(t *testing.T) { JobSettings: jobs.JobSettings{ Environments: []jobs.JobEnvironment{ environment0, + {Spec: &compute.Environment{ProjectEnvironment: "../pyproject.toml"}}, + {Spec: &compute.Environment{ProjectEnvironment: "/Workspace/shared/pyproject.toml"}}, }, }, } @@ -126,7 +128,10 @@ func TestVisitJobPaths_environments(t *testing.T) { } actual := collectVisitedPaths(t, root, VisitJobPaths) - var expected []dyn.Path + expected := []dyn.Path{ + dyn.MustPathFromString("resources.jobs.job0.environments[1].spec.project_environment"), + dyn.MustPathFromString("resources.jobs.job0.environments[2].spec.project_environment"), + } assert.ElementsMatch(t, expected, actual) } diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index 226a848b723..e237be3be81 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -48,6 +48,97 @@ func touchEmptyFile(t *testing.T, path string) { f.Close() } +func TestTranslatePathsProjectEnvironment(t *testing.T) { + for _, tc := range []struct { + name string + path string + localFile string + want string + wantErr string + gitSource bool + sourceLinked bool + }{ + { + name: "YAML directory", path: "./pyproject.toml", localFile: "resources/pyproject.toml", + want: "/Workspace/bundle/files/resources/pyproject.toml", + }, + { + name: "YAML parent", path: "../pyproject.toml", localFile: "pyproject.toml", + want: "/Workspace/bundle/files/pyproject.toml", + }, + { + name: "bare filename", path: "pyproject.toml", localFile: "resources/pyproject.toml", + want: "/Workspace/bundle/files/resources/pyproject.toml", + }, + { + name: "path with spaces", path: "../env files/pyproject.toml", localFile: "env files/pyproject.toml", + want: "/Workspace/bundle/files/env files/pyproject.toml", + }, + { + name: "absolute workspace path", path: "/Workspace/shared/pyproject.toml", + want: "/Workspace/shared/pyproject.toml", + }, + { + name: "source-linked deployment", path: "../pyproject.toml", localFile: "pyproject.toml", + want: "pyproject.toml", sourceLinked: true, + }, + { + name: "git source", path: "../pyproject.toml", + want: "../pyproject.toml", gitSource: true, + }, + { + name: "missing file", path: "../pyproject.toml", + wantErr: "file pyproject.toml not found", + }, + { + name: "outside sync root", path: "../../pyproject.toml", + wantErr: "is not contained in sync root path", + }, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + if tc.localFile != "" { + touchEmptyFile(t, filepath.Join(dir, tc.localFile)) + } + job := &resources.Job{ + JobSettings: jobs.JobSettings{ + Environments: []jobs.JobEnvironment{{ + EnvironmentKey: "project", + Spec: &compute.Environment{ProjectEnvironment: tc.path}, + }}, + }, + } + if tc.gitSource { + job.GitSource = &jobs.GitSource{GitUrl: "https://example.test/repo.git"} + } + b := &bundle.Bundle{ + BundleRootPath: dir, + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), + Config: config.Root{ + Workspace: config.Workspace{FilePath: "/Workspace/bundle/files"}, + Resources: config.Resources{Jobs: map[string]*resources.Job{"job": job}}, + Presets: config.Presets{SourceLinkedDeployment: &tc.sourceLinked}, + }, + } + locations := []dyn.Location{{File: filepath.Join(dir, "resources", "job.yml")}} + bundletest.SetLocation(b, ".", locations) + diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) + err := diags.Error() + if tc.wantErr != "" { + require.ErrorContains(t, err, tc.wantErr) + return + } + require.NoError(t, err) + want := tc.want + if tc.sourceLinked { + want = filepath.ToSlash(filepath.Join(dir, want)) + } + assert.Equal(t, want, b.Config.Resources.Jobs["job"].Environments[0].Spec.ProjectEnvironment) + }) + } +} + func TestTranslatePathsSkippedWithGitSource(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{