diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 617903e04da..5fa69b1d069 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -918,9 +918,10 @@ func runTest(t *testing.T, args := []string{"bash", "-euo", "pipefail", EntryPointScript} cmd := exec.CommandContext(ctx, args[0], args[1:]...) - cfg, user := internal.PrepareServerAndClient(t, config, LogRequests, outputDir, testEnv) + cfg, user, workspaceID := internal.PrepareServerAndClient(t, config, LogRequests, outputDir, testEnv) testdiff.PrepareReplacementsUser(t, &repls, user) testdiff.PrepareReplacementsWorkspaceConfig(t, &repls, cfg) + repls.Set(workspaceID, "[WORKSPACE_ID]") cmd.Env = auth.ProcessEnv(cfg) diff --git a/acceptance/air/cancel/output.txt b/acceptance/air/cancel/output.txt index b30dee255aa..58a33d92bea 100644 --- a/acceptance/air/cancel/output.txt +++ b/acceptance/air/cancel/output.txt @@ -24,6 +24,6 @@ Successfully requested cancellation for 2 run(s). === cancel --all >>> [CLI] air cancel --all -y Searching active runs for [USERNAME] in [DATABRICKS_URL]... -Successfully requested cancellation for run [NUMID] -Successfully requested cancellation for run [NUMID] +Successfully requested cancellation for run 334747067049496 +Successfully requested cancellation for run 566001814929041 Successfully requested cancellation for 2 run(s). diff --git a/acceptance/air/get-ai-runtime/output.txt b/acceptance/air/get-ai-runtime/output.txt index 900871ab6ef..096b127a6f1 100644 --- a/acceptance/air/get-ai-runtime/output.txt +++ b/acceptance/air/get-ai-runtime/output.txt @@ -31,7 +31,7 @@ │ │ ╰────────────────────────────────────────────────────────────────╯ -Run URL: [DATABRICKS_URL]/jobs/runs/123?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID] MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 === get (json) @@ -46,7 +46,7 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 "duration_seconds": 12, "attempt_number": 0, "experiment_name": "my-exp", - "dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[NUMID]", + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]", "mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/artifacts/logs/node_0", "est_remaining_seconds": null, "est_percent_complete": null diff --git a/acceptance/air/get/output.txt b/acceptance/air/get/output.txt index 7d5625108b8..2a2809cf255 100644 --- a/acceptance/air/get/output.txt +++ b/acceptance/air/get/output.txt @@ -31,7 +31,7 @@ │ │ ╰────────────────────────────────────────────────────────────────╯ -Run URL: [DATABRICKS_URL]/jobs/runs/123?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID] MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 === get (json) @@ -46,7 +46,7 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 "duration_seconds": 12, "attempt_number": 0, "experiment_name": "my-exp", - "dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[NUMID]", + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]", "mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/artifacts/logs/node_0", "est_remaining_seconds": null, "est_percent_complete": null diff --git a/acceptance/air/help/output.txt b/acceptance/air/help/output.txt index d9264f396fb..d95166bf83d 100644 --- a/acceptance/air/help/output.txt +++ b/acceptance/air/help/output.txt @@ -168,9 +168,9 @@ Usage: databricks air logs JOB_RUN_ID [flags] Examples: - databricks air logs [NUMID] - databricks air logs [NUMID] --node 1 --retry 0 - databricks air logs [NUMID] --download-to ./logs + databricks air logs 123456789 + databricks air logs 123456789 --node 1 --retry 0 + databricks air logs 123456789 --download-to ./logs Flags: --download-to string Download all logs to this directory instead of printing diff --git a/acceptance/air/image-push/output.txt b/acceptance/air/image-push/output.txt index 2d1a10b3ac7..03c49f36d40 100644 --- a/acceptance/air/image-push/output.txt +++ b/acceptance/air/image-push/output.txt @@ -3,12 +3,12 @@ >>> databricks air images push -p docker-test -s nvidia/cuda:13.4.1 --catalog my-catalog --schema my-schema --artifact cuda:13.4.1-cudnn-devel-ubuntu26.04 Warning: This feature is in Preview. APIs may change, and the workspace must enable the Preview features. Pushing nvidia/cuda:13.4.1 as my-catalog.my-schema.cuda:13.4.1-cudnn-devel-ubuntu26.04 using profile docker-test -Configured Docker credential helper for [NUMID].container.us-west-2.localhost +Configured Docker credential helper for [WORKSPACE_ID].container.us-west-2.localhost Updated Docker config: [TEST_TMP_DIR]/home/docker/config.json Installed Docker credential helper: [TEST_TMP_DIR]/home/bin/docker-credential-databricks Using local source image nvidia/cuda:13.4.1 -Tagging image as [NUMID].container.us-west-2.localhost/my-catalog.my-schema.cuda:13.4.1-cudnn-devel-ubuntu26.04 -Pushing image to [NUMID].container.us-west-2.localhost/my-catalog.my-schema.cuda:13.4.1-cudnn-devel-ubuntu26.04 +Tagging image as [WORKSPACE_ID].container.us-west-2.localhost/my-catalog.my-schema.cuda:13.4.1-cudnn-devel-ubuntu26.04 +Pushing image to [WORKSPACE_ID].container.us-west-2.localhost/my-catalog.my-schema.cuda:13.4.1-cudnn-devel-ubuntu26.04 Docker push progress Pushed image. Now you can set `environment.unity_catalog_image = my-catalog.my-schema.cuda:13.4.1-cudnn-devel-ubuntu26.04` in your YAML config and create a workload with `databricks air run -f config.yaml`. @@ -31,20 +31,20 @@ Now you can set `environment.unity_catalog_image = my-catalog.my-schema.cuda:13. === docker commands image ls --quiet nvidia/cuda:13.4.1 image inspect --format {{.Os}}/{{.Architecture}} nvidia/cuda:13.4.1 -tag nvidia/cuda:13.4.1 [NUMID].container.us-west-2.localhost/my-catalog.my-schema.cuda:13.4.1-cudnn-devel-ubuntu26.04 -push [NUMID].container.us-west-2.localhost/my-catalog.my-schema.cuda:13.4.1-cudnn-devel-ubuntu26.04 +tag nvidia/cuda:13.4.1 [WORKSPACE_ID].container.us-west-2.localhost/my-catalog.my-schema.cuda:13.4.1-cudnn-devel-ubuntu26.04 +push [WORKSPACE_ID].container.us-west-2.localhost/my-catalog.my-schema.cuda:13.4.1-cudnn-devel-ubuntu26.04 === missing image is pulled for AIR; destination defaults to source >>> databricks air images push -p docker-test -s example/image:v1 --catalog main --schema training Warning: This feature is in Preview. APIs may change, and the workspace must enable the Preview features. Pushing example/image:v1 as main.training.image:v1 using profile docker-test -Configured Docker credential helper for [NUMID].container.us-west-2.localhost +Configured Docker credential helper for [WORKSPACE_ID].container.us-west-2.localhost Updated Docker config: [TEST_TMP_DIR]/home/docker/config.json Installed Docker credential helper: [TEST_TMP_DIR]/home/bin/docker-credential-databricks Pulling source image example/image:v1 for linux/amd64 Docker pull progress -Tagging image as [NUMID].container.us-west-2.localhost/main.training.image:v1 -Pushing image to [NUMID].container.us-west-2.localhost/main.training.image:v1 +Tagging image as [WORKSPACE_ID].container.us-west-2.localhost/main.training.image:v1 +Pushing image to [WORKSPACE_ID].container.us-west-2.localhost/main.training.image:v1 Docker push progress Pushed image. Now you can set `environment.unity_catalog_image = main.training.image:v1` in your YAML config and create a workload with `databricks air run -f config.yaml`. @@ -53,13 +53,13 @@ Now you can set `environment.unity_catalog_image = main.training.image:v1` in yo >>> databricks air images push -p docker-test -s example/image:v1 --catalog main --schema training --pull -o json Warning: This feature is in Preview. APIs may change, and the workspace must enable the Preview features. Pushing example/image:v1 as main.training.image:v1 using profile docker-test -Configured Docker credential helper for [NUMID].container.us-west-2.localhost +Configured Docker credential helper for [WORKSPACE_ID].container.us-west-2.localhost Updated Docker config: [TEST_TMP_DIR]/home/docker/config.json Installed Docker credential helper: [TEST_TMP_DIR]/home/bin/docker-credential-databricks Pulling source image example/image:v1 for linux/amd64 Docker pull progress -Tagging image as [NUMID].container.us-west-2.localhost/main.training.image:v1 -Pushing image to [NUMID].container.us-west-2.localhost/main.training.image:v1 +Tagging image as [WORKSPACE_ID].container.us-west-2.localhost/main.training.image:v1 +Pushing image to [WORKSPACE_ID].container.us-west-2.localhost/main.training.image:v1 Docker push progress Pushed image. @@ -68,7 +68,7 @@ Pushed image. "v": 1, "data": { "unity_catalog_image": "main.training.image:v1", - "registry_image": "[NUMID].container.us-west-2.localhost/main.training.image:v1" + "registry_image": "[WORKSPACE_ID].container.us-west-2.localhost/main.training.image:v1" } } @@ -76,7 +76,7 @@ Pushed image. >>> musterr databricks air images push -p docker-test -s example/image:v1 --catalog main --schema training Warning: This feature is in Preview. APIs may change, and the workspace must enable the Preview features. Pushing example/image:v1 as main.training.image:v1 using profile docker-test -Configured Docker credential helper for [NUMID].container.us-west-2.localhost +Configured Docker credential helper for [WORKSPACE_ID].container.us-west-2.localhost Updated Docker config: [TEST_TMP_DIR]/home/docker/config.json Installed Docker credential helper: [TEST_TMP_DIR]/home/bin/docker-credential-databricks Cannot connect to the Docker daemon @@ -86,7 +86,7 @@ Error: check local source image example/image:v1 (ensure Docker is running and a >>> musterr databricks air images push -p docker-test -s example/image:v1 --catalog main --schema training Warning: This feature is in Preview. APIs may change, and the workspace must enable the Preview features. Pushing example/image:v1 as main.training.image:v1 using profile docker-test -Configured Docker credential helper for [NUMID].container.us-west-2.localhost +Configured Docker credential helper for [WORKSPACE_ID].container.us-west-2.localhost Updated Docker config: [TEST_TMP_DIR]/home/docker/config.json Installed Docker credential helper: [TEST_TMP_DIR]/home/bin/docker-credential-databricks Using local source image example/image:v1 @@ -97,7 +97,7 @@ Error: inspect source image example/image:v1: exit status 1 >>> musterr databricks air images push -p docker-test -s example/image:v1 --catalog main --schema training Warning: This feature is in Preview. APIs may change, and the workspace must enable the Preview features. Pushing example/image:v1 as main.training.image:v1 using profile docker-test -Configured Docker credential helper for [NUMID].container.us-west-2.localhost +Configured Docker credential helper for [WORKSPACE_ID].container.us-west-2.localhost Updated Docker config: [TEST_TMP_DIR]/home/docker/config.json Installed Docker credential helper: [TEST_TMP_DIR]/home/bin/docker-credential-databricks Using local source image example/image:v1 @@ -107,7 +107,7 @@ Error: source image example/image:v1 has platform "linux/arm64"; AIR requires li >>> musterr databricks air images push -p docker-test -s example/image:v1 --catalog main --schema training --pull Warning: This feature is in Preview. APIs may change, and the workspace must enable the Preview features. Pushing example/image:v1 as main.training.image:v1 using profile docker-test -Configured Docker credential helper for [NUMID].container.us-west-2.localhost +Configured Docker credential helper for [WORKSPACE_ID].container.us-west-2.localhost Updated Docker config: [TEST_TMP_DIR]/home/docker/config.json Installed Docker credential helper: [TEST_TMP_DIR]/home/bin/docker-credential-databricks Pulling source image example/image:v1 for linux/amd64 @@ -118,7 +118,7 @@ Error: source image example/image:v1 has platform "linux/arm64"; AIR requires li >>> musterr databricks air images push -p docker-test -s example/image:v1 --catalog main --schema training --pull Warning: This feature is in Preview. APIs may change, and the workspace must enable the Preview features. Pushing example/image:v1 as main.training.image:v1 using profile docker-test -Configured Docker credential helper for [NUMID].container.us-west-2.localhost +Configured Docker credential helper for [WORKSPACE_ID].container.us-west-2.localhost Updated Docker config: [TEST_TMP_DIR]/home/docker/config.json Installed Docker credential helper: [TEST_TMP_DIR]/home/bin/docker-credential-databricks Pulling source image example/image:v1 for linux/amd64 @@ -184,17 +184,17 @@ Error: invalid image repository name: catalog.schema.artifact must be 255 charac === all docker commands image ls --quiet nvidia/cuda:13.4.1 image inspect --format {{.Os}}/{{.Architecture}} nvidia/cuda:13.4.1 -tag nvidia/cuda:13.4.1 [NUMID].container.us-west-2.localhost/my-catalog.my-schema.cuda:13.4.1-cudnn-devel-ubuntu26.04 -push [NUMID].container.us-west-2.localhost/my-catalog.my-schema.cuda:13.4.1-cudnn-devel-ubuntu26.04 +tag nvidia/cuda:13.4.1 [WORKSPACE_ID].container.us-west-2.localhost/my-catalog.my-schema.cuda:13.4.1-cudnn-devel-ubuntu26.04 +push [WORKSPACE_ID].container.us-west-2.localhost/my-catalog.my-schema.cuda:13.4.1-cudnn-devel-ubuntu26.04 image ls --quiet example/image:v1 pull --platform linux/amd64 example/image:v1 image inspect --format {{.Os}}/{{.Architecture}} example/image:v1 -tag example/image:v1 [NUMID].container.us-west-2.localhost/main.training.image:v1 -push [NUMID].container.us-west-2.localhost/main.training.image:v1 +tag example/image:v1 [WORKSPACE_ID].container.us-west-2.localhost/main.training.image:v1 +push [WORKSPACE_ID].container.us-west-2.localhost/main.training.image:v1 pull --platform linux/amd64 example/image:v1 image inspect --format {{.Os}}/{{.Architecture}} example/image:v1 -tag example/image:v1 [NUMID].container.us-west-2.localhost/main.training.image:v1 -push [NUMID].container.us-west-2.localhost/main.training.image:v1 +tag example/image:v1 [WORKSPACE_ID].container.us-west-2.localhost/main.training.image:v1 +push [WORKSPACE_ID].container.us-west-2.localhost/main.training.image:v1 image ls --quiet example/image:v1 image ls --quiet example/image:v1 image inspect --format {{.Os}}/{{.Architecture}} example/image:v1 diff --git a/acceptance/air/list/output.txt b/acceptance/air/list/output.txt index 10889d0b2d6..9dfa0b22f24 100644 --- a/acceptance/air/list/output.txt +++ b/acceptance/air/list/output.txt @@ -2,7 +2,7 @@ === list (text) >>> [CLI] air list Run ID Experiment Status Started Duration Progress MLflow User Accelerators - [NUMID] qwen-train ● SUCCESS [TIMESTAMP] 12s - qwen-train-001 [USERNAME] 8x H100 + 334747067049496 qwen-train ● SUCCESS [TIMESTAMP] 12s - qwen-train-001 [USERNAME] 8x H100 === list (json) >>> [CLI] air list -o json @@ -12,7 +12,7 @@ "data": { "runs": [ { - "run_id": "[NUMID]", + "run_id": "334747067049496", "run_name": "qwen-train", "user": "[USERNAME]", "status": "SUCCESS", @@ -26,7 +26,7 @@ === list --all-status (text, via AiTrainingService index) >>> [CLI] air list --all-status Run ID Experiment Status Started Duration Progress MLflow User Accelerators - [NUMID] qwen-train ● SUCCESS [TIMESTAMP] 12s - qwen-train-001 [USERNAME] 8x H100 + 334747067049496 qwen-train ● SUCCESS [TIMESTAMP] 12s - qwen-train-001 [USERNAME] 8x H100 === list --all-status (json) >>> [CLI] air list --all-status -o json @@ -36,7 +36,7 @@ "data": { "runs": [ { - "run_id": "[NUMID]", + "run_id": "334747067049496", "run_name": "qwen-train", "user": "[USERNAME]", "status": "SUCCESS", diff --git a/acceptance/bin/repls.py b/acceptance/bin/repls.py index 1fffe2c949f..338127cf949 100644 --- a/acceptance/bin/repls.py +++ b/acceptance/bin/repls.py @@ -14,7 +14,7 @@ from pathlib import Path # Order of the replacements added by the scripts, so that they are applied before the ones -# from the harness: a job id must become [MY_JOB] rather than [NUMID]. +# from the harness: a job id must become [MY_JOB] even if a broader harness pattern matches it. USER_ORDER = -100 diff --git a/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.json b/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.json index dd31e902e64..9b81021ae27 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.json +++ b/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.json @@ -95,7 +95,7 @@ } ], "format": "MULTI_TASK", - "job_id": [NUMID], + "job_id": [TEST_JOB_ID], "max_concurrent_runs": 1, "name": "[default] My Wheel Job", "queue": { diff --git a/acceptance/bundle/artifacts/whl_dynamic/script b/acceptance/bundle/artifacts/whl_dynamic/script index 243fc65c164..b73202a65ec 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/script +++ b/acceptance/bundle/artifacts/whl_dynamic/script @@ -9,6 +9,7 @@ trace $CLI bundle validate -o json | nostamp | jq .artifacts $CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan nostamp < tmp.plan.json > out.plan_create.json $CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.deploy +replace_ids.py cat LOG.deploy | contains.py "patched_wheels/my_test_code_my_test_code/my_test_code-0.0.1+" "patched_wheels/my_prebuilt_whl_other_test_code/other_test_code-0.0.1+" > /dev/null title "There are 2 original wheels and 2 patched ones" diff --git a/acceptance/bundle/bundle_tag/id/output.txt b/acceptance/bundle/bundle_tag/id/output.txt index df1ac878cac..427b059bfb0 100644 --- a/acceptance/bundle/bundle_tag/id/output.txt +++ b/acceptance/bundle/bundle_tag/id/output.txt @@ -68,7 +68,7 @@ Resources: Jobs: foo: Name: Untitled - URL: [DATABRICKS_URL]/jobs/[NUMID]?w=[NUMID] + URL: [DATABRICKS_URL]/jobs/[FOO_ID]?w=[WORKSPACE_ID] >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: @@ -83,6 +83,6 @@ Destroy: 1 deleted "method": "POST", "path": "/api/2.2/jobs/delete", "body": { - "job_id": [NUMID] + "job_id": [FOO_ID] } } diff --git a/acceptance/bundle/bundle_tag/id/script b/acceptance/bundle/bundle_tag/id/script index 759cc89eba1..cb86bbae429 100644 --- a/acceptance/bundle/bundle_tag/id/script +++ b/acceptance/bundle/bundle_tag/id/script @@ -2,6 +2,7 @@ trace $CLI bundle validate trace $CLI bundle validate -o json | nostamp | jq .resources trace $CLI bundle plan trace $CLI bundle deploy +replace_ids.py trace print_requests.py --nostamp //jobs trace $CLI bundle summary trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/bundle_tag/url/output.txt b/acceptance/bundle/bundle_tag/url/output.txt index b217e3b6946..6bcd3f040e4 100644 --- a/acceptance/bundle/bundle_tag/url/output.txt +++ b/acceptance/bundle/bundle_tag/url/output.txt @@ -68,7 +68,7 @@ Resources: Jobs: foo: Name: Untitled - URL: [DATABRICKS_URL]/jobs/[NUMID]?w=[NUMID] + URL: [DATABRICKS_URL]/jobs/[FOO_ID]?w=[WORKSPACE_ID] >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: @@ -83,6 +83,6 @@ Destroy: 1 deleted "method": "POST", "path": "/api/2.2/jobs/delete", "body": { - "job_id": [NUMID] + "job_id": [FOO_ID] } } diff --git a/acceptance/bundle/bundle_tag/url/script b/acceptance/bundle/bundle_tag/url/script index 759cc89eba1..cb86bbae429 100644 --- a/acceptance/bundle/bundle_tag/url/script +++ b/acceptance/bundle/bundle_tag/url/script @@ -2,6 +2,7 @@ trace $CLI bundle validate trace $CLI bundle validate -o json | nostamp | jq .resources trace $CLI bundle plan trace $CLI bundle deploy +replace_ids.py trace print_requests.py --nostamp //jobs trace $CLI bundle summary trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/config-remote-sync/json_status_selectors/output.txt b/acceptance/bundle/config-remote-sync/json_status_selectors/output.txt index 2059b95c814..2b13979059e 100644 --- a/acceptance/bundle/config-remote-sync/json_status_selectors/output.txt +++ b/acceptance/bundle/config-remote-sync/json_status_selectors/output.txt @@ -14,7 +14,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === A malformed selector is reported as failed, exit 0 { "status": "failed", - "error": "invalid --select-ids value \"no-such-id-999\", expected \u003ctype\u003e:\u003cid\u003e (e.g. jobs:[NUMID])", + "error": "invalid --select-ids value \"no-such-id-999\", expected \u003ctype\u003e:\u003cid\u003e (e.g. jobs:123456789)", "files": null, "changes": {} } diff --git a/acceptance/bundle/config-remote-sync/select_basic/output.txt b/acceptance/bundle/config-remote-sync/select_basic/output.txt index d4cacb53c24..efc4894ccd5 100644 --- a/acceptance/bundle/config-remote-sync/select_basic/output.txt +++ b/acceptance/bundle/config-remote-sync/select_basic/output.txt @@ -73,7 +73,7 @@ Exit code: 1 === A selector without a type is still rejected >>> [CLI] bundle config-remote-sync --select-ids no-such-id-123 -Error: invalid --select-ids value "no-such-id-123", expected : (e.g. jobs:[NUMID]) +Error: invalid --select-ids value "no-such-id-123", expected : (e.g. jobs:123456789) Exit code: 1 diff --git a/acceptance/bundle/debug/fetch-repository-info-repos-error/output.txt b/acceptance/bundle/debug/fetch-repository-info-repos-error/output.txt index bdd6157ad68..3f8cdf11fc4 100644 --- a/acceptance/bundle/debug/fetch-repository-info-repos-error/output.txt +++ b/acceptance/bundle/debug/fetch-repository-info-repos-error/output.txt @@ -1,6 +1,6 @@ >>> [CLI] bundle debug fetch-repository-info --path /Workspace/Users/[USERNAME]/test-folder-[UNIQUE_NAME] -Warn: failed to load git info for repo [NUMID]: Fault injected by test. +Warn: failed to load git info for repo [REPO_ID]: Fault injected by test. { "worktree_root": "/Workspace/Users/[USERNAME]/test-folder-[UNIQUE_NAME]", "current_branch": "", diff --git a/acceptance/bundle/debug/fetch-repository-info-repos-error/script b/acceptance/bundle/debug/fetch-repository-info-repos-error/script index 0716206028a..4bebfdd19fd 100644 --- a/acceptance/bundle/debug/fetch-repository-info-repos-error/script +++ b/acceptance/bundle/debug/fetch-repository-info-repos-error/script @@ -11,6 +11,7 @@ fault.py "GET /api/2.0/repos/*" 403 0 1 export MSYS_NO_PATHCONV=1 folder=/Workspace/Users/$CURRENT_USER_NAME/test-folder-$UNIQUE_NAME -$CLI repos create https://github.com/databricks/bundle-examples gitHub --path "$folder" > /dev/null +repo_id=$($CLI repos create https://github.com/databricks/bundle-examples gitHub --path "$folder" | jq -r .id) +add_repl "$repo_id" REPO_ID trace $CLI bundle debug fetch-repository-info --path "$folder" diff --git a/acceptance/bundle/deploy/experimental-python/output.txt b/acceptance/bundle/deploy/experimental-python/output.txt index f40d99d1d2e..e66bb4ef967 100644 --- a/acceptance/bundle/deploy/experimental-python/output.txt +++ b/acceptance/bundle/deploy/experimental-python/output.txt @@ -10,7 +10,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged { "created_time": [UNIX_TIME_MILLIS], "creator_user_name": "[USERNAME]", - "job_id": [NUMID], + "job_id": [JOB1_ID], "settings": { "deployment": { "kind": "BUNDLE", diff --git a/acceptance/bundle/deploy/experimental-python/script b/acceptance/bundle/deploy/experimental-python/script index 09b90b28e2d..7d4df28d794 100644 --- a/acceptance/bundle/deploy/experimental-python/script +++ b/acceptance/bundle/deploy/experimental-python/script @@ -1,3 +1,6 @@ trace uv run --quiet --with $DATABRICKS_BUNDLES_WHEEL -- $CLI bundle deploy - -trace $CLI jobs list --output json | nostamp +trace $CLI jobs list --output json | tee jobs.json.tmp | nostamp +# Take the id from the listing rather than via replace_ids.py: that needs the bundles wheel +# (uv run), and uv cannot spawn a .py script on Windows. python3, not jq: the id exceeds 2^53. +add_repl "$(python3 -c 'import sys, json; print(json.load(sys.stdin)[0]["job_id"])' < jobs.json.tmp)" JOB1_ID +rm jobs.json.tmp diff --git a/acceptance/bundle/deploy/fail-on-active-runs/output.txt b/acceptance/bundle/deploy/fail-on-active-runs/output.txt index 753fe44e215..91445d88273 100644 --- a/acceptance/bundle/deploy/fail-on-active-runs/output.txt +++ b/acceptance/bundle/deploy/fail-on-active-runs/output.txt @@ -6,7 +6,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> errcode [CLI] bundle deploy --fail-on-active-runs -Error: job [NUMID] is running +Error: job [MY_JOB_ID] is running Exit code: 1 diff --git a/acceptance/bundle/deploy/fail-on-active-runs/script b/acceptance/bundle/deploy/fail-on-active-runs/script index 19e7b09a113..7bfaacef082 100644 --- a/acceptance/bundle/deploy/fail-on-active-runs/script +++ b/acceptance/bundle/deploy/fail-on-active-runs/script @@ -1,5 +1,6 @@ # We deploy the bundle first to create a job trace $CLI bundle deploy +replace_ids.py # We deploy the bundle again to check that the deploy is failing if the job is running trace errcode $CLI bundle deploy --fail-on-active-runs diff --git a/acceptance/bundle/deploy/mlops-stacks/output.txt b/acceptance/bundle/deploy/mlops-stacks/output.txt index 62b2dbcd8f4..1b62cc0093e 100644 --- a/acceptance/bundle/deploy/mlops-stacks/output.txt +++ b/acceptance/bundle/deploy/mlops-stacks/output.txt @@ -103,10 +103,10 @@ Warning: unknown field: description in resources/ml-artifacts-resource.yml:28:7 { - "experiment_id": "[NUMID]", - "model_id": "main.project_name_[UNIQUE_NAME].dev_[USERNAME]_project_name_[UNIQUE_NAME]-model", - "inference_job_id": "[NUMID]", - "training_job_id": "[NUMID]" + "experiment_id": "[EXPERIMENT_ID]", + "model_id": "[MODEL_ID]", + "inference_job_id": "[BATCH_INFERENCE_JOB_ID]", + "training_job_id": "[MODEL_TRAINING_JOB_ID]" } === Assert the batch inference job actually exists diff --git a/acceptance/bundle/deploy/mlops-stacks/script b/acceptance/bundle/deploy/mlops-stacks/script index f69ef7c9fca..a1770334913 100644 --- a/acceptance/bundle/deploy/mlops-stacks/script +++ b/acceptance/bundle/deploy/mlops-stacks/script @@ -43,6 +43,7 @@ cd "test_repo_mlops_stacks/project_name_${UNIQUE_NAME}" || exit 1 trace $CLI bundle summary --var catalog_name=main trace $CLI bundle validate --var catalog_name=main trace $CLI bundle deploy --var catalog_name=main +replace_ids.py trace $CLI bundle summary --var catalog_name=main -o json | jq -r '{experiment_id: .resources.experiments.experiment.id, model_id: .resources.registered_models.model.id, inference_job_id: .resources.jobs.batch_inference_job.id, training_job_id: .resources.jobs.model_training_job.id}' diff --git a/acceptance/bundle/deploy/python-notebook/output.txt b/acceptance/bundle/deploy/python-notebook/output.txt index 5b6e5413596..95a09e78d58 100644 --- a/acceptance/bundle/deploy/python-notebook/output.txt +++ b/acceptance/bundle/deploy/python-notebook/output.txt @@ -10,7 +10,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged { "created_time": [UNIX_TIME_MILLIS], "creator_user_name": "[USERNAME]", - "job_id": [NUMID], + "job_id": [MY_JOB_ID], "settings": { "deployment": { "kind": "BUNDLE", diff --git a/acceptance/bundle/deploy/python-notebook/script b/acceptance/bundle/deploy/python-notebook/script index 21318c58b66..b2b536d41b7 100644 --- a/acceptance/bundle/deploy/python-notebook/script +++ b/acceptance/bundle/deploy/python-notebook/script @@ -1,3 +1,4 @@ trace $CLI bundle deploy +replace_ids.py trace $CLI jobs list --output json | nostamp diff --git a/acceptance/bundle/deploy/readplan/basic/out.plan_skip.json b/acceptance/bundle/deploy/readplan/basic/out.plan_skip.json index c17133f90da..8982add5d75 100644 --- a/acceptance/bundle/deploy/readplan/basic/out.plan_skip.json +++ b/acceptance/bundle/deploy/readplan/basic/out.plan_skip.json @@ -16,7 +16,7 @@ "edit_mode": "UI_LOCKED", "email_notifications": {}, "format": "MULTI_TASK", - "job_id": [NUMID], + "job_id": [JOB_ID], "max_concurrent_runs": 1, "name": "test-job", "queue": { diff --git a/acceptance/bundle/deploy/readplan/basic/output.txt b/acceptance/bundle/deploy/readplan/basic/output.txt index 34e26568727..be446fb1b5a 100644 --- a/acceptance/bundle/deploy/readplan/basic/output.txt +++ b/acceptance/bundle/deploy/readplan/basic/output.txt @@ -35,7 +35,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "method": "GET", "path": "/api/2.2/jobs/get", "q": { - "job_id": "[NUMID]" + "job_id": "[JOB_ID]" } } diff --git a/acceptance/bundle/deploy/readplan/basic/script b/acceptance/bundle/deploy/readplan/basic/script index 690e34fdfeb..db73236d6a6 100644 --- a/acceptance/bundle/deploy/readplan/basic/script +++ b/acceptance/bundle/deploy/readplan/basic/script @@ -5,6 +5,7 @@ trace print_requests.py --get //jobs | nostamp | contains.py "!GET" "!POST" # First deploy to create the resource trace $CLI bundle deploy --plan tmp.plan_create.json +replace_ids.py trace print_requests.py --get //jobs | nostamp | contains.py "!GET" "POST" # Generate a plan (should show "skip" since nothing changed) diff --git a/acceptance/bundle/deploy/spark-jar-task/output.txt b/acceptance/bundle/deploy/spark-jar-task/output.txt index af7f0f5de21..44db681765f 100644 --- a/acceptance/bundle/deploy/spark-jar-task/output.txt +++ b/acceptance/bundle/deploy/spark-jar-task/output.txt @@ -8,7 +8,7 @@ Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle run jar_job -Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[JAR_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "[default] Test Spark Jar Job [UNIQUE_NAME]" RUNNING [TIMESTAMP] "[default] Test Spark Jar Job [UNIQUE_NAME]" TERMINATED SUCCESS diff --git a/acceptance/bundle/deploy/spark-jar-task/script b/acceptance/bundle/deploy/spark-jar-task/script index d736ca6bfae..3243e4767fc 100644 --- a/acceptance/bundle/deploy/spark-jar-task/script +++ b/acceptance/bundle/deploy/spark-jar-task/script @@ -1,4 +1,5 @@ envsubst < databricks.yml.tmpl > databricks.yml trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT trace $CLI bundle deploy +replace_ids.py trace $CLI bundle run jar_job diff --git a/acceptance/bundle/deploy/spark-jar-task/test.toml b/acceptance/bundle/deploy/spark-jar-task/test.toml index d4f48e2397e..6670a96c8c3 100644 --- a/acceptance/bundle/deploy/spark-jar-task/test.toml +++ b/acceptance/bundle/deploy/spark-jar-task/test.toml @@ -13,3 +13,7 @@ Response.Body = ''' "logs": "Hello from Jar!\n[]" } ''' + +[[Repls]] +Old = '/runs/\d+' +New = '/runs/[RUN_ID]' diff --git a/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt b/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt index 0fea6799b11..6923c5b3b79 100644 --- a/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt +++ b/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt @@ -97,10 +97,10 @@ Resources: Jobs: job_01: Name: job-01 - URL: [DATABRICKS_URL]/jobs/[JOB_01_ID]?w=[NUMID] + URL: [DATABRICKS_URL]/jobs/[JOB_01_ID]?w=[WORKSPACE_ID] job_02: Name: job-02 - URL: [DATABRICKS_URL]/jobs/[JOB_02_ID]?w=[NUMID] + URL: [DATABRICKS_URL]/jobs/[JOB_02_ID]?w=[WORKSPACE_ID] job_03: Name: job-03 URL: (not deployed) diff --git a/acceptance/bundle/deploy/wal/corrupted-wal-entry/output.txt b/acceptance/bundle/deploy/wal/corrupted-wal-entry/output.txt index b8a6fbaac80..5e382cadfce 100644 --- a/acceptance/bundle/deploy/wal/corrupted-wal-entry/output.txt +++ b/acceptance/bundle/deploy/wal/corrupted-wal-entry/output.txt @@ -24,10 +24,10 @@ Resources: Jobs: another_valid: Name: another-valid - URL: [DATABRICKS_URL]/jobs/[NUMID]?w=[NUMID] + URL: [DATABRICKS_URL]/jobs/[ANOTHER_VALID_ID]?w=[WORKSPACE_ID] valid_job: Name: valid-job - URL: [DATABRICKS_URL]/jobs/[NUMID]?w=[NUMID] + URL: [DATABRICKS_URL]/jobs/[VALID_JOB_ID]?w=[WORKSPACE_ID] >>> cat .databricks/bundle/default/resources.json.wal.corrupted {"k":"resources.jobs.partial_write","v":{"__id__":"33","state":{"name":"partial- diff --git a/acceptance/bundle/deploy/wal/corrupted-wal-entry/script b/acceptance/bundle/deploy/wal/corrupted-wal-entry/script index 3245bbdc377..a3584ddb809 100644 --- a/acceptance/bundle/deploy/wal/corrupted-wal-entry/script +++ b/acceptance/bundle/deploy/wal/corrupted-wal-entry/script @@ -11,6 +11,7 @@ envsubst < resources.json.wal.tmpl > .databricks/bundle/default/resources.json.w trace cat .databricks/bundle/default/resources.json.wal trace $CLI bundle deploy +replace_ids.py trace $CLI bundle summary trace cat .databricks/bundle/default/resources.json.wal.corrupted diff --git a/acceptance/bundle/deploy/wal/crash-after-create/output.txt b/acceptance/bundle/deploy/wal/crash-after-create/output.txt index 56822d5fbc0..7c07a54e9ff 100644 --- a/acceptance/bundle/deploy/wal/crash-after-create/output.txt +++ b/acceptance/bundle/deploy/wal/crash-after-create/output.txt @@ -20,7 +20,7 @@ Exit code: [KILLED] { "k": "resources.jobs.job_a", "v": { - "__id__": "[NUMID]", + "__id__": "[JOB_A_ID]", "state": { "deployment": { "kind": "BUNDLE", diff --git a/acceptance/bundle/deploy/wal/crash-after-create/script b/acceptance/bundle/deploy/wal/crash-after-create/script index 9dc54d806ec..f43b5378392 100644 --- a/acceptance/bundle/deploy/wal/crash-after-create/script +++ b/acceptance/bundle/deploy/wal/crash-after-create/script @@ -18,3 +18,4 @@ $CLI bundle $COMMAND &> LOG.COMMAND.txt trace assert_exists.py .databricks/bundle/default/resources.json trace assert_not_exists.py .databricks/bundle/default/resources.json.wal +replace_ids.py diff --git a/acceptance/bundle/deploy/wal/failed-plan-no-wal/output.txt b/acceptance/bundle/deploy/wal/failed-plan-no-wal/output.txt index 8cdc0e7c4c3..f1b616e23af 100644 --- a/acceptance/bundle/deploy/wal/failed-plan-no-wal/output.txt +++ b/acceptance/bundle/deploy/wal/failed-plan-no-wal/output.txt @@ -9,9 +9,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Deploy 2 (planning fails, must not leave a WAL) >>> errcode [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Error: cannot plan resources.jobs.test_job: reading id="[NUMID]": Fault injected by test. (403 INJECTED) +Error: cannot plan resources.jobs.test_job: reading id="[TEST_JOB_ID]": Fault injected by test. (403 INJECTED) -Endpoint: GET [DATABRICKS_URL]/api/2.2/jobs/get?job_id=[NUMID] +Endpoint: GET [DATABRICKS_URL]/api/2.2/jobs/get?job_id=[TEST_JOB_ID] HTTP Status: 403 Forbidden API error_code: INJECTED API message: Fault injected by test. @@ -27,9 +27,9 @@ Exit code: 1 === Deploy 3 (planning fails again, must not leave a WAL) >>> errcode [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Error: cannot plan resources.jobs.test_job: reading id="[NUMID]": Fault injected by test. (403 INJECTED) +Error: cannot plan resources.jobs.test_job: reading id="[TEST_JOB_ID]": Fault injected by test. (403 INJECTED) -Endpoint: GET [DATABRICKS_URL]/api/2.2/jobs/get?job_id=[NUMID] +Endpoint: GET [DATABRICKS_URL]/api/2.2/jobs/get?job_id=[TEST_JOB_ID] HTTP Status: 403 Forbidden API error_code: INJECTED API message: Fault injected by test. diff --git a/acceptance/bundle/deploy/wal/failed-plan-no-wal/script b/acceptance/bundle/deploy/wal/failed-plan-no-wal/script index b4fd4878627..381d8d647bc 100644 --- a/acceptance/bundle/deploy/wal/failed-plan-no-wal/script +++ b/acceptance/bundle/deploy/wal/failed-plan-no-wal/script @@ -13,6 +13,7 @@ title "Deploy 1 (normal: creates the job and the committed state)" trace $CLI bundle deploy +replace_ids.py # Fail the plan-stage refresh GET for the next two deploys only. fault.py "GET /api/2.2/jobs/get" 403 0 2 diff --git a/acceptance/bundle/deployment/bind/alert/test.toml b/acceptance/bundle/deployment/bind/alert/test.toml index b826f7af71d..abf4db36f6f 100644 --- a/acceptance/bundle/deployment/bind/alert/test.toml +++ b/acceptance/bundle/deployment/bind/alert/test.toml @@ -14,6 +14,6 @@ Ignore = [ # the case for the local testserver (127.0.0.1) and aws hosts; azure/gcp vanity hosts # embed the ID and omit it. Stripping it lets one golden match local and all clouds. [[Repls]] -Old = '\?[ow]=\d+' +Old = '\?[ow]=\[WORKSPACE_ID\]' New = '' -Order = 0 +Order = 1000 diff --git a/acceptance/bundle/deployment/bind/database_instance/output.txt b/acceptance/bundle/deployment/bind/database_instance/output.txt index f147c6eff1e..08a22ce7d59 100644 --- a/acceptance/bundle/deployment/bind/database_instance/output.txt +++ b/acceptance/bundle/deployment/bind/database_instance/output.txt @@ -13,7 +13,7 @@ Resources: Database instances: database_instance1: Name: test-instance-02 - URL: [DATABRICKS_URL]/compute/database-instances/test-instance-02?w=[NUMID] + URL: [DATABRICKS_URL]/compute/database-instances/test-instance-02?w=[WORKSPACE_ID] >>> [CLI] bundle deployment unbind database_instance1 diff --git a/acceptance/bundle/deployment/bind/experiment/output.txt b/acceptance/bundle/deployment/bind/experiment/output.txt index 1273ca405fb..639a001fe61 100644 --- a/acceptance/bundle/deployment/bind/experiment/output.txt +++ b/acceptance/bundle/deployment/bind/experiment/output.txt @@ -1,6 +1,6 @@ ->>> [CLI] bundle deployment bind experiment1 [NUMID] --auto-approve -Successfully bound experiment with an id '[NUMID]' +>>> [CLI] bundle deployment bind experiment1 [EXPERIMENT_ID] --auto-approve +Successfully bound experiment with an id '[EXPERIMENT_ID]' Run 'bundle deploy' to deploy changes to your workspace >>> [CLI] bundle deploy --force-lock --auto-approve @@ -8,7 +8,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQU Files: 2 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged ->>> [CLI] experiments get-experiment [NUMID] +>>> [CLI] experiments get-experiment [EXPERIMENT_ID] { "name": "/Users/[USERNAME]/test-experiment[UNIQUE_NAME]", "lifecycle_stage": "active" @@ -21,7 +21,7 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 0 deleted ->>> [CLI] experiments get-experiment [NUMID] +>>> [CLI] experiments get-experiment [EXPERIMENT_ID] { "name": "/Users/[USERNAME]/test-experiment[UNIQUE_NAME]", "lifecycle_stage": "active" diff --git a/acceptance/bundle/deployment/bind/experiment/script b/acceptance/bundle/deployment/bind/experiment/script index b2b41897e48..75eb9c590ce 100644 --- a/acceptance/bundle/deployment/bind/experiment/script +++ b/acceptance/bundle/deployment/bind/experiment/script @@ -4,6 +4,7 @@ export EXPERIMENT_NAME envsubst < databricks.yml.tmpl > databricks.yml EXPERIMENT_ID=$($CLI experiments create-experiment "${EXPERIMENT_NAME}" | jq -r '.experiment_id') +add_repl "$EXPERIMENT_ID" EXPERIMENT_ID cleanupRemoveExperiment() { title "Test cleanup:" diff --git a/acceptance/bundle/deployment/bind/job/engine-from-config/output.txt b/acceptance/bundle/deployment/bind/job/engine-from-config/output.txt index 8a91a9165d5..1c81c5d1ccc 100644 --- a/acceptance/bundle/deployment/bind/job/engine-from-config/output.txt +++ b/acceptance/bundle/deployment/bind/job/engine-from-config/output.txt @@ -11,8 +11,8 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] jobs create --json {"name": "My Job"} ->>> [CLI] bundle deployment bind job_1 [NUMID] --auto-approve -Successfully bound job with an id '[NUMID]' +>>> [CLI] bundle deployment bind job_1 [JOB_ID] --auto-approve +Successfully bound job with an id '[JOB_ID]' Run 'bundle deploy' to deploy changes to your workspace === State files after bind === .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/deployment/bind/job/engine-from-config/script b/acceptance/bundle/deployment/bind/job/engine-from-config/script index 62c0b6f7c97..7f62df189d7 100644 --- a/acceptance/bundle/deployment/bind/job/engine-from-config/script +++ b/acceptance/bundle/deployment/bind/job/engine-from-config/script @@ -15,6 +15,7 @@ trace $CLI bundle deployment unbind job_1 # Create a new job to bind to job_id=$(trace $CLI jobs create --json '{"name": "My Job"}' | jq -r '.job_id') +add_repl "$job_id" JOB_ID # Bind uses direct engine (reads bundle.engine from config). trace $CLI bundle deployment bind job_1 $job_id --auto-approve diff --git a/acceptance/bundle/deployment/bind/job/generate-and-bind/output.txt b/acceptance/bundle/deployment/bind/job/generate-and-bind/output.txt index a5be42b8953..c80c5d84598 100644 --- a/acceptance/bundle/deployment/bind/job/generate-and-bind/output.txt +++ b/acceptance/bundle/deployment/bind/job/generate-and-bind/output.txt @@ -1,12 +1,12 @@ === Create a pre-defined job: -Created job with ID: [NUMID] +Created job with ID: [JOB_ID] >>> [CLI] workspace mkdirs /Workspace/Users/[USERNAME]/python-[UNIQUE_NAME] >>> [CLI] workspace import /Workspace/Users/[USERNAME]/python-[UNIQUE_NAME]/test --file test.py --language PYTHON ->>> [CLI] bundle generate job --key test_job_key --existing-job-id [NUMID] --config-dir resources --source-dir src +>>> [CLI] bundle generate job --key test_job_key --existing-job-id [JOB_ID] --config-dir resources --source-dir src File successfully saved to src/test.py Job configuration successfully saved to resources/test_job_key.job.yml @@ -16,8 +16,8 @@ test.py >>> cat resources/test_job_key.job.yml name: generate-job-[UNIQUE_NAME] ->>> [CLI] bundle deployment bind test_job_key [NUMID] --auto-approve -Successfully bound job with an id '[NUMID]' +>>> [CLI] bundle deployment bind test_job_key [JOB_ID] --auto-approve +Successfully bound job with an id '[JOB_ID]' Run 'bundle deploy' to deploy changes to your workspace >>> [CLI] bundle deploy @@ -35,8 +35,8 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 1 deleted === Check that job is bound and does not exist after bundle is destroyed: ->>> errcode [CLI] jobs get [NUMID] --output json -Error: Job [NUMID] does not exist. +>>> errcode [CLI] jobs get [JOB_ID] --output json +Error: Job [JOB_ID] does not exist. Exit code: 1 diff --git a/acceptance/bundle/deployment/bind/job/generate-and-bind/script b/acceptance/bundle/deployment/bind/job/generate-and-bind/script index a1f6e2c34e2..a049d45a9b8 100644 --- a/acceptance/bundle/deployment/bind/job/generate-and-bind/script +++ b/acceptance/bundle/deployment/bind/job/generate-and-bind/script @@ -26,6 +26,7 @@ JOB_ID=$($CLI jobs create --json ' ] }' | jq -r '.job_id') +add_repl "$JOB_ID" JOB_ID echo "Created job with ID: $JOB_ID" envsubst < databricks.yml.tmpl > databricks.yml diff --git a/acceptance/bundle/deployment/bind/job/job-spark-python-task/output.txt b/acceptance/bundle/deployment/bind/job/job-spark-python-task/output.txt index ad9b242ac44..994bd66bd38 100644 --- a/acceptance/bundle/deployment/bind/job/job-spark-python-task/output.txt +++ b/acceptance/bundle/deployment/bind/job/job-spark-python-task/output.txt @@ -1,10 +1,10 @@ === Create a pre-defined job: -Created job with ID: [NUMID] +Created job with ID: [JOB_ID] === Bind job: ->>> [CLI] bundle deployment bind foo [NUMID] --auto-approve -Successfully bound job with an id '[NUMID]' +>>> [CLI] bundle deployment bind foo [JOB_ID] --auto-approve +Successfully bound job with an id '[JOB_ID]' Run 'bundle deploy' to deploy changes to your workspace === Remove .databricks directory to simulate fresh deployment: @@ -18,9 +18,9 @@ Files: 3 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === Read the pre-defined job: ->>> [CLI] jobs get [NUMID] +>>> [CLI] jobs get [JOB_ID] { - "job_id": [NUMID], + "job_id": [JOB_ID], "settings": { "name": "test-job-basic-[UNIQUE_NAME]", "tasks": [ @@ -47,9 +47,9 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 0 deleted === Read the pre-defined job again (expecting it still exists): ->>> [CLI] jobs get [NUMID] +>>> [CLI] jobs get [JOB_ID] { - "job_id": [NUMID], + "job_id": [JOB_ID], "settings": { "name": "test-job-basic-[UNIQUE_NAME]", "tasks": [ @@ -63,4 +63,4 @@ Destroy: 0 deleted } } -=== Delete the pre-defined job [NUMID]:0 +=== Delete the pre-defined job [JOB_ID]:0 diff --git a/acceptance/bundle/deployment/bind/job/job-spark-python-task/script b/acceptance/bundle/deployment/bind/job/job-spark-python-task/script index c0831899416..8562dab5647 100644 --- a/acceptance/bundle/deployment/bind/job/job-spark-python-task/script +++ b/acceptance/bundle/deployment/bind/job/job-spark-python-task/script @@ -19,6 +19,7 @@ JOB_ID=$($CLI jobs create --json ' } ] }' | jq -r '.job_id') +add_repl "$JOB_ID" JOB_ID echo "Created job with ID: $JOB_ID" diff --git a/acceptance/bundle/deployment/bind/job/noop-job/out.job.json b/acceptance/bundle/deployment/bind/job/noop-job/out.job.json index 80509b893bd..8eb7900d9f9 100644 --- a/acceptance/bundle/deployment/bind/job/noop-job/out.job.json +++ b/acceptance/bundle/deployment/bind/job/noop-job/out.job.json @@ -1,7 +1,7 @@ { "created_time": [UNIX_TIME_MILLIS], "creator_user_name": "[USERNAME]", - "job_id": [NUMID], + "job_id": [JOB_ID], "run_as_user_name": "[USERNAME]", "settings": { "deployment": { diff --git a/acceptance/bundle/deployment/bind/job/noop-job/output.txt b/acceptance/bundle/deployment/bind/job/noop-job/output.txt index c067058fe07..956f90977b4 100644 --- a/acceptance/bundle/deployment/bind/job/noop-job/output.txt +++ b/acceptance/bundle/deployment/bind/job/noop-job/output.txt @@ -1,8 +1,8 @@ >>> [CLI] jobs create --json {"name": "My Job"} ->>> [CLI] bundle deployment bind job_1 [NUMID] --auto-approve -Successfully bound job with an id '[NUMID]' +>>> [CLI] bundle deployment bind job_1 [JOB_ID] --auto-approve +Successfully bound job with an id '[JOB_ID]' Run 'bundle deploy' to deploy changes to your workspace >>> [CLI] bundle deploy @@ -11,4 +11,4 @@ Updated jobs.job_1 Files: 1 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged ->>> [CLI] jobs get [NUMID] --output json +>>> [CLI] jobs get [JOB_ID] --output json diff --git a/acceptance/bundle/deployment/bind/job/noop-job/script b/acceptance/bundle/deployment/bind/job/noop-job/script index c4689cd32b1..325a38f4726 100644 --- a/acceptance/bundle/deployment/bind/job/noop-job/script +++ b/acceptance/bundle/deployment/bind/job/noop-job/script @@ -1,4 +1,5 @@ job_id=$(trace $CLI jobs create --json '{"name": "My Job"}' | jq -r '.job_id') +add_repl "$job_id" JOB_ID trace $CLI bundle deployment bind job_1 $job_id --auto-approve diff --git a/acceptance/bundle/deployment/bind/job/python-job/out.job.json b/acceptance/bundle/deployment/bind/job/python-job/out.job.json index 80509b893bd..8eb7900d9f9 100644 --- a/acceptance/bundle/deployment/bind/job/python-job/out.job.json +++ b/acceptance/bundle/deployment/bind/job/python-job/out.job.json @@ -1,7 +1,7 @@ { "created_time": [UNIX_TIME_MILLIS], "creator_user_name": "[USERNAME]", - "job_id": [NUMID], + "job_id": [JOB_ID], "run_as_user_name": "[USERNAME]", "settings": { "deployment": { diff --git a/acceptance/bundle/deployment/bind/job/python-job/output.txt b/acceptance/bundle/deployment/bind/job/python-job/output.txt index e61fbd01226..22cb66acf04 100644 --- a/acceptance/bundle/deployment/bind/job/python-job/output.txt +++ b/acceptance/bundle/deployment/bind/job/python-job/output.txt @@ -1,8 +1,8 @@ >>> [CLI] jobs create --json {"name": "My Job"} ->>> uv run --with [DATABRICKS_BUNDLES_WHEEL] -q [CLI] bundle deployment bind job_1 [NUMID] --auto-approve -Successfully bound job with an id '[NUMID]' +>>> uv run --with [DATABRICKS_BUNDLES_WHEEL] -q [CLI] bundle deployment bind job_1 [JOB_ID] --auto-approve +Successfully bound job with an id '[JOB_ID]' Run 'bundle deploy' to deploy changes to your workspace >>> uv run --with [DATABRICKS_BUNDLES_WHEEL] -q [CLI] bundle deploy @@ -11,4 +11,4 @@ Updated jobs.job_1 Files: 2 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged ->>> [CLI] jobs get [NUMID] --output json +>>> [CLI] jobs get [JOB_ID] --output json diff --git a/acceptance/bundle/deployment/bind/job/python-job/script b/acceptance/bundle/deployment/bind/job/python-job/script index b827e22fd14..b915a7243f0 100644 --- a/acceptance/bundle/deployment/bind/job/python-job/script +++ b/acceptance/bundle/deployment/bind/job/python-job/script @@ -1,6 +1,7 @@ UV_RUN="uv run --with $DATABRICKS_BUNDLES_WHEEL -q" job_id=$(trace $CLI jobs create --json '{"name": "My Job"}' | jq -r '.job_id') +add_repl "$job_id" JOB_ID trace $UV_RUN $CLI bundle deployment bind job_1 $job_id --auto-approve diff --git a/acceptance/bundle/deployment/bind/mcp_service/output.txt b/acceptance/bundle/deployment/bind/mcp_service/output.txt index 39aac6934ba..63a9244a93c 100644 --- a/acceptance/bundle/deployment/bind/mcp_service/output.txt +++ b/acceptance/bundle/deployment/bind/mcp_service/output.txt @@ -13,7 +13,7 @@ Resources: MCP services: mcp1: Name: mysvc - URL: [DATABRICKS_URL]/explore/data/mcp-services/main/myschema/mysvc?w=[NUMID] + URL: [DATABRICKS_URL]/explore/data/mcp-services/main/myschema/mysvc?w=[WORKSPACE_ID] >>> [CLI] bundle deployment unbind mcp1 diff --git a/acceptance/bundle/deployment/bind/model_provider_service/output.txt b/acceptance/bundle/deployment/bind/model_provider_service/output.txt index cafdcf3d28b..ebb16ddd3b3 100644 --- a/acceptance/bundle/deployment/bind/model_provider_service/output.txt +++ b/acceptance/bundle/deployment/bind/model_provider_service/output.txt @@ -13,7 +13,7 @@ Resources: Model provider services: mps1: Name: mysvc - URL: [DATABRICKS_URL]/explore/data/model-provider-services/main/myschema/mysvc?w=[NUMID] + URL: [DATABRICKS_URL]/explore/data/model-provider-services/main/myschema/mysvc?w=[WORKSPACE_ID] >>> [CLI] bundle deployment unbind mps1 diff --git a/acceptance/bundle/deployment/bind/model_service/output.txt b/acceptance/bundle/deployment/bind/model_service/output.txt index bd402a482c2..7e49db6c1ea 100644 --- a/acceptance/bundle/deployment/bind/model_service/output.txt +++ b/acceptance/bundle/deployment/bind/model_service/output.txt @@ -13,7 +13,7 @@ Resources: Model services: ms1: Name: mysvc - URL: [DATABRICKS_URL]/explore/data/model-services/main/myschema/mysvc?w=[NUMID] + URL: [DATABRICKS_URL]/explore/data/model-services/main/myschema/mysvc?w=[WORKSPACE_ID] >>> [CLI] bundle deployment unbind ms1 diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/out.summary.json b/acceptance/bundle/deployment/bind/pipelines/recreate/out.summary.json index e67ca2304dc..6db8ad09cf9 100644 --- a/acceptance/bundle/deployment/bind/pipelines/recreate/out.summary.json +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/out.summary.json @@ -17,7 +17,7 @@ ], "name": "test-pipeline-[UNIQUE_NAME]", "storage": "/Shared/new_storage", - "url": "[DATABRICKS_URL]/pipelines/[NEW_PIPELINE_ID]?w=[NUMID]" + "url": "[DATABRICKS_URL]/pipelines/[NEW_PIPELINE_ID]?w=[WORKSPACE_ID]" } } } diff --git a/acceptance/bundle/deployment/bind/pipelines/update/out.summary.json b/acceptance/bundle/deployment/bind/pipelines/update/out.summary.json index fce55e66364..6d538af432a 100644 --- a/acceptance/bundle/deployment/bind/pipelines/update/out.summary.json +++ b/acceptance/bundle/deployment/bind/pipelines/update/out.summary.json @@ -26,7 +26,7 @@ } ], "name": "test-pipeline", - "url": "[DATABRICKS_URL]/pipelines/[NEW_PIPELINE_ID]?w=[NUMID]" + "url": "[DATABRICKS_URL]/pipelines/[NEW_PIPELINE_ID]?w=[WORKSPACE_ID]" } } }, diff --git a/acceptance/bundle/deployment/bind/sql_warehouse/output.txt b/acceptance/bundle/deployment/bind/sql_warehouse/output.txt index 858dec32105..60c3fe5f6b2 100644 --- a/acceptance/bundle/deployment/bind/sql_warehouse/output.txt +++ b/acceptance/bundle/deployment/bind/sql_warehouse/output.txt @@ -13,7 +13,7 @@ Resources: SQL Warehouses: sql_warehouse1: Name: DEFAULT Test SQL Warehouse - URL: [DATABRICKS_URL]/sql/warehouses/[SQL-WAREHOUSE-ID]?w=[NUMID] + URL: [DATABRICKS_URL]/sql/warehouses/[SQL-WAREHOUSE-ID]?w=[WORKSPACE_ID] >>> [CLI] bundle deployment unbind sql_warehouse1 diff --git a/acceptance/bundle/deployment/unbind/job/output.txt b/acceptance/bundle/deployment/unbind/job/output.txt index 93513dd6a38..0654ebfff39 100644 --- a/acceptance/bundle/deployment/unbind/job/output.txt +++ b/acceptance/bundle/deployment/unbind/job/output.txt @@ -5,8 +5,6 @@ Created jobs.job_1 Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> [CLI] bundle summary --var job_name=My Job --output json - >>> [CLI] bundle deployment unbind job_1 --var job_name=My Job >>> [CLI] bundle deploy --var job_name=Updated Job @@ -15,11 +13,11 @@ Created jobs.job_1 Files: 0 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> [CLI] jobs get [NUMID] --output json +>>> [CLI] jobs get [JOB_1_ID] --output json { "created_time": [UNIX_TIME_MILLIS], "creator_user_name": "[USERNAME]", - "job_id": [NUMID], + "job_id": [JOB_1_ID], "run_as_user_name": "[USERNAME]", "settings": { "deployment": { diff --git a/acceptance/bundle/deployment/unbind/job/script b/acceptance/bundle/deployment/unbind/job/script index 59ccd32d894..061aadcfabf 100644 --- a/acceptance/bundle/deployment/unbind/job/script +++ b/acceptance/bundle/deployment/unbind/job/script @@ -1,6 +1,6 @@ trace $CLI bundle deploy --var "job_name=My Job" -job_id=$(trace $CLI bundle summary --var "job_name=My Job" --output json | jq -r '.resources.jobs.job_1.id') +job_id=$(read_id.py job_1) trace $CLI bundle deployment unbind job_1 --var "job_name=My Job" diff --git a/acceptance/bundle/deployment/unbind/permissions/output.txt b/acceptance/bundle/deployment/unbind/permissions/output.txt index be6aa9f35b5..4c7e4bde615 100644 --- a/acceptance/bundle/deployment/unbind/permissions/output.txt +++ b/acceptance/bundle/deployment/unbind/permissions/output.txt @@ -6,9 +6,7 @@ Created jobs.job_1.permissions Files: 2 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged ->>> [CLI] bundle summary --output json - ->>> [CLI] jobs get-permissions [NUMID] --output json +>>> [CLI] jobs get-permissions [JOB_1_ID] --output json { "all_permissions": [ { @@ -29,7 +27,7 @@ Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Permissions should be the same as before unbind ->>> [CLI] jobs get-permissions [NUMID] --output json +>>> [CLI] jobs get-permissions [JOB_1_ID] --output json { "all_permissions": [ { @@ -48,5 +46,5 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 1 deleted ->>> [CLI] jobs delete [NUMID] +>>> [CLI] jobs delete [JOB_1_ID] 0 diff --git a/acceptance/bundle/deployment/unbind/permissions/script b/acceptance/bundle/deployment/unbind/permissions/script index 078e3c3af6b..c8cd6e4e6ac 100644 --- a/acceptance/bundle/deployment/unbind/permissions/script +++ b/acceptance/bundle/deployment/unbind/permissions/script @@ -8,7 +8,7 @@ cleanup() { trap cleanup EXIT trace $CLI bundle deploy -job_id=$(trace $CLI bundle summary --output json | jq -r '.resources.jobs.job_1.id') +job_id=$(read_id.py job_1) trace $CLI jobs get-permissions $job_id --output json | jq '.access_control_list[] | select(.group_name == "users")' trace $CLI bundle deployment unbind job_1 diff --git a/acceptance/bundle/deployment/unbind/python-job/output.txt b/acceptance/bundle/deployment/unbind/python-job/output.txt index b66dd78b0ef..ae85cdeb4b7 100644 --- a/acceptance/bundle/deployment/unbind/python-job/output.txt +++ b/acceptance/bundle/deployment/unbind/python-job/output.txt @@ -5,8 +5,6 @@ Created jobs.job_1 Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> uv run --with [DATABRICKS_BUNDLES_WHEEL] -q [CLI] bundle summary --var job_name=My Job --output json - >>> uv run --with [DATABRICKS_BUNDLES_WHEEL] -q [CLI] bundle deployment unbind job_1 --var job_name=My Job >>> uv run --with [DATABRICKS_BUNDLES_WHEEL] -q [CLI] bundle deploy --var job_name=Updated Job @@ -15,11 +13,11 @@ Created jobs.job_1 Files: 0 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> [CLI] jobs get [NUMID] --output json +>>> [CLI] jobs get [JOB_1_ID] --output json { "created_time": [UNIX_TIME_MILLIS], "creator_user_name": "[USERNAME]", - "job_id": [NUMID], + "job_id": [JOB_1_ID], "run_as_user_name": "[USERNAME]", "settings": { "deployment": { diff --git a/acceptance/bundle/deployment/unbind/python-job/script b/acceptance/bundle/deployment/unbind/python-job/script index 84d3a2616fb..c3b4dba3417 100644 --- a/acceptance/bundle/deployment/unbind/python-job/script +++ b/acceptance/bundle/deployment/unbind/python-job/script @@ -2,7 +2,7 @@ UV_RUN="uv run --with $DATABRICKS_BUNDLES_WHEEL -q" trace $UV_RUN $CLI bundle deploy --var "job_name=My Job" -job_id=$(trace $UV_RUN $CLI bundle summary --var "job_name=My Job" --output json | jq -r '.resources.jobs.job_1.id') +job_id=$(read_id.py job_1) trace $UV_RUN $CLI bundle deployment unbind job_1 --var "job_name=My Job" diff --git a/acceptance/bundle/destroy/jobs-and-pipeline/output.txt b/acceptance/bundle/destroy/jobs-and-pipeline/output.txt index 7946483f010..ac68178a52b 100644 --- a/acceptance/bundle/destroy/jobs-and-pipeline/output.txt +++ b/acceptance/bundle/destroy/jobs-and-pipeline/output.txt @@ -68,7 +68,7 @@ Error: The specified pipeline [UUID] was not found. Exit code: 1 === Assert job is deleted: -Error: Job [NUMID] does not exist. +Error: Job [FOO_ID] does not exist. Exit code: 1 diff --git a/acceptance/bundle/destroy/jobs-and-pipeline/script b/acceptance/bundle/destroy/jobs-and-pipeline/script index 95b093e851e..3d958e0c4a2 100644 --- a/acceptance/bundle/destroy/jobs-and-pipeline/script +++ b/acceptance/bundle/destroy/jobs-and-pipeline/script @@ -47,5 +47,5 @@ PIPELINE_ID=$($CLI bundle summary -o json | jq -r '.resources.pipelines.bar.id') trace $CLI pipelines get "${PIPELINE_ID}" | nostamp | jq "{spec}" title "Assert the job is created:\n" -JOB_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.foo.id') +JOB_ID=$(read_id.py foo) $CLI jobs get "${JOB_ID}" | jq '{name: .settings.name}' diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index eefd538e632..e1f52e355ff 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -22,7 +22,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": { "version_id": "1" }, @@ -39,7 +39,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } { "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/schemas.foo", "q": { "update_mask": "state,error_message,resource_id,status" }, @@ -53,7 +53,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } @@ -77,19 +77,19 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> print_requests.py --dms --get //api/2.0/bundle { "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]" + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]" } { "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]/resources" + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources" } { "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]" + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]" } { "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]/resources" + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources" } >>> print_state.py --no-dms @@ -121,15 +121,15 @@ Files: 2 uploaded, 0 deleted >>> print_requests.py --dms --get //api/2.0/bundle { "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]" + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]" } { "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]/resources" + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources" } === The recorded resources are unchanged by the decline: the schema is still at version 1 ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { "last_action_type": "OPERATION_ACTION_TYPE_CREATE", @@ -156,19 +156,19 @@ false >>> print_requests.py --dms --get //api/2.0/bundle { "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]/resources" + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources" } { "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]" + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]" } { "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]/resources" + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources" } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": { "version_id": "2" }, @@ -186,7 +186,7 @@ false } { "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/operations/schemas.foo", "q": { "update_mask": "state,error_message,status" }, @@ -198,12 +198,12 @@ false } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } } { "method": "DELETE", - "path": "/api/2.0/bundle/deployments/[NUMID]" + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]" } diff --git a/acceptance/bundle/dms/declined-deploy/script b/acceptance/bundle/dms/declined-deploy/script index de24d4c764b..e0fa60f0763 100644 --- a/acceptance/bundle/dms/declined-deploy/script +++ b/acceptance/bundle/dms/declined-deploy/script @@ -25,6 +25,7 @@ trace print_requests.py --dms --get //api/2.0/bundle title "The recorded resources are unchanged by the decline: the schema is still at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-declined-deploy-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +add_repl "$deployment_id" DEPLOYMENT_ID trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index 7f5d4327e92..d25afa59255 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -26,29 +26,29 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py --dms //versions/1/operations { "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.parent", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/jobs.parent", "q": { "update_mask": "state,error_message,resource_id,status" }, "body": { "error_message": "", - "resource_id": "[NUMID]", + "resource_id": "[PARENT_ID]", "sequence_id": 0, - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"parent\",\"queue\":{\"enabled\":true}}}", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[DEPLOYMENT_ID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"parent\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED" } } { "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.child", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/jobs.child", "q": { "update_mask": "state,error_message,resource_id,status" }, "body": { "error_message": "", - "resource_id": "[NUMID]", + "resource_id": "[CHILD_ID]", "sequence_id": 0, - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"description\":\"depends on [NUMID]\",\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"child\",\"queue\":{\"enabled\":true}},\"depends_on\":[{\"node\":\"resources.jobs.parent\",\"label\":\"${resources.jobs.parent.id}\"}]}", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[DEPLOYMENT_ID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"description\":\"depends on [PARENT_ID]\",\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"child\",\"queue\":{\"enabled\":true}},\"depends_on\":[{\"node\":\"resources.jobs.parent\",\"label\":\"${resources.jobs.parent.id}\"}]}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -69,7 +69,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === The service lists both jobs, each recorded at version 1 ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { "last_action_type": "OPERATION_ACTION_TYPE_CREATE", diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script index d795778f2d9..e8a2f2c69e5 100644 --- a/acceptance/bundle/dms/depends-on/script +++ b/acceptance/bundle/dms/depends-on/script @@ -12,10 +12,12 @@ dms_plan title "The service lists both jobs, each recorded at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-depends-on-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +add_repl "$deployment_id" DEPLOYMENT_ID trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state the service holds for this deployment" trace print_state.py --no-dms +replace_ids.py trace $CLI bundle destroy --auto-approve rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/dms/deployment-metadata-change/output.txt b/acceptance/bundle/dms/deployment-metadata-change/output.txt index 2bd9572e5d7..675d23ca70d 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/output.txt +++ b/acceptance/bundle/dms/deployment-metadata-change/output.txt @@ -22,7 +22,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": { "version_id": "1" }, @@ -39,7 +39,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } { "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/schemas.foo", "q": { "update_mask": "state,error_message,resource_id,status" }, @@ -53,7 +53,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } @@ -74,7 +74,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { "last_action_type": "OPERATION_ACTION_TYPE_CREATE", @@ -100,7 +100,7 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged >>> print_requests.py --dms //api/2.0/bundle { "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]", "q": { "update_mask": "display_name,target_name,deployment_mode,workspace_info" }, @@ -141,7 +141,7 @@ A common practice is to use a username or principal name in this path, i.e. use Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { "last_action_type": "OPERATION_ACTION_TYPE_CREATE", @@ -164,7 +164,7 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged >>> print_requests.py --dms //api/2.0/bundle ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { "last_action_type": "OPERATION_ACTION_TYPE_CREATE", diff --git a/acceptance/bundle/dms/deployment-metadata-change/script b/acceptance/bundle/dms/deployment-metadata-change/script index af091b2ce97..af66e148fdb 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/script +++ b/acceptance/bundle/dms/deployment-metadata-change/script @@ -7,6 +7,7 @@ trace print_requests.py --dms //api/2.0/bundle title "The plan records the deployment_history feature; the service holds the schema at version 1" dms_plan deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-metadata-pinned-${UNIQUE_NAME}/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +add_repl "$deployment_id" DEPLOYMENT_ID trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "Rename the bundle and switch target+mode. The state path is pinned, so this is the same deployment, updated in place - the mask carries every field that changed" diff --git a/acceptance/bundle/dms/empty-grants/output.txt b/acceptance/bundle/dms/empty-grants/output.txt index 6ea903f709a..4bf5a7efa43 100644 --- a/acceptance/bundle/dms/empty-grants/output.txt +++ b/acceptance/bundle/dms/empty-grants/output.txt @@ -23,7 +23,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": { "version_id": "1" }, @@ -44,7 +44,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } { "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/schemas.foo", "q": { "update_mask": "state,error_message,resource_id,status" }, @@ -58,7 +58,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } { "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo.grants", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/schemas.foo.grants", "q": { "update_mask": "state,error_message,resource_id,status" }, @@ -72,7 +72,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } @@ -90,7 +90,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged >>> print_requests.py --dms //api/2.0/bundle { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": { "version_id": "2" }, @@ -108,7 +108,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged } { "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo.grants", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/operations/schemas.foo.grants", "q": { "update_mask": "state,error_message,status" }, @@ -120,7 +120,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } @@ -146,7 +146,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The grants node emptied out, so the service no longer lists it - only the schema remains ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { "last_action_type": "OPERATION_ACTION_TYPE_CREATE", diff --git a/acceptance/bundle/dms/empty-grants/script b/acceptance/bundle/dms/empty-grants/script index 23cab279e51..0dc96e90c39 100644 --- a/acceptance/bundle/dms/empty-grants/script +++ b/acceptance/bundle/dms/empty-grants/script @@ -25,6 +25,7 @@ dms_plan title "The grants node emptied out, so the service no longer lists it - only the schema remains" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-empty-grants-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +add_repl "$deployment_id" DEPLOYMENT_ID trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state the service holds for this deployment" diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 0551f63fe66..4057525509b 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -63,9 +63,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py --dms //api/2.0/bundle --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-existing-state-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default"}}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_key": "jobs.one"}], "version_type": "VERSION_TYPE_DEPLOY"}} -{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_key": "jobs.one"}], "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[ONE_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[DEPLOYMENT_ID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} >>> find.py resources[.]json[.]wal --expect 0 diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index 7ec44e4a729..e64ca6febe6 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -29,6 +29,9 @@ trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --oneline trace find.py 'resources[.]json[.]wal' --expect 0 trace jq '{state_version, features}' .databricks/bundle/default/resources.json +DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true replace_ids.py +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-existing-state-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +add_repl "$deployment_id" DEPLOYMENT_ID # The recorded requests are asserted above via print_requests.py; drop whatever the run added after # that (telemetry, and the state probes of the non-recording variant) so nothing is left to compare. diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt index 6d5a6756c37..fd8d9a72e61 100644 --- a/acceptance/bundle/dms/failed-delete/output.txt +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -22,7 +22,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": { "version_id": "1" }, @@ -39,21 +39,21 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } { "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.stuck", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/jobs.stuck", "q": { "update_mask": "state,error_message,resource_id,status" }, "body": { "error_message": "", - "resource_id": "[NUMID]", + "resource_id": "[STUCK_ID]", "sequence_id": 0, - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"stuck\",\"queue\":{\"enabled\":true}}}", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[DEPLOYMENT_ID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"stuck\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED" } } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } @@ -75,7 +75,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === After deploy, the service lists the resource at version 1 ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { "last_action_type": "OPERATION_ACTION_TYPE_CREATE", @@ -105,7 +105,7 @@ The following resources will be deleted: All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default -Error: cannot delete resources.jobs.stuck: deleting id=[NUMID]: Fault injected by test. (400 INVALID_PARAMETER_VALUE) +Error: cannot delete resources.jobs.stuck: deleting id=[STUCK_ID]: Fault injected by test. (400 INVALID_PARAMETER_VALUE) Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/delete HTTP Status: 400 Bad Request @@ -116,7 +116,7 @@ API message: Fault injected by test. >>> print_requests.py --dms //api/2.0/bundle { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": { "version_id": "2" }, @@ -134,28 +134,28 @@ API message: Fault injected by test. } { "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.stuck", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/operations/jobs.stuck", "q": { "update_mask": "state,error_message,resource_id,status" }, "body": { - "error_message": "deleting id=[NUMID]: Fault injected by test. (400 INVALID_PARAMETER_VALUE)", - "resource_id": "[NUMID]", + "error_message": "deleting id=[STUCK_ID]: Fault injected by test. (400 INVALID_PARAMETER_VALUE)", + "resource_id": "[STUCK_ID]", "sequence_id": 0, - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"stuck\",\"queue\":{\"enabled\":true}}}", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[DEPLOYMENT_ID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"stuck\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_FAILED" } } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/complete", "body": { "completion_reason": "VERSION_COMPLETE_FAILURE" } } === After failure, the service shows the operation failed ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { "last_action_type": "OPERATION_ACTION_TYPE_DELETE", @@ -165,11 +165,11 @@ API message: Fault injected by test. ] === The failed operation records the error message ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[NUMID]/versions/2 -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/2 -o json [ { "action_type": "OPERATION_ACTION_TYPE_DELETE", - "error_message": "deleting id=[NUMID]: Fault injected by test. (400 INVALID_PARAMETER_VALUE)", + "error_message": "deleting id=[STUCK_ID]: Fault injected by test. (400 INVALID_PARAMETER_VALUE)", "resource_key": "jobs.stuck", "status": "OPERATION_STATUS_FAILED" } diff --git a/acceptance/bundle/dms/failed-delete/script b/acceptance/bundle/dms/failed-delete/script index fd72c7bd984..438b2925eea 100644 --- a/acceptance/bundle/dms/failed-delete/script +++ b/acceptance/bundle/dms/failed-delete/script @@ -9,10 +9,12 @@ dms_plan title "After deploy, the service lists the resource at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-delete-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +add_repl "$deployment_id" DEPLOYMENT_ID trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw state after successful deploy" trace print_state.py --no-dms +replace_ids.py title "A resource that fails to delete is recorded as a failed operation carrying the error, so the history says why rather than showing the resource gone" # The state entry survives too, so the next deploy still knows about the resource. diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index df298291ce6..ddbd5707b6c 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -22,7 +22,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": { "version_id": "1" }, @@ -39,7 +39,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } { "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/schemas.foo", "q": { "update_mask": "state,error_message,resource_id,status" }, @@ -53,7 +53,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } @@ -75,7 +75,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === After deploy, the service lists the resource at version 1 ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { "last_action_type": "OPERATION_ACTION_TYPE_CREATE", @@ -118,7 +118,7 @@ Files: 2 uploaded, 0 deleted >>> print_requests.py --dms //api/2.0/bundle { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": { "version_id": "2" }, @@ -136,7 +136,7 @@ Files: 2 uploaded, 0 deleted } { "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/operations/schemas.foo", "q": { "update_mask": "state,error_message,status" }, @@ -148,7 +148,7 @@ Files: 2 uploaded, 0 deleted } { "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/operations/schemas.foo", "q": { "update_mask": "state,error_message,status" }, @@ -160,14 +160,14 @@ Files: 2 uploaded, 0 deleted } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/complete", "body": { "completion_reason": "VERSION_COMPLETE_FAILURE" } } === The resource is not listed: state is what projects a resource, and the failed recreate left none ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [] === The raw state after failed recreate shows no resource diff --git a/acceptance/bundle/dms/failed-recreate/script b/acceptance/bundle/dms/failed-recreate/script index 07f802a39b5..60bcc99df10 100644 --- a/acceptance/bundle/dms/failed-recreate/script +++ b/acceptance/bundle/dms/failed-recreate/script @@ -9,6 +9,7 @@ dms_plan title "After deploy, the service lists the resource at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-recreate-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +add_repl "$deployment_id" DEPLOYMENT_ID trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw state after successful deploy" diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt index a45e4a21c1a..c73181e7247 100644 --- a/acceptance/bundle/dms/failed-update-permissions/output.txt +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -23,7 +23,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === The service lists the job at version 1 ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { "last_action_type": "OPERATION_ACTION_TYPE_CREATE", @@ -56,9 +56,9 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> musterr [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-update-permissions-[UNIQUE_NAME]/default/files... -Error: cannot update resources.jobs.foo.permissions: updating id=/jobs/[NUMID]: Fault injected by test. (403 PERMISSION_DENIED) +Error: cannot update resources.jobs.foo.permissions: updating id=/jobs/[FOO_ID]: Fault injected by test. (403 PERMISSION_DENIED) -Endpoint: PUT [DATABRICKS_URL]/api/2.0/permissions/jobs/[NUMID] +Endpoint: PUT [DATABRICKS_URL]/api/2.0/permissions/jobs/[FOO_ID] HTTP Status: 403 Forbidden API error_code: PERMISSION_DENIED API message: Fault injected by test. @@ -66,7 +66,7 @@ API message: Fault injected by test. Files: 1 uploaded, 0 deleted === The service lists the job with the failure ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { "last_action_type": "OPERATION_ACTION_TYPE_CREATE", @@ -81,11 +81,11 @@ Files: 1 uploaded, 0 deleted ] === The failed operation (version 2) records the error message ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[NUMID]/versions/2 -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/2 -o json [ { "action_type": "OPERATION_ACTION_TYPE_UPDATE", - "error_message": "updating id=/jobs/[NUMID]: Fault injected by test. (403 PERMISSION_DENIED)", + "error_message": "updating id=/jobs/[FOO_ID]: Fault injected by test. (403 PERMISSION_DENIED)", "resource_key": "jobs.foo.permissions", "status": "OPERATION_STATUS_FAILED" } @@ -104,6 +104,6 @@ Files: 1 uploaded, 0 deleted } >>> print_requests.py --dms //api/2.0/bundle --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.foo.permissions"}], "previous_version_id": "1", "version_type": "VERSION_TYPE_DEPLOY"}} -{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "updating id=/jobs/[NUMID]: Fault injected by test. (403 PERMISSION_DENIED)", "resource_id": "/jobs/[NUMID]", "sequence_id": 0, "state": "{\"state\":{\"object_id\":\"/jobs/[NUMID]\",\"__embed__\":[{\"level\":\"CAN_VIEW\",\"user_name\":\"viewer@example.com\"},{\"level\":\"IS_OWNER\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.jobs.foo\",\"label\":\"${resources.jobs.foo.id}\"}]}", "status": "OPERATION_STATUS_FAILED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_FAILURE"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.foo.permissions"}], "previous_version_id": "1", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/operations/jobs.foo.permissions", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "updating id=/jobs/[FOO_ID]: Fault injected by test. (403 PERMISSION_DENIED)", "resource_id": "/jobs/[FOO_ID]", "sequence_id": 0, "state": "{\"state\":{\"object_id\":\"/jobs/[FOO_ID]\",\"__embed__\":[{\"level\":\"CAN_VIEW\",\"user_name\":\"viewer@example.com\"},{\"level\":\"IS_OWNER\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.jobs.foo\",\"label\":\"${resources.jobs.foo.id}\"}]}", "status": "OPERATION_STATUS_FAILED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_FAILURE"}} diff --git a/acceptance/bundle/dms/failed-update-permissions/script b/acceptance/bundle/dms/failed-update-permissions/script index c8b57d339f9..8739bd4cc2b 100644 --- a/acceptance/bundle/dms/failed-update-permissions/script +++ b/acceptance/bundle/dms/failed-update-permissions/script @@ -2,6 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Deploy the job and its permissions" trace $CLI bundle deploy +replace_ids.py rm -f "$OUT_REQUESTS" title "The plan records the deployment_history feature and the job" @@ -9,6 +10,7 @@ dms_plan title "The service lists the job at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update-permissions-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +add_repl "$deployment_id" DEPLOYMENT_ID trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state after deploy" diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index e7c4d058f28..f090abba0fe 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -11,11 +11,11 @@ Files: 2 uploaded, 0 deleted Resources: 5 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py --dms //versions/1/operations --sort --del-body state --oneline -{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.five", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.four", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.three", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.two", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/jobs.five", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[FIVE_ID]", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/jobs.four", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[FOUR_ID]", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[ONE_ID]", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/jobs.three", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[THREE_ID]", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/jobs.two", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[TWO_ID]", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}} === The plan records the deployment_history feature >>> [CLI] bundle plan -o json @@ -33,7 +33,7 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged === The service lists all five resources at version 1 ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { "last_action_type": "OPERATION_ACTION_TYPE_CREATE", @@ -98,7 +98,7 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged === The service still lists all five resources, now at version 2 ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { "last_action_type": "OPERATION_ACTION_TYPE_CREATE", diff --git a/acceptance/bundle/dms/multiple-resources/script b/acceptance/bundle/dms/multiple-resources/script index 8c0145c2508..0f0276ff357 100644 --- a/acceptance/bundle/dms/multiple-resources/script +++ b/acceptance/bundle/dms/multiple-resources/script @@ -2,6 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; other bundle/dms tests cover it." trace $CLI bundle deploy +replace_ids.py trace print_requests.py --dms //versions/1/operations --sort --del-body state --oneline title "The plan records the deployment_history feature" @@ -9,6 +10,7 @@ dms_plan title "The service lists all five resources at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-multiple-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +add_repl "$deployment_id" DEPLOYMENT_ID trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state the service holds for this deployment" diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index 36893c2c23b..c0b7a35688e 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -7,7 +7,7 @@ Created pipelines.bar Files: 3 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { "last_action_type": "OPERATION_ACTION_TYPE_CREATE", @@ -23,7 +23,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle summary -o json { - "deployment_id": "[NUMID]", + "deployment_id": "[DEPLOYMENT_ID]", "latest_version_id": 1 } @@ -80,7 +80,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "catalog": "main", "channel": "CURRENT", "deployment": { - "deployment_id": "[NUMID]", + "deployment_id": "[DEPLOYMENT_ID]", "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", "version_id": "1" @@ -103,7 +103,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "path": "/api/2.2/jobs/create", "body": { "deployment": { - "deployment_id": "[NUMID]", + "deployment_id": "[DEPLOYMENT_ID]", "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", "version_id": "1" @@ -124,7 +124,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "catalog": "main", "channel": "CURRENT", "deployment": { - "deployment_id": "[NUMID]", + "deployment_id": "[DEPLOYMENT_ID]", "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", "version_id": "2" @@ -144,7 +144,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged } } ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { "last_action_type": "OPERATION_ACTION_TYPE_CREATE", @@ -164,7 +164,7 @@ Warning: The "development" property of pipelines is deprecated. Use "mode: devel in databricks.yml:18:20 { - "deployment_id": "[NUMID]", + "deployment_id": "[DEPLOYMENT_ID]", "latest_version_id": 2 } diff --git a/acceptance/bundle/dms/no-drift/script b/acceptance/bundle/dms/no-drift/script index 39dff73963d..66898d04cf9 100644 --- a/acceptance/bundle/dms/no-drift/script +++ b/acceptance/bundle/dms/no-drift/script @@ -4,6 +4,7 @@ title "Deploy job foo and pipeline bar; both are recorded at version 1" trace $CLI bundle deploy deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-drift-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +add_repl "$deployment_id" DEPLOYMENT_ID trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' trace $CLI bundle summary -o json | jq '.bundle.deployment | {deployment_id, latest_version_id}' diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index b3d38120851..884c22c6583 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -44,7 +44,7 @@ deployment_history_enabled true Plan: 0 to add, 0 to change, 0 to delete, 0 unchanged === The service lists no resources for the deployment ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [] === The raw recorded state the service holds for this deployment @@ -68,31 +68,31 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py --dms //api/2.0/bundle --get { "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]" + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]" } { "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]/resources" + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources" } { "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]" + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]" } { "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]/resources" + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources" } { "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]/resources" + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources" } { "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]" + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]" } { "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]/resources" + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources" } === The plan records the deployment_history feature @@ -110,7 +110,7 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 0 unchanged === The service still lists no resources ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [] === The raw recorded state after redeploy diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script index 247c1abcb3e..73c029962b9 100644 --- a/acceptance/bundle/dms/no-resources/script +++ b/acceptance/bundle/dms/no-resources/script @@ -11,6 +11,7 @@ dms_plan title "The service lists no resources for the deployment" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +add_repl "$deployment_id" DEPLOYMENT_ID trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state the service holds for this deployment" diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt index daaecd892f4..4d417d6adb7 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/output.txt +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -22,7 +22,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The service lists the schema at version 1 ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { "last_action_type": "OPERATION_ACTION_TYPE_CREATE", @@ -56,7 +56,7 @@ This action will result in the deletion or recreation of the following UC schema Recreated schemas.foo Error: recording operation for resources.schemas.foo: Fault injected by test. (500 INJECTED) -Endpoint: PATCH [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo?update_mask=state%2Cerror_message%2Cstatus +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/operations/schemas.foo?update_mask=state%2Cerror_message%2Cstatus HTTP Status: 500 Internal Server Error API error_code: INJECTED API message: Fault injected by test. @@ -64,7 +64,7 @@ API message: Fault injected by test. Files: 2 uploaded, 0 deleted === The service lists the schema with the failure ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { "last_action_type": "OPERATION_ACTION_TYPE_RECREATE", @@ -88,7 +88,7 @@ Files: 2 uploaded, 0 deleted >>> print_requests.py --dms //api/2.0/bundle { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": { "version_id": "2" }, @@ -106,7 +106,7 @@ Files: 2 uploaded, 0 deleted } { "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/operations/schemas.foo", "q": { "update_mask": "state,error_message,status" }, @@ -118,7 +118,7 @@ Files: 2 uploaded, 0 deleted } { "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/operations/schemas.foo", "q": { "update_mask": "state,error_message,resource_id,status" }, @@ -132,7 +132,7 @@ Files: 2 uploaded, 0 deleted } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/complete", "body": { "completion_reason": "VERSION_COMPLETE_FAILURE" } diff --git a/acceptance/bundle/dms/operation-upload-fails-once/script b/acceptance/bundle/dms/operation-upload-fails-once/script index dbe00b7b106..2b539134571 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/script +++ b/acceptance/bundle/dms/operation-upload-fails-once/script @@ -9,6 +9,7 @@ dms_plan title "The service lists the schema at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-upload-fails-once-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +add_repl "$deployment_id" DEPLOYMENT_ID trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state after deploy" diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index a2f7de239b9..7719a98aed7 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -9,7 +9,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py --dms //versions { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": { "version_id": "1" }, @@ -31,21 +31,21 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } { "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/jobs.foo", "q": { "update_mask": "state,error_message,resource_id,status" }, "body": { "error_message": "", - "resource_id": "[NUMID]", + "resource_id": "[FOO_ID]", "sequence_id": 0, - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":4,\"name\":\"[dev [USERNAME]] foo\",\"queue\":{\"enabled\":true},\"tags\":{\"dev\":\"[USERNAME]\"}}}", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[DEPLOYMENT_ID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":4,\"name\":\"[dev [USERNAME]] foo\",\"queue\":{\"enabled\":true},\"tags\":{\"dev\":\"[USERNAME]\"}}}", "status": "OPERATION_STATUS_SUCCEEDED" } } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } @@ -67,7 +67,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version ->>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID] +>>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[DEPLOYMENT_ID] { "target_name": "dev", "deployment_mode": "DEPLOYMENT_MODE_DEVELOPMENT", @@ -83,7 +83,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged } === The service lists the deployed job ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { "last_action_type": "OPERATION_ACTION_TYPE_CREATE", diff --git a/acceptance/bundle/dms/provenance/script b/acceptance/bundle/dms/provenance/script index 4da8761815d..8c15c861e19 100644 --- a/acceptance/bundle/dms/provenance/script +++ b/acceptance/bundle/dms/provenance/script @@ -6,6 +6,7 @@ git remote add origin https://github.com/databricks/bundle-examples.git trace $CLI bundle deploy # The commit SHA changes every run, so assert it is a 40-char hex string and drop it. add_repl.py "$(git rev-parse HEAD)" COMMIT +replace_ids.py -t dev trace print_requests.py --dms //versions title "The plan is a no-op and records the deployment_history feature" @@ -13,6 +14,7 @@ dms_plan title "The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-provenance-${UNIQUE_NAME}/dev/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') +add_repl "$deployment_id" DEPLOYMENT_ID # MSYS_NO_PATHCONV as above: Git Bash on Windows would rewrite the leading-'/' API # path into C:/Program Files/Git/api/2.0/... before the CLI saw it. trace MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/bundle/deployments/${deployment_id}" | jq '{target_name, deployment_mode, git_info, workspace_info}' diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index bf61dfb7392..a35bb951a32 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -43,7 +43,7 @@ Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": { "version_id": "1" }, @@ -60,7 +60,7 @@ Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged } { "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.doomed", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/jobs.doomed", "q": { "update_mask": "error_message,status" }, @@ -72,17 +72,17 @@ Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/complete", "body": { "completion_reason": "VERSION_COMPLETE_FAILURE" } } === The failed resource is not listed at all: state is what projects a resource, and a failed create records none, so a later deploy plans to create it rather than treating it as already deployed ->>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources +>>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources {} ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [] >>> print_state.py --no-dms diff --git a/acceptance/bundle/dms/record-failure/script b/acceptance/bundle/dms/record-failure/script index 51e785895f1..806e7ab8b15 100644 --- a/acceptance/bundle/dms/record-failure/script +++ b/acceptance/bundle/dms/record-failure/script @@ -9,6 +9,7 @@ title "The failed resource is not listed at all: state is what projects a resour # The deployment ID is the workspace node's ID; read it back the way the CLI does. # Extracted with python, not jq: the ID exceeds 2^53 and jq 1.6 rounds it. deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-failure-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') +add_repl "$deployment_id" DEPLOYMENT_ID # MSYS_NO_PATHCONV as above: Git Bash on Windows would rewrite the leading-'/' API # path into C:/Program Files/Git/api/2.0/... before the CLI saw it. trace MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/bundle/deployments/${deployment_id}/resources" diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index b96fa7dddf7..71c39e8e24b 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -18,7 +18,7 @@ Created jobs.foo Files: 3 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { "last_action_type": "OPERATION_ACTION_TYPE_CREATE", @@ -53,9 +53,9 @@ Error: plan lineage "" does not match state lineage "[UUID]"; the state may have === Only the first deploy recorded a version; the rejected replay created none >>> print_requests.py --dms //api/2.0/bundle --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-stale-plan-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default"}}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_key": "jobs.foo"}], "version_type": "VERSION_TYPE_DEPLOY"}} -{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_key": "jobs.foo"}], "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[FOO_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[DEPLOYMENT_ID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} === Save a plan against the recorded deployment, then move the deployment on >>> jq {serial} tmp.plan2.json diff --git a/acceptance/bundle/dms/stale-plan/script b/acceptance/bundle/dms/stale-plan/script index bb6d0b4a598..7c9d37630f0 100644 --- a/acceptance/bundle/dms/stale-plan/script +++ b/acceptance/bundle/dms/stale-plan/script @@ -8,6 +8,8 @@ dms_plan $CLI bundle deploy $(readplanarg tmp.plan.json) deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-stale-plan-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +add_repl "$deployment_id" DEPLOYMENT_ID +replace_ids.py trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state the service holds for this deployment" diff --git a/acceptance/bundle/dms/state-features/output.txt b/acceptance/bundle/dms/state-features/output.txt index 00451468b55..342fbfe6a44 100644 --- a/acceptance/bundle/dms/state-features/output.txt +++ b/acceptance/bundle/dms/state-features/output.txt @@ -20,7 +20,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> print_requests.py --dms //resources --get --oneline -{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} +{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"} === Disabling deployment history is refused by plan and deploy, while destroy keeps using DMS from state >>> env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle plan @@ -44,7 +44,7 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 1 deleted >>> print_requests.py --dms //api/2.0/bundle --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_DELETE", "resource_key": "jobs.foo"}], "previous_version_id": "1", "version_type": "VERSION_TYPE_DESTROY"}} -{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo", "q": {"update_mask": "state,error_message,status"}, "body": {"error_message": "", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} -{"method": "DELETE", "path": "/api/2.0/bundle/deployments/[NUMID]"} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_DELETE", "resource_key": "jobs.foo"}], "previous_version_id": "1", "version_type": "VERSION_TYPE_DESTROY"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/operations/jobs.foo", "q": {"update_mask": "state,error_message,status"}, "body": {"error_message": "", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} +{"method": "DELETE", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} diff --git a/acceptance/bundle/dms/state-features/script b/acceptance/bundle/dms/state-features/script index 4d49520f7c1..349845853cc 100644 --- a/acceptance/bundle/dms/state-features/script +++ b/acceptance/bundle/dms/state-features/script @@ -3,6 +3,8 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Deploy records version 3 and the deployment_history feature, with resources held in DMS" trace $CLI bundle deploy trace jq '{state_version, features, state}' .databricks/bundle/default/resources.json +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-state-features-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +add_repl "$deployment_id" DEPLOYMENT_ID rm -f "$OUT_REQUESTS" title "Reopening version 3 reads resources from DMS and produces a no-op plan" diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index 56eedb8531a..f7c42c00f00 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -22,7 +22,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": { "version_id": "1" }, @@ -39,7 +39,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } { "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/schemas.foo", "q": { "update_mask": "state,error_message,resource_id,status" }, @@ -53,7 +53,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } @@ -75,7 +75,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The service lists the schema, recorded at version 1 ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { "last_action_type": "OPERATION_ACTION_TYPE_CREATE", @@ -111,7 +111,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged >>> print_requests.py --dms //api/2.0/bundle { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": { "version_id": "2" }, @@ -129,7 +129,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged } { "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/operations/schemas.foo", "q": { "update_mask": "state,error_message,status" }, @@ -141,7 +141,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged } { "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/operations/schemas.foo", "q": { "update_mask": "state,error_message,resource_id,status" }, @@ -155,7 +155,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } @@ -177,7 +177,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The service lists the schema at version 2 with recreate action ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { "last_action_type": "OPERATION_ACTION_TYPE_RECREATE", @@ -213,7 +213,7 @@ Destroy: 1 deleted >>> print_requests.py --dms //api/2.0/bundle { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": { "version_id": "3" }, @@ -231,7 +231,7 @@ Destroy: 1 deleted } { "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/operations/schemas.foo", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/operations/schemas.foo", "q": { "update_mask": "state,error_message,status" }, @@ -243,12 +243,12 @@ Destroy: 1 deleted } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/complete", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } } { "method": "DELETE", - "path": "/api/2.0/bundle/deployments/[NUMID]" + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]" } diff --git a/acceptance/bundle/dms/successful-recreate/script b/acceptance/bundle/dms/successful-recreate/script index f916841af9e..87bdc98bc34 100644 --- a/acceptance/bundle/dms/successful-recreate/script +++ b/acceptance/bundle/dms/successful-recreate/script @@ -9,6 +9,7 @@ dms_plan title "The service lists the schema, recorded at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-successful-recreate-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +add_repl "$deployment_id" DEPLOYMENT_ID trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state the service holds for this deployment" diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt index ebc07afee93..989744df818 100644 --- a/acceptance/bundle/dms/version-never-created/output.txt +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -4,7 +4,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files... Error: failed to create deployment version: Internal error (500 INTERNAL_ERROR) -Endpoint: POST [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions?version_id=1 +Endpoint: POST [DATABRICKS_URL]/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions?version_id=1 HTTP Status: 500 Internal Server Error API error_code: INTERNAL_ERROR API message: Internal error @@ -32,13 +32,13 @@ Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged >>> print_requests.py --dms //api/2.0/bundle --get --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-version-never-created", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default"}}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_key": "jobs.foo"}], "version_type": "VERSION_TYPE_DEPLOY"}} -{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} -{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} -{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} -{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_key": "jobs.foo"}], "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} +{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"} +{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} +{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"} ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [] === The next deploy reuses the ID that node names and retries version 1, rather than creating a second deployment @@ -46,7 +46,7 @@ Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files... Error: failed to create deployment version: Internal error (500 INTERNAL_ERROR) -Endpoint: POST [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions?version_id=1 +Endpoint: POST [DATABRICKS_URL]/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions?version_id=1 HTTP Status: 500 Internal Server Error API error_code: INTERNAL_ERROR API message: Internal error @@ -54,7 +54,7 @@ API message: Internal error Files: 0 uploaded, 0 deleted >>> print_requests.py --dms //api/2.0/bundle --get --oneline -{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} -{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} -{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_key": "jobs.foo"}], "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"} +{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} +{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_key": "jobs.foo"}], "version_type": "VERSION_TYPE_DEPLOY"}} diff --git a/acceptance/bundle/dms/version-never-created/script b/acceptance/bundle/dms/version-never-created/script index 18d3b71c3b4..9498fa05eff 100644 --- a/acceptance/bundle/dms/version-never-created/script +++ b/acceptance/bundle/dms/version-never-created/script @@ -5,6 +5,7 @@ trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_U trace print_requests.py --dms //api/2.0/bundle --get --oneline deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-version-never-created/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +add_repl "$deployment_id" DEPLOYMENT_ID trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The next deploy reuses the ID that node names and retries version 1, rather than creating a second deployment" diff --git a/acceptance/bundle/generate/app_not_yet_deployed/output.txt b/acceptance/bundle/generate/app_not_yet_deployed/output.txt index 8ac4f1220c5..a1df279fd54 100644 --- a/acceptance/bundle/generate/app_not_yet_deployed/output.txt +++ b/acceptance/bundle/generate/app_not_yet_deployed/output.txt @@ -14,7 +14,7 @@ "id": "1000", "name": "my-app", "service_principal_client_id": "[UUID]", - "service_principal_id": [NUMID], + "service_principal_id": [SERVICE_PRINCIPAL_ID], "service_principal_name": "app-my-app", "url": "my-app-123.cloud.databricksapps.com" } diff --git a/acceptance/bundle/generate/app_not_yet_deployed/script b/acceptance/bundle/generate/app_not_yet_deployed/script index 8883d05d15d..1dae6245c35 100644 --- a/acceptance/bundle/generate/app_not_yet_deployed/script +++ b/acceptance/bundle/generate/app_not_yet_deployed/script @@ -1,2 +1,4 @@ -trace $CLI apps create my-app --no-compute --no-wait +trace $CLI apps create my-app --no-compute --no-wait | tee app.json.tmp +add_repl "$(sed -n 's/.*"service_principal_id": \([0-9]*\).*/\1/p' app.json.tmp)" SERVICE_PRINCIPAL_ID +rm app.json.tmp trace $CLI bundle generate app --existing-app-name my-app --config-dir . --key out diff --git a/acceptance/bundle/generate/auto-bind/output.txt b/acceptance/bundle/generate/auto-bind/output.txt index 6eb76255797..e7a43c9c50e 100644 --- a/acceptance/bundle/generate/auto-bind/output.txt +++ b/acceptance/bundle/generate/auto-bind/output.txt @@ -1,16 +1,16 @@ === Create a pre-defined job: -Created job with ID: [NUMID] +Created job with ID: [JOB_ID] >>> [CLI] workspace mkdirs /Workspace/Users/[USERNAME]/python-[UNIQUE_NAME] >>> [CLI] workspace import /Workspace/Users/[USERNAME]/python-[UNIQUE_NAME]/test --file test.py --language PYTHON === Generate and bind in one step: ->>> [CLI] bundle generate job --key test_job --existing-job-id [NUMID] --config-dir resources --source-dir src --bind +>>> [CLI] bundle generate job --key test_job --existing-job-id [JOB_ID] --config-dir resources --source-dir src --bind File successfully saved to src/test.py Job configuration successfully saved to resources/test_job.job.yml -Successfully bound job with an id '[NUMID]' +Successfully bound job with an id '[JOB_ID]' >>> ls src/ test.py @@ -35,8 +35,8 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 1 deleted === Check that job is bound and does not exist after bundle is destroyed: ->>> errcode [CLI] jobs get [NUMID] --output json -Error: Job [NUMID] does not exist. +>>> errcode [CLI] jobs get [JOB_ID] --output json +Error: Job [JOB_ID] does not exist. Exit code: 1 diff --git a/acceptance/bundle/generate/auto-bind/script b/acceptance/bundle/generate/auto-bind/script index f6517a8adb6..542dc0b96d6 100755 --- a/acceptance/bundle/generate/auto-bind/script +++ b/acceptance/bundle/generate/auto-bind/script @@ -20,6 +20,7 @@ JOB_ID=$($CLI jobs create --json ' } ] }' | jq -r '.job_id') +add_repl "$JOB_ID" JOB_ID echo "Created job with ID: $JOB_ID" diff --git a/acceptance/bundle/generate/dashboard-inplace/output.txt b/acceptance/bundle/generate/dashboard-inplace/output.txt index accc1b4eae4..52e5822e308 100644 --- a/acceptance/bundle/generate/dashboard-inplace/output.txt +++ b/acceptance/bundle/generate/dashboard-inplace/output.txt @@ -15,7 +15,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "create_time": "[TIMESTAMP]", "dashboard_id": "[DASHBOARD_ID]", "display_name": "test dashboard", - "etag": "[NUMID]", + "etag": "80611981", "lifecycle_state": "ACTIVE", "parent_path": "/Users/[USERNAME]/.bundle/dashboard update inplace/default/resources", "path": "/Users/[USERNAME]/.bundle/dashboard update inplace/default/resources/test dashboard.lvdash.json", diff --git a/acceptance/bundle/generate/dashboard/out/dashboard/test_dashboard.lvdash.json b/acceptance/bundle/generate/dashboard/out/dashboard/test_dashboard.lvdash.json index 5b9d2826f30..27ba4cd55f6 100644 --- a/acceptance/bundle/generate/dashboard/out/dashboard/test_dashboard.lvdash.json +++ b/acceptance/bundle/generate/dashboard/out/dashboard/test_dashboard.lvdash.json @@ -3,7 +3,7 @@ { "displayName": "New Page", "layout": [], - "name": "[NUMID]", + "name": "12345678", "pageType": "PAGE_TYPE_CANVAS" } ] diff --git a/acceptance/bundle/generate/dashboard_existing_path_nominal/out/dashboard/custom_key.lvdash.json b/acceptance/bundle/generate/dashboard_existing_path_nominal/out/dashboard/custom_key.lvdash.json index ea901f555c7..ff23ab6f19b 100644 --- a/acceptance/bundle/generate/dashboard_existing_path_nominal/out/dashboard/custom_key.lvdash.json +++ b/acceptance/bundle/generate/dashboard_existing_path_nominal/out/dashboard/custom_key.lvdash.json @@ -3,7 +3,7 @@ { "displayName": "New Page", "layout": [], - "name": "[NUMID]" + "name": "12345678" } ] } diff --git a/acceptance/bundle/generate/dashboard_existing_path_nominal/out/dashboard/this_is_a_test_dashboard.lvdash.json b/acceptance/bundle/generate/dashboard_existing_path_nominal/out/dashboard/this_is_a_test_dashboard.lvdash.json index ea901f555c7..ff23ab6f19b 100644 --- a/acceptance/bundle/generate/dashboard_existing_path_nominal/out/dashboard/this_is_a_test_dashboard.lvdash.json +++ b/acceptance/bundle/generate/dashboard_existing_path_nominal/out/dashboard/this_is_a_test_dashboard.lvdash.json @@ -3,7 +3,7 @@ { "displayName": "New Page", "layout": [], - "name": "[NUMID]" + "name": "12345678" } ] } diff --git a/acceptance/bundle/integration_whl/base/output.txt b/acceptance/bundle/integration_whl/base/output.txt index 0b12df225ae..e45eb854f37 100644 --- a/acceptance/bundle/integration_whl/base/output.txt +++ b/acceptance/bundle/integration_whl/base/output.txt @@ -39,7 +39,7 @@ Files: 17 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle run some_other_job -Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" RUNNING [TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS @@ -56,7 +56,7 @@ Files: 9 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged >>> [CLI] bundle run some_other_job -Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" RUNNING [TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS diff --git a/acceptance/bundle/integration_whl/custom_params/output.txt b/acceptance/bundle/integration_whl/custom_params/output.txt index 9b9290d024c..5da9e5cf0e7 100644 --- a/acceptance/bundle/integration_whl/custom_params/output.txt +++ b/acceptance/bundle/integration_whl/custom_params/output.txt @@ -39,7 +39,7 @@ Files: 14 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle run some_other_job --python-params param1,param2 -Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" RUNNING [TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS diff --git a/acceptance/bundle/integration_whl/interactive_cluster/output.txt b/acceptance/bundle/integration_whl/interactive_cluster/output.txt index ef257c455cd..e56247af31c 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster/output.txt +++ b/acceptance/bundle/integration_whl/interactive_cluster/output.txt @@ -40,7 +40,7 @@ Files: 14 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle run some_other_job -Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" RUNNING [TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS @@ -57,7 +57,7 @@ Files: 9 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> [CLI] bundle run some_other_job -Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" RUNNING [TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS diff --git a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/output.txt b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/output.txt index 6d871a6f1a4..160d5c1abe4 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/output.txt +++ b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/output.txt @@ -9,7 +9,7 @@ Files: 13 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle run some_other_job -Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" RUNNING [TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS @@ -21,7 +21,7 @@ Got arguments: >>> [CLI] bundle deploy -qq >>> [CLI] bundle run some_other_job -Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" RUNNING [TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS diff --git a/acceptance/bundle/integration_whl/interactive_single_user/output.txt b/acceptance/bundle/integration_whl/interactive_single_user/output.txt index 865f4f8743e..33047159fbd 100644 --- a/acceptance/bundle/integration_whl/interactive_single_user/output.txt +++ b/acceptance/bundle/integration_whl/interactive_single_user/output.txt @@ -40,7 +40,7 @@ Files: 13 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle run some_other_job -Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" RUNNING [TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS @@ -49,7 +49,7 @@ Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[NUMID]?o=[NUMID] >>> [CLI] bundle deploy -qq >>> [CLI] bundle run some_other_job -Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" RUNNING [TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS diff --git a/acceptance/bundle/integration_whl/serverless/output.txt b/acceptance/bundle/integration_whl/serverless/output.txt index f8b4d241499..99dfccb17a7 100644 --- a/acceptance/bundle/integration_whl/serverless/output.txt +++ b/acceptance/bundle/integration_whl/serverless/output.txt @@ -10,7 +10,7 @@ Files: 14 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle run some_other_job -Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "My Wheel Job" RUNNING [TIMESTAMP] "My Wheel Job" TERMINATED SUCCESS @@ -27,7 +27,7 @@ Files: 9 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged >>> [CLI] bundle run some_other_job -Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "My Wheel Job" RUNNING [TIMESTAMP] "My Wheel Job" TERMINATED SUCCESS diff --git a/acceptance/bundle/integration_whl/serverless_custom_params/output.txt b/acceptance/bundle/integration_whl/serverless_custom_params/output.txt index dd7dd93051f..2b93520215a 100644 --- a/acceptance/bundle/integration_whl/serverless_custom_params/output.txt +++ b/acceptance/bundle/integration_whl/serverless_custom_params/output.txt @@ -8,7 +8,7 @@ Files: 14 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle run some_other_job -Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "[default] Test Wheel Job With Environments [UNIQUE_NAME]" RUNNING [TIMESTAMP] "[default] Test Wheel Job With Environments [UNIQUE_NAME]" TERMINATED SUCCESS @@ -17,7 +17,7 @@ Got arguments: ['my_test_code', 'one', 'two'] >>> [CLI] bundle run some_other_job --python-params=param1,param2 -Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "[default] Test Wheel Job With Environments [UNIQUE_NAME]" RUNNING [TIMESTAMP] "[default] Test Wheel Job With Environments [UNIQUE_NAME]" TERMINATED SUCCESS diff --git a/acceptance/bundle/integration_whl/serverless_dynamic_version/output.txt b/acceptance/bundle/integration_whl/serverless_dynamic_version/output.txt index de3839bf308..1da7e8fcce6 100644 --- a/acceptance/bundle/integration_whl/serverless_dynamic_version/output.txt +++ b/acceptance/bundle/integration_whl/serverless_dynamic_version/output.txt @@ -10,7 +10,7 @@ Files: 14 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle run some_other_job -Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "My Wheel Job" RUNNING [TIMESTAMP] "My Wheel Job" TERMINATED SUCCESS @@ -28,7 +28,7 @@ Files: 9 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged >>> [CLI] bundle run some_other_job -Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "My Wheel Job" RUNNING [TIMESTAMP] "My Wheel Job" TERMINATED SUCCESS diff --git a/acceptance/bundle/integration_whl/serverless_extras/output.txt b/acceptance/bundle/integration_whl/serverless_extras/output.txt index 8d667991bdb..c9c5ac6554d 100644 --- a/acceptance/bundle/integration_whl/serverless_extras/output.txt +++ b/acceptance/bundle/integration_whl/serverless_extras/output.txt @@ -14,7 +14,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] >>> [CLI] bundle run some_other_job -Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "My Wheel Job" RUNNING [TIMESTAMP] "My Wheel Job" TERMINATED SUCCESS diff --git a/acceptance/bundle/integration_whl/serverless_extras_dynamic_version/output.txt b/acceptance/bundle/integration_whl/serverless_extras_dynamic_version/output.txt index 2ee70835a0f..ae76c6b53d9 100644 --- a/acceptance/bundle/integration_whl/serverless_extras_dynamic_version/output.txt +++ b/acceptance/bundle/integration_whl/serverless_extras_dynamic_version/output.txt @@ -14,7 +14,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] >>> [CLI] bundle run some_other_job -Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "My Wheel Job" RUNNING [TIMESTAMP] "My Wheel Job" TERMINATED SUCCESS diff --git a/acceptance/bundle/integration_whl/test.toml b/acceptance/bundle/integration_whl/test.toml index 6b36b680b92..b29abd8a7ec 100644 --- a/acceptance/bundle/integration_whl/test.toml +++ b/acceptance/bundle/integration_whl/test.toml @@ -26,3 +26,7 @@ Ignore = [ "python_wheel_wrapper.yml", "empty.yml", ] + +[[Repls]] +Old = '/runs/\d+' +New = '/runs/[RUN_ID]' diff --git a/acceptance/bundle/integration_whl/wrapper/output.txt b/acceptance/bundle/integration_whl/wrapper/output.txt index d00ab5fe102..418dcc45bda 100644 --- a/acceptance/bundle/integration_whl/wrapper/output.txt +++ b/acceptance/bundle/integration_whl/wrapper/output.txt @@ -39,7 +39,7 @@ Files: 15 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle run some_other_job -Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" RUNNING [TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS diff --git a/acceptance/bundle/integration_whl/wrapper_custom_params/output.txt b/acceptance/bundle/integration_whl/wrapper_custom_params/output.txt index 8861a6764d6..f7127244cd2 100644 --- a/acceptance/bundle/integration_whl/wrapper_custom_params/output.txt +++ b/acceptance/bundle/integration_whl/wrapper_custom_params/output.txt @@ -39,7 +39,7 @@ Files: 15 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle run some_other_job --python-params param1,param2 -Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" RUNNING [TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS diff --git a/acceptance/bundle/integration_whl/wrapper_custom_params/script b/acceptance/bundle/integration_whl/wrapper_custom_params/script index ab9c4503276..ebe783a86b8 100644 --- a/acceptance/bundle/integration_whl/wrapper_custom_params/script +++ b/acceptance/bundle/integration_whl/wrapper_custom_params/script @@ -5,4 +5,5 @@ cp -r $TESTDIR/../base/{$EXTRA_CONFIG,setup.py,my_test_code} . trace cat databricks.yml trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT trace $CLI bundle deploy +replace_ids.py trace $CLI bundle run some_other_job --python-params param1,param2 diff --git a/acceptance/bundle/local_state_staleness/output.txt b/acceptance/bundle/local_state_staleness/output.txt index abcc31c9164..133b867061c 100644 --- a/acceptance/bundle/local_state_staleness/output.txt +++ b/acceptance/bundle/local_state_staleness/output.txt @@ -21,7 +21,7 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged === Step 4: Verify only one job exists { "name": "test-job-basic-[UNIQUE_NAME]", - "id": [NUMID] + "id": [TEST_JOB_ID] } === Cleanup - destroy bundle A (bundle B does not have an active deployment) diff --git a/acceptance/bundle/local_state_staleness/script b/acceptance/bundle/local_state_staleness/script index 6c434629e2c..90faf6de288 100644 --- a/acceptance/bundle/local_state_staleness/script +++ b/acceptance/bundle/local_state_staleness/script @@ -30,6 +30,7 @@ trap cleanup EXIT title "Step 1: Deploy bundle A" cd "$BUNDLE_A_DIR" trace "$CLI" bundle deploy --force-lock --auto-approve +replace_ids.py title "Step 2: Deploy bundle B" cd "$BUNDLE_B_DIR" diff --git a/acceptance/bundle/migrate/auto/apply-failure/output.txt b/acceptance/bundle/migrate/auto/apply-failure/output.txt index 631211d3000..be5d6f0a284 100644 --- a/acceptance/bundle/migrate/auto/apply-failure/output.txt +++ b/acceptance/bundle/migrate/auto/apply-failure/output.txt @@ -9,7 +9,7 @@ Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Migrated 1 resource to direct deployment engine. -Error: cannot update resources.jobs.test_job: updating id=[NUMID]: Fault injected by test. (403 INJECTED) +Error: cannot update resources.jobs.test_job: updating id=[TEST_JOB_ID]: Fault injected by test. (403 INJECTED) Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/reset HTTP Status: 403 Forbidden @@ -31,9 +31,9 @@ direct_migrated_via_env true >>> MSYS_NO_PATHCONV=1 [CLI] workspace list /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state ID Type Language Path -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate.backup +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate.backup === Retry finishes the apply on the already-migrated direct state (no re-migration) >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy @@ -52,7 +52,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged >>> MSYS_NO_PATHCONV=1 [CLI] workspace list /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state ID Type Language Path -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate.backup +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate.backup diff --git a/acceptance/bundle/migrate/auto/apply-failure/script b/acceptance/bundle/migrate/auto/apply-failure/script index 910d871b677..78b838fa835 100644 --- a/acceptance/bundle/migrate/auto/apply-failure/script +++ b/acceptance/bundle/migrate/auto/apply-failure/script @@ -2,6 +2,7 @@ export DATABRICKS_BUNDLE_ENGINE= title "Replay a prior terraform deployment" trace replay_tfstate.py terraform-requests.json terraform.tfstate +replace_ids.py rm -f "$OUT_REQUESTS" # Change the job so the migrating deploy must update it (jobs/reset), giving us a resource apply diff --git a/acceptance/bundle/migrate/auto/artifacts/output.txt b/acceptance/bundle/migrate/auto/artifacts/output.txt index 1cbab27c9b9..0656827cd2d 100644 --- a/acceptance/bundle/migrate/auto/artifacts/output.txt +++ b/acceptance/bundle/migrate/auto/artifacts/output.txt @@ -20,10 +20,10 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged >>> MSYS_NO_PATHCONV=1 [CLI] workspace list /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state ID Type Language Path -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate.backup +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate.backup === Migrated state's library path (should be the resolved remote path, not the local glob) >>> print_state.py diff --git a/acceptance/bundle/migrate/auto/clean/out.plan_update.json b/acceptance/bundle/migrate/auto/clean/out.plan_update.json index 0ee7d34a8ca..1f79fdd0cb3 100644 --- a/acceptance/bundle/migrate/auto/clean/out.plan_update.json +++ b/acceptance/bundle/migrate/auto/clean/out.plan_update.json @@ -31,7 +31,7 @@ "edit_mode": "UI_LOCKED", "email_notifications": {}, "format": "MULTI_TASK", - "job_id": [NUMID], + "job_id": [TEST_JOB_ID], "max_concurrent_runs": 1, "name": "Test Auto-Migrate Job", "queue": { diff --git a/acceptance/bundle/migrate/auto/clean/script b/acceptance/bundle/migrate/auto/clean/script index d655a9cda1a..695759c1760 100644 --- a/acceptance/bundle/migrate/auto/clean/script +++ b/acceptance/bundle/migrate/auto/clean/script @@ -2,6 +2,7 @@ export DATABRICKS_BUNDLE_ENGINE= title "Replay a prior terraform deployment" trace replay_tfstate.py terraform-requests.json terraform.tfstate +replace_ids.py rm -f "$OUT_REQUESTS" # Each call to print_migration_telemetry inspects "$OUT_REQUESTS"; clear diff --git a/acceptance/bundle/migrate/auto/default/output.txt b/acceptance/bundle/migrate/auto/default/output.txt index 3755ca5dcde..30a5dd774e6 100644 --- a/acceptance/bundle/migrate/auto/default/output.txt +++ b/acceptance/bundle/migrate/auto/default/output.txt @@ -22,7 +22,7 @@ Resources: Jobs: test_job: Name: Test Auto-Migrate Job - URL: [DATABRICKS_URL]/jobs/[TEST_JOB_ID]?w=[NUMID] + URL: [DATABRICKS_URL]/jobs/[TEST_JOB_ID]?w=[WORKSPACE_ID] >>> [CLI] bundle summary -o json Warning: the Terraform deployment engine has been removed in Databricks CLI v1.20.0 @@ -35,7 +35,7 @@ Notice: migrating your bundle to direct deployment engine (https://github.com/da { "engine": "terraform", "id": "[TEST_JOB_ID]", - "url": "[DATABRICKS_URL]/jobs/[TEST_JOB_ID]?w=[NUMID]" + "url": "[DATABRICKS_URL]/jobs/[TEST_JOB_ID]?w=[WORKSPACE_ID]" } === Summary does not migrate: no state writes, terraform state is left untouched diff --git a/acceptance/bundle/migrate/auto/destroy/output.txt b/acceptance/bundle/migrate/auto/destroy/output.txt index 491efb78980..7022244297c 100644 --- a/acceptance/bundle/migrate/auto/destroy/output.txt +++ b/acceptance/bundle/migrate/auto/destroy/output.txt @@ -13,7 +13,7 @@ To proceed, use --auto-approve. >>> MSYS_NO_PATHCONV=1 [CLI] workspace list /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state ID Type Language Path -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate === Destroy on direct: migrate then destroy >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/migrate/auto/destroy/script b/acceptance/bundle/migrate/auto/destroy/script index d9a0e91ddf1..ec4954a9fd2 100644 --- a/acceptance/bundle/migrate/auto/destroy/script +++ b/acceptance/bundle/migrate/auto/destroy/script @@ -7,6 +7,7 @@ export DATABRICKS_BUNDLE_ENGINE= title "Replay a prior terraform deployment" trace replay_tfstate.py terraform-requests.json terraform.tfstate +replace_ids.py title "Destroy without approval: migration prepared in memory then dropped, stays on terraform" # The migration is prepared in memory (in process.go) before the destroy phase asks for approval, diff --git a/acceptance/bundle/migrate/auto/direct-only/output.txt b/acceptance/bundle/migrate/auto/direct-only/output.txt index 9ce063350da..10fcf4b0bc6 100644 --- a/acceptance/bundle/migrate/auto/direct-only/output.txt +++ b/acceptance/bundle/migrate/auto/direct-only/output.txt @@ -28,7 +28,7 @@ json.plan.resources.instance_pools.pool.new_state.value.node_type_id = "Standard >>> MSYS_NO_PATHCONV=1 [CLI] workspace list /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state ID Type Language Path -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate === Deploy commits the migration and creates the direct-only resource >>> [CLI] bundle deploy diff --git a/acceptance/bundle/migrate/auto/dms-migration/output.txt b/acceptance/bundle/migrate/auto/dms-migration/output.txt index 980c30afa4f..ba6f36cc8bf 100644 --- a/acceptance/bundle/migrate/auto/dms-migration/output.txt +++ b/acceptance/bundle/migrate/auto/dms-migration/output.txt @@ -17,4 +17,4 @@ Exit code: 1 >>> MSYS_NO_PATHCONV=1 [CLI] workspace list /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state ID Type Language Path -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate diff --git a/acceptance/bundle/migrate/auto/plan-failure/output.txt b/acceptance/bundle/migrate/auto/plan-failure/output.txt index d9a52d2ff9c..343ba610c1d 100644 --- a/acceptance/bundle/migrate/auto/plan-failure/output.txt +++ b/acceptance/bundle/migrate/auto/plan-failure/output.txt @@ -6,7 +6,7 @@ === Inject GET failure so the migration plan check fails: migration hard-errors (no terraform fallback) >>> DATABRICKS_BUNDLE_ENGINE=direct musterr [CLI] bundle deploy -Warn: migration to direct: cannot plan resources.jobs.test_job: reading id="[NUMID]": Fault injected by test. (403 INJECTED): Endpoint: GET [DATABRICKS_URL]/api/2.2/jobs/get?job_id=[NUMID] +Warn: migration to direct: cannot plan resources.jobs.test_job: reading id="[TEST_JOB_ID]": Fault injected by test. (403 INJECTED): Endpoint: GET [DATABRICKS_URL]/api/2.2/jobs/get?job_id=[TEST_JOB_ID] HTTP Status: 403 Forbidden API error_code: INJECTED API message: Fault injected by test. @@ -24,7 +24,7 @@ direct_migrate_plan_error true >>> MSYS_NO_PATHCONV=1 [CLI] workspace list /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state ID Type Language Path -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate === Retry: plan check passes this time, migration succeeds >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy @@ -44,7 +44,7 @@ direct_migrated_via_env true >>> MSYS_NO_PATHCONV=1 [CLI] workspace list /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state ID Type Language Path -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate.backup +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate.backup diff --git a/acceptance/bundle/migrate/auto/plan-failure/script b/acceptance/bundle/migrate/auto/plan-failure/script index e0f9eefeb71..0f56cae649c 100644 --- a/acceptance/bundle/migrate/auto/plan-failure/script +++ b/acceptance/bundle/migrate/auto/plan-failure/script @@ -2,6 +2,7 @@ export DATABRICKS_BUNDLE_ENGINE= title "Replay a prior terraform deployment (captured from a real terraform run)" trace replay_tfstate.py terraform-requests.json terraform.tfstate +replace_ids.py trace print_migration_telemetry rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/migrate/auto/plan-missing/output.txt b/acceptance/bundle/migrate/auto/plan-missing/output.txt index 5a121dfe386..74d958908d0 100644 --- a/acceptance/bundle/migrate/auto/plan-missing/output.txt +++ b/acceptance/bundle/migrate/auto/plan-missing/output.txt @@ -16,7 +16,7 @@ Exit code: 1 >>> MSYS_NO_PATHCONV=1 [CLI] workspace list /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state ID Type Language Path -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate === A plain deploy migrates the still-terraform state to direct >>> [CLI] bundle deploy diff --git a/acceptance/bundle/migrate/auto/push-failure/output.txt b/acceptance/bundle/migrate/auto/push-failure/output.txt index cea782a12cd..6e9a1c61b78 100644 --- a/acceptance/bundle/migrate/auto/push-failure/output.txt +++ b/acceptance/bundle/migrate/auto/push-failure/output.txt @@ -27,8 +27,8 @@ Exit code: 1 >>> MSYS_NO_PATHCONV=1 [CLI] workspace list /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state ID Type Language Path -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate === Retry: the local direct state is used (no re-migration) and the push completes >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy @@ -46,7 +46,7 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged >>> MSYS_NO_PATHCONV=1 [CLI] workspace list /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state ID Type Language Path -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate.backup +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate.backup diff --git a/acceptance/bundle/migrate/auto/run-no-commit/output.txt b/acceptance/bundle/migrate/auto/run-no-commit/output.txt index 449bbd59f9a..755265b41f1 100644 --- a/acceptance/bundle/migrate/auto/run-no-commit/output.txt +++ b/acceptance/bundle/migrate/auto/run-no-commit/output.txt @@ -5,7 +5,7 @@ === bundle run on direct migrates the terraform state IN MEMORY (no commit) to resolve the job >>> DATABRICKS_BUNDLE_ENGINE=direct errcode [CLI] bundle run test_job --no-wait Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). -Run URL: [DATABRICKS_URL]/jobs/[TEST_JOB_ID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[TEST_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] >>> print_migration_telemetry @@ -17,7 +17,7 @@ Run URL: [DATABRICKS_URL]/jobs/[TEST_JOB_ID]/runs/[NUMID]?o=[NUMID] >>> MSYS_NO_PATHCONV=1 [CLI] workspace list /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state ID Type Language Path -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate === bundle run works with engine: terraform: warns and runs the job from the terraform state >>> update_file.py databricks.yml {name: test-bundle} {name: test-bundle, engine: terraform} @@ -30,7 +30,7 @@ Remove the "bundle.engine" setting (or set it to "direct") to deploy with the di See https://docs.databricks.com/dev-tools/bundles/direct for details Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). -Run URL: [DATABRICKS_URL]/jobs/[TEST_JOB_ID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[TEST_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] >>> print_requests.py //jobs diff --git a/acceptance/bundle/migrate/auto/run-no-commit/test.toml b/acceptance/bundle/migrate/auto/run-no-commit/test.toml new file mode 100644 index 00000000000..3c887dd20bf --- /dev/null +++ b/acceptance/bundle/migrate/auto/run-no-commit/test.toml @@ -0,0 +1,3 @@ +[[Repls]] +Old = '/runs/\d+' +New = '/runs/[RUN_ID]' diff --git a/acceptance/bundle/migrate/auto/schema-storage-change/output.txt b/acceptance/bundle/migrate/auto/schema-storage-change/output.txt index 0902960a2a8..bc4083f4457 100644 --- a/acceptance/bundle/migrate/auto/schema-storage-change/output.txt +++ b/acceptance/bundle/migrate/auto/schema-storage-change/output.txt @@ -29,8 +29,8 @@ direct_migrate_recreate_planned true >>> MSYS_NO_PATHCONV=1 [CLI] workspace list /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state ID Type Language Path -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate === With --auto-approve the migration commits and the recreate is applied on direct >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy --auto-approve diff --git a/acceptance/bundle/migrate/auto/tfbackup-failure/output.txt b/acceptance/bundle/migrate/auto/tfbackup-failure/output.txt index 4a633e7d7d2..8475d5c21a0 100644 --- a/acceptance/bundle/migrate/auto/tfbackup-failure/output.txt +++ b/acceptance/bundle/migrate/auto/tfbackup-failure/output.txt @@ -22,11 +22,11 @@ direct_migrated_via_env true >>> MSYS_NO_PATHCONV=1 [CLI] workspace list /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state ID Type Language Path -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate.backup +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate +[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate.backup === Retry: deploys on already-migrated direct state >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy diff --git a/acceptance/bundle/migrate/command/basic/out.new_state.json b/acceptance/bundle/migrate/command/basic/out.new_state.json index e924cb4d005..81f4123c8c3 100644 --- a/acceptance/bundle/migrate/command/basic/out.new_state.json +++ b/acceptance/bundle/migrate/command/basic/out.new_state.json @@ -5,7 +5,7 @@ "serial": 6, "state": { "resources.jobs.test_job": { - "__id__": "[NUMID]", + "__id__": "[TEST_JOB_ID]", "state": { "deployment": { "kind": "BUNDLE", @@ -46,7 +46,7 @@ ], "name": "Test Migration Pipeline", "tags": { - "myjob_id": "[NUMID]", + "myjob_id": "[TEST_JOB_ID]", "myjob_name": "Test Migration Job", "myjob_timeout": "0", "myvolume_id": "mycat.myschema.myvol", diff --git a/acceptance/bundle/migrate/command/basic/out.original_state.json b/acceptance/bundle/migrate/command/basic/out.original_state.json index 147893321a1..89a3af519d6 100644 --- a/acceptance/bundle/migrate/command/basic/out.original_state.json +++ b/acceptance/bundle/migrate/command/basic/out.original_state.json @@ -44,7 +44,7 @@ "format": "MULTI_TASK", "git_source": [], "health": [], - "id": "[NUMID]", + "id": "[TEST_JOB_ID]", "job_cluster": [], "library": [], "max_concurrent_runs": 1, @@ -60,7 +60,7 @@ "pipeline_task": [], "provider_config": [ { - "workspace_id": "[NUMID]" + "workspace_id": "[WORKSPACE_ID]" } ], "python_wheel_task": [], @@ -147,7 +147,7 @@ "timeouts": null, "trigger": [], "triggers": [], - "url": "[DATABRICKS_URL]/#job/[NUMID]", + "url": "[DATABRICKS_URL]/#job/[TEST_JOB_ID]", "usage_policy_id": null, "webhook_notifications": [ { @@ -222,7 +222,7 @@ "photon": false, "provider_config": [ { - "workspace_id": "[NUMID]" + "workspace_id": "[WORKSPACE_ID]" } ], "restart_window": [], @@ -235,7 +235,7 @@ "state": "IDLE", "storage": "dbfs:/pipelines/[UUID]", "tags": { - "myjob_id": "[NUMID]", + "myjob_id": "[TEST_JOB_ID]", "myjob_name": "Test Migration Job", "myjob_timeout": "0", "myvolume_id": "mycat.myschema.myvol", @@ -273,7 +273,7 @@ "owner": "[USERNAME]", "provider_config": [ { - "workspace_id": "[NUMID]" + "workspace_id": "[WORKSPACE_ID]" } ], "schema_name": "myschema", diff --git a/acceptance/bundle/migrate/command/basic/out.plan_update.json b/acceptance/bundle/migrate/command/basic/out.plan_update.json index 0d3be4aacaa..b873a52a785 100644 --- a/acceptance/bundle/migrate/command/basic/out.plan_update.json +++ b/acceptance/bundle/migrate/command/basic/out.plan_update.json @@ -39,7 +39,7 @@ "edit_mode": "UI_LOCKED", "email_notifications": {}, "format": "MULTI_TASK", - "job_id": [NUMID], + "job_id": [TEST_JOB_ID], "max_concurrent_runs": 1, "name": "Test Migration Job", "queue": { @@ -150,7 +150,7 @@ ], "name": "Test Migrated Pipeline", "tags": { - "myjob_id": "[NUMID]", + "myjob_id": "[TEST_JOB_ID]", "myjob_name": "Test Migrated Job", "myjob_timeout": "${resources.jobs.test_job.timeout_seconds}", "myvolume_id": "mycat.myschema.myvol", @@ -186,7 +186,7 @@ "state": "IDLE", "storage": "dbfs:/pipelines/[UUID]", "tags": { - "myjob_id": "[NUMID]", + "myjob_id": "[TEST_JOB_ID]", "myjob_name": "Test Migration Job", "myjob_timeout": "0", "myvolume_id": "mycat.myschema.myvol", diff --git a/acceptance/bundle/migrate/command/basic/output.txt b/acceptance/bundle/migrate/command/basic/output.txt index d842285db84..5574c5fbbd7 100644 --- a/acceptance/bundle/migrate/command/basic/output.txt +++ b/acceptance/bundle/migrate/command/basic/output.txt @@ -42,7 +42,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged "method": "GET", "path": "/api/2.2/jobs/get", "q": { - "job_id": "[NUMID]" + "job_id": "[TEST_JOB_ID]" } } { @@ -54,7 +54,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged "method": "GET", "path": "/api/2.2/jobs/get", "q": { - "job_id": "[NUMID]" + "job_id": "[TEST_JOB_ID]" } } diff --git a/acceptance/bundle/migrate/command/basic/script b/acceptance/bundle/migrate/command/basic/script index 9922da55b67..fe089b00c11 100755 --- a/acceptance/bundle/migrate/command/basic/script +++ b/acceptance/bundle/migrate/command/basic/script @@ -10,6 +10,7 @@ mv terraform-prod-requests.json .databricks/terraform-prod-requests.json # Replay a prior terraform deployment (dev: a job, a volume and a pipeline). trace replay_tfstate.py terraform-requests.json terraform.tfstate -t dev +read_id.py -t dev test_job > /dev/null print_state.py -t dev > out.original_state.json trace $CLI bundle deployment migrate 2>&1 | contains.py 'Migrated 3 resources' diff --git a/acceptance/bundle/migrate/command/dashboards/out.new_state.json b/acceptance/bundle/migrate/command/dashboards/out.new_state.json index c1bdc4c7b78..c710e449216 100644 --- a/acceptance/bundle/migrate/command/dashboards/out.new_state.json +++ b/acceptance/bundle/migrate/command/dashboards/out.new_state.json @@ -9,7 +9,7 @@ "state": { "display_name": "my dashboard", "embed_credentials": true, - "etag": "[NUMID]", + "etag": "80611980", "parent_path": "/Workspace/Users/[USERNAME]", "published": true, "serialized_dashboard": "sha256:[HASH]", diff --git a/acceptance/bundle/migrate/command/dashboards/out.original_state.json b/acceptance/bundle/migrate/command/dashboards/out.original_state.json index 588895fb3c5..98213d4ec86 100644 --- a/acceptance/bundle/migrate/command/dashboards/out.original_state.json +++ b/acceptance/bundle/migrate/command/dashboards/out.original_state.json @@ -21,7 +21,7 @@ "dataset_schema": null, "display_name": "my dashboard", "embed_credentials": true, - "etag": "[NUMID]", + "etag": "80611980", "file_path": null, "id": "[DASHBOARD1_ID]", "lifecycle_state": "ACTIVE", @@ -30,7 +30,7 @@ "path": "/Users/[USERNAME]/my dashboard.lvdash.json", "provider_config": [ { - "workspace_id": "[NUMID]" + "workspace_id": "[WORKSPACE_ID]" } ], "serialized_dashboard": "{\"pages\":[{\"displayName\":\"Dashboard test bundle-deploy-dashboard\",\"name\":\"02724bf2\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n", diff --git a/acceptance/bundle/migrate/command/dashboards/out.plan_after_migrate.json b/acceptance/bundle/migrate/command/dashboards/out.plan_after_migrate.json index 37124a4c407..3d9a600d1d8 100644 --- a/acceptance/bundle/migrate/command/dashboards/out.plan_after_migrate.json +++ b/acceptance/bundle/migrate/command/dashboards/out.plan_after_migrate.json @@ -11,7 +11,7 @@ "dashboard_id": "[DASHBOARD1_ID]", "display_name": "my dashboard", "embed_credentials": true, - "etag": "[NUMID]", + "etag": "80611980", "lifecycle_state": "ACTIVE", "parent_path": "/Workspace/Users/[USERNAME]", "path": "/Users/[USERNAME]/my dashboard.lvdash.json", @@ -34,8 +34,8 @@ "etag": { "action": "skip", "reason": "custom", - "old": "[NUMID]", - "remote": "[NUMID]" + "old": "80611980", + "remote": "80611980" }, "lifecycle_state": { "action": "skip", diff --git a/acceptance/bundle/migrate/command/default-python/out.plan_after_deploy.json b/acceptance/bundle/migrate/command/default-python/out.plan_after_deploy.json index cc6b434e72c..5f9270249fa 100644 --- a/acceptance/bundle/migrate/command/default-python/out.plan_after_deploy.json +++ b/acceptance/bundle/migrate/command/default-python/out.plan_after_deploy.json @@ -96,7 +96,7 @@ } ], "pipeline_task": { - "pipeline_id": "[UUID]" + "pipeline_id": "[MY_DEFAULT_PYTHON_ETL_ID]" }, "task_key": "refresh_pipeline" } @@ -134,7 +134,7 @@ } } ], - "job_id": [NUMID], + "job_id": [SAMPLE_JOB_ID], "max_concurrent_runs": 4, "name": "[dev [USERNAME]] sample_job", "parameters": [ @@ -206,7 +206,7 @@ ], "email_notifications": {}, "pipeline_task": { - "pipeline_id": "[UUID]" + "pipeline_id": "[MY_DEFAULT_PYTHON_ETL_ID]" }, "run_if": "ALL_SUCCESS", "task_key": "refresh_pipeline", @@ -320,7 +320,7 @@ "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files" ] }, - "id": "[UUID]", + "id": "[MY_DEFAULT_PYTHON_ETL_ID]", "last_modified": [UNIX_TIME_MILLIS][1], "libraries": [ { @@ -330,12 +330,12 @@ } ], "name": "[dev [USERNAME]] my_default_python_etl", - "pipeline_id": "[UUID]", + "pipeline_id": "[MY_DEFAULT_PYTHON_ETL_ID]", "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl", "run_as_user_name": "[USERNAME]", "schema": "[USERNAME]", "state": "IDLE", - "storage": "dbfs:/pipelines/[UUID]", + "storage": "dbfs:/pipelines/[MY_DEFAULT_PYTHON_ETL_ID]", "tags": { "dev": "[USERNAME]" } @@ -344,7 +344,7 @@ "storage": { "action": "skip", "reason": "immutable_backend_value", - "remote": "dbfs:/pipelines/[UUID]" + "remote": "dbfs:/pipelines/[MY_DEFAULT_PYTHON_ETL_ID]" } } } diff --git a/acceptance/bundle/migrate/command/default-python/out.plan_after_migration.json b/acceptance/bundle/migrate/command/default-python/out.plan_after_migration.json index 05a4afe69d8..8df5a4684a9 100644 --- a/acceptance/bundle/migrate/command/default-python/out.plan_after_migration.json +++ b/acceptance/bundle/migrate/command/default-python/out.plan_after_migration.json @@ -96,7 +96,7 @@ } ], "pipeline_task": { - "pipeline_id": "[UUID]" + "pipeline_id": "[MY_DEFAULT_PYTHON_ETL_ID]" }, "task_key": "refresh_pipeline" } @@ -134,7 +134,7 @@ } } ], - "job_id": [NUMID], + "job_id": [SAMPLE_JOB_ID], "max_concurrent_runs": 4, "name": "[dev [USERNAME]] sample_job", "parameters": [ @@ -206,7 +206,7 @@ ], "email_notifications": {}, "pipeline_task": { - "pipeline_id": "[UUID]" + "pipeline_id": "[MY_DEFAULT_PYTHON_ETL_ID]" }, "run_if": "ALL_SUCCESS", "task_key": "refresh_pipeline", @@ -320,7 +320,7 @@ "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files" ] }, - "id": "[UUID]", + "id": "[MY_DEFAULT_PYTHON_ETL_ID]", "last_modified": [UNIX_TIME_MILLIS][1], "libraries": [ { @@ -330,12 +330,12 @@ } ], "name": "[dev [USERNAME]] my_default_python_etl", - "pipeline_id": "[UUID]", + "pipeline_id": "[MY_DEFAULT_PYTHON_ETL_ID]", "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl", "run_as_user_name": "[USERNAME]", "schema": "[USERNAME]", "state": "IDLE", - "storage": "dbfs:/pipelines/[UUID]", + "storage": "dbfs:/pipelines/[MY_DEFAULT_PYTHON_ETL_ID]", "tags": { "dev": "[USERNAME]" } @@ -344,7 +344,7 @@ "storage": { "action": "skip", "reason": "immutable_backend_value", - "remote": "dbfs:/pipelines/[UUID]" + "remote": "dbfs:/pipelines/[MY_DEFAULT_PYTHON_ETL_ID]" } } } diff --git a/acceptance/bundle/migrate/command/default-python/out.state_after_migration.json b/acceptance/bundle/migrate/command/default-python/out.state_after_migration.json index 484db23fd94..007548bcee5 100644 --- a/acceptance/bundle/migrate/command/default-python/out.state_after_migration.json +++ b/acceptance/bundle/migrate/command/default-python/out.state_after_migration.json @@ -5,7 +5,7 @@ "serial": 5, "state": { "resources.jobs.sample_job": { - "__id__": "[NUMID]", + "__id__": "[SAMPLE_JOB_ID]", "state": { "deployment": { "kind": "BUNDLE", @@ -89,7 +89,7 @@ } ], "pipeline_task": { - "pipeline_id": "[UUID]" + "pipeline_id": "[MY_DEFAULT_PYTHON_ETL_ID]" }, "task_key": "refresh_pipeline" } @@ -110,7 +110,7 @@ ] }, "resources.pipelines.my_default_python_etl": { - "__id__": "[UUID]", + "__id__": "[MY_DEFAULT_PYTHON_ETL_ID]", "state": { "channel": "CURRENT", "deployment": { diff --git a/acceptance/bundle/migrate/command/default-python/out.state_original.json b/acceptance/bundle/migrate/command/default-python/out.state_original.json index 1363591874f..049572d0136 100644 --- a/acceptance/bundle/migrate/command/default-python/out.state_original.json +++ b/acceptance/bundle/migrate/command/default-python/out.state_original.json @@ -44,7 +44,7 @@ "format": "MULTI_TASK", "git_source": [], "health": [], - "id": "[NUMID]", + "id": "[SAMPLE_JOB_ID]", "job_cluster": [ { "job_cluster_key": "job_cluster", @@ -123,7 +123,7 @@ "pipeline_task": [], "provider_config": [ { - "workspace_id": "[NUMID]" + "workspace_id": "[WORKSPACE_ID]" } ], "python_wheel_task": [], @@ -343,7 +343,7 @@ "full_refresh": false, "full_refresh_selection": null, "parameters": null, - "pipeline_id": "[UUID]", + "pipeline_id": "[MY_DEFAULT_PYTHON_ETL_ID]", "refresh_flow_selection": null, "refresh_selection": null, "reset_checkpoint_selection": null @@ -382,7 +382,7 @@ } ], "triggers": [], - "url": "[DATABRICKS_URL]/#job/[NUMID]", + "url": "[DATABRICKS_URL]/#job/[SAMPLE_JOB_ID]", "usage_policy_id": null, "webhook_notifications": [ { @@ -444,7 +444,7 @@ "filters": [], "gateway_definition": [], "health": null, - "id": "[UUID]", + "id": "[MY_DEFAULT_PYTHON_ETL_ID]", "ingestion_definition": [], "last_modified": [UNIX_TIME_MILLIS], "latest_updates": null, @@ -467,7 +467,7 @@ "photon": false, "provider_config": [ { - "workspace_id": "[NUMID]" + "workspace_id": "[WORKSPACE_ID]" } ], "restart_window": [], @@ -478,14 +478,14 @@ "serverless": false, "serverless_compute_id": null, "state": "IDLE", - "storage": "dbfs:/pipelines/[UUID]", + "storage": "dbfs:/pipelines/[MY_DEFAULT_PYTHON_ETL_ID]", "tags": { "dev": "[USERNAME]" }, "target": null, "timeouts": null, "trigger": [], - "url": "[DATABRICKS_URL]/#joblist/pipelines/[UUID]", + "url": "[DATABRICKS_URL]/#joblist/pipelines/[MY_DEFAULT_PYTHON_ETL_ID]", "usage_policy_id": null }, "sensitive_attributes": [], diff --git a/acceptance/bundle/migrate/command/default-python/output.txt b/acceptance/bundle/migrate/command/default-python/output.txt index 9a3bc0c6689..ed9dc1902b0 100644 --- a/acceptance/bundle/migrate/command/default-python/output.txt +++ b/acceptance/bundle/migrate/command/default-python/output.txt @@ -141,7 +141,7 @@ Building python_artifact... "storage": { "action": "skip", "reason": "immutable_backend_value", - "remote": "dbfs:/pipelines/[UUID]" + "remote": "dbfs:/pipelines/[MY_DEFAULT_PYTHON_ETL_ID]" } } @@ -260,6 +260,6 @@ Building python_artifact... "storage": { "action": "skip", "reason": "immutable_backend_value", - "remote": "dbfs:/pipelines/[UUID]" + "remote": "dbfs:/pipelines/[MY_DEFAULT_PYTHON_ETL_ID]" } } diff --git a/acceptance/bundle/migrate/command/default-python/script b/acceptance/bundle/migrate/command/default-python/script index 7f5a1078a20..ce5f97c944a 100755 --- a/acceptance/bundle/migrate/command/default-python/script +++ b/acceptance/bundle/migrate/command/default-python/script @@ -4,6 +4,7 @@ cd my_default_python # Replay a prior terraform deployment (a job and a DLT pipeline) trace replay_tfstate.py ../terraform-requests.json ../terraform.tfstate -t dev +replace_ids.py -t dev trace print_state.py > ../out.state_original.json trace $CLI bundle deployment migrate diff --git a/acceptance/bundle/migrate/command/grants/out.original_state.json b/acceptance/bundle/migrate/command/grants/out.original_state.json index 8aee4f3e101..b83be57aff1 100644 --- a/acceptance/bundle/migrate/command/grants/out.original_state.json +++ b/acceptance/bundle/migrate/command/grants/out.original_state.json @@ -36,7 +36,7 @@ "pipeline": null, "provider_config": [ { - "workspace_id": "[NUMID]" + "workspace_id": "[WORKSPACE_ID]" } ], "recipient": null, @@ -88,7 +88,7 @@ "pipeline": null, "provider_config": [ { - "workspace_id": "[NUMID]" + "workspace_id": "[WORKSPACE_ID]" } ], "recipient": null, @@ -139,7 +139,7 @@ "pipeline": null, "provider_config": [ { - "workspace_id": "[NUMID]" + "workspace_id": "[WORKSPACE_ID]" } ], "recipient": null, @@ -181,7 +181,7 @@ "owner": "[USERNAME]", "provider_config": [ { - "workspace_id": "[NUMID]" + "workspace_id": "[WORKSPACE_ID]" } ], "schema_name": "schema_grants", @@ -217,7 +217,7 @@ "properties": null, "provider_config": [ { - "workspace_id": "[NUMID]" + "workspace_id": "[WORKSPACE_ID]" } ], "schema_id": "[UUID]", @@ -244,7 +244,7 @@ "owner": "[USERNAME]", "provider_config": [ { - "workspace_id": "[NUMID]" + "workspace_id": "[WORKSPACE_ID]" } ], "schema_name": "schema_grants", diff --git a/acceptance/bundle/migrate/command/permissions/out.new_state.json b/acceptance/bundle/migrate/command/permissions/out.new_state.json index 53d41f1fe45..a2545a07093 100644 --- a/acceptance/bundle/migrate/command/permissions/out.new_state.json +++ b/acceptance/bundle/migrate/command/permissions/out.new_state.json @@ -5,7 +5,7 @@ "serial": 7, "state": { "resources.jobs.test_job": { - "__id__": "[NUMID]", + "__id__": "[TEST_JOB_ID]", "state": { "deployment": { "kind": "BUNDLE", @@ -29,9 +29,9 @@ } }, "resources.jobs.test_job.permissions": { - "__id__": "/jobs/[NUMID]", + "__id__": "/jobs/[TEST_JOB_ID]", "state": { - "object_id": "/jobs/[NUMID]", + "object_id": "/jobs/[TEST_JOB_ID]", "__embed__": [ { "level": "CAN_VIEW", diff --git a/acceptance/bundle/migrate/command/permissions/out.original_state.json b/acceptance/bundle/migrate/command/permissions/out.original_state.json index c7341d0ea88..e127e33e07b 100644 --- a/acceptance/bundle/migrate/command/permissions/out.original_state.json +++ b/acceptance/bundle/migrate/command/permissions/out.original_state.json @@ -44,7 +44,7 @@ "format": "MULTI_TASK", "git_source": [], "health": [], - "id": "[NUMID]", + "id": "[TEST_JOB_ID]", "job_cluster": [], "library": [], "max_concurrent_runs": 1, @@ -60,7 +60,7 @@ "pipeline_task": [], "provider_config": [ { - "workspace_id": "[NUMID]" + "workspace_id": "[WORKSPACE_ID]" } ], "python_wheel_task": [], @@ -147,7 +147,7 @@ "timeouts": null, "trigger": [], "triggers": [], - "url": "[DATABRICKS_URL]/#job/[NUMID]", + "url": "[DATABRICKS_URL]/#job/[TEST_JOB_ID]", "usage_policy_id": null, "webhook_notifications": [ { @@ -198,9 +198,9 @@ "directory_id": null, "directory_path": null, "experiment_id": null, - "id": "/jobs/[NUMID]", + "id": "/jobs/[TEST_JOB_ID]", "instance_pool_id": null, - "job_id": "[NUMID]", + "job_id": "[TEST_JOB_ID]", "knowledge_assistant_id": null, "notebook_id": null, "notebook_path": null, @@ -208,7 +208,7 @@ "pipeline_id": null, "provider_config": [ { - "workspace_id": "[NUMID]" + "workspace_id": "[WORKSPACE_ID]" } ], "registered_model_id": null, @@ -276,7 +276,7 @@ "pipeline_id": "[UUID]", "provider_config": [ { - "workspace_id": "[NUMID]" + "workspace_id": "[WORKSPACE_ID]" } ], "registered_model_id": null, @@ -358,7 +358,7 @@ "photon": false, "provider_config": [ { - "workspace_id": "[NUMID]" + "workspace_id": "[WORKSPACE_ID]" } ], "restart_window": [], diff --git a/acceptance/bundle/migrate/command/permissions/output.txt b/acceptance/bundle/migrate/command/permissions/output.txt index 182946cfffc..4b24a246e36 100644 --- a/acceptance/bundle/migrate/command/permissions/output.txt +++ b/acceptance/bundle/migrate/command/permissions/output.txt @@ -27,7 +27,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged "method": "GET", "path": "/api/2.2/jobs/get", "q": { - "job_id": "[NUMID]" + "job_id": "[TEST_JOB_ID]" } } { @@ -39,7 +39,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged "method": "GET", "path": "/api/2.2/jobs/get", "q": { - "job_id": "[NUMID]" + "job_id": "[TEST_JOB_ID]" } } diff --git a/acceptance/bundle/migrate/command/permissions/script b/acceptance/bundle/migrate/command/permissions/script index 75dcd39490b..8877d0da7d3 100755 --- a/acceptance/bundle/migrate/command/permissions/script +++ b/acceptance/bundle/migrate/command/permissions/script @@ -3,6 +3,7 @@ # this stands in for `DATABRICKS_BUNDLE_ENGINE=terraform bundle deploy` now that migrate tests must # not deploy with terraform. trace replay_tfstate.py terraform-requests.json terraform.tfstate +read_id.py test_job > /dev/null print_state.py > out.original_state.json trace $CLI bundle deployment migrate 2>&1 | contains.py 'Migrated 4 resources' diff --git a/acceptance/bundle/migrate/command/removed-all/output.txt b/acceptance/bundle/migrate/command/removed-all/output.txt index 886c4091c22..9084328ccd4 100644 --- a/acceptance/bundle/migrate/command/removed-all/output.txt +++ b/acceptance/bundle/migrate/command/removed-all/output.txt @@ -24,7 +24,7 @@ Deleted jobs.job_b Files: 1 uploaded, 0 deleted Resources: 0 created, 0 changed, 2 deleted, 0 unchanged ->>> print_requests.py //jobs/delete +>>> print_requests.py --sort //jobs/delete { "headers": { "User-Agent": [ @@ -34,7 +34,7 @@ Resources: 0 created, 0 changed, 2 deleted, 0 unchanged "method": "POST", "path": "/api/2.2/jobs/delete", "body": { - "job_id": [NUMID] + "job_id": [JOB_A_ID] } } { @@ -46,6 +46,6 @@ Resources: 0 created, 0 changed, 2 deleted, 0 unchanged "method": "POST", "path": "/api/2.2/jobs/delete", "body": { - "job_id": [NUMID] + "job_id": [JOB_B_ID] } } diff --git a/acceptance/bundle/migrate/command/removed-all/script b/acceptance/bundle/migrate/command/removed-all/script index 0a4cf0545e8..41d7fe33160 100644 --- a/acceptance/bundle/migrate/command/removed-all/script +++ b/acceptance/bundle/migrate/command/removed-all/script @@ -2,6 +2,7 @@ export DATABRICKS_BUNDLE_ENGINE= # Replay a prior terraform deployment (job_a and job_b) trace replay_tfstate.py terraform-requests.json terraform.tfstate +replace_ids.py # Remove every resource from config without deploying the deletion cat > databricks.yml <<'YAML' @@ -16,4 +17,4 @@ trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan | contains.py "2 to delet rm "$OUT_REQUESTS" trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy --auto-approve -trace print_requests.py //jobs/delete +trace print_requests.py --sort //jobs/delete diff --git a/acceptance/bundle/migrate/command/removed/output.txt b/acceptance/bundle/migrate/command/removed/output.txt index 948e659773e..a2326b43483 100644 --- a/acceptance/bundle/migrate/command/removed/output.txt +++ b/acceptance/bundle/migrate/command/removed/output.txt @@ -32,7 +32,7 @@ Resources: 0 created, 0 changed, 1 deleted, 1 unchanged "method": "POST", "path": "/api/2.2/jobs/delete", "body": { - "job_id": [NUMID] + "job_id": [JOB_B_ID] } } diff --git a/acceptance/bundle/migrate/command/removed/script b/acceptance/bundle/migrate/command/removed/script index 01a5d745c31..acae30b7549 100644 --- a/acceptance/bundle/migrate/command/removed/script +++ b/acceptance/bundle/migrate/command/removed/script @@ -2,6 +2,7 @@ export DATABRICKS_BUNDLE_ENGINE= # Replay a prior terraform deployment (both job_a and job_b) trace replay_tfstate.py terraform-requests.json terraform.tfstate +replace_ids.py # Remove job_b from config without deploying the deletion grep -v job_b databricks.yml > databricks_tmp.yml && mv databricks_tmp.yml databricks.yml diff --git a/acceptance/bundle/migrate/command/runas/out.old_state.json b/acceptance/bundle/migrate/command/runas/out.old_state.json index 1fa79c0b405..ffc6dcc50da 100644 --- a/acceptance/bundle/migrate/command/runas/out.old_state.json +++ b/acceptance/bundle/migrate/command/runas/out.old_state.json @@ -49,7 +49,7 @@ "pipeline_id": "[UUID]", "provider_config": [ { - "workspace_id": "[NUMID]" + "workspace_id": "[WORKSPACE_ID]" } ], "registered_model_id": null, @@ -131,7 +131,7 @@ "photon": false, "provider_config": [ { - "workspace_id": "[NUMID]" + "workspace_id": "[WORKSPACE_ID]" } ], "restart_window": [], diff --git a/acceptance/bundle/migrate/test.toml b/acceptance/bundle/migrate/test.toml index 7e9773a689b..ec69d990564 100644 --- a/acceptance/bundle/migrate/test.toml +++ b/acceptance/bundle/migrate/test.toml @@ -13,3 +13,8 @@ Env.DATABRICKS_BUNDLE_ENGINE = "direct" # These tests replay a prior terraform deployment (replay_tfstate.py) and then migrate to direct. # Recording is direct-only, and the DMS variant does not apply to the migration story. EnvMatrix.DMS = [""] + +# workspace list prints each state file's object id, minted per run. +[[Repls]] +Old = '(?m)^\d+(\s+FILE)' +New = '[OBJECT_ID]${1}' diff --git a/acceptance/bundle/open/output.txt b/acceptance/bundle/open/output.txt index 890b40998f4..603198f2c19 100644 --- a/acceptance/bundle/open/output.txt +++ b/acceptance/bundle/open/output.txt @@ -20,8 +20,8 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Use a fake browser that just prints the URL it would have opened === open after deployment >>> [CLI] bundle open foo -Opening browser at [DATABRICKS_URL]/jobs/[NUMID]?w=[NUMID] -[DATABRICKS_URL]/jobs/[NUMID]?w=[NUMID] +Opening browser at [DATABRICKS_URL]/jobs/[FOO_ID]?w=[WORKSPACE_ID] +[DATABRICKS_URL]/jobs/[FOO_ID]?w=[WORKSPACE_ID] === test auto-completion handler >>> [CLI] __complete bundle open , diff --git a/acceptance/bundle/open/script b/acceptance/bundle/open/script index 1fcc79ddbff..dfce55aa6da 100644 --- a/acceptance/bundle/open/script +++ b/acceptance/bundle/open/script @@ -5,6 +5,7 @@ title "not deployed yet" errcode trace $CLI bundle open foo errcode trace $CLI bundle deploy +replace_ids.py title "Use a fake browser that just prints the URL it would have opened" export BROWSER="echo_browser.py" diff --git a/acceptance/bundle/resource_deps/create_error/output.txt b/acceptance/bundle/resource_deps/create_error/output.txt index dcfc83f2ddf..9c44fd808d2 100644 --- a/acceptance/bundle/resource_deps/create_error/output.txt +++ b/acceptance/bundle/resource_deps/create_error/output.txt @@ -64,7 +64,7 @@ Resources: URL: (not deployed) independent: Name: independent - URL: [DATABRICKS_URL]/jobs/[INDEPENDENT_ID]?w=[NUMID] + URL: [DATABRICKS_URL]/jobs/[INDEPENDENT_ID]?w=[WORKSPACE_ID] === Plan should still contain foo and bar >>> [CLI] bundle plan diff --git a/acceptance/bundle/resource_deps/escaped_ref/output.txt b/acceptance/bundle/resource_deps/escaped_ref/output.txt index 612f722a335..0bc8c20e1e8 100644 --- a/acceptance/bundle/resource_deps/escaped_ref/output.txt +++ b/acceptance/bundle/resource_deps/escaped_ref/output.txt @@ -56,9 +56,9 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "notebook_task": { "base_parameters": { "escaped": "${resources.jobs.bar.id}", - "mixed": "${runtime} [NUMID]", - "real": "[NUMID]", - "same_name": "${resources.jobs.bar.id} [NUMID]" + "mixed": "${runtime} [BAR_ID]", + "real": "[BAR_ID]", + "same_name": "${resources.jobs.bar.id} [BAR_ID]" }, "notebook_path": "/notebook", "source": "WORKSPACE" diff --git a/acceptance/bundle/resource_deps/escaped_ref/script b/acceptance/bundle/resource_deps/escaped_ref/script index 459647cb1dc..ef8244e7eeb 100644 --- a/acceptance/bundle/resource_deps/escaped_ref/script +++ b/acceptance/bundle/resource_deps/escaped_ref/script @@ -12,6 +12,7 @@ trace jq '.plan | map_values(.depends_on)' plan.json | nostamp # that the payload below is identical either way. title "Payload: escaped stays literal, real is resolved\n" $CLI bundle deploy $(readplanarg plan.json) +replace_ids.py # --nostamp: under deployment-history recording the job payload carries the DMS deployment # stamp; drop it so the assertion is the same whether or not recording is on. trace print_requests.py //jobs --sort --nostamp diff --git a/acceptance/bundle/resource_deps/id_chain/out.plan_skip.json b/acceptance/bundle/resource_deps/id_chain/out.plan_skip.json index 216321f8b41..34e6950925c 100644 --- a/acceptance/bundle/resource_deps/id_chain/out.plan_skip.json +++ b/acceptance/bundle/resource_deps/id_chain/out.plan_skip.json @@ -17,7 +17,7 @@ "edit_mode": "UI_LOCKED", "email_notifications": {}, "format": "MULTI_TASK", - "job_id": [NUMID], + "job_id": [A_ID], "max_concurrent_runs": 1, "name": "aa", "queue": { @@ -60,11 +60,11 @@ "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" }, - "description": "new_prefix [NUMID]", + "description": "new_prefix [A_ID]", "edit_mode": "UI_LOCKED", "email_notifications": {}, "format": "MULTI_TASK", - "job_id": [NUMID], + "job_id": [B_ID], "max_concurrent_runs": 1, "name": "bb", "queue": { @@ -107,11 +107,11 @@ "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" }, - "description": "new_prefix [NUMID]", + "description": "new_prefix [B_ID]", "edit_mode": "UI_LOCKED", "email_notifications": {}, "format": "MULTI_TASK", - "job_id": [NUMID], + "job_id": [C_ID], "max_concurrent_runs": 1, "name": "cc", "queue": { @@ -154,11 +154,11 @@ "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" }, - "description": "new_prefix [NUMID]", + "description": "new_prefix [C_ID]", "edit_mode": "UI_LOCKED", "email_notifications": {}, "format": "MULTI_TASK", - "job_id": [NUMID], + "job_id": [D_ID], "max_concurrent_runs": 1, "name": "dd", "queue": { @@ -201,11 +201,11 @@ "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" }, - "description": "new_prefix [NUMID]", + "description": "new_prefix [D_ID]", "edit_mode": "UI_LOCKED", "email_notifications": {}, "format": "MULTI_TASK", - "job_id": [NUMID], + "job_id": [E_ID], "max_concurrent_runs": 1, "name": "ee", "queue": { diff --git a/acceptance/bundle/resource_deps/id_chain/out.plan_update.json b/acceptance/bundle/resource_deps/id_chain/out.plan_update.json index 991487aa832..182fe955fca 100644 --- a/acceptance/bundle/resource_deps/id_chain/out.plan_update.json +++ b/acceptance/bundle/resource_deps/id_chain/out.plan_update.json @@ -33,7 +33,7 @@ "edit_mode": "UI_LOCKED", "email_notifications": {}, "format": "MULTI_TASK", - "job_id": [NUMID], + "job_id": [A_ID], "max_concurrent_runs": 1, "name": "aa", "queue": { @@ -81,7 +81,7 @@ "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" }, - "description": "new_prefix [NUMID]", + "description": "new_prefix [A_ID]", "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, @@ -98,11 +98,11 @@ "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" }, - "description": "prefix [NUMID]", + "description": "prefix [A_ID]", "edit_mode": "UI_LOCKED", "email_notifications": {}, "format": "MULTI_TASK", - "job_id": [NUMID], + "job_id": [B_ID], "max_concurrent_runs": 1, "name": "bb", "queue": { @@ -115,9 +115,9 @@ "changes": { "description": { "action": "update", - "old": "prefix [NUMID]", - "new": "new_prefix [NUMID]", - "remote": "prefix [NUMID]" + "old": "prefix [A_ID]", + "new": "new_prefix [A_ID]", + "remote": "prefix [A_ID]" }, "email_notifications": { "action": "skip", @@ -150,7 +150,7 @@ "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" }, - "description": "new_prefix [NUMID]", + "description": "new_prefix [B_ID]", "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, @@ -167,11 +167,11 @@ "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" }, - "description": "prefix [NUMID]", + "description": "prefix [B_ID]", "edit_mode": "UI_LOCKED", "email_notifications": {}, "format": "MULTI_TASK", - "job_id": [NUMID], + "job_id": [C_ID], "max_concurrent_runs": 1, "name": "cc", "queue": { @@ -184,9 +184,9 @@ "changes": { "description": { "action": "update", - "old": "prefix [NUMID]", - "new": "new_prefix [NUMID]", - "remote": "prefix [NUMID]" + "old": "prefix [B_ID]", + "new": "new_prefix [B_ID]", + "remote": "prefix [B_ID]" }, "email_notifications": { "action": "skip", @@ -219,7 +219,7 @@ "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" }, - "description": "new_prefix [NUMID]", + "description": "new_prefix [C_ID]", "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, @@ -236,11 +236,11 @@ "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" }, - "description": "prefix [NUMID]", + "description": "prefix [C_ID]", "edit_mode": "UI_LOCKED", "email_notifications": {}, "format": "MULTI_TASK", - "job_id": [NUMID], + "job_id": [D_ID], "max_concurrent_runs": 1, "name": "dd", "queue": { @@ -253,9 +253,9 @@ "changes": { "description": { "action": "update", - "old": "prefix [NUMID]", - "new": "new_prefix [NUMID]", - "remote": "prefix [NUMID]" + "old": "prefix [C_ID]", + "new": "new_prefix [C_ID]", + "remote": "prefix [C_ID]" }, "email_notifications": { "action": "skip", @@ -288,7 +288,7 @@ "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" }, - "description": "new_prefix [NUMID]", + "description": "new_prefix [D_ID]", "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, @@ -305,11 +305,11 @@ "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" }, - "description": "prefix [NUMID]", + "description": "prefix [D_ID]", "edit_mode": "UI_LOCKED", "email_notifications": {}, "format": "MULTI_TASK", - "job_id": [NUMID], + "job_id": [E_ID], "max_concurrent_runs": 1, "name": "ee", "queue": { @@ -322,9 +322,9 @@ "changes": { "description": { "action": "update", - "old": "prefix [NUMID]", - "new": "new_prefix [NUMID]", - "remote": "prefix [NUMID]" + "old": "prefix [D_ID]", + "new": "new_prefix [D_ID]", + "remote": "prefix [D_ID]" }, "email_notifications": { "action": "skip", diff --git a/acceptance/bundle/resource_deps/id_chain/script b/acceptance/bundle/resource_deps/id_chain/script index 2a0c8fe3f4c..e24a3dd4612 100644 --- a/acceptance/bundle/resource_deps/id_chain/script +++ b/acceptance/bundle/resource_deps/id_chain/script @@ -20,6 +20,7 @@ $CLI bundle plan -o json | nostamp > out.plan_create.json trace print_requests trace $CLI bundle deploy +replace_ids.py trace print_requests_short trace update_file.py databricks.yml aa_desc aa_new_desc diff --git a/acceptance/bundle/resource_deps/model_id_ref/script b/acceptance/bundle/resource_deps/model_id_ref/script index 9fcec414295..3e2a1811b72 100644 --- a/acceptance/bundle/resource_deps/model_id_ref/script +++ b/acceptance/bundle/resource_deps/model_id_ref/script @@ -5,7 +5,7 @@ title "after deploy, registered_model_id is resolved to the model's numeric id" trace $CLI bundle deploy # Bind the model's exact numeric id to [MY_MODEL_ID] so the resolved reference is matched -# precisely rather than by the broad [NUMID] pattern. The id comes from the API (independent +# precisely. The id comes from the API (independent # of the deploy engine), so terraform and direct must both resolve to this exact value. model_id=$($CLI model-registry get-model my-model | jq -r '.registered_model_databricks.id') add_repl "$model_id" MY_MODEL_ID diff --git a/acceptance/bundle/resource_deps/run_state_resolution/jobs/output.txt b/acceptance/bundle/resource_deps/run_state_resolution/jobs/output.txt index cdf883c7fdf..51d09a97042 100644 --- a/acceptance/bundle/resource_deps/run_state_resolution/jobs/output.txt +++ b/acceptance/bundle/resource_deps/run_state_resolution/jobs/output.txt @@ -6,7 +6,7 @@ Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged, 2 not selected >>> [CLI] bundle run foo -Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[FOO_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "foo" RUNNING [TIMESTAMP] "foo" TERMINATED SUCCESS diff --git a/acceptance/bundle/resource_deps/run_state_resolution/jobs/script b/acceptance/bundle/resource_deps/run_state_resolution/jobs/script index 8e026519999..e5f979967c6 100644 --- a/acceptance/bundle/resource_deps/run_state_resolution/jobs/script +++ b/acceptance/bundle/resource_deps/run_state_resolution/jobs/script @@ -1,4 +1,5 @@ trace $CLI bundle deploy --select jobs.foo --auto-approve +replace_ids.py # bundle run resolves ${resources.*} only within the resource being run, so unrelated_job's # reference to the not-deployed bar is left untouched and no longer breaks the run. trace $CLI bundle run foo diff --git a/acceptance/bundle/resource_deps/run_state_resolution/jobs/test.toml b/acceptance/bundle/resource_deps/run_state_resolution/jobs/test.toml new file mode 100644 index 00000000000..3c887dd20bf --- /dev/null +++ b/acceptance/bundle/resource_deps/run_state_resolution/jobs/test.toml @@ -0,0 +1,3 @@ +[[Repls]] +Old = '/runs/\d+' +New = '/runs/[RUN_ID]' diff --git a/acceptance/bundle/resource_deps/unicode_reference/output.txt b/acceptance/bundle/resource_deps/unicode_reference/output.txt index 2d40580c0f2..969eade1b86 100644 --- a/acceptance/bundle/resource_deps/unicode_reference/output.txt +++ b/acceptance/bundle/resource_deps/unicode_reference/output.txt @@ -32,7 +32,7 @@ Resources: Jobs: foo: Name: ${resources.jobs.变量.id} - URL: [DATABRICKS_URL]/jobs/[NUMID]?w=[NUMID] + URL: [DATABRICKS_URL]/jobs/[FOO_ID]?w=[WORKSPACE_ID] 变量: Name: hello - URL: [DATABRICKS_URL]/jobs/[NUMID]?w=[NUMID] + URL: [DATABRICKS_URL]/jobs/[变量_ID]?w=[WORKSPACE_ID] diff --git a/acceptance/bundle/resource_deps/unicode_reference/script b/acceptance/bundle/resource_deps/unicode_reference/script index 1ae5d7992a2..b16b2154040 100644 --- a/acceptance/bundle/resource_deps/unicode_reference/script +++ b/acceptance/bundle/resource_deps/unicode_reference/script @@ -2,4 +2,5 @@ title "Unicode job names resolve in resources references" trace $CLI bundle validate trace $CLI bundle plan trace $CLI bundle deploy +replace_ids.py trace $CLI bundle summary diff --git a/acceptance/bundle/resources/alerts/with_file/test.toml b/acceptance/bundle/resources/alerts/with_file/test.toml index b9a91d1f9c4..3f3a5baee48 100644 --- a/acceptance/bundle/resources/alerts/with_file/test.toml +++ b/acceptance/bundle/resources/alerts/with_file/test.toml @@ -12,8 +12,8 @@ TimeoutCloud = "5m" # Setting this environment variable prevents that conversion on windows. Env.MSYS_NO_PATHCONV = "1" -# redact ?[ow]=[NUMID]. Different clouds can have different URL serialization, like [DATABRICKS_URL]/sql/alerts-v2/[ALERT_ID] vs [DATABRICKS_URL]/sql/alerts-v2/[ALERT_ID]?o=[NUMID]. +# redact ?[ow]=[WORKSPACE_ID]. Different clouds can have different URL serialization, like [DATABRICKS_URL]/sql/alerts-v2/[ALERT_ID] vs [DATABRICKS_URL]/sql/alerts-v2/[ALERT_ID]?o=[WORKSPACE_ID]. [[Repls]] -Old = '\?[ow]=\d+' +Old = '\?[ow]=\[WORKSPACE_ID\]' New = '' -Order = 0 +Order = 1000 diff --git a/acceptance/bundle/resources/alerts/with_file_run_from_subdir/output.txt b/acceptance/bundle/resources/alerts/with_file_run_from_subdir/output.txt index 337ef7876b3..af27a587c49 100644 --- a/acceptance/bundle/resources/alerts/with_file_run_from_subdir/output.txt +++ b/acceptance/bundle/resources/alerts/with_file_run_from_subdir/output.txt @@ -29,7 +29,7 @@ "quartz_cron_schedule": "44 19 */1 * * ?", "timezone_id": "Europe/Amsterdam" }, - "warehouse_id": "[NUMID]" + "warehouse_id": "0123456789012345" } === validate from a subdirectory of the bundle @@ -62,5 +62,5 @@ "quartz_cron_schedule": "44 19 */1 * * ?", "timezone_id": "Europe/Amsterdam" }, - "warehouse_id": "[NUMID]" + "warehouse_id": "0123456789012345" } diff --git a/acceptance/bundle/resources/apps/create_already_exists/output.txt b/acceptance/bundle/resources/apps/create_already_exists/output.txt index b6b3ccde8cc..b8440469192 100644 --- a/acceptance/bundle/resources/apps/create_already_exists/output.txt +++ b/acceptance/bundle/resources/apps/create_already_exists/output.txt @@ -2,7 +2,7 @@ >>> [CLI] apps create test-app-already-exists { "active_deployment": { - "deployment_id": "deploy-[NUMID]", + "deployment_id": "[APP_DEPLOYMENT_ID]", "source_code_path": "/Workspace/Users/[USERNAME]/test-app-already-exists", "status": { "message": "Deployment succeeded", @@ -23,7 +23,7 @@ "id": "1000", "name": "test-app-already-exists", "service_principal_client_id": "[UUID]", - "service_principal_id": [NUMID], + "service_principal_id": [SERVICE_PRINCIPAL_ID], "service_principal_name": "app-test-app-already-exists", "url": "test-app-already-exists-123.cloud.databricksapps.com" } diff --git a/acceptance/bundle/resources/apps/create_already_exists/script b/acceptance/bundle/resources/apps/create_already_exists/script index df26cb88959..845c56b830d 100644 --- a/acceptance/bundle/resources/apps/create_already_exists/script +++ b/acceptance/bundle/resources/apps/create_already_exists/script @@ -1,5 +1,8 @@ # Create an app with the same name outside of bundle -trace $CLI apps create test-app-already-exists +trace $CLI apps create test-app-already-exists | tee app.json.tmp +add_repl "$(sed -n 's/.*"deployment_id": "\(.*\)".*/\1/p' app.json.tmp)" APP_DEPLOYMENT_ID +add_repl "$(sed -n 's/.*"service_principal_id": \([0-9]*\).*/\1/p' app.json.tmp)" SERVICE_PRINCIPAL_ID +rm app.json.tmp # Bundle deploy should fail because app already exists and is not in DELETING state trace musterr $CLI bundle deploy diff --git a/acceptance/bundle/resources/apps/resource-refs/output.txt b/acceptance/bundle/resources/apps/resource-refs/output.txt index 2e4451621e8..1690bee41eb 100644 --- a/acceptance/bundle/resources/apps/resource-refs/output.txt +++ b/acceptance/bundle/resources/apps/resource-refs/output.txt @@ -54,7 +54,7 @@ You can access the app at data-app-123.cloud.databricksapps.com }, { "name": "MY_EXAMPLE_JOB_ID", - "value": "[NUMID]" + "value": "[EXAMPLE_JOB_ID]" }, { "name": "MY_EXAMPLE_VAR", diff --git a/acceptance/bundle/resources/apps/resource-refs/script b/acceptance/bundle/resources/apps/resource-refs/script index ace337470da..79ad39f683a 100644 --- a/acceptance/bundle/resources/apps/resource-refs/script +++ b/acceptance/bundle/resources/apps/resource-refs/script @@ -1,3 +1,4 @@ trace $CLI bundle deploy +read_id.py example_job > /dev/null trace $CLI bundle run data_app trace print_requests.py //apps diff --git a/acceptance/bundle/resources/cluster_policies/basic/output.txt b/acceptance/bundle/resources/cluster_policies/basic/output.txt index e2ef74525ba..0ba76cc3b55 100644 --- a/acceptance/bundle/resources/cluster_policies/basic/output.txt +++ b/acceptance/bundle/resources/cluster_policies/basic/output.txt @@ -55,7 +55,7 @@ Resources: Cluster Policies: test_cluster_policy: Name: my_cluster_policy-[UNIQUE_NAME] - URL: [DATABRICKS_URL]/compute/policies/[TEST_CLUSTER_POLICY_ID]?w=[NUMID] + URL: [DATABRICKS_URL]/compute/policies/[TEST_CLUSTER_POLICY_ID]?w=[WORKSPACE_ID] === Update the cluster policy name >>> update_file.py databricks.yml my_cluster_policy my_cluster_policy_2 @@ -88,7 +88,7 @@ Resources: Cluster Policies: test_cluster_policy: Name: my_cluster_policy_2-[UNIQUE_NAME] - URL: [DATABRICKS_URL]/compute/policies/[TEST_CLUSTER_POLICY_ID]?w=[NUMID] + URL: [DATABRICKS_URL]/compute/policies/[TEST_CLUSTER_POLICY_ID]?w=[WORKSPACE_ID] === Destroy the cluster policy >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/cluster_policies/job_ref/output.txt b/acceptance/bundle/resources/cluster_policies/job_ref/output.txt index 78d3b85ff01..1172c061bde 100644 --- a/acceptance/bundle/resources/cluster_policies/job_ref/output.txt +++ b/acceptance/bundle/resources/cluster_policies/job_ref/output.txt @@ -70,7 +70,7 @@ Resources: 0 created, 2 changed, 0 deleted, 0 unchanged "method": "POST", "path": "/api/2.2/jobs/reset", "body": { - "job_id": [NUMID], + "job_id": [J_ID], "new_settings": { "deployment": { "kind": "BUNDLE", @@ -112,7 +112,7 @@ Destroy: 2 deleted "method": "POST", "path": "/api/2.2/jobs/delete", "body": { - "job_id": [NUMID] + "job_id": [J_ID] } } { diff --git a/acceptance/bundle/resources/cluster_policies/job_ref/script b/acceptance/bundle/resources/cluster_policies/job_ref/script index 1117129a0eb..5b273fb62aa 100644 --- a/acceptance/bundle/resources/cluster_policies/job_ref/script +++ b/acceptance/bundle/resources/cluster_policies/job_ref/script @@ -1,7 +1,7 @@ trace $CLI bundle deploy -# Register [POL_ID] so the resolved reference in the job body is deterministic. -pol_id=`read_id.py pol` +# Register [POL_ID] and [J_ID] so the resolved references are deterministic. +replace_ids.py title "Deploy requests in dependency order: policy create precedes job create, job carries resolved policy id" trace print_requests.py //policies/clusters //jobs --nostamp diff --git a/acceptance/bundle/resources/clusters/deploy/instance_pool/output.txt b/acceptance/bundle/resources/clusters/deploy/instance_pool/output.txt index 11fac468d1f..3d95b26f400 100644 --- a/acceptance/bundle/resources/clusters/deploy/instance_pool/output.txt +++ b/acceptance/bundle/resources/clusters/deploy/instance_pool/output.txt @@ -15,7 +15,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "autotermination_minutes": 60, "aws_attributes": {}, "cluster_name": "test-cluster-1", - "instance_pool_id": "ip-[NUMID]", + "instance_pool_id": "ip-1234567890", "num_workers": 2, "spark_conf": { "spark.executor.memory": "2g" @@ -30,7 +30,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "autotermination_minutes": 60, "azure_attributes": {}, "cluster_name": "test-cluster-2", - "instance_pool_id": "ip-[NUMID]", + "instance_pool_id": "ip-1234567890", "num_workers": 2, "spark_conf": { "spark.executor.memory": "2g" @@ -45,7 +45,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "autotermination_minutes": 60, "cluster_name": "test-cluster-3", "gcp_attributes": {}, - "instance_pool_id": "ip-[NUMID]", + "instance_pool_id": "ip-1234567890", "num_workers": 2, "spark_conf": { "spark.executor.memory": "2g" diff --git a/acceptance/bundle/resources/clusters/deploy/instance_pool_and_node_type/output.txt b/acceptance/bundle/resources/clusters/deploy/instance_pool_and_node_type/output.txt index c06f5faab08..375dae7c781 100644 --- a/acceptance/bundle/resources/clusters/deploy/instance_pool_and_node_type/output.txt +++ b/acceptance/bundle/resources/clusters/deploy/instance_pool_and_node_type/output.txt @@ -13,7 +13,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "autotermination_minutes": 60, "aws_attributes": {}, "cluster_name": "test-cluster", - "instance_pool_id": "ip-[NUMID]", + "instance_pool_id": "ip-1234567890", "num_workers": 2, "spark_conf": { "spark.executor.memory": "2g" diff --git a/acceptance/bundle/resources/clusters/run/spark_python_task/output.txt b/acceptance/bundle/resources/clusters/run/spark_python_task/output.txt index e546270942d..b0453085244 100644 --- a/acceptance/bundle/resources/clusters/run/spark_python_task/output.txt +++ b/acceptance/bundle/resources/clusters/run/spark_python_task/output.txt @@ -11,7 +11,7 @@ Files: 3 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle run foo -Run URL: [DATABRICKS_URL]/jobs/[FOO_ID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[FOO_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "test-job-with-cluster-[UNIQUE_NAME]" RUNNING [TIMESTAMP] "test-job-with-cluster-[UNIQUE_NAME]" TERMINATED SUCCESS diff --git a/acceptance/bundle/resources/clusters/run/spark_python_task/test.toml b/acceptance/bundle/resources/clusters/run/spark_python_task/test.toml index 5172de730ad..8c6e5133c82 100644 --- a/acceptance/bundle/resources/clusters/run/spark_python_task/test.toml +++ b/acceptance/bundle/resources/clusters/run/spark_python_task/test.toml @@ -10,8 +10,8 @@ Ignore = [ ] [[Repls]] -Old = '\d{5,}' -New = "[NUMID]" +Old = '/runs/\d+' +New = '/runs/[RUN_ID]' [[Repls]] Old = '\\\\' diff --git a/acceptance/bundle/resources/dashboards/change-serialized-inline/out.plan.json b/acceptance/bundle/resources/dashboards/change-serialized-inline/out.plan.json index a6d717d34f5..f4532ccbb36 100644 --- a/acceptance/bundle/resources/dashboards/change-serialized-inline/out.plan.json +++ b/acceptance/bundle/resources/dashboards/change-serialized-inline/out.plan.json @@ -21,7 +21,7 @@ "dashboard_id": "[DASHBOARD_ID]", "display_name": "Inline Dash", "embed_credentials": false, - "etag": "[NUMID]", + "etag": "80611980", "lifecycle_state": "ACTIVE", "parent_path": "/Workspace/Users/[USERNAME]/.bundle/change-serialized-inline-[UNIQUE_NAME]/default/resources", "path": "/Users/[USERNAME]/.bundle/change-serialized-inline-[UNIQUE_NAME]/default/resources/Inline Dash.lvdash.json", @@ -44,8 +44,8 @@ "etag": { "action": "skip", "reason": "custom", - "old": "[NUMID]", - "remote": "[NUMID]" + "old": "80611980", + "remote": "80611980" }, "lifecycle_state": { "action": "skip", diff --git a/acceptance/bundle/resources/dashboards/detect-change/test.toml b/acceptance/bundle/resources/dashboards/detect-change/test.toml index 8d7d80eb679..590c5448008 100644 --- a/acceptance/bundle/resources/dashboards/detect-change/test.toml +++ b/acceptance/bundle/resources/dashboards/detect-change/test.toml @@ -21,7 +21,7 @@ New = "[ALPHANUMID]" [[Repls]] # clean up ?[ow]= suffix after URL since not all workspaces have that -Old = '\?[ow]=\[(NUMID|ALPHANUMID)\]' +Old = '\?[ow]=\[(WORKSPACE_ID|ALPHANUMID)\]' New = '' Order = 1000 diff --git a/acceptance/bundle/resources/dashboards/publish-failure-stale-content/test.toml b/acceptance/bundle/resources/dashboards/publish-failure-stale-content/test.toml index a60e4060d9f..b02f297e0e6 100644 --- a/acceptance/bundle/resources/dashboards/publish-failure-stale-content/test.toml +++ b/acceptance/bundle/resources/dashboards/publish-failure-stale-content/test.toml @@ -6,7 +6,3 @@ Ignore = [ ".databricks", "databricks.yml", ] - -[[Repls]] -Old = '\?[ow]=\d+' -New = "?[WSPARAM]=[NUMID]" diff --git a/acceptance/bundle/resources/database_instances/single-instance/test.toml b/acceptance/bundle/resources/database_instances/single-instance/test.toml index f620e35a30e..ef8d35d14b4 100644 --- a/acceptance/bundle/resources/database_instances/single-instance/test.toml +++ b/acceptance/bundle/resources/database_instances/single-instance/test.toml @@ -9,6 +9,6 @@ Ignore = [ [[Repls]] # clean up ?[ow]= suffix after URL since not all workspaces have that -Old = '\?[ow]=\[(NUMID|ALPHANUMID)\]' +Old = '\?[ow]=\[(WORKSPACE_ID|ALPHANUMID)\]' New = '' Order = 1000 diff --git a/acceptance/bundle/resources/experiments/basic/output.txt b/acceptance/bundle/resources/experiments/basic/output.txt index 9c1cf620c81..fc5ad8ebb98 100644 --- a/acceptance/bundle/resources/experiments/basic/output.txt +++ b/acceptance/bundle/resources/experiments/basic/output.txt @@ -13,7 +13,7 @@ Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged >>> [CLI] bundle deploy -qq ->>> [CLI] experiments get-experiment [NUMID] +>>> [CLI] experiments get-experiment [MY_EXPERIMENT_ID] { "name": "/Users/[USERNAME]/original-[UNIQUE_NAME]", "artifact_location": "s3://original-[UNIQUE_NAME]", @@ -57,7 +57,7 @@ Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged >>> [CLI] bundle deploy -qq ->>> [CLI] experiments get-experiment [NUMID] +>>> [CLI] experiments get-experiment [MY_EXPERIMENT_ID] { "name": "/Users/[USERNAME]/new-name-[UNIQUE_NAME]", "artifact_location": "s3://original-[UNIQUE_NAME]", @@ -101,7 +101,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged >>> [CLI] bundle deploy -qq ->>> [CLI] experiments get-experiment [NUMID] +>>> [CLI] experiments get-experiment [MY_EXPERIMENT_ID_3] { "name": "/Users/[USERNAME]/new-name-[UNIQUE_NAME]", "artifact_location": "s3://new-[UNIQUE_NAME]", @@ -140,7 +140,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged >>> [CLI] bundle deploy -qq ->>> [CLI] experiments get-experiment [NUMID] +>>> [CLI] experiments get-experiment [MY_EXPERIMENT_ID_3] { "name": "/Users/[USERNAME]/new-name-[UNIQUE_NAME]", "artifact_location": "s3://new-[UNIQUE_NAME]", diff --git a/acceptance/bundle/resources/experiments/basic/script b/acceptance/bundle/resources/experiments/basic/script index cdb522e9184..ab6ca3de78d 100755 --- a/acceptance/bundle/resources/experiments/basic/script +++ b/acceptance/bundle/resources/experiments/basic/script @@ -10,7 +10,7 @@ trace export EXPERIMENT_NAME_PREFIX="original" trace envsubst < templates/one_tag.tmpl > databricks.yml trace $CLI bundle plan trace $CLI bundle deploy -qq -experiment_id=$($CLI bundle summary --output json | jq -r '.resources.experiments.my_experiment.id') +experiment_id=$(read_id.py my_experiment) trace $CLI experiments get-experiment $experiment_id | jq '.experiment | {name, artifact_location, tags}' title "update the name" @@ -18,7 +18,7 @@ trace export EXPERIMENT_NAME_PREFIX="new-name" trace envsubst < templates/one_tag.tmpl > databricks.yml trace $CLI bundle plan trace $CLI bundle deploy -qq -experiment_id=$($CLI bundle summary --output json | jq -r '.resources.experiments.my_experiment.id') +experiment_id=$(read_id.py my_experiment) trace $CLI experiments get-experiment $experiment_id | jq '.experiment | {name, artifact_location, tags}' title "updating the artifact location should cause a recreation" @@ -26,12 +26,12 @@ trace export EXPERIMENT_ARTIFACT_LOCATION="s3://new-${UNIQUE_NAME}" trace envsubst < templates/one_tag.tmpl > databricks.yml trace $CLI bundle plan trace $CLI bundle deploy -qq -experiment_id=$($CLI bundle summary --output json | jq -r '.resources.experiments.my_experiment.id') +experiment_id=$(read_id.py my_experiment) trace $CLI experiments get-experiment $experiment_id | jq '.experiment | {name, artifact_location, tags}' title "adding a new tag should be a no-op" trace envsubst < templates/two_tag.tmpl > databricks.yml trace $CLI bundle plan > out.plan.txt trace $CLI bundle deploy -qq -experiment_id=$($CLI bundle summary --output json | jq -r '.resources.experiments.my_experiment.id') +experiment_id=$(read_id.py my_experiment) trace $CLI experiments get-experiment $experiment_id | jq '.experiment | {name, artifact_location, tags}' diff --git a/acceptance/bundle/resources/experiments/basic/test.toml b/acceptance/bundle/resources/experiments/basic/test.toml index d77ea9aa4c7..7d7ae48db25 100644 --- a/acceptance/bundle/resources/experiments/basic/test.toml +++ b/acceptance/bundle/resources/experiments/basic/test.toml @@ -1,7 +1,3 @@ Cloud = true RecordRequests = false RunsOnDbr = true - -[[Repls]] -Old = '\d{3,}' -New = "[NUMID]" diff --git a/acceptance/bundle/resources/instance_pools/output.txt b/acceptance/bundle/resources/instance_pools/output.txt index 724d71e4df3..950c7d70287 100644 --- a/acceptance/bundle/resources/instance_pools/output.txt +++ b/acceptance/bundle/resources/instance_pools/output.txt @@ -61,7 +61,7 @@ Resources: Instance Pools: test_instance_pool: Name: my_instance_pool - URL: [DATABRICKS_URL]/compute/instance-pools/[UUID]?w=[NUMID] + URL: [DATABRICKS_URL]/compute/instance-pools/[UUID]?w=[WORKSPACE_ID] === Update the instance pool name >>> update_file.py databricks.yml my_instance_pool my_instance_pool_2 @@ -97,7 +97,7 @@ Resources: Instance Pools: test_instance_pool: Name: my_instance_pool_2 - URL: [DATABRICKS_URL]/compute/instance-pools/[UUID]?w=[NUMID] + URL: [DATABRICKS_URL]/compute/instance-pools/[UUID]?w=[WORKSPACE_ID] === Destroy the instance pool >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/job_runs/basic/output.txt b/acceptance/bundle/resources/job_runs/basic/output.txt index dbd3c608a14..ba8bef3a0c8 100644 --- a/acceptance/bundle/resources/job_runs/basic/output.txt +++ b/acceptance/bundle/resources/job_runs/basic/output.txt @@ -34,7 +34,7 @@ Resources: >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-basic/default/files... Created jobs.my_job -Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[WORKSPACE_ID] Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Files: 1 uploaded, 0 deleted @@ -58,11 +58,11 @@ Resources: Job Runs: my_run: Name: - URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?w=[NUMID] + URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?w=[WORKSPACE_ID] Jobs: my_job: Name: my-job - URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]?w=[NUMID] + URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]?w=[WORKSPACE_ID] === exactly one run-now request was made >>> print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/interrupted_run/output.txt b/acceptance/bundle/resources/job_runs/interrupted_run/output.txt index e759bffe272..7b911a49c38 100644 --- a/acceptance/bundle/resources/job_runs/interrupted_run/output.txt +++ b/acceptance/bundle/resources/job_runs/interrupted_run/output.txt @@ -45,7 +45,7 @@ Exit code: 1 "job_id": [MY_JOB_ID], "run_id": [MY_RUN_ID], "run_name": "my-job", - "run_page_url": "[DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID]", + "run_page_url": "[DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[WORKSPACE_ID]", "run_type": "JOB_RUN", "state": { "life_cycle_state": "RUNNING" diff --git a/acceptance/bundle/resources/job_runs/job_parameters/output.txt b/acceptance/bundle/resources/job_runs/job_parameters/output.txt index e89fbc7094b..d3c5ff16368 100644 --- a/acceptance/bundle/resources/job_runs/job_parameters/output.txt +++ b/acceptance/bundle/resources/job_runs/job_parameters/output.txt @@ -3,7 +3,7 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-job-parameters/default/files... Created jobs.my_job -Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[WORKSPACE_ID] Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Files: 1 uploaded, 0 deleted @@ -36,11 +36,11 @@ Resources: Job Runs: my_run: Name: - URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?w=[NUMID] + URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?w=[WORKSPACE_ID] Jobs: my_job: Name: my-job - URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]?w=[NUMID] + URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]?w=[WORKSPACE_ID] >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt b/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt index dcf574a7d4f..f910dffcc05 100644 --- a/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt +++ b/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt @@ -3,7 +3,7 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-bundle-deploy/default/files... Created jobs.my_job -Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[WORKSPACE_ID] Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Files: 1 uploaded, 0 deleted @@ -46,7 +46,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged === bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-bundle-deploy/default/files... -Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[WORKSPACE_ID] Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS Recreated job_runs.my_run Files: 1 uploaded, 0 deleted @@ -62,11 +62,11 @@ Resources: Job Runs: my_run: Name: - URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?w=[NUMID] + URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?w=[WORKSPACE_ID] Jobs: my_job: Name: my-job - URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]?w=[NUMID] + URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]?w=[WORKSPACE_ID] === second run-now after recreate >>> print_requests.py --keep //jobs/runs/delete diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index 099497398b4..cd580a8efa4 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -3,7 +3,7 @@ >>> [CLI] bundle plan -o json Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... Created jobs.my_job -Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[WORKSPACE_ID] Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Files: 4 uploaded, 0 deleted @@ -66,7 +66,7 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged "result_state": "SUCCESS", "run_id": [MY_RUN_ID], "run_name": "my-job", - "run_page_url": "[DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID]", + "run_page_url": "[DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[WORKSPACE_ID]", "run_type": "JOB_RUN", "state": { "life_cycle_state": "TERMINATED", @@ -91,7 +91,7 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged === bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... -Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[WORKSPACE_ID] Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS Recreated job_runs.my_run Files: 2 uploaded, 0 deleted @@ -145,7 +145,7 @@ Warning: lifecycle.triggers.on_file_change: no synced files match "seed.txt" "result_state": "SUCCESS", "run_id": [MY_RUN_ID_2], "run_name": "my-job", - "run_page_url": "[DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID]", + "run_page_url": "[DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[WORKSPACE_ID]", "run_type": "JOB_RUN", "state": { "life_cycle_state": "TERMINATED", @@ -171,7 +171,7 @@ Warning: lifecycle.triggers.on_file_change: no synced files match "seed.txt" in databricks.yml:18:29 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... -Output from job_runs.my_run: id=[MY_RUN_ID_3]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_3]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_3]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_3]?o=[WORKSPACE_ID] Output from job_runs.my_run: id=[MY_RUN_ID_3]: SUCCESS Recreated job_runs.my_run Files: 2 uploaded, 1 deleted @@ -196,7 +196,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === restoring the file re-fires again >>> [CLI] bundle plan -o json Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... -Output from job_runs.my_run: id=[MY_RUN_ID_4]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_4]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_4]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_4]?o=[WORKSPACE_ID] Output from job_runs.my_run: id=[MY_RUN_ID_4]: SUCCESS Recreated job_runs.my_run Files: 2 uploaded, 1 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change/test.toml b/acceptance/bundle/resources/job_runs/on_file_change/test.toml index f5010a202d7..4585bcedd0c 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change/test.toml @@ -4,11 +4,9 @@ EnvMatrix.READPLAN = ["", "1"] Ignore = ["tmp.*.*"] -# Content hashes are stable but unreadable, and the parent's `\d{8,}` rule -# mangles any digit run inside them. Order=1 runs before it; Distinct keeps -# different hashes distinguishable. +# Content hashes are stable but unreadable. Distinct keeps different hashes +# distinguishable. [[Repls]] Old = '[0-9a-f]{64}' New = '[FILE_HASH]' -Order = 1 Distinct = true diff --git a/acceptance/bundle/resources/job_runs/on_file_change_build/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_build/output.txt index fa2cc712902..8cde92e29b9 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_build/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_build/output.txt @@ -4,7 +4,7 @@ Executing 'prebuild' script Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-build/default/files... Created jobs.my_job -Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[WORKSPACE_ID] Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Files: 3 uploaded, 0 deleted @@ -27,7 +27,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle deploy Executing 'prebuild' script Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-build/default/files... -Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[WORKSPACE_ID] Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS Recreated job_runs.my_run Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index 3963f98f4c1..ed04ecb79d3 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -3,7 +3,7 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... Created jobs.my_job -Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[WORKSPACE_ID] Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Files: 6 uploaded, 0 deleted @@ -42,7 +42,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... -Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[WORKSPACE_ID] Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS Recreated job_runs.my_run Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml index 45b345e15a7..98884820722 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -4,10 +4,8 @@ EnvMatrix.READPLAN = ["", "1"] Ignore = ["tmp.plan.json", "tmp.002.txt", "tmp.databricks.yml", ".gitignore"] -# See on_file_change/test.toml: mask content hashes before the parent's -# `\d{8,}` rule can mangle them, keeping distinct hashes distinguishable. +# See on_file_change/test.toml: mask content hashes, keeping distinct hashes distinguishable. [[Repls]] Old = '[0-9a-f]{64}' New = '[FILE_HASH]' -Order = 1 Distinct = true diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt index b7dd158c070..664e42bf0c0 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt @@ -3,8 +3,8 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-include-sync-root/default/files... Created jobs.my_job -Output from job_runs.my_run: id=[NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] -Output from job_runs.my_run: id=[NUMID]: SUCCESS +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[WORKSPACE_ID] +Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Files: 5 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script index 7f7934d00cf..2edf57688cd 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script @@ -13,6 +13,7 @@ file_triggers() { title "first deploy" withdir bundle trace $CLI bundle deploy +withdir bundle replace_ids.py trace file_triggers title "editing the file next to the included YAML re-fires" diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/test.toml index b0362c07bc0..19affe700d6 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/test.toml @@ -1,9 +1,8 @@ RecordRequests = false Ignore = [".databricks"] -# Mask hashes before the parent's `\d{8,}` rule; Distinct keeps them distinguishable. +# Mask content hashes; Distinct keeps them distinguishable. [[Repls]] Old = '[0-9a-f]{64}' New = '[FILE_HASH]' -Order = 1 Distinct = true diff --git a/acceptance/bundle/resources/job_runs/on_file_change_predeploy/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_predeploy/output.txt index 1ea9a3ff990..bbf77481367 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_predeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_predeploy/output.txt @@ -4,7 +4,7 @@ Executing 'predeploy' script Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-predeploy/default/files... Created jobs.my_job -Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[WORKSPACE_ID] Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Files: 3 uploaded, 0 deleted @@ -27,7 +27,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle deploy Executing 'predeploy' script Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-predeploy/default/files... -Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[WORKSPACE_ID] Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS Recreated job_runs.my_run Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt index 2413d50f7a5..577db05fae9 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -3,7 +3,7 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-redeploy/default/files... Created jobs.my_job -Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[WORKSPACE_ID] Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Files: 1 uploaded, 0 deleted @@ -19,11 +19,11 @@ Resources: Job Runs: my_run: Name: - URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?w=[NUMID] + URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?w=[WORKSPACE_ID] Jobs: my_job: Name: my-job - URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]?w=[NUMID] + URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]?w=[WORKSPACE_ID] >>> read_id.py my_job [MY_JOB_ID] @@ -70,7 +70,7 @@ Resources: "result_state": "SUCCESS", "run_id": [MY_RUN_ID], "run_name": "my-job", - "run_page_url": "[DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID]", + "run_page_url": "[DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[WORKSPACE_ID]", "run_type": "JOB_RUN", "state": { "life_cycle_state": "TERMINATED", @@ -96,7 +96,7 @@ Resources: >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-redeploy/default/files... -Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[WORKSPACE_ID] Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS Recreated job_runs.my_run Files: 1 uploaded, 0 deleted @@ -112,11 +112,11 @@ Resources: Job Runs: my_run: Name: - URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?w=[NUMID] + URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?w=[WORKSPACE_ID] Jobs: my_job: Name: my-job - URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]?w=[NUMID] + URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]?w=[WORKSPACE_ID] === the config change triggered a second, different run and left the previous one alone >>> print_requests.py --keep //jobs/runs/delete diff --git a/acceptance/bundle/resources/job_runs/retried_run_now/output.txt b/acceptance/bundle/resources/job_runs/retried_run_now/output.txt index 635f3c8ed7d..bf30c0cb2c0 100644 --- a/acceptance/bundle/resources/job_runs/retried_run_now/output.txt +++ b/acceptance/bundle/resources/job_runs/retried_run_now/output.txt @@ -3,7 +3,7 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-retried-run-now/default/files... Created jobs.my_job -Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[WORKSPACE_ID] Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Files: 1 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/jobs/big_id/out.state.json b/acceptance/bundle/resources/jobs/big_id/out.state.json index 7bbc03c50c2..b000f4ee5df 100644 --- a/acceptance/bundle/resources/jobs/big_id/out.state.json +++ b/acceptance/bundle/resources/jobs/big_id/out.state.json @@ -5,7 +5,7 @@ "serial": 1, "state": { "resources.jobs.foo": { - "__id__": "[NUMID]", + "__id__": "[FOO_ID]", "state": { "deployment": { "kind": "BUNDLE", diff --git a/acceptance/bundle/resources/jobs/big_id/output.txt b/acceptance/bundle/resources/jobs/big_id/output.txt index 79bb0c87994..f14ccc9e22e 100644 --- a/acceptance/bundle/resources/jobs/big_id/output.txt +++ b/acceptance/bundle/resources/jobs/big_id/output.txt @@ -74,6 +74,6 @@ Destroy: 1 deleted "method": "POST", "path": "/api/2.2/jobs/delete", "body": { - "job_id": [NUMID] + "job_id": [FOO_ID] } } diff --git a/acceptance/bundle/resources/jobs/big_id/script b/acceptance/bundle/resources/jobs/big_id/script index b3a3291d8f1..7463d436c45 100644 --- a/acceptance/bundle/resources/jobs/big_id/script +++ b/acceptance/bundle/resources/jobs/big_id/script @@ -4,6 +4,7 @@ trace $CLI bundle validate -o json | jq .resources > out.validate.json trace $CLI bundle plan -o json > .databricks/tmp.plan.json nostamp < .databricks/tmp.plan.json > out.plan.json $CLI bundle deploy $(readplanarg .databricks/tmp.plan.json) +replace_ids.py trace print_requests.py //jobs --nostamp print_state.py > out.state.json trace $CLI bundle plan | contains.py '0 to add, 0 to change, 0 to delete' diff --git a/acceptance/bundle/resources/jobs/check-metadata/test.toml b/acceptance/bundle/resources/jobs/check-metadata/test.toml index c748bb6ee8f..e4db39ccc44 100644 --- a/acceptance/bundle/resources/jobs/check-metadata/test.toml +++ b/acceptance/bundle/resources/jobs/check-metadata/test.toml @@ -15,6 +15,6 @@ Env.MSYS_NO_PATHCONV = "1" [[Repls]] # clean up ?[ow]= suffix after URL since not all workspaces have that -Old = '\?[ow]=\[NUMID\]' +Old = '\?[ow]=\[WORKSPACE_ID\]' New = '' Order = 1000 diff --git a/acceptance/bundle/resources/jobs/delete_job/out.plan.json b/acceptance/bundle/resources/jobs/delete_job/out.plan.json index f88c5ad5fe8..c2b396b5d61 100644 --- a/acceptance/bundle/resources/jobs/delete_job/out.plan.json +++ b/acceptance/bundle/resources/jobs/delete_job/out.plan.json @@ -16,7 +16,7 @@ "edit_mode": "UI_LOCKED", "email_notifications": {}, "format": "MULTI_TASK", - "job_id": [NUMID], + "job_id": [JOB_ID], "max_concurrent_runs": 1, "name": "Untitled", "queue": { diff --git a/acceptance/bundle/resources/jobs/delete_job/output.txt b/acceptance/bundle/resources/jobs/delete_job/output.txt index a8fb8384087..b57ef47f5d2 100644 --- a/acceptance/bundle/resources/jobs/delete_job/output.txt +++ b/acceptance/bundle/resources/jobs/delete_job/output.txt @@ -65,6 +65,6 @@ Resources: 0 created, 0 changed, 1 deleted, 0 unchanged "method": "POST", "path": "/api/2.2/jobs/delete", "body": { - "job_id": [NUMID] + "job_id": [JOB_ID] } } diff --git a/acceptance/bundle/resources/jobs/delete_job/script b/acceptance/bundle/resources/jobs/delete_job/script index feb32bce383..ca8a8cd58d3 100644 --- a/acceptance/bundle/resources/jobs/delete_job/script +++ b/acceptance/bundle/resources/jobs/delete_job/script @@ -1,4 +1,5 @@ trace $CLI bundle deploy +replace_ids.py cp empty.yml databricks.yml $CLI bundle plan -o json | nostamp > out.plan.json trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/jobs/delete_task/out.plan_update.json b/acceptance/bundle/resources/jobs/delete_task/out.plan_update.json index 86b91fc8ded..61124e44ae6 100644 --- a/acceptance/bundle/resources/jobs/delete_task/out.plan_update.json +++ b/acceptance/bundle/resources/jobs/delete_task/out.plan_update.json @@ -52,7 +52,7 @@ "edit_mode": "UI_LOCKED", "email_notifications": {}, "format": "MULTI_TASK", - "job_id": [NUMID], + "job_id": [JOB_ID], "max_concurrent_runs": 1, "name": "Untitled", "queue": { diff --git a/acceptance/bundle/resources/jobs/delete_task/script b/acceptance/bundle/resources/jobs/delete_task/script index 2ea14386cfe..c7ad0ba8405 100644 --- a/acceptance/bundle/resources/jobs/delete_task/script +++ b/acceptance/bundle/resources/jobs/delete_task/script @@ -4,6 +4,7 @@ mkdir -p .databricks $CLI bundle plan -o json > .databricks/tmp.plan_create.json nostamp < .databricks/tmp.plan_create.json > out.plan_create.json $CLI bundle deploy $(readplanarg .databricks/tmp.plan_create.json) +replace_ids.py grep -v TO_DELETE databricks.yml > tmp.yml && mv tmp.yml databricks.yml trace cat databricks.yml $CLI bundle plan -o json > .databricks/tmp.plan_update.json diff --git a/acceptance/bundle/resources/jobs/remote_delete/deploy/output.txt b/acceptance/bundle/resources/jobs/remote_delete/deploy/output.txt index 78a44f51f99..8a0866f2a8f 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/deploy/output.txt +++ b/acceptance/bundle/resources/jobs/remote_delete/deploy/output.txt @@ -10,7 +10,7 @@ Created jobs.foo Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> [CLI] jobs delete [NUMID] +>>> [CLI] jobs delete [FOO_ID] >>> [CLI] bundle plan create jobs.foo diff --git a/acceptance/bundle/resources/jobs/remote_delete/deploy/script b/acceptance/bundle/resources/jobs/remote_delete/deploy/script index fd985ad10d2..e89d8a8bc0a 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/deploy/script +++ b/acceptance/bundle/resources/jobs/remote_delete/deploy/script @@ -3,6 +3,7 @@ mkdir -p .databricks trace $CLI bundle plan trace $CLI bundle deploy +replace_ids.py job_id=`$CLI bundle summary -o json | jq -r .resources.jobs.foo.id` trace $CLI jobs delete $job_id trace $CLI bundle plan diff --git a/acceptance/bundle/resources/jobs/remote_delete/destroy/output.txt b/acceptance/bundle/resources/jobs/remote_delete/destroy/output.txt index 9fd70dd299a..e1d70157918 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/destroy/output.txt +++ b/acceptance/bundle/resources/jobs/remote_delete/destroy/output.txt @@ -10,7 +10,7 @@ Created jobs.foo Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> [CLI] jobs delete [NUMID] +>>> [CLI] jobs delete [FOO_ID] >>> errcode [CLI] bundle destroy --auto-approve All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resources/jobs/remote_delete/destroy/script b/acceptance/bundle/resources/jobs/remote_delete/destroy/script index 03f88082854..a77232ebf39 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/destroy/script +++ b/acceptance/bundle/resources/jobs/remote_delete/destroy/script @@ -1,5 +1,5 @@ trace $CLI bundle plan trace $CLI bundle deploy -job_id=`$CLI bundle summary -o json | jq -r .resources.jobs.foo.id` +job_id=$(read_id.py foo) trace $CLI jobs delete $job_id trace errcode $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/output.txt b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/output.txt index 2db3607d8a6..8421384be39 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/output.txt +++ b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/output.txt @@ -6,7 +6,7 @@ Created pipelines.bar Files: 3 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged ->>> [CLI] jobs delete [NUMID] +>>> [CLI] jobs delete [FOO_ID] >>> [CLI] pipelines delete [UUID] diff --git a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/script b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/script index 07562997192..eb7308ce7fd 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/script +++ b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/script @@ -1,5 +1,5 @@ trace $CLI bundle deploy -job_id=`$CLI bundle summary -o json | jq -r .resources.jobs.foo.id` +job_id=$(read_id.py foo) pipeline_id=`$CLI bundle summary -o json | jq -r .resources.pipelines.bar.id` trace $CLI jobs delete $job_id trace $CLI pipelines delete $pipeline_id diff --git a/acceptance/bundle/resources/jobs/task-source/out.git_job.txt b/acceptance/bundle/resources/jobs/task-source/out.git_job.txt index dcd89b2ec73..0a9a22942c8 100644 --- a/acceptance/bundle/resources/jobs/task-source/out.git_job.txt +++ b/acceptance/bundle/resources/jobs/task-source/out.git_job.txt @@ -90,7 +90,7 @@ ] } { - "job_id": [NUMID], + "job_id": [GIT_JOB_ID], "new_settings": { "deployment": { "kind": "BUNDLE", diff --git a/acceptance/bundle/resources/jobs/task-source/script b/acceptance/bundle/resources/jobs/task-source/script index bd249e805aa..a1117297442 100644 --- a/acceptance/bundle/resources/jobs/task-source/script +++ b/acceptance/bundle/resources/jobs/task-source/script @@ -1,4 +1,5 @@ trace $CLI bundle deploy +replace_ids.py # For terraform we expect the task source to be explicitly set always # For direct we do not expect it to be set unless explicitly specified in the bundle config diff --git a/acceptance/bundle/resources/mcp_services/basic/output.txt b/acceptance/bundle/resources/mcp_services/basic/output.txt index 1d7cdf33722..1e569e8b020 100644 --- a/acceptance/bundle/resources/mcp_services/basic/output.txt +++ b/acceptance/bundle/resources/mcp_services/basic/output.txt @@ -35,7 +35,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle summary -o json { "id": "main.myschema.myservice", - "url": "[DATABRICKS_URL]/explore/data/mcp-services/main/myschema/myservice?w=[NUMID]" + "url": "[DATABRICKS_URL]/explore/data/mcp-services/main/myschema/myservice?w=[WORKSPACE_ID]" } === Verify deployment diff --git a/acceptance/bundle/resources/model_provider_services/basic/output.txt b/acceptance/bundle/resources/model_provider_services/basic/output.txt index 378c1e5f070..6f61bef50bb 100644 --- a/acceptance/bundle/resources/model_provider_services/basic/output.txt +++ b/acceptance/bundle/resources/model_provider_services/basic/output.txt @@ -73,7 +73,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle summary -o json { "id": "main.myschema.myservice", - "url": "[DATABRICKS_URL]/explore/data/model-provider-services/main/myschema/myservice?w=[NUMID]" + "url": "[DATABRICKS_URL]/explore/data/model-provider-services/main/myschema/myservice?w=[WORKSPACE_ID]" } === Verify deployment diff --git a/acceptance/bundle/resources/model_services/basic/output.txt b/acceptance/bundle/resources/model_services/basic/output.txt index 04efffbe7bd..42d7686eee0 100644 --- a/acceptance/bundle/resources/model_services/basic/output.txt +++ b/acceptance/bundle/resources/model_services/basic/output.txt @@ -35,7 +35,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle summary -o json { "id": "main.myschema.myservice", - "url": "[DATABRICKS_URL]/explore/data/model-services/main/myschema/myservice?w=[NUMID]" + "url": "[DATABRICKS_URL]/explore/data/model-services/main/myschema/myservice?w=[WORKSPACE_ID]" } === Verify deployment diff --git a/acceptance/bundle/resources/permissions/_script b/acceptance/bundle/resources/permissions/_script index 7ef1a088d24..f388d49d4ed 100644 --- a/acceptance/bundle/resources/permissions/_script +++ b/acceptance/bundle/resources/permissions/_script @@ -11,6 +11,7 @@ print_requests() { rm "$OUT_REQUESTS" trace $CLI bundle deploy print_requests > out.requests.deploy.json +replace_ids.py trace $CLI bundle destroy --auto-approve print_requests > out.requests.destroy.json diff --git a/acceptance/bundle/resources/permissions/apps/current_can_manage/out.plan.json b/acceptance/bundle/resources/permissions/apps/current_can_manage/out.plan.json index 2184ca4027a..d7f485235ca 100644 --- a/acceptance/bundle/resources/permissions/apps/current_can_manage/out.plan.json +++ b/acceptance/bundle/resources/permissions/apps/current_can_manage/out.plan.json @@ -2,21 +2,21 @@ "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "plan": { - "resources.apps.foo": { + "resources.apps.[FOO_ID]": { "action": "create", "new_state": { "value": { "description": "", - "name": "foo", + "name": "[FOO_ID]", "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files" } } }, - "resources.apps.foo.permissions": { + "resources.apps.[FOO_ID].permissions": { "depends_on": [ { - "node": "resources.apps.foo", - "label": "${resources.apps.foo.id}" + "node": "resources.apps.[FOO_ID]", + "label": "${resources.apps.[FOO_ID].id}" } ], "action": "create", @@ -43,7 +43,7 @@ ] }, "vars": { - "object_id": "/apps/${resources.apps.foo.id}" + "object_id": "/apps/${resources.apps.[FOO_ID].id}" } } } diff --git a/acceptance/bundle/resources/permissions/apps/current_can_manage/out.requests.deploy.json b/acceptance/bundle/resources/permissions/apps/current_can_manage/out.requests.deploy.json index 831cda2417a..5e322e3e472 100644 --- a/acceptance/bundle/resources/permissions/apps/current_can_manage/out.requests.deploy.json +++ b/acceptance/bundle/resources/permissions/apps/current_can_manage/out.requests.deploy.json @@ -1,6 +1,6 @@ { "method": "PUT", - "path": "/api/2.0/permissions/apps/foo", + "path": "/api/2.0/permissions/apps/[FOO_ID]", "body": { "access_control_list": [ { diff --git a/acceptance/bundle/resources/permissions/apps/current_can_manage/output.txt b/acceptance/bundle/resources/permissions/apps/current_can_manage/output.txt index fdca9214b56..331a775ed9b 100644 --- a/acceptance/bundle/resources/permissions/apps/current_can_manage/output.txt +++ b/acceptance/bundle/resources/permissions/apps/current_can_manage/output.txt @@ -21,14 +21,14 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Created apps.foo -Created apps.foo.permissions +Created apps.[FOO_ID] +Created apps.[FOO_ID].permissions Files: 2 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: - delete resources.apps.foo + delete resources.apps.[FOO_ID] All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.plan.json b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.plan.json index 2184ca4027a..d7f485235ca 100644 --- a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.plan.json +++ b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.plan.json @@ -2,21 +2,21 @@ "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "plan": { - "resources.apps.foo": { + "resources.apps.[FOO_ID]": { "action": "create", "new_state": { "value": { "description": "", - "name": "foo", + "name": "[FOO_ID]", "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files" } } }, - "resources.apps.foo.permissions": { + "resources.apps.[FOO_ID].permissions": { "depends_on": [ { - "node": "resources.apps.foo", - "label": "${resources.apps.foo.id}" + "node": "resources.apps.[FOO_ID]", + "label": "${resources.apps.[FOO_ID].id}" } ], "action": "create", @@ -43,7 +43,7 @@ ] }, "vars": { - "object_id": "/apps/${resources.apps.foo.id}" + "object_id": "/apps/${resources.apps.[FOO_ID].id}" } } } diff --git a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.requests.deploy.json b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.requests.deploy.json index 831cda2417a..5e322e3e472 100644 --- a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.requests.deploy.json +++ b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.requests.deploy.json @@ -1,6 +1,6 @@ { "method": "PUT", - "path": "/api/2.0/permissions/apps/foo", + "path": "/api/2.0/permissions/apps/[FOO_ID]", "body": { "access_control_list": [ { diff --git a/acceptance/bundle/resources/permissions/apps/other_can_manage/output.txt b/acceptance/bundle/resources/permissions/apps/other_can_manage/output.txt index fdca9214b56..331a775ed9b 100644 --- a/acceptance/bundle/resources/permissions/apps/other_can_manage/output.txt +++ b/acceptance/bundle/resources/permissions/apps/other_can_manage/output.txt @@ -21,14 +21,14 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Created apps.foo -Created apps.foo.permissions +Created apps.[FOO_ID] +Created apps.[FOO_ID].permissions Files: 2 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: - delete resources.apps.foo + delete resources.apps.[FOO_ID] All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.requests.deploy.json b/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.requests.deploy.json index 63cadc29463..911f898ded8 100644 --- a/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.requests.deploy.json +++ b/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.requests.deploy.json @@ -1,6 +1,6 @@ { "method": "PUT", - "path": "/api/2.0/permissions/clusters/[UUID]", + "path": "/api/2.0/permissions/clusters/[FOO_ID]", "body": { "access_control_list": [ { diff --git a/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.plan.json b/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.plan.json index 6c46d4f1387..4f3da16cef9 100644 --- a/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.plan.json +++ b/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.plan.json @@ -6,7 +6,7 @@ "action": "create", "new_state": { "value": { - "name": "test-db-instance" + "name": "[FOO_ID]" } } }, diff --git a/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.requests.deploy.json b/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.requests.deploy.json index 401bb4d72af..bbdbbef2ae3 100644 --- a/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.requests.deploy.json +++ b/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.requests.deploy.json @@ -1,6 +1,6 @@ { "method": "PUT", - "path": "/api/2.0/permissions/database-instances/test-db-instance", + "path": "/api/2.0/permissions/database-instances/[FOO_ID]", "body": { "access_control_list": [ { diff --git a/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.requests.deploy.json b/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.requests.deploy.json index a8085e0a92c..624fdfe97c8 100644 --- a/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.requests.deploy.json +++ b/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.requests.deploy.json @@ -1,6 +1,6 @@ { "method": "PUT", - "path": "/api/2.0/permissions/experiments/[NUMID]", + "path": "/api/2.0/permissions/experiments/[FOO_ID]", "body": { "access_control_list": [ { diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.requests.deploy.json b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.requests.deploy.json index 5647c1a73b9..47781d53496 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.requests.deploy.json +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.requests.deploy.json @@ -1,6 +1,6 @@ { "method": "PUT", - "path": "/api/2.0/permissions/jobs/[NUMID]", + "path": "/api/2.0/permissions/jobs/[FOO_ID]", "body": { "access_control_list": [ { diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.deploy.requests.json b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.deploy.requests.json index 229ba30386a..d40073afd65 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.deploy.requests.json +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.deploy.requests.json @@ -17,7 +17,7 @@ } { "method": "PUT", - "path": "/api/2.0/permissions/jobs/[NUMID]", + "path": "/api/2.0/permissions/jobs/[INTERIM_GOLD_LAYER_JOB_ID]", "body": { "access_control_list": [ { diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.destroy.requests.json b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.destroy.requests.json index 6564b6f8bde..fa337fe39ce 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.destroy.requests.json +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.destroy.requests.json @@ -2,6 +2,6 @@ "method": "POST", "path": "/api/2.2/jobs/delete", "body": { - "job_id": [NUMID] + "job_id": [INTERIM_GOLD_LAYER_JOB_ID] } } diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.plan.txt b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.plan.txt index 92667a6f585..461e74b2bd6 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.plan.txt +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.plan.txt @@ -16,7 +16,7 @@ "edit_mode": "UI_LOCKED", "email_notifications": {}, "format": "MULTI_TASK", - "job_id": [NUMID], + "job_id": [INTERIM_GOLD_LAYER_JOB_ID], "max_concurrent_runs": 1, "name": "Untitled", "queue": { @@ -53,7 +53,7 @@ ], "action": "skip", "remote_state": { - "object_id": "/jobs/[NUMID]", + "object_id": "/jobs/[INTERIM_GOLD_LAYER_JOB_ID]", "__embed__": [ { "level": "IS_OWNER", diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script index 2c310df73fb..9fd1c142a98 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script @@ -2,6 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle validate -t green -o json | nostamp | jq .resources trace errcode $CLI bundle deploy -t green +replace_ids.py -t green print_requests.py //jobs --nostamp &> out.deploy.requests.json # check plan to ensure there is not drift diff --git a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.requests.deploy.json b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.requests.deploy.json index f7aa2dbaa0d..bbb8445b5a5 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.requests.deploy.json +++ b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.requests.deploy.json @@ -1,6 +1,6 @@ { "method": "PUT", - "path": "/api/2.0/permissions/jobs/[NUMID]", + "path": "/api/2.0/permissions/jobs/[FOO_ID]", "body": { "access_control_list": [ { diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.destroy.txt b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.destroy.txt index 45b95472392..f770ebfac73 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.destroy.txt +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.destroy.txt @@ -1,10 +1,10 @@ >>> errcode as-test-sp [CLI] bundle destroy --auto-approve -Warn: planning resources.jobs.foo: reading resources.jobs.foo id="[NUMID]": User [TEST_SP_APPLICATION_ID] does not have View or Admin or Manage Run or Owner permissions on job [NUMID] +Warn: planning resources.jobs.foo: reading resources.jobs.foo id="[FOO_ID]": User [TEST_SP_APPLICATION_ID] does not have View or Admin or Manage Run or Owner permissions on job [FOO_ID] The following resources will be deleted: delete resources.jobs.foo All files and directories at the following location will be deleted: /Workspace/Users/[TEST_SP_APPLICATION_ID]/.bundle/test-bundle-[UNIQUE_NAME]/default -Warn: destroying resources.jobs.foo: Ignoring permission error when deleting resources.jobs.foo id=[NUMID]: User [TEST_SP_APPLICATION_ID] does not have Admin or Owner permissions on job [NUMID] +Warn: destroying resources.jobs.foo: Ignoring permission error when deleting resources.jobs.foo id=[FOO_ID]: User [TEST_SP_APPLICATION_ID] does not have Admin or Owner permissions on job [FOO_ID] Destroy: 1 deleted diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/output.txt b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/output.txt index 747613ad9d9..f875db24c7a 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/output.txt +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/output.txt @@ -17,9 +17,9 @@ Created jobs.foo.permissions Files: 4 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged ->>> [CLI] permissions set jobs [NUMID] --json @take_ownership.json +>>> [CLI] permissions set jobs [FOO_ID] --json @take_ownership.json ->>> [CLI] jobs delete [NUMID] +>>> [CLI] jobs delete [FOO_ID] ->>> musterr [CLI] jobs get [NUMID] -Error: Job [NUMID] does not exist. +>>> musterr [CLI] jobs get [FOO_ID] +Error: Job [FOO_ID] does not exist. diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/script b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/script index 4f5531d9f6d..fae93344fc0 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/script +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/script @@ -25,6 +25,7 @@ trace as-test-sp $CLI bundle deploy # Read the summary as the deploying identity: under recording the state lives in that # identity's deployment record, which another identity does not resolve. job_id=$(as-test-sp $CLI bundle summary --output json | jq -r '.resources.jobs.foo.id') +add_repl "$job_id" FOO_ID trace $CLI permissions set jobs $job_id --json @take_ownership.json > /dev/null diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.destroy.txt b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.destroy.txt index 45b95472392..f770ebfac73 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.destroy.txt +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.destroy.txt @@ -1,10 +1,10 @@ >>> errcode as-test-sp [CLI] bundle destroy --auto-approve -Warn: planning resources.jobs.foo: reading resources.jobs.foo id="[NUMID]": User [TEST_SP_APPLICATION_ID] does not have View or Admin or Manage Run or Owner permissions on job [NUMID] +Warn: planning resources.jobs.foo: reading resources.jobs.foo id="[FOO_ID]": User [TEST_SP_APPLICATION_ID] does not have View or Admin or Manage Run or Owner permissions on job [FOO_ID] The following resources will be deleted: delete resources.jobs.foo All files and directories at the following location will be deleted: /Workspace/Users/[TEST_SP_APPLICATION_ID]/.bundle/test-bundle-[UNIQUE_NAME]/default -Warn: destroying resources.jobs.foo: Ignoring permission error when deleting resources.jobs.foo id=[NUMID]: User [TEST_SP_APPLICATION_ID] does not have Admin or Owner permissions on job [NUMID] +Warn: destroying resources.jobs.foo: Ignoring permission error when deleting resources.jobs.foo id=[FOO_ID]: User [TEST_SP_APPLICATION_ID] does not have Admin or Owner permissions on job [FOO_ID] Destroy: 1 deleted diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/output.txt b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/output.txt index f07e364b3e9..b9c92e84ee1 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/output.txt +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/output.txt @@ -16,9 +16,9 @@ Created jobs.foo Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> [CLI] permissions set jobs [NUMID] --json @take_ownership.json +>>> [CLI] permissions set jobs [FOO_ID] --json @take_ownership.json ->>> [CLI] jobs delete [NUMID] +>>> [CLI] jobs delete [FOO_ID] ->>> musterr [CLI] jobs get [NUMID] -Error: Job [NUMID] does not exist. +>>> musterr [CLI] jobs get [FOO_ID] +Error: Job [FOO_ID] does not exist. diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/script b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/script index 4f5531d9f6d..fae93344fc0 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/script +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/script @@ -25,6 +25,7 @@ trace as-test-sp $CLI bundle deploy # Read the summary as the deploying identity: under recording the state lives in that # identity's deployment record, which another identity does not resolve. job_id=$(as-test-sp $CLI bundle summary --output json | jq -r '.resources.jobs.foo.id') +add_repl "$job_id" FOO_ID trace $CLI permissions set jobs $job_id --json @take_ownership.json > /dev/null diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.requests.deploy.json b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.requests.deploy.json index 5647c1a73b9..47781d53496 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.requests.deploy.json +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.requests.deploy.json @@ -1,6 +1,6 @@ { "method": "PUT", - "path": "/api/2.0/permissions/jobs/[NUMID]", + "path": "/api/2.0/permissions/jobs/[FOO_ID]", "body": { "access_control_list": [ { diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.deploy.requests.json b/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.deploy.requests.json index f73e509a376..8c03b139400 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.deploy.requests.json +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.deploy.requests.json @@ -17,7 +17,7 @@ } { "method": "PUT", - "path": "/api/2.0/permissions/jobs/[NUMID]", + "path": "/api/2.0/permissions/jobs/[INTERIM_GOLD_LAYER_JOB_ID]", "body": { "access_control_list": [ { diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.destroy.requests.json b/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.destroy.requests.json index 6564b6f8bde..fa337fe39ce 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.destroy.requests.json +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.destroy.requests.json @@ -2,6 +2,6 @@ "method": "POST", "path": "/api/2.2/jobs/delete", "body": { - "job_id": [NUMID] + "job_id": [INTERIM_GOLD_LAYER_JOB_ID] } } diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/script b/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/script index c063be4b8d3..3befc57af80 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/script +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/script @@ -2,6 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle validate -t green -o json | jq .resources trace errcode $CLI bundle deploy -t green +replace_ids.py -t green print_requests.py //jobs --nostamp &> out.deploy.requests.json trace errcode $CLI bundle destroy -t green --auto-approve diff --git a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.requests.deploy.json b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.requests.deploy.json index b1eb1519e87..c43da51b594 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.requests.deploy.json +++ b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.requests.deploy.json @@ -1,6 +1,6 @@ { "method": "PUT", - "path": "/api/2.0/permissions/jobs/[NUMID]", + "path": "/api/2.0/permissions/jobs/[FOO_ID]", "body": { "access_control_list": [ { diff --git a/acceptance/bundle/resources/permissions/jobs/reorder_remotely/output.txt b/acceptance/bundle/resources/permissions/jobs/reorder_remotely/output.txt index 2a5f56804ec..faf5d38bdab 100644 --- a/acceptance/bundle/resources/permissions/jobs/reorder_remotely/output.txt +++ b/acceptance/bundle/resources/permissions/jobs/reorder_remotely/output.txt @@ -9,7 +9,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged ->>> [CLI] permissions set jobs [NUMID] --json @reorder.json +>>> [CLI] permissions set jobs [FOO_ID] --json @reorder.json >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/resources/permissions/jobs/reorder_remotely/script b/acceptance/bundle/resources/permissions/jobs/reorder_remotely/script index 71389de5c9c..14ab774473d 100644 --- a/acceptance/bundle/resources/permissions/jobs/reorder_remotely/script +++ b/acceptance/bundle/resources/permissions/jobs/reorder_remotely/script @@ -1,7 +1,7 @@ trace $CLI bundle deploy trace $CLI bundle plan | contains.py "2 unchanged" -job_id=$($CLI bundle summary --output json | jq -r '.resources.jobs.foo.id') +job_id=$(read_id.py foo) trace $CLI permissions set jobs $job_id --json @reorder.json > /dev/null diff --git a/acceptance/bundle/resources/permissions/jobs/viewers/out.requests.deploy.json b/acceptance/bundle/resources/permissions/jobs/viewers/out.requests.deploy.json index 45768bac0d5..6fcb88e628c 100644 --- a/acceptance/bundle/resources/permissions/jobs/viewers/out.requests.deploy.json +++ b/acceptance/bundle/resources/permissions/jobs/viewers/out.requests.deploy.json @@ -1,6 +1,6 @@ { "method": "PUT", - "path": "/api/2.0/permissions/jobs/[NUMID]", + "path": "/api/2.0/permissions/jobs/[FOO_ID]", "body": { "access_control_list": [ { diff --git a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.requests.deploy.json b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.requests.deploy.json index 42daa83bd86..3cf3626b1d8 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.requests.deploy.json +++ b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.requests.deploy.json @@ -1,6 +1,6 @@ { "method": "PUT", - "path": "/api/2.0/permissions/pipelines/[UUID]", + "path": "/api/2.0/permissions/pipelines/[FOO_ID]", "body": { "access_control_list": [ { diff --git a/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.requests.deploy.json b/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.requests.deploy.json index 578163835c1..432b096a2b0 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.requests.deploy.json +++ b/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.requests.deploy.json @@ -1,6 +1,6 @@ { "method": "PUT", - "path": "/api/2.0/permissions/pipelines/[UUID]", + "path": "/api/2.0/permissions/pipelines/[FOO_ID]", "body": { "access_control_list": [ { diff --git a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.requests.deploy.json b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.requests.deploy.json index 42daa83bd86..3cf3626b1d8 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.requests.deploy.json +++ b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.requests.deploy.json @@ -1,6 +1,6 @@ { "method": "PUT", - "path": "/api/2.0/permissions/pipelines/[UUID]", + "path": "/api/2.0/permissions/pipelines/[FOO_ID]", "body": { "access_control_list": [ { diff --git a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.requests.deploy.json b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.requests.deploy.json index 8925d4f66d4..374e454b56e 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.requests.deploy.json +++ b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.requests.deploy.json @@ -1,6 +1,6 @@ { "method": "PUT", - "path": "/api/2.0/permissions/pipelines/[UUID]", + "path": "/api/2.0/permissions/pipelines/[FOO_ID]", "body": { "access_control_list": [ { diff --git a/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.plan.json b/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.plan.json index 2c5a0f6b32b..3aa2eb78cf0 100644 --- a/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.plan.json +++ b/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.plan.json @@ -21,7 +21,7 @@ "action": "create", "new_state": { "value": { - "object_id": "/database-projects/test-project", + "object_id": "/database-[FOO_ID]", "__embed__": [ { "level": "CAN_USE", diff --git a/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.requests.deploy.json b/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.requests.deploy.json index 673b537f7f4..11f81a57efc 100644 --- a/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.requests.deploy.json +++ b/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.requests.deploy.json @@ -1,6 +1,6 @@ { "method": "PUT", - "path": "/api/2.0/permissions/database-projects/test-project", + "path": "/api/2.0/permissions/database-[FOO_ID]", "body": { "access_control_list": [ { diff --git a/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.requests.deploy.json b/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.requests.deploy.json index 2ef440a5941..78d825e1a16 100644 --- a/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.requests.deploy.json +++ b/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.requests.deploy.json @@ -1,6 +1,6 @@ { "method": "PUT", - "path": "/api/2.0/permissions/sql/warehouses/[UUID]", + "path": "/api/2.0/permissions/sql/warehouses/[FOO_ID]", "body": { "access_control_list": [ { diff --git a/acceptance/bundle/resources/permissions/target_permissions/out.requests_create.json b/acceptance/bundle/resources/permissions/target_permissions/out.requests_create.json index 3c028edebe1..828a03592a6 100644 --- a/acceptance/bundle/resources/permissions/target_permissions/out.requests_create.json +++ b/acceptance/bundle/resources/permissions/target_permissions/out.requests_create.json @@ -26,7 +26,7 @@ } { "method": "PUT", - "path": "/api/2.0/permissions/jobs/[NUMID]", + "path": "/api/2.0/permissions/jobs/[FOO_ID]", "body": { "access_control_list": [ { diff --git a/acceptance/bundle/resources/permissions/target_permissions/out.requests_delete.json b/acceptance/bundle/resources/permissions/target_permissions/out.requests_delete.json index 6564b6f8bde..466f93063f4 100644 --- a/acceptance/bundle/resources/permissions/target_permissions/out.requests_delete.json +++ b/acceptance/bundle/resources/permissions/target_permissions/out.requests_delete.json @@ -2,6 +2,6 @@ "method": "POST", "path": "/api/2.2/jobs/delete", "body": { - "job_id": [NUMID] + "job_id": [FOO_ID] } } diff --git a/acceptance/bundle/resources/permissions/target_permissions/script b/acceptance/bundle/resources/permissions/target_permissions/script index b6190da18d1..7a20b4d2447 100644 --- a/acceptance/bundle/resources/permissions/target_permissions/script +++ b/acceptance/bundle/resources/permissions/target_permissions/script @@ -1,6 +1,7 @@ trace $CLI bundle plan $CLI bundle plan -o json | nostamp > out.plan.json trace $CLI bundle deploy +replace_ids.py print_requests.py --nostamp //jobs/ > out.requests_create.json trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.plan.json b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.plan.json index e42a6290965..f24e8e5e618 100644 --- a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.plan.json +++ b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.plan.json @@ -7,7 +7,7 @@ "new_state": { "value": { "endpoint_type": "STANDARD", - "name": "vs-permissions-endpoint" + "name": "[FOO_ID]" } } }, diff --git a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.deploy.json b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.deploy.json index 9118a4da780..ecfc2c8ac50 100644 --- a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.deploy.json +++ b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.deploy.json @@ -1,6 +1,6 @@ { "method": "PUT", - "path": "/api/2.0/permissions/vector-search-endpoints/[UUID]", + "path": "/api/2.0/permissions[FOO_ID_2]", "body": { "access_control_list": [ { diff --git a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.destroy.json b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.destroy.json index 84c87416aa2..371018d13bd 100644 --- a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.destroy.json +++ b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.destroy.json @@ -1,4 +1,4 @@ { "method": "DELETE", - "path": "/api/2.0/vector-search/endpoints/vs-permissions-endpoint" + "path": "/api/2.0/vector-search/endpoints/[FOO_ID]" } diff --git a/acceptance/bundle/resources/postgres_branches/recreate/test.toml b/acceptance/bundle/resources/postgres_branches/recreate/test.toml index ea1db177791..76d7c84608e 100644 --- a/acceptance/bundle/resources/postgres_branches/recreate/test.toml +++ b/acceptance/bundle/resources/postgres_branches/recreate/test.toml @@ -1,5 +1,5 @@ [[Repls]] -# Normalize TraceId in error messages (run before default NUMID replacement at Order 9-10) +# Normalize TraceId in error messages Old = 'TraceId: [a-f0-9]+' New = 'TraceId: [TRACE_ID]' Order = 1 diff --git a/acceptance/bundle/resources/postgres_branches/test.toml b/acceptance/bundle/resources/postgres_branches/test.toml index c6e54bc122a..0f0b05f5fc6 100644 --- a/acceptance/bundle/resources/postgres_branches/test.toml +++ b/acceptance/bundle/resources/postgres_branches/test.toml @@ -13,7 +13,7 @@ Ignore = [ [[Repls]] # Clean up ?[ow]= suffix after URL since not all workspaces have that -Old = '\?[ow]=\[(NUMID|ALPHANUMID)\]' +Old = '\?[ow]=\[(WORKSPACE_ID|ALPHANUMID)\]' New = '' Order = 1000 diff --git a/acceptance/bundle/resources/postgres_catalogs/test.toml b/acceptance/bundle/resources/postgres_catalogs/test.toml index 9af9fcfff13..ee6c20062ab 100644 --- a/acceptance/bundle/resources/postgres_catalogs/test.toml +++ b/acceptance/bundle/resources/postgres_catalogs/test.toml @@ -11,7 +11,7 @@ Ignore = [ [[Repls]] # Clean up ?[ow]= suffix after URL since not all workspaces have that -Old = '\?[ow]=\[(NUMID|ALPHANUMID)\]' +Old = '\?[ow]=\[(WORKSPACE_ID|ALPHANUMID)\]' New = '' Order = 1000 diff --git a/acceptance/bundle/resources/postgres_databases/live_errors/test.toml b/acceptance/bundle/resources/postgres_databases/live_errors/test.toml index ee9c9b421ee..7881987de1f 100644 --- a/acceptance/bundle/resources/postgres_databases/live_errors/test.toml +++ b/acceptance/bundle/resources/postgres_databases/live_errors/test.toml @@ -20,7 +20,7 @@ Ignore = [ [[Repls]] # Clean up ?o= suffix after URL since not all workspaces have that -Old = '\?o=\[(NUMID|ALPHANUMID)\]' +Old = '\?o=\[(WORKSPACE_ID|ALPHANUMID)\]' New = '' Order = 1000 diff --git a/acceptance/bundle/resources/postgres_databases/test.toml b/acceptance/bundle/resources/postgres_databases/test.toml index f1afb0816e3..8a46fed9249 100644 --- a/acceptance/bundle/resources/postgres_databases/test.toml +++ b/acceptance/bundle/resources/postgres_databases/test.toml @@ -13,7 +13,7 @@ Ignore = [ [[Repls]] # Clean up ?o= suffix after URL since not all workspaces have that -Old = '\?o=\[(NUMID|ALPHANUMID)\]' +Old = '\?o=\[(WORKSPACE_ID|ALPHANUMID)\]' New = '' Order = 1000 diff --git a/acceptance/bundle/resources/postgres_endpoints/test.toml b/acceptance/bundle/resources/postgres_endpoints/test.toml index 53c6fa4bf5d..0b4642d0316 100644 --- a/acceptance/bundle/resources/postgres_endpoints/test.toml +++ b/acceptance/bundle/resources/postgres_endpoints/test.toml @@ -13,7 +13,7 @@ Ignore = [ [[Repls]] # Clean up ?[ow]= suffix after URL since not all workspaces have that -Old = '\?[ow]=\[(NUMID|ALPHANUMID)\]' +Old = '\?[ow]=\[(WORKSPACE_ID|ALPHANUMID)\]' New = '' Order = 1000 diff --git a/acceptance/bundle/resources/postgres_projects/add_default_endpoint_settings/out.plan.json b/acceptance/bundle/resources/postgres_projects/add_default_endpoint_settings/out.plan.json index 0b1f816b366..6e6d97aac27 100644 --- a/acceptance/bundle/resources/postgres_projects/add_default_endpoint_settings/out.plan.json +++ b/acceptance/bundle/resources/postgres_projects/add_default_endpoint_settings/out.plan.json @@ -23,7 +23,7 @@ "name": "projects/test-pg-proj-[UNIQUE_NAME]", "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { - "branch_logical_size_limit_bytes": [NUMID], + "branch_logical_size_limit_bytes": 8796093022208, "default_branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 1, diff --git a/acceptance/bundle/resources/postgres_projects/basic/output.txt b/acceptance/bundle/resources/postgres_projects/basic/output.txt index 6fe7c79ee6b..d73f3a8f118 100644 --- a/acceptance/bundle/resources/postgres_projects/basic/output.txt +++ b/acceptance/bundle/resources/postgres_projects/basic/output.txt @@ -30,7 +30,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "name": "projects/test-pg-proj-[UNIQUE_NAME]", "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { - "branch_logical_size_limit_bytes": [NUMID], + "branch_logical_size_limit_bytes": 8796093022208, "default_branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 4, diff --git a/acceptance/bundle/resources/postgres_projects/factcheck/update_mask_spec/output.txt b/acceptance/bundle/resources/postgres_projects/factcheck/update_mask_spec/output.txt index f61f5cf9033..7f5977e30ec 100644 --- a/acceptance/bundle/resources/postgres_projects/factcheck/update_mask_spec/output.txt +++ b/acceptance/bundle/resources/postgres_projects/factcheck/update_mask_spec/output.txt @@ -5,7 +5,7 @@ "name": "projects/factcheck-spec-[UNIQUE_NAME]", "project_id": "factcheck-spec-[UNIQUE_NAME]", "status": { - "branch_logical_size_limit_bytes": [NUMID], + "branch_logical_size_limit_bytes": 8796093022208, "default_branch": "projects/factcheck-spec-[UNIQUE_NAME]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 1, @@ -29,7 +29,7 @@ "name": "projects/factcheck-spec-[UNIQUE_NAME]", "project_id": "factcheck-spec-[UNIQUE_NAME]", "status": { - "branch_logical_size_limit_bytes": [NUMID], + "branch_logical_size_limit_bytes": 8796093022208, "default_branch": "projects/factcheck-spec-[UNIQUE_NAME]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 1, @@ -68,7 +68,7 @@ "name": "projects/factcheck-spec-[UNIQUE_NAME]", "project_id": "factcheck-spec-[UNIQUE_NAME]", "status": { - "branch_logical_size_limit_bytes": [NUMID], + "branch_logical_size_limit_bytes": 8796093022208, "default_branch": "projects/factcheck-spec-[UNIQUE_NAME]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 4, @@ -131,7 +131,7 @@ Exit code: 1 "name": "projects/factcheck-spec-[UNIQUE_NAME]", "project_id": "factcheck-spec-[UNIQUE_NAME]", "status": { - "branch_logical_size_limit_bytes": [NUMID], + "branch_logical_size_limit_bytes": 8796093022208, "default_branch": "projects/factcheck-spec-[UNIQUE_NAME]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 4, diff --git a/acceptance/bundle/resources/postgres_projects/factcheck/update_mask_star/output.txt b/acceptance/bundle/resources/postgres_projects/factcheck/update_mask_star/output.txt index 932e1d86fd8..2ccecba10f4 100644 --- a/acceptance/bundle/resources/postgres_projects/factcheck/update_mask_star/output.txt +++ b/acceptance/bundle/resources/postgres_projects/factcheck/update_mask_star/output.txt @@ -5,7 +5,7 @@ "name": "projects/factcheck-star-[UNIQUE_NAME]", "project_id": "factcheck-star-[UNIQUE_NAME]", "status": { - "branch_logical_size_limit_bytes": [NUMID], + "branch_logical_size_limit_bytes": 8796093022208, "default_branch": "projects/factcheck-star-[UNIQUE_NAME]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 1, @@ -50,7 +50,7 @@ Exit code: 1 "name": "projects/factcheck-star-[UNIQUE_NAME]", "project_id": "factcheck-star-[UNIQUE_NAME]", "status": { - "branch_logical_size_limit_bytes": [NUMID], + "branch_logical_size_limit_bytes": 8796093022208, "default_branch": "projects/factcheck-star-[UNIQUE_NAME]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 4, @@ -113,7 +113,7 @@ Exit code: 1 "name": "projects/factcheck-star-[UNIQUE_NAME]", "project_id": "factcheck-star-[UNIQUE_NAME]", "status": { - "branch_logical_size_limit_bytes": [NUMID], + "branch_logical_size_limit_bytes": 8796093022208, "default_branch": "projects/factcheck-star-[UNIQUE_NAME]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 4, diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete/output.txt b/acceptance/bundle/resources/postgres_projects/purge_on_delete/output.txt index 96e64343201..060aa7c997b 100644 --- a/acceptance/bundle/resources/postgres_projects/purge_on_delete/output.txt +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete/output.txt @@ -20,7 +20,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "name": "projects/test-pg-proj-hard-[UNIQUE_NAME]", "project_id": "test-pg-proj-hard-[UNIQUE_NAME]", "status": { - "branch_logical_size_limit_bytes": [NUMID], + "branch_logical_size_limit_bytes": 8796093022208, "default_branch": "projects/test-pg-proj-hard-[UNIQUE_NAME]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 1, @@ -43,7 +43,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "name": "projects/test-pg-proj-soft-[UNIQUE_NAME]", "project_id": "test-pg-proj-soft-[UNIQUE_NAME]", "status": { - "branch_logical_size_limit_bytes": [NUMID], + "branch_logical_size_limit_bytes": 8796093022208, "default_branch": "projects/test-pg-proj-soft-[UNIQUE_NAME]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 1, diff --git a/acceptance/bundle/resources/postgres_projects/recreate/out.get_project.txt b/acceptance/bundle/resources/postgres_projects/recreate/out.get_project.txt index 6b2ecab29d8..6bf20a18468 100644 --- a/acceptance/bundle/resources/postgres_projects/recreate/out.get_project.txt +++ b/acceptance/bundle/resources/postgres_projects/recreate/out.get_project.txt @@ -2,7 +2,7 @@ "name": "[MY_PROJECT_ID_2]", "project_id": "test-pg-new-[UNIQUE_NAME]", "status": { - "branch_logical_size_limit_bytes": [NUMID], + "branch_logical_size_limit_bytes": 8796093022208, "default_branch": "[MY_PROJECT_ID_2]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 4, diff --git a/acceptance/bundle/resources/postgres_projects/recreate/test.toml b/acceptance/bundle/resources/postgres_projects/recreate/test.toml index ea1db177791..76d7c84608e 100644 --- a/acceptance/bundle/resources/postgres_projects/recreate/test.toml +++ b/acceptance/bundle/resources/postgres_projects/recreate/test.toml @@ -1,5 +1,5 @@ [[Repls]] -# Normalize TraceId in error messages (run before default NUMID replacement at Order 9-10) +# Normalize TraceId in error messages Old = 'TraceId: [a-f0-9]+' New = 'TraceId: [TRACE_ID]' Order = 1 diff --git a/acceptance/bundle/resources/postgres_projects/remove_history_retention/out.project.txt b/acceptance/bundle/resources/postgres_projects/remove_history_retention/out.project.txt index eb446feebbf..50943239c89 100644 --- a/acceptance/bundle/resources/postgres_projects/remove_history_retention/out.project.txt +++ b/acceptance/bundle/resources/postgres_projects/remove_history_retention/out.project.txt @@ -2,7 +2,7 @@ "name": "projects/test-pg-proj-[UNIQUE_NAME]", "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { - "branch_logical_size_limit_bytes": [NUMID], + "branch_logical_size_limit_bytes": 8796093022208, "default_branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 1, diff --git a/acceptance/bundle/resources/postgres_projects/remove_history_retention/output.txt b/acceptance/bundle/resources/postgres_projects/remove_history_retention/output.txt index 13c9ce1019d..9de5206534c 100644 --- a/acceptance/bundle/resources/postgres_projects/remove_history_retention/output.txt +++ b/acceptance/bundle/resources/postgres_projects/remove_history_retention/output.txt @@ -11,7 +11,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "name": "projects/test-pg-proj-[UNIQUE_NAME]", "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { - "branch_logical_size_limit_bytes": [NUMID], + "branch_logical_size_limit_bytes": 8796093022208, "default_branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 1, diff --git a/acceptance/bundle/resources/postgres_projects/test.toml b/acceptance/bundle/resources/postgres_projects/test.toml index 7bec33c5e34..6ef50148760 100644 --- a/acceptance/bundle/resources/postgres_projects/test.toml +++ b/acceptance/bundle/resources/postgres_projects/test.toml @@ -13,7 +13,7 @@ Ignore = [ [[Repls]] # Clean up ?[ow]= suffix after URL since not all workspaces have that -Old = '\?[ow]=\[(NUMID|ALPHANUMID)\]' +Old = '\?[ow]=\[(WORKSPACE_ID|ALPHANUMID)\]' New = '' Order = 1000 diff --git a/acceptance/bundle/resources/postgres_projects/update_default_endpoint_suspend/output.txt b/acceptance/bundle/resources/postgres_projects/update_default_endpoint_suspend/output.txt index fe37d6db91d..a9ee77c8ce1 100644 --- a/acceptance/bundle/resources/postgres_projects/update_default_endpoint_suspend/output.txt +++ b/acceptance/bundle/resources/postgres_projects/update_default_endpoint_suspend/output.txt @@ -11,7 +11,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "name": "projects/test-pg-proj-[UNIQUE_NAME]", "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { - "branch_logical_size_limit_bytes": [NUMID], + "branch_logical_size_limit_bytes": 8796093022208, "default_branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 4, @@ -52,7 +52,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "name": "projects/test-pg-proj-[UNIQUE_NAME]", "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { - "branch_logical_size_limit_bytes": [NUMID], + "branch_logical_size_limit_bytes": 8796093022208, "default_branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 4, diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.no_change.json b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.no_change.json index 45989db3a77..c7a2dc7fc3e 100644 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.no_change.json +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.no_change.json @@ -5,7 +5,7 @@ "name": "[MY_PROJECT_ID]", "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { - "branch_logical_size_limit_bytes": [NUMID], + "branch_logical_size_limit_bytes": 8796093022208, "default_branch": "[MY_PROJECT_ID]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 4, diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.restore.json b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.restore.json index 39a17d8f975..b67022262b1 100644 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.restore.json +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.restore.json @@ -18,7 +18,7 @@ "name": "[MY_PROJECT_ID]", "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { - "branch_logical_size_limit_bytes": [NUMID], + "branch_logical_size_limit_bytes": 8796093022208, "default_branch": "[MY_PROJECT_ID]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 4, diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.update.json b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.update.json index 02da84cc6a9..a1cb80c5e19 100644 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.update.json +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.update.json @@ -18,7 +18,7 @@ "name": "[MY_PROJECT_ID]", "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { - "branch_logical_size_limit_bytes": [NUMID], + "branch_logical_size_limit_bytes": 8796093022208, "default_branch": "[MY_PROJECT_ID]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 4, diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/output.txt b/acceptance/bundle/resources/postgres_projects/update_display_name/output.txt index edb088c3a45..1ad3f9ed70f 100644 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/output.txt +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/output.txt @@ -29,7 +29,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "name": "[MY_PROJECT_ID]", "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { - "branch_logical_size_limit_bytes": [NUMID], + "branch_logical_size_limit_bytes": 8796093022208, "default_branch": "[MY_PROJECT_ID]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 4, @@ -102,7 +102,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "name": "[MY_PROJECT_ID]", "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { - "branch_logical_size_limit_bytes": [NUMID], + "branch_logical_size_limit_bytes": 8796093022208, "default_branch": "[MY_PROJECT_ID]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 4, @@ -143,7 +143,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "name": "[MY_PROJECT_ID]", "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { - "branch_logical_size_limit_bytes": [NUMID], + "branch_logical_size_limit_bytes": 8796093022208, "default_branch": "[MY_PROJECT_ID]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 4, diff --git a/acceptance/bundle/resources/postgres_projects/update_history_retention/output.txt b/acceptance/bundle/resources/postgres_projects/update_history_retention/output.txt index 7b842a5e5e5..63c086dc510 100644 --- a/acceptance/bundle/resources/postgres_projects/update_history_retention/output.txt +++ b/acceptance/bundle/resources/postgres_projects/update_history_retention/output.txt @@ -11,7 +11,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "name": "projects/test-pg-proj-[UNIQUE_NAME]", "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { - "branch_logical_size_limit_bytes": [NUMID], + "branch_logical_size_limit_bytes": 8796093022208, "default_branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 1, @@ -52,7 +52,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "name": "projects/test-pg-proj-[UNIQUE_NAME]", "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { - "branch_logical_size_limit_bytes": [NUMID], + "branch_logical_size_limit_bytes": 8796093022208, "default_branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production", "default_endpoint_settings": { "autoscaling_limit_max_cu": 1, diff --git a/acceptance/bundle/resources/postgres_roles/test.toml b/acceptance/bundle/resources/postgres_roles/test.toml index 035e2e8da61..8c1145246b1 100644 --- a/acceptance/bundle/resources/postgres_roles/test.toml +++ b/acceptance/bundle/resources/postgres_roles/test.toml @@ -13,7 +13,7 @@ Ignore = [ [[Repls]] # Clean up ?o= suffix after URL since not all workspaces have that -Old = '\?o=\[(NUMID|ALPHANUMID)\]' +Old = '\?o=\[(WORKSPACE_ID|ALPHANUMID)\]' New = '' Order = 1000 diff --git a/acceptance/bundle/resources/postgres_synced_tables/test.toml b/acceptance/bundle/resources/postgres_synced_tables/test.toml index d8b32172981..46260a85a59 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/test.toml +++ b/acceptance/bundle/resources/postgres_synced_tables/test.toml @@ -11,7 +11,7 @@ Ignore = [ [[Repls]] # Clean up ?[ow]= suffix after URL since not all workspaces have that -Old = '\?[ow]=\[(NUMID|ALPHANUMID)\]' +Old = '\?[ow]=\[(WORKSPACE_ID|ALPHANUMID)\]' New = '' Order = 1000 diff --git a/acceptance/bundle/resources/schemas/recreate/output.txt b/acceptance/bundle/resources/schemas/recreate/output.txt index 307cead0a63..8cbfb7e6731 100644 --- a/acceptance/bundle/resources/schemas/recreate/output.txt +++ b/acceptance/bundle/resources/schemas/recreate/output.txt @@ -23,7 +23,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "comment": "COMMENT1", "id": "main.myschema", "name": "myschema", - "url": "[DATABRICKS_URL]/explore/data/main/myschema?w=[NUMID]" + "url": "[DATABRICKS_URL]/explore/data/main/myschema?w=[WORKSPACE_ID]" } } diff --git a/acceptance/bundle/resources/sql_warehouses/output.txt b/acceptance/bundle/resources/sql_warehouses/output.txt index 2d1738cfe0b..5a109c71211 100644 --- a/acceptance/bundle/resources/sql_warehouses/output.txt +++ b/acceptance/bundle/resources/sql_warehouses/output.txt @@ -67,7 +67,7 @@ Resources: SQL Warehouses: test_sql_warehouse: Name: sql_warehouse_name - URL: [DATABRICKS_URL]/sql/warehouses/[UUID]?w=[NUMID] + URL: [DATABRICKS_URL]/sql/warehouses/[UUID]?w=[WORKSPACE_ID] === Update the warehouse name >>> update_file.py databricks.yml sql_warehouse_name sql_warehouse_name_2 @@ -105,7 +105,7 @@ Resources: SQL Warehouses: test_sql_warehouse: Name: sql_warehouse_name_2 - URL: [DATABRICKS_URL]/sql/warehouses/[UUID]?w=[NUMID] + URL: [DATABRICKS_URL]/sql/warehouses/[UUID]?w=[WORKSPACE_ID] === Destroy the warehouse >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/synced_database_tables/basic/test.toml b/acceptance/bundle/resources/synced_database_tables/basic/test.toml index 57cfc02fa64..41b51ae04a0 100644 --- a/acceptance/bundle/resources/synced_database_tables/basic/test.toml +++ b/acceptance/bundle/resources/synced_database_tables/basic/test.toml @@ -9,7 +9,7 @@ RunsOnDbr = false [[Repls]] # clean up ?[ow]= suffix after URL since not all workspaces have that -Old = '\?[ow]=\[(NUMID|ALPHANUMID)\]' +Old = '\?[ow]=\[(WORKSPACE_ID|ALPHANUMID)\]' New = '' Order = 1000 diff --git a/acceptance/bundle/resources/vector_search_endpoints/basic/output.txt b/acceptance/bundle/resources/vector_search_endpoints/basic/output.txt index 937623f7b77..df292d51699 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/basic/output.txt +++ b/acceptance/bundle/resources/vector_search_endpoints/basic/output.txt @@ -18,7 +18,7 @@ Resources: Vector Search Endpoints: my_endpoint: Name: vs-endpoint-[UNIQUE_NAME] - URL: [DATABRICKS_URL]/compute/vector-search/vs-endpoint-[UNIQUE_NAME]?w=[NUMID] + URL: [DATABRICKS_URL]/compute/vector-search/vs-endpoint-[UNIQUE_NAME]?w=[WORKSPACE_ID] >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-endpoint-[UNIQUE_NAME]/default/files... @@ -42,7 +42,7 @@ Resources: Vector Search Endpoints: my_endpoint: Name: vs-endpoint-[UNIQUE_NAME] - URL: [DATABRICKS_URL]/compute/vector-search/vs-endpoint-[UNIQUE_NAME]?w=[NUMID] + URL: [DATABRICKS_URL]/compute/vector-search/vs-endpoint-[UNIQUE_NAME]?w=[WORKSPACE_ID] >>> print_requests.py //vector-search/endpoints diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/output.txt b/acceptance/bundle/resources/vector_search_indexes/basic/output.txt index 71713fedfae..41a6096ef09 100644 --- a/acceptance/bundle/resources/vector_search_indexes/basic/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/basic/output.txt @@ -18,11 +18,11 @@ Resources: Vector Search Endpoints: my_endpoint: Name: vs-endpoint-[UNIQUE_NAME] - URL: [DATABRICKS_URL]/compute/vector-search/vs-endpoint-[UNIQUE_NAME]?w=[NUMID] + URL: [DATABRICKS_URL]/compute/vector-search/vs-endpoint-[UNIQUE_NAME]?w=[WORKSPACE_ID] Vector Search Indexes: my_index: Name: main.default.vs_index_[UNIQUE_NAME] - URL: [DATABRICKS_URL]/explore/data/main/default/vs_index_[UNIQUE_NAME]?w=[NUMID] + URL: [DATABRICKS_URL]/explore/data/main/default/vs_index_[UNIQUE_NAME]?w=[WORKSPACE_ID] >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default/files... @@ -49,11 +49,11 @@ Resources: Vector Search Endpoints: my_endpoint: Name: vs-endpoint-[UNIQUE_NAME] - URL: [DATABRICKS_URL]/compute/vector-search/vs-endpoint-[UNIQUE_NAME]?w=[NUMID] + URL: [DATABRICKS_URL]/compute/vector-search/vs-endpoint-[UNIQUE_NAME]?w=[WORKSPACE_ID] Vector Search Indexes: my_index: Name: main.default.vs_index_[UNIQUE_NAME] - URL: [DATABRICKS_URL]/explore/data/main/default/vs_index_[UNIQUE_NAME]?w=[NUMID] + URL: [DATABRICKS_URL]/explore/data/main/default/vs_index_[UNIQUE_NAME]?w=[WORKSPACE_ID] >>> print_requests.py //vector-search/indexes diff --git a/acceptance/bundle/resources/volumes/change-comment/output.txt b/acceptance/bundle/resources/volumes/change-comment/output.txt index 8f87f402732..275ccb5f70d 100644 --- a/acceptance/bundle/resources/volumes/change-comment/output.txt +++ b/acceptance/bundle/resources/volumes/change-comment/output.txt @@ -36,7 +36,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Summary should now show id and url "main.myschema.myvolume" -"[DATABRICKS_URL]/explore/data/volumes/main/myschema/myvolume?w=[NUMID]" +"[DATABRICKS_URL]/explore/data/volumes/main/myschema/myvolume?w=[WORKSPACE_ID]" === Verify deployment >>> [CLI] volumes read main.myschema.myvolume @@ -65,7 +65,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "id": "main.myschema.myvolume", "name": "myvolume", "schema_name": "myschema", - "url": "[DATABRICKS_URL]/explore/data/volumes/main/myschema/myvolume?w=[NUMID]", + "url": "[DATABRICKS_URL]/explore/data/volumes/main/myschema/myvolume?w=[WORKSPACE_ID]", "volume_path": "/Volumes/main/myschema/myvolume", "volume_type": "MANAGED" } diff --git a/acceptance/bundle/resources/volumes/change-name/output.txt b/acceptance/bundle/resources/volumes/change-name/output.txt index ca05f72fcef..417cd36f6fd 100644 --- a/acceptance/bundle/resources/volumes/change-name/output.txt +++ b/acceptance/bundle/resources/volumes/change-name/output.txt @@ -27,7 +27,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "id": "main.myschema.myvolume", "name": "myvolume", "schema_name": "myschema", - "url": "[DATABRICKS_URL]/explore/data/volumes/main/myschema/myvolume?w=[NUMID]", + "url": "[DATABRICKS_URL]/explore/data/volumes/main/myschema/myvolume?w=[WORKSPACE_ID]", "volume_path": "/Volumes/main/myschema/myvolume", "volume_type": "MANAGED" } diff --git a/acceptance/bundle/resources/volumes/change-schema-name/output.txt b/acceptance/bundle/resources/volumes/change-schema-name/output.txt index 0f0c4be1c7c..7791b740c98 100644 --- a/acceptance/bundle/resources/volumes/change-schema-name/output.txt +++ b/acceptance/bundle/resources/volumes/change-schema-name/output.txt @@ -27,7 +27,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "id": "main.myschema.myvolume", "name": "myvolume", "schema_name": "myschema", - "url": "[DATABRICKS_URL]/explore/data/volumes/main/myschema/myvolume?w=[NUMID]", + "url": "[DATABRICKS_URL]/explore/data/volumes/main/myschema/myvolume?w=[WORKSPACE_ID]", "volume_path": "/Volumes/main/myschema/myvolume", "volume_type": "MANAGED" } diff --git a/acceptance/bundle/run/basic/output.txt b/acceptance/bundle/run/basic/output.txt index 9afc827b839..9ae6068709a 100644 --- a/acceptance/bundle/run/basic/output.txt +++ b/acceptance/bundle/run/basic/output.txt @@ -13,7 +13,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle run foo -Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[FOO_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "foo" RUNNING [TIMESTAMP] "foo" TERMINATED SUCCESS @@ -27,7 +27,7 @@ Exit code: 1 === resource key with parameters >>> [CLI] bundle run foo -- arg1 arg2 -Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[FOO_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "foo" RUNNING [TIMESTAMP] "foo" TERMINATED SUCCESS diff --git a/acceptance/bundle/run/basic/script b/acceptance/bundle/run/basic/script index a7f6d361924..29fa197375b 100644 --- a/acceptance/bundle/run/basic/script +++ b/acceptance/bundle/run/basic/script @@ -3,6 +3,7 @@ errcode trace $CLI bundle run title "deploy and run resource" errcode trace $CLI bundle deploy +replace_ids.py errcode trace $CLI bundle run foo diff --git a/acceptance/bundle/run/basic/test.toml b/acceptance/bundle/run/basic/test.toml new file mode 100644 index 00000000000..3c887dd20bf --- /dev/null +++ b/acceptance/bundle/run/basic/test.toml @@ -0,0 +1,3 @@ +[[Repls]] +Old = '/runs/\d+' +New = '/runs/[RUN_ID]' diff --git a/acceptance/bundle/run/jobs/partial_run/output.txt b/acceptance/bundle/run/jobs/partial_run/output.txt index b7574fde41d..2fe779143a5 100644 --- a/acceptance/bundle/run/jobs/partial_run/output.txt +++ b/acceptance/bundle/run/jobs/partial_run/output.txt @@ -6,7 +6,7 @@ Files: 3 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle run my_job --only task_1 -Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "my_job" RUNNING [TIMESTAMP] "my_job" TERMINATED SUCCESS @@ -25,7 +25,7 @@ Hello from notebook2! "method": "POST", "path": "/api/2.2/jobs/run-now", "body": { - "job_id": [NUMID], + "job_id": [MY_JOB_ID], "only": [ "task_1" ] @@ -33,7 +33,7 @@ Hello from notebook2! } >>> [CLI] bundle run my_job --only task_1,task_2 -Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "my_job" RUNNING [TIMESTAMP] "my_job" TERMINATED SUCCESS @@ -52,7 +52,7 @@ Hello from notebook2! "method": "POST", "path": "/api/2.2/jobs/run-now", "body": { - "job_id": [NUMID], + "job_id": [MY_JOB_ID], "only": [ "task_1", "task_2" @@ -61,7 +61,7 @@ Hello from notebook2! } >>> [CLI] bundle run my_job -Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "my_job" RUNNING [TIMESTAMP] "my_job" TERMINATED SUCCESS @@ -80,7 +80,7 @@ Hello from notebook2! "method": "POST", "path": "/api/2.2/jobs/run-now", "body": { - "job_id": [NUMID] + "job_id": [MY_JOB_ID] } } @@ -91,7 +91,7 @@ Error: task "non_existent_task" not found in job "my_job" Error: task "non_existent_task" not found in job "my_job" >>> [CLI] bundle run my_job --only task1.table1,task2>,task3>table3 -Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "my_job" RUNNING [TIMESTAMP] "my_job" TERMINATED SUCCESS @@ -110,7 +110,7 @@ Hello from notebook2! "method": "POST", "path": "/api/2.2/jobs/run-now", "body": { - "job_id": [NUMID], + "job_id": [MY_JOB_ID], "only": [ "task1.table1", "task2>", @@ -120,7 +120,7 @@ Hello from notebook2! } >>> [CLI] bundle run my_job --only +task_1,task_2+ -Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "my_job" RUNNING [TIMESTAMP] "my_job" TERMINATED SUCCESS @@ -139,7 +139,7 @@ Hello from notebook2! "method": "POST", "path": "/api/2.2/jobs/run-now", "body": { - "job_id": [NUMID], + "job_id": [MY_JOB_ID], "only": [ "+task_1", "task_2+" diff --git a/acceptance/bundle/run/jobs/partial_run/script b/acceptance/bundle/run/jobs/partial_run/script index 78ce25d514a..9c1b6069942 100644 --- a/acceptance/bundle/run/jobs/partial_run/script +++ b/acceptance/bundle/run/jobs/partial_run/script @@ -1,4 +1,5 @@ trace $CLI bundle deploy +replace_ids.py trace $CLI bundle run my_job --only task_1 trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/run/jobs/partial_run/test.toml b/acceptance/bundle/run/jobs/partial_run/test.toml index c9137ee00b7..ee0a86da975 100644 --- a/acceptance/bundle/run/jobs/partial_run/test.toml +++ b/acceptance/bundle/run/jobs/partial_run/test.toml @@ -1,3 +1,7 @@ Cloud = false RecordRequests = true + +[[Repls]] +Old = '/runs/\d+' +New = '/runs/[RUN_ID]' diff --git a/acceptance/bundle/run/state-wiped/output.txt b/acceptance/bundle/run/state-wiped/output.txt index 91abb764f26..894e4148455 100644 --- a/acceptance/bundle/run/state-wiped/output.txt +++ b/acceptance/bundle/run/state-wiped/output.txt @@ -10,7 +10,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle run foo -Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[FOO_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "foo" RUNNING [TIMESTAMP] "foo" TERMINATED SUCCESS @@ -19,7 +19,7 @@ Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] >>> rm -fr .databricks >>> [CLI] bundle run foo -Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[FOO_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "foo" RUNNING [TIMESTAMP] "foo" TERMINATED SUCCESS diff --git a/acceptance/bundle/run/state-wiped/script b/acceptance/bundle/run/state-wiped/script index 537c9ca6ddf..e3b6ae29356 100644 --- a/acceptance/bundle/run/state-wiped/script +++ b/acceptance/bundle/run/state-wiped/script @@ -1,5 +1,6 @@ trace musterr $CLI bundle run foo trace $CLI bundle deploy +replace_ids.py trace $CLI bundle run foo diff --git a/acceptance/bundle/run/state-wiped/test.toml b/acceptance/bundle/run/state-wiped/test.toml new file mode 100644 index 00000000000..3c887dd20bf --- /dev/null +++ b/acceptance/bundle/run/state-wiped/test.toml @@ -0,0 +1,3 @@ +[[Repls]] +Old = '/runs/\d+' +New = '/runs/[RUN_ID]' diff --git a/acceptance/bundle/run_as/job_default/output.txt b/acceptance/bundle/run_as/job_default/output.txt index 41b7d64165a..e8fa5630934 100644 --- a/acceptance/bundle/run_as/job_default/output.txt +++ b/acceptance/bundle/run_as/job_default/output.txt @@ -41,7 +41,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } ->>> [CLI] jobs get [NUMID] +>>> [CLI] jobs get [JOB_WITH_RUN_AS_ID] { "user_name": "deco-test-user@databricks.com" } @@ -63,7 +63,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "method": "POST", "path": "/api/2.2/jobs/reset", "body": { - "job_id": [NUMID], + "job_id": [JOB_WITH_RUN_AS_ID], "new_settings": { "deployment": { "kind": "BUNDLE", @@ -93,7 +93,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } } ->>> [CLI] jobs get [NUMID] +>>> [CLI] jobs get [JOB_WITH_RUN_AS_ID] { "user_name": "deco-test-user@databricks.com" } diff --git a/acceptance/bundle/run_as/job_default/script b/acceptance/bundle/run_as/job_default/script index 784716b3c90..04355896bff 100644 --- a/acceptance/bundle/run_as/job_default/script +++ b/acceptance/bundle/run_as/job_default/script @@ -9,7 +9,7 @@ trap cleanup EXIT title "Deploy with run_as" trace $CLI bundle deploy trace print_requests.py --nostamp //jobs | contains.py "!GET" "POST" -JOB_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.job_with_run_as.id') +JOB_ID=$(read_id.py job_with_run_as) trace $CLI jobs get $JOB_ID | jq -r '.settings.run_as' update_file.py databricks.yml "run_as: diff --git a/acceptance/bundle/select/basic/output.txt b/acceptance/bundle/select/basic/output.txt index 9790c4ebc37..89f66b3d78f 100644 --- a/acceptance/bundle/select/basic/output.txt +++ b/acceptance/bundle/select/basic/output.txt @@ -68,7 +68,7 @@ select_used true "tasks": [ { "run_job_task": { - "job_id": [NUMID] + "job_id": [BAR_ID] }, "task_key": "run_bar" } @@ -86,13 +86,13 @@ Resources: Jobs: bar: Name: bar-[UNIQUE_NAME] - URL: [DATABRICKS_URL]/jobs/[NUMID]?w=[NUMID] + URL: [DATABRICKS_URL]/jobs/[BAR_ID]?w=[WORKSPACE_ID] baz: Name: baz-[UNIQUE_NAME] URL: (not deployed) foo: Name: foo-[UNIQUE_NAME] - URL: [DATABRICKS_URL]/jobs/[NUMID]?w=[NUMID] + URL: [DATABRICKS_URL]/jobs/[FOO_ID]?w=[WORKSPACE_ID] === Full plan after partial deploy >>> [CLI] bundle plan diff --git a/acceptance/bundle/select/basic/script b/acceptance/bundle/select/basic/script index f4fbd243a54..3b7aa10b34d 100644 --- a/acceptance/bundle/select/basic/script +++ b/acceptance/bundle/select/basic/script @@ -25,6 +25,7 @@ trace $CLI bundle plan --select jobs.foo $CLI bundle plan --select jobs.foo -o json > plan.json title "bundle deploy --select jobs.foo\n" $CLI bundle deploy --select jobs.foo $(readplanarg plan.json) +replace_ids.py # The deploy reports that --select was used via telemetry. title "Telemetry:\n" print_telemetry_bool_values | grep '^select_used ' diff --git a/acceptance/bundle/select/grants_permissions/output.txt b/acceptance/bundle/select/grants_permissions/output.txt index 3f1889891d5..6ada2c1689b 100644 --- a/acceptance/bundle/select/grants_permissions/output.txt +++ b/acceptance/bundle/select/grants_permissions/output.txt @@ -36,7 +36,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged, 2 not selected >>> print_requests.py --sort //permissions { "method": "PATCH", - "path": "/api/2.1/unity-catalog/permissions/schema/main.my_schema_[UNIQUE_NAME]", + "path": "/api/2.1/unity-catalog/permissions/schema/[MY_SCHEMA_ID]", "body": { "changes": [ { @@ -50,7 +50,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged, 2 not selected } { "method": "PUT", - "path": "/api/2.0/permissions/jobs/[NUMID]", + "path": "/api/2.0/permissions/jobs/[MY_JOB_ID]", "body": { "access_control_list": [ { diff --git a/acceptance/bundle/select/grants_permissions/script b/acceptance/bundle/select/grants_permissions/script index 230a9111291..7856742c6cf 100644 --- a/acceptance/bundle/select/grants_permissions/script +++ b/acceptance/bundle/select/grants_permissions/script @@ -25,6 +25,7 @@ $CLI bundle deploy --select jobs.my_job $(readplanarg plan-job.json) $CLI bundle plan --select schemas.my_schema -o json > plan-schema.json title "bundle deploy --select schemas.my_schema\n" $CLI bundle deploy --select schemas.my_schema $(readplanarg plan-schema.json) +replace_ids.py # Assert the permissions PUT and grants PATCH were emitted for the selected resources. trace print_requests.py --sort //permissions diff --git a/acceptance/bundle/state/force_pull_commands/output.txt b/acceptance/bundle/state/force_pull_commands/output.txt index 977e1a296e3..728a77b3e3f 100644 --- a/acceptance/bundle/state/force_pull_commands/output.txt +++ b/acceptance/bundle/state/force_pull_commands/output.txt @@ -25,12 +25,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === bundle open without --force-pull: no remote state read >>> [CLI] bundle open foo -Opening browser at [DATABRICKS_URL]/jobs/[NUMID]?w=[NUMID] +Opening browser at [DATABRICKS_URL]/jobs/[FOO_ID]?w=[WORKSPACE_ID] === bundle open --force-pull: remote state read >>> [CLI] bundle open foo --force-pull -Opening browser at [DATABRICKS_URL]/jobs/[NUMID]?w=[NUMID] +Opening browser at [DATABRICKS_URL]/jobs/[FOO_ID]?w=[WORKSPACE_ID] { "method": "GET", "path": "/api/2.0/workspace/export", diff --git a/acceptance/bundle/state/force_pull_commands/script b/acceptance/bundle/state/force_pull_commands/script index fc287979037..19b0748d61e 100644 --- a/acceptance/bundle/state/force_pull_commands/script +++ b/acceptance/bundle/state/force_pull_commands/script @@ -1,4 +1,5 @@ trace $CLI bundle deploy > /dev/null +replace_ids.py rm -f "$OUT_REQUESTS" title "Use a fake browser that just prints the URL it would have opened" diff --git a/acceptance/bundle/state/permission_level_migration/output.txt b/acceptance/bundle/state/permission_level_migration/output.txt index fb22c476b20..c642fb01f52 100644 --- a/acceptance/bundle/state/permission_level_migration/output.txt +++ b/acceptance/bundle/state/permission_level_migration/output.txt @@ -19,7 +19,7 @@ Resources: 1 created, 1 changed, 0 deleted, 0 unchanged "serial": 2, "state": { "resources.jobs.my_job": { - "__id__": "[NUMID]", + "__id__": "[MY_JOB_ID]", "state": { "deployment": { "kind": "BUNDLE", @@ -37,7 +37,7 @@ Resources: 1 created, 1 changed, 0 deleted, 0 unchanged "resources.jobs.my_job.permissions": { "__id__": "/jobs/123", "state": { - "object_id": "/jobs/[NUMID]", + "object_id": "/jobs/[MY_JOB_ID]", "__embed__": [ { "level": "CAN_VIEW", diff --git a/acceptance/bundle/state/permission_level_migration/script b/acceptance/bundle/state/permission_level_migration/script index bd0cdee18f5..e03dd05504e 100644 --- a/acceptance/bundle/state/permission_level_migration/script +++ b/acceptance/bundle/state/permission_level_migration/script @@ -13,6 +13,7 @@ trace $CLI bundle plan -o json | nostamp > out.plan.direct.json title "Deploy (migrates state)" trace $CLI bundle deploy +read_id.py my_job > /dev/null title "Print state after deploy" trace print_state.py | nostamp --indent 1 diff --git a/acceptance/bundle/summary/modified_status/output.txt b/acceptance/bundle/summary/modified_status/output.txt index 917ddf23b7e..59d922c8b85 100644 --- a/acceptance/bundle/summary/modified_status/output.txt +++ b/acceptance/bundle/summary/modified_status/output.txt @@ -100,7 +100,7 @@ Resources: 4 created, 0 changed, 0 deleted, 0 unchanged "quartz_cron_schedule": "44 19 */1 * * ?", "timezone_id": "Europe/Amsterdam" }, - "url": "[DATABRICKS_URL]/sql/alerts-v2/[UUID]?w=[NUMID]", + "url": "[DATABRICKS_URL]/sql/alerts-v2/[UUID]?w=[WORKSPACE_ID]", "warehouse_id": "aaaaaaaaaaaaaaaa" } }, @@ -121,7 +121,7 @@ Resources: 4 created, 0 changed, 0 deleted, 0 unchanged } ], "name": "test-pipeline", - "url": "[DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID]" + "url": "[DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID]" } }, "schemas": { @@ -130,7 +130,7 @@ Resources: 4 created, 0 changed, 0 deleted, 0 unchanged "comment": "COMMENT1", "id": "main.test-schema", "name": "test-schema", - "url": "[DATABRICKS_URL]/explore/data/main/test-schema?w=[NUMID]" + "url": "[DATABRICKS_URL]/explore/data/main/test-schema?w=[WORKSPACE_ID]" } }, "sql_warehouses": { @@ -143,7 +143,7 @@ Resources: 4 created, 0 changed, 0 deleted, 0 unchanged "max_num_clusters": 1, "name": "test-sql-warehouse", "spot_instance_policy": "COST_OPTIMIZED", - "url": "[DATABRICKS_URL]/sql/warehouses/[UUID]?w=[NUMID]" + "url": "[DATABRICKS_URL]/sql/warehouses/[UUID]?w=[WORKSPACE_ID]" } } } @@ -155,28 +155,28 @@ Resources: 4 created, 0 changed, 0 deleted, 0 unchanged "my_alert": { "id": "[UUID]", "modified_status": "deleted", - "url": "[DATABRICKS_URL]/sql/alerts-v2/[UUID]?w=[NUMID]" + "url": "[DATABRICKS_URL]/sql/alerts-v2/[UUID]?w=[WORKSPACE_ID]" } }, "pipelines": { "my_pipeline": { "id": "[UUID]", "modified_status": "deleted", - "url": "[DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID]" + "url": "[DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID]" } }, "schemas": { "my_schema": { "id": "main.test-schema", "modified_status": "deleted", - "url": "[DATABRICKS_URL]/explore/data/main/test-schema?w=[NUMID]" + "url": "[DATABRICKS_URL]/explore/data/main/test-schema?w=[WORKSPACE_ID]" } }, "sql_warehouses": { "my_sql_warehouse": { "id": "[UUID]", "modified_status": "deleted", - "url": "[DATABRICKS_URL]/sql/warehouses/[UUID]?w=[NUMID]" + "url": "[DATABRICKS_URL]/sql/warehouses/[UUID]?w=[WORKSPACE_ID]" } } } diff --git a/acceptance/bundle/telemetry/config-remote-sync-save/output.txt b/acceptance/bundle/telemetry/config-remote-sync-save/output.txt index 238fdfd64c7..530b2535442 100644 --- a/acceptance/bundle/telemetry/config-remote-sync-save/output.txt +++ b/acceptance/bundle/telemetry/config-remote-sync-save/output.txt @@ -35,7 +35,7 @@ Resource: resources.jobs.foo "states_available_count": 2, "state_resource_ids": { "resource_job_ids": [ - "[NUMID]" + "[FOO_ID]" ] } } diff --git a/acceptance/bundle/telemetry/config-remote-sync-save/script b/acceptance/bundle/telemetry/config-remote-sync-save/script index 43424f7cb26..95480e39d40 100644 --- a/acceptance/bundle/telemetry/config-remote-sync-save/script +++ b/acceptance/bundle/telemetry/config-remote-sync-save/script @@ -1,4 +1,5 @@ trace $CLI bundle deploy +replace_ids.py # Edit the job name locally (not deployed) so the local config diverges from the # deployed state. config-remote-sync --save then overwrites that pending edit, diff --git a/acceptance/bundle/telemetry/deploy/out.telemetry.txt b/acceptance/bundle/telemetry/deploy/out.telemetry.txt index 626565b3d77..45a366836f1 100644 --- a/acceptance/bundle/telemetry/deploy/out.telemetry.txt +++ b/acceptance/bundle/telemetry/deploy/out.telemetry.txt @@ -27,13 +27,13 @@ "resource_dashboard_count": 0, "resource_app_count": 0, "resource_job_ids": [ - "[NUMID]", - "[NUMID]", - "[NUMID]" + "[JOB_ONE_ID]", + "[JOB_THREE_ID]", + "[JOB_TWO_ID]" ], "resource_pipeline_ids": [ - "[UUID]", - "[UUID]" + "[PIPELINE_ONE_ID]", + "[PIPELINE_TWO_ID]" ], "experimental": { "configuration_file_count": 1, diff --git a/acceptance/bundle/telemetry/deploy/script b/acceptance/bundle/telemetry/deploy/script index f8c9dfa5a0b..ef428a814a1 100644 --- a/acceptance/bundle/telemetry/deploy/script +++ b/acceptance/bundle/telemetry/deploy/script @@ -1,4 +1,5 @@ trace $CLI bundle deploy +replace_ids.py trace cat "$OUT_REQUESTS" | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson' > telemetry.json @@ -28,7 +29,10 @@ cat telemetry.json | jq '[.entry.databricks_cli_log.bundle_deploy_event.experime # # upload_file_count and upload_file_sizes are asserted here and kept deterministic by # the sync.paths restriction in databricks.yml (see the comment there). -cat telemetry.json | jq 'del(.entry.databricks_cli_log.bundle_deploy_event.experimental.bundle_mutator_execution_time_ms, .entry.databricks_cli_log.bundle_deploy_event.resources_metadata) | .entry.databricks_cli_log.bundle_deploy_event.experimental.bool_values |= map(select(.key | startswith("engine_terraform_") | not))' > out.telemetry.txt +# The resource ids are listed in creation order, which varies because resources are created in +# parallel. Sort them by resource key so the order is stable. +ids=$($CLI bundle summary -o json | jq '[.resources[] | to_entries[] | {key: (.value.id | tostring), value: .key}] | from_entries') +cat telemetry.json | jq --argjson ids "$ids" 'del(.entry.databricks_cli_log.bundle_deploy_event.experimental.bundle_mutator_execution_time_ms, .entry.databricks_cli_log.bundle_deploy_event.resources_metadata) | .entry.databricks_cli_log.bundle_deploy_event.experimental.bool_values |= map(select(.key | startswith("engine_terraform_") | not)) | .entry.databricks_cli_log.bundle_deploy_event |= (.resource_job_ids |= sort_by($ids[.]) | .resource_pipeline_ids |= sort_by($ids[.]))' > out.telemetry.txt cmd_exec_id=$(extract_command_exec_id.py) deployment_id=$(cat .databricks/bundle/default/deployment.json | jq -r .id) diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.json b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.json index b1a746f29b9..b83830d569f 100644 --- a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.json +++ b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.json @@ -134,7 +134,7 @@ } } ], - "job_id": [NUMID], + "job_id": [SAMPLE_JOB_ID], "max_concurrent_runs": 4, "name": "[dev [USERNAME]] sample_job", "parameters": [ diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.json b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.json index 52e59958a1a..e781152f464 100644 --- a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.json +++ b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.json @@ -36,7 +36,7 @@ } } ], - "job_id": [NUMID], + "job_id": [SAMPLE_JOB_ID_2], "max_concurrent_runs": 1, "name": "sample_job", "parameters": [ @@ -199,7 +199,7 @@ ], "action": "skip", "remote_state": { - "object_id": "/jobs/[NUMID]", + "object_id": "/jobs/[SAMPLE_JOB_ID_2]", "__embed__": [ { "level": "IS_OWNER", diff --git a/acceptance/bundle/templates/default-python/classic/out.requests.prod.txt b/acceptance/bundle/templates/default-python/classic/out.requests.prod.txt index 0c08ea4e5d0..886e527e396 100644 --- a/acceptance/bundle/templates/default-python/classic/out.requests.prod.txt +++ b/acceptance/bundle/templates/default-python/classic/out.requests.prod.txt @@ -195,7 +195,7 @@ } { "method": "PUT", - "path": "/api/2.0/permissions/directories/[NUMID]", + "path": "/api/2.0/permissions/directories/[PROD_ROOT_DIR_ID]", "body": { "access_control_list": [ { @@ -207,7 +207,7 @@ } { "method": "PUT", - "path": "/api/2.0/permissions/jobs/[NUMID]", + "path": "/api/2.0/permissions/jobs/[SAMPLE_JOB_ID_2]", "body": { "access_control_list": [ { diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/sample_notebook.ipynb b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/sample_notebook.ipynb index 49bbd5935d7..ea9c0a8404a 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/sample_notebook.ipynb +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/sample_notebook.ipynb @@ -82,7 +82,7 @@ "widgets": { "catalog": { "currentValue": "hive_metastore", - "nuid": "c4t4l0g-w1dg-3t12-3456-[NUMID]", + "nuid": "c4t4l0g-w1dg-3t12-3456-789012345678", "typedWidgetInfo": { "autoCreated": false, "defaultValue": "hive_metastore", @@ -108,7 +108,7 @@ }, "schema": { "currentValue": "[USERNAME]", - "nuid": "5ch3m4-w1dg-3t98-7654-[NUMID]", + "nuid": "5ch3m4-w1dg-3t98-7654-321098765432", "typedWidgetInfo": { "autoCreated": false, "defaultValue": "default", diff --git a/acceptance/bundle/templates/default-python/classic/script b/acceptance/bundle/templates/default-python/classic/script index 13726d70cd0..3029c012d05 100644 --- a/acceptance/bundle/templates/default-python/classic/script +++ b/acceptance/bundle/templates/default-python/classic/script @@ -14,11 +14,16 @@ rm "$OUT_REQUESTS" # With --plan variant we don't build artifacts, so we can filter out relevant log lines $CLI bundle deploy -t dev $(readplanarg ../../out.plan_dev.json) 2>&1 | grep -vE '^Building python_artifact|^Uploading .databricks' print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.dev.txt +read_id.py -t dev sample_job > /dev/null trace $CLI bundle plan -t dev # check if if there is drift trace $CLI bundle plan -t dev -o json | nostamp > ../../out.plan_after_deploy_dev.json $CLI bundle deploy -t prod $(readplanarg ../../out.plan_prod.json) 2>&1 | grep -vE '^Building python_artifact|^Uploading dist' print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.prod.txt +read_id.py -t prod sample_job > /dev/null +# Extracted with python, not jq: the id exceeds 2^53 and jq 1.6 rounds it. +root_dir_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/my_default_python/prod" -o json | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +add_repl "$root_dir_id" PROD_ROOT_DIR_ID trace $CLI bundle plan -t prod # check if there is drift trace $CLI bundle plan -t prod -o json | nostamp > ../../out.plan_after_deploy_prod.json diff --git a/acceptance/bundle/templates/default-python/integration_classic/out.plan_dev.json b/acceptance/bundle/templates/default-python/integration_classic/out.plan_dev.json index 72311ab766a..9dadfe9cd8c 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/out.plan_dev.json +++ b/acceptance/bundle/templates/default-python/integration_classic/out.plan_dev.json @@ -55,7 +55,7 @@ "job_cluster_key": "job_cluster", "libraries": [ { - "whl": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/artifacts/.internal/project_name_[UNIQUE_NAME]-0.0.1+[NUMID]-py3-none-any.whl" + "whl": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/artifacts/.internal/project_name_[UNIQUE_NAME]-0.0.1+[WHEEL_BUILD]-py3-none-any.whl" } ], "notebook_task": { @@ -72,7 +72,7 @@ "job_cluster_key": "job_cluster", "libraries": [ { - "whl": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/artifacts/.internal/project_name_[UNIQUE_NAME]-0.0.1+[NUMID]-py3-none-any.whl" + "whl": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/artifacts/.internal/project_name_[UNIQUE_NAME]-0.0.1+[WHEEL_BUILD]-py3-none-any.whl" } ], "python_wheel_task": { diff --git a/acceptance/bundle/templates/default-python/integration_classic/output.txt b/acceptance/bundle/templates/default-python/integration_classic/output.txt index 6511805177c..d79f35fdb88 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/output.txt +++ b/acceptance/bundle/templates/default-python/integration_classic/output.txt @@ -30,7 +30,7 @@ Building python_artifact... >>> [CLI] bundle deploy -t dev Building python_artifact... -Uploading .databricks/bundle/dev/patched_wheels/python_artifact_project_name_[UNIQUE_NAME]/project_name_[UNIQUE_NAME]-0.0.1+[NUMID]-py3-none-any.whl... +Uploading .databricks/bundle/dev/patched_wheels/python_artifact_project_name_[UNIQUE_NAME]/project_name_[UNIQUE_NAME]-0.0.1+[WHEEL_BUILD]-py3-none-any.whl... Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files... Created jobs.sample_job Created pipelines.project_name_[UNIQUE_NAME]_etl @@ -48,7 +48,7 @@ Resources: Jobs: sample_job: Name: [dev [USERNAME]] sample_job - URL: [DATABRICKS_URL]/jobs/[NUMID] + URL: [DATABRICKS_URL]/jobs/[JOB_ID] Pipelines: project_name_[UNIQUE_NAME]_etl: Name: [dev [USERNAME]] project_name_[UNIQUE_NAME]_etl @@ -62,7 +62,7 @@ Resources: @@ -51,4 +51,5 @@ "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", -+ "id": "[NUMID]", ++ "id": "[JOB_ID]", "job_clusters": [ { @@ -138,5 +139,6 @@ @@ -70,7 +70,7 @@ Resources: } - } + }, -+ "url": "[DATABRICKS_URL]/jobs/[NUMID]" ++ "url": "[DATABRICKS_URL]/jobs/[JOB_ID]" } }, @@ -155,4 +157,5 @@ @@ -308,7 +308,7 @@ Resources: Jobs: sample_job: Name: sample_job - URL: [DATABRICKS_URL]/jobs/[NUMID] + URL: [DATABRICKS_URL]/jobs/[JOB_ID] Pipelines: project_name_[UNIQUE_NAME]_etl: Name: project_name_[UNIQUE_NAME]_etl diff --git a/acceptance/bundle/templates/default-python/integration_classic/test.toml b/acceptance/bundle/templates/default-python/integration_classic/test.toml index 4d358acdc89..bcdbe2b610e 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/test.toml +++ b/acceptance/bundle/templates/default-python/integration_classic/test.toml @@ -16,9 +16,18 @@ EnvMatrix.UV_PYTHON = [ EnvRepl.UV_PYTHON = true [[Repls]] -# for some reason leading number is lost converting "%H%M%S" 061234 to 61234 -Old = '\d{5,}' -New = '[NUMID]' +# The patched wheel version carries a build timestamp. +Old = '0\.0\.1\+\d+' +New = '0.0.1+[WHEEL_BUILD]' + +# Job ids; the resource keys embed $UNIQUE_NAME, so replace_ids.py names would vary per run. +[[Repls]] +Old = '/jobs/\d+' +New = '/jobs/[JOB_ID]' + +[[Repls]] +Old = '"id": "\d+"' +New = '"id": "[JOB_ID]"' [[Repls]] Old = 'Standard_D3_v2' @@ -35,7 +44,7 @@ New = "python3 setup.py bdist_wheel" [[Repls]] # clean up ?[ow]= suffix after URL since not all workspaces have that -Old = '\?[ow]=\[NUMID\]' +Old = '\?[ow]=\[WORKSPACE_ID\]' New = '' Order = 1000 diff --git a/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt b/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt index 5142d961643..3fe9b24b805 100644 --- a/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt +++ b/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt @@ -55,7 +55,7 @@ To get started, refer to the project README.md file and the documentation at htt "catalog": { - "currentValue": "hive_metastore", + "currentValue": "customcatalog", - "nuid": "c4t4l0g-w1dg-3t12-3456-[NUMID]", + "nuid": "c4t4l0g-w1dg-3t12-3456-789012345678", "typedWidgetInfo": { "autoCreated": false, - "defaultValue": "hive_metastore", diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.json b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.json index e50081e941d..4ab34f76543 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.json +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.json @@ -33,7 +33,7 @@ } ], "format": "MULTI_TASK", - "job_id": [NUMID], + "job_id": [SAMPLE_JOB_ID], "max_concurrent_runs": 4, "name": "[dev [USERNAME]] sample_job", "parameters": [ @@ -94,7 +94,7 @@ ], "email_notifications": {}, "pipeline_task": { - "pipeline_id": "[UUID]" + "pipeline_id": "[MY_DEFAULT_PYTHON_ETL_ID]" }, "run_if": "ALL_SUCCESS", "task_key": "refresh_pipeline", @@ -197,7 +197,7 @@ "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files" ] }, - "id": "[UUID]", + "id": "[MY_DEFAULT_PYTHON_ETL_ID]", "last_modified": [UNIX_TIME_MILLIS][1], "libraries": [ { @@ -207,7 +207,7 @@ } ], "name": "[dev [USERNAME]] my_default_python_etl", - "pipeline_id": "[UUID]", + "pipeline_id": "[MY_DEFAULT_PYTHON_ETL_ID]", "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl", "run_as_user_name": "[USERNAME]", "schema": "[USERNAME]", diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.json b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.json index 0746846796a..57c1c81b2af 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.json +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.json @@ -33,7 +33,7 @@ } ], "format": "MULTI_TASK", - "job_id": [NUMID], + "job_id": [SAMPLE_JOB_ID_2], "max_concurrent_runs": 1, "name": "sample_job", "parameters": [ @@ -91,7 +91,7 @@ ], "email_notifications": {}, "pipeline_task": { - "pipeline_id": "[UUID]" + "pipeline_id": "[MY_DEFAULT_PYTHON_ETL_ID_2]" }, "run_if": "ALL_SUCCESS", "task_key": "refresh_pipeline", @@ -185,7 +185,7 @@ ], "action": "skip", "remote_state": { - "object_id": "/jobs/[NUMID]", + "object_id": "/jobs/[SAMPLE_JOB_ID_2]", "__embed__": [ { "level": "IS_OWNER", @@ -211,7 +211,7 @@ "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files" ] }, - "id": "[UUID]", + "id": "[MY_DEFAULT_PYTHON_ETL_ID_2]", "last_modified": [UNIX_TIME_MILLIS][1], "libraries": [ { @@ -221,7 +221,7 @@ } ], "name": "my_default_python_etl", - "pipeline_id": "[UUID]", + "pipeline_id": "[MY_DEFAULT_PYTHON_ETL_ID_2]", "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl", "run_as_user_name": "[USERNAME]", "schema": "prod", @@ -238,7 +238,7 @@ ], "action": "skip", "remote_state": { - "object_id": "/pipelines/[UUID]", + "object_id": "/pipelines/[MY_DEFAULT_PYTHON_ETL_ID_2]", "__embed__": [ { "level": "IS_OWNER", diff --git a/acceptance/bundle/templates/default-python/serverless/out.requests.dev.txt b/acceptance/bundle/templates/default-python/serverless/out.requests.dev.txt index 55dae66f070..873fa7fd1df 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.requests.dev.txt +++ b/acceptance/bundle/templates/default-python/serverless/out.requests.dev.txt @@ -167,7 +167,7 @@ } ], "pipeline_task": { - "pipeline_id": "[UUID]" + "pipeline_id": "[MY_DEFAULT_PYTHON_ETL_ID]" }, "task_key": "refresh_pipeline" } diff --git a/acceptance/bundle/templates/default-python/serverless/out.requests.prod.txt b/acceptance/bundle/templates/default-python/serverless/out.requests.prod.txt index b5e1fda37a3..98a8721ee74 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.requests.prod.txt +++ b/acceptance/bundle/templates/default-python/serverless/out.requests.prod.txt @@ -167,7 +167,7 @@ } ], "pipeline_task": { - "pipeline_id": "[UUID]" + "pipeline_id": "[MY_DEFAULT_PYTHON_ETL_ID_2]" }, "task_key": "refresh_pipeline" } @@ -183,7 +183,7 @@ } { "method": "PUT", - "path": "/api/2.0/permissions/directories/[NUMID]", + "path": "/api/2.0/permissions/directories/[DIRECTORY_ID]", "body": { "access_control_list": [ { @@ -195,7 +195,7 @@ } { "method": "PUT", - "path": "/api/2.0/permissions/jobs/[NUMID]", + "path": "/api/2.0/permissions/jobs/[SAMPLE_JOB_ID_2]", "body": { "access_control_list": [ { @@ -207,7 +207,7 @@ } { "method": "PUT", - "path": "/api/2.0/permissions/pipelines/[UUID]", + "path": "/api/2.0/permissions/pipelines/[MY_DEFAULT_PYTHON_ETL_ID_2]", "body": { "access_control_list": [ { diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/sample_notebook.ipynb b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/sample_notebook.ipynb index 49bbd5935d7..ea9c0a8404a 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/sample_notebook.ipynb +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/sample_notebook.ipynb @@ -82,7 +82,7 @@ "widgets": { "catalog": { "currentValue": "hive_metastore", - "nuid": "c4t4l0g-w1dg-3t12-3456-[NUMID]", + "nuid": "c4t4l0g-w1dg-3t12-3456-789012345678", "typedWidgetInfo": { "autoCreated": false, "defaultValue": "hive_metastore", @@ -108,7 +108,7 @@ }, "schema": { "currentValue": "[USERNAME]", - "nuid": "5ch3m4-w1dg-3t98-7654-[NUMID]", + "nuid": "5ch3m4-w1dg-3t98-7654-321098765432", "typedWidgetInfo": { "autoCreated": false, "defaultValue": "default", diff --git a/acceptance/bundle/templates/default-python/serverless/script b/acceptance/bundle/templates/default-python/serverless/script index dc0c328c711..93e5048afec 100644 --- a/acceptance/bundle/templates/default-python/serverless/script +++ b/acceptance/bundle/templates/default-python/serverless/script @@ -12,11 +12,13 @@ $CLI bundle plan -o json -t prod | nostamp > ../../out.plan_prod.json rm "$OUT_REQUESTS" trace $CLI bundle deploy -t dev +replace_ids.py -t dev print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.dev.txt trace $CLI bundle plan -t dev # check if if there is drift trace $CLI bundle plan -t dev -o json | nostamp > ../../out.plan_after_deploy_dev.json trace $CLI bundle deploy -t prod +replace_ids.py -t prod print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.prod.txt trace $CLI bundle plan -t prod # check if there is drift trace $CLI bundle plan -t prod -o json | nostamp > ../../out.plan_after_deploy_prod.json diff --git a/acceptance/bundle/templates/default-python/serverless/test.toml b/acceptance/bundle/templates/default-python/serverless/test.toml index 159efe02696..2d59bea68ed 100644 --- a/acceptance/bundle/templates/default-python/serverless/test.toml +++ b/acceptance/bundle/templates/default-python/serverless/test.toml @@ -1 +1,5 @@ RecordRequests = true + +[[Repls]] +Old = '/permissions/directories/\d+' +New = '/permissions/directories/[DIRECTORY_ID]' diff --git a/acceptance/cmd/api/default-profile-vs-env/output.txt b/acceptance/cmd/api/default-profile-vs-env/output.txt index f126744e5ee..e274cafcc3f 100644 --- a/acceptance/cmd/api/default-profile-vs-env/output.txt +++ b/acceptance/cmd/api/default-profile-vs-env/output.txt @@ -14,7 +14,7 @@ "cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/api_get cmd-exec-id/[UUID] interactive/none auth/pat" ], "X-Databricks-Workspace-Id": [ - "[NUMID]" + "[WORKSPACE_ID]" ] }, "method": "GET", diff --git a/acceptance/cmd/api/default-profile/output.txt b/acceptance/cmd/api/default-profile/output.txt index 35bb35fcc23..68e21b2a638 100644 --- a/acceptance/cmd/api/default-profile/output.txt +++ b/acceptance/cmd/api/default-profile/output.txt @@ -14,7 +14,7 @@ "cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/api_get cmd-exec-id/[UUID] interactive/none auth/pat" ], "X-Databricks-Workspace-Id": [ - "[NUMID]" + "[WORKSPACE_ID]" ] }, "method": "GET", diff --git a/acceptance/cmd/api/integer-precision/test.toml b/acceptance/cmd/api/integer-precision/test.toml index 2f47242d1da..c4f96e4c59e 100644 --- a/acceptance/cmd/api/integer-precision/test.toml +++ b/acceptance/cmd/api/integer-precision/test.toml @@ -10,12 +10,10 @@ Response.Body = '{"name": "a>> musterr [CLI] auth docker configure docker-test --account-id [UUID] Error: --account-id is not supported for auth docker configure. Select the workspace with [PROFILE] or --profile instead ->>> musterr [CLI] auth docker configure docker-test --workspace-id [NUMID] +>>> musterr [CLI] auth docker configure docker-test --workspace-id [WORKSPACE_ID] Error: --workspace-id is not supported for auth docker configure. Select the workspace with [PROFILE] or --profile instead === Profile argument and flag @@ -48,7 +48,7 @@ Error: profile "blank-auth" uses auth_type "". Docker credential helper requires === Another profile already maps to the workspace >>> musterr [CLI] auth docker configure docker-test -Error: multiple Databricks profiles match workspace ID [NUMID]: duplicate and docker-test. Remove duplicate workspace_id entries before using Docker credential helper +Error: multiple Databricks profiles match workspace ID [WORKSPACE_ID]: duplicate and docker-test. Remove duplicate workspace_id entries before using Docker credential helper === Nothing changed Docker config not created; profiles unchanged diff --git a/acceptance/cmd/auth/docker/configure/output.txt b/acceptance/cmd/auth/docker/configure/output.txt index ac0b64def36..dd985e386a5 100644 --- a/acceptance/cmd/auth/docker/configure/output.txt +++ b/acceptance/cmd/auth/docker/configure/output.txt @@ -1,6 +1,6 @@ >>> databricks auth docker host --profile docker-test -Registry host: [NUMID].container.us-west-2.localhost +Registry host: [WORKSPACE_ID].container.us-west-2.localhost Credential helper configured: NO >>> print_requests.py --get //api @@ -14,7 +14,7 @@ Credential helper configured: NO { "headers": { "X-Databricks-Workspace-Id": [ - "[NUMID]" + "[WORKSPACE_ID]" ] }, "method": "GET", @@ -22,7 +22,7 @@ Credential helper configured: NO } >>> databricks auth docker configure docker-test -Configured Docker credential helper for [NUMID].container.us-west-2.localhost +Configured Docker credential helper for [WORKSPACE_ID].container.us-west-2.localhost Updated Docker config: [TEST_TMP_DIR]/home/docker/config.json Installed Docker credential helper: [TEST_TMP_DIR]/home/bin/docker-credential-databricks @@ -37,7 +37,7 @@ Installed Docker credential helper: [TEST_TMP_DIR]/home/bin/docker-credential-da { "headers": { "X-Databricks-Workspace-Id": [ - "[NUMID]" + "[WORKSPACE_ID]" ] }, "method": "GET", @@ -52,7 +52,7 @@ Installed Docker credential helper: [TEST_TMP_DIR]/home/bin/docker-credential-da } }, "credHelpers": { - "[NUMID].container.us-west-2.localhost": "databricks" + "[WORKSPACE_ID].container.us-west-2.localhost": "databricks" } } @@ -63,11 +63,11 @@ Installed Docker credential helper: [TEST_TMP_DIR]/home/bin/docker-credential-da [docker-test] host = [DATABRICKS_URL] auth_type = databricks-cli -workspace_id = [NUMID] +workspace_id = [WORKSPACE_ID] >>> databricks auth docker host --profile docker-test --output json { - "host": "[NUMID].container.us-west-2.localhost", + "host": "[WORKSPACE_ID].container.us-west-2.localhost", "configured": true } @@ -75,7 +75,7 @@ workspace_id = [NUMID] { "headers": { "X-Databricks-Workspace-Id": [ - "[NUMID]" + "[WORKSPACE_ID]" ] }, "method": "GET", @@ -103,7 +103,7 @@ Docker config unchanged { "headers": { "X-Databricks-Workspace-Id": [ - "[NUMID]" + "[WORKSPACE_ID]" ] }, "method": "GET", @@ -114,7 +114,7 @@ Docker config unchanged >>> databricks auth docker configure docker-test --region us-west-2 Flag --region has been deprecated, --region will be fully removed in the next release -Configured Docker credential helper for [NUMID].container.us-west-2.localhost +Configured Docker credential helper for [WORKSPACE_ID].container.us-west-2.localhost Updated Docker config: [TEST_TMP_DIR]/home/docker/config.json Installed Docker credential helper: [TEST_TMP_DIR]/home/bin/docker-credential-databricks No requests sent @@ -123,7 +123,7 @@ No requests sent >>> databricks auth docker configure docker-test --region us-west-2 Flag --region has been deprecated, --region will be fully removed in the next release -Configured Docker credential helper for [NUMID].container.us-west-2.localhost +Configured Docker credential helper for [WORKSPACE_ID].container.us-west-2.localhost Updated Docker config: [TEST_TMP_DIR]/home/docker/config.json Installed Docker credential helper: [TEST_TMP_DIR]/home/bin/docker-credential-databricks @@ -139,7 +139,7 @@ Installed Docker credential helper: [TEST_TMP_DIR]/home/bin/docker-credential-da === Configure with the installation directory absent from PATH >>> command [TEST_TMP_DIR]/home/bin/databricks auth docker configure docker-test -Configured Docker credential helper for [NUMID].container.us-west-2.localhost +Configured Docker credential helper for [WORKSPACE_ID].container.us-west-2.localhost Updated Docker config: [TEST_TMP_DIR]/home/docker/config.json Installed Docker credential helper: [TEST_TMP_DIR]/home/bin/docker-credential-databricks Warning: ensure [TEST_TMP_DIR]/home/bin is on PATH before any other docker-credential-databricks helper, and that .CMD is in PATHEXT on Windows @@ -148,7 +148,7 @@ Warning: ensure [TEST_TMP_DIR]/home/bin is on PATH before any other docker-crede { "headers": { "X-Databricks-Workspace-Id": [ - "[NUMID]" + "[WORKSPACE_ID]" ] }, "method": "GET", diff --git a/acceptance/cmd/auth/docker/host/output.txt b/acceptance/cmd/auth/docker/host/output.txt index 8c56b08c2ae..912c3ecfbd8 100644 --- a/acceptance/cmd/auth/docker/host/output.txt +++ b/acceptance/cmd/auth/docker/host/output.txt @@ -22,13 +22,13 @@ Error: --host is not supported for auth docker host. Select the workspace with - >>> musterr [CLI] auth docker host --profile docker-test --account-id [UUID] Error: --account-id is not supported for auth docker host. Select the workspace with --profile instead ->>> musterr [CLI] auth docker host --profile docker-test --workspace-id [NUMID] +>>> musterr [CLI] auth docker host --profile docker-test --workspace-id 111222333 Error: --workspace-id is not supported for auth docker host. Select the workspace with --profile instead === Credential helper not configured >>> [CLI] auth docker host --profile docker-test -Registry host: [NUMID].container.us-west-2.localhost +Registry host: [WORKSPACE_ID].container.us-west-2.localhost Credential helper configured: NO === Invalid Docker config @@ -39,12 +39,12 @@ Error: read Docker config [TEST_TMP_DIR]/home/docker/config.json: invalid charac === Default credential store without an installed helper >>> PATH=[TEST_TMP_DIR]/home [CLI] auth docker host --profile docker-test -Registry host: [NUMID].container.us-west-2.localhost +Registry host: [WORKSPACE_ID].container.us-west-2.localhost Credential helper configured: NO >>> PATH=[TEST_TMP_DIR]/home [CLI] auth docker host --profile docker-test --output json { - "host": "[NUMID].container.us-west-2.localhost", + "host": "[WORKSPACE_ID].container.us-west-2.localhost", "configured": false } diff --git a/acceptance/cmd/auth/docker/token/output.txt b/acceptance/cmd/auth/docker/token/output.txt index a4cb041803a..28262b1f2fe 100644 --- a/acceptance/cmd/auth/docker/token/output.txt +++ b/acceptance/cmd/auth/docker/token/output.txt @@ -18,7 +18,7 @@ === No profile for the workspace >>> musterr [CLI] auth docker token -Error: no Databricks profile found for workspace ID [NUMID] from registry host [NUMID].container.us-west-2.localhost. Run databricks auth login --host and set workspace_id for that profile +Error: no Databricks profile found for workspace ID 111222333 from registry host 111222333.container.us-west-2.localhost. Run databricks auth login --host and set workspace_id for that profile === Not an Artifact Registry host @@ -43,15 +43,15 @@ Error: auth docker token does not support --host >>> musterr [CLI] auth docker token --account-id [UUID] Error: auth docker token does not support --account-id ->>> musterr [CLI] auth docker token --workspace-id [NUMID] +>>> musterr [CLI] auth docker token --workspace-id 111222333 Error: auth docker token does not support --workspace-id === Test-server registry never matches a remote workspace >>> musterr [CLI] auth docker token -Error: no Databricks profile found for workspace ID [NUMID] from registry host [NUMID].container.us-west-2.localhost. Run databricks auth login --host and set workspace_id for that profile +Error: no Databricks profile found for workspace ID 444555666 from registry host 444555666.container.us-west-2.localhost. Run databricks auth login --host and set workspace_id for that profile === Multiple profiles for the workspace >>> musterr [CLI] auth docker token -Error: multiple Databricks profiles match workspace ID [NUMID]: docker-test and duplicate. Remove duplicate workspace_id entries before using Docker credential helper +Error: multiple Databricks profiles match workspace ID [WORKSPACE_ID]: docker-test and duplicate. Remove duplicate workspace_id entries before using Docker credential helper diff --git a/acceptance/cmd/auth/login/configure-serverless/out.databrickscfg b/acceptance/cmd/auth/login/configure-serverless/out.databrickscfg index b095b683c97..aa7cc83ac1e 100644 --- a/acceptance/cmd/auth/login/configure-serverless/out.databrickscfg +++ b/acceptance/cmd/auth/login/configure-serverless/out.databrickscfg @@ -1,5 +1,5 @@ [DEFAULT] host = [DATABRICKS_URL] serverless_compute_id = auto -workspace_id = [NUMID] +workspace_id = [WORKSPACE_ID] auth_type = databricks-cli diff --git a/acceptance/cmd/auth/login/configure-serverless/output.txt b/acceptance/cmd/auth/login/configure-serverless/output.txt index bb29a75f698..439f2f63638 100644 --- a/acceptance/cmd/auth/login/configure-serverless/output.txt +++ b/acceptance/cmd/auth/login/configure-serverless/output.txt @@ -12,5 +12,5 @@ Profile DEFAULT was successfully saved [DEFAULT] host = [DATABRICKS_URL] serverless_compute_id = auto -workspace_id = [NUMID] +workspace_id = [WORKSPACE_ID] auth_type = databricks-cli diff --git a/acceptance/cmd/auth/login/discovery/out.databrickscfg b/acceptance/cmd/auth/login/discovery/out.databrickscfg index c1769c4cba9..a88732b3057 100644 --- a/acceptance/cmd/auth/login/discovery/out.databrickscfg +++ b/acceptance/cmd/auth/login/discovery/out.databrickscfg @@ -4,7 +4,7 @@ [discovery-test] host = [DATABRICKS_URL] account_id = test-account-123 -workspace_id = [NUMID] +workspace_id = [WORKSPACE_ID] client_id = discovery-client-id auth_type = databricks-cli diff --git a/acceptance/cmd/auth/login/host-from-profile/out.databrickscfg b/acceptance/cmd/auth/login/host-from-profile/out.databrickscfg index e39250c3133..a94a2407cef 100644 --- a/acceptance/cmd/auth/login/host-from-profile/out.databrickscfg +++ b/acceptance/cmd/auth/login/host-from-profile/out.databrickscfg @@ -4,5 +4,5 @@ [existing-profile] host = [DATABRICKS_URL] auth_type = databricks-cli -workspace_id = [NUMID] +workspace_id = [WORKSPACE_ID] client_id = flag-client-id diff --git a/acceptance/cmd/auth/login/host-from-profile/output.txt b/acceptance/cmd/auth/login/host-from-profile/output.txt index 841ec08ade0..9585b8cd388 100644 --- a/acceptance/cmd/auth/login/host-from-profile/output.txt +++ b/acceptance/cmd/auth/login/host-from-profile/output.txt @@ -31,5 +31,5 @@ json.q.client_id = "flag-client-id"; [existing-profile] host = [DATABRICKS_URL] auth_type = databricks-cli -workspace_id = [NUMID] +workspace_id = [WORKSPACE_ID] client_id = flag-client-id diff --git a/acceptance/cmd/auth/login/nominal/out.databrickscfg b/acceptance/cmd/auth/login/nominal/out.databrickscfg index d94ee2221d5..cf14685849c 100644 --- a/acceptance/cmd/auth/login/nominal/out.databrickscfg +++ b/acceptance/cmd/auth/login/nominal/out.databrickscfg @@ -3,7 +3,7 @@ [test] host = [DATABRICKS_URL] -workspace_id = [NUMID] +workspace_id = [WORKSPACE_ID] auth_type = databricks-cli [__settings__] diff --git a/acceptance/cmd/auth/login/preserve-fields/output.txt b/acceptance/cmd/auth/login/preserve-fields/output.txt index 28c0ed56608..2e8cd590fc0 100644 --- a/acceptance/cmd/auth/login/preserve-fields/output.txt +++ b/acceptance/cmd/auth/login/preserve-fields/output.txt @@ -18,5 +18,5 @@ cluster_id = existing-cluster-123 warehouse_id = warehouse-456 azure_environment = USGOVERNMENT custom_key = my-custom-value -workspace_id = [NUMID] +workspace_id = [WORKSPACE_ID] auth_type = databricks-cli diff --git a/acceptance/cmd/auth/login/with-scopes/out.databrickscfg b/acceptance/cmd/auth/login/with-scopes/out.databrickscfg index c8af1832d70..be136f325dc 100644 --- a/acceptance/cmd/auth/login/with-scopes/out.databrickscfg +++ b/acceptance/cmd/auth/login/with-scopes/out.databrickscfg @@ -3,7 +3,7 @@ [scoped-test] host = [DATABRICKS_URL] -workspace_id = [NUMID] +workspace_id = [WORKSPACE_ID] scopes = jobs,pipelines,clusters auth_type = databricks-cli diff --git a/acceptance/cmd/auth/profiles/output.txt b/acceptance/cmd/auth/profiles/output.txt index 2dd737198d6..df472a2fd5e 100644 --- a/acceptance/cmd/auth/profiles/output.txt +++ b/acceptance/cmd/auth/profiles/output.txt @@ -21,7 +21,7 @@ "name": "unified-profile", "host": "https://unified.databricks.test", "account_id": "unified-account-456", - "workspace_id": "[NUMID]", + "workspace_id": "987654321", "cloud": "aws", "auth_type": "", "valid": false diff --git a/acceptance/cmd/sandbox/register/success/test.toml b/acceptance/cmd/sandbox/register/success/test.toml index fc90eb4b36f..57f55ffe246 100644 --- a/acceptance/cmd/sandbox/register/success/test.toml +++ b/acceptance/cmd/sandbox/register/success/test.toml @@ -9,9 +9,7 @@ Response.Body = ''' } ''' -# Pin the synthetic test hash so the framework's generic 3+-digit-run → -# [NUMID] regex doesn't mangle it. +# Pin the synthetic test hash to a stable placeholder. [[Repls]] Old = 'a0b1c2d3e4f5a0b1c2d3e4f5a0b1c2d3' New = '[KEY_HASH]' -Order = 5 diff --git a/acceptance/cmd/sandbox/ssh-key/delete/success/test.toml b/acceptance/cmd/sandbox/ssh-key/delete/success/test.toml index 056a1e4b69d..2cdf0a7f67b 100644 --- a/acceptance/cmd/sandbox/ssh-key/delete/success/test.toml +++ b/acceptance/cmd/sandbox/ssh-key/delete/success/test.toml @@ -6,4 +6,3 @@ Response.Body = '{}' [[Repls]] Old = '98b3b2bce82049769b4ba03990460d09' New = '[KEY_HASH]' -Order = 5 diff --git a/acceptance/cmd/sandbox/ssh-key/list/with-entries/test.toml b/acceptance/cmd/sandbox/ssh-key/list/with-entries/test.toml index 113d31a76fd..2baaabd30e1 100644 --- a/acceptance/cmd/sandbox/ssh-key/list/with-entries/test.toml +++ b/acceptance/cmd/sandbox/ssh-key/list/with-entries/test.toml @@ -20,14 +20,11 @@ Response.Body = ''' } ''' -# Pin our test hashes to stable placeholders so the framework's -# generic 3+-digit-run → [NUMID] regex doesn't mangle them. +# Pin our test hashes to stable placeholders. [[Repls]] Old = '98b3b2bce82049769b4ba03990460d09' New = '[KEY_HASH_1]' -Order = 5 [[Repls]] Old = '9e0fc7c899aeee6a462e3153aeae8f30' New = '[KEY_HASH_2]' -Order = 5 diff --git a/acceptance/cmd/workspace/apps/output.txt b/acceptance/cmd/workspace/apps/output.txt index e9931b1421b..3fe9bf974c0 100644 --- a/acceptance/cmd/workspace/apps/output.txt +++ b/acceptance/cmd/workspace/apps/output.txt @@ -3,7 +3,7 @@ >>> [CLI] apps create --json @input.json { "active_deployment": { - "deployment_id": "deploy-[NUMID]", + "deployment_id": "deploy-[DEPLOYMENT_ID]", "source_code_path": "/Workspace/Users/[USERNAME]/test-name", "status": { "message": "Deployment succeeded", @@ -36,7 +36,7 @@ } ], "service_principal_client_id": "[UUID]", - "service_principal_id": [NUMID], + "service_principal_id": [SERVICE_PRINCIPAL_ID], "service_principal_name": "app-test-name", "url": "test-name-123.cloud.databricksapps.com" } @@ -45,7 +45,7 @@ >>> [CLI] apps update test-name --json @input.json { "active_deployment": { - "deployment_id": "deploy-[NUMID]", + "deployment_id": "deploy-[DEPLOYMENT_ID]", "source_code_path": "/Workspace/Users/[USERNAME]/test-name", "status": { "message": "Deployment succeeded", @@ -78,7 +78,7 @@ } ], "service_principal_client_id": "[UUID]", - "service_principal_id": [NUMID], + "service_principal_id": [SERVICE_PRINCIPAL_ID], "service_principal_name": "app-test-name", "url": "test-name-123.cloud.databricksapps.com" } diff --git a/acceptance/cmd/workspace/apps/test.toml b/acceptance/cmd/workspace/apps/test.toml index 159efe02696..76b0f7f680b 100644 --- a/acceptance/cmd/workspace/apps/test.toml +++ b/acceptance/cmd/workspace/apps/test.toml @@ -1 +1,10 @@ RecordRequests = true + +# The testserver mints these per run. +[[Repls]] +Old = 'deploy-\d+' +New = 'deploy-[DEPLOYMENT_ID]' + +[[Repls]] +Old = '"service_principal_id": \d+' +New = '"service_principal_id": [SERVICE_PRINCIPAL_ID]' diff --git a/acceptance/cmd/workspace/jobs/runs-submit/output.txt b/acceptance/cmd/workspace/jobs/runs-submit/output.txt index ad78389be4c..a6ba8d2da18 100644 --- a/acceptance/cmd/workspace/jobs/runs-submit/output.txt +++ b/acceptance/cmd/workspace/jobs/runs-submit/output.txt @@ -21,7 +21,7 @@ ] } { - "run_id": [NUMID] + "run_id": [RUN_ID] } >>> print_requests.py //jobs/runs/submit diff --git a/acceptance/cmd/workspace/jobs/runs-submit/test.toml b/acceptance/cmd/workspace/jobs/runs-submit/test.toml index 0ddd580a96a..8ff027e8dfa 100644 --- a/acceptance/cmd/workspace/jobs/runs-submit/test.toml +++ b/acceptance/cmd/workspace/jobs/runs-submit/test.toml @@ -3,3 +3,7 @@ Cloud = false RecordRequests = true # This command does not touch the bundle engine; run it once. + +[[Repls]] +Old = '"run_id": \d+' +New = '"run_id": [RUN_ID]' diff --git a/acceptance/experimental/open/output.txt b/acceptance/experimental/open/output.txt index f9a43b801be..3be71a1c819 100644 --- a/acceptance/experimental/open/output.txt +++ b/acceptance/experimental/open/output.txt @@ -1,11 +1,11 @@ === print URL for a job >>> [CLI] experimental open --url jobs 123 -[DATABRICKS_URL]/jobs/123?w=[NUMID] +[DATABRICKS_URL]/jobs/123?w=[WORKSPACE_ID] === print URL for a notebook >>> [CLI] experimental open --url notebooks 12345 -[DATABRICKS_URL]/?w=[NUMID]#notebook/12345 +[DATABRICKS_URL]/?w=[WORKSPACE_ID]#notebook/12345 === unknown resource type >>> [CLI] experimental open --url unknown 123 diff --git a/acceptance/internal/prepare_server.go b/acceptance/internal/prepare_server.go index b99c7546eb7..239a4a904a1 100644 --- a/acceptance/internal/prepare_server.go +++ b/acceptance/internal/prepare_server.go @@ -7,12 +7,14 @@ import ( "os" "path/filepath" "slices" + "strconv" "strings" "sync" "testing" "time" "unicode/utf8" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/iamutil" "github.com/databricks/cli/libs/testproxy" @@ -86,7 +88,7 @@ func lookupEnv(testEnv []string, key string) (string, bool) { return "", false } -func PrepareServerAndClient(t *testing.T, config TestConfig, logRequests bool, outputDir string, testEnv []string) (*sdkconfig.Config, iam.User) { +func PrepareServerAndClient(t *testing.T, config TestConfig, logRequests bool, outputDir string, testEnv []string) (*sdkconfig.Config, iam.User, string) { cloudEnv := env.Get(t.Context(), "CLOUD_ENV") recordRequests := isTruePtr(config.RecordRequests) @@ -116,6 +118,9 @@ func PrepareServerAndClient(t *testing.T, config TestConfig, logRequests bool, o user, err := iamutil.GetCurrentUser(t.Context(), w) require.NoError(t, err, "Failed to get current user") + workspaceID, err := auth.ResolveWorkspaceID(t.Context(), w) + require.NoError(t, err, "Failed to get current workspace id") + cfg := w.Config // If we are running in a cloud environment AND we need to intercept requests @@ -135,7 +140,7 @@ func PrepareServerAndClient(t *testing.T, config TestConfig, logRequests bool, o } } - return cfg, *user + return cfg, *user, workspaceID } // Same topology as cloud, with the testserver as the upstream. Both servers see @@ -152,7 +157,7 @@ func PrepareServerAndClient(t *testing.T, config TestConfig, logRequests bool, o Token: token, } - return cfg, testUser + return cfg, testUser, strconv.Itoa(testserver.TestWorkspaceID) } // If we are not recording requests, and no custom server stubs are configured, @@ -163,7 +168,7 @@ func PrepareServerAndClient(t *testing.T, config TestConfig, logRequests bool, o Token: token, } - return cfg, testUser + return cfg, testUser, strconv.Itoa(testserver.TestWorkspaceID) } // Default case. Start a dedicated local server for the test with the server stubs configured @@ -176,7 +181,7 @@ func PrepareServerAndClient(t *testing.T, config TestConfig, logRequests bool, o // For the purposes of replacements, use testUser for local runs. // Note, users might have overridden /api/2.0/preview/scim/v2/Me but that should not affect the replacement: - return cfg, testUser + return cfg, testUser, strconv.Itoa(testserver.TestWorkspaceID) } func recordRequestsCallback(t *testing.T, includeHeaders []string, outputDir string) func(request *testserver.Request) { diff --git a/acceptance/localenv/test.toml b/acceptance/localenv/test.toml index 7ebbafd8c03..adc40108ecb 100644 --- a/acceptance/localenv/test.toml +++ b/acceptance/localenv/test.toml @@ -1,7 +1,5 @@ [[Repls]] # environments setup-local emits a real wall-time durationMs; normalize it so the -# JSON goldens are deterministic. Order must beat the generic numeric repls -# (Order = 10 in the root test.toml), which would otherwise rewrite the digits first. +# JSON goldens are deterministic. Old = '"durationMs": \d+' New = '"durationMs": [DURATION_MS]' -Order = 8 diff --git a/acceptance/pipelines/deploy/auto-approve/output.txt b/acceptance/pipelines/deploy/auto-approve/output.txt index c05caf9a441..ab4d3a151d7 100644 --- a/acceptance/pipelines/deploy/auto-approve/output.txt +++ b/acceptance/pipelines/deploy/auto-approve/output.txt @@ -4,7 +4,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-auto Created pipelines.foo Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -View your pipeline foo here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] +View your pipeline foo here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] === Remove resources from configuration to test auto-approve >>> rm resources.yml diff --git a/acceptance/pipelines/deploy/create-pipeline/output.txt b/acceptance/pipelines/deploy/create-pipeline/output.txt index c363c4091bd..ad253d3a5e8 100644 --- a/acceptance/pipelines/deploy/create-pipeline/output.txt +++ b/acceptance/pipelines/deploy/create-pipeline/output.txt @@ -4,7 +4,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-create-pipeli Created pipelines.foo Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -View your pipeline foo here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] +View your pipeline foo here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] >>> [CLI] pipelines get [UUID] { diff --git a/acceptance/pipelines/deploy/fail-on-active-runs/output.txt b/acceptance/pipelines/deploy/fail-on-active-runs/output.txt index bf36987c6ba..14ff2fd2e1b 100644 --- a/acceptance/pipelines/deploy/fail-on-active-runs/output.txt +++ b/acceptance/pipelines/deploy/fail-on-active-runs/output.txt @@ -4,7 +4,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-fail-on-a Created pipelines.my_pipeline Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] >>> [CLI] pipelines deploy --fail-on-active-runs Error: pipeline [UUID] is running diff --git a/acceptance/pipelines/deploy/force-lock/output.txt b/acceptance/pipelines/deploy/force-lock/output.txt index 2f538e14332..e3922e9a536 100644 --- a/acceptance/pipelines/deploy/force-lock/output.txt +++ b/acceptance/pipelines/deploy/force-lock/output.txt @@ -20,4 +20,4 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-forc Created pipelines.foo Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -View your pipeline foo here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] +View your pipeline foo here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] diff --git a/acceptance/pipelines/deploy/render-diagnostics-warning/output.txt b/acceptance/pipelines/deploy/render-diagnostics-warning/output.txt index dcfad6150c8..abc05d1d7d5 100644 --- a/acceptance/pipelines/deploy/render-diagnostics-warning/output.txt +++ b/acceptance/pipelines/deploy/render-diagnostics-warning/output.txt @@ -8,5 +8,5 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/render-diagnostics Created pipelines.test-pipeline Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -View your pipeline test-pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] +View your pipeline test-pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] diff --git a/acceptance/pipelines/deploy/var-flag/output.txt b/acceptance/pipelines/deploy/var-flag/output.txt index 7346a2e3b22..01fa4e82c5e 100644 --- a/acceptance/pipelines/deploy/var-flag/output.txt +++ b/acceptance/pipelines/deploy/var-flag/output.txt @@ -5,7 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-var- Created pipelines.foo Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -View your pipeline foo here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] +View your pipeline foo here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] === Verify: Check that variables were substituted correctly >>> [CLI] pipelines get [UUID] diff --git a/acceptance/pipelines/destroy/auto-approve/output.txt b/acceptance/pipelines/destroy/auto-approve/output.txt index 0599f367e15..d792a3a0942 100644 --- a/acceptance/pipelines/destroy/auto-approve/output.txt +++ b/acceptance/pipelines/destroy/auto-approve/output.txt @@ -5,7 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-dest Created pipelines.my_pipeline Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] >>> errcode [CLI] pipelines destroy Error: this command will destroy all resources deployed by this bundle, including workspace files in the deployment directory. diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade-state-divergence/output.txt b/acceptance/pipelines/destroy/destroy-pipeline-cascade-state-divergence/output.txt index c2d79364e2a..d2321f4fa70 100644 --- a/acceptance/pipelines/destroy/destroy-pipeline-cascade-state-divergence/output.txt +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade-state-divergence/output.txt @@ -5,7 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-dest Created pipelines.my_pipeline Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] >>> print_requests.py //api/2.0/pipelines { diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt b/acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt index 1dea2437f20..641d04b1af2 100644 --- a/acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt @@ -5,7 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-dest Created pipelines.my_pipeline Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] >>> print_requests.py //api/2.0/pipelines { @@ -28,7 +28,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-dest Updated pipelines.my_pipeline Files: 1 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged -View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] >>> print_requests.py //api/2.0/pipelines @@ -38,7 +38,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-dest Updated pipelines.my_pipeline Files: 1 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged -View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] >>> print_requests.py //api/2.0/pipelines @@ -48,7 +48,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-dest Updated pipelines.my_pipeline Files: 1 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged -View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] >>> print_requests.py //api/2.0/pipelines diff --git a/acceptance/pipelines/destroy/destroy-pipeline/output.txt b/acceptance/pipelines/destroy/destroy-pipeline/output.txt index 24ed46530bf..a1456cf87e9 100644 --- a/acceptance/pipelines/destroy/destroy-pipeline/output.txt +++ b/acceptance/pipelines/destroy/destroy-pipeline/output.txt @@ -5,7 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-dest Created pipelines.my_pipeline Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] >>> [CLI] pipelines destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/pipelines/destroy/force-lock/output.txt b/acceptance/pipelines/destroy/force-lock/output.txt index c82fc5aeb73..28c43d1bfd0 100644 --- a/acceptance/pipelines/destroy/force-lock/output.txt +++ b/acceptance/pipelines/destroy/force-lock/output.txt @@ -4,7 +4,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-forc Created pipelines.foo Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -View your pipeline foo here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] +View your pipeline foo here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] === upload lock file >>> [CLI] workspace import /Workspace/Users/[USERNAME]/.bundle/test-pipeline-force-lock/default/state/deploy.lock --format AUTO --file ./deploy.lock diff --git a/acceptance/pipelines/dry-run/dry-run-pipeline/output.txt b/acceptance/pipelines/dry-run/dry-run-pipeline/output.txt index faa627d5aa2..a4e2c1eb7a4 100644 --- a/acceptance/pipelines/dry-run/dry-run-pipeline/output.txt +++ b/acceptance/pipelines/dry-run/dry-run-pipeline/output.txt @@ -4,7 +4,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run/ Created pipelines.my_pipeline Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] === Dry running pipeline, should have validate_only set to true >>> [CLI] pipelines dry-run diff --git a/acceptance/pipelines/dry-run/no-wait/output.txt b/acceptance/pipelines/dry-run/no-wait/output.txt index bc248d837a3..50e609a6814 100644 --- a/acceptance/pipelines/dry-run/no-wait/output.txt +++ b/acceptance/pipelines/dry-run/no-wait/output.txt @@ -4,7 +4,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run- Created pipelines.my_pipeline Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] === Dry running pipeline with --no-wait flag >>> [CLI] pipelines dry-run --no-wait diff --git a/acceptance/pipelines/dry-run/restart/output.txt b/acceptance/pipelines/dry-run/restart/output.txt index bd7deba892f..cf77b26a8e3 100644 --- a/acceptance/pipelines/dry-run/restart/output.txt +++ b/acceptance/pipelines/dry-run/restart/output.txt @@ -4,7 +4,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run- Created pipelines.my_pipeline Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] === Dry running pipeline with --restart flag, should stop the current pipeline and start a new run >>> [CLI] pipelines dry-run --restart diff --git a/acceptance/pipelines/e2e/output.txt b/acceptance/pipelines/e2e/output.txt index e72dbfeb1fb..d0e9a696ddc 100644 --- a/acceptance/pipelines/e2e/output.txt +++ b/acceptance/pipelines/e2e/output.txt @@ -21,8 +21,8 @@ Created jobs.sample_job Created pipelines.lakeflow_project_etl Files: 14 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged -View your job sample_job here: [DATABRICKS_URL]/jobs/[NUMID]?w=[NUMID] -View your pipeline lakeflow_project_etl here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] +View your job sample_job here: [DATABRICKS_URL]/jobs/[SAMPLE_JOB_ID]?w=[WORKSPACE_ID] +View your pipeline lakeflow_project_etl here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] === Run pipeline >>> [CLI] pipelines run @@ -45,9 +45,9 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lakeflow_project/d Created pipelines.lakeflow_project_etl_2 Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 2 unchanged -View your job sample_job here: [DATABRICKS_URL]/jobs/[NUMID]?w=[NUMID] -View your pipeline lakeflow_project_etl here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] -View your pipeline lakeflow_project_etl_2 here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] +View your job sample_job here: [DATABRICKS_URL]/jobs/[SAMPLE_JOB_ID]?w=[WORKSPACE_ID] +View your pipeline lakeflow_project_etl here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] +View your pipeline lakeflow_project_etl_2 here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] === Assert the second pipeline is created >>> [CLI] pipelines get [UUID] diff --git a/acceptance/pipelines/e2e/script b/acceptance/pipelines/e2e/script index dd8a50ada10..d393e3334a8 100644 --- a/acceptance/pipelines/e2e/script +++ b/acceptance/pipelines/e2e/script @@ -6,6 +6,7 @@ trace $CLI pipelines init --output-dir output title "Deploy pipeline" cd output/lakeflow_project trace $CLI pipelines deploy +read_id.py sample_job > /dev/null title "Run pipeline" trace $CLI pipelines run diff --git a/acceptance/pipelines/open/open-after-deployment/output.txt b/acceptance/pipelines/open/open-after-deployment/output.txt index c75ccaabb6a..cf468e04dd6 100644 --- a/acceptance/pipelines/open/open-after-deployment/output.txt +++ b/acceptance/pipelines/open/open-after-deployment/output.txt @@ -10,15 +10,15 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipelines-open/def Created pipelines.test-pipelines-open Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -View your pipeline test-pipelines-open here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] +View your pipeline test-pipelines-open here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] === Modify PATH so that real open is not run === open after deployment. This will fail to open browser and complain, that's ok, we only want the message >>> [CLI] pipelines open -Opening browser at [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] +Opening browser at [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] Error: exec: "open": cannot run executable found relative to current directory === open with KEY, expect same output as opening without KEY >>> [CLI] pipelines open test-pipelines-open -Opening browser at [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] +Opening browser at [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] Error: exec: "open": cannot run executable found relative to current directory diff --git a/acceptance/pipelines/run/no-wait/output.txt b/acceptance/pipelines/run/no-wait/output.txt index 056034fc8cf..ff1d3d832b2 100644 --- a/acceptance/pipelines/run/no-wait/output.txt +++ b/acceptance/pipelines/run/no-wait/output.txt @@ -4,7 +4,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run- Created pipelines.my_pipeline Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] === test --no-wait flag >>> [CLI] pipelines run --no-wait diff --git a/acceptance/pipelines/run/refresh-flags/output.txt b/acceptance/pipelines/run/refresh-flags/output.txt index 828fb6c2095..68dcd0c9a23 100644 --- a/acceptance/pipelines/run/refresh-flags/output.txt +++ b/acceptance/pipelines/run/refresh-flags/output.txt @@ -4,7 +4,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run- Created pipelines.my_pipeline Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] === Running pipeline with --refresh flag and specific tables >>> [CLI] pipelines run --refresh table1,table2 diff --git a/acceptance/pipelines/run/restart/output.txt b/acceptance/pipelines/run/restart/output.txt index 75d9967f35b..07a3215d5dd 100644 --- a/acceptance/pipelines/run/restart/output.txt +++ b/acceptance/pipelines/run/restart/output.txt @@ -4,7 +4,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run- Created pipelines.my_pipeline Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] === Running pipeline with --restart flag >>> [CLI] pipelines run --restart diff --git a/acceptance/pipelines/run/run-info/output.txt b/acceptance/pipelines/run/run-info/output.txt index ba3eed0f52f..755270d2611 100644 --- a/acceptance/pipelines/run/run-info/output.txt +++ b/acceptance/pipelines/run/run-info/output.txt @@ -4,7 +4,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run/ Created pipelines.my_pipeline Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] === Run pipeline and display pipeline configuration summary and events durations >>> [CLI] pipelines run diff --git a/acceptance/pipelines/stop/output.txt b/acceptance/pipelines/stop/output.txt index 98217c8e70b..76587739aed 100644 --- a/acceptance/pipelines/stop/output.txt +++ b/acceptance/pipelines/stop/output.txt @@ -4,7 +4,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-stop Created pipelines.my_pipeline Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] >>> [CLI] pipelines run Recommendation: This command runs the last deployed version of the code diff --git a/acceptance/ssh/bulk-transfer/script b/acceptance/ssh/bulk-transfer/script index c5b239407d5..7e043dedf28 100644 --- a/acceptance/ssh/bulk-transfer/script +++ b/acceptance/ssh/bulk-transfer/script @@ -9,8 +9,7 @@ # with truncated output and, depending on which side noticed the drop first, exit code 0. # # 8 MiB is the size that matters: the remote IDE server pushes about 7 MiB while it starts, and -# it is eight times the window that broke. Keep the payload under 10,000,000 bytes - the root -# test.toml rewrites any run of 8 or more digits to [NUMID], which would erase the counts. +# it is eight times the window that broke. PAYLOAD_BYTES=8388608 # Generated rather than committed: 8 MiB does not belong in the repo, and the exact bytes do not diff --git a/acceptance/telemetry/failure/output.txt b/acceptance/telemetry/failure/output.txt index 3d754ba24d8..a4a56f92e22 100644 --- a/acceptance/telemetry/failure/output.txt +++ b/acceptance/telemetry/failure/output.txt @@ -7,10 +7,10 @@ HH:MM:SS Debug: GET /.well-known/databricks-config < HTTP/1.1 200 OK < { < "oidc_endpoint": "[DATABRICKS_URL]/oidc", -< "workspace_id": "[NUMID]" +< "workspace_id": "[WORKSPACE_ID]" < } pid=PID sdk=true HH:MM:SS Debug: [Local Cache] computed and stored result pid=PID -HH:MM:SS Debug: Resolved workspace_id from host metadata: "[NUMID]" pid=PID sdk=true +HH:MM:SS Debug: Resolved workspace_id from host metadata: "[WORKSPACE_ID]" pid=PID sdk=true HH:MM:SS Debug: Resolved cloud from hostname: "AWS" pid=PID sdk=true HH:MM:SS Debug: Resolved discovery_url from host metadata: "[DATABRICKS_URL]/oidc/.well-known/oauth-authorization-server" pid=PID sdk=true HH:MM:SS Info: completed execution pid=PID exit_code=0 diff --git a/acceptance/telemetry/partial-success/output.txt b/acceptance/telemetry/partial-success/output.txt index c43506faf8b..cfcda42e67d 100644 --- a/acceptance/telemetry/partial-success/output.txt +++ b/acceptance/telemetry/partial-success/output.txt @@ -7,10 +7,10 @@ HH:MM:SS Debug: GET /.well-known/databricks-config < HTTP/1.1 200 OK < { < "oidc_endpoint": "[DATABRICKS_URL]/oidc", -< "workspace_id": "[NUMID]" +< "workspace_id": "[WORKSPACE_ID]" < } pid=PID sdk=true HH:MM:SS Debug: [Local Cache] computed and stored result pid=PID -HH:MM:SS Debug: Resolved workspace_id from host metadata: "[NUMID]" pid=PID sdk=true +HH:MM:SS Debug: Resolved workspace_id from host metadata: "[WORKSPACE_ID]" pid=PID sdk=true HH:MM:SS Debug: Resolved cloud from hostname: "AWS" pid=PID sdk=true HH:MM:SS Debug: Resolved discovery_url from host metadata: "[DATABRICKS_URL]/oidc/.well-known/oauth-authorization-server" pid=PID sdk=true HH:MM:SS Info: completed execution pid=PID exit_code=0 diff --git a/acceptance/telemetry/skipped/output.txt b/acceptance/telemetry/skipped/output.txt index f72d02ed475..85a1de2dbdf 100644 --- a/acceptance/telemetry/skipped/output.txt +++ b/acceptance/telemetry/skipped/output.txt @@ -7,10 +7,10 @@ HH:MM:SS Debug: GET /.well-known/databricks-config < HTTP/1.1 200 OK < { < "oidc_endpoint": "[DATABRICKS_URL]/oidc", -< "workspace_id": "[NUMID]" +< "workspace_id": "[WORKSPACE_ID]" < } pid=PID sdk=true HH:MM:SS Debug: [Local Cache] computed and stored result pid=PID -HH:MM:SS Debug: Resolved workspace_id from host metadata: "[NUMID]" pid=PID sdk=true +HH:MM:SS Debug: Resolved workspace_id from host metadata: "[WORKSPACE_ID]" pid=PID sdk=true HH:MM:SS Debug: Resolved cloud from hostname: "AWS" pid=PID sdk=true HH:MM:SS Debug: Resolved discovery_url from host metadata: "[DATABRICKS_URL]/oidc/.well-known/oauth-authorization-server" pid=PID sdk=true HH:MM:SS Info: completed execution pid=PID exit_code=0 diff --git a/acceptance/telemetry/success/output.txt b/acceptance/telemetry/success/output.txt index 20aee525f7a..67a3874fe77 100644 --- a/acceptance/telemetry/success/output.txt +++ b/acceptance/telemetry/success/output.txt @@ -7,10 +7,10 @@ HH:MM:SS Debug: GET /.well-known/databricks-config < HTTP/1.1 200 OK < { < "oidc_endpoint": "[DATABRICKS_URL]/oidc", -< "workspace_id": "[NUMID]" +< "workspace_id": "[WORKSPACE_ID]" < } pid=PID sdk=true HH:MM:SS Debug: [Local Cache] computed and stored result pid=PID -HH:MM:SS Debug: Resolved workspace_id from host metadata: "[NUMID]" pid=PID sdk=true +HH:MM:SS Debug: Resolved workspace_id from host metadata: "[WORKSPACE_ID]" pid=PID sdk=true HH:MM:SS Debug: Resolved cloud from hostname: "AWS" pid=PID sdk=true HH:MM:SS Debug: Resolved discovery_url from host metadata: "[DATABRICKS_URL]/oidc/.well-known/oauth-authorization-server" pid=PID sdk=true HH:MM:SS Info: completed execution pid=PID exit_code=0 diff --git a/acceptance/telemetry/test.toml b/acceptance/telemetry/test.toml index eaf13403ffc..0b3a05238da 100644 --- a/acceptance/telemetry/test.toml +++ b/acceptance/telemetry/test.toml @@ -38,8 +38,6 @@ New = "(N more bytes)" # Host metadata cache keys vary per-test because the mock server URL changes. # Normalize them so the golden output is stable across runs. -# Order=1 so it runs before the parent's `\d{14,}` → `[NUMID]` replacement. [[Repls]] Old = '[a-f0-9]{64}' New = "[SHA256_HASH]" -Order = 1 diff --git a/acceptance/telemetry/timeout/output.txt b/acceptance/telemetry/timeout/output.txt index 1957ad448aa..550fe861652 100644 --- a/acceptance/telemetry/timeout/output.txt +++ b/acceptance/telemetry/timeout/output.txt @@ -7,10 +7,10 @@ HH:MM:SS Debug: GET /.well-known/databricks-config < HTTP/1.1 200 OK < { < "oidc_endpoint": "[DATABRICKS_URL]/oidc", -< "workspace_id": "[NUMID]" +< "workspace_id": "[WORKSPACE_ID]" < } pid=PID sdk=true HH:MM:SS Debug: [Local Cache] computed and stored result pid=PID -HH:MM:SS Debug: Resolved workspace_id from host metadata: "[NUMID]" pid=PID sdk=true +HH:MM:SS Debug: Resolved workspace_id from host metadata: "[WORKSPACE_ID]" pid=PID sdk=true HH:MM:SS Debug: Resolved cloud from hostname: "AWS" pid=PID sdk=true HH:MM:SS Debug: Resolved discovery_url from host metadata: "[DATABRICKS_URL]/oidc/.well-known/oauth-authorization-server" pid=PID sdk=true HH:MM:SS Info: completed execution pid=PID exit_code=0 diff --git a/acceptance/test.toml b/acceptance/test.toml index 3d965ec9729..83d92571f08 100644 --- a/acceptance/test.toml +++ b/acceptance/test.toml @@ -31,21 +31,11 @@ SortLines.deploy_actions = '^(Created|Updated|Deleted|Recreated|Resized) \S+$' # datetime.datetime(2030, 3, 17, 18, 46, 40) [[Repls]] -Old = '\d{20,}' -New = "[NUMID]" -Order = 10 - -[[Repls]] -Old = '1[78]\d{17}' +Old = '\b1[78]\d{17}\b' New = '[UNIX_TIME_NANOS]' Order = 10 Distinct = true -[[Repls]] -Old = '\d{17,}' -New = "[NUMID]" -Order = 10 - #[[Repls]] # This makes certain integration tests flaky as experiment IDs overlap in range with this regex #Old = '1[78]\d{14}' @@ -53,31 +43,16 @@ Order = 10 #Order = 10 [[Repls]] -Old = '\d{14,}' -New = "[NUMID]" -Order = 10 - -[[Repls]] -Old = '1[78]\d{11}' +Old = '\b1[78]\d{11}\b' New = '[UNIX_TIME_MILLIS]' Order = 10 Distinct = true [[Repls]] -Old = '\d{11,}' -New = "[NUMID]" -Order = 10 - -[[Repls]] -Old = '1[78]\d{8}' +Old = '\b1[78]\d{8}\b' New = '[UNIX_TIME_S]' Order = 10 -[[Repls]] -Old = '\d{8,}' -New = "[NUMID]" -Order = 10 - [[Repls]] # "2025-10-23T14:26:43.499488Z" Old = '2\d\d\d-\d\d-\d\d(T| )\d\d:\d\d:\d\d\.\d+(Z|[+-]\d\d:\d\d)?' diff --git a/acceptance/workspace/jobs/create/output.txt b/acceptance/workspace/jobs/create/output.txt index 80a25fcf5e0..df669116179 100644 --- a/acceptance/workspace/jobs/create/output.txt +++ b/acceptance/workspace/jobs/create/output.txt @@ -1,7 +1,7 @@ >>> [CLI] jobs create --json {"name":"abc"} { - "job_id": [NUMID] + "job_id": [JOB_ID] } >>> print_requests.py //jobs diff --git a/acceptance/workspace/jobs/create/script b/acceptance/workspace/jobs/create/script index 00344560ed8..9d579a353c5 100644 --- a/acceptance/workspace/jobs/create/script +++ b/acceptance/workspace/jobs/create/script @@ -1,3 +1,5 @@ -trace $CLI jobs create --json '{"name":"abc"}' +trace $CLI jobs create --json '{"name":"abc"}' | tee job.json.tmp +add_repl "$(sed -n 's/.*"job_id": \([0-9]*\).*/\1/p' job.json.tmp)" JOB_ID +rm job.json.tmp trace print_requests.py //jobs diff --git a/acceptance/workspace/repos/create_with_provider/output.txt b/acceptance/workspace/repos/create_with_provider/output.txt index ad4824ca645..07d9eac9623 100644 --- a/acceptance/workspace/repos/create_with_provider/output.txt +++ b/acceptance/workspace/repos/create_with_provider/output.txt @@ -2,11 +2,11 @@ >>> [CLI] repos create https://github.com/databricks/databricks-empty-ide-project.git gitHub --path /Repos/me@databricks.com/test-repo === Get by id should work ->>> [CLI] repos get [NUMID] -o json +>>> [CLI] repos get [REPO_ID] -o json { "branch": "main", "head_commit_id": "0a1b2c3d4e5f6a7b8e7dadd73e50a69d8ba47d8f", - "id": [NUMID], + "id": [REPO_ID], "path": "/Repos/me@databricks.com/test-repo", "provider": "gitHub", "url": "https://github.com/databricks/databricks-empty-ide-project.git" @@ -17,14 +17,14 @@ { "branch": "main", "head_commit_id": "0a1b2c3d4e5f6a7b8e7dadd73e50a69d8ba47d8f", - "id": [NUMID], + "id": [REPO_ID], "path": "/Repos/me@databricks.com/test-repo", "provider": "gitHub", "url": "https://github.com/databricks/databricks-empty-ide-project.git" } === Delete by id should work ->>> [CLI] repos delete [NUMID] +>>> [CLI] repos delete [REPO_ID] >>> print_requests.py --get //repos //workspace/get-status { @@ -38,7 +38,7 @@ } { "method": "GET", - "path": "/api/2.0/repos/[NUMID]" + "path": "/api/2.0/repos/[REPO_ID]" } { "method": "GET", @@ -49,9 +49,9 @@ } { "method": "GET", - "path": "/api/2.0/repos/[NUMID]" + "path": "/api/2.0/repos/[REPO_ID]" } { "method": "DELETE", - "path": "/api/2.0/repos/[NUMID]" + "path": "/api/2.0/repos/[REPO_ID]" } diff --git a/acceptance/workspace/repos/create_with_provider/script b/acceptance/workspace/repos/create_with_provider/script index 4f3a49d21b7..52de07f8921 100644 --- a/acceptance/workspace/repos/create_with_provider/script +++ b/acceptance/workspace/repos/create_with_provider/script @@ -3,6 +3,7 @@ provider=gitHub path=/Repos/me@databricks.com/test-repo repo_id=$(MSYS_NO_PATHCONV=1 trace $CLI repos create $url $provider --path $path | jq .id -r) +add_repl "$repo_id" REPO_ID title "Get by id should work" MSYS_NO_PATHCONV=1 trace $CLI repos get $repo_id -o json diff --git a/acceptance/workspace/repos/create_without_provider/output.txt b/acceptance/workspace/repos/create_without_provider/output.txt index b59b7cc614e..1945fb50252 100644 --- a/acceptance/workspace/repos/create_without_provider/output.txt +++ b/acceptance/workspace/repos/create_without_provider/output.txt @@ -3,7 +3,7 @@ { "branch": "main", "head_commit_id": "0a1b2c3d4e5f6a7b8e7dadd73e50a69d8ba47d8f", - "id": [NUMID], + "id": [REPO_ID], "path": "/Repos/me@databricks.com/test-repo", "provider": "gitHub", "url": "https://github.com/databricks/databricks-empty-ide-project.git" diff --git a/acceptance/workspace/repos/create_without_provider/test.toml b/acceptance/workspace/repos/create_without_provider/test.toml new file mode 100644 index 00000000000..17099bad1de --- /dev/null +++ b/acceptance/workspace/repos/create_without_provider/test.toml @@ -0,0 +1,3 @@ +[[Repls]] +Old = '"id": \d+,' +New = '"id": [REPO_ID],' diff --git a/acceptance/workspace/repos/delete_by_path/output.txt b/acceptance/workspace/repos/delete_by_path/output.txt index 6f8969139bc..d1cef31426b 100644 --- a/acceptance/workspace/repos/delete_by_path/output.txt +++ b/acceptance/workspace/repos/delete_by_path/output.txt @@ -1,12 +1,12 @@ >>> [CLI] repos create https://github.com/databricks/databricks-empty-ide-project.git gitHub --path /Repos/me@databricks.com/test-repo -[NUMID] +[REPO_ID] >>> [CLI] repos get /Repos/me@databricks.com/test-repo -o json { "branch": "main", "head_commit_id": "0a1b2c3d4e5f6a7b8e7dadd73e50a69d8ba47d8f", - "id": [NUMID], + "id": [REPO_ID], "path": "/Repos/me@databricks.com/test-repo", "provider": "gitHub", "url": "https://github.com/databricks/databricks-empty-ide-project.git" diff --git a/acceptance/workspace/repos/delete_by_path/script b/acceptance/workspace/repos/delete_by_path/script index 6c0d1e404fe..98e1928e2aa 100644 --- a/acceptance/workspace/repos/delete_by_path/script +++ b/acceptance/workspace/repos/delete_by_path/script @@ -2,7 +2,9 @@ url=https://github.com/databricks/databricks-empty-ide-project.git provider=gitHub path=/Repos/me@databricks.com/test-repo -MSYS_NO_PATHCONV=1 trace $CLI repos create $url $provider --path $path | jq .id -r +MSYS_NO_PATHCONV=1 trace $CLI repos create $url $provider --path $path | jq .id -r | tee repo_id.tmp +add_repl "$(cat repo_id.tmp)" REPO_ID +rm repo_id.tmp MSYS_NO_PATHCONV=1 trace $CLI repos get $path -o json diff --git a/acceptance/workspace/repos/git_cli_folder/output.txt b/acceptance/workspace/repos/git_cli_folder/output.txt index 956b55259d0..f27bd344833 100644 --- a/acceptance/workspace/repos/git_cli_folder/output.txt +++ b/acceptance/workspace/repos/git_cli_folder/output.txt @@ -1,13 +1,13 @@ >>> [CLI] repos create https://github.com/databricks/databricks-empty-ide-project.git gitHub --path /Workspace/Users/me@databricks.com/test-repo -[NUMID] +[REPO_ID] === Get by path resolves a Git CLI folder >>> [CLI] repos get /Workspace/Users/me@databricks.com/test-repo -o json { "branch": "main", "head_commit_id": "0a1b2c3d4e5f6a7b8e7dadd73e50a69d8ba47d8f", - "id": [NUMID], + "id": [REPO_ID], "path": "/Workspace/Users/me@databricks.com/test-repo", "provider": "gitHub", "url": "https://github.com/databricks/databricks-empty-ide-project.git" @@ -38,7 +38,7 @@ } { "method": "GET", - "path": "/api/2.0/repos/[NUMID]" + "path": "/api/2.0/repos/[REPO_ID]" } { "method": "GET", @@ -49,7 +49,7 @@ } { "method": "PATCH", - "path": "/api/2.0/repos/[NUMID]", + "path": "/api/2.0/repos/[REPO_ID]", "body": { "branch": "update-by-path" } @@ -63,5 +63,5 @@ } { "method": "DELETE", - "path": "/api/2.0/repos/[NUMID]" + "path": "/api/2.0/repos/[REPO_ID]" } diff --git a/acceptance/workspace/repos/git_cli_folder/script b/acceptance/workspace/repos/git_cli_folder/script index 46661bafb63..e64e97420d2 100644 --- a/acceptance/workspace/repos/git_cli_folder/script +++ b/acceptance/workspace/repos/git_cli_folder/script @@ -4,7 +4,9 @@ provider=gitHub # DIRECTORY, not a REPO. Path-based commands must still resolve it to a repo ID. path=/Workspace/Users/me@databricks.com/test-repo -MSYS_NO_PATHCONV=1 trace $CLI repos create $url $provider --path $path | jq .id -r +MSYS_NO_PATHCONV=1 trace $CLI repos create $url $provider --path $path | jq .id -r | tee repo_id.tmp +add_repl "$(cat repo_id.tmp)" REPO_ID +rm repo_id.tmp title "Get by path resolves a Git CLI folder" MSYS_NO_PATHCONV=1 trace $CLI repos get $path -o json diff --git a/acceptance/workspace/repos/update/output.txt b/acceptance/workspace/repos/update/output.txt index 9532d356c46..24414ac795d 100644 --- a/acceptance/workspace/repos/update/output.txt +++ b/acceptance/workspace/repos/update/output.txt @@ -1,13 +1,13 @@ >>> [CLI] repos create https://github.com/databricks/databricks-empty-ide-project.git gitHub --path /Repos/me@databricks.com/test-repo ->>> [CLI] repos update [NUMID] --branch update-by-id +>>> [CLI] repos update [REPO_ID] --branch update-by-id ->>> [CLI] repos get [NUMID] -o json +>>> [CLI] repos get [REPO_ID] -o json { "branch": "update-by-id", "head_commit_id": "0a1b2c3d4e5f6a7b8e7dadd73e50a69d8ba47d8f", - "id": [NUMID], + "id": [REPO_ID], "path": "/Repos/me@databricks.com/test-repo", "provider": "gitHub", "url": "https://github.com/databricks/databricks-empty-ide-project.git" @@ -15,11 +15,11 @@ >>> [CLI] repos update /Repos/me@databricks.com/test-repo --branch update-by-path ->>> [CLI] repos get [NUMID] -o json +>>> [CLI] repos get [REPO_ID] -o json { "branch": "update-by-path", "head_commit_id": "0a1b2c3d4e5f6a7b8e7dadd73e50a69d8ba47d8f", - "id": [NUMID], + "id": [REPO_ID], "path": "/Repos/me@databricks.com/test-repo", "provider": "gitHub", "url": "https://github.com/databricks/databricks-empty-ide-project.git" @@ -37,14 +37,14 @@ } { "method": "PATCH", - "path": "/api/2.0/repos/[NUMID]", + "path": "/api/2.0/repos/[REPO_ID]", "body": { "branch": "update-by-id" } } { "method": "GET", - "path": "/api/2.0/repos/[NUMID]" + "path": "/api/2.0/repos/[REPO_ID]" } { "method": "GET", @@ -55,12 +55,12 @@ } { "method": "PATCH", - "path": "/api/2.0/repos/[NUMID]", + "path": "/api/2.0/repos/[REPO_ID]", "body": { "branch": "update-by-path" } } { "method": "GET", - "path": "/api/2.0/repos/[NUMID]" + "path": "/api/2.0/repos/[REPO_ID]" } diff --git a/acceptance/workspace/repos/update/script b/acceptance/workspace/repos/update/script index db166f6f415..c6b449533bc 100644 --- a/acceptance/workspace/repos/update/script +++ b/acceptance/workspace/repos/update/script @@ -3,6 +3,7 @@ provider=gitHub path=/Repos/me@databricks.com/test-repo repo_id=$(MSYS_NO_PATHCONV=1 trace $CLI repos create $url $provider --path $path | jq .id -r) +add_repl "$repo_id" REPO_ID MSYS_NO_PATHCONV=1 trace $CLI repos update $repo_id --branch update-by-id diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index a56e2dc4c0b..a8217e3bb20 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -10,6 +10,7 @@ import ( "mime/multipart" "net/http" "path" + "strconv" "strings" "github.com/databricks/databricks-sdk-go/service/catalog" @@ -19,10 +20,13 @@ import ( "github.com/databricks/databricks-sdk-go/service/workspace" ) +// TestWorkspaceID is the id of the fake workspace. +const TestWorkspaceID = 900800700600 + var TestMetastore = catalog.MetastoreAssignment{ DefaultCatalogName: "hive_metastore", MetastoreId: "120efa64-9b68-46ba-be38-f319458430d2", - WorkspaceId: 900800700600, + WorkspaceId: TestWorkspaceID, } func AddDefaultHandlers(server *Server) { @@ -75,7 +79,7 @@ func AddDefaultHandlers(server *Server) { server.Handle("GET", "/api/2.0/preview/scim/v2/Me", func(req Request) any { return Response{ - Headers: map[string][]string{"X-Databricks-Org-Id": {"900800700600"}}, + Headers: map[string][]string{"X-Databricks-Org-Id": {strconv.Itoa(TestWorkspaceID)}}, Body: req.Workspace.MeUser(req.Token), } }) diff --git a/libs/testserver/jobs.go b/libs/testserver/jobs.go index ed2a7ac48b0..7fd1c359211 100644 --- a/libs/testserver/jobs.go +++ b/libs/testserver/jobs.go @@ -559,7 +559,7 @@ func (s *FakeWorkspace) JobsRunNow(req Request) Response { RunId: runId, JobId: request.JobId, State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, - RunPageUrl: fmt.Sprintf("%s/?o=900800700600#job/%d/run/%d", s.url, request.JobId, runId), + RunPageUrl: fmt.Sprintf("%s/?o=%d#job/%d/run/%d", s.url, TestWorkspaceID, request.JobId, runId), RunType: jobs.RunTypeJobRun, RunName: runName, Tasks: tasks, @@ -673,7 +673,7 @@ func (s *FakeWorkspace) JobsSubmit(req Request) Response { s.JobRuns[runId] = jobs.Run{ RunId: runId, State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, - RunPageUrl: fmt.Sprintf("%s/?o=900800700600#job/run/%d", s.url, runId), + RunPageUrl: fmt.Sprintf("%s/?o=%d#job/run/%d", s.url, TestWorkspaceID, runId), RunType: jobs.RunTypeSubmitRun, RunName: runName, Tasks: tasks, diff --git a/libs/testserver/repos.go b/libs/testserver/repos.go index 4686ba3103a..9e3f3d5bd18 100644 --- a/libs/testserver/repos.go +++ b/libs/testserver/repos.go @@ -11,8 +11,7 @@ import ( // fakeHeadCommitID is the commit the fake reports a clone is checked out to. // Nothing is cloned here, so the remote's real HEAD is unknowable. The counting -// prefix marks it as synthetic, so it is not mistaken for a real sha; it avoids -// runs of three digits, which the test replacements would rewrite to [NUMID]. +// prefix marks it as synthetic, so it is not mistaken for a real sha. const fakeHeadCommitID = "0a1b2c3d4e5f6a7b8e7dadd73e50a69d8ba47d8f" func (s *FakeWorkspace) ReposCreate(req Request) Response { diff --git a/libs/testserver/server.go b/libs/testserver/server.go index 7f949ebf5cd..1f56717c50a 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -319,7 +319,7 @@ Response.Body = '' s.Handle("GET", "/.well-known/databricks-config", func(_ Request) any { return map[string]any{ "oidc_endpoint": server.URL + "/oidc", - "workspace_id": "900800700600", + "workspace_id": strconv.Itoa(TestWorkspaceID), } })