-
Notifications
You must be signed in to change notification settings - Fork 156
Fix yaml config-remote-sync snapshot creation when bundle is deployed without any resources #4889
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from 2 commits
Commits
Show all changes
11 commits
Select commit
Hold shift + click to select a range
0eed91b
Fix yaml sync snapshot when bundle is deployed without any resources
ilyakuz-db f681927
Don't stop deploy when snapshot is not created
ilyakuz-db af91585
Use existing tests
ilyakuz-db 9c5b798
Remove diags
ilyakuz-db 6427e9d
Fix yaml sync snapshot when bundle is deployed without any resources
ilyakuz-db eccb2e7
Don't stop deploy when snapshot is not created
ilyakuz-db afe7735
Use existing tests
ilyakuz-db 9b5dae8
Remove diags
ilyakuz-db 06c92bc
Merge remote branch fix-yaml-sync-empty-deploy after rebase on main
ilyakuz-db 20430d3
Fix lint
ilyakuz-db 767b42e
Merge branch 'main' of github.com:databricks/cli into fix-yaml-sync-e…
ilyakuz-db File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
2 changes: 2 additions & 0 deletions
2
acceptance/bundle/config-remote-sync/empty_deploy/databricks.yml.tmpl
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| bundle: | ||
| name: test-bundle-$UNIQUE_NAME |
11 changes: 11 additions & 0 deletions
11
acceptance/bundle/config-remote-sync/empty_deploy/output.txt
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
|
|
||
| >>> [CLI] bundle deploy | ||
| Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... | ||
| Deploying resources... | ||
| Deployment complete! | ||
|
|
||
| >>> [CLI] bundle destroy --auto-approve | ||
| All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default | ||
|
|
||
| Deleting files... | ||
| Destroy complete! |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| #!/bin/bash | ||
|
|
||
| envsubst < databricks.yml.tmpl > databricks.yml | ||
|
|
||
| cleanup() { | ||
| trace $CLI bundle destroy --auto-approve | ||
| } | ||
| trap cleanup EXIT | ||
|
|
||
| trace $CLI bundle deploy |
10 changes: 10 additions & 0 deletions
10
acceptance/bundle/config-remote-sync/empty_deploy/test.toml
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| Cloud = true | ||
|
|
||
| RecordRequests = false | ||
| Ignore = [".databricks", "databricks.yml"] | ||
|
|
||
| [Env] | ||
| DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = "true" | ||
|
|
||
| [EnvMatrix] | ||
| DATABRICKS_BUNDLE_ENGINE = ["terraform"] | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -54,14 +54,17 @@ func (m *uploadStateForYamlSync) Apply(ctx context.Context, b *bundle.Bundle) di | |
|
|
||
| _, snapshotPath := b.StateFilenameConfigSnapshot(ctx) | ||
|
|
||
| diags := m.convertState(ctx, b, snapshotPath) | ||
| created, diags := m.convertState(ctx, b, snapshotPath) | ||
| if diags.HasError() { | ||
| return diags | ||
| return diag.Warningf("Failed to create config snapshot: %v", diags.Error()) | ||
| } | ||
| if !created { | ||
| return nil | ||
| } | ||
|
|
||
| err := uploadState(ctx, b) | ||
| if err != nil { | ||
| return diags.Extend(diag.Warningf("Failed to upload config snapshot to workspace: %v", err)) | ||
| return diag.Warningf("Failed to upload config snapshot to workspace: %v", err) | ||
| } | ||
|
|
||
| log.Infof(ctx, "Config snapshot created at %s", snapshotPath) | ||
|
|
@@ -92,10 +95,22 @@ func uploadState(ctx context.Context, b *bundle.Bundle) error { | |
| return nil | ||
| } | ||
|
|
||
| func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bundle, snapshotPath string) (diags diag.Diagnostics) { | ||
| func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bundle, snapshotPath string) (bool, diag.Diagnostics) { | ||
|
||
| terraformResources, err := terraform.ParseResourcesState(ctx, b) | ||
| if err != nil { | ||
| return diag.FromErr(err) | ||
| return false, diag.FromErr(fmt.Errorf("failed to parse terraform state for config snapshot: %w", err)) | ||
| } | ||
|
|
||
| // ParseResourcesState returns nil when the terraform state file doesn't exist | ||
| // (e.g. first deploy with no resources). | ||
| if terraformResources == nil { | ||
| return false, nil | ||
| } | ||
|
|
||
| _, localTerraformPath := b.StateFilenameTerraform(ctx) | ||
| data, err := os.ReadFile(localTerraformPath) | ||
| if err != nil { | ||
| return false, diag.FromErr(fmt.Errorf("failed to read terraform state for config snapshot: %w", err)) | ||
| } | ||
|
|
||
| state := make(map[string]dstate.ResourceEntry) | ||
|
|
@@ -111,18 +126,12 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun | |
| } | ||
| } | ||
|
|
||
| _, localTerraformPath := b.StateFilenameTerraform(ctx) | ||
| data, err := os.ReadFile(localTerraformPath) | ||
| if err != nil { | ||
| return diag.FromErr(err) | ||
| } | ||
|
|
||
| var tfState struct { | ||
| Lineage string `json:"lineage"` | ||
| Serial int `json:"serial"` | ||
| } | ||
| if err := json.Unmarshal(data, &tfState); err != nil { | ||
| return diag.FromErr(err) | ||
| return false, diag.FromErr(err) | ||
| } | ||
|
|
||
| migratedDB := dstate.NewDatabase(tfState.Lineage, tfState.Serial+1) | ||
|
|
@@ -141,26 +150,27 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun | |
| interpolatedRoot := b.Config.Value() | ||
| uninterpolatedRoot, err := reverseInterpolate(interpolatedRoot) | ||
| if err != nil { | ||
| return diag.FromErr(fmt.Errorf("failed to reverse interpolation: %w", err)) | ||
| return false, diag.FromErr(fmt.Errorf("failed to reverse interpolation: %w", err)) | ||
| } | ||
|
|
||
| var uninterpolatedConfig config.Root | ||
| err = uninterpolatedConfig.Mutate(func(_ dyn.Value) (dyn.Value, error) { | ||
| return uninterpolatedRoot, nil | ||
| }) | ||
| if err != nil { | ||
| return diag.FromErr(fmt.Errorf("failed to create uninterpolated config: %w", err)) | ||
| return false, diag.FromErr(fmt.Errorf("failed to create uninterpolated config: %w", err)) | ||
| } | ||
|
|
||
| plan, err := deploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(), &uninterpolatedConfig, snapshotPath) | ||
| if err != nil { | ||
| return diag.FromErr(err) | ||
| return false, diag.FromErr(err) | ||
| } | ||
|
|
||
| for _, entry := range plan.Plan { | ||
| entry.Action = deployplan.Update | ||
| } | ||
|
|
||
| var diags diag.Diagnostics | ||
| for key := range plan.Plan { | ||
| etag := etags[key] | ||
| if etag == "" { | ||
|
|
@@ -178,7 +188,7 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun | |
|
|
||
| deploymentBundle.Apply(ctx, b.WorkspaceClient(), plan, direct.MigrateMode(true)) | ||
|
|
||
| return diags | ||
| return true, diags | ||
| } | ||
|
|
||
| // reverseInterpolate reverses the terraform.Interpolate transformation. | ||
|
|
||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I think we should have a test with empty bundle already somewhere (if not, we should have it). We can then use EnvMatrix to run it both with DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC and without.
Perhaps there are more tests where we need to test this variant?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Added matrix to the acceptance/bundle/deploy/empty-bundle/test.toml