Skip to content

(WIP, not for merging) Standalone WASM+native build of "bundle init" #2491

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 51 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
51 commits
Select commit Hold shift + click to select a range
8f9635e
feat: Add main CLI entry point for WASM bundle initialization
denik Mar 13, 2025
f290418
feat: Add indented JSON output for template and render results
denik Mar 13, 2025
adfc063
Hack: standalone bundle init CLI, native & wasm
denik Mar 13, 2025
426a315
refactor: Rename formatJSON to RenderTemplate and parse JSON input
denik Mar 13, 2025
27a354a
fix: Remove unused renderFunc and undefined jsonFunc in WASM build
denik Mar 13, 2025
ab80d7f
feat: Add render button and template output display to index.html
denik Mar 13, 2025
2d849dc
feat: Enhance template rendering UI with JSON parsing and collapsible…
denik Mar 13, 2025
5e1f6b8
feat: Add file summary with total count and size to output
denik Mar 13, 2025
7b09af0
wip
denik Mar 13, 2025
982cbc2
feat: Add rendering time measurement to summary display
denik Mar 13, 2025
b1bfe2f
feat: Add syntax highlighting for Python and YAML files using Prism.js
denik Mar 13, 2025
10ee79d
feat: Enhance syntax highlighting with support for multiple file formats
denik Mar 13, 2025
46a512a
feat: Pre-open YAML files by default in file viewer
denik Mar 13, 2025
9a8bfda
feat: Add template name input field and support dynamic template sele…
denik Mar 13, 2025
6319b09
fix: Use provided template name instead of hardcoded default
denik Mar 13, 2025
cd742d3
refactor: simplify template handling and output in main_cli.go
denik Mar 13, 2025
efea237
feat: implement JSON parameter parsing in CLI with error handling
denik Mar 13, 2025
2e2d13d
wip
denik Mar 13, 2025
1c753a8
refactor: Modify loadHelpers to accept values map and add readValuesF…
denik Mar 13, 2025
61ede29
fix: Correct syntax error in readValuesFunc function
denik Mar 13, 2025
b003b00
allow overriding helpers like user_name
denik Mar 13, 2025
29812a4
pre open errors
denik Mar 13, 2025
c8b64ca
WIP; parameter passing works
denik Mar 13, 2025
696910f
feat: Add common parameters (helpers) input to template rendering int…
denik Mar 13, 2025
56f9265
update defaults
denik Mar 13, 2025
78ad800
add clean
denik Mar 13, 2025
5f06375
get rid of libs/filer and libs/git - they bring sdk dependency; size …
denik Mar 13, 2025
da2f5ea
add ldflags -w -s
denik Mar 13, 2025
4a02be8
disable inlining
denik Mar 13, 2025
71e238d
rewrite makefile; use optimized wasm
denik Mar 13, 2025
a481a15
fix: Add Prism language dependencies to resolve markdown component error
denik Mar 13, 2025
af179b8
fetch gzipped wasm and decompression using DecompressionStream
denik Mar 13, 2025
7e86f5f
clean up
denik Mar 13, 2025
091d06c
feat: Add URL hash-based parameter sharing for template renderer
denik Mar 13, 2025
90e3371
add 'make export'
denik Mar 13, 2025
f081538
Undo changes
denik Mar 13, 2025
dc0a303
refactor: Simplify clean target in Makefile to remove build directory
denik Mar 13, 2025
7a991ed
build: Reorganize Makefile to use build directory for generated files
denik Mar 13, 2025
3827551
cleanup
denik Mar 13, 2025
fd94535
clean up
denik Mar 13, 2025
12f2f59
update .gitignore
denik Mar 14, 2025
6220454
change title
denik Mar 14, 2025
ded49e2
rename experimental/render to experimental/template
denik Mar 14, 2025
62fc47f
rename wasm_bundle_init to wasm
denik Mar 14, 2025
df02864
clean up
denik Mar 14, 2025
3369c8c
refresh global uuid
denik Mar 14, 2025
6456e78
add RenderTemplateZip
denik Mar 14, 2025
e5d7fb7
feat: Add "Download ZIP" button with RenderTemplateZip functionality
denik Mar 14, 2025
6d8dc9a
feat: Update ZIP download filename to use template_project format
denik Mar 14, 2025
f3bf330
update 'make export' to git add & commit
denik Mar 14, 2025
7dfdfa9
create directory entries in zip
denik Mar 14, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 41 additions & 38 deletions libs/template/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,7 @@ import (
"fmt"
"io/fs"

"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/jsonschema"
"github.com/databricks/cli/libs/log"
"golang.org/x/exp/maps"
)

Expand Down Expand Up @@ -165,42 +163,45 @@ func (c *config) skipPrompt(p jsonschema.Property, r *renderer) (bool, error) {
}

func (c *config) promptOnce(property *jsonschema.Schema, name, defaultVal, description string) error {
var userInput string
if property.Enum != nil {
// List options for the user to select from
options, err := property.EnumStringSlice()
if err != nil {
return err
}
userInput, err = cmdio.AskSelect(c.ctx, description, options)
if err != nil {
return err
return nil
/*
var userInput string
if property.Enum != nil {
// List options for the user to select from
options, err := property.EnumStringSlice()
if err != nil {
return err
}
userInput, err = cmdio.AskSelect(c.ctx, description, options)
if err != nil {
return err
}
} else {
var err error
userInput, err = cmdio.Ask(c.ctx, description, defaultVal)
if err != nil {
return err
}
}
} else {

// Convert user input string back to a Go value
var err error
userInput, err = cmdio.Ask(c.ctx, description, defaultVal)
c.values[name], err = property.ParseString(userInput)
if err != nil {
return err
// Show error and retry if validation fails
cmdio.LogString(c.ctx, "Validation failed: "+err.Error())
return retriableError{err: err}
}
}

// Convert user input string back to a Go value
var err error
c.values[name], err = property.ParseString(userInput)
if err != nil {
// Show error and retry if validation fails
cmdio.LogString(c.ctx, "Validation failed: "+err.Error())
return retriableError{err: err}
}

// Validate the partial config which includes the new value
err = c.schema.ValidateInstance(c.values)
if err != nil {
// Show error and retry if validation fails
cmdio.LogString(c.ctx, "Validation failed: "+err.Error())
return retriableError{err: err}
}
return nil
// Validate the partial config which includes the new value
err = c.schema.ValidateInstance(c.values)
if err != nil {
// Show error and retry if validation fails
cmdio.LogString(c.ctx, "Validation failed: "+err.Error())
return retriableError{err: err}
}
return nil
*/
}

// Prompts user for values for properties that do not have a value set yet
Expand Down Expand Up @@ -255,11 +256,13 @@ func (c *config) promptForValues(r *renderer) error {
// Prompt user for any missing config values. Assign default values if
// terminal is not TTY
func (c *config) promptOrAssignDefaultValues(r *renderer) error {
// TODO: replace with IsPromptSupported call (requires fixing TestAccBundleInitErrorOnUnknownFields test)
if cmdio.IsOutTTY(c.ctx) && cmdio.IsInTTY(c.ctx) && !cmdio.IsGitBash(c.ctx) {
return c.promptForValues(r)
}
log.Debugf(c.ctx, "Terminal is not TTY. Assigning default values to template input parameters")
/*
// TODO: replace with IsPromptSupported call (requires fixing TestAccBundleInitErrorOnUnknownFields test)
if cmdio.IsOutTTY(c.ctx) && cmdio.IsInTTY(c.ctx) && !cmdio.IsGitBash(c.ctx) {
return c.promptForValues(r)
}
log.Debugf(c.ctx, "Terminal is not TTY. Assigning default values to template input parameters")
*/
return c.assignDefaultValues(r)
}

Expand Down
14 changes: 6 additions & 8 deletions libs/template/file.go
Original file line number Diff line number Diff line change
@@ -1,12 +1,8 @@
package template

import (
"bytes"
"context"
"io/fs"
"slices"

"github.com/databricks/cli/libs/filer"
)

// Interface representing a file to be materialized from a template into a project
Expand All @@ -18,7 +14,7 @@ type file interface {
RelPath() string

// Write file to disk at the destination path.
Write(ctx context.Context, out filer.Filer) error
// Write(ctx context.Context, out filer.Filer) error

// contents returns the file contents as a byte slice.
// This is used for testing purposes.
Expand All @@ -43,6 +39,7 @@ func (f *copyFile) RelPath() string {
return f.relPath
}

/*
func (f *copyFile) Write(ctx context.Context, out filer.Filer) error {
src, err := f.srcFS.Open(f.srcPath)
if err != nil {
Expand All @@ -51,6 +48,7 @@ func (f *copyFile) Write(ctx context.Context, out filer.Filer) error {
defer src.Close()
return out.Write(ctx, f.relPath, src, filer.CreateParentDirectories, filer.WriteMode(f.perm))
}
*/

func (f *copyFile) contents() ([]byte, error) {
return fs.ReadFile(f.srcFS, f.srcPath)
Expand All @@ -71,9 +69,9 @@ func (f *inMemoryFile) RelPath() string {
return f.relPath
}

func (f *inMemoryFile) Write(ctx context.Context, out filer.Filer) error {
return out.Write(ctx, f.relPath, bytes.NewReader(f.content), filer.CreateParentDirectories, filer.WriteMode(f.perm))
}
//func (f *inMemoryFile) Write(ctx context.Context, out filer.Filer) error {
// return out.Write(ctx, f.relPath, bytes.NewReader(f.content), filer.CreateParentDirectories, filer.WriteMode(f.perm))
//}

func (f *inMemoryFile) contents() ([]byte, error) {
return slices.Clone(f.content), nil
Expand Down
104 changes: 23 additions & 81 deletions libs/template/helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,15 @@ package template

import (
"context"
"errors"
"fmt"
"math/rand"
"net/url"
"os"
"regexp"
"text/template"

"github.com/databricks/cli/libs/command"
"github.com/databricks/cli/libs/iamutil"
"github.com/databricks/databricks-sdk-go/apierr"
"github.com/databricks/databricks-sdk-go/service/iam"
//"github.com/databricks/databricks-sdk-go/apierr"
//"github.com/databricks/databricks-sdk-go/service/iam"

"github.com/google/uuid"
)
Expand All @@ -32,9 +29,9 @@ type pair struct {
}

var (
cachedUser *iam.User
cachedIsServicePrincipal *bool
cachedCatalog *string
// cachedUser *iam.User
// cachedIsServicePrincipal *bool
// cachedCatalog *string
)

// UUID that is stable for the duration of the template execution. This can be used
Expand All @@ -44,8 +41,7 @@ var (
// is run and can be used to attribute DBU revenue to bundle templates.
var bundleUuid = uuid.New().String()

func loadHelpers(ctx context.Context) template.FuncMap {
w := command.WorkspaceClient(ctx)
func loadHelpers(ctx context.Context, values map[string]string) template.FuncMap {
return template.FuncMap{
"fail": func(format string, args ...any) (any, error) {
return nil, ErrFail{fmt.Sprintf(format, args...)}
Expand Down Expand Up @@ -89,83 +85,29 @@ func loadHelpers(ctx context.Context) template.FuncMap {
return result
},
// Get smallest node type (follows Terraform's GetSmallestNodeType)
"smallest_node_type": func() (string, error) {
if w.Config.Host == "" {
return "", errors.New("cannot determine target workspace, please first setup a configuration profile using 'databricks configure'")
}
if w.Config.IsAzure() {
return "Standard_D3_v2", nil
} else if w.Config.IsGcp() {
return "n1-standard-4", nil
}
return "i3.xlarge", nil
},
"smallest_node_type": readValuesFunc(values, "smallest_node_type", "i3.xlarge"),
"path_separator": func() string {
return string(os.PathSeparator)
},
"workspace_host": func() (string, error) {
if w.Config.Host == "" {
return "", errors.New("cannot determine target workspace, please first setup a configuration profile using 'databricks configure'")
}
return w.Config.Host, nil
},
"user_name": func() (string, error) {
if cachedUser == nil {
var err error
cachedUser, err = w.CurrentUser.Me(ctx)
if err != nil {
return "", err
}
}
result := cachedUser.UserName
if result == "" {
result = cachedUser.Id
}
return result, nil
},
"short_name": func() (string, error) {
if cachedUser == nil {
var err error
cachedUser, err = w.CurrentUser.Me(ctx)
if err != nil {
return "", err
}
}
return iamutil.GetShortUserName(cachedUser), nil
},
"workspace_host": readValuesFunc(values, "workspace_host", "<host>"),
"user_name": readValuesFunc(values, "user_name", "<user>"),
"short_name": readValuesFunc(values, "short_name", "<user>"),
// Get the default workspace catalog. If there is no default, or if
// Unity Catalog is not enabled, return an empty string.
"default_catalog": func() (string, error) {
if cachedCatalog == nil {
metastore, err := w.Metastores.Current(ctx)
if err != nil {
var aerr *apierr.APIError
if errors.As(err, &aerr) && (aerr.ErrorCode == "PERMISSION_DENIED" || aerr.ErrorCode == "METASTORE_DOES_NOT_EXIST") {
// Ignore: access denied or workspace doesn't have a metastore assigned
empty_default := ""
cachedCatalog = &empty_default
return "", nil
}
return "", err
}
cachedCatalog = &metastore.DefaultCatalogName
}
return *cachedCatalog, nil
},
"default_catalog": readValuesFunc(values, "default_catalog", ""),

"is_service_principal": func() (bool, error) {
if cachedIsServicePrincipal != nil {
return *cachedIsServicePrincipal, nil
}
if cachedUser == nil {
var err error
cachedUser, err = w.CurrentUser.Me(ctx)
if err != nil {
return false, err
}
}
result := iamutil.IsServicePrincipal(cachedUser)
cachedIsServicePrincipal = &result
return result, nil
return false, nil
},
}
}

func readValuesFunc(values map[string]string, name, defaultValue string) func() string {
return func() string {
x, ok := values[name]
if ok {
return x
}
return defaultValue
}
}
7 changes: 3 additions & 4 deletions libs/template/reader.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"path/filepath"
"strings"

"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/log"
)

Expand Down Expand Up @@ -83,11 +82,11 @@ func (r *gitReader) FS(ctx context.Context) (fs.FS, error) {
r.tmpRepoDir = repoDir

// start the spinner
promptSpinner := cmdio.Spinner(ctx)
promptSpinner <- "Downloading the template\n"
// promptSpinner := cmdio.Spinner(ctx)
// promptSpinner <- "Downloading the template\n"

err = r.cloneFunc(ctx, r.gitUrl, r.ref, repoDir)
close(promptSpinner)
// close(promptSpinner)
if err != nil {
return nil, err
}
Expand Down
Loading
Loading