Add --exclude-from and --include-from flags to sync command

NEXT_CHANGELOG.md

@@ -7,6 +7,7 @@
 ### Dependency updates
 
 ### CLI
+* Added `exclude-from` and `include-from` flags support to sync command ([#2660](https://github.com/databricks/cli/pull/2660))
 
 ### Bundles
 

acceptance/cmd/sync-from-file/gitignore.test-fixture

@@ -0,0 +1,4 @@
+ignored-folder/
+script
+output.txt
+repls.json

acceptance/cmd/sync-from-file/ignore.test-fixture

@@ -0,0 +1,5 @@
+project-folder/blob/*
+project-folder/static/**/*.txt
+project-folder/*.sql
+project-folder/app2.py
+ignored-folder2/

acceptance/cmd/sync-from-file/output.txt

@@ -0,0 +1,22 @@
+
+>>> [CLI] sync . /Users/[USERNAME] --exclude-from ignore.test-fixture
+Initial Sync Complete
+Uploaded .gitignore
+Uploaded ignore.test-fixture
+Uploaded project-folder
+Uploaded project-folder/app.py
+Uploaded project-folder/app.yaml
+
+>>> [CLI] sync . /Users/[USERNAME] --include-from ignore.test-fixture
+Initial Sync Complete
+Uploaded ignored-folder2
+Uploaded ignored-folder2/app.py
+Uploaded project-folder/app2.py
+Uploaded project-folder/blob
+Uploaded project-folder/blob/1
+Uploaded project-folder/blob/2
+Uploaded project-folder/query.sql
+Uploaded project-folder/static/folder1
+Uploaded project-folder/static/folder1/1.txt
+Uploaded project-folder/static/folder2
+Uploaded project-folder/static/folder2/2.txt

acceptance/cmd/sync-from-file/script

@@ -0,0 +1,14 @@
+mkdir "project-folder" "project-folder/blob" "project-folder/static" "project-folder/static/folder1" "project-folder/static/folder2" "ignored-folder" "ignored-folder/folder1" "ignored-folder2" ".git"
+touch "project-folder/app.yaml" "project-folder/app.py" "project-folder/app2.py" "project-folder/query.sql" "project-folder/blob/1" "project-folder/blob/2" "ignored-folder2/app.py"
+touch "project-folder/static/folder1/1.txt" "project-folder/static/folder2/2.txt"
+touch "ignored-folder/script.py" "ignored-folder/folder1/script.py" "ignored-folder/folder1/script.yaml" "ignored-folder/folder1/big-blob"
+mv gitignore.test-fixture .gitignore
+
+cleanup() {
+  rm -rf project-folder ignored-folder ignored-folder2 .git
+}
+trap cleanup EXIT
+
+# Note: output line starting with "Action: " lists files in non-deterministic order so we filter it out
+trace $CLI sync . /Users/$CURRENT_USER_NAME --exclude-from 'ignore.test-fixture' | grep -v "^Action: " | sort
+trace $CLI sync . /Users/$CURRENT_USER_NAME --include-from 'ignore.test-fixture' | grep -v "^Action: " | sort

cmd/sync/sync.go

@@ -1,12 +1,16 @@
 package sync
 
 import (
+	"bufio"
+	"bytes"
 	"context"
 	"errors"
 	"flag"
 	"fmt"
 	"io"
+	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/databricks/cli/bundle"
@@ -23,13 +27,39 @@ import (
 
 type syncFlags struct {
 	// project files polling interval
-	interval time.Duration
-	full     bool
-	watch    bool
-	output   flags.Output
-	exclude  []string
-	include  []string
-	dryRun   bool
+	interval    time.Duration
+	full        bool
+	watch       bool
+	output      flags.Output
+	exclude     []string
+	include     []string
+	dryRun      bool
+	excludeFrom string
+	includeFrom string
+}
+
+func readPatternsFile(filePath string) ([]string, error) {
+	if filePath == "" {
+		return nil, nil
+	}
+
+	data, err := os.ReadFile(filePath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read exclude-from file: %w", err)
+	}
+
+	var patterns []string
+	scanner := bufio.NewScanner(bytes.NewReader(data))
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		patterns = append(patterns, line)
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("error reading exclude-from file: %w", err)
+	}
+
+	return patterns, nil
 }
 
 func (f *syncFlags) syncOptionsFromBundle(cmd *cobra.Command, args []string, b *bundle.Bundle) (*sync.SyncOptions, error) {
@@ -42,11 +72,23 @@ func (f *syncFlags) syncOptionsFromBundle(cmd *cobra.Command, args []string, b *
 		return nil, fmt.Errorf("cannot get sync options: %w", err)
 	}
 
+	excludePatterns, err := readPatternsFile(f.excludeFrom)
+	if err != nil {
+		return nil, err
+	}
+
+	includePatterns, err := readPatternsFile(f.includeFrom)
+	if err != nil {
+		return nil, err
+	}
+
 	opts.Full = f.full
 	opts.PollInterval = f.interval
 	opts.WorktreeRoot = b.WorktreeRoot
 	opts.Exclude = append(opts.Exclude, f.exclude...)
+	opts.Exclude = append(opts.Exclude, excludePatterns...)
 	opts.Include = append(opts.Include, f.include...)
+	opts.Include = append(opts.Include, includePatterns...)
 	opts.DryRun = f.dryRun
 	return opts, nil
 }
@@ -78,6 +120,16 @@ func (f *syncFlags) syncOptionsFromArgs(cmd *cobra.Command, args []string) (*syn
 		log.Warnf(ctx, "Running in dry-run mode. No actual changes will be made.")
 	}
 
+	excludePatterns, err := readPatternsFile(f.excludeFrom)
+	if err != nil {
+		return nil, err
+	}
+
+	includePatterns, err := readPatternsFile(f.includeFrom)
+	if err != nil {
+		return nil, err
+	}
+
 	localRoot := vfs.MustNew(args[0])
 	info, err := git.FetchRepositoryInfo(ctx, localRoot.Native(), client)
 	if err != nil {
@@ -96,8 +148,8 @@ func (f *syncFlags) syncOptionsFromArgs(cmd *cobra.Command, args []string) (*syn
 		WorktreeRoot: worktreeRoot,
 		LocalRoot:    localRoot,
 		Paths:        []string{"."},
-		Include:      f.include,
-		Exclude:      f.exclude,
+		Include:      append(f.include, includePatterns...),
+		Exclude:      append(f.exclude, excludePatterns...),
 
 		RemotePath:   args[1],
 		Full:         f.full,
@@ -133,6 +185,8 @@ func New() *cobra.Command {
 	cmd.Flags().Var(&f.output, "output", "type of output format")
 	cmd.Flags().StringSliceVar(&f.exclude, "exclude", nil, "patterns to exclude from sync (can be specified multiple times)")
 	cmd.Flags().StringSliceVar(&f.include, "include", nil, "patterns to include in sync (can be specified multiple times)")
+	cmd.Flags().StringVar(&f.excludeFrom, "exclude-from", "", "file containing patterns to exclude from sync (one pattern per line)")
+	cmd.Flags().StringVar(&f.includeFrom, "include-from", "", "file containing patterns to include to sync (one pattern per line)")
 	cmd.Flags().BoolVar(&f.dryRun, "dry-run", false, "simulate sync execution without making actual changes")
 
 	// Wrapper for [root.MustWorkspaceClient] that disables loading authentication configuration from a bundle.

cmd/sync/sync_test.go

@@ -3,7 +3,9 @@ package sync
 import (
 	"context"
 	"flag"
+	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/databricks/cli/bundle"
@@ -64,3 +66,34 @@ func TestSyncOptionsFromArgs(t *testing.T) {
 	assert.Equal(t, local, opts.LocalRoot.Native())
 	assert.Equal(t, remote, opts.RemotePath)
 }
+
+func TestExcludeFromFlag(t *testing.T) {
+	// Create a temporary directory
+	tempDir := t.TempDir()
+	local := filepath.Join(tempDir, "local")
+	require.NoError(t, os.MkdirAll(local, 0o755))
+	remote := "/remote"
+
+	// Create a temporary exclude-from file
+	excludeFromPath := filepath.Join(tempDir, "exclude-patterns.txt")
+	excludePatterns := []string{
+		"*.log",
+		"build/",
+		"temp/*.tmp",
+	}
+	require.NoError(t, os.WriteFile(excludeFromPath, []byte(strings.Join(excludePatterns, "\n")), 0o644))
+
+	// Set up the flags
+	f := syncFlags{excludeFrom: excludeFromPath}
+	cmd := New()
+	cmd.SetContext(cmdctx.SetWorkspaceClient(context.Background(), nil))
+
+	// Test with both exclude flag and exclude-from flag
+	f.exclude = []string{"node_modules/"}
+	opts, err := f.syncOptionsFromArgs(cmd, []string{local, remote})
+	require.NoError(t, err)
+
+	// Should include both exclude flag and exclude-from patterns
+	expected := []string{"node_modules/", "*.log", "build/", "temp/*.tmp"}
+	assert.ElementsMatch(t, expected, opts.Exclude)
+}