Merge branch 'main' into lio-compose-file

Author: lionello

.github/workflows/install.yml

@@ -5,7 +5,7 @@ on:
     types: [published]
   push:
     branches:
-      - main
+      - "**"
     paths:
       - '.github/workflows/install.yml'
       - 'src/bin/install'
@@ -15,8 +15,19 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Install defang
+      - name: Install defang (latest)
         run: . <(curl -Ls https://s.defang.io/install)
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # avoid rate limiting
+
+      - name: Sanity check
+        run: defang --version
+
+      - name: Install defang (specific version)
+        run: . <(curl -Ls https://s.defang.io/install)
+        env:
+          DEFANG_INSTALL_VERSION: v0.5.36
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # alt name
 
       - name: Sanity check
         run: defang --version

pkgs/defang/default.nix

@@ -10,22 +10,22 @@ system ? builtins.currentSystem
 }:
 let
   shaMap = {
-    x86_64-linux = "0crcm1sr1zpi1cz6mbyhj6iivsz9akpln1zrk0flcy9dnkn4av9a";
-    aarch64-linux = "1nmmf7nw7s2d9q9y780nzgd3i0pz8dvyryjxccrfdp01sfzlks61";
-    x86_64-darwin = "1sncfvh315sp0n6chpvzdx5hp7hsrkx6m03aix8hfkb1hxjlg0hc";
-    aarch64-darwin = "1sncfvh315sp0n6chpvzdx5hp7hsrkx6m03aix8hfkb1hxjlg0hc";
+    x86_64-linux = "1qqbbmww2cd9pfignak4hyc1syvchg8lyqyjx5h6ir4zf6fsivs7";
+    aarch64-linux = "16yjplqhcd8qasyr85v6siw9nhriv4h46k19nyaqjw6v98lswnkv";
+    x86_64-darwin = "1z7663383q0cgv2yghk615szy4zjiby2vxr2dg79nzbzb024f55x";
+    aarch64-darwin = "1z7663383q0cgv2yghk615szy4zjiby2vxr2dg79nzbzb024f55x";
   };
 
   urlMap = {
-    x86_64-linux = "https://github.com/DefangLabs/defang/releases/download/v0.5.38/defang_0.5.38_linux_amd64.tar.gz";
-    aarch64-linux = "https://github.com/DefangLabs/defang/releases/download/v0.5.38/defang_0.5.38_linux_arm64.tar.gz";
-    x86_64-darwin = "https://github.com/DefangLabs/defang/releases/download/v0.5.38/defang_0.5.38_macOS.zip";
-    aarch64-darwin = "https://github.com/DefangLabs/defang/releases/download/v0.5.38/defang_0.5.38_macOS.zip";
+    x86_64-linux = "https://github.com/DefangLabs/defang/releases/download/v0.5.41/defang_0.5.41_linux_amd64.tar.gz";
+    aarch64-linux = "https://github.com/DefangLabs/defang/releases/download/v0.5.41/defang_0.5.41_linux_arm64.tar.gz";
+    x86_64-darwin = "https://github.com/DefangLabs/defang/releases/download/v0.5.41/defang_0.5.41_macOS.zip";
+    aarch64-darwin = "https://github.com/DefangLabs/defang/releases/download/v0.5.41/defang_0.5.41_macOS.zip";
   };
 in
 stdenvNoCC.mkDerivation {
   pname = "defang";
-  version = "0.5.38";
+  version = "0.5.41";
   src = fetchurl {
     url = urlMap.${system};
     sha256 = shaMap.${system};

pkgs/npm/package-lock.json

@@ -28,12 +28,24 @@ fi
 if [[ -z "$DEFANG_INSTALL_VERSION" ]]; then
     RELEASE_PATH="latest"
 else
-    RELEASE_PATH="tags/$DEFANG_INSTALL_VERSION"
+    RELEASE_PATH="tags/v${DEFANG_INSTALL_VERSION#v}"
 fi
 
-# Echo fetching the release path either latest or strip the tags/ from the version
-echo "Fetching the ${DEFANG_INSTALL_VERSION:-latest} release of defang..."
-RELEASE_JSON=$(curl -s -f -L https://api.github.com/repos/DefangLabs/defang/releases/$RELEASE_PATH)
+# Anonymous API request to GitHub are rate limited to 60 requests per hour.
+# Check whether the user has set a GitHub token to increase the rate limit.
+AUTH_HEADER=""
+if [[ -n "$GITHUB_TOKEN" ]]; then
+    AUTH_HEADER="Authorization: Bearer $GITHUB_TOKEN"
+elif [[ -n "$GH_TOKEN" ]]; then
+    AUTH_HEADER="Authorization: Bearer $GH_TOKEN"
+fi
+
+# Echo fetching the release path either latest or the version
+echo "Fetching the ${RELEASE_PATH#tags/} release of defang..."
+# Download the release information from GitHub, using the token if available, but falling back to anonymous access
+RELEASE_JSON=$([[ -n "$AUTH_HEADER" ]] &&
+    curl -sfL -H "$AUTH_HEADER" https://api.github.com/repos/DefangLabs/defang/releases/$RELEASE_PATH ||
+    curl -sfL https://api.github.com/repos/DefangLabs/defang/releases/$RELEASE_PATH)
 
 # Check for curl failure
 if [ -z "$RELEASE_JSON" ]; then
@@ -80,7 +92,7 @@ elif [ "$OS" = "Linux" ]; then
 fi
 
 # Download the file
-if ! curl -s -f -L "$DOWNLOAD_URL" -o "$FILENAME"; then
+if ! curl -sfL "$DOWNLOAD_URL" -o "$FILENAME"; then
     echo "Download failed. Please check your internet connection and try again."
     return 4
 fi
@@ -239,5 +251,5 @@ rm "$FILENAME"
 echo "Installation completed. You can now use defang by typing '$BINARY_NAME' in the terminal."
 
 # Unset the variables and functions to avoid polluting the user's environment
-unset EXTRACT_DIR DOWNLOAD_URL RELEASE_JSON RELEASE_PATH ARCH_SUFFIX ARCH OS FILENAME INSTALL_DIR BINARY_NAME REPLY EXPORT_PATH CURRENT_SHELL FOUND_PROFILE_FILE
+unset EXTRACT_DIR DOWNLOAD_URL RELEASE_JSON RELEASE_PATH ARCH_SUFFIX ARCH OS FILENAME INSTALL_DIR BINARY_NAME REPLY EXPORT_PATH CURRENT_SHELL FOUND_PROFILE_FILE AUTH_HEADER
 unset -f _prompt_and_append_to_file _generate_completion_script _install_completion_script

src/bin/install

@@ -64,6 +64,10 @@ func Execute(ctx context.Context) error {
 			term.Error("Error:", prettyError(err))
 		}
 
+		if err == cli.ErrDryRun {
+			return nil
+		}
+
 		var derr *cli.ComposeError
 		if errors.As(err, &derr) {
 			compose := "compose"
@@ -151,6 +155,9 @@ func SetupCommands(version string) {
 	tosCmd.Flags().Bool("agree-tos", false, "agree to the Defang terms of service")
 	RootCmd.AddCommand(tosCmd)
 
+	// Upgrade command
+	RootCmd.AddCommand(upgradeCmd)
+
 	// Token command
 	tokenCmd.Flags().Duration("expires", 24*time.Hour, "validity duration of the token")
 	tokenCmd.Flags().String("scope", "", fmt.Sprintf("scope of the token; one of %v (required)", scope.All())) // TODO: make it an Option
@@ -177,7 +184,7 @@ func SetupCommands(version string) {
 	// TODO: when we add multi-project support to the playground, differentiate
 	// between ls and ps
 	lsCommand.Aliases = []string{"getServices", "ps", "ls", "list"}
-	RootCmd.AddCommand(makeComposeLsCmd())
+	RootCmd.AddCommand(lsCommand)
 
 	// Get Status Command
 	RootCmd.AddCommand(getVersionCmd)
@@ -314,8 +321,8 @@ var RootCmd = &cobra.Command{
 			version := cmd.Root().Version // HACK to avoid circular dependency with RootCmd
 			term.Debug("Fabric:", v.Fabric, "CLI:", version, "CLI-Min:", v.CliMin)
 			if hasTty && isNewer(version, v.CliMin) {
-				term.Warn("Your CLI version is outdated. Please update to the latest version.")
-				os.Setenv("DEFANG_HIDE_UPDATE", "1") // hide the update hint at the end
+				term.Warn("Your CLI version is outdated. Please upgrade to the latest version by running:\n\ndefang upgrade")
+				os.Setenv("DEFANG_HIDE_UPDATE", "1") // hide the upgrade hint at the end
 			}
 		}
 
@@ -956,6 +963,16 @@ var tosCmd = &cobra.Command{
 	},
 }
 
+var upgradeCmd = &cobra.Command{
+	Use:     "upgrade",
+	Args:    cobra.NoArgs,
+	Aliases: []string{"update"},
+	Short:   "Upgrade the Defang CLI to the latest version",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return cli.Upgrade(cmd.Context())
+	},
+}
+
 func configureLoader(cmd *cobra.Command) compose.Loader {
 	f := cmd.Flags()
 	o := compose.LoaderOptions{}

src/cmd/cli/command/commands.go

@@ -18,7 +18,19 @@ import (
 	"github.com/spf13/cobra"
 )
 
+func getManagedResources(serviceInfos []*defangv1.ServiceInfo) []string {
+	var managedResources []string
+	for _, service := range serviceInfos {
+		if service.Type == defangv1.ServiceType_MANAGED {
+			managedResources = append(managedResources, service.Service.Name)
+		}
+	}
+	return managedResources
+}
+
 func makeComposeUpCmd() *cobra.Command {
+	mode := Mode(defangv1.DeploymentMode_DEVELOPMENT)
+
 	composeUpCmd := &cobra.Command{
 		Use:         "up",
 		Annotations: authNeededAnnotation,
@@ -29,7 +41,7 @@ func makeComposeUpCmd() *cobra.Command {
 			var detach, _ = cmd.Flags().GetBool("detach")
 
 			since := time.Now()
-			deploy, project, err := cli.ComposeUp(cmd.Context(), client, force)
+			deploy, project, err := cli.ComposeUp(cmd.Context(), client, force, mode.Value())
 			if err != nil {
 				if !errors.Is(err, types.ErrComposeFileNotFound) {
 					return err
@@ -44,6 +56,11 @@ func makeComposeUpCmd() *cobra.Command {
 
 			printPlaygroundPortalServiceURLs(deploy.Services)
 
+			managedResources := getManagedResources(deploy.Services)
+			if len(managedResources) > 0 {
+				term.Warnf("Defang cannot monitor status of the following managed service(s): %v.\n   To check if the managed service is up, check the status of the service which depends on it.", managedResources)
+			}
+
 			if detach {
 				term.Info("Detached.")
 				return nil
@@ -106,6 +123,8 @@ func makeComposeUpCmd() *cobra.Command {
 					if errors.As(context.Cause(tailCtx), &errDeploymentFailed) {
 						term.Warn(errDeploymentFailed)
 						failedServices = []string{errDeploymentFailed.Service}
+					} else if len(managedResources) > 0 {
+						term.Warn("Managed services have been deployed but not all services may be available yet.")
 					} else {
 						term.Warn("Deployment is not finished. Service(s) might not be running.")
 						// TODO: some services might be OK and we should only debug the ones that are not
@@ -151,6 +170,7 @@ func makeComposeUpCmd() *cobra.Command {
 	composeUpCmd.Flags().BoolP("detach", "d", false, "run in detached mode")
 	composeUpCmd.Flags().Bool("force", false, "force a build of the image even if nothing has changed")
 	composeUpCmd.Flags().Bool("tail", false, "tail the service logs after updating") // obsolete, but keep for backwards compatibility
+	composeUpCmd.Flags().VarP(&mode, "mode", "m", "deployment mode, possible values: "+strings.Join(allModes(), ", "))
 	_ = composeUpCmd.Flags().MarkHidden("tail")
 	return composeUpCmd
 }
@@ -165,7 +185,7 @@ func makeComposeStartCmd() *cobra.Command {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			var force, _ = cmd.Flags().GetBool("force")
 
-			deploy, _, err := cli.ComposeUp(cmd.Context(), client, force)
+			deploy, _, err := cli.ComposeUp(cmd.Context(), client, force, defangv1.DeploymentMode_UNSPECIFIED_MODE)
 			if err != nil {
 				return err
 			}
@@ -283,7 +303,7 @@ func makeComposeConfigCmd() *cobra.Command {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			cli.DoDryRun = true // config is like start in a dry run
 			// force=false to calculate the digest
-			if _, _, err := cli.ComposeUp(cmd.Context(), client, false); !errors.Is(err, cli.ErrDryRun) {
+			if _, _, err := cli.ComposeUp(cmd.Context(), client, false, defangv1.DeploymentMode_UNSPECIFIED_MODE); !errors.Is(err, cli.ErrDryRun) {
 				return err
 			}
 			return nil

src/cmd/cli/command/compose.go

@@ -0,0 +1,56 @@
+package command
+
+import (
+	"slices"
+	"testing"
+
+	defangv1 "github.com/DefangLabs/defang/src/protos/io/defang/v1"
+)
+
+func TestGetManagedResources(t *testing.T) {
+	tests := []struct {
+		name     string
+		project  []*defangv1.ServiceInfo
+		expected []string
+	}{
+		{
+			name:     "no services",
+			project:  []*defangv1.ServiceInfo{},
+			expected: []string{},
+		},
+		{
+			name:     "one service all managed",
+			project:  []*defangv1.ServiceInfo{{Service: &defangv1.ServiceID{Name: "redis"}, Type: defangv1.ServiceType_MANAGED}},
+			expected: []string{"redis"},
+		},
+		{
+			name:     "one service unmanaged",
+			project:  []*defangv1.ServiceInfo{{Service: &defangv1.ServiceID{Name: "web"}, Type: defangv1.ServiceType_SERVICE}},
+			expected: []string{},
+		},
+		{
+			name:     "one service unmanaged, one service managed",
+			project:  []*defangv1.ServiceInfo{{Service: &defangv1.ServiceID{Name: "web"}, Type: defangv1.ServiceType_SERVICE}, {Service: &defangv1.ServiceID{Name: "redis"}, Type: defangv1.ServiceType_MANAGED}},
+			expected: []string{"redis"},
+		},
+		{
+			name:     "two unmanaged",
+			project:  []*defangv1.ServiceInfo{{Service: &defangv1.ServiceID{Name: "web"}, Type: defangv1.ServiceType_SERVICE}, {Service: &defangv1.ServiceID{Name: "app"}, Type: defangv1.ServiceType_WORKER}},
+			expected: []string{},
+		},
+		{
+			name:     "two managed",
+			project:  []*defangv1.ServiceInfo{{Service: &defangv1.ServiceID{Name: "redis"}, Type: defangv1.ServiceType_MANAGED}, {Service: &defangv1.ServiceID{Name: "postgres"}, Type: defangv1.ServiceType_MANAGED}},
+			expected: []string{"redis", "postgres"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			actual := getManagedResources(tt.project)
+			if !slices.Equal(actual, tt.expected) {
+				t.Errorf("expected %v, got %v", tt.expected, actual)
+			}
+		})
+	}
+}

src/cmd/cli/command/compose_test.go

@@ -0,0 +1,40 @@
+package command
+
+import (
+	"fmt"
+	"strings"
+
+	defangv1 "github.com/DefangLabs/defang/src/protos/io/defang/v1"
+)
+
+type Mode defangv1.DeploymentMode
+
+func (b Mode) String() string {
+	return strings.ToLower(defangv1.DeploymentMode_name[int32(b)])
+}
+func (b *Mode) Set(s string) error {
+	mode, ok := defangv1.DeploymentMode_value[strings.ToUpper(s)]
+	if !ok {
+		return fmt.Errorf("invalid mode: %s, valid values are: %v", s, strings.Join(allModes(), ", "))
+	}
+	*b = Mode(mode)
+	return nil
+}
+func (b Mode) Type() string {
+	return "mode"
+}
+
+func (b Mode) Value() defangv1.DeploymentMode {
+	return defangv1.DeploymentMode(b)
+}
+
+func allModes() []string {
+	modes := make([]string, 0, len(defangv1.DeploymentMode_name)-1)
+	for i, mode := range defangv1.DeploymentMode_name {
+		if i == 0 {
+			continue
+		}
+		modes = append(modes, strings.ToLower(mode))
+	}
+	return modes
+}