Add security groups to new instances (openshift#154)

* Fetch the IP with the deployer (needs fixing)
* Apply a startup script to control plane nodes

Signed-off-by: Chuck Ha <[email protected]>

Author: chuckha

Gopkg.lock

@@ -96,7 +96,12 @@ func (a *Actuator) Create(cluster *clusterv1.Cluster, machine *clusterv1.Machine
 
 	status.InstanceID = &i.ID
 	status.InstanceState = aws.String(string(i.State))
-	// TODO: Set the machine.Status.NodeRef after the node has initialized
+
+	if machine.Annotations == nil {
+		machine.Annotations = map[string]string{}
+	}
+	machine.Annotations["cluster-api-provider-aws"] = "true"
+
 	return a.updateStatus(machine, status)
 }
 
@@ -151,12 +156,7 @@ func (a *Actuator) Update(cluster *clusterv1.Cluster, machine *clusterv1.Machine
 		return errors.Wrap(err, "failed to get machine status")
 	}
 
-	err = a.updateStatus(machine, status)
-	if err != nil {
-		return errors.Wrap(err, "failed to update machine status")
-	}
-
-	return nil
+	return a.updateStatus(machine, status)
 }
 
 // Exists test for the existence of a machine and is invoked by the Machine Controller

cloud/aws/actuators/machine/actuator.go

@@ -20,6 +20,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/golang/mock/gomock"
+	"k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
 	clientv1 "sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1"
@@ -50,6 +51,10 @@ func TestCreate(t *testing.T) {
 	// clusterapi calls
 	mg.mi.EXPECT().
 		UpdateStatus(&clusterv1.Machine{
+			ObjectMeta: v1.ObjectMeta{
+				Labels:      map[string]string{"set": "node"},
+				Annotations: map[string]string{"cluster-api-provider-aws": "true"},
+			},
 			Status: clusterv1.MachineStatus{
 				ProviderStatus: &runtime.RawExtension{
 					Raw: []byte(`{"kind":"AWSMachineProviderStatus","apiVersion":"awsproviderconfig/v1alpha1","instanceID":"1234","instanceState":"running"}
@@ -70,11 +75,12 @@ func TestCreate(t *testing.T) {
 	// ec2 calls
 	me.EXPECT().
 		RunInstances(&ec2.RunInstancesInput{
-			ImageId:      aws.String("aws-instance-id"),
-			InstanceType: aws.String(""),
-			MaxCount:     aws.Int64(1),
-			MinCount:     aws.Int64(1),
-			SubnetId:     aws.String("subnet-1"),
+			ImageId:          aws.String("aws-instance-id"),
+			InstanceType:     aws.String(""),
+			MaxCount:         aws.Int64(1),
+			MinCount:         aws.Int64(1),
+			SubnetId:         aws.String("subnet-1"),
+			SecurityGroupIds: aws.StringSlice([]string{"2"}),
 		}).
 		Return(&ec2.Reservation{
 			Instances: []*ec2.Instance{
@@ -109,11 +115,16 @@ func TestCreate(t *testing.T) {
 	if err := actuator.Create(&clusterv1.Cluster{
 		Status: clusterv1.ClusterStatus{
 			ProviderStatus: &runtime.RawExtension{
-				Raw: []byte(`{"kind":"AWSClusterProviderStatus","apiVersion":"awsproviderconfig/v1alpha1","network":{"subnets":[{"id": "subnet-1", "public": false}]}}
+				Raw: []byte(`{"kind":"AWSClusterProviderStatus","apiVersion":"awsproviderconfig/v1alpha1","network":{"subnets":[{"id": "subnet-1", "public": false}],"securityGroups":{"node":{"id":"2"}}}}
 `),
 			},
 		},
 	}, &clusterv1.Machine{
+		ObjectMeta: v1.ObjectMeta{
+			Labels: map[string]string{
+				"set": "node",
+			},
+		},
 		Spec: clusterv1.MachineSpec{
 			ProviderConfig: clusterv1.ProviderConfig{
 				Value: &runtime.RawExtension{

cloud/aws/actuators/machine/actuator_test.go

@@ -17,10 +17,15 @@ limitations under the License.
 package deployer
 
 import (
-	"errors"
+	"fmt"
 
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
 	clustercommon "sigs.k8s.io/cluster-api/pkg/apis/cluster/common"
 	clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
+
+	"github.com/pkg/errors"
+	"sigs.k8s.io/cluster-api-provider-aws/cloud/aws/providerconfig/v1alpha1"
 )
 
 // ProviderName is the name of the cloud provider
@@ -35,7 +40,34 @@ type AWSDeployer struct{}
 
 // GetIP returns the IP of a machine, but this is going away.
 func (*AWSDeployer) GetIP(cluster *clusterv1.Cluster, machine *clusterv1.Machine) (string, error) {
-	return "", errors.New("Not implemented")
+	codec, err := v1alpha1.NewCodec()
+	if err != nil {
+		return "", errors.Wrap(err, "Failed to create codec in deployer")
+	}
+
+	status := &v1alpha1.AWSMachineProviderStatus{}
+	if err := codec.DecodeProviderStatus(machine.Status.ProviderStatus, status); err != nil {
+		return "", errors.Wrap(err, "failed to decode machine provider status in deployer")
+	}
+
+	// will have credentials in environment
+	sess := session.Must(session.NewSession())
+	ec2client := ec2.New(sess)
+
+	dio, err := ec2client.DescribeInstances(&ec2.DescribeInstancesInput{
+		InstanceIds: []*string{status.InstanceID},
+	})
+	if err != nil {
+		return "", errors.Wrap(err, "failed to get instance description in deployer")
+	}
+
+	if len(dio.Reservations) == 0 {
+		return "", errors.New(fmt.Sprintf("no reservartions found for instance id %v", *status.InstanceID))
+	}
+	if len(dio.Reservations[0].Instances) == 0 {
+		return "", errors.New(fmt.Sprintf("instance was not found in a reserver for instance id %v", *status.InstanceID))
+	}
+	return *dio.Reservations[0].Instances[0].PublicIpAddress, nil
 }
 
 // GetKubeConfig returns the kubeconfig after the bootstrap process is complete.

cloud/aws/deployer/deployer.go

@@ -355,4 +355,10 @@ type Instance struct {
 
 	// Indicates whether the instance is optimized for Amazon EBS I/O.
 	EBSOptimized *bool `json:"ebsOptimized"`
+
+	// UserData defines the Base64-encoded user data to make available to the instance.
+	UserData *string
+
+	// SecurityGroupIDs are one or more security group IDs this instance belongs to.
+	SecurityGroupIDs []*string
 }

cloud/aws/providerconfig/v1alpha1/types.go

@@ -14,6 +14,7 @@
 package ec2
 
 import (
+	"encoding/base64"
 	"fmt"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -49,7 +50,8 @@ func (s *Service) InstanceIfExists(instanceID *string) (*v1alpha1.Instance, erro
 func (s *Service) CreateInstance(machine *clusterv1.Machine, config *v1alpha1.AWSMachineProviderConfig, clusterStatus *v1alpha1.AWSClusterProviderStatus) (*v1alpha1.Instance, error) {
 
 	input := &v1alpha1.Instance{
-		Type: config.InstanceType,
+		Type:             config.InstanceType,
+		SecurityGroupIDs: []*string{},
 	}
 
 	// Pick image from the machine configuration, or use a default one.
@@ -70,6 +72,18 @@ func (s *Service) CreateInstance(machine *clusterv1.Machine, config *v1alpha1.AW
 		input.SubnetID = sns[0].ID
 	}
 
+	fmt.Println(machine.ObjectMeta)
+
+	// apply values based on the role of the machine
+	if machine.ObjectMeta.Labels["set"] == "controlplane" {
+		input.UserData = aws.String(initControlPlaneScript())
+		input.SecurityGroupIDs = append(input.SecurityGroupIDs, aws.String(clusterStatus.Network.SecurityGroups[v1alpha1.SecurityGroupControlPlane].ID))
+	}
+
+	if machine.ObjectMeta.Labels["set"] == "node" {
+		input.SecurityGroupIDs = append(input.SecurityGroupIDs, aws.String(clusterStatus.Network.SecurityGroups[v1alpha1.SecurityGroupNode].ID))
+	}
+
 	// Pick SSH key, if any.
 	if config.KeyName != "" {
 		input.KeyName = aws.String(config.KeyName)
@@ -123,13 +137,15 @@ func (s *Service) CreateOrGetMachine(machine *clusterv1.Machine, status *v1alpha
 
 func (s *Service) runInstance(i *v1alpha1.Instance) (*v1alpha1.Instance, error) {
 	input := &ec2.RunInstancesInput{
-		InstanceType: aws.String(i.Type),
-		SubnetId:     aws.String(i.SubnetID),
-		ImageId:      aws.String(i.ImageID),
-		KeyName:      i.KeyName,
-		EbsOptimized: i.EBSOptimized,
-		MaxCount:     aws.Int64(1),
-		MinCount:     aws.Int64(1),
+		InstanceType:     aws.String(i.Type),
+		SubnetId:         aws.String(i.SubnetID),
+		ImageId:          aws.String(i.ImageID),
+		KeyName:          i.KeyName,
+		EbsOptimized:     i.EBSOptimized,
+		MaxCount:         aws.Int64(1),
+		MinCount:         aws.Int64(1),
+		UserData:         i.UserData,
+		SecurityGroupIds: i.SecurityGroupIDs,
 	}
 
 	if i.IAMProfile != nil {
@@ -151,6 +167,10 @@ func (s *Service) runInstance(i *v1alpha1.Instance) (*v1alpha1.Instance, error)
 }
 
 func fromSDKTypeToInstance(v *ec2.Instance) *v1alpha1.Instance {
+	securityGroupIDs := make([]*string, len(v.SecurityGroups))
+	for i, sg := range v.SecurityGroups {
+		securityGroupIDs[i] = sg.GroupId
+	}
 	i := &v1alpha1.Instance{
 		ID:           *v.InstanceId,
 		State:        v1alpha1.InstanceState(*v.State.Name),
@@ -162,6 +182,8 @@ func fromSDKTypeToInstance(v *ec2.Instance) *v1alpha1.Instance {
 		PublicIP:     v.PublicIpAddress,
 		ENASupport:   v.EnaSupport,
 		EBSOptimized: v.EbsOptimized,
+		// UserData is not defined on an instance
+		SecurityGroupIDs: securityGroupIDs,
 	}
 
 	if v.IamInstanceProfile != nil && v.IamInstanceProfile.Arn != nil {
@@ -172,3 +194,24 @@ func fromSDKTypeToInstance(v *ec2.Instance) *v1alpha1.Instance {
 
 	return i
 }
+
+// initControlPlaneScrip returns the b64 encoded script to run on start up.
+func initControlPlaneScript() string {
+	// The script must start with #!. If it goes on the next line Dedent will start the script with a \n.
+	script := []byte(`#!/usr/bin/env bash
+
+cat >/tmp/kubeadm.yaml <<EOF
+apiVersion: kubeadm.k8s.io/v1alpha3
+kind: InitConfiguration
+nodeRegistration:
+  criSocket: /var/run/containerd/containerd.sock
+EOF
+
+kubeadm init --config /tmp/kubeadm.yaml
+
+# Installation from https://docs.projectcalico.org/v3.2/getting-started/kubernetes/installation/calico
+kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f https://docs.projectcalico.org/v3.2/getting-started/kubernetes/installation/hosted/rbac-kdd.yaml
+kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f https://docs.projectcalico.org/v3.2/getting-started/kubernetes/installation/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml
+`)
+	return base64.StdEncoding.EncodeToString(script)
+}

cloud/aws/services/ec2/instances.go

@@ -20,7 +20,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/golang/mock/gomock"
 	"github.com/pkg/errors"
-	"k8s.io/apimachinery/pkg/runtime"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
 
 	"sigs.k8s.io/cluster-api-provider-aws/cloud/aws/providerconfig/v1alpha1"
@@ -184,48 +184,57 @@ func TestTerminateInstance(t *testing.T) {
 }
 
 func TestCreateInstance(t *testing.T) {
-	mockCtrl := gomock.NewController(t)
-	defer mockCtrl.Finish()
-
 	testcases := []struct {
-		name    string
-		machine clusterv1.Machine
-		expect  func(m *mock_ec2iface.MockEC2API)
-		check   func(instance *v1alpha1.Instance, err error)
+		name          string
+		machine       clusterv1.Machine
+		machineConfig *v1alpha1.AWSMachineProviderConfig
+		clusterStatus *v1alpha1.AWSClusterProviderStatus
+		expect        func(m *mock_ec2iface.MockEC2API)
+		check         func(instance *v1alpha1.Instance, err error)
 	}{
 		{
 			name: "simple",
 			machine: clusterv1.Machine{
-				Spec: clusterv1.MachineSpec{
-					ProviderConfig: clusterv1.ProviderConfig{
-						Value: &runtime.RawExtension{
-							Raw: []byte(`apiVersion: "cluster.k8s.io/v1alpha1"
-kind: Machine
-metadata:
-  generateName: aws-controlplane-
-  labels:
-    set: controlplane
-spec:
-  versions:
-    kubelet: v1.11.2
-    controlPlane: v1.11.2
-  providerConfig:
-    value:
-      apiVersion: awsproviderconfig/v1alpha1
-      kind: AWSMachineProviderConfig
-      instanceType: m5.large`),
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{"set": "node"},
+				},
+			},
+			machineConfig: &v1alpha1.AWSMachineProviderConfig{
+				AMI: v1alpha1.AWSResourceReference{
+					ID: aws.String("abc"),
+				},
+				InstanceType: "m5.large",
+			},
+			clusterStatus: &v1alpha1.AWSClusterProviderStatus{
+				Network: v1alpha1.Network{
+					Subnets: v1alpha1.Subnets{
+						&v1alpha1.Subnet{
+							ID:       "subnet-1",
+							IsPublic: false,
+						},
+						&v1alpha1.Subnet{
+							IsPublic: false,
+						},
+					},
+					SecurityGroups: map[v1alpha1.SecurityGroupRole]*v1alpha1.SecurityGroup{
+						v1alpha1.SecurityGroupControlPlane: &v1alpha1.SecurityGroup{
+							ID: "1",
+						},
+						v1alpha1.SecurityGroupNode: &v1alpha1.SecurityGroup{
+							ID: "2",
 						},
 					},
 				},
 			},
 			expect: func(m *mock_ec2iface.MockEC2API) {
 				m.EXPECT().
 					RunInstances(&ec2.RunInstancesInput{
-						ImageId:      aws.String("abc"),
-						InstanceType: aws.String("m5.large"),
-						MaxCount:     aws.Int64(1),
-						MinCount:     aws.Int64(1),
-						SubnetId:     aws.String("subnet-1"),
+						ImageId:          aws.String("abc"),
+						InstanceType:     aws.String("m5.large"),
+						MaxCount:         aws.Int64(1),
+						MinCount:         aws.Int64(1),
+						SubnetId:         aws.String("subnet-1"),
+						SecurityGroupIds: aws.StringSlice([]string{"2"}),
 					}).
 					Return(&ec2.Reservation{
 						Instances: []*ec2.Instance{
@@ -251,24 +260,12 @@ spec:
 
 	for _, tc := range testcases {
 		t.Run(tc.name, func(t *testing.T) {
+			mockCtrl := gomock.NewController(t)
+			defer mockCtrl.Finish()
 			ec2Mock := mock_ec2iface.NewMockEC2API(mockCtrl)
 			tc.expect(ec2Mock)
 			s := ec2svc.NewService(ec2Mock)
-			instance, err := s.CreateInstance(&tc.machine, &v1alpha1.AWSMachineProviderConfig{
-				AMI: v1alpha1.AWSResourceReference{
-					ID: aws.String("abc"),
-				},
-				InstanceType: "m5.large",
-			}, &v1alpha1.AWSClusterProviderStatus{
-				Network: v1alpha1.Network{
-					Subnets: v1alpha1.Subnets{
-						&v1alpha1.Subnet{
-							ID:       "subnet-1",
-							IsPublic: false,
-						},
-					},
-				},
-			})
+			instance, err := s.CreateInstance(&tc.machine, tc.machineConfig, tc.clusterStatus)
 			tc.check(instance, err)
 		})
 	}