cmd/openshift-install: Add 'destroy bootstrap' command

Using Terraform to remove all resources created by the bootstrap
modules.  For this to work, all platforms must define a bootstrap
module (and they all currently do).

This command moves the previous destroy-cluster into a new 'destroy
cluster' subcommand, because grouping different destroy flavors into
sub-commands makes the base command easier to understand.  We expect
both destroy flavors to be long-running, because it's hard to write
generic logic for "is the cluster sufficiently live for us to remove
the bootstrap".  We don't want to hang forever if the cluster dies
before coming up, but there's no solid rules for how long to wait
before deciding that it's never going to come up.  When we start
destroying the bootstrap resources automatically in the future, will
pick reasonable timeouts, but will want to still provide callers with
the ability to manually remove the bootstrap resources if we happen to
fall out of that timeout on a cluster that does eventually come up.

I've also created a LoadMetadata helper to share the "retrieve the
metadata from the asset directory" logic between the destroy-cluster
and destroy-bootstrap logic.  The new helper lives in the cluster
asset plackage close to the code that determines that file's location.

I've pushed the Terraform module unpacking and 'terraform init' call
down into a helper used by the Apply and Destroy functions to make
life easier on the callers.

I've also fixed a path.Join -> filepath.Join typo in Apply, which
dates back to ff5a57b (pkg/terraform: Modify some helper functions
for the new binary layout, 2018-09-19, #289).  These aren't network
paths ;).

Author: wking

README.md

@@ -53,5 +53,5 @@ export KUBECONFIG="${DIR}/auth/kubeconfig"
 Destroy the cluster and release associated resources with:
 
 ```sh
-openshift-install destroy-cluster
+openshift-install destroy cluster
 ```

cmd/openshift-install/destroy.go

@@ -6,14 +6,36 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/openshift/installer/pkg/destroy"
+	"github.com/openshift/installer/pkg/destroy/bootstrap"
 	_ "github.com/openshift/installer/pkg/destroy/libvirt"
 )
 
 func newDestroyCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "destroy",
+		Short: "Destroy part of an OpenShift cluster",
+		Long:  "",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return cmd.Help()
+		},
+	}
+	cmd.AddCommand(newDestroyBootstrapCmd())
+	cmd.AddCommand(newDestroyClusterCmd())
+	return cmd
+}
+
+func newLegacyDestroyClusterCmd() *cobra.Command {
 	return &cobra.Command{
 		Use:   "destroy-cluster",
+		Short: "DEPRECATED: Use 'destroy cluster' instead.",
+		RunE:  runDestroyCmd,
+	}
+}
+
+func newDestroyClusterCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   "cluster",
 		Short: "Destroy an OpenShift cluster",
-		Long:  "",
 		RunE:  runDestroyCmd,
 	}
 }
@@ -29,3 +51,13 @@ func runDestroyCmd(cmd *cobra.Command, args []string) error {
 	}
 	return nil
 }
+
+func newDestroyBootstrapCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   "bootstrap",
+		Short: "Destroy the bootstrap resources",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return bootstrap.Destroy(rootOpts.dir)
+		},
+	}
+}

cmd/openshift-install/main.go

@@ -22,6 +22,7 @@ func main() {
 	}
 	subCmds = append(subCmds,
 		newDestroyCmd(),
+		newLegacyDestroyClusterCmd(),
 		newVersionCmd(),
 		newGraphCmd(),
 	)

docs/dev/libvirt-howto.md

@@ -184,7 +184,7 @@ EOF
 ## Build and run the installer
 
 With [libvirt configured](#install-and-enable-libvirt), you can proceed with [the usual quick-start](../../README.md#quick-start).
-Set `TAGS` when building if you need `destroy-cluster` support for libvirt; this is not enabled by default because it requires [cgo][]:
+Set `TAGS` when building if you need `destroy cluster` support for libvirt; this is not enabled by default because it requires [cgo][]:
 
 ```sh
 TAGS=libvirt_destroy hack/build.sh
@@ -205,7 +205,7 @@ export OPENSHIFT_INSTALL_LIBVIRT_URI=qemu+tcp://192.168.122.1/system
 If you compiled with `libvirt_destroy`, you can use:
 
 ```sh
-openshift-install destroy-cluster
+openshift-install destroy cluster
 ```
 
 If you did not compile with `libvirt_destroy`, you can use [`virsh-cleanup.sh`](../../scripts/maintenance/virsh-cleanup.sh), but note it will currently destroy *all* libvirt resources.

pkg/asset/cluster/cluster.go

@@ -10,7 +10,6 @@ import (
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 
-	"github.com/openshift/installer/data"
 	"github.com/openshift/installer/pkg/asset"
 	"github.com/openshift/installer/pkg/asset/installconfig"
 	"github.com/openshift/installer/pkg/asset/kubeconfig"
@@ -19,8 +18,8 @@ import (
 )
 
 const (
-	// MetadataFilename is name of the file where clustermetadata is stored.
-	MetadataFilename = "metadata.json"
+	// metadataFileName is name of the file where clustermetadata is stored.
+	metadataFileName = "metadata.json"
 )
 
 // Cluster uses the terraform executable to launch a cluster
@@ -65,19 +64,14 @@ func (c *Cluster) Generate(parents asset.Parents) (err error) {
 		return errors.Wrap(err, "failed to write terraform.tfvars file")
 	}
 
-	platform := installConfig.Config.Platform.Name()
-	if err := data.Unpack(tmpDir, platform); err != nil {
-		return err
-	}
-
 	metadata := &types.ClusterMetadata{
 		ClusterName: installConfig.Config.ObjectMeta.Name,
 	}
 
 	defer func() {
 		if data, err2 := json.Marshal(metadata); err2 == nil {
 			c.FileList = append(c.FileList, &asset.File{
-				Filename: MetadataFilename,
+				Filename: metadataFileName,
 				Data:     data,
 			})
 		} else {
@@ -114,19 +108,8 @@ func (c *Cluster) Generate(parents asset.Parents) (err error) {
 		return fmt.Errorf("no known platform")
 	}
 
-	if err := data.Unpack(filepath.Join(tmpDir, "config.tf"), "config.tf"); err != nil {
-		return err
-	}
-
 	logrus.Infof("Using Terraform to create cluster...")
-
-	// This runs the terraform in a temp directory, the tfstate file will be returned
-	// to the asset store to persist it on the disk.
-	if err := terraform.Init(tmpDir); err != nil {
-		return errors.Wrap(err, "failed to initialize terraform")
-	}
-
-	stateFile, err := terraform.Apply(tmpDir)
+	stateFile, err := terraform.Apply(tmpDir, installConfig.Config.Platform.Name())
 	if err != nil {
 		err = errors.Wrap(err, "failed to run terraform")
 	}
@@ -167,3 +150,17 @@ func (c *Cluster) Load(f asset.FileFetcher) (found bool, err error) {
 
 	return true, fmt.Errorf("%q already exisits.  There may already be a running cluster", terraform.StateFileName)
 }
+
+// LoadMetadata loads the cluster metadata from an asset directory.
+func LoadMetadata(dir string) (cmetadata *types.ClusterMetadata, err error) {
+	raw, err := ioutil.ReadFile(filepath.Join(dir, metadataFileName))
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to read %s file", metadataFileName)
+	}
+
+	if err = json.Unmarshal(raw, &cmetadata); err != nil {
+		return nil, errors.Wrapf(err, "failed to Unmarshal data from %s file to types.ClusterMetadata", metadataFileName)
+	}
+
+	return cmetadata, err
+}

pkg/asset/cluster/tfvars.go

@@ -12,7 +12,8 @@ import (
 )
 
 const (
-	tfvarsFilename  = "terraform.tfvars"
+	// TfVarsFileName is the filename for Terraform variables.
+	TfVarsFileName  = "terraform.tfvars"
 	tfvarsAssetName = "Terraform Variables"
 )
 
@@ -62,7 +63,7 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 		return errors.Wrap(err, "failed to get Tfvars")
 	}
 	t.File = &asset.File{
-		Filename: tfvarsFilename,
+		Filename: TfVarsFileName,
 		Data:     data,
 	}
 
@@ -79,7 +80,7 @@ func (t *TerraformVariables) Files() []*asset.File {
 
 // Load reads the terraform.tfvars from disk.
 func (t *TerraformVariables) Load(f asset.FileFetcher) (found bool, err error) {
-	file, err := f.FetchByName(tfvarsFilename)
+	file, err := f.FetchByName(TfVarsFileName)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return false, nil

pkg/destroy/bootstrap/bootstrap.go

@@ -0,0 +1,56 @@
+// Package bootstrap uses Terraform to remove bootstrap resources.
+package bootstrap
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/openshift/installer/pkg/asset/cluster"
+	"github.com/openshift/installer/pkg/terraform"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+// Destroy uses Terraform to remove bootstrap resources.
+func Destroy(dir string) (err error) {
+	metadata, err := cluster.LoadMetadata(dir)
+	if err != nil {
+		return err
+	}
+
+	platform := metadata.Platform()
+	if platform == "" {
+		return errors.New("no platform configured in metadata")
+	}
+
+	tempDir, err := ioutil.TempDir("", "openshift-install-")
+	if err != nil {
+		return errors.Wrap(err, "failed to create temporary directory for Terraform execution")
+	}
+	defer os.RemoveAll(tempDir)
+
+	for _, filename := range []string{terraform.StateFileName, cluster.TfVarsFileName} {
+		err = copy(filepath.Join(dir, filename), filepath.Join(tempDir, filename))
+		if err != nil {
+			return errors.Wrapf(err, "failed to copy %s to the temporary directory", filename)
+		}
+	}
+
+	logrus.Infof("Using Terraform to destroy bootstrap resources...")
+	err = terraform.Destroy(tempDir, platform, "-target=module.bootstrap")
+	if err != nil {
+		err = errors.Wrap(err, "failed to run terraform")
+	}
+
+	return os.Rename(filepath.Join(dir, terraform.StateFileName), filepath.Join(tempDir, terraform.StateFileName))
+}
+
+func copy(from string, to string) error {
+	data, err := ioutil.ReadFile(from)
+	if err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(to, data, 0666)
+}

pkg/destroy/destroyer.go

@@ -1,10 +1,6 @@
 package destroy
 
 import (
-	"encoding/json"
-	"io/ioutil"
-	"path/filepath"
-
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 
@@ -26,25 +22,19 @@ var Registry = make(map[string]NewFunc)
 
 // New returns a Destroyer based on `metadata.json` in `rootDir`.
 func New(logger logrus.FieldLogger, rootDir string) (Destroyer, error) {
-	path := filepath.Join(rootDir, cluster.MetadataFilename)
-	raw, err := ioutil.ReadFile(filepath.Join(rootDir, cluster.MetadataFilename))
+	metadata, err := cluster.LoadMetadata(rootDir)
 	if err != nil {
-		return nil, errors.Wrapf(err, "failed to read %s file", cluster.MetadataFilename)
-	}
-
-	var cmetadata *types.ClusterMetadata
-	if err := json.Unmarshal(raw, &cmetadata); err != nil {
-		return nil, errors.Wrapf(err, "failed to Unmarshal data from %s file to types.ClusterMetadata", cluster.MetadataFilename)
+		return nil, err
 	}
 
-	platform := cmetadata.Platform()
+	platform := metadata.Platform()
 	if platform == "" {
-		return nil, errors.Errorf("no platform configured in %q", path)
+		return nil, errors.New("no platform configured in metadata")
 	}
 
 	creator, ok := Registry[platform]
 	if !ok {
 		return nil, errors.Errorf("no destroyers registered for %q", platform)
 	}
-	return creator(logger, cmetadata)
+	return creator(logger, metadata)
 }

pkg/terraform/terraform.go

@@ -2,8 +2,9 @@ package terraform
 
 import (
 	"fmt"
-	"path"
+	"path/filepath"
 
+	"github.com/openshift/installer/data"
 	"github.com/pkg/errors"
 )
 
@@ -26,10 +27,16 @@ func terraformExec(clusterDir string, args ...string) error {
 	return nil
 }
 
-// Apply runs "terraform apply" in the given directory. It returns the absolute
-// path of the tfstate file, rooted in the specified directory, along with any
-// errors from Terraform.
-func Apply(dir string, extraArgs ...string) (string, error) {
+// Apply unpacks the platform-specific Terraform modules into the
+// given directory and then runs 'terraform init' and 'terraform
+// apply'.  It returns the absolute path of the tfstate file, rooted
+// in the specified directory, along with any errors from Terraform.
+func Apply(dir string, platform string, extraArgs ...string) (path string, err error) {
+	err = unpackAndInit(dir, platform)
+	if err != nil {
+		return "", err
+	}
+
 	defaultArgs := []string{
 		"apply",
 		"-auto-approve",
@@ -39,10 +46,57 @@ func Apply(dir string, extraArgs ...string) (string, error) {
 	}
 	args := append(defaultArgs, extraArgs...)
 
-	return path.Join(dir, StateFileName), terraformExec(dir, args...)
+	return filepath.Join(dir, StateFileName), terraformExec(dir, args...)
+}
+
+// Destroy unpacks the platform-specific Terraform modules into the
+// given directory and then runs 'terraform init' and 'terraform
+// destroy'.
+func Destroy(dir string, platform string, extraArgs ...string) (err error) {
+	err = unpackAndInit(dir, platform)
+	if err != nil {
+		return err
+	}
+
+	defaultArgs := []string{
+		"destroy",
+		"-auto-approve",
+		"-no-color",
+		fmt.Sprintf("-state=%s", StateFileName),
+	}
+	args := append(defaultArgs, extraArgs...)
+
+	return terraformExec(dir, args...)
 }
 
-// Init runs "terraform init" in the given directory.
-func Init(dir string) error {
-	return terraformExec(dir, "init", "-input=false", "-no-color")
+// unpack unpacks the platform-specific Terraform modules into the
+// given directory.
+func unpack(dir string, platform string) (err error) {
+	err = data.Unpack(dir, platform)
+	if err != nil {
+		return err
+	}
+
+	err = data.Unpack(filepath.Join(dir, "config.tf"), "config.tf")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// unpackAndInit unpacks the platform-specific Terraform modules into
+// the given directory and then runs 'terraform init'.
+func unpackAndInit(dir string, platform string) (err error) {
+	err = unpack(dir, platform)
+	if err != nil {
+		return errors.Wrap(err, "failed to unpack Terraform modules")
+	}
+
+	err = terraformExec(dir, "init", "-input=false", "-no-color")
+	if err != nil {
+		return errors.Wrap(err, "failed to initialize Terraform")
+	}
+
+	return nil
 }