Merge pull request #1168 from liztio/lb-delete

🐛 Delete cloud provider provisioned load balancers and security groups

pkg/cloud/filter/ec2.go

@@ -80,6 +80,14 @@ func (ec2Filters) ProviderRole(role string) *ec2.Filter {
 	}
 }
 
+// ProviderOwned returns a filter using the cloud provider tag where the resource is owned.
+func (ec2Filters) ProviderOwned(clusterName string) *ec2.Filter {
+	return &ec2.Filter{
+		Name:   aws.String(fmt.Sprintf("tag:%s", infrav1.ClusterAWSCloudProviderTagKey(clusterName))),
+		Values: aws.StringSlice([]string{string(infrav1.ResourceLifecycleOwned)}),
+	}
+}
+
 // VPC returns a filter based on the id of the VPC.
 func (ec2Filters) VPC(vpcID string) *ec2.Filter {
 	return &ec2.Filter{

pkg/cloud/scope/clients.go

@@ -19,10 +19,12 @@ package scope
 import (
 	"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
 	"github.com/aws/aws-sdk-go/service/elb/elbiface"
+	"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface"
 )
 
 // AWSClients contains all the aws clients used by the scopes.
 type AWSClients struct {
-	EC2 ec2iface.EC2API
-	ELB elbiface.ELBAPI
+	EC2             ec2iface.EC2API
+	ELB             elbiface.ELBAPI
+	ResourceTagging resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI
 }

pkg/cloud/scope/cluster.go

@@ -24,6 +24,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/elb"
+	"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
 	"github.com/go-logr/logr"
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -75,6 +76,12 @@ func NewClusterScope(params ClusterScopeParams) (*ClusterScope, error) {
 		params.AWSClients.ELB = elbClient
 	}
 
+	if params.AWSClients.ResourceTagging == nil {
+		resourceTagging := resourcegroupstaggingapi.New(session)
+		resourceTagging.Handlers.Complete.PushBack(recordAWSPermissionsIssue(params.AWSCluster))
+		params.AWSClients.ResourceTagging = resourceTagging
+	}
+
 	helper, err := patch.NewHelper(params.AWSCluster, params.Client)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to init patch helper")

pkg/cloud/services/ec2/securitygroups.go

@@ -19,12 +19,12 @@ package ec2
 import (
 	"fmt"
 
-	"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/wait"
-	"sigs.k8s.io/cluster-api-provider-aws/pkg/record"
-
+	errlist "k8s.io/apimachinery/pkg/util/errors"
 	"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/converters"
 	"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/filter"
+	"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/wait"
 	"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/tags"
+	"sigs.k8s.io/cluster-api-provider-aws/pkg/record"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -160,22 +160,69 @@ func (s *Service) deleteSecurityGroups() error {
 	}
 
 	for _, sg := range s.scope.SecurityGroups() {
-		input := &ec2.DeleteSecurityGroupInput{
-			GroupId: aws.String(sg.ID),
-		}
+		s.deleteSecurityGroup(&sg, "managed")
+	}
+
+	clusterGroups, err := s.describeClusterOwnedSecurityGroups()
+	if err != nil {
+		return err
+	}
 
-		if _, err := s.scope.EC2.DeleteSecurityGroup(input); awserrors.IsIgnorableSecurityGroupError(err) != nil {
-			record.Warnf(s.scope.AWSCluster, "FailedDeleteSecurityGroup", "Failed to delete managed SecurityGroup %q: %v", sg.ID, err)
-			return errors.Wrapf(err, "failed to delete security group %q", sg.ID)
+	errs := []error{}
+	for _, sg := range clusterGroups {
+		if err := s.deleteSecurityGroup(&sg, "cluster managed"); err != nil {
+			errs = append(errs, err)
 		}
+	}
+
+	if len(errs) != 0 {
+		return errlist.NewAggregate(errs)
+	}
+
+	return nil
+}
 
-		record.Eventf(s.scope.AWSCluster, "SuccessfulDeleteSecurityGroup", "Deleted managed SecurityGroup %q", sg.ID)
-		s.scope.V(2).Info("Deleted security group security group", "security-group-id", sg.ID)
+func (s *Service) deleteSecurityGroup(sg *infrav1.SecurityGroup, typ string) error {
+	input := &ec2.DeleteSecurityGroupInput{
+		GroupId: aws.String(sg.ID),
 	}
 
+	if _, err := s.scope.EC2.DeleteSecurityGroup(input); awserrors.IsIgnorableSecurityGroupError(err) != nil {
+		record.Warnf(s.scope.AWSCluster, "FailedDeleteSecurityGroup", "Failed to delete %s SecurityGroup %q: %v", typ, sg.ID, err)
+		return errors.Wrapf(err, "failed to delete security group %q", sg.ID)
+	}
+
+	record.Eventf(s.scope.AWSCluster, "SuccessfulDeleteSecurityGroup", "Deleted %s SecurityGroup %q", typ, sg.ID)
+	s.scope.V(2).Info("Deleted security group", "security-group-id", sg.ID, "kind", typ)
+
 	return nil
 }
 
+func (s *Service) describeClusterOwnedSecurityGroups() ([]infrav1.SecurityGroup, error) {
+	input := &ec2.DescribeSecurityGroupsInput{
+		Filters: []*ec2.Filter{
+			filter.EC2.VPC(s.scope.VPC().ID),
+			filter.EC2.ProviderOwned(s.scope.Name()),
+		},
+	}
+
+	groups := []infrav1.SecurityGroup{}
+
+	err := s.scope.EC2.DescribeSecurityGroupsPages(input, func(out *ec2.DescribeSecurityGroupsOutput, last bool) bool {
+		for _, group := range out.SecurityGroups {
+			if group != nil {
+				groups = append(groups, makeInfraSecurityGroup(group))
+			}
+		}
+		return true
+	})
+
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to describe cluster-owned security groups in vpc %q", s.scope.VPC().ID)
+	}
+	return groups, nil
+}
+
 func (s *Service) describeSecurityGroupsByName() (map[string]infrav1.SecurityGroup, error) {
 	input := &ec2.DescribeSecurityGroupsInput{
 		Filters: []*ec2.Filter{
@@ -191,11 +238,7 @@ func (s *Service) describeSecurityGroupsByName() (map[string]infrav1.SecurityGro
 
 	res := make(map[string]infrav1.SecurityGroup, len(out.SecurityGroups))
 	for _, ec2sg := range out.SecurityGroups {
-		sg := infrav1.SecurityGroup{
-			ID:   *ec2sg.GroupId,
-			Name: *ec2sg.GroupName,
-			Tags: converters.TagsToMap(ec2sg.Tags),
-		}
+		sg := makeInfraSecurityGroup(ec2sg)
 
 		for _, ec2rule := range ec2sg.IpPermissions {
 			sg.IngressRules = append(sg.IngressRules, ingressRuleFromSDKType(ec2rule))
@@ -207,6 +250,14 @@ func (s *Service) describeSecurityGroupsByName() (map[string]infrav1.SecurityGro
 	return res, nil
 }
 
+func makeInfraSecurityGroup(ec2sg *ec2.SecurityGroup) infrav1.SecurityGroup {
+	return infrav1.SecurityGroup{
+		ID:   *ec2sg.GroupId,
+		Name: *ec2sg.GroupName,
+		Tags: converters.TagsToMap(ec2sg.Tags),
+	}
+}
+
 func (s *Service) createSecurityGroup(role infrav1.SecurityGroupRole, input *ec2.SecurityGroup) error {
 	out, err := s.scope.EC2.CreateSecurityGroup(&ec2.CreateSecurityGroupInput{
 		VpcId:       input.VpcId,

pkg/cloud/services/elb/loadbalancer.go

@@ -19,18 +19,24 @@ package elb
 import (
 	"fmt"
 	"reflect"
+	"strings"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/service/elb"
+	rgapi "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
 	"github.com/pkg/errors"
 	infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1alpha2"
 	"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/awserrors"
 	"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/converters"
 	"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/wait"
 )
 
+// ResourceGroups are filtered by ARN identifier: https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html#arns-syntax
+// this is the identifier for classic ELBs: https://docs.aws.amazon.com/IAM/latest/UserGuide/list_elasticloadbalancing.html#elasticloadbalancing-resources-for-iam-policies
+const elbResourceType = "elasticloadbalancing:loadbalancer"
+
 // ReconcileLoadbalancers reconciles the load balancers for the given cluster.
 func (s *Service) ReconcileLoadbalancers() error {
 	s.scope.V(2).Info("Reconciling load balancers")
@@ -89,19 +95,25 @@ func (s *Service) GetAPIServerDNSName() (string, error) {
 func (s *Service) DeleteLoadbalancers() error {
 	s.scope.V(2).Info("Deleting load balancers")
 
-	// Get default api server name.
-	elbName := GenerateELBName(s.scope.Name(), infrav1.APIServerRoleTagValue)
-
-	// Describe and delete if exists.
-	if _, err := s.describeClassicELB(elbName); err != nil {
-		if IsNotFound(err) {
-			return nil
-		}
+	elbs, err := s.listOwnedELBs()
+	if err != nil {
 		return err
 	}
 
-	if err := s.deleteClassicELBAndWait(elbName); err != nil {
-		return err
+	for _, elb := range elbs {
+		s.scope.V(3).Info("deleting load balancer", "arn", elb)
+		s.deleteClassicELB(elb)
+	}
+
+	if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (done bool, err error) {
+		elbs, err := s.listOwnedELBs()
+		if err != nil {
+			return false, err
+		}
+
+		return len(elbs) == 0, nil
+	}); err != nil {
+		return errors.Wrapf(err, "failed to wait for %q ELB deletions", s.scope.Name())
 	}
 
 	s.scope.V(2).Info("Deleting load balancers completed successfully")
@@ -291,34 +303,58 @@ func (s *Service) deleteClassicELB(name string) error {
 	return nil
 }
 
-func (s *Service) deleteClassicELBAndWait(name string) error {
-	if err := s.deleteClassicELB(name); err != nil {
-		return err
+func (s *Service) listByTag(tag string) ([]string, error) {
+	input := rgapi.GetResourcesInput{
+		ResourceTypeFilters: aws.StringSlice([]string{elbResourceType}),
+		TagFilters: []*rgapi.TagFilter{
+			{
+				Key:    aws.String(tag),
+				Values: aws.StringSlice([]string{string(infrav1.ResourceLifecycleOwned)}),
+			},
+		},
 	}
 
-	input := &elb.DescribeLoadBalancersInput{
-		LoadBalancerNames: aws.StringSlice([]string{name}),
-	}
+	names := []string{}
 
-	if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (done bool, err error) {
-		out, err := s.scope.ELB.DescribeLoadBalancers(input)
-		if err != nil {
-			if code, ok := awserrors.Code(err); ok && code == awserrors.LoadBalancerNotFound {
-				return true, nil
+	err := s.scope.ResourceTagging.GetResourcesPages(&input, func(r *rgapi.GetResourcesOutput, last bool) bool {
+		for _, tagmapping := range r.ResourceTagMappingList {
+			if tagmapping.ResourceARN != nil {
+				// We can't use arn.Parse because the "Resource" is loadbalancer/<name>
+				parts := strings.Split(*tagmapping.ResourceARN, "/")
+				name := parts[len(parts)-1]
+				if name == "" {
+					s.scope.Info("failed to parse ARN", "arn", *tagmapping.ResourceARN, "tag", tag)
+					continue
+				}
+				names = append(names, name)
 			}
-			return false, err
 		}
+		return true
+	})
 
-		// ELB already deleted.
-		if len(out.LoadBalancerDescriptions) == 0 {
-			return true, nil
-		}
-		return false, nil
-	}); err != nil {
-		return errors.Wrapf(err, "failed to wait for ELB deletion %q", name)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to list %s ELBs by tag group", s.scope.Name())
 	}
 
-	return nil
+	return names, nil
+}
+
+func (s *Service) listOwnedELBs() ([]string, error) {
+	// k8s.io/cluster/<name>, created by k/k cloud provider
+	serviceTag := infrav1.ClusterAWSCloudProviderTagKey(s.scope.Name())
+	arns, err := s.listByTag(serviceTag)
+	if err != nil {
+		return nil, err
+	}
+
+	// sigs.k8s.io/cluster-api-provider-aws/cluster/<name>, created by CAPA
+	capaTag := infrav1.ClusterTagKey(s.scope.Name())
+	clusterArns, err := s.listByTag(capaTag)
+	if err != nil {
+		return nil, err
+	}
+
+	return append(arns, clusterArns...), nil
 }
 
 func (s *Service) describeClassicELB(name string) (*infrav1.ClassicELB, error) {