Skip to content

Commit 16dfbb3

Browse files
crawfordcrawford
committed
data/aws: use nlbs instead of elbs
We've noticed an elevated rate of installation failures recently. The root cause appears to be 50-90 seconds of latency added to traffic going through the internal ELB on port 49500. This was causing Ignition's connections to timeout, resulting in the machines never provisioning. AWS's NLBs don't seem to have this high latency, so we've decided to move over to them instead. With the move to NLBs, we also get the ability to add individual health checks for each port instead of just a single health check for each load balancer. Also, NLBs are cheaper. This commit drops support for ingress and the console. Since the console and router aren't currently configured correctly, nobody should notice that this is gone. It was easier to drop support in this commit rather than continue to try to plumb through the existing implementation knowing that it was going to have to change in the future. Once the router has a strategy for ingress, we'll re-add this functionality using the new NLBs. This also drop support for the `<cluster-name>-k8s` DNS entry. We aren't aware of any consumers and it was going to be tedious to keep this working.
1 parent 8823176 commit 16dfbb3

File tree

9 files changed

+245
-302
lines changed

9 files changed

+245
-302
lines changed

‎data/data/aws/bootstrap/main.tf

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -120,8 +120,16 @@ resource "aws_instance" "bootstrap" {
120120
volume_tags = "${var.tags}"
121121
}
122122

123-
resource "aws_elb_attachment" "bootstrap" {
124-
count = "${var.elbs_length}"
125-
elb = "${var.elbs[count.index]}"
126-
instance = "${aws_instance.bootstrap.id}"
123+
resource "aws_lb_target_group_attachment" "public" {
124+
count = "${var.public_target_group_arns_length}"
125+
126+
target_group_arn = "${var.public_target_group_arns[count.index]}"
127+
target_id = "${aws_instance.bootstrap.private_ip}"
128+
}
129+
130+
resource "aws_lb_target_group_attachment" "private" {
131+
count = "${var.private_target_group_arns_length}"
132+
133+
target_group_arn = "${var.private_target_group_arns[count.index]}"
134+
target_id = "${aws_instance.bootstrap.private_ip}"
127135
}

‎data/data/aws/bootstrap/variables.tf

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,16 +18,6 @@ variable "cluster_name" {
1818
description = "The name of the cluster."
1919
}
2020

21-
variable "elbs" {
22-
type = "list"
23-
default = []
24-
description = "Elastic load balancer IDs to attach to the bootstrap node."
25-
}
26-
27-
variable "elbs_length" {
28-
description = "The length of the 'elbs' variable, to work around https://github.com/hashicorp/terraform/issues/12570."
29-
}
30-
3121
variable "iam_role" {
3222
type = "string"
3323
default = ""
@@ -45,6 +35,26 @@ variable "instance_type" {
4535
description = "The EC2 instance type for the bootstrap node."
4636
}
4737

38+
variable "private_target_group_arns" {
39+
type = "list"
40+
default = []
41+
description = "The list of target group ARNs for the private load balancer."
42+
}
43+
44+
variable "private_target_group_arns_length" {
45+
description = "The length of the 'private_target_group_arns' variable, to work around https://github.com/hashicorp/terraform/issues/12570."
46+
}
47+
48+
variable "public_target_group_arns" {
49+
type = "list"
50+
default = []
51+
description = "The list of target group ARNs for the public load balancer."
52+
}
53+
54+
variable "public_target_group_arns_length" {
55+
description = "The length of the 'public_target_group_arns' variable, to work around https://github.com/hashicorp/terraform/issues/12570."
56+
}
57+
4858
variable "subnet_id" {
4959
type = "string"
5060
description = "The subnet ID for the bootstrap node."

‎data/data/aws/main.tf

Lines changed: 45 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -17,16 +17,18 @@ provider "aws" {
1717
module "bootstrap" {
1818
source = "./bootstrap"
1919

20-
ami = "${var.tectonic_aws_ec2_ami_override}"
21-
associate_public_ip_address = "${var.tectonic_aws_endpoints != "private"}"
22-
bucket = "${aws_s3_bucket.bootstrap.id}"
23-
cluster_name = "${var.tectonic_cluster_name}"
24-
elbs = "${module.vpc.aws_lbs}"
25-
elbs_length = "${module.vpc.aws_lbs_length}"
26-
iam_role = "${var.tectonic_aws_master_iam_role_name}"
27-
ignition = "${var.ignition_bootstrap}"
28-
subnet_id = "${module.vpc.master_subnet_ids[0]}"
29-
vpc_security_group_ids = ["${concat(var.tectonic_aws_master_extra_sg_ids, list(module.vpc.master_sg_id))}"]
20+
ami = "${var.tectonic_aws_ec2_ami_override}"
21+
associate_public_ip_address = "${var.tectonic_aws_endpoints != "private"}"
22+
bucket = "${aws_s3_bucket.bootstrap.id}"
23+
cluster_name = "${var.tectonic_cluster_name}"
24+
public_target_group_arns = "${module.vpc.aws_lb_public_target_group_arns}"
25+
public_target_group_arns_length = "${module.vpc.aws_lb_public_target_group_arns_length}"
26+
private_target_group_arns = "${module.vpc.aws_lb_private_target_group_arns}"
27+
private_target_group_arns_length = "${module.vpc.aws_lb_private_target_group_arns_length}"
28+
iam_role = "${var.tectonic_aws_master_iam_role_name}"
29+
ignition = "${var.ignition_bootstrap}"
30+
subnet_id = "${module.vpc.master_subnet_ids[0]}"
31+
vpc_security_group_ids = ["${concat(var.tectonic_aws_master_extra_sg_ids, list(module.vpc.master_sg_id))}"]
3032

3133
tags = "${merge(map(
3234
"Name", "${var.tectonic_cluster_name}-bootstrap",
@@ -37,25 +39,26 @@ module "bootstrap" {
3739
module "masters" {
3840
source = "./master"
3941

40-
elb_api_internal_id = "${module.vpc.aws_elb_api_internal_id}"
41-
elb_api_external_id = "${module.vpc.aws_elb_api_external_id}"
42-
elb_console_id = "${module.vpc.aws_elb_console_id}"
43-
base_domain = "${var.tectonic_base_domain}"
44-
cluster_id = "${var.tectonic_cluster_id}"
45-
cluster_name = "${var.tectonic_cluster_name}"
46-
ec2_type = "${var.tectonic_aws_master_ec2_type}"
47-
extra_tags = "${var.tectonic_aws_extra_tags}"
48-
instance_count = "${var.tectonic_master_count}"
49-
master_iam_role = "${var.tectonic_aws_master_iam_role_name}"
50-
master_sg_ids = "${concat(var.tectonic_aws_master_extra_sg_ids, list(module.vpc.master_sg_id))}"
51-
private_endpoints = "${local.private_endpoints}"
52-
public_endpoints = "${local.public_endpoints}"
53-
root_volume_iops = "${var.tectonic_aws_master_root_volume_iops}"
54-
root_volume_size = "${var.tectonic_aws_master_root_volume_size}"
55-
root_volume_type = "${var.tectonic_aws_master_root_volume_type}"
56-
subnet_ids = "${module.vpc.master_subnet_ids}"
57-
ec2_ami = "${var.tectonic_aws_ec2_ami_override}"
58-
user_data_ign = "${var.ignition_master}"
42+
public_target_group_arns = "${module.vpc.aws_lb_public_target_group_arns}"
43+
public_target_group_arns_length = "${module.vpc.aws_lb_public_target_group_arns_length}"
44+
private_target_group_arns = "${module.vpc.aws_lb_private_target_group_arns}"
45+
private_target_group_arns_length = "${module.vpc.aws_lb_private_target_group_arns_length}"
46+
base_domain = "${var.tectonic_base_domain}"
47+
cluster_id = "${var.tectonic_cluster_id}"
48+
cluster_name = "${var.tectonic_cluster_name}"
49+
ec2_type = "${var.tectonic_aws_master_ec2_type}"
50+
extra_tags = "${var.tectonic_aws_extra_tags}"
51+
instance_count = "${var.tectonic_master_count}"
52+
master_iam_role = "${var.tectonic_aws_master_iam_role_name}"
53+
master_sg_ids = "${concat(var.tectonic_aws_master_extra_sg_ids, list(module.vpc.master_sg_id))}"
54+
private_endpoints = "${local.private_endpoints}"
55+
public_endpoints = "${local.public_endpoints}"
56+
root_volume_iops = "${var.tectonic_aws_master_root_volume_iops}"
57+
root_volume_size = "${var.tectonic_aws_master_root_volume_size}"
58+
root_volume_type = "${var.tectonic_aws_master_root_volume_type}"
59+
subnet_ids = "${module.vpc.master_subnet_ids}"
60+
ec2_ami = "${var.tectonic_aws_ec2_ami_override}"
61+
user_data_ign = "${var.ignition_master}"
5962
}
6063

6164
module "iam" {
@@ -68,22 +71,19 @@ module "iam" {
6871
module "dns" {
6972
source = "./route53"
7073

71-
api_external_elb_dns_name = "${module.vpc.aws_elb_api_external_dns_name}"
72-
api_external_elb_zone_id = "${module.vpc.aws_elb_api_external_zone_id}"
73-
api_internal_elb_dns_name = "${module.vpc.aws_elb_api_internal_dns_name}"
74-
api_internal_elb_zone_id = "${module.vpc.aws_elb_api_internal_zone_id}"
75-
api_ip_addresses = "${module.vpc.aws_lbs}"
76-
base_domain = "${var.tectonic_base_domain}"
77-
cluster_name = "${var.tectonic_cluster_name}"
78-
console_elb_dns_name = "${module.vpc.aws_console_dns_name}"
79-
console_elb_zone_id = "${module.vpc.aws_elb_console_zone_id}"
80-
elb_alias_enabled = true
81-
master_count = "${var.tectonic_master_count}"
82-
private_zone_id = "${local.private_zone_id}"
83-
external_vpc_id = "${module.vpc.vpc_id}"
84-
extra_tags = "${var.tectonic_aws_extra_tags}"
85-
private_endpoints = "${local.private_endpoints}"
86-
public_endpoints = "${local.public_endpoints}"
74+
api_external_lb_dns_name = "${module.vpc.aws_lb_api_external_dns_name}"
75+
api_external_lb_zone_id = "${module.vpc.aws_lb_api_external_zone_id}"
76+
api_internal_lb_dns_name = "${module.vpc.aws_lb_api_internal_dns_name}"
77+
api_internal_lb_zone_id = "${module.vpc.aws_lb_api_internal_zone_id}"
78+
base_domain = "${var.tectonic_base_domain}"
79+
cluster_name = "${var.tectonic_cluster_name}"
80+
elb_alias_enabled = true
81+
master_count = "${var.tectonic_master_count}"
82+
private_zone_id = "${local.private_zone_id}"
83+
external_vpc_id = "${module.vpc.vpc_id}"
84+
extra_tags = "${var.tectonic_aws_extra_tags}"
85+
private_endpoints = "${local.private_endpoints}"
86+
public_endpoints = "${local.public_endpoints}"
8787
}
8888

8989
module "vpc" {

‎data/data/aws/master/main.tf

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -113,20 +113,16 @@ resource "aws_instance" "master" {
113113
), var.extra_tags)}"
114114
}
115115

116-
resource "aws_elb_attachment" "masters_internal" {
117-
count = "${var.private_endpoints ? var.instance_count : 0}"
118-
elb = "${var.elb_api_internal_id}"
119-
instance = "${aws_instance.master.*.id[count.index]}"
120-
}
116+
resource "aws_lb_target_group_attachment" "public" {
117+
count = "${var.public_endpoints ? var.instance_count * var.public_target_group_arns_length : 0}"
121118

122-
resource "aws_elb_attachment" "masters_external" {
123-
count = "${var.public_endpoints ? var.instance_count : 0}"
124-
elb = "${var.elb_api_external_id}"
125-
instance = "${aws_instance.master.*.id[count.index]}"
119+
target_group_arn = "${var.public_target_group_arns[count.index % var.public_target_group_arns_length]}"
120+
target_id = "${aws_instance.master.*.private_ip[count.index / var.public_target_group_arns_length]}"
126121
}
127122

128-
resource "aws_elb_attachment" "masters_console" {
129-
count = "${var.instance_count}"
130-
elb = "${var.elb_console_id}"
131-
instance = "${aws_instance.master.*.id[count.index]}"
123+
resource "aws_lb_target_group_attachment" "private" {
124+
count = "${var.private_endpoints ? var.instance_count * var.private_target_group_arns_length : 0}"
125+
126+
target_group_arn = "${var.private_target_group_arns[count.index % var.private_target_group_arns_length]}"
127+
target_id = "${aws_instance.master.*.private_ip[count.index / var.private_target_group_arns_length]}"
132128
}

‎data/data/aws/master/variables.tf

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -46,21 +46,29 @@ variable "private_endpoints" {
4646
default = true
4747
}
4848

49+
variable "private_target_group_arns" {
50+
type = "list"
51+
default = []
52+
description = "The list of target group ARNs for the private load balancer."
53+
}
54+
55+
variable "private_target_group_arns_length" {
56+
description = "The length of the 'private_target_group_arns' variable, to work around https://github.com/hashicorp/terraform/issues/12570."
57+
}
58+
4959
variable "public_endpoints" {
5060
description = "If set to true, public-facing ingress resources are created."
5161
default = true
5262
}
5363

54-
variable "elb_api_internal_id" {
55-
type = "string"
56-
}
57-
58-
variable "elb_api_external_id" {
59-
type = "string"
64+
variable "public_target_group_arns" {
65+
type = "list"
66+
default = []
67+
description = "The list of target group ARNs for the public load balancer."
6068
}
6169

62-
variable "elb_console_id" {
63-
type = "string"
70+
variable "public_target_group_arns_length" {
71+
description = "The length of the 'public_target_group_arns' variable, to work around https://github.com/hashicorp/terraform/issues/12570."
6472
}
6573

6674
variable "root_volume_iops" {

‎data/data/aws/route53/tectonic.tf

Lines changed: 9 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -10,103 +10,33 @@ data "aws_route53_zone" "tectonic" {
1010
locals {
1111
public_zone_id = "${join("", data.aws_route53_zone.tectonic.*.zone_id)}"
1212

13-
zone_id = "${var.private_endpoints ?
14-
var.private_zone_id :
15-
local.public_zone_id}"
16-
}
17-
18-
resource "aws_route53_record" "tectonic_api" {
19-
count = "${var.elb_alias_enabled ? 0 : 1}"
20-
zone_id = "${local.public_zone_id}"
21-
name = "${var.cluster_name}-k8s"
22-
type = "A"
23-
ttl = "60"
24-
records = ["${var.api_ip_addresses}"]
13+
zone_id = "${var.private_endpoints ? var.private_zone_id : local.public_zone_id}"
2514
}
2615

2716
resource "aws_route53_record" "tectonic_api_external" {
28-
count = "${var.elb_alias_enabled ? local.public_endpoints_count : 0}"
17+
count = "${var.elb_alias_enabled ? local.public_endpoints_count : 0}"
18+
2919
zone_id = "${local.public_zone_id}"
3020
name = "${var.cluster_name}-api.${var.base_domain}"
3121
type = "A"
3222

3323
alias {
34-
name = "${var.api_external_elb_dns_name}"
35-
zone_id = "${var.api_external_elb_zone_id}"
24+
name = "${var.api_external_lb_dns_name}"
25+
zone_id = "${var.api_external_lb_zone_id}"
3626
evaluate_target_health = true
3727
}
3828
}
3929

4030
resource "aws_route53_record" "tectonic_api_internal" {
41-
count = "${var.elb_alias_enabled ? local.private_endpoints_count : 0}"
42-
zone_id = "${var.private_zone_id}"
43-
name = "${var.cluster_name}-api.${var.base_domain}"
44-
type = "A"
45-
46-
alias {
47-
name = "${var.api_internal_elb_dns_name}"
48-
zone_id = "${var.api_internal_elb_zone_id}"
49-
evaluate_target_health = true
50-
}
51-
}
52-
53-
resource "aws_route53_record" "tectonic-console" {
54-
count = "${var.elb_alias_enabled ? 0 : 1}"
55-
zone_id = "${local.public_zone_id}"
56-
name = "${var.cluster_name}"
57-
type = "A"
58-
ttl = "60"
59-
records = ["${var.worker_ip_addresses}"]
60-
}
61-
62-
resource "aws_route53_record" "tectonic_ingress_public" {
63-
count = "${var.elb_alias_enabled ? local.public_endpoints_count : 0}"
64-
zone_id = "${local.public_zone_id}"
65-
name = "${var.cluster_name}.${var.base_domain}"
66-
type = "A"
67-
68-
alias {
69-
name = "${var.console_elb_dns_name}"
70-
zone_id = "${var.console_elb_zone_id}"
71-
evaluate_target_health = true
72-
}
73-
}
31+
count = "${var.elb_alias_enabled ? local.private_endpoints_count : 0}"
7432

75-
resource "aws_route53_record" "tectonic_ingress_private" {
76-
count = "${var.elb_alias_enabled ? local.private_endpoints_count : 0}"
7733
zone_id = "${var.private_zone_id}"
78-
name = "${var.cluster_name}.${var.base_domain}"
79-
type = "A"
80-
81-
alias {
82-
name = "${var.console_elb_dns_name}"
83-
zone_id = "${var.console_elb_zone_id}"
84-
evaluate_target_health = true
85-
}
86-
}
87-
88-
resource "aws_route53_record" "routes_ingress_public" {
89-
count = "${var.elb_alias_enabled ? local.public_endpoints_count : 0}"
90-
zone_id = "${local.public_zone_id}"
91-
name = "*.${var.cluster_name}.${var.base_domain}"
92-
type = "A"
93-
94-
alias {
95-
name = "${var.console_elb_dns_name}"
96-
zone_id = "${var.console_elb_zone_id}"
97-
evaluate_target_health = true
98-
}
99-
}
100-
101-
resource "aws_route53_record" "routes_ingress_private" {
102-
count = "${var.elb_alias_enabled ? local.private_endpoints_count : 0}"
103-
zone_id = "${var.private_zone_id}"
104-
name = "*.${var.cluster_name}.${var.base_domain}"
34+
name = "${var.cluster_name}-api.${var.base_domain}"
10535
type = "A"
10636

10737
alias {
108-
name = "${var.console_elb_dns_name}"
109-
zone_id = "${var.console_elb_zone_id}"
38+
name = "${var.api_internal_lb_dns_name}"
39+
zone_id = "${var.api_internal_lb_zone_id}"
11040
evaluate_target_health = true
11141
}
11242
}

0 commit comments

Comments
 (0)