Automating more parts of the Packet NAT setup bits

Author: meatballhat

.gitignore

@@ -5,6 +5,7 @@
 .env
 .ruby-version
 .terraform/
+/tmp/
 assets/*.tar.bz2
 aws-*/config/
 crash.log

modules/packet_network/cloud-config.yml.tpl

@@ -5,6 +5,9 @@ write_files:
 - content: '${base64encode(github_users_env)}'
   encoding: b64
   path: /etc/default/github-users
+- content: '${base64encode(network_env)}'
+  encoding: b64
+  path: /etc/default/travis-network
 - content: '${base64encode(file("${assets}/rsyslog/rsyslog.conf"))}'
   encoding: b64
   path: /etc/rsyslog.conf

modules/packet_network/cloud-init.bash

@@ -13,7 +13,7 @@ main() {
   __setup_travis_user
   __install_packages
   __setup_sysctl
-  __setup_iptables
+  __setup_networking
 
   hostname >"${RUNDIR}/instance-hostname.tmpl"
 }
@@ -33,43 +33,52 @@ __install_packages() {
 }
 
 __setup_sysctl() {
-  # NOTE: we do this mostly to ensure file IO chatty services like mysql will
-  # play nicely with others, such as when multiple containers are running mysql,
-  # which is the default on precise + trusty.  The value we set here is 16^5,
-  # which is one power higher than the default of 16^4 :sparkles:.
   echo 1048576 >/proc/sys/fs/aio-max-nr
   sysctl -w fs.aio-max-nr=1048576
 
   echo 1 >/proc/sys/net/ipv4/ip_forward
   sysctl -w net.ipv4.ip_forward=1
 }
 
-__setup_iptables() {
+__setup_networking() {
   local pub_iface priv_iface elastic_ip
   pub_iface="$(__find_public_interface)"
   priv_iface="$(__find_private_interface)"
   elastic_ip="$(__find_elastic_ip)"
 
-  iptables -t nat -A POSTROUTING -o "${pub_iface}" -j SNAT --to "${elastic_ip}"
+  if [[ -n "${elastic_ip}" ]]; then
+    ip address add "${elastic_ip}/32" dev lo
+    iptables -t nat -A POSTROUTING -o "${pub_iface}" -j SNAT --to "${elastic_ip}"
+  fi
   iptables -t nat -A POSTROUTING -o "${pub_iface}" -j MASQUERADE
   iptables -A FORWARD -i "${pub_iface}" -o "${priv_iface}" \
     -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
   iptables -A FORWARD -i "${priv_iface}" -o "${pub_iface}" -j ACCEPT
 }
 
 __find_private_interface() {
-  # FIXME: dynamically do
-  echo enp2s0d1
+  local iface=enp2s0d1
+  iface="$(ip -o addr show | grep 'inet 192' | awk '{ print $2 }')"
+  echo "${iface:-enp2s0d1}"
 }
 
 __find_public_interface() {
-  # FIXME: dynamically do
-  echo bond0
+  local iface=bond0
+  iface="$(ip -o addr show | grep -vE 'inet (172|127|10|192)\.' | grep -v inet6 |
+             awk '{ print $2 }' | grep -v '^lo$' | head -n 1)"
+  echo "${iface:-bond0}"
 }
 
 __find_elastic_ip() {
   # FIXME: inject this from somewhere?
-  echo 127.0.0.1
+  local elastic_ip
+  if [[ -f /etc/default/travis-network ]]; then
+    # shellcheck source=/dev/null
+    source /etc/default/travis-network
+    elastic_ip="${TRAVIS_NETWORK_ELASTIC_IP}"
+  fi
+
+  echo "${elastic_ip}"
 }
 
 main "$@"

modules/packet_network/main.tf

@@ -17,6 +17,12 @@ variable "nat_server_plan" {
 variable "project_id" {}
 variable "syslog_address" {}
 
+resource "packet_reserved_ip_block" "ips" {
+  project_id = "${var.project_id}"
+  facility   = "${var.facility}"
+  quantity   = 1
+}
+
 data "template_file" "duo_config" {
   template = <<EOF
 # Written by cloud-init :heart:
@@ -28,18 +34,30 @@ failmode = secure
 EOF
 }
 
+data "template_file" "network_env" {
+  template = <<EOF
+export TRAVIS_NETWORK_ELASTIC_IP=${cidrhost(packet_reserved_ip_block.ips.cidr_notation, 0)}
+EOF
+}
+
 data "template_file" "cloud_config" {
   template = "${file("${path.module}/cloud-config.yml.tpl")}"
 
   vars {
     assets           = "${path.module}/../../assets"
     github_users_env = "export GITHUB_USERS='${var.github_users}'"
     here             = "${path.module}"
+    network_env      = "${data.template_file.network_env.rendered}"
     syslog_address   = "${var.syslog_address}"
     duo_config       = "${data.template_file.duo_config.rendered}"
   }
 }
 
+resource "local_file" "user_data_dump" {
+  filename = "${path.module}/../../tmp/packet-${var.env}-${var.index}-nat-user-data.yml"
+  content  = "${data.template_file.cloud_config.rendered}"
+}
+
 resource "packet_device" "nat" {
   billing_cycle    = "${var.billing_cycle}"
   facility         = "${var.facility}"
@@ -50,37 +68,30 @@ resource "packet_device" "nat" {
   user_data        = "${data.template_file.cloud_config.rendered}"
 }
 
-resource "packet_reserved_ip_block" "ips" {
-  project_id = "${var.project_id}"
-  facility   = "${var.facility}"
-  quantity   = 1
-}
-
-resource "packet_ip_attachment" "nat" {
-  device_id     = "${packet_device.nat.id}"
-  cidr_notation = "${packet_reserved_ip_block.ips.cidr_notation}"
-}
-
-resource "null_resource" "nat_post_provisioning_todo" {
+resource "null_resource" "user_data_copy" {
   triggers {
-    nat_public_ip = "${cidrhost(packet_ip_attachment.nat.cidr_notation, 0)}"
+    user_data_sha1 = "${sha1(data.template_file.cloud_config.rendered)}"
   }
 
-  provisioner "local-exec" {
-    command = <<EOF
-cat <<EOCAT
-TODO: finish configuring the nat with something like
+  depends_on = ["packet_device.nat", "local_file.user_data_dump"]
 
-    ip addr add ${cidrhost(packet_ip_attachment.nat.cidr_notation, 0)} dev bond0
-    ip route delete default
-    ip route add default via ${cidrhost(packet_ip_attachment.nat.cidr_notation, 0)} dev bond0
-    curl icanhazip.com  # <=== should be ${cidrhost(packet_ip_attachment.nat.cidr_notation, 0)}
+  provisioner "file" {
+    source      = "${local_file.user_data_dump.filename}"
+    destination = "/var/tmp/user-data.yml"
+  }
 
-EOCAT
-EOF
+  connection {
+    type = "ssh"
+    user = "root"
+    host = "${packet_device.nat.access_public_ipv4}"
   }
 }
 
+resource "packet_ip_attachment" "nat" {
+  device_id     = "${packet_device.nat.id}"
+  cidr_notation = "${packet_reserved_ip_block.ips.cidr_notation}"
+}
+
 output "nat_ip" {
   value = "${packet_device.nat.access_private_ipv4}"
 }
@@ -89,6 +100,10 @@ output "nat_public_ip" {
   value = "${cidrhost(packet_ip_attachment.nat.cidr_notation, 0)}"
 }
 
+output "nat_maint_ip" {
+  value = "${packet_device.nat.access_public_ipv4}"
+}
+
 output "facility" {
   value = "${var.facility}"
 }

modules/packet_worker/cloud-init.bash

@@ -42,8 +42,8 @@ main() {
   fi
 
   if [[ "${TRAVIS_NETWORK_NAT_IP}" ]]; then
-    # FIXME: halp
-    : something with iptables maybe
+    ip route del default
+    ip route add default via "${TRAVIS_NETWORK_NAT_IP}"
   fi
 
   __wait_for_docker

modules/packet_worker/main.tf

@@ -1,3 +1,4 @@
+variable "bastion_ip" {}
 variable "billing_cycle" {
   default = "hourly"
 }
@@ -13,6 +14,7 @@ variable "index" {}
 
 variable "project_id" {}
 variable "nat_ip" {}
+variable "nat_public_ip" {}
 variable "server_count" {}
 
 variable "server_plan" {
@@ -59,6 +61,7 @@ EOF
 data "template_file" "network_env" {
   template = <<EOF
 export TRAVIS_NETWORK_NAT_IP=${var.nat_ip}
+export TRAVIS_NETWORK_ELASTIC_IP=${var.nat_public_ip}
 EOF
 }
 
@@ -98,6 +101,11 @@ data "template_file" "cloud_config" {
   }
 }
 
+resource "local_file" "user_data_dump" {
+  filename = "${path.module}/../../tmp/packet-${var.env}-${var.index}-worker-user-data.yml"
+  content  = "${data.template_file.cloud_config.rendered}"
+}
+
 resource "packet_device" "worker" {
   count            = "${var.server_count}"
   billing_cycle    = "${var.billing_cycle}"
@@ -108,3 +116,24 @@ resource "packet_device" "worker" {
   project_id       = "${var.project_id}"
   user_data        = "${data.template_file.cloud_config.rendered}"
 }
+
+resource "null_resource" "user_data_copy" {
+  triggers {
+    user_data_sha1 = "${sha1(data.template_file.cloud_config.rendered)}"
+  }
+
+  depends_on = ["packet_device.worker", "local_file.user_data_dump"]
+
+  provisioner "file" {
+    source      = "${local_file.user_data_dump.filename}"
+    destination = "/var/tmp/user-data.yml"
+  }
+
+  connection {
+    type         = "ssh"
+    user         = "root"
+    host         = "${packet_device.worker.access_private_ipv4}"
+    bastion_host = "${var.bastion_ip}"
+    bastion_user = "root"
+  }
+}

packet-staging-1/main.tf

@@ -81,11 +81,17 @@ EOF
 
 module "packet_workers_com" {
   source                      = "../modules/packet_worker"
+  bastion_ip                  = "${module.packet_network_sjc1.nat_maint_ip}"
   env                         = "${var.env}"
   facility                    = "${module.packet_network_sjc1.facility}"
   github_users                = "${var.github_users}"
   index                       = "${var.index}"
   nat_ip                      = "${module.packet_network_sjc1.nat_ip}"
+  nat_public_ip               = "${module.packet_network_sjc1.nat_public_ip}"
+  project_id                  = "${var.project_id}"
+  server_count                = 1
+  site                        = "com"
+  syslog_address              = "${var.syslog_address_com}"
   worker_config               = "${data.template_file.worker_config_com.rendered}"
   worker_docker_image_android = "${var.latest_docker_image_amethyst}"
   worker_docker_image_default = "${var.latest_docker_image_garnet}"
@@ -99,19 +105,21 @@ module "packet_workers_com" {
   worker_docker_image_python  = "${var.latest_docker_image_garnet}"
   worker_docker_image_ruby    = "${var.latest_docker_image_garnet}"
   worker_docker_self_image    = "${var.latest_docker_image_worker}"
-  server_count                = 1
-  syslog_address              = "${var.syslog_address_com}"
-  site                        = "com"
-  project_id                  = "${var.project_id}"
 }
 
 module "packet_workers_org" {
   source                      = "../modules/packet_worker"
+  bastion_ip                  = "${module.packet_network_sjc1.nat_maint_ip}"
   env                         = "${var.env}"
   facility                    = "${module.packet_network_sjc1.facility}"
   github_users                = "${var.github_users}"
   index                       = "${var.index}"
   nat_ip                      = "${module.packet_network_sjc1.nat_ip}"
+  nat_public_ip               = "${module.packet_network_sjc1.nat_public_ip}"
+  project_id                  = "${var.project_id}"
+  server_count                = 1
+  site                        = "org"
+  syslog_address              = "${var.syslog_address_org}"
   worker_config               = "${data.template_file.worker_config_org.rendered}"
   worker_docker_image_android = "${var.latest_docker_image_amethyst}"
   worker_docker_image_default = "${var.latest_docker_image_garnet}"
@@ -125,10 +133,6 @@ module "packet_workers_org" {
   worker_docker_image_python  = "${var.latest_docker_image_garnet}"
   worker_docker_image_ruby    = "${var.latest_docker_image_garnet}"
   worker_docker_self_image    = "${var.latest_docker_image_worker}"
-  server_count                = 1
-  syslog_address              = "${var.syslog_address_org}"
-  site                        = "org"
-  project_id                  = "${var.project_id}"
 }
 
 resource "aws_route53_record" "nat" {