From eab6fcc8e636d7d22000d2f4d1da27e519f5081a Mon Sep 17 00:00:00 2001 From: jianglai Date: Tue, 17 Apr 2018 12:08:15 -0700 Subject: [PATCH] Add networking settings for canary proxies Canary proxies are not receiving real traffic but can be useful when testing Nomulus deployment (probers will probe canary proxy and compare metrics with production proxy). This CL added a separate load balancer for a canary proxy, running on the same clusters as production proxy. The canary proxies have their own IP addresses, but are not assigned domain names. Probers will directly connect to these endpoints by IP. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=193234937 --- .../proxy/terraform/example_config.tf | 5 +- .../registry/proxy/terraform/modules/dns.tf | 8 +- .../registry/proxy/terraform/modules/gke.tf | 7 - .../proxy/terraform/modules/gke/input.tf | 4 - .../registry/proxy/terraform/modules/input.tf | 11 ++ .../proxy/terraform/modules/networking.tf | 135 ++---------------- .../proxy/terraform/modules/output.tf | 17 ++- .../proxy/terraform/update_named_ports.sh | 3 +- 8 files changed, 38 insertions(+), 152 deletions(-) diff --git a/java/google/registry/proxy/terraform/example_config.tf b/java/google/registry/proxy/terraform/example_config.tf index 29fd94634..d19568bcd 100644 --- a/java/google/registry/proxy/terraform/example_config.tf +++ b/java/google/registry/proxy/terraform/example_config.tf @@ -28,8 +28,5 @@ output "proxy_instance_groups" { } output "proxy_ip_addresses" { - value = { - ipv4 = "${module.proxy.proxy_ipv4_address}" - ipv6 = "${module.proxy.proxy_ipv6_address}" - } + value = "${module.proxy.proxy_ip_addresses}" } diff --git a/java/google/registry/proxy/terraform/modules/dns.tf b/java/google/registry/proxy/terraform/modules/dns.tf index a59badbca..5ac480799 100644 --- a/java/google/registry/proxy/terraform/modules/dns.tf +++ b/java/google/registry/proxy/terraform/modules/dns.tf @@ -8,7 +8,7 @@ resource "google_dns_record_set" "proxy_epp_a_record" { type = "A" ttl = 300 managed_zone = "${google_dns_managed_zone.proxy_domain.name}" - rrdatas = ["${google_compute_global_address.proxy_ipv4_address.address}"] + rrdatas = ["${module.proxy_networking.proxy_ipv4_address}"] } resource "google_dns_record_set" "proxy_epp_aaaa_record" { @@ -16,7 +16,7 @@ resource "google_dns_record_set" "proxy_epp_aaaa_record" { type = "AAAA" ttl = 300 managed_zone = "${google_dns_managed_zone.proxy_domain.name}" - rrdatas = ["${google_compute_global_address.proxy_ipv6_address.address}"] + rrdatas = ["${module.proxy_networking.proxy_ipv6_address}"] } resource "google_dns_record_set" "proxy_whois_a_record" { @@ -24,7 +24,7 @@ resource "google_dns_record_set" "proxy_whois_a_record" { type = "A" ttl = 300 managed_zone = "${google_dns_managed_zone.proxy_domain.name}" - rrdatas = ["${google_compute_global_address.proxy_ipv4_address.address}"] + rrdatas = ["${module.proxy_networking.proxy_ipv4_address}"] } resource "google_dns_record_set" "proxy_whois_aaaa_record" { @@ -32,5 +32,5 @@ resource "google_dns_record_set" "proxy_whois_aaaa_record" { type = "AAAA" ttl = 300 managed_zone = "${google_dns_managed_zone.proxy_domain.name}" - rrdatas = ["${google_compute_global_address.proxy_ipv6_address.address}"] + rrdatas = ["${module.proxy_networking.proxy_ipv6_address}"] } diff --git a/java/google/registry/proxy/terraform/modules/gke.tf b/java/google/registry/proxy/terraform/modules/gke.tf index 787bdbb74..eb991360f 100644 --- a/java/google/registry/proxy/terraform/modules/gke.tf +++ b/java/google/registry/proxy/terraform/modules/gke.tf @@ -2,21 +2,18 @@ module "proxy_gke_americas" { source = "./gke" proxy_cluster_region = "americas" proxy_service_account_email = "${google_service_account.proxy_service_account.email}" - proxy_ports = "${var.proxy_ports}" } module "proxy_gke_emea" { source = "./gke" proxy_cluster_region = "emea" proxy_service_account_email = "${google_service_account.proxy_service_account.email}" - proxy_ports = "${var.proxy_ports}" } module "proxy_gke_apac" { source = "./gke" proxy_cluster_region = "apac" proxy_service_account_email = "${google_service_account.proxy_service_account.email}" - proxy_ports = "${var.proxy_ports}" } locals { @@ -26,7 +23,3 @@ locals { apac = "${module.proxy_gke_apac.proxy_instance_group}" } } - -output "proxy_instance_groups" { - value = "${local.proxy_instance_groups}" -} diff --git a/java/google/registry/proxy/terraform/modules/gke/input.tf b/java/google/registry/proxy/terraform/modules/gke/input.tf index 6482ef14c..94b1ad014 100644 --- a/java/google/registry/proxy/terraform/modules/gke/input.tf +++ b/java/google/registry/proxy/terraform/modules/gke/input.tf @@ -11,7 +11,3 @@ variable "proxy_cluster_zones" { apac = "asia-northeast1-c" } } - -variable "proxy_ports" { - type = "map" -} diff --git a/java/google/registry/proxy/terraform/modules/input.tf b/java/google/registry/proxy/terraform/modules/input.tf index 102dc54b3..a6236b7fb 100644 --- a/java/google/registry/proxy/terraform/modules/input.tf +++ b/java/google/registry/proxy/terraform/modules/input.tf @@ -33,3 +33,14 @@ variable "proxy_ports" { epp = 30002 } } + +# Node ports exposed by the canary proxy. +variable "proxy_ports_canary" { + type = "map" + + default = { + health_check = 40000 + whois = 40001 + epp = 40002 + } +} diff --git a/java/google/registry/proxy/terraform/modules/networking.tf b/java/google/registry/proxy/terraform/modules/networking.tf index 55c0d58d5..2c797cad8 100644 --- a/java/google/registry/proxy/terraform/modules/networking.tf +++ b/java/google/registry/proxy/terraform/modules/networking.tf @@ -1,129 +1,12 @@ -resource "google_compute_global_address" "proxy_ipv4_address" { - name = "proxy-ipv4-address" - ip_version = "IPV4" +module "proxy_networking" { + source = "./networking" + proxy_instance_groups = "${local.proxy_instance_groups}" + proxy_ports = "${var.proxy_ports}" } -resource "google_compute_global_address" "proxy_ipv6_address" { - name = "proxy-ipv6-address" - ip_version = "IPV6" -} - -resource "google_compute_firewall" "proxy_firewall" { - name = "proxy-firewall" - network = "default" - - allow { - protocol = "tcp" - - ports = [ - "${var.proxy_ports["epp"]}", - "${var.proxy_ports["whois"]}", - "${var.proxy_ports["health_check"]}", - ] - } - - source_ranges = [ - "130.211.0.0/22", - "35.191.0.0/16", - ] - - target_tags = [ - "proxy-cluster", - ] -} - -resource "google_compute_health_check" "proxy_health_check" { - name = "proxy-health-check" - - tcp_health_check { - port = "${var.proxy_ports["health_check"]}" - request = "HEALTH_CHECK_REQUEST" - response = "HEALTH_CHECK_RESPONSE" - } -} - -resource "google_compute_backend_service" "epp_backend_service" { - name = "epp-backend-service" - protocol = "TCP" - timeout_sec = 3600 - port_name = "epp" - - backend { - group = "${local.proxy_instance_groups["americas"]}" - } - - backend { - group = "${local.proxy_instance_groups["emea"]}" - } - - backend { - group = "${local.proxy_instance_groups["apac"]}" - } - - health_checks = [ - "${google_compute_health_check.proxy_health_check.self_link}", - ] -} - -resource "google_compute_backend_service" "whois_backend_service" { - name = "whois-backend-service" - protocol = "TCP" - timeout_sec = 60 - port_name = "whois" - - backend { - group = "${local.proxy_instance_groups["americas"]}" - } - - backend { - group = "${local.proxy_instance_groups["emea"]}" - } - - backend { - group = "${local.proxy_instance_groups["apac"]}" - } - - health_checks = [ - "${google_compute_health_check.proxy_health_check.self_link}", - ] -} - -resource "google_compute_target_tcp_proxy" "epp_tcp_proxy" { - name = "epp-tcp-proxy" - proxy_header = "PROXY_V1" - backend_service = "${google_compute_backend_service.epp_backend_service.self_link}" -} - -resource "google_compute_target_tcp_proxy" "whois_tcp_proxy" { - name = "whois-tcp-proxy" - proxy_header = "PROXY_V1" - backend_service = "${google_compute_backend_service.whois_backend_service.self_link}" -} - -resource "google_compute_global_forwarding_rule" "epp_ipv4_forwarding_rule" { - name = "epp-ipv4-forwarding-rule" - ip_address = "${google_compute_global_address.proxy_ipv4_address.address}" - target = "${google_compute_target_tcp_proxy.epp_tcp_proxy.self_link}" - port_range = "700" -} - -resource "google_compute_global_forwarding_rule" "epp_ipv6_forwarding_rule" { - name = "epp-ipv6-forwarding-rule" - ip_address = "${google_compute_global_address.proxy_ipv6_address.address}" - target = "${google_compute_target_tcp_proxy.epp_tcp_proxy.self_link}" - port_range = "700" -} - -resource "google_compute_global_forwarding_rule" "whois_ipv4_forwarding_rule" { - name = "whois-ipv4-forwarding-rule" - ip_address = "${google_compute_global_address.proxy_ipv4_address.address}" - target = "${google_compute_target_tcp_proxy.whois_tcp_proxy.self_link}" - port_range = "43" -} - -resource "google_compute_global_forwarding_rule" "whois_ipv6_forwarding_rule" { - name = "whois-ipv6-forwarding-rule" - ip_address = "${google_compute_global_address.proxy_ipv6_address.address}" - target = "${google_compute_target_tcp_proxy.whois_tcp_proxy.self_link}" - port_range = "43" +module "proxy_networking_canary" { + source = "./networking" + proxy_instance_groups = "${local.proxy_instance_groups}" + suffix = "-canary" + proxy_ports = "${var.proxy_ports_canary}" } diff --git a/java/google/registry/proxy/terraform/modules/output.tf b/java/google/registry/proxy/terraform/modules/output.tf index 7f3e76eec..d54c8c952 100644 --- a/java/google/registry/proxy/terraform/modules/output.tf +++ b/java/google/registry/proxy/terraform/modules/output.tf @@ -2,14 +2,19 @@ output "proxy_name_servers" { value = "${google_dns_managed_zone.proxy_domain.name_servers}" } +output "proxy_instance_groups" { + value = "${local.proxy_instance_groups}" +} + output "proxy_service_account_client_id" { value = "${google_service_account.proxy_service_account.unique_id}" } -output "proxy_ipv4_address" { - value = "${google_compute_global_address.proxy_ipv4_address.address}" -} - -output "proxy_ipv6_address" { - value = "${google_compute_global_address.proxy_ipv6_address.address}" +output "proxy_ip_addresses" { + value = { + ipv4 = "${module.proxy_networking.proxy_ipv4_address}" + ipv6 = "${module.proxy_networking.proxy_ipv6_address}" + ipv4_canary = "${module.proxy_networking_canary.proxy_ipv4_address}" + ipv6_canary = "${module.proxy_networking_canary.proxy_ipv6_address}" + } } diff --git a/java/google/registry/proxy/terraform/update_named_ports.sh b/java/google/registry/proxy/terraform/update_named_ports.sh index c384b9b79..f11eb69cb 100755 --- a/java/google/registry/proxy/terraform/update_named_ports.sh +++ b/java/google/registry/proxy/terraform/update_named_ports.sh @@ -21,6 +21,7 @@ while read line do gcloud compute instance-groups set-named-ports \ - --named-ports whois:30001,epp:30002 $line + --named-ports whois:30001,epp:30002,whois-canary:40001,epp-canary:40002 \ + $line done < <(terraform output proxy_instance_groups | awk '{print $3}' | \ awk -F '/' '{print "--project", $7, "--zone", $9, $11}')