From 6a994f320fad326de671ca63030f6625f9d7bf21 Mon Sep 17 00:00:00 2001 From: jianglai Date: Mon, 26 Feb 2018 10:51:26 -0800 Subject: [PATCH] Add GKE deployment config files for GCP proxy This CL setups up kubernetes configuration files necessary to deploy the proxy service to k8s (GKE to be specific). Because kubernetes service can only expose node ports higher than 30000, the default ports that the containers expose are also changed to >30000 so that they are consistent. This is *not* necessary, but makes it easier to remember which ports are for what purpose. Note that we are not setting up a load balancing service. The way it is set up now, the services are only visible within the clusters, on each node at the specified node ports. The load balancer k8s sets up uses GCP L4 load balancer that does not support IPv6 (because it does not do TCP termination at the LB, rather just routes packages to cluster nodes, and GCE VMs does not support IPv6 yet). The L4 load balancer also only provides regional IPs on the frontend, which means proxies running in different regions (Americas, EMEA, APAC) would all have different IPs, which in turn offload regional routing determination to the DNS system, adding complexity. A user of the proxy instead should set up TCP proxy load balancing in GCP separately and point traffic to the VM group(s) backing the k8s cluster. This allows for a single global anycast IP (IPv4 and IPv6) to be allocated at the load balancer frontend. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=187046521 --- .../registry/proxy/config/default-config.yaml | 2 +- .../proxy/{ => kubernetes}/Dockerfile | 4 +- .../proxy/{ => kubernetes}/build_image.sh | 4 +- .../proxy/kubernetes/proxy-deployment.yaml | 47 +++++++++++++++++++ .../proxy/kubernetes/proxy-service.yaml | 24 ++++++++++ 5 files changed, 77 insertions(+), 4 deletions(-) rename java/google/registry/proxy/{ => kubernetes}/Dockerfile (89%) rename java/google/registry/proxy/{ => kubernetes}/build_image.sh (95%) create mode 100644 java/google/registry/proxy/kubernetes/proxy-deployment.yaml create mode 100644 java/google/registry/proxy/kubernetes/proxy-service.yaml diff --git a/java/google/registry/proxy/config/default-config.yaml b/java/google/registry/proxy/config/default-config.yaml index c5e468747..da9f89e59 100644 --- a/java/google/registry/proxy/config/default-config.yaml +++ b/java/google/registry/proxy/config/default-config.yaml @@ -170,7 +170,7 @@ whois: customQuota: [] healthCheck: - port: 11111 + port: 30000 # Health checker request message, defined in GCP load balancer backend. checkRequest: HEALTH_CHECK_REQUEST diff --git a/java/google/registry/proxy/Dockerfile b/java/google/registry/proxy/kubernetes/Dockerfile similarity index 89% rename from java/google/registry/proxy/Dockerfile rename to java/google/registry/proxy/kubernetes/Dockerfile index df3688e55..2d74bb1bb 100644 --- a/java/google/registry/proxy/Dockerfile +++ b/java/google/registry/proxy/kubernetes/Dockerfile @@ -20,5 +20,5 @@ COPY ./proxy_server_deploy.jar /proxy/ ENTRYPOINT ["java", "-jar", "proxy_server_deploy.jar"] -# Ports used for health checking, EPP and WHOIS, respecitvely. -EXPOSE 11111 22222 12345 +# Ports used for health checking, WHOIS and EPP, respecitvely. +EXPOSE 30000 30001 30002 diff --git a/java/google/registry/proxy/build_image.sh b/java/google/registry/proxy/kubernetes/build_image.sh similarity index 95% rename from java/google/registry/proxy/build_image.sh rename to java/google/registry/proxy/kubernetes/build_image.sh index d2c375b46..eb117c8be 100755 --- a/java/google/registry/proxy/build_image.sh +++ b/java/google/registry/proxy/kubernetes/build_image.sh @@ -25,6 +25,8 @@ trap cleanup EXIT PROJECT=`gcloud config list 2>&1 | grep project | awk -F'= ' '{print $2}'`; +echo "PROJECT: ${PROJECT}" + PACKAGE_PREFIX="" PACKAGE=${PACKAGE_PREFIX}"java/google/registry/proxy" @@ -35,7 +37,7 @@ BUILD_TOOL=bazel WORKSPACE=`$BUILD_TOOL info workspace` -WORKDIR=${WORKSPACE}/${PACKAGE} +WORKDIR=${WORKSPACE}/${PACKAGE}/kubernetes BINDIR=${WORKSPACE}/${BUILD_TOOL}-bin/${PACKAGE} diff --git a/java/google/registry/proxy/kubernetes/proxy-deployment.yaml b/java/google/registry/proxy/kubernetes/proxy-deployment.yaml new file mode 100644 index 000000000..651a323ea --- /dev/null +++ b/java/google/registry/proxy/kubernetes/proxy-deployment.yaml @@ -0,0 +1,47 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: proxy-deployment + labels: + app: proxy +spec: + replicas: 3 + selector: + matchLabels: + app: proxy + template: + metadata: + labels: + app: proxy + spec: + volumes: + - name: service-account + secret: + secretName: proxy-account + containers: + - name: proxy + image: INSERT_YOUR_IMAGE_NAME_HERE + ports: + - containerPort: 30000 + name: health-check + - containerPort: 30001 + name: whois + - containerPort: 30002 + name: epp + readinessProbe: + tcpSocket: + port: health-check + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + tcpSocket: + port: health-check + initialDelaySeconds: 15 + periodSeconds: 20 + volumeMounts: + - name: service-account + mountPath: /var/secrets/google + args: ["--log"] + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /var/secrets/google/service-account.json diff --git a/java/google/registry/proxy/kubernetes/proxy-service.yaml b/java/google/registry/proxy/kubernetes/proxy-service.yaml new file mode 100644 index 000000000..6af8af5ba --- /dev/null +++ b/java/google/registry/proxy/kubernetes/proxy-service.yaml @@ -0,0 +1,24 @@ +kind: Service +apiVersion: v1 +metadata: + name: proxy-service +spec: + selector: + app: proxy + ports: + - protocol: TCP + port: 30000 + nodePort: 30000 + targetPort: health-check + name: health-check + - protocol: TCP + port: 30001 + nodePort: 30001 + targetPort: whois + name: whois + - protocol: TCP + port: 30002 + nodePort: 30002 + targetPort: epp + name: epp + type: NodePort