Merge pull request #29 from fayazg/gcm-cns

lb on cluster managers and affinity config

Author: Shubhra Kar

template_clustermanager.conf

@@ -10,6 +10,9 @@ name=global-cluster
 ## Should the cluster manager run in HA mode
 ##   Running in HA mode will start two node cluster manager with a database node
 ha=true
+## How many cluster manager nodes should this GCM set up (HA)
+gcm_node_count=2
+
 ## Comma separated names of triton networks to use
 ##   Cluster managers will have to be reachable by environment nodes
 networks=Joyent-SDC-Public,Joyent-SDC-Private

terraform/create-rancher-env.tf

@@ -137,14 +137,14 @@ module "gcp_example" {
   k8s_plane_isolation = "required"
 
   gcp_path_to_credentials = "/path/to/creds.json"
-  gcp_project_id = "k8s-test-187102"
+  gcp_project_id          = "k8s-test-187102"
 
   gcp_compute_region = "us-west1"
-  gcp_instance_zone = "us-west1-a"
+  gcp_instance_zone  = "us-west1-a"
 
-  etcd_gcp_instance_type = "n1-standard-1"
+  etcd_gcp_instance_type          = "n1-standard-1"
   orchestration_gcp_instance_type = "n1-standard-1"
-  compute_gcp_instance_type = "n1-standard-1"
+  compute_gcp_instance_type       = "n1-standard-1"
 
   # rancher_registry          = "docker-registry.joyent.com:5000"
   # rancher_registry_username = "username"

terraform/create-rancher.tf

@@ -21,6 +21,7 @@ module "create_rancher-example" {
   master_triton_machine_package  = "k4-highcpu-kvm-1.75G"
   mysqldb_triton_machine_package = "k4-highcpu-kvm-1.75G"
   ha                             = true
+  gcm_node_count                 = "2"
 
   # rancher_server_image      = "docker-registry.joyent.com:5000/rancher/server:v1.6.10"
   # rancher_agent_image       = "docker-registry.joyent.com:5000/rancher/agent:v1.2.6"

terraform/modules/triton-rancher-k8s/main.tf

@@ -109,6 +109,16 @@ resource "triton_machine" "etcd_host" {
   user_script = "${element(data.template_file.install_rancher_agent_etcd.*.rendered, count.index)}"
 
   networks = ["${data.triton_network.networks.*.id}"]
+
+  cns = {
+    services = ["etcd.${var.name}"]
+  }
+
+  affinity = ["role!=~etcd"]
+
+  tags = {
+    role = "etcd"
+  }
 }
 
 resource "rancher_registration_token" "orchestration" {
@@ -147,6 +157,16 @@ resource "triton_machine" "orchestration_host" {
   user_script = "${element(data.template_file.install_rancher_agent_orchestration.*.rendered, count.index)}"
 
   networks = ["${data.triton_network.networks.*.id}"]
+
+  cns = {
+    services = ["orchestration.${var.name}"]
+  }
+
+  affinity = ["role!=~orchestration"]
+
+  tags = {
+    role = "orchestration"
+  }
 }
 
 resource "rancher_registration_token" "compute" {

terraform/modules/triton-rancher/main.tf

@@ -48,7 +48,7 @@ resource "triton_machine" "rancher_mysqldb" {
   }
 
   provisioner "remote-exec" {
-    inline = <<EOF
+    inline = <<-EOF
       ${data.template_file.install_rancher_mysqldb.rendered}
       EOF
   }
@@ -79,7 +79,7 @@ data "template_file" "install_rancher_master" {
 }
 
 resource "triton_machine" "rancher_master" {
-  count = "${1 + var.ha}"
+  count = "${var.gcm_node_count}"
 
   package = "${var.master_triton_machine_package}"
   image   = "${data.triton_image.image.id}"
@@ -88,6 +88,16 @@ resource "triton_machine" "rancher_master" {
   user_script = "${data.template_file.install_rancher_master.rendered}"
 
   networks = ["${data.triton_network.networks.*.id}"]
+
+  cns = {
+    services = ["${var.name}"]
+  }
+
+  affinity = ["role!=~gcm"]
+
+  tags = {
+    role = "gcm"
+  }
 }
 
 data "template_file" "setup_rancher_k8s" {

terraform/modules/triton-rancher/variables.tf

@@ -75,6 +75,11 @@ variable "ha" {
   description = "Should Rancher be deployed in HA, if true a mysqldb node and 2 Rancher master nodes will be created."
 }
 
+variable "gcm_node_count" {
+  default     = "1"
+  description = "Number of Global Cluster Managers to cluster."
+}
+
 variable "mysqldb_triton_machine_package" {
   default     = ""
   description = "The Triton machine package to use for the Rancher mysqldb node. Defaults to master_triton_machine_package."

triton-kubernetes.sh

@@ -284,8 +284,13 @@ getClusterManagerConfig() {
     #	_mysqldb_triton_machine_package
 	#   _name
 	#   _ha
+	#   _gcm_node_count
 	_name="$(getArgument "Name your Global Cluster Manager" "global-cluster")"
 	_ha="$(getVerification "Do you want to set up the Global Cluster Manager in HA mode")"
+	if [ "$_ha" == "true" ]
+	then
+		_gcm_node_count="$(getArgument "Number of cluster manager nodes for $_name Global Cluster Manager" "2")"
+	fi
 	echo "From below options:"
 	echo "Joyent-SDC-Public"
 	echo "Joyent-SDC-Private"
@@ -625,7 +630,12 @@ readConfig() {
 		then
 			docker_engine_install_url=${line//docker_engine_install_url=/}
 			echo "docker-engine Install Script: $docker_engine_install_url"
+		elif echo "$line" | grep -q ^gcm_node_count=
+		then
+			_gcm_node_count=${line//gcm_node_count=/}
+			echo "Number of cluster manager Nodes: $_gcm_node_count"
 		fi
+
 		if [ "$_ha" == "true" ]
 		then
 			_etcd_node_count=3
@@ -653,7 +663,12 @@ setModuleClusterManager() {
 			rancher_registry_username_line="rancher_registry_username = \"${rancher_registry_username}\""
 			rancher_registry_password_line="rancher_registry_password = \"${rancher_registry_password}\""
 		fi
-		
+
+		if [ "$_ha" == "false" ]
+		then
+			_gcm_node_count="1"
+		fi
+
 		cd terraform
 		cat >>create-rancher.tf <<-EOF
 		
@@ -674,6 +689,7 @@ setModuleClusterManager() {
 		  master_triton_machine_package  = "${_master_triton_machine_package}"
 		  mysqldb_triton_machine_package = "${_mysqldb_triton_machine_package}"
 		  ha                             = ${_ha}
+		  gcm_node_count                 = ${_gcm_node_count}
 		  
 		  ${rancher_server_image_line}
 		  ${rancher_agent_image_line}
@@ -1027,8 +1043,17 @@ showClusterDetails() {
 	else
 		echo "This is a non-HA setup so there is only one cluster manager node."
 	fi
+
+	local __MasterCNS __name
+	__name=$_name
+	__MasterCNS=$(triton instance get "${__name}-master-1" 2>&1 | jq '.dns_names' 2>&1 | grep "${__name}.svc.*.triton.zone" | tr -d '"' | tr -d ' ' | tr -d ',' || true)
+	if [ "$__MasterCNS" ]
+	then
+		echo "    http://${__MasterCNS}:8080/"
+	else
+		echo "    http://${__MasterIP}:8080/"
+	fi
 	cat <<-EOM
-	    http://${__MasterIP}:8080/settings/env
 
 	Next step is adding Kubernetes environments to be managed here.
 	To start your first environment, run:
@@ -1047,13 +1072,32 @@ showEnvironmentDetails() {
 	else
 		echo "This is a non-HA setup so Kubernetes services could run on any of the compute nodes."
 	fi
+
+	local __MasterCNS __name
+	__name=$(triton ls -l 2>&1 | grep "$__MasterIP" | awk '{print $2}' 2>&1 | sed 's/-master-1//g' || true)
+	__MasterCNS=$(triton instance get "${__name}-master-1" 2>&1 | jq '.dns_names' 2>&1 | grep "${__name}.svc.*.triton.zone" | tr -d '"' | tr -d ' ' | tr -d ',' || true)
+	if [ "$__MasterCNS" ]
+	then
+		cat <<-EOM
+		Cluster Manager URL:
+		    http://${__MasterCNS}:8080/settings/env
+		Kubernetes Hosts URL:
+		    http://${__MasterCNS}:8080/env/$($TERRAFORM output -state=terraform/terraform.tfstate -module="${_name}" -json | jq '.environment_id.value' | tr -d '"')/infra/hosts?mode=dot
+		Kubernetes Health:
+		    http://${__MasterCNS}:8080/env/$($TERRAFORM output -state=terraform/terraform.tfstate -module="${_name}" -json | jq '.environment_id.value' | tr -d '"')/apps/stacks?which=cattle
+		EOM
+	else
+		cat <<-EOM
+		Cluster Manager URL:
+		    http://${__MasterIP}:8080/settings/env
+		Kubernetes Hosts URL:
+		    http://${__MasterIP}:8080/env/$($TERRAFORM output -state=terraform/terraform.tfstate -module="${_name}" -json | jq '.environment_id.value' | tr -d '"')/infra/hosts?mode=dot
+		Kubernetes Health:
+		    http://${__MasterIP}:8080/env/$($TERRAFORM output -state=terraform/terraform.tfstate -module="${_name}" -json | jq '.environment_id.value' | tr -d '"')/apps/stacks?which=cattle
+		EOM
+	fi
+
 	cat <<-EOM
-	Cluster Manager URL:
-	    http://${__MasterIP}:8080/settings/env
-	Kubernetes Hosts URL:
-	    http://${__MasterIP}:8080/env/$($TERRAFORM output -state=terraform/terraform.tfstate -module="${_name}" -json | jq '.environment_id.value' | tr -d '"')/infra/hosts?mode=dot
-	Kubernetes Health:
-	    http://${__MasterIP}:8080/env/$($TERRAFORM output -state=terraform/terraform.tfstate -module="${_name}" -json | jq '.environment_id.value' | tr -d '"')/apps/stacks?which=cattle
 	
 	NOTE: Nodes might take a few minutes to connect and come up.