measure-gcloud.rb

require 'bundler/inline'
gemfile do
  source 'https://rubygems.org'
  gem 'google-api-client', '~> 0.11'
end

require 'google/apis/compute_v1'

Google::Apis.logger = Logger.new(STDERR)
Google::Apis.logger.level = Logger::INFO

scopes = ['https://www.googleapis.com/auth/cloud-platform', 'https://www.googleapis.com/auth/compute']
auth = Google::Auth.get_application_default(scopes) # use system's service account

@compute_service = Google::Apis::ComputeV1::ComputeService.new
@compute_service.authorization = auth

def kubectl_get_nodes(context, filter)
  output = `kubectl --context #{context} get nodes -o json -l #{filter}`
  JSON.parse(output)
end

GCP_PROJECT = 'google-cloud-project-name'
GCP_ZONE = 'us-central1-a'

# kubectl context for the given cluster must be present
KUBECONTEXT = "gke_projectname_us-central1-a_kirs-autoscaling-cluster"

INSTANCE_GROUP_NAME = 'gke-kirs-autoscaling-clu-default-pool-adb41acc-grp'
NODEPOOL_NAME = 'default-pool'

new_nodes = 15

instance_group = @compute_service.get_instance_group_manager(GCP_PROJECT, GCP_ZONE, INSTANCE_GROUP_NAME )
instance_template_name = instance_group.instance_template.split("instanceTemplates/")[1]
instance_template = @compute_service.get_instance_template(GCP_PROJECT, instance_template_name)
puts "Machine type: #{instance_template.properties.machine_type}, adding new nodes = #{new_nodes}"

print "Confirm? (y/n): "
unless gets.strip == "y"
  puts ""
  exit
end

started_at = Time.now

# resize instance group to desired size
desired_size = instance_group.target_size + new_nodes
fields = { target_size: desired_size, fingerprint: instance_group.fingerprint }
@compute_service.patch_instance_group_manager(GCP_PROJECT, GCP_ZONE, instance_group.name, fields, {})

puts "=== RESIZED TO #{desired_size} nodes"

# watch nodes to become ready

loop do
  sleep 1
  instance_group = @compute_service.get_instance_group_manager(GCP_PROJECT, GCP_ZONE, instance_group.name)
  puts instance_group.current_actions.inspect

  instances = @compute_service.list_instances(GCP_PROJECT, GCP_ZONE)
  in_progress = instances.items.select { |i| i.status != "RUNNING" }
  break if in_progress.size.zero? && instance_group.target_size == desired_size && instance_group.current_actions.none == desired_size
  in_progress.each do |instance|
    puts [instance.name, instance.creation_timestamp, instance.status].join(" - ")
  end
end

puts "=== ALL VMS READY"
puts "vm time: #{Time.now - started_at}s"

puts "=== WAITING for kubelet"

vm_ready_at = Time.now

# watch K8S nodes to become ready

loop do
  sleep 1
  nodes = kubectl_get_nodes(KUBECONTEXT, "cloud.google.com/gke-nodepool=#{NODEPOOL_NAME}")
  actual_nodes = nodes.fetch('items').size
  if actual_nodes != desired_size
    puts "#{actual_nodes}/#{desired_size} nodes appeared"
    next
  end

  ready_nodes = 0
  nonready_nodes = 0
  nodes.fetch('items').each do |node|
    kubelet_status = node["status"]["conditions"].find { |c| c["type"] == "Ready" }.fetch("status")
    if kubelet_status == "True"
      ready_nodes += 1
    else
      nonready_nodes += 1
    end
  end
  break if nonready_nodes == 0

  puts "Nodes ready: #{ready_nodes}, nodes NOT ready: #{nonready_nodes}"
end

puts "=== KUBE READY"

puts "Machine type: #{instance_template.properties.machine_type}, added nodes= #{new_nodes}"
puts "vm time: #{vm_ready_at - started_at}s"
puts "kube time: #{Time.now - vm_ready_at}"
puts "total elapsed: #{Time.now - started_at}s"