vmpooler/vmware-host-pooler

#!/usr/bin/ruby

require 'rbvmomi'
require 'redis'
require 'yaml'

$:.unshift(File.dirname(__FILE__))
require 'lib/logger'
require 'lib/require_relative'
require 'lib/vsphere_helper'

$logger = Logger.new
$vsphere_helper = VsphereHelper.new

Dir.chdir(File.dirname(__FILE__))

# Load the configuration file
config_file = File.expand_path('vmware-host-pooler.yaml')
config = YAML.load_file(config_file)

pools   = config[:pools]
vsphere = config[:vsphere]

# Connect to Redis
$redis = Redis.new



# Clone a VM
def clone_vm template, pool, folder, datastore
  Thread.new {
    vm = {}

    if template =~ /\//
      templatefolders = template.split('/')
      vm['template'] = templatefolders.pop
    end

    if templatefolders
      vm[vm['template']] = $vsphere_helper.find_folder(templatefolders.join('/')).find(vm['template'])
    else
      raise "Please provide a full path to the template"
    end

    if vm['template'].length == 0
      raise "Unable to find template '#{vm['template']}'!"
    end

    # Generate a randomized hostname
    o = [('a'..'z'),('0'..'9')].map{|r| r.to_a}.flatten
    vm['hostname'] = o[rand(25)]+(0...14).map{o[rand(o.length)]}.join

    # Annotate with creation time, origin template, etc.
    configSpec = RbVmomi::VIM.VirtualMachineConfigSpec(
      :annotation =>
        'Base template: ' + vm['template'] + "\n" +
        'Creation time: ' + Time.now.strftime("%Y-%m-%d %H:%M")
    )

    # Put the VM in the specified folder and resource pool
    relocateSpec = RbVmomi::VIM.VirtualMachineRelocateSpec(
      :datastore    => $vsphere_helper.find_datastore(datastore),
      :pool         => $vsphere_helper.find_pool(pool),
      :diskMoveType => :moveChildMostDiskBacking
    )

    # Create a clone spec
    spec = RbVmomi::VIM.VirtualMachineCloneSpec(
      :location      => relocateSpec,
      :config        => configSpec,
      :powerOn       => true,
      :template      => false
    )

    # Clone the VM
    $logger.log('d', "[ ] '#{vm['hostname']}' is being cloned from '#{vm['template']}'")

    start = Time.now
    vm[vm['template']].CloneVM_Task(
      :folder => $vsphere_helper.find_folder(folder),
      :name => vm['hostname'],
      :spec => spec
    ).wait_for_completion
    finish = '%.2f' % (Time.now-start)

    # Add VM to Redis inventory ('pending' pool)
    $redis.sadd('vmware_host_pool__pending__'+vm['template'], vm['hostname'])

    $logger.log('s', "[+] '#{vm['hostname']}' cloned from '#{vm['template']}' in #{finish} seconds")

    # Metrics
    $redis.lpush('vmware_host_pool_metrics__deploy', finish)
    $redis.ltrim('vmware_host_pool_metrics__deploy', 0, 100)
  }
end

# Check the state of a VM
def check_vm vm, pool
  Thread.new {
    if (
      ($vsphere_helper.find_vms(vm)[vm]) and
      ($vsphere_helper.find_vms(vm)[vm].summary.guest.toolsRunningStatus == 'guestToolsRunning') and
      ($vsphere_helper.find_vms(vm)[vm].summary.guest.hostName) and
      ($vsphere_helper.find_vms(vm)[vm].summary.guest.hostName == vm) and
      ($vsphere_helper.find_vms(vm)[vm].summary.guest.ipAddress != nil)
    )
      begin
        Socket.getaddrinfo(vm, nil)
      rescue
        if (
          ($vsphere_helper.find_vms(vm)[vm].runtime) and
          ($vsphere_helper.find_vms(vm)[vm].runtime.bootTime) and
          (((( Time.now - $vsphere_helper.find_mvs(vm)[vm].runtime.bootTime ) / 60 ) / 60 ) >= 1)
        )
          $redis.srem('vmware_host_pool__pending__'+pool, vm)
          $redis.sadd('vmware_host_pool__failed__'+pool, vm)

          # Metrics
          $redis.lpush('vmware_host_pool_metrics__deploy_fail', '1')

          $logger.log('s', "[<] '#{vm}' moved to 'failed' queue")
        end

        next
      end

      $redis.sadd('vmware_host_pool__ready__'+pool, vm)
      $redis.srem('vmware_host_pool__pending__'+pool, vm)

      $logger.log('s', "[>] '#{vm}' moved to 'ready' queue")

      # Metrics
      $redis.lpush('vmware_host_pool_metrics__deploy_fail', '0')
    end
  }
end

# Destroy a VM
def destroy_vm vm
  Thread.new {
    host = $vsphere_helper.find_vms(vm)[vm]

    if (
      (host) and
      (host.runtime)
    )
      start = Time.now

      if host.runtime.powerState == 'poweredOn'
        $logger.log('d', "[ ] '#{vm}' is being shut down")
        host.PowerOffVM_Task.wait_for_completion
      end

      host.Destroy_Task.wait_for_completion
      finish = '%.2f' % (Time.now-start)

      $logger.log('s', "[-] '#{vm}' destroyed in #{finish} seconds")
    end
  }
end



# Update loop
loop do
  pools.each do |pool|
    inventory = {}
    total = 0

    # Locate the resource pool
    base = $vsphere_helper.find_pool(pool['pool'])

    # Make sure all VMs in resource pool are accounted-for
    base.vm.each do |vm|
      if (
        (! $redis.sismember('vmware_host_pool__pending__'+pool['name'], vm['name'])) and
        (! $redis.sismember('vmware_host_pool__ready__'+pool['name'], vm['name'])) and
        (! $redis.sismember('vmware_host_pool__running__'+pool['name'], vm['name'])) and
        (! $redis.sismember('vmware_host_pool__completed__'+pool['name'], vm['name'])) and
        (! $redis.sismember('vmware_host_pool__failed__'+pool['name'], vm['name']))
      )
        $redis.sadd('vmware_host_pool__pending__'+pool['name'], vm['name'])
      end

      inventory[vm['name']] = 1
      total = total+1
    end

    # Check 'pending' pool
    Thread.new {
      $redis.smembers('vmware_host_pool__pending__'+pool['name']).each do |vm|
        if ! inventory[vm]
          $redis.srem('vmware_host_pool__pending__'+pool['name'], vm)
        end

        Thread.new {
          check_vm(
            vm,
            pool['name']
          )
        }
      end
    }

    # Check 'ready' pool
    Thread.new {
      $redis.smembers('vmware_host_pool__ready__'+pool['name']).each do |vm|
        if ! inventory[vm]
          $redis.srem('vmware_host_pool__ready__'+pool['name'], vm)
        end

        if (
          (! $vsphere_helper.find_vms(vm)[vm]) or
          ($vsphere_helper.find_vms(vm)[vm].summary.guest.toolsRunningStatus != 'guestToolsRunning') or
          ($vsphere_helper.find_vms(vm)[vm].summary.guest.hostName != vm) or
          ($vsphere_helper.find_vms(vm)[vm].summary.guest.ipAddress == nil)
        )
          $redis.srem('vmware_host_pool__ready__'+pool['name'], vm)
          $redis.sadd('vmware_host_pool__failed__'+pool['name'], vm)

          $logger.log('s', "[<] '#{vm}' moved to 'failed' queue")

          next
        end

        begin
          Socket.getaddrinfo(vm, nil)
        rescue
          $redis.srem('vmware_host_pool__ready__'+pool['name'], vm)
          $redis.sadd('vmware_host_pool__failed__'+pool['name'], vm)

          $logger.log('s', "[<] '#{vm}' moved to 'failed' queue")

          # Metrics
          $redis.lpush('vmware_host_pool_metrics__deploy_fail', '1')
        end
      end
    }

    # Check 'completed' pool
    Thread.new {
      $redis.smembers('vmware_host_pool__completed__'+pool['name']).each do |vm|
        if ! inventory[vm]
          $redis.srem('vmware_host_pool__completed__'+pool['name'], vm)
        end

        Thread.new {
          destroy_vm(vm)
        }
      end
    }

    # Check 'failed' pool
    Thread.new {
      $redis.smembers('vmware_host_pool__failed__'+pool['name']).each do |vm|
        if ! inventory[vm]
          $redis.srem('vmware_host_pool__failed__'+pool['name'], vm)
        end

        if (
          ($vsphere_helper.find_vms(vm)[vm]) and
          ($vsphere_helper.find_vms(vm)[vm].summary.guest.toolsRunningStatus == 'guestToolsRunning') and
          ($vsphere_helper.find_vms(vm)[vm].summary.guest.hostName == vm) and
          ($vsphere_helper.find_vms(vm)[vm].summary.guest.ipAddress != nil)
        )
          begin
            Socket.getaddrinfo(vm, nil)
          rescue
            next
          end

          $redis.sadd('vmware_host_pool__ready__'+pool['name'], vm)
          $redis.srem('vmware_host_pool__failed__'+pool['name'], vm)

          $logger.log('s', "[>] '#{vm}' moved to 'ready' queue")
        else
          Thread.new {
            destroy_vm(vm)
          }
        end
      end
    }

    # Bring the pool up to the desired size
    Thread.new {
      if total < pool['size']
        # Provision VMs
        (1..(pool['size']-total)).each { |i|
          Thread.new {
            clone_vm(
              pool['template'],
              pool['pool'],
              pool['folder'],
              pool['datastore']
            )
          }
        }
      end
    }

    # Metrics
    $redis.ltrim('vmware_host_pool_metrics__deploy_fail', 0, 100)
  end
end