Reworked into a single namespaced Ruby application

lib/graphite.rb

@@ -1,18 +0,0 @@
-require 'rubygems' unless defined?(Gem)
-
-class Graphite
-  def initialize(
-    s = 'graphite'
-  )
-    @server = s
-  end
-
-  def log path, value
-    Thread.new {
-      socket = TCPSocket.new(@server, 2003)
-      socket.puts "#{path} #{value} #{Time.now.to_i}"
-      socket.close
-    }
-  end
-end
-

lib/logger.rb

@@ -1,20 +0,0 @@
-require 'rubygems' unless defined?(Gem)
-
-class Logger
-  def initialize(
-    f = '/var/log/vmpooler.log'
-  )
-    @file = f
-  end
-
-  def log level, string
-    time = Time.new
-    stamp = time.strftime('%Y-%m-%d %H:%M:%S')
-    puts "[#{stamp}] #{string}"
-
-    open(@file, 'a') do |f|
-      f.puts "[#{stamp}] #{string}"
-    end
-  end
-end
-

lib/require_relative.rb

@@ -1,21 +0,0 @@
-# require_relative was introduced in 1.9.2. This makes it
-# available to younger rubies.  It trys hard to not re-require
-# files.
-
-unless Kernel.respond_to?(:require_relative)
-  module Kernel
-    def require_relative(path)
-      desired_path = File.expand_path('../'+path.to_str, caller[0])
-      shortest = desired_path
-      $:.each do |path|
-        path += '/'
-        if desired_path.index(path) == 0
-          candidate = desired_path.sub(path, '')
-          shortest = candidate if candidate.size < shortest.size
-        end
-      end
-      require shortest
-    end
-  end
-end
-

lib/vmpooler.rb

@@ -0,0 +1,12 @@
+require 'rubygems' unless defined?(Gem)
+
+module Vmpooler
+  %w( api graphite logger pool_manager vsphere_helper ).each do |lib|
+    begin
+      require "vmpooler/#{lib}"
+    rescue LoadError
+      require File.expand_path(File.join(File.dirname(__FILE__), 'vmpooler', lib))
+    end
+  end
+end
+

lib/vmpooler/api.rb

@@ -0,0 +1,233 @@
+module Vmpooler
+  class API
+    def initialize
+      require 'sinatra/base'
+
+      require 'json'
+      require 'open-uri'
+      require 'redis'
+      require 'yaml'
+
+      # Load the configuration file
+      config_file = File.expand_path('vmpooler.yaml')
+      $config = YAML.load_file(config_file)
+
+      pools = $config[:pools]
+      redis = $config[:redis]
+
+      # Set some defaults
+      $config[:redis] ||= Hash.new
+      $config[:redis]['server'] ||= 'localhost'
+
+      # Connect to Redis
+      $redis = Redis.new(:host => $config[:redis]['server'])
+    end
+
+    def execute!
+      my_app = Sinatra.new {
+
+        set :environment, 'production'
+
+        get '/' do
+          erb :dashboard, locals: {
+            site_name: $config[:config]['site_name'] || '<b>vmpooler</b>',
+          }
+        end
+
+        get '/dashboard/stats/vmpooler/numbers/?' do
+          result = Hash.new
+          result['pending'] = 0
+          result['cloning'] = 0
+          result['booting'] = 0
+          result['ready'] = 0
+          result['running'] = 0
+          result['completed'] = 0
+
+          $config[:pools].each do |pool|
+            result['pending'] += $redis.scard( 'vmpooler__pending__' + pool['name'] )
+            result['ready'] += $redis.scard( 'vmpooler__ready__' + pool['name'] )
+            result['running'] += $redis.scard( 'vmpooler__running__' + pool['name'] )
+            result['completed'] += $redis.scard( 'vmpooler__completed__' + pool['name'] )
+          end
+
+          result['cloning'] = $redis.get( 'vmpooler__tasks__clone' )
+          result['booting'] = result['pending'].to_i - result['cloning'].to_i
+          result['booting'] = 0 if result['booting'] < 0
+          result['total'] = result['pending'].to_i + result['ready'].to_i + result['running'].to_i + result['completed'].to_i
+
+          content_type :json
+          JSON.pretty_generate(result)
+        end
+
+        get '/dashboard/stats/vmpooler/pool/?' do
+          result = Hash.new
+
+          $config[:pools].each do |pool|
+            result[pool['name']] ||= Hash.new
+            result[pool['name']]['size'] = pool['size']
+            result[pool['name']]['ready'] = $redis.scard( 'vmpooler__ready__' + pool['name'] )
+          end
+
+          if ( params[:history] )
+            if ( $config[:config]['graphite'] )
+              history ||= Hash.new
+
+              begin
+                buffer = open( 'http://'+$config[:config]['graphite']+'/render?target=vmpooler.ready.*&from=-1hour&format=json' ).read
+                history = JSON.parse( buffer )
+
+                history.each do |pool|
+                  if pool['target'] =~ /.*\.(.*)$/
+                    pool['name'] = $1
+
+                    if ( result[pool['name']] )
+                      pool['last'] = result[pool['name']]['size']
+                      result[pool['name']]['history'] ||= Array.new
+
+                      pool['datapoints'].each do |metric|
+                        8.times do |n|
+                          if ( metric[0] )
+                            pool['last'] = metric[0].to_i
+                            result[pool['name']]['history'].push( metric[0].to_i )
+                          else
+                            result[pool['name']]['history'].push( pool['last'] )
+                          end
+                        end
+                      end
+                    end
+                  end
+                end
+              rescue
+              end
+            else
+              $config[:pools].each do |pool|
+                result[pool['name']] ||= Hash.new
+                result[pool['name']]['history'] = [ $redis.scard( 'vmpooler__ready__' + pool['name'] ) ]
+              end
+            end
+          end
+
+          content_type :json
+          JSON.pretty_generate(result)
+        end
+
+        get '/dashboard/stats/vmpooler/running/?' do
+          result = Hash.new
+
+          $config[:pools].each do |pool|
+            running = $redis.scard( 'vmpooler__running__' + pool['name'] )
+            pool['major'] = $1 if pool['name'] =~ /^(\w+)\-/
+
+            result[pool['major']] ||= Hash.new
+
+            result[pool['major']]['running'] = result[pool['major']]['running'].to_i + running.to_i
+          end
+
+          if ( params[:history] )
+            if ( $config[:config]['graphite'] )
+              begin
+                buffer = open( 'http://'+$config[:config]['graphite']+'/render?target=vmpooler.running.*&from=-1hour&format=json' ).read
+                JSON.parse( buffer ).each do |pool|
+                  if pool['target'] =~ /.*\.(.*)$/
+                    pool['name'] = $1
+
+                    pool['major'] = $1 if pool['name'] =~ /^(\w+)\-/
+
+                    result[pool['major']]['history'] ||= Array.new
+
+                    for i in 0..pool['datapoints'].length
+                      if (
+                        pool['datapoints'][i] and
+                        pool['datapoints'][i][0]
+                      )
+                        pool['last'] = pool['datapoints'][i][0]
+
+                        result[pool['major']]['history'][i] ||= 0
+                        result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['datapoints'][i][0].to_i
+                      else
+                        result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['last'].to_i
+                      end
+                    end
+
+                  end
+                end
+              rescue
+              end
+            end
+          end
+
+          content_type :json
+          JSON.pretty_generate(result)
+        end
+
+        get '/vm/?' do
+          content_type :json
+
+          result = []
+
+          $config[:pools].each do |pool|
+            result.push(pool['name'])
+          end
+
+          JSON.pretty_generate(result)
+        end
+
+        get '/vm/:template/?' do
+          content_type :json
+
+          result = {}
+          result[params[:template]] = {}
+          result[params[:template]]['hosts'] = $redis.smembers('vmpooler__ready__'+params[:template])
+
+          JSON.pretty_generate(result)
+        end
+
+        post '/vm/:template/?' do
+          content_type :json
+
+          result = {}
+          result[params[:template]] = {}
+
+          if ( $redis.scard('vmpooler__ready__'+params[:template]) > 0 )
+            vm = $redis.spop('vmpooler__ready__'+params[:template])
+
+            unless (vm.nil?)
+              $redis.sadd('vmpooler__running__'+params[:template], vm)
+              $redis.hset('vmpooler__active__'+params[:template], vm, Time.now)
+
+              result[params[:template]]['ok'] = true
+              result[params[:template]]['hostname'] = vm
+            else
+              result[params[:template]]['ok'] = false
+            end
+          else
+            result[params[:template]]['ok'] = false
+          end
+
+          JSON.pretty_generate(result)
+        end
+
+        delete '/vm/:hostname/?' do
+          content_type :json
+
+          result = {}
+
+          result['ok'] = false
+
+          $config[:pools].each do |pool|
+            if $redis.sismember('vmpooler__running__'+pool['name'], params[:hostname])
+              $redis.srem('vmpooler__running__'+pool['name'], params[:hostname])
+              $redis.sadd('vmpooler__completed__'+pool['name'], params[:hostname])
+              result['ok'] = true
+            end
+          end
+
+          JSON.pretty_generate(result)
+        end
+      }
+
+      my_app.run!
+    end
+  end
+end
+

lib/vmpooler/graphite.rb

@@ -0,0 +1,20 @@
+require 'rubygems' unless defined?(Gem)
+
+module Vmpooler
+  class Graphite
+    def initialize(
+      s = 'graphite'
+    )
+      @server = s
+    end
+
+    def log path, value
+      Thread.new {
+        socket = TCPSocket.new(@server, 2003)
+        socket.puts "#{path} #{value} #{Time.now.to_i}"
+        socket.close
+      }
+    end
+  end
+end
+

lib/vmpooler/logger.rb

@@ -0,0 +1,21 @@
+require 'rubygems' unless defined?(Gem)
+
+module Vmpooler
+  class Logger
+    def initialize(
+      f = '/var/log/vmpooler.log'
+    )
+      @file = f
+    end
+
+    def log level, string
+      time = Time.new
+      stamp = time.strftime('%Y-%m-%d %H:%M:%S')
+
+      open(@file, 'a') do |f|
+        f.puts "[#{stamp}] #{string}"
+      end
+    end
+  end
+end
+

lib/vmpooler/pool_manager.rb

@@ -0,0 +1,459 @@
+module Vmpooler
+  class PoolManager
+    def initialize
+      require 'json'
+      require 'rbvmomi'
+      require 'redis'
+      require 'time'
+      require 'timeout'
+      require 'yaml'
+
+      # Load the configuration file
+      config_file = File.expand_path('vmpooler.yaml')
+      $config = YAML.load_file(config_file)
+
+      $pools   = $config[:pools]
+      vsphere = $config[:vsphere]
+      redis   = $config[:redis]
+
+      # Load logger library
+      $logger = Vmpooler::Logger.new $config[:config]['logfile']
+
+      # Load Graphite helper library (if configured)
+      if (defined? $config[:config]['graphite'])
+        $graphite = Vmpooler::Graphite.new $config[:config]['graphite']
+      end
+
+      # Set some defaults
+      $config[:config]['task_limit']   ||= 10
+      $config[:config]['vm_checktime'] ||= 15
+      $config[:config]['vm_lifetime']  ||= 24
+      $config[:redis] ||= Hash.new
+      $config[:redis]['server'] ||= 'localhost'
+
+      # Connect to Redis
+      $redis = Redis.new(:host => $config[:redis]['server'])
+
+      # vSphere object
+      $vsphere = {}
+
+      # Our thread-tracker object
+      $threads = {}
+    end
+
+
+    # Check the state of a VM
+    def check_pending_vm vm, pool, timeout
+      Thread.new {
+        host = $vsphere[pool].find_vm(vm)
+
+        if (host)
+          if (
+            (host.summary) and
+            (host.summary.guest) and
+            (host.summary.guest.hostName) and
+            (host.summary.guest.hostName == vm)
+          )
+            begin
+              Socket.getaddrinfo(vm, nil)
+            rescue
+            end
+
+            $redis.smove('vmpooler__pending__'+pool, 'vmpooler__ready__'+pool, vm)
+
+            $logger.log('s', "[>] [#{pool}] '#{vm}' moved to 'ready' queue")
+          end
+        else
+          clone_stamp = $redis.hget('vmpooler__vm__'+vm, 'clone')
+
+          if (
+            (clone_stamp) and
+            (((Time.now - Time.parse(clone_stamp))/60) > timeout)
+          )
+            $redis.smove('vmpooler__pending__'+pool, 'vmpooler__completed__'+pool, vm)
+
+            $logger.log('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes")
+          end
+        end
+      }
+    end
+
+    def check_ready_vm vm, pool, ttl
+      Thread.new {
+        if (ttl > 0)
+          if ((((Time.now - host.runtime.bootTime)/60).to_s[/^\d+\.\d{1}/].to_f) > ttl)
+            $redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm)
+
+            $logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue")
+          end
+        end
+
+        check_stamp = $redis.hget('vmpooler__vm__'+vm, 'check')
+
+        if (
+          (! check_stamp) or
+          (((Time.now - Time.parse(check_stamp))/60) > $config[:config]['vm_checktime'])
+        )
+          $redis.hset('vmpooler__vm__'+vm, 'check', Time.now)
+
+          host = $vsphere[pool].find_vm(vm) ||
+                 $vsphere[pool].find_vm_heavy(vm)[vm]
+
+          if (host)
+            if (
+              (host.runtime) and
+              (host.runtime.powerState) and
+              (host.runtime.powerState != 'poweredOn')
+            )
+              $redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm)
+
+              $logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off, removed from 'ready' queue")
+            end
+
+            if (
+              (host.summary.guest) and
+              (host.summary.guest.hostName) and
+              (host.summary.guest.hostName != vm)
+            )
+              $redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm)
+
+              $logger.log('d', "[!] [#{pool}] '#{vm}' has mismatched hostname, removed from 'ready' queue")
+            end
+          else
+            $redis.srem('vmpooler__ready__'+pool, vm)
+
+            $logger.log('s', "[!] [#{pool}] '#{vm}' not found in vCenter inventory, removed from 'ready' queue")
+          end
+
+          begin
+            Timeout::timeout(5) {
+              TCPSocket.new vm, 22
+            }
+          rescue
+            if ($redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm))
+              $logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue")
+            end
+          end
+        end
+      }
+    end
+
+    def check_running_vm vm, pool, ttl
+      Thread.new {
+        host = $vsphere[pool].find_vm(vm)
+
+        if (host)
+          if (
+            (host.runtime) and
+            (host.runtime.powerState != 'poweredOn')
+          )
+            $redis.smove('vmpooler__running__'+pool, 'vmpooler__completed__'+pool, vm)
+
+            $logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off or dead")
+          else
+            if (
+              (host.runtime) and
+              (host.runtime.bootTime)
+              ((((Time.now - host.runtime.bootTime)/60).to_s[/^\d+\.\d{1}/].to_f) > ttl)
+            )
+              $redis.smove('vmpooler__running__'+pool, 'vmpooler__completed__'+pool, vm)
+
+              $logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes")
+            end
+          end
+        end
+      }
+    end
+
+    # Clone a VM
+    def clone_vm template, pool, folder, datastore
+      Thread.new {
+        vm = {}
+
+        if template =~ /\//
+          templatefolders = template.split('/')
+          vm['template'] = templatefolders.pop
+        end
+
+        if templatefolders
+          vm[vm['template']] = $vsphere[vm['template']].find_folder(templatefolders.join('/')).find(vm['template'])
+        else
+          raise "Please provide a full path to the template"
+        end
+
+        if vm['template'].length == 0
+          raise "Unable to find template '#{vm['template']}'!"
+        end
+
+        # Generate a randomized hostname
+        o = [('a'..'z'),('0'..'9')].map{|r| r.to_a}.flatten
+        vm['hostname'] = o[rand(25)]+(0...14).map{o[rand(o.length)]}.join
+
+        # Add VM to Redis inventory ('pending' pool)
+        $redis.sadd('vmpooler__pending__'+vm['template'], vm['hostname'])
+        $redis.hset('vmpooler__vm__'+vm['hostname'], 'clone', Time.now)
+
+        # Annotate with creation time, origin template, etc.
+        configSpec = RbVmomi::VIM.VirtualMachineConfigSpec(
+          :annotation => JSON.pretty_generate({
+              name: vm['hostname'],
+              created_by: $config[:vsphere]['username'],
+              base_template: vm['template'],
+              creation_timestamp: Time.now.utc
+           })
+        )
+
+        # Put the VM in the specified folder and resource pool
+        relocateSpec = RbVmomi::VIM.VirtualMachineRelocateSpec(
+          :datastore    => $vsphere[vm['template']].find_datastore(datastore),
+          :pool         => $vsphere[vm['template']].find_pool(pool),
+          :diskMoveType => :moveChildMostDiskBacking
+        )
+
+        # Create a clone spec
+        spec = RbVmomi::VIM.VirtualMachineCloneSpec(
+          :location      => relocateSpec,
+          :config        => configSpec,
+          :powerOn       => true,
+          :template      => false
+        )
+
+        # Clone the VM
+        $logger.log('d', "[ ] [#{vm['template']}] '#{vm['hostname']}' is being cloned from '#{vm['template']}'")
+
+        begin
+          start = Time.now
+          vm[vm['template']].CloneVM_Task(
+            :folder => $vsphere[vm['template']].find_folder(folder),
+            :name => vm['hostname'],
+            :spec => spec
+          ).wait_for_completion
+          finish = '%.2f' % (Time.now-start)
+
+          $logger.log('s', "[+] [#{vm['template']}] '#{vm['hostname']}' cloned from '#{vm['template']}' in #{finish} seconds")
+        rescue
+          $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' clone appears to have failed")
+          $redis.srem('vmpooler__pending__'+vm['template'], vm['hostname'])
+        end
+
+        $redis.decr('vmpooler__tasks__clone')
+
+        begin
+          $graphite.log("vmpooler.clone.#{vm['template']}", finish) if defined? $graphite
+        rescue
+        end
+      }
+    end
+
+    # Destroy a VM
+    def destroy_vm vm, pool
+      Thread.new {
+        $redis.srem('vmpooler__completed__'+pool, vm)
+        $redis.hdel('vmpooler__active__'+pool, vm)
+        $redis.del('vmpooler__vm__'+vm)
+
+        host = $vsphere[pool].find_vm(vm) ||
+               $vsphere[pool].find_vm_heavy(vm)[vm]
+
+        if (host)
+          start = Time.now
+
+          if (
+            (host.runtime) and
+            (host.runtime.powerState) and
+            (host.runtime.powerState == 'poweredOn')
+          )
+            $logger.log('d', "[ ] [#{pool}] '#{vm}' is being shut down")
+            host.PowerOffVM_Task.wait_for_completion
+          end
+
+          host.Destroy_Task.wait_for_completion
+          finish = '%.2f' % (Time.now-start)
+
+          $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds")
+
+          $graphite.log("vmpooler.destroy.#{pool}", finish) if defined? $graphite
+        end
+      }
+    end
+
+    def check_pool pool
+      $logger.log('d', "[*] [#{pool['name']}] starting worker thread")
+
+      $threads[pool['name']] = Thread.new {
+        $vsphere[pool['name']] ||= Vmpooler::VsphereHelper.new
+
+        loop do
+          # INVENTORY
+          inventory = {}
+          begin
+            base = $vsphere[pool['name']].find_pool(pool['pool'])
+
+            base.vm.each do |vm|
+              if (
+                (! $redis.sismember('vmpooler__running__'+pool['name'], vm['name'])) and
+                (! $redis.sismember('vmpooler__ready__'+pool['name'], vm['name'])) and
+                (! $redis.sismember('vmpooler__pending__'+pool['name'], vm['name'])) and
+                (! $redis.sismember('vmpooler__completed__'+pool['name'], vm['name'])) and
+                (! $redis.sismember('vmpooler__discovered__'+pool['name'], vm['name']))
+              )
+                $redis.sadd('vmpooler__discovered__'+pool['name'], vm['name'])
+
+                $logger.log('s', "[?] [#{pool['name']}] '#{vm['name']}' added to 'discovered' queue")
+              end
+
+              inventory[vm['name']] = 1
+            end
+          rescue
+          end
+
+          # RUNNING
+          $redis.smembers('vmpooler__running__'+pool['name']).each do |vm|
+            if (inventory[vm])
+              if (pool['running_ttl'])
+                begin
+                  check_running_vm(vm, pool['name'], pool['running_ttl'])
+                rescue
+                end
+              else
+                begin
+                  check_running_vm(vm, pool['name'], '720')
+                rescue
+                end
+              end
+            end
+          end
+
+          # READY
+          $redis.smembers('vmpooler__ready__'+pool['name']).each do |vm|
+            if (inventory[vm])
+              begin
+                check_ready_vm(vm, pool['name'], pool['ready_ttl'] || 0)
+              rescue
+              end
+            end
+          end
+
+          # PENDING
+          $redis.smembers('vmpooler__pending__'+pool['name']).each do |vm|
+            pool['timeout'] ||= 15
+
+            if (inventory[vm])
+              begin
+                check_pending_vm(vm, pool['name'], pool['timeout'])
+              rescue
+              end
+            end
+          end
+
+          # COMPLETED
+          $redis.smembers('vmpooler__completed__'+pool['name']).each do |vm|
+            if (inventory[vm])
+              begin
+                destroy_vm(vm, pool['name'])
+              rescue
+                $logger.log('s', "[!] [#{pool['name']}] '#{vm}' destroy appears to have failed")
+                $redis.srem('vmpooler__completed__'+pool['name'], vm)
+                $redis.hdel('vmpooler__active__'+pool['name'], vm)
+                $redis.del('vmpooler__vm__'+vm)
+              end
+            else
+              $logger.log('s', "[!] [#{pool['name']}] '#{vm}' not found in inventory, removed from 'completed' queue")
+              $redis.srem('vmpooler__completed__'+pool['name'], vm)
+              $redis.hdel('vmpooler__active__'+pool['name'], vm)
+              $redis.del('vmpooler__vm__'+vm)
+            end
+          end
+
+          # DISCOVERED
+          $redis.smembers('vmpooler__discovered__'+pool['name']).each do |vm|
+            ['pending', 'ready', 'running', 'completed'].each do |queue|
+              if ($redis.sismember('vmpooler__'+queue+'__'+pool['name'], vm))
+                $logger.log('d', "[!] [#{pool['name']}] '#{vm}' found in '#{queue}', removed from 'discovered' queue")
+                $redis.srem('vmpooler__discovered__'+pool['name'], vm)
+              end
+            end
+
+            if ($redis.sismember('vmpooler__discovered__'+pool['name'], vm))
+              $redis.smove('vmpooler__discovered__'+pool['name'], 'vmpooler__completed__'+pool['name'], vm)
+            end
+          end
+
+          # LONG-RUNNING
+          $redis.smembers('vmpooler__running__'+pool['name']).each do |vm|
+            if ($redis.hget('vmpooler__active__'+pool['name'], vm))
+              running = (Time.now - Time.parse($redis.hget('vmpooler__active__'+pool['name'], vm)))/60/60
+              if (
+                ($config[:config]['vm_lifetime'] > 0) and
+                (running > $config[:config]['vm_lifetime'])
+              )
+                $redis.smove('vmpooler__running__'+pool['name'], 'vmpooler__completed__'+pool['name'], vm)
+
+                $logger.log('d', "[!] [#{pool['name']}] '#{vm}' reached end of TTL after #{$config[:config]['vm_lifetime']} hours")
+              end
+            end
+          end
+
+          # REPOPULATE
+          total = $redis.scard('vmpooler__ready__'+pool['name']) +
+                  $redis.scard('vmpooler__pending__'+pool['name'])
+
+          begin
+            if (defined? $graphite)
+              $graphite.log('vmpooler.ready.'+pool['name'], $redis.scard('vmpooler__ready__'+pool['name']))
+              $graphite.log('vmpooler.running.'+pool['name'], $redis.scard('vmpooler__running__'+pool['name']))
+            end
+          rescue
+          end
+
+          if (total < pool['size'])
+            (1..(pool['size'] - total)).each { |i|
+
+              if ($redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit'])
+                begin
+                  $redis.incr('vmpooler__tasks__clone')
+
+                  clone_vm(
+                    pool['template'],
+                    pool['pool'],
+                    pool['folder'],
+                    pool['datastore']
+                  )
+                rescue
+                  $logger.log('s', "[!] [#{pool['name']}] clone appears to have failed")
+                  $redis.decr('vmpooler__tasks__clone')
+                end
+              end
+            }
+          end
+
+          sleep(1)
+        end
+      }
+    end
+
+    def execute!
+      $logger.log('d', "starting vmpooler")
+
+      # Clear out the tasks manager, as we don't know about any tasks at this point
+      $redis.set('vmpooler__tasks__clone', 0)
+
+      loop do
+        $pools.each do |pool|
+          if (! $threads[pool['name']])
+            check_pool(pool)
+          else
+            if (! $threads[pool['name']].alive?)
+              $logger.log('d', "[!] [#{pool['name']}] worker thread died, restarting")
+              check_pool(pool)
+            end
+          end
+        end
+
+        sleep(1)
+      end
+    end
+
+  end
+end
+

lib/vmpooler/public/dashboard.css

@@ -0,0 +1,144 @@
+body,
+#content {
+  background: #fff;
+  margin-top: 0px;
+  margin-left: auto;
+  margin-right: auto;
+}
+
+#wrap {
+  margin-left: 1%;
+  margin-right: 1%;
+}
+
+#header {
+  border-bottom: solid 1px #888;
+  margin-bottom: 20px;
+}
+
+#header .logo,
+#header .text {
+  margin-left: 30px;
+  margin-bottom: 5px;
+  display: inline-block;
+}
+
+#header .text {
+  float: right;
+  position: relative;
+  right: 30px;
+  top: 90px;
+  font: 50px 'PT Sans', sans-serif;
+  letter-spacing: -0.05em;
+  color: #444;
+}
+
+.label {
+  text-transform: uppercase;
+  text-indent: 25px;
+  font: 12px 'PT Sans', sans-serif;
+  font-weight: bold;
+  line-height: 20px;
+  color: #888;
+  border-bottom: solid 1px #888;
+}
+
+.spinner {
+  background-image: url( '/img/spinner.svg' );
+  width: 15px;
+  height: 15px;
+  background-size: contain;
+  opacity: .7;
+  float: left;
+  margin-right: 5px;
+  -webkit-animation:rotate 1.5s infinite steps( 12 );
+  -moz-animation:rotate 1.5s infinite steps( 12 );
+  -ms-animation:rotate 1.5s infinite steps( 12 );
+  -o-animation:rotate 1.5s infinite steps( 12 );
+  animation:rotate 1.5s infinite steps( 12 );  
+}
+@keyframes 'rotate'  {
+  from {
+    -webkit-transform: rotate( 0deg );
+    -moz-transform: rotate( 0deg );
+    -o-transform: rotate( 0deg );
+    -ms-transform: rotate( 0deg );
+    transform: rotate( 0deg );
+  }
+  to {
+    -webkit-transform: rotate( 360deg );
+    -moz-transform: rotate( 360deg );
+    -o-transform: rotate( 360deg );
+    -ms-transform: rotate( 360deg );
+    transform: rotate( 360deg );
+  }
+}
+@-moz-keyframes rotate {
+  from {
+    -moz-transform: rotate( 0deg );
+    transform: rotate( 0deg );
+  }
+  to {
+    -moz-transform: rotate( 360deg );
+    transform: rotate( 360deg );
+  }
+}
+@-webkit-keyframes 'rotate' {
+  from {
+    -webkit-transform: rotate( 0deg );
+    transform: rotate( 0deg );
+  }
+  to {
+    -webkit-transform: rotate( 360deg );
+    transform: rotate( 360deg );
+  }
+}
+@-ms-keyframes 'rotate' {
+  from {
+    -ms-transform: rotate( 0deg );
+    transform: rotate( 0deg );
+  }
+  to {
+    -ms-transform: rotate( 360deg );
+    transform: rotate( 360deg );
+  }
+}
+@-o-keyframes 'rotate' {
+  from {
+    -o-transform: rotate( 0deg );
+    transform: rotate( 0deg );
+  }
+  to {
+    -o-transform: rotate( 360deg );
+    transform: rotate( 360deg );
+  }
+}
+
+.module {
+  margin-top: 5px;
+  margin-left: 25px;
+  margin-right: 25px;
+  font: 13px 'PT Sans', sans-serif;
+  line-height: 20px;
+  color: #888;
+}
+
+.line {
+  fill: none;
+}
+
+.area {
+  stroke-width: 0;
+}
+
+.axis path,
+.axis line {
+  fill: none;
+  stroke: #fff;
+  shape-rendering: crispEdges;
+}
+
+.tick {
+  fill: none;
+  stroke: #eee;
+}

lib/vmpooler/public/img/spinner.svg

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+   width="128px" height="128px" viewBox="0 0 128 128" enable-background="new 0 0 128 128" xml:space="preserve">
+<g id="_x31_6x16_Spinner.psd" display="none">
+  <defs>
+    <rect id="SVGID_1_" width="128" height="128"/>
+  </defs>
+  <clipPath id="SVGID_2_" display="inline">
+    <use xlink:href="#SVGID_1_"  overflow="visible"/>
+  </clipPath>
+  <g display="inline" clip-path="url(#SVGID_2_)">
+
+      <image overflow="visible" width="16" height="16" id="Layer_1_1_" xlink:href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAACXBIWXMAAAFiAAABYgFfJ9BTAAAA
+GXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAWVJREFUeNqM098rQ3EYx/Fzdhgh
+bcWW/GiKxKK2SKTcrP0PuFP+ABf+AP+Cf8OVe5dquMEVlgsZkhiFIys/3t/6HH2ds+GpV+1s5zzP
+s+f5HseJRsz6XMQmPOef0YMNZHW9ghslaFbCoUbVTHxgFmu6bkUN70qawrP9gGdVbsMtrrCMYwwg
+hx2M4RRlddMF31MXq1jCo27Oq5ttXffjBbvoxhxGcOaqgwwWMKUK60rwZnUaU+JhVHGkbn/EJBbr
+zCaIrCp/h+mggDQ+NawSLhokaMc44nrWb9IwUkpQ+2PnprNOtChBpNNeTOjHepHRPZE1mn1PY0br
+LGuI4cirgOniwXTsqlpR6zG7P9H/fMK5iiR1AktaaU7z2vKsYQaHJKl1VtCHUVyrsq8CFZ3IuyBB
+VV+Y6c7jFXsYRAL76FCySx24ezP48BQTmvCBthIT0+GhEqd/extdDdQeWsHaSjy85i8BBgAjbUmT
+DlLqrAAAAABJRU5ErkJggg==" transform="matrix(8 0 0 8 0 0)">
+    </image>
+  </g>
+</g>
+<line fill="none" stroke="#363636" stroke-width="12" stroke-linecap="round" stroke-miterlimit="10" x1="64.001" y1="8.125" x2="64.001" y2="27.762"/>
+<line fill="none" stroke="#484848" stroke-width="12" stroke-linecap="round" stroke-miterlimit="10" x1="36.062" y1="15.611" x2="45.88" y2="32.618"/>
+<line fill="none" stroke="#5A5A5A" stroke-width="12" stroke-linecap="round" stroke-miterlimit="10" x1="15.611" y1="36.062" x2="32.618" y2="45.882"/>
+<line fill="none" stroke="#6C6C6C" stroke-width="12" stroke-linecap="round" stroke-miterlimit="10" x1="8.126" y1="64" x2="27.764" y2="64"/>
+<line fill="none" stroke="#7E7E7E" stroke-width="12" stroke-linecap="round" stroke-miterlimit="10" x1="15.612" y1="91.938" x2="32.619" y2="82.117"/>
+<line fill="none" stroke="#909090" stroke-width="12" stroke-linecap="round" stroke-miterlimit="10" x1="36.064" y1="112.389" x2="45.883" y2="95.381"/>
+<line fill="none" stroke="#A2A2A2" stroke-width="12" stroke-linecap="round" stroke-miterlimit="10" x1="64.001" y1="119.873" x2="64.001" y2="100.237"/>
+<line fill="none" stroke="#B4B4B4" stroke-width="12" stroke-linecap="round" stroke-miterlimit="10" x1="91.939" y1="112.387" x2="82.121" y2="95.381"/>
+<line fill="none" stroke="#C6C6C6" stroke-width="12" stroke-linecap="round" stroke-miterlimit="10" x1="112.391" y1="91.935" x2="95.386" y2="82.116"/>
+<line fill="none" stroke="#000000" stroke-width="12" stroke-linecap="round" stroke-miterlimit="10" x1="119.875" y1="63.997" x2="100.241" y2="63.998"/>
+<line fill="none" stroke="#121212" stroke-width="12" stroke-linecap="round" stroke-miterlimit="10" x1="112.389" y1="36.06" x2="95.384" y2="45.877"/>
+<line fill="none" stroke="#242424" stroke-width="12" stroke-linecap="round" stroke-miterlimit="10" x1="91.937" y1="15.609" x2="82.118" y2="32.614"/>
+</svg>

lib/vmpooler/public/lib/stats-vmpooler-numbers.js

@@ -0,0 +1,261 @@
+var numbers_url = '/dashboard/stats/vmpooler/numbers';
+var numbers_width = 130;
+var numbers_height = 50;
+
+var stats_vmpooler_numbers__data  = {};
+var stats_vmpooler_numbers__svg   = {};
+
+d3.json( numbers_url,
+
+  function( stats_vmpooler_numbers__data ) {
+
+    ( function tick() {
+      setTimeout( function() {
+        var stats_vmpooler_numbers__data__live = ( function() {
+          var stats_vmpooler_numbers__data__live = null;
+
+          $.ajax( {
+            'url': numbers_url,
+            'async': false,
+            'global': false,
+            'dataType': 'json',
+            'success': function( data ) {
+              stats_vmpooler_numbers__data__live = data;
+            }
+          } );
+
+          return stats_vmpooler_numbers__data__live;
+        } )();
+
+        $( '#stats-vmpooler-numbers' ).empty();
+
+        stats_vmpooler_numbers__svg[ 'total' ] = d3.select( '#stats-vmpooler-numbers' )
+          .append( 'svg' )
+            .style( 'margin', '15px 0px 0px 0px' )
+            .style( 'padding', '0px 10px 20px 10px' )
+            .attr( 'width', numbers_width + 'px' )
+            .attr( 'height', numbers_height + 'px' );
+
+        stats_vmpooler_numbers__svg[ 'total' ]
+          .append( 'text' )
+            .text(
+              ( 'total VMs' )
+            )
+            .attr( {
+              'x': '5',
+              'y': numbers_height,
+              'font-face': '\'PT Sans\', sans-serif',
+              'font-size': '12px',
+              'font-weight': 'bold',
+              'fill': '#888'
+            } );
+
+        stats_vmpooler_numbers__svg[ 'total' ]
+          .append( 'text' )
+            .text(
+              ( stats_vmpooler_numbers__data__live[ 'total' ] )
+            )
+            .attr( {
+              'x': '0',
+              'y': '36',
+              'font-face': '\'PT Sans\', sans-serif',
+              'font-weight': 'bold',
+              'font-size': '50px',
+              'letter-spacing': '-0.05em',
+              'fill': '#444'
+            } );
+
+        stats_vmpooler_numbers__svg[ 'ready' ] = d3.select( '#stats-vmpooler-numbers' )
+          .append( 'svg' )
+            .style( 'margin', '15px 0px 0px 0px' )
+            .style( 'padding', '0px 10px 20px 10px' )
+            .attr( 'width', numbers_width + 'px' )
+            .attr( 'height', numbers_height + 'px' );
+
+        stats_vmpooler_numbers__svg[ 'ready' ]
+          .append( 'text' )
+            .text(
+              ( 'ready and waiting' )
+            )
+            .attr( {
+              'x': '5',
+              'y': numbers_height,
+              'font-face': '\'PT Sans\', sans-serif',
+              'font-size': '12px',
+              'font-weight': 'bold',
+              'fill': '#888'
+            } );
+
+        stats_vmpooler_numbers__svg[ 'ready' ]
+          .append( 'text' )
+            .text(
+              ( stats_vmpooler_numbers__data__live[ 'ready' ] )
+            )
+            .attr( {
+              'x': '0',
+              'y': '36',
+              'font-face': '\'PT Sans\', sans-serif',
+              'font-weight': 'bold',
+              'font-size': '50px',
+              'letter-spacing': '-0.05em',
+              'fill': '#444'
+            } );
+
+        stats_vmpooler_numbers__svg[ 'cloning' ] = d3.select( '#stats-vmpooler-numbers' )
+          .append( 'svg' )
+            .style( 'margin', '15px 0px 0px 0px' )
+            .style( 'padding', '0px 10px 20px 10px' )
+            .attr( 'width', numbers_width + 'px' )
+            .attr( 'height', numbers_height + 'px' );
+
+        stats_vmpooler_numbers__svg[ 'cloning' ]
+          .append( 'text' )
+            .text(
+              ( 'being cloned' )
+            )
+            .attr( {
+              'x': '5',
+              'y': numbers_height,
+              'font-face': '\'PT Sans\', sans-serif',
+              'font-size': '12px',
+              'font-weight': 'bold',
+              'fill': '#888'
+            } );
+
+        stats_vmpooler_numbers__svg[ 'cloning' ]
+          .append( 'text' )
+            .text(
+              ( stats_vmpooler_numbers__data__live[ 'cloning' ] )
+            )
+            .attr( {
+              'x': '0',
+              'y': '36',
+              'font-face': '\'PT Sans\', sans-serif',
+              'font-weight': 'bold',
+              'font-size': '50px',
+              'letter-spacing': '-0.05em',
+              'fill': '#444'
+            } );
+
+        stats_vmpooler_numbers__svg[ 'booting' ] = d3.select( '#stats-vmpooler-numbers' )
+          .append( 'svg' )
+            .style( 'margin', '15px 0px 0px 0px' )
+            .style( 'padding', '0px 10px 20px 10px' )
+            .attr( 'width', numbers_width + 'px' )
+            .attr( 'height', numbers_height + 'px' );
+
+        stats_vmpooler_numbers__svg[ 'booting' ]
+          .append( 'text' )
+            .text(
+              ( 'booting up' )
+            )
+            .attr( {
+              'x': '5',
+              'y': numbers_height,
+              'font-face': '\'PT Sans\', sans-serif',
+              'font-size': '12px',
+              'font-weight': 'bold',
+              'fill': '#888'
+            } );
+
+        stats_vmpooler_numbers__svg[ 'booting' ]
+          .append( 'text' )
+            .text(
+              ( stats_vmpooler_numbers__data__live[ 'booting' ] )
+            )
+            .attr( {
+              'x': '0',
+              'y': '36',
+              'font-face': '\'PT Sans\', sans-serif',
+              'font-weight': 'bold',
+              'font-size': '50px',
+              'letter-spacing': '-0.05em',
+              'fill': '#444'
+            } );
+
+
+
+
+
+
+
+        stats_vmpooler_numbers__svg[ 'running' ] = d3.select( '#stats-vmpooler-numbers' )
+          .append( 'svg' )
+            .style( 'margin', '15px 0px 0px 0px' )
+            .style( 'padding', '0px 10px 20px 10px' )
+            .attr( 'width', numbers_width + 'px' )
+            .attr( 'height', numbers_height + 'px' );
+
+        stats_vmpooler_numbers__svg[ 'running' ]
+          .append( 'text' )
+            .text(
+              ( 'running tests' )
+            )
+            .attr( {
+              'x': '5',
+              'y': numbers_height,
+              'font-face': '\'PT Sans\', sans-serif',
+              'font-size': '12px',
+              'font-weight': 'bold',
+              'fill': '#888'
+            } );
+
+        stats_vmpooler_numbers__svg[ 'running' ]
+          .append( 'text' )
+            .text(
+              ( stats_vmpooler_numbers__data__live[ 'running' ] )
+            )
+            .attr( {
+              'x': '0',
+              'y': '36',
+              'font-face': '\'PT Sans\', sans-serif',
+              'font-weight': 'bold',
+              'font-size': '50px',
+              'letter-spacing': '-0.05em',
+              'fill': '#444'
+            } );
+
+        stats_vmpooler_numbers__svg[ 'completed' ] = d3.select( '#stats-vmpooler-numbers' )
+          .append( 'svg' )
+            .style( 'margin', '15px 0px 0px 0px' )
+            .style( 'padding', '0px 10px 20px 10px' )
+            .attr( 'width', numbers_width + 'px' )
+            .attr( 'height', numbers_height + 'px' );
+
+        stats_vmpooler_numbers__svg[ 'completed' ]
+          .append( 'text' )
+            .text(
+              ( 'waiting to die' )
+            )
+            .attr( {
+              'x': '5',
+              'y': numbers_height,
+              'font-face': '\'PT Sans\', sans-serif',
+              'font-size': '12px',
+              'font-weight': 'bold',
+              'fill': '#888'
+            } );
+
+        stats_vmpooler_numbers__svg[ 'completed' ]
+          .append( 'text' )
+            .text(
+              ( stats_vmpooler_numbers__data__live[ 'completed' ] )
+            )
+            .attr( {
+              'x': '0',
+              'y': '36',
+              'font-face': '\'PT Sans\', sans-serif',
+              'font-weight': 'bold',
+              'font-size': '50px',
+              'letter-spacing': '-0.05em',
+              'fill': '#444'
+            } );
+
+        tick();
+      }, 5000 );
+    } )();
+
+  }
+
+);
+

lib/vmpooler/public/lib/stats-vmpooler-pool.js

@@ -0,0 +1,166 @@
+var pool_url = '/dashboard/stats/vmpooler/pool';
+var pool_width = 130;
+var pool_height = 80;
+
+var stats_vmpooler_pool__data  = {};
+var stats_vmpooler_pool__svg   = {};
+
+d3.json( pool_url+'?history=1',
+
+  function( stats_vmpooler_pool__data ) {
+
+    var stats_vmpooler_pool__data__keys = [];
+
+    for ( var key in stats_vmpooler_pool__data ) {
+      stats_vmpooler_pool__data__keys.push( key );
+    }
+
+    stats_vmpooler_pool__data__keys.sort().map(
+      function( pool ) {
+        stats_vmpooler_pool__data[ pool ][ 'r' ] = stats_vmpooler_pool__data[ pool ][ 'history' ];
+      }
+    );
+
+    ( function tick() {
+      setTimeout( function() {
+        var stats_vmpooler_pool__data__live = ( function() {
+          var stats_vmpooler_pool__data__live = null;
+
+          $.ajax( {
+            'url': pool_url,
+            'async': false,
+            'global': false,
+            'dataType': 'json',
+            'success': function( data ) {
+              stats_vmpooler_pool__data__live = data;
+            }
+          } );
+          
+          return stats_vmpooler_pool__data__live;
+        } )();
+
+        $( '#stats-vmpooler-pool' ).empty();
+
+        stats_vmpooler_pool__data__keys.sort().map(
+          function( pool ) {
+            var x = d3.scale.linear().domain( [ 0, 500 ] ).range( [ 0, pool_width ] );
+            var y = d3.scale.linear().domain( [ parseInt( stats_vmpooler_pool__data__live[ pool ][ 'size' ] ), 0 ] ).range( [ 0, pool_height - 15 ] );
+
+            var area = d3.svg.area()
+              .interpolate( 'basis' )
+              .x( function( d, i ) { return x( i ); } )
+              .y0( pool_height - 15 )
+              .y1( function( d ) { return y( d ); } );
+    
+            var path = d3.svg.line()
+              .interpolate( 'basis' )
+              .x( function( d, i ) { return x( i ); } )
+              .y( function( d ) { return y( d ); } );
+
+            stats_vmpooler_pool__data[ pool ][ 'r' ].push( parseInt( stats_vmpooler_pool__data__live[ pool ][ 'ready' ] ) );
+
+            var pool_current = stats_vmpooler_pool__data[ pool ][ 'r' ].slice( -1 )[ 0 ];
+            var pool_size    = stats_vmpooler_pool__data[ pool ][ 'size' ]
+            var pool_pct     = Math.floor( ( pool_current / pool_size ) * 100 );
+
+            var statuscolor = '#78a830';
+            if ( pool_pct < 50 ) { statuscolor = '#f0a800'; }
+            if ( pool_pct < 25 ) { statuscolor = '#d84830'; }
+
+            stats_vmpooler_pool__svg[ pool ] = d3.select( '#stats-vmpooler-pool' )
+              .append( 'svg' )
+                .style( 'margin', '15px 0px 0px 0px' )
+                .style( 'padding', '0px 10px 10px 10px' )
+                .attr( 'width', pool_width )
+                .attr( 'height', pool_height );
+
+            stats_vmpooler_pool__svg[ pool ]
+              .append( 'g' )
+              .attr( 'class', 'x tick' )
+              .attr( 'transform', 'translate( 0,' + ( pool_height - 15 ) + ')' )
+              .call(
+                d3.svg.axis()
+                  .scale( x )
+                  .ticks( 4 )
+                  .tickSize( -pool_height )
+                  .outerTickSize( 0 )
+                  .tickFormat( '' )
+                  .tickSubdivide( true )
+                  .orient( 'bottom' )
+              );
+
+            stats_vmpooler_pool__svg[ pool ]
+              .append( 'text' )
+                .text(
+                  ( pool )
+                )
+                .attr( {
+                  'x': '5',
+                  'y': pool_height - 2,
+                  'font-face': '\'PT Sans\', sans-serif',
+                  'font-weight': 'bold',
+                  'font-size': '12px',
+                  'fill': '#888'
+                } );
+
+            stats_vmpooler_pool__svg[ pool ]
+              .append( 'text' )
+                .text(
+                  ( pool_pct + '%' )
+                )
+                .attr( {
+                  'x': '5',
+                  'y': pool_height - 20,
+                  'font-face': '\'PT Sans\', sans-serif',
+                  'font-weight': 'bold',
+                  'font-size': '12px',
+                  'letter-spacing': '-0.05em',
+                  'fill': '#888'
+                } );
+
+            stats_vmpooler_pool__svg[ pool ]
+              .append( 'text' )
+                .text(
+                  ( '( ' ) +
+                  ( pool_current ) +
+                  ( '/' ) +
+                  ( pool_size ) +
+                  ( ' )' )
+                )
+                .attr( {
+                  'x': 40,
+                  'y': pool_height - 20,
+                  'font-face': '\'PT Sans\', sans-serif',
+                  'font-size': '12px',
+                  'letter-spacing': '-0.05em',
+                  'fill': '#888'
+                } );
+
+            stats_vmpooler_pool__svg[ pool ]
+              .append( 'path' )
+                .attr( 'class', 'area' )
+                .attr( 'fill', statuscolor )
+                .attr( 'opacity', '0.25' )
+                .attr( 'd', area( stats_vmpooler_pool__data[ pool ][ 'r' ] ) );
+
+            stats_vmpooler_pool__svg[ pool ]
+              .append( 'path' )
+                .attr( 'class', 'line' )
+                .attr( 'stroke', statuscolor )
+                .attr( 'stroke-width', '1' )
+                .attr( 'd', path( stats_vmpooler_pool__data[ pool ][ 'r' ] ) );
+
+            if ( stats_vmpooler_pool__data[ pool ][ 'r' ].length > 500 ) {
+              stats_vmpooler_pool__data[ pool ][ 'r' ].shift();
+            }
+          }
+        )
+
+        tick();
+      }, 5000 );
+    } )();
+
+  }
+
+);
+

lib/vmpooler/public/lib/stats-vmpooler-running.js

@@ -0,0 +1,189 @@
+var running_url = '/dashboard/stats/vmpooler/running';
+var running_height = 120;
+
+var colorscale = d3.scale.category20();
+var color = {};
+
+var stats_vmpooler_running__data  = {};
+var stats_vmpooler_running__svg   = {};
+
+var stats_vmpooler_running__data__total = 0;
+
+d3.json( running_url+'?history=1',
+
+  function( stats_vmpooler_running__data ) {
+
+    if ( typeof stats_vmpooler_running__data[ 'stack' ] === 'undefined' ) {
+      stats_vmpooler_running__data[ 'stack' ] = [];
+      stats_vmpooler_running__data[ 'stack_t' ] = [];
+    }
+
+    for ( var key in stats_vmpooler_running__data ) {
+      if ( stats_vmpooler_running__data[ key ][ 'history' ] ) {
+        for ( var c = 0; c < stats_vmpooler_running__data[ key ][ 'history' ].length; c++ ) {
+          if ( typeof stats_vmpooler_running__data[ 'stack' ][ c ] === 'undefined' ) {
+            stats_vmpooler_running__data[ 'stack' ][ c ] = {};
+          }
+
+          stats_vmpooler_running__data[ 'stack' ][ c ][ key ] = stats_vmpooler_running__data[ key ][ 'history' ][ c ];
+        }
+      }
+    }
+
+    for ( var c in stats_vmpooler_running__data[ 'stack' ] ) {
+      for ( var n = 0; n < 8; n++ ) {
+        stats_vmpooler_running__data[ 'stack_t' ].push( stats_vmpooler_running__data[ 'stack' ][ c ] );
+      }
+    }
+
+    stats_vmpooler_running__data[ 'stack' ] = stats_vmpooler_running__data[ 'stack_t' ];
+    delete stats_vmpooler_running__data[ 'stack_t' ];
+
+    ( function tick() {
+      setTimeout( function() {
+
+        var stats_vmpooler_running__data__live = ( function() {
+          var stats_vmpooler_running__data__live = null;
+
+          $.ajax( {
+            'url': running_url,
+            'async': false,
+            'global': false,
+            'dataType': 'json',
+            'success': function( data ) {
+              stats_vmpooler_running__data__live = data;
+            }
+          } );
+
+          return stats_vmpooler_running__data__live;
+        } )();
+
+        var stats_vmpooler_running__data__keys = [];
+
+        for ( var key in stats_vmpooler_running__data__live ) {
+          stats_vmpooler_running__data__keys.push( key );
+          for ( var c = 0; c < Object.keys(stats_vmpooler_running__data__keys).length; c++ ) { color[key] = colorscale( c ); }
+        }
+
+        $( '#stats-vmpooler-running' ).empty();
+
+        var x = d3.scale.linear().domain( [ 0, 500 ] ).range( [ 0, document.getElementById( 'stats-vmpooler-running' ).offsetWidth ] );
+        var y = d3.scale.linear().domain( [ 0, stats_vmpooler_running__data__total ] ).range( [ running_height, 0 ] );
+
+        var area = d3.svg.area()
+          .x( function( d, i ) { return x( i ); } )
+          .y0( function( d ) { return y( d.y0 ); } )
+          .y1( function( d ) { return y( d.y0 + d.y ); } );
+
+        var path = d3.svg.line()
+          .x( function( d, i ) { return x( i ); } )
+          .y( function( d ) { return y( d.y0 + d.y ); } );
+
+        var stack = d3.layout.stack()
+          .values( function( d ) { return d.values; } );
+
+        if ( typeof stats_vmpooler_running__data[ 'stack' ] === 'undefined' ) {
+          stats_vmpooler_running__data[ 'stack' ] = [];
+        }
+
+        stats_vmpooler_running__data[ 'tmp' ] = {};
+
+        for ( var key in stats_vmpooler_running__data__live ) {
+          stats_vmpooler_running__data[ 'tmp' ][ key ] = stats_vmpooler_running__data__live[ key ][ 'running' ];
+        }
+
+        stats_vmpooler_running__data[ 'stack' ].push( stats_vmpooler_running__data[ 'tmp' ] );
+
+        var stats_vmpooler_running__data__graph = stack(
+          stats_vmpooler_running__data__keys.sort().map(
+            function( name ) {
+              return {
+                name: name,
+                values: stats_vmpooler_running__data[ 'stack' ].map( function( d ) {
+                  return { y: d[ name ] };
+               })
+              }
+            }
+          )
+        );
+
+        stats_vmpooler_running__data__total = d3.max(
+          stats_vmpooler_running__data__graph, function( layer ) {
+            return d3.max( layer.values, function( d ) {
+              return d.y0 + d.y;
+            } );
+          }
+        );
+
+        var svg = d3.select( '#stats-vmpooler-running' )
+          .append( 'svg' )
+          .attr( 'height', running_height )
+          .attr( 'width', '100%' )
+          .style( 'margin-top', '15px' )
+          .style( 'margin-bottom', '10px' )
+          .append( 'g' );
+
+        var mysvg = svg.selectAll( '#stats-vmpooler-running' )
+          .data( stats_vmpooler_running__data__graph )
+          .enter()
+          .append( 'g' );
+
+        mysvg.append( 'path' )
+          .attr( 'd', function( d ) { return area( d.values ); } )
+          .attr( 'clas', 'area' )
+          .attr( 'opacity', '0.25' )
+          .style( 'fill', function( d ) { return color[ d.name ]; } );
+
+        mysvg.append( 'path' )
+          .attr( 'd', function( d ) { return path( d.values ); } )
+          .attr( 'class', 'line' )
+          .attr( 'stroke', function( d ) { return '#888'; } )
+          .attr( 'stroke-width', '1' );
+
+        stats_vmpooler_running__data__keys.sort().map(
+          function( key ) {
+            stats_vmpooler_running__svg[ key ] = d3.select( '#stats-vmpooler-running' )
+              .append( 'svg' )
+                .style( 'margin', '0px 0px 0px 0px' )
+                .style( 'padding', '0px 10px 10px 10px' )
+                .attr( 'width', '130px' )
+                .attr( 'height', '12px' );
+
+            stats_vmpooler_running__svg[ key ]
+              .append( 'rect' )
+                .attr( {
+                  'x': '5',
+                  'y': '3',
+                  'width': '10',
+                  'height': '10',
+                  'opacity': '0.25',
+                  'fill': function( d ) { return color[ key ]; }
+                } );
+
+            stats_vmpooler_running__svg[ key ]
+              .append( 'text' )
+                .text(
+                  ( key )
+                )
+                .attr( {
+                  'x': '20',
+                  'y': '12',
+                  'font-face': '\'PT Sans\', sans-serif',
+                  'font-size': '12px',
+                  'font-weight': 'bold',
+                  'fill': '#888'
+                } );
+          }
+        );
+
+        if ( stats_vmpooler_running__data[ 'stack' ].length > 500 ) {
+          stats_vmpooler_running__data[ 'stack' ].shift();
+        }
+
+        tick();
+      }, 5000 );
+    } )();
+
+  }
+
+);

lib/vmpooler/views/dashboard.erb

@@ -0,0 +1,33 @@
+<!-- stats-vmpooler-numbers -->
+
+<div class='label'>the numbers</div>
+
+<div id='stats-vmpooler-numbers' class='module'>
+  <div class='spinner'></div> Loading data...
+</div>
+
+<script src='/lib/stats-vmpooler-numbers.js'></script>
+
+<br />
+
+<!-- stats-vmpooler-running -->
+
+<div class='label'>VMs running tests</div>
+
+<div id='stats-vmpooler-running' class='module'>
+  <div class='spinner'></div> Loading data...
+</div>
+
+<script src='/lib/stats-vmpooler-running.js'></script>
+
+<br />
+
+<!-- stats-vmpooler-pool -->
+
+<div class='label'>individual pool capacity / fullness status</div>
+
+<div id='stats-vmpooler-pool' class='module'>
+  <div class='spinner'></div> Loading data...
+</div>
+
+<script src='/lib/stats-vmpooler-pool.js'></script>

lib/vmpooler/views/layout.erb

@@ -0,0 +1,31 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
+
+<head>
+  <meta charset='utf-8'>
+
+  <link rel='stylesheet' type='text/css' href='/dashboard.css' />
+
+  <script type='text/javascript' src='http://code.jquery.com/jquery-latest.min.js'></script>
+  <script type='text/javascript' src='http://d3js.org/d3.v3.min.js'></script>
+</head>
+
+<body>
+
+<div id='wrap'>
+<div id='content'>
+
+<div id='header'>
+  <embed src='/img/logo.jpg' width='200px' height='150px' class='logo' />
+  <div class='text'><%= site_name %></div>
+</div>
+
+<%= yield %>
+
+</div>
+</div>
+
+</body>
+
+</html>

lib/vmpooler/vsphere_helper.rb

@@ -0,0 +1,180 @@
+require 'rubygems' unless defined?(Gem)
+
+module Vmpooler
+  class VsphereHelper
+    def initialize vInfo = {}
+      begin
+        require 'rbvmomi'
+      rescue LoadError
+        raise "Unable to load RbVmomi, please ensure its installed"
+      end
+
+      config_file = File.expand_path('vmpooler.yaml')
+      vsphere = YAML.load_file(config_file)[:vsphere]
+
+      @connection = RbVmomi::VIM.connect :host     => vsphere['server'],
+                                         :user     => vsphere['username'],
+                                         :password => vsphere['password'],
+                                         :insecure => true
+    end
+
+
+
+    def find_datastore datastorename
+      begin
+        @connection.serviceInstance.CurrentTime
+      rescue
+        initialize()
+      end
+
+      datacenter = @connection.serviceInstance.find_datacenter
+      datacenter.find_datastore(datastorename)
+    end
+
+
+
+    def find_folder foldername
+      begin
+        @connection.serviceInstance.CurrentTime
+      rescue
+        initialize()
+      end
+
+      datacenter = @connection.serviceInstance.find_datacenter
+      base = datacenter.vmFolder
+      folders = foldername.split('/')
+      folders.each do |folder|
+        case base
+          when RbVmomi::VIM::Folder
+            base = base.childEntity.find { |f| f.name == folder }
+          else
+            abort "Unexpected object type encountered (#{base.class}) while finding folder"
+        end
+      end
+
+      base
+    end
+
+
+
+    def find_pool poolname
+      begin
+        @connection.serviceInstance.CurrentTime
+      rescue
+        initialize()
+      end
+
+      datacenter = @connection.serviceInstance.find_datacenter
+      base = datacenter.hostFolder
+      pools = poolname.split('/')
+      pools.each do |pool|
+        case base
+          when RbVmomi::VIM::Folder
+            base = base.childEntity.find { |f| f.name == pool }
+          when RbVmomi::VIM::ClusterComputeResource
+            base = base.resourcePool.resourcePool.find { |f| f.name == pool }
+          when RbVmomi::VIM::ResourcePool
+            base = base.resourcePool.find { |f| f.name == pool }
+          else
+            abort "Unexpected object type encountered (#{base.class}) while finding resource pool"
+        end
+      end
+
+      base = base.resourcePool unless base.is_a?(RbVmomi::VIM::ResourcePool) and base.respond_to?(:resourcePool)
+      base
+    end
+
+
+
+    def find_vm vmname
+      begin
+        @connection.serviceInstance.CurrentTime
+      rescue
+        initialize()
+      end
+
+      @connection.searchIndex.FindByDnsName(:vmSearch => true, :dnsName => vmname)
+    end
+
+
+
+    def find_vm_heavy vmname
+      begin
+        @connection.serviceInstance.CurrentTime
+      rescue
+        initialize()
+      end
+
+      vmname = vmname.is_a?(Array) ? vmname : [ vmname ]
+      containerView = get_base_vm_container_from @connection
+      propertyCollector = @connection.propertyCollector
+
+      objectSet = [{
+        :obj => containerView,
+        :skip => true,
+        :selectSet => [ RbVmomi::VIM::TraversalSpec.new({
+            :name => 'gettingTheVMs',
+            :path => 'view',
+           :skip => false,
+            :type => 'ContainerView'
+        }) ]
+      }]
+
+      propSet = [{
+        :pathSet => [ 'name' ],
+        :type => 'VirtualMachine'
+      }]
+
+      results = propertyCollector.RetrievePropertiesEx({
+        :specSet => [{
+          :objectSet => objectSet,
+          :propSet   => propSet
+        }],
+        :options => { :maxObjects => nil }
+      })
+
+      vms = {}
+      results.objects.each do |result|
+        name = result.propSet.first.val
+        next unless vmname.include? name
+        vms[name] = result.obj
+      end
+
+      while results.token do
+        results = propertyCollector.ContinueRetrievePropertiesEx({:token => results.token})
+        results.objects.each do |result|
+          name = result.propSet.first.val
+          next unless vmname.include? name
+          vms[name] = result.obj
+        end
+      end
+
+      vms
+    end
+
+
+
+    def get_base_vm_container_from connection
+      begin
+        connection.serviceInstance.CurrentTime
+      rescue
+        initialize()
+      end
+
+      viewManager = connection.serviceContent.viewManager
+      viewManager.CreateContainerView({
+        :container => connection.serviceContent.rootFolder,
+        :recursive => true,
+        :type      => [ 'VirtualMachine' ]
+      })
+    end
+
+
+
+    def close
+      @connection.close
+    end
+
+  end
+end
+

lib/vsphere_helper.rb

@@ -1,180 +0,0 @@
-require 'yaml' unless defined?(YAML)
-require 'rubygems' unless defined?(Gem)
-
-class VsphereHelper
-  def initialize vInfo = {}
-    begin
-      require 'rbvmomi'
-    rescue LoadError
-      raise "Unable to load RbVmomi, please ensure its installed"
-    end
-
-    Dir.chdir(File.dirname(__FILE__))
-
-    config_file = File.expand_path('../vmpooler.yaml')
-    vsphere = YAML.load_file(config_file)[:vsphere]
-
-    @connection = RbVmomi::VIM.connect :host     => vsphere['server'],
-                                       :user     => vsphere['username'],
-                                       :password => vsphere['password'],
-                                       :insecure => true
-  end
-
-
-
-  def find_datastore datastorename
-    begin
-      @connection.serviceInstance.CurrentTime
-    rescue
-      initialize()
-    end
-
-    datacenter = @connection.serviceInstance.find_datacenter
-    datacenter.find_datastore(datastorename)
-  end
-
-
-
-  def find_folder foldername
-    begin
-      @connection.serviceInstance.CurrentTime
-    rescue
-      initialize()
-    end
-
-    datacenter = @connection.serviceInstance.find_datacenter
-    base = datacenter.vmFolder
-    folders = foldername.split('/')
-    folders.each do |folder|
-      case base
-        when RbVmomi::VIM::Folder
-          base = base.childEntity.find { |f| f.name == folder }
-        else
-          abort "Unexpected object type encountered (#{base.class}) while finding folder"
-      end
-    end
-
-    base
-  end
-
-
-
-  def find_pool poolname
-    begin
-      @connection.serviceInstance.CurrentTime
-    rescue
-      initialize()
-    end
-
-    datacenter = @connection.serviceInstance.find_datacenter
-    base = datacenter.hostFolder
-    pools = poolname.split('/')
-    pools.each do |pool|
-      case base
-        when RbVmomi::VIM::Folder
-          base = base.childEntity.find { |f| f.name == pool }
-        when RbVmomi::VIM::ClusterComputeResource
-          base = base.resourcePool.resourcePool.find { |f| f.name == pool }
-        when RbVmomi::VIM::ResourcePool
-          base = base.resourcePool.find { |f| f.name == pool }
-        else
-          abort "Unexpected object type encountered (#{base.class}) while finding resource pool"
-      end
-    end
-
-    base = base.resourcePool unless base.is_a?(RbVmomi::VIM::ResourcePool) and base.respond_to?(:resourcePool)
-    base
-  end
-
-
-
-  def find_vm vmname
-    begin
-      @connection.serviceInstance.CurrentTime
-    rescue
-      initialize()
-    end
-
-    @connection.searchIndex.FindByDnsName(:vmSearch => true, :dnsName => vmname)
-  end
-
-
-
-  def find_vm_heavy vmname
-    begin
-      @connection.serviceInstance.CurrentTime
-    rescue
-      initialize()
-    end
-
-    vmname = vmname.is_a?(Array) ? vmname : [ vmname ]
-    containerView = get_base_vm_container_from @connection
-    propertyCollector = @connection.propertyCollector
-
-    objectSet = [{
-      :obj => containerView,
-      :skip => true,
-      :selectSet => [ RbVmomi::VIM::TraversalSpec.new({
-          :name => 'gettingTheVMs',
-          :path => 'view',
-          :skip => false,
-          :type => 'ContainerView'
-      }) ]
-    }]
-
-    propSet = [{
-      :pathSet => [ 'name' ],
-      :type => 'VirtualMachine'
-    }]
-
-    results = propertyCollector.RetrievePropertiesEx({
-      :specSet => [{
-        :objectSet => objectSet,
-        :propSet   => propSet
-      }],
-      :options => { :maxObjects => nil }
-    })
-
-    vms = {}
-    results.objects.each do |result|
-      name = result.propSet.first.val
-      next unless vmname.include? name
-      vms[name] = result.obj
-    end
-
-    while results.token do
-      results = propertyCollector.ContinueRetrievePropertiesEx({:token => results.token})
-      results.objects.each do |result|
-        name = result.propSet.first.val
-        next unless vmname.include? name
-        vms[name] = result.obj
-      end
-    end
-
-    vms
-  end
-
-
-
-  def get_base_vm_container_from connection
-    begin
-      connection.serviceInstance.CurrentTime
-    rescue
-      initialize()
-    end
-
-    viewManager = connection.serviceContent.viewManager
-    viewManager.CreateContainerView({
-      :container => connection.serviceContent.rootFolder,
-      :recursive => true,
-      :type      => [ 'VirtualMachine' ]
-    })
-  end
-
-
-
-  def close
-    @connection.close
-  end
-end
-