From 60cc1ef1783f416028a28066e2462813f524f393 Mon Sep 17 00:00:00 2001
From: Scott Schneider <sschneider@puppetlabs.com>
Date: Wed, 5 Mar 2014 12:57:25 -0800
Subject: [PATCH 1/2] Reworked into a single namespaced Ruby application

---
 lib/graphite.rb                               |  18 -
 lib/logger.rb                                 |  20 -
 lib/require_relative.rb                       |  21 -
 lib/vmpooler.rb                               |  12 +
 lib/vmpooler/api.rb                           | 233 +++++++++
 lib/vmpooler/graphite.rb                      |  20 +
 lib/vmpooler/logger.rb                        |  21 +
 lib/vmpooler/pool_manager.rb                  | 459 ++++++++++++++++++
 {public => lib/vmpooler/public}/dashboard.css |   0
 {public => lib/vmpooler/public}/img/logo.jpg  | Bin
 .../vmpooler/public}/img/spinner.svg          |   0
 .../public}/lib/stats-vmpooler-numbers.js     |   0
 .../public}/lib/stats-vmpooler-pool.js        |   0
 .../public}/lib/stats-vmpooler-running.js     |   0
 {views => lib/vmpooler/views}/dashboard.erb   |   0
 {views => lib/vmpooler/views}/layout.erb      |   0
 lib/vmpooler/vsphere_helper.rb                | 180 +++++++
 lib/vsphere_helper.rb                         | 180 -------
 vmpooler                                      | 459 +-----------------
 vmpooler-api                                  | 231 ---------
 20 files changed, 930 insertions(+), 924 deletions(-)
 delete mode 100755 lib/graphite.rb
 delete mode 100755 lib/logger.rb
 delete mode 100755 lib/require_relative.rb
 create mode 100644 lib/vmpooler.rb
 create mode 100644 lib/vmpooler/api.rb
 create mode 100644 lib/vmpooler/graphite.rb
 create mode 100644 lib/vmpooler/logger.rb
 create mode 100644 lib/vmpooler/pool_manager.rb
 rename {public => lib/vmpooler/public}/dashboard.css (100%)
 rename {public => lib/vmpooler/public}/img/logo.jpg (100%)
 rename {public => lib/vmpooler/public}/img/spinner.svg (100%)
 rename {public => lib/vmpooler/public}/lib/stats-vmpooler-numbers.js (100%)
 rename {public => lib/vmpooler/public}/lib/stats-vmpooler-pool.js (100%)
 rename {public => lib/vmpooler/public}/lib/stats-vmpooler-running.js (100%)
 rename {views => lib/vmpooler/views}/dashboard.erb (100%)
 rename {views => lib/vmpooler/views}/layout.erb (100%)
 create mode 100644 lib/vmpooler/vsphere_helper.rb
 delete mode 100755 lib/vsphere_helper.rb
 delete mode 100755 vmpooler-api

diff --git a/lib/graphite.rb b/lib/graphite.rb
deleted file mode 100755
index b52e011..0000000
--- a/lib/graphite.rb
+++ /dev/null
@@ -1,18 +0,0 @@
-require 'rubygems' unless defined?(Gem)
-
-class Graphite
-  def initialize(
-    s = 'graphite'
-  )
-    @server = s
-  end
-
-  def log path, value
-    Thread.new {
-      socket = TCPSocket.new(@server, 2003)
-      socket.puts "#{path} #{value} #{Time.now.to_i}"
-      socket.close
-    }
-  end
-end
-
diff --git a/lib/logger.rb b/lib/logger.rb
deleted file mode 100755
index 2ec8918..0000000
--- a/lib/logger.rb
+++ /dev/null
@@ -1,20 +0,0 @@
-require 'rubygems' unless defined?(Gem)
-
-class Logger
-  def initialize(
-    f = '/var/log/vmpooler.log'
-  )
-    @file = f
-  end
-
-  def log level, string
-    time = Time.new
-    stamp = time.strftime('%Y-%m-%d %H:%M:%S')
-    puts "[#{stamp}] #{string}"
-
-    open(@file, 'a') do |f|
-      f.puts "[#{stamp}] #{string}"
-    end
-  end
-end
-
diff --git a/lib/require_relative.rb b/lib/require_relative.rb
deleted file mode 100755
index ab9114c..0000000
--- a/lib/require_relative.rb
+++ /dev/null
@@ -1,21 +0,0 @@
-# require_relative was introduced in 1.9.2. This makes it
-# available to younger rubies.  It trys hard to not re-require
-# files.
-
-unless Kernel.respond_to?(:require_relative)
-  module Kernel
-    def require_relative(path)
-      desired_path = File.expand_path('../'+path.to_str, caller[0])
-      shortest = desired_path
-      $:.each do |path|
-        path += '/'
-        if desired_path.index(path) == 0
-          candidate = desired_path.sub(path, '')
-          shortest = candidate if candidate.size < shortest.size
-        end
-      end
-      require shortest
-    end
-  end
-end
-
diff --git a/lib/vmpooler.rb b/lib/vmpooler.rb
new file mode 100644
index 0000000..4e4d901
--- /dev/null
+++ b/lib/vmpooler.rb
@@ -0,0 +1,12 @@
+require 'rubygems' unless defined?(Gem)
+
+module Vmpooler
+  %w( api graphite logger pool_manager vsphere_helper ).each do |lib|
+    begin
+      require "vmpooler/#{lib}"
+    rescue LoadError
+      require File.expand_path(File.join(File.dirname(__FILE__), 'vmpooler', lib))
+    end
+  end
+end
+
diff --git a/lib/vmpooler/api.rb b/lib/vmpooler/api.rb
new file mode 100644
index 0000000..4a84234
--- /dev/null
+++ b/lib/vmpooler/api.rb
@@ -0,0 +1,233 @@
+module Vmpooler
+  class API
+    def initialize
+      require 'sinatra/base'
+
+      require 'json'
+      require 'open-uri'
+      require 'redis'
+      require 'yaml'
+
+      # Load the configuration file
+      config_file = File.expand_path('vmpooler.yaml')
+      $config = YAML.load_file(config_file)
+
+      pools = $config[:pools]
+      redis = $config[:redis]
+
+      # Set some defaults
+      $config[:redis] ||= Hash.new
+      $config[:redis]['server'] ||= 'localhost'
+
+      # Connect to Redis
+      $redis = Redis.new(:host => $config[:redis]['server'])
+    end
+
+    def execute!
+      my_app = Sinatra.new {
+
+        set :environment, 'production'
+
+        get '/' do
+          erb :dashboard, locals: {
+            site_name: $config[:config]['site_name'] || '<b>vmpooler</b>',
+          }
+        end
+
+        get '/dashboard/stats/vmpooler/numbers/?' do
+          result = Hash.new
+          result['pending'] = 0
+          result['cloning'] = 0
+          result['booting'] = 0
+          result['ready'] = 0
+          result['running'] = 0
+          result['completed'] = 0
+
+          $config[:pools].each do |pool|
+            result['pending'] += $redis.scard( 'vmpooler__pending__' + pool['name'] )
+            result['ready'] += $redis.scard( 'vmpooler__ready__' + pool['name'] )
+            result['running'] += $redis.scard( 'vmpooler__running__' + pool['name'] )
+            result['completed'] += $redis.scard( 'vmpooler__completed__' + pool['name'] )
+          end
+
+          result['cloning'] = $redis.get( 'vmpooler__tasks__clone' )
+          result['booting'] = result['pending'].to_i - result['cloning'].to_i
+          result['booting'] = 0 if result['booting'] < 0
+          result['total'] = result['pending'].to_i + result['ready'].to_i + result['running'].to_i + result['completed'].to_i
+
+          content_type :json
+          JSON.pretty_generate(result)
+        end
+
+        get '/dashboard/stats/vmpooler/pool/?' do
+          result = Hash.new
+
+          $config[:pools].each do |pool|
+            result[pool['name']] ||= Hash.new
+            result[pool['name']]['size'] = pool['size']
+            result[pool['name']]['ready'] = $redis.scard( 'vmpooler__ready__' + pool['name'] )
+          end
+
+          if ( params[:history] )
+            if ( $config[:config]['graphite'] )
+              history ||= Hash.new
+
+              begin
+                buffer = open( 'http://'+$config[:config]['graphite']+'/render?target=vmpooler.ready.*&from=-1hour&format=json' ).read
+                history = JSON.parse( buffer )
+
+                history.each do |pool|
+                  if pool['target'] =~ /.*\.(.*)$/
+                    pool['name'] = $1
+
+                    if ( result[pool['name']] )
+                      pool['last'] = result[pool['name']]['size']
+                      result[pool['name']]['history'] ||= Array.new
+
+                      pool['datapoints'].each do |metric|
+                        8.times do |n|
+                          if ( metric[0] )
+                            pool['last'] = metric[0].to_i
+                            result[pool['name']]['history'].push( metric[0].to_i )
+                          else
+                            result[pool['name']]['history'].push( pool['last'] )
+                          end
+                        end
+                      end
+                    end
+                  end
+                end
+              rescue
+              end
+            else
+              $config[:pools].each do |pool|
+                result[pool['name']] ||= Hash.new
+                result[pool['name']]['history'] = [ $redis.scard( 'vmpooler__ready__' + pool['name'] ) ]
+              end
+            end
+          end
+
+          content_type :json
+          JSON.pretty_generate(result)
+        end
+
+        get '/dashboard/stats/vmpooler/running/?' do
+          result = Hash.new
+
+          $config[:pools].each do |pool|
+            running = $redis.scard( 'vmpooler__running__' + pool['name'] )
+            pool['major'] = $1 if pool['name'] =~ /^(\w+)\-/
+
+            result[pool['major']] ||= Hash.new
+
+            result[pool['major']]['running'] = result[pool['major']]['running'].to_i + running.to_i
+          end
+
+          if ( params[:history] )
+            if ( $config[:config]['graphite'] )
+              begin
+                buffer = open( 'http://'+$config[:config]['graphite']+'/render?target=vmpooler.running.*&from=-1hour&format=json' ).read
+                JSON.parse( buffer ).each do |pool|
+                  if pool['target'] =~ /.*\.(.*)$/
+                    pool['name'] = $1
+
+                    pool['major'] = $1 if pool['name'] =~ /^(\w+)\-/
+
+                    result[pool['major']]['history'] ||= Array.new
+
+                    for i in 0..pool['datapoints'].length
+                      if (
+                        pool['datapoints'][i] and
+                        pool['datapoints'][i][0]
+                      )
+                        pool['last'] = pool['datapoints'][i][0]
+
+                        result[pool['major']]['history'][i] ||= 0
+                        result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['datapoints'][i][0].to_i
+                      else
+                        result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['last'].to_i
+                      end
+                    end
+
+                  end
+                end
+              rescue
+              end
+            end
+          end
+
+          content_type :json
+          JSON.pretty_generate(result)
+        end
+
+        get '/vm/?' do
+          content_type :json
+
+          result = []
+
+          $config[:pools].each do |pool|
+            result.push(pool['name'])
+          end
+
+          JSON.pretty_generate(result)
+        end
+
+        get '/vm/:template/?' do
+          content_type :json
+
+          result = {}
+          result[params[:template]] = {}
+          result[params[:template]]['hosts'] = $redis.smembers('vmpooler__ready__'+params[:template])
+
+          JSON.pretty_generate(result)
+        end
+
+        post '/vm/:template/?' do
+          content_type :json
+
+          result = {}
+          result[params[:template]] = {}
+
+          if ( $redis.scard('vmpooler__ready__'+params[:template]) > 0 )
+            vm = $redis.spop('vmpooler__ready__'+params[:template])
+
+            unless (vm.nil?)
+              $redis.sadd('vmpooler__running__'+params[:template], vm)
+              $redis.hset('vmpooler__active__'+params[:template], vm, Time.now)
+
+              result[params[:template]]['ok'] = true
+              result[params[:template]]['hostname'] = vm
+            else
+              result[params[:template]]['ok'] = false
+            end
+          else
+            result[params[:template]]['ok'] = false
+          end
+
+          JSON.pretty_generate(result)
+        end
+
+        delete '/vm/:hostname/?' do
+          content_type :json
+
+          result = {}
+
+          result['ok'] = false
+
+          $config[:pools].each do |pool|
+            if $redis.sismember('vmpooler__running__'+pool['name'], params[:hostname])
+              $redis.srem('vmpooler__running__'+pool['name'], params[:hostname])
+              $redis.sadd('vmpooler__completed__'+pool['name'], params[:hostname])
+              result['ok'] = true
+            end
+          end
+
+          JSON.pretty_generate(result)
+        end
+      }
+
+      my_app.run!
+    end
+  end
+end
+
diff --git a/lib/vmpooler/graphite.rb b/lib/vmpooler/graphite.rb
new file mode 100644
index 0000000..ecdb8ea
--- /dev/null
+++ b/lib/vmpooler/graphite.rb
@@ -0,0 +1,20 @@
+require 'rubygems' unless defined?(Gem)
+
+module Vmpooler
+  class Graphite
+    def initialize(
+      s = 'graphite'
+    )
+      @server = s
+    end
+
+    def log path, value
+      Thread.new {
+        socket = TCPSocket.new(@server, 2003)
+        socket.puts "#{path} #{value} #{Time.now.to_i}"
+        socket.close
+      }
+    end
+  end
+end
+
diff --git a/lib/vmpooler/logger.rb b/lib/vmpooler/logger.rb
new file mode 100644
index 0000000..de97481
--- /dev/null
+++ b/lib/vmpooler/logger.rb
@@ -0,0 +1,21 @@
+require 'rubygems' unless defined?(Gem)
+
+module Vmpooler
+  class Logger
+    def initialize(
+      f = '/var/log/vmpooler.log'
+    )
+      @file = f
+    end
+
+    def log level, string
+      time = Time.new
+      stamp = time.strftime('%Y-%m-%d %H:%M:%S')
+
+      open(@file, 'a') do |f|
+        f.puts "[#{stamp}] #{string}"
+      end
+    end
+  end
+end
+
diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb
new file mode 100644
index 0000000..c85ec21
--- /dev/null
+++ b/lib/vmpooler/pool_manager.rb
@@ -0,0 +1,459 @@
+module Vmpooler
+  class PoolManager
+    def initialize
+      require 'json'
+      require 'rbvmomi'
+      require 'redis'
+      require 'time'
+      require 'timeout'
+      require 'yaml'
+
+      # Load the configuration file
+      config_file = File.expand_path('vmpooler.yaml')
+      $config = YAML.load_file(config_file)
+
+      $pools   = $config[:pools]
+      vsphere = $config[:vsphere]
+      redis   = $config[:redis]
+
+      # Load logger library
+      $logger = Vmpooler::Logger.new $config[:config]['logfile']
+
+      # Load Graphite helper library (if configured)
+      if (defined? $config[:config]['graphite'])
+        $graphite = Vmpooler::Graphite.new $config[:config]['graphite']
+      end
+
+      # Set some defaults
+      $config[:config]['task_limit']   ||= 10
+      $config[:config]['vm_checktime'] ||= 15
+      $config[:config]['vm_lifetime']  ||= 24
+      $config[:redis] ||= Hash.new
+      $config[:redis]['server'] ||= 'localhost'
+
+      # Connect to Redis
+      $redis = Redis.new(:host => $config[:redis]['server'])
+
+      # vSphere object
+      $vsphere = {}
+
+      # Our thread-tracker object
+      $threads = {}
+    end
+
+
+    # Check the state of a VM
+    def check_pending_vm vm, pool, timeout
+      Thread.new {
+        host = $vsphere[pool].find_vm(vm)
+
+        if (host)
+          if (
+            (host.summary) and
+            (host.summary.guest) and
+            (host.summary.guest.hostName) and
+            (host.summary.guest.hostName == vm)
+          )
+            begin
+              Socket.getaddrinfo(vm, nil)
+            rescue
+            end
+
+            $redis.smove('vmpooler__pending__'+pool, 'vmpooler__ready__'+pool, vm)
+
+            $logger.log('s', "[>] [#{pool}] '#{vm}' moved to 'ready' queue")
+          end
+        else
+          clone_stamp = $redis.hget('vmpooler__vm__'+vm, 'clone')
+
+          if (
+            (clone_stamp) and
+            (((Time.now - Time.parse(clone_stamp))/60) > timeout)
+          )
+            $redis.smove('vmpooler__pending__'+pool, 'vmpooler__completed__'+pool, vm)
+
+            $logger.log('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes")
+          end
+        end
+      }
+    end
+
+    def check_ready_vm vm, pool, ttl
+      Thread.new {
+        if (ttl > 0)
+          if ((((Time.now - host.runtime.bootTime)/60).to_s[/^\d+\.\d{1}/].to_f) > ttl)
+            $redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm)
+
+            $logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue")
+          end
+        end
+
+        check_stamp = $redis.hget('vmpooler__vm__'+vm, 'check')
+
+        if (
+          (! check_stamp) or
+          (((Time.now - Time.parse(check_stamp))/60) > $config[:config]['vm_checktime'])
+        )
+          $redis.hset('vmpooler__vm__'+vm, 'check', Time.now)
+
+          host = $vsphere[pool].find_vm(vm) ||
+                 $vsphere[pool].find_vm_heavy(vm)[vm]
+
+          if (host)
+            if (
+              (host.runtime) and
+              (host.runtime.powerState) and
+              (host.runtime.powerState != 'poweredOn')
+            )
+              $redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm)
+
+              $logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off, removed from 'ready' queue")
+            end
+
+            if (
+              (host.summary.guest) and
+              (host.summary.guest.hostName) and
+              (host.summary.guest.hostName != vm)
+            )
+              $redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm)
+
+              $logger.log('d', "[!] [#{pool}] '#{vm}' has mismatched hostname, removed from 'ready' queue")
+            end
+          else
+            $redis.srem('vmpooler__ready__'+pool, vm)
+
+            $logger.log('s', "[!] [#{pool}] '#{vm}' not found in vCenter inventory, removed from 'ready' queue")
+          end
+
+          begin
+            Timeout::timeout(5) {
+              TCPSocket.new vm, 22
+            }
+          rescue
+            if ($redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm))
+              $logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue")
+            end
+          end
+        end
+      }
+    end
+
+    def check_running_vm vm, pool, ttl
+      Thread.new {
+        host = $vsphere[pool].find_vm(vm)
+
+        if (host)
+          if (
+            (host.runtime) and
+            (host.runtime.powerState != 'poweredOn')
+          )
+            $redis.smove('vmpooler__running__'+pool, 'vmpooler__completed__'+pool, vm)
+
+            $logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off or dead")
+          else
+            if (
+              (host.runtime) and
+              (host.runtime.bootTime)
+              ((((Time.now - host.runtime.bootTime)/60).to_s[/^\d+\.\d{1}/].to_f) > ttl)
+            )
+              $redis.smove('vmpooler__running__'+pool, 'vmpooler__completed__'+pool, vm)
+
+              $logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes")
+            end
+          end
+        end
+      }
+    end
+
+    # Clone a VM
+    def clone_vm template, pool, folder, datastore
+      Thread.new {
+        vm = {}
+
+        if template =~ /\//
+          templatefolders = template.split('/')
+          vm['template'] = templatefolders.pop
+        end
+
+        if templatefolders
+          vm[vm['template']] = $vsphere[vm['template']].find_folder(templatefolders.join('/')).find(vm['template'])
+        else
+          raise "Please provide a full path to the template"
+        end
+
+        if vm['template'].length == 0
+          raise "Unable to find template '#{vm['template']}'!"
+        end
+
+        # Generate a randomized hostname
+        o = [('a'..'z'),('0'..'9')].map{|r| r.to_a}.flatten
+        vm['hostname'] = o[rand(25)]+(0...14).map{o[rand(o.length)]}.join
+
+        # Add VM to Redis inventory ('pending' pool)
+        $redis.sadd('vmpooler__pending__'+vm['template'], vm['hostname'])
+        $redis.hset('vmpooler__vm__'+vm['hostname'], 'clone', Time.now)
+
+        # Annotate with creation time, origin template, etc.
+        configSpec = RbVmomi::VIM.VirtualMachineConfigSpec(
+          :annotation => JSON.pretty_generate({
+              name: vm['hostname'],
+              created_by: $config[:vsphere]['username'],
+              base_template: vm['template'],
+              creation_timestamp: Time.now.utc
+           })
+        )
+
+        # Put the VM in the specified folder and resource pool
+        relocateSpec = RbVmomi::VIM.VirtualMachineRelocateSpec(
+          :datastore    => $vsphere[vm['template']].find_datastore(datastore),
+          :pool         => $vsphere[vm['template']].find_pool(pool),
+          :diskMoveType => :moveChildMostDiskBacking
+        )
+
+        # Create a clone spec
+        spec = RbVmomi::VIM.VirtualMachineCloneSpec(
+          :location      => relocateSpec,
+          :config        => configSpec,
+          :powerOn       => true,
+          :template      => false
+        )
+
+        # Clone the VM
+        $logger.log('d', "[ ] [#{vm['template']}] '#{vm['hostname']}' is being cloned from '#{vm['template']}'")
+
+        begin
+          start = Time.now
+          vm[vm['template']].CloneVM_Task(
+            :folder => $vsphere[vm['template']].find_folder(folder),
+            :name => vm['hostname'],
+            :spec => spec
+          ).wait_for_completion
+          finish = '%.2f' % (Time.now-start)
+
+          $logger.log('s', "[+] [#{vm['template']}] '#{vm['hostname']}' cloned from '#{vm['template']}' in #{finish} seconds")
+        rescue
+          $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' clone appears to have failed")
+          $redis.srem('vmpooler__pending__'+vm['template'], vm['hostname'])
+        end
+
+        $redis.decr('vmpooler__tasks__clone')
+
+        begin
+          $graphite.log("vmpooler.clone.#{vm['template']}", finish) if defined? $graphite
+        rescue
+        end
+      }
+    end
+
+    # Destroy a VM
+    def destroy_vm vm, pool
+      Thread.new {
+        $redis.srem('vmpooler__completed__'+pool, vm)
+        $redis.hdel('vmpooler__active__'+pool, vm)
+        $redis.del('vmpooler__vm__'+vm)
+
+        host = $vsphere[pool].find_vm(vm) ||
+               $vsphere[pool].find_vm_heavy(vm)[vm]
+
+        if (host)
+          start = Time.now
+
+          if (
+            (host.runtime) and
+            (host.runtime.powerState) and
+            (host.runtime.powerState == 'poweredOn')
+          )
+            $logger.log('d', "[ ] [#{pool}] '#{vm}' is being shut down")
+            host.PowerOffVM_Task.wait_for_completion
+          end
+
+          host.Destroy_Task.wait_for_completion
+          finish = '%.2f' % (Time.now-start)
+
+          $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds")
+
+          $graphite.log("vmpooler.destroy.#{pool}", finish) if defined? $graphite
+        end
+      }
+    end
+
+    def check_pool pool
+      $logger.log('d', "[*] [#{pool['name']}] starting worker thread")
+
+      $threads[pool['name']] = Thread.new {
+        $vsphere[pool['name']] ||= Vmpooler::VsphereHelper.new
+
+        loop do
+          # INVENTORY
+          inventory = {}
+          begin
+            base = $vsphere[pool['name']].find_pool(pool['pool'])
+
+            base.vm.each do |vm|
+              if (
+                (! $redis.sismember('vmpooler__running__'+pool['name'], vm['name'])) and
+                (! $redis.sismember('vmpooler__ready__'+pool['name'], vm['name'])) and
+                (! $redis.sismember('vmpooler__pending__'+pool['name'], vm['name'])) and
+                (! $redis.sismember('vmpooler__completed__'+pool['name'], vm['name'])) and
+                (! $redis.sismember('vmpooler__discovered__'+pool['name'], vm['name']))
+              )
+                $redis.sadd('vmpooler__discovered__'+pool['name'], vm['name'])
+
+                $logger.log('s', "[?] [#{pool['name']}] '#{vm['name']}' added to 'discovered' queue")
+              end
+
+              inventory[vm['name']] = 1
+            end
+          rescue
+          end
+
+          # RUNNING
+          $redis.smembers('vmpooler__running__'+pool['name']).each do |vm|
+            if (inventory[vm])
+              if (pool['running_ttl'])
+                begin
+                  check_running_vm(vm, pool['name'], pool['running_ttl'])
+                rescue
+                end
+              else
+                begin
+                  check_running_vm(vm, pool['name'], '720')
+                rescue
+                end
+              end
+            end
+          end
+
+          # READY
+          $redis.smembers('vmpooler__ready__'+pool['name']).each do |vm|
+            if (inventory[vm])
+              begin
+                check_ready_vm(vm, pool['name'], pool['ready_ttl'] || 0)
+              rescue
+              end
+            end
+          end
+
+          # PENDING
+          $redis.smembers('vmpooler__pending__'+pool['name']).each do |vm|
+            pool['timeout'] ||= 15
+
+            if (inventory[vm])
+              begin
+                check_pending_vm(vm, pool['name'], pool['timeout'])
+              rescue
+              end
+            end
+          end
+
+          # COMPLETED
+          $redis.smembers('vmpooler__completed__'+pool['name']).each do |vm|
+            if (inventory[vm])
+              begin
+                destroy_vm(vm, pool['name'])
+              rescue
+                $logger.log('s', "[!] [#{pool['name']}] '#{vm}' destroy appears to have failed")
+                $redis.srem('vmpooler__completed__'+pool['name'], vm)
+                $redis.hdel('vmpooler__active__'+pool['name'], vm)
+                $redis.del('vmpooler__vm__'+vm)
+              end
+            else
+              $logger.log('s', "[!] [#{pool['name']}] '#{vm}' not found in inventory, removed from 'completed' queue")
+              $redis.srem('vmpooler__completed__'+pool['name'], vm)
+              $redis.hdel('vmpooler__active__'+pool['name'], vm)
+              $redis.del('vmpooler__vm__'+vm)
+            end
+          end
+
+          # DISCOVERED
+          $redis.smembers('vmpooler__discovered__'+pool['name']).each do |vm|
+            ['pending', 'ready', 'running', 'completed'].each do |queue|
+              if ($redis.sismember('vmpooler__'+queue+'__'+pool['name'], vm))
+                $logger.log('d', "[!] [#{pool['name']}] '#{vm}' found in '#{queue}', removed from 'discovered' queue")
+                $redis.srem('vmpooler__discovered__'+pool['name'], vm)
+              end
+            end
+
+            if ($redis.sismember('vmpooler__discovered__'+pool['name'], vm))
+              $redis.smove('vmpooler__discovered__'+pool['name'], 'vmpooler__completed__'+pool['name'], vm)
+            end
+          end
+
+          # LONG-RUNNING
+          $redis.smembers('vmpooler__running__'+pool['name']).each do |vm|
+            if ($redis.hget('vmpooler__active__'+pool['name'], vm))
+              running = (Time.now - Time.parse($redis.hget('vmpooler__active__'+pool['name'], vm)))/60/60
+              if (
+                ($config[:config]['vm_lifetime'] > 0) and
+                (running > $config[:config]['vm_lifetime'])
+              )
+                $redis.smove('vmpooler__running__'+pool['name'], 'vmpooler__completed__'+pool['name'], vm)
+
+                $logger.log('d', "[!] [#{pool['name']}] '#{vm}' reached end of TTL after #{$config[:config]['vm_lifetime']} hours")
+              end
+            end
+          end
+
+          # REPOPULATE
+          total = $redis.scard('vmpooler__ready__'+pool['name']) +
+                  $redis.scard('vmpooler__pending__'+pool['name'])
+
+          begin
+            if (defined? $graphite)
+              $graphite.log('vmpooler.ready.'+pool['name'], $redis.scard('vmpooler__ready__'+pool['name']))
+              $graphite.log('vmpooler.running.'+pool['name'], $redis.scard('vmpooler__running__'+pool['name']))
+            end
+          rescue
+          end
+
+          if (total < pool['size'])
+            (1..(pool['size'] - total)).each { |i|
+
+              if ($redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit'])
+                begin
+                  $redis.incr('vmpooler__tasks__clone')
+
+                  clone_vm(
+                    pool['template'],
+                    pool['pool'],
+                    pool['folder'],
+                    pool['datastore']
+                  )
+                rescue
+                  $logger.log('s', "[!] [#{pool['name']}] clone appears to have failed")
+                  $redis.decr('vmpooler__tasks__clone')
+                end
+              end
+            }
+          end
+
+          sleep(1)
+        end
+      }
+    end
+
+    def execute!
+      $logger.log('d', "starting vmpooler")
+
+      # Clear out the tasks manager, as we don't know about any tasks at this point
+      $redis.set('vmpooler__tasks__clone', 0)
+
+      loop do
+        $pools.each do |pool|
+          if (! $threads[pool['name']])
+            check_pool(pool)
+          else
+            if (! $threads[pool['name']].alive?)
+              $logger.log('d', "[!] [#{pool['name']}] worker thread died, restarting")
+              check_pool(pool)
+            end
+          end
+        end
+
+        sleep(1)
+      end
+    end
+
+  end
+end
+
diff --git a/public/dashboard.css b/lib/vmpooler/public/dashboard.css
similarity index 100%
rename from public/dashboard.css
rename to lib/vmpooler/public/dashboard.css
diff --git a/public/img/logo.jpg b/lib/vmpooler/public/img/logo.jpg
similarity index 100%
rename from public/img/logo.jpg
rename to lib/vmpooler/public/img/logo.jpg
diff --git a/public/img/spinner.svg b/lib/vmpooler/public/img/spinner.svg
similarity index 100%
rename from public/img/spinner.svg
rename to lib/vmpooler/public/img/spinner.svg
diff --git a/public/lib/stats-vmpooler-numbers.js b/lib/vmpooler/public/lib/stats-vmpooler-numbers.js
similarity index 100%
rename from public/lib/stats-vmpooler-numbers.js
rename to lib/vmpooler/public/lib/stats-vmpooler-numbers.js
diff --git a/public/lib/stats-vmpooler-pool.js b/lib/vmpooler/public/lib/stats-vmpooler-pool.js
similarity index 100%
rename from public/lib/stats-vmpooler-pool.js
rename to lib/vmpooler/public/lib/stats-vmpooler-pool.js
diff --git a/public/lib/stats-vmpooler-running.js b/lib/vmpooler/public/lib/stats-vmpooler-running.js
similarity index 100%
rename from public/lib/stats-vmpooler-running.js
rename to lib/vmpooler/public/lib/stats-vmpooler-running.js
diff --git a/views/dashboard.erb b/lib/vmpooler/views/dashboard.erb
similarity index 100%
rename from views/dashboard.erb
rename to lib/vmpooler/views/dashboard.erb
diff --git a/views/layout.erb b/lib/vmpooler/views/layout.erb
similarity index 100%
rename from views/layout.erb
rename to lib/vmpooler/views/layout.erb
diff --git a/lib/vmpooler/vsphere_helper.rb b/lib/vmpooler/vsphere_helper.rb
new file mode 100644
index 0000000..058e859
--- /dev/null
+++ b/lib/vmpooler/vsphere_helper.rb
@@ -0,0 +1,180 @@
+require 'rubygems' unless defined?(Gem)
+
+module Vmpooler
+  class VsphereHelper
+    def initialize vInfo = {}
+      begin
+        require 'rbvmomi'
+      rescue LoadError
+        raise "Unable to load RbVmomi, please ensure its installed"
+      end
+
+      config_file = File.expand_path('vmpooler.yaml')
+      vsphere = YAML.load_file(config_file)[:vsphere]
+
+      @connection = RbVmomi::VIM.connect :host     => vsphere['server'],
+                                         :user     => vsphere['username'],
+                                         :password => vsphere['password'],
+                                         :insecure => true
+    end
+
+
+
+    def find_datastore datastorename
+      begin
+        @connection.serviceInstance.CurrentTime
+      rescue
+        initialize()
+      end
+
+      datacenter = @connection.serviceInstance.find_datacenter
+      datacenter.find_datastore(datastorename)
+    end
+
+
+
+    def find_folder foldername
+      begin
+        @connection.serviceInstance.CurrentTime
+      rescue
+        initialize()
+      end
+
+      datacenter = @connection.serviceInstance.find_datacenter
+      base = datacenter.vmFolder
+      folders = foldername.split('/')
+      folders.each do |folder|
+        case base
+          when RbVmomi::VIM::Folder
+            base = base.childEntity.find { |f| f.name == folder }
+          else
+            abort "Unexpected object type encountered (#{base.class}) while finding folder"
+        end
+      end
+
+      base
+    end
+
+
+
+    def find_pool poolname
+      begin
+        @connection.serviceInstance.CurrentTime
+      rescue
+        initialize()
+      end
+
+      datacenter = @connection.serviceInstance.find_datacenter
+      base = datacenter.hostFolder
+      pools = poolname.split('/')
+      pools.each do |pool|
+        case base
+          when RbVmomi::VIM::Folder
+            base = base.childEntity.find { |f| f.name == pool }
+          when RbVmomi::VIM::ClusterComputeResource
+            base = base.resourcePool.resourcePool.find { |f| f.name == pool }
+          when RbVmomi::VIM::ResourcePool
+            base = base.resourcePool.find { |f| f.name == pool }
+          else
+            abort "Unexpected object type encountered (#{base.class}) while finding resource pool"
+        end
+      end
+
+      base = base.resourcePool unless base.is_a?(RbVmomi::VIM::ResourcePool) and base.respond_to?(:resourcePool)
+      base
+    end
+
+
+
+    def find_vm vmname
+      begin
+        @connection.serviceInstance.CurrentTime
+      rescue
+        initialize()
+      end
+
+      @connection.searchIndex.FindByDnsName(:vmSearch => true, :dnsName => vmname)
+    end
+
+
+
+    def find_vm_heavy vmname
+      begin
+        @connection.serviceInstance.CurrentTime
+      rescue
+        initialize()
+      end
+
+      vmname = vmname.is_a?(Array) ? vmname : [ vmname ]
+      containerView = get_base_vm_container_from @connection
+      propertyCollector = @connection.propertyCollector
+
+      objectSet = [{
+        :obj => containerView,
+        :skip => true,
+        :selectSet => [ RbVmomi::VIM::TraversalSpec.new({
+            :name => 'gettingTheVMs',
+            :path => 'view',
+           :skip => false,
+            :type => 'ContainerView'
+        }) ]
+      }]
+
+      propSet = [{
+        :pathSet => [ 'name' ],
+        :type => 'VirtualMachine'
+      }]
+
+      results = propertyCollector.RetrievePropertiesEx({
+        :specSet => [{
+          :objectSet => objectSet,
+          :propSet   => propSet
+        }],
+        :options => { :maxObjects => nil }
+      })
+
+      vms = {}
+      results.objects.each do |result|
+        name = result.propSet.first.val
+        next unless vmname.include? name
+        vms[name] = result.obj
+      end
+
+      while results.token do
+        results = propertyCollector.ContinueRetrievePropertiesEx({:token => results.token})
+        results.objects.each do |result|
+          name = result.propSet.first.val
+          next unless vmname.include? name
+          vms[name] = result.obj
+        end
+      end
+
+      vms
+    end
+
+
+
+    def get_base_vm_container_from connection
+      begin
+        connection.serviceInstance.CurrentTime
+      rescue
+        initialize()
+      end
+
+      viewManager = connection.serviceContent.viewManager
+      viewManager.CreateContainerView({
+        :container => connection.serviceContent.rootFolder,
+        :recursive => true,
+        :type      => [ 'VirtualMachine' ]
+      })
+    end
+
+
+
+    def close
+      @connection.close
+    end
+
+  end
+end
+
diff --git a/lib/vsphere_helper.rb b/lib/vsphere_helper.rb
deleted file mode 100755
index 1dfa018..0000000
--- a/lib/vsphere_helper.rb
+++ /dev/null
@@ -1,180 +0,0 @@
-require 'yaml' unless defined?(YAML)
-require 'rubygems' unless defined?(Gem)
-
-class VsphereHelper
-  def initialize vInfo = {}
-    begin
-      require 'rbvmomi'
-    rescue LoadError
-      raise "Unable to load RbVmomi, please ensure its installed"
-    end
-
-    Dir.chdir(File.dirname(__FILE__))
-
-    config_file = File.expand_path('../vmpooler.yaml')
-    vsphere = YAML.load_file(config_file)[:vsphere]
-
-    @connection = RbVmomi::VIM.connect :host     => vsphere['server'],
-                                       :user     => vsphere['username'],
-                                       :password => vsphere['password'],
-                                       :insecure => true
-  end
-
-
-
-  def find_datastore datastorename
-    begin
-      @connection.serviceInstance.CurrentTime
-    rescue
-      initialize()
-    end
-
-    datacenter = @connection.serviceInstance.find_datacenter
-    datacenter.find_datastore(datastorename)
-  end
-
-
-
-  def find_folder foldername
-    begin
-      @connection.serviceInstance.CurrentTime
-    rescue
-      initialize()
-    end
-
-    datacenter = @connection.serviceInstance.find_datacenter
-    base = datacenter.vmFolder
-    folders = foldername.split('/')
-    folders.each do |folder|
-      case base
-        when RbVmomi::VIM::Folder
-          base = base.childEntity.find { |f| f.name == folder }
-        else
-          abort "Unexpected object type encountered (#{base.class}) while finding folder"
-      end
-    end
-
-    base
-  end
-
-
-
-  def find_pool poolname
-    begin
-      @connection.serviceInstance.CurrentTime
-    rescue
-      initialize()
-    end
-
-    datacenter = @connection.serviceInstance.find_datacenter
-    base = datacenter.hostFolder
-    pools = poolname.split('/')
-    pools.each do |pool|
-      case base
-        when RbVmomi::VIM::Folder
-          base = base.childEntity.find { |f| f.name == pool }
-        when RbVmomi::VIM::ClusterComputeResource
-          base = base.resourcePool.resourcePool.find { |f| f.name == pool }
-        when RbVmomi::VIM::ResourcePool
-          base = base.resourcePool.find { |f| f.name == pool }
-        else
-          abort "Unexpected object type encountered (#{base.class}) while finding resource pool"
-      end
-    end
-
-    base = base.resourcePool unless base.is_a?(RbVmomi::VIM::ResourcePool) and base.respond_to?(:resourcePool)
-    base
-  end
-
-
-
-  def find_vm vmname
-    begin
-      @connection.serviceInstance.CurrentTime
-    rescue
-      initialize()
-    end
-
-    @connection.searchIndex.FindByDnsName(:vmSearch => true, :dnsName => vmname)
-  end
-
-
-
-  def find_vm_heavy vmname
-    begin
-      @connection.serviceInstance.CurrentTime
-    rescue
-      initialize()
-    end
-
-    vmname = vmname.is_a?(Array) ? vmname : [ vmname ]
-    containerView = get_base_vm_container_from @connection
-    propertyCollector = @connection.propertyCollector
-
-    objectSet = [{
-      :obj => containerView,
-      :skip => true,
-      :selectSet => [ RbVmomi::VIM::TraversalSpec.new({
-          :name => 'gettingTheVMs',
-          :path => 'view',
-          :skip => false,
-          :type => 'ContainerView'
-      }) ]
-    }]
-
-    propSet = [{
-      :pathSet => [ 'name' ],
-      :type => 'VirtualMachine'
-    }]
-
-    results = propertyCollector.RetrievePropertiesEx({
-      :specSet => [{
-        :objectSet => objectSet,
-        :propSet   => propSet
-      }],
-      :options => { :maxObjects => nil }
-    })
-
-    vms = {}
-    results.objects.each do |result|
-      name = result.propSet.first.val
-      next unless vmname.include? name
-      vms[name] = result.obj
-    end
-
-    while results.token do
-      results = propertyCollector.ContinueRetrievePropertiesEx({:token => results.token})
-      results.objects.each do |result|
-        name = result.propSet.first.val
-        next unless vmname.include? name
-        vms[name] = result.obj
-      end
-    end
-
-    vms
-  end
-
-
-
-  def get_base_vm_container_from connection
-    begin
-      connection.serviceInstance.CurrentTime
-    rescue
-      initialize()
-    end
-
-    viewManager = connection.serviceContent.viewManager
-    viewManager.CreateContainerView({
-      :container => connection.serviceContent.rootFolder,
-      :recursive => true,
-      :type      => [ 'VirtualMachine' ]
-    })
-  end
-
-
-
-  def close
-    @connection.close
-  end
-end
-
diff --git a/vmpooler b/vmpooler
index a304eca..0b720ad 100755
--- a/vmpooler
+++ b/vmpooler
@@ -1,463 +1,14 @@
-#!/usr/bin/ruby
-
-require 'json'
-require 'rbvmomi'
-require 'redis'
-require 'time'
-require 'timeout'
-require 'yaml'
+#!/usr/bin/env ruby
 
 $:.unshift(File.dirname(__FILE__))
-require 'lib/logger'
-require 'lib/require_relative'
-require 'lib/vsphere_helper'
 
-Dir.chdir(File.dirname(__FILE__))
+require 'rubygems' unless defined?(Gem)
+require 'lib/vmpooler'
 
-# Load the configuration file
-config_file = File.expand_path('vmpooler.yaml')
-$config = YAML.load_file(config_file)
-
-pools   = $config[:pools]
-vsphere = $config[:vsphere]
-redis   = $config[:redis]
-
-# Load logger library
-$logger = Logger.new $config[:config]['logfile']
-
-# Load Graphite helper library (if configured)
-if (defined? $config[:config]['graphite'])
-  require 'lib/graphite'
-  $graphite = Graphite.new $config[:config]['graphite']
-end
-
-# Set some defaults
-$config[:config]['task_limit']   ||= 10
-$config[:config]['vm_checktime'] ||= 15
-$config[:config]['vm_lifetime']  ||= 24
-$config[:redis] ||= Hash.new
-$config[:redis]['server'] ||= 'localhost'
-
-# Connect to Redis
-$redis = Redis.new(:host => $config[:redis]['server'])
-
-# vSphere object
-$vsphere = {}
-
-# Our thread-tracker object
-$threads = {}
-
-
-
-# Check the state of a VM
-def check_pending_vm vm, pool, timeout
-  Thread.new {
-    host = $vsphere[pool].find_vm(vm)
-
-    if (host)
-      if (
-        (host.summary) and
-        (host.summary.guest) and
-        (host.summary.guest.hostName) and
-        (host.summary.guest.hostName == vm)
-      )
-        begin
-          Socket.getaddrinfo(vm, nil)
-        rescue
-        end
-
-        $redis.smove('vmpooler__pending__'+pool, 'vmpooler__ready__'+pool, vm)
-
-        $logger.log('s', "[>] [#{pool}] '#{vm}' moved to 'ready' queue")
-      end
-    else
-      clone_stamp = $redis.hget('vmpooler__vm__'+vm, 'clone')
-
-      if (
-        (clone_stamp) and
-        (((Time.now - Time.parse(clone_stamp))/60) > timeout)
-      )
-        $redis.smove('vmpooler__pending__'+pool, 'vmpooler__completed__'+pool, vm)
-
-        $logger.log('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes")
-      end
-    end
-  }
-end
-
-def check_ready_vm vm, pool, ttl
-  Thread.new {
-    if (ttl > 0)
-      if ((((Time.now - host.runtime.bootTime)/60).to_s[/^\d+\.\d{1}/].to_f) > ttl)
-        $redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm)
-
-        $logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue")
-      end
-    end
-
-    check_stamp = $redis.hget('vmpooler__vm__'+vm, 'check')
-
-    if (
-      (! check_stamp) or
-      (((Time.now - Time.parse(check_stamp))/60) > $config[:config]['vm_checktime'])
-    )
-      $redis.hset('vmpooler__vm__'+vm, 'check', Time.now)
-
-      host = $vsphere[pool].find_vm(vm) ||
-             $vsphere[pool].find_vm_heavy(vm)[vm]
-
-      if (host)
-        if (
-          (host.runtime) and
-          (host.runtime.powerState) and
-          (host.runtime.powerState != 'poweredOn')
-        )
-          $redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm)
-
-          $logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off, removed from 'ready' queue")
-        end
-
-        if (
-          (host.summary.guest) and
-          (host.summary.guest.hostName) and
-          (host.summary.guest.hostName != vm)
-        )
-          $redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm)
-
-          $logger.log('d', "[!] [#{pool}] '#{vm}' has mismatched hostname, removed from 'ready' queue")
-        end
-      else
-        $redis.srem('vmpooler__ready__'+pool, vm)
-
-        $logger.log('s', "[!] [#{pool}] '#{vm}' not found in vCenter inventory, removed from 'ready' queue")
-      end
-
-      begin
-        Timeout::timeout(5) {
-          TCPSocket.new vm, 22
-        }
-      rescue
-        if ($redis.smove('vmpooler__ready__'+pool, 'vmpooler__completed__'+pool, vm))
-          $logger.log('d', "[!] [#{pool}] '#{vm}' is unreachable, removed from 'ready' queue")
-        end
-      end
-    end
-  }
-end
-
-def check_running_vm vm, pool, ttl
-  Thread.new {
-    host = $vsphere[pool].find_vm(vm)
-
-    if (host)
-      if (
-        (host.runtime) and
-        (host.runtime.powerState != 'poweredOn')
-      )
-        $redis.smove('vmpooler__running__'+pool, 'vmpooler__completed__'+pool, vm)
-
-        $logger.log('d', "[!] [#{pool}] '#{vm}' appears to be powered off or dead")
-      else
-        if (
-          (host.runtime) and
-          (host.runtime.bootTime)
-          ((((Time.now - host.runtime.bootTime)/60).to_s[/^\d+\.\d{1}/].to_f) > ttl)
-        )
-          $redis.smove('vmpooler__running__'+pool, 'vmpooler__completed__'+pool, vm)
-
-          $logger.log('d', "[!] [#{pool}] '#{vm}' reached end of TTL after #{ttl} minutes")
-        end
-      end
-    end
-  }
-end
-
-# Clone a VM
-def clone_vm template, pool, folder, datastore
-  Thread.new {
-    vm = {}
-
-    if template =~ /\//
-      templatefolders = template.split('/')
-      vm['template'] = templatefolders.pop
-    end
-
-    if templatefolders
-      vm[vm['template']] = $vsphere[vm['template']].find_folder(templatefolders.join('/')).find(vm['template'])
-    else
-      raise "Please provide a full path to the template"
-    end
-
-    if vm['template'].length == 0
-      raise "Unable to find template '#{vm['template']}'!"
-    end
-
-    # Generate a randomized hostname
-    o = [('a'..'z'),('0'..'9')].map{|r| r.to_a}.flatten
-    vm['hostname'] = o[rand(25)]+(0...14).map{o[rand(o.length)]}.join
-
-    # Add VM to Redis inventory ('pending' pool)
-    $redis.sadd('vmpooler__pending__'+vm['template'], vm['hostname'])
-    $redis.hset('vmpooler__vm__'+vm['hostname'], 'clone', Time.now)
-
-    # Annotate with creation time, origin template, etc.
-    configSpec = RbVmomi::VIM.VirtualMachineConfigSpec(
-      :annotation => JSON.pretty_generate({
-          name: vm['hostname'],
-          created_by: $config[:vsphere]['username'],
-          base_template: vm['template'],
-          creation_timestamp: Time.now.utc
-       })
-    )
-
-    # Put the VM in the specified folder and resource pool
-    relocateSpec = RbVmomi::VIM.VirtualMachineRelocateSpec(
-      :datastore    => $vsphere[vm['template']].find_datastore(datastore),
-      :pool         => $vsphere[vm['template']].find_pool(pool),
-      :diskMoveType => :moveChildMostDiskBacking
-    )
-
-    # Create a clone spec
-    spec = RbVmomi::VIM.VirtualMachineCloneSpec(
-      :location      => relocateSpec,
-      :config        => configSpec,
-      :powerOn       => true,
-      :template      => false
-    )
-
-    # Clone the VM
-    $logger.log('d', "[ ] [#{vm['template']}] '#{vm['hostname']}' is being cloned from '#{vm['template']}'")
-
-    begin
-      start = Time.now
-      vm[vm['template']].CloneVM_Task(
-        :folder => $vsphere[vm['template']].find_folder(folder),
-        :name => vm['hostname'],
-        :spec => spec
-      ).wait_for_completion
-      finish = '%.2f' % (Time.now-start)
-
-      $logger.log('s', "[+] [#{vm['template']}] '#{vm['hostname']}' cloned from '#{vm['template']}' in #{finish} seconds")
-    rescue
-      $logger.log('s', "[!] [#{vm['template']}] '#{vm['hostname']}' clone appears to have failed")
-      $redis.srem('vmpooler__pending__'+vm['template'], vm['hostname'])
-    end
-
-    $redis.decr('vmpooler__tasks__clone')
-
-    begin
-      $graphite.log("vmpooler.clone.#{vm['template']}", finish) if defined? $graphite
-    rescue
-    end
-  }
-end
-
-# Destroy a VM
-def destroy_vm vm, pool
-  Thread.new {
-    $redis.srem('vmpooler__completed__'+pool, vm)
-    $redis.hdel('vmpooler__active__'+pool, vm)
-    $redis.del('vmpooler__vm__'+vm)
-
-    host = $vsphere[pool].find_vm(vm) ||
-           $vsphere[pool].find_vm_heavy(vm)[vm]
-
-    if (host)
-      start = Time.now
-
-      if (
-        (host.runtime) and
-        (host.runtime.powerState) and
-        (host.runtime.powerState == 'poweredOn')
-      )
-        $logger.log('d', "[ ] [#{pool}] '#{vm}' is being shut down")
-        host.PowerOffVM_Task.wait_for_completion
-      end
-
-      host.Destroy_Task.wait_for_completion
-      finish = '%.2f' % (Time.now-start)
-
-      $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds")
-
-      $graphite.log("vmpooler.destroy.#{pool}", finish) if defined? $graphite
-    end
-  }
-end
-
-def check_pool pool
-  $logger.log('d', "[*] [#{pool['name']}] starting worker thread")
-
-  $threads[pool['name']] = Thread.new {
-    $vsphere[pool['name']] ||= VsphereHelper.new
-
-    loop do
-      # INVENTORY
-      inventory = {}
-      begin
-        base = $vsphere[pool['name']].find_pool(pool['pool'])
-
-        base.vm.each do |vm|
-          if (
-            (! $redis.sismember('vmpooler__running__'+pool['name'], vm['name'])) and
-            (! $redis.sismember('vmpooler__ready__'+pool['name'], vm['name'])) and
-            (! $redis.sismember('vmpooler__pending__'+pool['name'], vm['name'])) and
-            (! $redis.sismember('vmpooler__completed__'+pool['name'], vm['name'])) and
-            (! $redis.sismember('vmpooler__discovered__'+pool['name'], vm['name']))
-          )
-            $redis.sadd('vmpooler__discovered__'+pool['name'], vm['name'])
-
-            $logger.log('s', "[?] [#{pool['name']}] '#{vm['name']}' added to 'discovered' queue")
-          end
-
-          inventory[vm['name']] = 1
-        end
-      rescue
-      end
-
-      # RUNNING
-      $redis.smembers('vmpooler__running__'+pool['name']).each do |vm|
-        if (inventory[vm])
-          if (pool['running_ttl'])
-            begin
-              check_running_vm(vm, pool['name'], pool['running_ttl'])
-            rescue
-            end
-          else
-            begin
-              check_running_vm(vm, pool['name'], '720')
-            rescue
-            end
-          end
-        end
-      end
-
-      # READY
-      $redis.smembers('vmpooler__ready__'+pool['name']).each do |vm|
-        if (inventory[vm])
-          begin
-            check_ready_vm(vm, pool['name'], pool['ready_ttl'] || 0)
-          rescue
-          end
-        end
-      end
-
-      # PENDING
-      $redis.smembers('vmpooler__pending__'+pool['name']).each do |vm|
-        pool['timeout'] ||= 15
-
-        if (inventory[vm])
-          begin
-            check_pending_vm(vm, pool['name'], pool['timeout'])
-          rescue
-          end
-        end
-      end
-
-      # COMPLETED
-      $redis.smembers('vmpooler__completed__'+pool['name']).each do |vm|
-        if (inventory[vm])
-          begin
-            destroy_vm(vm, pool['name'])
-          rescue
-            $logger.log('s', "[!] [#{pool['name']}] '#{vm}' destroy appears to have failed")
-            $redis.srem('vmpooler__completed__'+pool['name'], vm)
-            $redis.hdel('vmpooler__active__'+pool['name'], vm)
-            $redis.del('vmpooler__vm__'+vm)
-          end
-        else
-          $logger.log('s', "[!] [#{pool['name']}] '#{vm}' not found in inventory, removed from 'completed' queue")
-          $redis.srem('vmpooler__completed__'+pool['name'], vm)
-          $redis.hdel('vmpooler__active__'+pool['name'], vm)
-          $redis.del('vmpooler__vm__'+vm)
-        end
-      end
-
-      # DISCOVERED
-      $redis.smembers('vmpooler__discovered__'+pool['name']).each do |vm|
-        ['pending', 'ready', 'running', 'completed'].each do |queue|
-          if ($redis.sismember('vmpooler__'+queue+'__'+pool['name'], vm))
-            $logger.log('d', "[!] [#{pool['name']}] '#{vm}' found in '#{queue}', removed from 'discovered' queue")
-            $redis.srem('vmpooler__discovered__'+pool['name'], vm)
-          end
-        end
-
-        if ($redis.sismember('vmpooler__discovered__'+pool['name'], vm))
-          $redis.smove('vmpooler__discovered__'+pool['name'], 'vmpooler__completed__'+pool['name'], vm)
-        end
-      end
-
-      # LONG-RUNNING
-      $redis.smembers('vmpooler__running__'+pool['name']).each do |vm|
-        if ($redis.hget('vmpooler__active__'+pool['name'], vm))
-          running = (Time.now - Time.parse($redis.hget('vmpooler__active__'+pool['name'], vm)))/60/60
-          if (
-            ($config[:config]['vm_lifetime'] > 0) and
-            (running > $config[:config]['vm_lifetime'])
-          )
-            $redis.smove('vmpooler__running__'+pool['name'], 'vmpooler__completed__'+pool['name'], vm)
-
-            $logger.log('d', "[!] [#{pool['name']}] '#{vm}' reached end of TTL after #{$config[:config]['vm_lifetime']} hours")
-          end
-        end
-      end
-
-      # REPOPULATE
-      total = $redis.scard('vmpooler__ready__'+pool['name']) +
-              $redis.scard('vmpooler__pending__'+pool['name'])
-
-      begin
-        if (defined? $graphite)
-          $graphite.log('vmpooler.ready.'+pool['name'], $redis.scard('vmpooler__ready__'+pool['name']))
-          $graphite.log('vmpooler.running.'+pool['name'], $redis.scard('vmpooler__running__'+pool['name']))
-        end
-      rescue
-      end
-
-      if (total < pool['size'])
-        (1..(pool['size'] - total)).each { |i|
-
-          if ($redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit'])
-            begin
-              $redis.incr('vmpooler__tasks__clone')
-
-              clone_vm(
-                pool['template'],
-                pool['pool'],
-                pool['folder'],
-                pool['datastore']
-              )
-            rescue
-              $logger.log('s', "[!] [#{pool['name']}] clone appears to have failed")
-              $redis.decr('vmpooler__tasks__clone')
-            end
-          end
-        }
-      end
-
-      sleep(1)
-    end
-  }
-end
-
-
-
-$logger.log('d', "starting vmpooler")
-
-# Clear out the tasks manager, as we don't know about any tasks at this point
-$redis.set('vmpooler__tasks__clone', 0)
+Thread.new { Vmpooler::API.new.execute! }
+Thread.new { Vmpooler::PoolManager.new.execute! }
 
 loop do
-  pools.each do |pool|
-    if (! $threads[pool['name']])
-      check_pool(pool)
-    else
-      if (! $threads[pool['name']].alive?)
-        $logger.log('d', "[!] [#{pool['name']}] worker thread died, restarting")
-        check_pool(pool)
-      end
-    end
-  end
-
   sleep(1)
 end
 
diff --git a/vmpooler-api b/vmpooler-api
deleted file mode 100755
index 6ecf611..0000000
--- a/vmpooler-api
+++ /dev/null
@@ -1,231 +0,0 @@
-#!/usr/bin/ruby
-
-require 'rubygems'
-
-require 'json'
-require 'open-uri'
-require 'redis'
-require 'sinatra'
-require 'yaml'
-
-$:.unshift(File.dirname(__FILE__))
-require 'lib/logger'
-require 'lib/require_relative'
-
-Dir.chdir(File.dirname(__FILE__))
-
-# Load the configuration file
-config_file = File.expand_path('vmpooler.yaml')
-$config = YAML.load_file(config_file)
-
-pools = $config[:pools]
-redis = $config[:redis]
-
-# Load logger library
-$logger = Logger.new $config[:config]['logfile']
-
-# Set some defaults
-$config[:redis] ||= Hash.new
-$config[:redis]['server'] ||= 'localhost'
-
-# Connect to Redis
-$redis = Redis.new(:host => $config[:redis]['server'])
-
-# Sinatra!
-get '/' do
-  erb :dashboard, locals: {
-    site_name: $config[:config]['site_name'] || '<b>vmpooler</b>',
-  }
-end
-
-get '/dashboard/stats/vmpooler/numbers/?' do
-  result = Hash.new
-  result['pending'] = 0
-  result['cloning'] = 0
-  result['booting'] = 0
-  result['ready'] = 0
-  result['running'] = 0
-  result['completed'] = 0
-
-  $config[:pools].each do |pool|
-    result['pending'] += $redis.scard( 'vmpooler__pending__' + pool['name'] )
-    result['ready'] += $redis.scard( 'vmpooler__ready__' + pool['name'] )
-    result['running'] += $redis.scard( 'vmpooler__running__' + pool['name'] )
-    result['completed'] += $redis.scard( 'vmpooler__completed__' + pool['name'] )
-  end
-
-  result['cloning'] = $redis.get( 'vmpooler__tasks__clone' )
-  result['booting'] = result['pending'].to_i - result['cloning'].to_i
-  result['booting'] = 0 if result['booting'] < 0
-  result['total'] = result['pending'].to_i + result['ready'].to_i + result['running'].to_i + result['completed'].to_i
-
-  content_type :json
-  JSON.pretty_generate(result)
-end
-
-get '/dashboard/stats/vmpooler/pool/?' do
-  result = Hash.new
-
-  $config[:pools].each do |pool|
-    result[pool['name']] ||= Hash.new
-    result[pool['name']]['size'] = pool['size']
-    result[pool['name']]['ready'] = $redis.scard( 'vmpooler__ready__' + pool['name'] )
-  end
-
-  if ( params[:history] )
-    if ( $config[:config]['graphite'] )
-      history ||= Hash.new
-
-      begin
-        buffer = open( 'http://'+$config[:config]['graphite']+'/render?target=vmpooler.ready.*&from=-1hour&format=json' ).read
-        history = JSON.parse( buffer )
-
-        history.each do |pool|
-          if pool['target'] =~ /.*\.(.*)$/
-            pool['name'] = $1
-
-            if ( result[pool['name']] )
-              pool['last'] = result[pool['name']]['size']
-              result[pool['name']]['history'] ||= Array.new
-
-              pool['datapoints'].each do |metric|
-                8.times do |n|
-                  if ( metric[0] )
-                    pool['last'] = metric[0].to_i
-                    result[pool['name']]['history'].push( metric[0].to_i )
-                  else
-                    result[pool['name']]['history'].push( pool['last'] )
-                  end
-                end
-              end
-            end
-          end
-        end
-      rescue
-      end
-    else
-      $config[:pools].each do |pool|
-        result[pool['name']] ||= Hash.new
-        result[pool['name']]['history'] = [ $redis.scard( 'vmpooler__ready__' + pool['name'] ) ]
-      end
-    end
-  end
-
-  content_type :json
-  JSON.pretty_generate(result)
-end
-
-get '/dashboard/stats/vmpooler/running/?' do
-  result = Hash.new
-
-  $config[:pools].each do |pool|
-    running = $redis.scard( 'vmpooler__running__' + pool['name'] )
-    pool['major'] = $1 if pool['name'] =~ /^(\w+)\-/
-
-    result[pool['major']] ||= Hash.new
-
-    result[pool['major']]['running'] = result[pool['major']]['running'].to_i + running.to_i
-  end
-
-  if ( params[:history] )
-    if ( $config[:config]['graphite'] )
-      begin
-        buffer = open( 'http://'+$config[:config]['graphite']+'/render?target=vmpooler.running.*&from=-1hour&format=json' ).read
-        JSON.parse( buffer ).each do |pool|
-          if pool['target'] =~ /.*\.(.*)$/
-            pool['name'] = $1
-
-            pool['major'] = $1 if pool['name'] =~ /^(\w+)\-/
-
-            result[pool['major']]['history'] ||= Array.new
-
-            for i in 0..pool['datapoints'].length
-              if (
-                pool['datapoints'][i] and
-                pool['datapoints'][i][0]
-              )
-                pool['last'] = pool['datapoints'][i][0]
-
-                result[pool['major']]['history'][i] ||= 0
-                result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['datapoints'][i][0].to_i
-              else
-                result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['last'].to_i
-              end
-            end
-
-          end
-        end
-      rescue
-      end
-    end
-  end
-
-  content_type :json
-  JSON.pretty_generate(result)
-end
-
-get '/vm/?' do
-  content_type :json
-
-  result = []
-
-  pools.each do |pool|
-    result.push(pool['name'])
-  end
-
-  JSON.pretty_generate(result)
-end
-
-get '/vm/:template/?' do
-  content_type :json
-
-  result = {}
-  result[params[:template]] = {}
-  result[params[:template]]['hosts'] = $redis.smembers('vmpooler__ready__'+params[:template])
-
-  JSON.pretty_generate(result)
-end
-
-post '/vm/:template/?' do
-  content_type :json
-
-  result = {}
-  result[params[:template]] = {}
-
-  if ( $redis.scard('vmpooler__ready__'+params[:template]) > 0 )
-    vm = $redis.spop('vmpooler__ready__'+params[:template])
-
-    unless (vm.nil?)
-      $redis.sadd('vmpooler__running__'+params[:template], vm)
-      $redis.hset('vmpooler__active__'+params[:template], vm, Time.now)
-
-      result[params[:template]]['ok'] = true
-      result[params[:template]]['hostname'] = vm
-    else
-      result[params[:template]]['ok'] = false
-    end
-  else
-    result[params[:template]]['ok'] = false
-  end
-
-  JSON.pretty_generate(result)
-end
-
-delete '/vm/:hostname/?' do
-  content_type :json
-
-  result = {}
-
-  result['ok'] = false
-
-  pools.each do |pool|
-    if $redis.sismember('vmpooler__running__'+pool['name'], params[:hostname])
-      $redis.srem('vmpooler__running__'+pool['name'], params[:hostname])
-      $redis.sadd('vmpooler__completed__'+pool['name'], params[:hostname])
-      result['ok'] = true
-    end
-  end
-
-  JSON.pretty_generate(result)
-end
-

From 60eead64558b804e9fe6f46695a53036cf81d397 Mon Sep 17 00:00:00 2001
From: Scott Schneider <sschneider@puppetlabs.com>
Date: Wed, 5 Mar 2014 13:19:43 -0800
Subject: [PATCH 2/2] Centralize external Gem loading

---
 lib/vmpooler.rb                | 9 +++++++++
 lib/vmpooler/api.rb            | 7 -------
 lib/vmpooler/pool_manager.rb   | 7 -------
 lib/vmpooler/vsphere_helper.rb | 6 ------
 4 files changed, 9 insertions(+), 20 deletions(-)

diff --git a/lib/vmpooler.rb b/lib/vmpooler.rb
index 4e4d901..ef820da 100644
--- a/lib/vmpooler.rb
+++ b/lib/vmpooler.rb
@@ -1,6 +1,15 @@
 require 'rubygems' unless defined?(Gem)
 
 module Vmpooler
+  require 'json'
+  require 'open-uri'
+  require 'rbvmomi'
+  require 'redis'
+  require 'sinatra/base'
+  require 'time'
+  require 'timeout'
+  require 'yaml'
+
   %w( api graphite logger pool_manager vsphere_helper ).each do |lib|
     begin
       require "vmpooler/#{lib}"
diff --git a/lib/vmpooler/api.rb b/lib/vmpooler/api.rb
index 4a84234..c2d6b57 100644
--- a/lib/vmpooler/api.rb
+++ b/lib/vmpooler/api.rb
@@ -1,13 +1,6 @@
 module Vmpooler
   class API
     def initialize
-      require 'sinatra/base'
-
-      require 'json'
-      require 'open-uri'
-      require 'redis'
-      require 'yaml'
-
       # Load the configuration file
       config_file = File.expand_path('vmpooler.yaml')
       $config = YAML.load_file(config_file)
diff --git a/lib/vmpooler/pool_manager.rb b/lib/vmpooler/pool_manager.rb
index c85ec21..20e25f3 100644
--- a/lib/vmpooler/pool_manager.rb
+++ b/lib/vmpooler/pool_manager.rb
@@ -1,13 +1,6 @@
 module Vmpooler
   class PoolManager
     def initialize
-      require 'json'
-      require 'rbvmomi'
-      require 'redis'
-      require 'time'
-      require 'timeout'
-      require 'yaml'
-
       # Load the configuration file
       config_file = File.expand_path('vmpooler.yaml')
       $config = YAML.load_file(config_file)
diff --git a/lib/vmpooler/vsphere_helper.rb b/lib/vmpooler/vsphere_helper.rb
index 058e859..402530b 100644
--- a/lib/vmpooler/vsphere_helper.rb
+++ b/lib/vmpooler/vsphere_helper.rb
@@ -3,12 +3,6 @@ require 'rubygems' unless defined?(Gem)
 module Vmpooler
   class VsphereHelper
     def initialize vInfo = {}
-      begin
-        require 'rbvmomi'
-      rescue LoadError
-        raise "Unable to load RbVmomi, please ensure its installed"
-      end
-
       config_file = File.expand_path('vmpooler.yaml')
       vsphere = YAML.load_file(config_file)[:vsphere]