(POOLER-112) Ensure a VM is only destroyed once

This commit implements a vm_mutex hash to allow synchronizing VM operations that should only happen once across threads. Without this change pool_manager will try to evaluate or destroy a VM multiple times, which results in an error being thrown by one of the destroy attempts as only one can succeed and a duplication of resources unnecessarily when there are no errors.

spec/unit/pool_manager_spec.rb

@@ -92,6 +92,19 @@ EOT
         subject._check_pending_vm(vm, pool, timeout, provider)
       end
     end
+
+    context 'with a locked vm mutex' do
+      let(:mutex) { Mutex.new }
+      before(:each) do
+        mutex.lock
+      end
+
+      it 'should return' do
+        expect(subject).to receive(:vm_mutex).and_return(mutex)
+
+        expect(subject._check_pending_vm(vm, pool, timeout, provider)).to be_nil
+      end
+    end
   end
 
   describe '#remove_nonexistent_vm' do
@@ -404,6 +417,19 @@ EOT
         end
       end
     end
+
+    context 'with a locked vm mutex' do
+      let(:mutex) { Mutex.new }
+      before(:each) do
+        mutex.lock
+      end
+
+      it 'should return' do
+        expect(subject).to receive(:vm_mutex).and_return(mutex)
+
+        expect(subject._check_ready_vm(vm, pool, ttl, provider)).to be_nil
+      end
+    end
   end
 
   describe '#check_running_vm' do
@@ -479,6 +505,19 @@ EOT
         expect(redis.sismember("vmpooler__completed__#{pool}", vm)).to be(true)
       end
     end
+
+    context 'with a locked vm mutex' do
+      let(:mutex) { Mutex.new }
+      before(:each) do
+        mutex.lock
+      end
+
+      it 'should return' do
+        expect(subject).to receive(:vm_mutex).and_return(mutex)
+
+        expect(subject._check_running_vm(vm, pool, timeout, provider)).to be_nil
+      end
+    end
   end
 
   describe '#move_vm_queue' do
@@ -681,7 +720,7 @@ EOT
       before(:each) do
         config[:redis] = nil
       end
-      
+
       it 'should raise an error' do
         expect{ subject._destroy_vm(vm,pool,provider) }.to raise_error(NoMethodError)
       end
@@ -732,6 +771,19 @@ EOT
         expect{ subject._destroy_vm(vm,pool,provider) }.to raise_error(/MockError/)
       end
     end
+
+    context 'when the VM mutex is locked' do
+      let(:mutex) { Mutex.new }
+      before(:each) do
+        mutex.lock
+      end
+
+      it 'should return' do
+        expect(subject).to receive(:vm_mutex).with(vm).and_return(mutex)
+
+        expect(subject._destroy_vm(vm,pool,provider)).to eq(nil)
+      end
+    end
   end
 
   describe '#create_vm_disk' do
@@ -1501,6 +1553,31 @@ EOT
         subject.migrate_vm(vm, pool, provider)
       end
     end
+
+    context 'with a locked vm mutex' do
+      let(:mutex) { Mutex.new }
+      before(:each) do
+        mutex.lock
+      end
+
+      it 'should return' do
+        expect(subject).to receive(:vm_mutex).and_return(mutex)
+
+        expect(subject.migrate_vm(vm, pool, provider)).to be_nil
+      end
+    end
+  end
+
+  describe '#vm_mutex' do
+    it 'should return a mutex' do
+      expect(subject.vm_mutex(vm)).to be_a(Mutex)
+    end
+
+    it 'should return the same mutex when called twice' do
+      first = subject.vm_mutex(vm)
+      second = subject.vm_mutex(vm)
+      expect(first).to be(second)
+    end
   end
 
   describe 'sync_pool_template' do