add error handling to container manager when invoker query fails (apache#5320)

* add error handling to container manager when invoker query fails

* fix tests

Co-authored-by: Brendan Doyle <[email protected]>

Author: 2 people

core/scheduler/src/main/scala/org/apache/openwhisk/core/scheduler/container/ContainerManager.scala

@@ -144,6 +144,11 @@ class ContainerManager(jobManagerFactory: ActorRefFactory => ActorRef,
     logging.info(this, s"received ${msgs.size} creation message [${msgs.head.invocationNamespace}:${msgs.head.action}]")
     ContainerManager
       .getAvailableInvokers(etcdClient, memory, invocationNamespace)
+      .recover({
+        case t: Throwable =>
+          logging.error(this, s"Unable to get available invokers: ${t.getMessage}.")
+          List.empty[InvokerHealth]
+      })
       .foreach { invokers =>
         if (invokers.isEmpty) {
           logging.error(this, "there is no available invoker to schedule.")

tests/src/test/scala/org/apache/openwhisk/core/scheduler/container/test/ContainerManagerTests.scala

@@ -864,7 +864,7 @@ class ContainerManagerTests
       ScheduledPair(msg4, None, Some(NoAvailableInvokersError)))
   }
 
-  it should "send FailedCreationJob to queue manager when no invokers are available" in {
+  it should "send FailedCreationJob to memory queue when no invokers are available" in {
     val mockEtcd = mock[EtcdClient]
     val probe = TestProbe()
     (mockEtcd
@@ -910,6 +910,50 @@ class ContainerManagerTests
         NoAvailableInvokersError))
   }
 
+  it should "send FailedCreationJob to memory queue when available invoker query fails" in {
+    val mockEtcd = mock[EtcdClient]
+    val probe = TestProbe()
+    (mockEtcd
+      .getPrefix(_: String))
+      .expects(InvokerKeys.prefix)
+      .returning(Future.failed(new Exception("etcd request failed.")))
+      .twice()
+
+    val fqn = FullyQualifiedEntityName(EntityPath("ns1"), EntityName(testAction))
+
+    QueuePool.put(
+      MemoryQueueKey(testInvocationNamespace, fqn.toDocId.asDocInfo(testRevision)),
+      MemoryQueueValue(probe.ref, true))
+
+    val mockJobManager = TestProbe()
+    val mockWatcher = TestProbe()
+
+    val manager =
+      system.actorOf(
+        ContainerManager.props(factory(mockJobManager), mockMessaging(), testsid, mockEtcd, config, mockWatcher.ref))
+
+    val msg =
+      ContainerCreationMessage(
+        TransactionId.testing,
+        testInvocationNamespace,
+        fqn,
+        testRevision,
+        actionMetadata,
+        testsid,
+        schedulerHost,
+        rpcPort)
+
+    manager ! ContainerCreation(List(msg), testMemory, testInvocationNamespace)
+    probe.expectMsg(
+      FailedCreationJob(
+        msg.creationId,
+        testInvocationNamespace,
+        msg.action,
+        testRevision,
+        NoAvailableInvokersError,
+        NoAvailableInvokersError))
+  }
+
   it should "schedule to the blackbox invoker when isBlackboxInvocation is true" in {
     stream.reset()
     val mockEtcd = mock[EtcdClient]