diff --git a/ansible/roles/invoker/tasks/deploy.yml b/ansible/roles/invoker/tasks/deploy.yml index b928eb0007b..83fc5197f2f 100644 --- a/ansible/roles/invoker/tasks/deploy.yml +++ b/ansible/roles/invoker/tasks/deploy.yml @@ -282,6 +282,7 @@ "CONFIG_whisk_invoker_https_keystorePassword": "{{ invoker.ssl.keystore.password }}" "CONFIG_whisk_invoker_https_keystoreFlavor": "{{ invoker.ssl.storeFlavor }}" "CONFIG_whisk_invoker_https_clientAuth": "{{ invoker.ssl.clientAuth }}" + "CONFIG_whisk_invoker_resource_tags": "{% if tags is defined %} '{{ tags | join(',') }}' {% else %} '' {% endif %}" "CONFIG_whisk_containerProxy_timeouts_idleContainer": "{{ whisk.containerProxy.timeouts.idleContainer }}" "CONFIG_whisk_containerProxy_timeouts_pauseGrace": "{{ whisk.containerProxy.timeouts.pauseGrace }}" "CONFIG_whisk_containerProxy_timeouts_keepingDuration": "{{ whisk.containerProxy.timeouts.keepingDuration }}" diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala index 57d4a8b0343..ab0495f0cf8 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala @@ -317,4 +317,6 @@ object ConfigKeys { val whiskInvokerUsername = "whisk.invoker.username" val whiskInvokerPassword = "whisk.invoker.password" + + val invokerResourceTags = "whisk.invoker.resource.tags" } diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/invoker/Invoker.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/invoker/Invoker.scala index 89656c98148..8ea33af9a8c 100644 --- a/core/invoker/src/main/scala/org/apache/openwhisk/core/invoker/Invoker.scala +++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/invoker/Invoker.scala @@ -109,7 +109,13 @@ object Invoker { implicit val logger = new AkkaLogging(akka.event.Logging.getLogger(actorSystem, this)) val poolConfig: ContainerPoolConfig = loadConfigOrThrow[ContainerPoolConfig](ConfigKeys.containerPool) val limitConfig: ConcurrencyLimitConfig = loadConfigOrThrow[ConcurrencyLimitConfig](ConfigKeys.concurrencyLimit) + val tags: Seq[String] = Some(loadConfigOrThrow[String](ConfigKeys.invokerResourceTags)) + .map(_.trim()) + .filter(_ != "") + .map(_.split(",").toSeq) + .getOrElse(Seq.empty[String]) + logger.info(this, s"invoker tags: (${tags.mkString(", ")})") // Prepare Kamon shutdown CoordinatedShutdown(actorSystem).addTask(CoordinatedShutdown.PhaseActorSystemTerminate, "shutdownKamon") { () => logger.info(this, s"Shutting down Kamon with coordinated shutdown") @@ -190,7 +196,13 @@ object Invoker { val maxMessageBytes = Some(ActivationEntityLimit.MAX_ACTIVATION_LIMIT) val invokerInstance = - InvokerInstanceId(assignedInvokerId, cmdLineArgs.uniqueName, cmdLineArgs.displayedName, poolConfig.userMemory) + InvokerInstanceId( + assignedInvokerId, + cmdLineArgs.uniqueName, + cmdLineArgs.displayedName, + poolConfig.userMemory, + None, + tags) val msgProvider = SpiLoader.get[MessagingProvider] if (msgProvider diff --git a/docs/tag-based-scheduling.md b/docs/tag-based-scheduling.md new file mode 100644 index 00000000000..de210e1c5f8 --- /dev/null +++ b/docs/tag-based-scheduling.md @@ -0,0 +1,57 @@ + + +# Tag-based Scheduling + +Invoker machines may have different resources such as GPU, high CPU, etc. +For those who want to take advantage of such resources, the system should be able to schedule activations to a certain invoker with the resources. + +## Tagging invokers + +Operators can configure any tags for invokers. + +```bash +invoker0 ansible_host=${INVOKER0} tags="['v1', 'gpu']" +invoker1 ansible_host=${INVOKER1} tags="['v1', 'cpu']" +invoker2 ansible_host=${INVOKER2} tags="['v2', 'gpu']" +invoker3 ansible_host=${INVOKER3} tags="['v2', 'cpu']" +invoker4 ansible_host=${INVOKER4} tags="['v1', 'mem']" +invoker5 ansible_host=${INVOKER5} tags="['v2', 'mem']" +invoker6 ansible_host=${INVOKER6} tags="['v2']" +invoker7 ansible_host=${INVOKER7} tags="['v2']" +invoker8 ansible_host=${INVOKER8} +invoker9 ansible_host=${INVOKER9} +``` + +Users can add the following annotations to their actions. + +``` +wsk action update params tests/dat/actions/params.js -i -a invoker-resources '["v2", "gpu"]' +``` + +Activation for this action will be delivered to invoker2. + +The annotations and the corresponding target invokers are as follows. + +* `["v1", "gpu"]` -> `invoker0` +* `["v2", "gpu"]` -> `invoker2` +* `["v1", "cpu"]` -> `invoker1` +* `["v2"]` -> One of `invoker2`, `invoker3`, `invoker5`, `invoker6`, and `invoker7` +* `["v1"]` -> One of `invoker0`, `invoker1`, `invoker4` +* `No annotation` -> One of `invoker8` and `invoker9` is chosen first. if they have no resource, choose one of the invokers with tags.