|
| 1 | +/* |
| 2 | + * Licensed to the Apache Software Foundation (ASF) under one or more |
| 3 | + * contributor license agreements. See the NOTICE file distributed with |
| 4 | + * this work for additional information regarding copyright ownership. |
| 5 | + * The ASF licenses this file to You under the Apache License, Version 2.0 |
| 6 | + * (the "License"); you may not use this file except in compliance with |
| 7 | + * the License. You may obtain a copy of the License at |
| 8 | + * |
| 9 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | + * |
| 11 | + * Unless required by applicable law or agreed to in writing, software |
| 12 | + * distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | + * See the License for the specific language governing permissions and |
| 15 | + * limitations under the License. |
| 16 | + */ |
| 17 | + |
| 18 | +package org.apache.openwhisk.core.service |
| 19 | + |
| 20 | +import akka.actor.Status.{Failure => FailureMessage} |
| 21 | +import akka.actor.{ActorRef, ActorSystem, Cancellable, FSM, Props, Stash} |
| 22 | +import akka.pattern.pipe |
| 23 | +import org.apache.openwhisk.common.{Logging, LoggingMarkers, MetricEmitter} |
| 24 | +import org.apache.openwhisk.core.ConfigKeys |
| 25 | +import org.apache.openwhisk.core.entity.InstanceId |
| 26 | +import org.apache.openwhisk.core.etcd.EtcdClient |
| 27 | +import org.apache.openwhisk.core.etcd.EtcdKV.InstanceKeys.instanceLease |
| 28 | +import pureconfig.loadConfigOrThrow |
| 29 | + |
| 30 | +import scala.concurrent.duration._ |
| 31 | +import scala.concurrent.{ExecutionContextExecutor, Future} |
| 32 | +import scala.util.{Failure, Success} |
| 33 | + |
| 34 | +// States |
| 35 | +sealed trait KeepAliveServiceState |
| 36 | +case object Ready extends KeepAliveServiceState |
| 37 | +case object Active extends KeepAliveServiceState |
| 38 | + |
| 39 | +// Data |
| 40 | +sealed trait KeepAliveServiceData |
| 41 | +case object NoData extends KeepAliveServiceData |
| 42 | +case class Lease(id: Long, ttl: Long) extends KeepAliveServiceData |
| 43 | +case class ActiveStates(worker: Cancellable, lease: Lease) extends KeepAliveServiceData |
| 44 | + |
| 45 | +// Events received by the actor |
| 46 | +case object RegrantLease |
| 47 | +case object GetLease |
| 48 | +case object GrantLease |
| 49 | + |
| 50 | +// Events internally used |
| 51 | +case class SetLease(lease: Lease) |
| 52 | +case class SetWatcher(worker: Cancellable) |
| 53 | + |
| 54 | +class LeaseKeepAliveService(etcdClient: EtcdClient, instanceId: InstanceId, watcherService: ActorRef)( |
| 55 | + implicit logging: Logging, |
| 56 | + actorSystem: ActorSystem) |
| 57 | + extends FSM[KeepAliveServiceState, KeepAliveServiceData] |
| 58 | + with Stash { |
| 59 | + |
| 60 | + implicit val ec: ExecutionContextExecutor = context.dispatcher |
| 61 | + |
| 62 | + private val leaseTimeout = loadConfigOrThrow[Int](ConfigKeys.etcdLeaseTimeout).seconds |
| 63 | + private val key = instanceLease(instanceId) |
| 64 | + private val watcherName = "lease-service" |
| 65 | + |
| 66 | + self ! GrantLease |
| 67 | + startWith(Ready, NoData) |
| 68 | + |
| 69 | + when(Ready) { |
| 70 | + case Event(GrantLease, NoData) => |
| 71 | + etcdClient |
| 72 | + .grant(leaseTimeout.toSeconds) |
| 73 | + .map { res => |
| 74 | + SetLease(Lease(res.getID, res.getTTL)) |
| 75 | + } |
| 76 | + .pipeTo(self) |
| 77 | + stay |
| 78 | + |
| 79 | + case Event(SetLease(lease), NoData) => |
| 80 | + startKeepAliveService(lease) |
| 81 | + .pipeTo(self) |
| 82 | + logging.info(this, s"Granted a new lease $lease") |
| 83 | + stay using lease |
| 84 | + |
| 85 | + case Event(SetWatcher(w), l: Lease) => |
| 86 | + goto(Active) using ActiveStates(w, l) |
| 87 | + |
| 88 | + case Event(t: FailureMessage, _) => |
| 89 | + logging.warn(this, s"Failed to grant new lease caused by: $t") |
| 90 | + self ! GrantLease |
| 91 | + stay() |
| 92 | + |
| 93 | + case _ => delay |
| 94 | + } |
| 95 | + |
| 96 | + when(Active) { |
| 97 | + case Event(WatchEndpointRemoved(`key`, `key`, _, false), ActiveStates(worker, lease)) => |
| 98 | + logging.info(this, s"endpoint ie removed so recreate a lease") |
| 99 | + recreateLease(worker, lease) |
| 100 | + |
| 101 | + case Event(RegrantLease, ActiveStates(worker, lease)) => |
| 102 | + logging.info(this, s"ReGrant a lease, old lease:${lease}") |
| 103 | + recreateLease(worker, lease) |
| 104 | + |
| 105 | + case Event(GetLease, ActiveStates(_, lease)) => |
| 106 | + logging.info(this, s"send the lease(${lease}) to ${sender()}") |
| 107 | + sender() ! lease |
| 108 | + stay() |
| 109 | + |
| 110 | + case _ => delay |
| 111 | + } |
| 112 | + |
| 113 | + initialize() |
| 114 | + |
| 115 | + private def startKeepAliveService(lease: Lease): Future[SetWatcher] = { |
| 116 | + val worker = |
| 117 | + actorSystem.scheduler.schedule(initialDelay = 0.second, interval = 500.milliseconds)(keepAliveOnce(lease)) |
| 118 | + |
| 119 | + /** |
| 120 | + * To verify that lease has been deleted since timeout, |
| 121 | + * create a key using lease, watch the key, and receive an event for deletion. |
| 122 | + */ |
| 123 | + etcdClient.put(key, s"${lease.id}", lease.id).map { _ => |
| 124 | + watcherService ! WatchEndpoint(key, s"${lease.id}", false, watcherName, Set(DeleteEvent)) |
| 125 | + SetWatcher(worker) |
| 126 | + } |
| 127 | + } |
| 128 | + |
| 129 | + private def keepAliveOnce(lease: Lease): Future[Long] = { |
| 130 | + etcdClient |
| 131 | + .keepAliveOnce(lease.id) |
| 132 | + .map(_.getID) |
| 133 | + .andThen { |
| 134 | + case Success(_) => MetricEmitter.emitCounterMetric(LoggingMarkers.SCHEDULER_KEEP_ALIVE(lease.id)) |
| 135 | + case Failure(t) => |
| 136 | + logging.warn(this, s"Failed to keep-alive of ${lease.id} caused by ${t}") |
| 137 | + self ! RegrantLease |
| 138 | + } |
| 139 | + } |
| 140 | + |
| 141 | + private def recreateLease(worker: Cancellable, lease: Lease) = { |
| 142 | + logging.info(this, s"recreate a lease, old lease: $lease") |
| 143 | + worker.cancel() // stop scheduler |
| 144 | + watcherService ! UnwatchEndpoint(key, false, watcherName) // stop watcher |
| 145 | + etcdClient |
| 146 | + .revoke(lease.id) // delete lease |
| 147 | + .onComplete(_ => self ! GrantLease) // create lease |
| 148 | + goto(Ready) using NoData |
| 149 | + } |
| 150 | + |
| 151 | + // Unstash all messages stashed while in intermediate state |
| 152 | + onTransition { |
| 153 | + case _ -> Ready => unstashAll() |
| 154 | + case _ -> Active => unstashAll() |
| 155 | + } |
| 156 | + |
| 157 | + /** Delays all incoming messages until unstashAll() is called */ |
| 158 | + def delay = { |
| 159 | + stash() |
| 160 | + stay |
| 161 | + } |
| 162 | + |
| 163 | + override def postStop(): Unit = { |
| 164 | + stateData match { |
| 165 | + case ActiveStates(w, _) => w.cancel() // stop scheduler if that exist |
| 166 | + case _ => // do nothing |
| 167 | + } |
| 168 | + watcherService ! UnwatchEndpoint(key, false, watcherName) |
| 169 | + } |
| 170 | +} |
| 171 | + |
| 172 | +object LeaseKeepAliveService { |
| 173 | + def props(etcdClient: EtcdClient, instanceId: InstanceId, watcherService: ActorRef)( |
| 174 | + implicit logging: Logging, |
| 175 | + actorSystem: ActorSystem): Props = { |
| 176 | + Props(new LeaseKeepAliveService(etcdClient, instanceId, watcherService)) |
| 177 | + .withDispatcher("dispatchers.lease-service-dispatcher") |
| 178 | + } |
| 179 | +} |
0 commit comments