Skip to content

Commit 992b0d0

Browse files
JesseStutlerJesseStutler
committed
Change the value of pause-grace for new scheduler
2 parents a490cd3 + 8d31e96 commit 992b0d0

File tree

59 files changed

+2440
-500
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

59 files changed

+2440
-500
lines changed

‎ansible/files/runtimes.json

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -14,20 +14,6 @@
1414
],
1515
"runtimes": {
1616
"nodejs": [
17-
{
18-
"kind": "nodejs:12",
19-
"default": false,
20-
"image": {
21-
"prefix": "openwhisk",
22-
"name": "action-nodejs-v12",
23-
"tag": "nightly"
24-
},
25-
"deprecated": false,
26-
"attached": {
27-
"attachmentName": "codefile",
28-
"attachmentType": "text/plain"
29-
}
30-
},
3117
{
3218
"kind": "nodejs:14",
3319
"default": true,
@@ -54,6 +40,20 @@
5440
}
5541
}
5642
]
43+
},
44+
{
45+
"kind": "nodejs:16",
46+
"default": false,
47+
"image": {
48+
"prefix": "openwhisk",
49+
"name": "action-nodejs-v16",
50+
"tag": "nightly"
51+
},
52+
"deprecated": false,
53+
"attached": {
54+
"attachmentName": "codefile",
55+
"attachmentType": "text/plain"
56+
}
5757
}
5858
],
5959
"python": [
@@ -195,7 +195,7 @@
195195
],
196196
"go": [
197197
{
198-
"kind": "go:1.15",
198+
"kind": "go:1.17",
199199
"default": true,
200200
"deprecated": false,
201201
"attached": {
@@ -204,7 +204,7 @@
204204
},
205205
"image": {
206206
"prefix": "openwhisk",
207-
"name": "action-golang-v1.15",
207+
"name": "action-golang-v1.17",
208208
"tag": "nightly"
209209
}
210210
}

‎ansible/group_vars/all

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,11 @@ whisk:
5252
require_api_key_annotation: "{{ require_api_key_annotation | default(true) | lower }}"
5353
require_response_payload: "{{ require_response_payload | default(true) | lower }}"
5454
cluster_name: "{{ whisk_cluster_name | default('whisk') }}"
55+
containerProxy:
56+
timeouts:
57+
idleContainer: "{{ containerProxy_timeouts_idleContainer | default('10 minutes') }}"
58+
pauseGrace: "{{ containerProxy_timeouts_pauseGrace | default('10 seconds') }}"
59+
keepingDuration: "{{ containerProxy_timeouts_keepingDuration | default('10 minutes') }}"
5560

5661
##
5762
# configuration parameters related to support runtimes (see org.apache.openwhisk.core.entity.ExecManifest for schema of the manifest).
@@ -213,6 +218,8 @@ invoker:
213218
{% endif %}"
214219
extraEnv: "{{ invoker_extraEnv | default({}) }}"
215220
protocol: "{{ invoker_protocol | default('https') }}"
221+
username: "{{ invoker_username | default('invoker.user') }}"
222+
password: "{{ invoker_password | default('invoker.pass') }}"
216223
ssl:
217224
cn: "openwhisk-invokers"
218225
keyPrefix: "{{ __invoker_ssl_keyPrefix }}"
@@ -511,8 +518,13 @@ scheduler:
511518
dataManagementService:
512519
retryInterval: "{{ scheduler_dataManagementService_retryInterval | default('1 second') }}"
513520
inProgressJobRetention: "{{ scheduler_inProgressJobRetention | default('20 seconds') }}"
521+
blackboxMultiple: "{{ scheduler_blackboxMultiple | default(15) }}"
514522
managedFraction: "{{ scheduler_managed_fraction | default(1.0 - (scheduler_blackbox_fraction | default(__scheduler_blackbox_fraction))) }}"
515523
blackboxFraction: "{{ scheduler_blackbox_fraction | default(__scheduler_blackbox_fraction) }}"
524+
scheduling:
525+
staleThreshold: "{{ scheduler_scheduling_staleThreshold | default('100 milliseconds') }}"
526+
checkInterval: "{{ scheduler_scheduling_checkInterval | default('100 milliseconds') }}"
527+
dropInterval: "{{ scheduler_scheduling_dropInterval | default('10 seconds') }}"
516528
queueManager:
517529
maxSchedulingTime: "{{ scheduler_maxSchedulingTime | default('20 second') }}"
518530
maxRetriesToGetQueue: "{{ scheduler_maxRetriesToGetQueue | default(13) }}"

‎ansible/roles/invoker/tasks/deploy.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -282,11 +282,17 @@
282282
"CONFIG_whisk_invoker_https_keystorePassword": "{{ invoker.ssl.keystore.password }}"
283283
"CONFIG_whisk_invoker_https_keystoreFlavor": "{{ invoker.ssl.storeFlavor }}"
284284
"CONFIG_whisk_invoker_https_clientAuth": "{{ invoker.ssl.clientAuth }}"
285+
"CONFIG_whisk_containerProxy_timeouts_idleContainer": "{{ whisk.containerProxy.timeouts.idleContainer }}"
286+
"CONFIG_whisk_containerProxy_timeouts_pauseGrace": "{{ whisk.containerProxy.timeouts.pauseGrace }}"
287+
"CONFIG_whisk_containerProxy_timeouts_keepingDuration": "{{ whisk.containerProxy.timeouts.keepingDuration }}"
285288
"CONFIG_whisk_containerPool_prewarmExpirationCheckInitDelay": "{{ container_pool_prewarm_expirationCheckInitDelay | default('10 minutes') }}"
286289
"CONFIG_whisk_containerPool_prewarmExpirationCheckInterval": "{{ container_pool_prewarm_expirationCheckInterval | default('10 minutes') }}"
287290
"CONFIG_whisk_containerPool_prewarmExpirationCheckIntervalVariance": "{{ container_pool_prewarm_expirationCheckIntervalVariance | default('10 seconds') }}"
288291
"CONFIG_whisk_containerPool_prewarmPromotion": "{{ container_pool_strict | default('false') | lower }}"
289292
"CONFIG_whisk_containerPool_prewarmMaxRetryLimit": "{{ container_pool_prewarm_max_retry_limit | default(5) }}"
293+
"CONFIG_whisk_containerPool_batchDeletionSize": "{{ container_pool_batchDeletionSize | default(10) }}"
294+
"CONFIG_whisk_invoker_username": "{{ invoker.username }}"
295+
"CONFIG_whisk_invoker_password": "{{ invoker.password }}"
290296

291297
- name: extend invoker dns env
292298
set_fact:

‎ansible/roles/schedulers/tasks/deploy.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,10 @@
113113
"CONFIG_whisk_scheduler_maxPeek": "{{ scheduler.maxPeek }}"
114114
"CONFIG_whisk_scheduler_dataManagementService_retryInterval": "{{ scheduler.dataManagementService.retryInterval }}"
115115
"CONFIG_whisk_scheduler_inProgressJobRetention": "{{ scheduler.inProgressJobRetention }}"
116+
"CONFIG_whisk_scheduler_blackboxMultiple": "{{ scheduler.blackboxMultiple }}"
117+
"CONFIG_whisk_scheduler_scheduling_staleThreshold": "{{ scheduler.scheduling.staleThreshold }}"
118+
"CONFIG_whisk_scheduler_scheduling_checkInterval": "{{ scheduler.scheduling.checkInterval }}"
119+
"CONFIG_whisk_scheduler_scheduling_dropInterval": "{{ scheduler.scheduling.dropInterval }}"
116120
"CONFIG_whisk_scheduler_queueManager_maxSchedulingTime": "{{ scheduler.queueManager.maxSchedulingTime }}"
117121
"CONFIG_whisk_scheduler_queueManager_maxRetriesToGetQueue": "{{ scheduler.queueManager.maxRetriesToGetQueue }}"
118122
"CONFIG_whisk_scheduler_queue_idleGrace": "{{ scheduler.queue.idleGrace }}"
@@ -121,6 +125,7 @@
121125
"CONFIG_whisk_scheduler_queue_gracefulShutdownTimeout": "{{ scheduler.queue.gracefulShutdownTimeout }}"
122126
"CONFIG_whisk_scheduler_queue_maxRetentionSize": "{{ scheduler.queue.maxRetentionSize }}"
123127
"CONFIG_whisk_scheduler_queue_maxRetentionMs": "{{ scheduler.queue.maxRetentionMs }}"
128+
"CONFIG_whisk_scheduler_queue_maxBlackboxRetentionMs": "{{ scheduler.queue.maxBlackboxRetentionMs }}"
124129
"CONFIG_whisk_scheduler_queue_throttlingFraction": "{{ scheduler.queue.throttlingFraction }}"
125130
"CONFIG_whisk_scheduler_queue_durationBufferSize": "{{ scheduler.queue.durationBufferSize }}"
126131
"CONFIG_whisk_durationChecker_timeWindow": "{{ durationChecker.timeWindow }}"

‎common/scala/src/main/scala/org/apache/openwhisk/common/Logging.scala

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -377,14 +377,14 @@ object LoggingMarkers {
377377
scheduler,
378378
"actionContainer",
379379
counter,
380-
Some(namespace),
380+
Some(action),
381381
Map("namespace" -> namespace, "action" -> action))(MeasurementUnit.none)
382382
def SCHEDULER_ACTION_INPROGRESS_CONTAINER(namespace: String, action: String) =
383383
LogMarkerToken(
384384
scheduler,
385385
"actionInProgressContainer",
386386
counter,
387-
Some(namespace),
387+
Some(action),
388388
Map("namespace" -> namespace, "action" -> action))(MeasurementUnit.none)
389389

390390
/*
@@ -412,6 +412,10 @@ object LoggingMarkers {
412412
LogMarkerToken(invoker, "sharedPackage", counter, None, Map("path" -> path))(MeasurementUnit.none)
413413
def INVOKER_CONTAINERPOOL_MEMORY(state: String) =
414414
LogMarkerToken(invoker, "containerPoolMemory", counter, Some(state), Map("state" -> state))(MeasurementUnit.none)
415+
def INVOKER_CONTAINERPOOL_CONTAINER(state: String, tags: Option[Map[String, String]] = None) = {
416+
val map = Map("state" -> state) ++: tags.getOrElse(Map.empty)
417+
LogMarkerToken(invoker, "containerPoolContainer", counter, Some(state), map)(MeasurementUnit.none)
418+
}
415419

416420
// System overload and random invoker assignment
417421
val MANAGED_SYSTEM_OVERLOAD =
@@ -493,12 +497,10 @@ object LoggingMarkers {
493497
val INVOKER_ACTIVATION = LogMarkerToken(invoker, activation, start)(MeasurementUnit.none)
494498
def INVOKER_DOCKER_CMD(cmd: String) =
495499
LogMarkerToken(invoker, "docker", start, Some(cmd), Map("cmd" -> cmd))(MeasurementUnit.time.milliseconds)
496-
def INVOKER_DOCKER_CMD_TIMEOUT(cmd: String) =
497-
LogMarkerToken(invoker, "docker", timeout, Some(cmd), Map("cmd" -> cmd))(MeasurementUnit.none)
498500
def INVOKER_RUNC_CMD(cmd: String) =
499501
LogMarkerToken(invoker, "runc", start, Some(cmd), Map("cmd" -> cmd))(MeasurementUnit.time.milliseconds)
500502
def INVOKER_KUBEAPI_CMD(cmd: String) =
501-
LogMarkerToken(invoker, "kubeapi", start, Some(cmd), Map("cmd" -> cmd))(MeasurementUnit.none)
503+
LogMarkerToken(invoker, "kubeapi", start, Some(cmd), Map("cmd" -> cmd))(MeasurementUnit.time.milliseconds)
502504
def INVOKER_CONTAINER_START(containerState: String, invocationNamespace: String, namespace: String, action: String) =
503505
LogMarkerToken(
504506
invoker,
@@ -568,6 +570,15 @@ object LoggingMarkers {
568570
val OFFLINE_INVOKER_BLACKBOX =
569571
LogMarkerToken(loadbalancer, "totalOfflineInvokerBlackBox", counter)(MeasurementUnit.none)
570572

573+
val HEALTHY_INVOKERS =
574+
LogMarkerToken(loadbalancer, "totalHealthyInvoker", counter)(MeasurementUnit.none)
575+
val UNHEALTHY_INVOKERS =
576+
LogMarkerToken(loadbalancer, "totalUnhealthyInvoker", counter)(MeasurementUnit.none)
577+
val OFFLINE_INVOKERS =
578+
LogMarkerToken(loadbalancer, "totalOfflineInvoker", counter)(MeasurementUnit.none)
579+
580+
val INVOKER_TOTALMEM = LogMarkerToken(loadbalancer, "totalCapacity", counter)(MeasurementUnit.none)
581+
571582
// Kafka related markers
572583
def KAFKA_QUEUE(topic: String) =
573584
if (TransactionId.metricsKamonTags)
@@ -580,17 +591,22 @@ object LoggingMarkers {
580591

581592
// Time that is needed to produce message in kafka
582593
val SCHEDULER_KAFKA = LogMarkerToken(scheduler, kafka, start)(MeasurementUnit.time.milliseconds)
583-
val SCHEDULER_WAIT_TIME =
584-
LogMarkerToken(scheduler, "waitTime", counter)(MeasurementUnit.none)
594+
val SCHEDULER_KAFKA_WAIT_TIME =
595+
LogMarkerToken(scheduler, "kafkaWaitTime", counter)(MeasurementUnit.time.milliseconds)
596+
def SCHEDULER_WAIT_TIME(action: String) =
597+
LogMarkerToken(scheduler, "waitTime", counter, Some(action), Map("action" -> action))(
598+
MeasurementUnit.time.milliseconds)
585599

586600
def SCHEDULER_KEEP_ALIVE(leaseId: Long) =
587601
LogMarkerToken(scheduler, "keepAlive", counter, None, Map("leaseId" -> leaseId.toString))(MeasurementUnit.none)
588602
def SCHEDULER_QUEUE = LogMarkerToken(scheduler, "queue", counter)(MeasurementUnit.none)
589603
def SCHEDULER_QUEUE_CREATE = LogMarkerToken(scheduler, "queueCreate", start)(MeasurementUnit.time.milliseconds)
604+
def SCHEDULER_QUEUE_RECOVER = LogMarkerToken(scheduler, "queueRecover", start)(MeasurementUnit.time.milliseconds)
590605
def SCHEDULER_QUEUE_UPDATE(reason: String) =
591606
LogMarkerToken(scheduler, "queueUpdate", counter, None, Map("reason" -> reason))(MeasurementUnit.none)
592607
def SCHEDULER_QUEUE_WAITING_ACTIVATION(action: String) =
593608
LogMarkerToken(scheduler, "queueActivation", counter, Some(action), Map("action" -> action))(MeasurementUnit.none)
609+
594610
/*
595611
* General markers
596612
*/

‎common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,7 @@ object WhiskConfig {
203203
object ConfigKeys {
204204
val cluster = "whisk.cluster"
205205
val loadbalancer = "whisk.loadbalancer"
206+
val fpcLoadBalancer = "whisk.loadbalancer.fpc"
206207
val fraction = "whisk.fraction"
207208
val buildInformation = "whisk.info"
208209

@@ -300,11 +301,20 @@ object ConfigKeys {
300301

301302
val schedulerGrpcService = "whisk.scheduler.grpc"
302303
val schedulerMaxPeek = "whisk.scheduler.max-peek"
304+
val schedulerScheduling = "whisk.scheduler.scheduling"
303305
val schedulerQueue = "whisk.scheduler.queue"
304306
val schedulerQueueManager = "whisk.scheduler.queue-manager"
305307
val schedulerInProgressJobRetention = "whisk.scheduler.in-progress-job-retention"
308+
val schedulerBlackboxMultiple = "whisk.scheduler.blackbox-multiple"
309+
val schedulerStaleThreshold = "whisk.scheduler.stale-threshold"
306310

307311
val whiskClusterName = "whisk.cluster.name"
308312

309313
val dataManagementServiceRetryInterval = "whisk.scheduler.data-management-service.retry-interval"
314+
315+
val whiskSchedulerUsername = "whisk.scheduler.username"
316+
val whiskSchedulerPassword = "whisk.scheduler.password"
317+
318+
val whiskInvokerUsername = "whisk.invoker.username"
319+
val whiskInvokerPassword = "whisk.invoker.password"
310320
}

‎common/scala/src/main/scala/org/apache/openwhisk/core/containerpool/ContainerFactory.scala

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,12 +54,14 @@ case class ContainerPoolConfig(userMemory: ByteSize,
5454
prewarmMaxRetryLimit: Int,
5555
prewarmPromotion: Boolean,
5656
memorySyncInterval: FiniteDuration,
57+
batchDeletionSize: Int,
5758
prewarmContainerCreationConfig: Option[PrewarmContainerCreationConfig] = None) {
5859
require(
5960
concurrentPeekFactor > 0 && concurrentPeekFactor <= 1.0,
6061
s"concurrentPeekFactor must be > 0 and <= 1.0; was $concurrentPeekFactor")
6162

6263
require(prewarmExpirationCheckInterval.toSeconds > 0, "prewarmExpirationCheckInterval must be > 0")
64+
require(batchDeletionSize > 0, "batch deletion size must be > 0")
6365

6466
/**
6567
* The shareFactor indicates the number of containers that would share a single core, on average.

‎common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/NoopActivationStore.scala

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,12 @@ package org.apache.openwhisk.core.database.memory
2020
import java.time.Instant
2121
import akka.actor.ActorSystem
2222
import org.apache.openwhisk.common.{Logging, PrintStreamLogging, TransactionId, WhiskInstants}
23-
import org.apache.openwhisk.core.database.{ActivationStore, ActivationStoreProvider, CacheChangeNotification, UserContext}
23+
import org.apache.openwhisk.core.database.{
24+
ActivationStore,
25+
ActivationStoreProvider,
26+
CacheChangeNotification,
27+
UserContext
28+
}
2429
import org.apache.openwhisk.core.entity.{ActivationId, DocInfo, EntityName, EntityPath, Subject, WhiskActivation}
2530
import spray.json.{JsNumber, JsObject}
2631

‎common/scala/src/main/scala/org/apache/openwhisk/core/database/mongodb/MongoDBArtifactStoreProvider.scala

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -37,13 +37,16 @@ case class MongoDBConfig(uri: String, database: String) {
3737
}
3838

3939
object MongoDBClient {
40-
private var _client: Option[MongoClient] = None
40+
private var _clientWithConfig: Option[(MongoClient, MongoDBConfig)] = None
4141

4242
def client(config: MongoDBConfig): MongoClient = {
43-
_client.getOrElse {
44-
val client = MongoClient(config.uri)
45-
_client = Some(client)
46-
client
43+
_clientWithConfig match {
44+
case Some((client, oldConfig)) if oldConfig == config =>
45+
client
46+
case _ =>
47+
val client = MongoClient(config.uri)
48+
_clientWithConfig = Some(client, config)
49+
client
4750
}
4851
}
4952
}

‎common/scala/src/main/scala/org/apache/openwhisk/core/etcd/EtcdClient.scala

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,10 +48,16 @@ object RichListenableFuture {
4848

4949
object EtcdClient {
5050
// hostAndPorts format: {HOST}:{PORT}[,{HOST}:{PORT},{HOST}:{PORT}, ...]
51-
def apply(hostAndPorts: String)(implicit ece: ExecutionContextExecutor): EtcdClient = {
52-
require(hostAndPorts != null)
53-
val client: Client = Client.forEndpoints(hostAndPorts).withPlainText().build()
54-
new EtcdClient(client)(ece)
51+
def apply(config: EtcdConfig)(implicit ece: ExecutionContextExecutor): EtcdClient = {
52+
require(config.hosts != null)
53+
require(
54+
(config.username.nonEmpty && config.password.nonEmpty) || (config.username.isEmpty && config.password.isEmpty))
55+
val clientBuilder = Client.forEndpoints(config.hosts).withPlainText()
56+
if (config.username.nonEmpty && config.password.nonEmpty) {
57+
new EtcdClient(clientBuilder.withCredentials(config.username.get, config.password.get).build())
58+
} else {
59+
new EtcdClient(clientBuilder.build())(ece)
60+
}
5561
}
5662

5763
def apply(client: Client)(implicit ece: ExecutionContextExecutor): EtcdClient = {

0 commit comments

Comments
 (0)