diff --git a/common/scala/src/main/scala/org/apache/openwhisk/common/Logging.scala b/common/scala/src/main/scala/org/apache/openwhisk/common/Logging.scala index caba798d724..3072ff33d57 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/common/Logging.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/common/Logging.scala @@ -412,6 +412,10 @@ object LoggingMarkers { LogMarkerToken(invoker, "sharedPackage", counter, None, Map("path" -> path))(MeasurementUnit.none) def INVOKER_CONTAINERPOOL_MEMORY(state: String) = LogMarkerToken(invoker, "containerPoolMemory", counter, Some(state), Map("state" -> state))(MeasurementUnit.none) + def INVOKER_CONTAINERPOOL_CONTAINER(state: String, tags: Option[Map[String, String]] = None) = { + val map = Map("state" -> state) ++: tags.getOrElse(Map.empty) + LogMarkerToken(invoker, "containerPoolContainer", counter, Some(state), map)(MeasurementUnit.none) + } // System overload and random invoker assignment val MANAGED_SYSTEM_OVERLOAD = @@ -493,12 +497,10 @@ object LoggingMarkers { val INVOKER_ACTIVATION = LogMarkerToken(invoker, activation, start)(MeasurementUnit.none) def INVOKER_DOCKER_CMD(cmd: String) = LogMarkerToken(invoker, "docker", start, Some(cmd), Map("cmd" -> cmd))(MeasurementUnit.time.milliseconds) - def INVOKER_DOCKER_CMD_TIMEOUT(cmd: String) = - LogMarkerToken(invoker, "docker", timeout, Some(cmd), Map("cmd" -> cmd))(MeasurementUnit.none) def INVOKER_RUNC_CMD(cmd: String) = LogMarkerToken(invoker, "runc", start, Some(cmd), Map("cmd" -> cmd))(MeasurementUnit.time.milliseconds) def INVOKER_KUBEAPI_CMD(cmd: String) = - LogMarkerToken(invoker, "kubeapi", start, Some(cmd), Map("cmd" -> cmd))(MeasurementUnit.none) + LogMarkerToken(invoker, "kubeapi", start, Some(cmd), Map("cmd" -> cmd))(MeasurementUnit.time.milliseconds) def INVOKER_CONTAINER_START(containerState: String, invocationNamespace: String, namespace: String, action: String) = LogMarkerToken( invoker, diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerClient.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerClient.scala index b594f74efab..4efdc2d6da8 100644 --- a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerClient.scala +++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerClient.scala @@ -34,7 +34,7 @@ import scala.util.Try import akka.event.Logging.{ErrorLevel, InfoLevel} import pureconfig._ import pureconfig.generic.auto._ -import org.apache.openwhisk.common.{Logging, LoggingMarkers, MetricEmitter, TransactionId} +import org.apache.openwhisk.common.{Logging, LoggingMarkers, TransactionId} import org.apache.openwhisk.core.ConfigKeys import org.apache.openwhisk.core.containerpool.ContainerId import org.apache.openwhisk.core.containerpool.ContainerAddress @@ -207,7 +207,6 @@ class DockerClient(dockerHost: Option[String] = None, case Success(_) => transid.finished(this, start) case Failure(pte: ProcessTimeoutException) => transid.failed(this, start, pte.getMessage, ErrorLevel) - MetricEmitter.emitCounterMetric(LoggingMarkers.INVOKER_DOCKER_CMD_TIMEOUT(args.head)) case Failure(t) => transid.failed(this, start, t.getMessage, ErrorLevel) } } diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/v2/FunctionPullingContainerPool.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/v2/FunctionPullingContainerPool.scala index 07029814cb7..94f7bd2f1c3 100644 --- a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/v2/FunctionPullingContainerPool.scala +++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/v2/FunctionPullingContainerPool.scala @@ -110,7 +110,23 @@ class FunctionPullingContainerPool( .emitHistogramMetric(LoggingMarkers.INVOKER_CONTAINERPOOL_MEMORY("busy"), memoryConsumptionOf(busyPool)) MetricEmitter .emitHistogramMetric(LoggingMarkers.INVOKER_CONTAINERPOOL_MEMORY("prewarmed"), memoryConsumptionOf(prewarmedPool)) + MetricEmitter + .emitHistogramMetric(LoggingMarkers.INVOKER_CONTAINERPOOL_MEMORY("warmed"), memoryConsumptionOf(warmedPool)) MetricEmitter.emitHistogramMetric(LoggingMarkers.INVOKER_CONTAINERPOOL_MEMORY("max"), poolConfig.userMemory.toMB) + val prewarmedSize = prewarmedPool.size + val busySize = busyPool.size + val warmedSize = warmedPool.size + val warmedPoolMap = warmedPool groupBy { + case (_, warmedData) => (warmedData.invocationNamespace, warmedData.action.toString) + } mapValues (_.size) + for ((data, size) <- warmedPoolMap) { + val tags: Option[Map[String, String]] = Some(Map("namespace" -> data._1, "action" -> data._2)) + MetricEmitter.emitHistogramMetric(LoggingMarkers.INVOKER_CONTAINERPOOL_CONTAINER("warmed", tags), size) + } + val allSize = prewarmedSize + busySize + warmedSize + MetricEmitter.emitHistogramMetric(LoggingMarkers.INVOKER_CONTAINERPOOL_CONTAINER("prewarmed"), prewarmedSize) + MetricEmitter.emitHistogramMetric(LoggingMarkers.INVOKER_CONTAINERPOOL_CONTAINER("busy"), busySize) + MetricEmitter.emitHistogramMetric(LoggingMarkers.INVOKER_CONTAINERPOOL_CONTAINER("all"), allSize) }) // Key is ColdStartKey, value is the number of cold Start in minute