Add containerPool container histogram metric (#5222)

* Add containerPool container histogram metric

* Add pod creation consume time metric

* Add docker container creation consume time metric

* Use transaction to calculate

* Add namespace and action tags for container metric

* Avoid send metric repeatedly

* Fix scala 2.13 compilation error

* Update according to review comment

Author: ningyougang

common/scala/src/main/scala/org/apache/openwhisk/common/Logging.scala

@@ -412,6 +412,10 @@ object LoggingMarkers {
     LogMarkerToken(invoker, "sharedPackage", counter, None, Map("path" -> path))(MeasurementUnit.none)
   def INVOKER_CONTAINERPOOL_MEMORY(state: String) =
     LogMarkerToken(invoker, "containerPoolMemory", counter, Some(state), Map("state" -> state))(MeasurementUnit.none)
+  def INVOKER_CONTAINERPOOL_CONTAINER(state: String, tags: Option[Map[String, String]] = None) = {
+    val map = Map("state" -> state) ++: tags.getOrElse(Map.empty)
+    LogMarkerToken(invoker, "containerPoolContainer", counter, Some(state), map)(MeasurementUnit.none)
+  }
 
   // System overload and random invoker assignment
   val MANAGED_SYSTEM_OVERLOAD =
@@ -493,12 +497,10 @@ object LoggingMarkers {
   val INVOKER_ACTIVATION = LogMarkerToken(invoker, activation, start)(MeasurementUnit.none)
   def INVOKER_DOCKER_CMD(cmd: String) =
     LogMarkerToken(invoker, "docker", start, Some(cmd), Map("cmd" -> cmd))(MeasurementUnit.time.milliseconds)
-  def INVOKER_DOCKER_CMD_TIMEOUT(cmd: String) =
-    LogMarkerToken(invoker, "docker", timeout, Some(cmd), Map("cmd" -> cmd))(MeasurementUnit.none)
   def INVOKER_RUNC_CMD(cmd: String) =
     LogMarkerToken(invoker, "runc", start, Some(cmd), Map("cmd" -> cmd))(MeasurementUnit.time.milliseconds)
   def INVOKER_KUBEAPI_CMD(cmd: String) =
-    LogMarkerToken(invoker, "kubeapi", start, Some(cmd), Map("cmd" -> cmd))(MeasurementUnit.none)
+    LogMarkerToken(invoker, "kubeapi", start, Some(cmd), Map("cmd" -> cmd))(MeasurementUnit.time.milliseconds)
   def INVOKER_CONTAINER_START(containerState: String, invocationNamespace: String, namespace: String, action: String) =
     LogMarkerToken(
       invoker,

core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerClient.scala

@@ -34,7 +34,7 @@ import scala.util.Try
 import akka.event.Logging.{ErrorLevel, InfoLevel}
 import pureconfig._
 import pureconfig.generic.auto._
-import org.apache.openwhisk.common.{Logging, LoggingMarkers, MetricEmitter, TransactionId}
+import org.apache.openwhisk.common.{Logging, LoggingMarkers, TransactionId}
 import org.apache.openwhisk.core.ConfigKeys
 import org.apache.openwhisk.core.containerpool.ContainerId
 import org.apache.openwhisk.core.containerpool.ContainerAddress
@@ -207,7 +207,6 @@ class DockerClient(dockerHost: Option[String] = None,
       case Success(_) => transid.finished(this, start)
       case Failure(pte: ProcessTimeoutException) =>
         transid.failed(this, start, pte.getMessage, ErrorLevel)
-        MetricEmitter.emitCounterMetric(LoggingMarkers.INVOKER_DOCKER_CMD_TIMEOUT(args.head))
       case Failure(t) => transid.failed(this, start, t.getMessage, ErrorLevel)
     }
   }

core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/v2/FunctionPullingContainerPool.scala

@@ -110,7 +110,23 @@ class FunctionPullingContainerPool(
       .emitHistogramMetric(LoggingMarkers.INVOKER_CONTAINERPOOL_MEMORY("busy"), memoryConsumptionOf(busyPool))
     MetricEmitter
       .emitHistogramMetric(LoggingMarkers.INVOKER_CONTAINERPOOL_MEMORY("prewarmed"), memoryConsumptionOf(prewarmedPool))
+    MetricEmitter
+      .emitHistogramMetric(LoggingMarkers.INVOKER_CONTAINERPOOL_MEMORY("warmed"), memoryConsumptionOf(warmedPool))
     MetricEmitter.emitHistogramMetric(LoggingMarkers.INVOKER_CONTAINERPOOL_MEMORY("max"), poolConfig.userMemory.toMB)
+    val prewarmedSize = prewarmedPool.size
+    val busySize = busyPool.size
+    val warmedSize = warmedPool.size
+    val warmedPoolMap = warmedPool groupBy {
+      case (_, warmedData) => (warmedData.invocationNamespace, warmedData.action.toString)
+    } mapValues (_.size)
+    for ((data, size) <- warmedPoolMap) {
+      val tags: Option[Map[String, String]] = Some(Map("namespace" -> data._1, "action" -> data._2))
+      MetricEmitter.emitHistogramMetric(LoggingMarkers.INVOKER_CONTAINERPOOL_CONTAINER("warmed", tags), size)
+    }
+    val allSize = prewarmedSize + busySize + warmedSize
+    MetricEmitter.emitHistogramMetric(LoggingMarkers.INVOKER_CONTAINERPOOL_CONTAINER("prewarmed"), prewarmedSize)
+    MetricEmitter.emitHistogramMetric(LoggingMarkers.INVOKER_CONTAINERPOOL_CONTAINER("busy"), busySize)
+    MetricEmitter.emitHistogramMetric(LoggingMarkers.INVOKER_CONTAINERPOOL_CONTAINER("all"), allSize)
   })
 
   // Key is ColdStartKey, value is the number of cold Start in minute