|
| 1 | +/* |
| 2 | + * Licensed to the Apache Software Foundation (ASF) under one or more |
| 3 | + * contributor license agreements. See the NOTICE file distributed with |
| 4 | + * this work for additional information regarding copyright ownership. |
| 5 | + * The ASF licenses this file to You under the Apache License, Version 2.0 |
| 6 | + * (the "License"); you may not use this file except in compliance with |
| 7 | + * the License. You may obtain a copy of the License at |
| 8 | + * |
| 9 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | + * |
| 11 | + * Unless required by applicable law or agreed to in writing, software |
| 12 | + * distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | + * See the License for the specific language governing permissions and |
| 15 | + * limitations under the License. |
| 16 | + */ |
| 17 | +package org.apache.openwhisk.core.scheduler.queue |
| 18 | + |
| 19 | +import akka.actor.ActorSystem |
| 20 | +import com.sksamuel.elastic4s.http.ElasticDsl._ |
| 21 | +import com.sksamuel.elastic4s.http.{ElasticClient, ElasticProperties, NoOpRequestConfigCallback} |
| 22 | +import com.sksamuel.elastic4s.searches.queries.Query |
| 23 | +import com.sksamuel.elastic4s.{ElasticDate, ElasticDateMath, Seconds} |
| 24 | +import org.apache.openwhisk.common.Logging |
| 25 | +import org.apache.openwhisk.core.ConfigKeys |
| 26 | +import org.apache.openwhisk.core.entity.WhiskActionMetaData |
| 27 | +import org.apache.openwhisk.spi.Spi |
| 28 | +import pureconfig.loadConfigOrThrow |
| 29 | +import spray.json.{JsArray, JsNumber, JsValue, RootJsonFormat, deserializationError, _} |
| 30 | + |
| 31 | +import scala.concurrent.Future |
| 32 | +import scala.concurrent.duration.FiniteDuration |
| 33 | +import scala.language.implicitConversions |
| 34 | +import scala.util.{Failure, Try} |
| 35 | +import pureconfig.generic.auto._ |
| 36 | + |
| 37 | +trait DurationChecker { |
| 38 | + def checkAverageDuration(invocationNamespace: String, actionMetaData: WhiskActionMetaData)( |
| 39 | + callback: DurationCheckResult => DurationCheckResult): Future[DurationCheckResult] |
| 40 | +} |
| 41 | + |
| 42 | +case class DurationCheckResult(averageDuration: Option[Double], hitCount: Long, took: Long) |
| 43 | + |
| 44 | +object ElasticSearchDurationChecker { |
| 45 | + val FilterAggregationName = "filterAggregation" |
| 46 | + val AverageAggregationName = "averageAggregation" |
| 47 | + |
| 48 | + implicit val serde = new ElasticSearchDurationCheckResultFormat() |
| 49 | + |
| 50 | + def getFromDate(timeWindow: FiniteDuration): ElasticDateMath = |
| 51 | + ElasticDate.now minus (timeWindow.toSeconds.toInt, Seconds) |
| 52 | +} |
| 53 | + |
| 54 | +class ElasticSearchDurationChecker(private val client: ElasticClient, val timeWindow: FiniteDuration)( |
| 55 | + implicit val actorSystem: ActorSystem, |
| 56 | + implicit val logging: Logging) |
| 57 | + extends DurationChecker { |
| 58 | + import ElasticSearchDurationChecker._ |
| 59 | + import org.apache.openwhisk.core.database.elasticsearch.ElasticSearchActivationStore.generateIndex |
| 60 | + |
| 61 | + implicit val ec = actorSystem.getDispatcher |
| 62 | + |
| 63 | + override def checkAverageDuration(invocationNamespace: String, actionMetaData: WhiskActionMetaData)( |
| 64 | + callback: DurationCheckResult => DurationCheckResult): Future[DurationCheckResult] = { |
| 65 | + val index = generateIndex(invocationNamespace) |
| 66 | + val fqn = actionMetaData.fullyQualifiedName(false) |
| 67 | + val fromDate = getFromDate(timeWindow) |
| 68 | + |
| 69 | + logging.info(this, s"check average duration for $fqn in $index for last $timeWindow") |
| 70 | + |
| 71 | + actionMetaData.binding match { |
| 72 | + case Some(binding) => |
| 73 | + val boolQueryResult = List( |
| 74 | + matchQuery("annotations.binding", s"$binding"), |
| 75 | + matchQuery("name", actionMetaData.name), |
| 76 | + rangeQuery("@timestamp").gte(fromDate)) |
| 77 | + |
| 78 | + executeQuery(boolQueryResult, callback, index) |
| 79 | + |
| 80 | + case None => |
| 81 | + val queryResult = List(matchQuery("path.keyword", fqn.toString), rangeQuery("@timestamp").gte(fromDate)) |
| 82 | + |
| 83 | + executeQuery(queryResult, callback, index) |
| 84 | + } |
| 85 | + } |
| 86 | + |
| 87 | + private def executeQuery(boolQueryResult: List[Query], |
| 88 | + callback: DurationCheckResult => DurationCheckResult, |
| 89 | + index: String) = { |
| 90 | + client |
| 91 | + .execute { |
| 92 | + (search(index) query { |
| 93 | + boolQuery must { |
| 94 | + boolQueryResult |
| 95 | + } |
| 96 | + } aggregations |
| 97 | + avgAgg(AverageAggregationName, "duration")).size(0) |
| 98 | + } |
| 99 | + .map { res => |
| 100 | + logging.debug(this, s"ElasticSearch query results: $res") |
| 101 | + Try(serde.read(res.body.getOrElse("").parseJson)) |
| 102 | + } |
| 103 | + .flatMap(Future.fromTry) |
| 104 | + .map(callback(_)) |
| 105 | + .andThen { |
| 106 | + case Failure(t) => |
| 107 | + logging.error(this, s"failed to check the average duration: ${t}") |
| 108 | + } |
| 109 | + } |
| 110 | +} |
| 111 | + |
| 112 | +object ElasticSearchDurationCheckerProvider extends DurationCheckerProvider { |
| 113 | + import org.apache.openwhisk.core.database.elasticsearch.ElasticSearchActivationStore._ |
| 114 | + |
| 115 | + override def instance(actorSystem: ActorSystem, log: Logging): ElasticSearchDurationChecker = { |
| 116 | + implicit val as: ActorSystem = actorSystem |
| 117 | + implicit val logging: Logging = log |
| 118 | + |
| 119 | + val elasticClient = |
| 120 | + ElasticClient( |
| 121 | + ElasticProperties(s"${elasticSearchConfig.protocol}://${elasticSearchConfig.hosts}"), |
| 122 | + NoOpRequestConfigCallback, |
| 123 | + httpClientCallback) |
| 124 | + |
| 125 | + new ElasticSearchDurationChecker(elasticClient, durationCheckerConfig.timeWindow) |
| 126 | + } |
| 127 | +} |
| 128 | + |
| 129 | +trait DurationCheckerProvider extends Spi { |
| 130 | + |
| 131 | + val durationCheckerConfig: DurationCheckerConfig = |
| 132 | + loadConfigOrThrow[DurationCheckerConfig](ConfigKeys.durationChecker) |
| 133 | + |
| 134 | + def instance(actorSystem: ActorSystem, logging: Logging): DurationChecker |
| 135 | +} |
| 136 | + |
| 137 | +class ElasticSearchDurationCheckResultFormat extends RootJsonFormat[DurationCheckResult] { |
| 138 | + import ElasticSearchDurationChecker._ |
| 139 | + import spray.json.DefaultJsonProtocol._ |
| 140 | + |
| 141 | + /** |
| 142 | + * Expected sample data |
| 143 | + { |
| 144 | + "_shards": { |
| 145 | + "failed": 0, |
| 146 | + "skipped": 0, |
| 147 | + "successful": 5, |
| 148 | + "total": 5 |
| 149 | + }, |
| 150 | + "aggregations": { |
| 151 | + "agg": { |
| 152 | + "value": 14 |
| 153 | + } |
| 154 | + }, |
| 155 | + "hits": { |
| 156 | + "hits": [], |
| 157 | + "max_score": 0, |
| 158 | + "total": 3 |
| 159 | + }, |
| 160 | + "timed_out": false, |
| 161 | + "took": 0 |
| 162 | + } |
| 163 | + */ |
| 164 | + /** |
| 165 | + * Expected sample data |
| 166 | + { |
| 167 | + "_shards": { |
| 168 | + "failed": 0, |
| 169 | + "skipped": 0, |
| 170 | + "successful": 5, |
| 171 | + "total": 5 |
| 172 | + }, |
| 173 | + "aggregations": { |
| 174 | + "pathAggregation": { |
| 175 | + "avg_duration": { |
| 176 | + "value": 13 |
| 177 | + }, |
| 178 | + "doc_count": 3 |
| 179 | + } |
| 180 | + }, |
| 181 | + "hits": { |
| 182 | + "hits": [], |
| 183 | + "max_score": 0, |
| 184 | + "total": 6 |
| 185 | + }, |
| 186 | + "timed_out": false, |
| 187 | + "took": 0 |
| 188 | + } |
| 189 | + */ |
| 190 | + implicit def read(json: JsValue) = { |
| 191 | + val jsObject = json.asJsObject |
| 192 | + |
| 193 | + jsObject.getFields("aggregations", "took", "hits") match { |
| 194 | + case Seq(aggregations, took, hits) => |
| 195 | + val hitCount = hits.asJsObject.getFields("total").headOption |
| 196 | + val filterAggregations = aggregations.asJsObject.getFields(FilterAggregationName) |
| 197 | + val averageAggregations = aggregations.asJsObject.getFields(AverageAggregationName) |
| 198 | + |
| 199 | + (filterAggregations, averageAggregations, hitCount) match { |
| 200 | + case (filterAggregations, _, Some(count)) if filterAggregations.nonEmpty => |
| 201 | + val averageDuration = |
| 202 | + filterAggregations.headOption.flatMap( |
| 203 | + _.asJsObject |
| 204 | + .getFields(AverageAggregationName) |
| 205 | + .headOption |
| 206 | + .flatMap(_.asJsObject.getFields("value").headOption)) |
| 207 | + |
| 208 | + averageDuration match { |
| 209 | + case Some(JsNull) => |
| 210 | + DurationCheckResult(None, count.convertTo[Long], took.convertTo[Long]) |
| 211 | + |
| 212 | + case Some(duration) => |
| 213 | + DurationCheckResult(Some(duration.convertTo[Double]), count.convertTo[Long], took.convertTo[Long]) |
| 214 | + |
| 215 | + case _ => deserializationError("Cannot deserialize ProductItem: invalid input. Raw input: ") |
| 216 | + } |
| 217 | + |
| 218 | + case (_, averageAggregations, Some(count)) if averageAggregations.nonEmpty => |
| 219 | + val averageDuration = averageAggregations.headOption.flatMap(_.asJsObject.getFields("value").headOption) |
| 220 | + |
| 221 | + averageDuration match { |
| 222 | + case Some(JsNull) => |
| 223 | + DurationCheckResult(None, count.convertTo[Long], took.convertTo[Long]) |
| 224 | + |
| 225 | + case Some(duration) => |
| 226 | + DurationCheckResult(Some(duration.convertTo[Double]), count.convertTo[Long], took.convertTo[Long]) |
| 227 | + |
| 228 | + case t => deserializationError(s"Cannot deserialize DurationCheckResult: invalid input. Raw input: $t") |
| 229 | + } |
| 230 | + |
| 231 | + case t => deserializationError(s"Cannot deserialize DurationCheckResult: invalid input. Raw input: $t") |
| 232 | + } |
| 233 | + |
| 234 | + case other => deserializationError(s"Cannot deserialize DurationCheckResult: invalid input. Raw input: $other") |
| 235 | + } |
| 236 | + |
| 237 | + } |
| 238 | + |
| 239 | + // This method would not be used. |
| 240 | + override def write(obj: DurationCheckResult): JsValue = { |
| 241 | + JsArray(JsNumber(obj.averageDuration.get), JsNumber(obj.hitCount), JsNumber(obj.took)) |
| 242 | + } |
| 243 | +} |
| 244 | + |
| 245 | +case class DurationCheckerConfig(timeWindow: FiniteDuration) |
0 commit comments