Skip to content
This repository was archived by the owner on Aug 2, 2022. It is now read-only.

Commit 684690a

Browse files
penghuopenghuo
authored
Add PPL stats endpoint (#706)
* Add PPL stats endpoint * update
1 parent 126eb4e commit 684690a

File tree

6 files changed

+207
-6
lines changed

6 files changed

+207
-6
lines changed

‎integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/MetricsIT.java

+85
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
/*
2+
*
3+
* Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License").
6+
* You may not use this file except in compliance with the License.
7+
* A copy of the License is located at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* or in the "license" file accompanying this file. This file is distributed
12+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
13+
* express or implied. See the License for the specific language governing
14+
* permissions and limitations under the License.
15+
*
16+
*/
17+
18+
package com.amazon.opendistroforelasticsearch.sql.ppl;
19+
20+
import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_BANK;
21+
import static org.hamcrest.Matchers.equalTo;
22+
23+
import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.MetricName;
24+
import java.io.BufferedReader;
25+
import java.io.IOException;
26+
import java.io.InputStream;
27+
import java.io.InputStreamReader;
28+
import java.util.concurrent.TimeUnit;
29+
import org.elasticsearch.client.Request;
30+
import org.elasticsearch.client.Response;
31+
import org.json.JSONObject;
32+
import org.junit.Assert;
33+
import org.junit.Test;
34+
35+
public class MetricsIT extends PPLIntegTestCase {
36+
37+
@Override
38+
protected void init() throws Exception {
39+
loadIndex(Index.BANK);
40+
}
41+
42+
@Test
43+
public void requestCount() throws IOException, InterruptedException {
44+
int beforeQueries = pplRequestTotal();
45+
multiQueries(3);
46+
TimeUnit.SECONDS.sleep(2L);
47+
48+
assertThat(pplRequestTotal(), equalTo(beforeQueries + 3));
49+
}
50+
51+
private void multiQueries(int n) throws IOException {
52+
for (int i = 0; i < n; ++i) {
53+
executeQuery(String.format("source=%s | where age = 31 + 1 | fields age", TEST_INDEX_BANK));
54+
}
55+
}
56+
57+
private Request makeStatRequest() {
58+
return new Request(
59+
"GET", "/_opendistro/_ppl/stats"
60+
);
61+
}
62+
63+
private int pplRequestTotal() throws IOException {
64+
JSONObject jsonObject = new JSONObject(executeStatRequest(makeStatRequest()));
65+
return jsonObject.getInt(MetricName.PPL_REQ_TOTAL.getName());
66+
}
67+
68+
private String executeStatRequest(final Request request) throws IOException {
69+
Response sqlResponse = client().performRequest(request);
70+
71+
Assert.assertTrue(sqlResponse.getStatusLine().getStatusCode() == 200);
72+
73+
InputStream is = sqlResponse.getEntity().getContent();
74+
StringBuilder sb = new StringBuilder();
75+
try (BufferedReader br = new BufferedReader(new InputStreamReader(is))) {
76+
String line = null;
77+
while ((line = br.readLine()) != null) {
78+
sb.append(line);
79+
}
80+
}
81+
82+
return sb.toString();
83+
}
84+
85+
}

‎legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/MetricFactory.java

+4
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ public static Metric createMetric(MetricName name) {
2525
case REQ_TOTAL:
2626
case DEFAULT_CURSOR_REQUEST_TOTAL:
2727
case DEFAULT:
28+
case PPL_REQ_TOTAL:
2829
return new NumericMetric<>(name.getName(), new BasicCounter());
2930
case CIRCUIT_BREAKER:
3031
return new GaugeMetric<>(name.getName(), BackOffRetryStrategy.GET_CB_STATE);
@@ -33,6 +34,9 @@ public static Metric createMetric(MetricName name) {
3334
case FAILED_REQ_COUNT_CUS:
3435
case FAILED_REQ_COUNT_SYS:
3536
case FAILED_REQ_COUNT_CB:
37+
case PPL_REQ_COUNT_TOTAL:
38+
case PPL_FAILED_REQ_COUNT_CUS:
39+
case PPL_FAILED_REQ_COUNT_SYS:
3640
return new NumericMetric<>(name.getName(), new RollingCounter());
3741
default:
3842
return new NumericMetric<>(name.getName(), new BasicCounter());

‎legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/MetricName.java

+19-4
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,10 @@
1515

1616
package com.amazon.opendistroforelasticsearch.sql.legacy.metrics;
1717

18+
import com.google.common.collect.ImmutableSet;
1819
import java.util.Arrays;
1920
import java.util.List;
21+
import java.util.Set;
2022
import java.util.stream.Collectors;
2123

2224
public enum MetricName {
@@ -29,7 +31,12 @@ public enum MetricName {
2931
DEFAULT_CURSOR_REQUEST_TOTAL("default_cursor_request_total"),
3032
DEFAULT_CURSOR_REQUEST_COUNT_TOTAL("default_cursor_request_count"),
3133
CIRCUIT_BREAKER("circuit_breaker"),
32-
DEFAULT("default");
34+
DEFAULT("default"),
35+
36+
PPL_REQ_TOTAL("ppl_request_total"),
37+
PPL_REQ_COUNT_TOTAL("ppl_request_count"),
38+
PPL_FAILED_REQ_COUNT_SYS("ppl_failed_request_count_syserr"),
39+
PPL_FAILED_REQ_COUNT_CUS("ppl_failed_request_count_cuserr");
3340

3441
private String name;
3542

@@ -45,10 +52,18 @@ public static List<String> getNames() {
4552
return Arrays.stream(MetricName.values()).map(v -> v.name).collect(Collectors.toList());
4653
}
4754

55+
56+
private static Set<MetricName> NUMERICAL_METRIC = new ImmutableSet.Builder<MetricName>()
57+
.add(PPL_REQ_TOTAL)
58+
.add(PPL_REQ_COUNT_TOTAL)
59+
.add(PPL_FAILED_REQ_COUNT_SYS)
60+
.add(PPL_FAILED_REQ_COUNT_CUS)
61+
.build();
62+
4863
public boolean isNumerical() {
4964
return this == REQ_TOTAL || this == REQ_COUNT_TOTAL || this == FAILED_REQ_COUNT_SYS
50-
|| this == FAILED_REQ_COUNT_CUS || this == FAILED_REQ_COUNT_CB || this == DEFAULT
51-
|| this == DEFAULT_CURSOR_REQUEST_TOTAL || this == DEFAULT_CURSOR_REQUEST_COUNT_TOTAL;
65+
|| this == FAILED_REQ_COUNT_CUS || this == FAILED_REQ_COUNT_CB || this == DEFAULT
66+
|| this == DEFAULT_CURSOR_REQUEST_TOTAL || this == DEFAULT_CURSOR_REQUEST_COUNT_TOTAL
67+
|| NUMERICAL_METRIC.contains(this);
5268
}
53-
5469
}

‎plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SQLPlugin.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlStatsAction;
2727
import com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings;
2828
import com.amazon.opendistroforelasticsearch.sql.plugin.rest.RestPPLQueryAction;
29+
import com.amazon.opendistroforelasticsearch.sql.plugin.rest.RestPPLStatsAction;
2930
import com.google.common.collect.ImmutableList;
3031
import java.util.Arrays;
3132
import java.util.Collection;
@@ -100,7 +101,8 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
100101
new RestPPLQueryAction(restController, clusterService, pluginSettings, settings),
101102
new RestSqlAction(settings, clusterService, pluginSettings),
102103
new RestSqlStatsAction(settings, restController),
103-
new RestSqlSettingsAction(settings, restController)
104+
new RestSqlSettingsAction(settings, restController),
105+
new RestPPLStatsAction(settings, restController)
104106
);
105107
}
106108

‎plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestPPLQueryAction.java

+15-1
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,9 @@
2929
import com.amazon.opendistroforelasticsearch.sql.exception.QueryEngineException;
3030
import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException;
3131
import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine.QueryResponse;
32+
import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.MetricName;
33+
import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.Metrics;
34+
import com.amazon.opendistroforelasticsearch.sql.legacy.utils.LogUtils;
3235
import com.amazon.opendistroforelasticsearch.sql.plugin.request.PPLQueryRequestFactory;
3336
import com.amazon.opendistroforelasticsearch.sql.ppl.PPLService;
3437
import com.amazon.opendistroforelasticsearch.sql.ppl.config.PPLServiceConfig;
@@ -97,6 +100,11 @@ public String getName() {
97100

98101
@Override
99102
protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient nodeClient) {
103+
Metrics.getInstance().getNumericalMetric(MetricName.PPL_REQ_TOTAL).increment();
104+
Metrics.getInstance().getNumericalMetric(MetricName.PPL_REQ_COUNT_TOTAL).increment();
105+
106+
LogUtils.addRequestId();
107+
100108
if (!pplEnabled.get()) {
101109
return channel -> reportError(channel, new IllegalAccessException(
102110
"Either opendistro.ppl.enabled or rest.action.multi.allow_explicit_index setting is false"
@@ -145,7 +153,13 @@ public void onResponse(QueryResponse response) {
145153
@Override
146154
public void onFailure(Exception e) {
147155
LOG.error("Error happened during query handling", e);
148-
reportError(channel, e, isClientError(e) ? BAD_REQUEST : SERVICE_UNAVAILABLE);
156+
if (isClientError(e)) {
157+
Metrics.getInstance().getNumericalMetric(MetricName.PPL_FAILED_REQ_COUNT_CUS).increment();
158+
reportError(channel, e, BAD_REQUEST);
159+
} else {
160+
Metrics.getInstance().getNumericalMetric(MetricName.PPL_FAILED_REQ_COUNT_SYS).increment();
161+
reportError(channel, e, SERVICE_UNAVAILABLE);
162+
}
149163
}
150164

151165
private void sendResponse(RestStatus status, String content) {

‎plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestPPLStatsAction.java

+81
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
/*
2+
*
3+
* Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License").
6+
* You may not use this file except in compliance with the License.
7+
* A copy of the License is located at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* or in the "license" file accompanying this file. This file is distributed
12+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
13+
* express or implied. See the License for the specific language governing
14+
* permissions and limitations under the License.
15+
*
16+
*/
17+
18+
package com.amazon.opendistroforelasticsearch.sql.plugin.rest;
19+
20+
import static org.elasticsearch.rest.RestStatus.SERVICE_UNAVAILABLE;
21+
22+
import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.ErrorMessageFactory;
23+
import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.Metrics;
24+
import com.amazon.opendistroforelasticsearch.sql.legacy.utils.LogUtils;
25+
import com.google.common.collect.ImmutableList;
26+
import java.util.List;
27+
import org.apache.logging.log4j.LogManager;
28+
import org.apache.logging.log4j.Logger;
29+
import org.elasticsearch.client.node.NodeClient;
30+
import org.elasticsearch.common.settings.Settings;
31+
import org.elasticsearch.rest.BaseRestHandler;
32+
import org.elasticsearch.rest.BytesRestResponse;
33+
import org.elasticsearch.rest.RestController;
34+
import org.elasticsearch.rest.RestRequest;
35+
import org.elasticsearch.rest.RestStatus;
36+
37+
/**
38+
* PPL Node level status.
39+
*/
40+
public class RestPPLStatsAction extends BaseRestHandler {
41+
42+
private static final Logger LOG = LogManager.getLogger(RestPPLStatsAction.class);
43+
44+
/**
45+
* API endpoint path.
46+
*/
47+
public static final String PPL_STATS_API_ENDPOINT = "/_opendistro/_ppl/stats";
48+
49+
public RestPPLStatsAction(Settings settings, RestController restController) {
50+
super();
51+
}
52+
53+
@Override
54+
public String getName() {
55+
return "ppl_stats_action";
56+
}
57+
58+
@Override
59+
public List<Route> routes() {
60+
return ImmutableList.of(
61+
new Route(RestRequest.Method.POST, PPL_STATS_API_ENDPOINT),
62+
new Route(RestRequest.Method.GET, PPL_STATS_API_ENDPOINT)
63+
);
64+
}
65+
66+
@Override
67+
protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
68+
69+
LogUtils.addRequestId();
70+
71+
try {
72+
return channel -> channel.sendResponse(new BytesRestResponse(RestStatus.OK,
73+
Metrics.getInstance().collectToJSON()));
74+
} catch (Exception e) {
75+
LOG.error("Failed during Query PPL STATS Action.", e);
76+
77+
return channel -> channel.sendResponse(new BytesRestResponse(SERVICE_UNAVAILABLE,
78+
ErrorMessageFactory.createErrorMessage(e, SERVICE_UNAVAILABLE.getStatus()).toString()));
79+
}
80+
}
81+
}

0 commit comments

Comments
 (0)