Skip to content

Commit 19ec5ba

Browse files
thinker0thinker0
committed
Merge branch 'master' into java_11_support
* master: Re-organize UI elements in Heorn UI topology page (apache#3397) Update the instruction for using the docker based development environment (apache#3395) Add tables for component running info (apache#3394) Helm chart Bookkeeper fix (apache#3381) Docker site (apache#3393) Refactor counters sections in Heron UI topology page (apache#3392) fix base url and counters flag; combine metrics requests (apache#3385) makemetricstriggerperiodic (apache#3384) Fix max capacity timeline in Heron UI (apache#3383) Avoid duplicated metrics requests in Heron UI (apache#3378) Add instance resource usage in container (apache#3375) Clean up license for healthHandler and jarjar help.txt (apache#3377) Fix license in health handler py (apache#3376) updating the NOTICE (apache#3374)
2 parents 4ca4810 + 06a55e9 commit 19ec5ba

File tree

21 files changed

+1278
-319
lines changed

21 files changed

+1278
-319
lines changed

‎LICENSE

+1-4
Original file line numberDiff line numberDiff line change
@@ -248,12 +248,9 @@ See project link for details.
248248
-> scripts/packages/tests_template_bin.sh
249249
-> scripts/packages/bin_common.sh
250250
-> third_party/java/bazel/extra_actions_base.proto
251-
Twitter
252-
-> heron/shell/src/python/handlers/healthhandler.py
253-
Google
251+
jarjar(https://github.com/google/jarjar/blob/master/LICENSE)
254252
-> third_party/java/jarjar/src/main/resources/com/tonicsystems/jarjar/help.txt
255253

256-
257254
========================================================================
258255
Third party MIT licenses
259256
========================================================================

‎NOTICE

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
Apache Heron
1+
Apache Heron (Incubating)
22
Copyright 2018-2019 The Apache Software Foundation
33

44
This product includes software developed at

‎deploy/kubernetes/helm/templates/bookie.yaml

+3
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,9 @@ data:
4141
BK_indexDirectories: "/bookkeeper/data/ledgers"
4242
BK_zkServers: {{ .Release.Name }}-zookeeper:{{ .Values.zookeeper.clientPort }}
4343
BK_autoRecoveryDaemonEnabled: "true"
44+
{{- if or (eq .Values.platform "gke") (eq .Values.platform "minikube") }}
45+
BK_useHostNameAsBookieID: "true"
46+
{{- end }}
4447
# TODO: Issue 458: https://github.com/apache/bookkeeper/issues/458
4548
{{- if eq .Values.bookkeeper.prometheus.enabled true }}
4649
BK_enableStatistics: "true"

‎docker/scripts/dev-env.sh

+7-2
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,15 @@
3030
# bazel build --config=ubuntu heron/...
3131
#
3232
# To enter an existing container with a new shell, find the container
33-
# id with this command:
33+
# ID with this command first:
3434
# docker ps -a
35-
# And then :
35+
# The image name looks like: "heron-dev:ubuntu18.04".=
36+
# After the container is found, execute the following commands to start
37+
# the container in case it is not started yet, and then start a new
38+
# terminal in the container:
39+
# docker container start CONTAINER_ID
3640
# docker exec -it CONTAINER_ID bash
41+
#
3742

3843
set -o nounset
3944
set -o errexit

‎heron/healthmgr/src/java/org/apache/heron/healthmgr/HealthManagerMetrics.java

+1-10
Original file line numberDiff line numberDiff line change
@@ -102,19 +102,10 @@ public HealthManagerMetrics(int metricsMgrPort) throws IOException {
102102
int interval = (int) systemConfig.getHeronMetricsExportInterval().getSeconds();
103103

104104

105-
looper.registerTimerEvent(Duration.ofSeconds(interval), new Runnable() {
105+
looper.registerPeriodicEvent(Duration.ofSeconds(interval), new Runnable() {
106106
@Override
107107
public void run() {
108108
sendMetrics();
109-
// next timer task
110-
if (looper != null) {
111-
looper.registerTimerEvent(Duration.ofSeconds(interval), new Runnable() {
112-
@Override
113-
public void run() {
114-
sendMetrics();
115-
}
116-
});
117-
}
118109
}
119110
});
120111
}

‎heron/shell/src/python/handlers/healthhandler.py

+13-10
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,22 @@
11
#!/usr/bin/env python
22
# -*- encoding: utf-8 -*-
33

4-
# Copyright 2016 Twitter. All rights reserved.
5-
#
6-
# Licensed under the Apache License, Version 2.0 (the "License");
7-
# you may not use this file except in compliance with the License.
8-
# You may obtain a copy of the License at
4+
# Licensed to the Apache Software Foundation (ASF) under one
5+
# or more contributor license agreements. See the NOTICE file
6+
# distributed with this work for additional information
7+
# regarding copyright ownership. The ASF licenses this file
8+
# to you under the Apache License, Version 2.0 (the
9+
# "License"); you may not use this file except in compliance
10+
# with the License. You may obtain a copy of the License at
911
#
1012
# http://www.apache.org/licenses/LICENSE-2.0
1113
#
12-
# Unless required by applicable law or agreed to in writing, software
13-
# distributed under the License is distributed on an "AS IS" BASIS,
14-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15-
# See the License for the specific language governing permissions and
16-
# limitations under the License.
14+
# Unless required by applicable law or agreed to in writing,
15+
# software distributed under the License is distributed on an
16+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17+
# KIND, either express or implied. See the License for the
18+
# specific language governing permissions and limitations
19+
# under the License.
1720

1821
""" healthhandler.py """
1922
import tornado.web

‎heron/tools/common/src/python/access/heron_api.py

+6-3
Original file line numberDiff line numberDiff line change
@@ -867,9 +867,12 @@ def compute_max(self, multi_ts):
867867
:param multi_ts:
868868
:return:
869869
'''
870-
if len(multi_ts) > 0 and len(multi_ts[0]["timeline"]) > 0:
871-
keys = multi_ts[0]["timeline"][0]["data"].keys()
872-
timelines = ([res["timeline"][0]["data"][key] for key in keys] for res in multi_ts)
870+
# Some components don't have specific metrics such as capacity hence the
871+
# key set is empty. These components are filtered out first.
872+
filtered_ts = [ts for ts in multi_ts if len(ts["timeline"][0]["data"]) > 0]
873+
if len(filtered_ts) > 0 and len(filtered_ts[0]["timeline"]) > 0:
874+
keys = filtered_ts[0]["timeline"][0]["data"].keys()
875+
timelines = ([res["timeline"][0]["data"][key] for key in keys] for res in filtered_ts)
873876
values = (max(v) for v in zip(*timelines))
874877
return dict(zip(keys, values))
875878
return {}

‎heron/tools/ui/resources/static/css/main.css

+38
Original file line numberDiff line numberDiff line change
@@ -2253,3 +2253,41 @@ nav {
22532253
.navbar-custom .navbar-link:hover {
22542254
color: #ffecb3;
22552255
}
2256+
2257+
/* Topology details */
2258+
div#display-navigator {
2259+
border-style: solid;
2260+
border-width: 1px;
2261+
border-color: #dddddd;
2262+
border-bottom-width: 0px;
2263+
}
2264+
2265+
#display-navigator .navbar-default button {
2266+
padding-left: 15px;
2267+
padding-right: 15px;
2268+
padding-top: 5px;
2269+
padding-bottom: 5px;
2270+
background-color: transparent;
2271+
border-width: 0px;
2272+
}
2273+
2274+
#display-navigator .navbar-default button.active {
2275+
background-color: lightskyblue;
2276+
}
2277+
2278+
/* Select the right topology details to show */
2279+
div#topologydetails {
2280+
border-style: solid;
2281+
border-width: 1px;
2282+
border-color: #dddddd;
2283+
}
2284+
2285+
div#topologydetails div.display-info {
2286+
display: none;
2287+
}
2288+
2289+
div#topologydetails.display-stats div.display-info.display-stats,
2290+
div#topologydetails.display-counters div.display-info.display-counters,
2291+
div#topologydetails.display-config div.display-info.display-config {
2292+
display: block;
2293+
}

‎heron/tools/ui/resources/static/js/config.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,7 @@ var ConfigTable = React.createClass({
166166
tableLayout: 'inherit'
167167
};
168168
return (
169-
<div>
169+
<div className="display-info display-config">
170170
<div className="widget-header">
171171
<div className="title">
172172
<h4>{title}</h4>

‎heron/tools/ui/resources/static/js/physical-plan.js

+38-9
Original file line numberDiff line numberDiff line change
@@ -63,16 +63,23 @@
6363
container.children = _.sortBy(container.children, 'name');
6464
// Parse index
6565
container.index = parseInt(container.id.split('-')[1]);
66-
// Search for resource config in packing plan
66+
// Search for container and instance resource config in packing plan
6767
container.required_resources = {
68-
'cpu': 0,
69-
'disk': 0,
70-
'ram': 0
68+
'cpu': 'unknown',
69+
'disk': 'unknown',
70+
'ram': 'unknown'
7171
};
7272
for (var i in containerPlan) {
7373
var packing = containerPlan[i];
7474
if (packing.id === container.index) {
7575
container.required_resources = packing.required_resources;
76+
// Get instance resources
77+
var instance_resources = {};
78+
for (var j in packing.instances) {
79+
var inst = packing.instances[j];
80+
instance_resources[inst.component_name + '_' + inst.task_id] = inst.instance_resources;
81+
}
82+
container.instance_resources = instance_resources;
7683
}
7784
}
7885
});
@@ -86,6 +93,18 @@
8693
return d.children.length;
8794
});
8895

96+
function formatByteSize(byteSize) {
97+
if (byteSize > 1024 * 1024 * 1024 / 2) {
98+
return (byteSize / (1024 * 1024 * 1024)).toFixed(2) + 'GB';
99+
} else if (byteSize > 1024 * 1024 / 2) {
100+
return (byteSize / (1024 * 1024)).toFixed(2) + 'MB';
101+
} else if (byteSize > 1024 / 2) {
102+
return (byteSize / 1024).toFixed(2) + 'KB';
103+
} else {
104+
return byteSize + 'B';
105+
}
106+
}
107+
89108
/**
90109
* Config paramaters for container/heron instance layout
91110
*/
@@ -159,15 +178,25 @@
159178
});
160179

161180
var container_tip = d3.tip()
162-
.attr('class', 'd3-tip main text-center container')
181+
.attr('class', 'd3-tip main container')
163182
.offset([8, 0])
164183
.direction('s')
165184
.html(function (container) {
185+
var instance_cpu = '';
186+
var instance_ram = '';
187+
var instance_disk = '';
188+
if (container.instance_resources) {
189+
for (var inst in container.instance_resources) {
190+
instance_cpu += '<li> - ' + inst + ': <span>' + container.instance_resources[inst].cpu + '</span></li>';
191+
instance_ram += '<li> - ' + inst + ': <span>' + formatByteSize(container.instance_resources[inst].ram) + '</span></li>';
192+
instance_disk += '<li> - ' + inst + ': <span>' + formatByteSize(container.instance_resources[inst].disk) + '</span></li>';
193+
}
194+
}
166195
return '<ul>' +
167-
'<li>cpu required: ' + container.required_resources.cpu + '</li>' +
168-
'<li>ram required: ' + (container.required_resources.ram / 1048576).toFixed(2) + 'MB</li>' +
169-
'<li>disk required: ' + (container.required_resources.disk / 1048576).toFixed(2) + 'MB</li>' +
170-
'</ul>';
196+
'<li>cpu required: <span>' + container.required_resources.cpu + '</span></li>' + instance_cpu +
197+
'<li>ram required: <span>' + formatByteSize(container.required_resources.ram) + '</span></li>' + instance_ram +
198+
'<li>disk required: <span>' + formatByteSize(container.required_resources.disk) + '</span></li>' + instance_disk +
199+
'</ul>';
171200
});
172201

173202
var containerGs = svg.selectAll('.physical-plan')

‎heron/tools/ui/resources/static/js/plan-controller.js

+1
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@ function PlanController(baseUrl, cluster, environ, toponame, physicalPlan, logic
110110
function clearFocus() {
111111
React.renderComponent(
112112
AllMetrics({
113+
baseUrl: baseUrl,
113114
cluster: cluster,
114115
environ: environ,
115116
topology: toponame

0 commit comments

Comments
 (0)