Skip to content

Commit ab7ba98

Browse files
dylan-fandylan-fan
authored
Merge pull request #204 from FederatedAI/dev-2.1.7
Dev 2.1.7
2 parents 8892c98 + 893b30c commit ab7ba98

File tree

62 files changed

+1108
-268
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

62 files changed

+1108
-268
lines changed

‎document/docs/service/adapter.md

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,10 @@ Context为上下文信息，用于传递请求所需参数，featureIds用于传
4040
#在host方的配置文件serving-server.properties中将其配置成自定义的类的全路径，如下所示
4141
feature.single.adaptor=com.webank.ai.fate.serving.adaptor.dataaccess.CustomAdapter
4242
feature.batch.adaptor=com.webank.ai.fate.serving.adaptor.dataaccess.CustomBatchAdapter
43+
feature.batch.single.adatpor=com.webank.ai.fate.serving.adaptor.dataaccess.CustomAdapter
4344
```
4445
可以根据需要实现Adapter中的逻辑，并修改serving-server.properties中feature.single.adaptor或feature.batch.adaptor配置项为新增Adapter的全类名即可。可以参考源码中的MockAdaptor
46+
注: feature.batch.single.adatpor与feature.batch.adatpor配套使用，feature.batch.single.adatpor可根据用户场景自行实现,fate-serving中目前支持httpAdaptor
4547

4648
## fate-serving-extension
4749
为了更好的代码解耦合，代码中将自定义adapter分离到fate-serving-extension模块中。用户可在此模块中开发自定义的adapter。
@@ -69,7 +71,7 @@ x0:1,x1:5,x2:13,x3:58,x4:95,x5:352,x6:418,x7:833,x8:888,x9:937,x10:32776
6971

7072
#### HttpAdapter
7173
在serving-server.properties文件中配置属性feature.single.adaptor和http.adapter.url，feature.single.adaptor为继承AbstractSingleFeatureDataAdaptor
72-
接口，url为调用获取数据接口地址。
74+
接口，url为调用获取数据接口地址。http.adapter.url中标明的用户接口，返回格式请定义为 {"code": 200, "data": xxx}标准格式即可，httpAdapter中会根据接口返回状态码是否为200判断用户数据拉取接口是否执行成功。
7375
```yaml
7476
feature.single.adaptor=com.webank.ai.fate.serving.adaptor.dataaccess.HttpAdapter
7577
http.adapter.url=http://127.0.0.1:9380/v1/http/adapter/getFeature

‎document/docs/service/admin.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ serving-admin提供了FATE-Serving集群的可视化操作界面，依赖zookeep
55
### 功能介绍
66
#### 用户管理
77
默认用户：admin，默认密码：admin，用户可在[conf/application.properties](config/admin.md)中修改预设用户。
8+
除此之外serving-admin提供一个基本的登录密码加解密功能，用户可在[conf/application.properties](config/application.properties)
9+
中通过设置admin.isEncrypt参数为true（默认为false关闭），同时根据spring.security中的BCryptPasswordEncoder库对密码进行提前处理并预设为默认密码。
810
serving-admin仅实现简单的用户登录，用户可业务需求，自行实现登录逻辑，或接入第三方平台。
911

1012
#### 节点管理

‎fate-serving-admin-ui/README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ fate-serving定制联邦服务管理端
44
## Build Setup
55

66
# Install dependencies
7-
npm install
7+
npm install --force
88

99
# Serve with hot reload at localhost:8080
1010
npm run serve

‎fate-serving-admin-ui/package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
"svg-sprite-loader": "^3.9.2",
4949
"vue-template-compiler": "^2.6.10",
5050
"webpack-bundle-analyzer": ">=3.3.2",
51-
"webpack-cli": "^3.2.3"
51+
"webpack-cli": "^3.2.3",
52+
"webpack": "^4.0.0"
5253
}
5354
}

‎fate-serving-admin-ui/pom.xml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@
5555
<goal>install-node-and-npm</goal>
5656
</goals>
5757
<configuration>
58-
<nodeVersion>v9.11.1</nodeVersion>
58+
<nodeVersion>v16.20.2</nodeVersion>
5959
</configuration>
6060
</execution>
6161
<!-- Install all project dependencies -->
@@ -68,7 +68,7 @@
6868
<phase>generate-resources</phase>
6969
<!-- Optional configuration which provides for running any npm command -->
7070
<configuration>
71-
<arguments>install</arguments>
71+
<arguments>install --force</arguments>
7272
</configuration>
7373
</execution>
7474
<!-- Build and minify static files -->

‎fate-serving-admin/bin/service.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ basepath=$(cd `dirname $0`;pwd)
2424
configpath=$(cd $basepath/conf;pwd)
2525
module=serving-admin
2626
main_class=com.webank.ai.fate.serving.admin.Bootstrap
27-
module_version=2.1.6
27+
module_version=2.1.7
2828

2929
case "$1" in
3030
start)

‎fate-serving-admin/pom.xml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,11 @@
5151
<version>${fate.version}</version>
5252
</dependency>
5353

54+
<dependency>
55+
<groupId>org.springframework.boot</groupId>
56+
<artifactId>spring-boot-starter-security</artifactId>
57+
</dependency>
58+
5459

5560

5661
<!--<dependency>

‎fate-serving-admin/src/main/java/com/webank/ai/fate/serving/admin/Bootstrap.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ public void stop() {
119119
} catch (InterruptedException e) {
120120
e.printStackTrace();
121121
}
122+
tryNum++;
122123
}
123124
}
124125
}

‎fate-serving-admin/src/main/java/com/webank/ai/fate/serving/admin/bean/ServiceConfiguration.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,7 @@ public static boolean isAllowModify(String project, String config) {
4444
return Boolean.FALSE;
4545
}
4646

47-
boolean match = Arrays.asList(value.config).contains(config);
48-
return match;
47+
return Arrays.asList(value.config).contains(config);
4948
}
5049

5150
}

‎fate-serving-admin/src/main/java/com/webank/ai/fate/serving/admin/config/WebConfig.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,8 +72,7 @@ public Cache cache() {
7272
Integer maxSize = MetaInfo.PROPERTY_LOCAL_CACHE_MAXSIZE;
7373
Integer expireTime = MetaInfo.PROPERTY_LOCAL_CACHE_EXPIRE;
7474
Integer interval = MetaInfo.PROPERTY_LOCAL_CACHE_INTERVAL;
75-
ExpiringLRUCache lruCache = new ExpiringLRUCache(maxSize, expireTime, interval);
76-
return lruCache;
75+
return new ExpiringLRUCache(maxSize, expireTime, interval);
7776
}
7877

7978

‎fate-serving-admin/src/main/java/com/webank/ai/fate/serving/admin/controller/ComponentController.java

Lines changed: 14 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,14 @@
1616

1717
package com.webank.ai.fate.serving.admin.controller;
1818

19+
import com.fasterxml.jackson.databind.ObjectMapper;
1920
import com.google.common.base.Preconditions;
2021
import com.google.common.collect.Maps;
2122
import com.webank.ai.fate.api.networking.common.CommonServiceGrpc;
2223
import com.webank.ai.fate.api.networking.common.CommonServiceProto;
2324
import com.webank.ai.fate.serving.admin.bean.ServiceConfiguration;
2425
import com.webank.ai.fate.serving.admin.services.ComponentService;
26+
import com.webank.ai.fate.serving.admin.utils.NetAddressChecker;
2527
import com.webank.ai.fate.serving.core.bean.*;
2628
import com.webank.ai.fate.serving.core.constant.StatusCode;
2729
import com.webank.ai.fate.serving.core.exceptions.RemoteRpcException;
@@ -53,24 +55,22 @@ public class ComponentController {
5355

5456
private static final Logger logger = LoggerFactory.getLogger(ComponentController.class);
5557

58+
private final ObjectMapper objectMapper = new ObjectMapper();
59+
5660
@Autowired
5761
ComponentService componentServices;
5862
GrpcConnectionPool grpcConnectionPool = GrpcConnectionPool.getPool();
5963

6064
@GetMapping("/component/list")
6165
public ReturnResult list() {
6266
ComponentService.NodeData cachedNodeData = componentServices.getCachedNodeData();
63-
return ReturnResult.build(StatusCode.SUCCESS, Dict.SUCCESS, JsonUtil.json2Object(JsonUtil.object2Json(cachedNodeData), Map.class));
67+
Map<String, Object> cachedNodeDataMap = objectMapper.convertValue(cachedNodeData, Map.class);
68+
return ReturnResult.build(StatusCode.SUCCESS, Dict.SUCCESS, cachedNodeDataMap);
6469
}
6570

6671
@GetMapping("/component/listProps")
6772
public ReturnResult listProps(String host, int port, String keyword) {
68-
if (!NetUtils.isValidAddress(host + ":" + port)) {
69-
throw new SysException("invalid address");
70-
}
71-
if (!componentServices.isAllowAccess(host, port)) {
72-
throw new RemoteRpcException("no allow access, target: " + host + ":" + port);
73-
}
73+
NetAddressChecker.check(host, port);
7474
ManagedChannel managedChannel = grpcConnectionPool.getManagedChannel(host, port);
7575
CommonServiceGrpc.CommonServiceBlockingStub blockingStub = CommonServiceGrpc.newBlockingStub(managedChannel);
7676
blockingStub = blockingStub.withDeadlineAfter(MetaInfo.PROPERTY_GRPC_TIMEOUT, TimeUnit.MILLISECONDS);
@@ -102,17 +102,15 @@ public ReturnResult listProps(String host, int port, String keyword) {
102102

103103
@PostMapping("/component/updateConfig")
104104
public ReturnResult updateConfig(@RequestBody RequestParamWrapper requestParams) {
105-
Preconditions.checkArgument(StringUtils.isNotBlank(requestParams.getFilePath()), "file path is blank");
106-
Preconditions.checkArgument(StringUtils.isNotBlank(requestParams.getData()), "data is blank");
105+
String filePath = requestParams.getFilePath();
106+
String data = requestParams.getData();
107+
Preconditions.checkArgument(StringUtils.isNotBlank(filePath), "file path is blank");
108+
Preconditions.checkArgument(StringUtils.isNotBlank(data), "data is blank");
107109

108110
String host = requestParams.getHost();
109111
int port = requestParams.getPort();
112+
NetAddressChecker.check(host, port);
110113

111-
if (!componentServices.isAllowAccess(host, port)) {
112-
throw new RemoteRpcException("no allow access, target: " + host + ":" + port);
113-
}
114-
115-
String filePath = requestParams.getFilePath();
116114
String fileName = filePath.substring(filePath.lastIndexOf(File.separator) + 1);
117115

118116
String project = componentServices.getProject(host, port);
@@ -124,8 +122,8 @@ public ReturnResult updateConfig(@RequestBody RequestParamWrapper requestParams)
124122
CommonServiceGrpc.CommonServiceBlockingStub blockingStub = CommonServiceGrpc.newBlockingStub(managedChannel)
125123
.withDeadlineAfter(MetaInfo.PROPERTY_GRPC_TIMEOUT, TimeUnit.MILLISECONDS);
126124
CommonServiceProto.UpdateConfigRequest.Builder builder = CommonServiceProto.UpdateConfigRequest.newBuilder();
127-
builder.setFilePath(requestParams.getFilePath());
128-
builder.setData(requestParams.getData());
125+
builder.setFilePath(filePath);
126+
builder.setData(data);
129127

130128
CommonServiceProto.CommonResponse response = blockingStub.updateConfig(builder.build());
131129
return ReturnResult.build(response.getStatusCode(), response.getMessage());

‎fate-serving-admin/src/main/java/com/webank/ai/fate/serving/admin/controller/DynamicLogController.java

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
package com.webank.ai.fate.serving.admin.controller;
2+
3+
import org.apache.logging.log4j.Level;
4+
import org.apache.logging.log4j.LogManager;
5+
import org.apache.logging.log4j.core.LoggerContext;
6+
import org.slf4j.Logger;
7+
import org.slf4j.LoggerFactory;
8+
import org.springframework.web.bind.annotation.*;
9+
10+
/**
11+
* @author hcy
12+
*/
13+
@RequestMapping("/admin")
14+
@RestController
15+
public class DynamicLogController {
16+
private static final Logger logger = LoggerFactory.getLogger(DynamicLogController.class);
17+
18+
@GetMapping("/alterSysLogLevel/{level}")
19+
public String alterSysLogLevel(@PathVariable String level){
20+
try {
21+
LoggerContext context = (LoggerContext) LogManager.getContext(false);
22+
context.getLogger("ROOT").setLevel(Level.valueOf(level));
23+
return "ok";
24+
} catch (Exception ex) {
25+
logger.error("admin alterSysLogLevel failed : " + ex);
26+
return "failed";
27+
}
28+
29+
}
30+
31+
@GetMapping("/alterPkgLogLevel")
32+
public String alterPkgLogLevel(@RequestParam String level, @RequestParam String pkgName){
33+
try {
34+
LoggerContext context = (LoggerContext) LogManager.getContext(false);
35+
context.getLogger(pkgName).setLevel(Level.valueOf(level));
36+
return "ok";
37+
} catch (Exception ex) {
38+
logger.error("admin alterPkgLogLevel failed : " + ex);
39+
return "failed";
40+
}
41+
}
42+
}

‎fate-serving-admin/src/main/java/com/webank/ai/fate/serving/admin/controller/LoginController.java

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.springframework.web.bind.annotation.RequestBody;
3131
import org.springframework.web.bind.annotation.RequestMapping;
3232
import org.springframework.web.bind.annotation.RestController;
33+
import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder;
3334

3435
import javax.servlet.http.HttpServletRequest;
3536
import java.util.Arrays;
@@ -51,7 +52,10 @@ public class LoginController {
5152
private String username;
5253

5354
@Value("${admin.password}")
54-
private String password;
55+
private String hashedPassword;
56+
57+
@Value("${admin.isEncrypt}")
58+
private Boolean isEncrypt;
5559

5660
@Autowired
5761
private Cache cache;
@@ -60,12 +64,20 @@ public class LoginController {
6064
public ReturnResult login(@RequestBody RequestParamWrapper requestParams) {
6165
String username = requestParams.getUsername();
6266
String password = requestParams.getPassword();
67+
boolean passwordIfCorrect;
6368

6469
Preconditions.checkArgument(StringUtils.isNotBlank(username), "parameter username is blank");
6570
Preconditions.checkArgument(StringUtils.isNotBlank(password), "parameter password is blank");
6671

6772
ReturnResult result = new ReturnResult();
68-
if (username.equals(this.username) && password.equals(this.password)) {
73+
74+
if (isEncrypt) {
75+
passwordIfCorrect = new BCryptPasswordEncoder().matches(password, this.hashedPassword);
76+
} else {
77+
passwordIfCorrect = password.equals(this.hashedPassword);
78+
}
79+
80+
if (username.equals(this.username) && passwordIfCorrect) {
6981
String userInfo = StringUtils.join(Arrays.asList(username, password), "_");
7082
String token = EncryptUtils.encrypt(Dict.USER_CACHE_KEY_PREFIX + userInfo, EncryptMethod.MD5);
7183
cache.put(token, userInfo, MetaInfo.PROPERTY_CACHE_TYPE.equalsIgnoreCase("local") ? MetaInfo.PROPERTY_LOCAL_CACHE_EXPIRE : MetaInfo.PROPERTY_REDIS_EXPIRE);
@@ -77,7 +89,7 @@ public ReturnResult login(@RequestBody RequestParamWrapper requestParams) {
7789
result.setRetcode(StatusCode.SUCCESS);
7890
result.setData(data);
7991
} else {
80-
logger.info("user {} login failure, username or password {} is wrong.", username,password);
92+
logger.error("user {} login failure, username or password {} is wrong.", username,password);
8193
result.setRetcode(StatusCode.PARAM_ERROR);
8294
result.setRetmsg("username or password is wrong");
8395
}
@@ -94,7 +106,7 @@ public ReturnResult logout(HttpServletRequest request) {
94106
cache.delete(sessionToken);
95107
result.setRetcode(StatusCode.SUCCESS);
96108
} else {
97-
logger.info("Session token unavailable");
109+
logger.error("Session token unavailable");
98110
result.setRetcode(StatusCode.PARAM_ERROR);
99111
result.setRetmsg("Session token unavailable");
100112
}

‎fate-serving-admin/src/main/java/com/webank/ai/fate/serving/admin/controller/ModelController.java

Lines changed: 11 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import com.webank.ai.fate.api.mlmodel.manager.ModelServiceGrpc;
2424
import com.webank.ai.fate.api.mlmodel.manager.ModelServiceProto;
2525
import com.webank.ai.fate.serving.admin.services.ComponentService;
26+
import com.webank.ai.fate.serving.admin.utils.NetAddressChecker;
2627
import com.webank.ai.fate.serving.core.bean.GrpcConnectionPool;
2728
import com.webank.ai.fate.serving.core.bean.MetaInfo;
2829
import com.webank.ai.fate.serving.core.bean.RequestParamWrapper;
@@ -66,12 +67,15 @@ public ReturnResult queryModel(String host, Integer port, String serviceId,Strin
6667
Preconditions.checkArgument(StringUtils.isNotBlank(host), "parameter host is blank");
6768
Preconditions.checkArgument(port != 0, "parameter port is blank");
6869

69-
if (page == null || page < 0) {
70-
page = 1;
70+
int defaultPage = 1;
71+
int defaultPageSize = 10;
72+
73+
if (page == null || page <= 0) {
74+
page = defaultPage;
7175
}
7276

73-
if (pageSize == null) {
74-
pageSize = 10;
77+
if (pageSize == null || pageSize <= 0) {
78+
pageSize = defaultPageSize;
7579
}
7680

7781
if (logger.isDebugEnabled()) {
@@ -153,7 +157,6 @@ public Callable<ReturnResult> transfer(@RequestBody RequestParamWrapper requestP
153157
return () -> {
154158
String host = requestParams.getHost();
155159
Integer port = requestParams.getPort();
156-
List<String> serviceIds = requestParams.getServiceIds();
157160
String tableName = requestParams.getTableName();
158161
String namespace = requestParams.getNamespace();
159162

@@ -176,13 +179,6 @@ public Callable<ReturnResult> transfer(@RequestBody RequestParamWrapper requestP
176179
//.setServiceId()
177180
.setNamespace(namespace).setTableName(tableName).setSourceIp(host).setSourcePort(port).build();
178181

179-
180-
181-
ModelServiceProto.UnloadRequest unloadRequest = ModelServiceProto.UnloadRequest.newBuilder()
182-
.setTableName(tableName)
183-
.setNamespace(namespace)
184-
.build();
185-
186182
ListenableFuture<ModelServiceProto.FetchModelResponse> future = futureStub.fetchModel(fetchModelRequest);
187183
ModelServiceProto.FetchModelResponse response = future.get(MetaInfo.PROPERTY_GRPC_TIMEOUT, TimeUnit.MILLISECONDS);
188184

@@ -281,13 +277,7 @@ private ModelServiceGrpc.ModelServiceBlockingStub getModelServiceBlockingStub(St
281277
Preconditions.checkArgument(StringUtils.isNotBlank(host), "parameter host is blank");
282278
Preconditions.checkArgument(port != null && port.intValue() != 0, "parameter port was wrong");
283279

284-
if (!NetUtils.isValidAddress(host + ":" + port)) {
285-
throw new SysException("invalid address");
286-
}
287-
288-
if (!componentService.isAllowAccess(host, port)) {
289-
throw new RemoteRpcException("no allow access, target: " + host + ":" + port);
290-
}
280+
NetAddressChecker.check(host, port);
291281

292282
ManagedChannel managedChannel = grpcConnectionPool.getManagedChannel(host, port);
293283
ModelServiceGrpc.ModelServiceBlockingStub blockingStub = ModelServiceGrpc.newBlockingStub(managedChannel);
@@ -299,17 +289,10 @@ private ModelServiceGrpc.ModelServiceFutureStub getModelServiceFutureStub(String
299289
Preconditions.checkArgument(StringUtils.isNotBlank(host), "parameter host is blank");
300290
Preconditions.checkArgument(port != null && port.intValue() != 0, "parameter port was wrong");
301291

302-
if (!NetUtils.isValidAddress(host + ":" + port)) {
303-
throw new SysException("invalid address");
304-
}
305-
306-
if (!componentService.isAllowAccess(host, port)) {
307-
throw new RemoteRpcException("no allow access, target: " + host + ":" + port);
308-
}
292+
NetAddressChecker.check(host, port);
309293

310294
ManagedChannel managedChannel = grpcConnectionPool.getManagedChannel(host, port);
311-
ModelServiceGrpc.ModelServiceFutureStub futureStub = ModelServiceGrpc.newFutureStub(managedChannel);
312-
return futureStub;
295+
return ModelServiceGrpc.newFutureStub(managedChannel);
313296
}
314297

315298
public void parseComponentInfo(ModelServiceProto.QueryModelResponse response){

0 commit comments

Comments
 (0)