Skip to content

Commit d90d239

Browse files
authored
CVL Changes #10: Custom Validation infra, Unit Test reorganization and new test cases addition (#39)
Adding Custom Validation Infrastructure Unit Test files reorganization Adding Test YANG Schemas Adding Unit Test for new CVL APIs Adding Unit Test for leafref, when, must expression evaluation
1 parent 4aa0899 commit d90d239

33 files changed

+4238
-838
lines changed

cvl/Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ schema:
5555

5656
test-schema: | schema
5757
$(MAKE) -C testdata/schema
58-
cp $(CVL_SCHEMA_DIR)/*.yin $(CVL_TEST_SCHEMA_DIR)/
58+
cp -n $(CVL_SCHEMA_DIR)/*.yin $(CVL_TEST_SCHEMA_DIR)/
5959

6060
tests:
6161
$(MAKE) -C tests

cvl/README.md

+13-5
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ Below steps need to be done to enable CVL logging.
2020

2121
2. Change the logging flags from "false" to "true" as below:
2222

23+
```
2324
{
2425
"TRACE_CACHE": "true",
2526
"TRACE_LIBYANG": "true",
@@ -29,15 +30,22 @@ Below steps need to be done to enable CVL logging.
2930
"TRACE_DELETE": "true",
3031
"TRACE_SEMANTIC": "true",
3132
"TRACE_SYNTAX": "true",
33+
"TRACE_ONERROR": "true",
3234
"__comment1__": "Set LOGTOSTDER to 'true' to log on standard error",
33-
"LOGTOSTDERR": "true",
34-
"__comment2__": "Display log upto INFO level",
35-
"STDERRTHRESHOLD": "INFO",
36-
"__comment3__": "Display log upto INFO level 8",
37-
"VERBOSITY": "8",
35+
"LOGTOSTDERR": "false",
36+
"__comment2__": "Log messages to standard error at or above this severity level",
37+
"STDERRTHRESHOLD": "ERROR",
38+
"__comment3__": "Log to /tmp/cvl.log file",
39+
"LOG_TO_FILE": "true",
40+
"__comment4__": "Limit log file size in bytes, 0 means no limit, default 10MB",
41+
"LOG_FILE_SIZE": "10485760",
42+
"__comment5__": "Set verbosity level(1 to 8) for verbose logs",
43+
"VERBOSITY": "0",
3844
"SKIP_VALIDATION": "false",
3945
"SKIP_SEMANTIC_VALIDATION": "false"
4046
}
47+
```
48+
4149
3. Below environment variables need to be set at the end in /usr/bin/rest-server.sh in mgmt-framework docker.
4250

4351
export CVL_DEBUG=1

cvl/conf/cvl_cfg.json

+8-5
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,16 @@
77
"TRACE_DELETE": "false",
88
"TRACE_SEMANTIC": "false",
99
"TRACE_SYNTAX": "false",
10-
"__comment1__": "Log trace data when error occurs",
11-
"TRACE_ONERROR": "true",
12-
"__comment2__": "Set LOGTOSTDER to 'true' to log on standard error",
10+
"TRACE_ONERROR": "false",
11+
"__comment1__": "Set LOGTOSTDER to 'true' to log on standard error",
1312
"LOGTOSTDERR": "false",
14-
"__comment3__": "Display log upto INFO level",
13+
"__comment2__": "Display log upto INFO level",
1514
"STDERRTHRESHOLD": "ERROR",
16-
"__comment4__": "Display log upto INFO level 8",
15+
"__comment3__": "Log to /tmp/cvl.log file",
16+
"LOG_TO_FILE": "false",
17+
"__comment4__": "Limit log file size in bytes, 0 means no limit, default 10MB",
18+
"LOG_FILE_SIZE": "10485760",
19+
"__comment5__": "Display log upto INFO level 8",
1720
"VERBOSITY": "0",
1821
"SKIP_VALIDATION": "false",
1922
"SKIP_SEMANTIC_VALIDATION": "false"

cvl/custom_validation/common.go

+132
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
////////////////////////////////////////////////////////////////////////////////
2+
// //
3+
// Copyright 2019 Broadcom. The term Broadcom refers to Broadcom Inc. and/or //
4+
// its subsidiaries. //
5+
// //
6+
// Licensed under the Apache License, Version 2.0 (the "License"); //
7+
// you may not use this file except in compliance with the License. //
8+
// You may obtain a copy of the License at //
9+
// //
10+
// http://www.apache.org/licenses/LICENSE-2.0 //
11+
// //
12+
// Unless required by applicable law or agreed to in writing, software //
13+
// distributed under the License is distributed on an "AS IS" BASIS, //
14+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. //
15+
// See the License for the specific language governing permissions and //
16+
// limitations under the License. //
17+
// //
18+
////////////////////////////////////////////////////////////////////////////////
19+
20+
package custom_validation
21+
22+
import (
23+
"reflect"
24+
"github.com/antchfx/xmlquery"
25+
"github.com/go-redis/redis/v7"
26+
"github.com/Azure/sonic-mgmt-common/cvl/internal/util"
27+
"github.com/Azure/sonic-mgmt-common/cvl/internal/yparser"
28+
)
29+
30+
type CustomValidation struct {}
31+
32+
type CVLValidateType uint
33+
const (
34+
VALIDATE_NONE CVLValidateType = iota //Data is used as dependent data
35+
VALIDATE_SYNTAX //Syntax is checked and data is used as dependent data
36+
VALIDATE_SEMANTICS //Semantics is checked
37+
VALIDATE_ALL //Syntax and Semantics are checked
38+
)
39+
40+
type CVLOperation uint
41+
const (
42+
OP_NONE CVLOperation = 0 //Used to just validate the config without any operation
43+
OP_CREATE = 1 << 0//For Create operation
44+
OP_UPDATE = 1 << 1//For Update operation
45+
OP_DELETE = 1 << 2//For Delete operation
46+
)
47+
48+
//CVLRetCode CVL Error codes
49+
type CVLRetCode int
50+
const (
51+
CVL_SUCCESS CVLRetCode = iota
52+
CVL_ERROR
53+
CVL_NOT_IMPLEMENTED
54+
CVL_INTERNAL_UNKNOWN
55+
CVL_FAILURE
56+
CVL_SYNTAX_ERROR = CVLRetCode(yparser.YP_SYNTAX_ERROR)
57+
CVL_SEMANTIC_ERROR = CVLRetCode(yparser.YP_SEMANTIC_ERROR)
58+
CVL_SYNTAX_MISSING_FIELD = CVLRetCode(yparser.YP_SYNTAX_MISSING_FIELD)
59+
CVL_SYNTAX_INVALID_FIELD = CVLRetCode(yparser.YP_SYNTAX_INVALID_FIELD) /* Invalid Field */
60+
CVL_SYNTAX_INVALID_INPUT_DATA = CVLRetCode(yparser.YP_SYNTAX_INVALID_INPUT_DATA) /*Invalid Input Data */
61+
CVL_SYNTAX_MULTIPLE_INSTANCE = CVLRetCode(yparser.YP_SYNTAX_MULTIPLE_INSTANCE) /* Multiple Field Instances */
62+
CVL_SYNTAX_DUPLICATE = CVLRetCode(yparser.YP_SYNTAX_DUPLICATE) /* Duplicate Fields */
63+
CVL_SYNTAX_ENUM_INVALID = CVLRetCode(yparser.YP_SYNTAX_ENUM_INVALID) /* Invalid enum value */
64+
CVL_SYNTAX_ENUM_INVALID_NAME = CVLRetCode(yparser.YP_SYNTAX_ENUM_INVALID_NAME) /* Invalid enum name */
65+
CVL_SYNTAX_ENUM_WHITESPACE = CVLRetCode(yparser.YP_SYNTAX_ENUM_WHITESPACE) /* Enum name with leading/trailing whitespaces */
66+
CVL_SYNTAX_OUT_OF_RANGE = CVLRetCode(yparser.YP_SYNTAX_OUT_OF_RANGE) /* Value out of range/length/pattern (data) */
67+
CVL_SYNTAX_MINIMUM_INVALID = CVLRetCode(yparser.YP_SYNTAX_MINIMUM_INVALID) /* min-elements constraint not honored */
68+
CVL_SYNTAX_MAXIMUM_INVALID = CVLRetCode(yparser.YP_SYNTAX_MAXIMUM_INVALID) /* max-elements constraint not honored */
69+
CVL_SEMANTIC_DEPENDENT_DATA_MISSING = CVLRetCode(yparser.YP_SEMANTIC_DEPENDENT_DATA_MISSING) /* Dependent Data is missing */
70+
CVL_SEMANTIC_MANDATORY_DATA_MISSING = CVLRetCode(yparser.YP_SEMANTIC_MANDATORY_DATA_MISSING) /* Mandatory Data is missing */
71+
CVL_SEMANTIC_KEY_ALREADY_EXIST = CVLRetCode(yparser.YP_SEMANTIC_KEY_ALREADY_EXIST) /* Key already existing. */
72+
CVL_SEMANTIC_KEY_NOT_EXIST = CVLRetCode(yparser.YP_SEMANTIC_KEY_NOT_EXIST) /* Key is missing. */
73+
CVL_SEMANTIC_KEY_DUPLICATE = CVLRetCode(yparser.YP_SEMANTIC_KEY_DUPLICATE) /* Duplicate key. */
74+
CVL_SEMANTIC_KEY_INVALID = CVLRetCode(yparser.YP_SEMANTIC_KEY_INVALID)
75+
)
76+
77+
//CVLEditConfigData Strcture for key and data in API
78+
type CVLEditConfigData struct {
79+
VType CVLValidateType //Validation type
80+
VOp CVLOperation //Operation type
81+
Key string //Key format : "PORT|Ethernet4"
82+
Data map[string]string //Value : {"alias": "40GE0/28", "mtu" : 9100, "admin_status": down}
83+
}
84+
85+
//CVLErrorInfo CVL Error Structure
86+
type CVLErrorInfo struct {
87+
TableName string /* Table having error */
88+
ErrCode CVLRetCode /* CVL Error return Code. */
89+
CVLErrDetails string /* CVL Error Message details. */
90+
Keys []string /* Keys of the Table having error. */
91+
Value string /* Field Value throwing error */
92+
Field string /* Field Name throwing error . */
93+
Msg string /* Detailed error message. */
94+
ConstraintErrMsg string /* Constraint error message. */
95+
ErrAppTag string
96+
}
97+
98+
type CustValidationCache struct {
99+
Data interface{}
100+
}
101+
102+
//CustValidationCtxt Custom validation context passed to custom validation function
103+
type CustValidationCtxt struct {
104+
ReqData []CVLEditConfigData //All request data
105+
CurCfg *CVLEditConfigData //Current request data for which validation should be done
106+
YNodeName string //YANG node name
107+
YNodeVal string //YANG node value, leaf-list will have "," separated value
108+
YCur *xmlquery.Node //YANG data tree
109+
SessCache *CustValidationCache //Session cache, can be used for storing data, persistent in session
110+
RClient *redis.Client //Redis client
111+
}
112+
113+
//InvokeCustomValidation Common function to invoke custom validation
114+
//TBD should we do this using GO plugin feature ?
115+
func InvokeCustomValidation(cv *CustomValidation, name string, args... interface{}) CVLErrorInfo {
116+
inputs := make([]reflect.Value, len(args))
117+
for i := range args {
118+
inputs[i] = reflect.ValueOf(args[i])
119+
}
120+
121+
f := reflect.ValueOf(cv).MethodByName(name)
122+
if !f.IsNil() {
123+
v := f.Call(inputs)
124+
util.TRACE_LEVEL_LOG(util.TRACE_SEMANTIC,
125+
"InvokeCustomValidation: %s(), CVLErrorInfo: %v", name, v[0])
126+
127+
return (v[0].Interface()).(CVLErrorInfo)
128+
}
129+
130+
return CVLErrorInfo{ErrCode: CVL_SUCCESS}
131+
}
132+

cvl/cvl.go

+62-1
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ import (
3434
"sync"
3535
"io/ioutil"
3636
"path/filepath"
37+
custv "github.com/Azure/sonic-mgmt-common/cvl/custom_validation"
38+
"unsafe"
3739
)
3840

3941
//DB number
@@ -106,6 +108,7 @@ type modelTableInfo struct {
106108
whenExpr map[string][]*whenInfo
107109
tablesForMustExp map[string]CVLOperation
108110
refFromTables []tblFieldPair //list of table or table/field referring to this table
111+
custValidation map[string]string // Map for custom validation node and function name
109112
dfltLeafVal map[string]string //map of leaf names and default value
110113
mandatoryNodes map[string]bool //map of leaf names and mandatory flag
111114
}
@@ -140,6 +143,7 @@ type CVL struct {
140143
maxTableElem map[string]int //max element count per table
141144
batchLeaf []*yparser.YParserLeafValue //field name and value
142145
yv *YValidator //Custom YANG validator for validating external dependencies
146+
custvCache custv.CustValidationCache //Custom validation cache per session
143147
}
144148

145149
// Struct for model namepsace and prefix
@@ -449,6 +453,7 @@ func storeModelInfo(modelFile string, module *yparser.YParserModule) {
449453
tInfo.redisTableSize = lInfo.RedisTableSize
450454
tInfo.keys = lInfo.Keys
451455
tInfo.mapLeaf = lInfo.MapLeaf
456+
tInfo.custValidation = lInfo.CustValidation
452457
tInfo.mandatoryNodes = lInfo.MandatoryNodes
453458

454459
//store default values used in must and when exp
@@ -955,6 +960,62 @@ func (c *CVL) addCfgDataItem(configData *map[string]interface{},
955960
return tblName, key
956961
}
957962

963+
//Perform user defined custom validation
964+
func (c *CVL) doCustomValidation(node *xmlquery.Node,
965+
custvCfg []custv.CVLEditConfigData,
966+
curCustvCfg *custv.CVLEditConfigData, yangListName,
967+
tbl, key string) CVLErrorInfo {
968+
969+
cvlErrObj := CVLErrorInfo{ErrCode : CVL_SUCCESS}
970+
971+
// yangListName provides the correct table name defined in sonic-yang
972+
// For ex. VLAN_INTERFACE_LIST and VLAN_INTERFACE_IPADDR_LIST are in same container
973+
for nodeName, custFunc := range modelInfo.tableInfo[yangListName].custValidation {
974+
//find the node value
975+
//node value is empty for custom validation function at list level
976+
nodeVal := ""
977+
if !strings.HasSuffix(nodeName, "_LIST") {
978+
for nodeLeaf := node.FirstChild; nodeLeaf != nil;
979+
nodeLeaf = nodeLeaf.NextSibling {
980+
if (nodeName != nodeLeaf.Data) {
981+
continue
982+
}
983+
984+
if (len(nodeLeaf.Attr) > 0) &&
985+
(nodeLeaf.Attr[0].Name.Local == "leaf-list") {
986+
nodeVal = curCustvCfg.Data[nodeName]
987+
} else {
988+
nodeVal = nodeLeaf.FirstChild.Data
989+
}
990+
}
991+
992+
}
993+
994+
//Call custom validation functions
995+
CVL_LOG(INFO_TRACE, "Calling custom validation function %s", custFunc)
996+
pCustv := &custv.CustValidationCtxt{
997+
ReqData: custvCfg,
998+
CurCfg: curCustvCfg,
999+
YNodeName: nodeName,
1000+
YNodeVal: nodeVal,
1001+
YCur: node,
1002+
SessCache: &(c.custvCache),
1003+
RClient: redisClient}
1004+
1005+
errObj := custv.InvokeCustomValidation(&custv.CustomValidation{},
1006+
custFunc, pCustv)
1007+
1008+
cvlErrObj = *(*CVLErrorInfo)(unsafe.Pointer(&errObj))
1009+
1010+
if (cvlErrObj.ErrCode != CVL_SUCCESS) {
1011+
CVL_LOG(WARNING, "Custom validation failed, Error = %v", cvlErrObj)
1012+
return cvlErrObj
1013+
}
1014+
}
1015+
1016+
return cvlErrObj
1017+
}
1018+
9581019
// getLeafRefInfo This function returns leafrefInfo structure based on table name,
9591020
// target table name and leaf node name where leafRef is present
9601021
func getLeafRefInfo(tblName, fldName, targetTblName string) *leafRefInfo {
@@ -978,4 +1039,4 @@ func isMandatoryTrueNode(tblName, field string) bool {
9781039
}
9791040

9801041
return false
981-
}
1042+
}

cvl/cvl_api.go

+14-2
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ package cvl
2121

2222
import (
2323
"fmt"
24+
"reflect"
2425
"encoding/json"
2526
"github.com/go-redis/redis/v7"
2627
toposort "github.com/philopon/go-toposort"
@@ -29,7 +30,9 @@ import (
2930
. "github.com/Azure/sonic-mgmt-common/cvl/internal/util"
3031
"strings"
3132
"github.com/antchfx/xmlquery"
33+
"unsafe"
3234
"runtime"
35+
custv "github.com/Azure/sonic-mgmt-common/cvl/custom_validation"
3336
"time"
3437
"sync"
3538
)
@@ -321,14 +324,16 @@ func (c *CVL) ValidateEditConfig(cfgData []CVLEditConfigData) (cvlErr CVLErrorIn
321324
caller = f.Name()
322325
}
323326

324-
CVL_LOG(INFO_DEBUG, "ValidateEditConfig() called from %s() : %v", caller, cfgData)
327+
CVL_LOG(INFO_DEBUG, "ValidateEditConfig() called from %s() : %v", caller, cfgData)
325328

326329
if SkipValidation() {
327330
CVL_LOG(INFO_TRACE, "Skipping CVL validation.")
328331
return cvlErrObj, CVL_SUCCESS
329332
}
330333

331334
//Type cast to custom validation cfg data
335+
sliceHeader := *(*reflect.SliceHeader)(unsafe.Pointer(&cfgData))
336+
custvCfg := *(*[]custv.CVLEditConfigData)(unsafe.Pointer(&sliceHeader))
332337

333338
c.clearTmpDbCache()
334339
//c.yv.root.FirstChild = nil
@@ -544,6 +549,13 @@ func (c *CVL) ValidateEditConfig(cfgData []CVLEditConfigData) (cvlErr CVLErrorIn
544549
continue
545550
}
546551

552+
//Step 3.2 : Run all custom validations
553+
cvlErrObj= c.doCustomValidation(node, custvCfg, &custvCfg[i], yangListName,
554+
tbl, key)
555+
if cvlErrObj.ErrCode != CVL_SUCCESS {
556+
return cvlErrObj,cvlErrObj.ErrCode
557+
}
558+
547559
//Step 3.3 : Perform semantic validation
548560
if cvlErrObj = c.validateSemantics(node, yangListName, key, &cfgData[i]);
549561
cvlErrObj.ErrCode != CVL_SUCCESS {
@@ -1087,4 +1099,4 @@ func (c *CVL) GetAllReferringTables(tableName string) (map[string][]string) {
10871099
}
10881100

10891101
return refTbls
1090-
}
1102+
}

0 commit comments

Comments
 (0)