Skip to content

Commit e2685dd

Browse files
rogpepperogpeppe
committed
encoding/jsonschema: run external tests in matrix
This change now runs the external tests in both the v2 and v3 evaluators and updates the respective test skip fields accordingly. There is a lot of duplication, naturally, but it seems better to keep the testing code simpler by having independently updated fields than to try to combine them when they're the same. It'll be straightforward to change the `teststats` helper program to summarise differences between v2 and v3 in a human-readable fashion as required. Signed-off-by: Roger Peppe <[email protected]> Change-Id: Id21625d2b3d3e442a64f91b1fac39b1fe418b2fd Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1200520 Unity-Result: CUE porcuepine <[email protected]> TryBot-Result: CUEcueckoo <[email protected]> Reviewed-by: Daniel Martí <[email protected]>
1 parent 9959a3e commit e2685dd

File tree

257 files changed

+9759
-4846
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

257 files changed

+9759
-4846
lines changed

‎encoding/jsonschema/external_test.go

Lines changed: 25 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,13 @@ import (
2626
"github.com/go-quicktest/qt"
2727

2828
"cuelang.org/go/cue"
29-
"cuelang.org/go/cue/cuecontext"
3029
"cuelang.org/go/cue/errors"
3130
"cuelang.org/go/cue/format"
3231
"cuelang.org/go/cue/token"
3332
"cuelang.org/go/encoding/json"
3433
"cuelang.org/go/encoding/jsonschema"
3534
"cuelang.org/go/encoding/jsonschema/internal/externaltest"
35+
"cuelang.org/go/internal/cuetdtest"
3636
"cuelang.org/go/internal/cuetest"
3737
)
3838

@@ -52,14 +52,14 @@ func TestExternal(t *testing.T) {
5252
// Group the tests under a single subtest so that we can use
5353
// t.Parallel and still guarantee that all tests have completed
5454
// by the end.
55-
t.Run("tests", func(t *testing.T) {
55+
cuetdtest.SmallMatrix.Run(t, "tests", func(t *testing.T, m *cuetdtest.M) {
5656
// Run tests in deterministic order so we get some consistency between runs.
5757
for _, filename := range sortedKeys(tests) {
5858
schemas := tests[filename]
5959
t.Run(testName(filename), func(t *testing.T) {
6060
for _, s := range schemas {
6161
t.Run(testName(s.Description), func(t *testing.T) {
62-
runExternalSchemaTests(t, filename, s)
62+
runExternalSchemaTests(t, m, filename, s)
6363
})
6464
}
6565
})
@@ -75,9 +75,9 @@ func TestExternal(t *testing.T) {
7575
qt.Assert(t, qt.IsNil(err))
7676
}
7777

78-
func runExternalSchemaTests(t *testing.T, filename string, s *externaltest.Schema) {
78+
func runExternalSchemaTests(t *testing.T, m *cuetdtest.M, filename string, s *externaltest.Schema) {
7979
t.Logf("file %v", path.Join("testdata/external", filename))
80-
ctx := cuecontext.New()
80+
ctx := m.CueContext()
8181
jsonAST, err := json.Extract("schema.json", s.Schema)
8282
qt.Assert(t, qt.IsNil(err))
8383
jsonValue := ctx.BuildExpr(jsonAST)
@@ -112,13 +112,13 @@ func runExternalSchemaTests(t *testing.T, filename string, s *externaltest.Schem
112112
t.Logf("txtar:\n%s", schemaFailureTxtar(s))
113113
for _, test := range s.Tests {
114114
t.Run("", func(t *testing.T) {
115-
testFailed(t, &test.Skip, test, "could not compile schema")
115+
testFailed(t, m, &test.Skip, test, "could not compile schema")
116116
})
117117
}
118-
testFailed(t, &s.Skip, s, fmt.Sprintf("extract error: %v", extractErr))
118+
testFailed(t, m, &s.Skip, s, fmt.Sprintf("extract error: %v", extractErr))
119119
return
120120
}
121-
testSucceeded(t, &s.Skip, s)
121+
testSucceeded(t, m, &s.Skip, s)
122122

123123
for _, test := range s.Tests {
124124
t.Run(testName(test.Description), func(t *testing.T) {
@@ -140,15 +140,15 @@ func runExternalSchemaTests(t *testing.T, filename string, s *externaltest.Schem
140140
err = instValue.Unify(schemaValue).Err()
141141
if test.Valid {
142142
if err != nil {
143-
testFailed(t, &test.Skip, test, errors.Details(err, nil))
143+
testFailed(t, m, &test.Skip, test, errors.Details(err, nil))
144144
} else {
145-
testSucceeded(t, &test.Skip, test)
145+
testSucceeded(t, m, &test.Skip, test)
146146
}
147147
} else {
148148
if err == nil {
149-
testFailed(t, &test.Skip, test, "unexpected success")
149+
testFailed(t, m, &test.Skip, test, "unexpected success")
150150
} else {
151-
testSucceeded(t, &test.Skip, test)
151+
testSucceeded(t, m, &test.Skip, test)
152152
}
153153
}
154154
})
@@ -203,28 +203,34 @@ func testName(s string) string {
203203
// testFailed marks the current test as failed with the
204204
// given error message, and updates the
205205
// skip field pointed to by skipField if necessary.
206-
func testFailed(t *testing.T, skipField *externaltest.Skip, p positioner, errStr string) {
206+
func testFailed(t *testing.T, m *cuetdtest.M, skipField *externaltest.Skip, p positioner, errStr string) {
207207
if cuetest.UpdateGoldenFiles {
208208
if *skipField == nil && !allowRegressions() {
209209
t.Fatalf("test regression; was succeeding, now failing: %v", errStr)
210210
}
211-
*skipField = externaltest.Skip{"v2": errStr}
211+
if *skipField == nil {
212+
*skipField = make(externaltest.Skip)
213+
}
214+
(*skipField)[m.Name()] = errStr
212215
return
213216
}
214-
if *skipField != nil {
215-
t.Skipf("skipping due to known error: %v", *skipField)
217+
if reason := (*skipField)[m.Name()]; reason != "" {
218+
t.Skipf("skipping due to known error: %v", reason)
216219
}
217220
t.Fatal(errStr)
218221
}
219222

220223
// testFails marks the current test as succeeded and updates the
221224
// skip field pointed to by skipField if necessary.
222-
func testSucceeded(t *testing.T, skipField *externaltest.Skip, p positioner) {
225+
func testSucceeded(t *testing.T, m *cuetdtest.M, skipField *externaltest.Skip, p positioner) {
223226
if cuetest.UpdateGoldenFiles {
224-
*skipField = nil
227+
delete(*skipField, m.Name())
228+
if len(*skipField) == 0 {
229+
*skipField = nil
230+
}
225231
return
226232
}
227-
if *skipField != nil {
233+
if reason := (*skipField)[m.Name()]; reason != "" {
228234
t.Fatalf("unexpectedly more correct behavior (test success) on skipped test")
229235
}
230236
}

‎encoding/jsonschema/testdata/external/tests/draft2019-09/additionalItems.json

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,8 @@
103103
"data": [],
104104
"valid": true,
105105
"skip": {
106-
"v2": "5 errors in empty disjunction:\nconflicting values [] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:45\n instance.json:1:1\nconflicting values bool and [] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n"
106+
"v2": "5 errors in empty disjunction:\nconflicting values [] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:45\n instance.json:1:1\nconflicting values bool and [] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n",
107+
"v3": "conflicting values [] and {...} (mismatched types list and struct):\n generated.cue:2:45\n instance.json:1:1\nconflicting values bool and [] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\nincompatible list lengths (0 and 3):\n instance.json:1:1\n"
107108
}
108109
},
109110
{
@@ -113,7 +114,8 @@
113114
],
114115
"valid": true,
115116
"skip": {
116-
"v2": "5 errors in empty disjunction:\nconflicting values [1] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:45\n instance.json:1:1\nconflicting values bool and [1] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n"
117+
"v2": "5 errors in empty disjunction:\nconflicting values [1] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:45\n instance.json:1:1\nconflicting values bool and [1] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n",
118+
"v3": "conflicting values [1] and {...} (mismatched types list and struct):\n generated.cue:2:45\n instance.json:1:1\nconflicting values bool and [1] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\nincompatible list lengths (1 and 3):\n instance.json:1:1\n"
117119
}
118120
},
119121
{
@@ -124,7 +126,8 @@
124126
],
125127
"valid": true,
126128
"skip": {
127-
"v2": "5 errors in empty disjunction:\nconflicting values [1,2] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:45\n instance.json:1:1\nconflicting values bool and [1,2] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1,2] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1,2] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1,2] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n"
129+
"v2": "5 errors in empty disjunction:\nconflicting values [1,2] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:45\n instance.json:1:1\nconflicting values bool and [1,2] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1,2] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1,2] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1,2] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n",
130+
"v3": "conflicting values [1,2] and {...} (mismatched types list and struct):\n generated.cue:2:45\n instance.json:1:1\nconflicting values bool and [1,2] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1,2] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1,2] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1,2] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\nincompatible list lengths (2 and 3):\n instance.json:1:1\n"
128131
}
129132
},
130133
{
@@ -195,7 +198,8 @@
195198
],
196199
"valid": true,
197200
"skip": {
198-
"v2": "5 errors in empty disjunction:\nconflicting values [1,\"foo\",false] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:41\n instance.json:1:1\nconflicting values bool and [1,\"foo\",false] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1,\"foo\",false] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1,\"foo\",false] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1,\"foo\",false] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n"
201+
"v2": "5 errors in empty disjunction:\nconflicting values [1,\"foo\",false] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:41\n instance.json:1:1\nconflicting values bool and [1,\"foo\",false] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1,\"foo\",false] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1,\"foo\",false] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1,\"foo\",false] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n",
202+
"v3": "conflicting values [1,\"foo\",false] and {...} (mismatched types list and struct):\n generated.cue:2:41\n instance.json:1:1\nconflicting values bool and [1,\"foo\",false] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1,\"foo\",false] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1,\"foo\",false] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1,\"foo\",false] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\nincompatible list lengths (1 and 3):\n instance.json:1:1\n"
199203
}
200204
}
201205
]
@@ -226,7 +230,8 @@
226230
],
227231
"valid": true,
228232
"skip": {
229-
"v2": "5 errors in empty disjunction:\nconflicting values [1,null] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:41\n instance.json:1:1\nconflicting values bool and [1,null] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1,null] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1,null] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1,null] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n"
233+
"v2": "5 errors in empty disjunction:\nconflicting values [1,null] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:41\n instance.json:1:1\nconflicting values bool and [1,null] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1,null] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1,null] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1,null] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n",
234+
"v3": "conflicting values [1,null] and {...} (mismatched types list and struct):\n generated.cue:2:41\n instance.json:1:1\nconflicting values bool and [1,null] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1,null] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1,null] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1,null] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\nincompatible list lengths (1 and 2):\n instance.json:1:1\n"
230235
}
231236
}
232237
]

‎encoding/jsonschema/testdata/external/tests/draft2019-09/additionalProperties.json

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,8 @@
2929
},
3030
"valid": false,
3131
"skip": {
32-
"v2": "unexpected success"
32+
"v2": "unexpected success",
33+
"v3": "unexpected success"
3334
}
3435
},
3536
{
@@ -85,7 +86,8 @@
8586
},
8687
"valid": false,
8788
"skip": {
88-
"v2": "unexpected success"
89+
"v2": "unexpected success",
90+
"v3": "unexpected success"
8991
}
9092
}
9193
]
@@ -232,7 +234,8 @@
232234
}
233235
},
234236
"skip": {
235-
"v2": "extract error: cannot compile resulting schema: reference \"strings\" in label expression refers to field against which it would be matched:\n generated.cue:5:3\n"
237+
"v2": "extract error: cannot compile resulting schema: reference \"strings\" in label expression refers to field against which it would be matched:\n generated.cue:5:3\n",
238+
"v3": "extract error: cannot compile resulting schema: reference \"strings\" in label expression refers to field against which it would be matched:\n generated.cue:5:3\n"
236239
},
237240
"tests": [
238241
{
@@ -242,7 +245,8 @@
242245
},
243246
"valid": true,
244247
"skip": {
245-
"v2": "could not compile schema"
248+
"v2": "could not compile schema",
249+
"v3": "could not compile schema"
246250
}
247251
},
248252
{
@@ -253,7 +257,8 @@
253257
},
254258
"valid": false,
255259
"skip": {
256-
"v2": "could not compile schema"
260+
"v2": "could not compile schema",
261+
"v3": "could not compile schema"
257262
}
258263
}
259264
]
@@ -276,7 +281,8 @@
276281
"additionalProperties": false
277282
},
278283
"skip": {
279-
"v2": "extract error: unsupported constraint \"dependentSchemas\""
284+
"v2": "extract error: unsupported constraint \"dependentSchemas\"",
285+
"v3": "extract error: unsupported constraint \"dependentSchemas\""
280286
},
281287
"tests": [
282288
{
@@ -286,7 +292,8 @@
286292
},
287293
"valid": false,
288294
"skip": {
289-
"v2": "could not compile schema"
295+
"v2": "could not compile schema",
296+
"v3": "could not compile schema"
290297
}
291298
},
292299
{
@@ -296,7 +303,8 @@
296303
},
297304
"valid": false,
298305
"skip": {
299-
"v2": "could not compile schema"
306+
"v2": "could not compile schema",
307+
"v3": "could not compile schema"
300308
}
301309
},
302310
{
@@ -307,7 +315,8 @@
307315
},
308316
"valid": false,
309317
"skip": {
310-
"v2": "could not compile schema"
318+
"v2": "could not compile schema",
319+
"v3": "could not compile schema"
311320
}
312321
}
313322
]

‎encoding/jsonschema/testdata/external/tests/draft2019-09/allOf.json

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,8 @@
4242
},
4343
"valid": false,
4444
"skip": {
45-
"v2": "unexpected success"
45+
"v2": "unexpected success",
46+
"v3": "unexpected success"
4647
}
4748
},
4849
{
@@ -52,7 +53,8 @@
5253
},
5354
"valid": false,
5455
"skip": {
55-
"v2": "unexpected success"
56+
"v2": "unexpected success",
57+
"v3": "unexpected success"
5658
}
5759
},
5860
{
@@ -118,7 +120,8 @@
118120
},
119121
"valid": false,
120122
"skip": {
121-
"v2": "unexpected success"
123+
"v2": "unexpected success",
124+
"v3": "unexpected success"
122125
}
123126
},
124127
{
@@ -129,7 +132,8 @@
129132
},
130133
"valid": false,
131134
"skip": {
132-
"v2": "unexpected success"
135+
"v2": "unexpected success",
136+
"v3": "unexpected success"
133137
}
134138
},
135139
{
@@ -140,7 +144,8 @@
140144
},
141145
"valid": false,
142146
"skip": {
143-
"v2": "unexpected success"
147+
"v2": "unexpected success",
148+
"v3": "unexpected success"
144149
}
145150
},
146151
{
@@ -150,7 +155,8 @@
150155
},
151156
"valid": false,
152157
"skip": {
153-
"v2": "unexpected success"
158+
"v2": "unexpected success",
159+
"v3": "unexpected success"
154160
}
155161
}
156162
]
@@ -213,7 +219,8 @@
213219
"data": "foo",
214220
"valid": false,
215221
"skip": {
216-
"v2": "unexpected success"
222+
"v2": "unexpected success",
223+
"v3": "unexpected success"
217224
}
218225
}
219226
]
@@ -233,7 +240,8 @@
233240
"data": "foo",
234241
"valid": false,
235242
"skip": {
236-
"v2": "unexpected success"
243+
"v2": "unexpected success",
244+
"v3": "unexpected success"
237245
}
238246
}
239247
]

0 commit comments

Comments
 (0)