test: add testing for generic builder multi subject (#162)

* test: add testing for generic builder multi subject

Signed-off-by: Asra Ali <[email protected]>

* update comments

Signed-off-by: Asra Ali <[email protected]>

Author: asraa

main_test.go

@@ -8,6 +8,7 @@ import (
 	"testing"
 
 	"github.com/slsa-framework/slsa-verifier/pkg"
+	"golang.org/x/mod/semver"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
@@ -28,6 +29,8 @@ var generatorVersions = map[string][]string{
 	"v1.1.1": {"go"},
 	"v1.2.0": {"generic"}}
 
+const TEST_DIR = "./testdata"
+
 func Test_runVerify(t *testing.T) {
 	t.Parallel()
 	tests := []struct {
@@ -43,6 +46,10 @@ func Test_runVerify(t *testing.T) {
 		// or testdata from malicious untrusted builders.
 		// When true, this does not iterate over all builder versions.
 		noversion bool
+		// minversion is a special case to test a newly added feature into a builder
+		minversion string
+		// specifying builders will restrict builders to only the specified ones.
+		builders []string
 	}{
 		{
 			name:     "valid main branch default",
@@ -300,6 +307,21 @@ func Test_runVerify(t *testing.T) {
 			pversiontag: pString("v15.1"),
 			err:         pkg.ErrorMismatchVersionedTag,
 		},
+		// Multiple subjects in version v1.2.0+
+		{
+			name:       "multiple subject first match",
+			artifact:   "binary-linux-amd64-multi-subject-first",
+			source:     "github.com/slsa-framework/example-package",
+			minversion: "v1.2.0",
+			builders:   []string{"generic"},
+		},
+		{
+			name:       "multiple subject second match",
+			artifact:   "binary-linux-amd64-multi-subject-second",
+			source:     "github.com/slsa-framework/example-package",
+			minversion: "v1.2.0",
+			builders:   []string{"generic"},
+		},
 		// Special case of the e2e test repository building builder from head.
 		{
 			name:      "e2e test repository verified with builder at head",
@@ -343,35 +365,39 @@ func Test_runVerify(t *testing.T) {
 		tt := tt // Re-initializing variable so it is not changed while executing the closure below
 		t.Run(tt.name, func(t *testing.T) {
 			// t.Parallel()
-			getBuildersAndVersions := func() []string {
+			getBuildersAndVersions := func(minversion string, ttBuilders []string) []string {
 				res := []string{}
-				builders := []string{}
-				testdataDir, err := ioutil.ReadDir("./testdata")
-				if err != nil {
-					t.Fatal(err)
-				}
-				for _, f := range testdataDir {
-					if f.IsDir() {
-						// These are the builder subfolders
-						builders = append(builders, f.Name())
+				builders := tt.builders
+				if len(builders) == 0 {
+					testdataDir, err := ioutil.ReadDir(TEST_DIR)
+					if err != nil {
+						t.Error(err)
+					}
+					for _, f := range testdataDir {
+						if f.IsDir() {
+							// These are the builder subfolders
+							builders = append(builders, f.Name())
+						}
 					}
 				}
 				for _, builder := range builders {
-					builderDir, err := ioutil.ReadDir(fmt.Sprintf("./testdata/%s", builder))
+					builderDir, err := ioutil.ReadDir(filepath.Join(TEST_DIR, builder))
 					if err != nil {
-						t.Fatal(err)
+						t.Error(err)
 					}
 					for _, f := range builderDir {
-						if f.IsDir() {
+						// Builder subfolders are semantic version strings.
+						// Compare if a min version is given.
+						if f.IsDir() && semver.Compare(minversion, f.Name()) <= 0 {
 							// These are the supported versions of the builder
-							res = append(res, fmt.Sprintf("%s/%s", builder, f.Name()))
+							res = append(res, filepath.Join(builder, f.Name()))
 						}
 					}
 				}
 				return res
 			}
 
-			checkVersions := getBuildersAndVersions()
+			checkVersions := getBuildersAndVersions(tt.minversion, tt.builders)
 			if tt.noversion {
 				checkVersions = []string{""}
 			}
@@ -382,7 +408,7 @@ func Test_runVerify(t *testing.T) {
 					branch = "main"
 				}
 
-				artifactPath = filepath.Clean(fmt.Sprintf("./testdata/%v/%s", v, tt.artifact))
+				artifactPath = filepath.Clean(filepath.Join(TEST_DIR, v, tt.artifact))
 				provenancePath = fmt.Sprintf("%s.intoto.jsonl", artifactPath)
 
 				_, err := runVerify(artifactPath,

testdata/generic/v1.2.0/binary-linux-amd64-multi-subject-first

@@ -0,0 +1 @@
+artifact1

testdata/generic/v1.2.0/binary-linux-amd64-multi-subject-first.intoto.jsonl

@@ -0,0 +1 @@
+{"payloadType":"application/vnd.in-toto+json","payload":"eyJfdHlwZSI6Imh0dHBzOi8vaW4tdG90by5pby9TdGF0ZW1lbnQvdjAuMSIsInByZWRpY2F0ZVR5cGUiOiJodHRwczovL3Nsc2EuZGV2L3Byb3ZlbmFuY2UvdjAuMiIsInN1YmplY3QiOlt7Im5hbWUiOiJhcnRpZmFjdDEiLCJkaWdlc3QiOnsic2hhMjU2IjoiNDgyY2U4YzhmN2U4NjdkYTNhM2MwNWE5YWVlNjM3NzAzZTE3NDcwZWQxY2Y4ODJhOWU1YjQwNWU4ZjgyNjE5ZCJ9fSx7Im5hbWUiOiJhcnRpZmFjdDIiLCJkaWdlc3QiOnsic2hhMjU2IjoiODljZmM2OTU0ZTg4YjJmOTJhN2MyODc5ZDllYjA4NWM0MmYzYzcwNjVkMDEyYTUwNjZmNDUwZGJlNTliMmMwMCJ9fSx7Im5hbWUiOiJhcnRpZmFjdDMiLCJkaWdlc3QiOnsic2hhMjU2IjoiN2E1ZDIxYTZhZGFjOTQ1NTYxZDg1OWJkMWRlY2ZjMzdiMjQwODc4OGNmMzIwNmRmMzUxOWUyODFhZmQzMWI2ZSJ9fV0sInByZWRpY2F0ZSI6eyJidWlsZGVyIjp7ImlkIjoiaHR0cHM6Ly9naXRodWIuY29tL3Nsc2EtZnJhbWV3b3JrL3Nsc2EtZ2l0aHViLWdlbmVyYXRvci8uZ2l0aHViL3dvcmtmbG93cy9nZW5lcmF0b3JfZ2VuZXJpY19zbHNhMy55bWxAcmVmcy9oZWFkcy9tYWluIn0sImJ1aWxkVHlwZSI6Imh0dHBzOi8vZ2l0aHViLmNvbS9zbHNhLWZyYW1ld29yay9zbHNhLWdpdGh1Yi1nZW5lcmF0b3JAdjEiLCJpbnZvY2F0aW9uIjp7ImNvbmZpZ1NvdXJjZSI6eyJ1cmkiOiJnaXQraHR0cHM6Ly9naXRodWIuY29tL3Nsc2EtZnJhbWV3b3JrL2V4YW1wbGUtcGFja2FnZUByZWZzL2hlYWRzL21haW4iLCJkaWdlc3QiOnsic2hhMSI6IjJjYjEwZjQ4NWRjMzZkZDk5NDc2ZjNlM2I3ZWVjOTlkZmQ3OTZkYTIifSwiZW50cnlQb2ludCI6Ii5naXRodWIvd29ya2Zsb3dzL2UyZS5nZW5lcmljLnNjaGVkdWxlLm1haW4ubXVsdGktc3ViamVjdHMuc2xzYTMueW1sIn0sInBhcmFtZXRlcnMiOnt9LCJlbnZpcm9ubWVudCI6eyJnaXRodWJfYWN0b3IiOiJsYXVyZW50c2ltb24iLCJnaXRodWJfYWN0b3JfaWQiOiI2NDUwNTA5OSIsImdpdGh1Yl9iYXNlX3JlZiI6IiIsImdpdGh1Yl9ldmVudF9uYW1lIjoic2NoZWR1bGUiLCJnaXRodWJfZXZlbnRfcGF5bG9hZCI6eyJzY2hlZHVsZSI6IjAgNCAqICogKiJ9LCJnaXRodWJfaGVhZF9yZWYiOiIiLCJnaXRodWJfcmVmIjoicmVmcy9oZWFkcy9tYWluIiwiZ2l0aHViX3JlZl90eXBlIjoiYnJhbmNoIiwiZ2l0aHViX3JlcG9zaXRvcnlfaWQiOiI0ODYzMjU4MDkiLCJnaXRodWJfcmVwb3NpdG9yeV9vd25lciI6InNsc2EtZnJhbWV3b3JrIiwiZ2l0aHViX3JlcG9zaXRvcnlfb3duZXJfaWQiOiI4MDQzMTE4NyIsImdpdGh1Yl9ydW5fYXR0ZW1wdCI6IjEiLCJnaXRodWJfcnVuX2lkIjoiMjcxNjM3NDA3NiIsImdpdGh1Yl9ydW5fbnVtYmVyIjoiMTMiLCJnaXRodWJfc2hhMSI6IjJjYjEwZjQ4NWRjMzZkZDk5NDc2ZjNlM2I3ZWVjOTlkZmQ3OTZkYTIifX0sIm1ldGFkYXRhIjp7ImJ1aWxkSW52b2NhdGlvbklEIjoiMjcxNjM3NDA3Ni0xIiwiY29tcGxldGVuZXNzIjp7InBhcmFtZXRlcnMiOnRydWUsImVudmlyb25tZW50IjpmYWxzZSwibWF0ZXJpYWxzIjpmYWxzZX0sInJlcHJvZHVjaWJsZSI6ZmFsc2V9LCJtYXRlcmlhbHMiOlt7InVyaSI6ImdpdCtodHRwczovL2dpdGh1Yi5jb20vc2xzYS1mcmFtZXdvcmsvZXhhbXBsZS1wYWNrYWdlQHJlZnMvaGVhZHMvbWFpbiIsImRpZ2VzdCI6eyJzaGExIjoiMmNiMTBmNDg1ZGMzNmRkOTk0NzZmM2UzYjdlZWM5OWRmZDc5NmRhMiJ9fV19fQ==","signatures":[{"keyid":"","sig":"MEUCICNze+GG7blYjmJmXMXFGJaFQaGNHNz7e2XvBjwn11rrAiEArCto8K9MLReCB9wkb5pdreVBPbm7RHNoKv9BND8AJxQ=","cert":"-----BEGIN CERTIFICATE-----\nMIIDdjCCAvygAwIBAgITKyNvAShskkDKQt7Nibo+TTfnpDAKBggqhkjOPQQDAzAq\nMRUwEwYDVQQKEwxzaWdzdG9yZS5kZXYxETAPBgNVBAMTCHNpZ3N0b3JlMB4XDTIy\nMDcyMjA0NDkyM1oXDTIyMDcyMjA0NTkyMlowADBZMBMGByqGSM49AgEGCCqGSM49\nAwEHA0IABPTJ64TMtITPS0wyq/aov6KAaOQo/NwWnFbbpAjJuADM0TZyshD+r+/X\nojJndYY+k0dN46ZwgrsosvGl1m8NcuSjggIpMIICJTAOBgNVHQ8BAf8EBAMCB4Aw\nEwYDVR0lBAwwCgYIKwYBBQUHAwMwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUW7RY\nPTTIaNKtq3xILv9tfHvYdD0wHwYDVR0jBBgwFoAUWMAeX5FFpWapesyQoZMi0CrF\nxfowgYMGA1UdEQEB/wR5MHeGdWh0dHBzOi8vZ2l0aHViLmNvbS9zbHNhLWZyYW1l\nd29yay9zbHNhLWdpdGh1Yi1nZW5lcmF0b3IvLmdpdGh1Yi93b3JrZmxvd3MvZ2Vu\nZXJhdG9yX2dlbmVyaWNfc2xzYTMueW1sQHJlZnMvaGVhZHMvbWFpbjBSBgorBgEE\nAYO/MAEEBEQuZ2l0aHViL3dvcmtmbG93cy9lMmUuZ2VuZXJpYy5zY2hlZHVsZS5t\nYWluLm11bHRpLXN1YmplY3RzLnNsc2EzLnltbDA5BgorBgEEAYO/MAEBBCtodHRw\nczovL3Rva2VuLmFjdGlvbnMuZ2l0aHVidXNlcmNvbnRlbnQuY29tMB0GCisGAQQB\ng78wAQYED3JlZnMvaGVhZHMvbWFpbjA2BgorBgEEAYO/MAEDBCgyY2IxMGY0ODVk\nYzM2ZGQ5OTQ3NmYzZTNiN2VlYzk5ZGZkNzk2ZGEyMCwGCisGAQQBg78wAQUEHnNs\nc2EtZnJhbWV3b3JrL2V4YW1wbGUtcGFja2FnZTAWBgorBgEEAYO/MAECBAhzY2hl\nZHVsZTAKBggqhkjOPQQDAwNoADBlAjEA8cIq5/AEeKOEfZ0Eqduz0a5/aYXjyJJG\nH2ZvNwTtkvtM8LghCpj3aCgCGKZ98zZ4AjB/yhvDwdw7fysJ1k4aQDOoXGWvpJpp\n3XN26KG8pkCkOzjANAgytUMT4WMsDTEWsZI=\n-----END CERTIFICATE-----\n"}]}

testdata/generic/v1.2.0/binary-linux-amd64-multi-subject-second

@@ -0,0 +1 @@
+artifact2

testdata/generic/v1.2.0/binary-linux-amd64-multi-subject-second.intoto.jsonl

@@ -0,0 +1 @@
+{"payloadType":"application/vnd.in-toto+json","payload":"eyJfdHlwZSI6Imh0dHBzOi8vaW4tdG90by5pby9TdGF0ZW1lbnQvdjAuMSIsInByZWRpY2F0ZVR5cGUiOiJodHRwczovL3Nsc2EuZGV2L3Byb3ZlbmFuY2UvdjAuMiIsInN1YmplY3QiOlt7Im5hbWUiOiJhcnRpZmFjdDEiLCJkaWdlc3QiOnsic2hhMjU2IjoiNDgyY2U4YzhmN2U4NjdkYTNhM2MwNWE5YWVlNjM3NzAzZTE3NDcwZWQxY2Y4ODJhOWU1YjQwNWU4ZjgyNjE5ZCJ9fSx7Im5hbWUiOiJhcnRpZmFjdDIiLCJkaWdlc3QiOnsic2hhMjU2IjoiODljZmM2OTU0ZTg4YjJmOTJhN2MyODc5ZDllYjA4NWM0MmYzYzcwNjVkMDEyYTUwNjZmNDUwZGJlNTliMmMwMCJ9fSx7Im5hbWUiOiJhcnRpZmFjdDMiLCJkaWdlc3QiOnsic2hhMjU2IjoiN2E1ZDIxYTZhZGFjOTQ1NTYxZDg1OWJkMWRlY2ZjMzdiMjQwODc4OGNmMzIwNmRmMzUxOWUyODFhZmQzMWI2ZSJ9fV0sInByZWRpY2F0ZSI6eyJidWlsZGVyIjp7ImlkIjoiaHR0cHM6Ly9naXRodWIuY29tL3Nsc2EtZnJhbWV3b3JrL3Nsc2EtZ2l0aHViLWdlbmVyYXRvci8uZ2l0aHViL3dvcmtmbG93cy9nZW5lcmF0b3JfZ2VuZXJpY19zbHNhMy55bWxAcmVmcy9oZWFkcy9tYWluIn0sImJ1aWxkVHlwZSI6Imh0dHBzOi8vZ2l0aHViLmNvbS9zbHNhLWZyYW1ld29yay9zbHNhLWdpdGh1Yi1nZW5lcmF0b3JAdjEiLCJpbnZvY2F0aW9uIjp7ImNvbmZpZ1NvdXJjZSI6eyJ1cmkiOiJnaXQraHR0cHM6Ly9naXRodWIuY29tL3Nsc2EtZnJhbWV3b3JrL2V4YW1wbGUtcGFja2FnZUByZWZzL2hlYWRzL21haW4iLCJkaWdlc3QiOnsic2hhMSI6IjJjYjEwZjQ4NWRjMzZkZDk5NDc2ZjNlM2I3ZWVjOTlkZmQ3OTZkYTIifSwiZW50cnlQb2ludCI6Ii5naXRodWIvd29ya2Zsb3dzL2UyZS5nZW5lcmljLnNjaGVkdWxlLm1haW4ubXVsdGktc3ViamVjdHMuc2xzYTMueW1sIn0sInBhcmFtZXRlcnMiOnt9LCJlbnZpcm9ubWVudCI6eyJnaXRodWJfYWN0b3IiOiJsYXVyZW50c2ltb24iLCJnaXRodWJfYWN0b3JfaWQiOiI2NDUwNTA5OSIsImdpdGh1Yl9iYXNlX3JlZiI6IiIsImdpdGh1Yl9ldmVudF9uYW1lIjoic2NoZWR1bGUiLCJnaXRodWJfZXZlbnRfcGF5bG9hZCI6eyJzY2hlZHVsZSI6IjAgNCAqICogKiJ9LCJnaXRodWJfaGVhZF9yZWYiOiIiLCJnaXRodWJfcmVmIjoicmVmcy9oZWFkcy9tYWluIiwiZ2l0aHViX3JlZl90eXBlIjoiYnJhbmNoIiwiZ2l0aHViX3JlcG9zaXRvcnlfaWQiOiI0ODYzMjU4MDkiLCJnaXRodWJfcmVwb3NpdG9yeV9vd25lciI6InNsc2EtZnJhbWV3b3JrIiwiZ2l0aHViX3JlcG9zaXRvcnlfb3duZXJfaWQiOiI4MDQzMTE4NyIsImdpdGh1Yl9ydW5fYXR0ZW1wdCI6IjEiLCJnaXRodWJfcnVuX2lkIjoiMjcxNjM3NDA3NiIsImdpdGh1Yl9ydW5fbnVtYmVyIjoiMTMiLCJnaXRodWJfc2hhMSI6IjJjYjEwZjQ4NWRjMzZkZDk5NDc2ZjNlM2I3ZWVjOTlkZmQ3OTZkYTIifX0sIm1ldGFkYXRhIjp7ImJ1aWxkSW52b2NhdGlvbklEIjoiMjcxNjM3NDA3Ni0xIiwiY29tcGxldGVuZXNzIjp7InBhcmFtZXRlcnMiOnRydWUsImVudmlyb25tZW50IjpmYWxzZSwibWF0ZXJpYWxzIjpmYWxzZX0sInJlcHJvZHVjaWJsZSI6ZmFsc2V9LCJtYXRlcmlhbHMiOlt7InVyaSI6ImdpdCtodHRwczovL2dpdGh1Yi5jb20vc2xzYS1mcmFtZXdvcmsvZXhhbXBsZS1wYWNrYWdlQHJlZnMvaGVhZHMvbWFpbiIsImRpZ2VzdCI6eyJzaGExIjoiMmNiMTBmNDg1ZGMzNmRkOTk0NzZmM2UzYjdlZWM5OWRmZDc5NmRhMiJ9fV19fQ==","signatures":[{"keyid":"","sig":"MEUCICNze+GG7blYjmJmXMXFGJaFQaGNHNz7e2XvBjwn11rrAiEArCto8K9MLReCB9wkb5pdreVBPbm7RHNoKv9BND8AJxQ=","cert":"-----BEGIN CERTIFICATE-----\nMIIDdjCCAvygAwIBAgITKyNvAShskkDKQt7Nibo+TTfnpDAKBggqhkjOPQQDAzAq\nMRUwEwYDVQQKEwxzaWdzdG9yZS5kZXYxETAPBgNVBAMTCHNpZ3N0b3JlMB4XDTIy\nMDcyMjA0NDkyM1oXDTIyMDcyMjA0NTkyMlowADBZMBMGByqGSM49AgEGCCqGSM49\nAwEHA0IABPTJ64TMtITPS0wyq/aov6KAaOQo/NwWnFbbpAjJuADM0TZyshD+r+/X\nojJndYY+k0dN46ZwgrsosvGl1m8NcuSjggIpMIICJTAOBgNVHQ8BAf8EBAMCB4Aw\nEwYDVR0lBAwwCgYIKwYBBQUHAwMwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUW7RY\nPTTIaNKtq3xILv9tfHvYdD0wHwYDVR0jBBgwFoAUWMAeX5FFpWapesyQoZMi0CrF\nxfowgYMGA1UdEQEB/wR5MHeGdWh0dHBzOi8vZ2l0aHViLmNvbS9zbHNhLWZyYW1l\nd29yay9zbHNhLWdpdGh1Yi1nZW5lcmF0b3IvLmdpdGh1Yi93b3JrZmxvd3MvZ2Vu\nZXJhdG9yX2dlbmVyaWNfc2xzYTMueW1sQHJlZnMvaGVhZHMvbWFpbjBSBgorBgEE\nAYO/MAEEBEQuZ2l0aHViL3dvcmtmbG93cy9lMmUuZ2VuZXJpYy5zY2hlZHVsZS5t\nYWluLm11bHRpLXN1YmplY3RzLnNsc2EzLnltbDA5BgorBgEEAYO/MAEBBCtodHRw\nczovL3Rva2VuLmFjdGlvbnMuZ2l0aHVidXNlcmNvbnRlbnQuY29tMB0GCisGAQQB\ng78wAQYED3JlZnMvaGVhZHMvbWFpbjA2BgorBgEEAYO/MAEDBCgyY2IxMGY0ODVk\nYzM2ZGQ5OTQ3NmYzZTNiN2VlYzk5ZGZkNzk2ZGEyMCwGCisGAQQBg78wAQUEHnNs\nc2EtZnJhbWV3b3JrL2V4YW1wbGUtcGFja2FnZTAWBgorBgEEAYO/MAECBAhzY2hl\nZHVsZTAKBggqhkjOPQQDAwNoADBlAjEA8cIq5/AEeKOEfZ0Eqduz0a5/aYXjyJJG\nH2ZvNwTtkvtM8LghCpj3aCgCGKZ98zZ4AjB/yhvDwdw7fysJ1k4aQDOoXGWvpJpp\n3XN26KG8pkCkOzjANAgytUMT4WMsDTEWsZI=\n-----END CERTIFICATE-----\n"}]}