Merge pull request #133 from mattfenwick/parse-netpol-list

add ability to parse network policy list from file

Author: mattfenwick

networkpolicies/yaml-syntax/yaml-list.yaml

@@ -0,0 +1,47 @@
+apiVersion: v1
+kind: List
+items:
+- apiVersion: networking.k8s.io/v1
+  kind: NetworkPolicy
+  metadata:
+    name: pol1
+    namespace: ns-y
+  spec:
+    egress:
+    - {}
+    ingress:
+    - from:
+      - podSelector: {}
+    podSelector: {}
+    policyTypes:
+    - Ingress
+    - Egress
+- apiVersion: networking.k8s.io/v1
+  kind: NetworkPolicy
+  metadata:
+    name: pol2
+    namespace: ns-y
+  spec:
+    ingress:
+    - from:
+      - namespaceSelector: {}
+      ports:
+      - port: 8080
+        protocol: TCP
+    podSelector: {}
+    policyTypes:
+    - Ingress
+- apiVersion: networking.k8s.io/v1
+  kind: NetworkPolicy
+  metadata:
+    name: pol3
+    namespace: ns-y
+  spec:
+    ingress:
+    - from:
+      - namespaceSelector: {}
+    podSelector:
+      matchLabels:
+        app: qrs
+    policyTypes:
+    - Ingress

pkg/cli/analyze.go

@@ -2,11 +2,12 @@ package cli
 
 import (
 	"fmt"
+	"strings"
+
 	"github.com/mattfenwick/collections/pkg/set"
 	"github.com/mattfenwick/cyclonus/pkg/connectivity/probe"
 	"github.com/mattfenwick/cyclonus/pkg/generator"
 	"github.com/mattfenwick/cyclonus/pkg/linter"
-	"strings"
 
 	"github.com/mattfenwick/cyclonus/pkg/kube"
 	"github.com/mattfenwick/cyclonus/pkg/kube/netpol"
@@ -102,12 +103,18 @@ func RunAnalyzeCommand(args *AnalyzeArgs) {
 			kubeNamespaces = nsList.Items
 			namespaces = []string{v1.NamespaceAll}
 		}
-		kubePolicies, err = readPoliciesFromKube(kubeClient, namespaces)
+		kubePolicies, err = kube.ReadNetworkPoliciesFromKube(kubeClient, namespaces)
+		if err != nil {
+			logrus.Errorf("unable to read network policies from kube, ns '%s': %+v", namespaces, err)
+		}
 		kubePods, err = kube.GetPodsInNamespaces(kubeClient, namespaces)
+		if err != nil {
+			logrus.Errorf("unable to read pods from kube, ns '%s': %+v", namespaces, err)
+		}
 	}
 	// 2. read policies from file
 	if args.PolicyPath != "" {
-		policiesFromPath, err := readPoliciesFromPath(args.PolicyPath)
+		policiesFromPath, err := kube.ReadNetworkPoliciesFromPath(args.PolicyPath)
 		utils.DoOrDie(err)
 		kubePolicies = append(kubePolicies, policiesFromPath...)
 	}

pkg/cli/utils.go pkg/kube/read.go

@@ -1,16 +1,18 @@
-package cli
+package kube
 
 import (
-	"github.com/mattfenwick/cyclonus/pkg/kube"
+	"os"
+	"path/filepath"
+
+	"github.com/mattfenwick/collections/pkg/builtin"
+	"github.com/mattfenwick/collections/pkg/slice"
 	"github.com/mattfenwick/cyclonus/pkg/utils"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	networkingv1 "k8s.io/api/networking/v1"
-	"os"
-	"path/filepath"
 )
 
-func readPoliciesFromPath(policyPath string) ([]*networkingv1.NetworkPolicy, error) {
+func ReadNetworkPoliciesFromPath(policyPath string) ([]*networkingv1.NetworkPolicy, error) {
 	var allPolicies []*networkingv1.NetworkPolicy
 	err := filepath.Walk(policyPath, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
@@ -26,15 +28,27 @@ func readPoliciesFromPath(policyPath string) ([]*networkingv1.NetworkPolicy, err
 			return err
 		}
 
-		// try parsing a list first
-		policies, err := utils.ParseYaml[[]*networkingv1.NetworkPolicy](bytes)
+		// TODO try parsing plain yaml list (that is: not a NetworkPolicyList)
+		// policies, err := utils.ParseYaml[[]*networkingv1.NetworkPolicy](bytes)
+
+		// TODO try parsing multiple policies separated by '---' lines
+		// policies, err := yaml.ParseMany[networkingv1.NetworkPolicy](bytes)
+		// if err == nil {
+		// 	log.Debugf("parsed %d policies from %s", len(policies), path)
+		// 	allPolicies = append(allPolicies, refNetpolList(policies)...)
+		// 	return nil
+		// }
+		// log.Errorf("unable to parse multiple policies separated by '---' lines: %+v", err)
+
+		// try parsing a NetworkPolicyList
+		policyList, err := utils.ParseYamlStrict[networkingv1.NetworkPolicyList](bytes)
 		if err == nil {
-			log.Debugf("parsed %d policies from %s", len(*policies), path)
-			allPolicies = append(allPolicies, *policies...)
+			allPolicies = append(allPolicies, refNetpolList(policyList.Items)...)
 			return nil
 		}
 
-		log.Debugf("failed to parse list from %s, falling back to parsing single policy", path)
+		log.Debugf("unable to parse list of policies: %+v", err)
+
 		policy, err := utils.ParseYamlStrict[networkingv1.NetworkPolicy](bytes)
 		if err != nil {
 			return errors.WithMessagef(err, "unable to parse single policy from yaml at %s", path)
@@ -56,18 +70,14 @@ func readPoliciesFromPath(policyPath string) ([]*networkingv1.NetworkPolicy, err
 	return allPolicies, nil
 }
 
-func readPoliciesFromKube(kubeClient *kube.Kubernetes, namespaces []string) ([]*networkingv1.NetworkPolicy, error) {
-	netpols, err := kube.GetNetworkPoliciesInNamespaces(kubeClient, namespaces)
+func ReadNetworkPoliciesFromKube(kubeClient *Kubernetes, namespaces []string) ([]*networkingv1.NetworkPolicy, error) {
+	netpols, err := GetNetworkPoliciesInNamespaces(kubeClient, namespaces)
 	if err != nil {
 		return nil, err
 	}
 	return refNetpolList(netpols), nil
 }
 
 func refNetpolList(refs []networkingv1.NetworkPolicy) []*networkingv1.NetworkPolicy {
-	policies := make([]*networkingv1.NetworkPolicy, len(refs))
-	for i := 0; i < len(refs); i++ {
-		policies[i] = &refs[i]
-	}
-	return policies
+	return slice.Map(builtin.Reference[networkingv1.NetworkPolicy], refs)
 }

pkg/kube/read_tests.go

@@ -0,0 +1,42 @@
+package kube
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func RunReadNetworkPolicyTests() {
+	Describe("ReadNetworkPolicies", func() {
+		It("Should read a single policy from a single file", func() {
+			policies, err := ReadNetworkPoliciesFromPath("../../networkpolicies/features/portrange1.yaml")
+			Expect(err).To(BeNil())
+			Expect(len(policies)).To(Equal(1))
+		})
+		It("Should read a list of policies from a single file", func() {
+			policies, err := ReadNetworkPoliciesFromPath("../../networkpolicies/yaml-syntax/yaml-list.yaml")
+			Expect(err).To(BeNil())
+			Expect(len(policies)).To(Equal(3))
+		})
+
+		// TODO test case to read multiple policies from plain yaml list
+
+		// TODO
+		// It("Should read multiple policies separated by '---' lines from a single file", func() {
+		// 	policies, err := ReadNetworkPoliciesFromPath("../../networkpolicies/yaml-syntax/triple-dash-separated.yaml")
+		// 	Expect(err).To(BeNil())
+		// 	Expect(len(policies)).To(Equal(3))
+		// })
+
+		It("Should read multiple policies from all files in a directory", func() {
+			policies, err := ReadNetworkPoliciesFromPath("../../networkpolicies/simple-example")
+			Expect(err).To(BeNil())
+			Expect(len(policies)).To(Equal(7))
+
+			policies, err = ReadNetworkPoliciesFromPath("../../networkpolicies/")
+			Expect(err).To(BeNil())
+			Expect(len(policies)).To(Equal(14))
+		})
+
+		// TODO test to show what happens for duplicate names
+	})
+}

pkg/kube/suite_test.go

@@ -11,5 +11,6 @@ func TestModel(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunIPAddressTests()
 	RunLabelSelectorTests()
+	RunReadNetworkPolicyTests()
 	RunSpecs(t, "network policy matcher suite")
 }

pkg/matcher/simplifier_tests.go

@@ -123,7 +123,7 @@ func RunSimplifierTests() {
 			}
 			port99OnUdp := &PortProtocolMatcher{
 				Port:     &port99,
-				Protocol: v1.ProtocolSCTP, // TODO should this be udp?
+				Protocol: v1.ProtocolUDP,
 			}
 
 			allMatcher := &AllPortMatcher{}