Skip to content

Commit 3b23583

Browse files
hackerainhackerain
committed
delete legacy acls when upgrading to v1.13.x (#4742)
the acls in v1.13.x are in tier 2 rather than tier 0 in v1.12.x, the legacy acls may cause some unexpected behaviors because acls in tier 0 have the higest priority. we should delete legacy acls and recreate them when upgrading to v1.13.x. Signed-off-by: Rain Suo <[email protected]>
1 parent 7b63325 commit 3b23583

14 files changed

+173
-46
lines changed

‎mocks/pkg/ovs/interface.go

+4-4
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/controller/admin_network_policy.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,7 @@ func (c *Controller) handleAddAnp(key string) (err error) {
190190
return err
191191
}
192192

193-
ingressACLOps, err := c.OVNNbClient.DeleteAclsOps(pgName, portGroupKey, "to-lport", nil)
193+
ingressACLOps, err := c.OVNNbClient.DeleteAclsOps(pgName, portGroupKey, "to-lport", nil, -1)
194194
if err != nil {
195195
klog.Errorf("failed to generate clear operations for anp %s ingress acls: %v", key, err)
196196
return err
@@ -266,7 +266,7 @@ func (c *Controller) handleAddAnp(key string) (err error) {
266266
return fmt.Errorf("failed to delete unused ingress address set for anp %s: %w", key, err)
267267
}
268268

269-
egressACLOps, err := c.OVNNbClient.DeleteAclsOps(pgName, portGroupKey, "from-lport", nil)
269+
egressACLOps, err := c.OVNNbClient.DeleteAclsOps(pgName, portGroupKey, "from-lport", nil, -1)
270270
if err != nil {
271271
klog.Errorf("failed to generate clear operations for anp %s egress acls: %v", key, err)
272272
return err

‎pkg/controller/baseline_admin_network_policy.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ func (c *Controller) handleAddBanp(key string) (err error) {
148148
return err
149149
}
150150

151-
ingressACLOps, err := c.OVNNbClient.DeleteAclsOps(pgName, portGroupKey, "to-lport", nil)
151+
ingressACLOps, err := c.OVNNbClient.DeleteAclsOps(pgName, portGroupKey, "to-lport", nil, -1)
152152
if err != nil {
153153
klog.Errorf("failed to generate clear operations for banp %s ingress acls: %v", key, err)
154154
return err
@@ -225,7 +225,7 @@ func (c *Controller) handleAddBanp(key string) (err error) {
225225
return fmt.Errorf("failed to delete unused ingress address set for banp %s: %w", key, err)
226226
}
227227

228-
egressACLOps, err := c.OVNNbClient.DeleteAclsOps(pgName, portGroupKey, "from-lport", nil)
228+
egressACLOps, err := c.OVNNbClient.DeleteAclsOps(pgName, portGroupKey, "from-lport", nil, -1)
229229
if err != nil {
230230
klog.Errorf("failed to generate clear operations for banp %s egress acls: %v", key, err)
231231
return err

‎pkg/controller/controller.go

+19
Original file line numberDiff line numberDiff line change
@@ -945,6 +945,10 @@ func (c *Controller) Run(ctx context.Context) {
945945
}
946946
}
947947

948+
// for upgrading from v1.12.x to v1.13.x
949+
// see https://github.com/kubeovn/kube-ovn/issues/4742
950+
c.handleUpgrading()
951+
948952
// start workers to do all the network operations
949953
c.startWorkers(ctx)
950954

@@ -1078,6 +1082,21 @@ func (c *Controller) shutdown() {
10781082
}
10791083
}
10801084

1085+
func (c *Controller) handleUpgrading() {
1086+
if err := c.upgradeSecurityGroups(); err != nil {
1087+
util.LogFatalAndExit(err, "failed to upgrade security groups")
1088+
}
1089+
if err := c.upgradeSubnets(); err != nil {
1090+
util.LogFatalAndExit(err, "failed to upgrade subnets")
1091+
}
1092+
if err := c.upgradeNetworkPolicies(); err != nil {
1093+
util.LogFatalAndExit(err, "failed to upgrade network policies")
1094+
}
1095+
if err := c.upgradeNodes(); err != nil {
1096+
util.LogFatalAndExit(err, "failed to upgrade nodes")
1097+
}
1098+
}
1099+
10811100
func (c *Controller) startWorkers(ctx context.Context) {
10821101
klog.Info("Starting workers")
10831102

‎pkg/controller/network_policy.go

+31-4
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,33 @@ func (c *Controller) enqueueUpdateNp(oldObj, newObj interface{}) {
6262
}
6363
}
6464

65+
func (c *Controller) upgradeNetworkPolicies() error {
66+
// for upgrading from v1.12.x to v1.13.x
67+
68+
// clear legacy acls in tier 0 for all network policies
69+
// including ingress, egress and subnet gateway acls
70+
nps, err := c.npsLister.NetworkPolicies(corev1.NamespaceAll).List(labels.Everything())
71+
if err != nil {
72+
klog.Errorf("failed to list network policies %v", err)
73+
return err
74+
}
75+
76+
for _, np := range nps {
77+
npName := np.Name
78+
nameArray := []rune(np.Name)
79+
if !unicode.IsLetter(nameArray[0]) {
80+
npName = "np" + np.Name
81+
}
82+
pgName := strings.ReplaceAll(fmt.Sprintf("%s.%s", npName, np.Namespace), "-", ".")
83+
84+
if err = c.OVNNbClient.DeleteAcls(pgName, portGroupKey, "", nil, util.DefaultACLTier); err != nil {
85+
klog.Errorf("clear legacy network policy %s acls: %v", pgName, err)
86+
return err
87+
}
88+
}
89+
return nil
90+
}
91+
6592
func (c *Controller) createAsForNetpol(ns, name, direction, asName string, addresses []string) error {
6693
if err := c.OVNNbClient.CreateAddressSet(asName, map[string]string{
6794
networkPolicyKey: fmt.Sprintf("%s/%s/%s", ns, name, direction),
@@ -165,7 +192,7 @@ func (c *Controller) handleUpdateNp(key string) error {
165192
return err
166193
}
167194

168-
ingressACLOps, err := c.OVNNbClient.DeleteAclsOps(pgName, portGroupKey, "to-lport", nil)
195+
ingressACLOps, err := c.OVNNbClient.DeleteAclsOps(pgName, portGroupKey, "to-lport", nil, -1)
169196
if err != nil {
170197
klog.Errorf("generate operations that clear np %s ingress acls: %v", key, err)
171198
return err
@@ -281,7 +308,7 @@ func (c *Controller) handleUpdateNp(key string) error {
281308
}
282309
}
283310
} else {
284-
if err = c.OVNNbClient.DeleteAcls(pgName, portGroupKey, "to-lport", nil); err != nil {
311+
if err = c.OVNNbClient.DeleteAcls(pgName, portGroupKey, "to-lport", nil, -1); err != nil {
285312
klog.Errorf("delete np %s ingress acls: %v", key, err)
286313
return err
287314
}
@@ -294,7 +321,7 @@ func (c *Controller) handleUpdateNp(key string) error {
294321
}
295322
}
296323

297-
egressACLOps, err := c.OVNNbClient.DeleteAclsOps(pgName, portGroupKey, "from-lport", nil)
324+
egressACLOps, err := c.OVNNbClient.DeleteAclsOps(pgName, portGroupKey, "from-lport", nil, -1)
298325
if err != nil {
299326
klog.Errorf("generate operations that clear np %s egress acls: %v", key, err)
300327
return err
@@ -408,7 +435,7 @@ func (c *Controller) handleUpdateNp(key string) error {
408435
}
409436
}
410437
} else {
411-
if err = c.OVNNbClient.DeleteAcls(pgName, portGroupKey, "from-lport", nil); err != nil {
438+
if err = c.OVNNbClient.DeleteAcls(pgName, portGroupKey, "from-lport", nil, -1); err != nil {
412439
klog.Errorf("delete np %s egress acls: %v", key, err)
413440
return err
414441
}

‎pkg/controller/node.go

+27-1
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,32 @@ func nodeUnderlayAddressSetName(node string, af int) string {
9494
return fmt.Sprintf("node_%s_underlay_v%d", strings.ReplaceAll(node, "-", "_"), af)
9595
}
9696

97+
func (c *Controller) upgradeNodes() error {
98+
// for upgrading from v1.12.x to v1.13.x
99+
100+
// clear legacy acls in tier 0 for node port group
101+
nodes, err := c.nodesLister.List(labels.Everything())
102+
if err != nil {
103+
klog.Errorf("failed to list nodes: %v", err)
104+
return err
105+
}
106+
107+
for _, node := range nodes {
108+
pgName := strings.ReplaceAll(node.Annotations[util.PortNameAnnotation], "-", ".")
109+
110+
if pgName == "" {
111+
continue
112+
}
113+
114+
if err = c.OVNNbClient.DeleteAcls(pgName, portGroupKey, "", nil, util.DefaultACLTier); err != nil {
115+
klog.Errorf("delete legacy node acl for node pg %s: %v", pgName, err)
116+
return err
117+
}
118+
}
119+
120+
return nil
121+
}
122+
97123
func (c *Controller) handleAddNode(key string) error {
98124
c.nodeKeyMutex.LockKey(key)
99125
defer func() { _ = c.nodeKeyMutex.UnlockKey(key) }()
@@ -786,7 +812,7 @@ func (c *Controller) checkAndUpdateNodePortGroup() error {
786812
}
787813
} else {
788814
// clear all acl
789-
if err = c.OVNNbClient.DeleteAcls(pgName, portGroupKey, "", nil); err != nil {
815+
if err = c.OVNNbClient.DeleteAcls(pgName, portGroupKey, "", nil, -1); err != nil {
790816
klog.Errorf("delete node acl for node pg %s: %v", pgName, err)
791817
}
792818
}

‎pkg/controller/security_group.go

+27
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,33 @@ func (c *Controller) enqueueDeleteSg(obj interface{}) {
6262
c.delSgQueue.Add(key)
6363
}
6464

65+
func (c *Controller) upgradeSecurityGroups() error {
66+
// for upgrading from v1.12.x to v1.13.x
67+
68+
// clear legacy acls in tier 0 for deny all port group
69+
pgName := ovs.GetSgPortGroupName(util.DenyAllSecurityGroup)
70+
if err := c.OVNNbClient.DeleteAcls(pgName, portGroupKey, "", nil, util.DefaultACLTier); err != nil {
71+
klog.Error(err)
72+
return fmt.Errorf("delete legacy acls from port group %s: %w", pgName, err)
73+
}
74+
75+
// clear legacy acls in tier 0 for all sg port groups
76+
sgs, err := c.sgsLister.List(labels.Everything())
77+
if err != nil {
78+
klog.Errorf("failed to list security groups: %v", err)
79+
return err
80+
}
81+
for _, sg := range sgs {
82+
pgName := ovs.GetSgPortGroupName(sg.Name)
83+
if err := c.OVNNbClient.DeleteAcls(pgName, portGroupKey, "", nil, util.DefaultACLTier); err != nil {
84+
klog.Error(err)
85+
return fmt.Errorf("delete legacy acls from port group %s: %w", pgName, err)
86+
}
87+
}
88+
89+
return nil
90+
}
91+
6592
func (c *Controller) initDefaultDenyAllSecurityGroup() error {
6693
pgName := ovs.GetSgPortGroupName(util.DenyAllSecurityGroup)
6794
if err := c.OVNNbClient.CreatePortGroup(pgName, map[string]string{

‎pkg/controller/subnet.go

+21-2
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,25 @@ func (c *Controller) enqueueUpdateSubnet(oldObj, newObj interface{}) {
152152
}
153153
}
154154

155+
func (c *Controller) upgradeSubnets() error {
156+
// for upgrading from v1.12.x to v1.13.x
157+
158+
// clear legacy acls in tier 0 for all subnets
159+
subnets, err := c.subnetsLister.List(labels.Everything())
160+
if err != nil {
161+
klog.Errorf("failed to list subnets %v", err)
162+
return err
163+
}
164+
165+
for _, subnet := range subnets {
166+
if err = c.OVNNbClient.DeleteAcls(subnet.Name, logicalSwitchKey, "", nil, util.DefaultACLTier); err != nil {
167+
klog.Errorf("clear legacy logical switch %s acls: %v", subnet.Name, err)
168+
return err
169+
}
170+
}
171+
return nil
172+
}
173+
155174
func (c *Controller) formatSubnet(subnet *kubeovnv1.Subnet) (*kubeovnv1.Subnet, error) {
156175
var (
157176
changed bool
@@ -790,7 +809,7 @@ func (c *Controller) handleAddOrUpdateSubnet(key string) error {
790809
}
791810
} else {
792811
// clear acl when direction is ""
793-
if err = c.OVNNbClient.DeleteAcls(subnet.Name, logicalSwitchKey, "", nil); err != nil {
812+
if err = c.OVNNbClient.DeleteAcls(subnet.Name, logicalSwitchKey, "", nil, -1); err != nil {
794813
if err = c.patchSubnetStatus(subnet, "ResetLogicalSwitchAclFailed", err.Error()); err != nil {
795814
klog.Error(err)
796815
return err
@@ -890,7 +909,7 @@ func (c *Controller) handleDeleteLogicalSwitch(key string) (err error) {
890909
}
891910

892911
// clear acl when direction is ""
893-
if err = c.OVNNbClient.DeleteAcls(key, logicalSwitchKey, "", nil); err != nil {
912+
if err = c.OVNNbClient.DeleteAcls(key, logicalSwitchKey, "", nil, -1); err != nil {
894913
klog.Errorf("clear logical switch %s acls: %v", key, err)
895914
return err
896915
}

‎pkg/ovs/interface.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -160,8 +160,8 @@ type ACL interface {
160160
SetACLLog(pgName string, logEnable, isIngress bool) error
161161
SetLogicalSwitchPrivate(lsName, cidrBlock, nodeSwitchCIDR string, allowSubnets []string) error
162162
SGLostACL(sg *kubeovnv1.SecurityGroup) (bool, error)
163-
DeleteAcls(parentName, parentType, direction string, externalIDs map[string]string) error
164-
DeleteAclsOps(parentName, parentType, direction string, externalIDs map[string]string) ([]ovsdb.Operation, error)
163+
DeleteAcls(parentName, parentType, direction string, externalIDs map[string]string, tier int) error
164+
DeleteAclsOps(parentName, parentType, direction string, externalIDs map[string]string, tier int) ([]ovsdb.Operation, error)
165165
UpdateAnpRuleACLOps(pgName, asName, protocol, aclName string, priority int, aclAction ovnnb.ACLAction, logACLActions []ovnnb.ACLAction, rulePorts []v1alpha1.AdminNetworkPolicyPort, isIngress, isBanp bool) ([]ovsdb.Operation, error)
166166
}
167167

‎pkg/ovs/ovn-nb-acl.go

+18-10
Original file line numberDiff line numberDiff line change
@@ -384,7 +384,7 @@ func (c *OVNNbClient) UpdateSgACL(sg *kubeovnv1.SecurityGroup, direction string)
384384
pgName := GetSgPortGroupName(sg.Name)
385385

386386
// clear acl
387-
if err := c.DeleteAcls(pgName, portGroupKey, direction, nil); err != nil {
387+
if err := c.DeleteAcls(pgName, portGroupKey, direction, nil, -1); err != nil {
388388
klog.Error(err)
389389
return fmt.Errorf("delete direction '%s' acls from port group %s: %w", direction, pgName, err)
390390
}
@@ -441,7 +441,7 @@ func (c *OVNNbClient) UpdateSgACL(sg *kubeovnv1.SecurityGroup, direction string)
441441
}
442442

443443
func (c *OVNNbClient) UpdateLogicalSwitchACL(lsName, cidrBlock string, subnetAcls []kubeovnv1.ACL, allowEWTraffic bool) error {
444-
if err := c.DeleteAcls(lsName, logicalSwitchKey, "", map[string]string{"subnet": lsName}); err != nil {
444+
if err := c.DeleteAcls(lsName, logicalSwitchKey, "", map[string]string{"subnet": lsName}, -1); err != nil {
445445
klog.Error(err)
446446
return fmt.Errorf("delete subnet acls from %s: %w", lsName, err)
447447
}
@@ -530,7 +530,7 @@ func (c *OVNNbClient) UpdateACL(acl *ovnnb.ACL, fields ...interface{}) error {
530530
// SetLogicalSwitchPrivate will drop all ingress traffic except allow subnets, same subnet and node subnet
531531
func (c *OVNNbClient) SetLogicalSwitchPrivate(lsName, cidrBlock, nodeSwitchCIDR string, allowSubnets []string) error {
532532
// clear acls
533-
if err := c.DeleteAcls(lsName, logicalSwitchKey, "", nil); err != nil {
533+
if err := c.DeleteAcls(lsName, logicalSwitchKey, "", nil, -1); err != nil {
534534
klog.Error(err)
535535
return fmt.Errorf("clear logical switch %s acls: %w", lsName, err)
536536
}
@@ -731,8 +731,8 @@ func (c *OVNNbClient) CreateBareACL(parentName, direction, priority, match, acti
731731
// DeleteAcls delete several acl once,
732732
// delete to-lport and from-lport direction acl when direction is empty, otherwise one-way
733733
// parentType is 'ls' or 'pg'
734-
func (c *OVNNbClient) DeleteAcls(parentName, parentType, direction string, externalIDs map[string]string) error {
735-
ops, err := c.DeleteAclsOps(parentName, parentType, direction, externalIDs)
734+
func (c *OVNNbClient) DeleteAcls(parentName, parentType, direction string, externalIDs map[string]string, tier int) error {
735+
ops, err := c.DeleteAclsOps(parentName, parentType, direction, externalIDs, tier)
736736
if err != nil {
737737
klog.Error(err)
738738
return err
@@ -822,14 +822,16 @@ func (c *OVNNbClient) GetACL(parent, direction, priority, match string, ignoreNo
822822
// result should include all to-lport and from-lport acls when direction is empty,
823823
// result should include all acls when externalIDs is empty,
824824
// result should include all acls which externalIDs[key] is not empty when externalIDs[key] is ""
825+
// result should include all acls when tier is -1
826+
// result should include all acls in specific tier when tier is not -1
825827
// TODO: maybe add other filter conditions(priority or match)
826-
func (c *OVNNbClient) ListAcls(direction string, externalIDs map[string]string) ([]ovnnb.ACL, error) {
828+
func (c *OVNNbClient) ListAcls(direction string, externalIDs map[string]string, tier int) ([]ovnnb.ACL, error) {
827829
ctx, cancel := context.WithTimeout(context.Background(), c.Timeout)
828830
defer cancel()
829831

830832
aclList := make([]ovnnb.ACL, 0)
831833

832-
if err := c.WhereCache(aclFilter(direction, externalIDs)).List(ctx, &aclList); err != nil {
834+
if err := c.WhereCache(aclFilter(direction, externalIDs, tier)).List(ctx, &aclList); err != nil {
833835
klog.Error(err)
834836
return nil, fmt.Errorf("list acls: %w", err)
835837
}
@@ -1091,8 +1093,10 @@ func newNetworkPolicyACLMatch(pgName, asAllowName, asExceptName, protocol, direc
10911093
// result should include all to-lport and from-lport acls when direction is empty,
10921094
// result should include all acls when externalIDs is empty,
10931095
// result should include all acls which externalIDs[key] is not empty when externalIDs[key] is ""
1096+
// result should include all acls when tier is -1
1097+
// result should include all acls in specific tier when tier is not -1
10941098
// TODO: maybe add other filter conditions(priority or match)
1095-
func aclFilter(direction string, externalIDs map[string]string) func(acl *ovnnb.ACL) bool {
1099+
func aclFilter(direction string, externalIDs map[string]string, tier int) func(acl *ovnnb.ACL) bool {
10961100
return func(acl *ovnnb.ACL) bool {
10971101
if len(acl.ExternalIDs) < len(externalIDs) {
10981102
return false
@@ -1118,6 +1122,10 @@ func aclFilter(direction string, externalIDs map[string]string) func(acl *ovnnb.
11181122
return false
11191123
}
11201124

1125+
if tier != -1 && acl.Tier != tier {
1126+
return false
1127+
}
1128+
11211129
return true
11221130
}
11231131
}
@@ -1173,7 +1181,7 @@ func (c *OVNNbClient) CreateAclsOps(parentName, parentType string, acls ...*ovnn
11731181
// DeleteAcls return operation which delete several acl once,
11741182
// delete to-lport and from-lport direction acl when direction is empty, otherwise one-way
11751183
// parentType is 'ls' or 'pg'
1176-
func (c *OVNNbClient) DeleteAclsOps(parentName, parentType, direction string, externalIDs map[string]string) ([]ovsdb.Operation, error) {
1184+
func (c *OVNNbClient) DeleteAclsOps(parentName, parentType, direction string, externalIDs map[string]string, tier int) ([]ovsdb.Operation, error) {
11771185
if parentName == "" {
11781186
return nil, errors.New("the port group name or logical switch name is required")
11791187
}
@@ -1185,7 +1193,7 @@ func (c *OVNNbClient) DeleteAclsOps(parentName, parentType, direction string, ex
11851193
externalIDs[aclParentKey] = parentName
11861194

11871195
/* delete acls from port group or logical switch */
1188-
acls, err := c.ListAcls(direction, externalIDs)
1196+
acls, err := c.ListAcls(direction, externalIDs, tier)
11891197
if err != nil {
11901198
klog.Error(err)
11911199
return nil, fmt.Errorf("list type %s %s acls: %w", parentType, parentName, err)

0 commit comments

Comments
 (0)