kube-ovn部署为underlay模式； ubuntu@a-master-1:~/yaml$ alias k = kubectl ubuntu@a-master-1:~/yaml$ k get svc NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE kubernetes ClusterIP 10.96.0.1 443/TCP 10d test-svc-cluster ClusterIP 10.109.26.124 80/TCP 9d ============================================= ubuntu@a-master-1:~/yaml$ k get pod NAME READY STATUS RESTARTS AGE client 1/1 Running 0 52m frontend 1/1 Running 0 53m ============================================= apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: client-workloads namespace: default spec: podSelector: matchLabels: tag: client egress: # Allows outgoing connections to the vcluster control plane - ports: - port: 443 protocol: TCP - port: 8443 protocol: TCP - port: 80 ## 这里有80端口，在client pod中，应该可以访问 svc （10.109.26.124）的80端口，但实际不通。但如果端口号改为 8000 ，就可以通了。 （8000 是pod本身开的端口） protocol: TCP ## 这个现象和k8s规范是不符合的，访问svc的端口，networkpolicy中就应该设置的是 svc的端口；是否和kube-ovn underlay模式有关系？ policyTypes: - Egress ============================================= ubuntu@a-master-1:~/yaml$ k ko trace client 10.109.26.124 tcp 80 + kubectl exec ovn-central-65bbcd766-s5vlc -n kube-system -c ovn-central -- ovn-trace ovn-default 'inport == "client.default" && ip.ttl == 64 && eth.src == 7e:70:24:0d:00:cc && ip4.src == 172.16.1.29 && eth.dst == fa:16:3e:87:da:6b && ip4.dst == 10.109.26.124 && tcp.src == 10000 && tcp.dst == 80 && ct.new' # ct_state=new|trk,tcp,reg14=0x28,vlan_tci=0x0000,dl_src=7e:70:24:0d:00:cc,dl_dst=fa:16:3e:87:da:6b,nw_src=172.16.1.29,nw_dst=10.109.26.124,nw_tos=0,nw_ecn=0,nw_ttl=64,nw_frag=no,tp_src=10000,tp_dst=80,tcp_flags=0 ingress(dp="ovn-default", inport="client.default") -------------------------------------------------- 0. ls_in_check_port_sec (northd.c:8960): 1, priority 50, uuid 310366b2 reg0[15] = check_in_port_sec(); next; 4. ls_in_pre_acl (northd.c:6026): ip, priority 100, uuid fa0507ee reg0[0] = 1; next; 5. ls_in_pre_lb (northd.c:6230): ip, priority 100, uuid 39395862 reg0[2] = 1; next; 6. ls_in_pre_stateful (northd.c:7365): reg0[2] == 1 && ip4.dst == 10.109.26.124 && tcp.dst == 80, priority 120, uuid ce213b01 reg1 = 10.109.26.124; reg2[0..15] = 80; ct_lb_mark; ct_lb_mark /* default (use --ct to customize) */ ------------------------------------------------ 7. ls_in_acl_hint (northd.c:6514): ct.est && ct_mark.blocked == 0, priority 1, uuid 3f893ce2 reg0[10] = 1; next; 9. ls_in_acl_action (northd.c:6879): reg8[30..31] == 0, priority 500, uuid a43a7c6c reg8[30..31] = 1; next(8); 9. ls_in_acl_action (northd.c:6879): reg8[30..31] == 1, priority 500, uuid ee718b06 reg8[30..31] = 2; next(8); 9. ls_in_acl_action (northd.c:6868): 1, priority 0, uuid 35cd54e9 reg8[16] = 0; reg8[17] = 0; reg8[18] = 0; reg8[30..31] = 0; next; 13. ls_in_lb (northd.c:7863): ct.new && ip4.dst == 10.109.26.124 && tcp.dst == 80, priority 120, uuid 92a3278d reg0[1] = 0; reg1 = 10.109.26.124; reg2[0..15] = 80; ct_lb_mark(backends=172.16.1.28:8000); ct_lb_mark /* default (use --ct to customize) */ ------------------------------------------------ 15. ls_in_pre_hairpin (northd.c:8032): ip && ct.trk, priority 100, uuid eda0563a reg0[6] = chk_lb_hairpin(); reg0[12] = chk_lb_hairpin_reply(); next; 19. ls_in_acl_after_lb_action (northd.c:6879): reg8[30..31] == 0, priority 500, uuid 9e3b4cb3 reg8[30..31] = 1; next(18); 19. ls_in_acl_after_lb_action (northd.c:6879): reg8[30..31] == 1, priority 500, uuid 87cedb5e reg8[30..31] = 2; next(18); 18. ls_in_acl_after_lb_eval (northd.c:6727): reg8[30..31] == 2 && reg0[10] == 1 && (inport == @client.workloads.default && ip), priority 3000, uuid a923c237 reg8[17] = 1; ct_commit { ct_mark.blocked = 1; }; next; 19. ls_in_acl_after_lb_action (northd.c:6845): reg8[17] == 1, priority 1000, uuid 6cf828c6 reg8[16] = 0; reg8[17] = 0; reg8[18] = 0; reg8[30..31] = 0; + set +x -------- Start OVS Tracing + kubectl exec ovs-ovn-rkc8w -c openvswitch -n kube-system -- ovs-appctl ofproto/trace br-int in_port=492,tcp,nw_ttl=64,nw_src=172.16.1.29,nw_dst=10.109.26.124,dl_src=7e:70:24:0d:00:cc,dl_dst=fa:16:3e:87:da:6b,tcp_src=1000,tcp_dst=80 Flow: tcp,in_port=492,vlan_tci=0x0000,dl_src=7e:70:24:0d:00:cc,dl_dst=fa:16:3e:87:da:6b,nw_src=172.16.1.29,nw_dst=10.109.26.124,nw_tos=0,nw_ecn=0,nw_ttl=64,nw_frag=no,tp_src=1000,tp_dst=80,tcp_flags=0 bridge("br-int") ---------------- 0. in_port=492, priority 100, cookie 0x11a60f39 set_field:0x1f/0xffff->reg13 set_field:0x5->reg11 set_field:0x2->reg12 set_field:0x3->metadata set_field:0x28->reg14 set_field:0/0xffff0000->reg13 resubmit(,8) 8. metadata=0x3, priority 50, cookie 0x310366b2 set_field:0/0x1000->reg10 resubmit(,73) 73. No match. drop move:NXM_NX_REG10[12]->NXM_NX_XXREG0[111] -> NXM_NX_XXREG0[111] is now 0 resubmit(,9) 9. metadata=0x3, priority 0, cookie 0x4dc3653c resubmit(,10) 10. metadata=0x3, priority 0, cookie 0x95f9e4f1 resubmit(,11) 11. metadata=0x3, priority 0, cookie 0x94e772f3 resubmit(,12) 12. ip,metadata=0x3, priority 100, cookie 0xfa0507ee set_field:0x1000000000000000000000000/0x1000000000000000000000000->xxreg0 resubmit(,13) 13. ip,metadata=0x3, priority 100, cookie 0x39395862 set_field:0x4000000000000000000000000/0x4000000000000000000000000->xxreg0 resubmit(,14) 14. tcp,reg0=0x4/0x4,metadata=0x3,nw_dst=10.109.26.124,tp_dst=80, priority 120, cookie 0xce213b01 set_field:0xa6d1a7c0000000000000000/0xffffffff0000000000000000->xxreg0 set_field:0x5000000000/0xffff00000000->xxreg0 ct(table=15,zone=NXM_NX_REG13[0..15],nat) nat -> A clone of the packet is forked to recirculate. The forked pipeline will be resumed at table 15. -> Sets the packet to an untracked state, and clears all the conntrack fields. Final flow: tcp,reg0=0x5,reg1=0xa6d1a7c,reg2=0x50,reg11=0x5,reg12=0x2,reg13=0x1f,reg14=0x28,metadata=0x3,in_port=492,vlan_tci=0x0000,dl_src=7e:70:24:0d:00:cc,dl_dst=fa:16:3e:87:da:6b,nw_src=172.16.1.29,nw_dst=10.109.26.124,nw_tos=0,nw_ecn=0,nw_ttl=64,nw_frag=no,tp_src=1000,tp_dst=80,tcp_flags=0 Megaflow: recirc_id=0,eth,tcp,in_port=492,dl_src=7e:70:24:0d:00:cc,dl_dst=fa:16:3e:87:da:6b,nw_dst=10.109.26.124,nw_frag=no,tp_dst=80 Datapath actions: ct(zone=31,nat),recirc(0x3b0) =============================================================================== recirc(0x3b0) - resume conntrack with default ct_state=trk|new (use --ct-next to customize) Replacing src/dst IP/ports to simulate NAT: Initial flow: Modified flow: =============================================================================== Flow: recirc_id=0x3b0,ct_state=new|trk,ct_zone=31,eth,tcp,reg0=0x5,reg1=0xa6d1a7c,reg2=0x50,reg11=0x5,reg12=0x2,reg13=0x1f,reg14=0x28,metadata=0x3,in_port=492,vlan_tci=0x0000,dl_src=7e:70:24:0d:00:cc,dl_dst=fa:16:3e:87:da:6b,nw_src=172.16.1.29,nw_dst=10.109.26.124,nw_tos=0,nw_ecn=0,nw_ttl=64,nw_frag=no,tp_src=1000,tp_dst=80,tcp_flags=0 bridge("br-int") ---------------- thaw Resuming from table 15 15. ct_state=+new-est+trk,metadata=0x3, priority 7, cookie 0x17d768f2 set_field:0x80000000000000000000000000/0x80000000000000000000000000->xxreg0 set_field:0x200000000000000000000000000/0x200000000000000000000000000->xxreg0 resubmit(,16) 16. ct_state=-est+trk,ip,metadata=0x3, priority 1, cookie 0xe53bbcc1 set_field:0x2000000000000000000000000/0x2000000000000000000000000->xxreg0 resubmit(,17) 17. reg8=0/0xc0000000,metadata=0x3, priority 500, cookie 0xa43a7c6c set_field:0x4000000000000000/0xc000000000000000->xreg4 resubmit(,16) 16. ct_state=-est+trk,ip,metadata=0x3, priority 1, cookie 0xe53bbcc1 set_field:0x2000000000000000000000000/0x2000000000000000000000000->xxreg0 resubmit(,17) 17. reg8=0x40000000/0xc0000000,metadata=0x3, priority 500, cookie 0xee718b06 set_field:0x8000000000000000/0xc000000000000000->xreg4 resubmit(,16) 16. ct_state=-est+trk,ip,metadata=0x3, priority 1, cookie 0xe53bbcc1 set_field:0x2000000000000000000000000/0x2000000000000000000000000->xxreg0 resubmit(,17) 17. metadata=0x3, priority 0, cookie 0x35cd54e9 set_field:0/0x1000000000000->xreg4 set_field:0/0x2000000000000->xreg4 set_field:0/0x4000000000000->xreg4 set_field:0/0xc000000000000000->xreg4 resubmit(,18) 18. metadata=0x3, priority 0, cookie 0x84cef7a8 resubmit(,19) 19. metadata=0x3, priority 0, cookie 0x364ad647 resubmit(,20) 20. metadata=0x3, priority 0, cookie 0xb68cf6ac resubmit(,21) 21. ct_state=+new+trk,tcp,metadata=0x3,nw_dst=10.109.26.124,tp_dst=80, priority 120, cookie 0x92a3278d set_field:0/0x2000000000000000000000000->xxreg0 set_field:0xa6d1a7c0000000000000000/0xffffffff0000000000000000->xxreg0 set_field:0x5000000000/0xffff00000000->xxreg0 group:10 -> no live bucket Final flow: recirc_id=0x3b0,ct_state=new|trk,ct_zone=31,eth,tcp,reg0=0x285,reg1=0xa6d1a7c,reg2=0x50,reg11=0x5,reg12=0x2,reg13=0x1f,reg14=0x28,metadata=0x3,in_port=492,vlan_tci=0x0000,dl_src=7e:70:24:0d:00:cc,dl_dst=fa:16:3e:87:da:6b,nw_src=172.16.1.29,nw_dst=10.109.26.124,nw_tos=0,nw_ecn=0,nw_ttl=64,nw_frag=no,tp_src=1000,tp_dst=80,tcp_flags=0 Megaflow: recirc_id=0x3b0,ct_state=+new-est-rel-rpl+trk,ct_mark=0/0x1,eth,tcp,in_port=492,dl_dst=fa:16:3e:87:da:6b,nw_dst=10.109.26.124,nw_frag=no,tp_dst=80 Datapath actions: hash(l4(0)),recirc(0x3b1)