diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 71d4fffa2..d7b8361fe 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -31,7 +31,7 @@ }, { "ImportPath": "github.com/contiv/ofnet", - "Rev": "745d8b6ec38fcff41e7123f9db524ab329a91444" + "Rev": "2ed83a9c24b58c6bbdb807a771947958ad352bbd" }, { "ImportPath": "github.com/coreos/etcd/etcdserver/etcdhttp/httptypes", diff --git a/Godeps/_workspace/src/github.com/contiv/ofnet/ofnet_test.go b/Godeps/_workspace/src/github.com/contiv/ofnet/ofnet_test.go index e29677763..0e88f22e6 100644 --- a/Godeps/_workspace/src/github.com/contiv/ofnet/ofnet_test.go +++ b/Godeps/_workspace/src/github.com/contiv/ofnet/ofnet_test.go @@ -196,97 +196,147 @@ func TestOfnetSetupVtep(t *testing.T) { // Test adding/deleting Vrouter routes func TestOfnetVrouteAddDelete(t *testing.T) { - for i := 0; i < NUM_AGENT; i++ { - j := i + 1 - macAddr, _ := net.ParseMAC(fmt.Sprintf("02:02:02:%02x:%02x:%02x", j, j, j)) - ipAddr := net.ParseIP(fmt.Sprintf("10.10.%d.%d", j, j)) - endpoint := EndpointInfo{ - PortNo : uint32(NUM_AGENT + 2), - MacAddr : macAddr, - Vlan : 1, - IpAddr : ipAddr, - } - - log.Infof("Installing local vrouter endpoint: %+v", endpoint) - - // Install the local endpoint - err := vrtrAgents[i].AddLocalEndpoint(endpoint) - if err != nil { - t.Errorf("Error installing endpoint: %+v. Err: %v", endpoint, err) - } - } - - log.Infof("Finished adding local vrouter endpoint") - - // verify all ovs switches have this route - for i := 0; i < NUM_AGENT; i++ { - brName := "ovsbr1" + fmt.Sprintf("%d", i) - - flowList, err := ofctlFlowDump(brName) - if err != nil { - t.Errorf("Error getting flow entries. Err: %v", err) - } + for iter := 0; iter < 4; iter++ { + for i := 0; i < NUM_AGENT; i++ { + j := i + 1 + macAddr, _ := net.ParseMAC(fmt.Sprintf("02:02:02:%02x:%02x:%02x", j, j, j)) + ipAddr := net.ParseIP(fmt.Sprintf("10.10.%d.%d", j, j)) + endpoint := EndpointInfo{ + PortNo : uint32(NUM_AGENT + 2), + MacAddr : macAddr, + Vlan : 1, + IpAddr : ipAddr, + } + + log.Infof("Installing local vrouter endpoint: %+v", endpoint) + + // Install the local endpoint + err := vrtrAgents[i].AddLocalEndpoint(endpoint) + if err != nil { + t.Fatalf("Error installing endpoint: %+v. Err: %v", endpoint, err) + return + } + } + + log.Infof("Finished adding local vrouter endpoint") + + // verify all ovs switches have this route + for i := 0; i < NUM_AGENT; i++ { + brName := "ovsbr1" + fmt.Sprintf("%d", i) + + flowList, err := ofctlFlowDump(brName) + if err != nil { + t.Errorf("Error getting flow entries. Err: %v", err) + } + + // verify flow entry exists + for j := 0; j < NUM_AGENT; j++ { + k := j + 1 + ipFlowMatch := fmt.Sprintf("priority=100,ip,nw_dst=10.10.%d.%d", k, k) + ipTableId := 2 + if !ofctlFlowMatch(flowList, ipTableId, ipFlowMatch) { + t.Errorf("Could not find the route %s on ovs %s", ipFlowMatch, brName) + } + + log.Infof("Found ipflow %s on ovs %s", ipFlowMatch, brName) + } + } + + log.Infof("Adding Vrouter endpoint successful.\n Testing Delete") + + for i := 0; i < NUM_AGENT; i++ { + j := i + 1 + macAddr, _ := net.ParseMAC(fmt.Sprintf("02:02:02:%02x:%02x:%02x", j, j, j)) + ipAddr := net.ParseIP(fmt.Sprintf("10.10.%d.%d", j, j)) + endpoint := EndpointInfo{ + PortNo : uint32(NUM_AGENT + 2), + MacAddr : macAddr, + Vlan : 1, + IpAddr : ipAddr, + } - // verify flow entry exists - for j := 0; j < NUM_AGENT; j++ { - k := j + 1 - ipFlowMatch := fmt.Sprintf("priority=100,ip,nw_dst=10.10.%d.%d", k, k) - ipTableId := 2 - if !ofctlFlowMatch(flowList, ipTableId, ipFlowMatch) { - t.Errorf("Could not find the route %s on ovs %s", ipFlowMatch, brName) - } + log.Infof("Deleting local vrouter endpoint: %+v", endpoint) - log.Infof("Found ipflow %s on ovs %s", ipFlowMatch, brName) - } - } + // Install the local endpoint + err := vrtrAgents[i].RemoveLocalEndpoint(uint32(NUM_AGENT + 2)) + if err != nil { + t.Fatalf("Error deleting endpoint: %+v. Err: %v", endpoint, err) + return + } + } + } } // Test adding/deleting Vxlan routes func TestOfnetVxlanAddDelete(t *testing.T) { - for i := 0; i < NUM_AGENT; i++ { - j := i + 1 - macAddr, _ := net.ParseMAC(fmt.Sprintf("02:02:02:%02x:%02x:%02x", j, j, j)) - ipAddr := net.ParseIP(fmt.Sprintf("10.10.%d.%d", j, j)) - endpoint := EndpointInfo{ - PortNo : uint32(NUM_AGENT + 2), - MacAddr : macAddr, - Vlan : 1, - IpAddr : ipAddr, - } - - log.Infof("Installing local vxlan endpoint: %+v", endpoint) - - // Install the local endpoint - err := vxlanAgents[i].AddLocalEndpoint(endpoint) - if err != nil { - t.Errorf("Error installing endpoint: %+v. Err: %v", endpoint, err) - } - } - - log.Infof("Finished adding local vxlan endpoint") - - // verify all ovs switches have this route - for i := 0; i < NUM_AGENT; i++ { - brName := "ovsbr2" + fmt.Sprintf("%d", i) - - flowList, err := ofctlFlowDump(brName) - if err != nil { - t.Errorf("Error getting flow entries. Err: %v", err) - } - - // verify flow entry exists - for j := 0; j < NUM_AGENT; j++ { - k := j + 1 - macFlowMatch := fmt.Sprintf("priority=100,dl_vlan=1,dl_dst=02:02:02:%02x:%02x:%02x", k, k, k) - - macTableId := 3 - if !ofctlFlowMatch(flowList, macTableId, macFlowMatch) { - t.Errorf("Could not find the mac flow %s on ovs %s", macFlowMatch, brName) - } - - log.Infof("Found macFlow %s on ovs %s", macFlowMatch, brName) - } - } + for iter := 0; iter < 4; iter++ { + for i := 0; i < NUM_AGENT; i++ { + j := i + 1 + macAddr, _ := net.ParseMAC(fmt.Sprintf("02:02:02:%02x:%02x:%02x", j, j, j)) + ipAddr := net.ParseIP(fmt.Sprintf("10.10.%d.%d", j, j)) + endpoint := EndpointInfo{ + PortNo : uint32(NUM_AGENT + 2), + MacAddr : macAddr, + Vlan : 1, + IpAddr : ipAddr, + } + + log.Infof("Installing local vxlan endpoint: %+v", endpoint) + + // Install the local endpoint + err := vxlanAgents[i].AddLocalEndpoint(endpoint) + if err != nil { + t.Errorf("Error installing endpoint: %+v. Err: %v", endpoint, err) + } + } + + log.Infof("Finished adding local vxlan endpoint") + + // verify all ovs switches have this route + for i := 0; i < NUM_AGENT; i++ { + brName := "ovsbr2" + fmt.Sprintf("%d", i) + + flowList, err := ofctlFlowDump(brName) + if err != nil { + t.Errorf("Error getting flow entries. Err: %v", err) + } + + // verify flow entry exists + for j := 0; j < NUM_AGENT; j++ { + k := j + 1 + macFlowMatch := fmt.Sprintf("priority=100,dl_vlan=1,dl_dst=02:02:02:%02x:%02x:%02x", k, k, k) + + macTableId := 3 + if !ofctlFlowMatch(flowList, macTableId, macFlowMatch) { + t.Errorf("Could not find the mac flow %s on ovs %s", macFlowMatch, brName) + } + + log.Infof("Found macFlow %s on ovs %s", macFlowMatch, brName) + } + } + + log.Infof("Add vxlan endpoint successful.\n Testing Delete") + + for i := 0; i < NUM_AGENT; i++ { + j := i + 1 + macAddr, _ := net.ParseMAC(fmt.Sprintf("02:02:02:%02x:%02x:%02x", j, j, j)) + ipAddr := net.ParseIP(fmt.Sprintf("10.10.%d.%d", j, j)) + endpoint := EndpointInfo{ + PortNo : uint32(NUM_AGENT + 2), + MacAddr : macAddr, + Vlan : 1, + IpAddr : ipAddr, + } + + log.Infof("Deleting local vxlan endpoint: %+v", endpoint) + + // Install the local endpoint + err := vxlanAgents[i].RemoveLocalEndpoint(uint32(NUM_AGENT + 2)) + if err != nil { + t.Errorf("Error deleting endpoint: %+v. Err: %v", endpoint, err) + } + } + } } // Wait for debug and cleanup diff --git a/Godeps/_workspace/src/github.com/contiv/ofnet/vrouter.go b/Godeps/_workspace/src/github.com/contiv/ofnet/vrouter.go index 8542e74cc..12d8d6d36 100644 --- a/Godeps/_workspace/src/github.com/contiv/ofnet/vrouter.go +++ b/Godeps/_workspace/src/github.com/contiv/ofnet/vrouter.go @@ -46,6 +46,7 @@ type Vrouter struct { // Flow Database flowDb map[string]*ofctrl.Flow // Database of flow entries + portVlanFlowDb map[uint32]*ofctrl.Flow // Database of flow entries // Router Mac to be used myRouterMac net.HardwareAddr @@ -70,6 +71,7 @@ func NewVrouter(agent *OfnetAgent, rpcServ *rpc.Server) *Vrouter { // Create a route table and my router mac vrouter.routeTable = make(map[string]*OfnetRoute) vrouter.flowDb = make(map[string]*ofctrl.Flow) + vrouter.portVlanFlowDb = make(map[uint32]*ofctrl.Flow) vrouter.myRouterMac, _ = net.ParseMAC("00:00:11:11:11:11") // Register for Route rpc callbacks @@ -161,6 +163,9 @@ func (self *Vrouter) AddLocalEndpoint(endpoint EndpointInfo) error { return err } + // save the flow entry + self.portVlanFlowDb[endpoint.PortNo] = portVlanFlow + // build the route to add route := OfnetRoute{ IpAddr: endpoint.IpAddr, @@ -212,7 +217,8 @@ func (self *Vrouter) AddLocalEndpoint(endpoint EndpointInfo) error { // Note: Works only for local ports func (self *Vrouter) findLocalRouteByPortno(portNo uint32) *OfnetRoute { for _, route := range self.routeTable { - if route.PortNo == portNo { + if (route.OriginatorIp.String() == self.agent.localIp.String()) && + (route.PortNo == portNo) { return route } } @@ -228,6 +234,15 @@ func (self *Vrouter) RemoveLocalEndpoint(portNo uint32) error { log.Errorf("Could not find local route ") } + // Remove the port vlan flow. + portVlanFlow := self.portVlanFlowDb[portNo] + if portVlanFlow != nil { + err := portVlanFlow.Delete() + if err != nil { + log.Errorf("Error deleting portvlan flow. Err: %v", err) + } + } + // Uninstall the route err := self.uninstallRoute(route) if err != nil { diff --git a/Godeps/_workspace/src/github.com/contiv/ofnet/vxlanBridge.go b/Godeps/_workspace/src/github.com/contiv/ofnet/vxlanBridge.go index 4f3fbc1ed..e84d539d6 100644 --- a/Godeps/_workspace/src/github.com/contiv/ofnet/vxlanBridge.go +++ b/Godeps/_workspace/src/github.com/contiv/ofnet/vxlanBridge.go @@ -13,21 +13,22 @@ See the License for the specific language governing permissions and limitations under the License. */ package ofnet + // This file implements the vxlan bridging datapath import ( - //"fmt" - "net" - "net/rpc" - "time" - "errors" - - //"github.com/shaleman/libOpenflow/openflow13" - //"github.com/shaleman/libOpenflow/protocol" - "github.com/contiv/ofnet/ofctrl" - "github.com/contiv/ofnet/rpcHub" - - log "github.com/Sirupsen/logrus" + //"fmt" + "errors" + "net" + "net/rpc" + "time" + + //"github.com/shaleman/libOpenflow/openflow13" + //"github.com/shaleman/libOpenflow/protocol" + "github.com/contiv/ofnet/ofctrl" + "github.com/contiv/ofnet/rpcHub" + + log "github.com/Sirupsen/logrus" ) // VXLAN tables are structured as follows @@ -35,10 +36,10 @@ import ( // +-------+ // | Valid | // | Pkts +-->+-------+ -// +-------+ | Vlan | +----------+ -// | Table +-->| Mac Src | +---------+ -// +-------+ | Learning +-->| Mac Dst | +--------------+ -// +----------+ | Lookup +--+-->| Ucast Output | +// +-------+ | Vlan | +// | Table +-------+ +---------+ +// +-------+ +--------->| Mac Dst | +--------------+ +// | Lookup +--+-->| Ucast Output | // +---------+ | +--------------+ // | // | @@ -51,587 +52,605 @@ import ( // Vxlan state. type Vxlan struct { - agent *OfnetAgent // Pointer back to ofnet agent that owns this - ofSwitch *ofctrl.OFSwitch // openflow switch we are talking to + agent *OfnetAgent // Pointer back to ofnet agent that owns this + ofSwitch *ofctrl.OFSwitch // openflow switch we are talking to - vlanDb map[uint16]*Vlan // Database of known vlans + vlanDb map[uint16]*Vlan // Database of known vlans - // Mac route table - macRouteDb map[string]*MacRoute + // Mac route table + macRouteDb map[string]*MacRoute - // Fgraph tables - inputTable *ofctrl.Table // Packet lookup starts here - vlanTable *ofctrl.Table // Vlan Table. map port or VNI to vlan - macDestTable *ofctrl.Table // Destination mac lookup + // Fgraph tables + inputTable *ofctrl.Table // Packet lookup starts here + vlanTable *ofctrl.Table // Vlan Table. map port or VNI to vlan + macDestTable *ofctrl.Table // Destination mac lookup - // Flow Database - macFlowDb map[string]*ofctrl.Flow // Database of flow entries + // Flow Database + macFlowDb map[string]*ofctrl.Flow // Database of flow entries + portVlanFlowDb map[uint32]*ofctrl.Flow // Database of flow entries } // Vlan info type Vlan struct { - Vni uint32 // Vxlan VNI - localPortList map[uint32]*uint32 // List of local ports only - allPortList map[uint32]*uint32 // List of local + remote(vtep) ports - localFlood *ofctrl.Flood // local only flood list - allFlood *ofctrl.Flood // local + remote flood list + Vni uint32 // Vxlan VNI + localPortList map[uint32]*uint32 // List of local ports only + allPortList map[uint32]*uint32 // List of local + remote(vtep) ports + localFlood *ofctrl.Flood // local only flood list + allFlood *ofctrl.Flood // local + remote flood list } // Mac address info type MacRoute struct { - MacAddrStr string // Mac address of the end point(in string format) - Vni uint32 // Vxlan VNI - OriginatorIp net.IP // Originating switch - PortNo uint32 // Port number on originating switch - Timestamp time.Time // Timestamp of the last event + MacAddrStr string // Mac address of the end point(in string format) + Vni uint32 // Vxlan VNI + OriginatorIp net.IP // Originating switch + PortNo uint32 // Port number on originating switch + Timestamp time.Time // Timestamp of the last event } const METADATA_RX_VTEP = 0x1 // Create a new vxlan instance func NewVxlan(agent *OfnetAgent, rpcServ *rpc.Server) *Vxlan { - vxlan := new(Vxlan) + vxlan := new(Vxlan) - // Keep a reference to the agent - vxlan.agent = agent + // Keep a reference to the agent + vxlan.agent = agent - // init DBs - vxlan.macRouteDb = make(map[string]*MacRoute) - vxlan.vlanDb = make(map[uint16]*Vlan) - vxlan.macFlowDb = make(map[string]*ofctrl.Flow) + // init DBs + vxlan.macRouteDb = make(map[string]*MacRoute) + vxlan.vlanDb = make(map[uint16]*Vlan) + vxlan.macFlowDb = make(map[string]*ofctrl.Flow) + vxlan.portVlanFlowDb = make(map[uint32]*ofctrl.Flow) - log.Infof("Registering vxlan RPC calls") + log.Infof("Registering vxlan RPC calls") - // Register for Route rpc callbacks - err := rpcServ.Register(vxlan) - if err != nil { - log.Fatalf("Error registering vxlan RPC") - } + // Register for Route rpc callbacks + err := rpcServ.Register(vxlan) + if err != nil { + log.Fatalf("Error registering vxlan RPC") + } - return vxlan + return vxlan } // Handle new master added event func (self *Vxlan) MasterAdded(master *OfnetNode) error { - // Send all local routes to new master. - for _, macRoute := range self.macRouteDb { - if macRoute.OriginatorIp.String() == self.agent.localIp.String() { - var resp bool - - log.Infof("Sending macRoute %+v to master %+v", macRoute, master) - - // Make the RPC call to add the route to master - client := rpcHub.Client(master.HostAddr, master.HostPort) - err := client.Call("OfnetMaster.MacRouteAdd", macRoute, &resp) - if (err != nil) { - log.Errorf("Failed to add route %+v to master %+v. Err: %v", macRoute, master, err) - return err - } - } - } - - return nil + // Send all local routes to new master. + for _, macRoute := range self.macRouteDb { + if macRoute.OriginatorIp.String() == self.agent.localIp.String() { + var resp bool + + log.Infof("Sending macRoute %+v to master %+v", macRoute, master) + + // Make the RPC call to add the route to master + client := rpcHub.Client(master.HostAddr, master.HostPort) + err := client.Call("OfnetMaster.MacRouteAdd", macRoute, &resp) + if err != nil { + log.Errorf("Failed to add route %+v to master %+v. Err: %v", macRoute, master, err) + return err + } + } + } + + return nil } // Handle switch connected notification func (self *Vxlan) SwitchConnected(sw *ofctrl.OFSwitch) { - // Keep a reference to the switch - self.ofSwitch = sw - // Init the Fgraph - self.initFgraph() + // Keep a reference to the switch + self.ofSwitch = sw + // Init the Fgraph + self.initFgraph() - log.Infof("Switch connected(vxlan)") + log.Infof("Switch connected(vxlan)") } + // Handle switch disconnected notification func (self *Vxlan) SwitchDisconnected(sw *ofctrl.OFSwitch) { - // FIXME: ?? + // FIXME: ?? } // Handle incoming packet func (self *Vxlan) PacketRcvd(sw *ofctrl.OFSwitch, pkt *ofctrl.PacketIn) { - // Ignore all incoming packets for now + // Ignore all incoming packets for now } // Add a local endpoint and install associated local route func (self *Vxlan) AddLocalEndpoint(endpoint EndpointInfo) error { - log.Infof("Adding local endpoint: %+v", endpoint) - - vni := self.agent.vlanVniMap[endpoint.Vlan] - if vni == nil { - log.Errorf("VNI for vlan %d is not known", endpoint.Vlan) - return errors.New("Unknown Vlan") - } - - // Install a flow entry for vlan mapping and point it to IP table - portVlanFlow, err := self.vlanTable.NewFlow(ofctrl.FlowMatch{ - Priority: FLOW_MATCH_PRIORITY, - InputPort: endpoint.PortNo, - }) - if err != nil { - log.Errorf("Error creating portvlan entry. Err: %v", err) - return err - } - - // Set the vlan and install it - portVlanFlow.SetVlan(endpoint.Vlan) - err = portVlanFlow.Next(self.macDestTable) - if err != nil { - log.Errorf("Error installing portvlan entry. Err: %v", err) - return err - } - - // Add the port to local and remote flood list - output, _ := self.ofSwitch.OutputPort(endpoint.PortNo) - vlan := self.vlanDb[endpoint.Vlan] - if vlan != nil { - vlan.localFlood.AddOutput(output) - vlan.allFlood.AddOutput(output) - } - - // Finally install the mac address - macFlow, err := self.macDestTable.NewFlow(ofctrl.FlowMatch{ - Priority: FLOW_MATCH_PRIORITY, - VlanId: endpoint.Vlan, - MacDa: &endpoint.MacAddr, - }) - if err != nil { - log.Errorf("Error creating mac flow for endpoint %+v. Err: %v", endpoint, err) - return err - } - - // Remove vlan tag and point it to local port - macFlow.PopVlan() - macFlow.Next(output) - - // Save the flow in DB - self.macFlowDb[endpoint.MacAddr.String()] = macFlow - - // Build the mac route - macRoute := MacRoute{ - MacAddrStr: endpoint.MacAddr.String(), - Vni: *vni, - OriginatorIp: self.agent.localIp, - PortNo: endpoint.PortNo, - Timestamp: time.Now(), - } - - // Advertize the route to master - err = self.localMacRouteAdd(&macRoute) - if (err != nil) { - log.Errorf("Failed to add route %+v to master. Err: %v", macRoute, err) - return err - } - - return nil + log.Infof("Adding local endpoint: %+v", endpoint) + + vni := self.agent.vlanVniMap[endpoint.Vlan] + if vni == nil { + log.Errorf("VNI for vlan %d is not known", endpoint.Vlan) + return errors.New("Unknown Vlan") + } + + // Install a flow entry for vlan mapping and point it to Mac table + portVlanFlow, err := self.vlanTable.NewFlow(ofctrl.FlowMatch{ + Priority: FLOW_MATCH_PRIORITY, + InputPort: endpoint.PortNo, + }) + if err != nil { + log.Errorf("Error creating portvlan entry. Err: %v", err) + return err + } + + // Set the vlan and install it + portVlanFlow.SetVlan(endpoint.Vlan) + err = portVlanFlow.Next(self.macDestTable) + if err != nil { + log.Errorf("Error installing portvlan entry. Err: %v", err) + return err + } + + // save the flow entry + self.portVlanFlowDb[endpoint.PortNo] = portVlanFlow + + // Add the port to local and remote flood list + output, _ := self.ofSwitch.OutputPort(endpoint.PortNo) + vlan := self.vlanDb[endpoint.Vlan] + if vlan != nil { + vlan.localFlood.AddOutput(output) + vlan.allFlood.AddOutput(output) + } + + // Finally install the mac address + macFlow, err := self.macDestTable.NewFlow(ofctrl.FlowMatch{ + Priority: FLOW_MATCH_PRIORITY, + VlanId: endpoint.Vlan, + MacDa: &endpoint.MacAddr, + }) + if err != nil { + log.Errorf("Error creating mac flow for endpoint %+v. Err: %v", endpoint, err) + return err + } + + // Remove vlan tag and point it to local port + macFlow.PopVlan() + macFlow.Next(output) + + // Save the flow in DB + self.macFlowDb[endpoint.MacAddr.String()] = macFlow + + // Build the mac route + macRoute := MacRoute{ + MacAddrStr: endpoint.MacAddr.String(), + Vni: *vni, + OriginatorIp: self.agent.localIp, + PortNo: endpoint.PortNo, + Timestamp: time.Now(), + } + + // Advertize the route to master + err = self.localMacRouteAdd(&macRoute) + if err != nil { + log.Errorf("Failed to add route %+v to master. Err: %v", macRoute, err) + return err + } + + return nil } // Find a mac by output port number // Note: Works only for local ports // FIXME: remove this function and add a mapping between local portNo and macRoute func (self *Vxlan) findLocalMacRouteByPortno(portNo uint32) *MacRoute { - for _, macRoute := range self.macRouteDb { - if macRoute.PortNo == portNo { - return macRoute - } - } - - return nil + for _, macRoute := range self.macRouteDb { + if (macRoute.OriginatorIp.String() == self.agent.localIp.String()) && + (macRoute.PortNo == portNo) { + return macRoute + } + } + + return nil } // Remove local endpoint func (self *Vxlan) RemoveLocalEndpoint(portNo uint32) error { - // find the mac route - macRoute := self.findLocalMacRouteByPortno(portNo) - if macRoute == nil { - log.Errorf("Local mac route not found for port: %d", portNo) - return errors.New("Local mac route not found") - } - - // Remove the port from flood lists - vlanId := self.agent.vniVlanMap[macRoute.Vni] - vlan := self.vlanDb[*vlanId] - output, _ := self.ofSwitch.OutputPort(portNo) - vlan.localFlood.RemoveOutput(output) - vlan.allFlood.RemoveOutput(output) - - // Uninstall the flow - err := self.uninstallMacRoute(macRoute) - if err != nil { - log.Errorf("Error Uninstalling mac route: %+v. Err: %v", macRoute, err) - } - - // Remove the route from local DB and Advertize delete - return self.localMacRouteDel(macRoute) + // find the mac route + macRoute := self.findLocalMacRouteByPortno(portNo) + if macRoute == nil { + log.Errorf("Local mac route not found for port: %d", portNo) + return errors.New("Local mac route not found") + } + + // Remove the port from flood lists + vlanId := self.agent.vniVlanMap[macRoute.Vni] + vlan := self.vlanDb[*vlanId] + output, _ := self.ofSwitch.OutputPort(portNo) + vlan.localFlood.RemoveOutput(output) + vlan.allFlood.RemoveOutput(output) + + // Remove the port vlan flow. + portVlanFlow := self.portVlanFlowDb[portNo] + if portVlanFlow != nil { + err := portVlanFlow.Delete() + if err != nil { + log.Errorf("Error deleting portvlan flow. Err: %v", err) + } + } + + // Uninstall the flow + err := self.uninstallMacRoute(macRoute) + if err != nil { + log.Errorf("Error Uninstalling mac route: %+v. Err: %v", macRoute, err) + } + + // Remove the route from local DB and Advertize delete + return self.localMacRouteDel(macRoute) } // Add virtual tunnel end point. This is mainly used for mapping remote vtep IP // to ofp port number. func (self *Vxlan) AddVtepPort(portNo uint32, remoteIp net.IP) error { - // Install VNI to vlan mapping for each vni - for vni, vlan := range self.agent.vniVlanMap { - // Install a flow entry for VNI/vlan and point it to macDest table - portVlanFlow, _ := self.vlanTable.NewFlow(ofctrl.FlowMatch{ - Priority: FLOW_MATCH_PRIORITY, - InputPort: portNo, - TunnelId: uint64(vni), - }) - portVlanFlow.SetVlan(*vlan) - - // Set the metadata to indicate packet came in from VTEP port - portVlanFlow.SetMetadata(METADATA_RX_VTEP, METADATA_RX_VTEP) - - // Point to next table - portVlanFlow.Next(self.macDestTable) - } - - // Walk all vlans and add vtep port to the vlan - for vlanId, vlan := range self.vlanDb { - vni := self.agent.vlanVniMap[vlanId] - if vni == nil { - log.Errorf("Can not find vni for vlan: %d", vlanId) - } - output, _ := self.ofSwitch.OutputPort(portNo) - vlan.allFlood.AddTunnelOutput(output, uint64(*vni)) - } - - return nil + // Install VNI to vlan mapping for each vni + for vni, vlan := range self.agent.vniVlanMap { + // Install a flow entry for VNI/vlan and point it to macDest table + portVlanFlow, _ := self.vlanTable.NewFlow(ofctrl.FlowMatch{ + Priority: FLOW_MATCH_PRIORITY, + InputPort: portNo, + TunnelId: uint64(vni), + }) + portVlanFlow.SetVlan(*vlan) + + // Set the metadata to indicate packet came in from VTEP port + portVlanFlow.SetMetadata(METADATA_RX_VTEP, METADATA_RX_VTEP) + + // Point to next table + portVlanFlow.Next(self.macDestTable) + } + + // Walk all vlans and add vtep port to the vlan + for vlanId, vlan := range self.vlanDb { + vni := self.agent.vlanVniMap[vlanId] + if vni == nil { + log.Errorf("Can not find vni for vlan: %d", vlanId) + } + output, _ := self.ofSwitch.OutputPort(portNo) + vlan.allFlood.AddTunnelOutput(output, uint64(*vni)) + } + + return nil } // Remove a VTEP port func (self *Vxlan) RemoveVtepPort(portNo uint32, remoteIp net.IP) error { - // Remove the VTEP from flood lists - output, _ := self.ofSwitch.OutputPort(portNo) - for _, vlan := range self.vlanDb { - // Walk all vlans and remove from flood lists - vlan.allFlood.RemoveOutput(output) - } - - // FIXME: uninstall vlan-vni mapping. - - // Walk all routes and remove anything pointing at this VTEP - for _, macRoute := range self.macRouteDb { - // If it originated from this remote host, uninstall the flow - if macRoute.OriginatorIp.String() == remoteIp.String() { - err := self.uninstallMacRoute(macRoute) - if err != nil { - log.Errorf("Error uninstalling mac route: %+v. Err: %v", macRoute, err) - } - } - } - - return nil + // Remove the VTEP from flood lists + output, _ := self.ofSwitch.OutputPort(portNo) + for _, vlan := range self.vlanDb { + // Walk all vlans and remove from flood lists + vlan.allFlood.RemoveOutput(output) + } + + // FIXME: uninstall vlan-vni mapping. + + // Walk all routes and remove anything pointing at this VTEP + for _, macRoute := range self.macRouteDb { + // If it originated from this remote host, uninstall the flow + if macRoute.OriginatorIp.String() == remoteIp.String() { + err := self.uninstallMacRoute(macRoute) + if err != nil { + log.Errorf("Error uninstalling mac route: %+v. Err: %v", macRoute, err) + } + } + } + + return nil } // Add a vlan. func (self *Vxlan) AddVlan(vlanId uint16, vni uint32) error { - // check if the vlan already exists. if it does, we are done - if self.vlanDb[vlanId] != nil { - return nil - } - - // create new vlan object - vlan := new(Vlan) - vlan.Vni = vni - vlan.localPortList = make(map[uint32]*uint32) - vlan.allPortList = make(map[uint32]*uint32) - - // Create flood entries - vlan.localFlood, _ = self.ofSwitch.NewFlood() - vlan.allFlood, _ = self.ofSwitch.NewFlood() - - // Walk all VTEP ports and add vni-vlan mapping for new VNI - for _, vtepPort := range self.agent.vtepTable { - // Install a flow entry for VNI/vlan and point it to macDest table - portVlanFlow, err := self.vlanTable.NewFlow(ofctrl.FlowMatch{ - Priority: FLOW_MATCH_PRIORITY, - InputPort: *vtepPort, - TunnelId: uint64(vni), - }) - if err != nil { - log.Errorf("Error creating port vlan flow for vlan %d. Err: %v", vlanId, err) - return err - } - - // Set vlan id - portVlanFlow.SetVlan(vlanId) - - // Set the metadata to indicate packet came in from VTEP port - portVlanFlow.SetMetadata(METADATA_RX_VTEP, METADATA_RX_VTEP) - - // Point to next table - portVlanFlow.Next(self.macDestTable) - } - - // Walk all VTEP ports and add it to the allFlood list - for _, vtepPort := range self.agent.vtepTable { - output, _ := self.ofSwitch.OutputPort(*vtepPort) - vlan.allFlood.AddTunnelOutput(output, uint64(vni)) - } - - log.Infof("Installing vlan flood entry for vlan: %d", vlanId) - - // Install local flood and remote flood entries in macDestTable - var metadataLclRx uint64 = 0 - var metadataVtepRx uint64 = METADATA_RX_VTEP - vlanFlood, err := self.macDestTable.NewFlow(ofctrl.FlowMatch{ - Priority: FLOW_FLOOD_PRIORITY, - VlanId: vlanId, - Metadata: &metadataLclRx, - MetadataMask: &metadataVtepRx, - }) - if err != nil { - log.Errorf("Error creating local+remote flood. Err: %v", err) - return err - } - - vlanFlood.Next(vlan.allFlood) - vlanLclFlood, err := self.macDestTable.NewFlow(ofctrl.FlowMatch{ - Priority: FLOW_FLOOD_PRIORITY, - VlanId: vlanId, - Metadata: &metadataVtepRx, - MetadataMask: &metadataVtepRx, - }) - if err != nil { - log.Errorf("Error creating local flood. Err: %v", err) - return err - } - vlanLclFlood.Next(vlan.localFlood) - - // store it in DB - self.vlanDb[vlanId] = vlan - - return nil + // check if the vlan already exists. if it does, we are done + if self.vlanDb[vlanId] != nil { + return nil + } + + // create new vlan object + vlan := new(Vlan) + vlan.Vni = vni + vlan.localPortList = make(map[uint32]*uint32) + vlan.allPortList = make(map[uint32]*uint32) + + // Create flood entries + vlan.localFlood, _ = self.ofSwitch.NewFlood() + vlan.allFlood, _ = self.ofSwitch.NewFlood() + + // Walk all VTEP ports and add vni-vlan mapping for new VNI + for _, vtepPort := range self.agent.vtepTable { + // Install a flow entry for VNI/vlan and point it to macDest table + portVlanFlow, err := self.vlanTable.NewFlow(ofctrl.FlowMatch{ + Priority: FLOW_MATCH_PRIORITY, + InputPort: *vtepPort, + TunnelId: uint64(vni), + }) + if err != nil { + log.Errorf("Error creating port vlan flow for vlan %d. Err: %v", vlanId, err) + return err + } + + // Set vlan id + portVlanFlow.SetVlan(vlanId) + + // Set the metadata to indicate packet came in from VTEP port + portVlanFlow.SetMetadata(METADATA_RX_VTEP, METADATA_RX_VTEP) + + // Point to next table + portVlanFlow.Next(self.macDestTable) + } + + // Walk all VTEP ports and add it to the allFlood list + for _, vtepPort := range self.agent.vtepTable { + output, _ := self.ofSwitch.OutputPort(*vtepPort) + vlan.allFlood.AddTunnelOutput(output, uint64(vni)) + } + + log.Infof("Installing vlan flood entry for vlan: %d", vlanId) + + // Install local flood and remote flood entries in macDestTable + var metadataLclRx uint64 = 0 + var metadataVtepRx uint64 = METADATA_RX_VTEP + vlanFlood, err := self.macDestTable.NewFlow(ofctrl.FlowMatch{ + Priority: FLOW_FLOOD_PRIORITY, + VlanId: vlanId, + Metadata: &metadataLclRx, + MetadataMask: &metadataVtepRx, + }) + if err != nil { + log.Errorf("Error creating local+remote flood. Err: %v", err) + return err + } + + vlanFlood.Next(vlan.allFlood) + vlanLclFlood, err := self.macDestTable.NewFlow(ofctrl.FlowMatch{ + Priority: FLOW_FLOOD_PRIORITY, + VlanId: vlanId, + Metadata: &metadataVtepRx, + MetadataMask: &metadataVtepRx, + }) + if err != nil { + log.Errorf("Error creating local flood. Err: %v", err) + return err + } + vlanLclFlood.Next(vlan.localFlood) + + // store it in DB + self.vlanDb[vlanId] = vlan + + return nil } // Remove a vlan func (self *Vxlan) RemoveVlan(vlanId uint16, vni uint32) error { - vlan := self.vlanDb[vlanId] - if vlan == nil { - log.Fatalf("Could not find the vlan %d", vlanId) - } - - // Make sure the flood lists are empty - if (vlan.allFlood.NumOutput() != 0) || (vlan.localFlood.NumOutput() != 0) { - log.Fatalf("VLAN flood list is not empty") - } - - // make sure there are no mac routes still installed in this vlan - for _, macRoute := range self.macRouteDb { - if macRoute.Vni == vni { - log.Fatalf("Vlan %d still has routes. Route: %+v", vlanId, macRoute) - } - } - - // Uninstall the flood lists - vlan.allFlood.Delete() - vlan.localFlood.Delete() - - // Remove it from DB - delete(self.vlanDb, vlanId) - - return nil + vlan := self.vlanDb[vlanId] + if vlan == nil { + log.Fatalf("Could not find the vlan %d", vlanId) + } + + // Make sure the flood lists are empty + if (vlan.allFlood.NumOutput() != 0) || (vlan.localFlood.NumOutput() != 0) { + log.Fatalf("VLAN flood list is not empty") + } + + // make sure there are no mac routes still installed in this vlan + for _, macRoute := range self.macRouteDb { + if macRoute.Vni == vni { + log.Fatalf("Vlan %d still has routes. Route: %+v", vlanId, macRoute) + } + } + + // Uninstall the flood lists + vlan.allFlood.Delete() + vlan.localFlood.Delete() + + // Remove it from DB + delete(self.vlanDb, vlanId) + + return nil } // Mac route add rpc call from master func (self *Vxlan) MacRouteAdd(macRoute *MacRoute, ret *bool) error { - log.Infof("Received mac route: %+v", macRoute) - - // If this is a local route we are done - if (macRoute.OriginatorIp.String() == self.agent.localIp.String()) { - return nil - } - - // Check if we have the route already and which is more recent - oldRoute := self.macRouteDb[macRoute.MacAddrStr] - if (oldRoute != nil) { - // If old route has more recent timestamp, nothing to do - if !macRoute.Timestamp.After(oldRoute.Timestamp) { - return nil - } - } - - // First, add the route to local routing table - self.macRouteDb[macRoute.MacAddrStr] = macRoute - - // Lookup the VTEP for the route - vtepPort := self.agent.vtepTable[macRoute.OriginatorIp.String()] - if (vtepPort == nil) { - log.Errorf("Could not find the VTEP for mac route: %+v", macRoute) - - return errors.New("VTEP not found") - } - - // map VNI to vlan Id - vlanId := self.agent.vniVlanMap[macRoute.Vni] - if vlanId == nil { - log.Errorf("Macroute %+v on unknown VNI: %d", macRoute, macRoute.Vni) - return errors.New("Unknown VNI") - } - - macAddr, _ := net.ParseMAC(macRoute.MacAddrStr) - - // Install the route in OVS - // Create an output port for the vtep - outPort, err := self.ofSwitch.OutputPort(*vtepPort) - if (err != nil) { - log.Errorf("Error creating output port %d. Err: %v", *vtepPort, err) - return err - } - - // Finally install the mac address - macFlow, _ := self.macDestTable.NewFlow(ofctrl.FlowMatch{ - Priority: FLOW_MATCH_PRIORITY, - VlanId: *vlanId, - MacDa: &macAddr, - }) - macFlow.PopVlan() - macFlow.SetTunnelId(uint64(macRoute.Vni)) - macFlow.Next(outPort) - - return nil + log.Infof("Received mac route: %+v", macRoute) + + // If this is a local route we are done + if macRoute.OriginatorIp.String() == self.agent.localIp.String() { + return nil + } + + // Check if we have the route already and which is more recent + oldRoute := self.macRouteDb[macRoute.MacAddrStr] + if oldRoute != nil { + // If old route has more recent timestamp, nothing to do + if !macRoute.Timestamp.After(oldRoute.Timestamp) { + return nil + } + } + + // First, add the route to local routing table + self.macRouteDb[macRoute.MacAddrStr] = macRoute + + // Lookup the VTEP for the route + vtepPort := self.agent.vtepTable[macRoute.OriginatorIp.String()] + if vtepPort == nil { + log.Errorf("Could not find the VTEP for mac route: %+v", macRoute) + + return errors.New("VTEP not found") + } + + // map VNI to vlan Id + vlanId := self.agent.vniVlanMap[macRoute.Vni] + if vlanId == nil { + log.Errorf("Macroute %+v on unknown VNI: %d", macRoute, macRoute.Vni) + return errors.New("Unknown VNI") + } + + macAddr, _ := net.ParseMAC(macRoute.MacAddrStr) + + // Install the route in OVS + // Create an output port for the vtep + outPort, err := self.ofSwitch.OutputPort(*vtepPort) + if err != nil { + log.Errorf("Error creating output port %d. Err: %v", *vtepPort, err) + return err + } + + // Finally install the mac address + macFlow, _ := self.macDestTable.NewFlow(ofctrl.FlowMatch{ + Priority: FLOW_MATCH_PRIORITY, + VlanId: *vlanId, + MacDa: &macAddr, + }) + macFlow.PopVlan() + macFlow.SetTunnelId(uint64(macRoute.Vni)) + macFlow.Next(outPort) + + // Save the flow in DB + self.macFlowDb[macRoute.MacAddrStr] = macFlow + + return nil } // Mac route delete rpc call from master -func (self *Vxlan) MacRouteDel (macRoute *MacRoute, ret *bool) error { - log.Infof("Received DELETE mac route: %+v", macRoute) - - // If this is a local route we are done - if (macRoute.OriginatorIp.String() == self.agent.localIp.String()) { - return nil - } - - // Ignore duplicate delete requests we might receive from multiple - // Ofnet masters - if self.macRouteDb[macRoute.MacAddrStr] == nil { - return nil - } - - // Uninstall the route - err := self.uninstallMacRoute(macRoute) - if err != nil { - log.Errorf("Error uninstalling mac route %+v. Err: %v", macRoute, err) - } - - // Remove it from route table - delete(self.macRouteDb, macRoute.MacAddrStr) - - return nil +func (self *Vxlan) MacRouteDel(macRoute *MacRoute, ret *bool) error { + log.Infof("Received DELETE mac route: %+v", macRoute) + + // If this is a local route we are done + if macRoute.OriginatorIp.String() == self.agent.localIp.String() { + return nil + } + + // Ignore duplicate delete requests we might receive from multiple + // Ofnet masters + if self.macRouteDb[macRoute.MacAddrStr] == nil { + return nil + } + + // Uninstall the route + err := self.uninstallMacRoute(macRoute) + if err != nil { + log.Errorf("Error uninstalling mac route %+v. Err: %v", macRoute, err) + } + + // Remove it from route table + delete(self.macRouteDb, macRoute.MacAddrStr) + + return nil } // Add a local route to routing table and distribute it func (self *Vxlan) localMacRouteAdd(macRoute *MacRoute) error { - // First, add the route to local routing table - self.macRouteDb[macRoute.MacAddrStr] = macRoute + // First, add the route to local routing table + self.macRouteDb[macRoute.MacAddrStr] = macRoute - // Send the route to all known masters - for _, master := range self.agent.masterDb { - var resp bool + // Send the route to all known masters + for _, master := range self.agent.masterDb { + var resp bool - log.Infof("Sending macRoute %+v to master %+v", macRoute, master) + log.Infof("Sending macRoute %+v to master %+v", macRoute, master) - // Make the RPC call to add the route to master - client := rpcHub.Client(master.HostAddr, master.HostPort) - err := client.Call("OfnetMaster.MacRouteAdd", macRoute, &resp) - if (err != nil) { - log.Errorf("Failed to add route %+v to master %+v. Err: %v", macRoute, master, err) - return err - } - } + // Make the RPC call to add the route to master + client := rpcHub.Client(master.HostAddr, master.HostPort) + err := client.Call("OfnetMaster.MacRouteAdd", macRoute, &resp) + if err != nil { + log.Errorf("Failed to add route %+v to master %+v. Err: %v", macRoute, master, err) + return err + } + } - return nil + return nil } // Delete a local route and inform the master func (self *Vxlan) localMacRouteDel(macRoute *MacRoute) error { - // delete the route from local routing table - delete(self.macRouteDb, macRoute.MacAddrStr) + // delete the route from local routing table + delete(self.macRouteDb, macRoute.MacAddrStr) - // Send the DELETE to all known masters - for _, master := range self.agent.masterDb { - var resp bool + // Send the DELETE to all known masters + for _, master := range self.agent.masterDb { + var resp bool - log.Infof("Sending DELETE macRoute %+v to master %+v", macRoute, master) + log.Infof("Sending DELETE macRoute %+v to master %+v", macRoute, master) - // Make the RPC call to add the route to master - client := rpcHub.Client(master.HostAddr, master.HostPort) - err := client.Call("OfnetMaster.MacRouteDel", macRoute, &resp) - if (err != nil) { - log.Errorf("Failed to DELETE route %+v to master %+v. Err: %v", macRoute, master, err) - return err - } - } + // Make the RPC call to add the route to master + client := rpcHub.Client(master.HostAddr, master.HostPort) + err := client.Call("OfnetMaster.MacRouteDel", macRoute, &resp) + if err != nil { + log.Errorf("Failed to DELETE route %+v to master %+v. Err: %v", macRoute, master, err) + return err + } + } - return nil + return nil } // Uninstall mac route from OVS func (self *Vxlan) uninstallMacRoute(macRoute *MacRoute) error { - // find the flow - macFlow := self.macFlowDb[macRoute.MacAddrStr] - if macFlow == nil { - log.Errorf("Could not find the flow for macRoute: %+v", macRoute) - return errors.New("Mac flow not found") - } - - // Delete the flow - err := macFlow.Delete() - if err != nil { - log.Errorf("Error deleting mac flow: %+v. Err: %v", macFlow, err) - } - - return err + // find the flow + macFlow := self.macFlowDb[macRoute.MacAddrStr] + if macFlow == nil { + log.Errorf("Could not find the flow for macRoute: %+v", macRoute) + return errors.New("Mac flow not found") + } + + // Delete the flow + err := macFlow.Delete() + if err != nil { + log.Errorf("Error deleting mac flow: %+v. Err: %v", macFlow, err) + } + + return err } - const MAC_DEST_TBL_ID = 3 // initialize Fgraph on the switch func (self *Vxlan) initFgraph() error { - sw := self.ofSwitch - - log.Infof("Installing initial flow entries") - - // Create all tables - self.inputTable = sw.DefaultTable() - self.vlanTable, _ = sw.NewTable(VLAN_TBL_ID) - self.macDestTable, _ = sw.NewTable(MAC_DEST_TBL_ID) - - //Create all drop entries - // Drop mcast source mac - bcastMac, _ := net.ParseMAC("01:00:00:00:00:00") - bcastSrcFlow, _ := self.inputTable.NewFlow(ofctrl.FlowMatch{ - Priority: FLOW_MATCH_PRIORITY, - MacSa: &bcastMac, - MacSaMask: &bcastMac, - }) - bcastSrcFlow.Next(sw.DropAction()) - - // FIXME: Add additional checks on: - // Drop STP packets - // Send LLDP packets to controller - // Send LACP packets to controller - // Drop all other reserved mcast packets in 01-80-C2 range. - - // Send all valid packets to vlan table - // This is installed at lower priority so that all packets that miss above - // flows will match entry - validPktFlow, _ := self.inputTable.NewFlow(ofctrl.FlowMatch{ - Priority: FLOW_MISS_PRIORITY, - }) - validPktFlow.Next(self.vlanTable) - - // Drop all packets that miss Vlan lookup - vlanMissFlow, _ := self.vlanTable.NewFlow(ofctrl.FlowMatch{ - Priority: FLOW_MISS_PRIORITY, - }) - vlanMissFlow.Next(sw.DropAction()) - - // Drop all packets that miss mac dest lookup AND vlan flood lookup - floodMissFlow, _ := self.macDestTable.NewFlow(ofctrl.FlowMatch{ - Priority: FLOW_MISS_PRIORITY, - }) - floodMissFlow.Next(sw.DropAction()) - - // Drop all - return nil + sw := self.ofSwitch + + log.Infof("Installing initial flow entries") + + // Create all tables + self.inputTable = sw.DefaultTable() + self.vlanTable, _ = sw.NewTable(VLAN_TBL_ID) + self.macDestTable, _ = sw.NewTable(MAC_DEST_TBL_ID) + + //Create all drop entries + // Drop mcast source mac + bcastMac, _ := net.ParseMAC("01:00:00:00:00:00") + bcastSrcFlow, _ := self.inputTable.NewFlow(ofctrl.FlowMatch{ + Priority: FLOW_MATCH_PRIORITY, + MacSa: &bcastMac, + MacSaMask: &bcastMac, + }) + bcastSrcFlow.Next(sw.DropAction()) + + // FIXME: Add additional checks on: + // Drop STP packets + // Send LLDP packets to controller + // Send LACP packets to controller + // Drop all other reserved mcast packets in 01-80-C2 range. + + // Send all valid packets to vlan table + // This is installed at lower priority so that all packets that miss above + // flows will match entry + validPktFlow, _ := self.inputTable.NewFlow(ofctrl.FlowMatch{ + Priority: FLOW_MISS_PRIORITY, + }) + validPktFlow.Next(self.vlanTable) + + // Drop all packets that miss Vlan lookup + vlanMissFlow, _ := self.vlanTable.NewFlow(ofctrl.FlowMatch{ + Priority: FLOW_MISS_PRIORITY, + }) + vlanMissFlow.Next(sw.DropAction()) + + // Drop all packets that miss mac dest lookup AND vlan flood lookup + floodMissFlow, _ := self.macDestTable.NewFlow(ofctrl.FlowMatch{ + Priority: FLOW_MISS_PRIORITY, + }) + floodMissFlow.Next(sw.DropAction()) + + // Drop all + return nil } diff --git a/Makefile b/Makefile index 13ac40281..da92c5683 100644 --- a/Makefile +++ b/Makefile @@ -43,15 +43,27 @@ unit-test: build CONTIV_HOST_GOPATH=$(GOPATH) CONTIV_HOST_GOBIN=$(HOST_GOBIN) \ CONTIV_HOST_GOROOT=$(HOST_GOROOT) ./scripts/unittests -vagrant +unit-test-centos: build + CONTIV_NODE_OS=centos CONTIV_HOST_GOPATH=$(GOPATH) CONTIV_HOST_GOBIN=$(HOST_GOBIN) \ + CONTIV_HOST_GOROOT=$(HOST_GOROOT) ./scripts/unittests -vagrant + # setting CONTIV_SOE=1 while calling 'make system-test' will stop the test # on first failure and leave setup in that state. This can be useful for debugging # as part of development. system-test: build CONTIV_HOST_GOPATH=$(GOPATH) godep go test --timeout 30m -v -run "sanity" \ + github.com/contiv/netplugin/systemtests/singlehost + CONTIV_HOST_GOPATH=$(GOPATH) godep go test --timeout 80m -v -run "sanity" \ + github.com/contiv/netplugin/systemtests/twohosts + +system-test-centos: build + CONTIV_NODE_OS=centos CONTIV_HOST_GOPATH=$(GOPATH) godep go test --timeout 30m -v -run "sanity" \ github.com/contiv/netplugin/systemtests/singlehost - CONTIV_HOST_GOPATH=$(GOPATH) godep go test --timeout 60m -v -run "sanity" \ + CONTIV_NODE_OS=centos CONTIV_HOST_GOPATH=$(GOPATH) godep go test --timeout 90m -v -run "sanity" \ github.com/contiv/netplugin/systemtests/twohosts +centos-tests: unit-test-centos system-test-centos + # setting CONTIV_SOE=1 while calling 'make regress-test' will stop the test # on first failure and leave setup in that state. This can be useful for debugging # as part of development. diff --git a/Vagrantfile b/Vagrantfile index 2d1b51499..05b26bd06 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -21,15 +21,15 @@ source /etc/profile.d/envvar.sh ## set the mounted host filesystems to be read-only.Just a safety check ## to prevent inadvertent modifications from vm. -(mount -o remount,ro,exec /vagrant) || exit 1 +(mount -o remount,ro,exec,norelatime /vagrant) || exit 1 if [ -e #{host_gobin_path} ]; then - (mount -o remount,ro,exec #{host_gobin_path}) || exit 1 + (mount -o remount,ro,exec,norelatime #{host_gobin_path}) || exit 1 fi if [ -e #{host_goroot_path} ]; then - (mount -o remount,ro,exec #{host_goroot_path}) || exit 1 + (mount -o remount,ro,exec,norelatime #{host_goroot_path}) || exit 1 fi if [ -e #{netplugin_synced_gopath} ]; then - (mount -o remount,ro,exec #{netplugin_synced_gopath}) || exit 1 + (mount -o remount,ro,exec,norelatime #{netplugin_synced_gopath}) || exit 1 fi ### install basic packages @@ -81,9 +81,13 @@ SCRIPT VAGRANTFILE_API_VERSION = "2" Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.box = "contiv/ubuntu-v4" - # Commenting out the url since we host the image on Atlas. - # config.vm.box_url = "https://cisco.box.com/shared/static/27u8utb1em5730rzprhr5szeuv2p0wir.box" + if ENV['CONTIV_NODE_OS'] && ENV['CONTIV_NODE_OS'] == "centos" then + config.vm.box = "contiv/centos" + else + config.vm.box = "contiv/ubuntu-v4" + # Commenting out the url since we host the image on Atlas. + # config.vm.box_url = "https://cisco.box.com/shared/static/27u8utb1em5730rzprhr5szeuv2p0wir.box" + end num_nodes = 2 if ENV['CONTIV_NODES'] && ENV['CONTIV_NODES'] != "" then num_nodes = ENV['CONTIV_NODES'].to_i diff --git a/drivers/ovsSwitch.go b/drivers/ovsSwitch.go index e465b8ed6..93f9addfc 100644 --- a/drivers/ovsSwitch.go +++ b/drivers/ovsSwitch.go @@ -125,6 +125,9 @@ func (sw *OvsSwitch) Delete() { } if sw.ovsdbDriver != nil { sw.ovsdbDriver.Delete() + + // Wait a little for OVS switch to be deleted + time.Sleep(300 * time.Millisecond) } } diff --git a/drivers/ovsdbDriver.go b/drivers/ovsdbDriver.go index d518ad843..42658bf86 100644 --- a/drivers/ovsdbDriver.go +++ b/drivers/ovsdbDriver.go @@ -83,6 +83,7 @@ func NewOvsdbDriver(bridgeName string, failMode string) (*OvsdbDriver, error) { func (d *OvsdbDriver) Delete() error { if d.ovs != nil { d.createDeleteBridge(d.bridgeName, "", operDeleteBridge) + log.Infof("Deleting OVS bridge: %s", d.bridgeName) (*d.ovs).Disconnect() } diff --git a/drivers/ovsendpointstate.go b/drivers/ovsendpointstate.go index c1f20139c..c3a323a59 100644 --- a/drivers/ovsendpointstate.go +++ b/drivers/ovsendpointstate.go @@ -70,6 +70,7 @@ type OvsOperEndpointState struct { core.CommonState NetID string `json:"netID"` ContName string `json:"contName"` + ContUUID string `json:"contUUID"` AttachUUID string `json:"attachUUID"` IPAddress string `json:"ipAddress"` MacAddress string `json:"macAddress"` diff --git a/netplugin/netd.go b/netplugin/netd.go index 80f76e430..b6d246f32 100644 --- a/netplugin/netd.go +++ b/netplugin/netd.go @@ -18,6 +18,7 @@ package main import ( "bufio" "encoding/json" + "errors" "flag" "fmt" "io/ioutil" @@ -46,14 +47,15 @@ import ( // network provisioning interfaces type cliOpts struct { - hostLabel string - nativeInteg bool - cfgFile string - debug bool - syslog string - jsonLog bool - vtepIP string - vlanIntf string + hostLabel string + nativeInteg bool + cfgFile string + debug bool + syslog string + jsonLog bool + vtepIP string + vlanIntf string + forceDeleteEp bool } func skipHost(vtepIP, homingHost, myHostLabel string) bool { @@ -193,7 +195,6 @@ func getContainerEPContextByContName(stateDriver core.StateDriver, contName stri epCtxs []crtclient.ContainerEPContext, err error) { var epCtx *crtclient.ContainerEPContext - contName = strings.TrimPrefix(contName, "/") readEp := &drivers.OvsCfgEndpointState{} readEp.StateDriver = stateDriver epCfgs, err := readEp.ReadAll() @@ -373,7 +374,95 @@ func attachContainer(stateDriver core.StateDriver, crt *crt.CRT, contName string return nil } -func handleContainerStart(netPlugin *plugin.NetPlugin, crt *crt.CRT, +// search cfg eps to find epid of a matching container name +func getEpIDByContainerName(netPlugin *plugin.NetPlugin, contName string) ([]string, error) { + + epIDs := []string{} + readEp := &drivers.OvsCfgEndpointState{} + readEp.StateDriver = netPlugin.StateDriver + epCfgs, err := readEp.ReadAll() + if err != nil { + log.Errorf("got err %v when reading all cfg eps \n", err) + return epIDs, err + } + for _, epCfg := range epCfgs { + ep := epCfg.(*drivers.OvsCfgEndpointState) + if ep.ContName == contName { + epIDs = append(epIDs, ep.ID) + } + } + + if len(epIDs) == 0 { + err = errors.New("failed to find epCfg for contName") + log.Errorf("error getting endpoint id from container name '%s': %v \n", contName, err) + } + + return epIDs, err +} + +// search oper eps to find epid of a matching container uuid +func getEpIDByContainerUUID(netPlugin *plugin.NetPlugin, contUUID string) ([]string, error) { + epIDs := []string{} + readOperEp := &drivers.OvsOperEndpointState{} + readOperEp.StateDriver = netPlugin.StateDriver + epOpers, err := readOperEp.ReadAll() + if err != nil { + log.Errorf("error reading all oper eps: %v \n", err) + return epIDs, err + } + for _, epOper := range epOpers { + ep := epOper.(*drivers.OvsOperEndpointState) + if ep.ContUUID == contUUID { + log.Infof("VJ - deleting ep with following info: %v \n", ep) + epIDs = append(epIDs, ep.ID) + } + } + + if len(epIDs) == 0 { + err = errors.New("UUID not found") + log.Errorf("getting endpoint id from uuid %v \n", err) + } + + return epIDs, err +} + +func createContainerEpOper(netPlugin *plugin.NetPlugin, contUUID, contName string) error { + epIDs, err := getEpIDByContainerName(netPlugin, contName) + if err != nil { + log.Debugf("unable to find ep for container %s, error %v\n", contName, err) + return err + } + + for _, epID := range epIDs { + operEp := &drivers.OvsOperEndpointState{} + operEp.StateDriver = netPlugin.StateDriver + err = operEp.Read(epID) + if core.ErrIfKeyExists(err) != nil { + return err + } + + if err == nil { + operEp.ContUUID = contUUID + err = operEp.Write() + if err != nil { + log.Errorf("error updating oper state for ep %s \n", contName) + return err + } + + log.Infof("updating container '%s' with id '%s' \n", contName, contUUID) + } else { + err = netPlugin.CreateEndpoint(epID) + if err != nil { + log.Errorf("Endpoint creation failed. Error: %s", err) + return err + } + log.Infof("Endpoint operation create succeeded") + } + } + return err +} + +func handleContainerStart(netPlugin *plugin.NetPlugin, crt *crt.CRT, opts *cliOpts, contID string) error { // var epContexts []crtclient.ContainerEPContext @@ -382,6 +471,15 @@ func handleContainerStart(netPlugin *plugin.NetPlugin, crt *crt.CRT, log.Errorf("Could not find container name from container id %s \n", contID) return err } + contName = strings.TrimPrefix(contName, "/") + + if opts.forceDeleteEp { + err = createContainerEpOper(netPlugin, contID, contName) + if err != nil { + log.Errorf("error updating container's uuid: %v \n", err) + return err + } + } err = attachContainer(netPlugin.StateDriver, crt, contName) if err != nil { @@ -391,7 +489,7 @@ func handleContainerStart(netPlugin *plugin.NetPlugin, crt *crt.CRT, return err } -func handleContainerStop(netPlugin *plugin.NetPlugin, crt *crt.CRT, +func handleContainerStop(netPlugin *plugin.NetPlugin, crt *crt.CRT, opts *cliOpts, contID string) error { // If CONTIV_DIND_HOST_GOPATH env variable is set we can assume we are in docker in docker testbed // Here we need to set the network namespace of the ports created by netplugin back to NS of the docker host @@ -407,6 +505,23 @@ func handleContainerStop(netPlugin *plugin.NetPlugin, crt *crt.CRT, } return err } + + if opts.forceDeleteEp { + log.Infof("deleting operEp for container with uuid %s \n", contID) + epIDs, err := getEpIDByContainerUUID(netPlugin, contID) + if err != nil { + log.Errorf("error obtaining container's epid for uuid %s: %v \n", contID, err) + return err + } + for _, epID := range epIDs { + err = netPlugin.DeleteEndpoint(epID) + if err != nil { + log.Errorf("error deleting an endpoint upon container stop: %v \n", err) + return err + } + } + } + return nil } @@ -424,20 +539,25 @@ func handleDockerEvents(event *dockerclient.Event, retErr chan error, log.Errorf("error decoding netplugin in handleDocker \n") } + opts, ok := args[2].(*cliOpts) + if !ok { + log.Errorf("error decoding global opts in Docker handler \n") + } + log.Infof("Received event: %#v, for netPlugin %v \n", *event, netPlugin) // XXX: with plugin (in a lib) this code will handle these events // this cod will need to go away then switch event.Status { case "start": - err = handleContainerStart(netPlugin, crt, event.Id) + err = handleContainerStart(netPlugin, crt, opts, event.Id) if err != nil { log.Errorf("error '%s' handling container %s \n", err, event.Id) } case "die": log.Debugf("received die event for container \n") - err = handleContainerStop(netPlugin, crt, event.Id) + err = handleContainerStop(netPlugin, crt, opts, event.Id) if err != nil { log.Errorf("error '%s' handling container %s \n", err, event.Id) } @@ -567,7 +687,7 @@ func startDockerEventPoll(netPlugin *plugin.NetPlugin, crt *crt.CRT, recvErr cha // start docker client and handle docker events // wait on error chan for problems handling the docker events dockerCRT := crt.ContainerIf.(*docker.Docker) - dockerCRT.Client.StartMonitorEvents(handleDockerEvents, recvErr, netPlugin, crt) + dockerCRT.Client.StartMonitorEvents(handleDockerEvents, recvErr, netPlugin, crt, &opts) } } @@ -622,6 +742,10 @@ func main() { "debug", false, "Show debugging information generated by netplugin") + flagSet.BoolVar(&opts.forceDeleteEp, + "force-delete-ep", + false, + "force ep deletion upon container deletion") flagSet.BoolVar(&opts.jsonLog, "json-log", false, @@ -669,6 +793,14 @@ func main() { log.Infof("host-label not specified, using default (%s)", opts.hostLabel) } + if utils.FetchSysAttrs() != nil { + log.Fatalf("Error reading system attributes \n") + } else { + if utils.SysAttrs.OsType == "centos" { + opts.forceDeleteEp = true + } + } + defConfigStr := fmt.Sprintf(`{ "drivers" : { "network": %q, diff --git a/systemtests/twohosts/sanity_test.go b/systemtests/twohosts/sanity_test.go index 270870307..59e363773 100644 --- a/systemtests/twohosts/sanity_test.go +++ b/systemtests/twohosts/sanity_test.go @@ -800,7 +800,7 @@ func TestTwoHostsMultiVxlanPingFailureStatefulStart_sanity(t *testing.T) { }() } -func TestTwoHostsVxlanDeltaConfig_sanity_sanity(t *testing.T) { +func TestTwoHostsVxlanDeltaConfig_sanity(t *testing.T) { defer func() { utils.ConfigCleanupCommon(t, testbed.GetNodes()) utils.StopOnError(t.Failed()) diff --git a/utils/sysutils.go b/utils/sysutils.go new file mode 100644 index 000000000..414187d0d --- /dev/null +++ b/utils/sysutils.go @@ -0,0 +1,64 @@ +/*** +Copyright 2014 Cisco Systems Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "io/ioutil" + "strings" + + log "github.com/Sirupsen/logrus" +) + +// Implement utilities to fetch System specific information and capabilities. +// These capabilities could be used natively by netplugin or exported. +// In most cases the os capabilities do not change over the course of uptime +// of the system, however if it does, a registry mechanism should be used +// to notify the interested threads + +// SystemAttributes enlist the system specific attributes and are read upon +// the system start +type SystemAttributes struct { + OsType string + TotalRAM int + TotalDiskGB int + TotalNetBw int +} + +// Exported system attributes +var SysAttrs SystemAttributes + +// FetchSysAttrs would read the system attributes and store them in the +// exported vars for the plugin to use; some of the attributes may need OS +// spefici methods to fetch, thus the first attribute to fetch is the OS type +func FetchSysAttrs() error { + output, err := ioutil.ReadFile("/etc/os-release") + if err != nil { + log.Errorf("Error reading the /etc/os-release Error: %s Output: \n%s\n", err, output) + return err + } + + strOutput := string(output) + if strings.Contains(strOutput, "CentOS") { + SysAttrs.OsType = "centos" + } else if strings.Contains(strOutput, "Ubuntu") { + SysAttrs.OsType = "ubuntu" + } else { + SysAttrs.OsType = "unsupported" + } + + // fetch the system memory, disk, and other attributes + return err +} diff --git a/utils/sysutils_test.go b/utils/sysutils_test.go new file mode 100644 index 000000000..b2149dce1 --- /dev/null +++ b/utils/sysutils_test.go @@ -0,0 +1,36 @@ +/*** +Copyright 2014 Cisco Systems Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "testing" +) + +// InitSysAttrs tests fetching the system parameters +func InitSysattrs(t *testing.T) { + err := FetchSysAttrs() + if err != nil { + t.Fatalf("failed to fetch system attributes, err %s \n", err) + } +} + +// GetOsType tests getting os type from the system +func GetOsType(t *testing.T) { + + if (SysAttrs.OsType == "unspecified") || (SysAttrs.OsType == "") { + t.Fatalf("failed to get the proper os type \n") + } +}