Skip to content

Commit d05de09

Browse files
unclejackunclejack
authored
Merge pull request #1043 from tiewei/CNTV-157
Init global settings from object store
2 parents 06fa667 + 6db58c4 commit d05de09

File tree

6 files changed

+199
-138
lines changed

6 files changed

+199
-138
lines changed

‎netmaster/objApi/apiController.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ func NewAPIController(router *mux.Router, objdbClient objdb.API, storeURL string
116116
NetworkInfraType: "default",
117117
Vlans: "1-4094",
118118
Vxlans: "1-10000",
119-
FwdMode: "bridge",
119+
FwdMode: "", // set empty fwd mode by default
120120
ArpMode: "proxy",
121121
PvtSubnet: defHostPvtNet,
122122
})

‎netplugin/plugin/netplugin.go

+43-17
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,14 @@ limitations under the License.
1616
package plugin
1717

1818
import (
19+
"fmt"
1920
"github.com/Sirupsen/logrus"
2021
"github.com/contiv/netplugin/core"
2122
"github.com/contiv/netplugin/netmaster/mastercfg"
2223
"github.com/contiv/netplugin/utils"
2324
"github.com/contiv/netplugin/utils/netutils"
2425
"sync"
26+
"time"
2527
)
2628

2729
// implements the generic Plugin interface
@@ -50,8 +52,6 @@ type NetPlugin struct {
5052
PluginConfig Config
5153
}
5254

53-
const defaultPvtSubnet = 0xac130000
54-
5555
// Init initializes the NetPlugin instance via the configuration string passed.
5656
func (p *NetPlugin) Init(pluginConfig Config) error {
5757
var err error
@@ -75,8 +75,10 @@ func (p *NetPlugin) Init(pluginConfig Config) error {
7575

7676
// set state driver in instance info
7777
pluginConfig.Instance.StateDriver = p.StateDriver
78-
79-
InitGlobalSettings(p.StateDriver, &pluginConfig.Instance)
78+
err = InitGlobalSettings(p.StateDriver, &pluginConfig.Instance)
79+
if err != nil {
80+
return err
81+
}
8082

8183
// initialize network driver
8284
p.NetworkDriver, err = utils.NewNetworkDriver(pluginConfig.Drivers.Network, &pluginConfig.Instance)
@@ -303,27 +305,51 @@ func (p *NetPlugin) Reinit(cfg Config) {
303305
}
304306

305307
//InitGlobalSettings initializes cluster-wide settings (e.g. fwd-mode)
306-
func InitGlobalSettings(stateDriver core.StateDriver, inst *core.InstanceInfo) {
308+
func InitGlobalSettings(stateDriver core.StateDriver, inst *core.InstanceInfo) error {
309+
310+
/*
311+
Query global settings from state store
312+
1. if forward mode or private net is empty, retry after 1 seccond
313+
2. if forward mode doesn't match, return error
314+
3. if stored private net is wrong, return error
315+
*/
307316

308317
gCfg := mastercfg.GlobConfig{}
309318
gCfg.StateDriver = stateDriver
310-
err := gCfg.Read("")
311-
if err != nil {
312-
logrus.Errorf("Error reading forwarding mode from cluster store")
313-
inst.FwdMode = "bridge"
314-
inst.HostPvtNW = defaultPvtSubnet
315-
return
316-
}
317319

320+
// wait until able to get fwd mode and private subnet
321+
for {
322+
if err := gCfg.Read(""); err != nil {
323+
logrus.Errorf("Error reading global settings from cluster store, error: %v", err.Error())
324+
} else {
325+
if gCfg.FwdMode == "" || gCfg.PvtSubnet == "" {
326+
if gCfg.FwdMode == "" {
327+
logrus.Warnf("No forwarding mode found from cluster store")
328+
}
329+
if gCfg.PvtSubnet == "" {
330+
logrus.Warnf("No private subnet found from cluster store")
331+
}
332+
333+
} else {
334+
logrus.Infof("Got global forwarding mode: %v", gCfg.FwdMode)
335+
logrus.Infof("Got global private subnet: %v", gCfg.PvtSubnet)
336+
break
337+
}
338+
}
339+
logrus.Warnf("Sleep 1 second and retry pulling global settings")
340+
time.Sleep(1 * time.Second)
341+
}
318342
inst.FwdMode = gCfg.FwdMode
343+
logrus.Infof("Using forwarding mode: %v", inst.FwdMode)
319344
net, err := netutils.CIDRToMask(gCfg.PvtSubnet)
320345
if err != nil {
321-
logrus.Errorf("%s %v, will use default", gCfg.PvtSubnet, err)
322-
inst.HostPvtNW = defaultPvtSubnet
323-
} else {
324-
inst.HostPvtNW = net
325-
logrus.Infof("HostPvtNW: %v", net)
346+
err := fmt.Errorf("Error convert private subnet %v from CIDR to mask, error %v", gCfg.PvtSubnet, err.Error())
347+
logrus.Errorf(err.Error())
348+
return err
326349
}
350+
inst.HostPvtNW = net
351+
logrus.Infof("Using host private subnet: %v", gCfg.PvtSubnet)
352+
return nil
327353
}
328354

329355
//AddSvcSpec adds k8 service spec

‎netplugin/plugin/netplugin_test.go

+119-70
Original file line numberDiff line numberDiff line change
@@ -18,25 +18,52 @@ package plugin
1818
import (
1919
"encoding/json"
2020
"fmt"
21-
"testing"
22-
21+
"github.com/contiv/netplugin/core"
22+
"github.com/contiv/netplugin/netmaster/mastercfg"
2323
"github.com/contiv/netplugin/state"
24+
"github.com/contiv/netplugin/utils"
25+
"testing"
2426
)
2527

2628
var fakeStateDriver *state.FakeStateDriver
2729

30+
func initFakeStateDriver(t *testing.T) {
31+
// init fake state driver
32+
instInfo := core.InstanceInfo{}
33+
d, err := utils.NewStateDriver("fakedriver", &instInfo)
34+
if err != nil {
35+
t.Fatalf("failed to init statedriver. Error: %s", err)
36+
}
37+
38+
fakeStateDriver = d.(*state.FakeStateDriver)
39+
}
40+
41+
func deinitFakeStateDriver() {
42+
// release fake state driver
43+
utils.ReleaseStateDriver()
44+
}
45+
2846
func TestNetPluginInit(t *testing.T) {
47+
// Testing init NetPlugin
48+
initFakeStateDriver(t)
49+
defer deinitFakeStateDriver()
50+
gCfg := mastercfg.GlobConfig{
51+
FwdMode: "bridge",
52+
PvtSubnet: "172.19.0.0/16"}
53+
gCfg.StateDriver = fakeStateDriver
54+
gCfg.Write()
55+
2956
configStr := `{
30-
"drivers" : {
31-
"network": "ovs",
32-
"endpoint": "ovs",
33-
"state": "fakedriver"
34-
},
35-
"plugin-instance": {
36-
"host-label": "testHost",
37-
"fwd-mode":"bridge"
38-
}
39-
}`
57+
"drivers" : {
58+
"network": "ovs",
59+
"endpoint": "ovs",
60+
"state": "fakedriver"
61+
},
62+
"plugin-instance": {
63+
"host-label": "testHost",
64+
"fwd-mode":"bridge"
65+
}
66+
}`
4067

4168
// Parse the config
4269
pluginConfig := Config{}
@@ -56,6 +83,7 @@ func TestNetPluginInit(t *testing.T) {
5683
}
5784

5885
func TestNetPluginInitInvalidConfigEmptyString(t *testing.T) {
86+
// Test NetPlugin init failure when no config provided
5987
pluginConfig := Config{}
6088

6189
plugin := NetPlugin{}
@@ -66,13 +94,14 @@ func TestNetPluginInitInvalidConfigEmptyString(t *testing.T) {
6694
}
6795

6896
func TestNetPluginInitInvalidConfigMissingInstance(t *testing.T) {
97+
// Test NetPlugin init failure when missing instance config
6998
configStr := `{
70-
"drivers" : {
71-
"network": "ovs",
72-
"endpoint": "ovs",
73-
"state": "fakedriver"
74-
}
75-
}`
99+
"drivers" : {
100+
"network": "ovs",
101+
"endpoint": "ovs",
102+
"state": "fakedriver"
103+
}
104+
}`
76105

77106
// Parse the config
78107
pluginConfig := Config{}
@@ -86,18 +115,19 @@ func TestNetPluginInitInvalidConfigMissingInstance(t *testing.T) {
86115
}
87116

88117
func TestNetPluginInitInvalidConfigEmptyHostLabel(t *testing.T) {
118+
// Test NetPlugin init failure when empty HostLabel provided
89119
configStr := `{
90-
"drivers" : {
91-
"network": "ovs",
92-
"endpoint": "ovs",
93-
"state": "fakedriver"
94-
},
95-
"plugin-instance": {
96-
"host-label": "",
97-
"fwd-mode":"bridge",
98-
"db-url": "etcd://127.0.0.1:4001"
99-
}
100-
}`
120+
"drivers" : {
121+
"network": "ovs",
122+
"endpoint": "ovs",
123+
"state": "fakedriver"
124+
},
125+
"plugin-instance": {
126+
"host-label": "",
127+
"fwd-mode":"bridge",
128+
"db-url": "etcd://127.0.0.1:4001"
129+
}
130+
}`
101131

102132
// Parse the config
103133
pluginConfig := Config{}
@@ -114,17 +144,18 @@ func TestNetPluginInitInvalidConfigEmptyHostLabel(t *testing.T) {
114144
}
115145

116146
func TestNetPluginInitInvalidConfigMissingStateDriverName(t *testing.T) {
147+
// Test NetPlugin init failure when missing state driver name
117148
configStr := `{
118-
"drivers" : {
119-
"network": "ovs",
120-
"endpoint": "ovs"
121-
},
122-
"plugin-instance": {
123-
"host-label": "testHost",
124-
"fwd-mode":"bridge",
125-
"db-url": "etcd://127.0.0.1:4001"
126-
}
127-
}`
149+
"drivers" : {
150+
"network": "ovs",
151+
"endpoint": "ovs"
152+
},
153+
"plugin-instance": {
154+
"host-label": "testHost",
155+
"fwd-mode":"bridge",
156+
"db-url": "etcd://127.0.0.1:4001"
157+
}
158+
}`
128159

129160
// Parse the config
130161
pluginConfig := Config{}
@@ -141,17 +172,18 @@ func TestNetPluginInitInvalidConfigMissingStateDriverName(t *testing.T) {
141172
}
142173

143174
func TestNetPluginInitInvalidConfigMissingStateDriverURL(t *testing.T) {
175+
// Test NetPlugin init failure when missing state driver url
144176
configStr := `{
145-
"drivers" : {
146-
"network": "ovs",
147-
"endpoint": "ovs",
148-
"state": "etcd"
149-
},
150-
"plugin-instance": {
151-
"host-label": "testHost",
152-
"fwd-mode":"bridge"
153-
}
154-
}`
177+
"drivers" : {
178+
"network": "ovs",
179+
"endpoint": "ovs",
180+
"state": "etcd"
181+
},
182+
"plugin-instance": {
183+
"host-label": "testHost",
184+
"fwd-mode":"bridge"
185+
}
186+
}`
155187

156188
// Parse the config
157189
pluginConfig := Config{}
@@ -169,18 +201,26 @@ func TestNetPluginInitInvalidConfigMissingStateDriverURL(t *testing.T) {
169201
}
170202

171203
func TestNetPluginInitInvalidConfigMissingNetworkDriverName(t *testing.T) {
204+
// Test NetPlugin init failure when missing network driver name
205+
initFakeStateDriver(t)
206+
defer deinitFakeStateDriver()
207+
gCfg := mastercfg.GlobConfig{
208+
FwdMode: "bridge",
209+
PvtSubnet: "172.19.0.0/16"}
210+
gCfg.StateDriver = fakeStateDriver
211+
gCfg.Write()
172212
configStr := `{
173-
"drivers" : {
174-
"endpoint": "ovs",
175-
"state": "fakedriver",
176-
"container": "docker"
177-
},
178-
"plugin-instance": {
179-
"host-label": "testHost",
180-
"fwd-mode":"bridge",
181-
"db-url": "etcd://127.0.0.1:4001"
182-
}
183-
}`
213+
"drivers" : {
214+
"endpoint": "ovs",
215+
"state": "fakedriver",
216+
"container": "docker"
217+
},
218+
"plugin-instance": {
219+
"host-label": "testHost",
220+
"fwd-mode":"bridge",
221+
"db-url": "etcd://127.0.0.1:4001"
222+
}
223+
}`
184224

185225
// Parse the config
186226
pluginConfig := Config{}
@@ -196,19 +236,28 @@ func TestNetPluginInitInvalidConfigMissingNetworkDriverName(t *testing.T) {
196236
}
197237
}
198238

199-
func TestNetPluginInitInvalidConfigMissingFwdMode(t *testing.T) {
239+
func TestNetPluginInitInvalidConfigInvalidPrivateSubnet(t *testing.T) {
240+
// Test NetPlugin init failure when private subnet is not valid
241+
initFakeStateDriver(t)
242+
defer deinitFakeStateDriver()
243+
gCfg := mastercfg.GlobConfig{
244+
FwdMode: "routing",
245+
PvtSubnet: "172.19.0.0"}
246+
gCfg.StateDriver = fakeStateDriver
247+
gCfg.Write()
200248
configStr := `{
201-
"drivers" : {
202-
"network": "ovs",
203-
"endpoint": "ovs",
204-
"state": "fakedriver",
205-
"container": "docker",
206-
},
249+
"drivers" : {
250+
"network": "ovs",
251+
"endpoint": "ovs",
252+
"state": "fakedriver",
253+
"container": "docker",
254+
},
207255
"plugin-instance": {
208-
"host-label": "testHost",
209-
"db-url": "etcd://127.0.0.1:4001"
256+
"host-label": "testHost",
257+
"db-url": "etcd://127.0.0.1:4001",
258+
"fwd-mode":"routing",
210259
}
211-
}`
260+
}`
212261

213262
// Parse the config
214263
pluginConfig := Config{}

0 commit comments

Comments
 (0)