Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

netmaster cli refactor #1088

Merged
merged 5 commits into from
Dec 5, 2017
Merged

netmaster cli refactor #1088

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d190d61
netmaster cli refactor
Nov 22, 2017
7cd63b4
Address review comments
Nov 30, 2017
d851259
Update README message
tiewei Dec 1, 2017
23636b2
Wait up to 5 mins on getting pod name of running pods
tiewei Dec 1, 2017
eef09ad
Polish CLI helpers
tiewei Dec 5, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions Vagrantfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ https_proxy = ENV['HTTPS_PROXY'] || ENV['https_proxy'] || ''
build_version = ENV['BUILD_VERSION'] || ''
cluster_ip_nodes = ''
v2plugin_name = ENV['CONTIV_V2PLUGIN_NAME'] || 'contiv/v2netplugin:0.1'
cluster_store = ENV['CONTIV_CLUSTER_STORE'] || 'etcd://localhost:2379'
cluster_store_driver = ENV['CONTIV_CLUSTER_STORE_DRIVER'] || 'etcd'
cluster_store_url = ENV['CONTIV_CLUSTER_STORE_URL'] || 'http://localhost:2379'
nightly_release = ENV['NIGHTLY_RELEASE'] || ''
node_os = ENV['CONTIV_NODE_OS'] || 'centos'
base_ip = ENV['CONTIV_IP_PREFIX'] || '192.168.2.'
Expand All @@ -46,7 +47,8 @@ export https_proxy='#{https_proxy}'
export NIGHTLY_RELEASE=#{nightly_release}
export no_proxy=%{cluster_ip_nodes},127.0.0.1,localhost,netmaster
export CLUSTER_NODE_IPS=%{cluster_ip_nodes}
export CONTIV_CLUSTER_STORE=#{cluster_store}
export CONTIV_CLUSTER_STORE_DRIVER=#{cluster_store_driver}
export CONTIV_CLUSTER_STORE_URL=#{cluster_store_url}
export CONTIV_V2PLUGIN_NAME=#{v2plugin_name}
export CONTIV_DOCKER_SWARM=#{docker_swarm}
export BUILD_VERSION=#{build_version}
Expand Down Expand Up @@ -119,7 +121,7 @@ fi
if [[ #{docker_swarm} == "swarm_mode" ]]; then
perl -i -lpe 's!^ExecStart(.+)$!ExecStart$1 !' /lib/systemd/system/docker.service
else
if [[ "$CONTIV_CLUSTER_STORE" == *"consul:"* ]]
if [[ "$CONTIV_CLUSTER_STORE_DRIVER" == "consul" ]]
then
perl -i -lpe 's!^ExecStart(.+)$!ExecStart$1 -H tcp://0.0.0.0:2375 -H unix:///var/run/docker.sock --cluster-store=consul://localhost:8500!' /lib/systemd/system/docker.service
else
Expand Down Expand Up @@ -225,7 +227,7 @@ module VagrantPlugins

def self.up_hook(arg)
unless File.exist?(STATEFILE) # prevent it from writing more than once.
f = File.open(STATEFILE, "w")
f = File.open(STATEFILE, "w")
ENV.each do |x,y|
f.puts "%s=%s" % [x,y]
end
Expand Down Expand Up @@ -256,7 +258,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
config.vm.box_check_update = false
if Vagrant.has_plugin?("vagrant-vbguest")
config.vbguest.auto_update = false
end
end
if node_os == "ubuntu" then
config.vm.box = "contiv/ubuntu1604-netplugin"
config.vm.box_version = "0.7.0"
Expand Down
9 changes: 9 additions & 0 deletions core/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
package core

// Plugin modes
const (
Docker = "docker"
Kubernetes = "kubernetes"
SwarmMode = "swarm-mode"
Test = "test"
)
2 changes: 1 addition & 1 deletion core/error_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ func TestErrorStackTrace(t *testing.T) {
}

if e.Error() == "an error\n" {
t.Fatal("Error message did not yield stack trace with CONTIV_TRACE set")
t.Fatal("Error message did not yield stack trace")
}

lines := strings.Split(e.Error(), "\n")
Expand Down
84 changes: 51 additions & 33 deletions install/HowtoSetupContiv.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,33 +14,52 @@ Download tarball for the version you want to try.
We recommend starting netmaster on master node in your master slave architecture. You can start netmaster in HA mode and appropriate netmaster node will handle all the queries from end user.

```
netmaster --help
Usage: netmaster [OPTION]...
-cluster-mode string
{docker, kubernetes} (default "docker")
-cluster-store string
Etcd or Consul cluster store url. (default "etcd://127.0.0.1:2379") << we support consul as well, you need to change it here.
-debug
Turn on debugging information
-help
prints this message
-listen-url string
Url to listen http requests on (default ":9999")
-version
prints current version
netplugin --help
NAME:
netplugin - Contiv netplugin service

USAGE:
netplugin [global options] command [command options] [arguments...]

VERSION:

Version: <netplugin-version>
GitCommit: <netplugin-commit-sha>
BuildTime: <netplugin-build-time>


COMMANDS:
help, h Shows a list of commands or help for one command

GLOBAL OPTIONS:
--consul-endpoints value, --consul value a comma-delimited list of netplugin consul endpoints [$CONTIV_NETPLUGIN_CONSUL_ENDPOINTS]
--ctrl-ip value set netplugin control ip for control plane communication (default: <host-ip-from-local-resolver>) [$CONTIV_NETPLUGIN_CONTROL_IP]
--etcd-endpoints value, --etcd value a comma-delimited list of netplugin etcd endpoints [$CONTIV_NETPLUGIN_ETCD_ENDPOINTS]
--fwdmode value, --forward-mode value set netplugin forwarding network mode, options: [bridge, routing] [$CONTIV_NETPLUGIN_FORWARD_MODE]
--host value, --host-label value set netplugin host to identify itself (default: <host-name-reported-by-the-kernel>) [$CONTIV_NETPLUGIN_HOST]
--log-level value set netplugin log level, options: [DEBUG, INFO, WARN, ERROR] (default: "INFO") [$CONTIV_NETPLUGIN_LOG_LEVEL]
--mode value, --plugin-mode value, --cluster-mode value set netplugin mode, options: [docker, kubernetes, swarm-mode] [$CONTIV_NETPLUGIN_MODE]
--netmode value, --network-mode value set netplugin network mode, options: [vlan, vxlan] [$CONTIV_NETPLUGIN_NET_MODE]
--syslog-url value set netplugin syslog url in format protocol://ip:port (default: "udp://127.0.0.1:514") [$CONTIV_NETPLUGIN_SYSLOG_URL]
--use-json-log, --json-log set netplugin log format to json if this flag is provided [$CONTIV_NETPLUGIN_USE_JSON_LOG]
--use-syslog, --syslog set netplugin send log to syslog if this flag is provided [$CONTIV_NETPLUGIN_USE_SYSLOG]
--vlan-uplinks value, --vlan-if value a comma-delimited list of netplugin uplink interfaces [$CONTIV_NETPLUGIN_VLAN_UPLINKS]
--vtep-ip value set netplugin vtep ip for vxlan communication (default: <host-ip-from-local-resolver>) [$CONTIV_NETPLUGIN_VTEP_IP]
--vxlan-port value set netplugin VXLAN port (default: 4789) [$CONTIV_NETPLUGIN_VXLAN_PORT]
--help, -h show help
--version, -v print the version
```

## How to start netplugin ?

You start netplugin on each node in your cluster.

```
netplugin --help
NAME:
netplugin - Contiv netplugin service
netmaster - Contiv netmaster service

USAGE:
netplugin [global options] command [command options] [arguments...]
netmaster [global options] command [command options] [arguments...]

VERSION:

Expand All @@ -53,22 +72,21 @@ COMMANDS:
help, h Shows a list of commands or help for one command

GLOBAL OPTIONS:
--consul-endpoints value, --consul value set netplugin consul endpoints [$CONTIV_NETPLUGIN_CONSUL_ENDPOINTS]
--ctrl-ip value set netplugin control ip for control plane communication (default: <current-ip-address>) [$CONTIV_NETPLUGIN_CONTROL_IP]
--etcd-endpoints value, --etcd value set netplugin etcd endpoints [$CONTIV_NETPLUGIN_ETCD_ENDPOINTS]
--fwdmode value, --forward-mode value set netplugin forwarding network mode, options: [bridge, routing] [$CONTIV_NETPLUGIN_NET_MODE]
--host value, --host-label value set netplugin host to identify itself (default: "8994bb27e5fb") [$CONTIV_NETPLUGIN_HOST]
--log-level value set netplugin log level, options: [DEBUG, INFO, WARN, ERROR] (default: "INFO") [$CONTIV_NETPLUGIN_LOG_LEVEL]
--mode value, --plugin-mode value set netplugin mode, options: [docker, kubernetes, swarm-mode] [$CONTIV_NETPLUGIN_MODE]
--netmode value, --network-mode value set netplugin network mode, options: [vlan, vxlan] [$CONTIV_NETPLUGIN_NET_MODE]
--syslog-url value set netplugin syslog url in format protocol://ip:port (default: "udp://127.0.0.1:514") [$CONTIV_NETPLUGIN_SYSLOG_URL]
--use-json-log, --json-log set netplugin log format to json [$CONTIV_NETPLUGIN_USE_JSON_LOG]
--use-syslog, --syslog set netplugin send log to syslog or not [$CONTIV_NETPLUGIN_USE_SYSLOG]
--vlan-uplinks value, --vlan-if value set netplugin uplink interfaces [$CONTIV_NETPLUGIN_VLAN_UPLINKS]
--vtep-ip value set netplugin vtep ip for vxlan communication (default: <current-ip-address>) [$CONTIV_NETPLUGIN_VTEP_IP]
--vxlan-port value set netplugin VXLAN port (default: 4789) [$CONTIV_NETPLUGIN_VXLAN_PORT]
--help, -h show help
--version, -v print the version
--consul-endpoints value, --consul value a comma-delimited list of netmaster consul endpoints [$CONTIV_NETMASTER_CONSUL_ENDPOINTS]
--etcd-endpoints value, --etcd value a comma-delimited list of netmaster etcd endpoints [$CONTIV_NETMASTER_ETCD_ENDPOINTS]
--external-address value, --listen-url value set netmaster external address to listen on, used for general API service (default: "0.0.0.0:9999") [$CONTIV_NETMASTER_EXTERNAL_ADDRESS]
--fwdmode value, --forward-mode value set netmaster forwarding network mode, options: [bridge, routing] [$CONTIV_NETMASTER_FORWARD_MODE]
--infra value, --infra-type value set netmaster infra type, options [aci, default] (default: "default") [$CONTIV_NETMASTER_INFRA]
--internal-address value, --control-url value set netmaster internal address to listen on, used for RPC and leader election (default: <host-ip-from-local-resolver>:<port-of-external-address>) [$CONTIV_NETMASTER_INTERNAL_ADDRESS]
--log-level value set netmaster log level, options: [DEBUG, INFO, WARN, ERROR] (default: "INFO") [$CONTIV_NETMASTER_LOG_LEVEL]
--mode value, --plugin-mode value, --cluster-mode value set netmaster mode, options: [docker, kubernetes, swarm-mode] [$CONTIV_NETMASTER_MODE]
--name value, --plugin-name value set netmaster plugin name for docker v2 plugin (default: "netplugin") [$CONTIV_NETMASTER_PLUGIN_NAME]
--netmode value, --network-mode value set netmaster network mode, options: [vlan, vxlan] [$CONTIV_NETMASTER_NET_MODE]
--syslog-url value set netmaster syslog url in format protocol://ip:port (default: "udp://127.0.0.1:514") [$CONTIV_NETMASTER_SYSLOG_URL]
--use-json-log, --json-log set netmaster log format to json if this flag is provided [$CONTIV_NETMASTER_USE_JSON_LOG]
--use-syslog, --syslog set netmaster send log to syslog if this flag is provided [$CONTIV_NETMASTER_USE_SYSLOG]
--help, -h show help
--version, -v print the version
```

for example we can start netplugin in following manner :
Expand Down
7 changes: 4 additions & 3 deletions mgmtfn/dockplugin/netDriver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (

log "github.com/Sirupsen/logrus"
"github.com/contiv/netplugin/contivmodel/client"
"github.com/contiv/netplugin/core"
"github.com/contiv/netplugin/netmaster/docknet"
"github.com/contiv/netplugin/netmaster/intent"
"github.com/contiv/netplugin/netmaster/master"
Expand Down Expand Up @@ -399,7 +400,7 @@ func allocateNetwork(w http.ResponseWriter, r *http.Request) {
// node where a container in the network is instantiated.
// We process only allocateNetwork in this case.
//
if pluginMode == master.SwarmMode {
if pluginMode == core.SwarmMode {
tag := ""
log.Infof("Options: %+v", anreq.Options)
if _, tagOk := anreq.Options["contiv-tag"]; tagOk {
Expand Down Expand Up @@ -441,7 +442,7 @@ func freeNetwork(w http.ResponseWriter, r *http.Request) {
return
}

if pluginMode == master.SwarmMode {
if pluginMode == core.SwarmMode {
err = deleteNetworkHelper(req.NetworkID)
if err != nil {
httpError(w, "Could not delete network", err)
Expand Down Expand Up @@ -536,7 +537,7 @@ func GetDockerNetworkName(nwID string) (string, string, string, error) {
if err == nil {
return dnetOper.TenantName, dnetOper.NetworkName, dnetOper.ServiceName, nil
}
if pluginMode == master.SwarmMode {
if pluginMode == core.SwarmMode {
log.Errorf("Unable to find docknet info in objstore")
return "", "", "", err
}
Expand Down
61 changes: 38 additions & 23 deletions netmaster/daemon/daemon.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import (
"github.com/contiv/netplugin/netmaster/resources"
"github.com/contiv/netplugin/objdb"
"github.com/contiv/netplugin/utils"
"github.com/contiv/netplugin/utils/netutils"
"github.com/contiv/ofnet"
"github.com/gorilla/mux"

Expand All @@ -46,10 +47,14 @@ const leaderLockTTL = 30
// MasterDaemon runs the daemon FSM
type MasterDaemon struct {
// Public state
ListenURL string // URL where netmaster listens for ext requests
ControlURL string // URL where netmaster listens for ctrl pkts
ClusterStore string // state store URL
ClusterMode string // cluster scheduler used docker/kubernetes/mesos etc
ListenURL string // URL where netmaster listens for ext requests
ControlURL string // URL where netmaster listens for ctrl pkts
ClusterStoreDriver string // state store driver name
ClusterStoreURL string // state store endpoint
ClusterMode string // cluster scheduler used docker/kubernetes/mesos etc
NetworkMode string // network mode (vlan or vxlan)
NetForwardMode string // forwarding mode (bridge or routing)
NetInfraType string // infra type (aci or default)

// Private state
currState string // Current state of the daemon
Expand All @@ -70,13 +75,13 @@ func (d *MasterDaemon) Init() {
// set cluster mode
err := master.SetClusterMode(d.ClusterMode)
if err != nil {
log.Fatalf("Failed to set cluster-mode. Error: %s", err)
log.Fatalf("Failed to set cluster-mode %q. Error: %s", d.ClusterMode, err)
}

// initialize state driver
d.stateDriver, err = initStateDriver(d.ClusterStore)
d.stateDriver, err = utils.NewStateDriver(d.ClusterStoreDriver, &core.InstanceInfo{DbURL: d.ClusterStoreURL})
if err != nil {
log.Fatalf("Failed to init state-store. Error: %s", err)
log.Fatalf("Failed to init state-store: driver %q, URLs %q. Error: %s", d.ClusterStoreDriver, d.ClusterStoreURL, err)
}

// Initialize resource manager
Expand All @@ -86,9 +91,9 @@ func (d *MasterDaemon) Init() {
}

// Create an objdb client
d.objdbClient, err = objdb.NewClient(d.ClusterStore)
d.objdbClient, err = objdb.InitClient(d.ClusterStoreDriver, []string{d.ClusterStoreURL})
if err != nil {
log.Fatalf("Error connecting to state store: %v. Err: %v", d.ClusterStore, err)
log.Fatalf("Error connecting to state store: driver %q, URLs %q. Err: %v", d.ClusterStoreDriver, d.ClusterStoreURL, err)
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i think we should check for ':' in ControlURL and ListenURL since those are parsed later in startListeners

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it actually got checked in the CLI part

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ah I see it now

}

Expand Down Expand Up @@ -404,7 +409,11 @@ func (d *MasterDaemon) runLeader() {
router := mux.NewRouter()

// Create a new api controller
d.apiController = objApi.NewAPIController(router, d.objdbClient, d.ClusterStore)
apiConfig := &objApi.APIControllerConfig{
NetForwardMode: d.NetForwardMode,
NetInfraType: d.NetInfraType,
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

AFACT this isn't supposed to be modifiable, why not have NewAPIController accept APIControllerConfig by value?

Copy link
Contributor Author

@tiewei tiewei Dec 5, 2017 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this object only got used here (and integ testings), I used object was because of intending to add more parameters from configs .

have NewAPIController accept APIControllerConfig by value

this is troublesome in golang, especially when having the parameters in different type

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i think the struct is fine, just dunno why you need a pointer to it

d.apiController = objApi.NewAPIController(router, d.objdbClient, apiConfig)

//Restore state from clusterStore
d.restoreCache()
Expand Down Expand Up @@ -447,6 +456,7 @@ func (d *MasterDaemon) startListeners(router *mux.Router, stopChan chan bool) {
server := &http.Server{Handler: router}
server.SetKeepAlivesEnabled(false)

// bind on external address
listener, err := net.Listen("tcp", d.ListenURL)
if nil != err {
log.Fatalln(err)
Expand All @@ -457,20 +467,25 @@ func (d *MasterDaemon) startListeners(router *mux.Router, stopChan chan bool) {

go server.Serve(listener)

listenURL := strings.Split(d.ListenURL, ":")
controlURL := strings.Split(d.ControlURL, ":")
if d.ControlURL != d.ListenURL {
externalAddr := strings.Split(d.ListenURL, ":")
internalAddr := strings.Split(d.ControlURL, ":")
if externalAddr[0] == "0.0.0.0" && externalAddr[1] == internalAddr[1] {
// ignore internal bind if external and internal are on the same port and external bind on 0.0.0.0
log.Infof("Ignore creating API listener on %q because %q covers it", d.ControlURL, d.ListenURL)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

seems we could have external and internal on same IP as well?
externalAddr[1] == internalAddr[1] && (externalAddr[0] == "0.0.0.0" || externalAddr[0] == internalAddr[0])

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if they are on the same ip but different port will go to else where create a listener on different port with same ip. Only got ignored when they have overlapping

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

oh nvm, I wasn't looking at the outer if restriction

} else {
// it should fail-fast if ControlURL and ListenURL have other overlapping
ctrlListener, err := net.Listen("tcp", d.ControlURL)
if nil != err {
log.Fatalln(err)
}
log.Infof("Netmaster listening on %s for control packets", d.ControlURL)
ctrlListener = utils.ListenWrapper(ctrlListener)
defer ctrlListener.Close()

if (strings.Compare(listenURL[1], controlURL[1]) != 0) || (len(listenURL[0]) != 0 && strings.Compare(listenURL[0], "0.0.0.0") != 0 && strings.Compare(listenURL[0], controlURL[0]) != 0) {
ctrlListener, err := net.Listen("tcp", d.ControlURL)
if nil != err {
log.Fatalln(err)
// start server
go server.Serve(ctrlListener)
}
log.Infof("Netmaster listening on %s for control packets", d.ControlURL)
ctrlListener = utils.ListenWrapper(ctrlListener)
defer ctrlListener.Close()

// start server
go server.Serve(ctrlListener)
}

// Wait till we are asked to stop
Expand Down Expand Up @@ -584,7 +599,7 @@ func (d *MasterDaemon) getMasterInfo() (map[string]interface{}, error) {
info := make(map[string]interface{})

// get local ip
localIP, err := GetLocalAddr()
localIP, err := netutils.GetDefaultAddr()
if err != nil {
return nil, errors.New("error getting local IP address")
}
Expand Down
40 changes: 1 addition & 39 deletions netmaster/daemon/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (
"net/http"
"net/http/httputil"
"net/url"
"strings"

"github.com/contiv/netplugin/core"
"github.com/contiv/netplugin/netmaster/mastercfg"
Expand All @@ -48,48 +47,11 @@ func getVersion(w http.ResponseWriter, r *http.Request) {
return
}

// initStateDriver creates a state driver based on the cluster store URL
func initStateDriver(clusterStore string) (core.StateDriver, error) {
// parse the state store URL
parts := strings.Split(clusterStore, "://")
if len(parts) < 2 {
return nil, core.Errorf("Invalid state-store URL %q", clusterStore)
}
stateStore := parts[0]

// Make sure we support the statestore type
switch stateStore {
case utils.EtcdNameStr:
case utils.ConsulNameStr:
default:
return nil, core.Errorf("Unsupported state-store %q", stateStore)
}

// Setup instance info
instInfo := core.InstanceInfo{
DbURL: clusterStore,
}

return utils.NewStateDriver(stateStore, &instInfo)
}

// GetLocalAddr gets local address to be used
func GetLocalAddr() (string, error) {
// get the ip address by local hostname
localIP, err := netutils.GetMyAddr()
if err == nil && netutils.IsAddrLocal(localIP) {
return localIP, nil
}

// Return first available address if we could not find by hostname
return netutils.GetFirstLocalAddr()
}

// slaveProxyHandler redirects to current master
func slaveProxyHandler(w http.ResponseWriter, r *http.Request) {
log.Infof("proxy handler for %q ", r.URL.Path)

localIP, err := GetLocalAddr()
localIP, err := netutils.GetDefaultAddr()
if err != nil {
log.Fatalf("Error getting local IP address. Err: %v", err)
}
Expand Down
Loading