why gateway proxy can not distribute the http request to local server? prompt 404

[toranger]

did‘t trigger the grpc call at all

[johanbrandhorst]

I'm gonna need a lot more information to be able to help you. What does your protofile look like? What is the exact curl command you ran? It looks to me from that screenshot that you're just calling the wrong URL. A gRPC-Gateway would not reply with an error like this, (notice the Powered By Jetty:// error).

[toranger]

Pre: I run two local raft master node srv and one local grpc-gateway to distribute http request
such like this rpc function, protofile like this:


Ps: the http request can arrived at gateway node, but also tell me transport is closing. ^

[achew22]

@toranger, could you include not a screenshot, but a copy/paste'd version of the go code you're using to register the grpc gateway handlers and also to start the server?

[toranger]

@achew22,
svr.txt

package server

import (
	"context"
	"cos-config/common"
	"net"
	"strings"
	"sync"
	"time"

	"github.com/coreos/etcd/clientv3"
	"github.com/golang/glog"
	"google.golang.org/grpc"
)

type ConfigMaster struct {
	etcdClient *clientv3.Client
	grpcServer *grpc.Server
	lis        net.Listener
	addr       string
	started    bool
	mu         sync.Mutex
	wg         *sync.WaitGroup
	s          *ConfigSvrMasterService
}


func (m *ConfigMaster) Init(etcdEndpoints, listenEndpoints, etcdUserName, etcdUserPasswd string, wg *sync.WaitGroup) error {
	etcd_eps := strings.Split(etcdEndpoints, ",")
	c, err := clientv3.New(clientv3.Config{
		Endpoints:   etcd_eps,
		DialTimeout: 2 * time.Second,
		Username:    etcdUserName,
		Password:    etcdUserPasswd,
	})

	if err != nil {
		glog.Error("Can not create etcd client Error", err.Error())
		return err
	}
	m.etcdClient = c
	m.s = NewConfigSvrMasterService(c, common.CosClient)

	m.lis, err = net.Listen("tcp", listenEndpoints)
	if err != nil {
		glog.Error("failed to listen ", err.Error())
		return err
	}

	m.grpcServer = grpc.NewServer()
	RegisterConfigSvrMasterServiceServer(m.grpcServer, m.s)
	m.addr = listenEndpoints
	m.wg = wg
	return nil
}

func (m *ConfigMaster) Start() error {
	glog.Info("Begin to start config master")
	defer m.wg.Done()

	m.mu.Lock()
	if m.started {
		glog.Info("Master has been started")
		m.mu.Unlock()
		return nil
	}
	m.mu.Unlock()

	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()
	retryTimes := 1
	var err error
	for retryTimes <= 5 {
		_, errMsg, err := putKeyToEtcd(ctx, m.etcdClient, MasterEtcdKey, m.addr)
		if err != nil {
			glog.Info("Failed to update master key ", errMsg)
			retryTimes += 1
			continue
		} else {
			break
		}
	}

	if retryTimes > 5 {
		glog.Fatal("Unable to put master endpoint to etcd, ", err)
	}

	err = common.SyncConfigDataFromETCD(m.etcdClient, m.s.GetConfigData())
	if err != nil {
		glog.Fatal("Failed to sync config data from etcd ", err.Error())
	}

	err = common.SyncConfigMetaFromETCD(m.etcdClient, m.s.GetConfigMeta())
	if err != nil {
		glog.Fatal("Failed to sync config Meta from etcd ", err.Error())
	}

	err = common.SyncUpgradeInfoFromETCD(m.etcdClient, m.s.GetConfigEvent())
	if err != nil {
		glog.Fatal("Failed to sync config upgrade info from etcd ", err.Error())
	}

	glog.Info("begin to start master, current status is ", m.started)
	m.started = true
	return m.grpcServer.Serve(m.lis)
}

func (m *ConfigMaster) Stop() error {
	m.mu.Lock()
	defer m.mu.Unlock()
	if !m.started {
		glog.Info("Config master has not been started yet. ignore")
		return nil
	}

	glog.Info("Stopping config master")
	m.grpcServer.Stop()
	m.started = false
	return nil
}

gateway.txt

package main

import (
	"context"
	"cos-config/common"
	server "cos-config/master"
	"errors"
	"flag"
	"net/http"
	"os"
	"os/signal"
	"sync"
	"syscall"
	"time"

	"cos-config/storage"
	"github.com/coreos/etcd/clientv3"
	"github.com/golang/glog"
	////	"github.com/golang/protobuf/proto"
	"github.com/grpc-ecosystem/grpc-gateway/runtime"
	"google.golang.org/grpc"
)

var (
	restfulServerEndpoint = flag.String("rest_server_port", ":8090", "rest server listen endpoint")
	etcdUrls              = flag.String("etcd_endpoint", "119.29.166.185:2379", "multiple etcd service endpoint, sepeated by comma")
	ctx                   context.Context
	cancel                context.CancelFunc
	svr                   http.Server
	etcdClient            *clientv3.Client
	masterEndpoint        string
	wg                    sync.WaitGroup
	watcher               *common.Watcher
)

func run(backendEndpoint string) error {
	glog.Info("get the backend point", backendEndpoint)
	defer wg.Done()
	ctx = context.Background()
	ctx, cancel = context.WithCancel(ctx)
	defer cancel()

	mux := runtime.NewServeMux()

	opts := []grpc.DialOption{grpc.WithInsecure()}
	err := server.RegisterConfigSvrMasterServiceHandlerFromEndpoint(ctx, mux, backendEndpoint, opts)

	if err != nil {
		glog.Error("register config svr failed ", err.Error())
		return err
	}

	svr = http.Server{
		Addr:    *restfulServerEndpoint,
		Handler: mux,
	}
	return svr.ListenAndServe()
}

func getBackendServiceMasterEndpoint() (string, bool) {
	item, _, _, err := common.GetKeyFromETCD(context.Background(), etcdClient, server.MasterEtcdKey)
	if err != nil {
		glog.Error("Failed to get master endpoint ", err.Error())
		return "", false
	}

	return item.ConfigValue, true
}

type HttpServerWatchHandler struct {
}

func (h HttpServerWatchHandler) HandlePut(event *clientv3.Event) error {
	if nil == event {
		return errors.New("put event is nil.")
	}
	if clientv3.EventTypePut != event.Type {
		return errors.New("This function can only handle events of type put")
	}

	if len(event.Kv.Value) > 0 && string(event.Kv.Value) != masterEndpoint {
		glog.Infof("Backend master endpoint changed from %s to %s", masterEndpoint, string(event.Kv.Value))
		masterEndpoint = string(event.Kv.Value)
	}

	svr.Shutdown(ctx)

	wg.Add(1)
	go run(masterEndpoint)
	return nil
}

func (h HttpServerWatchHandler) HandleDelete(event *clientv3.Event) error {
	return nil
}

func main() {
	flag.Parse()
	defer glog.Flush()

	properties := make(map[string]string)
	properties[storage.PropertiesEtcdUrls] = *etcdUrls
	var err error
	etcdClient, err = storage.NewEtcdClient(properties)
	if err != nil {
		glog.Fatalf("Init etcd client failed. error: %s", err.Error())
	}

	watcher, err = common.NewWatcher(etcdClient, []string{server.MasterEtcdKey}, HttpServerWatchHandler{})
	if err != nil {
		glog.Fatalf("Failed to new watcher. error: %s", err.Error())
	}

	for {
		if masterEndpoint, ok := getBackendServiceMasterEndpoint(); ok {
			wg.Add(1)
			go run(masterEndpoint)
			break
		} else {
			glog.Error("Failed to get master endpoint, retrying....")
			time.Sleep(1 * time.Second)
		}
	}

	
}

[achew22]

Can you write a bit about what it is you've tried and what you think might be wrong too? I'm happy to help with a specific problem but I don't have time to dig through a couple hundred lines of code and dozens of linked libraries -- sorry

[toranger]

Sorry, the problem is I start two svr ( 0.0.0.0:9090 and 0.0.0.0:9092 ) and gateway( listen 8090), though http request can arrive at gateway but svr did't trigger the grpc method. It always present "transport is closing". Does it mean svr has problem with grpc? or?

[achew22]

Can you write a bit about what it is you've tried and what you think might be wrong too? Are you getting any other error messages for example?

[toranger]

There are no other error messages, I tried use curl to call the rpc function, but all remind me "transport is closing", if gateway can't connect server it will remind "all sub connection failed", so I think maybe use gateway to trigger rpc failed.

[achew22]

Are you curling to the gRPC endpoint or to the http exposed endpoint? What is the curl command you're running?

[toranger]

curl -H "Content-Type:application/json" -d '{"module_name":"cos-cgi", "env_name":"dev", "category":"ap-shanghai"}' -X POST -k http://118.89.226.213:8090/v1/getagentinfo/module, all did not work

[achew22]

when your run that command with the additional arguments -vvv what do you get?

[toranger]

[achew22]

In general unless you're working with a GUI application it is not helpful to post screenshots of your terminal. Can you please copy/paste the text instead in the future?

[achew22]

In your go code where you call return svr.ListenAndServe(), can you add a print statement that indicates the port/address it is listening on and verify that in the output of the program?

[toranger]

A long time later, it will show:

• upload completely sent off: 69 out of 69 bytes
  < HTTP/1.1 503 Service Unavailable
  < Content-Type: application/json
  < Date: Mon, 13 Aug 2018 02:27:50 GMT
  < Content-Length: 42
  <
• Connection #0 to host 118.89.226.213 left intact
  Add the log to the svr port, It seems no problem. I start all server on one machine to test.
  I0813 10:36:48.370241 15174 main.go:67] current gateway server is: {:8090 0xc42007bb00 0s 0s 0s 0s 0 map[] ... }