Merge branch 'master' into throttler-updateMySQLClusterProbes-log-typo

Author: timvaillancourt

.github/workflows/release.yml

@@ -0,0 +1,29 @@
+name: goreleaser
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  goreleaser:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      -
+        name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.14
+      -
+        name: Run GoReleaser
+        uses: goreleaser/goreleaser-action@v2
+        with:
+          version: latest
+          args: release --rm-dist
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -1,5 +1,6 @@
 .gopath/
 bin/
+dist/
 main
 conf/freno.local.conf.json
 .vendor/

.goreleaser.yml

@@ -0,0 +1,37 @@
+before:
+  hooks:
+    - go mod download
+    - go test -v -covermode=atomic ./pkg/...
+builds:
+- env:
+    - CGO_ENABLED=0
+  goos:
+    - linux
+    - darwin
+  ldflags:
+    - -w -s
+    - -X main.AppVersion={{.Version}} -X main.GitCommit={{.Commit}}
+  main: ./cmd/freno/main.go
+checksum:
+  name_template: 'checksums.txt'
+changelog:
+  sort: asc
+nfpms:
+  - vendor: GitHub
+    homepage: https://github.com/github/freno
+    description: Cooperative, highly available throttler service
+    license: MIT
+    bindir: /usr/bin
+    config_files:
+      "resources/freno.conf.skeleton.json": "/etc/freno.conf.json"
+    files:
+      "resources/etc/init.d/freno": "/etc/init.d/freno"
+    empty_folders:
+      - "/var/lib/freno"
+    formats:
+      - deb
+      - rpm
+release:
+  github:
+    owner: github
+    name: freno

.travis.yml

@@ -100,7 +100,7 @@ function oinstall() {
   prefix="$2"
 
   cd  $mydir
-  gofmt -s -w  go/
+  gofmt -s -w pkg/
   cp ./resources/etc/init.d/freno $builddir/freno/etc/init.d/freno
   chmod +x $builddir/freno/etc/init.d/freno
   cp ./resources/freno.conf.skeleton.json $builddir/freno/etc/freno.conf.json
@@ -143,7 +143,7 @@ function build() {
   prefix="$4"
   ldflags="-X main.AppVersion=${RELEASE_VERSION} -X main.GitCommit=${GIT_COMMIT}"
   echo "Building via $(go version)"
-  gobuild="go build -i ${opt_race} -ldflags \"$ldflags\" -o $builddir/freno${prefix}/freno go/cmd/freno/main.go"
+  gobuild="go build -i ${opt_race} -ldflags \"$ldflags\" -o $builddir/freno${prefix}/freno cmd/freno/main.go"
 
   case $os in
     'linux')

build.sh

@@ -58,6 +58,7 @@ type MySQLConfigurationSettings struct {
 	HttpCheckPort        int      // port for HTTP check. -1 to disable.
 	HttpCheckPath        string   // If non-empty, requires HttpCheckPort
 	IgnoreHosts          []string // If non empty, substrings to indicate hosts to be ignored/skipped
+	VitessCells          []string // Name of the Vitess cells for polling tablet hosts
 
 	Clusters map[string](*MySQLClusterConfigurationSettings) // cluster name -> cluster config
 }
@@ -114,6 +115,9 @@ func (settings *MySQLConfigurationSettings) postReadAdjustments() error {
 		if len(clusterSettings.IgnoreHosts) == 0 {
 			clusterSettings.IgnoreHosts = settings.IgnoreHosts
 		}
+		if !clusterSettings.VitessSettings.IsEmpty() && len(clusterSettings.VitessSettings.Cells) < 1 {
+			clusterSettings.VitessSettings.Cells = settings.VitessCells
+		}
 	}
 	return nil
 }

pkg/config/mysql_config.go

@@ -6,6 +6,7 @@ package config
 
 type VitessConfigurationSettings struct {
 	API         string
+	Cells       []string
 	Keyspace    string
 	Shard       string
 	TimeoutSecs uint

pkg/config/vitess_config.go

@@ -327,7 +327,9 @@ func (throttler *Throttler) refreshMySQLInventory() error {
 				if err != nil {
 					return log.Errorf("Unable to get vitess hosts from %s, %s/%s: %+v", clusterSettings.VitessSettings.API, keyspace, shard, err)
 				}
-				log.Debugf("Read %+v hosts from vitess %s, %s/%s", len(tablets), clusterSettings.VitessSettings.API, keyspace, shard)
+				log.Debugf("Read %+v hosts from vitess %s, %s/%s, cells=%s", len(tablets), clusterSettings.VitessSettings.API,
+					keyspace, shard, strings.Join(vitess.ParseCells(clusterSettings.VitessSettings), ","),
+				)
 				clusterProbes := &mysql.ClusterProbes{
 					ClusterName:      clusterName,
 					IgnoreHostsCount: clusterSettings.IgnoreHostsCount,

pkg/throttle/throttler.go

@@ -16,9 +16,23 @@ const defaultTimeout = time.Duration(5) * time.Second
 
 // Tablet represents information about a running instance of vttablet.
 type Tablet struct {
-	MysqlHostname string              `json:"mysql_hostname,omitempty"`
-	MysqlPort     int32               `json:"mysql_port,omitempty"`
-	Type          topodata.TabletType `json:"type,omitempty"`
+	Alias         *topodata.TabletAlias `json:"alias,omitempty"`
+	MysqlHostname string                `json:"mysql_hostname,omitempty"`
+	MysqlPort     int32                 `json:"mysql_port,omitempty"`
+	Type          topodata.TabletType   `json:"type,omitempty"`
+}
+
+// HasValidCell returns a bool reflecting if a tablet is in a valid Vitess cell
+func (t Tablet) HasValidCell(validCells []string) bool {
+	if len(validCells) == 0 {
+		return true
+	}
+	for _, cell := range validCells {
+		if t.Alias.GetCell() == cell {
+			return true
+		}
+	}
+	return false
 }
 
 // IsValidReplica returns a bool reflecting if a tablet type is REPLICA
@@ -40,17 +54,29 @@ func constructAPIURL(settings config.VitessConfigurationSettings) (url string) {
 	return url
 }
 
+// ParseCells returns a slice of non-empty Vitess cell names
+func ParseCells(settings config.VitessConfigurationSettings) (cells []string) {
+	for _, cell := range settings.Cells {
+		cell = strings.TrimSpace(cell)
+		if cell != "" {
+			cells = append(cells, cell)
+		}
+	}
+	return cells
+}
+
 // filterReplicaTablets parses a list of tablets, returning replica tablets only
-func filterReplicaTablets(tablets []Tablet) (replicas []Tablet) {
+func filterReplicaTablets(settings config.VitessConfigurationSettings, tablets []Tablet) (replicas []Tablet) {
+	validCells := ParseCells(settings)
 	for _, tablet := range tablets {
-		if tablet.IsValidReplica() {
+		if tablet.HasValidCell(validCells) && tablet.IsValidReplica() {
 			replicas = append(replicas, tablet)
 		}
 	}
 	return replicas
 }
 
-// ParseTablets reads from vitess /api/ks_tablets/<keyspace>/[shard] and returns a
+// ParseTablets reads from vitess /api/keyspace/<keyspace>/tablets/[shard] and returns a
 // listing (mysql_hostname, mysql_port, type) of REPLICA tablets
 func ParseTablets(settings config.VitessConfigurationSettings) (tablets []Tablet, err error) {
 	if settings.TimeoutSecs == 0 {
@@ -64,13 +90,16 @@ func ParseTablets(settings config.VitessConfigurationSettings) (tablets []Tablet
 	if err != nil {
 		return tablets, err
 	}
-
 	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return tablets, fmt.Errorf("%v", resp.Status)
+	}
+
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return tablets, err
 	}
-
 	err = json.Unmarshal(body, &tablets)
-	return filterReplicaTablets(tablets), err
+	return filterReplicaTablets(settings, tablets), err
 }

pkg/vitess/api_client.go

@@ -18,35 +18,44 @@ func TestParseTablets(t *testing.T) {
 		case "/api/keyspace/test/tablets/00":
 			data, _ := json.Marshal([]Tablet{
 				{
+					Alias:         &topodata.TabletAlias{Cell: "cell1"},
 					MysqlHostname: "master",
 					Type:          topodata.TabletType_MASTER,
 				},
 				{
-					MysqlHostname: "replica",
+					Alias:         &topodata.TabletAlias{Cell: "cell2"},
+					MysqlHostname: "replica1",
 					Type:          topodata.TabletType_REPLICA,
 				},
 				{
+					Alias:         &topodata.TabletAlias{Cell: "cell3"},
+					MysqlHostname: "replica2",
+					Type:          topodata.TabletType_REPLICA,
+				},
+				{
+					Alias:         &topodata.TabletAlias{Cell: "cell2"},
 					MysqlHostname: "spare",
 					Type:          topodata.TabletType_SPARE,
 				},
 				{
+					Alias:         &topodata.TabletAlias{Cell: "cell3"},
 					MysqlHostname: "batch",
 					Type:          topodata.TabletType_BATCH,
 				},
 				{
+					Alias:         &topodata.TabletAlias{Cell: "cell2"},
 					MysqlHostname: "backup",
 					Type:          topodata.TabletType_BACKUP,
 				},
 				{
-
+					Alias:         &topodata.TabletAlias{Cell: "cell1"},
 					MysqlHostname: "restore",
 					Type:          topodata.TabletType_RESTORE,
 				},
 			})
 			fmt.Fprint(w, string(data))
 		default:
 			w.WriteHeader(http.StatusNotFound)
-			fmt.Fprint(w, "[]")
 		}
 	}))
 	defer vitessApi.Close()
@@ -62,30 +71,64 @@ func TestParseTablets(t *testing.T) {
 			t.Fatalf("Expected no error, got %q", err)
 		}
 
-		if len(tablets) != 1 {
-			t.Fatalf("Expected 1 tablet, got %d", len(tablets))
+		if len(tablets) != 2 {
+			t.Fatalf("Expected 2 tablets, got %d", len(tablets))
 		}
 
-		if tablets[0].MysqlHostname != "replica" {
-			t.Fatalf("Expected hostname %q, got %q", "replica", tablets[0].MysqlHostname)
+		if tablets[0].MysqlHostname != "replica1" {
+			t.Fatalf("Expected hostname %q, got %q", "replica1", tablets[0].MysqlHostname)
+		}
+		if tablets[1].MysqlHostname != "replica2" {
+			t.Fatalf("Expected hostname %q, got %q", "replica2", tablets[1].MysqlHostname)
 		}
 
 		if httpClient.Timeout != time.Second {
 			t.Fatalf("Expected vitess client timeout of %v, got %v", time.Second, httpClient.Timeout)
 		}
 	})
 
+	t.Run("with-cell", func(t *testing.T) {
+		settings := config.VitessConfigurationSettings{
+			API:      vitessApi.URL,
+			Cells:    []string{"cell2"},
+			Keyspace: "test",
+			Shard:    "00",
+		}
+		tablets, err := ParseTablets(settings)
+		if err != nil {
+			t.Fatalf("Expected no error, got %q", err)
+		}
+
+		if len(tablets) != 1 {
+			t.Fatalf("Expected 1 tablet, got %d", len(tablets))
+		}
+
+		if tablets[0].MysqlHostname != "replica1" {
+			t.Fatalf("Expected hostname %q, got %q", "replica1", tablets[0].MysqlHostname)
+		}
+		if tablets[0].Alias.GetCell() != "cell2" {
+			t.Fatalf("Expected vitess cell %s, got %s", "cell2", tablets[0].Alias.GetCell())
+		}
+
+		// empty cell names should cause no filtering
+		settings.Cells = []string{"", ""}
+		tablets, _ = ParseTablets(settings)
+		if len(tablets) != 2 {
+			t.Fatalf("Expected 2 tablet, got %d", len(tablets))
+		}
+	})
+
 	t.Run("not-found", func(t *testing.T) {
 		tablets, err := ParseTablets(config.VitessConfigurationSettings{
 			API:      vitessApi.URL,
 			Keyspace: "not-found",
 			Shard:    "40-80",
 		})
-		if err != nil {
-			t.Fatalf("Expected no error, got %q", err)
+		if err == nil || err.Error() != "404 Not Found" {
+			t.Fatalf("Expected %q error, got %q", "404 Not Found", err)
 		}
 
-		if len(tablets) > 0 {
+		if len(tablets) != 0 {
 			t.Fatalf("Expected 0 tablets, got %d", len(tablets))
 		}