Add Support for Mapbox Vector Tiles in the grid server (#680)

* Replace deprecated package go.geojson with orb
Replace deprecated package io/ioutil with os

* Add support for Mapbox Vector Tile in tile_server

* Add tests for MVT tile generation in gridserver

* Update tile_server README to include Mapbox Vector Tile support
Format tile_server README

* Specify Mapbox Vector Tile version and update test data

* Updated README to explicitly state Mapbox Vector Tile version as 2.1.
* Modified MVT layer version to 2.
* Updated test data files to match the new layer version.

---------

Co-authored-by: Doug Rinckes <[email protected]>

Author: broot5

tile_server/README.md

@@ -2,42 +2,44 @@
 
 This code provides a Go server to handle
 [Tile Map Service](https://en.wikipedia.org/wiki/Tile_Map_Service) requests. It
-is able to respond with [GeoJSON](https://geojson.org) or image tiles, either of
+is able to respond with [GeoJSON](https://geojson.org), image tiles or
+[Mapbox Vector Tiles](https://github.com/mapbox/vector-tile-spec) (version 2.1), any of
 which can be added as an overlay to a map.
 
 ## Limitations
 
-1.  This server does not implement any GetCapabilities methods.
-1.  A fixed image tile size of 256x256 pixels is used.
+1. This server does not implement any GetCapabilities methods.
+1. A fixed image tile size of 256x256 pixels is used.
 
 ## Tile Requests
 
 The server responds to tile requests. These send a zoom level, and x and y tile
-numbers. The request URL determines whether the response should be GeoJSON or an
-image.
+numbers. The request URL determines whether the response should be GeoJSON, an
+image, or a Mapbox Vector Tile.
 
 The format of the requests is:
 
 ```
 //hostname:port/grid/[tilespec]/z/x/y.[format]?[options]
 ```
 
-*   `tilespec` must be either `wms` or `tms`. The only difference in these is
+* `tilespec` must be either `wms` or `tms`. The only difference in these is
     that the Y tiles are numbered from north to south (`wms`) or from south to
     north (`tms`).
-*   `format` must be either `json` for a GeoJSON FeatureCollection, or `png`
-    for a PNG image tile.
-*   The optional parameters are:
-    *   `linecol`: this defines the RGBA colour to use for lines in the PNG
+* `format` must be either `json` for a GeoJSON FeatureCollection, `png`
+    for a PNG image tile, or `mvt` for a Mapbox Vector Tile.
+* The optional parameters are:
+  * `linecol`: this defines the RGBA colour to use for lines in the PNG
         tiles.
-    *   `labelcol`: this defines the RGBA colour to use for the labels in the
+  * `labelcol`: this defines the RGBA colour to use for the labels in the
         PNG tiles.
-    *   `zoomadjust`: this is added to the map zoom value, to cause the returned
-        grid to be finer or coarser. This affects both GeoJSON and image tiles.
-    *   `projection`: this can be used to change the map projection from the
+  * `zoomadjust`: this is added to the map zoom value, to cause the returned
+        grid to be finer or coarser. This affects both GeoJSON, image tiles,
+        and Mapbox Vector Tile.
+  * `projection`: this can be used to change the map projection from the
         default, spherical mercator, to geodetic. Valid values are:
-        *   `mercator` or `epsg:3857`: selects spherical mercator (default)
-        *   `geodetic` or `epsg:4326`: selects geodetic projection
+    * `mercator` or `epsg:3857`: selects spherical mercator (default)
+    * `geodetic` or `epsg:4326`: selects geodetic projection
 
 An example request could be:
 
@@ -147,18 +149,18 @@ go test ./tile_server/gridserver -v --logtostderr
 
 The following other projects need to be installed:
 
-[GeoJSON](https://github.com/paulmach/go.geojson) provides the definition for
-the GeoJSON response objects. Install with:
+[orb](https://github.com/paulmach/orb) provides the definition for
+the GeoJSON and Mapbox Vector Tile response objects. Install with:
 
 ```
-go get github.com/paulmach/go.geojson
+go get github.com/paulmach/orb
 ```
 
 [Freetype](https://github.com/golang/freetype) is used for the labels in the PNG
 tiles. Install with:
 
 ```
-$ go get github.com/golang/freetype
+go get github.com/golang/freetype
 ```
 
 [Open Location Code](https://github.com/open-location-code/) generates the codes

tile_server/gridserver/go.mod

@@ -11,4 +11,8 @@ require (
 	golang.org/x/image v0.18.0
 )
 
-require go.mongodb.org/mongo-driver v1.11.4 // indirect
+require (
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/paulmach/protoscan v0.2.1 // indirect
+	go.mongodb.org/mongo-driver v1.11.4 // indirect
+)

tile_server/gridserver/go.sum

@@ -1,6 +1,7 @@
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
@@ -23,6 +24,7 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU=
 github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU=
+github.com/paulmach/protoscan v0.2.1 h1:rM0FpcTjUMvPUNk2BhPJrreDKetq43ChnL+x1sRg8O8=
 github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -123,6 +125,7 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

tile_server/gridserver/gridserver.go

@@ -16,6 +16,7 @@ import (
 const (
 	outputJSON       = "json"
 	outputPNG        = "png"
+	outputMVT        = "mvt"
 	tileNumberingWMS = "wms"
 	tileNumberingTMS = "tms"
 	lineColorOption  = "linecol"
@@ -25,7 +26,7 @@ const (
 )
 
 var (
-	pathSpec = regexp.MustCompile(fmt.Sprintf(`^/grid/(%s|%s)/(\d+)/(\d+)/(\d+)\.(%s|%s)`, tileNumberingWMS, tileNumberingTMS, outputJSON, outputPNG))
+	pathSpec = regexp.MustCompile(fmt.Sprintf(`^/grid/(%s|%s)/(\d+)/(\d+)/(\d+)\.(%s|%s|%s)`, tileNumberingWMS, tileNumberingTMS, outputJSON, outputPNG, outputMVT))
 )
 
 // LatLng represents a latitude and longitude in degrees.
@@ -71,6 +72,8 @@ func Parse(r *http.Request) (*TileRef, error) {
 		opts.Format = JSONTile
 	} else if g[5] == outputPNG {
 		opts.Format = ImageTile
+	} else if g[5] == outputMVT {
+		opts.Format = VectorTile
 	} else {
 		return nil, fmt.Errorf("Tile output type not specified: %v", g[5])
 	}

tile_server/gridserver/mvt.go

@@ -0,0 +1,43 @@
+package gridserver
+
+import (
+	log "github.com/golang/glog"
+	"github.com/paulmach/orb/encoding/mvt"
+	"github.com/paulmach/orb/maptile"
+)
+
+const (
+	layerName    = "grid"
+	layerVersion = 2
+	layerExtent  = 4096
+)
+
+// MVT returns a Mapbox Vector Tile (MVT) marshalled as bytes.
+func (t *TileRef) MVT() ([]byte, error) {
+	log.Infof("Producing mvt for tile z/x/y %v/%v/%v (%s)", t.Z, t.X, t.Y, t.Path())
+	gj, err := t.GeoJSON()
+	if err != nil {
+		return nil, err
+	}
+
+	layer := &mvt.Layer{
+		Name:     layerName,
+		Version:  layerVersion,
+		Extent:   layerExtent,
+		Features: gj.Features,
+	}
+
+	// Since GeoJSON stores geometries in latitude and longitude (WGS84),
+	// we only need to project the coordinates if the desired output projection is Mercator.
+	if t.Options.Projection.String() == "mercator" {
+		// Convert TMS coordinates to WMS coordinates
+		wmsY := (1 << uint(t.Z)) - t.Y - 1
+		layer.ProjectToTile(maptile.New(uint32(t.X), uint32(wmsY), maptile.Zoom(t.Z)))
+	}
+
+	data, err := mvt.Marshal(mvt.Layers{layer})
+	if err != nil {
+		return nil, err
+	}
+	return data, nil
+}

tile_server/gridserver/mvt_test.go

@@ -0,0 +1,47 @@
+package gridserver
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestMVT(t *testing.T) {
+	var tests = []struct {
+		x, y, z  int
+		opts     *TileOptions
+		testFile string
+	}{
+		{x: 17, y: 19, z: 5, testFile: testDataPath + "5_17_19.mvt"},
+		{
+			x: 17, y: 19, z: 5,
+			opts:     &TileOptions{Format: VectorTile, LineColor: lineColor, LabelColor: labelColor, Projection: NewMercatorTMS(), ZoomAdjust: 2},
+			testFile: testDataPath + "5_17_19_zoom_2.mvt",
+		},
+		{
+			x: 1098232, y: 1362659, z: 21,
+			opts:     &TileOptions{Format: VectorTile, LineColor: lineColor, LabelColor: labelColor, Projection: NewMercatorTMS(), ZoomAdjust: 0},
+			testFile: testDataPath + "21_1098232_1362659.mvt",
+		},
+		{
+			x: 17, y: 19, z: 5,
+			opts:     &TileOptions{Format: VectorTile, LineColor: lineColor, LabelColor: labelColor, Projection: NewGeodeticTMS(), ZoomAdjust: 0},
+			testFile: testDataPath + "5_17_19_geodetic.mvt",
+		},
+		{
+			x: 1098232, y: 1362659, z: 21,
+			opts:     &TileOptions{Format: VectorTile, LineColor: lineColor, LabelColor: labelColor, Projection: NewGeodeticTMS(), ZoomAdjust: 0},
+			testFile: testDataPath + "21_1098232_1362659_geodetic.mvt",
+		},
+	}
+	for n, td := range tests {
+		want := readTestData(td.testFile)
+		tr := MakeTileRef(td.x, td.y, td.z, td.opts)
+		got, err := tr.MVT()
+		if err != nil {
+			t.Errorf("Test %d: MVT failed: %v", n, err)
+		}
+		if !bytes.Equal(got, want) {
+			t.Errorf("Test %d: got MVT != want MVT", n)
+		}
+	}
+}