Early key release (#706)

* Allow for v1.5+ early key release. Multiple keys can be provided that all have the same start interval
* Add configuration params for this
* Add tests for new pieces. 100% coverage on exposure model transform.
* Add documentation.
* add same day release as an optional feature to the generate server

Fixes #705
Part of #663

Author: mikehelmick

docs/getting-started/publishing-temporary-exposure-keys.md

@@ -19,7 +19,42 @@ exposed, their keys are shared to the server in order for applications to
 download and [determine if other users interacted with any of the now exposed
 keys](https://blog.google/documents/69/Exposure_Notification_-_Cryptography_Specification_v1.2.1.pdf).
 
-## Publishing Keys
+## The Publish API Call
+
+TEKs are published by sending the appropriate JSON document in the body of
+an HTTP POST request to the `exposure` server.
+
+The structure of the API is defined in [pkg/api/v1alpha1/exposure_types.go](https://github.com/google/exposure-notifications-server/blob/main/pkg/api/v1alpha1/exposure_types.go),
+in the `Publish` type. Please see the documentation in the source file
+for details of the fields themselves.
+
+Here, we point out some non-obvious validation that is applied to the keys.
+
+* All keys must be valid! If there are any validation errors, the entire batch
+  is rejected.
+* Max keys per publish: Default is `20` and can be adjusted with the
+  `MAX_KEYS_ON_PUBLISH` environment variable.
+* Max overlapping keys with same start interval: Default is `3` and can be
+  adjusted with the `MAX_SAME_START_INTERVAL_KEYS` environment variable.
+	In practical terms, this means that if you are obtaining TEK history on a
+	mobile device with >= v1.5 of the device API, it will stop the validity
+	of the current day's TEK and issue a new now. Both keys will have the same
+	start iterval.
+* Max age: How old keys can be. The default is `360h` (15 days) and can be
+  adjusted with the `MAX_INTERVAL_AGE_ON_PUBLISH`. All provided keys must have
+	a `rollingStartNumber` that is >= to the max age.
+* Keys with a future start time (`rollingStartNumber` indicates time > now),
+  are rejected.
+* Keys that are "sill valid" are accepted by the server, but they are embargoed
+  until after they key could no longer be replayed usefully. A stall valid key
+	is one where the `rollingStartNumber` is in the past, but the
+	`rollingStartNumber` + the `rollingPeriod` indicates a future time.
+* When using health authority verification certificates
+  (__strongly recommended__), the TEK data in the publish request and the
+	`hmackey` must be able to be used to calculate the HMAC value as present in
+	the certificate.
+
+## Server Access Configuration
 
 In order for your application to publish keys to the server, the server
 requires the registration of the Application Name (for Android) or the Bundle ID

internal/generate/config.go

@@ -36,13 +36,15 @@ type Config struct {
 	SecretManager         secrets.Config
 	ObservabilityExporter observability.Config
 
-	Port             string        `env:"PORT, default=8080"`
-	NumExposures     int           `env:"NUM_EXPOSURES_GENERATED, default=10"`
-	KeysPerExposure  int           `env:"KEYS_PER_EXPOSURE, default=14"`
-	MaxKeysOnPublish int           `env:"MAX_KEYS_ON_PUBLISH, default=15"`
-	MaxIntervalAge   time.Duration `env:"MAX_INTERVAL_AGE_ON_PUBLISH, default=360h"`
-	TruncateWindow   time.Duration `env:"TRUNCATE_WINDOW, default=1h"`
-	DefaultRegion    string        `env:"DEFAULT_REGOIN, default=US"`
+	Port                     string        `env:"PORT, default=8080"`
+	NumExposures             int           `env:"NUM_EXPOSURES_GENERATED, default=10"`
+	KeysPerExposure          int           `env:"KEYS_PER_EXPOSURE, default=14"`
+	MaxKeysOnPublish         int           `env:"MAX_KEYS_ON_PUBLISH, default=15"`
+	MaxSameStartIntervalKeys int           `env:"MAX_SAME_START_INTERVAL_KEYS, default=2"`
+	SimulateSameDayRelease   bool          `env:"SIMULATE_SAME_DAY_RELEASE, default=false"`
+	MaxIntervalAge           time.Duration `env:"MAX_INTERVAL_AGE_ON_PUBLISH, default=360h"`
+	TruncateWindow           time.Duration `env:"TRUNCATE_WINDOW, default=1h"`
+	DefaultRegion            string        `env:"DEFAULT_REGOIN, default=US"`
 }
 
 func (c *Config) DatabaseConfig() *database.Config {

internal/generate/handler.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"sort"
 	"strings"
 	"time"
 
@@ -38,7 +39,7 @@ func NewHandler(ctx context.Context, config *Config, env *serverenv.ServerEnv) (
 		return nil, fmt.Errorf("missing database in server environment")
 	}
 
-	transformer, err := model.NewTransformer(config.MaxKeysOnPublish, config.MaxIntervalAge, config.TruncateWindow, false)
+	transformer, err := model.NewTransformer(config.MaxKeysOnPublish, config.MaxSameStartIntervalKeys, config.MaxIntervalAge, config.TruncateWindow, false)
 	if err != nil {
 		return nil, fmt.Errorf("model.NewTransformer: %w", err)
 	}
@@ -86,6 +87,19 @@ func (h *generateHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			Regions:        regions,
 			AppPackageName: "generated.data",
 		}
+		if h.config.SimulateSameDayRelease {
+			sort.Slice(publish.Keys, func(i int, j int) bool {
+				return publish.Keys[i].IntervalNumber < publish.Keys[j].IntervalNumber
+			})
+			lastKey := &publish.Keys[len(publish.Keys)-1]
+			newLastDayKey, err := util.RandomExposureKey(lastKey.IntervalNumber, 144, lastKey.TransmissionRisk)
+			if err != nil {
+				logger.Errorf("unable to simulate same day key release: %v", err)
+			} else {
+				lastKey.IntervalCount = lastKey.IntervalCount / 2
+				publish.Keys = append(publish.Keys, newLastDayKey)
+			}
+		}
 
 		exposures, err := h.transformer.TransformPublish(ctx, &publish, batchTime)
 		if err != nil {

internal/integration/integration.go

@@ -151,10 +151,11 @@ func testServer(tb testing.TB) (*serverenv.ServerEnv, *http.Client) {
 
 	// Publish
 	publishConfig := &publish.Config{
-		MaxKeysOnPublish:        15,
-		MaxIntervalAge:          360 * time.Hour,
-		TruncateWindow:          1 * time.Second,
-		DebugReleaseSameDayKeys: true,
+		MaxKeysOnPublish:         15,
+		MaxSameStartIntervalKeys: 2,
+		MaxIntervalAge:           360 * time.Hour,
+		TruncateWindow:           1 * time.Second,
+		DebugReleaseSameDayKeys:  true,
 	}
 
 	publishHandler, err := publish.NewHandler(ctx, publishConfig, env)

internal/publish/config.go

@@ -41,12 +41,16 @@ type Config struct {
 	Verification          verification.Config
 	ObservabilityExporter observability.Config
 
-	Port             string        `env:"PORT, default=8080"`
-	MaxKeysOnPublish int           `env:"MAX_KEYS_ON_PUBLISH, default=15"`
-	MaxIntervalAge   time.Duration `env:"MAX_INTERVAL_AGE_ON_PUBLISH, default=360h"`
-	TruncateWindow   time.Duration `env:"TRUNCATE_WINDOW, default=1h"`
+	Port             string `env:"PORT, default=8080"`
+	MaxKeysOnPublish int    `env:"MAX_KEYS_ON_PUBLISH, default=20"`
+	// Provides compatibility w/ 1.5 release.
+	MaxSameStartIntervalKeys int           `env:"MAX_SAME_START_INTERVAL_KEYS, default=3"`
+	MaxIntervalAge           time.Duration `env:"MAX_INTERVAL_AGE_ON_PUBLISH, default=360h"`
+	TruncateWindow           time.Duration `env:"TRUNCATE_WINDOW, default=1h"`
 
-	// Flags for local development and testing.
+	// Flags for local development and testing. This will cause still valid keys
+	// to not be embargoed.
+	// Normallly "still valid" keys can be accepted, but are embargoed.
 	DebugReleaseSameDayKeys bool `env:"DEBUG_RELEASE_SAME_DAY_KEYS"`
 }
 

internal/publish/model/exposure_model.go

@@ -103,7 +103,8 @@ func TimeForIntervalNumber(interval int32) time.Time {
 
 // Transformer represents a configured Publish -> Exposure[] transformer.
 type Transformer struct {
-	maxExposureKeys     int
+	maxExposureKeys     int           // Overall maximum number of keys.
+	maxSameDayKeys      int           // Number of keys that are allowed to have the same start interval.
 	maxIntervalStartAge time.Duration // How many intervals old does this server accept?
 	truncateWindow      time.Duration
 	debugReleaseSameDay bool // If true, still valid keys are not embargoed.
@@ -112,18 +113,23 @@ type Transformer struct {
 // NewTransformer creates a transformer for turning publish API requests into
 // records for insertion into the database. On the call to TransformPublish
 // all data is validated according to the transformer that is used.
-func NewTransformer(maxExposureKeys int, maxIntervalStartAge time.Duration, truncateWindow time.Duration, releaseSameDayKeys bool) (*Transformer, error) {
+func NewTransformer(maxExposureKeys int, maxSameDayKeys int, maxIntervalStartAge time.Duration, truncateWindow time.Duration, releaseSameDayKeys bool) (*Transformer, error) {
 	if maxExposureKeys <= 0 {
 		return nil, fmt.Errorf("maxExposureKeys must be > 0, got %v", maxExposureKeys)
 	}
+	if maxSameDayKeys < 1 {
+		return nil, fmt.Errorf("maxSameDayKeys must be >= 1, got %v", maxSameDayKeys)
+	}
 	return &Transformer{
 		maxExposureKeys:     maxExposureKeys,
+		maxSameDayKeys:      maxSameDayKeys,
 		maxIntervalStartAge: maxIntervalStartAge,
 		truncateWindow:      truncateWindow,
 		debugReleaseSameDay: releaseSameDayKeys,
 	}, nil
 }
 
+// KeyTransform represents the settings to apply when transforming an individual key on a publish request.
 type KeyTransform struct {
 	MinStartInterval      int32
 	MaxStartInterval      int32
@@ -165,7 +171,7 @@ func TransformExposureKey(exposureKey verifyapi.ExposureKey, appPackageName stri
 	createdAt := settings.CreatedAt
 	// If the key is valid beyond the current interval number. Adjust the createdAt time for the key.
 	if exposureKey.IntervalNumber+exposureKey.IntervalCount > settings.MaxStartInterval {
-		// key is still valid. The created At for this key needs to be adjusted unless debuggin is enabled.
+		// key is still valid. The created At for this key needs to be adjusted unless debugging is enabled.
 		if !settings.ReleaseStillValidKeys {
 			createdAt = TimeForIntervalNumber(exposureKey.IntervalNumber + exposureKey.IntervalCount).Truncate(settings.BatchWindow)
 		}
@@ -238,25 +244,51 @@ func (t *Transformer) TransformPublish(ctx context.Context, inData *verifyapi.Pu
 		entities = append(entities, exposure)
 	}
 
-	// Ensure that the uploaded keys are for a consecutive time period. No
-	// overlaps and no gaps.
-	// 1) Sort by interval number.
+	// Validate the uploaded data meets configuration parameters.
+	// In v1.5+, it is possible to have multiple keys that overlap. They
+	// take the form of the same start interval with variable rolling period numbers.
+	// Sort by interval number to make necessary checks easier.
 	sort.Slice(entities, func(i int, j int) bool {
+		if entities[i].IntervalNumber == entities[j].IntervalNumber {
+			return entities[i].IntervalCount == entities[j].IntervalCount
+		}
 		return entities[i].IntervalNumber < entities[j].IntervalNumber
 	})
-	// 2) Walk the slice and verify no gaps/overlaps.
-	// We know the slice isn't empty, seed w/ the first interval.
-	nextInterval := entities[0].IntervalNumber
+	// Check that any overlapping keys meet configuration.
+	// Overlapping keys must have the same start interval. And there is a max number
+	// of "same day" keys that are allowed.
+	// We do not enforce that keys have UTC midnight aligned start intervals.
+
+	// Running count of start intervals.
+	startIntervals := make(map[int32]int)
+	lastInterval := entities[0].IntervalNumber
+	nextInterval := entities[0].IntervalNumber + entities[0].IntervalCount
+
 	for _, ex := range entities {
+		// Relies on the default value of 0 for the map value type.
+		startIntervals[ex.IntervalNumber] = startIntervals[ex.IntervalNumber] + 1
+
+		if ex.IntervalNumber == lastInterval {
+			// OK, overlaps by start interval. But move out the nextInterval
+			nextInterval = ex.IntervalNumber + ex.IntervalCount
+			continue
+		}
+
 		if ex.IntervalNumber < nextInterval {
-			if t.debugReleaseSameDay {
-				logging.FromContext(ctx).Errorf("exposure keys have overlapping intervals")
-				break
-			}
-			return nil, fmt.Errorf("exposure keys have overlapping intervals")
+			msg := fmt.Sprintf("exposure keys have non aligned overlapping intervals. %v overlaps with previous key that is good from %v to %v.", ex.IntervalNumber, lastInterval, nextInterval)
+			logging.FromContext(ctx).Errorf(msg)
+			return nil, fmt.Errorf(msg)
 		}
+		// OK, current key starts at or after the end of the previous one. Advance both variables.
+		lastInterval = ex.IntervalNumber
 		nextInterval = ex.IntervalNumber + ex.IntervalCount
 	}
 
+	for k, v := range startIntervals {
+		if v > t.maxSameDayKeys {
+			return nil, fmt.Errorf("too many overlapping keys for start interval: %v want: <= %v, got: %v", k, t.maxSameDayKeys, v)
+		}
+	}
+
 	return entities, nil
 }