CBG-4767: Make _raw endpoint return actual state of persisted doc (#7649)

Author: bbrks

base/constants.go

@@ -157,6 +157,15 @@ const (
 	FromConnStrWarningThreshold = 10 * time.Second
 )
 
+// SyncGatewayRawDocXattrs is a list of xattrs that Sync Gateway will fetch when reading a raw document.
+var SyncGatewayRawDocXattrs = []string{
+	SyncXattrName,
+	GlobalXattrName,
+	VvXattrName,
+	MouXattrName,
+	VirtualDocumentXattr,
+}
+
 const (
 	DefaultScope             = "_default"
 	DefaultCollection        = "_default"

base/error.go

@@ -104,7 +104,7 @@ func (err *HTTPError) Error() string {
 
 // HTTPErrorf creates an HTTPError with an http status code and a formatted message.
 func HTTPErrorf(status int, format string, args ...interface{}) *HTTPError {
-	return NewHTTPError(status, fmt.Sprintf(format, args...))
+	return NewHTTPError(status, fmt.Errorf(format, args...).Error())
 }
 
 // NewHTTPError creates an HTTPError with an http status code and a message. Use HTTPErrorf for printf style messages.

channels/main_test.go

@@ -24,7 +24,7 @@ func TestMain(m *testing.M) {
 	teardownFuncs := make([]func(), 0)
 	teardownFuncs = append(teardownFuncs, base.SetUpGlobalTestLogging(ctx))
 	teardownFuncs = append(teardownFuncs, base.SetUpGlobalTestProfiling(m))
-	teardownFuncs = append(teardownFuncs, base.SetUpGlobalTestMemoryWatermark(m, 256))
+	teardownFuncs = append(teardownFuncs, base.SetUpGlobalTestMemoryWatermark(m, 512))
 
 	base.SkipPrometheusStatsRegistration = true
 

db/database_collection.go

@@ -10,12 +10,15 @@ package db
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"slices"
 	"time"
 
 	"github.com/couchbase/sync_gateway/auth"
 	"github.com/couchbase/sync_gateway/base"
 	"github.com/couchbase/sync_gateway/channels"
+	"github.com/google/uuid"
 )
 
 // DatabaseCollection provides a representation of a single collection of a database.
@@ -59,6 +62,79 @@ func (c *DatabaseCollection) AllowConflicts() bool {
 	return base.DefaultAllowConflicts
 }
 
+type GetRawDocOpts struct {
+	IncludeDoc bool // If true, the document body will be returned as well as the metadata.
+	Redact     bool
+	RedactSalt string
+}
+
+// GetRawDoc returns the persisted version of a document, including its SG-related xattrs.
+func (c *DatabaseCollection) GetRawDoc(ctx context.Context, docID string, opts *GetRawDocOpts) (docBody json.RawMessage, xattrs map[string]json.RawMessage, err error) {
+	if opts == nil {
+		// default
+		opts = &GetRawDocOpts{
+			IncludeDoc: true,
+		}
+	}
+
+	if opts.Redact && opts.RedactSalt == "" {
+		// generate a random salt if one is not provided
+		opts.RedactSalt = uuid.New().String()
+	} else if !opts.Redact && opts.RedactSalt != "" {
+		return nil, nil, fmt.Errorf("RedactSalt provided to GetRawDoc but Redact is not enabled")
+	}
+
+	// collect the set of xattrs to fetch that Sync Gateway is interested in.
+	xattrKeys := slices.Clone(base.SyncGatewayRawDocXattrs)
+	userXattrKey := c.dbCtx.Options.UserXattrKey
+	if userXattrKey != "" {
+		// we'll redact this later after fetching - it's still useful to know it was present even if we can't see the contents.
+		xattrKeys = append(xattrKeys, userXattrKey)
+	}
+
+	var xattrValuesBytes map[string][]byte
+	if opts.IncludeDoc {
+		docBody, xattrValuesBytes, _, err = c.dataStore.GetWithXattrs(ctx, docID, xattrKeys)
+	} else {
+		xattrValuesBytes, _, err = c.dataStore.GetXattrs(ctx, docID, xattrKeys)
+	}
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// stamp all requested xattrKeys and populate with values where appropriate (ensures null values are present)
+	xattrs = make(map[string]json.RawMessage, len(xattrValuesBytes))
+	for _, k := range xattrKeys {
+		if opts.Redact {
+			switch k {
+			case base.SyncXattrName:
+				redactedV, err := RedactRawSyncData(xattrValuesBytes[k], opts.RedactSalt)
+				if err != nil {
+					return nil, nil, fmt.Errorf("couldn't redact sync data: %w", err)
+				}
+				xattrs[k] = redactedV
+			case base.GlobalXattrName:
+				redactedV, err := RedactRawGlobalSyncData(xattrValuesBytes[k], opts.RedactSalt)
+				if err != nil {
+					return nil, nil, fmt.Errorf("couldn't redact global sync data: %w", err)
+				}
+				xattrs[k] = redactedV
+			case userXattrKey:
+				// include the key but not the value so we can see _something_ was present.
+				xattrs[k] = []byte(`"redacted"`)
+			default:
+				// no redaction for this key
+				xattrs[k] = xattrValuesBytes[k]
+			}
+		} else {
+			// redaction disabled
+			xattrs[k] = xattrValuesBytes[k]
+		}
+	}
+
+	return docBody, xattrs, nil
+}
+
 func (c *DatabaseCollection) GetCollectionDatastore() base.DataStore {
 	return c.dataStore
 }

db/document.go

@@ -13,6 +13,7 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/base64"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"math"
@@ -131,71 +132,105 @@ func (sd *SyncData) getCurrentChannels() base.Set {
 	return ch
 }
 
-func (sd *SyncData) HashRedact(salt string) SyncData {
-
-	// Creating a new SyncData with the redacted info. We copy all the information which stays the same and create new
-	// items for the redacted data. The data to be redacted is populated below.
-	redactedSyncData := SyncData{
-		CurrentRev:      sd.CurrentRev,
-		NewestRev:       sd.NewestRev,
-		Flags:           sd.Flags,
-		Sequence:        sd.Sequence,
-		UnusedSequences: sd.UnusedSequences,
-		RecentSequences: sd.RecentSequences,
-		History:         RevTree{},
-		Channels:        channels.ChannelMap{},
-		Access:          UserAccessMap{},
-		RoleAccess:      UserAccessMap{},
-		Expiry:          sd.Expiry,
-		Cas:             sd.Cas,
-		Crc32c:          sd.Crc32c,
-		TombstonedAt:    sd.TombstonedAt,
-		Attachments:     AttachmentsMeta{},
-	}
-
-	// Populate and redact channels
+// RedactRawGlobalSyncData runs HashRedact on the given global sync data.
+func RedactRawGlobalSyncData(syncData []byte, redactSalt string) ([]byte, error) {
+	if redactSalt == "" {
+		return nil, fmt.Errorf("redact salt must be set")
+	}
+
+	var gsd GlobalSyncData
+	if err := json.Unmarshal(syncData, &gsd); err != nil {
+		return nil, fmt.Errorf("couldn't unmarshal sync data: %w", err)
+	}
+
+	if err := gsd.HashRedact(redactSalt); err != nil {
+		return nil, fmt.Errorf("couldn't redact global sync data: %w", err)
+	}
+
+	return json.Marshal(gsd)
+}
+
+// HashRedact does in-place redaction of UserData inside GlobalSyncData, by hashing attachment names.
+func (gsd *GlobalSyncData) HashRedact(salt string) error {
+	for k, v := range gsd.GlobalAttachments {
+		gsd.GlobalAttachments[base.Sha1HashString(k, salt)] = v
+		delete(gsd.GlobalAttachments, k)
+	}
+	return nil
+}
+
+// RedactRawSyncData runs HashRedact on the given sync data.
+func RedactRawSyncData(syncData []byte, redactSalt string) ([]byte, error) {
+	if redactSalt == "" {
+		return nil, fmt.Errorf("redact salt must be set")
+	}
+
+	var sd SyncDataAlias
+	if err := json.Unmarshal(syncData, &sd); err != nil {
+		return nil, fmt.Errorf("couldn't unmarshal sync data: %w", err)
+	}
+
+	if err := sd.HashRedact(redactSalt); err != nil {
+		return nil, fmt.Errorf("couldn't redact sync data: %w", err)
+	}
+
+	return json.Marshal(sd)
+}
+
+// HashRedact does in-place redaction of UserData inside SyncData, by hashing channel names, user access, role access, and attachment names.
+func (sd *SyncDataAlias) HashRedact(salt string) error {
+
+	// Redact channel names
 	for k, v := range sd.Channels {
-		redactedSyncData.Channels[base.Sha1HashString(k, salt)] = v
+		sd.Channels[base.Sha1HashString(k, salt)] = v
+		delete(sd.Channels, k)
+	}
+	for i, v := range sd.ChannelSet {
+		sd.ChannelSet[i].Name = base.Sha1HashString(v.Name, salt)
+	}
+	for i, v := range sd.ChannelSetHistory {
+		sd.ChannelSetHistory[i].Name = base.Sha1HashString(v.Name, salt)
 	}
 
-	// Populate and redact history. This is done as it also includes channel names
+	// Redact history. This is done as it also includes channel names
 	for k, revInfo := range sd.History {
-
 		if revInfo.Channels != nil {
-			redactedChannels := base.Set{}
+			redactedChannels := make(base.Set, len(revInfo.Channels))
 			for existingChanKey := range revInfo.Channels {
 				redactedChannels.Add(base.Sha1HashString(existingChanKey, salt))
 			}
 			revInfo.Channels = redactedChannels
 		}
-
-		redactedSyncData.History[k] = revInfo
+		sd.History[k] = revInfo
 	}
 
-	// Populate and redact user access
+	// Redact user access
 	for k, v := range sd.Access {
 		accessTimerSet := map[string]channels.VbSequence{}
 		for channelName, vbStats := range v {
 			accessTimerSet[base.Sha1HashString(channelName, salt)] = vbStats
 		}
-		redactedSyncData.Access[base.Sha1HashString(k, salt)] = accessTimerSet
+		sd.Access[base.Sha1HashString(k, salt)] = accessTimerSet
+		delete(sd.Access, k)
 	}
 
-	// Populate and redact user role access
+	// Redact user role access
 	for k, v := range sd.RoleAccess {
 		accessTimerSet := map[string]channels.VbSequence{}
 		for channelName, vbStats := range v {
 			accessTimerSet[base.Sha1HashString(channelName, salt)] = vbStats
 		}
-		redactedSyncData.RoleAccess[base.Sha1HashString(k, salt)] = accessTimerSet
+		sd.RoleAccess[base.Sha1HashString(k, salt)] = accessTimerSet
+		delete(sd.RoleAccess, k)
 	}
 
-	// Populate and redact attachment names
+	// Redact attachment names (pre-4.0 attachment location)
 	for k, v := range sd.Attachments {
-		redactedSyncData.Attachments[base.Sha1HashString(k, salt)] = v
+		sd.Attachments[base.Sha1HashString(k, salt)] = v
+		delete(sd.Attachments, k)
 	}
 
-	return redactedSyncData
+	return nil
 }
 
 // A document as stored in Couchbase. Contains the body of the current revision plus metadata.