Skip to content

Commit 9b2e2ec

Browse files
chlowellchlowell
committed
Default test recording sanitizers
1 parent a45cf26 commit 9b2e2ec

File tree

4 files changed

+388
-5
lines changed

4 files changed

+388
-5
lines changed

‎sdk/internal/recording/default_sanitizers.go

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
// this file contains a set of default sanitizers applied to all recordings
5+
6+
package recording
7+
8+
import (
9+
"bytes"
10+
"encoding/json"
11+
"fmt"
12+
"io"
13+
"net/http"
14+
)
15+
16+
// sanitizer represents a single sanitizer configured via the test proxy's /Admin/AddSanitizers endpoint
17+
type sanitizer struct {
18+
// Name is the name of a sanitizer type e.g. "BodyKeySanitizer"
19+
Name string `json:"Name,omitempty"`
20+
Body sanitizerBody `json:"Body,omitempty"`
21+
}
22+
23+
type sanitizerBody struct {
24+
// GroupForReplace is the name of the regex group to replace
25+
GroupForReplace string `json:"groupForReplace,omitempty"`
26+
// JSONPath is the JSON path to the value to replace
27+
JSONPath string `json:"jsonPath,omitempty"`
28+
// Key is the name of a header to sanitize
29+
Key string `json:"key,omitempty"`
30+
// Regex is the regular expression to match a value to sanitize
31+
Regex string `json:"regex,omitempty"`
32+
// Value is the string that replaces the matched value. The sanitizers in
33+
// this file accept the test proxy's default Value, "Sanitized".
34+
Value string `json:"value,omitempty"`
35+
}
36+
37+
func newBodyKeySanitizer(jsonPath string) sanitizer {
38+
return sanitizer{
39+
Name: "BodyKeySanitizer",
40+
Body: sanitizerBody{
41+
JSONPath: jsonPath,
42+
},
43+
}
44+
}
45+
46+
func newBodyRegexSanitizer(regex, groupForReplace string) sanitizer {
47+
return sanitizer{
48+
Name: "BodyRegexSanitizer",
49+
Body: sanitizerBody{
50+
GroupForReplace: groupForReplace,
51+
Regex: regex,
52+
},
53+
}
54+
}
55+
56+
func newGeneralRegexSanitizer(regex, groupForReplace string) sanitizer {
57+
return sanitizer{
58+
Name: "GeneralRegexSanitizer",
59+
Body: sanitizerBody{
60+
GroupForReplace: groupForReplace,
61+
Regex: regex,
62+
},
63+
}
64+
}
65+
66+
func newHeaderRegexSanitizer(key, regex, groupForReplace string) sanitizer {
67+
return sanitizer{
68+
Name: "HeaderRegexSanitizer",
69+
Body: sanitizerBody{
70+
GroupForReplace: groupForReplace,
71+
Key: key,
72+
Regex: regex,
73+
},
74+
}
75+
}
76+
77+
// addSanitizers adds an arbitrary number of sanitizers with a single request. It
78+
// isn't exported because SDK modules don't add enough sanitizers to benefit from it.
79+
func addSanitizers(s []sanitizer, options *RecordingOptions) error {
80+
if options == nil {
81+
options = defaultOptions()
82+
}
83+
url := fmt.Sprintf("%s/Admin/AddSanitizers", options.baseURL())
84+
req, err := http.NewRequest(http.MethodPost, url, nil)
85+
if err != nil {
86+
return err
87+
}
88+
handleTestLevelSanitizer(req, options)
89+
b, err := json.Marshal(s)
90+
if err != nil {
91+
return err
92+
}
93+
req.Body = io.NopCloser(bytes.NewReader(b))
94+
req.ContentLength = int64(len(b))
95+
req.Header.Set("Content-Type", "application/json")
96+
return handleProxyResponse(client.Do(req))
97+
}
98+
99+
var defaultSanitizers = []sanitizer{
100+
newGeneralRegexSanitizer(`("|;)Secret=(?<secret>[^;]+)`, "secret"),
101+
newBodyKeySanitizer("$..refresh_token"),
102+
newHeaderRegexSanitizer("api-key", "", ""),
103+
newBodyKeySanitizer("$..access_token"),
104+
newBodyKeySanitizer("$..connectionString"),
105+
newBodyKeySanitizer("$..applicationSecret"),
106+
newBodyKeySanitizer("$..apiKey"),
107+
newBodyRegexSanitizer(`client_secret=(?<secret>[^&"]+)`, "secret"),
108+
newBodyRegexSanitizer(`client_assertion=(?<secret>[^&"]+)`, "secret"),
109+
newHeaderRegexSanitizer("x-ms-rename-source", "", ""),
110+
newHeaderRegexSanitizer("x-ms-file-rename-source-authorization", "", ""),
111+
newHeaderRegexSanitizer("x-ms-file-rename-source", "", ""),
112+
newHeaderRegexSanitizer("x-ms-encryption-key-sha256", "", ""),
113+
newHeaderRegexSanitizer("x-ms-encryption-key", "", ""),
114+
newHeaderRegexSanitizer("x-ms-copy-source-authorization", "", ""),
115+
newHeaderRegexSanitizer("x-ms-copy-source", "", ""),
116+
newBodyRegexSanitizer("token=(?<token>[^&]+)($|&)", "token"),
117+
newHeaderRegexSanitizer("subscription-key", "", ""),
118+
newBodyKeySanitizer("$..sshPassword"),
119+
newBodyKeySanitizer("$..secondaryKey"),
120+
newBodyKeySanitizer("$..runAsPassword"),
121+
newBodyKeySanitizer("$..primaryKey"),
122+
newHeaderRegexSanitizer("Location", "", ""),
123+
newGeneralRegexSanitizer(`("|;)[Aa]ccess[Kk]ey=(?<secret>[^;]+)`, "secret"),
124+
newGeneralRegexSanitizer(`("|;)[Aa]ccount[Kk]ey=(?<secret>[^;]+)`, "secret"),
125+
newBodyKeySanitizer("$..aliasSecondaryConnectionString"),
126+
newGeneralRegexSanitizer(`("|;)[Ss]hared[Aa]ccess[Kk]ey=(?<secret>[^;\"]+)`, "secret"),
127+
newHeaderRegexSanitizer("aeg-sas-token", "", ""),
128+
newHeaderRegexSanitizer("aeg-sas-key", "", ""),
129+
newHeaderRegexSanitizer("aeg-channel-name", "", ""),
130+
newBodyKeySanitizer("$..adminPassword"),
131+
newBodyKeySanitizer("$..administratorLoginPassword"),
132+
newBodyKeySanitizer("$..accessToken"),
133+
newBodyKeySanitizer("$..accessSAS"),
134+
newGeneralRegexSanitizer(`(?:(sv|sig|se|srt|ss|sp)=)(?<secret>[^&\"]+)`, "secret"), // SAS tokens
135+
newBodyKeySanitizer("$.value[*].key"),
136+
newBodyKeySanitizer("$.key"),
137+
newBodyKeySanitizer("$..userId"),
138+
newBodyKeySanitizer("$..urlSource"),
139+
newBodyKeySanitizer("$..uploadUrl"),
140+
newBodyKeySanitizer("$..token"),
141+
newBodyKeySanitizer("$..to"),
142+
newBodyKeySanitizer("$..tenantId"),
143+
newBodyKeySanitizer("$..targetResourceId"),
144+
newBodyKeySanitizer("$..targetModelLocation"),
145+
newBodyKeySanitizer("$..storageContainerWriteSas"),
146+
newBodyKeySanitizer("$..storageContainerUri"),
147+
newBodyKeySanitizer("$..storageContainerReadListSas"),
148+
newBodyKeySanitizer("$..storageAccountPrimaryKey"),
149+
newBodyKeySanitizer("$..storageAccount"),
150+
newBodyKeySanitizer("$..source"),
151+
newBodyKeySanitizer("$..secondaryReadonlyMasterKey"),
152+
newBodyKeySanitizer("$..secondaryMasterKey"),
153+
newBodyKeySanitizer("$..secondaryConnectionString"),
154+
newBodyKeySanitizer("$..scriptUrlSasToken"),
155+
newBodyKeySanitizer("$..scan"),
156+
newBodyKeySanitizer("$..sasUri"),
157+
newBodyKeySanitizer("$..resourceGroup"),
158+
newBodyKeySanitizer("$..privateKey"),
159+
newBodyKeySanitizer("$..principalId"),
160+
newBodyKeySanitizer("$..primaryReadonlyMasterKey"),
161+
newBodyKeySanitizer("$..primaryMasterKey"),
162+
newBodyKeySanitizer("$..primaryConnectionString"),
163+
newBodyKeySanitizer("$..password"),
164+
newBodyKeySanitizer("$..outputDataUri"),
165+
newBodyKeySanitizer("$..managedResourceGroupName"),
166+
newBodyKeySanitizer("$..logLink"),
167+
newBodyKeySanitizer("$..lastModifiedBy"),
168+
newBodyKeySanitizer("$..keyVaultClientSecret"),
169+
newBodyKeySanitizer("$..inputDataUri"),
170+
newBodyKeySanitizer("$..id"),
171+
newBodyKeySanitizer("$..httpHeader"),
172+
newBodyKeySanitizer("$..guardian"),
173+
newBodyKeySanitizer("$..functionKey"),
174+
newBodyKeySanitizer("$..from"),
175+
newBodyKeySanitizer("$..fencingClientPassword"),
176+
newBodyKeySanitizer("$..encryptedCredential"),
177+
newBodyKeySanitizer("$..credential"),
178+
newBodyKeySanitizer("$..createdBy"),
179+
newBodyKeySanitizer("$..containerUri"),
180+
newBodyKeySanitizer("$..clientSecret"),
181+
newBodyKeySanitizer("$..certificatePassword"),
182+
newBodyKeySanitizer("$..catalog"),
183+
newBodyKeySanitizer("$..azureBlobSource.containerUrl"),
184+
newBodyKeySanitizer("$..authHeader"),
185+
newBodyKeySanitizer("$..atlasKafkaSecondaryEndpoint"),
186+
newBodyKeySanitizer("$..atlasKafkaPrimaryEndpoint"),
187+
newBodyKeySanitizer("$..appkey"),
188+
newBodyKeySanitizer("$..appId"),
189+
newBodyKeySanitizer("$..acrToken"),
190+
newBodyKeySanitizer("$..accountKey"),
191+
newBodyKeySanitizer("$..AccessToken"),
192+
newBodyKeySanitizer("$..WEBSITE_AUTH_ENCRYPTION_KEY"),
193+
newBodyRegexSanitizer("-----BEGIN PRIVATE KEY-----\\\\n(?<key>.+\\\\n)*-----END PRIVATE KEY-----\\\\n", "key"),
194+
newBodyKeySanitizer("$..adminPassword.value"),
195+
newBodyKeySanitizer("$..decryptionKey"),
196+
}

‎sdk/internal/recording/default_sanitizers_test.go

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
package recording
5+
6+
import (
7+
"bytes"
8+
cryptorand "crypto/rand"
9+
"crypto/rsa"
10+
"crypto/tls"
11+
"crypto/x509"
12+
"encoding/json"
13+
"encoding/pem"
14+
"fmt"
15+
"io"
16+
"math/big"
17+
"net/http"
18+
"os"
19+
"path/filepath"
20+
"strings"
21+
"testing"
22+
"time"
23+
24+
"github.com/Azure/azure-sdk-for-go/sdk/internal/mock"
25+
"github.com/stretchr/testify/require"
26+
)
27+
28+
// newMockServerForProxy creates a mock.Server with TLS enabled and configures the test proxy to trust the Server's
29+
// cert. It assumes the test proxy is running and reachable using defaultOptions(), and handles closing the Server.
30+
func newMockServerForProxy(t *testing.T) *mock.Server {
31+
pk, err := rsa.GenerateKey(cryptorand.Reader, 2048)
32+
require.NoError(t, err)
33+
template := &x509.Certificate{
34+
NotAfter: time.Now().Add(time.Minute),
35+
SerialNumber: big.NewInt(1),
36+
}
37+
certBytes, err := x509.CreateCertificate(cryptorand.Reader, template, template, &pk.PublicKey, pk)
38+
require.NoError(t, err)
39+
srv, close := mock.NewTLSServer(
40+
mock.WithTLSConfig(&tls.Config{
41+
Certificates: []tls.Certificate{{
42+
Certificate: [][]byte{certBytes},
43+
PrivateKey: pk,
44+
}},
45+
}),
46+
)
47+
t.Cleanup(close)
48+
49+
// configure the proxy to trust the mock.Server's TLS cert
50+
c := *http.DefaultClient
51+
c.Transport = &http.Transport{
52+
TLSClientConfig: &tls.Config{
53+
InsecureSkipVerify: true,
54+
},
55+
}
56+
certPEM := string(pem.EncodeToMemory(&pem.Block{
57+
Bytes: certBytes,
58+
Type: "CERTIFICATE",
59+
}))
60+
res, err := c.Post(
61+
fmt.Sprintf("https://localhost:%d/Admin/SetRecordingOptions", defaultOptions().ProxyPort),
62+
"application/json",
63+
io.NopCloser(strings.NewReader(
64+
fmt.Sprintf(`{"Transport":{"TLSValidationCert":%q}}`, strings.ReplaceAll(certPEM, "\n", "")),
65+
)),
66+
)
67+
require.NoError(t, err)
68+
require.NotNil(t, res)
69+
if res.StatusCode != http.StatusOK {
70+
d, err := io.ReadAll(res.Body)
71+
require.NoError(t, err)
72+
require.Failf(t, "failed to configure proxy to trust mock server's TLS cert: %s", string(d))
73+
}
74+
return srv
75+
}
76+
77+
func TestDefaultSanitizers(t *testing.T) {
78+
before := recordMode
79+
defer func() { recordMode = before }()
80+
recordMode = RecordingMode
81+
82+
t.Setenv(proxyManualStartEnv, "false")
83+
proxy, err := StartTestProxy("", nil)
84+
require.NoError(t, err)
85+
defer func() {
86+
err := StopTestProxy(proxy)
87+
require.NoError(t, err)
88+
_ = os.Remove(filepath.Join("testdata", "recordings", t.Name()+".json"))
89+
}()
90+
91+
client, err := NewRecordingHTTPClient(t, nil)
92+
require.NoError(t, err)
93+
94+
srv := newMockServerForProxy(t)
95+
96+
// build a request and response containing all the values that should be sanitized by default
97+
fail := "FAIL"
98+
failSAS := strings.ReplaceAll("sv=*&sig=*&se=*&srt=*&ss=*&sp=*", "*", fail)
99+
q := "?sig=" + fail
100+
req, err := http.NewRequest(http.MethodGet, srv.URL()+q, nil)
101+
require.NoError(t, err)
102+
req.Header.Set("Content-Type", "application/json")
103+
resOpts := []mock.ResponseOption{mock.WithStatusCode(http.StatusOK)}
104+
body := map[string]any{}
105+
for _, s := range defaultSanitizers {
106+
switch s.Name {
107+
case "BodyKeySanitizer":
108+
k := strings.TrimLeft(s.Body.JSONPath, "$.")
109+
var v any = fail
110+
if before, after, found := strings.Cut(k, "."); found {
111+
// path is e.g. $..foo.bar, so this value would be in a nested object
112+
k = before
113+
if strings.HasSuffix(k, "[*]") {
114+
// path is e.g. $..foo[*].bar, so this value would be in an object array
115+
k = strings.TrimSuffix(k, "[*]")
116+
v = []map[string]string{{after: fail}}
117+
} else {
118+
v = map[string]string{after: fail}
119+
}
120+
}
121+
body[k] = v
122+
case "HeaderRegexSanitizer":
123+
// if there's no group specified, we can generate a matching value because this sanitizer
124+
// performs a simple replacement (this works provided the default regex sanitizers continue
125+
// to follow the convention of always naming a group)
126+
if s.Body.GroupForReplace == "" {
127+
req.Header.Set(s.Body.Key, fail)
128+
resOpts = append(resOpts, mock.WithHeader(s.Body.Key, fail))
129+
}
130+
default:
131+
// handle regex sanitizers below because generating matching values is tricky
132+
}
133+
}
134+
// add values matching body regex sanitizers
135+
for i, v := range []string{
136+
"client_secret=" + fail + "&client_assertion=" + fail,
137+
strings.ReplaceAll("-----BEGIN PRIVATE KEY-----\n*\n*\n*\n-----END PRIVATE KEY-----\n", "*", fail),
138+
failSAS,
139+
strings.Join([]string{"AccessKey", "accesskey", "Accesskey", "AccountKey", "SharedAccessKey"}, "="+fail+";") + "=" + fail,
140+
} {
141+
k := fmt.Sprint(i)
142+
require.NotContains(t, body, k, "test bug: body already has key %q", k)
143+
body[k] = v
144+
}
145+
// add values matching header regex sanitizers
146+
for _, h := range []string{"ServiceBusDlqSupplementaryAuthorization", "ServiceBusSupplementaryAuthorization", "SupplementaryAuthorization"} {
147+
req.Header.Set(h, failSAS)
148+
}
149+
150+
// set request and response bodies
151+
j, err := json.Marshal(body)
152+
require.NoError(t, err)
153+
req.Body = io.NopCloser(bytes.NewReader(j))
154+
srv.SetResponse(append(resOpts, mock.WithBody(j))...)
155+
156+
err = Start(t, packagePath, nil)
157+
require.NoError(t, err)
158+
resp, err := client.Do(req)
159+
require.NoError(t, err)
160+
err = Stop(t, nil)
161+
require.NoError(t, err)
162+
if resp.StatusCode != http.StatusOK {
163+
b, err := io.ReadAll(resp.Body)
164+
require.NoError(t, err)
165+
t.Fatal(string(b))
166+
}
167+
168+
b, err := os.ReadFile(fmt.Sprintf("./testdata/recordings/%s.json", t.Name()))
169+
require.NoError(t, err)
170+
if bytes.Contains(b, []byte(fail)) {
171+
var buf bytes.Buffer
172+
require.NoError(t, json.Indent(&buf, b, "", " "))
173+
t.Fatalf("%q shouldn't appear in this recording:\n%s%q shouldn't appear in the above recording", fail, buf.String(), fail)
174+
}
175+
}

0 commit comments

Comments
 (0)