Skip to content

Commit 0fe57b1

Browse files
authored
feat: exception rate limiter by type (#1994)
1 parent 93168cb commit 0fe57b1

File tree

15 files changed

+247
-116
lines changed

15 files changed

+247
-116
lines changed

playground/nextjs/README.md

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,21 +8,14 @@ NEXT_PUBLIC_POSTHOG_KEY='<your-local-api-key>' pnpm dev
88

99
Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
1010

11-
12-
### Against a locally running PostHog instance
13-
14-
```bash
15-
NEXT_PUBLIC_POSTHOG_KEY='<your-local-api-key>' NEXT_PUBLIC_POSTHOG_HOST='http://localhost:8010' pnpm dev
16-
```
17-
1811
### Testing local changes to posthog-js
1912

2013
Running `pnpm dev` will run an additional script that uses pnpm to link `posthog-js` locally to this package.
2114

2215
If you need to provide environment variables, you can do so:
2316

2417
```bash
25-
NEXT_PUBLIC_POSTHOG_KEY='<your-local-api-key>' NEXT_PUBLIC_POSTHOG_HOST='http://localhost:8000' pnpm dev
18+
NEXT_PUBLIC_POSTHOG_KEY='<your-local-api-key>' NEXT_PUBLIC_POSTHOG_HOST='http://localhost:8010' pnpm dev
2619
```
2720

2821
### Testing cross-subdomain tracking
@@ -32,12 +25,14 @@ subdomains to localhost. There are a few steps required to do this, these are th
3225
with Chrome:
3326

3427
Add the following to your /etc/host file:
28+
3529
```
3630
127.0.0.1 www.posthog.dev
3731
127.0.0.1 app.posthog.dev
3832
```
3933

4034
To restart your DNS server on MacOS, run:
35+
4136
```bash
4237
sudo killall -HUP mDNSResponder
4338
```
@@ -50,5 +45,6 @@ NEXT_PUBLIC_POSTHOG_KEY='<your-local-api-key>' NEXT_PUBLIC_POSTHOG_HOST='http://
5045

5146
You can now open the subdomains we added to the host file, but you will likely see a warning about unsafe certificates. To get around this, in Chrome you can type `thisisunsafe` to bypass the warning.
5247
The subdomains are:
53-
* [https://www.posthog.dev:3000](https://www.posthog.dev:3000)
54-
* [https://app.posthog.dev:3000](https://app.posthog.dev:3000)
48+
49+
- [https://www.posthog.dev:3000](https://www.posthog.dev:3000)
50+
- [https://app.posthog.dev:3000](https://app.posthog.dev:3000)

src/__tests__/__snapshots__/config-snapshot.test.ts.snap

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -314,11 +314,21 @@ exports[`config snapshot for PostHogConfig 1`] = `
314314
\\"undefined\\",
315315
\\"number\\"
316316
],
317-
\\"__mutationRateLimiterRefillRate\\": [
317+
\\"__mutationThrottlerRefillRate\\": [
318318
\\"undefined\\",
319319
\\"number\\"
320320
],
321-
\\"__mutationRateLimiterBucketSize\\": [
321+
\\"__mutationThrottlerBucketSize\\": [
322+
\\"undefined\\",
323+
\\"number\\"
324+
]
325+
},
326+
\\"error_tracking\\": {
327+
\\"__exceptionRateLimiterRefillRate\\": [
328+
\\"undefined\\",
329+
\\"number\\"
330+
],
331+
\\"__exceptionRateLimiterBucketSize\\": [
322332
\\"undefined\\",
323333
\\"number\\"
324334
]

src/__tests__/extensions/replay/mutation-rate-limiter.test.ts renamed to src/__tests__/extensions/replay/mutation-throttler.test.ts

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { MutationRateLimiter } from '../../../extensions/replay/mutation-rate-limiter'
1+
import { MutationThrottler } from '../../../extensions/replay/mutation-throttler'
22
import {
33
INCREMENTAL_SNAPSHOT_EVENT_TYPE,
44
MUTATION_SOURCE_TYPE,
@@ -25,7 +25,7 @@ const makeEvent = (mutations: {
2525
timestamp: 1,
2626
})
2727

28-
describe('MutationRateLimiter', () => {
28+
describe('MutationThrottler', () => {
2929
const mockGetNode = jest.fn()
3030
const mockGetId = jest.fn()
3131
const rrwebMock: jest.Mock<rrwebRecord> = {
@@ -35,15 +35,15 @@ describe('MutationRateLimiter', () => {
3535
},
3636
} as unknown as jest.Mock<rrwebRecord>
3737

38-
let mutationRateLimiter: MutationRateLimiter
38+
let mutationThrottler: MutationThrottler
3939
let onBlockedNodeMock: (id: number, node: Node | null) => void
4040

4141
beforeEach(() => {
4242
mockGetNode.mockReturnValueOnce({ nodeName: 'div' })
4343
mockGetId.mockReturnValueOnce(1)
4444

4545
onBlockedNodeMock = jest.fn()
46-
mutationRateLimiter = new MutationRateLimiter(rrwebMock as unknown as rrwebRecord, {
46+
mutationThrottler = new MutationThrottler(rrwebMock as unknown as rrwebRecord, {
4747
onBlockedNode: onBlockedNodeMock,
4848
})
4949
})
@@ -55,17 +55,17 @@ describe('MutationRateLimiter', () => {
5555
test('event is passed through unchanged when not throttled', () => {
5656
const event = makeEvent({})
5757

58-
const result = mutationRateLimiter.throttleMutations(event)
58+
const result = mutationThrottler.throttleMutations(event)
5959

6060
expect(result).toBe(event)
6161
})
6262

6363
test('returns undefined if no mutations are left', () => {
6464
const event = makeEvent({ attributes: [{ id: 1, attributes: { a: 'ttribute' } }] })
6565

66-
mutationRateLimiter['_mutationBuckets']['1'] = 0
66+
mutationThrottler['_rateLimiter']['_buckets']['1'] = 0
6767

68-
const result = mutationRateLimiter.throttleMutations(event)
68+
const result = mutationThrottler.throttleMutations(event)
6969

7070
expect(result).toBeUndefined()
7171
})
@@ -77,9 +77,9 @@ describe('MutationRateLimiter', () => {
7777
attributes: [{ id: 1, attributes: { a: 'ttribute' } }],
7878
})
7979

80-
mutationRateLimiter['_mutationBuckets']['1'] = 0
80+
mutationThrottler['_rateLimiter']['_buckets']['1'] = 0
8181

82-
const result = mutationRateLimiter.throttleMutations(event)
82+
const result = mutationThrottler.throttleMutations(event)
8383

8484
expect(result).toStrictEqual(
8585
makeEvent({
@@ -96,9 +96,9 @@ describe('MutationRateLimiter', () => {
9696
attributes: [{ id: 1, attributes: { a: 'ttribute' } }],
9797
})
9898

99-
mutationRateLimiter['_mutationBuckets']['1'] = 0
99+
mutationThrottler['_rateLimiter']['_buckets']['1'] = 0
100100

101-
const result = mutationRateLimiter.throttleMutations(event)
101+
const result = mutationThrottler.throttleMutations(event)
102102

103103
expect(result).toStrictEqual(
104104
makeEvent({
@@ -114,7 +114,7 @@ describe('MutationRateLimiter', () => {
114114
data: {},
115115
}
116116

117-
const result = mutationRateLimiter.throttleMutations(event as unknown as eventWithTime)
117+
const result = mutationThrottler.throttleMutations(event as unknown as eventWithTime)
118118

119119
expect(result).toBe(event)
120120
})

src/__tests__/posthog-core.loaded.test.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ describe('loaded() with flags', () => {
167167
const receivedFeatureFlagsSpy = jest.spyOn(instance.featureFlags, 'receivedFeatureFlags')
168168

169169
instance.featureFlags._callDecideEndpoint()
170-
jest.runAllTimers()
170+
jest.runOnlyPendingTimers()
171171

172172
if (expectedCall) {
173173
expect(receivedFeatureFlagsSpy).toHaveBeenCalledWith(expectedArgs, false)
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
import { jest } from '@jest/globals'
2+
import { BucketedRateLimiter } from '../../utils/bucketed-rate-limiter'
3+
4+
jest.useFakeTimers()
5+
6+
describe('BucketedRateLimiter', () => {
7+
let rateLimiter: BucketedRateLimiter<string>
8+
9+
beforeEach(() => {
10+
rateLimiter = new BucketedRateLimiter({
11+
bucketSize: 10,
12+
refillRate: 1,
13+
refillInterval: 1000,
14+
})
15+
})
16+
17+
afterEach(() => {
18+
jest.clearAllMocks()
19+
})
20+
21+
test('it is not rate limited by default', () => {
22+
const result = rateLimiter.consumeRateLimit('ResizeObserver')
23+
expect(result).toBe(false)
24+
})
25+
26+
test('returns true if no mutations are left', () => {
27+
rateLimiter['_buckets']['ResizeObserver'] = 0
28+
29+
const result = rateLimiter.consumeRateLimit('ResizeObserver')
30+
expect(result).toBe(true)
31+
})
32+
})

src/entrypoints/exception-autocapture.ts

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,15 @@
1-
import { errorToProperties, unhandledRejectionToProperties } from '../extensions/exception-autocapture/error-conversion'
1+
import {
2+
ErrorProperties,
3+
errorToProperties,
4+
unhandledRejectionToProperties,
5+
} from '../extensions/exception-autocapture/error-conversion'
26
import { assignableWindow, window } from '../utils/globals'
3-
import { ErrorEventArgs, Properties } from '../types'
7+
import { ErrorEventArgs } from '../types'
48
import { createLogger } from '../utils/logger'
59

610
const logger = createLogger('[ExceptionAutocapture]')
711

8-
const wrapOnError = (captureFn: (props: Properties) => void) => {
12+
const wrapOnError = (captureFn: (props: ErrorProperties) => void) => {
913
const win = window as any
1014
if (!win) {
1115
logger.info('window not available, cannot wrap onerror')
@@ -25,7 +29,7 @@ const wrapOnError = (captureFn: (props: Properties) => void) => {
2529
}
2630
}
2731

28-
const wrapUnhandledRejection = (captureFn: (props: Properties) => void) => {
32+
const wrapUnhandledRejection = (captureFn: (props: ErrorProperties) => void) => {
2933
const win = window as any
3034
if (!win) {
3135
logger.info('window not available, cannot wrap onUnhandledRejection')
@@ -46,7 +50,7 @@ const wrapUnhandledRejection = (captureFn: (props: Properties) => void) => {
4650
}
4751
}
4852

49-
const wrapConsoleError = (captureFn: (props: Properties) => void) => {
53+
const wrapConsoleError = (captureFn: (props: ErrorProperties) => void) => {
5054
const con = console as any
5155
if (!con) {
5256
logger.info('console not available, cannot wrap console.error')

src/extensions/exception-autocapture/error-conversion.ts

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ import {
1414
import { defaultStackParser, StackFrame, StackParser } from './stack-trace'
1515

1616
import { isEmptyString, isString, isUndefined } from '../../utils/type-utils'
17-
import { ErrorEventArgs, SeverityLevel, severityLevels } from '../../types'
17+
import { SeverityLevel, severityLevels } from '../../types'
1818
import { getFilenameToChunkIdMap } from './chunk-ids'
1919

2020
type ErrorConversionArgs = {
@@ -63,10 +63,6 @@ export interface Exception {
6363
}
6464
}
6565

66-
export interface ErrorConversions {
67-
errorToProperties: (args: ErrorEventArgs, metadata?: ErrorMetadata) => ErrorProperties
68-
unhandledRejectionToProperties: (args: [ev: PromiseRejectionEvent]) => ErrorProperties
69-
}
7066
/**
7167
* based on the very wonderful MIT licensed Sentry SDK
7268
*/

src/extensions/exception-autocapture/index.ts

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,18 @@
11
import { assignableWindow, window } from '../../utils/globals'
22
import { PostHog } from '../../posthog-core'
3-
import { ExceptionAutoCaptureConfig, Properties, RemoteConfig } from '../../types'
3+
import { ExceptionAutoCaptureConfig, RemoteConfig } from '../../types'
44

55
import { createLogger } from '../../utils/logger'
66
import { EXCEPTION_CAPTURE_ENABLED_SERVER_SIDE } from '../../constants'
77
import { isObject, isUndefined } from '../../utils/type-utils'
8+
import { ErrorProperties } from './error-conversion'
9+
import { BucketedRateLimiter } from '../../utils/bucketed-rate-limiter'
810

911
const logger = createLogger('[ExceptionAutocapture]')
1012

1113
export class ExceptionObserver {
1214
private _instance: PostHog
15+
private _rateLimiter: BucketedRateLimiter<string>
1316
private _remoteEnabled: boolean | undefined
1417
private _config: Required<ExceptionAutoCaptureConfig>
1518
private _unwrapOnError: (() => void) | undefined
@@ -21,6 +24,15 @@ export class ExceptionObserver {
2124
this._remoteEnabled = !!this._instance.persistence?.props[EXCEPTION_CAPTURE_ENABLED_SERVER_SIDE]
2225
this._config = this._requiredConfig()
2326

27+
// by default captures ten exceptions before rate limiting by exception type
28+
// refills at a rate of one token / 10 second period
29+
// e.g. will capture 1 exception rate limited exception every 10 seconds until burst ends
30+
this._rateLimiter = new BucketedRateLimiter({
31+
refillRate: this._instance.config.error_tracking.__exceptionRateLimiterRefillRate ?? 1,
32+
bucketSize: this._instance.config.error_tracking.__exceptionRateLimiterBucketSize ?? 10,
33+
refillInterval: 10000, // ten seconds in milliseconds
34+
})
35+
2436
this.startIfEnabled()
2537
}
2638

@@ -127,13 +139,23 @@ export class ExceptionObserver {
127139
this.startIfEnabled()
128140
}
129141

130-
captureException(errorProperties: Properties) {
142+
captureException(errorProperties: ErrorProperties) {
131143
const posthogHost = this._instance.requestRouter.endpointFor('ui')
132144

133145
errorProperties.$exception_personURL = `${posthogHost}/project/${
134146
this._instance.config.token
135147
}/person/${this._instance.get_distinct_id()}`
136148

149+
const exceptionType = errorProperties.$exception_list[0].type ?? 'Exception'
150+
const isRateLimited = this._rateLimiter.consumeRateLimit(exceptionType)
151+
152+
if (isRateLimited) {
153+
logger.info('Skipping exception capture because of client rate limiting.', {
154+
exception: errorProperties.$exception_list[0].type,
155+
})
156+
return
157+
}
158+
137159
this._instance.exceptions.sendExceptionEvent(errorProperties)
138160
}
139161
}

0 commit comments

Comments
 (0)