Skip to content
This repository was archived by the owner on Oct 3, 2023. It is now read-only.

Commit 04f1fee

Browse files
mayurkale22mayurkale22
authored
Add support for Binary Format serializer for TagMap (#431)
* Add support for Binary Format serializer for TagMap * fix review comments 1. Add comments for MSB and REST constants 2. Remove VERSION_ID_OFFSET -> VERSION_ID_INDEX and b -> currentByte 3. Add backwards-compatible check on versionId * Add tests for the varint encoding/decoding
1 parent d3e2d7c commit 04f1fee

File tree

4 files changed

+374
-0
lines changed

4 files changed

+374
-0
lines changed

‎packages/opencensus-core/src/tags/propagation/binary-serializer.ts

+150
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
/**
2+
* Copyright 2019, OpenCensus Authors
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
/**
18+
* This module contains the functions for serializing and deserializing
19+
* TagMap (TagContext) with the binary format. It allows tags to propagate
20+
* across requests.
21+
*
22+
* <p>OpenCensus tag context encoding:
23+
*
24+
* <ul>
25+
* <li>Tags are encoded in single byte sequence. The version 0 format is:
26+
* <li>{@code <version_id><encoded_tags>}
27+
* <li>{@code <version_id> -> a single byte, value 0}
28+
* <li>{@code <encoded_tags> -> (<tag_field_id><tag_encoding>)*}
29+
* <ul>
30+
* <li>{@code <tag_field_id>} -> a single byte, value 0
31+
* <li>{@code <tag_encoding>}:
32+
* <ul>
33+
* <li>{@code <tag_key_len><tag_key><tag_val_len><tag_val>}
34+
* <ul>
35+
* <li>{@code <tag_key_len>} -> varint encoded integer
36+
* <li>{@code <tag_key>} -> tag_key_len bytes comprising tag name
37+
* <li>{@code <tag_val_len>} -> varint encoded integer
38+
* <li>{@code <tag_val>} -> tag_val_len bytes comprising tag value
39+
* </ul>
40+
* </li>
41+
* </ul>
42+
* </li>
43+
* </ul>
44+
* </ul>
45+
*/
46+
47+
import {TagMap} from '../tag-map';
48+
import {TagKey, TagValue} from '../types';
49+
import {DecodeVarint, EncodeVarint} from './variant-encoding';
50+
51+
// This size limit only applies to the bytes representing tag keys and values.
52+
export const TAG_MAP_SERIALIZED_SIZE_LIMIT = 8192;
53+
54+
const ENCODING = 'utf8';
55+
const VERSION_ID = 0;
56+
const TAG_FIELD_ID = 0;
57+
const VERSION_ID_INDEX = 0;
58+
59+
/**
60+
* Serializes a given TagMap to the on-the-wire format.
61+
* @param tagMap The TagMap to serialize.
62+
*/
63+
export function serializeBinary(tagMap: TagMap): Buffer {
64+
const byteArray: number[] = [];
65+
byteArray.push(VERSION_ID);
66+
let totalChars = 0;
67+
const tags = tagMap.tags;
68+
tags.forEach((tagValue: TagValue, tagKey: TagKey) => {
69+
totalChars += tagKey.name.length;
70+
totalChars += tagValue.value.length;
71+
encodeTag(tagKey, tagValue, byteArray);
72+
});
73+
74+
if (totalChars > TAG_MAP_SERIALIZED_SIZE_LIMIT) {
75+
throw new Error(`Size of TagMap exceeds the maximum serialized size ${
76+
TAG_MAP_SERIALIZED_SIZE_LIMIT}`);
77+
}
78+
return Buffer.from(byteArray);
79+
}
80+
81+
/**
82+
* Deserializes input to TagMap based on the binary format standard.
83+
* @param buffer The TagMap to deserialize.
84+
*/
85+
export function deserializeBinary(buffer: Buffer): TagMap {
86+
if (buffer.length === 0) {
87+
throw new Error('Input buffer can not be empty.');
88+
}
89+
const versionId = buffer.readInt8(VERSION_ID_INDEX);
90+
if (versionId > VERSION_ID) {
91+
throw new Error(`Wrong Version ID: ${
92+
versionId}. Currently supports version up to: ${VERSION_ID}`);
93+
}
94+
return parseTags(buffer);
95+
}
96+
97+
function encodeTag(tagKey: TagKey, tagValue: TagValue, byteArray: number[]) {
98+
byteArray.push(TAG_FIELD_ID);
99+
encodeString(tagKey.name, byteArray);
100+
encodeString(tagValue.value, byteArray);
101+
}
102+
103+
function encodeString(input: string, byteArray: number[]) {
104+
byteArray.push(...EncodeVarint(input.length));
105+
byteArray.push(...input.split('').map(unicode));
106+
return byteArray;
107+
}
108+
109+
function parseTags(buffer: Buffer): TagMap {
110+
const tags = new TagMap();
111+
const limit = buffer.length;
112+
let totalChars = 0;
113+
let currentIndex = 1;
114+
115+
while (currentIndex < limit) {
116+
const fieldId = buffer.readInt8(currentIndex);
117+
if (fieldId > TAG_FIELD_ID) {
118+
// Stop parsing at the first unknown field ID, since there is no way to
119+
// know its length.
120+
break;
121+
}
122+
currentIndex += 1;
123+
const key = decodeString(buffer, currentIndex);
124+
currentIndex += key.length;
125+
totalChars += key.length;
126+
127+
currentIndex += 1;
128+
const val = decodeString(buffer, currentIndex);
129+
currentIndex += val.length;
130+
totalChars += val.length;
131+
132+
currentIndex += 1;
133+
if (totalChars > TAG_MAP_SERIALIZED_SIZE_LIMIT) {
134+
throw new Error(`Size of TagMap exceeds the maximum serialized size ${
135+
TAG_MAP_SERIALIZED_SIZE_LIMIT}`);
136+
} else {
137+
tags.set({name: key}, {value: val});
138+
}
139+
}
140+
return tags;
141+
}
142+
143+
function decodeString(buffer: Buffer, offset: number): string {
144+
const length = DecodeVarint(buffer, offset);
145+
return buffer.toString(ENCODING, offset + 1, offset + 1 + length);
146+
}
147+
148+
function unicode(x: string) {
149+
return x.charCodeAt(0);
150+
}

‎packages/opencensus-core/src/tags/propagation/variant-encoding.ts

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
/**
2+
* Copyright 2019, OpenCensus Authors
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
// The MSB (most significant bit) indicates whether we've reached the end of
18+
// the number. Set means there is more than one byte in the varint.
19+
const MSB = 0x80;
20+
21+
// The REST indicates the lower 7 bits of each byte.
22+
const REST = 0x7F;
23+
24+
/**
25+
* Encodes a number in a variable-length encoding, 7 bits per byte.
26+
* @param value The input number.
27+
*/
28+
export function EncodeVarint(value: number) {
29+
const ret: number[] = [];
30+
do {
31+
const bits = value & REST;
32+
value >>>= 7;
33+
const b = bits + ((value !== 0) ? MSB : 0);
34+
ret.push(b);
35+
} while (value !== 0);
36+
return ret;
37+
}
38+
39+
/**
40+
* Decodes a varint from buffer.
41+
* @param buffer The source buffer.
42+
* @param offset The offset within buffer.
43+
*/
44+
export function DecodeVarint(buffer: Buffer, offset: number) {
45+
let ret = 0;
46+
let shift = 0;
47+
let currentByte;
48+
let counter = offset;
49+
do {
50+
if (shift >= 32) {
51+
throw new Error('varint too long');
52+
}
53+
currentByte = buffer.readInt8(counter++);
54+
ret |= (currentByte & REST) << shift;
55+
shift += 7;
56+
} while ((currentByte & MSB) !== 0);
57+
return ret;
58+
}

‎packages/opencensus-core/test/test-binary-serializer.ts

+116
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
/**
2+
* Copyright 2019, OpenCensus Authors
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
import * as assert from 'assert';
18+
import {deserializeBinary, serializeBinary, TAG_MAP_SERIALIZED_SIZE_LIMIT} from '../src/tags/propagation/binary-serializer';
19+
import {TagMap} from '../src/tags/tag-map';
20+
21+
const K1 = {
22+
name: 'k1'
23+
};
24+
const K2 = {
25+
name: 'k2'
26+
};
27+
const K3 = {
28+
name: 'k3'
29+
};
30+
const K4 = {
31+
name: 'k4'
32+
};
33+
34+
const V1 = {
35+
value: 'v1'
36+
};
37+
const V2 = {
38+
value: 'v2'
39+
};
40+
const V3 = {
41+
value: 'v3'
42+
};
43+
const V4 = {
44+
value: 'v4'
45+
};
46+
47+
describe('Binary Format Serializer', () => {
48+
const emptyTagMap = new TagMap();
49+
50+
const singleTagMap = new TagMap();
51+
singleTagMap.set(K1, V1);
52+
53+
const multipleTagMap = new TagMap();
54+
multipleTagMap.set(K1, V1);
55+
multipleTagMap.set(K2, V2);
56+
multipleTagMap.set(K3, V3);
57+
multipleTagMap.set(K4, V4);
58+
59+
describe('serializeBinary', () => {
60+
it('should serialize empty tag map', () => {
61+
const binary = serializeBinary(emptyTagMap);
62+
assert.deepEqual(deserializeBinary(binary), emptyTagMap);
63+
});
64+
65+
it('should serialize with one tag map', () => {
66+
const binary = serializeBinary(singleTagMap);
67+
assert.deepEqual(deserializeBinary(binary), singleTagMap);
68+
});
69+
70+
it('should serialize with multiple tag', () => {
71+
const binary = serializeBinary(multipleTagMap);
72+
assert.deepEqual(deserializeBinary(binary), multipleTagMap);
73+
});
74+
75+
it('should throw an error when exceeds the max serialized size', () => {
76+
const tags = new TagMap();
77+
for (let i = 0; i < TAG_MAP_SERIALIZED_SIZE_LIMIT / 8 - 1; i++) {
78+
// Each tag will be with format {key : "0123", value : "0123"}, so the
79+
// length of it is 8.
80+
const pad = '0000'.substring(0, 4 - `${i}`.length);
81+
const str = `${pad}${i}`;
82+
tags.set({name: str}, {value: str});
83+
}
84+
// The last tag will be of size 9, so the total size of the TagMap
85+
// (8193) will be one byte more than limit.
86+
tags.set({name: 'last'}, {value: 'last1'});
87+
88+
assert.throws(() => {
89+
serializeBinary(tags);
90+
}, /^Error: Size of TagMap exceeds the maximum serialized size 8192/);
91+
});
92+
});
93+
94+
describe('deserializeBinary', () => {
95+
it('should throw an error when invalid tagKey', () => {
96+
const buff =
97+
Buffer.from([0x01, 0x00, 0x02, 0x6b, 0x31, 0x02, 0x76, 0x31]);
98+
assert.throws(() => {
99+
deserializeBinary(buff);
100+
}, /^Error: Wrong Version ID: 1. Currently supports version up to: 0/);
101+
});
102+
103+
it('should stop parsing at first unknown field ID', () => {
104+
const expectedTags = new TagMap();
105+
expectedTags.set(K1, V1);
106+
107+
const buff = Buffer.from([
108+
0x00, 0x00, 0x02, 0x6b, 0x31, 0x02, 0x76, 0x31, 0x01, 0x02, 0x6b, 0x32,
109+
0x02, 0x76, 0x32
110+
]);
111+
const tags = deserializeBinary(buff);
112+
assert.equal(tags.tags.size, 1);
113+
assert.deepStrictEqual(tags, expectedTags);
114+
});
115+
});
116+
});

‎packages/opencensus-core/test/test-variant-encoding.ts

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
/**
2+
* Copyright 2019, OpenCensus Authors
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
import * as assert from 'assert';
18+
import {DecodeVarint, EncodeVarint} from '../src/tags/propagation/variant-encoding';
19+
20+
const testCases =
21+
[0, 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000];
22+
23+
function randint(range: number) {
24+
return Math.floor(Math.random() * range);
25+
}
26+
27+
describe('variant encoding', () => {
28+
it('should encode single byte', () => {
29+
const expected = randint(127);
30+
assert.deepEqual(EncodeVarint(expected), [expected]);
31+
});
32+
33+
it('should encode/decode multiple bytes', () => {
34+
const num = 300;
35+
const expectedBytes = [0xAC, 0x02]; // [172, 2]
36+
37+
const variant = EncodeVarint(num);
38+
assert.deepEqual(variant, expectedBytes);
39+
const buff = Buffer.from(variant);
40+
assert.equal(DecodeVarint(buff, 0), num);
41+
});
42+
43+
for (const testCase of testCases) {
44+
it(`should encode and decode ${testCase} correctly`, () => {
45+
const variant = EncodeVarint(testCase);
46+
const buff = Buffer.from(variant);
47+
assert.equal(DecodeVarint(buff, 0), testCase);
48+
});
49+
}
50+
});

0 commit comments

Comments
 (0)