Skip to content
This repository was archived by the owner on Oct 3, 2023. It is now read-only.

Add support for recording HTTP stats #370

Merged
merged 3 commits into from
Mar 1, 2019
Merged

Add support for recording HTTP stats #370

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
647a791
Add support for recording HTTP stats
mayurkale22 Feb 28, 2019
ae0c276
fix review comment
mayurkale22 Mar 1, 2019
790284a
enable noUnusedLocals
mayurkale22 Mar 1, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,10 @@ All notable changes to this project will be documented in this file.
- Add support for ```tags```, ```status``` and ```annotation``` in Zipkin exporter.
- Add support for Binary propagation format.
- Add support for object(```SpanOptions```) as an argument for ```startChildSpan``` function, similar to ```startRootSpan```.
- Add proto files to exporter-ocagent package. Fixes issue [#174](https://github.com/census-instrumentation/opencensus-node/issues/174).
- Add proto files to exporter-ocagent package. Fixes issue [#174](https://github.com/census-instrumentation/opencensus-node/issues/174).
- Remove `ConfigStream` behavior from exporter-ocagent. This was unstable and is not currently supported by any other language instrumentation.
- Change default exporter-ocagent port to `55678` to match the default OC Agent port.
- Add support for recording HTTP stats.

## 0.0.9 - 2019-02-12
- Add Metrics API.
Expand Down
9 changes: 1 addition & 8 deletions packages/opencensus-instrumentation-grpc/src/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,4 @@
*/

export * from './grpc';

import {registerAllGrpcViews, registerClientGrpcBasicViews, registerServerGrpcBasicViews} from './grpc-stats/stats-common';

export {
registerAllGrpcViews,
registerClientGrpcBasicViews,
registerServerGrpcBasicViews
};
export {registerAllGrpcViews, registerClientGrpcBasicViews, registerServerGrpcBasicViews} from './grpc-stats/stats-common';
24 changes: 13 additions & 11 deletions ...-instrumentation-http/src/http-metrics.ts → ...us-instrumentation-http/src/http-stats.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,13 @@
* limitations under the License.
*/

import {AggregationType, globalStats, MeasureUnit, Stats, View} from '@opencensus/core';
import {AggregationType, globalStats, Measure, MeasureUnit, Stats, View} from '@opencensus/core';

/**
* {@link Measure} for the client-side total bytes sent in request body (not
* including headers). This is uncompressed bytes.
*/
export const HTTP_CLIENT_SENT_BYTES = globalStats.createMeasureInt64(
export const HTTP_CLIENT_SENT_BYTES: Measure = globalStats.createMeasureInt64(
'opencensus.io/http/client/sent_bytes', MeasureUnit.BYTE,
'Client-side total bytes sent in request body (uncompressed)');

Expand All @@ -31,25 +31,27 @@ export const HTTP_CLIENT_SENT_BYTES = globalStats.createMeasureInt64(
* the Content-Length header. This is uncompressed bytes. Responses with no
* body should record 0 for this value.
*/
export const HTTP_CLIENT_RECEIVED_BYTES = globalStats.createMeasureInt64(
'opencensus.io/http/client/received_bytes', MeasureUnit.BYTE,
'Client-side total bytes received in response bodies (uncompressed)');
export const HTTP_CLIENT_RECEIVED_BYTES: Measure =
globalStats.createMeasureInt64(
'opencensus.io/http/client/received_bytes', MeasureUnit.BYTE,
'Client-side total bytes received in response bodies (uncompressed)');

/**
* {@link Measure} for the client-side time between first byte of request
* headers sent to last byte of response received, or terminal error.
*/
export const HTTP_CLIENT_ROUNDTRIP_LATENCY = globalStats.createMeasureDouble(
export const HTTP_CLIENT_ROUNDTRIP_LATENCY: Measure = globalStats.createMeasureDouble(
'opencensus.io/http/client/roundtrip_latency', MeasureUnit.MS,
'Client-side time between first byte of request headers sent to last byte of response received, or terminal error');

/**
* {@link Measure} for the server-side total bytes received in request body
* (not including headers). This is uncompressed bytes.
*/
export const HTTP_SERVER_RECEIVED_BYTES = globalStats.createMeasureInt64(
'opencensus.io/http/server/received_bytes', MeasureUnit.BYTE,
'Server-side total bytes received in request body (uncompressed)');
export const HTTP_SERVER_RECEIVED_BYTES: Measure =
globalStats.createMeasureInt64(
'opencensus.io/http/server/received_bytes', MeasureUnit.BYTE,
'Server-side total bytes received in request body (uncompressed)');

/**
* {@link Measure} for the server-side total bytes sent in response bodies
Expand All @@ -58,15 +60,15 @@ export const HTTP_SERVER_RECEIVED_BYTES = globalStats.createMeasureInt64(
* Content-Length header. This is uncompressed bytes. Responses with no
* body should record 0 for this value.
*/
export const HTTP_SERVER_SENT_BYTES = globalStats.createMeasureInt64(
export const HTTP_SERVER_SENT_BYTES: Measure = globalStats.createMeasureInt64(
'opencensus.io/http/server/sent_bytes', MeasureUnit.BYTE,
'Server-side total bytes sent in response bodies (uncompressed)');

/**
* {@link Measure} for the server-side time between first byte of request
* headers received to last byte of response sent, or terminal error.
*/
export const HTTP_SERVER_LATENCY = globalStats.createMeasureDouble(
export const HTTP_SERVER_LATENCY: Measure = globalStats.createMeasureDouble(
'opencensus.io/http/server/server_latency', MeasureUnit.MS,
'Server-side time between first byte of request headers received to last byte of response sent, or terminal error');

Expand Down
115 changes: 72 additions & 43 deletions packages/opencensus-instrumentation-http/src/http.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,22 @@
* limitations under the License.
*/

import {BasePlugin, CanonicalCode, Func, HeaderGetter, HeaderSetter, MessageEventType, RootSpan, Span, SpanKind, TraceOptions} from '@opencensus/core';
import * as httpModule from 'http';
import {BasePlugin, CanonicalCode, Func, HeaderGetter, HeaderSetter, MessageEventType, Span, SpanKind, TagMap, TraceOptions} from '@opencensus/core';
import {ClientRequest, ClientResponse, IncomingMessage, request, RequestOptions, ServerResponse} from 'http';
import * as semver from 'semver';
import * as shimmer from 'shimmer';
import * as url from 'url';
import * as uuid from 'uuid';
import {HttpPluginConfig, IgnoreMatcher} from './types';
import * as stats from './http-stats';
import {IgnoreMatcher} from './types';

export type HttpGetCallback = (res: httpModule.IncomingMessage) => void;
export type HttpModule = typeof httpModule;
export type RequestFunction = typeof httpModule.request;
export type HttpGetCallback = (res: IncomingMessage) => void;
export type RequestFunction = typeof request;

function isOpenCensusRequest(options: RequestOptions) {
return options && options.headers &&
!!options.headers['x-opencensus-outgoing-request'];
}

/** Http instrumentation plugin for Opencensus */
export class HttpPlugin extends BasePlugin {
Expand All @@ -47,10 +52,7 @@ export class HttpPlugin extends BasePlugin {
super(moduleName);
}


/**
* Patches HTTP incoming and outcoming request functions.
*/
/** Patches HTTP incoming and outcoming request functions. */
protected applyPatch() {
this.logger.debug('applying patch to %s@%s', this.moduleName, this.version);

Expand All @@ -72,9 +74,8 @@ export class HttpPlugin extends BasePlugin {
// https://nodejs.org/dist/latest/docs/api/http.html#http_http_get_options_callback
// https://github.com/googleapis/cloud-trace-nodejs/blob/master/src/plugins/plugin-http.ts#L198
return function getTrace(
options: httpModule.RequestOptions|string,
callback: HttpGetCallback) {
const req = httpModule.request(options, callback);
options: RequestOptions|string, callback: HttpGetCallback) {
const req = request(options, callback);
req.end();
return req;
};
Expand All @@ -95,7 +96,6 @@ export class HttpPlugin extends BasePlugin {
return this.moduleExports;
}


/** Unpatches all HTTP patched function. */
protected applyUnpatch(): void {
shimmer.unwrap(this.moduleExports, 'request');
Expand Down Expand Up @@ -149,7 +149,6 @@ export class HttpPlugin extends BasePlugin {
}
}


/**
* Creates spans for incoming requests, restoring spans' context if applied.
*/
Expand All @@ -164,10 +163,11 @@ export class HttpPlugin extends BasePlugin {
if (event !== 'request') {
return original.apply(this, arguments);
}

const request: httpModule.IncomingMessage = args[0];
const response: httpModule.ServerResponse = args[1];
const startTime = Date.now();
const request: IncomingMessage = args[0];
const response: ServerResponse = args[1];
const path = request.url ? url.parse(request.url).pathname || '' : '';
const method = request.method || 'GET';
plugin.logger.debug('%s plugin incomingRequest', plugin.moduleName);

if (plugin.isIgnored(
Expand Down Expand Up @@ -201,14 +201,15 @@ export class HttpPlugin extends BasePlugin {
// https://github.com/GoogleCloudPlatform/cloud-trace-nodejs/blob/master/src/plugins/plugin-connect.ts#L75)
const originalEnd = response.end;

response.end = function(this: httpModule.ServerResponse) {
response.end = function(this: ServerResponse) {
response.end = originalEnd;
const returned = response.end.apply(this, arguments);

const requestUrl = request.url ? url.parse(request.url) : null;
const host = headers.host || 'localhost';
const userAgent =
(headers['user-agent'] || headers['User-Agent']) as string;
const tags = new TagMap();

rootSpan.addAttribute(
HttpPlugin.ATTRIBUTE_HTTP_HOST,
Expand All @@ -217,21 +218,18 @@ export class HttpPlugin extends BasePlugin {
'$1',
));

rootSpan.addAttribute(
HttpPlugin.ATTRIBUTE_HTTP_METHOD, request.method || 'GET');

rootSpan.addAttribute(HttpPlugin.ATTRIBUTE_HTTP_METHOD, method);
if (requestUrl) {
rootSpan.addAttribute(
HttpPlugin.ATTRIBUTE_HTTP_PATH, requestUrl.pathname || '');
rootSpan.addAttribute(
HttpPlugin.ATTRIBUTE_HTTP_ROUTE, requestUrl.path || '');
tags.set(stats.HTTP_SERVER_ROUTE, {value: requestUrl.path || ''});
}

if (userAgent) {
rootSpan.addAttribute(
HttpPlugin.ATTRIBUTE_HTTP_USER_AGENT, userAgent);
}

rootSpan.addAttribute(
HttpPlugin.ATTRIBUTE_HTTP_STATUS_CODE,
response.statusCode.toString());
Expand All @@ -243,6 +241,12 @@ export class HttpPlugin extends BasePlugin {
rootSpan.addMessageEvent(
MessageEventType.RECEIVED, uuid.v4().split('-').join(''));

tags.set(stats.HTTP_SERVER_METHOD, {value: method});
tags.set(
stats.HTTP_SERVER_STATUS,
{value: response.statusCode.toString()});
HttpPlugin.recordStats(rootSpan.kind, tags, Date.now() - startTime);

rootSpan.end();
return returned;
};
Expand All @@ -253,20 +257,17 @@ export class HttpPlugin extends BasePlugin {
};
}


/**
* Creates spans for outgoing requests, sending spans' context for distributed
* tracing.
*/
protected getPatchOutgoingRequestFunction() {
return (original: Func<httpModule.ClientRequest>): Func<
httpModule.ClientRequest> => {
return (original: Func<ClientRequest>): Func<ClientRequest> => {
const plugin = this;
return function outgoingRequest(
options: httpModule.RequestOptions|string,
callback): httpModule.ClientRequest {
options: RequestOptions|string, callback): ClientRequest {
if (!options) {
return original.apply(this, arguments);
return original.apply(this, [options, callback]);
}

// Makes sure the url is an url object
Expand All @@ -280,11 +281,10 @@ export class HttpPlugin extends BasePlugin {
origin = `${parsedUrl.protocol || 'http:'}//${parsedUrl.host}`;
} else {
// Do not trace ourselves
if (options.headers &&
options.headers['x-opencensus-outgoing-request']) {
if (isOpenCensusRequest(options)) {
plugin.logger.debug(
'header with "x-opencensus-outgoing-request" - do not trace');
return original.apply(this, arguments);
return original.apply(this, [options, callback]);
}

try {
Expand All @@ -294,12 +294,12 @@ export class HttpPlugin extends BasePlugin {
}
method = options.method || 'GET';
origin = `${options.protocol || 'http:'}//${options.host}`;
} catch (e) {
} catch (ignore) {
}
}

const request: httpModule.ClientRequest =
original.apply(this, arguments);
const request: ClientRequest =
original.apply(this, [options, callback]);

if (plugin.isIgnored(
origin + pathname, request,
Expand All @@ -315,7 +315,6 @@ export class HttpPlugin extends BasePlugin {
kind: SpanKind.CLIENT,
};


// Checks if this outgoing request is part of an operation by checking
// if there is a current root span, if so, we create a child span. In
// case there is no root span, this means that the outgoing request is
Expand Down Expand Up @@ -343,10 +342,10 @@ export class HttpPlugin extends BasePlugin {
* @param options The arguments to the original function.
*/
private getMakeRequestTraceFunction(
// tslint:disable-next-line:no-any
request: httpModule.ClientRequest, options: httpModule.RequestOptions,
plugin: HttpPlugin): Func<httpModule.ClientRequest> {
return (span: Span): httpModule.ClientRequest => {
request: ClientRequest, options: RequestOptions,
plugin: HttpPlugin): Func<ClientRequest> {
return (span: Span): ClientRequest => {
const startTime = Date.now();
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could this be a high-accuracy timestamp? I believe some Node versions support performance.now(), which would be an easy swap in, or else you could use process.hrtime and do some conversion / use a utility.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For stats recoding, we want to record ms level granularity. Looks like Date.now() gives us that. WDYT?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, that's fine. Didn't realize the stats were only at ms granularity.

plugin.logger.debug('makeRequestTrace');

if (!span) {
Expand All @@ -365,17 +364,19 @@ export class HttpPlugin extends BasePlugin {
propagation.inject(setter, span.spanContext);
}

request.on('response', (response: httpModule.ClientResponse) => {
request.on('response', (response: ClientResponse) => {
plugin.tracer.wrapEmitter(response);
plugin.logger.debug('outgoingRequest on response()');

response.on('end', () => {
plugin.logger.debug('outgoingRequest on end()');
const method = response.method ? response.method : 'GET';
const headers = options.headers;
const userAgent =
headers ? (headers['user-agent'] || headers['User-Agent']) : null;

const tags = new TagMap();
tags.set(stats.HTTP_CLIENT_METHOD, {value: method});

if (options.hostname) {
span.addAttribute(HttpPlugin.ATTRIBUTE_HTTP_HOST, options.hostname);
}
Expand All @@ -394,12 +395,16 @@ export class HttpPlugin extends BasePlugin {
HttpPlugin.ATTRIBUTE_HTTP_STATUS_CODE,
response.statusCode.toString());
span.setStatus(HttpPlugin.parseResponseStatus(response.statusCode));
tags.set(
stats.HTTP_CLIENT_STATUS,
{value: response.statusCode.toString()});
}

// Message Event ID is not defined
span.addMessageEvent(
MessageEventType.SENT, uuid.v4().split('-').join(''));

HttpPlugin.recordStats(span.kind, tags, Date.now() - startTime);
span.end();
});

Expand Down Expand Up @@ -457,6 +462,30 @@ export class HttpPlugin extends BasePlugin {
}
}
}

/** Method to record stats for client and server. */
static recordStats(kind: SpanKind, tags: TagMap, ms: number) {
if (!plugin.stats) {
return;
}

try {
const measureList = [];
switch (kind) {
case SpanKind.CLIENT:
measureList.push(
{measure: stats.HTTP_CLIENT_ROUNDTRIP_LATENCY, value: ms});
break;
case SpanKind.SERVER:
measureList.push({measure: stats.HTTP_SERVER_LATENCY, value: ms});
break;
default:
break;
}
plugin.stats.record(measureList, tags);
} catch (ignore) {
}
}
}

const plugin = new HttpPlugin('http');
Expand Down
1 change: 1 addition & 0 deletions packages/opencensus-instrumentation-http/src/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,4 @@
*/

export * from './http';
export {registerAllClientViews, registerAllServerViews, registerAllViews} from './http-stats';
Loading