Repository URL to install this package:
|
Version:
0.6.5 ▾
|
/*
* (c) Copyright IBM Corp. 2021
* (c) Copyright Instana Inc. and contributors 2019
*/
'use strict';
// eslint-disable-next-line import/order
const environmentUtil = require('./environment');
const uninstrumented = require('./uninstrumentedHttp');
const constants = require('./constants');
let logger = require('./console_logger');
const layerExtensionHostname = 'localhost';
const layerExtensionPort = 7365;
let useLambdaExtension = false;
const timeoutEnvVar = 'INSTANA_TIMEOUT';
let defaultTimeout = 500;
const layerExtensionTimeout = 300;
let backendTimeout = defaultTimeout;
const proxyEnvVar = 'INSTANA_ENDPOINT_PROXY';
let proxyAgent;
let stopSendingOnFailure = true;
let propagateErrorsUpstream = false;
let requestHasFailed = false;
let warningsHaveBeenLogged = false;
const disableCaCheckEnvVar = 'INSTANA_DISABLE_CA_CHECK';
const disableCaCheck = process.env[disableCaCheckEnvVar] === 'true';
if (process.env[proxyEnvVar] && !environmentUtil.sendUnencrypted) {
const proxyUrl = process.env[proxyEnvVar];
logger.info(
`The environment variable ${proxyEnvVar} is set. Requests to the Instana back end will be routed via a proxy ` +
`server: ${proxyUrl}.`
);
const HttpsProxyAgent = require('https-proxy-agent');
proxyAgent = new HttpsProxyAgent(proxyUrl);
} else if (process.env[proxyEnvVar] && environmentUtil.sendUnencrypted) {
logger.warn(
`Both ${proxyEnvVar} and ${environmentUtil.sendUnencryptedEnvVar} are set, but this combination is not supported.` +
' Requests to the Instana back end will not be routed via a proxy server.'
);
}
let hostHeader;
exports.init = function init(
identityProvider,
_logger,
_stopSendingOnFailure,
_propagateErrorsUpstream,
_defaultTimeout,
_useLambdaExtension
) {
stopSendingOnFailure = _stopSendingOnFailure == null ? true : _stopSendingOnFailure;
propagateErrorsUpstream = _propagateErrorsUpstream == null ? false : _propagateErrorsUpstream;
defaultTimeout = _defaultTimeout == null ? defaultTimeout : _defaultTimeout;
useLambdaExtension = _useLambdaExtension;
backendTimeout = defaultTimeout;
if (process.env[timeoutEnvVar]) {
backendTimeout = parseInt(process.env[timeoutEnvVar], 10);
if (isNaN(backendTimeout) || backendTimeout < 0) {
logger.warn(
`The value of ${timeoutEnvVar} (${process.env[timeoutEnvVar]}) cannot be parsed to a valid numerical value. ` +
`Will fall back to the default timeout (${defaultTimeout} ms).`
);
backendTimeout = defaultTimeout;
}
}
if (identityProvider) {
hostHeader = identityProvider.getHostHeader();
if (hostHeader == null) {
hostHeader = 'nodejs-serverless';
}
} else {
hostHeader = 'nodejs-serverless';
}
if (_logger) {
logger = _logger;
}
requestHasFailed = false;
// Heartbeat is only for the AWS Lambda extension
// IMPORTANT: the @supertenant/aws-lambda package will not
// send data once. It can happen all the time till the Lambda handler dies!
// SpanBuffer sends data asap and when the handler is finished the rest is sent.
if (useLambdaExtension) {
scheduleLambdaExtensionHeartbeatRequest();
}
};
exports.setLogger = function setLogger(_logger) {
logger = _logger;
};
exports.sendBundle = function sendBundle(bundle, finalLambdaRequest, callback) {
send('/bundle', bundle, finalLambdaRequest, callback);
};
exports.sendMetrics = function sendMetrics(metrics, callback) {
send('/metrics', metrics, false, callback);
};
exports.sendSpans = function sendSpans(spans, callback) {
send('/traces', spans, false, callback);
};
let heartbeatInterval;
function scheduleLambdaExtensionHeartbeatRequest() {
const executeHeartbeat = () => {
logger.debug('Executing Heartbeat request to Lambda extension.');
const req = uninstrumented.http.request(
{
hostname: layerExtensionHostname,
port: layerExtensionPort,
path: '/heartbeat',
method: 'POST',
// This sets a timeout for establishing the socket connection, see setTimeout below for a timeout for an
// idle connection after the socket has been opened.
timeout: layerExtensionTimeout
},
res => {
if (res.statusCode === 200) {
logger.debug('The Instana Lambda extension Heartbeat request has succeeded.');
} else {
handleHeartbeatError(
new Error(
`The Instana Lambda extension Heartbeat request has returned an unexpected status code: ${res.statusCode}`
)
);
}
}
);
function handleHeartbeatError(e) {
// Make sure we do not try to talk to the Lambda extension again.
useLambdaExtension = false;
clearInterval(heartbeatInterval);
logger.debug(
'The Instana Lambda extension Heartbeat request did not succeed. Falling back to talking to the Instana back ' +
'end directly.',
e
);
}
req.on('error', e => {
// req.destroyed indicates that we have run into a timeout and have already handled the timeout error.
if (req.destroyed) {
return;
}
handleHeartbeatError(e);
});
// Handle timeouts that occur after connecting to the socket (no response from the extension).
req.setTimeout(layerExtensionTimeout, () => {
handleHeartbeatError(new Error('The Lambda extension Heartbeat request timed out.'));
// Destroy timed out request manually as mandated in https://nodejs.org/api/http.html#event-timeout.
if (req && !req.destroyed) {
try {
destroyRequest(req);
} catch (e) {
// ignore
}
}
});
req.end();
};
// call immediately
executeHeartbeat();
// NOTE: it is fine to use interval, because the req timeout is 300ms and the interval is 500
heartbeatInterval = setInterval(executeHeartbeat, 500);
heartbeatInterval.unref();
}
function getTransport(localUseLambdaExtension) {
if (localUseLambdaExtension) {
// The Lambda extension is always HTTP without TLS on localhost.
return uninstrumented.http;
} else {
return environmentUtil.sendUnencrypted ? uninstrumented.http : uninstrumented.https;
}
}
function getBackendTimeout(localUseLambdaExtension) {
return localUseLambdaExtension ? layerExtensionTimeout : backendTimeout;
}
function send(resourcePath, payload, finalLambdaRequest, callback) {
// We need a local copy of the global useLambdaExtension variable, otherwise it might be changed concurrently by
// scheduleLambdaExtensionHeartbeatRequest. But we need to remember the value at the time we _started_ the request to
// decide whether to fall back to sending to the back end directly or give up sending data completely.
let localUseLambdaExtension = useLambdaExtension;
if (requestHasFailed && stopSendingOnFailure) {
logger.info(
`Not attempting to send data to ${resourcePath} as a previous request has already timed out or failed.`
);
callback();
return;
} else {
logger.debug(`Sending data to Instana (${resourcePath}).`);
}
if (!warningsHaveBeenLogged) {
warningsHaveBeenLogged = true;
if (environmentUtil.sendUnencrypted) {
logger.error(
`${environmentUtil.sendUnencryptedEnvVar} is set, which means that all traffic to Instana is send ` +
'unencrypted via plain HTTP, not via HTTPS. This will effectively make that traffic public. This setting ' +
'should never be used in production.'
);
}
if (disableCaCheck) {
logger.warn(
`${disableCaCheckEnvVar} is set, which means that the server certificate will not be verified against ` +
'the list of known CAs. This makes your service vulnerable to MITM attacks when connecting to Instana. ' +
'This setting should never be used in production, unless you use our on-premises product and are unable to ' +
'operate the Instana back end with a certificate with a known root CA.'
);
}
}
// prepend backend's path if the configured URL has a path component
const requestPath =
localUseLambdaExtension || environmentUtil.getBackendPath() === '/'
? resourcePath
: environmentUtil.getBackendPath() + resourcePath;
// serialize the payload object
const serializedPayload = JSON.stringify(payload);
const options = {
hostname: localUseLambdaExtension ? layerExtensionHostname : environmentUtil.getBackendHost(),
port: localUseLambdaExtension ? layerExtensionPort : environmentUtil.getBackendPort(),
path: requestPath,
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Content-Length': Buffer.byteLength(serializedPayload),
[constants.xInstanaHost]: hostHeader,
[constants.xInstanaKey]: environmentUtil.getInstanaAgentKey()
},
rejectUnauthorized: !disableCaCheck
};
options.timeout = getBackendTimeout(localUseLambdaExtension);
if (proxyAgent && !localUseLambdaExtension) {
options.agent = proxyAgent;
}
let req;
const skipWaitingForHttpResponse = !proxyAgent && !localUseLambdaExtension;
const transport = getTransport(localUseLambdaExtension);
if (skipWaitingForHttpResponse) {
// If the Lambda extension is not available to act as a proxy between the Lambda and serverless-acceptor (and
// additionally, if no user-configured proxy is in place), we change the HTTP handling a bit to reduce the time
// we keep the Lamdba alive: We deliberately do not pass a callback when calling https.request but instead we pass
// the callback to req.end. This way, we do not wait for the HTTP _response_, but we still make sure the request
// data is written to the network completely. This reduces the delay we add to the Lambda execution time to
// report metrics and traces quite a bit. The (acceptable) downside is that we do not get to examine the response
// for HTTP status codes.
req = transport.request(options);
} else {
// If (a) our Lambda extension is available, or if (b) a user-provided proxy is in use, we do *not* apply the
// optimization outlined above. Instead, we opt for the more traditional workflow of waiting until the HTTP response
// has been received. For the case (a) it is simply not necessary because the request to the Lambda extension is
// happening on localhost and will be very fast. For case (b), the reason is that some proxies interact in weird
// ways with the HTTP flow.
//
// See the req.end(serializedPayload) call below, too. In the no-extension/no-proxy case, that call has the callback
// to end the processing. Otherwise, the callback is provided here to http.request().
req = transport.request(options, () => {
// This is the final request from an AWS Lambda. In some scenarios we might get a stale timeout event on the
// socket object from a previous invocation of the Lambda handler, that is, from before the AWS Lambda runtime
// froze the Node.js process. Explicitly removing all listeners on the req object helps with that.
if (finalLambdaRequest) {
req.removeAllListeners();
req.on('error', () => {});
}
callback();
});
}
// See above for the difference between the timeout attribute in the request options and handling the 'timeout'
// event. This only adds a read timeout after the connection has been established and we need the timout attribute
// in the request options additionally for protection against cases where *connecting* to the socket takes too long,
// see https://nodejs.org/api/http.html#http_request_settimeout_timeout_callback:
// > Once a socket is assigned to this request **and is connected**
// > socket.setTimeout() will be called.
req.on('timeout', () => onTimeout(localUseLambdaExtension, req, resourcePath, payload, finalLambdaRequest, callback));
req.on('error', e => {
// CASE: we manually destroy streams, skip these errors
// Otherwise we will produce `Error: socket hang up` errors in the logs
// We already print the warning that a timeout happened
// https://nodejs.org/api/http.html#requestdestroyed
if (req.destroyed) {
return;
}
if (localUseLambdaExtension) {
// This is a failure from talking to the Lambda extension on localhost. Most probably it is simply not available
// because @supertenant/aws-lambda has been installed as a normal npm dependency instead of using Instana's
// Lambda layer. We use this failure as a signal to not try to the extension again and instead fall back to
// talking to serverless-acceptor directly. We also immediately retry the current request with that new downstream
// target in place.
logger.debug(
'Could not connect to the Instana Lambda extension. Falling back to talking to the Instana back end directly.',
e
);
// Make sure we do not try to talk to the Lambda extension again.
useLambdaExtension = localUseLambdaExtension = false;
clearInterval(heartbeatInterval);
// Retry the request immediately, this time sending it to serverless-acceptor directly.
send(resourcePath, payload, finalLambdaRequest, callback);
} else {
// We are not using the Lambda extension, because we are either not in an AWS Lambda, or a previous request to the
// extension has already failed. Thus, this is a failure from talking directly to serverless-acceptor
// (or a user-provided proxy).
requestHasFailed = true;
if (!propagateErrorsUpstream) {
if (proxyAgent) {
logger.warn(
'Could not send traces and metrics to Instana. Could not connect to the configured proxy ' +
`${process.env[proxyEnvVar]}.`,
e
);
} else {
logger.warn('Could not send traces and metrics to Instana. The Instana back end seems to be unavailable.', e);
}
}
callback(propagateErrorsUpstream ? e : undefined);
}
});
req.on('finish', () => {
logger.debug(`Sent data to Instana (${resourcePath}).`);
if (useLambdaExtension && finalLambdaRequest) {
clearInterval(heartbeatInterval);
}
});
if (skipWaitingForHttpResponse) {
req.end(serializedPayload, () => {
if (finalLambdaRequest) {
// This is the final request from an AWS Lambda, directly before the Lambda returns its response to the client.
// The Node.js process might be frozen by the AWS Lambda runtime machinery after that and thawed again later for
// another invocation. When the Node.js process is frozen while the request is pending, and then thawed later,
// this can trigger a stale, bogus timeout event (because from the perspective of the freshly thawed Node.js
// runtime, the request has been pending and inactive since a long time). To avoid that, we remove all listeners
// (including the timeout listener) on the request. Since the Lambda runtime will be frozen afterwards (or
// reused for a different, unrelated invocation), it is safe to assume that we are no longer interested in any
// events emitted by the request or the underlying socket.
req.removeAllListeners();
// We need to have a listener for errors that ignores everything, otherwise aborting the request/socket will
// produce an "Unhandled 'error' event"
req.on('error', () => {});
// Finally, abort the request because from our end we are no longer interested in the response and we also do
// not want to let pending IO actions linger in the event loop. This will also call request.destoy and
// req.socket.destroy() internally.
req.abort();
}
// We finish as soon as the request has been flushed, without waiting for the response.
callback();
});
} else {
// See above for why the proxy case has no callback on req.end. Instead, it uses the more traditional callback on
// request creation.
req.end(serializedPayload);
}
}
function onTimeout(localUseLambdaExtension, req, resourcePath, payload, finalLambdaRequest, callback) {
if (localUseLambdaExtension) {
// This is a timeout from talking to the Lambda extension on localhost. Most probably it is simply not available
// because @supertenant/aws-lambda has been installed as a normal npm dependency instead of using Instana's
// Lambda layer. We use this failure as a signal to not try to the extension again and instead fall back to
// talking to serverless-acceptor directly. We also immediately retry the current request with that new downstream
// target in place.
logger.debug(
'Request timed out while trying to talk to Instana Lambda extension. Falling back to talking to the Instana ' +
'back end directly.'
);
// Make sure we do not try to talk to the Lambda extension again.
useLambdaExtension = localUseLambdaExtension = false;
clearInterval(heartbeatInterval);
if (req && !req.destroyed) {
try {
destroyRequest(req);
} catch (e) {
// ignore
}
}
// Retry the request immediately, this time sending it to serverless-acceptor directly.
send(resourcePath, payload, finalLambdaRequest, callback);
} else {
// We are not using the Lambda extension, because we are either not in an AWS Lambda, or a previous request to the
// extension has already failed. Thus, this is a timeout from talking directly to serverless-acceptor
// (or a user-provided proxy).
requestHasFailed = true;
// We need to destroy the request manually, otherwise it keeps the runtime running (and timing out) when
// (a) the wrapped Lambda handler uses the callback API, and
// (b) context.callbackWaitsForEmptyEventLoop = false is not set.
// Also, the Node.js documentation mandates to destroy the request manually in case of a timeout. See
// https://nodejs.org/api/http.html#http_event_timeout.
if (req && !req.destroyed) {
try {
destroyRequest(req);
} catch (e) {
// ignore
}
}
const message =
'Could not send traces and metrics to Instana. The Instana back end did not respond in the configured timeout ' +
`of ${backendTimeout} ms. The timeout can be configured by setting the environment variable ${timeoutEnvVar}.`;
if (!propagateErrorsUpstream) {
logger.warn(message);
}
callback(propagateErrorsUpstream ? new Error(message) : undefined);
}
}
function destroyRequest(req) {
req.destroy();
// Before Node.js 14, req.destroy does not set the req.destroyed property,
// see https://github.com/nodejs/node/pull/33131. We rely on that property in some scenarios. Specifically, when
// deciding whether to fall back to sending data to the back end directly after a request to the extension has failed.
// Without this workaround, we would send the data to the back end twice after a request to the extension has failed:
// Once from the onTimeout handler and once from the req.on('error') handler.
//
// This workaround can be removed as soon as we drop support for Node.js 12.
req.destroyed = true;
}