Skip to content

Commit

Permalink
fix: changes to metadata.cloud.project.* fields collected for GCP
Browse files Browse the repository at this point in the history
- cloud.project.id per the shared APM spec change
- fix the bignum parsing of instance.id (we were losing precision before
  this change)
- add instance.name (missed in the original impl?)

Closes: #3614
  • Loading branch information
trentm committed Sep 8, 2023
1 parent b959464 commit 19f96ca
Show file tree
Hide file tree
Showing 7 changed files with 105 additions and 269 deletions.
22 changes: 22 additions & 0 deletions CHANGELOG.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,28 @@ Notes:
See the <<upgrade-to-v4>> guide.
==== Unreleased
[float]
===== Breaking changes
[float]
===== Features
[float]
===== Bug fixes
[float]
===== Chores
- Changes to cloud metadata collection for Google Cloud (GCP). Most notably
the `cloud.project.id` field is now the `project-id` from
https://cloud.google.com/compute/docs/metadata/default-metadata-values#project_metadata
rather than the `numeric-project-id`. This matches the value produced by
Elastic Beats (like filebeat). {issues}3614[#3614]
[[release-notes-4.0.0]]
==== 4.0.0 - 2023/09/07
Expand Down
101 changes: 40 additions & 61 deletions lib/cloud-metadata/gcp.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,13 @@
*/

'use strict';

const URL = require('url').URL;
const JSONBigInt = require('json-bigint');
const { httpRequest } = require('../http-request');

const DEFAULT_BASE_URL = new URL('/', 'http://metadata.google.internal:80');

/**
* Checks for metadata server then fetches data
*
Expand Down Expand Up @@ -44,11 +48,25 @@ function getMetadataGcp(
});

res.on('end', function (data) {
if (res.statusCode !== 200) {
logger.debug('gcp metadata: unexpected statusCode: %s', res.statusCode);
cb(
new Error(
'error fetching gcp metadata: unexpected statusCode: ' +
res.statusCode,
),
);
return;
}
// Note: We could also guard on the response having the
// 'Metadata-Flavor: Google' header as done by:
// https://github.com/googleapis/gcp-metadata/blob/v6.0.0/src/index.ts#L109-L112

let result;
try {
result = formatMetadataStringIntoObject(finalData.join(''));
} catch (err) {
logger.trace(
logger.debug(
'gcp metadata server responded, but there was an ' +
'error parsing the result: %o',
err,
Expand Down Expand Up @@ -78,76 +96,37 @@ function getMetadataGcp(
/**
* Builds metadata object
*
* Takes the response from a /computeMetadata/v1/?recursive=true
* service request and formats it into the cloud metadata object
* Convert a GCP Cloud Engine VM metadata response
* (https://cloud.google.com/compute/docs/metadata/default-metadata-values)
* to the APM intake cloud metadata object
* (https://github.com/elastic/apm/blob/main/specs/agents/metadata.md#gcp-metadata).
*
* See discussion about big int values here:
* https://github.com/googleapis/gcp-metadata#take-care-with-large-number-valued-properties
* This implementation is using the same 'json-bigint' library as 'gcp-metadata'.
*/
function formatMetadataStringIntoObject(string) {
const data = JSON.parse(string);
// cast string manipulation fields as strings "just in case"
if (data.instance) {
data.instance.machineType = String(data.instance.machineType);
data.instance.zone = String(data.instance.zone);
}
const data = JSONBigInt.parse(string);

// E.g., 'projects/513326162531/zones/us-west1-b' -> 'us-west1-b'
const az = data.instance.zone.split('/').pop();

const metadata = {
availability_zone: null,
region: null,
provider: 'gcp',
instance: {
id: null,
id: data.instance.id.toString(), // We expect this to be a BigInt.
name: data.instance.name,
},
machine: {
type: null,
},
provider: null,
project: {
id: null,
name: null,
id: data.project.projectId,
},
availability_zone: az,
region: az.slice(0, az.lastIndexOf('-')), // 'us-west1-b' -> 'us-west1'
machine: {
type: data.instance.machineType.split('/').pop(),
},
};

metadata.availability_zone = null;
metadata.region = null;
if (data.instance && data.instance.zone) {
// `projects/513326162531/zones/us-west1-b` manipuated into
// `us-west1-b`, and then `us-west1`
const regionWithZone = data.instance.zone.split('/').pop();
const parts = regionWithZone.split('-');
parts.pop();
metadata.region = parts.join('-');
metadata.availability_zone = regionWithZone;
}

if (data.instance) {
metadata.instance = {
id: String(data.instance.id),
};

metadata.machine = {
type: String(data.instance.machineType.split('/').pop()),
};
} else {
metadata.instance = {
id: null,
};

metadata.machine = {
type: null,
};
}

metadata.provider = 'gcp';

if (data.project) {
metadata.project = {
id: String(data.project.numericProjectId),
name: String(data.project.projectId),
};
} else {
metadata.project = {
id: null,
name: null,
};
}
return metadata;
}

Expand Down
21 changes: 18 additions & 3 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@
"fast-stream-to-buffer": "^1.0.0",
"http-headers": "^3.0.2",
"import-in-the-middle": "1.4.2",
"json-bigint": "^1.0.0",
"lru-cache": "^10.0.1",
"measured-reporting": "^1.51.1",
"module-details-from-path": "^1.0.3",
Expand Down
123 changes: 3 additions & 120 deletions test/cloud-metadata/_fixtures.js
Original file line number Diff line number Diff line change
Expand Up @@ -64,128 +64,11 @@ module.exports = {
},
],
gcp: [
{
name: 'gcp does not crash on empty response',
response: {},
},
{
name: 'gcp unexpected string fixture',
response: {
instance: {
zone: 123456,
machineType: 123456,
},
},
},
{
name: 'default gcp fixture',
response: {
instance: {
attributes: {},
cpuPlatform: 'Intel Broadwell',
description: '',
disks: [
{
deviceName: 'username-temp-delete-me-cloud-metadata',
index: 0,
interface: 'SCSI',
mode: 'READ_WRITE',
type: 'PERSISTENT',
},
],
guestAttributes: {},
hostname:
'username-temp-delete-me-cloud-metadata.c.elastic-apm.internal',
id: 7684572792595385000,
image:
'projects/debian-cloud/global/images/debian-10-buster-v20201216',
legacyEndpointAccess: {
0.1: 0,
v1beta1: 0,
},
licenses: [
{
id: '5543610867827062957',
},
],
machineType: 'projects/513326162531/machineTypes/e2-micro',
maintenanceEvent: 'NONE',
name: 'username-temp-delete-me-cloud-metadata',
networkInterfaces: [
{
accessConfigs: [
{
externalIp: '35.247.28.180',
type: 'ONE_TO_ONE_NAT',
},
],
dnsServers: ['169.254.169.254'],
forwardedIps: [],
gateway: '10.138.0.1',
ip: '10.138.0.2',
ipAliases: [],
mac: '42:01:0a:8a:00:02',
mtu: 1460,
network: 'projects/513326162531/networks/default',
subnetmask: '255.255.240.0',
targetInstanceIps: [],
},
],
preempted: 'FALSE',
remainingCpuTime: -1,
scheduling: {
automaticRestart: 'TRUE',
onHostMaintenance: 'MIGRATE',
preemptible: 'FALSE',
},
serviceAccounts: {
'[email protected]': {
aliases: ['default'],
email: '[email protected]',
scopes: [
'https://www.googleapis.com/auth/devstorage.read_only',
'https://www.googleapis.com/auth/logging.write',
'https://www.googleapis.com/auth/monitoring.write',
'https://www.googleapis.com/auth/servicecontrol',
'https://www.googleapis.com/auth/service.management.readonly',
'https://www.googleapis.com/auth/trace.append',
],
},
default: {
aliases: ['default'],
email: '[email protected]',
scopes: [
'https://www.googleapis.com/auth/devstorage.read_only',
'https://www.googleapis.com/auth/logging.write',
'https://www.googleapis.com/auth/monitoring.write',
'https://www.googleapis.com/auth/servicecontrol',
'https://www.googleapis.com/auth/service.management.readonly',
'https://www.googleapis.com/auth/trace.append',
],
},
},
tags: ['http-server'],
virtualClock: {
driftToken: '0',
},
zone: 'projects/513326162531/zones/us-west1-b',
},
oslogin: {
authenticate: {
sessions: {},
},
},
project: {
attributes: {
'gke-smith-de35da35-secondary-ranges':
'services:default:default:gke-smith-services-de35da35,pods:default:default:gke-smith-pods-de35da35',
'serial-port-enable': '1',
'ssh-keys': '... public keys snipped ...',
},
numericProjectId: 513326162531,
projectId: 'elastic-apm',
},
},
// This is an actual response from a dev VM, edited slightly for size and privacy.
response:
'{"instance":{"attributes":{},"cpuPlatform":"Intel Broadwell","description":"","disks":[{"deviceName":"trentm-play-vm0","index":0,"interface":"SCSI","mode":"READ_WRITE","type":"PERSISTENT-BALANCED"}],"guestAttributes":{},"hostname":"trentm-play-vm0.c.acme-eng.internal","id":5737554347302044216,"image":"projects/debian-cloud/global/images/debian-11-bullseye-v20230814","licenses":[{"id":"3853522013536123851"}],"machineType":"projects/523926462582/machineTypes/e2-medium","maintenanceEvent":"NONE","name":"trentm-play-vm0","networkInterfaces":[{"accessConfigs":[{"externalIp":"33.162.212.82","type":"ONE_TO_ONE_NAT"}],"dnsServers":["169.254.169.254"],"forwardedIps":[],"gateway":"10.138.0.1","ip":"10.138.0.7","ipAliases":[],"mac":"42:01:0a:9a:0e:27","mtu":1460,"network":"projects/523926462582/networks/default","subnetmask":"255.255.240.0","targetInstanceIps":[]}],"preempted":"FALSE","remainingCpuTime":-1,"scheduling":{"automaticRestart":"TRUE","onHostMaintenance":"MIGRATE","preemptible":"FALSE"},"serviceAccounts":{"[email protected]":{"aliases":["default"],"email":"[email protected]","scopes":["https://www.googleapis.com/auth/devstorage.read_only","https://www.googleapis.com/auth/logging.write","https://www.googleapis.com/auth/monitoring.write","https://www.googleapis.com/auth/servicecontrol","https://www.googleapis.com/auth/service.management.readonly","https://www.googleapis.com/auth/trace.append"]},"default":{"aliases":["default"],"email":"[email protected]","scopes":["https://www.googleapis.com/auth/devstorage.read_only","https://www.googleapis.com/auth/logging.write","https://www.googleapis.com/auth/monitoring.write","https://www.googleapis.com/auth/servicecontrol","https://www.googleapis.com/auth/service.management.readonly","https://www.googleapis.com/auth/trace.append"]}},"tags":[],"virtualClock":{"driftToken":"0"},"zone":"projects/523926462582/zones/us-west1-b"},"oslogin":{"authenticate":{"sessions":{}}},"project":{"attributes":{"gke-kk-dev-cluster-8264c0ad-secondary-ranges":"services:default:gke-kk-dev-cluster-subnet-8264c0ad:gke-kk-dev-cluster-services-8264c0ad,pods:default:gke-kk-dev-cluster-subnet-8264c0ad:gke-kk-dev-cluster-pods-8264c0ad","serial-port-enable":"1","ssh-keys":"[REDACTED]","sshKeys":"[REDACTED]"},"numericProjectId":523926462582,"projectId":"acme-eng"}}',
},
],
azure: [
Expand Down
8 changes: 6 additions & 2 deletions test/cloud-metadata/_lib.js
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,10 @@ function addGcpRoute(app, fixture) {
throw new Error('Metadata-Flavor: Google header required');
}

res.status(200);
res.set('Metadata-Flavor', 'Google');
res.set('Content-Type', 'application/json');
res.set('Server', 'Metadata Server for VM');
res.send(fixture.response);
});

Expand Down Expand Up @@ -205,7 +209,7 @@ function addRoutesToExpressApp(app, provider, fixture) {
function createTestServer(provider, fixtureName) {
const fixture = loadFixtureData(provider, fixtureName);
if (!fixture) {
throw new Error(`Unknown ${provider} fixtured named ${fixtureName}`);
throw new Error(`Unknown ${provider} fixture named ${fixtureName}`);
}
const app = express();
return addRoutesToExpressApp(app, provider, fixture);
Expand All @@ -223,7 +227,7 @@ function createTestServer(provider, fixtureName) {
function createSlowTestServer(provider, fixtureName) {
const fixture = loadFixtureData(provider, fixtureName);
if (!fixture) {
throw new Error(`Unknown ${provider} fixtured named ${fixtureName}`);
throw new Error(`Unknown ${provider} fixture named ${fixtureName}`);
}
const app = express();
return addSlowRoutesToExpressApp(app, provider, fixture);
Expand Down
Loading

0 comments on commit 19f96ca

Please sign in to comment.