diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index c1a27bfbb9..76a84de9f3 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -33,6 +33,28 @@ Notes: See the <> guide. + +==== Unreleased + +[float] +===== Breaking changes + +[float] +===== Features + +[float] +===== Bug fixes + +[float] +===== Chores + +- Changes to cloud metadata collection for Google Cloud (GCP). Most notably + the `cloud.project.id` field is now the `project-id` from + https://cloud.google.com/compute/docs/metadata/default-metadata-values#project_metadata + rather than the `numeric-project-id`. This matches the value produced by + Elastic Beats (like filebeat). {issues}3614[#3614] + + [[release-notes-4.0.0]] ==== 4.0.0 - 2023/09/07 diff --git a/lib/cloud-metadata/gcp.js b/lib/cloud-metadata/gcp.js index 936a7673b7..e7694902ed 100644 --- a/lib/cloud-metadata/gcp.js +++ b/lib/cloud-metadata/gcp.js @@ -5,9 +5,13 @@ */ 'use strict'; + const URL = require('url').URL; +const JSONBigInt = require('json-bigint'); const { httpRequest } = require('../http-request'); + const DEFAULT_BASE_URL = new URL('/', 'http://metadata.google.internal:80'); + /** * Checks for metadata server then fetches data * @@ -44,11 +48,25 @@ function getMetadataGcp( }); res.on('end', function (data) { + if (res.statusCode !== 200) { + logger.debug('gcp metadata: unexpected statusCode: %s', res.statusCode); + cb( + new Error( + 'error fetching gcp metadata: unexpected statusCode: ' + + res.statusCode, + ), + ); + return; + } + // Note: We could also guard on the response having the + // 'Metadata-Flavor: Google' header as done by: + // https://github.com/googleapis/gcp-metadata/blob/v6.0.0/src/index.ts#L109-L112 + let result; try { result = formatMetadataStringIntoObject(finalData.join('')); } catch (err) { - logger.trace( + logger.debug( 'gcp metadata server responded, but there was an ' + 'error parsing the result: %o', err, @@ -78,76 +96,37 @@ function getMetadataGcp( /** * Builds metadata object * - * Takes the response from a /computeMetadata/v1/?recursive=true - * service request and formats it into the cloud metadata object + * Convert a GCP Cloud Engine VM metadata response + * (https://cloud.google.com/compute/docs/metadata/default-metadata-values) + * to the APM intake cloud metadata object + * (https://github.com/elastic/apm/blob/main/specs/agents/metadata.md#gcp-metadata). + * + * See discussion about big int values here: + * https://github.com/googleapis/gcp-metadata#take-care-with-large-number-valued-properties + * This implementation is using the same 'json-bigint' library as 'gcp-metadata'. */ function formatMetadataStringIntoObject(string) { - const data = JSON.parse(string); - // cast string manipulation fields as strings "just in case" - if (data.instance) { - data.instance.machineType = String(data.instance.machineType); - data.instance.zone = String(data.instance.zone); - } + const data = JSONBigInt.parse(string); + + // E.g., 'projects/513326162531/zones/us-west1-b' -> 'us-west1-b' + const az = data.instance.zone.split('/').pop(); const metadata = { - availability_zone: null, - region: null, + provider: 'gcp', instance: { - id: null, + id: data.instance.id.toString(), // We expect this to be a BigInt. + name: data.instance.name, }, - machine: { - type: null, - }, - provider: null, project: { - id: null, - name: null, + id: data.project.projectId, + }, + availability_zone: az, + region: az.slice(0, az.lastIndexOf('-')), // 'us-west1-b' -> 'us-west1' + machine: { + type: data.instance.machineType.split('/').pop(), }, }; - metadata.availability_zone = null; - metadata.region = null; - if (data.instance && data.instance.zone) { - // `projects/513326162531/zones/us-west1-b` manipuated into - // `us-west1-b`, and then `us-west1` - const regionWithZone = data.instance.zone.split('/').pop(); - const parts = regionWithZone.split('-'); - parts.pop(); - metadata.region = parts.join('-'); - metadata.availability_zone = regionWithZone; - } - - if (data.instance) { - metadata.instance = { - id: String(data.instance.id), - }; - - metadata.machine = { - type: String(data.instance.machineType.split('/').pop()), - }; - } else { - metadata.instance = { - id: null, - }; - - metadata.machine = { - type: null, - }; - } - - metadata.provider = 'gcp'; - - if (data.project) { - metadata.project = { - id: String(data.project.numericProjectId), - name: String(data.project.projectId), - }; - } else { - metadata.project = { - id: null, - name: null, - }; - } return metadata; } diff --git a/package-lock.json b/package-lock.json index 6ac125c6e9..330d6ec4f6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,6 +28,7 @@ "fast-stream-to-buffer": "^1.0.0", "http-headers": "^3.0.2", "import-in-the-middle": "1.4.2", + "json-bigint": "^1.0.0", "lru-cache": "^10.0.1", "measured-reporting": "^1.51.1", "module-details-from-path": "^1.0.3", @@ -7073,7 +7074,6 @@ "version": "9.0.0", "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.0.tgz", "integrity": "sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A==", - "dev": true, "engines": { "node": "*" } @@ -12339,6 +12339,14 @@ "node": ">=4" } }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -23841,8 +23849,7 @@ "bignumber.js": { "version": "9.0.0", "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.0.tgz", - "integrity": "sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A==", - "dev": true + "integrity": "sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A==" }, "binary-extensions": { "version": "2.2.0", @@ -27800,6 +27807,14 @@ "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", "dev": true }, + "json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "requires": { + "bignumber.js": "^9.0.0" + } + }, "json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", diff --git a/package.json b/package.json index 746dd95e7b..8f36e49425 100644 --- a/package.json +++ b/package.json @@ -106,6 +106,7 @@ "fast-stream-to-buffer": "^1.0.0", "http-headers": "^3.0.2", "import-in-the-middle": "1.4.2", + "json-bigint": "^1.0.0", "lru-cache": "^10.0.1", "measured-reporting": "^1.51.1", "module-details-from-path": "^1.0.3", diff --git a/test/cloud-metadata/_fixtures.js b/test/cloud-metadata/_fixtures.js index 3ad885f3bd..7eb50c7851 100644 --- a/test/cloud-metadata/_fixtures.js +++ b/test/cloud-metadata/_fixtures.js @@ -64,128 +64,11 @@ module.exports = { }, ], gcp: [ - { - name: 'gcp does not crash on empty response', - response: {}, - }, - { - name: 'gcp unexpected string fixture', - response: { - instance: { - zone: 123456, - machineType: 123456, - }, - }, - }, { name: 'default gcp fixture', - response: { - instance: { - attributes: {}, - cpuPlatform: 'Intel Broadwell', - description: '', - disks: [ - { - deviceName: 'username-temp-delete-me-cloud-metadata', - index: 0, - interface: 'SCSI', - mode: 'READ_WRITE', - type: 'PERSISTENT', - }, - ], - guestAttributes: {}, - hostname: - 'username-temp-delete-me-cloud-metadata.c.elastic-apm.internal', - id: 7684572792595385000, - image: - 'projects/debian-cloud/global/images/debian-10-buster-v20201216', - legacyEndpointAccess: { - 0.1: 0, - v1beta1: 0, - }, - licenses: [ - { - id: '5543610867827062957', - }, - ], - machineType: 'projects/513326162531/machineTypes/e2-micro', - maintenanceEvent: 'NONE', - name: 'username-temp-delete-me-cloud-metadata', - networkInterfaces: [ - { - accessConfigs: [ - { - externalIp: '35.247.28.180', - type: 'ONE_TO_ONE_NAT', - }, - ], - dnsServers: ['169.254.169.254'], - forwardedIps: [], - gateway: '10.138.0.1', - ip: '10.138.0.2', - ipAliases: [], - mac: '42:01:0a:8a:00:02', - mtu: 1460, - network: 'projects/513326162531/networks/default', - subnetmask: '255.255.240.0', - targetInstanceIps: [], - }, - ], - preempted: 'FALSE', - remainingCpuTime: -1, - scheduling: { - automaticRestart: 'TRUE', - onHostMaintenance: 'MIGRATE', - preemptible: 'FALSE', - }, - serviceAccounts: { - '513326162531-compute@developer.gserviceaccount.com': { - aliases: ['default'], - email: '513326162531-compute@developer.gserviceaccount.com', - scopes: [ - 'https://www.googleapis.com/auth/devstorage.read_only', - 'https://www.googleapis.com/auth/logging.write', - 'https://www.googleapis.com/auth/monitoring.write', - 'https://www.googleapis.com/auth/servicecontrol', - 'https://www.googleapis.com/auth/service.management.readonly', - 'https://www.googleapis.com/auth/trace.append', - ], - }, - default: { - aliases: ['default'], - email: '513326162531-compute@developer.gserviceaccount.com', - scopes: [ - 'https://www.googleapis.com/auth/devstorage.read_only', - 'https://www.googleapis.com/auth/logging.write', - 'https://www.googleapis.com/auth/monitoring.write', - 'https://www.googleapis.com/auth/servicecontrol', - 'https://www.googleapis.com/auth/service.management.readonly', - 'https://www.googleapis.com/auth/trace.append', - ], - }, - }, - tags: ['http-server'], - virtualClock: { - driftToken: '0', - }, - zone: 'projects/513326162531/zones/us-west1-b', - }, - oslogin: { - authenticate: { - sessions: {}, - }, - }, - project: { - attributes: { - 'gke-smith-de35da35-secondary-ranges': - 'services:default:default:gke-smith-services-de35da35,pods:default:default:gke-smith-pods-de35da35', - 'serial-port-enable': '1', - 'ssh-keys': '... public keys snipped ...', - }, - numericProjectId: 513326162531, - projectId: 'elastic-apm', - }, - }, + // This is an actual response from a dev VM, edited slightly for size and privacy. + response: + '{"instance":{"attributes":{},"cpuPlatform":"Intel Broadwell","description":"","disks":[{"deviceName":"trentm-play-vm0","index":0,"interface":"SCSI","mode":"READ_WRITE","type":"PERSISTENT-BALANCED"}],"guestAttributes":{},"hostname":"trentm-play-vm0.c.acme-eng.internal","id":5737554347302044216,"image":"projects/debian-cloud/global/images/debian-11-bullseye-v20230814","licenses":[{"id":"3853522013536123851"}],"machineType":"projects/523926462582/machineTypes/e2-medium","maintenanceEvent":"NONE","name":"trentm-play-vm0","networkInterfaces":[{"accessConfigs":[{"externalIp":"33.162.212.82","type":"ONE_TO_ONE_NAT"}],"dnsServers":["169.254.169.254"],"forwardedIps":[],"gateway":"10.138.0.1","ip":"10.138.0.7","ipAliases":[],"mac":"42:01:0a:9a:0e:27","mtu":1460,"network":"projects/523926462582/networks/default","subnetmask":"255.255.240.0","targetInstanceIps":[]}],"preempted":"FALSE","remainingCpuTime":-1,"scheduling":{"automaticRestart":"TRUE","onHostMaintenance":"MIGRATE","preemptible":"FALSE"},"serviceAccounts":{"523926462582-compute@developer.gserviceaccount.com":{"aliases":["default"],"email":"523926462582-compute@developer.gserviceaccount.com","scopes":["https://www.googleapis.com/auth/devstorage.read_only","https://www.googleapis.com/auth/logging.write","https://www.googleapis.com/auth/monitoring.write","https://www.googleapis.com/auth/servicecontrol","https://www.googleapis.com/auth/service.management.readonly","https://www.googleapis.com/auth/trace.append"]},"default":{"aliases":["default"],"email":"523926462582-compute@developer.gserviceaccount.com","scopes":["https://www.googleapis.com/auth/devstorage.read_only","https://www.googleapis.com/auth/logging.write","https://www.googleapis.com/auth/monitoring.write","https://www.googleapis.com/auth/servicecontrol","https://www.googleapis.com/auth/service.management.readonly","https://www.googleapis.com/auth/trace.append"]}},"tags":[],"virtualClock":{"driftToken":"0"},"zone":"projects/523926462582/zones/us-west1-b"},"oslogin":{"authenticate":{"sessions":{}}},"project":{"attributes":{"gke-kk-dev-cluster-8264c0ad-secondary-ranges":"services:default:gke-kk-dev-cluster-subnet-8264c0ad:gke-kk-dev-cluster-services-8264c0ad,pods:default:gke-kk-dev-cluster-subnet-8264c0ad:gke-kk-dev-cluster-pods-8264c0ad","serial-port-enable":"1","ssh-keys":"[REDACTED]","sshKeys":"[REDACTED]"},"numericProjectId":523926462582,"projectId":"acme-eng"}}', }, ], azure: [ diff --git a/test/cloud-metadata/_lib.js b/test/cloud-metadata/_lib.js index 0b83c59250..bb0c73e256 100644 --- a/test/cloud-metadata/_lib.js +++ b/test/cloud-metadata/_lib.js @@ -95,6 +95,10 @@ function addGcpRoute(app, fixture) { throw new Error('Metadata-Flavor: Google header required'); } + res.status(200); + res.set('Metadata-Flavor', 'Google'); + res.set('Content-Type', 'application/json'); + res.set('Server', 'Metadata Server for VM'); res.send(fixture.response); }); @@ -205,7 +209,7 @@ function addRoutesToExpressApp(app, provider, fixture) { function createTestServer(provider, fixtureName) { const fixture = loadFixtureData(provider, fixtureName); if (!fixture) { - throw new Error(`Unknown ${provider} fixtured named ${fixtureName}`); + throw new Error(`Unknown ${provider} fixture named ${fixtureName}`); } const app = express(); return addRoutesToExpressApp(app, provider, fixture); @@ -223,7 +227,7 @@ function createTestServer(provider, fixtureName) { function createSlowTestServer(provider, fixtureName) { const fixture = loadFixtureData(provider, fixtureName); if (!fixture) { - throw new Error(`Unknown ${provider} fixtured named ${fixtureName}`); + throw new Error(`Unknown ${provider} fixture named ${fixtureName}`); } const app = express(); return addSlowRoutesToExpressApp(app, provider, fixture); diff --git a/test/cloud-metadata/index.test.js b/test/cloud-metadata/index.test.js index a1edf7a6b4..15d7e38806 100644 --- a/test/cloud-metadata/index.test.js +++ b/test/cloud-metadata/index.test.js @@ -254,28 +254,6 @@ tape('cloud metadata: aws empty data', function (t) { }); }); -tape('cloud metadata: gcp empty data', function (t) { - t.plan(2); - - const provider = 'gcp'; - const fixtureName = 'gcp does not crash on empty response'; - const serverGcp = createTestServer(provider, fixtureName); - - const cloudProvider = 'auto'; - const listener = serverGcp.listen(0, function () { - providerUrls.aws.port = listener.address().port; - providerUrls.gcp.port = listener.address().port; - providerUrls.azure.port = listener.address().port; - - const cloudMetadata = new CloudMetadata(cloudProvider, logger); - cloudMetadata.getCloudMetadata(providerUrls, function (err, metadata) { - t.error(err, 'no errors expected'); - t.ok(metadata, 'returned data'); - listener.close(); - }); - }); -}); - tape('cloud metadata: azure empty data', function (t) { t.plan(2); @@ -323,12 +301,9 @@ tape('cloud metadata: azure empty data', function (t) { tape( 'cloud metadata: main function returns data with gcp server', function (t) { - t.plan(9); - const provider = 'gcp'; const fixtureName = 'default gcp fixture'; const serverGcp = createTestServer(provider, fixtureName); - const fixture = loadFixtureData(provider, fixtureName); const cloudProvider = 'auto'; const listener = serverGcp.listen(0, function () { @@ -337,41 +312,23 @@ tape( providerUrls.azure.port = listener.address().port; const cloudMetadata = new CloudMetadata(cloudProvider, logger); - cloudMetadata.getCloudMetadata(providerUrls, function (err, metadata) { - t.error(err, 'no errors expected'); - t.ok(metadata, 'returned data'); - t.equals( - metadata.instance.id, - fixture.response.instance.id + '', - 'instance id is set and is a string', - ); - t.equals( - metadata.provider, - provider + '', - 'provider is set and is a string', - ); - t.equals( - metadata.project.id, - fixture.response.project.numericProjectId + '', - 'project id is set and is a string', - ); - t.equals( - metadata.project.name, - fixture.response.project.projectId + '', - 'project name is set and is a string', - ); - - // for properties we create via manipulation, just test hard coded - // string constants rather than re-manipulate in that same, possibly - // buggy, way - t.equals(metadata.region, 'us-west1', 'region is set'); - t.equals( - metadata.availability_zone, - 'us-west1-b', - 'availability_zone is set', + cloudMetadata.getCloudMetadata(providerUrls, function (err, cmeta) { + t.error(err, 'getCloudMetadata did not error'); + t.ok(cmeta, 'getCloudMetadata returned metadata'); + t.deepEqual( + cmeta, + { + provider: 'gcp', + instance: { id: '5737554347302044216', name: 'trentm-play-vm0' }, + project: { id: 'acme-eng' }, + availability_zone: 'us-west1-b', + region: 'us-west1', + machine: { type: 'e2-medium' }, + }, + 'metadata.cloud', ); - t.equals(metadata.machine.type, 'e2-micro', 'machine type is set'); listener.close(); + t.end(); }); }); }, @@ -443,31 +400,6 @@ tape( }, ); -tape( - 'cloud metadata: gcp string manipulation does not fail on non-strings', - function (t) { - t.plan(2); - - const provider = 'gcp'; - const fixtureName = 'gcp unexpected string fixture'; - const serverGcp = createTestServer(provider, fixtureName); - - const cloudProvider = 'auto'; - const listener = serverGcp.listen(0, function () { - providerUrls.aws.port = listener.address().port; - providerUrls.gcp.port = listener.address().port; - providerUrls.azure.port = listener.address().port; - - const cloudMetadata = new CloudMetadata(cloudProvider, logger); - cloudMetadata.getCloudMetadata(providerUrls, function (err, metadata) { - t.error(err, 'no errors expected'); - t.ok(metadata, 'returned data'); - listener.close(); - }); - }); - }, -); - tape('gcp metadata: no gcp server', function (t) { t.plan(1);