[Feature] - Support proxy requests (#55)

* Support for proxying requests to proxy server

* .

* fix types

* .

* updating

* some more cleanup

* .

* updating tests

* condition test if less than node 18

* adding dist

* .

* remove jest updates

* reverting gitignore

* removed unused

.vscode/launch.json

@@ -1,22 +1,28 @@
 {
   "configurations": [
-  {
-    "command": "yarn worker:data",
-    "name": "Run Data Worker",
-    "request": "launch",
-    "type": "node-terminal"
-  },
-  {
-    "command": "yarn worker:cron",
-    "name": "Run Cron Worker",
-    "request": "launch",
-    "type": "node-terminal"
-  },
-  {
-    "command": "yarn server",
-    "name": "Run Server",
-    "request": "launch",
-    "type": "node-terminal"
-  },
+    {
+      "command": "yarn worker:data",
+      "name": "Run Data Worker",
+      "request": "launch",
+      "type": "node-terminal"
+    },
+    {
+      "command": "yarn worker:cron",
+      "name": "Run Cron Worker",
+      "request": "launch",
+      "type": "node-terminal"
+    },
+    {
+      "command": "yarn server",
+      "name": "Run Server",
+      "request": "launch",
+      "type": "node-terminal"
+    },
+    {
+      "command": "yarn test:e2e",
+      "name": "Run test",
+      "request": "launch",
+      "type": "node-terminal"
+    }
   ]
 }

src/constants.ts

@@ -4,7 +4,7 @@ const defaultConfig = {
   timeout: 5000,
   eventSinkEndpoint: '/events',
   errorSinkEndpoint: '/errors',
-  remoteConfigFetchEndpoint: '/config',
+  remoteConfigFetchEndpoint: '/v2/config',
   telemetryEndpoint: '/telemetry',
   useRemoteConfig: true,
   useTelemetry: true,
@@ -19,10 +19,11 @@ const defaultConfig = {
   redactByDefault: false,
   allowedDomains: [],
   cacheTtl: 0,
+  proxyConfig: {},
 
   // After the close command is sent, wait for this many milliseconds before
   // exiting. This gives any hanging responses a chance to return.
-  waitAfterClose: 1000,
+  waitAfterClose: 1000
 };
 
 const errors = {
@@ -39,7 +40,7 @@ const errors = {
   NO_CLIENT_ID:
     'No Client ID Provided, set SUPERGOOD_CLIENT_ID or pass it as an argument',
   NO_CLIENT_SECRET:
-    'No Client Secret Provided, set SUPERGOOD_CLIENT_SECRET or pass it as an argument',
+    'No Client Secret Provided, set SUPERGOOD_CLIENT_SECRET or pass it as an argument'
 };
 
 const SensitiveKeyActions = {
@@ -50,7 +51,7 @@ const SensitiveKeyActions = {
 const EndpointActions = {
   ALLOW: 'Allow',
   IGNORE: 'Ignore'
-}
+};
 
 const TestErrorPath = '/api/supergood-test-error';
 const LocalClientId = 'local-client-id';
@@ -59,7 +60,7 @@ const LocalClientSecret = 'local-client-secret';
 const ContentType = {
   Json: 'application/json',
   Text: 'text/plain',
-  EventStream: 'text/event-stream',
+  EventStream: 'text/event-stream'
 };
 
 export {

src/index.ts

@@ -88,7 +88,9 @@ const Supergood = () => {
     },
     baseUrl = process.env.SUPERGOOD_BASE_URL || 'https://api.supergood.ai',
     baseTelemetryUrl = process.env.SUPERGOOD_TELEMETRY_BASE_URL ||
-      'https://telemetry.supergood.ai'
+      'https://telemetry.supergood.ai',
+    baseProxyURL = process.env.SUPERGOOD_PROXY_BASE_URL ||
+      'https://proxy.supergood.ai'
   ): TConfig extends { useRemoteConfig: false } ? void : Promise<void> => {
     if (!clientId) throw new Error(errors.NO_CLIENT_ID);
     if (!clientSecret) throw new Error(errors.NO_CLIENT_SECRET);
@@ -124,6 +126,7 @@ const Supergood = () => {
       ignoredDomains: supergoodConfig.ignoredDomains,
       allowLocalUrls: supergoodConfig.allowLocalUrls,
       allowIpAddresses: supergoodConfig.allowIpAddresses,
+      proxyConfig: supergoodConfig.proxyConfig,
       baseUrl
     };
 
@@ -153,10 +156,18 @@ const Supergood = () => {
           remoteConfigFetchUrl,
           headerOptions
         );
-        supergoodConfig = {
-          ...supergoodConfig,
-          remoteConfig: processRemoteConfig(remoteConfigPayload)
-        };
+        const { endpointConfig, proxyConfig } =
+          processRemoteConfig(remoteConfigPayload);
+
+        // NOTE: The supergood should not change its reference via spread
+        // e.g supergoodConfig = {...supergoodConfig, remoteConfig: endpointConfig, proxyConfig: ...proxyConfig }
+        // This is because we require reference to the mutated object for the proxy
+        supergoodConfig.remoteConfig = endpointConfig;
+        supergoodConfig.proxyConfig.vendorCredentialConfig =
+          proxyConfig?.vendorCredentialConfig;
+        supergoodConfig.proxyConfig.proxyURL = new URL(baseProxyURL);
+        supergoodConfig.proxyConfig.clientId = clientId;
+        supergoodConfig.proxyConfig.clientSecret = clientSecret;
       } catch (e) {
         log.error(
           errors.FETCHING_CONFIG,

src/interceptor/FetchInterceptor.ts

@@ -3,7 +3,9 @@ import crypto from 'crypto';
 import { IsomorphicRequest } from './utils/IsomorphicRequest';
 import { IsomorphicResponse } from './utils/IsomorphicResponse';
 import { isInterceptable } from './utils/isInterceptable';
+import { SupergoodProxyHeaders, shouldProxyRequest } from './utils/proxyUtils';
 import { Interceptor, NodeRequestInterceptorOptions } from './Interceptor';
+import { ProxyConfigType } from '../types';
 
 export class FetchInterceptor extends Interceptor {
   constructor(options?: NodeRequestInterceptorOptions) {
@@ -17,20 +19,21 @@ export class FetchInterceptor extends Interceptor {
     );
   }
 
-  public setup({ isWithinContext }: { isWithinContext: () => boolean }){
+  public setup({ isWithinContext }: { isWithinContext: () => boolean }) {
     const pureFetch = globalThis.fetch;
 
     globalThis.fetch = async (input, init) => {
       const requestId = crypto.randomUUID();
-      const request = new Request(input, init);
+      let request = new Request(input, init);
+      const requestURL = new URL(request.url);
       const _isInterceptable = isInterceptable({
-        url: new URL(request.url),
+        url: requestURL,
         ignoredDomains: this.options.ignoredDomains ?? [],
         allowedDomains: this.options.allowedDomains ?? [],
         baseUrl: this.options.baseUrl ?? '',
         allowLocalUrls: this.options.allowLocalUrls ?? false,
         allowIpAddresses: this.options.allowIpAddresses ?? false,
-        isWithinContext: isWithinContext ?? (() => true),
+        isWithinContext: isWithinContext ?? (() => true)
       });
 
       if (_isInterceptable) {
@@ -40,6 +43,10 @@ export class FetchInterceptor extends Interceptor {
         this.emitter.emit('request', isomorphicRequest, requestId);
       }
 
+      if (shouldProxyRequest(requestURL, this.options.proxyConfig)) {
+        request = modifyRequest(request, requestURL, this.options?.proxyConfig);
+      }
+
       return pureFetch(request).then(async (response) => {
         if (_isInterceptable) {
           const isomorphicResponse = await IsomorphicResponse.fromFetchResponse(
@@ -56,3 +63,26 @@ export class FetchInterceptor extends Interceptor {
     });
   }
 }
+
+const modifyRequest = (
+  originalRequest: Request,
+  originalRequestURL: URL,
+  proxyConfig?: ProxyConfigType
+) => {
+  const headers = originalRequest.headers;
+  headers.set(
+    SupergoodProxyHeaders.upstreamHeader,
+    originalRequestURL.protocol + '//' + originalRequestURL.host
+  );
+  headers.set(SupergoodProxyHeaders.clientId, proxyConfig?.clientId || '');
+  headers.set(
+    SupergoodProxyHeaders.clientSecret,
+    proxyConfig?.clientSecret || ''
+  );
+
+  const proxyURL = proxyConfig?.proxyURL as URL;
+  proxyURL.pathname = originalRequestURL.pathname;
+  proxyURL.search = originalRequestURL.search;
+
+  return new Request(proxyURL as URL, { headers });
+};

src/interceptor/Interceptor.ts

@@ -1,11 +1,13 @@
 import { EventEmitter } from 'events';
+import { ConfigType, ProxyConfigType } from '../types';
 
 export interface NodeRequestInterceptorOptions {
   ignoredDomains?: string[];
   allowedDomains?: string[];
   allowLocalUrls?: boolean;
   allowIpAddresses?: boolean;
   baseUrl?: string;
+  proxyConfig?: ProxyConfigType;
 }
 
 export class Interceptor {

src/interceptor/NodeClientRequest.ts

@@ -13,6 +13,9 @@ import { getArrayBuffer } from './utils/bufferUtils';
 import { isInterceptable } from './utils/isInterceptable';
 import { IsomorphicResponse } from './utils/IsomorphicResponse';
 import { cloneIncomingMessage } from './utils/cloneIncomingMessage';
+import { shouldProxyRequest, SupergoodProxyHeaders } from './utils/proxyUtils';
+import { ProxyConfigType } from '../types';
+import { ResolvedRequestOptions } from './utils/getUrlByRequestOptions';
 
 export type NodeClientOptions = {
   emitter: EventEmitter;
@@ -21,6 +24,7 @@ export type NodeClientOptions = {
   ignoredDomains?: string[];
   allowedDomains?: string[];
   allowIpAddresses?: boolean;
+  proxyConfig?: ProxyConfigType;
   isWithinContext?: () => boolean;
 };
 
@@ -33,11 +37,21 @@ export class NodeClientRequest extends ClientRequest {
   public requestBuffer: Buffer | null;
   public requestId: string | null = null;
   public isInterceptable: boolean;
+  private originalUrl?: URL;
 
   constructor(
     [url, requestOptions, callback]: NormalizedClientRequestArgs,
     options: NodeClientOptions
   ) {
+    const tmpURL = new URL(url);
+    if (shouldProxyRequest(url, options.proxyConfig)) {
+      requestOptions = modifyRequestOptionsWithProxyConfig(
+        requestOptions,
+        url,
+        options?.proxyConfig
+      );
+    }
+
     super(requestOptions, callback);
 
     this.requestId = crypto.randomUUID();
@@ -57,6 +71,34 @@ export class NodeClientRequest extends ClientRequest {
       allowIpAddresses: options.allowIpAddresses ?? false,
       isWithinContext: options.isWithinContext ?? (() => true)
     });
+
+    if (shouldProxyRequest(this.url, options?.proxyConfig)) {
+      this.modifyRequestWithProxyConfig(tmpURL, options?.proxyConfig);
+    }
+  }
+
+  private modifyRequestWithProxyConfig(
+    tmpUrl: URL,
+    proxyConfig?: ProxyConfigType
+  ): void {
+    this.originalUrl = tmpUrl;
+    this.setHeader(
+      SupergoodProxyHeaders.upstreamHeader,
+      this.url.protocol + '//' + this.url.host
+    );
+    this.setHeader(SupergoodProxyHeaders.clientId, proxyConfig?.clientId || '');
+    this.setHeader(
+      SupergoodProxyHeaders.clientSecret,
+      proxyConfig?.clientSecret || ''
+    );
+    this.setHeader('host', proxyConfig?.proxyURL?.host || '');
+    if (proxyConfig?.proxyURL) {
+      this.url.protocol = proxyConfig?.proxyURL.protocol;
+      this.url.hostname = proxyConfig?.proxyURL.hostname;
+      this.url.host = proxyConfig?.proxyURL.host;
+      this.url.protocol = proxyConfig.proxyURL.protocol;
+      this.url.port = proxyConfig.proxyURL.port;
+    }
   }
 
   private writeRequestBodyChunk(
@@ -124,14 +166,18 @@ export class NodeClientRequest extends ClientRequest {
           message: IncomingMessage,
           emitter: EventEmitter
         ) {
-          const isomorphicResponse = await IsomorphicResponse.fromIncomingMessage(
-            message
-          );
+          const isomorphicResponse =
+            await IsomorphicResponse.fromIncomingMessage(message);
           emitter.emit(event, isomorphicResponse, requestId);
         }
 
-        emitResponse('response', this.requestId as string, secondClone, this.emitter)
-        return super.emit(event, firstClone, ...args.slice(1))
+        emitResponse(
+          'response',
+          this.requestId as string,
+          secondClone,
+          this.emitter
+        );
+        return super.emit(event, firstClone, ...args.slice(1));
       } catch (e) {
         return super.emit(event as string, ...args);
       }
@@ -151,7 +197,8 @@ export class NodeClientRequest extends ClientRequest {
       headers.set(headerName.toLowerCase(), headerValue.toString());
     }
 
-    const isomorphicRequest = new IsomorphicRequest(this.url, {
+    const url = this.originalUrl || this.url;
+    const isomorphicRequest = new IsomorphicRequest(url, {
       body,
       method: this.method || 'GET',
       credentials: 'same-origin',
@@ -161,3 +208,29 @@ export class NodeClientRequest extends ClientRequest {
     return isomorphicRequest;
   }
 }
+
+// NOTE: this function lives outside of the class since it
+// must be run prior to calling super()
+const modifyRequestOptionsWithProxyConfig = (
+  requestOptions: ResolvedRequestOptions,
+  url: URL,
+  proxyConfig?: ProxyConfigType
+): ResolvedRequestOptions => {
+  const modifiedRequestOptions = { ...requestOptions };
+  if (!modifiedRequestOptions.headers) {
+    modifiedRequestOptions.headers = {};
+  }
+
+  modifiedRequestOptions.headers[SupergoodProxyHeaders.upstreamHeader] =
+    url.protocol + '//' + url.host;
+  modifiedRequestOptions.headers[SupergoodProxyHeaders.clientId] =
+    proxyConfig?.clientId;
+  modifiedRequestOptions.headers[SupergoodProxyHeaders.clientSecret] =
+    proxyConfig?.clientSecret;
+
+  modifiedRequestOptions.protocol = proxyConfig?.proxyURL?.protocol;
+  modifiedRequestOptions.host = proxyConfig?.proxyURL?.host;
+  modifiedRequestOptions.hostname = proxyConfig?.proxyURL?.hostname;
+  modifiedRequestOptions.port = proxyConfig?.proxyURL?.port;
+  return modifiedRequestOptions;
+};

src/interceptor/NodeRequestInterceptor.ts

@@ -6,8 +6,8 @@ import { Protocol } from './NodeClientRequest';
 import { Interceptor, NodeRequestInterceptorOptions } from './Interceptor';
 import {
   ClientRequestArgs,
-  normalizeClientRequestArgs,
-} from './utils/request-args'
+  normalizeClientRequestArgs
+} from './utils/request-args';
 
 export type ClientRequestModules = Map<Protocol, typeof http | typeof https>;
 
@@ -38,15 +38,15 @@ export class NodeRequestInterceptor extends Interceptor {
         allowLocalUrls: this.options.allowLocalUrls,
         allowIpAddresses: this.options.allowIpAddresses,
         baseUrl: this.options.baseUrl,
-        isWithinContext,
+        proxyConfig: this.options.proxyConfig,
+        isWithinContext
       };
 
       // @ts-ignore
       requestModule.request = request(protocol, options);
 
       // @ts-ignore
       requestModule.get = get(protocol, options);
-
     }
   }
 }