server: update http fetch usage, add support for relative require

koush · Jan 10, 2024 · aef218c · aef218c
1 parent aea24a8
commit aef218c
Show file tree

Hide file tree

Showing 4 changed files with 51 additions and 21 deletions.
diff --git a/server/src/http-fetch-helpers.ts b/server/src/http-fetch-helpers.ts
@@ -1,11 +1,11 @@
 import { once } from 'events';
 import { http, https } from 'follow-redirects';
-import { IncomingHttpHeaders, IncomingMessage } from 'http';
-import { Readable } from 'stream';
+import { IncomingMessage } from 'http';
+import type { Readable } from 'stream';
 
 export type HttpFetchResponseType = 'json' | 'text' | 'buffer' | 'readable';
 export interface HttpFetchOptions<T extends HttpFetchResponseType> {
-    url: string|URL;
+    url: string | URL;
     family?: 4 | 6;
     method?: string;
     headers?: HeadersInit;
@@ -71,12 +71,22 @@ export async function getNpmPackageInfo(pkg: string) {
         url: `https://registry.npmjs.org/${pkg}`,
         // force ipv4 in case of busted ipv6.
         family: 4,
+        responseType: 'json',
     });
     return body;
 }
 
-export function setFetchAcceptOptions(accept: string, headers: Headers) {
-    headers.set('Accept', accept);
+export function getHttpFetchAccept(responseType: HttpFetchResponseType) {
+    const { accept } = getHttpFetchParser(responseType);
+    return accept;
+}
+
+export function setDefaultHttpFetchAccept(headers: Headers, responseType: HttpFetchResponseType) {
+    if (headers.has('Accept'))
+        return;
+    const { accept } = getHttpFetchParser(responseType);
+    if (accept)
+        headers.set('Accept', accept);
 }
 
 export function fetchStatusCodeOk(statusCode: number) {
@@ -90,7 +100,7 @@ export function checkStatus(statusCode: number) {
 
 export interface HttpFetchResponse<T> {
     statusCode: number;
-    headers: IncomingHttpHeaders;
+    headers: Headers;
     body: T;
 }
 
@@ -106,6 +116,10 @@ export function getHttpFetchParser(responseType: HttpFetchResponseType) {
     return BufferParser;
 }
 
+export function parseResponseType(readable: IncomingMessage, responseType: HttpFetchResponseType) {
+    return getHttpFetchParser(responseType).parse(readable);
+}
+
 export async function httpFetch<T extends HttpFetchOptions<HttpFetchResponseType>>(options: T): Promise<HttpFetchResponse<
     // first one serves as default.
     T extends HttpFetchBufferOptions ? Buffer
@@ -114,10 +128,9 @@ export async function httpFetch<T extends HttpFetchOptions<HttpFetchResponseType
     : T extends HttpFetchJsonOptions ? any : Buffer
 >> {
     const headers = new Headers(options.headers);
-    const parser = getHttpFetchParser(options.responseType);
+    setDefaultHttpFetchAccept(headers, options.responseType);
 
-    if (parser.accept)
-        setFetchAcceptOptions(parser.accept, headers);
+    const parser = getHttpFetchParser(options.responseType);
 
     const { url } = options;
     const isSecure = url.toString().startsWith('https:');
@@ -144,7 +157,7 @@ export async function httpFetch<T extends HttpFetchOptions<HttpFetchResponseType
         options.body.pipe(request);
     else
         request.end();
-    const [response] = await once(request, 'response') as IncomingMessage[];
+    const [response] = await once(request, 'response') as [IncomingMessage];
 
     if (!options?.ignoreStatusCode) {
         try {
@@ -156,9 +169,16 @@ export async function httpFetch<T extends HttpFetchOptions<HttpFetchResponseType
         }
     }
 
+    const incomingHeaders = new Headers();
+    for (const [k, v] of Object.entries(response.headers)) {
+        for (const vv of (typeof v === 'string' ? [v] : v)) {
+            incomingHeaders.append(k, vv)
+        }
+    }
+
     return {
         statusCode: response.statusCode,
-        headers: response.headers,
+        headers: incomingHeaders,
         body: await parser.parse(response),
     };
 }

diff --git a/server/src/plugin/plugin-remote-worker.ts b/server/src/plugin/plugin-remote-worker.ts
@@ -1,16 +1,16 @@
 import { ScryptedStatic, SystemManager } from '@scrypted/types';
 import AdmZip from 'adm-zip';
-import crypto from 'crypto';
 import { once } from 'events';
 import fs from 'fs';
 import { Volume } from 'memfs';
 import net from 'net';
 import path from 'path';
 import { install as installSourceMapSupport } from 'source-map-support';
+import { computeClusterObjectHash } from '../cluster/cluster-hash';
+import { ClusterObject, ConnectRPCObject } from '../cluster/connect-rpc-object';
 import { listenZero } from '../listen-zero';
 import { RpcMessage, RpcPeer } from '../rpc';
 import { createDuplexRpcPeer } from '../rpc-serializer';
-import { ClusterObject, ConnectRPCObject } from '../cluster/connect-rpc-object';
 import { MediaManagerImpl } from './media';
 import { PluginAPI, PluginAPIProxy, PluginRemote, PluginRemoteLoadZipOptions } from './plugin-api';
 import { prepareConsoles } from './plugin-console';
@@ -19,7 +19,6 @@ import { DeviceManagerImpl, PluginReader, attachPluginRemote, setupPluginRemote
 import { PluginStats, startStatsUpdater } from './plugin-remote-stats';
 import { createREPLServer } from './plugin-repl';
 import { NodeThreadWorker } from './runtime/node-thread-worker';
-import { computeClusterObjectHash } from '../cluster/cluster-hash';
 const { link } = require('linkfs');
 
 const serverVersion = require('../../package.json').version;
@@ -243,6 +242,15 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
                     return require('fs');
                 }
                 try {
+                    if (name.startsWith('.') && zipOptions?.unzippedPath) {
+                        try {
+                            const c = path.join(zipOptions.unzippedPath, name);
+                            const module = require(c);
+                            return module;
+                        }
+                        catch (e) {
+                        }
+                    }
                     const module = require(name);
                     return module;
                 }

diff --git a/server/src/runtime.ts b/server/src/runtime.ts
@@ -1,4 +1,3 @@
-import net from 'net';
 import { Device, DeviceInformation, DeviceProvider, EngineIOHandler, HttpRequest, HttpRequestHandler, ScryptedDevice, ScryptedInterface, ScryptedInterfaceMethod, ScryptedInterfaceProperty, ScryptedNativeId, ScryptedUser as SU } from '@scrypted/types';
 import AdmZip from 'adm-zip';
 import crypto from 'crypto';
@@ -9,6 +8,7 @@ import { ParamsDictionary } from 'express-serve-static-core';
 import fs from 'fs';
 import http, { ServerResponse } from 'http';
 import https from 'https';
+import net from 'net';
 import type { spawn as ptySpawn } from 'node-pty-prebuilt-multiarch';
 import path from 'path';
 import { ParsedQs } from 'qs';
@@ -17,8 +17,10 @@ import { PassThrough } from 'stream';
 import tar from 'tar';
 import { URL } from "url";
 import WebSocket, { Server as WebSocketServer } from "ws";
+import { computeClusterObjectHash } from './cluster/cluster-hash';
+import { ClusterObject } from './cluster/connect-rpc-object';
 import { Plugin, PluginDevice, ScryptedAlert, ScryptedUser } from './db-types';
-import { fetchBuffer, getNpmPackageInfo } from './http-fetch-helpers';
+import { getNpmPackageInfo, httpFetch } from './http-fetch-helpers';
 import { createResponseInterface } from './http-interfaces';
 import { getDisplayName, getDisplayRoom, getDisplayType, getProvidedNameOrDefault, getProvidedRoomOrDefault, getProvidedTypeOrDefault } from './infer-defaults';
 import { IOServer } from './io';
@@ -35,7 +37,6 @@ import { getPluginVolume } from './plugin/plugin-volume';
 import { NodeForkWorker } from './plugin/runtime/node-fork-worker';
 import { PythonRuntimeWorker } from './plugin/runtime/python-worker';
 import { RuntimeWorker, RuntimeWorkerOptions } from './plugin/runtime/runtime-worker';
-import { ClusterObject } from './cluster/connect-rpc-object';
 import { getIpAddress, SCRYPTED_INSECURE_PORT, SCRYPTED_SECURE_PORT } from './server-settings';
 import { AddressSettings } from './services/addresses';
 import { Alerts } from './services/alerts';
@@ -45,7 +46,6 @@ import { PluginComponent } from './services/plugin';
 import { ServiceControl } from './services/service-control';
 import { UsersService } from './services/users';
 import { getState, ScryptedStateManager, setState } from './state';
-import { computeClusterObjectHash } from './cluster/cluster-hash';
 
 interface DeviceProxyPair {
     handler: PluginDeviceProxyHandler;
@@ -619,7 +619,8 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
         }
         console.log('installing package', pkg, version);
 
-        const { body: tarball } = await fetchBuffer(`${registry.versions[version].dist.tarball}`, {
+        const { body: tarball } = await httpFetch( {
+            url: `${registry.versions[version].dist.tarball}`,
             // force ipv4 in case of busted ipv6.
             family: 4,
         });

diff --git a/server/src/services/service-control.ts b/server/src/services/service-control.ts
@@ -1,5 +1,5 @@
 import fs from 'fs';
-import { fetchJSON } from "../http-fetch-helpers";
+import { httpFetch } from "../http-fetch-helpers";
 import { ScryptedRuntime } from "../runtime";
 
 export class ServiceControl {
@@ -23,7 +23,8 @@ export class ServiceControl {
         const webhookUpdate = process.env.SCRYPTED_WEBHOOK_UPDATE;
         if (webhookUpdate) {
             const webhookUpdateAuthorization = process.env.SCRYPTED_WEBHOOK_UPDATE_AUTHORIZATION;
-            await fetchJSON(webhookUpdate, {
+            await httpFetch({
+                url: webhookUpdate,
                 headers: {
                     Authorization: webhookUpdateAuthorization,
                 }