From b34debcd096b50360ff82db0269d40f21a4c0948 Mon Sep 17 00:00:00 2001 From: Richard Rowlands Date: Thu, 9 May 2024 15:14:45 -0600 Subject: [PATCH] Support for 'maxColSizeMb' autoscaling config parameter --- admincli.js | 4 ++-- docs/aws.md | 13 ++++++------ libs/asr-providers/aws.js | 12 +++++------ libs/asr-providers/digitalocean.js | 12 +++++------ libs/asr-providers/hetzner.js | 12 +++++------ libs/asr-providers/scaleway.js | 12 +++++------ libs/classes/AbstractASRProvider.js | 14 ++++++------ libs/proxy.js | 33 ++++++++++++++++++++++++++++- libs/taskNew.js | 6 +++--- 9 files changed, 75 insertions(+), 43 deletions(-) diff --git a/admincli.js b/admincli.js index b7b4241..86ce9b2 100644 --- a/admincli.js +++ b/admincli.js @@ -232,8 +232,8 @@ module.exports = { const subcommand = args[0].toLocaleUpperCase(); args = args.slice(1, args.length); if (subcommand === "VIEWCMD"){ - const [ numImages ] = args; - const cmd = await asr.debugCreateDockerMachineCmd(numImages); + const [ numImages, colSizeMb ] = args; + const cmd = await asr.debugCreateDockerMachineCmd(numImages, colSizeMb); socket.write(`${cmd}\r\n`); }else{ invalid(); diff --git a/docs/aws.md b/docs/aws.md index b81cb0b..aa99a19 100644 --- a/docs/aws.md +++ b/docs/aws.md @@ -95,9 +95,10 @@ instance able to process the requested number of images is always selected. [EC2Instances.info](https://www.ec2instances.info) is a useful resource to help in selecting the appropriate instance type. -| Field | Description | -|-----------|---------------------------------------------------------------------------------------------------| -| maxImages | The maximum number of images this instance size can handle. | -| slug | EC2 instance type to request (for example, `t3.medium`). | -| storage | Amount of storage to allocate to this instance's EBS root volume, in GB. | -| spotPrice | The maximum hourly price you're willing to bid for this instance (if spot instances are enabled). | \ No newline at end of file +| Field | Description | +|--------------|---------------------------------------------------------------------------------------------------| +| maxImages | The maximum number of images this instance size can handle. | +| maxColSizeMb | Optional. The maximum size of an image collection this instance size can handle. | +| slug | EC2 instance type to request (for example, `t3.medium`). | +| storage | Amount of storage to allocate to this instance's EBS root volume, in GB. | +| spotPrice | The maximum hourly price you're willing to bid for this instance (if spot instances are enabled). | \ No newline at end of file diff --git a/libs/asr-providers/aws.js b/libs/asr-providers/aws.js index 8a9143d..1c1dd42 100644 --- a/libs/asr-providers/aws.js +++ b/libs/asr-providers/aws.js @@ -99,8 +99,8 @@ module.exports = class AWSAsrProvider extends AbstractASRProvider{ return `https://${this.getConfig("s3.bucket")}.${this.getConfig("s3.endpoint")}`; } - canHandle(imagesCount){ - return this.getImagePropertiesFor(imagesCount) !== null; + canHandle(imagesCount, colSizeMb){ + return this.getImagePropertiesFor(imagesCount, colSizeMb) !== null; } async setupMachine(req, token, dm, nodeToken){ @@ -138,13 +138,13 @@ module.exports = class AWSAsrProvider extends AbstractASRProvider{ `--token ${nodeToken}`].join(" ")); } - getImagePropertiesFor(imagesCount){ + getImagePropertiesFor(imagesCount, colSizeMb){ const im = this.getConfig("imageSizeMapping"); let props = null; for (var k in im){ const mapping = im[k]; - if (mapping['maxImages'] >= imagesCount){ + if (mapping['maxImages'] >= imagesCount && (mapping['maxColSizeMb'] == null || mapping['maxColSizeMb'] >= colSizeMb)){ props = mapping; break; } @@ -161,8 +161,8 @@ module.exports = class AWSAsrProvider extends AbstractASRProvider{ return this.getConfig("maxUploadTime"); } - async getCreateArgs(imagesCount){ - const image_props = this.getImagePropertiesFor(imagesCount); + async getCreateArgs(imagesCount, colSizeMb){ + const image_props = this.getImagePropertiesFor(imagesCount, colSizeMb); const args = [ "--amazonec2-access-key", this.getConfig("accessKey"), "--amazonec2-secret-key", this.getConfig("secretKey"), diff --git a/libs/asr-providers/digitalocean.js b/libs/asr-providers/digitalocean.js index b78916f..619463b 100644 --- a/libs/asr-providers/digitalocean.js +++ b/libs/asr-providers/digitalocean.js @@ -103,10 +103,10 @@ module.exports = class DigitalOceanAsrProvider extends AbstractASRProvider{ return `https://${this.getConfig("s3.bucket")}.${this.getConfig("s3.endpoint")}`; } - canHandle(imagesCount){ + canHandle(imagesCount, colSizeMb){ const minImages = this.getConfig("minImages", -1); - return this.getImageSlugFor(imagesCount) !== null && + return this.getImageSlugFor(imagesCount, colSizeMb) !== null && (minImages === -1 || imagesCount >= minImages); } @@ -137,13 +137,13 @@ module.exports = class DigitalOceanAsrProvider extends AbstractASRProvider{ `--token ${nodeToken}`].join(" ")); } - getImageSlugFor(imagesCount){ + getImageSlugFor(imagesCount, colSizeMb){ const im = this.getConfig("imageSizeMapping"); let slug = null; for (var k in im){ const mapping = im[k]; - if (mapping['maxImages'] >= imagesCount){ + if (mapping['maxImages'] >= imagesCount && (mapping['maxColSizeMb'] == null || mapping['maxColSizeMb'] >= colSizeMb)){ slug = mapping['slug']; break; } @@ -204,14 +204,14 @@ module.exports = class DigitalOceanAsrProvider extends AbstractASRProvider{ }; } - async getCreateArgs(imagesCount){ + async getCreateArgs(imagesCount, colSizeMb){ const imageInfo = await this.getImageInfo(); const args = [ "--digitalocean-access-token", this.getConfig("accessToken"), "--digitalocean-region", imageInfo.region, "--digitalocean-image", imageInfo.image, - "--digitalocean-size", this.getImageSlugFor(imagesCount) + "--digitalocean-size", this.getImageSlugFor(imagesCount, colSizeMb) ]; if (this.getConfig("monitoring")){ diff --git a/libs/asr-providers/hetzner.js b/libs/asr-providers/hetzner.js index 3e035bb..08efebc 100644 --- a/libs/asr-providers/hetzner.js +++ b/libs/asr-providers/hetzner.js @@ -103,10 +103,10 @@ module.exports = class HetznerAsrProvider extends AbstractASRProvider{ return `https://${this.getConfig("s3.bucket")}.${this.getConfig("s3.endpoint")}`; } - canHandle(imagesCount){ + canHandle(imagesCount, colSizeMb){ const minImages = this.getConfig("minImages", -1); - return this.getImageSlugFor(imagesCount) !== null && + return this.getImageSlugFor(imagesCount, colSizeMb) !== null && (minImages === -1 || imagesCount >= minImages); } @@ -188,13 +188,13 @@ module.exports = class HetznerAsrProvider extends AbstractASRProvider{ `--token ${nodeToken}`].join(" ")); } - getImageSlugFor(imagesCount){ + getImageSlugFor(imagesCount, colSizeMb){ const im = this.getConfig("imageSizeMapping"); let slug = null; for (var k in im){ const mapping = im[k]; - if (mapping['maxImages'] >= imagesCount){ + if (mapping['maxImages'] >= imagesCount && (mapping['maxColSizeMb'] == null || mapping['maxColSizeMb'] >= colSizeMb)){ slug = mapping['slug']; break; } @@ -211,11 +211,11 @@ module.exports = class HetznerAsrProvider extends AbstractASRProvider{ return this.getConfig("maxUploadTime"); } - async getCreateArgs(imagesCount){ + async getCreateArgs(imagesCount, colSizeMb){ const args = [ "--hetzner-api-token", this.getConfig("apiToken"), "--hetzner-server-location", this.getConfig("location"), - "--hetzner-server-type", this.getImageSlugFor(imagesCount) + "--hetzner-server-type", this.getImageSlugFor(imagesCount, colSizeMb) ]; if (this.getConfig("snapshot")){ diff --git a/libs/asr-providers/scaleway.js b/libs/asr-providers/scaleway.js index 5ed1250..664a47e 100644 --- a/libs/asr-providers/scaleway.js +++ b/libs/asr-providers/scaleway.js @@ -86,10 +86,10 @@ module.exports = class ScalewayAsrProvider extends AbstractASRProvider{ return `https://${this.getConfig("s3.bucket")}.${this.getConfig("s3.endpoint")}`; } - canHandle(imagesCount){ + canHandle(imagesCount, colSizeMb){ const minImages = this.getConfig("minImages", -1); - return this.getImageSlugFor(imagesCount) !== null && + return this.getImageSlugFor(imagesCount, colSizeMb) !== null && (minImages === -1 || imagesCount >= minImages); } @@ -120,13 +120,13 @@ module.exports = class ScalewayAsrProvider extends AbstractASRProvider{ `--token ${nodeToken}`].join(" ")); } - getImageSlugFor(imagesCount){ + getImageSlugFor(imagesCount, colSizeMb){ const im = this.getConfig("imageSizeMapping"); let slug = null; for (var k in im){ const mapping = im[k]; - if (mapping['maxImages'] >= imagesCount){ + if (mapping['maxImages'] >= imagesCount && (mapping['maxColSizeMb'] == null || mapping['maxColSizeMb'] >= colSizeMb)){ slug = mapping['slug']; break; } @@ -143,13 +143,13 @@ module.exports = class ScalewayAsrProvider extends AbstractASRProvider{ return this.getConfig("maxUploadTime"); } - async getCreateArgs(imagesCount){ + async getCreateArgs(imagesCount, colSizeMb){ const args = [ "--scaleway-organization", this.getConfig("organization"), "--scaleway-token", this.getConfig("secretToken"), "--scaleway-region", this.getConfig("region"), "--scaleway-image", this.getConfig("image"), - "--scaleway-commercial-type", this.getImageSlugFor(imagesCount) + "--scaleway-commercial-type", this.getImageSlugFor(imagesCount, colSizeMb) ]; if (this.getConfig("engineInstallUrl")){ diff --git a/libs/classes/AbstractASRProvider.js b/libs/classes/AbstractASRProvider.js index 76082d8..25d11aa 100644 --- a/libs/classes/AbstractASRProvider.js +++ b/libs/classes/AbstractASRProvider.js @@ -46,11 +46,11 @@ module.exports = class AbstractASRProvider{ throw new Error("Not implemented"); } - async getCreateArgs(imagesCount){ + async getCreateArgs(imagesCount, colSizeMb){ throw new Error("Not implemented"); } - canHandle(imagesCount){ + canHandle(imagesCount, colSizeMb){ throw new Error("Not implemented"); } @@ -98,8 +98,8 @@ module.exports = class AbstractASRProvider{ } // Helper function for debugging - async debugCreateDockerMachineCmd(imagesCount){ - const args = await this.getCreateArgs(imagesCount); + async debugCreateDockerMachineCmd(imagesCount, colSizeMb){ + const args = await this.getCreateArgs(imagesCount, colSizeMb); return `docker-machine create --driver ${this.getDriverName()} ${args.join(" ")} debug-machine`; } @@ -110,12 +110,12 @@ module.exports = class AbstractASRProvider{ // @param hostname {String} docker-machine hostname // @param status {Object} status information about the task being created // @return {Node} a new Node instance - async createNode(req, imagesCount, token, hostname, status){ - if (!this.canHandle(imagesCount)) throw new Error(`Cannot handle ${imagesCount} images.`); + async createNode(req, imagesCount, colSizeMb, token, hostname, status){ + if (!this.canHandle(imagesCount, colSizeMb)) throw new Error(`Cannot handle ${imagesCount} images.`); const dm = new DockerMachine(hostname); const args = ["--driver", this.getDriverName()] - .concat(await this.getCreateArgs(imagesCount)); + .concat(await this.getCreateArgs(imagesCount, colSizeMb)); const nodeToken = short.generate(); try{ diff --git a/libs/proxy.js b/libs/proxy.js index db93be1..9596e70 100644 --- a/libs/proxy.js +++ b/libs/proxy.js @@ -389,12 +389,43 @@ module.exports = { if (err) cb(err); else cb(null, files.filter(f => f.toLowerCase() !== "body.json")); }); + }, + + cb => { + let total = 0; // this field in bytes + + fs.readdir(tmpPath, (err, names) => { + if (err) return cb(err); + + names = names.filter(f => f.toLowerCase() !== "body.json"); + + let left = names.length; + + if (left === 0) return cb(null, total); + + function done(size) + { + total += size + + left-- + if (left === 0) cb(null, total/1024/1024); + } + + for (let name of names) + { + fs.lstat(path.join(tmpPath, name), (err, stats) => { + if (err) return cb(err); + done(stats.size) + }) + } + }); } - ], async (err, [ body, files ]) => { + ], async (err, [ body, files, colSizeMb ]) => { if (err) json(res, {error: err.message}); else{ body.fileNames = files; body.imagesCount = files.length; + body.colSizeMb = colSizeMb; try{ await taskNew.process(req, res, cloudProvider, taskId, body, query.token, limits, getLimitedOptions); diff --git a/libs/taskNew.js b/libs/taskNew.js index 6bd8c9e..9b0b1f1 100644 --- a/libs/taskNew.js +++ b/libs/taskNew.js @@ -286,7 +286,7 @@ module.exports = { process: async function(req, res, cloudProvider, uuid, params, token, limits, getLimitedOptions){ const tmpPath = path.join("tmp", uuid); - const { options, taskName, skipPostProcessing, outputs, dateCreated, fileNames, imagesCount, webhook } = params; + const { options, taskName, skipPostProcessing, outputs, dateCreated, fileNames, imagesCount, colSizeMb, webhook } = params; if (fileNames.length < 1){ throw new Error(`Not enough images (${fileNames.length} files uploaded)`); @@ -307,7 +307,7 @@ module.exports = { // Do we need to / can we create a new node via autoscaling? const autoscale = (!node || node.availableSlots() === 0) && asrProvider.isAllowedToCreateNewNodes() && - asrProvider.canHandle(fileNames.length); + asrProvider.canHandle(fileNames.length, colSizeMb); if (autoscale) node = nodes.referenceNode(); // Use the reference node for task options purposes @@ -560,7 +560,7 @@ module.exports = { const asr = asrProvider.get(); try{ dmHostname = asr.generateHostname(imagesCount); - node = await asr.createNode(req, imagesCount, token, dmHostname, status); + node = await asr.createNode(req, imagesCount, colSizeMb, token, dmHostname, status); if (!status.aborted) nodes.add(node); else return; }catch(e){