Skip to content

Commit

Permalink
Support for 'maxColSizeMb' autoscaling config parameter
Browse files Browse the repository at this point in the history
  • Loading branch information
@rrowlands
rrowlands committed May 9, 2024
1 parent 0562a47 commit b34debc
Show file tree
Hide file tree
Showing 9 changed files with 75 additions and 43 deletions.
4 changes: 2 additions & 2 deletions admincli.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,8 +232,8 @@ module.exports = {
const subcommand = args[0].toLocaleUpperCase();
args = args.slice(1, args.length);
if (subcommand === "VIEWCMD"){
const [ numImages ] = args;
const cmd = await asr.debugCreateDockerMachineCmd(numImages);
const [ numImages, colSizeMb ] = args;
const cmd = await asr.debugCreateDockerMachineCmd(numImages, colSizeMb);
socket.write(`${cmd}\r\n`);
}else{
invalid();
Expand Down
13 changes: 7 additions & 6 deletions docs/aws.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,9 +95,10 @@ instance able to process the requested number of images is always selected.

[EC2Instances.info](https://www.ec2instances.info) is a useful resource to help in selecting the appropriate instance type.

| Field | Description |
|-----------|---------------------------------------------------------------------------------------------------|
| maxImages | The maximum number of images this instance size can handle. |
| slug | EC2 instance type to request (for example, `t3.medium`). |
| storage | Amount of storage to allocate to this instance's EBS root volume, in GB. |
| spotPrice | The maximum hourly price you're willing to bid for this instance (if spot instances are enabled). |
| Field | Description |
|--------------|---------------------------------------------------------------------------------------------------|
| maxImages | The maximum number of images this instance size can handle. |
| maxColSizeMb | Optional. The maximum size of an image collection this instance size can handle. |
| slug | EC2 instance type to request (for example, `t3.medium`). |
| storage | Amount of storage to allocate to this instance's EBS root volume, in GB. |
| spotPrice | The maximum hourly price you're willing to bid for this instance (if spot instances are enabled). |
12 changes: 6 additions & 6 deletions libs/asr-providers/aws.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,8 @@ module.exports = class AWSAsrProvider extends AbstractASRProvider{
return `https://${this.getConfig("s3.bucket")}.${this.getConfig("s3.endpoint")}`;
}

canHandle(imagesCount){
return this.getImagePropertiesFor(imagesCount) !== null;
canHandle(imagesCount, colSizeMb){
return this.getImagePropertiesFor(imagesCount, colSizeMb) !== null;
}

async setupMachine(req, token, dm, nodeToken){
Expand Down Expand Up @@ -138,13 +138,13 @@ module.exports = class AWSAsrProvider extends AbstractASRProvider{
`--token ${nodeToken}`].join(" "));
}

getImagePropertiesFor(imagesCount){
getImagePropertiesFor(imagesCount, colSizeMb){
const im = this.getConfig("imageSizeMapping");

let props = null;
for (var k in im){
const mapping = im[k];
if (mapping['maxImages'] >= imagesCount){
if (mapping['maxImages'] >= imagesCount && (mapping['maxColSizeMb'] == null || mapping['maxColSizeMb'] >= colSizeMb)){
props = mapping;
break;
}
Expand All @@ -161,8 +161,8 @@ module.exports = class AWSAsrProvider extends AbstractASRProvider{
return this.getConfig("maxUploadTime");
}

async getCreateArgs(imagesCount){
const image_props = this.getImagePropertiesFor(imagesCount);
async getCreateArgs(imagesCount, colSizeMb){
const image_props = this.getImagePropertiesFor(imagesCount, colSizeMb);
const args = [
"--amazonec2-access-key", this.getConfig("accessKey"),
"--amazonec2-secret-key", this.getConfig("secretKey"),
Expand Down
12 changes: 6 additions & 6 deletions libs/asr-providers/digitalocean.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,10 +103,10 @@ module.exports = class DigitalOceanAsrProvider extends AbstractASRProvider{
return `https://${this.getConfig("s3.bucket")}.${this.getConfig("s3.endpoint")}`;
}

canHandle(imagesCount){
canHandle(imagesCount, colSizeMb){
const minImages = this.getConfig("minImages", -1);

return this.getImageSlugFor(imagesCount) !== null &&
return this.getImageSlugFor(imagesCount, colSizeMb) !== null &&
(minImages === -1 || imagesCount >= minImages);
}

Expand Down Expand Up @@ -137,13 +137,13 @@ module.exports = class DigitalOceanAsrProvider extends AbstractASRProvider{
`--token ${nodeToken}`].join(" "));
}

getImageSlugFor(imagesCount){
getImageSlugFor(imagesCount, colSizeMb){
const im = this.getConfig("imageSizeMapping");

let slug = null;
for (var k in im){
const mapping = im[k];
if (mapping['maxImages'] >= imagesCount){
if (mapping['maxImages'] >= imagesCount && (mapping['maxColSizeMb'] == null || mapping['maxColSizeMb'] >= colSizeMb)){
slug = mapping['slug'];
break;
}
Expand Down Expand Up @@ -204,14 +204,14 @@ module.exports = class DigitalOceanAsrProvider extends AbstractASRProvider{
};
}

async getCreateArgs(imagesCount){
async getCreateArgs(imagesCount, colSizeMb){
const imageInfo = await this.getImageInfo();

const args = [
"--digitalocean-access-token", this.getConfig("accessToken"),
"--digitalocean-region", imageInfo.region,
"--digitalocean-image", imageInfo.image,
"--digitalocean-size", this.getImageSlugFor(imagesCount)
"--digitalocean-size", this.getImageSlugFor(imagesCount, colSizeMb)
];

if (this.getConfig("monitoring")){
Expand Down
12 changes: 6 additions & 6 deletions libs/asr-providers/hetzner.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,10 +103,10 @@ module.exports = class HetznerAsrProvider extends AbstractASRProvider{
return `https://${this.getConfig("s3.bucket")}.${this.getConfig("s3.endpoint")}`;
}

canHandle(imagesCount){
canHandle(imagesCount, colSizeMb){
const minImages = this.getConfig("minImages", -1);

return this.getImageSlugFor(imagesCount) !== null &&
return this.getImageSlugFor(imagesCount, colSizeMb) !== null &&
(minImages === -1 || imagesCount >= minImages);
}

Expand Down Expand Up @@ -188,13 +188,13 @@ module.exports = class HetznerAsrProvider extends AbstractASRProvider{
`--token ${nodeToken}`].join(" "));
}

getImageSlugFor(imagesCount){
getImageSlugFor(imagesCount, colSizeMb){
const im = this.getConfig("imageSizeMapping");

let slug = null;
for (var k in im){
const mapping = im[k];
if (mapping['maxImages'] >= imagesCount){
if (mapping['maxImages'] >= imagesCount && (mapping['maxColSizeMb'] == null || mapping['maxColSizeMb'] >= colSizeMb)){
slug = mapping['slug'];
break;
}
Expand All @@ -211,11 +211,11 @@ module.exports = class HetznerAsrProvider extends AbstractASRProvider{
return this.getConfig("maxUploadTime");
}

async getCreateArgs(imagesCount){
async getCreateArgs(imagesCount, colSizeMb){
const args = [
"--hetzner-api-token", this.getConfig("apiToken"),
"--hetzner-server-location", this.getConfig("location"),
"--hetzner-server-type", this.getImageSlugFor(imagesCount)
"--hetzner-server-type", this.getImageSlugFor(imagesCount, colSizeMb)
];

if (this.getConfig("snapshot")){
Expand Down
12 changes: 6 additions & 6 deletions libs/asr-providers/scaleway.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,10 +86,10 @@ module.exports = class ScalewayAsrProvider extends AbstractASRProvider{
return `https://${this.getConfig("s3.bucket")}.${this.getConfig("s3.endpoint")}`;
}

canHandle(imagesCount){
canHandle(imagesCount, colSizeMb){
const minImages = this.getConfig("minImages", -1);

return this.getImageSlugFor(imagesCount) !== null &&
return this.getImageSlugFor(imagesCount, colSizeMb) !== null &&
(minImages === -1 || imagesCount >= minImages);
}

Expand Down Expand Up @@ -120,13 +120,13 @@ module.exports = class ScalewayAsrProvider extends AbstractASRProvider{
`--token ${nodeToken}`].join(" "));
}

getImageSlugFor(imagesCount){
getImageSlugFor(imagesCount, colSizeMb){
const im = this.getConfig("imageSizeMapping");

let slug = null;
for (var k in im){
const mapping = im[k];
if (mapping['maxImages'] >= imagesCount){
if (mapping['maxImages'] >= imagesCount && (mapping['maxColSizeMb'] == null || mapping['maxColSizeMb'] >= colSizeMb)){
slug = mapping['slug'];
break;
}
Expand All @@ -143,13 +143,13 @@ module.exports = class ScalewayAsrProvider extends AbstractASRProvider{
return this.getConfig("maxUploadTime");
}

async getCreateArgs(imagesCount){
async getCreateArgs(imagesCount, colSizeMb){
const args = [
"--scaleway-organization", this.getConfig("organization"),
"--scaleway-token", this.getConfig("secretToken"),
"--scaleway-region", this.getConfig("region"),
"--scaleway-image", this.getConfig("image"),
"--scaleway-commercial-type", this.getImageSlugFor(imagesCount)
"--scaleway-commercial-type", this.getImageSlugFor(imagesCount, colSizeMb)
];

if (this.getConfig("engineInstallUrl")){
Expand Down
14 changes: 7 additions & 7 deletions libs/classes/AbstractASRProvider.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,11 @@ module.exports = class AbstractASRProvider{
throw new Error("Not implemented");
}

async getCreateArgs(imagesCount){
async getCreateArgs(imagesCount, colSizeMb){
throw new Error("Not implemented");
}

canHandle(imagesCount){
canHandle(imagesCount, colSizeMb){
throw new Error("Not implemented");
}

Expand Down Expand Up @@ -98,8 +98,8 @@ module.exports = class AbstractASRProvider{
}

// Helper function for debugging
async debugCreateDockerMachineCmd(imagesCount){
const args = await this.getCreateArgs(imagesCount);
async debugCreateDockerMachineCmd(imagesCount, colSizeMb){
const args = await this.getCreateArgs(imagesCount, colSizeMb);
return `docker-machine create --driver ${this.getDriverName()} ${args.join(" ")} debug-machine`;
}

Expand All @@ -110,12 +110,12 @@ module.exports = class AbstractASRProvider{
// @param hostname {String} docker-machine hostname
// @param status {Object} status information about the task being created
// @return {Node} a new Node instance
async createNode(req, imagesCount, token, hostname, status){
if (!this.canHandle(imagesCount)) throw new Error(`Cannot handle ${imagesCount} images.`);
async createNode(req, imagesCount, colSizeMb, token, hostname, status){
if (!this.canHandle(imagesCount, colSizeMb)) throw new Error(`Cannot handle ${imagesCount} images.`);

const dm = new DockerMachine(hostname);
const args = ["--driver", this.getDriverName()]
.concat(await this.getCreateArgs(imagesCount));
.concat(await this.getCreateArgs(imagesCount, colSizeMb));
const nodeToken = short.generate();

try{
Expand Down
33 changes: 32 additions & 1 deletion libs/proxy.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -389,12 +389,43 @@ module.exports = {
if (err) cb(err);
else cb(null, files.filter(f => f.toLowerCase() !== "body.json"));
});
},

cb => {
let total = 0; // this field in bytes

fs.readdir(tmpPath, (err, names) => {
if (err) return cb(err);

names = names.filter(f => f.toLowerCase() !== "body.json");

let left = names.length;

if (left === 0) return cb(null, total);

function done(size)
{
total += size

left--
if (left === 0) cb(null, total/1024/1024);
}

for (let name of names)
{
fs.lstat(path.join(tmpPath, name), (err, stats) => {
if (err) return cb(err);
done(stats.size)
})
}
});
}
], async (err, [ body, files ]) => {
], async (err, [ body, files, colSizeMb ]) => {
if (err) json(res, {error: err.message});
else{
body.fileNames = files;
body.imagesCount = files.length;
body.colSizeMb = colSizeMb;

try{
await taskNew.process(req, res, cloudProvider, taskId, body, query.token, limits, getLimitedOptions);
Expand Down
6 changes: 3 additions & 3 deletions libs/taskNew.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,7 @@ module.exports = {

process: async function(req, res, cloudProvider, uuid, params, token, limits, getLimitedOptions){
const tmpPath = path.join("tmp", uuid);
const { options, taskName, skipPostProcessing, outputs, dateCreated, fileNames, imagesCount, webhook } = params;
const { options, taskName, skipPostProcessing, outputs, dateCreated, fileNames, imagesCount, colSizeMb, webhook } = params;

if (fileNames.length < 1){
throw new Error(`Not enough images (${fileNames.length} files uploaded)`);
Expand All @@ -307,7 +307,7 @@ module.exports = {
// Do we need to / can we create a new node via autoscaling?
const autoscale = (!node || node.availableSlots() === 0) &&
asrProvider.isAllowedToCreateNewNodes() &&
asrProvider.canHandle(fileNames.length);
asrProvider.canHandle(fileNames.length, colSizeMb);

if (autoscale) node = nodes.referenceNode(); // Use the reference node for task options purposes

Expand Down Expand Up @@ -560,7 +560,7 @@ module.exports = {
const asr = asrProvider.get();
try{
dmHostname = asr.generateHostname(imagesCount);
node = await asr.createNode(req, imagesCount, token, dmHostname, status);
node = await asr.createNode(req, imagesCount, colSizeMb, token, dmHostname, status);
if (!status.aborted) nodes.add(node);
else return;
}catch(e){
Expand Down

0 comments on commit b34debc

Please sign in to comment.