Skip to content

Commit

Permalink
fix(aws): add user-data support and change spot instance type
Browse files Browse the repository at this point in the history
  • Loading branch information
@JacieChao
JacieChao committed Jun 1, 2022
1 parent dfb8b3c commit adfa12a
Show file tree
Hide file tree
Showing 4 changed files with 57 additions and 103 deletions.
6 changes: 0 additions & 6 deletions pkg/cluster/base.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,12 +84,6 @@ func NewBaseProvider() *ProviderBase {
// GetCreateOptions get create command flag options.
func (p *ProviderBase) GetCreateOptions() []types.Flag {
return []types.Flag{
{
Name: "ui",
P: &p.UI,
V: p.UI,
Usage: "(deprecated) Enable K3s UI(kubernetes/dashboard). Will remove at v0.5.0. For how to login to UI, please see: https://github.com/kubernetes/dashboard/blob/master/docs/user/access-control/creating-sample-user.md",
},
{
Name: "cluster",
P: &p.Cluster,
Expand Down
140 changes: 43 additions & 97 deletions pkg/providers/aws/aws.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"os"
"reflect"
"strconv"
Expand Down Expand Up @@ -39,7 +40,6 @@ const (
defaultSecurityGroupName = "autok3s"
defaultDeviceName = "/dev/sda1"
requestSpotInstance = false
defaultSpotPrice = "0.50"
)

const (
Expand All @@ -51,8 +51,6 @@ type Amazon struct {
*cluster.ProviderBase `json:",inline"`
typesaws.Options `json:",inline"`
client *ec2.EC2

spotInstanceRequestIDs []string
}

func init() {
Expand All @@ -76,7 +74,6 @@ func newProvider() *Amazon {
RequestSpotInstance: requestSpotInstance,
CloudControllerManager: false,
},
spotInstanceRequestIDs: []string{},
}
}

Expand Down Expand Up @@ -241,13 +238,6 @@ func (p *Amazon) rollbackInstance(ids []string) error {
if err := p.terminateInstance(ids); err != nil {
return err
}

// cancel unfulfilled spot instance request
if len(p.spotInstanceRequestIDs) > 0 {
if err := p.cancelSpotInstance(); err != nil {
return err
}
}
return nil
}

Expand All @@ -268,6 +258,16 @@ func (p *Amazon) generateInstance(ssh *types.SSH) (*types.Cluster, error) {
}
}

if p.UserDataPath != "" {
userDataBytes, err := ioutil.ReadFile(p.UserDataPath)
if err != nil {
return nil, err
}
if len(userDataBytes) > 0 {
p.UserDataContent = base64.StdEncoding.EncodeToString(userDataBytes)
}
}

// run ecs master instances.
if masterNum > 0 {
p.Logger.Infof("[%s] prepare for %d of master instances", p.GetProviderName(), masterNum)
Expand Down Expand Up @@ -350,89 +350,38 @@ func (p *Amazon) runInstances(num int, master bool, ssh *types.SSH) error {
}
}

var instanceList []*ec2.Instance
input := &ec2.RunInstancesInput{
ImageId: &p.AMI,
MinCount: aws.Int64(int64(num)),
MaxCount: aws.Int64(int64(num)),
Placement: &ec2.Placement{
AvailabilityZone: &p.Zone,
},
KeyName: &p.KeypairName,
InstanceType: &p.InstanceType,
NetworkInterfaces: netSpecs,
IamInstanceProfile: iamProfile,
BlockDeviceMappings: []*ec2.BlockDeviceMapping{bdm},
UserData: aws.String(p.UserDataContent),
}
if p.RequestSpotInstance {
if p.SpotPrice == "" {
p.SpotPrice = defaultSpotPrice
input.InstanceMarketOptions = &ec2.InstanceMarketOptionsRequest{
MarketType: aws.String("spot"),
}
req := ec2.RequestSpotInstancesInput{
LaunchSpecification: &ec2.RequestSpotLaunchSpecification{
ImageId: &p.AMI,
Placement: &ec2.SpotPlacement{
AvailabilityZone: &p.Zone,
},
KeyName: &p.KeypairName,
InstanceType: &p.InstanceType,
NetworkInterfaces: netSpecs,
IamInstanceProfile: iamProfile,
BlockDeviceMappings: []*ec2.BlockDeviceMapping{bdm},
},
InstanceCount: aws.Int64(int64(num)),
SpotPrice: &p.SpotPrice,
}

spotInstanceRequest, err := p.client.RequestSpotInstances(&req)
if err != nil {
return fmt.Errorf("[%s] failed request spot instance: %v", p.GetProviderName(), err)
}
for _, spotRequest := range spotInstanceRequest.SpotInstanceRequests {
requestID := spotRequest.SpotInstanceRequestId
p.spotInstanceRequestIDs = append(p.spotInstanceRequestIDs, aws.StringValue(requestID))
p.Logger.Infof("[%s] waiting for spot instance full filled", p.GetProviderName())
err = p.client.WaitUntilSpotInstanceRequestFulfilled(&ec2.DescribeSpotInstanceRequestsInput{
SpotInstanceRequestIds: []*string{requestID},
})
if err != nil {
return err
if p.SpotPrice != "" {
input.InstanceMarketOptions.SpotOptions = &ec2.SpotMarketOptions{
MaxPrice: aws.String(p.SpotPrice),
}

p.Logger.Infof("[%s] resolve instance information by spot request id %s", p.GetProviderName(), *requestID)

spotInstance, err := p.client.DescribeSpotInstanceRequests(&ec2.DescribeSpotInstanceRequestsInput{
SpotInstanceRequestIds: []*string{requestID},
})
if err != nil {
return err
}
if spotInstance != nil && spotInstance.SpotInstanceRequests != nil {
instanceIDs := make([]*string, 0)
for _, spotIns := range spotInstance.SpotInstanceRequests {
instanceIDs = append(instanceIDs, spotIns.InstanceId)
}
output, err := p.client.DescribeInstances(&ec2.DescribeInstancesInput{
InstanceIds: instanceIDs,
})
if err != nil {
return err
}
for _, ins := range output.Reservations {
instanceList = append(instanceList, ins.Instances[0])
}
}
}
} else {
input := &ec2.RunInstancesInput{
ImageId: &p.AMI,
MinCount: aws.Int64(int64(num)),
MaxCount: aws.Int64(int64(num)),
Placement: &ec2.Placement{
AvailabilityZone: &p.Zone,
},
KeyName: &p.KeypairName,
InstanceType: &p.InstanceType,
NetworkInterfaces: netSpecs,
IamInstanceProfile: iamProfile,
BlockDeviceMappings: []*ec2.BlockDeviceMapping{bdm},
}
}

inst, err := p.client.RunInstances(input)
inst, err := p.client.RunInstances(input)

if err != nil || len(inst.Instances) != num {
return fmt.Errorf("[%s] calling runInstances error. region: %s, zone: %s, msg: [%v]",
p.GetProviderName(), p.Region, p.Zone, err)
}
instanceList = inst.Instances
if err != nil || len(inst.Instances) != num {
return fmt.Errorf("[%s] calling runInstances error. region: %s, zone: %s, msg: [%v]",
p.GetProviderName(), p.Region, p.Zone, err)
}
instanceList := inst.Instances

ids := make([]*string, 0)
for _, ins := range instanceList {
Expand Down Expand Up @@ -795,6 +744,13 @@ func (p *Amazon) CreateCheck() error {
p.SubnetID = *subnets.Subnets[0].SubnetId
}
}
// check user-data
if p.UserDataPath != "" {
if _, err := os.Stat(p.UserDataPath); err != nil {
return err
}
}

return nil
}

Expand Down Expand Up @@ -1226,16 +1182,6 @@ func (p *Amazon) terminateInstance(ids []string) error {
return nil
}

func (p *Amazon) cancelSpotInstance() error {
if p.client == nil {
p.newClient()
}
_, err := p.client.CancelSpotInstanceRequests(&ec2.CancelSpotInstanceRequestsInput{
SpotInstanceRequestIds: aws.StringSlice(p.spotInstanceRequestIDs),
})
return err
}

func (p *Amazon) isInstanceRunning(state string) bool {
return state == ec2.InstanceStateNameRunning
}
12 changes: 12 additions & 0 deletions pkg/providers/aws/flag.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -294,5 +294,17 @@ func (p *Amazon) sharedFlags() []types.Flag {
V: p.CloudControllerManager,
Usage: "Enable cloud-controller-manager component, for more information, please check https://github.com/kubernetes/cloud-provider-aws/blob/master/docs/getting_started.md",
},
{
Name: "user-data-content",
P: &p.UserDataContent,
V: p.UserDataContent,
Usage: "The user data content to make available to the instance, must be base64-encoded text. For more information, see running commands on your Linux instance at launch: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/user-data.html",
},
{
Name: "user-data-path",
P: &p.UserDataPath,
V: p.UserDataPath,
Usage: "The user data to make available to the instance, provide user data file path. For more information, see running commands on your Linux instance at launch: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/user-data.html",
},
}
}
2 changes: 2 additions & 0 deletions pkg/types/aws/aws.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,4 +20,6 @@ type Options struct {
SpotPrice string `json:"spot-price,omitempty" yaml:"spot-price,omitempty"`
Tags []string `json:"tags,omitempty" yaml:"tags,omitempty"`
CloudControllerManager bool `json:"cloud-controller-manager" yaml:"cloud-controller-manager"`
UserDataContent string `json:"user-data-content,omitempty" yaml:"user-data-content,omitempty"`
UserDataPath string `json:"user-data-path,omitempty" yaml:"user-data-path,omitempty"`
}

0 comments on commit adfa12a

Please sign in to comment.