Merge pull request #542 from codescalers/development_updates

update pkgs and fix units for resources

server/app/app.go

@@ -47,7 +47,10 @@ func NewApp(ctx context.Context, configFile string) (app *App, err error) {
 		return
 	}
 
-	tfPluginClient, err := deployer.NewTFPluginClient(config.Account.Mnemonics, "sr25519", config.Account.Network, "", "", "", 0, false)
+	tfPluginClient, err := deployer.NewTFPluginClient(
+		config.Account.Mnemonics,
+		deployer.WithNetwork(config.Account.Network),
+	)
 	if err != nil {
 		return
 	}
@@ -58,9 +61,6 @@ func NewApp(ctx context.Context, configFile string) (app *App, err error) {
 	}
 
 	server := newServer(config.Server.Host, config.Server.Port)
-	if err != nil {
-		return
-	}
 
 	return &App{
 		config:   config,

server/app/setup.go

@@ -85,7 +85,7 @@ func SetUp(t testing.TB) *App {
 	err = db.Migrate()
 	assert.NoError(t, err)
 
-	tfPluginClient, err := deployer.NewTFPluginClient(configuration.Account.Mnemonics, "sr25519", configuration.Account.Network, "", "", "", 0, false)
+	tfPluginClient, err := deployer.NewTFPluginClient(configuration.Account.Mnemonics, deployer.WithNetwork(configuration.Account.Network))
 	assert.NoError(t, err)
 
 	newDeployer, err := c4sDeployer.NewDeployer(db, streams.RedisClient{}, tfPluginClient)

server/cmd/root.go

@@ -53,7 +53,7 @@ func Execute() {
 
 	err := rootCmd.Execute()
 	if err != nil {
-		log.Err(err).Send()
+		log.Error().Err(err).Send()
 		os.Exit(1)
 	}
 }

server/deployer/deployer.go

@@ -13,7 +13,6 @@ import (
 	"github.com/rs/zerolog/log"
 	"github.com/threefoldtech/tfgrid-sdk-go/grid-client/deployer"
 	"github.com/threefoldtech/tfgrid-sdk-go/grid-client/workloads"
-	"github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types"
 	"github.com/threefoldtech/zos/pkg/gridtypes"
 	"gopkg.in/validator.v2"
 )
@@ -96,12 +95,12 @@ func (d *Deployer) PeriodicDeploy(ctx context.Context, sec int) {
 	ticker := time.NewTicker(time.Second * time.Duration(sec))
 
 	for range ticker.C {
-		vmNets, vms, err := d.consumeVMs(ctx)
+		vmNets, vms, err := d.consumeVMs()
 		if err != nil {
 			log.Error().Err(err).Msg("failed to consume vms")
 		}
 
-		k8sNets, clusters, err := d.consumeK8s(ctx)
+		k8sNets, clusters, err := d.consumeK8s()
 		if err != nil {
 			log.Error().Err(err).Msg("failed to consume clusters")
 		}
@@ -205,25 +204,6 @@ func calcNodeResources(resources string, public bool) (uint64, uint64, uint64, u
 	return cru, mru, sru, ips, nil
 }
 
-// choose suitable nodes based on needed resources
-func filterNode(resource string, public bool) (types.NodeFilter, error) {
-	cru, mru, sru, ips, err := calcNodeResources(resource, public)
-	if err != nil {
-		return types.NodeFilter{}, err
-	}
-
-	return types.NodeFilter{
-		FarmIDs:  []uint64{1},
-		TotalCRU: &cru,
-		FreeSRU:  &sru,
-		FreeMRU:  &mru,
-		FreeIPs:  &ips,
-		IPv4:     &trueVal,
-		Status:   &statusUp,
-		IPv6:     &trueVal,
-	}, nil
-}
-
 func calcNeededQuota(resources string) (int, error) {
 	var neededQuota int
 	switch resources {

server/deployer/deployment_consumer.go

@@ -145,7 +145,7 @@ func (d *Deployer) ConsumeK8sRequest(ctx context.Context, pending bool) {
 	}
 }
 
-func (d *Deployer) consumeVMs(ctx context.Context) (nets []*workloads.ZNet, vms []*workloads.Deployment, err error) {
+func (d *Deployer) consumeVMs() (nets []*workloads.ZNet, vms []*workloads.Deployment, err error) {
 	result, err := d.Redis.Read(streams.DeployVMStreamName, streams.DeployVMConsumerGroupName, 5, false)
 	if err != nil {
 		if errors.Is(err, redis.Nil) {
@@ -179,7 +179,7 @@ func (d *Deployer) consumeVMs(ctx context.Context) (nets []*workloads.ZNet, vms
 	return
 }
 
-func (d *Deployer) consumeK8s(ctx context.Context) (nets []*workloads.ZNet, clusters []*workloads.K8sCluster, err error) {
+func (d *Deployer) consumeK8s() (nets []*workloads.ZNet, clusters []*workloads.K8sCluster, err error) {
 	result, err := d.Redis.Read(streams.DeployK8sStreamName, streams.DeployK8sConsumerGroupName, 5, false)
 	if err != nil {
 		if errors.Is(err, redis.Nil) {

server/deployer/k8s_deployer.go

@@ -97,22 +97,22 @@ func (d *Deployer) deployK8sClusterWithNetwork(ctx context.Context, k8sDeployInp
 	}
 
 	// checks that network and k8s are deployed successfully
-	loadedNet, err := d.tfPluginClient.State.LoadNetworkFromGrid(cluster.NetworkName)
+	loadedNet, err := d.tfPluginClient.State.LoadNetworkFromGrid(ctx, cluster.NetworkName)
 	if err != nil {
 		return 0, 0, 0, errors.Wrapf(err, "failed to load network '%s' on nodes %v", cluster.NetworkName, network.Nodes)
 	}
 
-	loadedCluster, err := d.tfPluginClient.State.LoadK8sFromGrid([]uint32{node}, cluster.Master.Name)
+	loadedCluster, err := d.tfPluginClient.State.LoadK8sFromGrid(ctx, []uint32{node}, cluster.Master.Name)
 	if err != nil {
 		return 0, 0, 0, errors.Wrapf(err, "failed to load kubernetes cluster '%s' on nodes %v", cluster.Master.Name, network.Nodes)
 	}
 
 	return node, loadedNet.NodeDeploymentID[node], loadedCluster.NodeDeploymentID[node], nil
 }
 
-func (d *Deployer) loadK8s(k8sDeployInput models.K8sDeployInput, userID string, node uint32, networkContractID uint64, k8sContractID uint64) (models.K8sCluster, error) {
+func (d *Deployer) loadK8s(ctx context.Context, k8sDeployInput models.K8sDeployInput, userID string, node uint32, networkContractID uint64, k8sContractID uint64) (models.K8sCluster, error) {
 	// load cluster
-	resCluster, err := d.tfPluginClient.State.LoadK8sFromGrid([]uint32{node}, k8sDeployInput.MasterName)
+	resCluster, err := d.tfPluginClient.State.LoadK8sFromGrid(ctx, []uint32{node}, k8sDeployInput.MasterName)
 	if err != nil {
 		return models.K8sCluster{}, err
 	}
@@ -129,7 +129,7 @@ func (d *Deployer) loadK8s(k8sDeployInput models.K8sDeployInput, userID string,
 		Public:    k8sDeployInput.Public,
 		PublicIP:  resCluster.Master.ComputedIP,
 		Name:      k8sDeployInput.MasterName,
-		YggIP:     resCluster.Master.YggIP,
+		YggIP:     resCluster.Master.PlanetaryIP,
 		Resources: k8sDeployInput.Resources,
 	}
 	workers := []models.Worker{}
@@ -179,18 +179,17 @@ func (d *Deployer) getK8sAvailableNode(ctx context.Context, k models.K8sDeployIn
 		rootfs = append(rootfs, *convertGBToBytes(uint64(2)))
 	}
 
-	freeMRU := mru
-	freeSRU := sru
+	freeSRU := convertGBToBytes(sru)
 	filter := types.NodeFilter{
 		Status:  &statusUp,
-		FreeMRU: &freeMRU,
-		FreeSRU: &freeSRU,
+		FreeMRU: convertGBToBytes(mru),
+		FreeSRU: freeSRU,
 		FreeIPs: &ips,
 		FarmIDs: []uint64{1},
 		IPv6:    &trueVal,
 	}
 
-	nodes, err := deployer.FilterNodes(ctx, d.tfPluginClient, filter, nil, nil, rootfs)
+	nodes, err := deployer.FilterNodes(ctx, d.tfPluginClient, filter, []uint64{*freeSRU}, nil, rootfs, 1)
 	if err != nil {
 		return 0, err
 	}
@@ -248,7 +247,7 @@ func (d *Deployer) deployK8sRequest(ctx context.Context, user models.User, k8sDe
 		return http.StatusInternalServerError, errors.New(internalServerErrorMsg)
 	}
 
-	k8sCluster, err := d.loadK8s(k8sDeployInput, user.ID.String(), node, networkContractID, k8sContractID)
+	k8sCluster, err := d.loadK8s(ctx, k8sDeployInput, user.ID.String(), node, networkContractID, k8sContractID)
 	if err != nil {
 		log.Error().Err(err).Send()
 		return http.StatusInternalServerError, errors.New(internalServerErrorMsg)

server/deployer/vms_deployer.go

@@ -13,17 +13,30 @@ import (
 	"github.com/rs/zerolog/log"
 	"github.com/threefoldtech/tfgrid-sdk-go/grid-client/deployer"
 	"github.com/threefoldtech/tfgrid-sdk-go/grid-client/workloads"
+	"github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types"
 	"gorm.io/gorm"
 )
 
 func (d *Deployer) deployVM(ctx context.Context, vmInput models.DeployVMInput, sshKey string, adminSSHKey string) (*workloads.VM, uint64, uint64, uint64, error) {
 	// filter nodes
-	filter, err := filterNode(vmInput.Resources, vmInput.Public)
+	cru, mru, sru, ips, err := calcNodeResources(vmInput.Resources, vmInput.Public)
 	if err != nil {
 		return nil, 0, 0, 0, err
 	}
 
-	nodeIDs, err := deployer.FilterNodes(ctx, d.tfPluginClient, filter, nil, nil, nil)
+	freeSRU := convertGBToBytes(sru)
+	filter := types.NodeFilter{
+		FarmIDs:  []uint64{1},
+		TotalCRU: &cru,
+		FreeSRU:  freeSRU,
+		FreeMRU:  convertGBToBytes(mru),
+		FreeIPs:  &ips,
+		IPv4:     &trueVal,
+		Status:   &statusUp,
+		IPv6:     &trueVal,
+	}
+
+	nodeIDs, err := deployer.FilterNodes(ctx, d.tfPluginClient, filter, []uint64{*freeSRU}, nil, nil, 1)
 	if err != nil {
 		return nil, 0, 0, 0, err
 	}
@@ -35,7 +48,7 @@ func (d *Deployer) deployVM(ctx context.Context, vmInput models.DeployVMInput, s
 	// create disk
 	disk := workloads.Disk{
 		Name:   "disk",
-		SizeGB: int(*filter.FreeSRU),
+		SizeGB: int(sru),
 	}
 
 	// create vm workload
@@ -45,7 +58,7 @@ func (d *Deployer) deployVM(ctx context.Context, vmInput models.DeployVMInput, s
 		CPU:       int(*filter.TotalCRU),
 		PublicIP:  vmInput.Public,
 		Planetary: true,
-		Memory:    int(*filter.FreeMRU) * 1024,
+		Memory:    int(mru) * 1024,
 		Mounts: []workloads.Mount{
 			{DiskName: disk.Name, MountPoint: "/disk"},
 		},
@@ -73,12 +86,12 @@ func (d *Deployer) deployVM(ctx context.Context, vmInput models.DeployVMInput, s
 	}
 
 	// checks that network and vm are deployed successfully
-	loadedNet, err := d.tfPluginClient.State.LoadNetworkFromGrid(dl.NetworkName)
+	loadedNet, err := d.tfPluginClient.State.LoadNetworkFromGrid(ctx, dl.NetworkName)
 	if err != nil {
 		return nil, 0, 0, 0, errors.Wrapf(err, "failed to load network '%s' on node %v", dl.NetworkName, dl.NodeID)
 	}
 
-	loadedDl, err := d.tfPluginClient.State.LoadDeploymentFromGrid(nodeID, dl.Name)
+	loadedDl, err := d.tfPluginClient.State.LoadDeploymentFromGrid(ctx, nodeID, dl.Name)
 	if err != nil {
 		return nil, 0, 0, 0, errors.Wrapf(err, "failed to load vm '%s' on node %v", dl.Name, dl.NodeID)
 	}
@@ -128,7 +141,7 @@ func (d *Deployer) deployVMRequest(ctx context.Context, user models.User, input
 	userVM := models.VM{
 		UserID:            user.ID.String(),
 		Name:              vm.Name,
-		YggIP:             vm.YggIP,
+		YggIP:             vm.PlanetaryIP,
 		Resources:         input.Resources,
 		Public:            input.Public,
 		PublicIP:          vm.ComputedIP,

server/go.mod

@@ -6,22 +6,22 @@ require (
 	github.com/caitlin615/nist-password-validator v0.0.0-20190321104149-45ab5d3140de
 	github.com/go-redis/redis v6.15.9+incompatible
 	github.com/golang-jwt/jwt/v4 v4.5.0
-	github.com/google/uuid v1.4.0
+	github.com/google/uuid v1.6.0
 	github.com/gorilla/mux v1.8.1
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.17.0
-	github.com/rs/zerolog v1.31.0
+	github.com/rs/zerolog v1.32.0
 	github.com/sendgrid/sendgrid-go v3.13.0+incompatible
 	github.com/spf13/cobra v1.8.0
-	github.com/stretchr/testify v1.8.4
-	github.com/threefoldtech/tfgrid-sdk-go/grid-client v0.11.7
-	github.com/threefoldtech/tfgrid-sdk-go/grid-proxy v0.11.7
-	github.com/threefoldtech/zos v0.5.6-0.20231029102142-baf8d9ee77fe
-	golang.org/x/crypto v0.15.0
+	github.com/stretchr/testify v1.9.0
+	github.com/threefoldtech/tfgrid-sdk-go/grid-client v0.14.13
+	github.com/threefoldtech/tfgrid-sdk-go/grid-proxy v0.14.13
+	github.com/threefoldtech/zos v0.5.6-0.20240226114056-364e04acbed3
+	golang.org/x/crypto v0.21.0
 	golang.org/x/text v0.14.0
 	gopkg.in/validator.v2 v2.0.1
 	gorm.io/driver/sqlite v1.4.4
-	gorm.io/gorm v1.25.5
+	gorm.io/gorm v1.25.7
 )
 
 require (
@@ -32,7 +32,7 @@ require (
 	github.com/centrifuge/go-substrate-rpc-client/v4 v4.0.12 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cosmos/go-bip39 v1.0.0 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/deckarep/golang-set v1.8.0 // indirect
 	github.com/decred/base58 v1.0.5 // indirect
 	github.com/decred/dcrd/crypto/blake256 v1.0.1 // indirect
@@ -42,42 +42,42 @@ require (
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/gomodule/redigo v2.0.0+incompatible // indirect
+	github.com/gorilla/schema v1.2.1 // indirect
 	github.com/gorilla/websocket v1.5.1 // indirect
 	github.com/gtank/merlin v0.1.1 // indirect
 	github.com/gtank/ristretto255 v0.1.2 // indirect
-	github.com/hashicorp/errwrap v1.0.0 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c // indirect
+	github.com/holiman/uint256 v1.2.3 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jbenet/go-base58 v0.0.0-20150317085156-6237cf65f3a6 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-sqlite3 v1.14.17 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mimoo/StrobeGo v0.0.0-20220103164710-9a04d6ca976b // indirect
 	github.com/onsi/ginkgo v1.16.5 // indirect
-	github.com/onsi/gomega v1.28.1 // indirect
-	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
+	github.com/onsi/gomega v1.31.1 // indirect
 	github.com/pierrec/xxHash v0.1.5 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
 	github.com/prometheus/common v0.44.0 // indirect
 	github.com/prometheus/procfs v0.11.1 // indirect
-	github.com/rs/cors v1.9.0 // indirect
+	github.com/rogpeppe/go-internal v1.12.0 // indirect
+	github.com/rs/cors v1.10.1 // indirect
 	github.com/sendgrid/rest v2.6.9+incompatible // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/threefoldtech/tfchain/clients/tfchain-client-go v0.0.0-20231017113908-9d56a805ead0 // indirect
-	github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go v0.11.7 // indirect
+	github.com/threefoldtech/tfchain/clients/tfchain-client-go v0.0.0-20240227171040-f2a20ee3e965 // indirect
+	github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go v0.14.12 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
-	github.com/tyler-smith/go-bip39 v1.1.0 // indirect
 	github.com/vedhavyas/go-subkey v1.0.3 // indirect
 	github.com/yusufpapurcu/wmi v1.2.3 // indirect
-	golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect
-	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/sync v0.5.0 // indirect
-	golang.org/x/sys v0.14.0 // indirect
+	golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect
+	golang.org/x/net v0.21.0 // indirect
+	golang.org/x/sync v0.6.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
 	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20200609130330-bd2cb7843e1b // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect