diff --git a/server/app/app.go b/server/app/app.go index cc6dc3bf..1a6a310e 100644 --- a/server/app/app.go +++ b/server/app/app.go @@ -87,6 +87,8 @@ func (a *App) startBackgroundWorkers(ctx context.Context) { go a.deployer.PeriodicRequests(ctx, substrateBlockDiffInSeconds) go a.deployer.PeriodicDeploy(ctx, substrateBlockDiffInSeconds) + // remove expired vms and k8s + // check pending deployments a.deployer.ConsumeVMRequest(ctx, true) a.deployer.ConsumeK8sRequest(ctx, true) diff --git a/server/app/k8s_handler.go b/server/app/k8s_handler.go index 3ae5e66e..7d555f26 100644 --- a/server/app/k8s_handler.go +++ b/server/app/k8s_handler.go @@ -54,7 +54,16 @@ func (a *App) K8sDeployHandler(req *http.Request) (interface{}, Response) { return nil, InternalServerError(errors.New(internalServerErrorMsg)) } - _, err = deployer.ValidateK8sQuota(k8sDeployInput, quota.Vms, quota.PublicIPs) + allQuotaVMs, err := a.db.ListUserQuotaVMs(quota.ID.String()) + if err == gorm.ErrRecordNotFound { + return nil, NotFound(errors.New("user quota vms are not found")) + } + if err != nil { + log.Error().Err(err).Send() + return nil, InternalServerError(errors.New(internalServerErrorMsg)) + } + + _, _, err = deployer.ValidateK8sQuota(k8sDeployInput, allQuotaVMs, quota.PublicIPs) if err != nil { log.Error().Err(err).Send() return nil, BadRequest(errors.New(err.Error())) diff --git a/server/app/quota_handler_test.go b/server/app/quota_handler_test.go index b1a22348..f31aea0a 100644 --- a/server/app/quota_handler_test.go +++ b/server/app/quota_handler_test.go @@ -5,7 +5,6 @@ import ( "fmt" "net/http" "testing" - "time" "github.com/codescalers/cloud4students/internal" "github.com/codescalers/cloud4students/models" @@ -45,7 +44,6 @@ func TestQuotaRouter(t *testing.T) { err = app.db.CreateQuota( &models.Quota{ UserID: user.ID.String(), - Vms: map[time.Time]int{time.Now().Add(time.Hour): 10}, PublicIPs: 1, }, ) diff --git a/server/app/setup.go b/server/app/setup.go index 0d83c327..06325454 100644 --- a/server/app/setup.go +++ b/server/app/setup.go @@ -8,7 +8,6 @@ import ( "net/http/httptest" "os" "path/filepath" - "testing" c4sDeployer "github.com/codescalers/cloud4students/deployer" @@ -73,23 +72,35 @@ func SetUp(t testing.TB) *App { `, dbPath) err := os.WriteFile(configPath, []byte(config), 0644) - assert.NoError(t, err) + if !assert.NoError(t, err) { + return &App{} + } configuration, err := internal.ReadConfFile(configPath) - assert.NoError(t, err) + if !assert.NoError(t, err) { + return &App{} + } db := models.NewDB() err = db.Connect(configuration.Database.File) - assert.NoError(t, err) + if !assert.NoError(t, err) { + return &App{} + } err = db.Migrate() - assert.NoError(t, err) + if !assert.NoError(t, err) { + return &App{} + } tfPluginClient, err := deployer.NewTFPluginClient(configuration.Account.Mnemonics, "sr25519", configuration.Account.Network, "", "", "", 0, false) - assert.NoError(t, err) + if !assert.NoError(t, err) { + return &App{} + } newDeployer, err := c4sDeployer.NewDeployer(db, streams.RedisClient{}, tfPluginClient) - assert.NoError(t, err) + if !assert.NoError(t, err) { + return &App{} + } app := &App{ config: configuration, diff --git a/server/app/user_handler.go b/server/app/user_handler.go index f50464af..fbc70250 100644 --- a/server/app/user_handler.go +++ b/server/app/user_handler.go @@ -573,6 +573,7 @@ func (a *App) ApplyForVoucherHandler(req *http.Request) (interface{}, Response) return nil, BadRequest(errors.New("invalid voucher data")) } + // make sure the requested duration is less that the maximum allowed duration if input.VoucherDuration > a.config.VouchersMaxDuration { return nil, BadRequest(fmt.Errorf("invalid voucher duration, max duration is %d", a.config.VouchersMaxDuration)) } @@ -580,12 +581,12 @@ func (a *App) ApplyForVoucherHandler(req *http.Request) (interface{}, Response) // generate voucher for user but can't use it until admin approves it v := internal.GenerateRandomVoucher(5) voucher := models.Voucher{ - Voucher: v, - UserID: userID, - VMs: input.VMs, - Reason: input.Reason, - PublicIPs: input.PublicIPs, - VoucherDuration: input.VoucherDuration, + Voucher: v, + UserID: userID, + VMs: input.VMs, + Reason: input.Reason, + PublicIPs: input.PublicIPs, + Duration: input.VoucherDuration, } err = a.db.CreateVoucher(&voucher) @@ -630,9 +631,7 @@ func (a *App) ActivateVoucherHandler(req *http.Request) (interface{}, Response) return nil, InternalServerError(errors.New(internalServerErrorMsg)) } - expirationDate := time.Now().Add(time.Duration(voucherQuota.VoucherDuration) * 30 * 24 * time.Hour) - - userQuotaVMs, err := a.db.GetUserQuotaVMs(quota.ID.String(), expirationDate) + userQuotaVMs, err := a.db.GetUserQuotaVMs(quota.ID.String(), voucherQuota.Duration) if err == gorm.ErrRecordNotFound { return nil, NotFound(errors.New("user quota vms is not found")) } @@ -665,7 +664,7 @@ func (a *App) ActivateVoucherHandler(req *http.Request) (interface{}, Response) return nil, InternalServerError(errors.New(internalServerErrorMsg)) } - err = a.db.UpdateUserQuotaVMs(quota.ID.String(), expirationDate, userQuotaVMs.Vms+voucherQuota.VMs) + err = a.db.UpdateUserQuotaVMs(quota.ID.String(), voucherQuota.Duration, userQuotaVMs.Vms+voucherQuota.VMs) if err != nil { log.Error().Err(err).Send() return nil, InternalServerError(errors.New(internalServerErrorMsg)) diff --git a/server/app/voucher_handler.go b/server/app/voucher_handler.go index a46a62e8..11fdd2ec 100644 --- a/server/app/voucher_handler.go +++ b/server/app/voucher_handler.go @@ -50,11 +50,11 @@ func (a *App) GenerateVoucherHandler(req *http.Request) (interface{}, Response) } v := models.Voucher{ - Voucher: voucher, - VMs: input.VMs, - PublicIPs: input.PublicIPs, - VoucherDuration: input.VoucherDuration, - Approved: true, + Voucher: voucher, + VMs: input.VMs, + PublicIPs: input.PublicIPs, + Approved: true, + Duration: input.VoucherDuration, } err = a.db.CreateVoucher(&v) diff --git a/server/deployer/k8s_deployer.go b/server/deployer/k8s_deployer.go index 17f5f28e..bb059ac3 100644 --- a/server/deployer/k8s_deployer.go +++ b/server/deployer/k8s_deployer.go @@ -199,28 +199,31 @@ func (d *Deployer) getK8sAvailableNode(ctx context.Context, k models.K8sDeployIn } // ValidateK8sQuota validates the quota a k8s deployment need -func ValidateK8sQuota(k models.K8sDeployInput, availableResourcesQuota, availablePublicIPsQuota int) (int, error) { +func ValidateK8sQuota(k models.K8sDeployInput, availableResourcesQuota []models.QuotaVM, availablePublicIPsQuota int) (int, int, error) { neededQuota, err := calcNeededQuota(k.Resources) if err != nil { - return 0, err + return 0, 0, err + } + + if k.Public && availablePublicIPsQuota < publicQuota { + return 0, 0, fmt.Errorf("no available quota %d for public ips", availablePublicIPsQuota) } for _, worker := range k.Workers { workerQuota, err := calcNeededQuota(worker.Resources) if err != nil { - return 0, err + return 0, 0, err } neededQuota += workerQuota } - if availableResourcesQuota < neededQuota { - return 0, fmt.Errorf("no available quota %d for kubernetes deployment, you can request a new voucher", availableResourcesQuota) - } - if k.Public && availablePublicIPsQuota < publicQuota { - return 0, fmt.Errorf("no available quota %d for public ips", availablePublicIPsQuota) + for _, quotaVMs := range availableResourcesQuota { + if quotaVMs.Duration >= k.Duration && quotaVMs.Vms >= neededQuota { + return quotaVMs.Duration, neededQuota, nil + } } - return neededQuota, nil + return 0, 0, fmt.Errorf("no available quota %v for kubernetes deployment, you can request a new voucher", availableResourcesQuota) } func (d *Deployer) deployK8sRequest(ctx context.Context, user models.User, k8sDeployInput models.K8sDeployInput, adminSSHKey string) (int, error) { @@ -235,12 +238,30 @@ func (d *Deployer) deployK8sRequest(ctx context.Context, user models.User, k8sDe return http.StatusInternalServerError, errors.New(internalServerErrorMsg) } - neededQuota, err := ValidateK8sQuota(k8sDeployInput, quota.Vms, quota.PublicIPs) + allQuotaVMs, err := d.db.ListUserQuotaVMs(quota.ID.String()) + if err == gorm.ErrRecordNotFound { + return http.StatusNotFound, errors.New("user quota vms are not found") + } + if err != nil { + log.Error().Err(err).Send() + return http.StatusInternalServerError, errors.New(internalServerErrorMsg) + } + + neededQuotaDuration, neededQuota, err := ValidateK8sQuota(k8sDeployInput, allQuotaVMs, quota.PublicIPs) if err != nil { log.Error().Err(err).Send() return http.StatusBadRequest, err } + quotaVMs, err := d.db.GetUserQuotaVMs(quota.ID.String(), neededQuotaDuration) + if err == gorm.ErrRecordNotFound { + return http.StatusNotFound, errors.New("user quota vm is not found") + } + if err != nil { + log.Error().Err(err).Send() + return http.StatusInternalServerError, errors.New(internalServerErrorMsg) + } + // deploy network and cluster node, networkContractID, k8sContractID, err := d.deployK8sClusterWithNetwork(ctx, k8sDeployInput, user.SSHKey, adminSSHKey) if err != nil { @@ -253,12 +274,14 @@ func (d *Deployer) deployK8sRequest(ctx context.Context, user models.User, k8sDe log.Error().Err(err).Send() return http.StatusInternalServerError, errors.New(internalServerErrorMsg) } + publicIPsQuota := quota.PublicIPs if k8sDeployInput.Public { publicIPsQuota -= publicQuota } + // update quota - err = d.db.UpdateUserQuota(user.ID.String(), quota.Vms-neededQuota, publicIPsQuota) + err = d.db.UpdateUserQuota(user.ID.String(), publicIPsQuota) if err == gorm.ErrRecordNotFound { return http.StatusNotFound, errors.New("user quota is not found") } @@ -267,6 +290,15 @@ func (d *Deployer) deployK8sRequest(ctx context.Context, user models.User, k8sDe return http.StatusInternalServerError, errors.New(internalServerErrorMsg) } + err = d.db.UpdateUserQuotaVMs(quota.ID.String(), neededQuotaDuration, quotaVMs.Vms-neededQuota) + if err == gorm.ErrRecordNotFound { + return http.StatusNotFound, errors.New("User quota vms is not found") + } + if err != nil { + log.Error().Err(err).Send() + return http.StatusInternalServerError, errors.New(internalServerErrorMsg) + } + err = d.db.CreateK8s(&k8sCluster) if err != nil { log.Error().Err(err).Send() diff --git a/server/deployer/vms_deployer.go b/server/deployer/vms_deployer.go index 14aecb8c..c4f7567b 100644 --- a/server/deployer/vms_deployer.go +++ b/server/deployer/vms_deployer.go @@ -88,24 +88,23 @@ func (d *Deployer) deployVM(ctx context.Context, vmInput models.DeployVMInput, s } // ValidateVMQuota validates the quota a vm deployment need -func ValidateVMQuota(vm models.DeployVMInput, availableResourcesQuota []models.QuotaVM, availablePublicIPsQuota int) (time.Time, int, error) { +func ValidateVMQuota(vm models.DeployVMInput, availableResourcesQuota []models.QuotaVM, availablePublicIPsQuota int) (int, int, error) { neededQuota, err := calcNeededQuota(vm.Resources) if err != nil { - return time.Now(), 0, err + return 0, 0, err } if vm.Public && availablePublicIPsQuota < publicQuota { - return time.Now(), 0, fmt.Errorf("no available quota %d for public ips", availablePublicIPsQuota) + return 0, 0, fmt.Errorf("no available quota %d for public ips", availablePublicIPsQuota) } - requestedExpirationDate := time.Now().Add(time.Duration(vm.Duration) * 30 * 24 * time.Hour) - for _, vms := range availableResourcesQuota { - if requestedExpirationDate.Before(vms.ExpirationDate) && neededQuota <= vms.Vms { - return vms.ExpirationDate, vms.Vms - neededQuota, nil + for _, quotaVMs := range availableResourcesQuota { + if quotaVMs.Duration >= vm.Duration && quotaVMs.Vms >= neededQuota { + return quotaVMs.Duration, neededQuota, nil } } - return time.Now(), 0, fmt.Errorf("no available quota %v for deployment for resources %s, you can request a new voucher", availableResourcesQuota, vm.Resources) + return 0, 0, fmt.Errorf("no available quota %v for deployment for resources %s, you can request a new voucher", availableResourcesQuota, vm.Resources) } func (d *Deployer) deployVMRequest(ctx context.Context, user models.User, input models.DeployVMInput, adminSSHKey string) (int, error) { @@ -119,7 +118,7 @@ func (d *Deployer) deployVMRequest(ctx context.Context, user models.User, input return http.StatusInternalServerError, errors.New(internalServerErrorMsg) } - quotaVM, err := d.db.ListUserQuotaVMs(quota.ID.String()) + allQuotaVMs, err := d.db.ListUserQuotaVMs(quota.ID.String()) if err == gorm.ErrRecordNotFound { return http.StatusNotFound, errors.New("user quota vm is not found") } @@ -128,11 +127,21 @@ func (d *Deployer) deployVMRequest(ctx context.Context, user models.User, input return http.StatusInternalServerError, errors.New(internalServerErrorMsg) } - expirationDate, newQuotaVMs, err := ValidateVMQuota(input, quotaVM, quota.PublicIPs) + neededQuotaDuration, neededQuota, err := ValidateVMQuota(input, allQuotaVMs, quota.PublicIPs) if err != nil { return http.StatusBadRequest, err } + quotaVMs, err := d.db.GetUserQuotaVMs(quota.ID.String(), neededQuotaDuration) + if err == gorm.ErrRecordNotFound { + return http.StatusNotFound, errors.New("user quota vm is not found") + } + if err != nil { + log.Error().Err(err).Send() + return http.StatusInternalServerError, errors.New(internalServerErrorMsg) + } + + // deploy network and vm vm, contractID, networkContractID, diskSize, err := d.deployVM(ctx, input, user.SSHKey, adminSSHKey) if err != nil { log.Error().Err(err).Send() @@ -151,7 +160,7 @@ func (d *Deployer) deployVMRequest(ctx context.Context, user models.User, input MRU: uint64(vm.Memory), ContractID: contractID, NetworkContractID: networkContractID, - ExpirationDate: expirationDate, + ExpirationDate: time.Now().Add(time.Duration(quotaVMs.Duration) * 30 * 24 * time.Hour).Truncate(24 * time.Hour), } err = d.db.CreateVM(&userVM) @@ -174,7 +183,7 @@ func (d *Deployer) deployVMRequest(ctx context.Context, user models.User, input return http.StatusInternalServerError, errors.New(internalServerErrorMsg) } - err = d.db.UpdateUserQuotaVMs(quota.ID.String(), expirationDate, newQuotaVMs) + err = d.db.UpdateUserQuotaVMs(quota.ID.String(), neededQuotaDuration, quotaVMs.Vms-neededQuota) if err == gorm.ErrRecordNotFound { return http.StatusNotFound, errors.New("User quota vms is not found") } diff --git a/server/models/database.go b/server/models/database.go index 87add831..fc67d65a 100644 --- a/server/models/database.go +++ b/server/models/database.go @@ -32,7 +32,7 @@ func (d *DB) Connect(file string) error { // Migrate migrates db schema func (d *DB) Migrate() error { - err := d.db.AutoMigrate(&User{}, &Quota{}, &VM{}, &K8sCluster{}, &Master{}, &Worker{}, &Voucher{}, &Maintenance{}, &Notification{}) + err := d.db.AutoMigrate(&User{}, &Quota{}, &QuotaVM{}, &VM{}, &K8sCluster{}, &Master{}, &Worker{}, &Voucher{}, &Maintenance{}, &Notification{}) if err != nil { return err } @@ -68,11 +68,12 @@ func (d *DB) GetUserByID(id string) (User, error) { func (d *DB) ListAllUsers() ([]UserUsedQuota, error) { var res []UserUsedQuota query := d.db.Table("users"). - Select("*, users.id as user_id, sum(vouchers.vms) as vms, sum(vouchers.public_ips) as public_ips, sum(vouchers.vms) - quota.vms as used_vms, sum(vouchers.public_ips) - quota.public_ips as used_public_ips"). + Select("*, users.id as user_id, sum(vouchers.vms) as vms, sum(vouchers.public_ips) as public_ips, sum(vouchers.vms) - sum(quota_vms.vms) as used_vms, sum(vouchers.public_ips) - quota.public_ips as used_public_ips"). Joins("left join quota on quota.user_id = users.id"). + Joins("left join quota_vms on quota.id = quota_vms.quota_id"). Joins("left join vouchers on vouchers.used = true and vouchers.user_id = users.id"). Where("verified = true"). - Group("users.id"). + Group("users.id, quota.id"). Scan(&res) return res, query.Error } @@ -227,10 +228,15 @@ func (d *DB) UpdateUserQuota(userID string, publicIPs int) error { } // UpdateUserQuotaVMs updates quota vms -func (d *DB) UpdateUserQuotaVMs(QuotaID string, expirationDate time.Time, vms int) error { - return d.db.Model(&QuotaVM{}). - Where(&QuotaVM{QoutaID: QuotaID, ExpirationDate: expirationDate}). - Update("vms", vms).Error +func (d *DB) UpdateUserQuotaVMs(QuotaID string, duration int, vms int) error { + query := d.db.Model(&QuotaVM{}). + Where(&QuotaVM{QuotaID: QuotaID, Duration: duration}). + Update("vms", vms) + + if query.RowsAffected == 0 { + return d.CreateQuotaVM(&QuotaVM{QuotaID: QuotaID, Duration: duration, Vms: vms}) + } + return query.Error } // GetUserQuota gets user quota available publicIPs @@ -241,10 +247,9 @@ func (d *DB) GetUserQuota(userID string) (Quota, error) { } // GetUserQuotaVMs gets user quota available vms (vms will be used for both vms and k8s clusters) -func (d *DB) GetUserQuotaVMs(quotaID string, expirationDate time.Time) (QuotaVM, error) { +func (d *DB) GetUserQuotaVMs(quotaID string, duration int) (QuotaVM, error) { var res QuotaVM - query := d.db.Select("expiration_date", "vms"). - FirstOrCreate(&res, &QuotaVM{QoutaID: quotaID, ExpirationDate: expirationDate}) + query := d.db.FirstOrCreate(&res, &QuotaVM{QuotaID: quotaID, Duration: duration}) return res, query.Error } diff --git a/server/models/database_test.go b/server/models/database_test.go index 05f80b05..f13fb6a1 100644 --- a/server/models/database_test.go +++ b/server/models/database_test.go @@ -3,7 +3,6 @@ package models import ( "testing" - "time" "github.com/stretchr/testify/assert" "gorm.io/gorm" @@ -482,7 +481,7 @@ func TestCreateQuota(t *testing.T) { func TestUpdateUserQuota(t *testing.T) { db := setupDB(t) t.Run("quota not found so no updates", func(t *testing.T) { - err := db.UpdateUserQuota("user", map[time.Time]int{time.Now(): 5}, 0) + err := db.UpdateUserQuota("user", 0) assert.NoError(t, err) }) t.Run("quota found", func(t *testing.T) { @@ -494,24 +493,33 @@ func TestUpdateUserQuota(t *testing.T) { err = db.CreateQuota("a2) assert.NoError(t, err) - err = db.UpdateUserQuota("user", map[time.Time]int{time.Now().Add(time.Hour): 5}, 10) + err = db.UpdateUserQuota("user", 10) assert.NoError(t, err) var q Quota err = db.db.First(&q, "user_id = 'user'").Error assert.NoError(t, err) - assert.Equal(t, q.Vms, 5) + assert.Equal(t, q.PublicIPs, 10) + err = db.UpdateUserQuotaVMs(q.ID.String(), 1, 5) + assert.NoError(t, err) + + var qvm QuotaVM + err = db.db.First(&qvm, "quota_id = ? AND duration = ?", q.ID.String(), 1).Error + assert.NoError(t, err) + assert.Equal(t, qvm.Vms, 5) + + q = Quota{} err = db.db.First(&q, "user_id = 'new-user'").Error assert.NoError(t, err) - assert.Equal(t, q.Vms, 0) + assert.Equal(t, q.PublicIPs, 0) }) t.Run("quota found with zero values", func(t *testing.T) { quota := Quota{UserID: "1"} err := db.CreateQuota("a) assert.NoError(t, err) - err = db.UpdateUserQuota("1", map[time.Time]int{time.Now(): 0}, 0) + err = db.UpdateUserQuota("1", 0) assert.NoError(t, err) }) } diff --git a/server/models/quota.go b/server/models/quota.go index 1f5f3e6a..63e7eaa1 100644 --- a/server/models/quota.go +++ b/server/models/quota.go @@ -1,11 +1,26 @@ // Package models for database models package models -import "github.com/google/uuid" +import ( + "github.com/google/uuid" + "gorm.io/gorm" +) // Quota struct holds available vms for each user type Quota struct { ID uuid.UUID `gorm:"primary_key; unique; type:uuid; column:id"` UserID string `json:"user_id"` PublicIPs int `json:"public_ips"` + VMs []QuotaVM `json:"vms"` +} + +// BeforeCreate generates a new uuid +func (quota *Quota) BeforeCreate(tx *gorm.DB) (err error) { + id, err := uuid.NewUUID() + if err != nil { + return err + } + + quota.ID = id + return } diff --git a/server/models/quota_vms.go b/server/models/quota_vms.go index 5470b27f..9c7ce8ad 100644 --- a/server/models/quota_vms.go +++ b/server/models/quota_vms.go @@ -1,11 +1,9 @@ // Package models for database models package models -import "time" - // QuotaVM struct holds available vms and their expiration date for each user type QuotaVM struct { - QoutaID string `json:"qouta_id"` - Vms int `json:"vms"` - ExpirationDate time.Time `json:"expiration_date"` + QuotaID string `json:"qouta_id"` + Vms int `json:"vms"` + Duration int `json:"duration"` } diff --git a/server/models/voucher.go b/server/models/voucher.go index e3b3208c..46fdca65 100644 --- a/server/models/voucher.go +++ b/server/models/voucher.go @@ -5,16 +5,16 @@ import "time" // Voucher struct holds data of vouchers type Voucher struct { - ID int `json:"id" gorm:"primaryKey"` - UserID string `json:"user_id" binding:"required"` - Voucher string `json:"voucher" gorm:"unique"` - VMs int `json:"vms" binding:"required"` - PublicIPs int `json:"public_ips" binding:"required"` - Reason string `json:"reason" binding:"required"` - Used bool `json:"used" binding:"required"` - Approved bool `json:"approved" binding:"required"` - Rejected bool `json:"rejected" binding:"required"` - VoucherDuration int `json:"voucher_duration" binding:"required"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` + ID int `json:"id" gorm:"primaryKey"` + UserID string `json:"user_id" binding:"required"` + Voucher string `json:"voucher" gorm:"unique"` + VMs int `json:"vms" binding:"required"` + PublicIPs int `json:"public_ips" binding:"required"` + Reason string `json:"reason" binding:"required"` + Used bool `json:"used" binding:"required"` + Approved bool `json:"approved" binding:"required"` + Rejected bool `json:"rejected" binding:"required"` + Duration int `json:"duration" binding:"required"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` }