Skip to content

Commit

Permalink
Suport vm cpu pinning
Browse files Browse the repository at this point in the history 
Signed-off-by: Cooper Tseng <[email protected]>
  • Loading branch information
@brandboat
brandboat committed Sep 26, 2024
1 parent 41bb2f4 commit c46ed10
Show file tree
Hide file tree
Showing 5 changed files with 210 additions and 10 deletions.
12 changes: 10 additions & 2 deletions internal/provider/virtualmachine/resource_virtualmachine_constructor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,14 @@ import (
"errors"
"fmt"

"github.com/harvester/harvester/pkg/builder"
harvesterutil "github.com/harvester/harvester/pkg/util"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/pointer"
kubevirtv1 "kubevirt.io/api/core/v1"

"github.com/harvester/harvester/pkg/builder"
harvesterutil "github.com/harvester/harvester/pkg/util"

"github.com/harvester/terraform-provider-harvester/internal/util"
"github.com/harvester/terraform-provider-harvester/pkg/client"
"github.com/harvester/terraform-provider-harvester/pkg/constants"
Expand Down Expand Up @@ -306,6 +307,13 @@ func (c *Constructor) Setup() util.Processors {
return nil
},
},
{
Field: constants.FieldVirtualMachineCPUPinning,
Parser: func(i interface{}) error {
vmBuilder.VirtualMachine.Spec.Template.Spec.Domain.CPU.DedicatedCPUPlacement = i.(bool)
return nil
},
},
}
return append(processors, customProcessors...)
}
Expand Down
6 changes: 6 additions & 0 deletions internal/provider/virtualmachine/schema_virtualmachine.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,12 @@ please use %s instead of this deprecated field:
Optional: true,
Default: false,
},
constants.FieldVirtualMachineCPUPinning: {
Type: schema.TypeBool,
Description: "To enable VM CPU pinning, ensure that at least one node has the CPU manager enabled",
Optional: true,
Default: false,
},
}
util.NamespacedSchemaWrap(s, false)
return s
Expand Down
191 changes: 185 additions & 6 deletions internal/tests/resource_virtualmachine_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,20 @@ package tests

import (
"context"
"crypto/tls"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"testing"
"time"

"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
"github.com/hashicorp/terraform-plugin-sdk/v2/terraform"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/rest"
kubevirtv1 "kubevirt.io/api/core/v1"

"github.com/harvester/terraform-provider-harvester/pkg/client"
Expand All @@ -30,6 +38,7 @@ resource %s "%s" {
cpu = 1
%s = "%s"
%s = %s
run_strategy = "RerunOnFailure"
machine_type = "q35"
Expand All @@ -50,16 +59,17 @@ resource %s "%s" {
`
)

func buildVirtualMachineConfig(name, description, memory string) string {
func buildVirtualMachineConfig(name, description, memory string, cpuPinning bool) string {
return fmt.Sprintf(testAccVirtualMachineConfigTemplate, constants.ResourceTypeVirtualMachine, name,
constants.FieldCommonName, name,
constants.FieldCommonDescription, description,
constants.FieldVirtualMachineMemory, memory)
constants.FieldVirtualMachineMemory, memory,
constants.FieldVirtualMachineCPUPinning, strconv.FormatBool(cpuPinning))
}

func TestAccVirtualMachine_basic(t *testing.T) {
var (
vm *kubevirtv1.VirtualMachine
vm = &kubevirtv1.VirtualMachine{}
ctx = context.Background()
)
resource.Test(t, resource.TestCase{
Expand All @@ -68,21 +78,74 @@ func TestAccVirtualMachine_basic(t *testing.T) {
CheckDestroy: testAccCheckVirtualMachineDestroy(ctx),
Steps: []resource.TestStep{
{
Config: buildVirtualMachineConfig(testAccVirtualMachineName, testAccVirtualMachineDescription, testAccVirtualMachineMemory),
Config: buildVirtualMachineConfig(testAccVirtualMachineName, testAccVirtualMachineDescription, testAccVirtualMachineMemory, false),
Check: resource.ComposeTestCheckFunc(
testAccVirtualMachineExists(ctx, testAccVirtualMachineResourceName, vm),
resource.TestCheckResourceAttr(testAccVirtualMachineResourceName, constants.FieldCommonName, testAccVirtualMachineName),
resource.TestCheckResourceAttr(testAccVirtualMachineResourceName, constants.FieldCommonDescription, testAccVirtualMachineDescription),
resource.TestCheckResourceAttr(testAccVirtualMachineResourceName, constants.FieldVirtualMachineMemory, testAccVirtualMachineMemory),
resource.TestCheckResourceAttr(testAccVirtualMachineResourceName, constants.FieldVirtualMachineCPUPinning, "false"),
),
},
{
Config: buildVirtualMachineConfig(testAccVirtualMachineName, testAccVirtualMachineDescription, testAccVirtualMachineMemoryUpdate),
Config: buildVirtualMachineConfig(testAccVirtualMachineName, testAccVirtualMachineDescription, testAccVirtualMachineMemoryUpdate, false),
Check: resource.ComposeTestCheckFunc(
testAccVirtualMachineExists(ctx, testAccVirtualMachineResourceName, vm),
resource.TestCheckResourceAttr(testAccVirtualMachineResourceName, constants.FieldCommonName, testAccVirtualMachineName),
resource.TestCheckResourceAttr(testAccVirtualMachineResourceName, constants.FieldCommonDescription, testAccVirtualMachineDescription),
resource.TestCheckResourceAttr(testAccVirtualMachineResourceName, constants.FieldVirtualMachineMemory, testAccVirtualMachineMemoryUpdate),
resource.TestCheckResourceAttr(testAccVirtualMachineResourceName, constants.FieldVirtualMachineCPUPinning, "false"),
),
},
},
})
}

func TestAccVirtualMachine_cpu_pinning(t *testing.T) {
var (
vm = &kubevirtv1.VirtualMachine{}
ctx = context.Background()
nodeName = ""
)

resource.Test(t, resource.TestCase{
PreCheck: func() {
testAccPreCheck(t)
nodes, err := testAccProvider.Meta().(*client.Client).KubeClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
if err != nil {
t.Fatal(err)
}
if len(nodes.Items) == 0 {
t.Fatal("failed to find any node")
}
nodeName = nodes.Items[0].Name
t.Log("enable cpu manager on node " + nodeName)
enableCPUManager(t, ctx, nodeName)
},
Providers: testAccProviders,
CheckDestroy: func(s *terraform.State) error {
err := testAccCheckVirtualMachineDestroy(ctx)(s)
defer func() {
t.Log("disable cpu manager on node " + nodeName)
disableCPUManager(t, ctx, nodeName)
}()
return err
},
Steps: []resource.TestStep{
{
Config: buildVirtualMachineConfig(testAccVirtualMachineName, testAccVirtualMachineDescription, testAccVirtualMachineMemory, true),
Check: resource.ComposeTestCheckFunc(
testAccVirtualMachineExists(ctx, testAccVirtualMachineResourceName, vm),
func(s *terraform.State) error {
if vm.Spec.Template == nil || vm.Spec.Template.Spec.Domain.CPU == nil || !vm.Spec.Template.Spec.Domain.CPU.DedicatedCPUPlacement {
return errors.New("DedicatedCPUPlacement should be true")
}
return nil
},
resource.TestCheckResourceAttr(testAccVirtualMachineResourceName, constants.FieldCommonName, testAccVirtualMachineName),
resource.TestCheckResourceAttr(testAccVirtualMachineResourceName, constants.FieldCommonDescription, testAccVirtualMachineDescription),
resource.TestCheckResourceAttr(testAccVirtualMachineResourceName, constants.FieldVirtualMachineMemory, testAccVirtualMachineMemory),
resource.TestCheckResourceAttr(testAccVirtualMachineResourceName, constants.FieldVirtualMachineCPUPinning, "true"),
),
},
},
Expand Down Expand Up @@ -111,7 +174,7 @@ func testAccVirtualMachineExists(ctx context.Context, n string, vm *kubevirtv1.V
if err != nil {
return err
}
vm = foundVM
*vm = *foundVM
return nil
}
}
Expand Down Expand Up @@ -141,3 +204,119 @@ func testAccCheckVirtualMachineDestroy(ctx context.Context) resource.TestCheckFu
return nil
}
}

func enableCPUManager(t *testing.T, ctx context.Context, nodeName string) {
if err := updateCPUManagerPolicy(ctx, nodeName, true); err != nil {
t.Fatalf("failed to enable cpu manager: %v", err)
}
}

func disableCPUManager(t *testing.T, ctx context.Context, nodeName string) {
if err := updateCPUManagerPolicy(ctx, nodeName, false); err != nil {
t.Fatalf("failed to disable cpu manager: %v", err)
}
}

func updateCPUManagerPolicy(ctx context.Context, nodeName string, enableCPUManager bool) error {
c := testAccProvider.Meta().(*client.Client)
action := "disableCPUManager"
if enableCPUManager {
action = "enableCPUManager"
}
apiURL, err := buildNodeActionURL(c.RestConfig, nodeName, action)
if err != nil {
return fmt.Errorf("failed to build node action url: %v", err)
}
req, err := createRequest(apiURL, c.RestConfig.BearerToken)
if err != nil {
return fmt.Errorf("failed to create request: %v", err)
}
client := &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true, // #nosec G402
},
},
}

resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to send request: %v", err)
}
defer resp.Body.Close()

if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK {
bodyBytes, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("failed to read response, status code: %d, err: %v", resp.StatusCode, err)
}
return fmt.Errorf("status code: %d, err: %s", resp.StatusCode, string(bodyBytes))
}

err = waitForCPUMangerLabel(ctx, c, nodeName, enableCPUManager)
if err != nil {
return fmt.Errorf("wait cpu manager label failed: %v", err)
}
return nil
}

func buildNodeActionURL(config *rest.Config, nodeName, action string) (string, error) {
parsedURL, err := url.Parse(config.Host)
if err != nil {
return "", fmt.Errorf("failed to parse restconfig host to url: %v", err)
}

parsedURL.Path = "/v1/harvester/nodes/" + nodeName
query := parsedURL.Query()
query.Set("action", action)
parsedURL.RawQuery = query.Encode()

return parsedURL.String(), nil
}

func createRequest(apiURL, bearerToken string) (*http.Request, error) {
req, err := http.NewRequest(http.MethodPost, apiURL, nil)
if err != nil {
return nil, fmt.Errorf("failed to create request: %v", err)
}
req.Header.Set("Authorization", "Bearer "+bearerToken)
return req, nil
}

func waitForCPUMangerLabel(ctx context.Context, c *client.Client, nodeName string, enableCPUManager bool) error {
return waitUntil(func() (bool, error) {
node, err := c.KubeClient.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
if err != nil {
return false, fmt.Errorf("failed to get node: %v", err)
}

expectedValue := strconv.FormatBool(enableCPUManager)
if value, exists := node.Labels["cpumanager"]; exists && value == expectedValue {
return true, nil
}
return false, nil
})
}

func waitUntil(fun func() (bool, error)) error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()

ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()

for {
select {
case <-ctx.Done():
return fmt.Errorf("timeout reached, the condition was not met within 5 minutes")
case <-ticker.C:
ok, err := fun()
if err != nil {
return err
}
if ok {
return nil
}
}
}
}
1 change: 1 addition & 0 deletions pkg/constants/constants_virtualmachine.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ const (
FieldVirtualMachineInstanceNodeName = "node_name"
FieldVirtualMachineEFI = "efi"
FieldVirtualMachineSecureBoot = "secure_boot"
FieldVirtualMachineCPUPinning = "cpu_pinning"

StateVirtualMachineStarting = "Starting"
StateVirtualMachineRunning = "Running"
Expand Down
10 changes: 8 additions & 2 deletions pkg/importer/resource_virtualmachine_importer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,12 @@ import (
"encoding/json"
"fmt"

"github.com/harvester/harvester/pkg/builder"
harvesterutil "github.com/harvester/harvester/pkg/util"
corev1 "k8s.io/api/core/v1"
kubevirtv1 "kubevirt.io/api/core/v1"

"github.com/harvester/harvester/pkg/builder"
harvesterutil "github.com/harvester/harvester/pkg/util"

"github.com/harvester/terraform-provider-harvester/pkg/constants"
"github.com/harvester/terraform-provider-harvester/pkg/helper"
)
Expand Down Expand Up @@ -50,6 +51,10 @@ func (v *VMImporter) CPU() int {
return int(v.VirtualMachine.Spec.Template.Spec.Domain.CPU.Cores)
}

func (v *VMImporter) DedicatedCPUPlacement() bool {
return bool(v.VirtualMachine.Spec.Template.Spec.Domain.CPU.DedicatedCPUPlacement)
}

func (v *VMImporter) EFI() bool {
firmware := v.VirtualMachine.Spec.Template.Spec.Domain.Firmware
return firmware != nil && firmware.Bootloader != nil && firmware.Bootloader.EFI != nil
Expand Down Expand Up @@ -380,6 +385,7 @@ func ResourceVirtualMachineStateGetter(vm *kubevirtv1.VirtualMachine, vmi *kubev
constants.FieldVirtualMachineInstanceNodeName: vmImporter.NodeName(),
constants.FieldVirtualMachineEFI: vmImporter.EFI(),
constants.FieldVirtualMachineSecureBoot: vmImporter.SecureBoot(),
constants.FieldVirtualMachineCPUPinning: vmImporter.DedicatedCPUPlacement(),
},
}, nil
}

0 comments on commit c46ed10

Please sign in to comment.