diff --git a/inttest/Makefile.variables b/inttest/Makefile.variables index ac5c579d84b1..5764c17a033c 100644 --- a/inttest/Makefile.variables +++ b/inttest/Makefile.variables @@ -52,6 +52,7 @@ smoketests := \ check-noderole \ check-noderole-no-taints \ check-noderole-single \ + check-openebs\ check-psp \ check-singlenode \ check-statussocket \ diff --git a/inttest/common/bootloosesuite.go b/inttest/common/bootloosesuite.go index ecd9540452df..4e6cf4762ad0 100644 --- a/inttest/common/bootloosesuite.go +++ b/inttest/common/bootloosesuite.go @@ -43,6 +43,7 @@ import ( "github.com/k0sproject/k0s/internal/pkg/file" apclient "github.com/k0sproject/k0s/pkg/client/clientset" + helmclient "github.com/k0sproject/k0s/pkg/client/clientset/typed/helm/v1beta1" "github.com/k0sproject/k0s/pkg/constant" "github.com/k0sproject/k0s/pkg/k0scontext" "github.com/k0sproject/k0s/pkg/kubernetes/watch" @@ -905,6 +906,15 @@ func (s *BootlooseSuite) ExtensionsClient(node string, k0sKubeconfigArgs ...stri return extclient.NewForConfig(cfg) } +// KubeClient return kube client by loading the admin access config from given node +func (s *BootlooseSuite) HelmClient(node string, k0sKubeconfigArgs ...string) (*helmclient.HelmV1beta1Client, error) { + cfg, err := s.GetKubeConfig(node, k0sKubeconfigArgs...) + if err != nil { + return nil, err + } + return helmclient.NewForConfig(cfg) +} + // WaitForNodeReady wait that we see the given node in "Ready" state in kubernetes API func (s *BootlooseSuite) WaitForNodeReady(name string, kc kubernetes.Interface) error { s.T().Logf("waiting to see %s ready in kube API", name) diff --git a/inttest/openebs/openebs_test.go b/inttest/openebs/openebs_test.go new file mode 100644 index 000000000000..ad1e783eaefa --- /dev/null +++ b/inttest/openebs/openebs_test.go @@ -0,0 +1,181 @@ +/* +Copyright 2024 k0s authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package openebs + +import ( + "context" + "testing" + "time" + + "github.com/k0sproject/k0s/inttest/common" + "github.com/k0sproject/k0s/pkg/kubernetes/watch" + "k8s.io/apimachinery/pkg/api/errors" + + helmv1beta1 "github.com/k0sproject/k0s/pkg/apis/helm/v1beta1" + "github.com/stretchr/testify/suite" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type OpenEBSSuite struct { + common.BootlooseSuite +} + +func (s *OpenEBSSuite) TestK0sGetsUp() { + ctx := s.Context() + s.T().Run("Both storage and helm extensions enabled", func(t *testing.T) { + s.PutFile(s.ControllerNode(0), "/tmp/k0s.yaml", k0sConfigWithBoth) + s.Require().NoError(s.InitController(0, "--config=/tmp/k0s.yaml", "--disable-components=konnectivity-server,metrics-server")) + s.Require().NoError(s.RunWorkers()) + + kc, err := s.KubeClient("controller0", "") + s.Require().NoError(err) + + err = s.WaitForNodeReady("worker0", kc) + s.NoError(err) + + err = s.WaitForNodeReady("worker1", kc) + s.NoError(err) + + // When both storage and helm are enabled, there should be no action. + // Unfortunately to check that, the best we can do is seeing that it's not + // created after a grace period + s.T().Log("Waiting 30 additional seconds of grace period to see if charts are created") + time.Sleep(1 * time.Second) + + s.T().Log("Checking that the chart isn't created") + hc, err := s.HelmClient("controller0", "") + s.Require().NoError(err) + _, err = hc.Charts("kube-system").Get(ctx, openEBSChart, metav1.GetOptions{}) + s.Require().True(errors.IsNotFound(err), "Chart was created when it shouldn't have been") + + s.T().Log("Stopping k0s") + s.Require().NoError(s.StopController(s.ControllerNode(0))) + }) + + s.T().Run("Only storage extension enabled", func(t *testing.T) { + s.PutFile(s.ControllerNode(0), "/tmp/k0s.yaml", k0sConfigWithStorage) + s.Require().NoError(s.InitController(0, "--config=/tmp/k0s.yaml", "--disable-components=konnectivity-server,metrics-server")) + + kc, err := s.KubeClient("controller0", "") + s.Require().NoError(err) + + err = s.WaitForNodeReady("worker0", kc) + s.NoError(err) + + // When both storage and helm are enabled, there should be no action. + // Unfortunately to check that, the best we can do is seeing that it's not + // created after a grace period + s.T().Log("Checking that the chart is created") + s.Require().NoError(s.waitForChartUpdated(ctx, "3.3.0")) + + s.T().Log("Stopping k0s") + s.Require().NoError(s.StopController(s.ControllerNode(0))) + }) + + s.T().Run("Only helm extension enabled", func(t *testing.T) { + s.PutFile(s.ControllerNode(0), "/tmp/k0s.yaml", k0sConfigWithHelm) + s.Require().NoError(s.InitController(0, "--config=/tmp/k0s.yaml", "--disable-components=konnectivity-server,metrics-server")) + + kc, err := s.KubeClient("controller0", "") + s.Require().NoError(err) + + err = s.WaitForNodeReady("worker0", kc) + s.NoError(err) + + // When both storage and helm are enabled, there should be no action. + // Unfortunately to check that, the best we can do is seeing that it's not + // created after a grace period + s.T().Log("Checking that the chart is updated to 3.9.0") + s.Require().NoError(s.waitForChartUpdated(ctx, "3.9.0")) + }) +} + +func TestOpenEBSSuite(t *testing.T) { + s := OpenEBSSuite{ + common.BootlooseSuite{ + ControllerCount: 1, + WorkerCount: 2, + }, + } + suite.Run(t, &s) +} + +func (s *OpenEBSSuite) waitForChartUpdated(ctx context.Context, version string) error { + hc, err := s.HelmClient("controller0", "") + s.Require().NoError(err) + + hc.Charts("kube-system").Watch(ctx, metav1.ListOptions{}) + return watch.Charts(hc.Charts("kube-system")). + WithObjectName(openEBSChart). + Until(ctx, func(chart *helmv1beta1.Chart) (done bool, err error) { + // We don't need to actually deploy helm in this test + // we're just validation that the spec is correct + return chart.Spec.Version == version, nil + }) + +} + +const k0sConfigWithBoth = ` +spec: + extensions: + storage: + type: openebs_local_storage + helm: + repositories: + - name: openebs-internal + url: https://openebs.github.io/charts + charts: + - name: openebs + chartname: openebs-internal/openebs + version: "3.9.0" + namespace: openebs + order: 1 + values: | + localprovisioner: + hostpathClass: + enabled: true + isDefaultClass: false +` + +const k0sConfigWithStorage = ` +spec: + extensions: + storage: + type: openebs_local_storage +` + +const k0sConfigWithHelm = ` +spec: + extensions: + helm: + repositories: + - name: openebs-internal + url: https://openebs.github.io/charts + charts: + - name: openebs + chartname: openebs-internal/openebs + version: "3.9.0" + namespace: openebs + order: 1 + values: | + localprovisioner: + hostpathClass: + enabled: true + isDefaultClass: false +` +const openEBSChartPath = "/apis/helm.k0sproject.io/v1beta1/namespaces/kube-system/charts/k0s-addon-chart-openebs" +const openEBSChart = "k0s-addon-chart-openebs" diff --git a/pkg/kubernetes/watch/k0s.go b/pkg/kubernetes/watch/k0s.go index 2cf7e3681e2b..fb9ac3236503 100644 --- a/pkg/kubernetes/watch/k0s.go +++ b/pkg/kubernetes/watch/k0s.go @@ -18,6 +18,7 @@ package watch import ( autopilotv1beta2 "github.com/k0sproject/k0s/pkg/apis/autopilot/v1beta2" + helmv1beta1 "github.com/k0sproject/k0s/pkg/apis/helm/v1beta1" k0sv1beta1 "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" ) @@ -28,3 +29,7 @@ func ClusterConfigs(client Provider[*k0sv1beta1.ClusterConfigList]) *Watcher[k0s func Plans(client Provider[*autopilotv1beta2.PlanList]) *Watcher[autopilotv1beta2.Plan] { return FromClient[*autopilotv1beta2.PlanList, autopilotv1beta2.Plan](client) } + +func Charts(client Provider[*helmv1beta1.ChartList]) *Watcher[helmv1beta1.Chart] { + return FromClient[*helmv1beta1.ChartList, helmv1beta1.Chart](client) +}