diff --git a/cmd/config/validate_test.go b/cmd/config/validate_test.go index 87537d32a453..40955c92afbb 100644 --- a/cmd/config/validate_test.go +++ b/cmd/config/validate_test.go @@ -35,9 +35,9 @@ func validConfig() []byte { func invalidConfig() []byte { config := v1beta1.DefaultClusterConfig() - config.Spec.Extensions.Storage.Type = "external_storage" - // CreateDefaultStorageClass is not supported with external_storage - config.Spec.Extensions.Storage.CreateDefaultStorageClass = true + // NLLB cannot be used with external address + config.Spec.Network.NodeLocalLoadBalancing.Enabled = true + config.Spec.API.ExternalAddress = "k0s.example.com" cfg, _ := yaml.Marshal(config) return cfg } @@ -50,7 +50,7 @@ func TestValidateCmd(t *testing.T) { errOut := bytes.NewBuffer(nil) cmd.SetErr(errOut) assert.Error(t, cmd.Execute()) - assert.Contains(t, errOut.String(), "default storage class for external_storage") + assert.Contains(t, errOut.String(), "node-local load balancing cannot be used in conjunction with an external Kubernetes API server address") errOut.Reset() cmd.SetIn(bytes.NewReader(validConfig())) assert.NoError(t, cmd.Execute()) diff --git a/docs/configuration.md b/docs/configuration.md index 5850f5e883e8..f8f8d4ec7a4f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -63,9 +63,6 @@ spec: concurrencyLevel: 5 charts: null repositories: null - storage: - create_default_storage_class: false - type: external_storage installConfig: users: etcdUser: etcd @@ -526,20 +523,6 @@ In the runtime the image names are calculated as `my.own.repo/calico/kube-contro `spec.extensions.helm` is the config file key in which you configure the list of [Helm](https://helm.sh) repositories and charts to deploy during cluster bootstrap (for more information, refer to [Helm Charts](helm-charts.md)). -### `spec.extensions.storage` - -`spec.extensions.storage` controls bundled storage provider. -The default value `external` makes no storage deployed. - -To enable [embedded host-local storage provider](examples/openebs.md) use the following configuration: - -```yaml -spec: - extensions: - storage: - type: openebs_local_storage -``` - ### `spec.konnectivity` The `spec.konnectivity` key is the config file key in which you configure Konnectivity-related settings. diff --git a/docs/examples/openebs.md b/docs/examples/openebs.md index e71513d0f8e6..210a0b376d51 100644 --- a/docs/examples/openebs.md +++ b/docs/examples/openebs.md @@ -1,16 +1,8 @@ # OpenEBS -This tutorial covers the installation of OpenEBS as a Helm extension, both from -scratch and how to migrate it from a storage extension. - -## Installing OpenEBS from scratch - -**WARNING**: Do not configure OpenEBS as both a storage extension and a Helm -extension. It's considered an invalid configuration and k0s will entirely ignore -the configuration to prevent accidental upgrades or downgrades. The chart -objects defined in the API will still behave normally. - -OpenEBS can be installed as a helm chart by adding it as an extension to your configuration: +This tutorial covers the installation of OpenEBS as a Helm extension. OpenEBS +can be installed as a helm chart by adding it as an extension to your +configuration: ```yaml extensions: @@ -33,69 +25,6 @@ OpenEBS can be installed as a helm chart by adding it as an extension to your co If you want OpenEBS to be your default storage class, set `isDefaultClass` to `true`. -## Migrating bundled OpenEBS to helm extension - -The bundled OpenEBS extension is already a helm extension installed as a -`chart.helm.k0sproject.io`. For this reason, all we have to do is to remove the -manifests and to clean up the object. However, this must be done in a specific order -to prevent data loss. - -**WARNING**: Not following the steps in the precise order presented by the -documentation may provoke data loss. - -The first step to perform the migration is to disable the `applier-manager` -component on all controllers. For each controller, restart the controller -with the flag `--disable-components=applier-manager`. If you already had this flag, -set it to `--disable-components=,applier-manager`. - -Once the `applier-manager` is disabled in every running controller, you need to modify -the configuration to use `external_storage` instead of `openebs_local_storage`. - -If you are using [dynamic configuration](../dynamic-configuration.md), you can -change it with this command: - -```shell -kubectl patch clusterconfig -n kube-system k0s --patch '{"spec":{"extensions":{"storage":{"type":"external_storage"}}}}' --type=merge -``` - -If you are using a static configuration file, replace `spec.extensions.storage.type` -from `openebs_local_storage` to `external_storage` in all control plane nodes and -restart all the control plane nodes one by one. - -When the configuration is set to `external_storage` and the servers are -restarted, you must manage the it as a chart object in the API: - -```shell -kubectl get chart -n kube-system k0s-addon-chart-openebs -o yaml -``` - -First, remove the labels and annotations related to the stack applier: - -```shell -k0s kc annotate -n kube-system chart k0s-addon-chart-openebs k0s.k0sproject.io/stack-checksum- -k0s kc label -n kube-system chart k0s-addon-chart-openebs k0s.k0sproject.io/stack- -``` - -After the annotations and labels are removed, remove the manifest file **on each -controller**. This file is located in -`/manifests/helm/_helm_extension_openebs.yaml`, which in -most installations defaults to -`/var/lib/k0s/manifests/helm/0_helm_extension_openebs.yaml`. - -**WARNING**: Not removing the old manifest file from all controllers may cause -the manifest to be reapplied, reverting your changes and potentially casuing -data loss. - -Finally, we want to re-enable the `applier-manager` and restart all controllers -without the `--disable-components=applier-manager` flag. - -Once the migration is coplete, you'll be able to update the OpenEBS chart. -Let's take v3.9.0 as an example: - -```shell -kubectl patch chart -n kube-system k0s-addon-chart-openebs --patch '{"spec":{"version":"3.9.0"}}' --type=merge -``` - ## Usage Once installed, the cluster will have two storage classes available for you to use: diff --git a/docs/storage.md b/docs/storage.md index 16a445098aef..faacdab4d40e 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -26,110 +26,3 @@ Different Kubernetes storage solutions are explained in the [official Kubernetes - Portworx If you are looking for a fault-tolerant storage with data replication, you can find a k0s tutorial for configuring Ceph storage with Rook [in here](examples/rook-ceph.md). - -## Bundled OpenEBS storage (deprecated) - -Bundled OpenEBS was deprecated in favor of running it [as a helm extension](./examples/openebs.md), -this documentation is maintained as a reference for existing installations. - -This was done for three reasons: - -1. By installing it as a helm extension, users have more control and flexibility without adding complexity. -2. It allows users to choose the OpenEBS version independent of their k0s version. -3. It makes the k0s configuration more consistent. - -For new installations or to migrate existing installations, please refer to the [OpenEBS extension page](./examples/openebs.md). - -The OpenEBS extension is enabled by setting [`spec.extensions.storage.type`](configuration.md#specextensionsstorage) to``openebs_local_storage`: - -```yaml -spec: - extensions: - storage: - type: openebs_local_storage -``` - -The cluster will have two storage classes available for you to use: - -```shell -k0s kubectl get storageclass -``` - -```shell -NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE -openebs-device openebs.io/local Delete WaitForFirstConsumer false 24s -openebs-hostpath openebs.io/local Delete WaitForFirstConsumer false 24s -``` - -The `openebs-hostpath` is the storage class that maps to the `/var/openebs/local` - -The `openebs-device` is not configured and could be configured by [manifest deployer](manifests.md) accordingly to the [OpenEBS documentation](https://docs.openebs.io/) - -### Example usage - -Use following manifests as an example of pod with mounted volume: - -```yaml -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: nginx-pvc - namespace: default -spec: - accessModes: - - ReadWriteOnce - storageClassName: openebs-hostpath - resources: - requests: - storage: 5Gi ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nginx - namespace: default - labels: - app: nginx -spec: - selector: - matchLabels: - app: nginx - strategy: - type: Recreate - template: - metadata: - labels: - app: nginx - spec: - containers: - - image: nginx - name: nginx - volumeMounts: - - name: persistent-storage - mountPath: /var/lib/nginx - volumes: - - name: persistent-storage - persistentVolumeClaim: - claimName: nginx-pvc -``` - -```shell -k0s kubectl apply -f nginx.yaml -``` - -```shell -persistentvolumeclaim/nginx-pvc created -deployment.apps/nginx created -bash-5.1# k0s kc get pods -NAME READY STATUS RESTARTS AGE -nginx-d95bcb7db-gzsdt 1/1 Running 0 30s -``` - -```shell -k0s kubectl get pv -``` - -```shell -NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE -pvc-9a7fae2d-eb03-42c3-aaa9-1a807d5df12f 5Gi RWO Delete Bound default/nginx-pvc openebs-hostpath 30s -``` diff --git a/inttest/Makefile b/inttest/Makefile index 448933c240fb..ef8da20ba86f 100644 --- a/inttest/Makefile +++ b/inttest/Makefile @@ -107,8 +107,6 @@ check-metricsscraper-singlenode: TEST_PACKAGE=metricsscraper check-nllb: TIMEOUT=15m -check-openebs: TIMEOUT=7m - .PHONY: $(smoketests) include Makefile.variables diff --git a/inttest/Makefile.variables b/inttest/Makefile.variables index 800480b593f9..90a249262643 100644 --- a/inttest/Makefile.variables +++ b/inttest/Makefile.variables @@ -30,7 +30,6 @@ smoketests := \ check-customdomain \ check-customports \ check-customports-dynamicconfig \ - check-defaultstorage \ check-disabledcomponents \ check-dualstack \ check-dualstack-dynamicconfig \ @@ -55,7 +54,6 @@ smoketests := \ check-noderole \ check-noderole-no-taints \ check-noderole-single \ - check-openebs\ check-psp \ check-reset \ check-singlenode \ diff --git a/inttest/defaultstorage/defaultstorage_test.go b/inttest/defaultstorage/defaultstorage_test.go deleted file mode 100644 index 7201c891cfe2..000000000000 --- a/inttest/defaultstorage/defaultstorage_test.go +++ /dev/null @@ -1,123 +0,0 @@ -/* -Copyright 2022 k0s authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package defaultstorage - -import ( - "testing" - - "github.com/stretchr/testify/suite" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/k0sproject/k0s/inttest/common" -) - -type DefaultStorageSuite struct { - common.BootlooseSuite -} - -func (s *DefaultStorageSuite) TestK0sGetsUp() { - s.PutFile(s.ControllerNode(0), "/tmp/k0s.yaml", k0sConfig) - s.Require().NoError(s.InitController(0, "--config=/tmp/k0s.yaml")) - s.Require().NoError(s.RunWorkers()) - - kc, err := s.KubeClient(s.ControllerNode(0), "") - s.Require().NoError(err) - - err = s.WaitForNodeReady(s.WorkerNode(0), kc) - s.NoError(err) - - s.T().Log("waiting to see default storage class") - err = common.WaitForDefaultStorageClass(s.Context(), kc) - s.NoError(err) - - // We need to create the pvc only after default storage class is set, otherwise k8s will not be able to set it on the PVC - s.T().Log("default SC found, creating a deployment with PVC and waiting for it to be ready") - s.MakeDir(s.ControllerNode(0), "/var/lib/k0s/manifests/test") - s.PutFile(s.ControllerNode(0), "/var/lib/k0s/manifests/test/pvc.yaml", pvcManifest) - s.PutFile(s.ControllerNode(0), "/var/lib/k0s/manifests/test/deployment.yaml", deploymentManifest) - err = common.WaitForDeployment(s.Context(), kc, "nginx", "kube-system") - s.NoError(err) - - s.AssertSomeKubeSystemPods(kc) - - pv, err := kc.CoreV1().PersistentVolumes().List(s.Context(), v1.ListOptions{}) - s.Require().NoError(err) - s.NotEmpty(pv.Items, "At least one persistent volume must be created for the deployment with claims") -} - -func TestDefaultStorageSuite(t *testing.T) { - s := DefaultStorageSuite{ - common.BootlooseSuite{ - ControllerCount: 1, - WorkerCount: 1, - }, - } - suite.Run(t, &s) -} - -const pvcManifest = ` -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: nginx-pvc - namespace: kube-system -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 5Gi -` - -const deploymentManifest = ` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nginx - namespace: kube-system - labels: - app: nginx -spec: - selector: - matchLabels: - app: nginx - strategy: - type: Recreate - template: - metadata: - labels: - app: nginx - spec: - containers: - - image: nginx - name: nginx - volumeMounts: - - name: persistent-storage - mountPath: /var/lib/nginx - volumes: - - name: persistent-storage - persistentVolumeClaim: - claimName: nginx-pvc -` - -const k0sConfig = ` -spec: - extensions: - storage: - type: openebs_local_storage - create_default_storage_class: true -` diff --git a/inttest/openebs/openebs_test.go b/inttest/openebs/openebs_test.go deleted file mode 100644 index cc1333c6a989..000000000000 --- a/inttest/openebs/openebs_test.go +++ /dev/null @@ -1,211 +0,0 @@ -/* -Copyright 2024 k0s authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package openebs - -import ( - "context" - "testing" - "time" - - "github.com/k0sproject/bootloose/pkg/config" - "github.com/k0sproject/k0s/inttest/common" - "github.com/k0sproject/k0s/pkg/kubernetes/watch" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/client-go/kubernetes" - - helmv1beta1 "github.com/k0sproject/k0s/pkg/apis/helm/v1beta1" - helmclient "github.com/k0sproject/k0s/pkg/client/clientset/typed/helm/v1beta1" - "github.com/stretchr/testify/suite" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -type OpenEBSSuite struct { - common.BootlooseSuite -} - -func (s *OpenEBSSuite) TestK0sGetsUp() { - ctx := s.Context() - - s.T().Log("Start k0s with both storage and helm extensions enabled") - s.PutFile(s.ControllerNode(0), "/tmp/k0s.yaml", k0sConfigWithBoth) - s.Require().NoError(s.InitController(0, "--config=/tmp/k0s.yaml", "--disable-components=konnectivity-server,metrics-server")) - s.Require().NoError(s.RunWorkers()) - - kc, err := s.KubeClient(s.ControllerNode(0), "") - s.Require().NoError(err) - - s.Require().NoError(s.WaitForNodeReady(s.WorkerNode(0), kc)) - - // When both storage and helm are enabled, there should be no action. - // Unfortunately to check that, the best we can do is seeing that it's not - // created after a grace period - s.T().Log("Waiting 30 additional seconds of grace period to see if charts are created") - s.sleep(ctx, 30*time.Second) - - s.T().Log("Checking that the chart isn't created") - hc, err := s.HelmClient(s.ControllerNode(0), "") - s.Require().NoError(err) - _, err = hc.Charts("kube-system").Get(ctx, openEBSChart, metav1.GetOptions{}) - s.Require().True(errors.IsNotFound(err), "Chart was created when it shouldn't have been") - - // Verify Test as a storage extension - s.T().Log("Retarting k0s with only storage extension enabled") - s.Require().NoError(s.StopController(s.ControllerNode(0))) - s.PutFile(s.ControllerNode(0), "/tmp/k0s.yaml", k0sConfigWithStorage) - s.Require().NoError(s.StartController(s.ControllerNode(0))) - s.Require().NoError(s.WaitForNodeReady(s.WorkerNode(0), kc)) - - s.T().Log("Checking that the chart is created and ready") - s.Require().NoError(s.waitForChartUpdated(ctx, "3.3.0")) - s.waitForOpenEBSReady(ctx, kc) - - // Migrate to helm chart - s.T().Log("Restarting k0s without applier-manager and without extension") - s.Require().NoError(s.StopController(s.ControllerNode(0))) - s.PutFile(s.ControllerNode(0), "/tmp/k0s.yaml", k0sConfigNoExtension) - s.Require().NoError(s.InitController(0, "--config=/tmp/k0s.yaml", "--disable-components=konnectivity-server,metrics-server,applier-manager")) - s.Require().NoError(s.WaitForNodeReady(s.WorkerNode(0), kc)) - - s.T().Log("Removing Label and annotation") - c, err := hc.Charts("kube-system").Get(ctx, openEBSChart, metav1.GetOptions{}) - s.Require().NoError(err, "Error getting OpenEBS chart after removing the storage extension") - delete(c.Annotations, "k0s.k0sproject.io/stack-checksum") - delete(c.Labels, "k0s.k0sproject.io/stack") - _, err = hc.Charts("kube-system").Update(ctx, c, metav1.UpdateOptions{}) - s.Require().NoError(err, "Error removing stack applier information in OpenEBS chart") - - s.T().Log("Removing the manifest") - ssh, err := s.SSH(s.Context(), s.ControllerNode(0)) - s.Require().NoError(err) - defer ssh.Disconnect() - s.Require().NoError(ssh.Exec(ctx, "rm -f /var/lib/k0s/manifests/helm/0_helm_extension_openebs.yaml", common.SSHStreams{})) - - s.T().Log("Upgrading to 3.9.0") - c, err = hc.Charts("kube-system").Get(ctx, openEBSChart, metav1.GetOptions{}) - s.Require().NoError(err, "Error getting OpenEBS chart after removing the storage extension") - c.Spec.Version = "3.9.0" - _, err = hc.Charts("kube-system").Update(ctx, c, metav1.UpdateOptions{}) - s.Require().NoError(err, "Error upgrading OpenEBS chart") - - s.T().Log("Checking that the chart is upgrades to 3.9.0 and becomes ready") - s.Require().NoError(s.waitForChartUpdated(ctx, "3.9.0")) - s.waitForOpenEBSReady(ctx, kc) - - // Test that applier doesn't revert it back to 3.9.0 - s.T().Log("Restarting the controller with manifest applier") - s.Require().NoError(s.InitController(0, "--config=/tmp/k0s.yaml", "--disable-components=konnectivity-server,metrics-server")) - s.Require().NoError(s.WaitForNodeReady(s.WorkerNode(0), kc)) - - s.T().Log("Waiting 30 additional seconds of grace period to see if charts is deleted") - s.sleep(ctx, 30*time.Second) - - s.T().Log("Checking that the chart is still to 3.9.0 and ready") - s.Require().NoError(s.waitForChartUpdated(ctx, "3.9.0")) - s.waitForOpenEBSReady(ctx, kc) -} - -func TestOpenEBSSuite(t *testing.T) { - s := OpenEBSSuite{ - common.BootlooseSuite{ - ControllerCount: 1, - WorkerCount: 1, - ExtraVolumes: []config.Volume{{ - Type: "bind", - Source: "/run/udev", - Destination: "/run/udev", - ReadOnly: false, - }}, - }, - } - suite.Run(t, &s) -} - -func (s *OpenEBSSuite) waitForChartUpdated(ctx context.Context, version string) error { - hc, err := s.HelmClient(s.ControllerNode(0), "") - s.Require().NoError(err) - - return watch.Charts(hc.Charts("kube-system")). - WithObjectName(openEBSChart). - WithErrorCallback(common.RetryWatchErrors(s.T().Logf)). - Until(ctx, func(chart *helmv1beta1.Chart) (done bool, err error) { - // We don't need to actually deploy helm in this test - // we're just validation that the spec is correct - return chart.Spec.Version == version && - chart.Status.AppVersion == version && - chart.Status.Version == version, nil - }) - -} - -func (s *OpenEBSSuite) waitForOpenEBSReady(ctx context.Context, kc *kubernetes.Clientset) { - s.T().Log("Waiting for openEBS to be ready") - s.Require().NoError(common.WaitForDeployment(ctx, kc, "openebs-localpv-provisioner", "openebs")) - s.Require().NoError(common.WaitForDeployment(ctx, kc, "openebs-ndm-operator", "openebs")) - s.Require().NoError(common.WaitForDaemonSet(ctx, kc, "openebs-ndm", "openebs")) -} - -// HelmClient returns HelmV1beta1Client by loading the admin access config from given node -func (s *OpenEBSSuite) HelmClient(node string, k0sKubeconfigArgs ...string) (*helmclient.HelmV1beta1Client, error) { - cfg, err := s.GetKubeConfig(node, k0sKubeconfigArgs...) - if err != nil { - return nil, err - } - return helmclient.NewForConfig(cfg) -} - -func (s *OpenEBSSuite) sleep(ctx context.Context, d time.Duration) { - select { - case <-ctx.Done(): - s.Require().NoError(ctx.Err()) - case <-time.After(30 * time.Second): - } -} - -const k0sConfigWithBoth = ` -spec: - extensions: - storage: - type: openebs_local_storage - helm: - repositories: - - name: openebs-internal - url: https://openebs.github.io/charts - charts: - - name: openebs - chartname: openebs-internal/openebs - version: "3.9.0" - namespace: openebs - order: 1 - values: | - localprovisioner: - hostpathClass: - enabled: true - isDefaultClass: false -` - -const k0sConfigWithStorage = ` -spec: - extensions: - storage: - type: openebs_local_storage -` - -const k0sConfigNoExtension = ` -spec: - extensions: {} -` -const openEBSChart = "k0s-addon-chart-openebs" diff --git a/pkg/apis/k0s/v1beta1/clusterconfig_types.go b/pkg/apis/k0s/v1beta1/clusterconfig_types.go index 65c880a4dc28..8afbcfe28ea1 100644 --- a/pkg/apis/k0s/v1beta1/clusterconfig_types.go +++ b/pkg/apis/k0s/v1beta1/clusterconfig_types.go @@ -235,9 +235,6 @@ func (c *ClusterConfig) UnmarshalJSON(data []byte) error { if jc.Spec.Extensions == nil { jc.Spec.Extensions = DefaultExtensions() } - if jc.Spec.Extensions.Storage == nil { - jc.Spec.Extensions.Storage = DefaultStorageExtension() - } if jc.Spec.Network == nil { jc.Spec.Network = DefaultNetwork() } diff --git a/pkg/apis/k0s/v1beta1/extensions.go b/pkg/apis/k0s/v1beta1/extensions.go index 1f0bfab4129d..9af1cf7f7156 100644 --- a/pkg/apis/k0s/v1beta1/extensions.go +++ b/pkg/apis/k0s/v1beta1/extensions.go @@ -27,9 +27,22 @@ var _ Validateable = (*ClusterExtensions)(nil) // ClusterExtensions specifies cluster extensions type ClusterExtensions struct { - //+kubebuilder:deprecatedversion:warning="storage is deprecated and will be ignored in 1.30. https://docs.k0sproject.io/stable/examples/openebs". - Storage *StorageExtension `json:"storage"` - Helm *HelmExtensions `json:"helm"` + // Deprecated: storage is deprecated and will be ignored starting from k0s + // 1.31 and onwards: https://docs.k0sproject.io/stable/examples/openebs + // +optional + Storage *StorageExtension `json:"storage,omitempty"` + + Helm *HelmExtensions `json:"helm"` +} + +// Deprecated: No-op; kept for backwards compatibility. +type StorageExtension struct { + // Deprecated: No-op; kept for backwards compatibility. + // +optional + Type string `json:"type,omitempty"` + // Deprecated: No-op; kept for backwards compatibility. + // +optional + CreateDefaultStorageClass bool `json:"create_default_storage_class"` } // HelmExtensions specifies settings for cluster helm based extensions @@ -166,23 +179,12 @@ func (e *ClusterExtensions) Validate() []error { if e.Helm != nil { errs = append(errs, e.Helm.Validate()...) } - if e.Storage != nil { - errs = append(errs, e.Storage.Validate()...) - } return errs } -func DefaultStorageExtension() *StorageExtension { - return &StorageExtension{ - Type: ExternalStorage, - CreateDefaultStorageClass: false, - } -} - // DefaultExtensions default values func DefaultExtensions() *ClusterExtensions { return &ClusterExtensions{ - Storage: DefaultStorageExtension(), Helm: &HelmExtensions{ ConcurrencyLevel: 5, }, diff --git a/pkg/apis/k0s/v1beta1/storageextensions.go b/pkg/apis/k0s/v1beta1/storageextensions.go deleted file mode 100644 index a79b990d038a..000000000000 --- a/pkg/apis/k0s/v1beta1/storageextensions.go +++ /dev/null @@ -1,46 +0,0 @@ -/* -Copyright 2021 k0s authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import "fmt" - -// StorageExtenstion specifies cluster default storage -type StorageExtension struct { - Type string `json:"type"` - CreateDefaultStorageClass bool `json:"create_default_storage_class"` -} - -var _ Validateable = (*StorageExtension)(nil) - -const ( - ExternalStorage = "external_storage" - OpenEBSLocal = "openebs_local_storage" -) - -func (se *StorageExtension) Validate() []error { - var errs []error - switch se.Type { - case ExternalStorage, OpenEBSLocal: - // do nothing on valid types - default: - errs = append(errs, fmt.Errorf("unknown storage mode `%s`", se.Type)) - } - if se.CreateDefaultStorageClass && se.Type == ExternalStorage { - errs = append(errs, fmt.Errorf("can't create default storage class for external_storage")) - } - return errs -} diff --git a/pkg/component/controller/extensions_controller.go b/pkg/component/controller/extensions_controller.go index 8eb1c3f4c23c..63c46f99e705 100644 --- a/pkg/component/controller/extensions_controller.go +++ b/pkg/component/controller/extensions_controller.go @@ -37,14 +37,12 @@ import ( "github.com/k0sproject/k0s/pkg/component/controller/leaderelector" "github.com/k0sproject/k0s/pkg/component/manager" "github.com/k0sproject/k0s/pkg/config" - "github.com/k0sproject/k0s/pkg/constant" "github.com/k0sproject/k0s/pkg/helm" kubeutil "github.com/k0sproject/k0s/pkg/kubernetes" "github.com/sirupsen/logrus" "helm.sh/helm/v3/pkg/release" "helm.sh/helm/v3/pkg/storage/driver" apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/clientcmd" @@ -57,7 +55,6 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/yaml" ) // Helm watch for Chart crd @@ -91,79 +88,7 @@ const ( func (ec *ExtensionsController) Reconcile(ctx context.Context, clusterConfig *k0sv1beta1.ClusterConfig) error { ec.L.Info("Extensions reconciliation started") defer ec.L.Info("Extensions reconciliation finished") - - var errs []error - helmSettings, err := ec.configureStorage(clusterConfig) - if err != nil { - errs = append(errs, fmt.Errorf("cannot configure storage: %w", err)) - } - - if err := ec.reconcileHelmExtensions(helmSettings); err != nil { - errs = append(errs, fmt.Errorf("can't reconcile helm based extensions: %w", err)) - } - - return errors.Join(errs...) -} - -func (ec *ExtensionsController) configureStorage(clusterConfig *k0sv1beta1.ClusterConfig) (*k0sv1beta1.HelmExtensions, error) { - helmSettings := clusterConfig.Spec.Extensions.Helm - if clusterConfig.Spec.Extensions.Storage.Type != k0sv1beta1.OpenEBSLocal { - return helmSettings, nil - } - - for _, chart := range helmSettings.Charts { - if chart.ChartName == "openebs-internal/openebs" { - return nil, fmt.Errorf("openebs-internal/openebs is defined in spec.extensions.helm.charts and spec.extensions.storage.type is set to openebs_local_storage. https://docs.k0sproject.io/stable/examples/openebs") - } - } - helmSettings, err := addOpenEBSHelmExtension(helmSettings, clusterConfig.Spec.Extensions.Storage) - if err != nil { - return nil, fmt.Errorf("can't add openebs helm extension") - } - return helmSettings, nil -} - -func addOpenEBSHelmExtension(helmSpec *k0sv1beta1.HelmExtensions, storageExtension *k0sv1beta1.StorageExtension) (*k0sv1beta1.HelmExtensions, error) { - openEBSValues := map[string]interface{}{ - "localprovisioner": map[string]interface{}{ - "hostpathClass": map[string]interface{}{ - "enabled": true, - "isDefaultClass": storageExtension.CreateDefaultStorageClass, - }, - }, - } - values, err := yamlifyValues(openEBSValues) - if err != nil { - logrus.Errorf("can't yamlify openebs values: %v", err) - return nil, err - } - if helmSpec == nil { - helmSpec = &k0sv1beta1.HelmExtensions{ - Repositories: k0sv1beta1.RepositoriesSettings{}, - Charts: k0sv1beta1.ChartsSettings{}, - } - } - helmSpec.Repositories = append(helmSpec.Repositories, k0sv1beta1.Repository{ - Name: "openebs-internal", - URL: constant.OpenEBSRepository, - }) - helmSpec.Charts = append(helmSpec.Charts, k0sv1beta1.Chart{ - Name: "openebs", - ChartName: "openebs-internal/openebs", - TargetNS: "openebs", - Version: constant.OpenEBSVersion, - Values: values, - Timeout: metav1.Duration{Duration: time.Duration(time.Minute * 30)}, // it takes a while to install openebs - }) - return helmSpec, nil -} - -func yamlifyValues(values map[string]interface{}) (string, error) { - bytes, err := yaml.Marshal(values) - if err != nil { - return "", err - } - return string(bytes), nil + return ec.reconcileHelmExtensions(clusterConfig.Spec.Extensions.Helm) } // reconcileHelmExtensions creates instance of Chart CR for each chart of the config file diff --git a/pkg/component/controller/extensions_controller_test.go b/pkg/component/controller/extensions_controller_test.go index dc4da3b246d1..499c68f9fdfb 100644 --- a/pkg/component/controller/extensions_controller_test.go +++ b/pkg/component/controller/extensions_controller_test.go @@ -136,69 +136,6 @@ func TestChartNeedsUpgrade(t *testing.T) { } } -func addHelmExtension(config *k0sv1beta1.ClusterConfig) *k0sv1beta1.ClusterConfig { - config.Spec.Extensions.Storage.Type = k0sv1beta1.OpenEBSLocal - return config -} - -func addStorageExtension(config *k0sv1beta1.ClusterConfig) *k0sv1beta1.ClusterConfig { - config.Spec.Extensions.Helm, _ = addOpenEBSHelmExtension(config.Spec.Extensions.Helm, config.Spec.Extensions.Storage) - return config -} - -func TestConfigureStorage(t *testing.T) { - var testCases = []struct { - description string - clusterConfig *k0sv1beta1.ClusterConfig - expectedErr bool - expectedOpenEBS bool - }{ - { - "no_openebs", - k0sv1beta1.DefaultClusterConfig(), - false, - false, - }, - { - "openebs_helm_extension", - addHelmExtension(k0sv1beta1.DefaultClusterConfig()), - false, - true, - }, - { - "openebs_storage_extension", - addStorageExtension(k0sv1beta1.DefaultClusterConfig()), - false, - true, - }, - { - "openebs_both", - addStorageExtension(addHelmExtension(k0sv1beta1.DefaultClusterConfig())), - true, - false, - }, - } - - for _, tc := range testCases { - t.Run(tc.description, func(t *testing.T) { - ec := ExtensionsController{} - helmSettings, err := ec.configureStorage(tc.clusterConfig) - - if tc.expectedErr { - assert.Error(t, err) - } else { - assert.NoError(t, err) - } - - if tc.expectedOpenEBS { - assert.Equal(t, 1, len(helmSettings.Charts)) - assert.Equal(t, "openebs", helmSettings.Charts[0].Name) - } - }) - } - -} - func TestChartManifestFileName(t *testing.T) { chart := k0sv1beta1.Chart{ Name: "release", diff --git a/static/manifests/k0s/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml b/static/manifests/k0s/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml index b81c5b7b5443..5cef1473f867 100644 --- a/static/manifests/k0s/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml +++ b/static/manifests/k0s/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml @@ -157,11 +157,15 @@ spec: type: array type: object storage: - description: StorageExtenstion specifies cluster default storage + description: |- + Deprecated: storage is deprecated and will be ignored starting from k0s + 1.31 and onwards: https://docs.k0sproject.io/stable/examples/openebs properties: create_default_storage_class: + description: 'Deprecated: No-op; kept for backwards compatibility.' type: boolean type: + description: 'Deprecated: No-op; kept for backwards compatibility.' type: string type: object type: object