diff --git a/deploy/csi-driver.yaml b/deploy/ecs/csi-driver.yaml similarity index 100% rename from deploy/csi-driver.yaml rename to deploy/ecs/csi-driver.yaml diff --git a/deploy/csi-plugin.yaml b/deploy/ecs/csi-plugin.yaml similarity index 100% rename from deploy/csi-plugin.yaml rename to deploy/ecs/csi-plugin.yaml diff --git a/deploy/csi-provisioner.yaml b/deploy/ecs/csi-provisioner.yaml similarity index 100% rename from deploy/csi-provisioner.yaml rename to deploy/ecs/csi-provisioner.yaml diff --git a/deploy/rbac.yaml b/deploy/ecs/rbac.yaml similarity index 100% rename from deploy/rbac.yaml rename to deploy/ecs/rbac.yaml diff --git a/deploy/nonecs/csi-driver.yaml b/deploy/nonecs/csi-driver.yaml new file mode 100644 index 000000000..967564a14 --- /dev/null +++ b/deploy/nonecs/csi-driver.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: nasplugin.csi.alibabacloud.com +spec: + attachRequired: false + podInfoOnMount: true +--- +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: ossplugin.csi.alibabacloud.com +spec: + attachRequired: false + podInfoOnMount: true \ No newline at end of file diff --git a/deploy/nonecs/csi-plugin.yaml b/deploy/nonecs/csi-plugin.yaml new file mode 100644 index 000000000..29a575297 --- /dev/null +++ b/deploy/nonecs/csi-plugin.yaml @@ -0,0 +1,181 @@ +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-plugin + namespace: kube-system +spec: + selector: + matchLabels: + app: csi-plugin + template: + metadata: + labels: + app: csi-plugin + spec: + tolerations: + - operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: type + operator: NotIn + values: + - virtual-kubelet + nodeSelector: + kubernetes.io/os: linux + serviceAccount: csi-admin + priorityClassName: system-node-critical + hostNetwork: true + hostPID: true + dnsPolicy: ClusterFirst + containers: + - name: nas-driver-registrar + image: registry-cn-hangzhou.ack.aliyuncs.com/acs/csi-node-driver-registrar:v2.3.1-038aeb6-aliyun + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 500m + memory: 1024Mi + args: + - "--v=5" + - "--csi-address=/var/lib/kubelet/csi-plugins/nasplugin.csi.alibabacloud.com/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/csi-plugins/nasplugin.csi.alibabacloud.com/csi.sock" + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet/ + - name: registration-dir + mountPath: /registration + - name: oss-driver-registrar + image: registry-cn-hangzhou.ack.aliyuncs.com/acs/csi-node-driver-registrar:v2.3.1-038aeb6-aliyun + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 500m + memory: 1024Mi + args: + - "--v=5" + - "--csi-address=/var/lib/kubelet/csi-plugins/ossplugin.csi.alibabacloud.com/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/csi-plugins/ossplugin.csi.alibabacloud.com/csi.sock" + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet/ + - name: registration-dir + mountPath: /registration + - name: csi-plugin + securityContext: + privileged: true + allowPrivilegeEscalation: true + image: registry-cn-hangzhou.ack.aliyuncs.com/acs/csi-plugin:v1.24.9-74f8490-aliyun + args: + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=2" + - "--nodeid=$(KUBE_NODE_NAME)" + - "--driver=oss,nas" + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix://var/lib/kubelet/csi-plugins/driverplugin.csi.alibabacloud.com-replace/csi.sock + - name: MAX_VOLUMES_PERNODE + value: "15" + - name: SERVICE_TYPE + value: "plugin" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 1024Mi + livenessProbe: + httpGet: + path: /healthz + port: healthz + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 5 + ports: + - name: healthz + containerPort: 11260 + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet/ + mountPropagation: "Bidirectional" + - name: etc + mountPath: /host/etc + - name: host-log + mountPath: /var/log/ + - name: ossconnectordir + mountPath: /host/usr/ + - name: container-dir + mountPath: /var/lib/container + mountPropagation: "Bidirectional" + - name: host-dev + mountPath: /dev + mountPropagation: "HostToContainer" + - mountPath: /var/addon + name: addon-token + readOnly: true + - mountPath: /host/var/run/ + name: fuse-metrics-dir + volumes: + - name: fuse-metrics-dir + hostPath: + path: /var/run/ + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry + type: DirectoryOrCreate + - name: container-dir + hostPath: + path: /var/lib/container + type: DirectoryOrCreate + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: host-dev + hostPath: + path: /dev + - name: host-log + hostPath: + path: /var/log/ + - name: etc + hostPath: + path: /etc + - name: ossconnectordir + hostPath: + path: /usr/ + - name: addon-token + secret: + defaultMode: 420 + optional: true + items: + - key: addon.token.config + path: token-config + secretName: addon.csi.token + updateStrategy: + rollingUpdate: + maxUnavailable: 20% + type: RollingUpdate \ No newline at end of file diff --git a/deploy/nonecs/csi-provisioner.yaml b/deploy/nonecs/csi-provisioner.yaml new file mode 100644 index 000000000..066c075d1 --- /dev/null +++ b/deploy/nonecs/csi-provisioner.yaml @@ -0,0 +1,210 @@ +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: csi-provisioner + namespace: kube-system +spec: + selector: + matchLabels: + app: csi-provisioner + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + replicas: 2 + template: + metadata: + labels: + app: csi-provisioner + spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: type + operator: NotIn + values: + - virtual-kubelet + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - csi-provisioner + topologyKey: kubernetes.io/hostname + tolerations: + - effect: NoSchedule + operator: Exists + key: node-role.kubernetes.io/master + - effect: NoSchedule + operator: Exists + key: node.cloudprovider.kubernetes.io/uninitialized + serviceAccount: csi-admin + hostPID: true + priorityClassName: system-node-critical + containers: + - name: external-nas-provisioner + image: registry-cn-hangzhou.ack.aliyuncs.com/acs/csi-provisioner:v3.0.0-080f01e64-aliyun + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 500m + memory: 1024Mi + args: + - "--csi-address=$(ADDRESS)" + - "--volume-name-prefix=nas" + - "--timeout=150s" + - "--leader-election=true" + - "--retry-interval-start=500ms" + - "--default-fstype=nfs" + - "--v=5" + env: + - name: ADDRESS + value: /var/lib/kubelet/csi-provisioner/nasplugin.csi.alibabacloud.com/csi.sock + volumeMounts: + - name: nas-provisioner-dir + mountPath: /var/lib/kubelet/csi-provisioner/nasplugin.csi.alibabacloud.com + - name: external-nas-resizer + image: registry-cn-hangzhou.ack.aliyuncs.com/acs/csi-resizer:v1.3-ca84e84-aliyun + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 500m + memory: 8Gi + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--leader-election" + env: + - name: ADDRESS + value: /var/lib/kubelet/csi-provisioner/nasplugin.csi.alibabacloud.com/csi.sock + volumeMounts: + - name: nas-provisioner-dir + mountPath: /var/lib/kubelet/csi-provisioner/nasplugin.csi.alibabacloud.com + - name: external-oss-provisioner + args: + - --csi-address=$(ADDRESS) + - --volume-name-prefix=oss + - --timeout=150s + - --leader-election=true + - --retry-interval-start=500ms + - --default-fstype=ossfs + - --v=5 + env: + - name: ADDRESS + value: /var/lib/kubelet/csi-provisioner/ossplugin.csi.alibabacloud.com/csi.sock + image: registry-cn-hangzhou.ack.aliyuncs.com/acs/csi-provisioner:v3.0.0-080f01e64-aliyun + resources: + limits: + cpu: 500m + memory: 1Gi + requests: + cpu: 10m + memory: 16Mi + volumeMounts: + - mountPath: /var/lib/kubelet/csi-provisioner/ossplugin.csi.alibabacloud.com + name: oss-provisioner-dir + - name: csi-provisioner + securityContext: + privileged: true + image: registry-cn-hangzhou.ack.aliyuncs.com/acs/csi-plugin:v1.24.9-74f8490-aliyun + args: + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=2" + - "--nodeid=$(KUBE_NODE_NAME)" + - "--driver=nas,oss" + env: + - name: CSI_ENDPOINT + value: unix://var/lib/kubelet/csi-provisioner/driverplugin.csi.alibabacloud.com-replace/csi.sock + - name: MAX_VOLUMES_PERNODE + value: "15" + - name: SERVICE_TYPE + value: "provisioner" + - name: "CLUSTER_ID" + value: "{{.ClusterID}}" + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + livenessProbe: + httpGet: + path: /healthz + port: healthz + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 5 + periodSeconds: 20 + ports: + - name: healthz + containerPort: 11270 + volumeMounts: + - name: host-log + mountPath: /var/log/ + - name: nas-provisioner-dir + mountPath: /var/lib/kubelet/csi-provisioner/nasplugin.csi.alibabacloud.com + - name: oss-provisioner-dir + mountPath: /var/lib/kubelet/csi-provisioner/ossplugin.csi.alibabacloud.com + - mountPath: /var/addon + name: addon-token + readOnly: true + - mountPath: /mnt + mountPropagation: Bidirectional + name: host-dev + - mountPath: /host/etc + name: etc + resources: + limits: + cpu: 500m + memory: 1024Mi + requests: + cpu: 100m + memory: 128Mi + volumes: + - name: nas-provisioner-dir + emptyDir: {} + - name: oss-provisioner-dir + emptyDir: {} + - name: host-log + hostPath: + path: /var/log/ + - name: etc + hostPath: + path: /etc + type: "" + - name: host-dev + hostPath: + path: /mnt + type: "" + - name: addon-token + secret: + defaultMode: 420 + optional: true + items: + - key: addon.token.config + path: token-config + secretName: addon.csi.token \ No newline at end of file diff --git a/deploy/nonecs/rbac.yaml b/deploy/nonecs/rbac.yaml new file mode 100644 index 000000000..93b77bfe5 --- /dev/null +++ b/deploy/nonecs/rbac.yaml @@ -0,0 +1,125 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-admin + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: kube-system + name: alicloud-csi-plugin +rules: +- apiGroups: [""] + resources: ["secrets"] + resourceNames: ["csi-local-plugin-cert"] + verbs: ["get"] +- apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] +- apiGroups: [""] + resources: ["endpoints"] + resourceNames: ["cnfs-cache-ds-service"] + verbs: ["get"] +- apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["csi-plugin", "ack-cluster-profile"] + verbs: ["get"] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: alicloud-csi-plugin +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "create", "delete", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "update", "list"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch", "create"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update", "create"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "create", "list", "watch", "delete", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list"] + - apiGroups: ["storage.alibabacloud.com"] + resources: ["rules"] + verbs: ["get"] + - apiGroups: ["storage.alibabacloud.com"] + resources: ["containernetworkfilesystems"] + verbs: ["get","list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: alicloud-csi-plugin + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: alicloud-csi-plugin +subjects: +- kind: ServiceAccount + name: csi-admin + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: alicloud-csi-plugin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: alicloud-csi-plugin +subjects: +- kind: ServiceAccount + name: csi-admin + namespace: kube-system +--- \ No newline at end of file diff --git a/pkg/disk/disk.go b/pkg/disk/disk.go index 455360e8b..5fcaac403 100644 --- a/pkg/disk/disk.go +++ b/pkg/disk/disk.go @@ -345,10 +345,20 @@ func GlobalConfigSet(nodeID string) *restclient.Config { log.Log.Infof("Describe node %s and Set RunTimeClass to %s", nodeName, runtimeValue) regionID, zoneID, vmID = getMeta(nodeInfo) + log.Log.Infof("NewNodeServer: get instance meta info from metadataserver, regionID: %s, zoneID: %s, vmID: %s", regionID, zoneID, vmID) + if nodeID == "" { nodeID = vmID } } + if zoneID == "" || !strings.HasPrefix(vmID, "i-") { + doc, err := retryGetInstanceDoc() + log.Log.Infof("NewNodeServer: get instance meta info failed from metadataserver, err: %v, doc: %v", err, doc) + if err == nil { + zoneID = doc.ZoneID + nodeID = doc.InstanceID + } + } runtimeEnv := os.Getenv("RUNTIME") if runtimeEnv == MixRunTimeMode { runtimeValue = MixRunTimeMode diff --git a/pkg/disk/nodeserver.go b/pkg/disk/nodeserver.go index 8e1b4a7d4..3a2ca2e51 100644 --- a/pkg/disk/nodeserver.go +++ b/pkg/disk/nodeserver.go @@ -157,13 +157,7 @@ func NewNodeServer(d *csicommon.CSIDriver, c *ecs.Client) csi.NodeServer { zoneID, nodeID = getZoneID(c, nodeID) } else { if zoneID == "" || nodeID == "" { - doc, err := retryGetInstanceDoc() - if err != nil { - log.Log.Infof("NewNodeServer: get instance meta info failed from metadataserver, err: %v", err) - zoneID, nodeID = getZoneID(c, nodeID) - } - zoneID = doc.ZoneID - nodeID = doc.InstanceID + zoneID, nodeID = getZoneID(c, nodeID) } } log.Log.Infof("NewNodeServer: zone id: %+v, GlobalConfigVar.zoneID: %s", zoneID, GlobalConfigVar.ZoneID) @@ -743,6 +737,7 @@ func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol } } } + return &csi.NodeStageVolumeResponse{}, nil }