From b675587a91349f03a7be8b13015e2dcbe1dd4298 Mon Sep 17 00:00:00 2001
From: Aleksandr Rigachnyi <a.rigachnyy@gmail.com>
Date: Thu, 4 Apr 2024 12:54:37 +0200
Subject: [PATCH] NBSNEBIUS-85: add manifests for nbs; support volume attach
 via nbd (#875)

---
 cloud/blockstore/tools/csi_driver/README.md   |  6 +-
 .../deploy/manifests/5-nbs-configmap.yaml     | 45 +++++++++++
 .../deploy/manifests/6-nbs-daemonset.yaml     | 78 +++++++++++++++++++
 ...-deployment.yaml => 7-csi-deployment.yaml} |  0
 ...{6-daemonset.yaml => 8-csi-daemonset.yaml} |  6 ++
 5 files changed, 133 insertions(+), 2 deletions(-)
 create mode 100644 cloud/blockstore/tools/csi_driver/deploy/manifests/5-nbs-configmap.yaml
 create mode 100644 cloud/blockstore/tools/csi_driver/deploy/manifests/6-nbs-daemonset.yaml
 rename cloud/blockstore/tools/csi_driver/deploy/manifests/{5-deployment.yaml => 7-csi-deployment.yaml} (100%)
 rename cloud/blockstore/tools/csi_driver/deploy/manifests/{6-daemonset.yaml => 8-csi-daemonset.yaml} (95%)

diff --git a/cloud/blockstore/tools/csi_driver/README.md b/cloud/blockstore/tools/csi_driver/README.md
index bd6883bb1d2..091ac668af6 100644
--- a/cloud/blockstore/tools/csi_driver/README.md
+++ b/cloud/blockstore/tools/csi_driver/README.md
@@ -67,8 +67,10 @@ kubectl apply -f ./deploy/manifests/2-rbac.yaml
 kubectl apply -f ./deploy/manifests/3-csidriver.yaml
 kubectl apply -f ./deploy/manifests/4-storageclass.yaml
 # TODO: kubectl secret nbs-puller-secret
-kubectl apply -f ./deploy/manifests/5-deployment.yaml
-kubectl apply -f ./deploy/manifests/6-daemonset.yaml
+kubectl apply -f ./deploy/manifests/5-nbs-configmap.yaml
+kubectl apply -f ./deploy/manifests/6-nbs-daemonset.yaml
+kubectl apply -f ./deploy/manifests/7-csi-deployment.yaml
+kubectl apply -f ./deploy/manifests/8-csi-daemonset.yaml
 ```
 
 # copy nbsd and blockstore-nbd to minikube and run them
diff --git a/cloud/blockstore/tools/csi_driver/deploy/manifests/5-nbs-configmap.yaml b/cloud/blockstore/tools/csi_driver/deploy/manifests/5-nbs-configmap.yaml
new file mode 100644
index 00000000000..38c28a7dd08
--- /dev/null
+++ b/cloud/blockstore/tools/csi_driver/deploy/manifests/5-nbs-configmap.yaml
@@ -0,0 +1,45 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nbs-server-config-node
+  namespace: nbs-csi-ns
+data:
+  nbs-server.txt: |-
+    ServerConfig {
+      Host: "localhost"
+      Port: 9766
+      DataHost: "localhost"
+      DataPort: 9767
+      StrictContractValidation: true
+      UnixSocketPath: "/run/nbsd/grpc.sock"
+      ThreadsCount: 4
+      VhostEnabled: true
+      VhostThreadsCount: 4
+      NodeType: "nbs"
+      EndpointStorageType: ENDPOINT_STORAGE_FILE
+      EndpointStorageDir: "/Berkanavt/nbs-server/endpoints"
+      NbdEnabled: true
+      NbdThreadsCount: 4
+      GrpcKeepAliveTime: 10000
+      GrpcKeepAliveTimeout: 1000
+      GrpcKeepAlivePermitWithoutCalls: true
+      GrpcHttp2MinRecvPingIntervalWithoutData: 1000
+      NbdDevicePrefix: "/mnt/dev/nbd"
+    }
+    LocalServiceConfig {
+      DataDir: "/Berkanavt/nbs-server/data"
+    }
+
+
+
+  nbs_server.cfg: |-
+    nbs_home="${nbs_home:-"/Berkanavt/nbs-server"}"
+    nbs_config="${nbs_home}/cfg"
+    nbs_server_file="${nbs_config}/nbs-server.txt"
+
+    nbs_arg="${nbs_arg}${nbs_server_file:+ --server-file ${nbs_server_file}}"
+
+    nbs_arg="${nbs_arg} --service=local"
+    nbs_arg="${nbs_arg} --mon-port=8766"
+    nbs_arg="${nbs_arg} --verbose"
diff --git a/cloud/blockstore/tools/csi_driver/deploy/manifests/6-nbs-daemonset.yaml b/cloud/blockstore/tools/csi_driver/deploy/manifests/6-nbs-daemonset.yaml
new file mode 100644
index 00000000000..4012263e559
--- /dev/null
+++ b/cloud/blockstore/tools/csi_driver/deploy/manifests/6-nbs-daemonset.yaml
@@ -0,0 +1,78 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: nbs-server-node
+  namespace: nbs-csi-ns
+spec:
+  selector:
+    matchLabels:
+      app: "nbs-server-app"
+  template:
+    metadata:
+      labels:
+        app: "nbs-server-app"
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/os: linux
+      initContainers:
+        - name: modprobe-nbd
+          image: cr.ai.nebius.cloud/mirror/io/docker/library/busybox:1.32.1
+          imagePullPolicy: IfNotPresent
+          command: ["modprobe", "nbd", "nbds_max=1024"]
+          securityContext:
+            privileged: true
+          resources:
+            requests:
+              cpu: "100m"
+              memory: "16Mi"
+          volumeMounts:
+            - name: lib-modules
+              mountPath: /lib/modules
+              readOnly: false
+      containers:
+        - name: nbs-server
+          securityContext:
+            privileged: true
+          image: cr.ai.nebius.cloud/crnvrq2hg2rgmbj3ebr9/nebius-blockstore-server:85.stable-23-3-21
+          imagePullPolicy: Always
+          volumeMounts:
+            - name: nbs-server
+              mountPath: /Berkanavt/nbs-server
+              readOnly: false
+            - name: nbs-configs
+              mountPath: /Berkanavt/nbs-server/cfg
+              readOnly: true
+            - name: nbs-run
+              mountPath: /run/nbsd
+              readOnly: false
+            - name: dev
+              mountPath: /mnt/dev
+              readOnly: false
+      volumes:
+        - name: nbs-server-backups
+          hostPath:
+            path: /Berkanavt/nbs-server/backups
+            type: DirectoryOrCreate
+        - name: nbs-server-endpoints
+          hostPath:
+            path: /Berkanavt/nbs-server/endpoints
+            type: DirectoryOrCreate
+        - name: nbs-configs
+          configMap:
+            name: nbs-server-config-node
+        - name: nbs-run
+          hostPath:
+            path: /run/nbsd
+            type: DirectoryOrCreate
+        - name: dev
+          hostPath:
+            path: /dev
+            type: DirectoryOrCreate
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules
+            type: DirectoryOrCreate
+      imagePullSecrets:
+        - name: nbs-puller-secret
\ No newline at end of file
diff --git a/cloud/blockstore/tools/csi_driver/deploy/manifests/5-deployment.yaml b/cloud/blockstore/tools/csi_driver/deploy/manifests/7-csi-deployment.yaml
similarity index 100%
rename from cloud/blockstore/tools/csi_driver/deploy/manifests/5-deployment.yaml
rename to cloud/blockstore/tools/csi_driver/deploy/manifests/7-csi-deployment.yaml
diff --git a/cloud/blockstore/tools/csi_driver/deploy/manifests/6-daemonset.yaml b/cloud/blockstore/tools/csi_driver/deploy/manifests/8-csi-daemonset.yaml
similarity index 95%
rename from cloud/blockstore/tools/csi_driver/deploy/manifests/6-daemonset.yaml
rename to cloud/blockstore/tools/csi_driver/deploy/manifests/8-csi-daemonset.yaml
index fb9f9936e2e..49e9eb30f42 100644
--- a/cloud/blockstore/tools/csi_driver/deploy/manifests/6-daemonset.yaml
+++ b/cloud/blockstore/tools/csi_driver/deploy/manifests/8-csi-daemonset.yaml
@@ -78,6 +78,8 @@ spec:
               mountPath: /csi
             - name: sockets-dir
               mountPath: /nbsd-sockets
+            - name: dev
+              mountPath: /mnt/dev
       volumes:
         - name: registration-dir
           hostPath:
@@ -95,5 +97,9 @@ spec:
           hostPath:
             path: /run/nbsd/sockets
             type: DirectoryOrCreate
+        - name: dev
+          hostPath:
+            path: /dev
+            type: DirectoryOrCreate
       imagePullSecrets:
         - name: nbs-puller-secret