From 4d26aac221fcd1f6c7fbc8fa52398de9b30f89be Mon Sep 17 00:00:00 2001 From: Boris Glimcher Date: Wed, 31 May 2023 02:28:20 +0300 Subject: [PATCH] ci: add support for TLS connection Signed-off-by: Boris Glimcher --- docker-compose.yml | 10 +++++++--- go.mod | 2 +- go.sum | 2 ++ pkg/backend/nvme.go | 1 + pkg/frontend/nvme.go | 3 ++- scripts/tests.sh | 4 ++++ 6 files changed, 17 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 453459bc..d99ebeaa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,7 @@ services: ports: - "9009:9009" - "4444:4444" + - "5555:5555" privileged: true networks: - opi @@ -25,6 +26,8 @@ services: grep hugetlbfs /proc/mounts || mount -t hugetlbfs nodev /mnt/huge && \ echo 1024 > /proc/sys/vm/nr_hugepages && \ grep "" /sys/kernel/mm/hugepages/hugepages-*/nr_hugepages && \ + echo -n NVMeTLSkey-1:01:MDAxMTIyMzM0NDU1NjY3Nzg4OTlhYWJiY2NkZGVlZmZwJEiQ: > /tmp/opikey.txt && \ + chmod 0600 /tmp/opikey.txt && \ dd if=/dev/zero of=/tmp/aio_bdev_file bs=512 count=64 && \ /usr/local/bin/spdk_tgt -m 0x1 -s 512 --no-pci -S /var/tmp |& tee /tmp/spdk.log & \ for i in `seq 1 10`; do ./rpc.py spdk_get_version && break || sleep 1; done && \ @@ -33,9 +36,10 @@ services: ./rpc.py nvmf_create_transport -t TCP -u 8192 -m 4 -c 0 && \ ./rpc.py nvmf_create_transport -t VFIOUSER && \ ./rpc.py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001 -d SPDK_Controller1 && \ - ./rpc.py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t tcp -a `hostname -i` -f ipv4 -s 4444 && \ ./rpc.py nvmf_subsystem_allow_any_host nqn.2016-06.io.spdk:cnode1 --disable && \ - ./rpc.py nvmf_subsystem_add_host nqn.2016-06.io.spdk:cnode1 nqn.2014-08.org.nvmexpress:uuid:feb98abe-d51f-40c8-b348-2753f3571d3c && \ + ./rpc.py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t tcp -a `hostname -i` -f ipv4 -s 4444 && \ + ./rpc.py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t tcp -a `hostname -i` -f ipv4 -s 5555 --secure-channel && \ + ./rpc.py nvmf_subsystem_add_host nqn.2016-06.io.spdk:cnode1 nqn.2014-08.org.nvmexpress:uuid:feb98abe-d51f-40c8-b348-2753f3571d3c --psk /tmp/opikey.txt && \ ./rpc_http_proxy.py 0.0.0.0 9009 spdkuser spdkpass' healthcheck: test: ["CMD-SHELL", "python3 /usr/libexec/spdk/scripts/rpc.py spdk_get_version || exit 1"] @@ -56,7 +60,7 @@ services: depends_on: spdk: condition: service_healthy - command: sh -c "/opi-spdk-bridge -port=50051 -spdk_addr=/var/tmp/spdk.sock -tcp_trid=$$(getent hosts spdk | awk '{ print $$1 }'):4444" + command: sh -c "echo -n 'NVMeTLSkey-1:01:MDAxMTIyMzM0NDU1NjY3Nzg4OTlhYWJiY2NkZGVlZmZwJEiQ:' > /tmp/opikey.txt && chmod 0600 /tmp/opikey.txt && /opi-spdk-bridge -port=50051 -spdk_addr=/var/tmp/spdk.sock -tcp_trid=$$(getent hosts spdk | awk '{ print $$1 }'):4444" healthcheck: test: grpcurl -plaintext localhost:50051 list || exit 1 diff --git a/go.mod b/go.mod index afa997a6..a8760b68 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.19 require ( github.com/digitalocean/go-qemu v0.0.0-20221209210016-f035778c97f7 github.com/google/uuid v1.3.0 - github.com/opiproject/gospdk v0.0.0-20230515120524-37c85998ff39 + github.com/opiproject/gospdk v0.0.0-20230601215713-d912b55f1d0a github.com/opiproject/opi-api v0.0.0-20230531200807-4aa6c73d421b google.golang.org/grpc v1.55.0 google.golang.org/protobuf v1.30.0 diff --git a/go.sum b/go.sum index d18c5d41..5475e879 100644 --- a/go.sum +++ b/go.sum @@ -13,6 +13,8 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/opiproject/gospdk v0.0.0-20230515120524-37c85998ff39 h1:PS5lPK4scy8AUY3UTa2rnCXRN/9ExHWldf1Gaqk0+7E= github.com/opiproject/gospdk v0.0.0-20230515120524-37c85998ff39/go.mod h1:Unf3idtaelGKnTbIbJ+oKsxrBOFdVdFlDnwUKKVSxPQ= +github.com/opiproject/gospdk v0.0.0-20230601215713-d912b55f1d0a h1:Bap/OuTiR/OqSWCyntyFmVKPPSbKTaulKD/QdPJ5ojs= +github.com/opiproject/gospdk v0.0.0-20230601215713-d912b55f1d0a/go.mod h1:Unf3idtaelGKnTbIbJ+oKsxrBOFdVdFlDnwUKKVSxPQ= github.com/opiproject/opi-api v0.0.0-20230531200807-4aa6c73d421b h1:+W/+F5eDs5ZKyg+2dCvu6KI7FEXFcptwMvuBAPuWLUY= github.com/opiproject/opi-api v0.0.0-20230531200807-4aa6c73d421b/go.mod h1:92pv4ulvvPMuxCJ9ND3aYbmBfEMLx0VCjpkiR7ZTqPY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/pkg/backend/nvme.go b/pkg/backend/nvme.go index c77a0f7d..a073c4bf 100644 --- a/pkg/backend/nvme.go +++ b/pkg/backend/nvme.go @@ -56,6 +56,7 @@ func (s *Server) CreateNVMfRemoteController(_ context.Context, in *pb.CreateNVMf Hostnqn: in.NvMfRemoteController.Hostnqn, Hdgst: in.NvMfRemoteController.Hdgst, Ddgst: in.NvMfRemoteController.Ddgst, + Psk: "/tmp/opikey.txt", } var result []spdk.BdevNvmeAttachControllerResult err := s.rpc.Call("bdev_nvme_attach_controller", ¶ms, &result) diff --git a/pkg/frontend/nvme.go b/pkg/frontend/nvme.go index fcec7679..6d5d41d3 100644 --- a/pkg/frontend/nvme.go +++ b/pkg/frontend/nvme.go @@ -89,6 +89,7 @@ func (c *tcpSubsystemListener) Params(_ *pb.NvmeController, nqn string) spdk.Nvm result.ListenAddress.Traddr = c.listenAddr.String() result.ListenAddress.Trsvcid = c.listenPort result.ListenAddress.Adrfam = c.protocol + result.SecureChannel = true return result } @@ -114,7 +115,7 @@ func (s *Server) CreateNvmeSubsystem(_ context.Context, in *pb.CreateNvmeSubsyst Nqn: in.NvmeSubsystem.Spec.Nqn, SerialNumber: in.NvmeSubsystem.Spec.SerialNumber, ModelNumber: in.NvmeSubsystem.Spec.ModelNumber, - AllowAnyHost: true, + AllowAnyHost: false, MaxNamespaces: int(in.NvmeSubsystem.Spec.MaxNamespaces), } var result spdk.NvmfCreateSubsystemResult diff --git a/scripts/tests.sh b/scripts/tests.sh index b06323d4..a7775057 100755 --- a/scripts/tests.sh +++ b/scripts/tests.sh @@ -51,9 +51,13 @@ grpc_cli=(docker run --network=opi-spdk-bridge_opi --rm docker.io/namely/grpc-cl "${grpc_cli[@]}" call --json_input --json_output opi-spdk-server:50051 GetNvmeSubsystem "{name : 'subsystem1'}" "${grpc_cli[@]}" call --json_input --json_output opi-spdk-server:50051 GetNvmeController "{name : 'controller1'}" "${grpc_cli[@]}" call --json_input --json_output opi-spdk-server:50051 GetNvmeNamespace "{name : 'namespace1'}" +echo -n NVMeTLSkey-1:01:MDAxMTIyMzM0NDU1NjY3Nzg4OTlhYWJiY2NkZGVlZmZwJEiQ: > /tmp/opikey.txt +chmod 0600 /tmp/opikey.txt docker run --rm --network=host --privileged -v /dev/hugepages:/dev/hugepages ghcr.io/opiproject/spdk:main spdk_nvme_identify -r 'traddr:127.0.0.1 trtype:TCP adrfam:IPv4 trsvcid:4444' docker run --rm --network=host --privileged -v /dev/hugepages:/dev/hugepages ghcr.io/opiproject/spdk:main spdk_nvme_perf -r 'traddr:127.0.0.1 trtype:TCP adrfam:IPv4 trsvcid:4444 subnqn:nqn.2022-09.io.spdk:opitest1 hostnqn:nqn.2014-08.org.nvmexpress:uuid:feb98abe-d51f-40c8-b348-2753f3571d3c' -c 0x1 -q 1 -o 4096 -w randread -t 10 | tee log.txt grep "Total" log.txt +docker run --rm --network=host --privileged -v /dev/hugepages:/dev/hugepages ghcr.io/opiproject/spdk:main spdk_nvme_perf -r 'traddr:127.0.0.1 trtype:TCP adrfam:IPv4 trsvcid:5555 subnqn:nqn.2022-09.io.spdk:opitest1 hostnqn:nqn.2014-08.org.nvmexpress:uuid:feb98abe-d51f-40c8-b348-2753f3571d3c' -c 0x1 -q 1 -o 4096 -w randread -t 10 --psk-path /tmp/opikey.txt | tee log.txt +grep "Total" log.txt "${grpc_cli[@]}" call --json_input --json_output opi-spdk-server:50051 DeleteNvmeNamespace "{name : 'namespace1'}" "${grpc_cli[@]}" call --json_input --json_output opi-spdk-server:50051 DeleteNvmeController "{name : 'controller1'}" "${grpc_cli[@]}" call --json_input --json_output opi-spdk-server:50051 DeleteNvmeSubsystem "{name : 'subsystem1'}"