Kozev stable sync (#435)

* NBS-4808 make build and easy

* Add actors tests to build

* NBS-4876 fix compilation in arcadia (#426)

* NBS-4876 fix compilation in arcadia

* Fix compilation

* Remove spaces

* CLOUDDUTY-25033

---------

Co-authored-by: Kirill Pleshivtsev <[email protected]>

cloud/blockstore/buildall/.gitignore

@@ -0,0 +1,2 @@
+cloud
+contrib

cloud/blockstore/buildall/ya.make

@@ -0,0 +1,13 @@
+OWNER(g:cloud-nbs)
+
+PACKAGE()
+
+PEERDIR(
+    contrib/ydb/apps/ydbd
+    cloud/blockstore/apps/client
+    cloud/blockstore/apps/disk_agent
+    cloud/blockstore/apps/server
+    cloud/blockstore/tools/nbd
+)
+
+END()

cloud/blockstore/libs/storage/partition/part_actor_compaction.cpp

@@ -310,10 +310,8 @@ void TCompactionActor::InitBlockDigests()
         }
 
         if (rc.ZeroBlobId) {
-            ui32 skipped = 0;
             for (const auto blockIndex: xrange(rc.BlockRange)) {
                 if (rc.ZeroBlobSkipMask.Get(blockIndex - rc.BlockRange.Start)) {
-                    ++skipped;
                     continue;
                 }
 

cloud/blockstore/libs/storage/partition/part_state_ut.cpp

@@ -598,7 +598,6 @@ Y_UNIT_TEST_SUITE(TPartitionStateTest)
             UNIT_ASSERT_VALUES_EQUAL(1, state.GetAlmostFullChannelCount());
 
             ui32 firstChannelSelected = 0;
-            ui32 secondChannelSelected = 0;
             for (ui32 i = 0; i < 1000; ++i) {
                 auto blobId = state.GenerateBlobId(
                     kind,
@@ -613,7 +612,6 @@ Y_UNIT_TEST_SUITE(TPartitionStateTest)
                         ui32(TPartitionSchema::FirstDataChannel + 1),
                         blobId.Channel()
                     );
-                    ++secondChannelSelected;
                 }
             }
 
@@ -630,7 +628,6 @@ Y_UNIT_TEST_SUITE(TPartitionStateTest)
             UNIT_ASSERT_VALUES_EQUAL(2, state.GetAlmostFullChannelCount());
 
             firstChannelSelected = 0;
-            secondChannelSelected = 0;
             for (ui32 i = 0; i < 1000; ++i) {
                 auto blobId = state.GenerateBlobId(
                     kind,
@@ -645,7 +642,6 @@ Y_UNIT_TEST_SUITE(TPartitionStateTest)
                         ui32(TPartitionSchema::FirstDataChannel + 1),
                         blobId.Channel()
                     );
-                    ++secondChannelSelected;
                 }
             }
 

cloud/blockstore/libs/storage/partition2/part2_state_ut.cpp

@@ -1264,7 +1264,6 @@ Y_UNIT_TEST_SUITE(TPartition2StateTest)
             UNIT_ASSERT_VALUES_EQUAL(1, state.GetAlmostFullChannelCount());
 
             ui32 firstChannelSelected = 0;
-            ui32 secondChannelSelected = 0;
             for (ui32 i = 0; i < 1000; ++i) {
                 auto blobId = state.GenerateBlobId(
                     kind,
@@ -1279,7 +1278,6 @@ Y_UNIT_TEST_SUITE(TPartition2StateTest)
                         ui32(TPartitionSchema::FirstDataChannel + 1),
                         blobId.Channel()
                     );
-                    ++secondChannelSelected;
                 }
             }
 
@@ -1296,7 +1294,6 @@ Y_UNIT_TEST_SUITE(TPartition2StateTest)
             UNIT_ASSERT_VALUES_EQUAL(2, state.GetAlmostFullChannelCount());
 
             firstChannelSelected = 0;
-            secondChannelSelected = 0;
             for (ui32 i = 0; i < 1000; ++i) {
                 auto blobId = state.GenerateBlobId(
                     kind,
@@ -1311,7 +1308,6 @@ Y_UNIT_TEST_SUITE(TPartition2StateTest)
                         ui32(TPartitionSchema::FirstDataChannel + 1),
                         blobId.Channel()
                     );
-                    ++secondChannelSelected;
                 }
             }
 

cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_migration_actor.cpp

@@ -203,7 +203,7 @@ void TNonreplicatedPartitionMigrationActor::HandleMigrationStateUpdated(
     const TActorContext& ctx)
 {
     Y_UNUSED(ev);
-    
+
     UpdatingMigrationState = false;
     if (MigrationFinished) {
         FinishMigration(ctx, false);

cloud/blockstore/libs/storage/volume/model/checkpoint_ut.cpp

@@ -292,7 +292,7 @@ Y_UNIT_TEST_SUITE(TCheckpointStore)
             EShadowDiskState::Preparing,
             512,
             1024);
-        
+
         checkpoint = store.GetCheckpoint(checkpointId);
 
         UNIT_ASSERT_VALUES_EQUAL(

cloud/blockstore/tools/testing/eternal_tests/range-validator/main.cpp

@@ -2,8 +2,9 @@
 
 #include <library/cpp/getopt/small/last_getopt.h>
 
-#include <util/generic/vector.h>
+#include <util/datetime/base.h>
 #include <util/generic/size_literals.h>
+#include <util/generic/vector.h>
 
 using namespace NCloud::NBlockStore;
 
@@ -75,17 +76,17 @@ int main(int argc, const char** argv)
     for (ui32 rangeIdx: options.Ranges) {
         Cout << "Start validation of " << rangeIdx << " range" << Endl;
 
-        const auto& config = configHolder->GetConfig().Ranges()[rangeIdx];
+        const auto& config = configHolder->GetConfig().GetRanges()[rangeIdx];
 
-        ui64 len = config.RequestCount();
-        ui64 startOffset = config.StartOffset();
-        ui64 requestSize = config.RequestBlockCount() * 4_KB;
+        ui64 len = config.GetRequestCount();
+        ui64 startOffset = config.GetStartOffset();
+        ui64 requestSize = config.GetRequestBlockCount() * 4_KB;
 
         TVector<ui64> expected(len);
 
-        ui64 step = config.Step();
-        ui64 curBlockIdx = config.LastBlockIdx();
-        ui64 curNum = config.NumberToWrite();
+        ui64 step = config.GetStep();
+        ui64 curBlockIdx = config.GetLastBlockIdx();
+        ui64 curNum = config.GetNumberToWrite();
         ui64 cnt = 0;
         while (cnt < len && curNum != 0) {
             curBlockIdx = (curBlockIdx + len - step) % len;

cloud/storage/core/libs/actors/ya.make

@@ -10,3 +10,5 @@ PEERDIR(
 )
 
 END()
+
+RECURSE_FOR_TESTS(ut)

example/.gitignore

@@ -0,0 +1,9 @@
+certs
+data
+logs
+
+blockstore-client
+blockstore-nbd
+diskagentd
+nbsd
+ydbd

example/0-setup.sh

@@ -9,14 +9,33 @@ find_bin_dir() {
     readlink -e `dirname $0`
 }
 
+find_blockstore_all_dir() {
+    readlink -e `dirname $0`/../cloud/blockstore/buildall
+}
+
 BIN_DIR=`find_bin_dir`
+BLOCKSTORE_ALL_DIR=`find_blockstore_all_dir`
 PERSISTENT_TMP_DIR=${PERSISTENT_TMP_DIR:-$HOME/tmp/nbs}
 
+BUILD_FILES=" \
+    contrib/ydb/apps/ydbd/ydbd \
+    cloud/blockstore/apps/server/nbsd \
+    cloud/blockstore/apps/disk_agent/diskagentd \
+    cloud/blockstore/apps/client/blockstore-client \
+    cloud/blockstore/tools/nbd/blockstore-nbd \
+    "
+
+# create symlinks
+for file in $BUILD_FILES; do
+    ln -svf $BLOCKSTORE_ALL_DIR/$file $BIN_DIR/
+done
+
 for dir in $DATA_DIRS; do
     mkdir -p "$PERSISTENT_TMP_DIR/$dir"
     ln -svfT "$PERSISTENT_TMP_DIR/$dir" "$BIN_DIR/$dir"
 done
 
+# check symlinks
 for bin in ydbd nbsd blockstore-nbd blockstore-client diskagentd
 do
   if ! test -f "$BIN_DIR/$bin"; then

example/1-start_storage.sh

@@ -4,7 +4,7 @@ NODE=${NODE:-1}
 GRPC_PORT=${GRPC_PORT:-9001}
 MON_PORT=${MON_PORT:-8765}
 YDBD="./ydbd"
-
+export LD_LIBRARY_PATH=$(dirname $(readlink ydbd))
 $YDBD server \
     --tcp \
     --node              $NODE \

example/2-init_storage.sh

@@ -2,6 +2,7 @@
 
 DATA_DIR="data"
 YDBD="./ydbd"
+export LD_LIBRARY_PATH=$(dirname $(readlink ydbd))
 
 echo "DefineBox"
 $YDBD -s grpc://localhost:9001 admin bs config invoke --proto-file dynamic/DefineBox.txt

example/3-start_nbs.sh

@@ -7,6 +7,7 @@ DATA_SERVER_PORT=${DATA_SERVER_PORT:-9767}
 SECURE_SERVER_PORT=${SECURE_SERVER_PORT:-9768}
 MON_PORT=${MON_PORT:-8766}
 NBSD="./nbsd"
+export LD_LIBRARY_PATH=$(dirname $(readlink nbsd))
 
 $NBSD \
     --domain             Root \

example/4-start_nbs_storage.sh

@@ -12,6 +12,7 @@ find_bin_dir() {
 BIN_DIR=`find_bin_dir`
 CLIENT="blockstore-client"
 DAGENT="diskagentd"
+export LD_LIBRARY_PATH=$(dirname $(readlink $BIN_DIR/diskagentd))
 
 $BIN_DIR/$CLIENT ExecuteAction --action DiskRegistrySetWritableState --verbose error --input-bytes '{"State":true}'
 if [ $? -ne 0 ]; then

example/5-create_and_attach_disk.sh

@@ -72,6 +72,7 @@ fi
 # create disk
 echo "Creating disk $id in $kind mode"
 CLIENT="./blockstore-client"
+export LD_LIBRARY_PATH=$(dirname $(readlink $BIN_DIR/$CLIENT))
 $BIN_DIR/$CLIENT createvolume \
     --storage-media-kind $kind --blocks-count $blocks_count --disk-id $id
 
@@ -89,8 +90,11 @@ SOCK="$BIN_DIR/$id.sock"
 
 sudo modprobe nbd
 touch $SOCK
-sudo $BIN_DIR/$NBD --device-mode endpoint --disk-id $id --access-mode rw \
-    --mount-mode local --connect-device $device --listen-path $SOCK
+cmd="LD_LIBRARY_PATH=$(dirname $(readlink $BIN_DIR/$NBD)) \
+    $BIN_DIR/${NBD} --device-mode endpoint --disk-id $id --access-mode rw \
+    --mount-mode local --connect-device $device --listen-path $SOCK"
+
+sudo $cmd
 
 if [ $? -ne 0 ]; then
     echo "Attaching disk $id to $device failed"

example/6-run_qemu.sh

@@ -54,12 +54,12 @@ do
     esac
 done
 
-if [ -z "$diskid" ] then
+if [ -z "$diskid" ] ; then
     echo "Disk id shouldn't be empty"
     exit 1
 fi
 
-if [ -z "$socket" ] then
+if [ -z "$socket" ] ; then
     echo "Socket path shouldn't be empty"
     exit 1
 fi

example/BUILD.md

@@ -9,19 +9,6 @@ nbsd, diskagentd, blockstore-client amd blockstore-nbd can be built for Ubuntu 1
 
 ## Prerequisites
 
-Below is a list of packages that need to be installed before building NBS. [How to Build](#how-to-build) section contains step by step instructions to obtain these packages.
-
- - cmake 3.22+
- - clang-14
- - lld-14
- - git 2.20+
- - python3.8
- - pip3
- - antlr3
- - libaio-dev
- - libidn11-dev
- - ninja 1.10+
-
 We run multiple clang instances in parallel to speed up the process by default. Each instance of clang may use up to 1GB of RAM, and linking the binary may use up to 16GB of RAM, please make sure your build host has enough resources.
 
 ## Runtime Requirements
@@ -32,14 +19,6 @@ We run multiple clang instances in parallel to speed up the process by default.
 
 # How to Build
 
-## Install dependencies
-
-```bash
-sudo apt-get -y install git cmake python3-pip ninja-build antlr3 m4 clang-14 lld-14 libidn11-dev libaio1 libaio-dev llvm-14
-sudo pip3 install conan==1.59 grpcio-tools==1.57.0
-
-```
-
 ## Create the work directory.
 > :warning: Please make sure you have at least 80Gb of free space. We also recommend placing this directory on SSD to reduce build times.
 
@@ -63,6 +42,12 @@ ninja -j 8
 
 ## Build - ya make
 
+### Build all
+To build all binaries at once in release mode, run:
+```bash
+./ya make cloud/blockstore/buildall -r
+```
+
 ### Build nbsd
 
 To build nbsd run:

example/README.md

@@ -1,9 +1,10 @@
 # Setup for local debugging
 
 ## Build binaries from sources
-1. Build ydbd using the instructions from https://github.com/ydb-platform/ydb/blob/main/BUILD.md
-2. Build nbsd, blockstore-client, diskagentd, blockstore-nbd using the instructions from https://github.com/ydb-platform/nbs/blob/main/example/BUILD.md
-3. Copy all those tools to current directory
+1. Build nbsd, blockstore-client, diskagentd, blockstore-nbd
+```bash
+ya make cloud/blockstore/buildall -r
+```
 
 ## Prepare current directory
 ```bash

example/nbs/.gitignore

@@ -0,0 +1,3 @@
+nbs-disk-agent-*.txt
+nbs-disk-registry.txt
+nbs-location-*.txt