issue #354: fix issue with pushing empty values to ydb tables + check…

… this in integration test (#355) * issue #354: fix issue with pushing empty values to ydb tables + check this in integration test * cleanup
ydb-platform · Feb 8, 2024 · 2659e8f · 2659e8f
1 parent d68a1db
commit 2659e8f
Show file tree

Hide file tree

Showing 2 changed files with 50 additions and 12 deletions.
diff --git a/cloud/blockstore/libs/ydbstats/ydbstats.cpp b/cloud/blockstore/libs/ydbstats/ydbstats.cpp
@@ -436,27 +436,36 @@ TFuture<NProto::TError> TYdbStatsUploader::DoUploadStats(
 
     TVector<TUploadData> dataForUpload;
 
-    TValueBuilder rows;
-    rows.BeginList();
-    for (const auto& stat: stats) {
-        rows.AddListItem(stat.GetYdbValues());
-    }
-    rows.EndList();
+    auto buildYdbRows = [&] () {
+        TValueBuilder rows;
 
-    auto result = rows.Build();
+        rows.BeginList();
+        for (const auto& stat: stats) {
+            rows.AddListItem(stat.GetYdbValues());
+        }
+        rows.EndList();
+
+        return rows.Build();
+    };
 
     if (setupResult.StatsTableName) {
-        dataForUpload.emplace_back(setupResult.StatsTableName, result);
+        // TODO: avoid explicitly making copies
+        // of stats after https://github.com/ydb-platform/ydb/issues/1659
+        dataForUpload.emplace_back(setupResult.StatsTableName, buildYdbRows());
     }
 
     if (setupResult.HistoryTableName) {
-        dataForUpload.emplace_back(setupResult.HistoryTableName, result);
+        // TODO: avoid explicitly making copies
+        // of stats after https://github.com/ydb-platform/ydb/issues/1659
+        dataForUpload.emplace_back(setupResult.HistoryTableName, buildYdbRows());
     }
 
     if (setupResult.ArchiveStatsTableName) {
+        // TODO: avoid explicitly making copies
+        // of stats after https://github.com/ydb-platform/ydb/issues/1659
         dataForUpload.emplace_back(
             setupResult.ArchiveStatsTableName,
-            std::move(result));
+            buildYdbRows());
     }
 
     if (setupResult.BlobLoadMetricsTableName) {

diff --git a/cloud/blockstore/tests/loadtest/local-metrics/test.py b/cloud/blockstore/tests/loadtest/local-metrics/test.py
@@ -22,11 +22,17 @@
 
 from subprocess import call, check_output
 
+from datetime import date
+
 STORAGE_POOL = [
     dict(name="dynamic_storage_pool:1", kind="rot", pdisk_user_kind=0),
     dict(name="dynamic_storage_pool:2", kind="ssd", pdisk_user_kind=0),
 ]
 
+StatsTableName = "cloud-local-perfstats"
+HistoryTablePrefix = "cloud-local-perfstats"
+ArchiveStatsTableName = "cloud-local-archive-perfstats"
+
 
 def kikimr_start():
     kikimr_binary_path = common.binary_path("ydb/apps/ydbd/ydbd")
@@ -61,8 +67,9 @@ def nbs_server_start(kikimr_cluster, configurator):
     # in current realization StatsTableName and HistoryTablePrefix are necessary
     ydbstat = TYdbStatsConfig()
     ydbstat.BlobLoadMetricsTableName = "cloud-local-metrics"
-    ydbstat.StatsTableName = "cloud-local-perfstats"
-    ydbstat.HistoryTablePrefix = "cloud-local-perfstats"
+    ydbstat.StatsTableName = StatsTableName
+    ydbstat.HistoryTablePrefix = HistoryTablePrefix
+    ydbstat.ArchiveStatsTableName = ArchiveStatsTableName
     ydbstat.DatabaseName = "/Root"
     ydbstat.ServerAddress = "localhost:" + str(kikimr_port)
     ydbstat.HistoryTableLifetimeDays = 2
@@ -117,6 +124,26 @@ def get_load_data(kikimr_port):
     return read_data_size, read_data_iops, write_data_size, write_data_iops
 
 
+def check_ydb_volume_metrics(kikimr_port):
+    ydb_binary_path = common.binary_path("contrib/ydb/apps/ydb/ydb")
+
+    dailyTable = "{}-{}".format(HistoryTablePrefix, date.today())
+
+    for table in [StatsTableName, ArchiveStatsTableName, dailyTable]:
+        json_string = check_output([
+            ydb_binary_path,
+            "--endpoint", "grpc://localhost:" + str(kikimr_port),
+            "--database", "/Root",
+            "yql",
+            "--format", "json-unicode-array",
+            "--script", "SELECT * FROM `/Root/{}`;".format(table),
+        ])
+
+        json_data = json.loads(json_string)
+        assert len(json_data) != 0
+        assert json_data[0]["DiskId"] == "vol0"
+
+
 def test_metrics():
     kikimr_cluster, configurator = kikimr_start()
     nbs = nbs_server_start(kikimr_cluster, configurator)
@@ -192,6 +219,8 @@ def test_metrics():
     read_data_size, read_data_iops, \
         write_data_size, write_data_iops = get_load_data(kikimr_port)
 
+    check_ydb_volume_metrics(kikimr_port)
+
     nbs.stop()
 
     assert read_data_size >= raw_data_size