Skip to content

Commit

Permalink
merge to stable-23-3 (#314)
Browse files Browse the repository at this point in the history 
* loadtest env tear_down should happen inside finally blocks because otherwise we may get an exception because blockstore-loadtest run failed and will never know that kikimr daemon crashed (#290)

* issue #95 NBSNEBIUS-70: if Flush/Compaction/Cleanup/FlushBytes ops are enqueued before CompactionMap is completely loaded, their EOperationStates should not get stuck in the Enqueued state (#303)
  • Loading branch information
@qkrorlqr
qkrorlqr authored Feb 1, 2024
1 parent 557c562 commit 3e3896f
Show file tree
Hide file tree
Showing 23 changed files with 350 additions and 186 deletions.
25 changes: 13 additions & 12 deletions cloud/blockstore/tests/loadtest/local-auth/test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,17 +43,18 @@ def test_load():
"cloud/blockstore/tests/certs/server.crt")
client.AuthToken = "test_auth_token"

ret = run_test(
"load",
common.source_path(
"cloud/blockstore/tests/loadtest/local-auth/local.txt"),
env.nbs_secure_port,
env.mon_port,
client_config=client,
enable_tls=True,
env_processes=[env.nbs],
)

env.tear_down()
try:
ret = run_test(
"load",
common.source_path(
"cloud/blockstore/tests/loadtest/local-auth/local.txt"),
env.nbs_secure_port,
env.mon_port,
client_config=client,
enable_tls=True,
env_processes=[env.nbs],
)
finally:
env.tear_down()

return ret
21 changes: 11 additions & 10 deletions cloud/blockstore/tests/loadtest/local-checkpoint/test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,16 +123,17 @@ def __run_test(test_case):
use_in_memory_pdisks=True,
)

ret = run_test(
test_case.name,
test_case.config_path,
env.nbs_port,
env.mon_port,
nbs_log_path=env.nbs_log_path,
env_processes=[env.nbs],
)

env.tear_down()
try:
ret = run_test(
test_case.name,
test_case.config_path,
env.nbs_port,
env.mon_port,
nbs_log_path=env.nbs_log_path,
env_processes=[env.nbs],
)
finally:
env.tear_down()

return ret

Expand Down
39 changes: 20 additions & 19 deletions cloud/blockstore/tests/loadtest/local-discovery/test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,27 +115,28 @@ def test_load():
use_in_memory_pdisks=True,
)

ret = run_test(
"load",
common.source_path(
"cloud/blockstore/tests/loadtest/local-discovery/local.txt"),
env.nbs_port,
env.mon_port,
env_processes=[env.nbs],
)

results_path = os.path.join(common.output_path(), "results.txt")
try:
ret = run_test(
"load",
common.source_path(
"cloud/blockstore/tests/loadtest/local-discovery/local.txt"),
env.nbs_port,
env.mon_port,
env_processes=[env.nbs],
)

def is_good(port):
return port in good_ports
results_path = os.path.join(common.output_path(), "results.txt")

with open(results_path, "r") as f:
for line in f:
data = json.loads(line)
ports = [x['Port'] for x in data['Instances']]
for port in ports:
assert is_good(port)
def is_good(port):
return port in good_ports

env.tear_down()
with open(results_path, "r") as f:
for line in f:
data = json.loads(line)
ports = [x['Port'] for x in data['Instances']]
for port in ports:
assert is_good(port)
finally:
env.tear_down()

return ret
19 changes: 10 additions & 9 deletions cloud/blockstore/tests/loadtest/local-edgecase/test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,15 +131,16 @@ def __run_test(test_case):
use_in_memory_pdisks=True,
)

ret = run_test(
test_case.name,
test_case.config_path,
env.nbs_port,
env.mon_port,
env_processes=[env.nbs],
)

env.tear_down()
try:
ret = run_test(
test_case.name,
test_case.config_path,
env.nbs_port,
env.mon_port,
env_processes=[env.nbs],
)
finally:
env.tear_down()

return ret

Expand Down
19 changes: 10 additions & 9 deletions cloud/blockstore/tests/loadtest/local-emergency/test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,15 +99,16 @@ def __run_test(test_case):
env.kikimr_cluster.restart_nodes()
env.nbs.restart()

ret = run_test(
"emergency-%s" % test_case.name,
common.source_path(test_case.config_path),
env.nbs_port,
env.mon_port,
env_processes=[env.nbs],
)

env.tear_down()
try:
ret = run_test(
"emergency-%s" % test_case.name,
common.source_path(test_case.config_path),
env.nbs_port,
env.mon_port,
env_processes=[env.nbs],
)
finally:
env.tear_down()

return ret

Expand Down
23 changes: 12 additions & 11 deletions cloud/blockstore/tests/loadtest/local-encryption/test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,17 +121,18 @@ def __run_test(test_case):
client = TClientConfig()
client.NbdSocketSuffix = nbd_socket_suffix

ret = run_test(
test_case.name,
config_path,
env.nbs_port,
env.mon_port,
nbs_log_path=env.nbs_log_path,
client_config=client,
env_processes=[env.nbs],
)

env.tear_down()
try:
ret = run_test(
test_case.name,
config_path,
env.nbs_port,
env.mon_port,
nbs_log_path=env.nbs_log_path,
client_config=client,
env_processes=[env.nbs],
)
finally:
env.tear_down()

return ret

Expand Down
23 changes: 12 additions & 11 deletions cloud/blockstore/tests/loadtest/local-endpoints/test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,17 +118,18 @@ def __run_test(test_case):
client_config.NbdSocketSuffix = nbd_socket_suffix
client_config.NbdStructuredReply = test_case.nbd_structured_reply

ret = run_test(
test_case.name,
test_case.config_path,
env.nbs_port,
env.mon_port,
nbs_log_path=env.nbs_log_path,
client_config=client_config,
env_processes=[env.nbs],
)

env.tear_down()
try:
ret = run_test(
test_case.name,
test_case.config_path,
env.nbs_port,
env.mon_port,
nbs_log_path=env.nbs_log_path,
client_config=client_config,
env_processes=[env.nbs],
)
finally:
env.tear_down()

return ret

Expand Down
31 changes: 16 additions & 15 deletions cloud/blockstore/tests/loadtest/local-mirror/test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,21 +259,22 @@ def __run_test(test_case):

config_path = __process_config(test_case.config_path, devices_per_agent)

ret = run_test(
test_case.name,
config_path,
nbs.nbs_port,
nbs.mon_port,
nbs_log_path=nbs.stderr_file_name,
client_config=client,
env_processes=disk_agents + [nbs],
)

for disk_agent in disk_agents:
disk_agent.stop()

nbs.stop()
kikimr_cluster.stop()
try:
ret = run_test(
test_case.name,
config_path,
nbs.nbs_port,
nbs.mon_port,
nbs_log_path=nbs.stderr_file_name,
client_config=client,
env_processes=disk_agents + [nbs],
)
finally:
for disk_agent in disk_agents:
disk_agent.stop()

nbs.stop()
kikimr_cluster.stop()
finally:
__cleanup_file_devices(devices)

Expand Down
25 changes: 13 additions & 12 deletions cloud/blockstore/tests/loadtest/local-nemesis/test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,18 +107,19 @@ def __run_test(test_case):
client.NbdSocketSuffix = nbd_socket_suffix
client.NbdStructuredReply = test_case.nbd_structured_reply

ret = run_test(
test_case.name,
test_case.config_path,
env.nbs_port,
env.mon_port,
nbs_log_path=env.nbs_log_path,
client_config=client,
endpoint_storage_dir=endpoint_storage_dir,
env_processes=[env.nbs],
)

env.tear_down()
try:
ret = run_test(
test_case.name,
test_case.config_path,
env.nbs_port,
env.mon_port,
nbs_log_path=env.nbs_log_path,
client_config=client,
endpoint_storage_dir=endpoint_storage_dir,
env_processes=[env.nbs],
)
finally:
env.tear_down()

return ret

Expand Down
23 changes: 12 additions & 11 deletions cloud/blockstore/tests/loadtest/local-newfeatures/test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -280,17 +280,18 @@ def __run_test(test_case):
client = TClientConfig()
client.NbdSocketSuffix = nbd_socket_suffix

ret = run_test(
test_case.name,
test_case.config_path,
env.nbs_port,
env.mon_port,
client_config=client,
endpoint_storage_dir=endpoint_storage_dir,
env_processes=[env.nbs],
)

env.tear_down()
try:
ret = run_test(
test_case.name,
test_case.config_path,
env.nbs_port,
env.mon_port,
client_config=client,
endpoint_storage_dir=endpoint_storage_dir,
env_processes=[env.nbs],
)
finally:
env.tear_down()

return ret

Expand Down
29 changes: 15 additions & 14 deletions cloud/blockstore/tests/loadtest/local-nonrepl/test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -318,20 +318,21 @@ def __run_test(test_case):
client = TClientConfig()
client.NbdSocketSuffix = nbd_socket_suffix

ret = run_test(
test_case.name,
config_path,
nbs.nbs_port,
nbs.mon_port,
nbs_log_path=nbs.stderr_file_name,
client_config=client,
env_processes=disk_agents + [nbs])

for disk_agent in disk_agents:
disk_agent.stop()

nbs.stop()
kikimr_cluster.stop()
try:
ret = run_test(
test_case.name,
config_path,
nbs.nbs_port,
nbs.mon_port,
nbs_log_path=nbs.stderr_file_name,
client_config=client,
env_processes=disk_agents + [nbs])
finally:
for disk_agent in disk_agents:
disk_agent.stop()

nbs.stop()
kikimr_cluster.stop()
finally:
if not test_case.use_memory_devices:
cleanup_file_devices(devices)
Expand Down
21 changes: 11 additions & 10 deletions cloud/blockstore/tests/loadtest/local-overflow/test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,16 +72,17 @@ def __run_test(test_case):
],
)

ret = run_test(
"overflow-%s" % test_case.name,
common.source_path(test_case.config_path),
env.nbs_port,
env.mon_port,
stat_filter=test_case.stat_filter,
env_processes=[env.nbs],
)

env.tear_down()
try:
ret = run_test(
"overflow-%s" % test_case.name,
common.source_path(test_case.config_path),
env.nbs_port,
env.mon_port,
stat_filter=test_case.stat_filter,
env_processes=[env.nbs],
)
finally:
env.tear_down()

return ret

Expand Down
Loading

0 comments on commit 3e3896f

Please sign in to comment.