diff --git a/spec/cnf_conformance_all/cnf_conformance_container_chaos_spec.cr b/spec/cnf_conformance_all/cnf_conformance_container_chaos_spec.cr new file mode 100644 index 000000000..06c33a7f5 --- /dev/null +++ b/spec/cnf_conformance_all/cnf_conformance_container_chaos_spec.cr @@ -0,0 +1,25 @@ +require "../spec_helper" +require "../../src/tasks/utils/utils.cr" +require "colorize" + +describe "CNF Conformance all Container Chaos" do + before_all do + `./cnf-conformance setup` + $?.success?.should be_true + end + + after_all do + `./cnf-conformance samples_cleanup` + $?.success?.should be_true + end + + it "'all' should run the whole test suite", tags: "happy-path" do + `./cnf-conformance samples_cleanup` + response_s = `./cnf-conformance all ~platform ~compatibilty ~statelessness ~security ~scalability ~configuration_lifecycle ~observability ~installability ~hardware_affinity ~microservice ~chaos_network_loss cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-conformance.yml verbose` + LOGGING.info response_s + (/Final workload score:/ =~ response_s).should_not be_nil + (/Final score:/ =~ response_s).should_not be_nil + (all_result_test_names(CNFManager.final_cnf_results_yml)).should eq([ "chaos_cpu_hog", "chaos_container_kill"]) + $?.success?.should be_true + end +end diff --git a/spec/cnf_conformance_all/cnf_conformance_network_chaos_spec.cr b/spec/cnf_conformance_all/cnf_conformance_network_chaos_spec.cr new file mode 100644 index 000000000..1d9e1fed5 --- /dev/null +++ b/spec/cnf_conformance_all/cnf_conformance_network_chaos_spec.cr @@ -0,0 +1,25 @@ +require "../spec_helper" +require "../../src/tasks/utils/utils.cr" +require "colorize" + +describe "CNF Conformance all Network Chaos" do + before_all do + `./cnf-conformance setup` + $?.success?.should be_true + end + + after_all do + `./cnf-conformance samples_cleanup` + $?.success?.should be_true + end + + it "'all' should run the whole test suite", tags: "happy-path" do + `./cnf-conformance samples_cleanup` + response_s = `./cnf-conformance all ~platform ~compatibilty ~statelessness ~security ~scalability ~configuration_lifecycle ~observability ~installability ~hardware_affinity ~microservice ~chaos_cpu_hog ~chaos_container_kill cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-conformance.yml verbose` + LOGGING.info response_s + (/Final workload score:/ =~ response_s).should_not be_nil + (/Final score:/ =~ response_s).should_not be_nil + (all_result_test_names(CNFManager.final_cnf_results_yml)).should eq([ "chaos_network_loss"]) + $?.success?.should be_true + end +end diff --git a/spec/cnf_conformance_spec.cr b/spec/cnf_conformance_all/cnf_conformance_spec.cr similarity index 81% rename from spec/cnf_conformance_spec.cr rename to spec/cnf_conformance_all/cnf_conformance_spec.cr index 6e5e24e3c..e4911f190 100644 --- a/spec/cnf_conformance_spec.cr +++ b/spec/cnf_conformance_all/cnf_conformance_spec.cr @@ -1,5 +1,5 @@ -require "./spec_helper" -require "../src/tasks/utils/utils.cr" +require "../spec_helper" +require "../../src/tasks/utils/utils.cr" require "colorize" describe CnfConformance do @@ -15,7 +15,7 @@ describe CnfConformance do it "'all' should run the whole test suite", tags: "happy-path" do `./cnf-conformance samples_cleanup` - response_s = `./cnf-conformance all ~platform cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-conformance.yml verbose` + response_s = `./cnf-conformance all ~platform ~resilience cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-conformance.yml verbose` LOGGING.info response_s (/PASSED: Helm readiness probe found/ =~ response_s).should_not be_nil (/PASSED: Helm liveness probe/ =~ response_s).should_not be_nil @@ -25,7 +25,7 @@ describe CnfConformance do (/PASSED: Published Helm Chart Found/ =~ response_s).should_not be_nil (/Final workload score:/ =~ response_s).should_not be_nil (/Final score:/ =~ response_s).should_not be_nil - (all_result_test_names(CNFManager.final_cnf_results_yml)).should eq(["volume_hostpath_not_found", "privileged", "increase_capacity", "decrease_capacity", "ip_addresses", "liveness", "readiness", "rolling_update", "nodeport_not_used", "hardcoded_ip_addresses_in_k8s_runtime_configuration", "install_script_helm", "helm_chart_valid", "helm_chart_published","helm_deploy", "reasonable_image_size", "reasonable_startup_time", "chaos_network_loss", "chaos_cpu_hog", "chaos_container_kill"]) + (all_result_test_names(CNFManager.final_cnf_results_yml)).should eq(["volume_hostpath_not_found", "privileged", "increase_capacity", "decrease_capacity", "ip_addresses", "liveness", "readiness", "rolling_update", "nodeport_not_used", "hardcoded_ip_addresses_in_k8s_runtime_configuration", "install_script_helm", "helm_chart_valid", "helm_chart_published","helm_deploy", "reasonable_image_size", "reasonable_startup_time" ]) $?.success?.should be_true end end diff --git a/spec/microservice_spec.cr b/spec/microservice_spec.cr index 7260037e3..c65fba819 100644 --- a/spec/microservice_spec.cr +++ b/spec/microservice_spec.cr @@ -61,7 +61,7 @@ describe "Microservice" do it "'reasonable_image_size' should fail if image is larger than 5gb", tags: "reasonable_image_size" do begin `./cnf-conformance cnf_cleanup cnf-path=sample-cnfs/sample-large-cnf force=true` - `./cnf-conformance cnf_setup cnf-path=sample-cnfs/sample-large-cnf deploy_with_chart=false` + `./cnf-conformance cnf_setup cnf-path=sample-cnfs/sample-large-cnf deploy_with_chart=false wait_count=0` response_s = `./cnf-conformance reasonable_image_size verbose` LOGGING.info response_s $?.success?.should be_true diff --git a/spec/resilience_spec.cr b/spec/resilience/container_chaos_spec.cr similarity index 50% rename from spec/resilience_spec.cr rename to spec/resilience/container_chaos_spec.cr index aed517e5b..69ff79bad 100644 --- a/spec/resilience_spec.cr +++ b/spec/resilience/container_chaos_spec.cr @@ -1,11 +1,11 @@ -require "./spec_helper" +require "../spec_helper" require "colorize" -require "../src/tasks/utils/utils.cr" -require "../src/tasks/utils/system_information/helm.cr" +require "../../src/tasks/utils/utils.cr" +require "../../src/tasks/utils/system_information/helm.cr" require "file_utils" require "sam" -describe "Resilience" do +describe "Resilience Container Chaos" do before_all do `./cnf-conformance configuration_file_setup` $?.success?.should be_true @@ -38,32 +38,4 @@ describe "Resilience" do $?.success?.should be_true end end - - it "'chaos_network_loss' A 'Good' CNF should not crash when network loss occurs", tags: ["chaos_network_loss"] do - begin - `./cnf-conformance cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-conformance.yml` - $?.success?.should be_true - response_s = `./cnf-conformance chaos_network_loss verbose` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: Replicas available match desired count after network chaos test/ =~ response_s).should_not be_nil - ensure - `./cnf-conformance cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-conformance.yml` - $?.success?.should be_true - end - end - - it "'chaos_network_loss' A 'Bad' CNF should crash when network loss occurs", tags: ["chaos_network_loss"] do - begin - `./cnf-conformance cnf_setup cnf-path=sample-cnfs/sample_network_loss deploy_with_chart=false` - $?.success?.should be_true - response_s = `./cnf-conformance chaos_network_loss verbose` - LOGGING.info response_s - $?.success?.should be_true - (/FAILURE: Replicas did not return desired count after network chaos test/ =~ response_s).should_not be_nil - ensure - `./cnf-conformance cnf_cleanup cnf-path=sample-cnfs/sample_network_loss deploy_with_chart=false` - $?.success?.should be_true - end - end end diff --git a/spec/resilience/network_chaos_spec.cr b/spec/resilience/network_chaos_spec.cr new file mode 100644 index 000000000..7aa09ede9 --- /dev/null +++ b/spec/resilience/network_chaos_spec.cr @@ -0,0 +1,41 @@ +require "../spec_helper" +require "colorize" +require "../../src/tasks/utils/utils.cr" +require "../../src/tasks/utils/system_information/helm.cr" +require "file_utils" +require "sam" + +describe "Resilience Network Chaos" do + before_all do + `./cnf-conformance configuration_file_setup` + $?.success?.should be_true + end + + it "'chaos_network_loss' A 'Good' CNF should not crash when network loss occurs", tags: ["chaos_network_loss"] do + begin + `./cnf-conformance cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-conformance.yml` + $?.success?.should be_true + response_s = `./cnf-conformance chaos_network_loss verbose` + LOGGING.info response_s + $?.success?.should be_true + (/PASSED: Replicas available match desired count after network chaos test/ =~ response_s).should_not be_nil + ensure + `./cnf-conformance cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-conformance.yml` + $?.success?.should be_true + end + end + + it "'chaos_network_loss' A 'Bad' CNF should crash when network loss occurs", tags: ["chaos_network_loss"] do + begin + `./cnf-conformance cnf_setup cnf-path=sample-cnfs/sample_network_loss deploy_with_chart=false` + $?.success?.should be_true + response_s = `./cnf-conformance chaos_network_loss verbose` + LOGGING.info response_s + $?.success?.should be_true + (/FAILURE: Replicas did not return desired count after network chaos test/ =~ response_s).should_not be_nil + ensure + `./cnf-conformance cnf_cleanup cnf-path=sample-cnfs/sample_network_loss deploy_with_chart=false` + $?.success?.should be_true + end + end +end