Merge pull request #458 from cncf/split_specs

Split specs for faster builds

spec/cnf_conformance_all/cnf_conformance_container_chaos_spec.cr

@@ -0,0 +1,25 @@
+require "../spec_helper"
+require "../../src/tasks/utils/utils.cr"
+require "colorize"
+
+describe "CNF Conformance all Container Chaos" do
+  before_all do
+    `./cnf-conformance setup`
+    $?.success?.should be_true
+  end
+
+  after_all do
+    `./cnf-conformance samples_cleanup`
+    $?.success?.should be_true
+  end
+
+  it "'all' should run the whole test suite", tags: "happy-path" do
+    `./cnf-conformance samples_cleanup`
+    response_s = `./cnf-conformance all ~platform ~compatibilty ~statelessness ~security ~scalability ~configuration_lifecycle ~observability ~installability ~hardware_affinity ~microservice ~chaos_network_loss cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-conformance.yml verbose`
+    LOGGING.info response_s
+    (/Final workload score:/ =~ response_s).should_not be_nil
+    (/Final score:/ =~ response_s).should_not be_nil
+    (all_result_test_names(CNFManager.final_cnf_results_yml)).should eq([  "chaos_cpu_hog", "chaos_container_kill"])
+    $?.success?.should be_true
+  end
+end

spec/cnf_conformance_all/cnf_conformance_network_chaos_spec.cr

@@ -0,0 +1,25 @@
+require "../spec_helper"
+require "../../src/tasks/utils/utils.cr"
+require "colorize"
+
+describe "CNF Conformance all Network Chaos" do
+  before_all do
+    `./cnf-conformance setup`
+    $?.success?.should be_true
+  end
+
+  after_all do
+    `./cnf-conformance samples_cleanup`
+    $?.success?.should be_true
+  end
+
+  it "'all' should run the whole test suite", tags: "happy-path" do
+    `./cnf-conformance samples_cleanup`
+    response_s = `./cnf-conformance all ~platform ~compatibilty ~statelessness ~security ~scalability ~configuration_lifecycle ~observability ~installability ~hardware_affinity ~microservice ~chaos_cpu_hog ~chaos_container_kill cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-conformance.yml verbose`
+    LOGGING.info response_s
+    (/Final workload score:/ =~ response_s).should_not be_nil
+    (/Final score:/ =~ response_s).should_not be_nil
+    (all_result_test_names(CNFManager.final_cnf_results_yml)).should eq([ "chaos_network_loss"])
+    $?.success?.should be_true
+  end
+end

spec/cnf_conformance_spec.cr → spec/cnf_conformance_all/cnf_conformance_spec.cr

@@ -1,5 +1,5 @@
-require "./spec_helper"
-require "../src/tasks/utils/utils.cr"
+require "../spec_helper"
+require "../../src/tasks/utils/utils.cr"
 require "colorize"
 
 describe CnfConformance do
@@ -15,7 +15,7 @@ describe CnfConformance do
 
   it "'all' should run the whole test suite", tags: "happy-path" do
     `./cnf-conformance samples_cleanup`
-    response_s = `./cnf-conformance all ~platform cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-conformance.yml verbose`
+    response_s = `./cnf-conformance all ~platform ~resilience cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-conformance.yml verbose`
     LOGGING.info response_s
     (/PASSED: Helm readiness probe found/ =~ response_s).should_not be_nil
     (/PASSED: Helm liveness probe/ =~ response_s).should_not be_nil
@@ -25,7 +25,7 @@ describe CnfConformance do
     (/PASSED: Published Helm Chart Found/ =~ response_s).should_not be_nil
     (/Final workload score:/ =~ response_s).should_not be_nil
     (/Final score:/ =~ response_s).should_not be_nil
-    (all_result_test_names(CNFManager.final_cnf_results_yml)).should eq(["volume_hostpath_not_found", "privileged", "increase_capacity", "decrease_capacity", "ip_addresses", "liveness", "readiness", "rolling_update", "nodeport_not_used", "hardcoded_ip_addresses_in_k8s_runtime_configuration", "install_script_helm", "helm_chart_valid", "helm_chart_published","helm_deploy", "reasonable_image_size", "reasonable_startup_time", "chaos_network_loss", "chaos_cpu_hog", "chaos_container_kill"])
+    (all_result_test_names(CNFManager.final_cnf_results_yml)).should eq(["volume_hostpath_not_found", "privileged", "increase_capacity", "decrease_capacity", "ip_addresses", "liveness", "readiness", "rolling_update", "nodeport_not_used", "hardcoded_ip_addresses_in_k8s_runtime_configuration", "install_script_helm", "helm_chart_valid", "helm_chart_published","helm_deploy", "reasonable_image_size", "reasonable_startup_time" ])
     $?.success?.should be_true
   end
 end

spec/microservice_spec.cr

@@ -61,7 +61,7 @@ describe "Microservice" do
   it "'reasonable_image_size' should fail if image is larger than 5gb", tags: "reasonable_image_size" do
     begin
       `./cnf-conformance cnf_cleanup cnf-path=sample-cnfs/sample-large-cnf force=true`
-      `./cnf-conformance cnf_setup cnf-path=sample-cnfs/sample-large-cnf deploy_with_chart=false`
+      `./cnf-conformance cnf_setup cnf-path=sample-cnfs/sample-large-cnf deploy_with_chart=false wait_count=0`
       response_s = `./cnf-conformance reasonable_image_size verbose`
       LOGGING.info response_s
       $?.success?.should be_true

spec/resilience_spec.cr → spec/resilience/container_chaos_spec.cr

@@ -1,11 +1,11 @@
-require "./spec_helper"
+require "../spec_helper"
 require "colorize"
-require "../src/tasks/utils/utils.cr"
-require "../src/tasks/utils/system_information/helm.cr"
+require "../../src/tasks/utils/utils.cr"
+require "../../src/tasks/utils/system_information/helm.cr"
 require "file_utils"
 require "sam"
 
-describe "Resilience" do
+describe "Resilience Container Chaos" do
   before_all do
     `./cnf-conformance configuration_file_setup`
     $?.success?.should be_true
@@ -38,32 +38,4 @@ describe "Resilience" do
       $?.success?.should be_true
     end
   end
-
-  it "'chaos_network_loss' A 'Good' CNF should not crash when network loss occurs", tags: ["chaos_network_loss"]  do
-    begin
-      `./cnf-conformance cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-conformance.yml`
-      $?.success?.should be_true
-      response_s = `./cnf-conformance chaos_network_loss verbose`
-      LOGGING.info response_s
-      $?.success?.should be_true
-      (/PASSED: Replicas available match desired count after network chaos test/ =~ response_s).should_not be_nil
-    ensure
-      `./cnf-conformance cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-conformance.yml`
-      $?.success?.should be_true
-    end
-  end
-
-  it "'chaos_network_loss' A 'Bad' CNF should crash when network loss occurs", tags: ["chaos_network_loss"]  do
-    begin
-      `./cnf-conformance cnf_setup cnf-path=sample-cnfs/sample_network_loss deploy_with_chart=false`
-      $?.success?.should be_true
-      response_s = `./cnf-conformance chaos_network_loss verbose`
-      LOGGING.info response_s
-      $?.success?.should be_true
-      (/FAILURE: Replicas did not return desired count after network chaos test/ =~ response_s).should_not be_nil
-    ensure
-      `./cnf-conformance cnf_cleanup cnf-path=sample-cnfs/sample_network_loss deploy_with_chart=false`
-      $?.success?.should be_true
-    end
-  end
 end

spec/resilience/network_chaos_spec.cr

@@ -0,0 +1,41 @@
+require "../spec_helper"
+require "colorize"
+require "../../src/tasks/utils/utils.cr"
+require "../../src/tasks/utils/system_information/helm.cr"
+require "file_utils"
+require "sam"
+
+describe "Resilience Network Chaos" do
+  before_all do
+    `./cnf-conformance configuration_file_setup`
+    $?.success?.should be_true
+  end
+
+  it "'chaos_network_loss' A 'Good' CNF should not crash when network loss occurs", tags: ["chaos_network_loss"]  do
+    begin
+      `./cnf-conformance cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-conformance.yml`
+      $?.success?.should be_true
+      response_s = `./cnf-conformance chaos_network_loss verbose`
+      LOGGING.info response_s
+      $?.success?.should be_true
+      (/PASSED: Replicas available match desired count after network chaos test/ =~ response_s).should_not be_nil
+    ensure
+      `./cnf-conformance cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-conformance.yml`
+      $?.success?.should be_true
+    end
+  end
+
+  it "'chaos_network_loss' A 'Bad' CNF should crash when network loss occurs", tags: ["chaos_network_loss"]  do
+    begin
+      `./cnf-conformance cnf_setup cnf-path=sample-cnfs/sample_network_loss deploy_with_chart=false`
+      $?.success?.should be_true
+      response_s = `./cnf-conformance chaos_network_loss verbose`
+      LOGGING.info response_s
+      $?.success?.should be_true
+      (/FAILURE: Replicas did not return desired count after network chaos test/ =~ response_s).should_not be_nil
+    ensure
+      `./cnf-conformance cnf_cleanup cnf-path=sample-cnfs/sample_network_loss deploy_with_chart=false`
+      $?.success?.should be_true
+    end
+  end
+end