Merge pull request #2675 from fermyon/conformance-test-try-again

.github/workflows/build.yml

@@ -170,6 +170,8 @@ jobs:
       - name: Run Integration Tests
         run: |
           make test-integration
+        env:
+          SPIN_CONFORMANCE_TESTS_DOCKER_OPT_OUT: true
         # Only run integration tests on macOS as they will be run on ubuntu separately
         if: ${{ matrix.runner == 'macos-14' }}
 

Cargo.toml

@@ -100,6 +100,8 @@ runtime-tests = { path = "tests/runtime-tests" }
 test-components = { path = "tests/test-components" }
 test-codegen-macro = { path = "crates/test-codegen-macro" }
 test-environment = { git = "https://github.com/fermyon/conformance-tests", rev = "d2129a3fd73140a76c77f15a030a5273b37cbd11" }
+conformance-tests = { git = "https://github.com/fermyon/conformance-tests", rev = "d2129a3fd73140a76c77f15a030a5273b37cbd11" }
+conformance = { path = "tests/conformance-tests" }
 
 [build-dependencies]
 cargo-target-dep = { git = "https://github.com/fermyon/cargo-target-dep", rev = "482f269eceb7b1a7e8fc618bf8c082dd24979cf1" }

tests/conformance-tests/src/lib.rs

@@ -0,0 +1,78 @@
+use anyhow::Context as _;
+use testing_framework::runtimes::spin_cli::{SpinCli, SpinConfig};
+
+/// Run a single conformance test against the supplied spin binary.
+pub fn run_test(
+    test: conformance_tests::Test,
+    spin_binary: &std::path::Path,
+) -> anyhow::Result<()> {
+    let mut services = Vec::new();
+    for precondition in test.config.preconditions {
+        match precondition {
+            conformance_tests::config::Precondition::HttpEcho => {
+                services.push("http-echo");
+            }
+            conformance_tests::config::Precondition::TcpEcho => {
+                services.push("tcp-echo");
+            }
+            conformance_tests::config::Precondition::Redis => {
+                if should_run_docker_based_tests() {
+                    services.push("redis")
+                } else {
+                    // Skip the test if docker is not installed.
+                    return Ok(());
+                }
+            }
+            conformance_tests::config::Precondition::Mqtt => {
+                if should_run_docker_based_tests() {
+                    services.push("mqtt")
+                } else {
+                    // Skip the test if docker is not installed.
+                    return Ok(());
+                }
+            }
+            conformance_tests::config::Precondition::KeyValueStore(_) => {}
+            conformance_tests::config::Precondition::Sqlite => {}
+        }
+    }
+    let env_config = SpinCli::config(
+        SpinConfig {
+            binary_path: spin_binary.to_owned(),
+            spin_up_args: Vec::new(),
+            app_type: testing_framework::runtimes::SpinAppType::Http,
+        },
+        test_environment::services::ServicesConfig::new(services)?,
+        move |e| {
+            let mut manifest =
+                test_environment::manifest_template::EnvTemplate::from_file(&test.manifest)?;
+            manifest.substitute(e, |_| None)?;
+            e.write_file("spin.toml", manifest.contents())?;
+            e.copy_into(&test.component, test.component.file_name().unwrap())?;
+            Ok(())
+        },
+    );
+    let mut env = test_environment::TestEnvironment::up(env_config, |_| Ok(()))?;
+    for invocation in test.config.invocations {
+        let conformance_tests::config::Invocation::Http(mut invocation) = invocation;
+        invocation.request.substitute_from_env(&mut env)?;
+        let spin = env.runtime_mut();
+        let actual = invocation
+            .request
+            .send(|request| spin.make_http_request(request))?;
+
+        conformance_tests::assertions::assert_response(&invocation.response, &actual)
+            .with_context(|| {
+                format!(
+                    "Failed assertion.\nstdout: {}\nstderr: {}",
+                    spin.stdout().to_owned(),
+                    spin.stderr()
+                )
+            })?;
+    }
+    Ok(())
+}
+
+/// Whether or not docker is installed on the system.
+fn should_run_docker_based_tests() -> bool {
+    std::env::var("SPIN_CONFORMANCE_TESTS_DOCKER_OPT_OUT").is_err()
+}

tests/conformance-tests/src/main.rs

@@ -1,63 +1,7 @@
-use anyhow::Context as _;
-use testing_framework::runtimes::spin_cli::{SpinCli, SpinConfig};
-
 fn main() {
     let spin_binary: std::path::PathBuf = std::env::args()
         .nth(1)
         .expect("expected first argument to be path to spin binary")
         .into();
-    conformance_tests::run_tests(move |test| run_test(test, &spin_binary)).unwrap();
-}
-
-fn run_test(test: conformance_tests::Test, spin_binary: &std::path::Path) -> anyhow::Result<()> {
-    let mut services = Vec::new();
-    for precondition in test.config.preconditions {
-        match precondition {
-            conformance_tests::config::Precondition::HttpEcho => {
-                services.push("http-echo");
-            }
-            conformance_tests::config::Precondition::TcpEcho => {
-                services.push("tcp-echo");
-            }
-            conformance_tests::config::Precondition::Redis => services.push("redis"),
-            conformance_tests::config::Precondition::Mqtt => services.push("mqtt"),
-            conformance_tests::config::Precondition::KeyValueStore(_) => {}
-            conformance_tests::config::Precondition::Sqlite => {}
-        }
-    }
-    let env_config = SpinCli::config(
-        SpinConfig {
-            binary_path: spin_binary.to_owned(),
-            spin_up_args: Vec::new(),
-            app_type: testing_framework::runtimes::SpinAppType::Http,
-        },
-        test_environment::services::ServicesConfig::new(services)?,
-        move |e| {
-            let mut manifest =
-                test_environment::manifest_template::EnvTemplate::from_file(&test.manifest)?;
-            manifest.substitute(e, |_| None)?;
-            e.write_file("spin.toml", manifest.contents())?;
-            e.copy_into(&test.component, test.component.file_name().unwrap())?;
-            Ok(())
-        },
-    );
-    let mut env = test_environment::TestEnvironment::up(env_config, |_| Ok(()))?;
-    for invocation in test.config.invocations {
-        let conformance_tests::config::Invocation::Http(mut invocation) = invocation;
-        invocation.request.substitute_from_env(&mut env)?;
-        let spin = env.runtime_mut();
-        let actual = invocation
-            .request
-            .send(|request| spin.make_http_request(request))?;
-
-        conformance_tests::assertions::assert_response(&invocation.response, &actual)
-            .with_context(|| {
-                format!(
-                    "Failed assertion.\nstdout: {}\nstderr: {}",
-                    spin.stdout().to_owned(),
-                    spin.stderr()
-                )
-            })?;
-    }
-    Ok(())
+    conformance_tests::run_tests(move |test| conformance::run_test(test, &spin_binary)).unwrap();
 }