diff --git a/README.md b/README.md index da837d1..72da0a7 100644 --- a/README.md +++ b/README.md @@ -458,12 +458,8 @@ All test specs have the following fields: * `name`: (optional) string describing the test unit. * `description`: (optional) string with longer description of the test unit. -* `timeout`: (optional) an object containing [timeout information][timeout] for the test - unit. -* `timeout.after`: a string duration of time the test unit is expected to +* `timeout`: (optional) a string duration of time the test unit is expected to complete within. -* `timeout.expected`: a bool indicating that the test unit is expected to not - complete before `timeout.after`. This is really only useful in unit testing. * `retry`: (optional) an object containing retry configurationu for the test unit. Some plugins will automatically attempt to retry the test action when an assertion fails. This field allows you to control this retry behaviour for @@ -497,7 +493,6 @@ All test specs have the following fields: [exec-plugin]: https://github.com/gdt-dev/gdt/tree/ecee17249e1fa10147cf9191be0358923da44094/plugin/exec [http-plugin]: https://github.com/gdt-dev/http [kube-plugin]: https://github.com/gdt-dev/kube -[timeout]: https://github.com/gdt-dev/gdt/blob/2791e11105fd3c36d1f11a7d111e089be7cdc84c/types/timeout.go#L11-L22 [wait]: https://github.com/gdt-dev/gdt/blob/2791e11105fd3c36d1f11a7d111e089be7cdc84c/types/wait.go#L11-L25 #### `exec` test spec structure diff --git a/plugin/exec/eval_test.go b/plugin/exec/eval_test.go index 62820e1..f433ea2 100644 --- a/plugin/exec/eval_test.go +++ b/plugin/exec/eval_test.go @@ -16,10 +16,15 @@ import ( "testing" gdtcontext "github.com/gdt-dev/gdt/context" + execplugin "github.com/gdt-dev/gdt/plugin/exec" "github.com/gdt-dev/gdt/scenario" "github.com/stretchr/testify/require" ) +func init() { + execplugin.OverrideDefaultTimeout("0.5s") +} + var failFlag = flag.Bool("fail", false, "run tests expected to fail") func TestNoExitCodeSimpleCommand(t *testing.T) { @@ -161,6 +166,45 @@ func TestContainsNoneOf(t *testing.T) { require.Nil(err) } +func TestFailExecTimeoutPluginDefault(t *testing.T) { + if !*failFlag { + t.Skip("skipping without -fail flag") + } + require := require.New(t) + + fp := filepath.Join("testdata", "timeout-plugin-default.yaml") + f, err := os.Open(fp) + require.Nil(err) + + s, err := scenario.FromReader( + f, + scenario.WithPath(fp), + ) + require.Nil(err) + require.NotNil(s) + + ctx := gdtcontext.New(gdtcontext.WithDebug()) + err = s.Run(ctx, t) + require.Nil(err) +} + +func TestExecTimeoutPluginDefault(t *testing.T) { + require := require.New(t) + target := os.Args[0] + failArgs := []string{ + "-test.v", + "-test.run=FailExecTimeoutPluginDefault", + "-fail", + } + outerr, err := exec.Command(target, failArgs...).CombinedOutput() + + // The test should have failed... + require.NotNil(err) + debugout := string(outerr) + require.Contains(debugout, "using timeout of 0.5s [plugin default]") + require.Contains(debugout, "assertion failed: timeout exceeded") +} + func TestFailExecSleepTimeout(t *testing.T) { if !*failFlag { t.Skip("skipping without -fail flag") @@ -289,8 +333,8 @@ func TestExecTimeoutCascade(t *testing.T) { require.NotNil(err) debugout := string(outerr) - require.Contains(debugout, "using timeout of 500ms (expected: false) [scenario default]") - require.Contains(debugout, "using timeout of 20ms (expected: true)") + require.Contains(debugout, "using timeout of 500ms [scenario default]") + require.Contains(debugout, "using timeout of 20ms") } func TestFailExecOnFail(t *testing.T) { diff --git a/plugin/exec/plugin.go b/plugin/exec/plugin.go index 6fb0c9d..222c86d 100644 --- a/plugin/exec/plugin.go +++ b/plugin/exec/plugin.go @@ -11,6 +11,15 @@ import ( gdttypes "github.com/gdt-dev/gdt/types" ) +var ( + DefaultTimeout = "10s" +) + +// OverrideDefaultTimeout is only used in testing... +func OverrideDefaultTimeout(d string) { + DefaultTimeout = d +} + func init() { gdtplugin.Register(Plugin()) } @@ -24,6 +33,9 @@ type plugin struct{} func (p *plugin) Info() gdttypes.PluginInfo { return gdttypes.PluginInfo{ Name: pluginName, + Timeout: &gdttypes.Timeout{ + After: DefaultTimeout, + }, } } diff --git a/plugin/exec/testdata/sleep-timeout.yaml b/plugin/exec/testdata/sleep-timeout.yaml index 5880423..d0d9725 100644 --- a/plugin/exec/testdata/sleep-timeout.yaml +++ b/plugin/exec/testdata/sleep-timeout.yaml @@ -7,4 +7,3 @@ tests: - exec: sleep 5 timeout: after: 50ms - expected: true diff --git a/plugin/exec/testdata/timeout-cascade.yaml b/plugin/exec/testdata/timeout-cascade.yaml index 6a0cc5c..af243c7 100644 --- a/plugin/exec/testdata/timeout-cascade.yaml +++ b/plugin/exec/testdata/timeout-cascade.yaml @@ -10,4 +10,3 @@ tests: exec: sleep .25 timeout: after: 20ms - expected: true diff --git a/plugin/exec/testdata/timeout-plugin-default.yaml b/plugin/exec/testdata/timeout-plugin-default.yaml new file mode 100644 index 0000000..499bda4 --- /dev/null +++ b/plugin/exec/testdata/timeout-plugin-default.yaml @@ -0,0 +1,5 @@ +name: timeout-plugin-default +description: a scenario that tests the plugin default timeout +tests: + - name: uses plugin default timeout + exec: sleep 1 diff --git a/scenario/parse_test.go b/scenario/parse_test.go index 8991ea9..14bb855 100644 --- a/scenario/parse_test.go +++ b/scenario/parse_test.go @@ -92,6 +92,18 @@ func TestUnknownSpec(t *testing.T) { assert.Nil(s) } +func TestTimeoutScalarOrMap(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + fp := filepath.Join("testdata", "parse", "timeout-scalar-or-map.yaml") + f, err := os.Open(fp) + require.Nil(err) + + _, err = scenario.FromReader(f, scenario.WithPath(fp)) + assert.Nil(err) +} + func TestBadTimeout(t *testing.T) { assert := assert.New(t) require := require.New(t) @@ -101,7 +113,7 @@ func TestBadTimeout(t *testing.T) { require.Nil(err) s, err := scenario.FromReader(f, scenario.WithPath(fp)) - assert.ErrorIs(err, errors.ErrExpectedMap) + assert.ErrorIs(err, errors.ErrExpectedScalarOrMap) assert.Nil(s) } diff --git a/scenario/run.go b/scenario/run.go index 877035d..36bb477 100644 --- a/scenario/run.go +++ b/scenario/run.go @@ -73,6 +73,7 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error { plugin := s.evalPlugins[idx] pinfo := plugin.Info() pretry := pinfo.Retry + ptimeout := pinfo.Timeout // Create a brand new context that inherits the top-level context's // cancel func. We want to set deadlines for each test spec and if @@ -88,7 +89,7 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error { time.Sleep(wait.BeforeDuration()) } - to := getTimeout(ctx, sb.Timeout, scDefaults) + to := getTimeout(ctx, sb.Timeout, ptimeout, scDefaults) if to != nil { var cancel context.CancelFunc specCtx, cancel = context.WithTimeout(specCtx, to.Duration()) @@ -191,26 +192,36 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error { // getTimeout returns the timeout value for the test spec. If the spec has a // timeout override, we use that. Otherwise, we inspect the scenario's defaults -// and, if present, use that timeout. +// and, if present, use that timeout. If the scenario's defaults for not +// indicate a timeout configuration, we ask the plugin if it has timeout +// defaults and use that. func getTimeout( ctx context.Context, specTimeout *gdttypes.Timeout, + pluginTimeout *gdttypes.Timeout, scenDefaults *Defaults, ) *gdttypes.Timeout { if specTimeout != nil { debug.Println( - ctx, "using timeout of %s (expected: %t)", - specTimeout.After, specTimeout.Expected, + ctx, "using timeout of %s", + specTimeout.After, ) return specTimeout } if scenDefaults != nil && scenDefaults.Timeout != nil { debug.Println( - ctx, "using timeout of %s (expected: %t) [scenario default]", - scenDefaults.Timeout.After, scenDefaults.Timeout.Expected, + ctx, "using timeout of %s [scenario default]", + scenDefaults.Timeout.After, ) return scenDefaults.Timeout } + if pluginTimeout != nil { + debug.Println( + ctx, "using timeout of %s [plugin default]", + pluginTimeout.After, + ) + return pluginTimeout + } return nil } diff --git a/scenario/testdata/parse/fail/bad-timeout.yaml b/scenario/testdata/parse/fail/bad-timeout.yaml index 220ecd9..1c56936 100644 --- a/scenario/testdata/parse/fail/bad-timeout.yaml +++ b/scenario/testdata/parse/fail/bad-timeout.yaml @@ -2,4 +2,5 @@ name: bad-timeout description: a scenario with an invalid timeout spec tests: - foo: baz - timeout: notatimeout + timeout: + - one diff --git a/scenario/testdata/parse/timeout-scalar-or-map.yaml b/scenario/testdata/parse/timeout-scalar-or-map.yaml new file mode 100644 index 0000000..06ac989 --- /dev/null +++ b/scenario/testdata/parse/timeout-scalar-or-map.yaml @@ -0,0 +1,8 @@ +name: timeout-scalar-or-map +description: a scenario with both scalar and object timeout spec +tests: + - foo: baz + timeout: 1s + - foo: bar + timeout: + after: 1s diff --git a/types/plugin.go b/types/plugin.go index ff4043b..151d7b5 100644 --- a/types/plugin.go +++ b/types/plugin.go @@ -15,6 +15,9 @@ type PluginInfo struct { Aliases []string // Description describes what types of tests the plugin can handle. Description string + // Timeout is a Timeout that should be used by default for test specs of + // this plugin. + Timeout *Timeout // Retry is a Retry that should be used by default for test specs of this // plugin. Retry *Retry diff --git a/types/spec.go b/types/spec.go index 6d090ec..ebd6762 100644 --- a/types/spec.go +++ b/types/spec.go @@ -105,12 +105,20 @@ func (s *Spec) UnmarshalYAML(node *yaml.Node) error { } s.Description = valNode.Value case "timeout": - if valNode.Kind != yaml.MappingNode { - return errors.ExpectedMapAt(valNode) - } var to *Timeout - if err := valNode.Decode(&to); err != nil { - return errors.ExpectedTimeoutAt(valNode) + switch valNode.Kind { + case yaml.MappingNode: + // We support the old-style timeout:after + if err := valNode.Decode(&to); err != nil { + return errors.ExpectedTimeoutAt(valNode) + } + case yaml.ScalarNode: + // We also support a straight string duration + to = &Timeout{ + After: valNode.Value, + } + default: + return errors.ExpectedScalarOrMapAt(valNode) } _, err := time.ParseDuration(to.After) if err != nil { diff --git a/types/timeout.go b/types/timeout.go index d14e889..bdd32e9 100644 --- a/types/timeout.go +++ b/types/timeout.go @@ -16,9 +16,6 @@ type Timeout struct { // Specify a duration using Go's time duration string. // See https://pkg.go.dev/time#ParseDuration After string `yaml:"after,omitempty"` - // Expected indicates whether the timeout is expected to be exceeded. This - // is mostly useful for unit testing of the timeout functionality itself. - Expected bool `yaml:"expected,omitempty"` } // Duration returns the time duration of the Timeout