nextest_runner/config/elements/
slow_timeout.rs

1// Copyright (c) The nextest Contributors
2// SPDX-License-Identifier: MIT OR Apache-2.0
3
4use crate::time::far_future_duration;
5use serde::{Deserialize, Serialize, de::IntoDeserializer};
6use std::{fmt, num::NonZeroUsize, time::Duration};
7
8/// Type for the slow-timeout config key.
9#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)]
10#[serde(rename_all = "kebab-case")]
11pub struct SlowTimeout {
12    #[serde(with = "humantime_serde")]
13    pub(crate) period: Duration,
14    #[serde(default)]
15    pub(crate) terminate_after: Option<NonZeroUsize>,
16    #[serde(with = "humantime_serde", default = "default_grace_period")]
17    pub(crate) grace_period: Duration,
18    #[serde(default)]
19    pub(crate) on_timeout: SlowTimeoutResult,
20}
21
22impl SlowTimeout {
23    /// A reasonable value for "maximum slow timeout".
24    pub(crate) const VERY_LARGE: Self = Self {
25        // See far_future() in pausable_sleep.rs for why this is roughly 30 years.
26        period: far_future_duration(),
27        terminate_after: None,
28        grace_period: Duration::from_secs(10),
29        on_timeout: SlowTimeoutResult::Fail,
30    };
31}
32
33fn default_grace_period() -> Duration {
34    Duration::from_secs(10)
35}
36
37pub(in crate::config) fn deserialize_slow_timeout<'de, D>(
38    deserializer: D,
39) -> Result<Option<SlowTimeout>, D::Error>
40where
41    D: serde::Deserializer<'de>,
42{
43    struct V;
44
45    impl<'de2> serde::de::Visitor<'de2> for V {
46        type Value = Option<SlowTimeout>;
47
48        fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
49            write!(
50                formatter,
51                "a table ({{ period = \"60s\", terminate-after = 2 }}) or a string (\"60s\")"
52            )
53        }
54
55        fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
56        where
57            E: serde::de::Error,
58        {
59            if v.is_empty() {
60                Ok(None)
61            } else {
62                let period = humantime_serde::deserialize(v.into_deserializer())?;
63                Ok(Some(SlowTimeout {
64                    period,
65                    terminate_after: None,
66                    grace_period: default_grace_period(),
67                    on_timeout: SlowTimeoutResult::Fail,
68                }))
69            }
70        }
71
72        fn visit_map<A>(self, map: A) -> Result<Self::Value, A::Error>
73        where
74            A: serde::de::MapAccess<'de2>,
75        {
76            SlowTimeout::deserialize(serde::de::value::MapAccessDeserializer::new(map)).map(Some)
77        }
78    }
79
80    deserializer.deserialize_any(V)
81}
82
83/// The result of controlling slow timeout behavior.
84///
85/// In most situations a timed out test should be marked failing. However, there are certain
86/// classes of tests which are expected to run indefinitely long, like fuzzing, which explores a
87/// huge state space. For these tests it's nice to be able to treat a timeout as a success since
88/// they generally check for invariants and other properties of the code under test during their
89/// execution. A timeout in this context doesn't mean that there are no failing inputs, it just
90/// means that they weren't found up until that moment, which is still valuable information.
91#[derive(Clone, Copy, Debug, Deserialize, Serialize, Default, PartialEq, Eq)]
92#[serde(rename_all = "kebab-case")]
93pub enum SlowTimeoutResult {
94    #[default]
95    /// The test is marked as failed.
96    Fail,
97
98    /// The test is marked as passed.
99    Pass,
100}
101
102#[cfg(test)]
103mod tests {
104    use super::*;
105    use crate::{
106        config::{core::NextestConfig, utils::test_helpers::*},
107        run_mode::NextestRunMode,
108    };
109    use camino_tempfile::tempdir;
110    use indoc::indoc;
111    use nextest_filtering::ParseContext;
112    use test_case::test_case;
113
114    #[test_case(
115        "",
116        Ok(SlowTimeout {
117            period: Duration::from_secs(60),
118            terminate_after: None,
119            grace_period: Duration::from_secs(10),
120            on_timeout: SlowTimeoutResult::Fail,
121        }),
122        None
123        ; "empty config is expected to use the hardcoded values"
124    )]
125    #[test_case(
126        indoc! {r#"
127            [profile.default]
128            slow-timeout = "30s"
129        "#},
130        Ok(SlowTimeout {
131            period: Duration::from_secs(30),
132            terminate_after: None,
133            grace_period: Duration::from_secs(10),
134            on_timeout: SlowTimeoutResult::Fail,
135        }),
136        None
137        ; "overrides the default profile"
138    )]
139    #[test_case(
140        indoc! {r#"
141            [profile.default]
142            slow-timeout = "30s"
143
144            [profile.ci]
145            slow-timeout = { period = "60s", terminate-after = 3 }
146        "#},
147        Ok(SlowTimeout {
148            period: Duration::from_secs(30),
149            terminate_after: None,
150            grace_period: Duration::from_secs(10),
151            on_timeout: SlowTimeoutResult::Fail,
152        }),
153        Some(SlowTimeout {
154            period: Duration::from_secs(60),
155            terminate_after: Some(NonZeroUsize::new(3).unwrap()),
156            grace_period: Duration::from_secs(10),
157            on_timeout: SlowTimeoutResult::Fail,
158        })
159        ; "adds a custom profile 'ci'"
160    )]
161    #[test_case(
162        indoc! {r#"
163            [profile.default]
164            slow-timeout = { period = "60s", terminate-after = 3 }
165
166            [profile.ci]
167            slow-timeout = "30s"
168        "#},
169        Ok(SlowTimeout {
170            period: Duration::from_secs(60),
171            terminate_after: Some(NonZeroUsize::new(3).unwrap()),
172            grace_period: Duration::from_secs(10),
173            on_timeout: SlowTimeoutResult::Fail,
174        }),
175        Some(SlowTimeout {
176            period: Duration::from_secs(30),
177            terminate_after: None,
178            grace_period: Duration::from_secs(10),
179            on_timeout: SlowTimeoutResult::Fail,
180        })
181        ; "ci profile uses string notation"
182    )]
183    #[test_case(
184        indoc! {r#"
185            [profile.default]
186            slow-timeout = { period = "60s", terminate-after = 3, grace-period = "1s" }
187
188            [profile.ci]
189            slow-timeout = "30s"
190        "#},
191        Ok(SlowTimeout {
192            period: Duration::from_secs(60),
193            terminate_after: Some(NonZeroUsize::new(3).unwrap()),
194            grace_period: Duration::from_secs(1),
195            on_timeout: SlowTimeoutResult::Fail,
196        }),
197        Some(SlowTimeout {
198            period: Duration::from_secs(30),
199            terminate_after: None,
200            grace_period: Duration::from_secs(10),
201            on_timeout: SlowTimeoutResult::Fail,
202        })
203        ; "timeout grace period"
204    )]
205    #[test_case(
206        indoc! {r#"
207            [profile.default]
208            slow-timeout = { period = "60s" }
209        "#},
210        Ok(SlowTimeout {
211            period: Duration::from_secs(60),
212            terminate_after: None,
213            grace_period: Duration::from_secs(10),
214            on_timeout: SlowTimeoutResult::Fail,
215        }),
216        None
217        ; "partial table"
218    )]
219    #[test_case(
220        indoc! {r#"
221            [profile.default]
222            slow-timeout = { period = "60s", terminate-after = 0 }
223        "#},
224        Err("original: invalid value: integer `0`, expected a nonzero usize"),
225        None
226        ; "zero terminate-after should fail"
227    )]
228    #[test_case(
229        indoc! {r#"
230            [profile.default]
231            slow-timeout = { period = "60s", on-timeout = "pass" }
232        "#},
233        Ok(SlowTimeout {
234            period: Duration::from_secs(60),
235            terminate_after: None,
236            grace_period: Duration::from_secs(10),
237            on_timeout: SlowTimeoutResult::Pass,
238        }),
239        None
240        ; "timeout result success"
241    )]
242    #[test_case(
243        indoc! {r#"
244            [profile.default]
245            slow-timeout = { period = "60s", on-timeout = "fail" }
246        "#},
247        Ok(SlowTimeout {
248            period: Duration::from_secs(60),
249            terminate_after: None,
250            grace_period: Duration::from_secs(10),
251            on_timeout: SlowTimeoutResult::Fail,
252        }),
253        None
254        ; "timeout result failure"
255    )]
256    #[test_case(
257        indoc! {r#"
258            [profile.default]
259            slow-timeout = { period = "60s", on-timeout = "pass" }
260
261            [profile.ci]
262            slow-timeout = { period = "30s", on-timeout = "fail" }
263        "#},
264        Ok(SlowTimeout {
265            period: Duration::from_secs(60),
266            terminate_after: None,
267            grace_period: Duration::from_secs(10),
268            on_timeout: SlowTimeoutResult::Pass,
269        }),
270        Some(SlowTimeout {
271            period: Duration::from_secs(30),
272            terminate_after: None,
273            grace_period: Duration::from_secs(10),
274            on_timeout: SlowTimeoutResult::Fail,
275        })
276        ; "override on-timeout option"
277    )]
278    #[test_case(
279        indoc! {r#"
280            [profile.default]
281            slow-timeout = "60s"
282
283            [profile.ci]
284            slow-timeout = { terminate-after = 3 }
285        "#},
286        Err("original: missing configuration field \"profile.ci.slow-timeout.period\""),
287        None
288
289        ; "partial slow-timeout table should error"
290    )]
291    fn slowtimeout_adheres_to_hierarchy(
292        config_contents: &str,
293        expected_default: Result<SlowTimeout, &str>,
294        maybe_expected_ci: Option<SlowTimeout>,
295    ) {
296        let workspace_dir = tempdir().unwrap();
297
298        let graph = temp_workspace(&workspace_dir, config_contents);
299
300        let pcx = ParseContext::new(&graph);
301
302        let nextest_config_result = NextestConfig::from_sources(
303            graph.workspace().root(),
304            &pcx,
305            None,
306            &[][..],
307            &Default::default(),
308        );
309
310        match expected_default {
311            Ok(expected_default) => {
312                let nextest_config = nextest_config_result.expect("config file should parse");
313
314                assert_eq!(
315                    nextest_config
316                        .profile("default")
317                        .expect("default profile should exist")
318                        .apply_build_platforms(&build_platforms())
319                        .slow_timeout(NextestRunMode::Test),
320                    expected_default,
321                );
322
323                if let Some(expected_ci) = maybe_expected_ci {
324                    assert_eq!(
325                        nextest_config
326                            .profile("ci")
327                            .expect("ci profile should exist")
328                            .apply_build_platforms(&build_platforms())
329                            .slow_timeout(NextestRunMode::Test),
330                        expected_ci,
331                    );
332                }
333            }
334
335            Err(expected_err_str) => {
336                let err_str = format!("{:?}", nextest_config_result.unwrap_err());
337
338                assert!(
339                    err_str.contains(expected_err_str),
340                    "expected error string not found: {err_str}",
341                )
342            }
343        }
344    }
345
346    // Default test slow-timeout is 60 seconds.
347    const DEFAULT_TEST_SLOW_TIMEOUT: SlowTimeout = SlowTimeout {
348        period: Duration::from_secs(60),
349        terminate_after: None,
350        grace_period: Duration::from_secs(10),
351        on_timeout: SlowTimeoutResult::Fail,
352    };
353
354    /// Expected bench timeout: either a specific value or "very large" (default).
355    #[derive(Debug)]
356    enum ExpectedBenchTimeout {
357        /// Expect a specific timeout value.
358        Exact(SlowTimeout),
359        /// Expect the default very large timeout (>= VERY_LARGE, accounting for
360        /// leap years in humantime parsing).
361        VeryLarge,
362    }
363
364    #[test_case(
365        "",
366        DEFAULT_TEST_SLOW_TIMEOUT,
367        ExpectedBenchTimeout::VeryLarge
368        ; "empty config uses defaults for both modes"
369    )]
370    #[test_case(
371        indoc! {r#"
372            [profile.default]
373            slow-timeout = { period = "10s", terminate-after = 2 }
374        "#},
375        SlowTimeout {
376            period: Duration::from_secs(10),
377            terminate_after: Some(NonZeroUsize::new(2).unwrap()),
378            grace_period: Duration::from_secs(10),
379            on_timeout: SlowTimeoutResult::Fail,
380        },
381        // bench.slow-timeout should still be 30 years (default).
382        ExpectedBenchTimeout::VeryLarge
383        ; "slow-timeout does not affect bench.slow-timeout"
384    )]
385    #[test_case(
386        indoc! {r#"
387            [profile.default]
388            bench.slow-timeout = { period = "20s", terminate-after = 3 }
389        "#},
390        // slow-timeout should still be 60s (default).
391        DEFAULT_TEST_SLOW_TIMEOUT,
392        ExpectedBenchTimeout::Exact(SlowTimeout {
393            period: Duration::from_secs(20),
394            terminate_after: Some(NonZeroUsize::new(3).unwrap()),
395            grace_period: Duration::from_secs(10),
396            on_timeout: SlowTimeoutResult::Fail,
397        })
398        ; "bench.slow-timeout does not affect slow-timeout"
399    )]
400    #[test_case(
401        indoc! {r#"
402            [profile.default]
403            slow-timeout = { period = "10s", terminate-after = 2 }
404            bench.slow-timeout = { period = "20s", terminate-after = 3 }
405        "#},
406        SlowTimeout {
407            period: Duration::from_secs(10),
408            terminate_after: Some(NonZeroUsize::new(2).unwrap()),
409            grace_period: Duration::from_secs(10),
410            on_timeout: SlowTimeoutResult::Fail,
411        },
412        ExpectedBenchTimeout::Exact(SlowTimeout {
413            period: Duration::from_secs(20),
414            terminate_after: Some(NonZeroUsize::new(3).unwrap()),
415            grace_period: Duration::from_secs(10),
416            on_timeout: SlowTimeoutResult::Fail,
417        })
418        ; "both slow-timeout and bench.slow-timeout can be set independently"
419    )]
420    #[test_case(
421        indoc! {r#"
422            [profile.default]
423            bench.slow-timeout = "30s"
424        "#},
425        DEFAULT_TEST_SLOW_TIMEOUT,
426        ExpectedBenchTimeout::Exact(SlowTimeout {
427            period: Duration::from_secs(30),
428            terminate_after: None,
429            grace_period: Duration::from_secs(10),
430            on_timeout: SlowTimeoutResult::Fail,
431        })
432        ; "bench.slow-timeout string notation"
433    )]
434    fn bench_slowtimeout_is_independent(
435        config_contents: &str,
436        expected_test_timeout: SlowTimeout,
437        expected_bench_timeout: ExpectedBenchTimeout,
438    ) {
439        let workspace_dir = tempdir().unwrap();
440
441        let graph = temp_workspace(&workspace_dir, config_contents);
442
443        let pcx = ParseContext::new(&graph);
444
445        let nextest_config = NextestConfig::from_sources(
446            graph.workspace().root(),
447            &pcx,
448            None,
449            &[][..],
450            &Default::default(),
451        )
452        .expect("config file should parse");
453
454        let profile = nextest_config
455            .profile("default")
456            .expect("default profile should exist")
457            .apply_build_platforms(&build_platforms());
458
459        assert_eq!(
460            profile.slow_timeout(NextestRunMode::Test),
461            expected_test_timeout,
462            "Test mode slow-timeout mismatch"
463        );
464
465        let actual_bench_timeout = profile.slow_timeout(NextestRunMode::Benchmark);
466        match expected_bench_timeout {
467            ExpectedBenchTimeout::Exact(expected) => {
468                assert_eq!(
469                    actual_bench_timeout, expected,
470                    "Benchmark mode slow-timeout mismatch"
471                );
472            }
473            ExpectedBenchTimeout::VeryLarge => {
474                // The default is "30y" which humantime parses accounting for
475                // leap years, so it is slightly larger than VERY_LARGE.
476                assert!(
477                    actual_bench_timeout.period >= SlowTimeout::VERY_LARGE.period,
478                    "Benchmark mode slow-timeout should be >= VERY_LARGE, got {:?}",
479                    actual_bench_timeout.period
480                );
481                assert_eq!(
482                    actual_bench_timeout.terminate_after, None,
483                    "Benchmark mode terminate_after should be None"
484                );
485            }
486        }
487    }
488}
nextest_runner/config/elements/slow_timeout.rs

nextest_runner/config/elements/
slow_timeout.rs
