nextest_runner/config/
retry_policy.rs

1// Copyright (c) The nextest Contributors
2// SPDX-License-Identifier: MIT OR Apache-2.0
3
4use serde::Deserialize;
5use std::{cmp::Ordering, fmt, time::Duration};
6
7/// Type for the retry config key.
8#[derive(Debug, Copy, Clone, Deserialize, PartialEq, Eq)]
9#[serde(tag = "backoff", rename_all = "kebab-case", deny_unknown_fields)]
10pub enum RetryPolicy {
11    /// Fixed backoff.
12    #[serde(rename_all = "kebab-case")]
13    Fixed {
14        /// Maximum retry count.
15        count: usize,
16
17        /// Delay between retries.
18        #[serde(default, with = "humantime_serde")]
19        delay: Duration,
20
21        /// If set to true, randomness will be added to the delay on each retry attempt.
22        #[serde(default)]
23        jitter: bool,
24    },
25
26    /// Exponential backoff.
27    #[serde(rename_all = "kebab-case")]
28    Exponential {
29        /// Maximum retry count.
30        count: usize,
31
32        /// Delay between retries. Not optional for exponential backoff.
33        #[serde(with = "humantime_serde")]
34        delay: Duration,
35
36        /// If set to true, randomness will be added to the delay on each retry attempt.
37        #[serde(default)]
38        jitter: bool,
39
40        /// If set, limits the delay between retries.
41        #[serde(default, with = "humantime_serde")]
42        max_delay: Option<Duration>,
43    },
44}
45
46impl Default for RetryPolicy {
47    #[inline]
48    fn default() -> Self {
49        Self::new_without_delay(0)
50    }
51}
52
53impl RetryPolicy {
54    /// Create new policy with no delay between retries.
55    pub fn new_without_delay(count: usize) -> Self {
56        Self::Fixed {
57            count,
58            delay: Duration::ZERO,
59            jitter: false,
60        }
61    }
62
63    /// Returns the number of retries.
64    pub fn count(&self) -> usize {
65        match self {
66            Self::Fixed { count, .. } | Self::Exponential { count, .. } => *count,
67        }
68    }
69}
70
71pub(super) fn deserialize_retry_policy<'de, D>(
72    deserializer: D,
73) -> Result<Option<RetryPolicy>, D::Error>
74where
75    D: serde::Deserializer<'de>,
76{
77    struct V;
78
79    impl<'de2> serde::de::Visitor<'de2> for V {
80        type Value = Option<RetryPolicy>;
81
82        fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
83            write!(
84                formatter,
85                "a table ({{ count = 5, backoff = \"exponential\", delay = \"1s\", max-delay = \"10s\", jitter = true }}) or a number (5)"
86            )
87        }
88
89        // Note that TOML uses i64, not u64.
90        fn visit_i64<E>(self, v: i64) -> Result<Self::Value, E>
91        where
92            E: serde::de::Error,
93        {
94            match v.cmp(&0) {
95                Ordering::Greater | Ordering::Equal => {
96                    Ok(Some(RetryPolicy::new_without_delay(v as usize)))
97                }
98                Ordering::Less => Err(serde::de::Error::invalid_value(
99                    serde::de::Unexpected::Signed(v),
100                    &self,
101                )),
102            }
103        }
104
105        fn visit_map<A>(self, map: A) -> Result<Self::Value, A::Error>
106        where
107            A: serde::de::MapAccess<'de2>,
108        {
109            RetryPolicy::deserialize(serde::de::value::MapAccessDeserializer::new(map)).map(Some)
110        }
111    }
112
113    // Post-deserialize validation of retry policy.
114    let retry_policy = deserializer.deserialize_any(V)?;
115    match &retry_policy {
116        Some(RetryPolicy::Fixed {
117            count: _,
118            delay,
119            jitter,
120        }) => {
121            // Jitter can't be specified if delay is 0.
122            if delay.is_zero() && *jitter {
123                return Err(serde::de::Error::custom(
124                    "`jitter` cannot be true if `delay` isn't specified or is zero",
125                ));
126            }
127        }
128        Some(RetryPolicy::Exponential {
129            count,
130            delay,
131            jitter: _,
132            max_delay,
133        }) => {
134            // Count can't be zero.
135            if *count == 0 {
136                return Err(serde::de::Error::custom(
137                    "`count` cannot be zero with exponential backoff",
138                ));
139            }
140            // Delay can't be zero.
141            if delay.is_zero() {
142                return Err(serde::de::Error::custom(
143                    "`delay` cannot be zero with exponential backoff",
144                ));
145            }
146            // Max delay, if specified, can't be zero.
147            if max_delay.is_some_and(|f| f.is_zero()) {
148                return Err(serde::de::Error::custom(
149                    "`max-delay` cannot be zero with exponential backoff",
150                ));
151            }
152            // Max delay can't be less than delay.
153            if max_delay.is_some_and(|max_delay| max_delay < *delay) {
154                return Err(serde::de::Error::custom(
155                    "`max-delay` cannot be less than delay with exponential backoff",
156                ));
157            }
158        }
159        None => {}
160    }
161
162    Ok(retry_policy)
163}
164
165#[cfg(test)]
166mod tests {
167    use super::*;
168    use crate::{
169        config::{
170            NextestConfig,
171            test_helpers::{binary_query, build_platforms, temp_workspace},
172        },
173        errors::ConfigParseErrorKind,
174    };
175    use camino_tempfile::tempdir;
176    use config::ConfigError;
177    use guppy::graph::cargo::BuildPlatform;
178    use indoc::indoc;
179    use nextest_filtering::{ParseContext, TestQuery};
180    use test_case::test_case;
181
182    #[test]
183    fn parse_retries_valid() {
184        let config_contents = indoc! {r#"
185            [profile.default]
186            retries = { backoff = "fixed", count = 3 }
187
188            [profile.no-retries]
189            retries = 0
190
191            [profile.fixed-with-delay]
192            retries = { backoff = "fixed", count = 3, delay = "1s" }
193
194            [profile.exp]
195            retries = { backoff = "exponential", count = 4, delay = "2s" }
196
197            [profile.exp-with-max-delay]
198            retries = { backoff = "exponential", count = 5, delay = "3s", max-delay = "10s" }
199
200            [profile.exp-with-max-delay-and-jitter]
201            retries = { backoff = "exponential", count = 6, delay = "4s", max-delay = "1m", jitter = true }
202        "#};
203
204        let workspace_dir = tempdir().unwrap();
205
206        let graph = temp_workspace(&workspace_dir, config_contents);
207        let pcx = ParseContext::new(&graph);
208
209        let config = NextestConfig::from_sources(
210            graph.workspace().root(),
211            &pcx,
212            None,
213            [],
214            &Default::default(),
215        )
216        .expect("config is valid");
217        assert_eq!(
218            config
219                .profile("default")
220                .expect("default profile exists")
221                .apply_build_platforms(&build_platforms())
222                .retries(),
223            RetryPolicy::Fixed {
224                count: 3,
225                delay: Duration::ZERO,
226                jitter: false,
227            },
228            "default retries matches"
229        );
230
231        assert_eq!(
232            config
233                .profile("no-retries")
234                .expect("profile exists")
235                .apply_build_platforms(&build_platforms())
236                .retries(),
237            RetryPolicy::new_without_delay(0),
238            "no-retries retries matches"
239        );
240
241        assert_eq!(
242            config
243                .profile("fixed-with-delay")
244                .expect("profile exists")
245                .apply_build_platforms(&build_platforms())
246                .retries(),
247            RetryPolicy::Fixed {
248                count: 3,
249                delay: Duration::from_secs(1),
250                jitter: false,
251            },
252            "fixed-with-delay retries matches"
253        );
254
255        assert_eq!(
256            config
257                .profile("exp")
258                .expect("profile exists")
259                .apply_build_platforms(&build_platforms())
260                .retries(),
261            RetryPolicy::Exponential {
262                count: 4,
263                delay: Duration::from_secs(2),
264                jitter: false,
265                max_delay: None,
266            },
267            "exp retries matches"
268        );
269
270        assert_eq!(
271            config
272                .profile("exp-with-max-delay")
273                .expect("profile exists")
274                .apply_build_platforms(&build_platforms())
275                .retries(),
276            RetryPolicy::Exponential {
277                count: 5,
278                delay: Duration::from_secs(3),
279                jitter: false,
280                max_delay: Some(Duration::from_secs(10)),
281            },
282            "exp-with-max-delay retries matches"
283        );
284
285        assert_eq!(
286            config
287                .profile("exp-with-max-delay-and-jitter")
288                .expect("profile exists")
289                .apply_build_platforms(&build_platforms())
290                .retries(),
291            RetryPolicy::Exponential {
292                count: 6,
293                delay: Duration::from_secs(4),
294                jitter: true,
295                max_delay: Some(Duration::from_secs(60)),
296            },
297            "exp-with-max-delay-and-jitter retries matches"
298        );
299    }
300
301    #[test_case(
302        indoc!{r#"
303            [profile.default]
304            retries = { backoff = "foo" }
305        "#},
306        "unknown variant `foo`, expected `fixed` or `exponential`"
307        ; "invalid value for backoff")]
308    #[test_case(
309        indoc!{r#"
310            [profile.default]
311            retries = { backoff = "fixed" }
312        "#},
313        "missing field `count`"
314        ; "fixed specified without count")]
315    #[test_case(
316        indoc!{r#"
317            [profile.default]
318            retries = { backoff = "fixed", count = 1, delay = "foobar" }
319        "#},
320        "invalid value: string \"foobar\", expected a duration"
321        ; "delay is not a valid duration")]
322    #[test_case(
323        indoc!{r#"
324            [profile.default]
325            retries = { backoff = "fixed", count = 1, jitter = true }
326        "#},
327        "`jitter` cannot be true if `delay` isn't specified or is zero"
328        ; "jitter specified without delay")]
329    #[test_case(
330        indoc!{r#"
331            [profile.default]
332            retries = { backoff = "fixed", count = 1, max-delay = "10s" }
333        "#},
334        "unknown field `max-delay`, expected one of `count`, `delay`, `jitter`"
335        ; "max-delay is incompatible with fixed backoff")]
336    #[test_case(
337        indoc!{r#"
338            [profile.default]
339            retries = { backoff = "exponential", count = 1 }
340        "#},
341        "missing field `delay`"
342        ; "exponential backoff must specify delay")]
343    #[test_case(
344        indoc!{r#"
345            [profile.default]
346            retries = { backoff = "exponential", delay = "1s" }
347        "#},
348        "missing field `count`"
349        ; "exponential backoff must specify count")]
350    #[test_case(
351        indoc!{r#"
352            [profile.default]
353            retries = { backoff = "exponential", count = 0, delay = "1s" }
354        "#},
355        "`count` cannot be zero with exponential backoff"
356        ; "exponential backoff must have a non-zero count")]
357    #[test_case(
358        indoc!{r#"
359            [profile.default]
360            retries = { backoff = "exponential", count = 1, delay = "0s" }
361        "#},
362        "`delay` cannot be zero with exponential backoff"
363        ; "exponential backoff must have a non-zero delay")]
364    #[test_case(
365        indoc!{r#"
366            [profile.default]
367            retries = { backoff = "exponential", count = 1, delay = "1s", max-delay = "0s" }
368        "#},
369        "`max-delay` cannot be zero with exponential backoff"
370        ; "exponential backoff must have a non-zero max delay")]
371    #[test_case(
372        indoc!{r#"
373            [profile.default]
374            retries = { backoff = "exponential", count = 1, delay = "4s", max-delay = "2s", jitter = true }
375        "#},
376        "`max-delay` cannot be less than delay"
377        ; "max-delay greater than delay")]
378    fn parse_retries_invalid(config_contents: &str, expected_message: &str) {
379        let workspace_dir = tempdir().unwrap();
380
381        let graph = temp_workspace(&workspace_dir, config_contents);
382        let pcx = ParseContext::new(&graph);
383
384        let config_err = NextestConfig::from_sources(
385            graph.workspace().root(),
386            &pcx,
387            None,
388            [],
389            &Default::default(),
390        )
391        .expect_err("config expected to be invalid");
392
393        let message = match config_err.kind() {
394            ConfigParseErrorKind::DeserializeError(path_error) => match path_error.inner() {
395                ConfigError::Message(message) => message,
396                other => {
397                    panic!(
398                        "for config error {config_err:?}, expected ConfigError::Message for inner error {other:?}"
399                    );
400                }
401            },
402            other => {
403                panic!(
404                    "for config error {other:?}, expected ConfigParseErrorKind::DeserializeError"
405                );
406            }
407        };
408
409        assert!(
410            message.contains(expected_message),
411            "expected message \"{message}\" to contain \"{expected_message}\""
412        );
413    }
414
415    #[test_case(
416        indoc! {r#"
417            [[profile.default.overrides]]
418            filter = "test(=my_test)"
419            retries = 2
420
421            [profile.ci]
422        "#},
423        BuildPlatform::Target,
424        RetryPolicy::new_without_delay(2)
425
426        ; "my_test matches exactly"
427    )]
428    #[test_case(
429        indoc! {r#"
430            [[profile.default.overrides]]
431            filter = "!test(=my_test)"
432            retries = 2
433
434            [profile.ci]
435        "#},
436        BuildPlatform::Target,
437        RetryPolicy::new_without_delay(0)
438
439        ; "not match"
440    )]
441    #[test_case(
442        indoc! {r#"
443            [[profile.default.overrides]]
444            filter = "test(=my_test)"
445
446            [profile.ci]
447        "#},
448        BuildPlatform::Target,
449        RetryPolicy::new_without_delay(0)
450
451        ; "no retries specified"
452    )]
453    #[test_case(
454        indoc! {r#"
455            [[profile.default.overrides]]
456            filter = "test(test)"
457            retries = 2
458
459            [[profile.default.overrides]]
460            filter = "test(=my_test)"
461            retries = 3
462
463            [profile.ci]
464        "#},
465        BuildPlatform::Target,
466        RetryPolicy::new_without_delay(2)
467
468        ; "earlier configs override later ones"
469    )]
470    #[test_case(
471        indoc! {r#"
472            [[profile.default.overrides]]
473            filter = "test(test)"
474            retries = 2
475
476            [profile.ci]
477
478            [[profile.ci.overrides]]
479            filter = "test(=my_test)"
480            retries = 3
481        "#},
482        BuildPlatform::Target,
483        RetryPolicy::new_without_delay(3)
484
485        ; "profile-specific configs override default ones"
486    )]
487    #[test_case(
488        indoc! {r#"
489            [[profile.default.overrides]]
490            filter = "(!package(test-package)) and test(test)"
491            retries = 2
492
493            [profile.ci]
494
495            [[profile.ci.overrides]]
496            filter = "!test(=my_test_2)"
497            retries = 3
498        "#},
499        BuildPlatform::Target,
500        RetryPolicy::new_without_delay(3)
501
502        ; "no overrides match my_test exactly"
503    )]
504    #[test_case(
505        indoc! {r#"
506            [[profile.default.overrides]]
507            platform = "x86_64-unknown-linux-gnu"
508            filter = "test(test)"
509            retries = 2
510
511            [[profile.default.overrides]]
512            filter = "test(=my_test)"
513            retries = 3
514
515            [profile.ci]
516        "#},
517        BuildPlatform::Host,
518        RetryPolicy::new_without_delay(2)
519
520        ; "earlier config applied because it matches host triple"
521    )]
522    #[test_case(
523        indoc! {r#"
524            [[profile.default.overrides]]
525            platform = "aarch64-apple-darwin"
526            filter = "test(test)"
527            retries = 2
528
529            [[profile.default.overrides]]
530            filter = "test(=my_test)"
531            retries = 3
532
533            [profile.ci]
534        "#},
535        BuildPlatform::Host,
536        RetryPolicy::new_without_delay(3)
537
538        ; "earlier config ignored because it doesn't match host triple"
539    )]
540    #[test_case(
541        indoc! {r#"
542            [[profile.default.overrides]]
543            platform = "aarch64-apple-darwin"
544            filter = "test(test)"
545            retries = 2
546
547            [[profile.default.overrides]]
548            filter = "test(=my_test)"
549            retries = 3
550
551            [profile.ci]
552        "#},
553        BuildPlatform::Target,
554        RetryPolicy::new_without_delay(2)
555
556        ; "earlier config applied because it matches target triple"
557    )]
558    #[test_case(
559        indoc! {r#"
560            [[profile.default.overrides]]
561            platform = "x86_64-unknown-linux-gnu"
562            filter = "test(test)"
563            retries = 2
564
565            [[profile.default.overrides]]
566            filter = "test(=my_test)"
567            retries = 3
568
569            [profile.ci]
570        "#},
571        BuildPlatform::Target,
572        RetryPolicy::new_without_delay(3)
573
574        ; "earlier config ignored because it doesn't match target triple"
575    )]
576    #[test_case(
577        indoc! {r#"
578            [[profile.default.overrides]]
579            platform = 'cfg(target_os = "macos")'
580            filter = "test(test)"
581            retries = 2
582
583            [[profile.default.overrides]]
584            filter = "test(=my_test)"
585            retries = 3
586
587            [profile.ci]
588        "#},
589        BuildPlatform::Target,
590        RetryPolicy::new_without_delay(2)
591
592        ; "earlier config applied because it matches target cfg expr"
593    )]
594    #[test_case(
595        indoc! {r#"
596            [[profile.default.overrides]]
597            platform = 'cfg(target_arch = "x86_64")'
598            filter = "test(test)"
599            retries = 2
600
601            [[profile.default.overrides]]
602            filter = "test(=my_test)"
603            retries = 3
604
605            [profile.ci]
606        "#},
607        BuildPlatform::Target,
608        RetryPolicy::new_without_delay(3)
609
610        ; "earlier config ignored because it doesn't match target cfg expr"
611    )]
612    fn overrides_retries(
613        config_contents: &str,
614        build_platform: BuildPlatform,
615        retries: RetryPolicy,
616    ) {
617        let workspace_dir = tempdir().unwrap();
618
619        let graph = temp_workspace(&workspace_dir, config_contents);
620        let package_id = graph.workspace().iter().next().unwrap().id();
621        let pcx = ParseContext::new(&graph);
622
623        let config = NextestConfig::from_sources(
624            graph.workspace().root(),
625            &pcx,
626            None,
627            &[][..],
628            &Default::default(),
629        )
630        .unwrap();
631        let binary_query = binary_query(&graph, package_id, "lib", "my-binary", build_platform);
632        let query = TestQuery {
633            binary_query: binary_query.to_query(),
634            test_name: "my_test",
635        };
636        let profile = config
637            .profile("ci")
638            .expect("ci profile is defined")
639            .apply_build_platforms(&build_platforms());
640        let settings_for = profile.settings_for(&query);
641        assert_eq!(
642            settings_for.retries(),
643            retries,
644            "actual retries don't match expected retries"
645        );
646    }
647}
nextest_runner/config/retry_policy.rs

nextest_runner/config/
retry_policy.rs
