nextest_runner/config/elements/
retry_policy.rs

1// Copyright (c) The nextest Contributors
2// SPDX-License-Identifier: MIT OR Apache-2.0
3
4use serde::Deserialize;
5use std::{cmp::Ordering, fmt, time::Duration};
6
7/// Type for the retry config key.
8#[derive(Debug, Copy, Clone, Deserialize, PartialEq, Eq)]
9#[serde(tag = "backoff", rename_all = "kebab-case", deny_unknown_fields)]
10pub enum RetryPolicy {
11    /// Fixed backoff.
12    #[serde(rename_all = "kebab-case")]
13    Fixed {
14        /// Maximum retry count.
15        count: usize,
16
17        /// Delay between retries.
18        #[serde(default, with = "humantime_serde")]
19        delay: Duration,
20
21        /// If set to true, randomness will be added to the delay on each retry attempt.
22        #[serde(default)]
23        jitter: bool,
24    },
25
26    /// Exponential backoff.
27    #[serde(rename_all = "kebab-case")]
28    Exponential {
29        /// Maximum retry count.
30        count: usize,
31
32        /// Delay between retries. Not optional for exponential backoff.
33        #[serde(with = "humantime_serde")]
34        delay: Duration,
35
36        /// If set to true, randomness will be added to the delay on each retry attempt.
37        #[serde(default)]
38        jitter: bool,
39
40        /// If set, limits the delay between retries.
41        #[serde(default, with = "humantime_serde")]
42        max_delay: Option<Duration>,
43    },
44}
45
46impl Default for RetryPolicy {
47    #[inline]
48    fn default() -> Self {
49        Self::new_without_delay(0)
50    }
51}
52
53impl RetryPolicy {
54    /// Create new policy with no delay between retries.
55    pub fn new_without_delay(count: usize) -> Self {
56        Self::Fixed {
57            count,
58            delay: Duration::ZERO,
59            jitter: false,
60        }
61    }
62
63    /// Returns the number of retries.
64    pub fn count(&self) -> usize {
65        match self {
66            Self::Fixed { count, .. } | Self::Exponential { count, .. } => *count,
67        }
68    }
69}
70
71pub(in crate::config) fn deserialize_retry_policy<'de, D>(
72    deserializer: D,
73) -> Result<Option<RetryPolicy>, D::Error>
74where
75    D: serde::Deserializer<'de>,
76{
77    struct V;
78
79    impl<'de2> serde::de::Visitor<'de2> for V {
80        type Value = Option<RetryPolicy>;
81
82        fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
83            write!(
84                formatter,
85                "a table ({{ count = 5, backoff = \"exponential\", delay = \"1s\", max-delay = \"10s\", jitter = true }}) or a number (5)"
86            )
87        }
88
89        // Note that TOML uses i64, not u64.
90        fn visit_i64<E>(self, v: i64) -> Result<Self::Value, E>
91        where
92            E: serde::de::Error,
93        {
94            match v.cmp(&0) {
95                Ordering::Greater | Ordering::Equal => {
96                    Ok(Some(RetryPolicy::new_without_delay(v as usize)))
97                }
98                Ordering::Less => Err(serde::de::Error::invalid_value(
99                    serde::de::Unexpected::Signed(v),
100                    &self,
101                )),
102            }
103        }
104
105        fn visit_map<A>(self, map: A) -> Result<Self::Value, A::Error>
106        where
107            A: serde::de::MapAccess<'de2>,
108        {
109            RetryPolicy::deserialize(serde::de::value::MapAccessDeserializer::new(map)).map(Some)
110        }
111    }
112
113    // Post-deserialize validation of retry policy.
114    let retry_policy = deserializer.deserialize_any(V)?;
115    match &retry_policy {
116        Some(RetryPolicy::Fixed {
117            count: _,
118            delay,
119            jitter,
120        }) => {
121            // Jitter can't be specified if delay is 0.
122            if delay.is_zero() && *jitter {
123                return Err(serde::de::Error::custom(
124                    "`jitter` cannot be true if `delay` isn't specified or is zero",
125                ));
126            }
127        }
128        Some(RetryPolicy::Exponential {
129            count,
130            delay,
131            jitter: _,
132            max_delay,
133        }) => {
134            // Count can't be zero.
135            if *count == 0 {
136                return Err(serde::de::Error::custom(
137                    "`count` cannot be zero with exponential backoff",
138                ));
139            }
140            // Delay can't be zero.
141            if delay.is_zero() {
142                return Err(serde::de::Error::custom(
143                    "`delay` cannot be zero with exponential backoff",
144                ));
145            }
146            // Max delay, if specified, can't be zero.
147            if max_delay.is_some_and(|f| f.is_zero()) {
148                return Err(serde::de::Error::custom(
149                    "`max-delay` cannot be zero with exponential backoff",
150                ));
151            }
152            // Max delay can't be less than delay.
153            if max_delay.is_some_and(|max_delay| max_delay < *delay) {
154                return Err(serde::de::Error::custom(
155                    "`max-delay` cannot be less than delay with exponential backoff",
156                ));
157            }
158        }
159        None => {}
160    }
161
162    Ok(retry_policy)
163}
164
165#[cfg(test)]
166mod tests {
167    use super::*;
168    use crate::{
169        config::{core::NextestConfig, utils::test_helpers::*},
170        errors::ConfigParseErrorKind,
171    };
172    use camino_tempfile::tempdir;
173    use config::ConfigError;
174    use guppy::graph::cargo::BuildPlatform;
175    use indoc::indoc;
176    use nextest_filtering::{ParseContext, TestQuery};
177    use test_case::test_case;
178
179    #[test]
180    fn parse_retries_valid() {
181        let config_contents = indoc! {r#"
182            [profile.default]
183            retries = { backoff = "fixed", count = 3 }
184
185            [profile.no-retries]
186            retries = 0
187
188            [profile.fixed-with-delay]
189            retries = { backoff = "fixed", count = 3, delay = "1s" }
190
191            [profile.exp]
192            retries = { backoff = "exponential", count = 4, delay = "2s" }
193
194            [profile.exp-with-max-delay]
195            retries = { backoff = "exponential", count = 5, delay = "3s", max-delay = "10s" }
196
197            [profile.exp-with-max-delay-and-jitter]
198            retries = { backoff = "exponential", count = 6, delay = "4s", max-delay = "1m", jitter = true }
199        "#};
200
201        let workspace_dir = tempdir().unwrap();
202
203        let graph = temp_workspace(&workspace_dir, config_contents);
204        let pcx = ParseContext::new(&graph);
205
206        let config = NextestConfig::from_sources(
207            graph.workspace().root(),
208            &pcx,
209            None,
210            [],
211            &Default::default(),
212        )
213        .expect("config is valid");
214        assert_eq!(
215            config
216                .profile("default")
217                .expect("default profile exists")
218                .apply_build_platforms(&build_platforms())
219                .retries(),
220            RetryPolicy::Fixed {
221                count: 3,
222                delay: Duration::ZERO,
223                jitter: false,
224            },
225            "default retries matches"
226        );
227
228        assert_eq!(
229            config
230                .profile("no-retries")
231                .expect("profile exists")
232                .apply_build_platforms(&build_platforms())
233                .retries(),
234            RetryPolicy::new_without_delay(0),
235            "no-retries retries matches"
236        );
237
238        assert_eq!(
239            config
240                .profile("fixed-with-delay")
241                .expect("profile exists")
242                .apply_build_platforms(&build_platforms())
243                .retries(),
244            RetryPolicy::Fixed {
245                count: 3,
246                delay: Duration::from_secs(1),
247                jitter: false,
248            },
249            "fixed-with-delay retries matches"
250        );
251
252        assert_eq!(
253            config
254                .profile("exp")
255                .expect("profile exists")
256                .apply_build_platforms(&build_platforms())
257                .retries(),
258            RetryPolicy::Exponential {
259                count: 4,
260                delay: Duration::from_secs(2),
261                jitter: false,
262                max_delay: None,
263            },
264            "exp retries matches"
265        );
266
267        assert_eq!(
268            config
269                .profile("exp-with-max-delay")
270                .expect("profile exists")
271                .apply_build_platforms(&build_platforms())
272                .retries(),
273            RetryPolicy::Exponential {
274                count: 5,
275                delay: Duration::from_secs(3),
276                jitter: false,
277                max_delay: Some(Duration::from_secs(10)),
278            },
279            "exp-with-max-delay retries matches"
280        );
281
282        assert_eq!(
283            config
284                .profile("exp-with-max-delay-and-jitter")
285                .expect("profile exists")
286                .apply_build_platforms(&build_platforms())
287                .retries(),
288            RetryPolicy::Exponential {
289                count: 6,
290                delay: Duration::from_secs(4),
291                jitter: true,
292                max_delay: Some(Duration::from_secs(60)),
293            },
294            "exp-with-max-delay-and-jitter retries matches"
295        );
296    }
297
298    #[test_case(
299        indoc!{r#"
300            [profile.default]
301            retries = { backoff = "foo" }
302        "#},
303        "unknown variant `foo`, expected `fixed` or `exponential`"
304        ; "invalid value for backoff")]
305    #[test_case(
306        indoc!{r#"
307            [profile.default]
308            retries = { backoff = "fixed" }
309        "#},
310        "missing field `count`"
311        ; "fixed specified without count")]
312    #[test_case(
313        indoc!{r#"
314            [profile.default]
315            retries = { backoff = "fixed", count = 1, delay = "foobar" }
316        "#},
317        "invalid value: string \"foobar\", expected a duration"
318        ; "delay is not a valid duration")]
319    #[test_case(
320        indoc!{r#"
321            [profile.default]
322            retries = { backoff = "fixed", count = 1, jitter = true }
323        "#},
324        "`jitter` cannot be true if `delay` isn't specified or is zero"
325        ; "jitter specified without delay")]
326    #[test_case(
327        indoc!{r#"
328            [profile.default]
329            retries = { backoff = "fixed", count = 1, max-delay = "10s" }
330        "#},
331        "unknown field `max-delay`, expected one of `count`, `delay`, `jitter`"
332        ; "max-delay is incompatible with fixed backoff")]
333    #[test_case(
334        indoc!{r#"
335            [profile.default]
336            retries = { backoff = "exponential", count = 1 }
337        "#},
338        "missing field `delay`"
339        ; "exponential backoff must specify delay")]
340    #[test_case(
341        indoc!{r#"
342            [profile.default]
343            retries = { backoff = "exponential", delay = "1s" }
344        "#},
345        "missing field `count`"
346        ; "exponential backoff must specify count")]
347    #[test_case(
348        indoc!{r#"
349            [profile.default]
350            retries = { backoff = "exponential", count = 0, delay = "1s" }
351        "#},
352        "`count` cannot be zero with exponential backoff"
353        ; "exponential backoff must have a non-zero count")]
354    #[test_case(
355        indoc!{r#"
356            [profile.default]
357            retries = { backoff = "exponential", count = 1, delay = "0s" }
358        "#},
359        "`delay` cannot be zero with exponential backoff"
360        ; "exponential backoff must have a non-zero delay")]
361    #[test_case(
362        indoc!{r#"
363            [profile.default]
364            retries = { backoff = "exponential", count = 1, delay = "1s", max-delay = "0s" }
365        "#},
366        "`max-delay` cannot be zero with exponential backoff"
367        ; "exponential backoff must have a non-zero max delay")]
368    #[test_case(
369        indoc!{r#"
370            [profile.default]
371            retries = { backoff = "exponential", count = 1, delay = "4s", max-delay = "2s", jitter = true }
372        "#},
373        "`max-delay` cannot be less than delay"
374        ; "max-delay greater than delay")]
375    fn parse_retries_invalid(config_contents: &str, expected_message: &str) {
376        let workspace_dir = tempdir().unwrap();
377
378        let graph = temp_workspace(&workspace_dir, config_contents);
379        let pcx = ParseContext::new(&graph);
380
381        let config_err = NextestConfig::from_sources(
382            graph.workspace().root(),
383            &pcx,
384            None,
385            [],
386            &Default::default(),
387        )
388        .expect_err("config expected to be invalid");
389
390        let message = match config_err.kind() {
391            ConfigParseErrorKind::DeserializeError(path_error) => match path_error.inner() {
392                ConfigError::Message(message) => message,
393                other => {
394                    panic!(
395                        "for config error {config_err:?}, expected ConfigError::Message for inner error {other:?}"
396                    );
397                }
398            },
399            other => {
400                panic!(
401                    "for config error {other:?}, expected ConfigParseErrorKind::DeserializeError"
402                );
403            }
404        };
405
406        assert!(
407            message.contains(expected_message),
408            "expected message \"{message}\" to contain \"{expected_message}\""
409        );
410    }
411
412    #[test_case(
413        indoc! {r#"
414            [[profile.default.overrides]]
415            filter = "test(=my_test)"
416            retries = 2
417
418            [profile.ci]
419        "#},
420        BuildPlatform::Target,
421        RetryPolicy::new_without_delay(2)
422
423        ; "my_test matches exactly"
424    )]
425    #[test_case(
426        indoc! {r#"
427            [[profile.default.overrides]]
428            filter = "!test(=my_test)"
429            retries = 2
430
431            [profile.ci]
432        "#},
433        BuildPlatform::Target,
434        RetryPolicy::new_without_delay(0)
435
436        ; "not match"
437    )]
438    #[test_case(
439        indoc! {r#"
440            [[profile.default.overrides]]
441            filter = "test(=my_test)"
442
443            [profile.ci]
444        "#},
445        BuildPlatform::Target,
446        RetryPolicy::new_without_delay(0)
447
448        ; "no retries specified"
449    )]
450    #[test_case(
451        indoc! {r#"
452            [[profile.default.overrides]]
453            filter = "test(test)"
454            retries = 2
455
456            [[profile.default.overrides]]
457            filter = "test(=my_test)"
458            retries = 3
459
460            [profile.ci]
461        "#},
462        BuildPlatform::Target,
463        RetryPolicy::new_without_delay(2)
464
465        ; "earlier configs override later ones"
466    )]
467    #[test_case(
468        indoc! {r#"
469            [[profile.default.overrides]]
470            filter = "test(test)"
471            retries = 2
472
473            [profile.ci]
474
475            [[profile.ci.overrides]]
476            filter = "test(=my_test)"
477            retries = 3
478        "#},
479        BuildPlatform::Target,
480        RetryPolicy::new_without_delay(3)
481
482        ; "profile-specific configs override default ones"
483    )]
484    #[test_case(
485        indoc! {r#"
486            [[profile.default.overrides]]
487            filter = "(!package(test-package)) and test(test)"
488            retries = 2
489
490            [profile.ci]
491
492            [[profile.ci.overrides]]
493            filter = "!test(=my_test_2)"
494            retries = 3
495        "#},
496        BuildPlatform::Target,
497        RetryPolicy::new_without_delay(3)
498
499        ; "no overrides match my_test exactly"
500    )]
501    #[test_case(
502        indoc! {r#"
503            [[profile.default.overrides]]
504            platform = "x86_64-unknown-linux-gnu"
505            filter = "test(test)"
506            retries = 2
507
508            [[profile.default.overrides]]
509            filter = "test(=my_test)"
510            retries = 3
511
512            [profile.ci]
513        "#},
514        BuildPlatform::Host,
515        RetryPolicy::new_without_delay(2)
516
517        ; "earlier config applied because it matches host triple"
518    )]
519    #[test_case(
520        indoc! {r#"
521            [[profile.default.overrides]]
522            platform = "aarch64-apple-darwin"
523            filter = "test(test)"
524            retries = 2
525
526            [[profile.default.overrides]]
527            filter = "test(=my_test)"
528            retries = 3
529
530            [profile.ci]
531        "#},
532        BuildPlatform::Host,
533        RetryPolicy::new_without_delay(3)
534
535        ; "earlier config ignored because it doesn't match host triple"
536    )]
537    #[test_case(
538        indoc! {r#"
539            [[profile.default.overrides]]
540            platform = "aarch64-apple-darwin"
541            filter = "test(test)"
542            retries = 2
543
544            [[profile.default.overrides]]
545            filter = "test(=my_test)"
546            retries = 3
547
548            [profile.ci]
549        "#},
550        BuildPlatform::Target,
551        RetryPolicy::new_without_delay(2)
552
553        ; "earlier config applied because it matches target triple"
554    )]
555    #[test_case(
556        indoc! {r#"
557            [[profile.default.overrides]]
558            platform = "x86_64-unknown-linux-gnu"
559            filter = "test(test)"
560            retries = 2
561
562            [[profile.default.overrides]]
563            filter = "test(=my_test)"
564            retries = 3
565
566            [profile.ci]
567        "#},
568        BuildPlatform::Target,
569        RetryPolicy::new_without_delay(3)
570
571        ; "earlier config ignored because it doesn't match target triple"
572    )]
573    #[test_case(
574        indoc! {r#"
575            [[profile.default.overrides]]
576            platform = 'cfg(target_os = "macos")'
577            filter = "test(test)"
578            retries = 2
579
580            [[profile.default.overrides]]
581            filter = "test(=my_test)"
582            retries = 3
583
584            [profile.ci]
585        "#},
586        BuildPlatform::Target,
587        RetryPolicy::new_without_delay(2)
588
589        ; "earlier config applied because it matches target cfg expr"
590    )]
591    #[test_case(
592        indoc! {r#"
593            [[profile.default.overrides]]
594            platform = 'cfg(target_arch = "x86_64")'
595            filter = "test(test)"
596            retries = 2
597
598            [[profile.default.overrides]]
599            filter = "test(=my_test)"
600            retries = 3
601
602            [profile.ci]
603        "#},
604        BuildPlatform::Target,
605        RetryPolicy::new_without_delay(3)
606
607        ; "earlier config ignored because it doesn't match target cfg expr"
608    )]
609    fn overrides_retries(
610        config_contents: &str,
611        build_platform: BuildPlatform,
612        retries: RetryPolicy,
613    ) {
614        let workspace_dir = tempdir().unwrap();
615
616        let graph = temp_workspace(&workspace_dir, config_contents);
617        let package_id = graph.workspace().iter().next().unwrap().id();
618        let pcx = ParseContext::new(&graph);
619
620        let config = NextestConfig::from_sources(
621            graph.workspace().root(),
622            &pcx,
623            None,
624            &[][..],
625            &Default::default(),
626        )
627        .unwrap();
628        let binary_query = binary_query(&graph, package_id, "lib", "my-binary", build_platform);
629        let query = TestQuery {
630            binary_query: binary_query.to_query(),
631            test_name: "my_test",
632        };
633        let profile = config
634            .profile("ci")
635            .expect("ci profile is defined")
636            .apply_build_platforms(&build_platforms());
637        let settings_for = profile.settings_for(&query);
638        assert_eq!(
639            settings_for.retries(),
640            retries,
641            "actual retries don't match expected retries"
642        );
643    }
644}
nextest_runner/config/elements/retry_policy.rs

nextest_runner/config/elements/
retry_policy.rs
