nextest_runner/config/elements/
retry_policy.rs

1// Copyright (c) The nextest Contributors
2// SPDX-License-Identifier: MIT OR Apache-2.0
3
4use serde::Deserialize;
5use std::{cmp::Ordering, fmt, time::Duration};
6
7/// Type for the retry config key.
8#[derive(Debug, Copy, Clone, Deserialize, PartialEq, Eq)]
9#[serde(tag = "backoff", rename_all = "kebab-case", deny_unknown_fields)]
10pub enum RetryPolicy {
11    /// Fixed backoff.
12    #[serde(rename_all = "kebab-case")]
13    Fixed {
14        /// Maximum retry count.
15        count: u32,
16
17        /// Delay between retries.
18        #[serde(default, with = "humantime_serde")]
19        delay: Duration,
20
21        /// If set to true, randomness will be added to the delay on each retry attempt.
22        #[serde(default)]
23        jitter: bool,
24    },
25
26    /// Exponential backoff.
27    #[serde(rename_all = "kebab-case")]
28    Exponential {
29        /// Maximum retry count.
30        count: u32,
31
32        /// Delay between retries. Not optional for exponential backoff.
33        #[serde(with = "humantime_serde")]
34        delay: Duration,
35
36        /// If set to true, randomness will be added to the delay on each retry attempt.
37        #[serde(default)]
38        jitter: bool,
39
40        /// If set, limits the delay between retries.
41        #[serde(default, with = "humantime_serde")]
42        max_delay: Option<Duration>,
43    },
44}
45
46impl Default for RetryPolicy {
47    #[inline]
48    fn default() -> Self {
49        Self::new_without_delay(0)
50    }
51}
52
53impl RetryPolicy {
54    /// Create new policy with no delay between retries.
55    pub fn new_without_delay(count: u32) -> Self {
56        Self::Fixed {
57            count,
58            delay: Duration::ZERO,
59            jitter: false,
60        }
61    }
62
63    /// Returns the number of retries.
64    pub fn count(&self) -> u32 {
65        match self {
66            Self::Fixed { count, .. } | Self::Exponential { count, .. } => *count,
67        }
68    }
69}
70
71pub(in crate::config) fn deserialize_retry_policy<'de, D>(
72    deserializer: D,
73) -> Result<Option<RetryPolicy>, D::Error>
74where
75    D: serde::Deserializer<'de>,
76{
77    struct V;
78
79    impl<'de2> serde::de::Visitor<'de2> for V {
80        type Value = Option<RetryPolicy>;
81
82        fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
83            write!(
84                formatter,
85                "a table ({{ count = 5, backoff = \"exponential\", delay = \"1s\", max-delay = \"10s\", jitter = true }}) or a number (5)"
86            )
87        }
88
89        // Note that TOML uses i64, not u64.
90        fn visit_i64<E>(self, v: i64) -> Result<Self::Value, E>
91        where
92            E: serde::de::Error,
93        {
94            match v.cmp(&0) {
95                Ordering::Greater | Ordering::Equal => {
96                    let v = u32::try_from(v).map_err(|_| {
97                        serde::de::Error::invalid_value(
98                            serde::de::Unexpected::Signed(v),
99                            &"a positive u32",
100                        )
101                    })?;
102                    Ok(Some(RetryPolicy::new_without_delay(v)))
103                }
104                Ordering::Less => Err(serde::de::Error::invalid_value(
105                    serde::de::Unexpected::Signed(v),
106                    &self,
107                )),
108            }
109        }
110
111        fn visit_map<A>(self, map: A) -> Result<Self::Value, A::Error>
112        where
113            A: serde::de::MapAccess<'de2>,
114        {
115            RetryPolicy::deserialize(serde::de::value::MapAccessDeserializer::new(map)).map(Some)
116        }
117    }
118
119    // Post-deserialize validation of retry policy.
120    let retry_policy = deserializer.deserialize_any(V)?;
121    match &retry_policy {
122        Some(RetryPolicy::Fixed {
123            count: _,
124            delay,
125            jitter,
126        }) => {
127            // Jitter can't be specified if delay is 0.
128            if delay.is_zero() && *jitter {
129                return Err(serde::de::Error::custom(
130                    "`jitter` cannot be true if `delay` isn't specified or is zero",
131                ));
132            }
133        }
134        Some(RetryPolicy::Exponential {
135            count,
136            delay,
137            jitter: _,
138            max_delay,
139        }) => {
140            // Count can't be zero.
141            if *count == 0 {
142                return Err(serde::de::Error::custom(
143                    "`count` cannot be zero with exponential backoff",
144                ));
145            }
146            // Delay can't be zero.
147            if delay.is_zero() {
148                return Err(serde::de::Error::custom(
149                    "`delay` cannot be zero with exponential backoff",
150                ));
151            }
152            // Max delay, if specified, can't be zero.
153            if max_delay.is_some_and(|f| f.is_zero()) {
154                return Err(serde::de::Error::custom(
155                    "`max-delay` cannot be zero with exponential backoff",
156                ));
157            }
158            // Max delay can't be less than delay.
159            if max_delay.is_some_and(|max_delay| max_delay < *delay) {
160                return Err(serde::de::Error::custom(
161                    "`max-delay` cannot be less than delay with exponential backoff",
162                ));
163            }
164        }
165        None => {}
166    }
167
168    Ok(retry_policy)
169}
170
171#[cfg(test)]
172mod tests {
173    use super::*;
174    use crate::{
175        config::{core::NextestConfig, utils::test_helpers::*},
176        errors::ConfigParseErrorKind,
177        run_mode::NextestRunMode,
178    };
179    use camino_tempfile::tempdir;
180    use config::ConfigError;
181    use guppy::graph::cargo::BuildPlatform;
182    use indoc::indoc;
183    use nextest_filtering::{ParseContext, TestQuery};
184    use nextest_metadata::TestCaseName;
185    use test_case::test_case;
186
187    #[test]
188    fn parse_retries_valid() {
189        let config_contents = indoc! {r#"
190            [profile.default]
191            retries = { backoff = "fixed", count = 3 }
192
193            [profile.no-retries]
194            retries = 0
195
196            [profile.fixed-with-delay]
197            retries = { backoff = "fixed", count = 3, delay = "1s" }
198
199            [profile.exp]
200            retries = { backoff = "exponential", count = 4, delay = "2s" }
201
202            [profile.exp-with-max-delay]
203            retries = { backoff = "exponential", count = 5, delay = "3s", max-delay = "10s" }
204
205            [profile.exp-with-max-delay-and-jitter]
206            retries = { backoff = "exponential", count = 6, delay = "4s", max-delay = "1m", jitter = true }
207        "#};
208
209        let workspace_dir = tempdir().unwrap();
210
211        let graph = temp_workspace(&workspace_dir, config_contents);
212        let pcx = ParseContext::new(&graph);
213
214        let config = NextestConfig::from_sources(
215            graph.workspace().root(),
216            &pcx,
217            None,
218            [],
219            &Default::default(),
220        )
221        .expect("config is valid");
222        assert_eq!(
223            config
224                .profile("default")
225                .expect("default profile exists")
226                .apply_build_platforms(&build_platforms())
227                .retries(),
228            RetryPolicy::Fixed {
229                count: 3,
230                delay: Duration::ZERO,
231                jitter: false,
232            },
233            "default retries matches"
234        );
235
236        assert_eq!(
237            config
238                .profile("no-retries")
239                .expect("profile exists")
240                .apply_build_platforms(&build_platforms())
241                .retries(),
242            RetryPolicy::new_without_delay(0),
243            "no-retries retries matches"
244        );
245
246        assert_eq!(
247            config
248                .profile("fixed-with-delay")
249                .expect("profile exists")
250                .apply_build_platforms(&build_platforms())
251                .retries(),
252            RetryPolicy::Fixed {
253                count: 3,
254                delay: Duration::from_secs(1),
255                jitter: false,
256            },
257            "fixed-with-delay retries matches"
258        );
259
260        assert_eq!(
261            config
262                .profile("exp")
263                .expect("profile exists")
264                .apply_build_platforms(&build_platforms())
265                .retries(),
266            RetryPolicy::Exponential {
267                count: 4,
268                delay: Duration::from_secs(2),
269                jitter: false,
270                max_delay: None,
271            },
272            "exp retries matches"
273        );
274
275        assert_eq!(
276            config
277                .profile("exp-with-max-delay")
278                .expect("profile exists")
279                .apply_build_platforms(&build_platforms())
280                .retries(),
281            RetryPolicy::Exponential {
282                count: 5,
283                delay: Duration::from_secs(3),
284                jitter: false,
285                max_delay: Some(Duration::from_secs(10)),
286            },
287            "exp-with-max-delay retries matches"
288        );
289
290        assert_eq!(
291            config
292                .profile("exp-with-max-delay-and-jitter")
293                .expect("profile exists")
294                .apply_build_platforms(&build_platforms())
295                .retries(),
296            RetryPolicy::Exponential {
297                count: 6,
298                delay: Duration::from_secs(4),
299                jitter: true,
300                max_delay: Some(Duration::from_secs(60)),
301            },
302            "exp-with-max-delay-and-jitter retries matches"
303        );
304    }
305
306    #[test_case(
307        indoc!{r#"
308            [profile.default]
309            retries = { backoff = "foo" }
310        "#},
311        ConfigErrorKind::Message,
312        "unknown variant `foo`, expected `fixed` or `exponential`"
313        ; "invalid value for backoff")]
314    #[test_case(
315        indoc!{r#"
316            [profile.default]
317            retries = { backoff = "fixed" }
318        "#},
319        ConfigErrorKind::NotFound,
320        "profile.default.retries.count"
321        ; "fixed specified without count")]
322    #[test_case(
323        indoc!{r#"
324            [profile.default]
325            retries = { backoff = "fixed", count = 1, delay = "foobar" }
326        "#},
327        ConfigErrorKind::Message,
328        "invalid value: string \"foobar\", expected a duration"
329        ; "delay is not a valid duration")]
330    #[test_case(
331        indoc!{r#"
332            [profile.default]
333            retries = { backoff = "fixed", count = 1, jitter = true }
334        "#},
335        ConfigErrorKind::Message,
336        "`jitter` cannot be true if `delay` isn't specified or is zero"
337        ; "jitter specified without delay")]
338    #[test_case(
339        indoc!{r#"
340            [profile.default]
341            retries = { backoff = "fixed", count = 1, max-delay = "10s" }
342        "#},
343        ConfigErrorKind::Message,
344        "unknown field `max-delay`, expected one of `count`, `delay`, `jitter`"
345        ; "max-delay is incompatible with fixed backoff")]
346    #[test_case(
347        indoc!{r#"
348            [profile.default]
349            retries = { backoff = "exponential", count = 1 }
350        "#},
351        ConfigErrorKind::NotFound,
352        "profile.default.retries.delay"
353        ; "exponential backoff must specify delay")]
354    #[test_case(
355        indoc!{r#"
356            [profile.default]
357            retries = { backoff = "exponential", delay = "1s" }
358        "#},
359        ConfigErrorKind::NotFound,
360        "profile.default.retries.count"
361        ; "exponential backoff must specify count")]
362    #[test_case(
363        indoc!{r#"
364            [profile.default]
365            retries = { backoff = "exponential", count = 0, delay = "1s" }
366        "#},
367        ConfigErrorKind::Message,
368        "`count` cannot be zero with exponential backoff"
369        ; "exponential backoff must have a non-zero count")]
370    #[test_case(
371        indoc!{r#"
372            [profile.default]
373            retries = { backoff = "exponential", count = 1, delay = "0s" }
374        "#},
375        ConfigErrorKind::Message,
376        "`delay` cannot be zero with exponential backoff"
377        ; "exponential backoff must have a non-zero delay")]
378    #[test_case(
379        indoc!{r#"
380            [profile.default]
381            retries = { backoff = "exponential", count = 1, delay = "1s", max-delay = "0s" }
382        "#},
383        ConfigErrorKind::Message,
384        "`max-delay` cannot be zero with exponential backoff"
385        ; "exponential backoff must have a non-zero max delay")]
386    #[test_case(
387        indoc!{r#"
388            [profile.default]
389            retries = { backoff = "exponential", count = 1, delay = "4s", max-delay = "2s", jitter = true }
390        "#},
391        ConfigErrorKind::Message,
392        "`max-delay` cannot be less than delay"
393        ; "max-delay greater than delay")]
394    fn parse_retries_invalid(
395        config_contents: &str,
396        expected_kind: ConfigErrorKind,
397        expected_message: &str,
398    ) {
399        let workspace_dir = tempdir().unwrap();
400
401        let graph = temp_workspace(&workspace_dir, config_contents);
402        let pcx = ParseContext::new(&graph);
403
404        let config_err = NextestConfig::from_sources(
405            graph.workspace().root(),
406            &pcx,
407            None,
408            [],
409            &Default::default(),
410        )
411        .expect_err("config expected to be invalid");
412
413        let message = match config_err.kind() {
414            ConfigParseErrorKind::DeserializeError(path_error) => {
415                match (path_error.inner(), expected_kind) {
416                    (ConfigError::Message(message), ConfigErrorKind::Message) => message,
417                    (ConfigError::NotFound(message), ConfigErrorKind::NotFound) => message,
418                    (other, expected) => {
419                        panic!(
420                            "for config error {config_err:?}, expected \
421                             ConfigErrorKind::{expected:?} for inner error {other:?}"
422                        );
423                    }
424                }
425            }
426            other => {
427                panic!(
428                    "for config error {other:?}, expected ConfigParseErrorKind::DeserializeError"
429                );
430            }
431        };
432
433        assert!(
434            message.contains(expected_message),
435            "expected message \"{message}\" to contain \"{expected_message}\""
436        );
437    }
438
439    #[test_case(
440        indoc! {r#"
441            [[profile.default.overrides]]
442            filter = "test(=my_test)"
443            retries = 2
444
445            [profile.ci]
446        "#},
447        BuildPlatform::Target,
448        RetryPolicy::new_without_delay(2)
449
450        ; "my_test matches exactly"
451    )]
452    #[test_case(
453        indoc! {r#"
454            [[profile.default.overrides]]
455            filter = "!test(=my_test)"
456            retries = 2
457
458            [profile.ci]
459        "#},
460        BuildPlatform::Target,
461        RetryPolicy::new_without_delay(0)
462
463        ; "not match"
464    )]
465    #[test_case(
466        indoc! {r#"
467            [[profile.default.overrides]]
468            filter = "test(=my_test)"
469
470            [profile.ci]
471        "#},
472        BuildPlatform::Target,
473        RetryPolicy::new_without_delay(0)
474
475        ; "no retries specified"
476    )]
477    #[test_case(
478        indoc! {r#"
479            [[profile.default.overrides]]
480            filter = "test(test)"
481            retries = 2
482
483            [[profile.default.overrides]]
484            filter = "test(=my_test)"
485            retries = 3
486
487            [profile.ci]
488        "#},
489        BuildPlatform::Target,
490        RetryPolicy::new_without_delay(2)
491
492        ; "earlier configs override later ones"
493    )]
494    #[test_case(
495        indoc! {r#"
496            [[profile.default.overrides]]
497            filter = "test(test)"
498            retries = 2
499
500            [profile.ci]
501
502            [[profile.ci.overrides]]
503            filter = "test(=my_test)"
504            retries = 3
505        "#},
506        BuildPlatform::Target,
507        RetryPolicy::new_without_delay(3)
508
509        ; "profile-specific configs override default ones"
510    )]
511    #[test_case(
512        indoc! {r#"
513            [[profile.default.overrides]]
514            filter = "(!package(test-package)) and test(test)"
515            retries = 2
516
517            [profile.ci]
518
519            [[profile.ci.overrides]]
520            filter = "!test(=my_test_2)"
521            retries = 3
522        "#},
523        BuildPlatform::Target,
524        RetryPolicy::new_without_delay(3)
525
526        ; "no overrides match my_test exactly"
527    )]
528    #[test_case(
529        indoc! {r#"
530            [[profile.default.overrides]]
531            platform = "x86_64-unknown-linux-gnu"
532            filter = "test(test)"
533            retries = 2
534
535            [[profile.default.overrides]]
536            filter = "test(=my_test)"
537            retries = 3
538
539            [profile.ci]
540        "#},
541        BuildPlatform::Host,
542        RetryPolicy::new_without_delay(2)
543
544        ; "earlier config applied because it matches host triple"
545    )]
546    #[test_case(
547        indoc! {r#"
548            [[profile.default.overrides]]
549            platform = "aarch64-apple-darwin"
550            filter = "test(test)"
551            retries = 2
552
553            [[profile.default.overrides]]
554            filter = "test(=my_test)"
555            retries = 3
556
557            [profile.ci]
558        "#},
559        BuildPlatform::Host,
560        RetryPolicy::new_without_delay(3)
561
562        ; "earlier config ignored because it doesn't match host triple"
563    )]
564    #[test_case(
565        indoc! {r#"
566            [[profile.default.overrides]]
567            platform = "aarch64-apple-darwin"
568            filter = "test(test)"
569            retries = 2
570
571            [[profile.default.overrides]]
572            filter = "test(=my_test)"
573            retries = 3
574
575            [profile.ci]
576        "#},
577        BuildPlatform::Target,
578        RetryPolicy::new_without_delay(2)
579
580        ; "earlier config applied because it matches target triple"
581    )]
582    #[test_case(
583        indoc! {r#"
584            [[profile.default.overrides]]
585            platform = "x86_64-unknown-linux-gnu"
586            filter = "test(test)"
587            retries = 2
588
589            [[profile.default.overrides]]
590            filter = "test(=my_test)"
591            retries = 3
592
593            [profile.ci]
594        "#},
595        BuildPlatform::Target,
596        RetryPolicy::new_without_delay(3)
597
598        ; "earlier config ignored because it doesn't match target triple"
599    )]
600    #[test_case(
601        indoc! {r#"
602            [[profile.default.overrides]]
603            platform = 'cfg(target_os = "macos")'
604            filter = "test(test)"
605            retries = 2
606
607            [[profile.default.overrides]]
608            filter = "test(=my_test)"
609            retries = 3
610
611            [profile.ci]
612        "#},
613        BuildPlatform::Target,
614        RetryPolicy::new_without_delay(2)
615
616        ; "earlier config applied because it matches target cfg expr"
617    )]
618    #[test_case(
619        indoc! {r#"
620            [[profile.default.overrides]]
621            platform = 'cfg(target_arch = "x86_64")'
622            filter = "test(test)"
623            retries = 2
624
625            [[profile.default.overrides]]
626            filter = "test(=my_test)"
627            retries = 3
628
629            [profile.ci]
630        "#},
631        BuildPlatform::Target,
632        RetryPolicy::new_without_delay(3)
633
634        ; "earlier config ignored because it doesn't match target cfg expr"
635    )]
636    fn overrides_retries(
637        config_contents: &str,
638        build_platform: BuildPlatform,
639        retries: RetryPolicy,
640    ) {
641        let workspace_dir = tempdir().unwrap();
642
643        let graph = temp_workspace(&workspace_dir, config_contents);
644        let package_id = graph.workspace().iter().next().unwrap().id();
645        let pcx = ParseContext::new(&graph);
646
647        let config = NextestConfig::from_sources(
648            graph.workspace().root(),
649            &pcx,
650            None,
651            &[][..],
652            &Default::default(),
653        )
654        .unwrap();
655        let binary_query = binary_query(&graph, package_id, "lib", "my-binary", build_platform);
656        let test_name = TestCaseName::new("my_test");
657        let query = TestQuery {
658            binary_query: binary_query.to_query(),
659            test_name: &test_name,
660        };
661        let profile = config
662            .profile("ci")
663            .expect("ci profile is defined")
664            .apply_build_platforms(&build_platforms());
665        let settings_for = profile.settings_for(NextestRunMode::Test, &query);
666        assert_eq!(
667            settings_for.retries(),
668            retries,
669            "actual retries don't match expected retries"
670        );
671    }
672}
nextest_runner/config/elements/retry_policy.rs

nextest_runner/config/elements/
retry_policy.rs
