nextest_runner/config/elements/
retry_policy.rs

1// Copyright (c) The nextest Contributors
2// SPDX-License-Identifier: MIT OR Apache-2.0
3
4use serde::Deserialize;
5use std::{cmp::Ordering, fmt, time::Duration};
6
7/// Type for the retry config key.
8#[derive(Debug, Copy, Clone, Deserialize, PartialEq, Eq)]
9#[serde(tag = "backoff", rename_all = "kebab-case", deny_unknown_fields)]
10pub enum RetryPolicy {
11    /// Fixed backoff.
12    #[serde(rename_all = "kebab-case")]
13    Fixed {
14        /// Maximum retry count.
15        count: u32,
16
17        /// Delay between retries.
18        #[serde(default, with = "humantime_serde")]
19        delay: Duration,
20
21        /// If set to true, randomness will be added to the delay on each retry attempt.
22        #[serde(default)]
23        jitter: bool,
24    },
25
26    /// Exponential backoff.
27    #[serde(rename_all = "kebab-case")]
28    Exponential {
29        /// Maximum retry count.
30        count: u32,
31
32        /// Delay between retries. Not optional for exponential backoff.
33        #[serde(with = "humantime_serde")]
34        delay: Duration,
35
36        /// If set to true, randomness will be added to the delay on each retry attempt.
37        #[serde(default)]
38        jitter: bool,
39
40        /// If set, limits the delay between retries.
41        #[serde(default, with = "humantime_serde")]
42        max_delay: Option<Duration>,
43    },
44}
45
46impl Default for RetryPolicy {
47    #[inline]
48    fn default() -> Self {
49        Self::new_without_delay(0)
50    }
51}
52
53impl RetryPolicy {
54    /// Create new policy with no delay between retries.
55    pub fn new_without_delay(count: u32) -> Self {
56        Self::Fixed {
57            count,
58            delay: Duration::ZERO,
59            jitter: false,
60        }
61    }
62
63    /// Returns the number of retries.
64    pub fn count(&self) -> u32 {
65        match self {
66            Self::Fixed { count, .. } | Self::Exponential { count, .. } => *count,
67        }
68    }
69}
70
71pub(in crate::config) fn deserialize_retry_policy<'de, D>(
72    deserializer: D,
73) -> Result<Option<RetryPolicy>, D::Error>
74where
75    D: serde::Deserializer<'de>,
76{
77    struct V;
78
79    impl<'de2> serde::de::Visitor<'de2> for V {
80        type Value = Option<RetryPolicy>;
81
82        fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
83            write!(
84                formatter,
85                "a table ({{ count = 5, backoff = \"exponential\", delay = \"1s\", max-delay = \"10s\", jitter = true }}) or a number (5)"
86            )
87        }
88
89        // Note that TOML uses i64, not u64.
90        fn visit_i64<E>(self, v: i64) -> Result<Self::Value, E>
91        where
92            E: serde::de::Error,
93        {
94            match v.cmp(&0) {
95                Ordering::Greater | Ordering::Equal => {
96                    let v = u32::try_from(v).map_err(|_| {
97                        serde::de::Error::invalid_value(
98                            serde::de::Unexpected::Signed(v),
99                            &"a positive u32",
100                        )
101                    })?;
102                    Ok(Some(RetryPolicy::new_without_delay(v)))
103                }
104                Ordering::Less => Err(serde::de::Error::invalid_value(
105                    serde::de::Unexpected::Signed(v),
106                    &self,
107                )),
108            }
109        }
110
111        fn visit_map<A>(self, map: A) -> Result<Self::Value, A::Error>
112        where
113            A: serde::de::MapAccess<'de2>,
114        {
115            RetryPolicy::deserialize(serde::de::value::MapAccessDeserializer::new(map)).map(Some)
116        }
117    }
118
119    // Post-deserialize validation of retry policy.
120    let retry_policy = deserializer.deserialize_any(V)?;
121    match &retry_policy {
122        Some(RetryPolicy::Fixed {
123            count: _,
124            delay,
125            jitter,
126        }) => {
127            // Jitter can't be specified if delay is 0.
128            if delay.is_zero() && *jitter {
129                return Err(serde::de::Error::custom(
130                    "`jitter` cannot be true if `delay` isn't specified or is zero",
131                ));
132            }
133        }
134        Some(RetryPolicy::Exponential {
135            count,
136            delay,
137            jitter: _,
138            max_delay,
139        }) => {
140            // Count can't be zero.
141            if *count == 0 {
142                return Err(serde::de::Error::custom(
143                    "`count` cannot be zero with exponential backoff",
144                ));
145            }
146            // Delay can't be zero.
147            if delay.is_zero() {
148                return Err(serde::de::Error::custom(
149                    "`delay` cannot be zero with exponential backoff",
150                ));
151            }
152            // Max delay, if specified, can't be zero.
153            if max_delay.is_some_and(|f| f.is_zero()) {
154                return Err(serde::de::Error::custom(
155                    "`max-delay` cannot be zero with exponential backoff",
156                ));
157            }
158            // Max delay can't be less than delay.
159            if max_delay.is_some_and(|max_delay| max_delay < *delay) {
160                return Err(serde::de::Error::custom(
161                    "`max-delay` cannot be less than delay with exponential backoff",
162                ));
163            }
164        }
165        None => {}
166    }
167
168    Ok(retry_policy)
169}
170
171#[cfg(test)]
172mod tests {
173    use super::*;
174    use crate::{
175        config::{core::NextestConfig, utils::test_helpers::*},
176        errors::ConfigParseErrorKind,
177    };
178    use camino_tempfile::tempdir;
179    use config::ConfigError;
180    use guppy::graph::cargo::BuildPlatform;
181    use indoc::indoc;
182    use nextest_filtering::{ParseContext, TestQuery};
183    use test_case::test_case;
184
185    #[test]
186    fn parse_retries_valid() {
187        let config_contents = indoc! {r#"
188            [profile.default]
189            retries = { backoff = "fixed", count = 3 }
190
191            [profile.no-retries]
192            retries = 0
193
194            [profile.fixed-with-delay]
195            retries = { backoff = "fixed", count = 3, delay = "1s" }
196
197            [profile.exp]
198            retries = { backoff = "exponential", count = 4, delay = "2s" }
199
200            [profile.exp-with-max-delay]
201            retries = { backoff = "exponential", count = 5, delay = "3s", max-delay = "10s" }
202
203            [profile.exp-with-max-delay-and-jitter]
204            retries = { backoff = "exponential", count = 6, delay = "4s", max-delay = "1m", jitter = true }
205        "#};
206
207        let workspace_dir = tempdir().unwrap();
208
209        let graph = temp_workspace(&workspace_dir, config_contents);
210        let pcx = ParseContext::new(&graph);
211
212        let config = NextestConfig::from_sources(
213            graph.workspace().root(),
214            &pcx,
215            None,
216            [],
217            &Default::default(),
218        )
219        .expect("config is valid");
220        assert_eq!(
221            config
222                .profile("default")
223                .expect("default profile exists")
224                .apply_build_platforms(&build_platforms())
225                .retries(),
226            RetryPolicy::Fixed {
227                count: 3,
228                delay: Duration::ZERO,
229                jitter: false,
230            },
231            "default retries matches"
232        );
233
234        assert_eq!(
235            config
236                .profile("no-retries")
237                .expect("profile exists")
238                .apply_build_platforms(&build_platforms())
239                .retries(),
240            RetryPolicy::new_without_delay(0),
241            "no-retries retries matches"
242        );
243
244        assert_eq!(
245            config
246                .profile("fixed-with-delay")
247                .expect("profile exists")
248                .apply_build_platforms(&build_platforms())
249                .retries(),
250            RetryPolicy::Fixed {
251                count: 3,
252                delay: Duration::from_secs(1),
253                jitter: false,
254            },
255            "fixed-with-delay retries matches"
256        );
257
258        assert_eq!(
259            config
260                .profile("exp")
261                .expect("profile exists")
262                .apply_build_platforms(&build_platforms())
263                .retries(),
264            RetryPolicy::Exponential {
265                count: 4,
266                delay: Duration::from_secs(2),
267                jitter: false,
268                max_delay: None,
269            },
270            "exp retries matches"
271        );
272
273        assert_eq!(
274            config
275                .profile("exp-with-max-delay")
276                .expect("profile exists")
277                .apply_build_platforms(&build_platforms())
278                .retries(),
279            RetryPolicy::Exponential {
280                count: 5,
281                delay: Duration::from_secs(3),
282                jitter: false,
283                max_delay: Some(Duration::from_secs(10)),
284            },
285            "exp-with-max-delay retries matches"
286        );
287
288        assert_eq!(
289            config
290                .profile("exp-with-max-delay-and-jitter")
291                .expect("profile exists")
292                .apply_build_platforms(&build_platforms())
293                .retries(),
294            RetryPolicy::Exponential {
295                count: 6,
296                delay: Duration::from_secs(4),
297                jitter: true,
298                max_delay: Some(Duration::from_secs(60)),
299            },
300            "exp-with-max-delay-and-jitter retries matches"
301        );
302    }
303
304    #[test_case(
305        indoc!{r#"
306            [profile.default]
307            retries = { backoff = "foo" }
308        "#},
309        ConfigErrorKind::Message,
310        "unknown variant `foo`, expected `fixed` or `exponential`"
311        ; "invalid value for backoff")]
312    #[test_case(
313        indoc!{r#"
314            [profile.default]
315            retries = { backoff = "fixed" }
316        "#},
317        ConfigErrorKind::NotFound,
318        "profile.default.retries.count"
319        ; "fixed specified without count")]
320    #[test_case(
321        indoc!{r#"
322            [profile.default]
323            retries = { backoff = "fixed", count = 1, delay = "foobar" }
324        "#},
325        ConfigErrorKind::Message,
326        "invalid value: string \"foobar\", expected a duration"
327        ; "delay is not a valid duration")]
328    #[test_case(
329        indoc!{r#"
330            [profile.default]
331            retries = { backoff = "fixed", count = 1, jitter = true }
332        "#},
333        ConfigErrorKind::Message,
334        "`jitter` cannot be true if `delay` isn't specified or is zero"
335        ; "jitter specified without delay")]
336    #[test_case(
337        indoc!{r#"
338            [profile.default]
339            retries = { backoff = "fixed", count = 1, max-delay = "10s" }
340        "#},
341        ConfigErrorKind::Message,
342        "unknown field `max-delay`, expected one of `count`, `delay`, `jitter`"
343        ; "max-delay is incompatible with fixed backoff")]
344    #[test_case(
345        indoc!{r#"
346            [profile.default]
347            retries = { backoff = "exponential", count = 1 }
348        "#},
349        ConfigErrorKind::NotFound,
350        "profile.default.retries.delay"
351        ; "exponential backoff must specify delay")]
352    #[test_case(
353        indoc!{r#"
354            [profile.default]
355            retries = { backoff = "exponential", delay = "1s" }
356        "#},
357        ConfigErrorKind::NotFound,
358        "profile.default.retries.count"
359        ; "exponential backoff must specify count")]
360    #[test_case(
361        indoc!{r#"
362            [profile.default]
363            retries = { backoff = "exponential", count = 0, delay = "1s" }
364        "#},
365        ConfigErrorKind::Message,
366        "`count` cannot be zero with exponential backoff"
367        ; "exponential backoff must have a non-zero count")]
368    #[test_case(
369        indoc!{r#"
370            [profile.default]
371            retries = { backoff = "exponential", count = 1, delay = "0s" }
372        "#},
373        ConfigErrorKind::Message,
374        "`delay` cannot be zero with exponential backoff"
375        ; "exponential backoff must have a non-zero delay")]
376    #[test_case(
377        indoc!{r#"
378            [profile.default]
379            retries = { backoff = "exponential", count = 1, delay = "1s", max-delay = "0s" }
380        "#},
381        ConfigErrorKind::Message,
382        "`max-delay` cannot be zero with exponential backoff"
383        ; "exponential backoff must have a non-zero max delay")]
384    #[test_case(
385        indoc!{r#"
386            [profile.default]
387            retries = { backoff = "exponential", count = 1, delay = "4s", max-delay = "2s", jitter = true }
388        "#},
389        ConfigErrorKind::Message,
390        "`max-delay` cannot be less than delay"
391        ; "max-delay greater than delay")]
392    fn parse_retries_invalid(
393        config_contents: &str,
394        expected_kind: ConfigErrorKind,
395        expected_message: &str,
396    ) {
397        let workspace_dir = tempdir().unwrap();
398
399        let graph = temp_workspace(&workspace_dir, config_contents);
400        let pcx = ParseContext::new(&graph);
401
402        let config_err = NextestConfig::from_sources(
403            graph.workspace().root(),
404            &pcx,
405            None,
406            [],
407            &Default::default(),
408        )
409        .expect_err("config expected to be invalid");
410
411        let message = match config_err.kind() {
412            ConfigParseErrorKind::DeserializeError(path_error) => {
413                match (path_error.inner(), expected_kind) {
414                    (ConfigError::Message(message), ConfigErrorKind::Message) => message,
415                    (ConfigError::NotFound(message), ConfigErrorKind::NotFound) => message,
416                    (other, expected) => {
417                        panic!(
418                            "for config error {config_err:?}, expected \
419                             ConfigErrorKind::{expected:?} for inner error {other:?}"
420                        );
421                    }
422                }
423            }
424            other => {
425                panic!(
426                    "for config error {other:?}, expected ConfigParseErrorKind::DeserializeError"
427                );
428            }
429        };
430
431        assert!(
432            message.contains(expected_message),
433            "expected message \"{message}\" to contain \"{expected_message}\""
434        );
435    }
436
437    #[test_case(
438        indoc! {r#"
439            [[profile.default.overrides]]
440            filter = "test(=my_test)"
441            retries = 2
442
443            [profile.ci]
444        "#},
445        BuildPlatform::Target,
446        RetryPolicy::new_without_delay(2)
447
448        ; "my_test matches exactly"
449    )]
450    #[test_case(
451        indoc! {r#"
452            [[profile.default.overrides]]
453            filter = "!test(=my_test)"
454            retries = 2
455
456            [profile.ci]
457        "#},
458        BuildPlatform::Target,
459        RetryPolicy::new_without_delay(0)
460
461        ; "not match"
462    )]
463    #[test_case(
464        indoc! {r#"
465            [[profile.default.overrides]]
466            filter = "test(=my_test)"
467
468            [profile.ci]
469        "#},
470        BuildPlatform::Target,
471        RetryPolicy::new_without_delay(0)
472
473        ; "no retries specified"
474    )]
475    #[test_case(
476        indoc! {r#"
477            [[profile.default.overrides]]
478            filter = "test(test)"
479            retries = 2
480
481            [[profile.default.overrides]]
482            filter = "test(=my_test)"
483            retries = 3
484
485            [profile.ci]
486        "#},
487        BuildPlatform::Target,
488        RetryPolicy::new_without_delay(2)
489
490        ; "earlier configs override later ones"
491    )]
492    #[test_case(
493        indoc! {r#"
494            [[profile.default.overrides]]
495            filter = "test(test)"
496            retries = 2
497
498            [profile.ci]
499
500            [[profile.ci.overrides]]
501            filter = "test(=my_test)"
502            retries = 3
503        "#},
504        BuildPlatform::Target,
505        RetryPolicy::new_without_delay(3)
506
507        ; "profile-specific configs override default ones"
508    )]
509    #[test_case(
510        indoc! {r#"
511            [[profile.default.overrides]]
512            filter = "(!package(test-package)) and test(test)"
513            retries = 2
514
515            [profile.ci]
516
517            [[profile.ci.overrides]]
518            filter = "!test(=my_test_2)"
519            retries = 3
520        "#},
521        BuildPlatform::Target,
522        RetryPolicy::new_without_delay(3)
523
524        ; "no overrides match my_test exactly"
525    )]
526    #[test_case(
527        indoc! {r#"
528            [[profile.default.overrides]]
529            platform = "x86_64-unknown-linux-gnu"
530            filter = "test(test)"
531            retries = 2
532
533            [[profile.default.overrides]]
534            filter = "test(=my_test)"
535            retries = 3
536
537            [profile.ci]
538        "#},
539        BuildPlatform::Host,
540        RetryPolicy::new_without_delay(2)
541
542        ; "earlier config applied because it matches host triple"
543    )]
544    #[test_case(
545        indoc! {r#"
546            [[profile.default.overrides]]
547            platform = "aarch64-apple-darwin"
548            filter = "test(test)"
549            retries = 2
550
551            [[profile.default.overrides]]
552            filter = "test(=my_test)"
553            retries = 3
554
555            [profile.ci]
556        "#},
557        BuildPlatform::Host,
558        RetryPolicy::new_without_delay(3)
559
560        ; "earlier config ignored because it doesn't match host triple"
561    )]
562    #[test_case(
563        indoc! {r#"
564            [[profile.default.overrides]]
565            platform = "aarch64-apple-darwin"
566            filter = "test(test)"
567            retries = 2
568
569            [[profile.default.overrides]]
570            filter = "test(=my_test)"
571            retries = 3
572
573            [profile.ci]
574        "#},
575        BuildPlatform::Target,
576        RetryPolicy::new_without_delay(2)
577
578        ; "earlier config applied because it matches target triple"
579    )]
580    #[test_case(
581        indoc! {r#"
582            [[profile.default.overrides]]
583            platform = "x86_64-unknown-linux-gnu"
584            filter = "test(test)"
585            retries = 2
586
587            [[profile.default.overrides]]
588            filter = "test(=my_test)"
589            retries = 3
590
591            [profile.ci]
592        "#},
593        BuildPlatform::Target,
594        RetryPolicy::new_without_delay(3)
595
596        ; "earlier config ignored because it doesn't match target triple"
597    )]
598    #[test_case(
599        indoc! {r#"
600            [[profile.default.overrides]]
601            platform = 'cfg(target_os = "macos")'
602            filter = "test(test)"
603            retries = 2
604
605            [[profile.default.overrides]]
606            filter = "test(=my_test)"
607            retries = 3
608
609            [profile.ci]
610        "#},
611        BuildPlatform::Target,
612        RetryPolicy::new_without_delay(2)
613
614        ; "earlier config applied because it matches target cfg expr"
615    )]
616    #[test_case(
617        indoc! {r#"
618            [[profile.default.overrides]]
619            platform = 'cfg(target_arch = "x86_64")'
620            filter = "test(test)"
621            retries = 2
622
623            [[profile.default.overrides]]
624            filter = "test(=my_test)"
625            retries = 3
626
627            [profile.ci]
628        "#},
629        BuildPlatform::Target,
630        RetryPolicy::new_without_delay(3)
631
632        ; "earlier config ignored because it doesn't match target cfg expr"
633    )]
634    fn overrides_retries(
635        config_contents: &str,
636        build_platform: BuildPlatform,
637        retries: RetryPolicy,
638    ) {
639        let workspace_dir = tempdir().unwrap();
640
641        let graph = temp_workspace(&workspace_dir, config_contents);
642        let package_id = graph.workspace().iter().next().unwrap().id();
643        let pcx = ParseContext::new(&graph);
644
645        let config = NextestConfig::from_sources(
646            graph.workspace().root(),
647            &pcx,
648            None,
649            &[][..],
650            &Default::default(),
651        )
652        .unwrap();
653        let binary_query = binary_query(&graph, package_id, "lib", "my-binary", build_platform);
654        let query = TestQuery {
655            binary_query: binary_query.to_query(),
656            test_name: "my_test",
657        };
658        let profile = config
659            .profile("ci")
660            .expect("ci profile is defined")
661            .apply_build_platforms(&build_platforms());
662        let settings_for = profile.settings_for(&query);
663        assert_eq!(
664            settings_for.retries(),
665            retries,
666            "actual retries don't match expected retries"
667        );
668    }
669}
nextest_runner/config/elements/retry_policy.rs

nextest_runner/config/elements/
retry_policy.rs
