nextest_runner/list/
test_list.rs

1// Copyright (c) The nextest Contributors
2// SPDX-License-Identifier: MIT OR Apache-2.0
3
4use super::{DisplayFilterMatcher, TestListDisplayFilter};
5use crate::{
6    cargo_config::EnvironmentMap,
7    config::{
8        core::EvaluatableProfile,
9        overrides::{ListSettings, TestSettings},
10        scripts::{WrapperScriptConfig, WrapperScriptTargetRunner},
11    },
12    double_spawn::DoubleSpawnInfo,
13    errors::{CreateTestListError, FromMessagesError, WriteTestListError},
14    helpers::{convert_build_platform, dylib_path, dylib_path_envvar, write_test_name},
15    indenter::indented,
16    list::{BinaryList, OutputFormat, RustBuildMeta, Styles, TestListState},
17    reuse_build::PathMapper,
18    target_runner::{PlatformRunner, TargetRunner},
19    test_command::{LocalExecuteContext, TestCommand, TestCommandPhase},
20    test_filter::{BinaryMismatchReason, FilterBinaryMatch, FilterBound, TestFilterBuilder},
21    write_str::WriteStr,
22};
23use camino::{Utf8Path, Utf8PathBuf};
24use debug_ignore::DebugIgnore;
25use futures::prelude::*;
26use guppy::{
27    PackageId,
28    graph::{PackageGraph, PackageMetadata},
29};
30use iddqd::{IdOrdItem, IdOrdMap, id_upcast};
31use nextest_filtering::{BinaryQuery, EvalContext, TestQuery};
32use nextest_metadata::{
33    BuildPlatform, FilterMatch, MismatchReason, RustBinaryId, RustNonTestBinaryKind,
34    RustTestBinaryKind, RustTestBinarySummary, RustTestCaseSummary, RustTestSuiteStatusSummary,
35    RustTestSuiteSummary, TestListSummary,
36};
37use owo_colors::OwoColorize;
38use std::{
39    borrow::Cow,
40    collections::{BTreeMap, BTreeSet},
41    ffi::{OsStr, OsString},
42    fmt, io,
43    path::PathBuf,
44    sync::{Arc, OnceLock},
45};
46use tokio::runtime::Runtime;
47use tracing::debug;
48
49/// A Rust test binary built by Cargo. This artifact hasn't been run yet so there's no information
50/// about the tests within it.
51///
52/// Accepted as input to [`TestList::new`].
53#[derive(Clone, Debug)]
54pub struct RustTestArtifact<'g> {
55    /// A unique identifier for this test artifact.
56    pub binary_id: RustBinaryId,
57
58    /// Metadata for the package this artifact is a part of. This is used to set the correct
59    /// environment variables.
60    pub package: PackageMetadata<'g>,
61
62    /// The path to the binary artifact.
63    pub binary_path: Utf8PathBuf,
64
65    /// The unique binary name defined in `Cargo.toml` or inferred by the filename.
66    pub binary_name: String,
67
68    /// The kind of Rust test binary this is.
69    pub kind: RustTestBinaryKind,
70
71    /// Non-test binaries to be exposed to this artifact at runtime (name, path).
72    pub non_test_binaries: BTreeSet<(String, Utf8PathBuf)>,
73
74    /// The working directory that this test should be executed in.
75    pub cwd: Utf8PathBuf,
76
77    /// The platform for which this test artifact was built.
78    pub build_platform: BuildPlatform,
79}
80
81impl<'g> RustTestArtifact<'g> {
82    /// Constructs a list of test binaries from the list of built binaries.
83    pub fn from_binary_list(
84        graph: &'g PackageGraph,
85        binary_list: Arc<BinaryList>,
86        rust_build_meta: &RustBuildMeta<TestListState>,
87        path_mapper: &PathMapper,
88        platform_filter: Option<BuildPlatform>,
89    ) -> Result<Vec<Self>, FromMessagesError> {
90        let mut binaries = vec![];
91
92        for binary in &binary_list.rust_binaries {
93            if platform_filter.is_some() && platform_filter != Some(binary.build_platform) {
94                continue;
95            }
96
97            // Look up the executable by package ID.
98            let package_id = PackageId::new(binary.package_id.clone());
99            let package = graph
100                .metadata(&package_id)
101                .map_err(FromMessagesError::PackageGraph)?;
102
103            // Tests are run in the directory containing Cargo.toml
104            let cwd = package
105                .manifest_path()
106                .parent()
107                .unwrap_or_else(|| {
108                    panic!(
109                        "manifest path {} doesn't have a parent",
110                        package.manifest_path()
111                    )
112                })
113                .to_path_buf();
114
115            let binary_path = path_mapper.map_binary(binary.path.clone());
116            let cwd = path_mapper.map_cwd(cwd);
117
118            // Non-test binaries are only exposed to integration tests and benchmarks.
119            let non_test_binaries = if binary.kind == RustTestBinaryKind::TEST
120                || binary.kind == RustTestBinaryKind::BENCH
121            {
122                // Note we must use the TestListState rust_build_meta here to ensure we get remapped
123                // paths.
124                match rust_build_meta.non_test_binaries.get(package_id.repr()) {
125                    Some(binaries) => binaries
126                        .iter()
127                        .filter(|binary| {
128                            // Only expose BIN_EXE non-test files.
129                            binary.kind == RustNonTestBinaryKind::BIN_EXE
130                        })
131                        .map(|binary| {
132                            // Convert relative paths to absolute ones by joining with the target directory.
133                            let abs_path = rust_build_meta.target_directory.join(&binary.path);
134                            (binary.name.clone(), abs_path)
135                        })
136                        .collect(),
137                    None => BTreeSet::new(),
138                }
139            } else {
140                BTreeSet::new()
141            };
142
143            binaries.push(RustTestArtifact {
144                binary_id: binary.id.clone(),
145                package,
146                binary_path,
147                binary_name: binary.name.clone(),
148                kind: binary.kind.clone(),
149                cwd,
150                non_test_binaries,
151                build_platform: binary.build_platform,
152            })
153        }
154
155        Ok(binaries)
156    }
157
158    /// Returns a [`BinaryQuery`] corresponding to this test artifact.
159    pub fn to_binary_query(&self) -> BinaryQuery<'_> {
160        BinaryQuery {
161            package_id: self.package.id(),
162            binary_id: &self.binary_id,
163            kind: &self.kind,
164            binary_name: &self.binary_name,
165            platform: convert_build_platform(self.build_platform),
166        }
167    }
168
169    // ---
170    // Helper methods
171    // ---
172    fn into_test_suite(self, status: RustTestSuiteStatus) -> RustTestSuite<'g> {
173        let Self {
174            binary_id,
175            package,
176            binary_path,
177            binary_name,
178            kind,
179            non_test_binaries,
180            cwd,
181            build_platform,
182        } = self;
183
184        RustTestSuite {
185            binary_id,
186            binary_path,
187            package,
188            binary_name,
189            kind,
190            non_test_binaries,
191            cwd,
192            build_platform,
193            status,
194        }
195    }
196}
197
198/// Information about skipped tests and binaries.
199#[derive(Clone, Debug, Eq, PartialEq)]
200pub struct SkipCounts {
201    /// The number of skipped tests.
202    pub skipped_tests: usize,
203
204    /// The number of tests skipped due to not being in the default set.
205    pub skipped_tests_default_filter: usize,
206
207    /// The number of skipped binaries.
208    pub skipped_binaries: usize,
209
210    /// The number of binaries skipped due to not being in the default set.
211    pub skipped_binaries_default_filter: usize,
212}
213
214/// List of test instances, obtained by querying the [`RustTestArtifact`] instances generated by Cargo.
215#[derive(Clone, Debug)]
216pub struct TestList<'g> {
217    test_count: usize,
218    rust_build_meta: RustBuildMeta<TestListState>,
219    rust_suites: IdOrdMap<RustTestSuite<'g>>,
220    workspace_root: Utf8PathBuf,
221    env: EnvironmentMap,
222    updated_dylib_path: OsString,
223    // Computed on first access.
224    skip_counts: OnceLock<SkipCounts>,
225}
226
227impl<'g> TestList<'g> {
228    /// Creates a new test list by running the given command and applying the specified filter.
229    #[expect(clippy::too_many_arguments)]
230    pub fn new<I>(
231        ctx: &TestExecuteContext<'_>,
232        test_artifacts: I,
233        rust_build_meta: RustBuildMeta<TestListState>,
234        filter: &TestFilterBuilder,
235        workspace_root: Utf8PathBuf,
236        env: EnvironmentMap,
237        profile: &impl ListProfile,
238        bound: FilterBound,
239        list_threads: usize,
240    ) -> Result<Self, CreateTestListError>
241    where
242        I: IntoIterator<Item = RustTestArtifact<'g>>,
243        I::IntoIter: Send,
244    {
245        let updated_dylib_path = Self::create_dylib_path(&rust_build_meta)?;
246        debug!(
247            "updated {}: {}",
248            dylib_path_envvar(),
249            updated_dylib_path.to_string_lossy(),
250        );
251        let lctx = LocalExecuteContext {
252            phase: TestCommandPhase::List,
253            // Note: this is the remapped workspace root, not the original one.
254            // (We really should have newtypes for this.)
255            workspace_root: &workspace_root,
256            rust_build_meta: &rust_build_meta,
257            double_spawn: ctx.double_spawn,
258            dylib_path: &updated_dylib_path,
259            profile_name: ctx.profile_name,
260            env: &env,
261        };
262
263        let ecx = profile.filterset_ecx();
264
265        let runtime = Runtime::new().map_err(CreateTestListError::TokioRuntimeCreate)?;
266
267        let stream = futures::stream::iter(test_artifacts).map(|test_binary| {
268            async {
269                let binary_query = test_binary.to_binary_query();
270                let binary_match = filter.filter_binary_match(&test_binary, &ecx, bound);
271                match binary_match {
272                    FilterBinaryMatch::Definite | FilterBinaryMatch::Possible => {
273                        debug!(
274                            "executing test binary to obtain test list \
275                            (match result is {binary_match:?}): {}",
276                            test_binary.binary_id,
277                        );
278                        // Run the binary to obtain the test list.
279                        let list_settings = profile.list_settings_for(&binary_query);
280                        let (non_ignored, ignored) = test_binary
281                            .exec(&lctx, &list_settings, ctx.target_runner)
282                            .await?;
283                        let info = Self::process_output(
284                            test_binary,
285                            filter,
286                            &ecx,
287                            bound,
288                            non_ignored.as_str(),
289                            ignored.as_str(),
290                        )?;
291                        Ok::<_, CreateTestListError>(info)
292                    }
293                    FilterBinaryMatch::Mismatch { reason } => {
294                        debug!("skipping test binary: {reason}: {}", test_binary.binary_id,);
295                        Ok(Self::process_skipped(test_binary, reason))
296                    }
297                }
298            }
299        });
300        let fut = stream.buffer_unordered(list_threads).try_collect();
301
302        let rust_suites: IdOrdMap<_> = runtime.block_on(fut)?;
303
304        // Ensure that the runtime doesn't stay hanging even if a custom test framework misbehaves
305        // (can be an issue on Windows).
306        runtime.shutdown_background();
307
308        let test_count = rust_suites
309            .iter()
310            .map(|suite| suite.status.test_count())
311            .sum();
312
313        Ok(Self {
314            rust_suites,
315            workspace_root,
316            env,
317            rust_build_meta,
318            updated_dylib_path,
319            test_count,
320            skip_counts: OnceLock::new(),
321        })
322    }
323
324    /// Creates a new test list with the given binary names and outputs.
325    #[cfg(test)]
326    fn new_with_outputs(
327        test_bin_outputs: impl IntoIterator<
328            Item = (RustTestArtifact<'g>, impl AsRef<str>, impl AsRef<str>),
329        >,
330        workspace_root: Utf8PathBuf,
331        rust_build_meta: RustBuildMeta<TestListState>,
332        filter: &TestFilterBuilder,
333        env: EnvironmentMap,
334        ecx: &EvalContext<'_>,
335        bound: FilterBound,
336    ) -> Result<Self, CreateTestListError> {
337        let mut test_count = 0;
338
339        let updated_dylib_path = Self::create_dylib_path(&rust_build_meta)?;
340
341        let rust_suites = test_bin_outputs
342            .into_iter()
343            .map(|(test_binary, non_ignored, ignored)| {
344                let binary_match = filter.filter_binary_match(&test_binary, ecx, bound);
345                match binary_match {
346                    FilterBinaryMatch::Definite | FilterBinaryMatch::Possible => {
347                        debug!(
348                            "processing output for binary \
349                            (match result is {binary_match:?}): {}",
350                            test_binary.binary_id,
351                        );
352                        let info = Self::process_output(
353                            test_binary,
354                            filter,
355                            ecx,
356                            bound,
357                            non_ignored.as_ref(),
358                            ignored.as_ref(),
359                        )?;
360                        test_count += info.status.test_count();
361                        Ok(info)
362                    }
363                    FilterBinaryMatch::Mismatch { reason } => {
364                        debug!("skipping test binary: {reason}: {}", test_binary.binary_id,);
365                        Ok(Self::process_skipped(test_binary, reason))
366                    }
367                }
368            })
369            .collect::<Result<IdOrdMap<_>, _>>()?;
370
371        Ok(Self {
372            rust_suites,
373            workspace_root,
374            env,
375            rust_build_meta,
376            updated_dylib_path,
377            test_count,
378            skip_counts: OnceLock::new(),
379        })
380    }
381
382    /// Returns the total number of tests across all binaries.
383    pub fn test_count(&self) -> usize {
384        self.test_count
385    }
386
387    /// Returns the Rust build-related metadata for this test list.
388    pub fn rust_build_meta(&self) -> &RustBuildMeta<TestListState> {
389        &self.rust_build_meta
390    }
391
392    /// Returns the total number of skipped tests.
393    pub fn skip_counts(&self) -> &SkipCounts {
394        self.skip_counts.get_or_init(|| {
395            let mut skipped_tests_default_filter = 0;
396            let skipped_tests = self
397                .iter_tests()
398                .filter(|instance| match instance.test_info.filter_match {
399                    FilterMatch::Mismatch {
400                        reason: MismatchReason::DefaultFilter,
401                    } => {
402                        skipped_tests_default_filter += 1;
403                        true
404                    }
405                    FilterMatch::Mismatch { .. } => true,
406                    FilterMatch::Matches => false,
407                })
408                .count();
409
410            let mut skipped_binaries_default_filter = 0;
411            let skipped_binaries = self
412                .rust_suites
413                .iter()
414                .filter(|suite| match suite.status {
415                    RustTestSuiteStatus::Skipped {
416                        reason: BinaryMismatchReason::DefaultSet,
417                    } => {
418                        skipped_binaries_default_filter += 1;
419                        true
420                    }
421                    RustTestSuiteStatus::Skipped { .. } => true,
422                    RustTestSuiteStatus::Listed { .. } => false,
423                })
424                .count();
425
426            SkipCounts {
427                skipped_tests,
428                skipped_tests_default_filter,
429                skipped_binaries,
430                skipped_binaries_default_filter,
431            }
432        })
433    }
434
435    /// Returns the total number of tests that aren't skipped.
436    ///
437    /// It is always the case that `run_count + skip_count == test_count`.
438    pub fn run_count(&self) -> usize {
439        self.test_count - self.skip_counts().skipped_tests
440    }
441
442    /// Returns the total number of binaries that contain tests.
443    pub fn binary_count(&self) -> usize {
444        self.rust_suites.len()
445    }
446
447    /// Returns the total number of binaries that were listed (not skipped).
448    pub fn listed_binary_count(&self) -> usize {
449        self.binary_count() - self.skip_counts().skipped_binaries
450    }
451
452    /// Returns the mapped workspace root.
453    pub fn workspace_root(&self) -> &Utf8Path {
454        &self.workspace_root
455    }
456
457    /// Returns the environment variables to be used when running tests.
458    pub fn cargo_env(&self) -> &EnvironmentMap {
459        &self.env
460    }
461
462    /// Returns the updated dynamic library path used for tests.
463    pub fn updated_dylib_path(&self) -> &OsStr {
464        &self.updated_dylib_path
465    }
466
467    /// Constructs a serializble summary for this test list.
468    pub fn to_summary(&self) -> TestListSummary {
469        let rust_suites = self
470            .rust_suites
471            .iter()
472            .map(|test_suite| {
473                let (status, test_cases) = test_suite.status.to_summary();
474                let testsuite = RustTestSuiteSummary {
475                    package_name: test_suite.package.name().to_owned(),
476                    binary: RustTestBinarySummary {
477                        binary_name: test_suite.binary_name.clone(),
478                        package_id: test_suite.package.id().repr().to_owned(),
479                        kind: test_suite.kind.clone(),
480                        binary_path: test_suite.binary_path.clone(),
481                        binary_id: test_suite.binary_id.clone(),
482                        build_platform: test_suite.build_platform,
483                    },
484                    cwd: test_suite.cwd.clone(),
485                    status,
486                    test_cases,
487                };
488                (test_suite.binary_id.clone(), testsuite)
489            })
490            .collect();
491        let mut summary = TestListSummary::new(self.rust_build_meta.to_summary());
492        summary.test_count = self.test_count;
493        summary.rust_suites = rust_suites;
494        summary
495    }
496
497    /// Outputs this list to the given writer.
498    pub fn write(
499        &self,
500        output_format: OutputFormat,
501        writer: &mut dyn WriteStr,
502        colorize: bool,
503    ) -> Result<(), WriteTestListError> {
504        match output_format {
505            OutputFormat::Human { verbose } => self
506                .write_human(writer, verbose, colorize)
507                .map_err(WriteTestListError::Io),
508            OutputFormat::Serializable(format) => format.to_writer(&self.to_summary(), writer),
509        }
510    }
511
512    /// Iterates over all the test suites.
513    pub fn iter(&self) -> impl Iterator<Item = &RustTestSuite<'_>> + '_ {
514        self.rust_suites.iter()
515    }
516
517    /// Iterates over the list of tests, returning the path and test name.
518    pub fn iter_tests(&self) -> impl Iterator<Item = TestInstance<'_>> + '_ {
519        self.rust_suites.iter().flat_map(|test_suite| {
520            test_suite
521                .status
522                .test_cases()
523                .map(move |(name, test_info)| TestInstance::new(name, test_suite, test_info))
524        })
525    }
526
527    /// Produces a priority queue of tests based on the given profile.
528    pub fn to_priority_queue(
529        &'g self,
530        profile: &'g EvaluatableProfile<'g>,
531    ) -> TestPriorityQueue<'g> {
532        TestPriorityQueue::new(self, profile)
533    }
534
535    /// Outputs this list as a string with the given format.
536    pub fn to_string(&self, output_format: OutputFormat) -> Result<String, WriteTestListError> {
537        let mut s = String::with_capacity(1024);
538        self.write(output_format, &mut s, false)?;
539        Ok(s)
540    }
541
542    // ---
543    // Helper methods
544    // ---
545
546    // Empty list for tests.
547    #[cfg(test)]
548    pub(crate) fn empty() -> Self {
549        Self {
550            test_count: 0,
551            workspace_root: Utf8PathBuf::new(),
552            rust_build_meta: RustBuildMeta::empty(),
553            env: EnvironmentMap::empty(),
554            updated_dylib_path: OsString::new(),
555            rust_suites: IdOrdMap::new(),
556            skip_counts: OnceLock::new(),
557        }
558    }
559
560    pub(crate) fn create_dylib_path(
561        rust_build_meta: &RustBuildMeta<TestListState>,
562    ) -> Result<OsString, CreateTestListError> {
563        let dylib_path = dylib_path();
564        let dylib_path_is_empty = dylib_path.is_empty();
565        let new_paths = rust_build_meta.dylib_paths();
566
567        let mut updated_dylib_path: Vec<PathBuf> =
568            Vec::with_capacity(dylib_path.len() + new_paths.len());
569        updated_dylib_path.extend(
570            new_paths
571                .iter()
572                .map(|path| path.clone().into_std_path_buf()),
573        );
574        updated_dylib_path.extend(dylib_path);
575
576        // On macOS, these are the defaults when DYLD_FALLBACK_LIBRARY_PATH isn't set or set to an
577        // empty string. (This is relevant if nextest is invoked as its own process and not
578        // a Cargo subcommand.)
579        //
580        // This copies the logic from
581        // https://cs.github.com/rust-lang/cargo/blob/7d289b171183578d45dcabc56db6db44b9accbff/src/cargo/core/compiler/compilation.rs#L292.
582        if cfg!(target_os = "macos") && dylib_path_is_empty {
583            if let Some(home) = home::home_dir() {
584                updated_dylib_path.push(home.join("lib"));
585            }
586            updated_dylib_path.push("/usr/local/lib".into());
587            updated_dylib_path.push("/usr/lib".into());
588        }
589
590        std::env::join_paths(updated_dylib_path)
591            .map_err(move |error| CreateTestListError::dylib_join_paths(new_paths, error))
592    }
593
594    fn process_output(
595        test_binary: RustTestArtifact<'g>,
596        filter: &TestFilterBuilder,
597        ecx: &EvalContext<'_>,
598        bound: FilterBound,
599        non_ignored: impl AsRef<str>,
600        ignored: impl AsRef<str>,
601    ) -> Result<RustTestSuite<'g>, CreateTestListError> {
602        let mut test_cases = BTreeMap::new();
603
604        // Treat ignored and non-ignored as separate sets of single filters, so that partitioning
605        // based on one doesn't affect the other.
606        let mut non_ignored_filter = filter.build();
607        for test_name in Self::parse(&test_binary.binary_id, non_ignored.as_ref())? {
608            test_cases.insert(
609                test_name.into(),
610                RustTestCaseSummary {
611                    ignored: false,
612                    filter_match: non_ignored_filter.filter_match(
613                        &test_binary,
614                        test_name,
615                        ecx,
616                        bound,
617                        false,
618                    ),
619                },
620            );
621        }
622
623        let mut ignored_filter = filter.build();
624        for test_name in Self::parse(&test_binary.binary_id, ignored.as_ref())? {
625            // Note that libtest prints out:
626            // * just ignored tests if --ignored is passed in
627            // * all tests, both ignored and non-ignored, if --ignored is not passed in
628            // Adding ignored tests after non-ignored ones makes everything resolve correctly.
629            test_cases.insert(
630                test_name.into(),
631                RustTestCaseSummary {
632                    ignored: true,
633                    filter_match: ignored_filter.filter_match(
634                        &test_binary,
635                        test_name,
636                        ecx,
637                        bound,
638                        true,
639                    ),
640                },
641            );
642        }
643
644        Ok(test_binary.into_test_suite(RustTestSuiteStatus::Listed {
645            test_cases: test_cases.into(),
646        }))
647    }
648
649    fn process_skipped(
650        test_binary: RustTestArtifact<'g>,
651        reason: BinaryMismatchReason,
652    ) -> RustTestSuite<'g> {
653        test_binary.into_test_suite(RustTestSuiteStatus::Skipped { reason })
654    }
655
656    /// Parses the output of --list --message-format terse and returns a sorted list.
657    fn parse<'a>(
658        binary_id: &'a RustBinaryId,
659        list_output: &'a str,
660    ) -> Result<Vec<&'a str>, CreateTestListError> {
661        let mut list = Self::parse_impl(binary_id, list_output).collect::<Result<Vec<_>, _>>()?;
662        list.sort_unstable();
663        Ok(list)
664    }
665
666    fn parse_impl<'a>(
667        binary_id: &'a RustBinaryId,
668        list_output: &'a str,
669    ) -> impl Iterator<Item = Result<&'a str, CreateTestListError>> + 'a + use<'a> {
670        // The output is in the form:
671        // <test name>: test
672        // <test name>: test
673        // ...
674
675        list_output.lines().map(move |line| {
676            line.strip_suffix(": test")
677                .or_else(|| line.strip_suffix(": benchmark"))
678                .ok_or_else(|| {
679                    CreateTestListError::parse_line(
680                        binary_id.clone(),
681                        format!(
682                            "line '{line}' did not end with the string ': test' or ': benchmark'"
683                        ),
684                        list_output,
685                    )
686                })
687        })
688    }
689
690    /// Writes this test list out in a human-friendly format.
691    pub fn write_human(
692        &self,
693        writer: &mut dyn WriteStr,
694        verbose: bool,
695        colorize: bool,
696    ) -> io::Result<()> {
697        self.write_human_impl(None, writer, verbose, colorize)
698    }
699
700    /// Writes this test list out in a human-friendly format with the given filter.
701    pub(crate) fn write_human_with_filter(
702        &self,
703        filter: &TestListDisplayFilter<'_>,
704        writer: &mut dyn WriteStr,
705        verbose: bool,
706        colorize: bool,
707    ) -> io::Result<()> {
708        self.write_human_impl(Some(filter), writer, verbose, colorize)
709    }
710
711    fn write_human_impl(
712        &self,
713        filter: Option<&TestListDisplayFilter<'_>>,
714        mut writer: &mut dyn WriteStr,
715        verbose: bool,
716        colorize: bool,
717    ) -> io::Result<()> {
718        let mut styles = Styles::default();
719        if colorize {
720            styles.colorize();
721        }
722
723        for info in &self.rust_suites {
724            let matcher = match filter {
725                Some(filter) => match filter.matcher_for(&info.binary_id) {
726                    Some(matcher) => matcher,
727                    None => continue,
728                },
729                None => DisplayFilterMatcher::All,
730            };
731
732            // Skip this binary if there are no tests within it that will be run, and this isn't
733            // verbose output.
734            if !verbose
735                && info
736                    .status
737                    .test_cases()
738                    .all(|(_, test_case)| !test_case.filter_match.is_match())
739            {
740                continue;
741            }
742
743            writeln!(writer, "{}:", info.binary_id.style(styles.binary_id))?;
744            if verbose {
745                writeln!(
746                    writer,
747                    "  {} {}",
748                    "bin:".style(styles.field),
749                    info.binary_path
750                )?;
751                writeln!(writer, "  {} {}", "cwd:".style(styles.field), info.cwd)?;
752                writeln!(
753                    writer,
754                    "  {} {}",
755                    "build platform:".style(styles.field),
756                    info.build_platform,
757                )?;
758            }
759
760            let mut indented = indented(writer).with_str("    ");
761
762            match &info.status {
763                RustTestSuiteStatus::Listed { test_cases } => {
764                    let matching_tests: Vec<_> = test_cases
765                        .iter()
766                        .filter(|(name, _)| matcher.is_match(name))
767                        .collect();
768                    if matching_tests.is_empty() {
769                        writeln!(indented, "(no tests)")?;
770                    } else {
771                        for (name, info) in matching_tests {
772                            match (verbose, info.filter_match.is_match()) {
773                                (_, true) => {
774                                    write_test_name(name, &styles, &mut indented)?;
775                                    writeln!(indented)?;
776                                }
777                                (true, false) => {
778                                    write_test_name(name, &styles, &mut indented)?;
779                                    writeln!(indented, " (skipped)")?;
780                                }
781                                (false, false) => {
782                                    // Skip printing this test entirely if it isn't a match.
783                                }
784                            }
785                        }
786                    }
787                }
788                RustTestSuiteStatus::Skipped { reason } => {
789                    writeln!(indented, "(test binary {reason}, skipped)")?;
790                }
791            }
792
793            writer = indented.into_inner();
794        }
795        Ok(())
796    }
797}
798
799/// Profile implementation for test lists.
800pub trait ListProfile {
801    /// Returns the evaluation context.
802    fn filterset_ecx(&self) -> EvalContext<'_>;
803
804    /// Returns list-time settings for a test binary.
805    fn list_settings_for(&self, query: &BinaryQuery<'_>) -> ListSettings<'_>;
806}
807
808impl<'g> ListProfile for EvaluatableProfile<'g> {
809    fn filterset_ecx(&self) -> EvalContext<'_> {
810        self.filterset_ecx()
811    }
812
813    fn list_settings_for(&self, query: &BinaryQuery<'_>) -> ListSettings<'_> {
814        self.list_settings_for(query)
815    }
816}
817
818/// A test list that has been sorted and has had priorities applied to it.
819pub struct TestPriorityQueue<'a> {
820    tests: Vec<TestInstanceWithSettings<'a>>,
821}
822
823impl<'a> TestPriorityQueue<'a> {
824    fn new(test_list: &'a TestList<'a>, profile: &'a EvaluatableProfile<'a>) -> Self {
825        let mut tests = test_list
826            .iter_tests()
827            .map(|instance| {
828                let settings = profile.settings_for(&instance.to_test_query());
829                TestInstanceWithSettings { instance, settings }
830            })
831            .collect::<Vec<_>>();
832        // Note: this is a stable sort so that tests with the same priority are
833        // sorted by what `iter_tests` produced.
834        tests.sort_by_key(|test| test.settings.priority());
835
836        Self { tests }
837    }
838}
839
840impl<'a> IntoIterator for TestPriorityQueue<'a> {
841    type Item = TestInstanceWithSettings<'a>;
842    type IntoIter = std::vec::IntoIter<Self::Item>;
843
844    fn into_iter(self) -> Self::IntoIter {
845        self.tests.into_iter()
846    }
847}
848
849/// A test instance, along with computed settings from a profile.
850///
851/// Returned from [`TestPriorityQueue`].
852#[derive(Debug)]
853pub struct TestInstanceWithSettings<'a> {
854    /// The test instance.
855    pub instance: TestInstance<'a>,
856
857    /// The settings for this test.
858    pub settings: TestSettings<'a>,
859}
860
861/// A suite of tests within a single Rust test binary.
862///
863/// This is a representation of [`nextest_metadata::RustTestSuiteSummary`] used internally by the runner.
864#[derive(Clone, Debug, Eq, PartialEq)]
865pub struct RustTestSuite<'g> {
866    /// A unique identifier for this binary.
867    pub binary_id: RustBinaryId,
868
869    /// The path to the binary.
870    pub binary_path: Utf8PathBuf,
871
872    /// Package metadata.
873    pub package: PackageMetadata<'g>,
874
875    /// The unique binary name defined in `Cargo.toml` or inferred by the filename.
876    pub binary_name: String,
877
878    /// The kind of Rust test binary this is.
879    pub kind: RustTestBinaryKind,
880
881    /// The working directory that this test binary will be executed in. If None, the current directory
882    /// will not be changed.
883    pub cwd: Utf8PathBuf,
884
885    /// The platform the test suite is for (host or target).
886    pub build_platform: BuildPlatform,
887
888    /// Non-test binaries corresponding to this test suite (name, path).
889    pub non_test_binaries: BTreeSet<(String, Utf8PathBuf)>,
890
891    /// Test suite status and test case names.
892    pub status: RustTestSuiteStatus,
893}
894
895impl IdOrdItem for RustTestSuite<'_> {
896    type Key<'a>
897        = &'a RustBinaryId
898    where
899        Self: 'a;
900
901    fn key(&self) -> Self::Key<'_> {
902        &self.binary_id
903    }
904
905    id_upcast!();
906}
907
908impl RustTestArtifact<'_> {
909    /// Run this binary with and without --ignored and get the corresponding outputs.
910    async fn exec(
911        &self,
912        lctx: &LocalExecuteContext<'_>,
913        list_settings: &ListSettings<'_>,
914        target_runner: &TargetRunner,
915    ) -> Result<(String, String), CreateTestListError> {
916        // This error situation has been known to happen with reused builds. It produces
917        // a really terrible and confusing "file not found" message if allowed to prceed.
918        if !self.cwd.is_dir() {
919            return Err(CreateTestListError::CwdIsNotDir {
920                binary_id: self.binary_id.clone(),
921                cwd: self.cwd.clone(),
922            });
923        }
924        let platform_runner = target_runner.for_build_platform(self.build_platform);
925
926        let non_ignored = self.exec_single(false, lctx, list_settings, platform_runner);
927        let ignored = self.exec_single(true, lctx, list_settings, platform_runner);
928
929        let (non_ignored_out, ignored_out) = futures::future::join(non_ignored, ignored).await;
930        Ok((non_ignored_out?, ignored_out?))
931    }
932
933    async fn exec_single(
934        &self,
935        ignored: bool,
936        lctx: &LocalExecuteContext<'_>,
937        list_settings: &ListSettings<'_>,
938        runner: Option<&PlatformRunner>,
939    ) -> Result<String, CreateTestListError> {
940        let mut cli = TestCommandCli::default();
941        cli.apply_wrappers(
942            list_settings.list_wrapper(),
943            runner,
944            lctx.workspace_root,
945            &lctx.rust_build_meta.target_directory,
946        );
947        cli.push(self.binary_path.as_str());
948
949        cli.extend(["--list", "--format", "terse"]);
950        if ignored {
951            cli.push("--ignored");
952        }
953
954        let cmd = TestCommand::new(
955            lctx,
956            cli.program
957                .clone()
958                .expect("at least one argument passed in")
959                .into_owned(),
960            &cli.args,
961            &self.cwd,
962            &self.package,
963            &self.non_test_binaries,
964        );
965
966        let output =
967            cmd.wait_with_output()
968                .await
969                .map_err(|error| CreateTestListError::CommandExecFail {
970                    binary_id: self.binary_id.clone(),
971                    command: cli.to_owned_cli(),
972                    error,
973                })?;
974
975        if output.status.success() {
976            String::from_utf8(output.stdout).map_err(|err| CreateTestListError::CommandNonUtf8 {
977                binary_id: self.binary_id.clone(),
978                command: cli.to_owned_cli(),
979                stdout: err.into_bytes(),
980                stderr: output.stderr,
981            })
982        } else {
983            Err(CreateTestListError::CommandFail {
984                binary_id: self.binary_id.clone(),
985                command: cli.to_owned_cli(),
986                exit_status: output.status,
987                stdout: output.stdout,
988                stderr: output.stderr,
989            })
990        }
991    }
992}
993
994/// Serializable information about the status of and test cases within a test suite.
995///
996/// Part of a [`RustTestSuiteSummary`].
997#[derive(Clone, Debug, Eq, PartialEq)]
998pub enum RustTestSuiteStatus {
999    /// The test suite was executed with `--list` and the list of test cases was obtained.
1000    Listed {
1001        /// The test cases contained within this test suite.
1002        test_cases: DebugIgnore<BTreeMap<String, RustTestCaseSummary>>,
1003    },
1004
1005    /// The test suite was not executed.
1006    Skipped {
1007        /// The reason why the test suite was skipped.
1008        reason: BinaryMismatchReason,
1009    },
1010}
1011
1012static EMPTY_TEST_CASE_MAP: BTreeMap<String, RustTestCaseSummary> = BTreeMap::new();
1013
1014impl RustTestSuiteStatus {
1015    /// Returns the number of test cases within this suite.
1016    pub fn test_count(&self) -> usize {
1017        match self {
1018            RustTestSuiteStatus::Listed { test_cases } => test_cases.len(),
1019            RustTestSuiteStatus::Skipped { .. } => 0,
1020        }
1021    }
1022
1023    /// Returns the list of test cases within this suite.
1024    pub fn test_cases(&self) -> impl Iterator<Item = (&str, &RustTestCaseSummary)> + '_ {
1025        match self {
1026            RustTestSuiteStatus::Listed { test_cases } => test_cases.iter(),
1027            RustTestSuiteStatus::Skipped { .. } => {
1028                // Return an empty test case.
1029                EMPTY_TEST_CASE_MAP.iter()
1030            }
1031        }
1032        .map(|(name, case)| (name.as_str(), case))
1033    }
1034
1035    /// Converts this status to its serializable form.
1036    pub fn to_summary(
1037        &self,
1038    ) -> (
1039        RustTestSuiteStatusSummary,
1040        BTreeMap<String, RustTestCaseSummary>,
1041    ) {
1042        match self {
1043            Self::Listed { test_cases } => {
1044                (RustTestSuiteStatusSummary::LISTED, test_cases.clone().0)
1045            }
1046            Self::Skipped {
1047                reason: BinaryMismatchReason::Expression,
1048            } => (RustTestSuiteStatusSummary::SKIPPED, BTreeMap::new()),
1049            Self::Skipped {
1050                reason: BinaryMismatchReason::DefaultSet,
1051            } => (
1052                RustTestSuiteStatusSummary::SKIPPED_DEFAULT_FILTER,
1053                BTreeMap::new(),
1054            ),
1055        }
1056    }
1057}
1058
1059/// Represents a single test with its associated binary.
1060#[derive(Clone, Copy, Debug, Eq, PartialEq)]
1061pub struct TestInstance<'a> {
1062    /// The name of the test.
1063    pub name: &'a str,
1064
1065    /// Information about the test suite.
1066    pub suite_info: &'a RustTestSuite<'a>,
1067
1068    /// Information about the test.
1069    pub test_info: &'a RustTestCaseSummary,
1070}
1071
1072impl<'a> TestInstance<'a> {
1073    /// Creates a new `TestInstance`.
1074    pub(crate) fn new(
1075        name: &'a (impl AsRef<str> + ?Sized),
1076        suite_info: &'a RustTestSuite,
1077        test_info: &'a RustTestCaseSummary,
1078    ) -> Self {
1079        Self {
1080            name: name.as_ref(),
1081            suite_info,
1082            test_info,
1083        }
1084    }
1085
1086    /// Return an identifier for test instances, including being able to sort
1087    /// them.
1088    #[inline]
1089    pub fn id(&self) -> TestInstanceId<'a> {
1090        TestInstanceId {
1091            binary_id: &self.suite_info.binary_id,
1092            test_name: self.name,
1093        }
1094    }
1095
1096    /// Returns the corresponding [`TestQuery`] for this `TestInstance`.
1097    pub fn to_test_query(&self) -> TestQuery<'a> {
1098        TestQuery {
1099            binary_query: BinaryQuery {
1100                package_id: self.suite_info.package.id(),
1101                binary_id: &self.suite_info.binary_id,
1102                kind: &self.suite_info.kind,
1103                binary_name: &self.suite_info.binary_name,
1104                platform: convert_build_platform(self.suite_info.build_platform),
1105            },
1106            test_name: self.name,
1107        }
1108    }
1109
1110    /// Creates the command for this test instance.
1111    pub(crate) fn make_command(
1112        &self,
1113        ctx: &TestExecuteContext<'_>,
1114        test_list: &TestList<'_>,
1115        wrapper_script: Option<&'a WrapperScriptConfig>,
1116        extra_args: &[String],
1117    ) -> TestCommand {
1118        // TODO: non-rust tests
1119
1120        let platform_runner = ctx
1121            .target_runner
1122            .for_build_platform(self.suite_info.build_platform);
1123
1124        let mut cli = TestCommandCli::default();
1125        cli.apply_wrappers(
1126            wrapper_script,
1127            platform_runner,
1128            test_list.workspace_root(),
1129            &test_list.rust_build_meta().target_directory,
1130        );
1131        cli.push(self.suite_info.binary_path.as_str());
1132
1133        cli.extend(["--exact", self.name, "--nocapture"]);
1134        if self.test_info.ignored {
1135            cli.push("--ignored");
1136        }
1137        cli.extend(extra_args.iter().map(String::as_str));
1138
1139        let lctx = LocalExecuteContext {
1140            phase: TestCommandPhase::Run,
1141            workspace_root: test_list.workspace_root(),
1142            rust_build_meta: &test_list.rust_build_meta,
1143            double_spawn: ctx.double_spawn,
1144            dylib_path: test_list.updated_dylib_path(),
1145            profile_name: ctx.profile_name,
1146            env: &test_list.env,
1147        };
1148
1149        TestCommand::new(
1150            &lctx,
1151            cli.program
1152                .expect("at least one argument is guaranteed")
1153                .into_owned(),
1154            &cli.args,
1155            &self.suite_info.cwd,
1156            &self.suite_info.package,
1157            &self.suite_info.non_test_binaries,
1158        )
1159    }
1160}
1161
1162#[derive(Clone, Debug, Default)]
1163struct TestCommandCli<'a> {
1164    program: Option<Cow<'a, str>>,
1165    args: Vec<Cow<'a, str>>,
1166}
1167
1168impl<'a> TestCommandCli<'a> {
1169    fn apply_wrappers(
1170        &mut self,
1171        wrapper_script: Option<&'a WrapperScriptConfig>,
1172        platform_runner: Option<&'a PlatformRunner>,
1173        workspace_root: &Utf8Path,
1174        target_dir: &Utf8Path,
1175    ) {
1176        // Apply the wrapper script if it's enabled.
1177        if let Some(wrapper) = wrapper_script {
1178            match wrapper.target_runner {
1179                WrapperScriptTargetRunner::Ignore => {
1180                    // Ignore the platform runner.
1181                    self.push(wrapper.command.program(workspace_root, target_dir));
1182                    self.extend(wrapper.command.args.iter().map(String::as_str));
1183                }
1184                WrapperScriptTargetRunner::AroundWrapper => {
1185                    // Platform runner goes first.
1186                    if let Some(runner) = platform_runner {
1187                        self.push(runner.binary());
1188                        self.extend(runner.args());
1189                    }
1190                    self.push(wrapper.command.program(workspace_root, target_dir));
1191                    self.extend(wrapper.command.args.iter().map(String::as_str));
1192                }
1193                WrapperScriptTargetRunner::WithinWrapper => {
1194                    // Wrapper script goes first.
1195                    self.push(wrapper.command.program(workspace_root, target_dir));
1196                    self.extend(wrapper.command.args.iter().map(String::as_str));
1197                    if let Some(runner) = platform_runner {
1198                        self.push(runner.binary());
1199                        self.extend(runner.args());
1200                    }
1201                }
1202                WrapperScriptTargetRunner::OverridesWrapper => {
1203                    // Target runner overrides wrapper.
1204                    if let Some(runner) = platform_runner {
1205                        self.push(runner.binary());
1206                        self.extend(runner.args());
1207                    }
1208                }
1209            }
1210        } else {
1211            // If no wrapper script is enabled, use the platform runner.
1212            if let Some(runner) = platform_runner {
1213                self.push(runner.binary());
1214                self.extend(runner.args());
1215            }
1216        }
1217    }
1218
1219    fn push(&mut self, arg: impl Into<Cow<'a, str>>) {
1220        if self.program.is_none() {
1221            self.program = Some(arg.into());
1222        } else {
1223            self.args.push(arg.into());
1224        }
1225    }
1226
1227    fn extend(&mut self, args: impl IntoIterator<Item = &'a str>) {
1228        for arg in args {
1229            self.push(arg);
1230        }
1231    }
1232
1233    fn to_owned_cli(&self) -> Vec<String> {
1234        let mut owned_cli = Vec::new();
1235        if let Some(program) = &self.program {
1236            owned_cli.push(program.to_string());
1237        }
1238        owned_cli.extend(self.args.iter().map(|arg| arg.clone().into_owned()));
1239        owned_cli
1240    }
1241}
1242
1243/// A key for identifying and sorting test instances.
1244///
1245/// Returned by [`TestInstance::id`].
1246#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)]
1247pub struct TestInstanceId<'a> {
1248    /// The binary ID.
1249    pub binary_id: &'a RustBinaryId,
1250
1251    /// The name of the test.
1252    pub test_name: &'a str,
1253}
1254
1255impl fmt::Display for TestInstanceId<'_> {
1256    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1257        write!(f, "{} {}", self.binary_id, self.test_name)
1258    }
1259}
1260
1261/// Context required for test execution.
1262#[derive(Clone, Debug)]
1263pub struct TestExecuteContext<'a> {
1264    /// The name of the profile.
1265    pub profile_name: &'a str,
1266
1267    /// Double-spawn info.
1268    pub double_spawn: &'a DoubleSpawnInfo,
1269
1270    /// Target runner.
1271    pub target_runner: &'a TargetRunner,
1272}
1273
1274#[cfg(test)]
1275mod tests {
1276    use super::*;
1277    use crate::{
1278        cargo_config::{TargetDefinitionLocation, TargetTriple, TargetTripleSource},
1279        config::scripts::{ScriptCommand, ScriptCommandRelativeTo},
1280        list::SerializableFormat,
1281        platform::{BuildPlatforms, HostPlatform, PlatformLibdir, TargetPlatform},
1282        target_runner::PlatformRunnerSource,
1283        test_filter::{RunIgnored, TestFilterPatterns},
1284    };
1285    use guppy::CargoMetadata;
1286    use iddqd::id_ord_map;
1287    use indoc::indoc;
1288    use maplit::btreemap;
1289    use nextest_filtering::{CompiledExpr, Filterset, FiltersetKind, ParseContext};
1290    use nextest_metadata::{FilterMatch, MismatchReason, PlatformLibdirUnavailable};
1291    use pretty_assertions::assert_eq;
1292    use std::sync::LazyLock;
1293    use target_spec::Platform;
1294
1295    #[test]
1296    fn test_parse_test_list() {
1297        // Lines ending in ': benchmark' (output by the default Rust bencher) should be skipped.
1298        let non_ignored_output = indoc! {"
1299            tests::foo::test_bar: test
1300            tests::baz::test_quux: test
1301            benches::bench_foo: benchmark
1302        "};
1303        let ignored_output = indoc! {"
1304            tests::ignored::test_bar: test
1305            tests::baz::test_ignored: test
1306            benches::ignored_bench_foo: benchmark
1307        "};
1308
1309        let cx = ParseContext::new(&PACKAGE_GRAPH_FIXTURE);
1310
1311        let test_filter = TestFilterBuilder::new(
1312            RunIgnored::Default,
1313            None,
1314            TestFilterPatterns::default(),
1315            // Test against the platform() predicate because this is the most important one here.
1316            vec![
1317                Filterset::parse("platform(target)".to_owned(), &cx, FiltersetKind::Test).unwrap(),
1318            ],
1319        )
1320        .unwrap();
1321        let fake_cwd: Utf8PathBuf = "/fake/cwd".into();
1322        let fake_binary_name = "fake-binary".to_owned();
1323        let fake_binary_id = RustBinaryId::new("fake-package::fake-binary");
1324
1325        let test_binary = RustTestArtifact {
1326            binary_path: "/fake/binary".into(),
1327            cwd: fake_cwd.clone(),
1328            package: package_metadata(),
1329            binary_name: fake_binary_name.clone(),
1330            binary_id: fake_binary_id.clone(),
1331            kind: RustTestBinaryKind::LIB,
1332            non_test_binaries: BTreeSet::new(),
1333            build_platform: BuildPlatform::Target,
1334        };
1335
1336        let skipped_binary_name = "skipped-binary".to_owned();
1337        let skipped_binary_id = RustBinaryId::new("fake-package::skipped-binary");
1338        let skipped_binary = RustTestArtifact {
1339            binary_path: "/fake/skipped-binary".into(),
1340            cwd: fake_cwd.clone(),
1341            package: package_metadata(),
1342            binary_name: skipped_binary_name.clone(),
1343            binary_id: skipped_binary_id.clone(),
1344            kind: RustTestBinaryKind::PROC_MACRO,
1345            non_test_binaries: BTreeSet::new(),
1346            build_platform: BuildPlatform::Host,
1347        };
1348
1349        let fake_triple = TargetTriple {
1350            platform: Platform::new(
1351                "aarch64-unknown-linux-gnu",
1352                target_spec::TargetFeatures::Unknown,
1353            )
1354            .unwrap(),
1355            source: TargetTripleSource::CliOption,
1356            location: TargetDefinitionLocation::Builtin,
1357        };
1358        let fake_host_libdir = "/home/fake/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/x86_64-unknown-linux-gnu/lib";
1359        let build_platforms = BuildPlatforms {
1360            host: HostPlatform {
1361                platform: TargetTriple::x86_64_unknown_linux_gnu().platform,
1362                libdir: PlatformLibdir::Available(fake_host_libdir.into()),
1363            },
1364            target: Some(TargetPlatform {
1365                triple: fake_triple,
1366                // Test an unavailable libdir.
1367                libdir: PlatformLibdir::Unavailable(PlatformLibdirUnavailable::new_const("test")),
1368            }),
1369        };
1370
1371        let fake_env = EnvironmentMap::empty();
1372        let rust_build_meta =
1373            RustBuildMeta::new("/fake", build_platforms).map_paths(&PathMapper::noop());
1374        let ecx = EvalContext {
1375            default_filter: &CompiledExpr::ALL,
1376        };
1377        let test_list = TestList::new_with_outputs(
1378            [
1379                (test_binary, &non_ignored_output, &ignored_output),
1380                (
1381                    skipped_binary,
1382                    &"should-not-show-up-stdout",
1383                    &"should-not-show-up-stderr",
1384                ),
1385            ],
1386            Utf8PathBuf::from("/fake/path"),
1387            rust_build_meta,
1388            &test_filter,
1389            fake_env,
1390            &ecx,
1391            FilterBound::All,
1392        )
1393        .expect("valid output");
1394        assert_eq!(
1395            test_list.rust_suites,
1396            id_ord_map! {
1397                RustTestSuite {
1398                    status: RustTestSuiteStatus::Listed {
1399                        test_cases: btreemap! {
1400                            "tests::foo::test_bar".to_owned() => RustTestCaseSummary {
1401                                ignored: false,
1402                                filter_match: FilterMatch::Matches,
1403                            },
1404                            "tests::baz::test_quux".to_owned() => RustTestCaseSummary {
1405                                ignored: false,
1406                                filter_match: FilterMatch::Matches,
1407                            },
1408                            "benches::bench_foo".to_owned() => RustTestCaseSummary {
1409                                ignored: false,
1410                                filter_match: FilterMatch::Matches,
1411                            },
1412                            "tests::ignored::test_bar".to_owned() => RustTestCaseSummary {
1413                                ignored: true,
1414                                filter_match: FilterMatch::Mismatch { reason: MismatchReason::Ignored },
1415                            },
1416                            "tests::baz::test_ignored".to_owned() => RustTestCaseSummary {
1417                                ignored: true,
1418                                filter_match: FilterMatch::Mismatch { reason: MismatchReason::Ignored },
1419                            },
1420                            "benches::ignored_bench_foo".to_owned() => RustTestCaseSummary {
1421                                ignored: true,
1422                                filter_match: FilterMatch::Mismatch { reason: MismatchReason::Ignored },
1423                            },
1424                        }.into(),
1425                    },
1426                    cwd: fake_cwd.clone(),
1427                    build_platform: BuildPlatform::Target,
1428                    package: package_metadata(),
1429                    binary_name: fake_binary_name,
1430                    binary_id: fake_binary_id,
1431                    binary_path: "/fake/binary".into(),
1432                    kind: RustTestBinaryKind::LIB,
1433                    non_test_binaries: BTreeSet::new(),
1434                },
1435                RustTestSuite {
1436                    status: RustTestSuiteStatus::Skipped {
1437                        reason: BinaryMismatchReason::Expression,
1438                    },
1439                    cwd: fake_cwd,
1440                    build_platform: BuildPlatform::Host,
1441                    package: package_metadata(),
1442                    binary_name: skipped_binary_name,
1443                    binary_id: skipped_binary_id,
1444                    binary_path: "/fake/skipped-binary".into(),
1445                    kind: RustTestBinaryKind::PROC_MACRO,
1446                    non_test_binaries: BTreeSet::new(),
1447                },
1448            }
1449        );
1450
1451        // Check that the expected outputs are valid.
1452        static EXPECTED_HUMAN: &str = indoc! {"
1453        fake-package::fake-binary:
1454            benches::bench_foo
1455            tests::baz::test_quux
1456            tests::foo::test_bar
1457        "};
1458        static EXPECTED_HUMAN_VERBOSE: &str = indoc! {"
1459            fake-package::fake-binary:
1460              bin: /fake/binary
1461              cwd: /fake/cwd
1462              build platform: target
1463                benches::bench_foo
1464                benches::ignored_bench_foo (skipped)
1465                tests::baz::test_ignored (skipped)
1466                tests::baz::test_quux
1467                tests::foo::test_bar
1468                tests::ignored::test_bar (skipped)
1469            fake-package::skipped-binary:
1470              bin: /fake/skipped-binary
1471              cwd: /fake/cwd
1472              build platform: host
1473                (test binary didn't match filtersets, skipped)
1474        "};
1475        static EXPECTED_JSON_PRETTY: &str = indoc! {r#"
1476            {
1477              "rust-build-meta": {
1478                "target-directory": "/fake",
1479                "base-output-directories": [],
1480                "non-test-binaries": {},
1481                "build-script-out-dirs": {},
1482                "linked-paths": [],
1483                "platforms": {
1484                  "host": {
1485                    "platform": {
1486                      "triple": "x86_64-unknown-linux-gnu",
1487                      "target-features": "unknown"
1488                    },
1489                    "libdir": {
1490                      "status": "available",
1491                      "path": "/home/fake/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/x86_64-unknown-linux-gnu/lib"
1492                    }
1493                  },
1494                  "targets": [
1495                    {
1496                      "platform": {
1497                        "triple": "aarch64-unknown-linux-gnu",
1498                        "target-features": "unknown"
1499                      },
1500                      "libdir": {
1501                        "status": "unavailable",
1502                        "reason": "test"
1503                      }
1504                    }
1505                  ]
1506                },
1507                "target-platforms": [
1508                  {
1509                    "triple": "aarch64-unknown-linux-gnu",
1510                    "target-features": "unknown"
1511                  }
1512                ],
1513                "target-platform": "aarch64-unknown-linux-gnu"
1514              },
1515              "test-count": 6,
1516              "rust-suites": {
1517                "fake-package::fake-binary": {
1518                  "package-name": "metadata-helper",
1519                  "binary-id": "fake-package::fake-binary",
1520                  "binary-name": "fake-binary",
1521                  "package-id": "metadata-helper 0.1.0 (path+file:///Users/fakeuser/local/testcrates/metadata/metadata-helper)",
1522                  "kind": "lib",
1523                  "binary-path": "/fake/binary",
1524                  "build-platform": "target",
1525                  "cwd": "/fake/cwd",
1526                  "status": "listed",
1527                  "testcases": {
1528                    "benches::bench_foo": {
1529                      "ignored": false,
1530                      "filter-match": {
1531                        "status": "matches"
1532                      }
1533                    },
1534                    "benches::ignored_bench_foo": {
1535                      "ignored": true,
1536                      "filter-match": {
1537                        "status": "mismatch",
1538                        "reason": "ignored"
1539                      }
1540                    },
1541                    "tests::baz::test_ignored": {
1542                      "ignored": true,
1543                      "filter-match": {
1544                        "status": "mismatch",
1545                        "reason": "ignored"
1546                      }
1547                    },
1548                    "tests::baz::test_quux": {
1549                      "ignored": false,
1550                      "filter-match": {
1551                        "status": "matches"
1552                      }
1553                    },
1554                    "tests::foo::test_bar": {
1555                      "ignored": false,
1556                      "filter-match": {
1557                        "status": "matches"
1558                      }
1559                    },
1560                    "tests::ignored::test_bar": {
1561                      "ignored": true,
1562                      "filter-match": {
1563                        "status": "mismatch",
1564                        "reason": "ignored"
1565                      }
1566                    }
1567                  }
1568                },
1569                "fake-package::skipped-binary": {
1570                  "package-name": "metadata-helper",
1571                  "binary-id": "fake-package::skipped-binary",
1572                  "binary-name": "skipped-binary",
1573                  "package-id": "metadata-helper 0.1.0 (path+file:///Users/fakeuser/local/testcrates/metadata/metadata-helper)",
1574                  "kind": "proc-macro",
1575                  "binary-path": "/fake/skipped-binary",
1576                  "build-platform": "host",
1577                  "cwd": "/fake/cwd",
1578                  "status": "skipped",
1579                  "testcases": {}
1580                }
1581              }
1582            }"#};
1583
1584        assert_eq!(
1585            test_list
1586                .to_string(OutputFormat::Human { verbose: false })
1587                .expect("human succeeded"),
1588            EXPECTED_HUMAN
1589        );
1590        assert_eq!(
1591            test_list
1592                .to_string(OutputFormat::Human { verbose: true })
1593                .expect("human succeeded"),
1594            EXPECTED_HUMAN_VERBOSE
1595        );
1596        println!(
1597            "{}",
1598            test_list
1599                .to_string(OutputFormat::Serializable(SerializableFormat::JsonPretty))
1600                .expect("json-pretty succeeded")
1601        );
1602        assert_eq!(
1603            test_list
1604                .to_string(OutputFormat::Serializable(SerializableFormat::JsonPretty))
1605                .expect("json-pretty succeeded"),
1606            EXPECTED_JSON_PRETTY
1607        );
1608    }
1609
1610    #[test]
1611    fn apply_wrappers_examples() {
1612        cfg_if::cfg_if! {
1613            if #[cfg(windows)]
1614            {
1615                let workspace_root = Utf8Path::new("D:\\workspace\\root");
1616                let target_dir = Utf8Path::new("C:\\foo\\bar");
1617            } else {
1618                let workspace_root = Utf8Path::new("/workspace/root");
1619                let target_dir = Utf8Path::new("/foo/bar");
1620            }
1621        };
1622
1623        // Test with no wrappers
1624        {
1625            let mut cli_no_wrappers = TestCommandCli::default();
1626            cli_no_wrappers.apply_wrappers(None, None, workspace_root, target_dir);
1627            cli_no_wrappers.extend(["binary", "arg"]);
1628            assert_eq!(cli_no_wrappers.to_owned_cli(), vec!["binary", "arg"]);
1629        }
1630
1631        // Test with platform runner only
1632        {
1633            let runner = PlatformRunner::debug_new(
1634                "runner".into(),
1635                Vec::new(),
1636                PlatformRunnerSource::Env("fake".to_owned()),
1637            );
1638            let mut cli_runner_only = TestCommandCli::default();
1639            cli_runner_only.apply_wrappers(None, Some(&runner), workspace_root, target_dir);
1640            cli_runner_only.extend(["binary", "arg"]);
1641            assert_eq!(
1642                cli_runner_only.to_owned_cli(),
1643                vec!["runner", "binary", "arg"],
1644            );
1645        }
1646
1647        // Test wrapper with ignore target runner
1648        {
1649            let runner = PlatformRunner::debug_new(
1650                "runner".into(),
1651                Vec::new(),
1652                PlatformRunnerSource::Env("fake".to_owned()),
1653            );
1654            let wrapper_ignore = WrapperScriptConfig {
1655                command: ScriptCommand {
1656                    program: "wrapper".into(),
1657                    args: Vec::new(),
1658                    relative_to: ScriptCommandRelativeTo::None,
1659                },
1660                target_runner: WrapperScriptTargetRunner::Ignore,
1661            };
1662            let mut cli_wrapper_ignore = TestCommandCli::default();
1663            cli_wrapper_ignore.apply_wrappers(
1664                Some(&wrapper_ignore),
1665                Some(&runner),
1666                workspace_root,
1667                target_dir,
1668            );
1669            cli_wrapper_ignore.extend(["binary", "arg"]);
1670            assert_eq!(
1671                cli_wrapper_ignore.to_owned_cli(),
1672                vec!["wrapper", "binary", "arg"],
1673            );
1674        }
1675
1676        // Test wrapper with around wrapper (runner first)
1677        {
1678            let runner = PlatformRunner::debug_new(
1679                "runner".into(),
1680                Vec::new(),
1681                PlatformRunnerSource::Env("fake".to_owned()),
1682            );
1683            let wrapper_around = WrapperScriptConfig {
1684                command: ScriptCommand {
1685                    program: "wrapper".into(),
1686                    args: Vec::new(),
1687                    relative_to: ScriptCommandRelativeTo::None,
1688                },
1689                target_runner: WrapperScriptTargetRunner::AroundWrapper,
1690            };
1691            let mut cli_wrapper_around = TestCommandCli::default();
1692            cli_wrapper_around.apply_wrappers(
1693                Some(&wrapper_around),
1694                Some(&runner),
1695                workspace_root,
1696                target_dir,
1697            );
1698            cli_wrapper_around.extend(["binary", "arg"]);
1699            assert_eq!(
1700                cli_wrapper_around.to_owned_cli(),
1701                vec!["runner", "wrapper", "binary", "arg"],
1702            );
1703        }
1704
1705        // Test wrapper with within wrapper (wrapper first)
1706        {
1707            let runner = PlatformRunner::debug_new(
1708                "runner".into(),
1709                Vec::new(),
1710                PlatformRunnerSource::Env("fake".to_owned()),
1711            );
1712            let wrapper_within = WrapperScriptConfig {
1713                command: ScriptCommand {
1714                    program: "wrapper".into(),
1715                    args: Vec::new(),
1716                    relative_to: ScriptCommandRelativeTo::None,
1717                },
1718                target_runner: WrapperScriptTargetRunner::WithinWrapper,
1719            };
1720            let mut cli_wrapper_within = TestCommandCli::default();
1721            cli_wrapper_within.apply_wrappers(
1722                Some(&wrapper_within),
1723                Some(&runner),
1724                workspace_root,
1725                target_dir,
1726            );
1727            cli_wrapper_within.extend(["binary", "arg"]);
1728            assert_eq!(
1729                cli_wrapper_within.to_owned_cli(),
1730                vec!["wrapper", "runner", "binary", "arg"],
1731            );
1732        }
1733
1734        // Test wrapper with overrides wrapper (runner only)
1735        {
1736            let runner = PlatformRunner::debug_new(
1737                "runner".into(),
1738                Vec::new(),
1739                PlatformRunnerSource::Env("fake".to_owned()),
1740            );
1741            let wrapper_overrides = WrapperScriptConfig {
1742                command: ScriptCommand {
1743                    program: "wrapper".into(),
1744                    args: Vec::new(),
1745                    relative_to: ScriptCommandRelativeTo::None,
1746                },
1747                target_runner: WrapperScriptTargetRunner::OverridesWrapper,
1748            };
1749            let mut cli_wrapper_overrides = TestCommandCli::default();
1750            cli_wrapper_overrides.apply_wrappers(
1751                Some(&wrapper_overrides),
1752                Some(&runner),
1753                workspace_root,
1754                target_dir,
1755            );
1756            cli_wrapper_overrides.extend(["binary", "arg"]);
1757            assert_eq!(
1758                cli_wrapper_overrides.to_owned_cli(),
1759                vec!["runner", "binary", "arg"],
1760            );
1761        }
1762
1763        // Test wrapper with args
1764        {
1765            let wrapper_with_args = WrapperScriptConfig {
1766                command: ScriptCommand {
1767                    program: "wrapper".into(),
1768                    args: vec!["--flag".to_string(), "value".to_string()],
1769                    relative_to: ScriptCommandRelativeTo::None,
1770                },
1771                target_runner: WrapperScriptTargetRunner::Ignore,
1772            };
1773            let mut cli_wrapper_args = TestCommandCli::default();
1774            cli_wrapper_args.apply_wrappers(
1775                Some(&wrapper_with_args),
1776                None,
1777                workspace_root,
1778                target_dir,
1779            );
1780            cli_wrapper_args.extend(["binary", "arg"]);
1781            assert_eq!(
1782                cli_wrapper_args.to_owned_cli(),
1783                vec!["wrapper", "--flag", "value", "binary", "arg"],
1784            );
1785        }
1786
1787        // Test platform runner with args
1788        {
1789            let runner_with_args = PlatformRunner::debug_new(
1790                "runner".into(),
1791                vec!["--runner-flag".into(), "value".into()],
1792                PlatformRunnerSource::Env("fake".to_owned()),
1793            );
1794            let mut cli_runner_args = TestCommandCli::default();
1795            cli_runner_args.apply_wrappers(
1796                None,
1797                Some(&runner_with_args),
1798                workspace_root,
1799                target_dir,
1800            );
1801            cli_runner_args.extend(["binary", "arg"]);
1802            assert_eq!(
1803                cli_runner_args.to_owned_cli(),
1804                vec!["runner", "--runner-flag", "value", "binary", "arg"],
1805            );
1806        }
1807
1808        // Test wrapper with ScriptCommandRelativeTo::WorkspaceRoot
1809        {
1810            let wrapper_relative_to_workspace_root = WrapperScriptConfig {
1811                command: ScriptCommand {
1812                    program: "abc/def/my-wrapper".into(),
1813                    args: vec!["--verbose".to_string()],
1814                    relative_to: ScriptCommandRelativeTo::WorkspaceRoot,
1815                },
1816                target_runner: WrapperScriptTargetRunner::Ignore,
1817            };
1818            let mut cli_wrapper_relative = TestCommandCli::default();
1819            cli_wrapper_relative.apply_wrappers(
1820                Some(&wrapper_relative_to_workspace_root),
1821                None,
1822                workspace_root,
1823                target_dir,
1824            );
1825            cli_wrapper_relative.extend(["binary", "arg"]);
1826
1827            cfg_if::cfg_if! {
1828                if #[cfg(windows)] {
1829                    let wrapper_path = "D:\\workspace\\root\\abc\\def\\my-wrapper";
1830                } else {
1831                    let wrapper_path = "/workspace/root/abc/def/my-wrapper";
1832                }
1833            }
1834            assert_eq!(
1835                cli_wrapper_relative.to_owned_cli(),
1836                vec![wrapper_path, "--verbose", "binary", "arg"],
1837            );
1838        }
1839
1840        // Test wrapper with ScriptCommandRelativeTo::Target
1841        {
1842            let wrapper_relative_to_target = WrapperScriptConfig {
1843                command: ScriptCommand {
1844                    program: "abc/def/my-wrapper".into(),
1845                    args: vec!["--verbose".to_string()],
1846                    relative_to: ScriptCommandRelativeTo::Target,
1847                },
1848                target_runner: WrapperScriptTargetRunner::Ignore,
1849            };
1850            let mut cli_wrapper_relative = TestCommandCli::default();
1851            cli_wrapper_relative.apply_wrappers(
1852                Some(&wrapper_relative_to_target),
1853                None,
1854                workspace_root,
1855                target_dir,
1856            );
1857            cli_wrapper_relative.extend(["binary", "arg"]);
1858            cfg_if::cfg_if! {
1859                if #[cfg(windows)] {
1860                    let wrapper_path = "C:\\foo\\bar\\abc\\def\\my-wrapper";
1861                } else {
1862                    let wrapper_path = "/foo/bar/abc/def/my-wrapper";
1863                }
1864            }
1865            assert_eq!(
1866                cli_wrapper_relative.to_owned_cli(),
1867                vec![wrapper_path, "--verbose", "binary", "arg"],
1868            );
1869        }
1870    }
1871
1872    static PACKAGE_GRAPH_FIXTURE: LazyLock<PackageGraph> = LazyLock::new(|| {
1873        static FIXTURE_JSON: &str = include_str!("../../../fixtures/cargo-metadata.json");
1874        let metadata = CargoMetadata::parse_json(FIXTURE_JSON).expect("fixture is valid JSON");
1875        metadata
1876            .build_graph()
1877            .expect("fixture is valid PackageGraph")
1878    });
1879
1880    static PACKAGE_METADATA_ID: &str = "metadata-helper 0.1.0 (path+file:///Users/fakeuser/local/testcrates/metadata/metadata-helper)";
1881    fn package_metadata() -> PackageMetadata<'static> {
1882        PACKAGE_GRAPH_FIXTURE
1883            .metadata(&PackageId::new(PACKAGE_METADATA_ID))
1884            .expect("package ID is valid")
1885    }
1886}