nextest_runner/list/
test_list.rs

1// Copyright (c) The nextest Contributors
2// SPDX-License-Identifier: MIT OR Apache-2.0
3
4use super::{DisplayFilterMatcher, TestListDisplayFilter};
5use crate::{
6    cargo_config::EnvironmentMap,
7    config::{
8        EvaluatableProfile, ListSettings, TestSettings, WrapperScriptConfig,
9        WrapperScriptTargetRunner,
10    },
11    double_spawn::DoubleSpawnInfo,
12    errors::{CreateTestListError, FromMessagesError, WriteTestListError},
13    helpers::{convert_build_platform, dylib_path, dylib_path_envvar, write_test_name},
14    indenter::indented,
15    list::{BinaryList, OutputFormat, RustBuildMeta, Styles, TestListState},
16    reuse_build::PathMapper,
17    target_runner::{PlatformRunner, TargetRunner},
18    test_command::{LocalExecuteContext, TestCommand, TestCommandPhase},
19    test_filter::{BinaryMismatchReason, FilterBinaryMatch, FilterBound, TestFilterBuilder},
20    write_str::WriteStr,
21};
22use camino::{Utf8Path, Utf8PathBuf};
23use debug_ignore::DebugIgnore;
24use futures::prelude::*;
25use guppy::{
26    PackageId,
27    graph::{PackageGraph, PackageMetadata},
28};
29use nextest_filtering::{BinaryQuery, EvalContext, TestQuery};
30use nextest_metadata::{
31    BuildPlatform, FilterMatch, MismatchReason, RustBinaryId, RustNonTestBinaryKind,
32    RustTestBinaryKind, RustTestBinarySummary, RustTestCaseSummary, RustTestSuiteStatusSummary,
33    RustTestSuiteSummary, TestListSummary,
34};
35use owo_colors::OwoColorize;
36use std::{
37    borrow::Cow,
38    collections::{BTreeMap, BTreeSet},
39    ffi::{OsStr, OsString},
40    fmt, io,
41    path::PathBuf,
42    sync::{Arc, OnceLock},
43};
44use tokio::runtime::Runtime;
45use tracing::debug;
46
47/// A Rust test binary built by Cargo. This artifact hasn't been run yet so there's no information
48/// about the tests within it.
49///
50/// Accepted as input to [`TestList::new`].
51#[derive(Clone, Debug)]
52pub struct RustTestArtifact<'g> {
53    /// A unique identifier for this test artifact.
54    pub binary_id: RustBinaryId,
55
56    /// Metadata for the package this artifact is a part of. This is used to set the correct
57    /// environment variables.
58    pub package: PackageMetadata<'g>,
59
60    /// The path to the binary artifact.
61    pub binary_path: Utf8PathBuf,
62
63    /// The unique binary name defined in `Cargo.toml` or inferred by the filename.
64    pub binary_name: String,
65
66    /// The kind of Rust test binary this is.
67    pub kind: RustTestBinaryKind,
68
69    /// Non-test binaries to be exposed to this artifact at runtime (name, path).
70    pub non_test_binaries: BTreeSet<(String, Utf8PathBuf)>,
71
72    /// The working directory that this test should be executed in.
73    pub cwd: Utf8PathBuf,
74
75    /// The platform for which this test artifact was built.
76    pub build_platform: BuildPlatform,
77}
78
79impl<'g> RustTestArtifact<'g> {
80    /// Constructs a list of test binaries from the list of built binaries.
81    pub fn from_binary_list(
82        graph: &'g PackageGraph,
83        binary_list: Arc<BinaryList>,
84        rust_build_meta: &RustBuildMeta<TestListState>,
85        path_mapper: &PathMapper,
86        platform_filter: Option<BuildPlatform>,
87    ) -> Result<Vec<Self>, FromMessagesError> {
88        let mut binaries = vec![];
89
90        for binary in &binary_list.rust_binaries {
91            if platform_filter.is_some() && platform_filter != Some(binary.build_platform) {
92                continue;
93            }
94
95            // Look up the executable by package ID.
96            let package_id = PackageId::new(binary.package_id.clone());
97            let package = graph
98                .metadata(&package_id)
99                .map_err(FromMessagesError::PackageGraph)?;
100
101            // Tests are run in the directory containing Cargo.toml
102            let cwd = package
103                .manifest_path()
104                .parent()
105                .unwrap_or_else(|| {
106                    panic!(
107                        "manifest path {} doesn't have a parent",
108                        package.manifest_path()
109                    )
110                })
111                .to_path_buf();
112
113            let binary_path = path_mapper.map_binary(binary.path.clone());
114            let cwd = path_mapper.map_cwd(cwd);
115
116            // Non-test binaries are only exposed to integration tests and benchmarks.
117            let non_test_binaries = if binary.kind == RustTestBinaryKind::TEST
118                || binary.kind == RustTestBinaryKind::BENCH
119            {
120                // Note we must use the TestListState rust_build_meta here to ensure we get remapped
121                // paths.
122                match rust_build_meta.non_test_binaries.get(package_id.repr()) {
123                    Some(binaries) => binaries
124                        .iter()
125                        .filter(|binary| {
126                            // Only expose BIN_EXE non-test files.
127                            binary.kind == RustNonTestBinaryKind::BIN_EXE
128                        })
129                        .map(|binary| {
130                            // Convert relative paths to absolute ones by joining with the target directory.
131                            let abs_path = rust_build_meta.target_directory.join(&binary.path);
132                            (binary.name.clone(), abs_path)
133                        })
134                        .collect(),
135                    None => BTreeSet::new(),
136                }
137            } else {
138                BTreeSet::new()
139            };
140
141            binaries.push(RustTestArtifact {
142                binary_id: binary.id.clone(),
143                package,
144                binary_path,
145                binary_name: binary.name.clone(),
146                kind: binary.kind.clone(),
147                cwd,
148                non_test_binaries,
149                build_platform: binary.build_platform,
150            })
151        }
152
153        Ok(binaries)
154    }
155
156    /// Returns a [`BinaryQuery`] corresponding to this test artifact.
157    pub fn to_binary_query(&self) -> BinaryQuery<'_> {
158        BinaryQuery {
159            package_id: self.package.id(),
160            binary_id: &self.binary_id,
161            kind: &self.kind,
162            binary_name: &self.binary_name,
163            platform: convert_build_platform(self.build_platform),
164        }
165    }
166
167    // ---
168    // Helper methods
169    // ---
170    fn into_test_suite(self, status: RustTestSuiteStatus) -> (RustBinaryId, RustTestSuite<'g>) {
171        let Self {
172            binary_id,
173            package,
174            binary_path,
175            binary_name,
176            kind,
177            non_test_binaries,
178            cwd,
179            build_platform,
180        } = self;
181        (
182            binary_id.clone(),
183            RustTestSuite {
184                binary_id,
185                binary_path,
186                package,
187                binary_name,
188                kind,
189                non_test_binaries,
190                cwd,
191                build_platform,
192                status,
193            },
194        )
195    }
196}
197
198/// Information about skipped tests and binaries.
199#[derive(Clone, Debug, Eq, PartialEq)]
200pub struct SkipCounts {
201    /// The number of skipped tests.
202    pub skipped_tests: usize,
203
204    /// The number of tests skipped due to not being in the default set.
205    pub skipped_tests_default_filter: usize,
206
207    /// The number of skipped binaries.
208    pub skipped_binaries: usize,
209
210    /// The number of binaries skipped due to not being in the default set.
211    pub skipped_binaries_default_filter: usize,
212}
213
214/// List of test instances, obtained by querying the [`RustTestArtifact`] instances generated by Cargo.
215#[derive(Clone, Debug)]
216pub struct TestList<'g> {
217    test_count: usize,
218    rust_build_meta: RustBuildMeta<TestListState>,
219    rust_suites: BTreeMap<RustBinaryId, RustTestSuite<'g>>,
220    workspace_root: Utf8PathBuf,
221    env: EnvironmentMap,
222    updated_dylib_path: OsString,
223    // Computed on first access.
224    skip_counts: OnceLock<SkipCounts>,
225}
226
227impl<'g> TestList<'g> {
228    /// Creates a new test list by running the given command and applying the specified filter.
229    #[expect(clippy::too_many_arguments)]
230    pub fn new<I>(
231        ctx: &TestExecuteContext<'_>,
232        test_artifacts: I,
233        rust_build_meta: RustBuildMeta<TestListState>,
234        filter: &TestFilterBuilder,
235        workspace_root: Utf8PathBuf,
236        env: EnvironmentMap,
237        profile: &impl ListProfile,
238        bound: FilterBound,
239        list_threads: usize,
240    ) -> Result<Self, CreateTestListError>
241    where
242        I: IntoIterator<Item = RustTestArtifact<'g>>,
243        I::IntoIter: Send,
244    {
245        let updated_dylib_path = Self::create_dylib_path(&rust_build_meta)?;
246        debug!(
247            "updated {}: {}",
248            dylib_path_envvar(),
249            updated_dylib_path.to_string_lossy(),
250        );
251        let lctx = LocalExecuteContext {
252            phase: TestCommandPhase::List,
253            // Note: this is the remapped workspace root, not the original one.
254            // (We really should have newtypes for this.)
255            workspace_root: &workspace_root,
256            rust_build_meta: &rust_build_meta,
257            double_spawn: ctx.double_spawn,
258            dylib_path: &updated_dylib_path,
259            profile_name: ctx.profile_name,
260            env: &env,
261        };
262
263        let ecx = profile.filterset_ecx();
264
265        let runtime = Runtime::new().map_err(CreateTestListError::TokioRuntimeCreate)?;
266
267        let stream = futures::stream::iter(test_artifacts).map(|test_binary| {
268            async {
269                let binary_query = test_binary.to_binary_query();
270                let binary_match = filter.filter_binary_match(&test_binary, &ecx, bound);
271                match binary_match {
272                    FilterBinaryMatch::Definite | FilterBinaryMatch::Possible => {
273                        debug!(
274                            "executing test binary to obtain test list \
275                            (match result is {binary_match:?}): {}",
276                            test_binary.binary_id,
277                        );
278                        // Run the binary to obtain the test list.
279                        let list_settings = profile.list_settings_for(&binary_query);
280                        let (non_ignored, ignored) = test_binary
281                            .exec(&lctx, &list_settings, ctx.target_runner)
282                            .await?;
283                        let (bin, info) = Self::process_output(
284                            test_binary,
285                            filter,
286                            &ecx,
287                            bound,
288                            non_ignored.as_str(),
289                            ignored.as_str(),
290                        )?;
291                        Ok::<_, CreateTestListError>((bin, info))
292                    }
293                    FilterBinaryMatch::Mismatch { reason } => {
294                        debug!("skipping test binary: {reason}: {}", test_binary.binary_id,);
295                        Ok(Self::process_skipped(test_binary, reason))
296                    }
297                }
298            }
299        });
300        let fut = stream.buffer_unordered(list_threads).try_collect();
301
302        let rust_suites: BTreeMap<_, _> = runtime.block_on(fut)?;
303
304        // Ensure that the runtime doesn't stay hanging even if a custom test framework misbehaves
305        // (can be an issue on Windows).
306        runtime.shutdown_background();
307
308        let test_count = rust_suites
309            .values()
310            .map(|suite| suite.status.test_count())
311            .sum();
312
313        Ok(Self {
314            rust_suites,
315            workspace_root,
316            env,
317            rust_build_meta,
318            updated_dylib_path,
319            test_count,
320            skip_counts: OnceLock::new(),
321        })
322    }
323
324    /// Creates a new test list with the given binary names and outputs.
325    #[cfg(test)]
326    fn new_with_outputs(
327        test_bin_outputs: impl IntoIterator<
328            Item = (RustTestArtifact<'g>, impl AsRef<str>, impl AsRef<str>),
329        >,
330        workspace_root: Utf8PathBuf,
331        rust_build_meta: RustBuildMeta<TestListState>,
332        filter: &TestFilterBuilder,
333        env: EnvironmentMap,
334        ecx: &EvalContext<'_>,
335        bound: FilterBound,
336    ) -> Result<Self, CreateTestListError> {
337        let mut test_count = 0;
338
339        let updated_dylib_path = Self::create_dylib_path(&rust_build_meta)?;
340
341        let rust_suites = test_bin_outputs
342            .into_iter()
343            .map(|(test_binary, non_ignored, ignored)| {
344                let binary_match = filter.filter_binary_match(&test_binary, ecx, bound);
345                match binary_match {
346                    FilterBinaryMatch::Definite | FilterBinaryMatch::Possible => {
347                        debug!(
348                            "processing output for binary \
349                            (match result is {binary_match:?}): {}",
350                            test_binary.binary_id,
351                        );
352                        let (bin, info) = Self::process_output(
353                            test_binary,
354                            filter,
355                            ecx,
356                            bound,
357                            non_ignored.as_ref(),
358                            ignored.as_ref(),
359                        )?;
360                        test_count += info.status.test_count();
361                        Ok((bin, info))
362                    }
363                    FilterBinaryMatch::Mismatch { reason } => {
364                        debug!("skipping test binary: {reason}: {}", test_binary.binary_id,);
365                        Ok(Self::process_skipped(test_binary, reason))
366                    }
367                }
368            })
369            .collect::<Result<BTreeMap<_, _>, _>>()?;
370
371        Ok(Self {
372            rust_suites,
373            workspace_root,
374            env,
375            rust_build_meta,
376            updated_dylib_path,
377            test_count,
378            skip_counts: OnceLock::new(),
379        })
380    }
381
382    /// Returns the total number of tests across all binaries.
383    pub fn test_count(&self) -> usize {
384        self.test_count
385    }
386
387    /// Returns the Rust build-related metadata for this test list.
388    pub fn rust_build_meta(&self) -> &RustBuildMeta<TestListState> {
389        &self.rust_build_meta
390    }
391
392    /// Returns the total number of skipped tests.
393    pub fn skip_counts(&self) -> &SkipCounts {
394        self.skip_counts.get_or_init(|| {
395            let mut skipped_tests_default_filter = 0;
396            let skipped_tests = self
397                .iter_tests()
398                .filter(|instance| match instance.test_info.filter_match {
399                    FilterMatch::Mismatch {
400                        reason: MismatchReason::DefaultFilter,
401                    } => {
402                        skipped_tests_default_filter += 1;
403                        true
404                    }
405                    FilterMatch::Mismatch { .. } => true,
406                    FilterMatch::Matches => false,
407                })
408                .count();
409
410            let mut skipped_binaries_default_filter = 0;
411            let skipped_binaries = self
412                .rust_suites
413                .values()
414                .filter(|suite| match suite.status {
415                    RustTestSuiteStatus::Skipped {
416                        reason: BinaryMismatchReason::DefaultSet,
417                    } => {
418                        skipped_binaries_default_filter += 1;
419                        true
420                    }
421                    RustTestSuiteStatus::Skipped { .. } => true,
422                    RustTestSuiteStatus::Listed { .. } => false,
423                })
424                .count();
425
426            SkipCounts {
427                skipped_tests,
428                skipped_tests_default_filter,
429                skipped_binaries,
430                skipped_binaries_default_filter,
431            }
432        })
433    }
434
435    /// Returns the total number of tests that aren't skipped.
436    ///
437    /// It is always the case that `run_count + skip_count == test_count`.
438    pub fn run_count(&self) -> usize {
439        self.test_count - self.skip_counts().skipped_tests
440    }
441
442    /// Returns the total number of binaries that contain tests.
443    pub fn binary_count(&self) -> usize {
444        self.rust_suites.len()
445    }
446
447    /// Returns the total number of binaries that were listed (not skipped).
448    pub fn listed_binary_count(&self) -> usize {
449        self.binary_count() - self.skip_counts().skipped_binaries
450    }
451
452    /// Returns the mapped workspace root.
453    pub fn workspace_root(&self) -> &Utf8Path {
454        &self.workspace_root
455    }
456
457    /// Returns the environment variables to be used when running tests.
458    pub fn cargo_env(&self) -> &EnvironmentMap {
459        &self.env
460    }
461
462    /// Returns the updated dynamic library path used for tests.
463    pub fn updated_dylib_path(&self) -> &OsStr {
464        &self.updated_dylib_path
465    }
466
467    /// Constructs a serializble summary for this test list.
468    pub fn to_summary(&self) -> TestListSummary {
469        let rust_suites = self
470            .rust_suites
471            .values()
472            .map(|test_suite| {
473                let (status, test_cases) = test_suite.status.to_summary();
474                let testsuite = RustTestSuiteSummary {
475                    package_name: test_suite.package.name().to_owned(),
476                    binary: RustTestBinarySummary {
477                        binary_name: test_suite.binary_name.clone(),
478                        package_id: test_suite.package.id().repr().to_owned(),
479                        kind: test_suite.kind.clone(),
480                        binary_path: test_suite.binary_path.clone(),
481                        binary_id: test_suite.binary_id.clone(),
482                        build_platform: test_suite.build_platform,
483                    },
484                    cwd: test_suite.cwd.clone(),
485                    status,
486                    test_cases,
487                };
488                (test_suite.binary_id.clone(), testsuite)
489            })
490            .collect();
491        let mut summary = TestListSummary::new(self.rust_build_meta.to_summary());
492        summary.test_count = self.test_count;
493        summary.rust_suites = rust_suites;
494        summary
495    }
496
497    /// Outputs this list to the given writer.
498    pub fn write(
499        &self,
500        output_format: OutputFormat,
501        writer: &mut dyn WriteStr,
502        colorize: bool,
503    ) -> Result<(), WriteTestListError> {
504        match output_format {
505            OutputFormat::Human { verbose } => self
506                .write_human(writer, verbose, colorize)
507                .map_err(WriteTestListError::Io),
508            OutputFormat::Serializable(format) => format.to_writer(&self.to_summary(), writer),
509        }
510    }
511
512    /// Iterates over all the test suites.
513    pub fn iter(&self) -> impl Iterator<Item = &RustTestSuite<'_>> + '_ {
514        self.rust_suites.values()
515    }
516
517    /// Iterates over the list of tests, returning the path and test name.
518    pub fn iter_tests(&self) -> impl Iterator<Item = TestInstance<'_>> + '_ {
519        self.rust_suites.values().flat_map(|test_suite| {
520            test_suite
521                .status
522                .test_cases()
523                .map(move |(name, test_info)| TestInstance::new(name, test_suite, test_info))
524        })
525    }
526
527    /// Produces a priority queue of tests based on the given profile.
528    pub fn to_priority_queue(
529        &'g self,
530        profile: &'g EvaluatableProfile<'g>,
531    ) -> TestPriorityQueue<'g> {
532        TestPriorityQueue::new(self, profile)
533    }
534
535    /// Outputs this list as a string with the given format.
536    pub fn to_string(&self, output_format: OutputFormat) -> Result<String, WriteTestListError> {
537        let mut s = String::with_capacity(1024);
538        self.write(output_format, &mut s, false)?;
539        Ok(s)
540    }
541
542    // ---
543    // Helper methods
544    // ---
545
546    // Empty list for tests.
547    #[cfg(test)]
548    pub(crate) fn empty() -> Self {
549        Self {
550            test_count: 0,
551            workspace_root: Utf8PathBuf::new(),
552            rust_build_meta: RustBuildMeta::empty(),
553            env: EnvironmentMap::empty(),
554            updated_dylib_path: OsString::new(),
555            rust_suites: BTreeMap::new(),
556            skip_counts: OnceLock::new(),
557        }
558    }
559
560    pub(crate) fn create_dylib_path(
561        rust_build_meta: &RustBuildMeta<TestListState>,
562    ) -> Result<OsString, CreateTestListError> {
563        let dylib_path = dylib_path();
564        let dylib_path_is_empty = dylib_path.is_empty();
565        let new_paths = rust_build_meta.dylib_paths();
566
567        let mut updated_dylib_path: Vec<PathBuf> =
568            Vec::with_capacity(dylib_path.len() + new_paths.len());
569        updated_dylib_path.extend(
570            new_paths
571                .iter()
572                .map(|path| path.clone().into_std_path_buf()),
573        );
574        updated_dylib_path.extend(dylib_path);
575
576        // On macOS, these are the defaults when DYLD_FALLBACK_LIBRARY_PATH isn't set or set to an
577        // empty string. (This is relevant if nextest is invoked as its own process and not
578        // a Cargo subcommand.)
579        //
580        // This copies the logic from
581        // https://cs.github.com/rust-lang/cargo/blob/7d289b171183578d45dcabc56db6db44b9accbff/src/cargo/core/compiler/compilation.rs#L292.
582        if cfg!(target_os = "macos") && dylib_path_is_empty {
583            if let Some(home) = home::home_dir() {
584                updated_dylib_path.push(home.join("lib"));
585            }
586            updated_dylib_path.push("/usr/local/lib".into());
587            updated_dylib_path.push("/usr/lib".into());
588        }
589
590        std::env::join_paths(updated_dylib_path)
591            .map_err(move |error| CreateTestListError::dylib_join_paths(new_paths, error))
592    }
593
594    fn process_output(
595        test_binary: RustTestArtifact<'g>,
596        filter: &TestFilterBuilder,
597        ecx: &EvalContext<'_>,
598        bound: FilterBound,
599        non_ignored: impl AsRef<str>,
600        ignored: impl AsRef<str>,
601    ) -> Result<(RustBinaryId, RustTestSuite<'g>), CreateTestListError> {
602        let mut test_cases = BTreeMap::new();
603
604        // Treat ignored and non-ignored as separate sets of single filters, so that partitioning
605        // based on one doesn't affect the other.
606        let mut non_ignored_filter = filter.build();
607        for test_name in Self::parse(&test_binary.binary_id, non_ignored.as_ref())? {
608            test_cases.insert(
609                test_name.into(),
610                RustTestCaseSummary {
611                    ignored: false,
612                    filter_match: non_ignored_filter.filter_match(
613                        &test_binary,
614                        test_name,
615                        ecx,
616                        bound,
617                        false,
618                    ),
619                },
620            );
621        }
622
623        let mut ignored_filter = filter.build();
624        for test_name in Self::parse(&test_binary.binary_id, ignored.as_ref())? {
625            // Note that libtest prints out:
626            // * just ignored tests if --ignored is passed in
627            // * all tests, both ignored and non-ignored, if --ignored is not passed in
628            // Adding ignored tests after non-ignored ones makes everything resolve correctly.
629            test_cases.insert(
630                test_name.into(),
631                RustTestCaseSummary {
632                    ignored: true,
633                    filter_match: ignored_filter.filter_match(
634                        &test_binary,
635                        test_name,
636                        ecx,
637                        bound,
638                        true,
639                    ),
640                },
641            );
642        }
643
644        Ok(test_binary.into_test_suite(RustTestSuiteStatus::Listed {
645            test_cases: test_cases.into(),
646        }))
647    }
648
649    fn process_skipped(
650        test_binary: RustTestArtifact<'g>,
651        reason: BinaryMismatchReason,
652    ) -> (RustBinaryId, RustTestSuite<'g>) {
653        test_binary.into_test_suite(RustTestSuiteStatus::Skipped { reason })
654    }
655
656    /// Parses the output of --list --message-format terse and returns a sorted list.
657    fn parse<'a>(
658        binary_id: &'a RustBinaryId,
659        list_output: &'a str,
660    ) -> Result<Vec<&'a str>, CreateTestListError> {
661        let mut list = Self::parse_impl(binary_id, list_output).collect::<Result<Vec<_>, _>>()?;
662        list.sort_unstable();
663        Ok(list)
664    }
665
666    fn parse_impl<'a>(
667        binary_id: &'a RustBinaryId,
668        list_output: &'a str,
669    ) -> impl Iterator<Item = Result<&'a str, CreateTestListError>> + 'a + use<'a> {
670        // The output is in the form:
671        // <test name>: test
672        // <test name>: test
673        // ...
674
675        list_output.lines().map(move |line| {
676            line.strip_suffix(": test")
677                .or_else(|| line.strip_suffix(": benchmark"))
678                .ok_or_else(|| {
679                    CreateTestListError::parse_line(
680                        binary_id.clone(),
681                        format!(
682                            "line '{line}' did not end with the string ': test' or ': benchmark'"
683                        ),
684                        list_output,
685                    )
686                })
687        })
688    }
689
690    /// Writes this test list out in a human-friendly format.
691    pub fn write_human(
692        &self,
693        writer: &mut dyn WriteStr,
694        verbose: bool,
695        colorize: bool,
696    ) -> io::Result<()> {
697        self.write_human_impl(None, writer, verbose, colorize)
698    }
699
700    /// Writes this test list out in a human-friendly format with the given filter.
701    pub(crate) fn write_human_with_filter(
702        &self,
703        filter: &TestListDisplayFilter<'_>,
704        writer: &mut dyn WriteStr,
705        verbose: bool,
706        colorize: bool,
707    ) -> io::Result<()> {
708        self.write_human_impl(Some(filter), writer, verbose, colorize)
709    }
710
711    fn write_human_impl(
712        &self,
713        filter: Option<&TestListDisplayFilter<'_>>,
714        mut writer: &mut dyn WriteStr,
715        verbose: bool,
716        colorize: bool,
717    ) -> io::Result<()> {
718        let mut styles = Styles::default();
719        if colorize {
720            styles.colorize();
721        }
722
723        for info in self.rust_suites.values() {
724            let matcher = match filter {
725                Some(filter) => match filter.matcher_for(&info.binary_id) {
726                    Some(matcher) => matcher,
727                    None => continue,
728                },
729                None => DisplayFilterMatcher::All,
730            };
731
732            // Skip this binary if there are no tests within it that will be run, and this isn't
733            // verbose output.
734            if !verbose
735                && info
736                    .status
737                    .test_cases()
738                    .all(|(_, test_case)| !test_case.filter_match.is_match())
739            {
740                continue;
741            }
742
743            writeln!(writer, "{}:", info.binary_id.style(styles.binary_id))?;
744            if verbose {
745                writeln!(
746                    writer,
747                    "  {} {}",
748                    "bin:".style(styles.field),
749                    info.binary_path
750                )?;
751                writeln!(writer, "  {} {}", "cwd:".style(styles.field), info.cwd)?;
752                writeln!(
753                    writer,
754                    "  {} {}",
755                    "build platform:".style(styles.field),
756                    info.build_platform,
757                )?;
758            }
759
760            let mut indented = indented(writer).with_str("    ");
761
762            match &info.status {
763                RustTestSuiteStatus::Listed { test_cases } => {
764                    let matching_tests: Vec<_> = test_cases
765                        .iter()
766                        .filter(|(name, _)| matcher.is_match(name))
767                        .collect();
768                    if matching_tests.is_empty() {
769                        writeln!(indented, "(no tests)")?;
770                    } else {
771                        for (name, info) in matching_tests {
772                            match (verbose, info.filter_match.is_match()) {
773                                (_, true) => {
774                                    write_test_name(name, &styles, &mut indented)?;
775                                    writeln!(indented)?;
776                                }
777                                (true, false) => {
778                                    write_test_name(name, &styles, &mut indented)?;
779                                    writeln!(indented, " (skipped)")?;
780                                }
781                                (false, false) => {
782                                    // Skip printing this test entirely if it isn't a match.
783                                }
784                            }
785                        }
786                    }
787                }
788                RustTestSuiteStatus::Skipped { reason } => {
789                    writeln!(indented, "(test binary {reason}, skipped)")?;
790                }
791            }
792
793            writer = indented.into_inner();
794        }
795        Ok(())
796    }
797}
798
799/// Profile implementation for test lists.
800pub trait ListProfile {
801    /// Returns the evaluation context.
802    fn filterset_ecx(&self) -> EvalContext<'_>;
803
804    /// Returns list-time settings for a test binary.
805    fn list_settings_for(&self, query: &BinaryQuery<'_>) -> ListSettings<'_>;
806}
807
808impl<'g> ListProfile for EvaluatableProfile<'g> {
809    fn filterset_ecx(&self) -> EvalContext<'_> {
810        self.filterset_ecx()
811    }
812
813    fn list_settings_for(&self, query: &BinaryQuery<'_>) -> ListSettings<'_> {
814        self.list_settings_for(query)
815    }
816}
817
818/// A test list that has been sorted and has had priorities applied to it.
819pub struct TestPriorityQueue<'a> {
820    tests: Vec<TestInstanceWithSettings<'a>>,
821}
822
823impl<'a> TestPriorityQueue<'a> {
824    fn new(test_list: &'a TestList<'a>, profile: &'a EvaluatableProfile<'a>) -> Self {
825        let mut tests = test_list
826            .iter_tests()
827            .map(|instance| {
828                let settings = profile.settings_for(&instance.to_test_query());
829                TestInstanceWithSettings { instance, settings }
830            })
831            .collect::<Vec<_>>();
832        // Note: this is a stable sort so that tests with the same priority are
833        // sorted by what `iter_tests` produced.
834        tests.sort_by_key(|test| test.settings.priority());
835
836        Self { tests }
837    }
838}
839
840impl<'a> IntoIterator for TestPriorityQueue<'a> {
841    type Item = TestInstanceWithSettings<'a>;
842    type IntoIter = std::vec::IntoIter<Self::Item>;
843
844    fn into_iter(self) -> Self::IntoIter {
845        self.tests.into_iter()
846    }
847}
848
849/// A test instance, along with computed settings from a profile.
850///
851/// Returned from [`TestPriorityQueue`].
852#[derive(Debug)]
853pub struct TestInstanceWithSettings<'a> {
854    /// The test instance.
855    pub instance: TestInstance<'a>,
856
857    /// The settings for this test.
858    pub settings: TestSettings<'a>,
859}
860
861/// A suite of tests within a single Rust test binary.
862///
863/// This is a representation of [`nextest_metadata::RustTestSuiteSummary`] used internally by the runner.
864#[derive(Clone, Debug, Eq, PartialEq)]
865pub struct RustTestSuite<'g> {
866    /// A unique identifier for this binary.
867    pub binary_id: RustBinaryId,
868
869    /// The path to the binary.
870    pub binary_path: Utf8PathBuf,
871
872    /// Package metadata.
873    pub package: PackageMetadata<'g>,
874
875    /// The unique binary name defined in `Cargo.toml` or inferred by the filename.
876    pub binary_name: String,
877
878    /// The kind of Rust test binary this is.
879    pub kind: RustTestBinaryKind,
880
881    /// The working directory that this test binary will be executed in. If None, the current directory
882    /// will not be changed.
883    pub cwd: Utf8PathBuf,
884
885    /// The platform the test suite is for (host or target).
886    pub build_platform: BuildPlatform,
887
888    /// Non-test binaries corresponding to this test suite (name, path).
889    pub non_test_binaries: BTreeSet<(String, Utf8PathBuf)>,
890
891    /// Test suite status and test case names.
892    pub status: RustTestSuiteStatus,
893}
894
895impl RustTestArtifact<'_> {
896    /// Run this binary with and without --ignored and get the corresponding outputs.
897    async fn exec(
898        &self,
899        lctx: &LocalExecuteContext<'_>,
900        list_settings: &ListSettings<'_>,
901        target_runner: &TargetRunner,
902    ) -> Result<(String, String), CreateTestListError> {
903        // This error situation has been known to happen with reused builds. It produces
904        // a really terrible and confusing "file not found" message if allowed to prceed.
905        if !self.cwd.is_dir() {
906            return Err(CreateTestListError::CwdIsNotDir {
907                binary_id: self.binary_id.clone(),
908                cwd: self.cwd.clone(),
909            });
910        }
911        let platform_runner = target_runner.for_build_platform(self.build_platform);
912
913        let non_ignored = self.exec_single(false, lctx, list_settings, platform_runner);
914        let ignored = self.exec_single(true, lctx, list_settings, platform_runner);
915
916        let (non_ignored_out, ignored_out) = futures::future::join(non_ignored, ignored).await;
917        Ok((non_ignored_out?, ignored_out?))
918    }
919
920    async fn exec_single(
921        &self,
922        ignored: bool,
923        lctx: &LocalExecuteContext<'_>,
924        list_settings: &ListSettings<'_>,
925        runner: Option<&PlatformRunner>,
926    ) -> Result<String, CreateTestListError> {
927        let mut cli = TestCommandCli::default();
928        cli.apply_wrappers(
929            list_settings.list_wrapper(),
930            runner,
931            lctx.workspace_root,
932            &lctx.rust_build_meta.target_directory,
933        );
934        cli.push(self.binary_path.as_str());
935
936        cli.extend(["--list", "--format", "terse"]);
937        if ignored {
938            cli.push("--ignored");
939        }
940
941        let cmd = TestCommand::new(
942            lctx,
943            cli.program
944                .clone()
945                .expect("at least one argument passed in")
946                .into_owned(),
947            &cli.args,
948            &self.cwd,
949            &self.package,
950            &self.non_test_binaries,
951        );
952
953        let output =
954            cmd.wait_with_output()
955                .await
956                .map_err(|error| CreateTestListError::CommandExecFail {
957                    binary_id: self.binary_id.clone(),
958                    command: cli.to_owned_cli(),
959                    error,
960                })?;
961
962        if output.status.success() {
963            String::from_utf8(output.stdout).map_err(|err| CreateTestListError::CommandNonUtf8 {
964                binary_id: self.binary_id.clone(),
965                command: cli.to_owned_cli(),
966                stdout: err.into_bytes(),
967                stderr: output.stderr,
968            })
969        } else {
970            Err(CreateTestListError::CommandFail {
971                binary_id: self.binary_id.clone(),
972                command: cli.to_owned_cli(),
973                exit_status: output.status,
974                stdout: output.stdout,
975                stderr: output.stderr,
976            })
977        }
978    }
979}
980
981/// Serializable information about the status of and test cases within a test suite.
982///
983/// Part of a [`RustTestSuiteSummary`].
984#[derive(Clone, Debug, Eq, PartialEq)]
985pub enum RustTestSuiteStatus {
986    /// The test suite was executed with `--list` and the list of test cases was obtained.
987    Listed {
988        /// The test cases contained within this test suite.
989        test_cases: DebugIgnore<BTreeMap<String, RustTestCaseSummary>>,
990    },
991
992    /// The test suite was not executed.
993    Skipped {
994        /// The reason why the test suite was skipped.
995        reason: BinaryMismatchReason,
996    },
997}
998
999static EMPTY_TEST_CASE_MAP: BTreeMap<String, RustTestCaseSummary> = BTreeMap::new();
1000
1001impl RustTestSuiteStatus {
1002    /// Returns the number of test cases within this suite.
1003    pub fn test_count(&self) -> usize {
1004        match self {
1005            RustTestSuiteStatus::Listed { test_cases } => test_cases.len(),
1006            RustTestSuiteStatus::Skipped { .. } => 0,
1007        }
1008    }
1009
1010    /// Returns the list of test cases within this suite.
1011    pub fn test_cases(&self) -> impl Iterator<Item = (&str, &RustTestCaseSummary)> + '_ {
1012        match self {
1013            RustTestSuiteStatus::Listed { test_cases } => test_cases.iter(),
1014            RustTestSuiteStatus::Skipped { .. } => {
1015                // Return an empty test case.
1016                EMPTY_TEST_CASE_MAP.iter()
1017            }
1018        }
1019        .map(|(name, case)| (name.as_str(), case))
1020    }
1021
1022    /// Converts this status to its serializable form.
1023    pub fn to_summary(
1024        &self,
1025    ) -> (
1026        RustTestSuiteStatusSummary,
1027        BTreeMap<String, RustTestCaseSummary>,
1028    ) {
1029        match self {
1030            Self::Listed { test_cases } => {
1031                (RustTestSuiteStatusSummary::LISTED, test_cases.clone().0)
1032            }
1033            Self::Skipped {
1034                reason: BinaryMismatchReason::Expression,
1035            } => (RustTestSuiteStatusSummary::SKIPPED, BTreeMap::new()),
1036            Self::Skipped {
1037                reason: BinaryMismatchReason::DefaultSet,
1038            } => (
1039                RustTestSuiteStatusSummary::SKIPPED_DEFAULT_FILTER,
1040                BTreeMap::new(),
1041            ),
1042        }
1043    }
1044}
1045
1046/// Represents a single test with its associated binary.
1047#[derive(Clone, Copy, Debug, Eq, PartialEq)]
1048pub struct TestInstance<'a> {
1049    /// The name of the test.
1050    pub name: &'a str,
1051
1052    /// Information about the test suite.
1053    pub suite_info: &'a RustTestSuite<'a>,
1054
1055    /// Information about the test.
1056    pub test_info: &'a RustTestCaseSummary,
1057}
1058
1059impl<'a> TestInstance<'a> {
1060    /// Creates a new `TestInstance`.
1061    pub(crate) fn new(
1062        name: &'a (impl AsRef<str> + ?Sized),
1063        suite_info: &'a RustTestSuite,
1064        test_info: &'a RustTestCaseSummary,
1065    ) -> Self {
1066        Self {
1067            name: name.as_ref(),
1068            suite_info,
1069            test_info,
1070        }
1071    }
1072
1073    /// Return an identifier for test instances, including being able to sort
1074    /// them.
1075    #[inline]
1076    pub fn id(&self) -> TestInstanceId<'a> {
1077        TestInstanceId {
1078            binary_id: &self.suite_info.binary_id,
1079            test_name: self.name,
1080        }
1081    }
1082
1083    /// Returns the corresponding [`TestQuery`] for this `TestInstance`.
1084    pub fn to_test_query(&self) -> TestQuery<'a> {
1085        TestQuery {
1086            binary_query: BinaryQuery {
1087                package_id: self.suite_info.package.id(),
1088                binary_id: &self.suite_info.binary_id,
1089                kind: &self.suite_info.kind,
1090                binary_name: &self.suite_info.binary_name,
1091                platform: convert_build_platform(self.suite_info.build_platform),
1092            },
1093            test_name: self.name,
1094        }
1095    }
1096
1097    /// Creates the command for this test instance.
1098    pub(crate) fn make_command(
1099        &self,
1100        ctx: &TestExecuteContext<'_>,
1101        test_list: &TestList<'_>,
1102        wrapper_script: Option<&'a WrapperScriptConfig>,
1103        extra_args: &[String],
1104    ) -> TestCommand {
1105        // TODO: non-rust tests
1106
1107        let platform_runner = ctx
1108            .target_runner
1109            .for_build_platform(self.suite_info.build_platform);
1110
1111        let mut cli = TestCommandCli::default();
1112        cli.apply_wrappers(
1113            wrapper_script,
1114            platform_runner,
1115            test_list.workspace_root(),
1116            &test_list.rust_build_meta().target_directory,
1117        );
1118        cli.push(self.suite_info.binary_path.as_str());
1119
1120        cli.extend(["--exact", self.name, "--nocapture"]);
1121        if self.test_info.ignored {
1122            cli.push("--ignored");
1123        }
1124        cli.extend(extra_args.iter().map(String::as_str));
1125
1126        let lctx = LocalExecuteContext {
1127            phase: TestCommandPhase::Run,
1128            workspace_root: test_list.workspace_root(),
1129            rust_build_meta: &test_list.rust_build_meta,
1130            double_spawn: ctx.double_spawn,
1131            dylib_path: test_list.updated_dylib_path(),
1132            profile_name: ctx.profile_name,
1133            env: &test_list.env,
1134        };
1135
1136        TestCommand::new(
1137            &lctx,
1138            cli.program
1139                .expect("at least one argument is guaranteed")
1140                .into_owned(),
1141            &cli.args,
1142            &self.suite_info.cwd,
1143            &self.suite_info.package,
1144            &self.suite_info.non_test_binaries,
1145        )
1146    }
1147}
1148
1149#[derive(Clone, Debug, Default)]
1150struct TestCommandCli<'a> {
1151    program: Option<Cow<'a, str>>,
1152    args: Vec<Cow<'a, str>>,
1153}
1154
1155impl<'a> TestCommandCli<'a> {
1156    fn apply_wrappers(
1157        &mut self,
1158        wrapper_script: Option<&'a WrapperScriptConfig>,
1159        platform_runner: Option<&'a PlatformRunner>,
1160        workspace_root: &Utf8Path,
1161        target_dir: &Utf8Path,
1162    ) {
1163        // Apply the wrapper script if it's enabled.
1164        if let Some(wrapper) = wrapper_script {
1165            match wrapper.target_runner {
1166                WrapperScriptTargetRunner::Ignore => {
1167                    // Ignore the platform runner.
1168                    self.push(wrapper.command.program(workspace_root, target_dir));
1169                    self.extend(wrapper.command.args.iter().map(String::as_str));
1170                }
1171                WrapperScriptTargetRunner::AroundWrapper => {
1172                    // Platform runner goes first.
1173                    if let Some(runner) = platform_runner {
1174                        self.push(runner.binary());
1175                        self.extend(runner.args());
1176                    }
1177                    self.push(wrapper.command.program(workspace_root, target_dir));
1178                    self.extend(wrapper.command.args.iter().map(String::as_str));
1179                }
1180                WrapperScriptTargetRunner::WithinWrapper => {
1181                    // Wrapper script goes first.
1182                    self.push(wrapper.command.program(workspace_root, target_dir));
1183                    self.extend(wrapper.command.args.iter().map(String::as_str));
1184                    if let Some(runner) = platform_runner {
1185                        self.push(runner.binary());
1186                        self.extend(runner.args());
1187                    }
1188                }
1189                WrapperScriptTargetRunner::OverridesWrapper => {
1190                    // Target runner overrides wrapper.
1191                    if let Some(runner) = platform_runner {
1192                        self.push(runner.binary());
1193                        self.extend(runner.args());
1194                    }
1195                }
1196            }
1197        } else {
1198            // If no wrapper script is enabled, use the platform runner.
1199            if let Some(runner) = platform_runner {
1200                self.push(runner.binary());
1201                self.extend(runner.args());
1202            }
1203        }
1204    }
1205
1206    fn push(&mut self, arg: impl Into<Cow<'a, str>>) {
1207        if self.program.is_none() {
1208            self.program = Some(arg.into());
1209        } else {
1210            self.args.push(arg.into());
1211        }
1212    }
1213
1214    fn extend(&mut self, args: impl IntoIterator<Item = &'a str>) {
1215        for arg in args {
1216            self.push(arg);
1217        }
1218    }
1219
1220    fn to_owned_cli(&self) -> Vec<String> {
1221        let mut owned_cli = Vec::new();
1222        if let Some(program) = &self.program {
1223            owned_cli.push(program.to_string());
1224        }
1225        owned_cli.extend(self.args.iter().map(|arg| arg.clone().into_owned()));
1226        owned_cli
1227    }
1228}
1229
1230/// A key for identifying and sorting test instances.
1231///
1232/// Returned by [`TestInstance::id`].
1233#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)]
1234pub struct TestInstanceId<'a> {
1235    /// The binary ID.
1236    pub binary_id: &'a RustBinaryId,
1237
1238    /// The name of the test.
1239    pub test_name: &'a str,
1240}
1241
1242impl fmt::Display for TestInstanceId<'_> {
1243    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1244        write!(f, "{} {}", self.binary_id, self.test_name)
1245    }
1246}
1247
1248/// Context required for test execution.
1249#[derive(Clone, Debug)]
1250pub struct TestExecuteContext<'a> {
1251    /// The name of the profile.
1252    pub profile_name: &'a str,
1253
1254    /// Double-spawn info.
1255    pub double_spawn: &'a DoubleSpawnInfo,
1256
1257    /// Target runner.
1258    pub target_runner: &'a TargetRunner,
1259}
1260
1261#[cfg(test)]
1262mod tests {
1263    use super::*;
1264    use crate::{
1265        cargo_config::{TargetDefinitionLocation, TargetTriple, TargetTripleSource},
1266        config::{ScriptCommand, ScriptCommandRelativeTo},
1267        list::SerializableFormat,
1268        platform::{BuildPlatforms, HostPlatform, PlatformLibdir, TargetPlatform},
1269        target_runner::PlatformRunnerSource,
1270        test_filter::{RunIgnored, TestFilterPatterns},
1271    };
1272    use guppy::CargoMetadata;
1273    use indoc::indoc;
1274    use maplit::btreemap;
1275    use nextest_filtering::{CompiledExpr, Filterset, FiltersetKind, ParseContext};
1276    use nextest_metadata::{FilterMatch, MismatchReason, PlatformLibdirUnavailable};
1277    use pretty_assertions::assert_eq;
1278    use std::sync::LazyLock;
1279    use target_spec::Platform;
1280
1281    #[test]
1282    fn test_parse_test_list() {
1283        // Lines ending in ': benchmark' (output by the default Rust bencher) should be skipped.
1284        let non_ignored_output = indoc! {"
1285            tests::foo::test_bar: test
1286            tests::baz::test_quux: test
1287            benches::bench_foo: benchmark
1288        "};
1289        let ignored_output = indoc! {"
1290            tests::ignored::test_bar: test
1291            tests::baz::test_ignored: test
1292            benches::ignored_bench_foo: benchmark
1293        "};
1294
1295        let cx = ParseContext::new(&PACKAGE_GRAPH_FIXTURE);
1296
1297        let test_filter = TestFilterBuilder::new(
1298            RunIgnored::Default,
1299            None,
1300            TestFilterPatterns::default(),
1301            // Test against the platform() predicate because this is the most important one here.
1302            vec![
1303                Filterset::parse("platform(target)".to_owned(), &cx, FiltersetKind::Test).unwrap(),
1304            ],
1305        )
1306        .unwrap();
1307        let fake_cwd: Utf8PathBuf = "/fake/cwd".into();
1308        let fake_binary_name = "fake-binary".to_owned();
1309        let fake_binary_id = RustBinaryId::new("fake-package::fake-binary");
1310
1311        let test_binary = RustTestArtifact {
1312            binary_path: "/fake/binary".into(),
1313            cwd: fake_cwd.clone(),
1314            package: package_metadata(),
1315            binary_name: fake_binary_name.clone(),
1316            binary_id: fake_binary_id.clone(),
1317            kind: RustTestBinaryKind::LIB,
1318            non_test_binaries: BTreeSet::new(),
1319            build_platform: BuildPlatform::Target,
1320        };
1321
1322        let skipped_binary_name = "skipped-binary".to_owned();
1323        let skipped_binary_id = RustBinaryId::new("fake-package::skipped-binary");
1324        let skipped_binary = RustTestArtifact {
1325            binary_path: "/fake/skipped-binary".into(),
1326            cwd: fake_cwd.clone(),
1327            package: package_metadata(),
1328            binary_name: skipped_binary_name.clone(),
1329            binary_id: skipped_binary_id.clone(),
1330            kind: RustTestBinaryKind::PROC_MACRO,
1331            non_test_binaries: BTreeSet::new(),
1332            build_platform: BuildPlatform::Host,
1333        };
1334
1335        let fake_triple = TargetTriple {
1336            platform: Platform::new(
1337                "aarch64-unknown-linux-gnu",
1338                target_spec::TargetFeatures::Unknown,
1339            )
1340            .unwrap(),
1341            source: TargetTripleSource::CliOption,
1342            location: TargetDefinitionLocation::Builtin,
1343        };
1344        let fake_host_libdir = "/home/fake/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/x86_64-unknown-linux-gnu/lib";
1345        let build_platforms = BuildPlatforms {
1346            host: HostPlatform {
1347                platform: TargetTriple::x86_64_unknown_linux_gnu().platform,
1348                libdir: PlatformLibdir::Available(fake_host_libdir.into()),
1349            },
1350            target: Some(TargetPlatform {
1351                triple: fake_triple,
1352                // Test an unavailable libdir.
1353                libdir: PlatformLibdir::Unavailable(PlatformLibdirUnavailable::new_const("test")),
1354            }),
1355        };
1356
1357        let fake_env = EnvironmentMap::empty();
1358        let rust_build_meta =
1359            RustBuildMeta::new("/fake", build_platforms).map_paths(&PathMapper::noop());
1360        let ecx = EvalContext {
1361            default_filter: &CompiledExpr::ALL,
1362        };
1363        let test_list = TestList::new_with_outputs(
1364            [
1365                (test_binary, &non_ignored_output, &ignored_output),
1366                (
1367                    skipped_binary,
1368                    &"should-not-show-up-stdout",
1369                    &"should-not-show-up-stderr",
1370                ),
1371            ],
1372            Utf8PathBuf::from("/fake/path"),
1373            rust_build_meta,
1374            &test_filter,
1375            fake_env,
1376            &ecx,
1377            FilterBound::All,
1378        )
1379        .expect("valid output");
1380        assert_eq!(
1381            test_list.rust_suites,
1382            btreemap! {
1383                fake_binary_id.clone() => RustTestSuite {
1384                    status: RustTestSuiteStatus::Listed {
1385                        test_cases: btreemap! {
1386                            "tests::foo::test_bar".to_owned() => RustTestCaseSummary {
1387                                ignored: false,
1388                                filter_match: FilterMatch::Matches,
1389                            },
1390                            "tests::baz::test_quux".to_owned() => RustTestCaseSummary {
1391                                ignored: false,
1392                                filter_match: FilterMatch::Matches,
1393                            },
1394                            "benches::bench_foo".to_owned() => RustTestCaseSummary {
1395                                ignored: false,
1396                                filter_match: FilterMatch::Matches,
1397                            },
1398                            "tests::ignored::test_bar".to_owned() => RustTestCaseSummary {
1399                                ignored: true,
1400                                filter_match: FilterMatch::Mismatch { reason: MismatchReason::Ignored },
1401                            },
1402                            "tests::baz::test_ignored".to_owned() => RustTestCaseSummary {
1403                                ignored: true,
1404                                filter_match: FilterMatch::Mismatch { reason: MismatchReason::Ignored },
1405                            },
1406                            "benches::ignored_bench_foo".to_owned() => RustTestCaseSummary {
1407                                ignored: true,
1408                                filter_match: FilterMatch::Mismatch { reason: MismatchReason::Ignored },
1409                            },
1410                        }.into(),
1411                    },
1412                    cwd: fake_cwd.clone(),
1413                    build_platform: BuildPlatform::Target,
1414                    package: package_metadata(),
1415                    binary_name: fake_binary_name,
1416                    binary_id: fake_binary_id,
1417                    binary_path: "/fake/binary".into(),
1418                    kind: RustTestBinaryKind::LIB,
1419                    non_test_binaries: BTreeSet::new(),
1420                },
1421                skipped_binary_id.clone() => RustTestSuite {
1422                    status: RustTestSuiteStatus::Skipped {
1423                        reason: BinaryMismatchReason::Expression,
1424                    },
1425                    cwd: fake_cwd,
1426                    build_platform: BuildPlatform::Host,
1427                    package: package_metadata(),
1428                    binary_name: skipped_binary_name,
1429                    binary_id: skipped_binary_id,
1430                    binary_path: "/fake/skipped-binary".into(),
1431                    kind: RustTestBinaryKind::PROC_MACRO,
1432                    non_test_binaries: BTreeSet::new(),
1433                },
1434            }
1435        );
1436
1437        // Check that the expected outputs are valid.
1438        static EXPECTED_HUMAN: &str = indoc! {"
1439        fake-package::fake-binary:
1440            benches::bench_foo
1441            tests::baz::test_quux
1442            tests::foo::test_bar
1443        "};
1444        static EXPECTED_HUMAN_VERBOSE: &str = indoc! {"
1445            fake-package::fake-binary:
1446              bin: /fake/binary
1447              cwd: /fake/cwd
1448              build platform: target
1449                benches::bench_foo
1450                benches::ignored_bench_foo (skipped)
1451                tests::baz::test_ignored (skipped)
1452                tests::baz::test_quux
1453                tests::foo::test_bar
1454                tests::ignored::test_bar (skipped)
1455            fake-package::skipped-binary:
1456              bin: /fake/skipped-binary
1457              cwd: /fake/cwd
1458              build platform: host
1459                (test binary didn't match filtersets, skipped)
1460        "};
1461        static EXPECTED_JSON_PRETTY: &str = indoc! {r#"
1462            {
1463              "rust-build-meta": {
1464                "target-directory": "/fake",
1465                "base-output-directories": [],
1466                "non-test-binaries": {},
1467                "build-script-out-dirs": {},
1468                "linked-paths": [],
1469                "platforms": {
1470                  "host": {
1471                    "platform": {
1472                      "triple": "x86_64-unknown-linux-gnu",
1473                      "target-features": "unknown"
1474                    },
1475                    "libdir": {
1476                      "status": "available",
1477                      "path": "/home/fake/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/x86_64-unknown-linux-gnu/lib"
1478                    }
1479                  },
1480                  "targets": [
1481                    {
1482                      "platform": {
1483                        "triple": "aarch64-unknown-linux-gnu",
1484                        "target-features": "unknown"
1485                      },
1486                      "libdir": {
1487                        "status": "unavailable",
1488                        "reason": "test"
1489                      }
1490                    }
1491                  ]
1492                },
1493                "target-platforms": [
1494                  {
1495                    "triple": "aarch64-unknown-linux-gnu",
1496                    "target-features": "unknown"
1497                  }
1498                ],
1499                "target-platform": "aarch64-unknown-linux-gnu"
1500              },
1501              "test-count": 6,
1502              "rust-suites": {
1503                "fake-package::fake-binary": {
1504                  "package-name": "metadata-helper",
1505                  "binary-id": "fake-package::fake-binary",
1506                  "binary-name": "fake-binary",
1507                  "package-id": "metadata-helper 0.1.0 (path+file:///Users/fakeuser/local/testcrates/metadata/metadata-helper)",
1508                  "kind": "lib",
1509                  "binary-path": "/fake/binary",
1510                  "build-platform": "target",
1511                  "cwd": "/fake/cwd",
1512                  "status": "listed",
1513                  "testcases": {
1514                    "benches::bench_foo": {
1515                      "ignored": false,
1516                      "filter-match": {
1517                        "status": "matches"
1518                      }
1519                    },
1520                    "benches::ignored_bench_foo": {
1521                      "ignored": true,
1522                      "filter-match": {
1523                        "status": "mismatch",
1524                        "reason": "ignored"
1525                      }
1526                    },
1527                    "tests::baz::test_ignored": {
1528                      "ignored": true,
1529                      "filter-match": {
1530                        "status": "mismatch",
1531                        "reason": "ignored"
1532                      }
1533                    },
1534                    "tests::baz::test_quux": {
1535                      "ignored": false,
1536                      "filter-match": {
1537                        "status": "matches"
1538                      }
1539                    },
1540                    "tests::foo::test_bar": {
1541                      "ignored": false,
1542                      "filter-match": {
1543                        "status": "matches"
1544                      }
1545                    },
1546                    "tests::ignored::test_bar": {
1547                      "ignored": true,
1548                      "filter-match": {
1549                        "status": "mismatch",
1550                        "reason": "ignored"
1551                      }
1552                    }
1553                  }
1554                },
1555                "fake-package::skipped-binary": {
1556                  "package-name": "metadata-helper",
1557                  "binary-id": "fake-package::skipped-binary",
1558                  "binary-name": "skipped-binary",
1559                  "package-id": "metadata-helper 0.1.0 (path+file:///Users/fakeuser/local/testcrates/metadata/metadata-helper)",
1560                  "kind": "proc-macro",
1561                  "binary-path": "/fake/skipped-binary",
1562                  "build-platform": "host",
1563                  "cwd": "/fake/cwd",
1564                  "status": "skipped",
1565                  "testcases": {}
1566                }
1567              }
1568            }"#};
1569
1570        assert_eq!(
1571            test_list
1572                .to_string(OutputFormat::Human { verbose: false })
1573                .expect("human succeeded"),
1574            EXPECTED_HUMAN
1575        );
1576        assert_eq!(
1577            test_list
1578                .to_string(OutputFormat::Human { verbose: true })
1579                .expect("human succeeded"),
1580            EXPECTED_HUMAN_VERBOSE
1581        );
1582        println!(
1583            "{}",
1584            test_list
1585                .to_string(OutputFormat::Serializable(SerializableFormat::JsonPretty))
1586                .expect("json-pretty succeeded")
1587        );
1588        assert_eq!(
1589            test_list
1590                .to_string(OutputFormat::Serializable(SerializableFormat::JsonPretty))
1591                .expect("json-pretty succeeded"),
1592            EXPECTED_JSON_PRETTY
1593        );
1594    }
1595
1596    #[test]
1597    fn apply_wrappers_examples() {
1598        cfg_if::cfg_if! {
1599            if #[cfg(windows)]
1600            {
1601                let workspace_root = Utf8Path::new("D:\\workspace\\root");
1602                let target_dir = Utf8Path::new("C:\\foo\\bar");
1603            } else {
1604                let workspace_root = Utf8Path::new("/workspace/root");
1605                let target_dir = Utf8Path::new("/foo/bar");
1606            }
1607        };
1608
1609        // Test with no wrappers
1610        {
1611            let mut cli_no_wrappers = TestCommandCli::default();
1612            cli_no_wrappers.apply_wrappers(None, None, workspace_root, target_dir);
1613            cli_no_wrappers.extend(["binary", "arg"]);
1614            assert_eq!(cli_no_wrappers.to_owned_cli(), vec!["binary", "arg"]);
1615        }
1616
1617        // Test with platform runner only
1618        {
1619            let runner = PlatformRunner::debug_new(
1620                "runner".into(),
1621                Vec::new(),
1622                PlatformRunnerSource::Env("fake".to_owned()),
1623            );
1624            let mut cli_runner_only = TestCommandCli::default();
1625            cli_runner_only.apply_wrappers(None, Some(&runner), workspace_root, target_dir);
1626            cli_runner_only.extend(["binary", "arg"]);
1627            assert_eq!(
1628                cli_runner_only.to_owned_cli(),
1629                vec!["runner", "binary", "arg"],
1630            );
1631        }
1632
1633        // Test wrapper with ignore target runner
1634        {
1635            let runner = PlatformRunner::debug_new(
1636                "runner".into(),
1637                Vec::new(),
1638                PlatformRunnerSource::Env("fake".to_owned()),
1639            );
1640            let wrapper_ignore = WrapperScriptConfig {
1641                command: ScriptCommand {
1642                    program: "wrapper".into(),
1643                    args: Vec::new(),
1644                    relative_to: ScriptCommandRelativeTo::None,
1645                },
1646                target_runner: WrapperScriptTargetRunner::Ignore,
1647            };
1648            let mut cli_wrapper_ignore = TestCommandCli::default();
1649            cli_wrapper_ignore.apply_wrappers(
1650                Some(&wrapper_ignore),
1651                Some(&runner),
1652                workspace_root,
1653                target_dir,
1654            );
1655            cli_wrapper_ignore.extend(["binary", "arg"]);
1656            assert_eq!(
1657                cli_wrapper_ignore.to_owned_cli(),
1658                vec!["wrapper", "binary", "arg"],
1659            );
1660        }
1661
1662        // Test wrapper with around wrapper (runner first)
1663        {
1664            let runner = PlatformRunner::debug_new(
1665                "runner".into(),
1666                Vec::new(),
1667                PlatformRunnerSource::Env("fake".to_owned()),
1668            );
1669            let wrapper_around = WrapperScriptConfig {
1670                command: ScriptCommand {
1671                    program: "wrapper".into(),
1672                    args: Vec::new(),
1673                    relative_to: ScriptCommandRelativeTo::None,
1674                },
1675                target_runner: WrapperScriptTargetRunner::AroundWrapper,
1676            };
1677            let mut cli_wrapper_around = TestCommandCli::default();
1678            cli_wrapper_around.apply_wrappers(
1679                Some(&wrapper_around),
1680                Some(&runner),
1681                workspace_root,
1682                target_dir,
1683            );
1684            cli_wrapper_around.extend(["binary", "arg"]);
1685            assert_eq!(
1686                cli_wrapper_around.to_owned_cli(),
1687                vec!["runner", "wrapper", "binary", "arg"],
1688            );
1689        }
1690
1691        // Test wrapper with within wrapper (wrapper first)
1692        {
1693            let runner = PlatformRunner::debug_new(
1694                "runner".into(),
1695                Vec::new(),
1696                PlatformRunnerSource::Env("fake".to_owned()),
1697            );
1698            let wrapper_within = WrapperScriptConfig {
1699                command: ScriptCommand {
1700                    program: "wrapper".into(),
1701                    args: Vec::new(),
1702                    relative_to: ScriptCommandRelativeTo::None,
1703                },
1704                target_runner: WrapperScriptTargetRunner::WithinWrapper,
1705            };
1706            let mut cli_wrapper_within = TestCommandCli::default();
1707            cli_wrapper_within.apply_wrappers(
1708                Some(&wrapper_within),
1709                Some(&runner),
1710                workspace_root,
1711                target_dir,
1712            );
1713            cli_wrapper_within.extend(["binary", "arg"]);
1714            assert_eq!(
1715                cli_wrapper_within.to_owned_cli(),
1716                vec!["wrapper", "runner", "binary", "arg"],
1717            );
1718        }
1719
1720        // Test wrapper with overrides wrapper (runner only)
1721        {
1722            let runner = PlatformRunner::debug_new(
1723                "runner".into(),
1724                Vec::new(),
1725                PlatformRunnerSource::Env("fake".to_owned()),
1726            );
1727            let wrapper_overrides = WrapperScriptConfig {
1728                command: ScriptCommand {
1729                    program: "wrapper".into(),
1730                    args: Vec::new(),
1731                    relative_to: ScriptCommandRelativeTo::None,
1732                },
1733                target_runner: WrapperScriptTargetRunner::OverridesWrapper,
1734            };
1735            let mut cli_wrapper_overrides = TestCommandCli::default();
1736            cli_wrapper_overrides.apply_wrappers(
1737                Some(&wrapper_overrides),
1738                Some(&runner),
1739                workspace_root,
1740                target_dir,
1741            );
1742            cli_wrapper_overrides.extend(["binary", "arg"]);
1743            assert_eq!(
1744                cli_wrapper_overrides.to_owned_cli(),
1745                vec!["runner", "binary", "arg"],
1746            );
1747        }
1748
1749        // Test wrapper with args
1750        {
1751            let wrapper_with_args = WrapperScriptConfig {
1752                command: ScriptCommand {
1753                    program: "wrapper".into(),
1754                    args: vec!["--flag".to_string(), "value".to_string()],
1755                    relative_to: ScriptCommandRelativeTo::None,
1756                },
1757                target_runner: WrapperScriptTargetRunner::Ignore,
1758            };
1759            let mut cli_wrapper_args = TestCommandCli::default();
1760            cli_wrapper_args.apply_wrappers(
1761                Some(&wrapper_with_args),
1762                None,
1763                workspace_root,
1764                target_dir,
1765            );
1766            cli_wrapper_args.extend(["binary", "arg"]);
1767            assert_eq!(
1768                cli_wrapper_args.to_owned_cli(),
1769                vec!["wrapper", "--flag", "value", "binary", "arg"],
1770            );
1771        }
1772
1773        // Test platform runner with args
1774        {
1775            let runner_with_args = PlatformRunner::debug_new(
1776                "runner".into(),
1777                vec!["--runner-flag".into(), "value".into()],
1778                PlatformRunnerSource::Env("fake".to_owned()),
1779            );
1780            let mut cli_runner_args = TestCommandCli::default();
1781            cli_runner_args.apply_wrappers(
1782                None,
1783                Some(&runner_with_args),
1784                workspace_root,
1785                target_dir,
1786            );
1787            cli_runner_args.extend(["binary", "arg"]);
1788            assert_eq!(
1789                cli_runner_args.to_owned_cli(),
1790                vec!["runner", "--runner-flag", "value", "binary", "arg"],
1791            );
1792        }
1793
1794        // Test wrapper with ScriptCommandRelativeTo::WorkspaceRoot
1795        {
1796            let wrapper_relative_to_workspace_root = WrapperScriptConfig {
1797                command: ScriptCommand {
1798                    program: "abc/def/my-wrapper".into(),
1799                    args: vec!["--verbose".to_string()],
1800                    relative_to: ScriptCommandRelativeTo::WorkspaceRoot,
1801                },
1802                target_runner: WrapperScriptTargetRunner::Ignore,
1803            };
1804            let mut cli_wrapper_relative = TestCommandCli::default();
1805            cli_wrapper_relative.apply_wrappers(
1806                Some(&wrapper_relative_to_workspace_root),
1807                None,
1808                workspace_root,
1809                target_dir,
1810            );
1811            cli_wrapper_relative.extend(["binary", "arg"]);
1812
1813            cfg_if::cfg_if! {
1814                if #[cfg(windows)] {
1815                    let wrapper_path = "D:\\workspace\\root\\abc\\def\\my-wrapper";
1816                } else {
1817                    let wrapper_path = "/workspace/root/abc/def/my-wrapper";
1818                }
1819            }
1820            assert_eq!(
1821                cli_wrapper_relative.to_owned_cli(),
1822                vec![wrapper_path, "--verbose", "binary", "arg"],
1823            );
1824        }
1825
1826        // Test wrapper with ScriptCommandRelativeTo::Target
1827        {
1828            let wrapper_relative_to_target = WrapperScriptConfig {
1829                command: ScriptCommand {
1830                    program: "abc/def/my-wrapper".into(),
1831                    args: vec!["--verbose".to_string()],
1832                    relative_to: ScriptCommandRelativeTo::Target,
1833                },
1834                target_runner: WrapperScriptTargetRunner::Ignore,
1835            };
1836            let mut cli_wrapper_relative = TestCommandCli::default();
1837            cli_wrapper_relative.apply_wrappers(
1838                Some(&wrapper_relative_to_target),
1839                None,
1840                workspace_root,
1841                target_dir,
1842            );
1843            cli_wrapper_relative.extend(["binary", "arg"]);
1844            cfg_if::cfg_if! {
1845                if #[cfg(windows)] {
1846                    let wrapper_path = "C:\\foo\\bar\\abc\\def\\my-wrapper";
1847                } else {
1848                    let wrapper_path = "/foo/bar/abc/def/my-wrapper";
1849                }
1850            }
1851            assert_eq!(
1852                cli_wrapper_relative.to_owned_cli(),
1853                vec![wrapper_path, "--verbose", "binary", "arg"],
1854            );
1855        }
1856    }
1857
1858    static PACKAGE_GRAPH_FIXTURE: LazyLock<PackageGraph> = LazyLock::new(|| {
1859        static FIXTURE_JSON: &str = include_str!("../../../fixtures/cargo-metadata.json");
1860        let metadata = CargoMetadata::parse_json(FIXTURE_JSON).expect("fixture is valid JSON");
1861        metadata
1862            .build_graph()
1863            .expect("fixture is valid PackageGraph")
1864    });
1865
1866    static PACKAGE_METADATA_ID: &str = "metadata-helper 0.1.0 (path+file:///Users/fakeuser/local/testcrates/metadata/metadata-helper)";
1867    fn package_metadata() -> PackageMetadata<'static> {
1868        PACKAGE_GRAPH_FIXTURE
1869            .metadata(&PackageId::new(PACKAGE_METADATA_ID))
1870            .expect("package ID is valid")
1871    }
1872}