nextest_runner/reporter/structured/
libtest.rs

1// Copyright (c) The nextest Contributors
2// SPDX-License-Identifier: MIT OR Apache-2.0
3
4//! libtest compatible output support
5//!
6//! Before 1.70.0 it was possible to send `--format json` to test executables and
7//! they would print out a JSON line to stdout for various events. This format
8//! was however not intended to be stabilized, so 1.70.0 made it nightly only as
9//! intended. However, machine readable output is immensely useful to other
10//! tooling that can much more easily consume it than parsing the output meant
11//! for humans.
12//!
13//! Since there already existed tooling using the libtest output format, this
14//! event aggregator replicates that format so that projects can seamlessly
15//! integrate cargo-nextest into their project, as well as get the benefit of
16//! running their tests on stable instead of being forced to use nightly.
17//!
18//! This implementation will attempt to follow the libtest format as it changes,
19//! but the rate of changes is quite low (see <https://github.com/rust-lang/rust/blob/master/library/test/src/formatters/json.rs>)
20//! so this should not be a big issue to users, however, if the format is changed,
21//! the changes will be replicated in this file with a new minor version allowing
22//! users to move to the new format or stick to the format version(s) they were
23//! using before
24
25use crate::{
26    config::elements::LeakTimeoutResult,
27    errors::{DisplayErrorChain, FormatVersionError, FormatVersionErrorInner, WriteEventError},
28    list::RustTestSuite,
29    reporter::events::{ExecutionResult, StressIndex, TestEvent, TestEventKind},
30    test_output::{ChildExecutionOutput, ChildOutput, ChildSingleOutput},
31};
32use bstr::ByteSlice;
33use iddqd::{IdOrdItem, IdOrdMap, id_ord_map, id_upcast};
34use nextest_metadata::{MismatchReason, RustBinaryId};
35use std::fmt::Write as _;
36
37/// To support pinning the version of the output, we just use this simple enum
38/// to document changes as libtest output changes
39#[derive(Copy, Clone)]
40#[repr(u8)]
41enum FormatMinorVersion {
42    /// The libtest output as of `rustc 1.75.0-nightly (aa1a71e9e 2023-10-26)` with `--format json --report-time`
43    ///
44    /// * `{ "type": "suite", "event": "started", "test_count": <u32> }` - Start of a test binary run, always printed
45    ///   * `{ "type": "test", "event": "started", "name": "<name>" }` - Start of a single test, always printed
46    ///   * `{ "type": "test", "name": "<name>", "event": "ignored" }` - Printed if a test is ignored
47    ///     * Will have an additional `"message" = "<message>"` field if the there is a message in the ignore attribute eg. `#[ignore = "not yet implemented"]`
48    ///   * `{ "type": "test", "name": "<name>", "event": "ok", "exec_time": <f32> }` - Printed if a test runs successfully
49    ///   * `{ "type": "test", "name": "<name>", "event": "failed", "exec_time": <f32>, "stdout": "<escaped output collected during test execution>" }` - Printed if a test fails, note the stdout field actually contains both stdout and stderr despite the name
50    ///     * If `--ensure-time` is passed, libtest will add `"reason": "time limit exceeded"` if the test passes, but exceeds the time limit.
51    ///     * If `#[should_panic = "<expected message>"]` is used and message doesn't match, an additional `"message": "panic did not contain expected string\n<panic message>"` field is added
52    /// * `{ "type": "suite", "event": "<overall_status>", "passed": <u32>, "failed": <u32>, "ignored": <u32>, "measured": <u32>, "filtered_out": <u32>, "exec_time": <f32> }`
53    ///   * `event` will be `"ok"` if no failures occurred, or `"failed"` if `"failed" > 0`
54    ///   * `ignored` will be > 0 if there are `#[ignore]` tests and `--ignored` was not passed
55    ///   * `filtered_out` with be > 0 if there were tests not marked `#[ignore]` and `--ignored` was passed OR a test filter was passed and 1 or more tests were not executed
56    ///   * `measured` is only > 0 if running benchmarks
57    First = 1,
58    #[doc(hidden)]
59    _Max,
60}
61
62/// If libtest output is ever stabilized, this would most likely become the single
63/// version and we could get rid of the minor version, but who knows if that
64/// will ever happen
65#[derive(Copy, Clone)]
66#[repr(u8)]
67enum FormatMajorVersion {
68    /// The libtest output is unstable
69    Unstable = 0,
70    #[doc(hidden)]
71    _Max,
72}
73
74/// The accumulated stats for a single test binary
75struct LibtestSuite<'cfg> {
76    /// The number of tests that failed
77    failed: usize,
78    /// The number of tests that succeeded
79    succeeded: usize,
80    /// The number of tests that were ignored
81    ignored: usize,
82    /// The number of tests that were not executed due to filters
83    filtered: usize,
84    /// The number of tests in this suite that are still running
85    running: usize,
86
87    stress_index: Option<StressIndex>,
88    meta: &'cfg RustTestSuite<'cfg>,
89    /// The accumulated duration of every test that has been executed
90    total: std::time::Duration,
91    /// Libtest outputs outputs a `started` event for every test that isn't
92    /// filtered, including ignored tests, then outputs `ignored` events after
93    /// all the started events, so we just mimic that with a temporary buffer
94    ignore_block: Option<bytes::BytesMut>,
95    /// The single block of output accumulated for all tests executed in the binary,
96    /// this needs to be emitted as a single block to emulate how cargo test works,
97    /// executing each test binary serially and outputting a json line for each
98    /// event, as otherwise consumers would not be able to associate a single test
99    /// with its parent suite
100    output_block: bytes::BytesMut,
101}
102
103impl IdOrdItem for LibtestSuite<'_> {
104    type Key<'a>
105        = &'a RustBinaryId
106    where
107        Self: 'a;
108
109    fn key(&self) -> Self::Key<'_> {
110        &self.meta.binary_id
111    }
112
113    id_upcast!();
114}
115
116/// Determines whether the `nextest` subobject is added with additional metadata
117/// to events
118#[derive(Copy, Clone, Debug)]
119pub enum EmitNextestObject {
120    /// The `nextest` subobject is added
121    Yes,
122    /// The `nextest` subobject is not added
123    No,
124}
125
126const KIND_TEST: &str = "test";
127const KIND_SUITE: &str = "suite";
128
129const EVENT_STARTED: &str = "started";
130const EVENT_IGNORED: &str = "ignored";
131const EVENT_OK: &str = "ok";
132const EVENT_FAILED: &str = "failed";
133
134#[inline]
135fn fmt_err(err: std::fmt::Error) -> WriteEventError {
136    WriteEventError::Io(std::io::Error::new(std::io::ErrorKind::OutOfMemory, err))
137}
138
139/// A reporter that reports test runs in the same line-by-line JSON format as
140/// libtest itself
141pub struct LibtestReporter<'cfg> {
142    _minor: FormatMinorVersion,
143    _major: FormatMajorVersion,
144    test_suites: IdOrdMap<LibtestSuite<'cfg>>,
145    /// If true, we emit a `nextest` subobject with additional metadata in it
146    /// that consumers can use for easier integration if they wish
147    emit_nextest_obj: bool,
148}
149
150impl<'cfg> LibtestReporter<'cfg> {
151    /// Creates a new libtest reporter
152    ///
153    /// The version string is used to allow the reporter to evolve along with
154    /// libtest, but still be able to output a stable format for consumers. If
155    /// it is not specified the latest version of the format will be produced.
156    ///
157    /// If [`EmitNextestObject::Yes`] is passed, an additional `nextest` subobject
158    /// will be added to some events that includes additional metadata not produced
159    /// by libtest, but most consumers should still be able to consume them as
160    /// the base format itself is not changed
161    pub fn new(
162        version: Option<&str>,
163        emit_nextest_obj: EmitNextestObject,
164    ) -> Result<Self, FormatVersionError> {
165        let emit_nextest_obj = matches!(emit_nextest_obj, EmitNextestObject::Yes);
166
167        let Some(version) = version else {
168            return Ok(Self {
169                _minor: FormatMinorVersion::First,
170                _major: FormatMajorVersion::Unstable,
171                test_suites: IdOrdMap::new(),
172                emit_nextest_obj,
173            });
174        };
175        let Some((major, minor)) = version.split_once('.') else {
176            return Err(FormatVersionError {
177                input: version.into(),
178                error: FormatVersionErrorInner::InvalidFormat {
179                    expected: "<major>.<minor>",
180                },
181            });
182        };
183
184        let major: u8 = major.parse().map_err(|err| FormatVersionError {
185            input: version.into(),
186            error: FormatVersionErrorInner::InvalidInteger {
187                which: "major",
188                err,
189            },
190        })?;
191
192        let minor: u8 = minor.parse().map_err(|err| FormatVersionError {
193            input: version.into(),
194            error: FormatVersionErrorInner::InvalidInteger {
195                which: "minor",
196                err,
197            },
198        })?;
199
200        let major = match major {
201            0 => FormatMajorVersion::Unstable,
202            o => {
203                return Err(FormatVersionError {
204                    input: version.into(),
205                    error: FormatVersionErrorInner::InvalidValue {
206                        which: "major",
207                        value: o,
208                        range: (FormatMajorVersion::Unstable as u8)
209                            ..(FormatMajorVersion::_Max as u8),
210                    },
211                });
212            }
213        };
214
215        let minor = match minor {
216            1 => FormatMinorVersion::First,
217            o => {
218                return Err(FormatVersionError {
219                    input: version.into(),
220                    error: FormatVersionErrorInner::InvalidValue {
221                        which: "minor",
222                        value: o,
223                        range: (FormatMinorVersion::First as u8)..(FormatMinorVersion::_Max as u8),
224                    },
225                });
226            }
227        };
228
229        Ok(Self {
230            _major: major,
231            _minor: minor,
232            test_suites: IdOrdMap::new(),
233            emit_nextest_obj,
234        })
235    }
236
237    pub(crate) fn write_event(&mut self, event: &TestEvent<'cfg>) -> Result<(), WriteEventError> {
238        let mut retries = None;
239
240        // Write the pieces of data that are the same across all events
241        let (kind, eve, stress_index, test_instance) = match &event.kind {
242            TestEventKind::TestStarted {
243                stress_index,
244                test_instance,
245                ..
246            } => (KIND_TEST, EVENT_STARTED, stress_index, test_instance),
247            TestEventKind::TestSkipped {
248                stress_index,
249                test_instance,
250                reason: MismatchReason::Ignored,
251            } => {
252                // Note: unfortunately, libtest does not expose the message test in `#[ignore = "<message>"]`
253                // so we can't replicate the behavior of libtest exactly by emitting
254                // that message as additional metadata
255                (KIND_TEST, EVENT_STARTED, stress_index, test_instance)
256            }
257            TestEventKind::TestFinished {
258                stress_index,
259                test_instance,
260                run_statuses,
261                ..
262            } => {
263                if run_statuses.len() > 1 {
264                    retries = Some(run_statuses.len());
265                }
266
267                (
268                    KIND_TEST,
269                    match run_statuses.last_status().result {
270                        ExecutionResult::Pass
271                        | ExecutionResult::Leak {
272                            result: LeakTimeoutResult::Pass,
273                        } => EVENT_OK,
274                        ExecutionResult::Leak {
275                            result: LeakTimeoutResult::Fail,
276                        }
277                        | ExecutionResult::Fail { .. }
278                        | ExecutionResult::ExecFail
279                        | ExecutionResult::Timeout => EVENT_FAILED,
280                    },
281                    stress_index,
282                    test_instance,
283                )
284            }
285            TestEventKind::StressSubRunFinished { .. } | TestEventKind::RunFinished { .. } => {
286                for test_suite in std::mem::take(&mut self.test_suites) {
287                    self.finalize(test_suite)?;
288                }
289
290                return Ok(());
291            }
292            _ => return Ok(()),
293        };
294
295        let suite_info = test_instance.suite_info;
296        let crate_name = suite_info.package.name();
297        let binary_name = &suite_info.binary_name;
298
299        // Emit the suite start if this is the first test of the suite
300        let mut test_suite = match self.test_suites.entry(&suite_info.binary_id) {
301            id_ord_map::Entry::Vacant(e) => {
302                let (running, ignored, filtered) =
303                    suite_info.status.test_cases().fold((0, 0, 0), |acc, tc| {
304                        if tc.1.ignored {
305                            (acc.0, acc.1 + 1, acc.2)
306                        } else if tc.1.filter_match.is_match() {
307                            (acc.0 + 1, acc.1, acc.2)
308                        } else {
309                            (acc.0, acc.1, acc.2 + 1)
310                        }
311                    });
312
313                let mut out = bytes::BytesMut::with_capacity(1024);
314                write!(
315                    &mut out,
316                    r#"{{"type":"{KIND_SUITE}","event":"{EVENT_STARTED}","test_count":{}"#,
317                    running + ignored,
318                )
319                .map_err(fmt_err)?;
320
321                if self.emit_nextest_obj {
322                    write!(
323                        out,
324                        r#","nextest":{{"crate":"{crate_name}","test_binary":"{binary_name}","kind":"{}""#,
325                        suite_info.kind,
326                    )
327                    .map_err(fmt_err)?;
328
329                    if let Some(stress_index) = stress_index {
330                        write!(out, r#","stress_index":{}"#, stress_index.current)
331                            .map_err(fmt_err)?;
332                        if let Some(total) = stress_index.total {
333                            write!(out, r#","stress_total":{total}"#).map_err(fmt_err)?;
334                        }
335                    }
336
337                    write!(out, "}}").map_err(fmt_err)?;
338                }
339
340                out.extend_from_slice(b"}\n");
341
342                e.insert(LibtestSuite {
343                    running,
344                    failed: 0,
345                    succeeded: 0,
346                    ignored,
347                    filtered,
348                    stress_index: *stress_index,
349                    meta: test_instance.suite_info,
350                    total: std::time::Duration::new(0, 0),
351                    ignore_block: None,
352                    output_block: out,
353                })
354            }
355            id_ord_map::Entry::Occupied(e) => e.into_mut(),
356        };
357
358        let test_suite_mut = &mut *test_suite;
359        let out = &mut test_suite_mut.output_block;
360
361        // After all the tests have been started or ignored, put the block of
362        // tests that were ignored just as libtest does
363        if matches!(event.kind, TestEventKind::TestFinished { .. }) {
364            if let Some(ib) = test_suite_mut.ignore_block.take() {
365                out.extend_from_slice(&ib);
366            }
367        }
368
369        // This is one place where we deviate from the behavior of libtest, by
370        // always prefixing the test name with both the crate and the binary name,
371        // as this information is quite important to distinguish tests from each
372        // other when testing inside a large workspace with hundreds or thousands
373        // of tests
374        //
375        // Additionally, a `#<n>` is used as a suffix if the test was retried,
376        // as libtest does not support that functionality
377        write!(
378            out,
379            r#"{{"type":"{kind}","event":"{eve}","name":"{}::{}"#,
380            suite_info.package.name(),
381            suite_info.binary_name,
382        )
383        .map_err(fmt_err)?;
384
385        if let Some(stress_index) = stress_index {
386            write!(out, "@stress-{}", stress_index.current).map_err(fmt_err)?;
387        }
388        write!(out, "${}", test_instance.name).map_err(fmt_err)?;
389        if let Some(retry_count) = retries {
390            write!(out, "#{retry_count}\"").map_err(fmt_err)?;
391        } else {
392            out.extend_from_slice(b"\"");
393        }
394
395        match &event.kind {
396            TestEventKind::TestFinished { run_statuses, .. } => {
397                let last_status = run_statuses.last_status();
398
399                test_suite_mut.total += last_status.time_taken;
400                test_suite_mut.running -= 1;
401
402                // libtest actually requires an additional `--report-time` flag to be
403                // passed for the exec_time information to be written. This doesn't
404                // really make sense when outputting structured output so we emit it
405                // unconditionally
406                write!(
407                    out,
408                    r#","exec_time":{}"#,
409                    last_status.time_taken.as_secs_f64()
410                )
411                .map_err(fmt_err)?;
412
413                match last_status.result {
414                    ExecutionResult::Fail { .. } | ExecutionResult::ExecFail => {
415                        test_suite_mut.failed += 1;
416
417                        // Write the output from the test into the `stdout` (even
418                        // though it could contain stderr output as well).
419                        write!(out, r#","stdout":""#).map_err(fmt_err)?;
420
421                        strip_human_output_from_failed_test(
422                            &last_status.output,
423                            out,
424                            test_instance.name,
425                        )?;
426                        out.extend_from_slice(b"\"");
427                    }
428                    ExecutionResult::Timeout => {
429                        test_suite_mut.failed += 1;
430                        out.extend_from_slice(br#","reason":"time limit exceeded""#);
431                    }
432                    _ => {
433                        test_suite_mut.succeeded += 1;
434                    }
435                }
436            }
437            TestEventKind::TestSkipped { .. } => {
438                test_suite_mut.running -= 1;
439
440                if test_suite_mut.ignore_block.is_none() {
441                    test_suite_mut.ignore_block = Some(bytes::BytesMut::with_capacity(1024));
442                }
443
444                let ib = test_suite_mut
445                    .ignore_block
446                    .get_or_insert_with(|| bytes::BytesMut::with_capacity(1024));
447
448                writeln!(
449                    ib,
450                    r#"{{"type":"{kind}","event":"{EVENT_IGNORED}","name":"{}::{}${}"}}"#,
451                    suite_info.package.name(),
452                    suite_info.binary_name,
453                    test_instance.name,
454                )
455                .map_err(fmt_err)?;
456            }
457            _ => {}
458        };
459
460        out.extend_from_slice(b"}\n");
461
462        if self.emit_nextest_obj {
463            {
464                use std::io::Write as _;
465
466                let mut stdout = std::io::stdout().lock();
467                stdout.write_all(out).map_err(WriteEventError::Io)?;
468                stdout.flush().map_err(WriteEventError::Io)?;
469                out.clear();
470            }
471
472            if test_suite_mut.running == 0 {
473                std::mem::drop(test_suite);
474
475                if let Some(test_suite) = self.test_suites.remove(&suite_info.binary_id) {
476                    self.finalize(test_suite)?;
477                }
478            }
479        } else {
480            // If this is the last test of the suite, emit the test suite summary
481            // before emitting the entire block
482            if test_suite_mut.running > 0 {
483                return Ok(());
484            }
485
486            std::mem::drop(test_suite);
487
488            if let Some(test_suite) = self.test_suites.remove(&suite_info.binary_id) {
489                self.finalize(test_suite)?;
490            }
491        }
492
493        Ok(())
494    }
495
496    fn finalize(&self, mut test_suite: LibtestSuite) -> Result<(), WriteEventError> {
497        let event = if test_suite.failed > 0 {
498            EVENT_FAILED
499        } else {
500            EVENT_OK
501        };
502
503        let out = &mut test_suite.output_block;
504        let suite_info = test_suite.meta;
505
506        // It's possible that a test failure etc has cancelled the run, in which
507        // case we might still have tests that are "running", even ones that are
508        // actually skipped, so we just add those to the filtered list
509        if test_suite.running > 0 {
510            test_suite.filtered += test_suite.running;
511        }
512
513        write!(
514            out,
515            r#"{{"type":"{KIND_SUITE}","event":"{event}","passed":{},"failed":{},"ignored":{},"measured":0,"filtered_out":{},"exec_time":{}"#,
516            test_suite.succeeded,
517            test_suite.failed,
518            test_suite.ignored,
519            test_suite.filtered,
520            test_suite.total.as_secs_f64(),
521        )
522        .map_err(fmt_err)?;
523
524        if self.emit_nextest_obj {
525            let crate_name = suite_info.package.name();
526            let binary_name = &suite_info.binary_name;
527            write!(
528                out,
529                r#","nextest":{{"crate":"{crate_name}","test_binary":"{binary_name}","kind":"{}""#,
530                suite_info.kind,
531            )
532            .map_err(fmt_err)?;
533
534            if let Some(stress_index) = test_suite.stress_index {
535                write!(out, r#","stress_index":{}"#, stress_index.current).map_err(fmt_err)?;
536                if let Some(total) = stress_index.total {
537                    write!(out, r#","stress_total":{total}"#).map_err(fmt_err)?;
538                }
539            }
540
541            write!(out, "}}").map_err(fmt_err)?;
542        }
543
544        out.extend_from_slice(b"}\n");
545
546        {
547            use std::io::Write as _;
548
549            let mut stdout = std::io::stdout().lock();
550            stdout.write_all(out).map_err(WriteEventError::Io)?;
551            stdout.flush().map_err(WriteEventError::Io)?;
552        }
553
554        Ok(())
555    }
556}
557
558/// Unfortunately, to replicate the libtest json output, we need to do our own
559/// filtering of the output to strip out the data emitted by libtest in the
560/// human format.
561///
562/// This function relies on the fact that nextest runs every individual test in
563/// isolation.
564fn strip_human_output_from_failed_test(
565    output: &ChildExecutionOutput,
566    out: &mut bytes::BytesMut,
567    test_name: &str,
568) -> Result<(), WriteEventError> {
569    match output {
570        ChildExecutionOutput::Output {
571            result: _,
572            output,
573            errors,
574        } => {
575            match output {
576                ChildOutput::Combined { output } => {
577                    strip_human_stdout_or_combined(output, out, test_name)?;
578                }
579                ChildOutput::Split(split) => {
580                    // This is not a case that we hit because we always set CaptureStrategy to Combined. But
581                    // handle it in a reasonable fashion. (We do have a unit test for this case, so gate the
582                    // assertion with cfg(not(test)).)
583                    #[cfg(not(test))]
584                    {
585                        debug_assert!(false, "libtest output requires CaptureStrategy::Combined");
586                    }
587                    if let Some(stdout) = &split.stdout {
588                        if !stdout.is_empty() {
589                            write!(out, "--- STDOUT ---\\n").map_err(fmt_err)?;
590                            strip_human_stdout_or_combined(stdout, out, test_name)?;
591                        }
592                    } else {
593                        write!(out, "(stdout not captured)").map_err(fmt_err)?;
594                    }
595                    // If stderr is not empty, just write all of it in.
596                    if let Some(stderr) = &split.stderr {
597                        if !stderr.is_empty() {
598                            write!(out, "\\n--- STDERR ---\\n").map_err(fmt_err)?;
599                            write!(out, "{}", EscapedString(stderr.as_str_lossy()))
600                                .map_err(fmt_err)?;
601                        }
602                    } else {
603                        writeln!(out, "\\n(stderr not captured)").map_err(fmt_err)?;
604                    }
605                }
606            }
607
608            if let Some(errors) = errors {
609                write!(out, "\\n--- EXECUTION ERRORS ---\\n").map_err(fmt_err)?;
610                write!(
611                    out,
612                    "{}",
613                    EscapedString(&DisplayErrorChain::new(errors).to_string())
614                )
615                .map_err(fmt_err)?;
616            }
617        }
618        ChildExecutionOutput::StartError(error) => {
619            write!(out, "--- EXECUTION ERROR ---\\n").map_err(fmt_err)?;
620            write!(
621                out,
622                "{}",
623                EscapedString(&DisplayErrorChain::new(error).to_string())
624            )
625            .map_err(fmt_err)?;
626        }
627    }
628    Ok(())
629}
630
631fn strip_human_stdout_or_combined(
632    output: &ChildSingleOutput,
633    out: &mut bytes::BytesMut,
634    test_name: &str,
635) -> Result<(), WriteEventError> {
636    if output.buf.contains_str("running 1 test\n") {
637        // This is most likely the default test harness.
638        let lines = output
639            .lines()
640            .skip_while(|line| line != b"running 1 test")
641            .skip(1)
642            .take_while(|line| {
643                if let Some(name) = line
644                    .strip_prefix(b"test ")
645                    .and_then(|np| np.strip_suffix(b" ... FAILED"))
646                {
647                    if test_name.as_bytes() == name {
648                        return false;
649                    }
650                }
651
652                true
653            })
654            .map(|line| line.to_str_lossy());
655
656        for line in lines {
657            // This will never fail unless we are OOM
658            write!(out, "{}\\n", EscapedString(&line)).map_err(fmt_err)?;
659        }
660    } else {
661        // This is most likely a custom test harness. Just write out the entire
662        // output.
663        write!(out, "{}", EscapedString(output.as_str_lossy())).map_err(fmt_err)?;
664    }
665
666    Ok(())
667}
668
669/// Copy of the same string escaper used in libtest
670///
671/// <https://github.com/rust-lang/rust/blob/f440b5f0ea042cb2087a36631b20878f9847ee28/library/test/src/formatters/json.rs#L222-L285>
672struct EscapedString<'s>(&'s str);
673
674impl std::fmt::Display for EscapedString<'_> {
675    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> ::std::fmt::Result {
676        let mut start = 0;
677        let s = self.0;
678
679        for (i, byte) in s.bytes().enumerate() {
680            let escaped = match byte {
681                b'"' => "\\\"",
682                b'\\' => "\\\\",
683                b'\x00' => "\\u0000",
684                b'\x01' => "\\u0001",
685                b'\x02' => "\\u0002",
686                b'\x03' => "\\u0003",
687                b'\x04' => "\\u0004",
688                b'\x05' => "\\u0005",
689                b'\x06' => "\\u0006",
690                b'\x07' => "\\u0007",
691                b'\x08' => "\\b",
692                b'\t' => "\\t",
693                b'\n' => "\\n",
694                b'\x0b' => "\\u000b",
695                b'\x0c' => "\\f",
696                b'\r' => "\\r",
697                b'\x0e' => "\\u000e",
698                b'\x0f' => "\\u000f",
699                b'\x10' => "\\u0010",
700                b'\x11' => "\\u0011",
701                b'\x12' => "\\u0012",
702                b'\x13' => "\\u0013",
703                b'\x14' => "\\u0014",
704                b'\x15' => "\\u0015",
705                b'\x16' => "\\u0016",
706                b'\x17' => "\\u0017",
707                b'\x18' => "\\u0018",
708                b'\x19' => "\\u0019",
709                b'\x1a' => "\\u001a",
710                b'\x1b' => "\\u001b",
711                b'\x1c' => "\\u001c",
712                b'\x1d' => "\\u001d",
713                b'\x1e' => "\\u001e",
714                b'\x1f' => "\\u001f",
715                b'\x7f' => "\\u007f",
716                _ => {
717                    continue;
718                }
719            };
720
721            if start < i {
722                f.write_str(&s[start..i])?;
723            }
724
725            f.write_str(escaped)?;
726
727            start = i + 1;
728        }
729
730        if start != self.0.len() {
731            f.write_str(&s[start..])?;
732        }
733
734        Ok(())
735    }
736}
737
738#[cfg(test)]
739mod test {
740    use crate::{
741        errors::ChildStartError,
742        reporter::structured::libtest::strip_human_output_from_failed_test,
743        test_output::{ChildExecutionOutput, ChildOutput, ChildSplitOutput},
744    };
745    use bytes::BytesMut;
746    use color_eyre::eyre::eyre;
747    use std::{io, sync::Arc};
748
749    /// Validates that the human output portion from a failed test is stripped
750    /// out when writing a JSON string, as it is not part of the output when
751    /// libtest itself outputs the JSON, so we have 100% identical output to libtest
752    #[test]
753    fn strips_human_output() {
754        const TEST_OUTPUT: &[&str] = &[
755            "\n",
756            "running 1 test\n",
757            "[src/index.rs:185] \"boop\" = \"boop\"\n",
758            "this is stdout\n",
759            "this i stderr\nok?\n",
760            "thread 'index::test::download_url_crates_io'",
761            r" panicked at src/index.rs:206:9:
762oh no
763stack backtrace:
764    0: rust_begin_unwind
765                at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1/library/std/src/panicking.rs:597:5
766    1: core::panicking::panic_fmt
767                at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1/library/core/src/panicking.rs:72:14
768    2: tame_index::index::test::download_url_crates_io
769                at ./src/index.rs:206:9
770    3: tame_index::index::test::download_url_crates_io::{{closure}}
771                at ./src/index.rs:179:33
772    4: core::ops::function::FnOnce::call_once
773                at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1/library/core/src/ops/function.rs:250:5
774    5: core::ops::function::FnOnce::call_once
775                at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1/library/core/src/ops/function.rs:250:5
776note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.
777",
778            "test index::test::download_url_crates_io ... FAILED\n",
779            "\n\nfailures:\n\nfailures:\n    index::test::download_url_crates_io\n\ntest result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 13 filtered out; finished in 0.01s\n",
780        ];
781
782        let output = {
783            let mut acc = BytesMut::new();
784            for line in TEST_OUTPUT {
785                acc.extend_from_slice(line.as_bytes());
786            }
787
788            ChildOutput::Combined {
789                output: acc.freeze().into(),
790            }
791        };
792
793        let mut actual = bytes::BytesMut::new();
794        strip_human_output_from_failed_test(
795            &ChildExecutionOutput::Output {
796                result: None,
797                output,
798                errors: None,
799            },
800            &mut actual,
801            "index::test::download_url_crates_io",
802        )
803        .unwrap();
804
805        insta::assert_snapshot!(std::str::from_utf8(&actual).unwrap());
806    }
807
808    #[test]
809    fn strips_human_output_custom_test_harness() {
810        // For a custom test harness, we don't strip the human output at all.
811        const TEST_OUTPUT: &[&str] = &["\n", "this is a custom test harness!!!\n", "1 test passed"];
812
813        let output = {
814            let mut acc = BytesMut::new();
815            for line in TEST_OUTPUT {
816                acc.extend_from_slice(line.as_bytes());
817            }
818
819            ChildOutput::Combined {
820                output: acc.freeze().into(),
821            }
822        };
823
824        let mut actual = bytes::BytesMut::new();
825        strip_human_output_from_failed_test(
826            &ChildExecutionOutput::Output {
827                result: None,
828                output,
829                errors: None,
830            },
831            &mut actual,
832            "non-existent",
833        )
834        .unwrap();
835
836        insta::assert_snapshot!(std::str::from_utf8(&actual).unwrap());
837    }
838
839    #[test]
840    fn strips_human_output_start_error() {
841        let inner_error = eyre!("inner error");
842        let error = io::Error::other(inner_error);
843
844        let output = ChildExecutionOutput::StartError(ChildStartError::Spawn(Arc::new(error)));
845
846        let mut actual = bytes::BytesMut::new();
847        strip_human_output_from_failed_test(&output, &mut actual, "non-existent").unwrap();
848
849        insta::assert_snapshot!(std::str::from_utf8(&actual).unwrap());
850    }
851
852    #[test]
853    fn strips_human_output_none() {
854        let mut actual = bytes::BytesMut::new();
855        strip_human_output_from_failed_test(
856            &ChildExecutionOutput::Output {
857                result: None,
858                output: ChildOutput::Split(ChildSplitOutput {
859                    stdout: None,
860                    stderr: None,
861                }),
862                errors: None,
863            },
864            &mut actual,
865            "non-existent",
866        )
867        .unwrap();
868
869        insta::assert_snapshot!(std::str::from_utf8(&actual).unwrap());
870    }
871}