nextest_runner/reporter/structured/
libtest.rs

1// Copyright (c) The nextest Contributors
2// SPDX-License-Identifier: MIT OR Apache-2.0
3
4//! libtest compatible output support
5//!
6//! Before 1.70.0 it was possible to send `--format json` to test executables and
7//! they would print out a JSON line to stdout for various events. This format
8//! was however not intended to be stabilized, so 1.70.0 made it nightly only as
9//! intended. However, machine readable output is immensely useful to other
10//! tooling that can much more easily consume it than parsing the output meant
11//! for humans.
12//!
13//! Since there already existed tooling using the libtest output format, this
14//! event aggregator replicates that format so that projects can seamlessly
15//! integrate cargo-nextest into their project, as well as get the benefit of
16//! running their tests on stable instead of being forced to use nightly.
17//!
18//! This implementation will attempt to follow the libtest format as it changes,
19//! but the rate of changes is quite low (see <https://github.com/rust-lang/rust/blob/master/library/test/src/formatters/json.rs>)
20//! so this should not be a big issue to users, however, if the format is changed,
21//! the changes will be replicated in this file with a new minor version allowing
22//! users to move to the new format or stick to the format version(s) they were
23//! using before
24
25use crate::{
26    config::elements::LeakTimeoutResult,
27    errors::{DisplayErrorChain, FormatVersionError, FormatVersionErrorInner, WriteEventError},
28    list::RustTestSuite,
29    reporter::events::{ExecutionResult, StressIndex, TestEvent, TestEventKind},
30    test_output::{ChildExecutionOutput, ChildOutput, ChildSingleOutput},
31};
32use bstr::ByteSlice;
33use nextest_metadata::MismatchReason;
34use std::{collections::BTreeMap, fmt::Write as _};
35
36/// To support pinning the version of the output, we just use this simple enum
37/// to document changes as libtest output changes
38#[derive(Copy, Clone)]
39#[repr(u8)]
40enum FormatMinorVersion {
41    /// The libtest output as of `rustc 1.75.0-nightly (aa1a71e9e 2023-10-26)` with `--format json --report-time`
42    ///
43    /// * `{ "type": "suite", "event": "started", "test_count": <u32> }` - Start of a test binary run, always printed
44    ///   * `{ "type": "test", "event": "started", "name": "<name>" }` - Start of a single test, always printed
45    ///   * `{ "type": "test", "name": "<name>", "event": "ignored" }` - Printed if a test is ignored
46    ///     * Will have an additional `"message" = "<message>"` field if the there is a message in the ignore attribute eg. `#[ignore = "not yet implemented"]`
47    ///   * `{ "type": "test", "name": "<name>", "event": "ok", "exec_time": <f32> }` - Printed if a test runs successfully
48    ///   * `{ "type": "test", "name": "<name>", "event": "failed", "exec_time": <f32>, "stdout": "<escaped output collected during test execution>" }` - Printed if a test fails, note the stdout field actually contains both stdout and stderr despite the name
49    ///     * If `--ensure-time` is passed, libtest will add `"reason": "time limit exceeded"` if the test passes, but exceeds the time limit.
50    ///     * If `#[should_panic = "<expected message>"]` is used and message doesn't match, an additional `"message": "panic did not contain expected string\n<panic message>"` field is added
51    /// * `{ "type": "suite", "event": "<overall_status>", "passed": <u32>, "failed": <u32>, "ignored": <u32>, "measured": <u32>, "filtered_out": <u32>, "exec_time": <f32> }`
52    ///   * `event` will be `"ok"` if no failures occurred, or `"failed"` if `"failed" > 0`
53    ///   * `ignored` will be > 0 if there are `#[ignore]` tests and `--ignored` was not passed
54    ///   * `filtered_out` with be > 0 if there were tests not marked `#[ignore]` and `--ignored` was passed OR a test filter was passed and 1 or more tests were not executed
55    ///   * `measured` is only > 0 if running benchmarks
56    First = 1,
57    #[doc(hidden)]
58    _Max,
59}
60
61/// If libtest output is ever stabilized, this would most likely become the single
62/// version and we could get rid of the minor version, but who knows if that
63/// will ever happen
64#[derive(Copy, Clone)]
65#[repr(u8)]
66enum FormatMajorVersion {
67    /// The libtest output is unstable
68    Unstable = 0,
69    #[doc(hidden)]
70    _Max,
71}
72
73/// The accumulated stats for a single test binary
74struct LibtestSuite<'cfg> {
75    /// The number of tests that failed
76    failed: usize,
77    /// The number of tests that succeeded
78    succeeded: usize,
79    /// The number of tests that were ignored
80    ignored: usize,
81    /// The number of tests that were not executed due to filters
82    filtered: usize,
83    /// The number of tests in this suite that are still running
84    running: usize,
85
86    stress_index: Option<StressIndex>,
87    meta: &'cfg RustTestSuite<'cfg>,
88    /// The accumulated duration of every test that has been executed
89    total: std::time::Duration,
90    /// Libtest outputs outputs a `started` event for every test that isn't
91    /// filtered, including ignored tests, then outputs `ignored` events after
92    /// all the started events, so we just mimic that with a temporary buffer
93    ignore_block: Option<bytes::BytesMut>,
94    /// The single block of output accumulated for all tests executed in the binary,
95    /// this needs to be emitted as a single block to emulate how cargo test works,
96    /// executing each test binary serially and outputting a json line for each
97    /// event, as otherwise consumers would not be able to associate a single test
98    /// with its parent suite
99    output_block: bytes::BytesMut,
100}
101
102/// Determines whether the `nextest` subobject is added with additional metadata
103/// to events
104#[derive(Copy, Clone, Debug)]
105pub enum EmitNextestObject {
106    /// The `nextest` subobject is added
107    Yes,
108    /// The `nextest` subobject is not added
109    No,
110}
111
112const KIND_TEST: &str = "test";
113const KIND_SUITE: &str = "suite";
114
115const EVENT_STARTED: &str = "started";
116const EVENT_IGNORED: &str = "ignored";
117const EVENT_OK: &str = "ok";
118const EVENT_FAILED: &str = "failed";
119
120#[inline]
121fn fmt_err(err: std::fmt::Error) -> WriteEventError {
122    WriteEventError::Io(std::io::Error::new(std::io::ErrorKind::OutOfMemory, err))
123}
124
125/// A reporter that reports test runs in the same line-by-line JSON format as
126/// libtest itself
127pub struct LibtestReporter<'cfg> {
128    _minor: FormatMinorVersion,
129    _major: FormatMajorVersion,
130    test_suites: BTreeMap<&'cfg str, LibtestSuite<'cfg>>,
131    /// If true, we emit a `nextest` subobject with additional metadata in it
132    /// that consumers can use for easier integration if they wish
133    emit_nextest_obj: bool,
134}
135
136impl<'cfg> LibtestReporter<'cfg> {
137    /// Creates a new libtest reporter
138    ///
139    /// The version string is used to allow the reporter to evolve along with
140    /// libtest, but still be able to output a stable format for consumers. If
141    /// it is not specified the latest version of the format will be produced.
142    ///
143    /// If [`EmitNextestObject::Yes`] is passed, an additional `nextest` subobject
144    /// will be added to some events that includes additional metadata not produced
145    /// by libtest, but most consumers should still be able to consume them as
146    /// the base format itself is not changed
147    pub fn new(
148        version: Option<&str>,
149        emit_nextest_obj: EmitNextestObject,
150    ) -> Result<Self, FormatVersionError> {
151        let emit_nextest_obj = matches!(emit_nextest_obj, EmitNextestObject::Yes);
152
153        let Some(version) = version else {
154            return Ok(Self {
155                _minor: FormatMinorVersion::First,
156                _major: FormatMajorVersion::Unstable,
157                test_suites: BTreeMap::new(),
158                emit_nextest_obj,
159            });
160        };
161        let Some((major, minor)) = version.split_once('.') else {
162            return Err(FormatVersionError {
163                input: version.into(),
164                error: FormatVersionErrorInner::InvalidFormat {
165                    expected: "<major>.<minor>",
166                },
167            });
168        };
169
170        let major: u8 = major.parse().map_err(|err| FormatVersionError {
171            input: version.into(),
172            error: FormatVersionErrorInner::InvalidInteger {
173                which: "major",
174                err,
175            },
176        })?;
177
178        let minor: u8 = minor.parse().map_err(|err| FormatVersionError {
179            input: version.into(),
180            error: FormatVersionErrorInner::InvalidInteger {
181                which: "minor",
182                err,
183            },
184        })?;
185
186        let major = match major {
187            0 => FormatMajorVersion::Unstable,
188            o => {
189                return Err(FormatVersionError {
190                    input: version.into(),
191                    error: FormatVersionErrorInner::InvalidValue {
192                        which: "major",
193                        value: o,
194                        range: (FormatMajorVersion::Unstable as u8)
195                            ..(FormatMajorVersion::_Max as u8),
196                    },
197                });
198            }
199        };
200
201        let minor = match minor {
202            1 => FormatMinorVersion::First,
203            o => {
204                return Err(FormatVersionError {
205                    input: version.into(),
206                    error: FormatVersionErrorInner::InvalidValue {
207                        which: "minor",
208                        value: o,
209                        range: (FormatMinorVersion::First as u8)..(FormatMinorVersion::_Max as u8),
210                    },
211                });
212            }
213        };
214
215        Ok(Self {
216            _major: major,
217            _minor: minor,
218            test_suites: BTreeMap::new(),
219            emit_nextest_obj,
220        })
221    }
222
223    pub(crate) fn write_event(&mut self, event: &TestEvent<'cfg>) -> Result<(), WriteEventError> {
224        let mut retries = None;
225
226        // Write the pieces of data that are the same across all events
227        let (kind, eve, stress_index, test_instance) = match &event.kind {
228            TestEventKind::TestStarted {
229                stress_index,
230                test_instance,
231                ..
232            } => (KIND_TEST, EVENT_STARTED, stress_index, test_instance),
233            TestEventKind::TestSkipped {
234                stress_index,
235                test_instance,
236                reason: MismatchReason::Ignored,
237            } => {
238                // Note: unfortunately, libtest does not expose the message test in `#[ignore = "<message>"]`
239                // so we can't replicate the behavior of libtest exactly by emitting
240                // that message as additional metadata
241                (KIND_TEST, EVENT_STARTED, stress_index, test_instance)
242            }
243            TestEventKind::TestFinished {
244                stress_index,
245                test_instance,
246                run_statuses,
247                ..
248            } => {
249                if run_statuses.len() > 1 {
250                    retries = Some(run_statuses.len());
251                }
252
253                (
254                    KIND_TEST,
255                    match run_statuses.last_status().result {
256                        ExecutionResult::Pass
257                        | ExecutionResult::Leak {
258                            result: LeakTimeoutResult::Pass,
259                        } => EVENT_OK,
260                        ExecutionResult::Leak {
261                            result: LeakTimeoutResult::Fail,
262                        }
263                        | ExecutionResult::Fail { .. }
264                        | ExecutionResult::ExecFail
265                        | ExecutionResult::Timeout => EVENT_FAILED,
266                    },
267                    stress_index,
268                    test_instance,
269                )
270            }
271            TestEventKind::StressSubRunFinished { .. } | TestEventKind::RunFinished { .. } => {
272                for test_suite in std::mem::take(&mut self.test_suites).into_values() {
273                    self.finalize(test_suite)?;
274                }
275
276                return Ok(());
277            }
278            _ => return Ok(()),
279        };
280
281        let suite_info = test_instance.suite_info;
282        let crate_name = suite_info.package.name();
283        let binary_name = &suite_info.binary_name;
284
285        // Emit the suite start if this is the first test of the suite
286        let test_suite = match self.test_suites.entry(suite_info.binary_id.as_str()) {
287            std::collections::btree_map::Entry::Vacant(e) => {
288                let (running, ignored, filtered) =
289                    suite_info.status.test_cases().fold((0, 0, 0), |acc, tc| {
290                        if tc.1.ignored {
291                            (acc.0, acc.1 + 1, acc.2)
292                        } else if tc.1.filter_match.is_match() {
293                            (acc.0 + 1, acc.1, acc.2)
294                        } else {
295                            (acc.0, acc.1, acc.2 + 1)
296                        }
297                    });
298
299                let mut out = bytes::BytesMut::with_capacity(1024);
300                write!(
301                    &mut out,
302                    r#"{{"type":"{KIND_SUITE}","event":"{EVENT_STARTED}","test_count":{}"#,
303                    running + ignored,
304                )
305                .map_err(fmt_err)?;
306
307                if self.emit_nextest_obj {
308                    write!(
309                        out,
310                        r#","nextest":{{"crate":"{crate_name}","test_binary":"{binary_name}","kind":"{}""#,
311                        suite_info.kind,
312                    )
313                    .map_err(fmt_err)?;
314
315                    if let Some(stress_index) = stress_index {
316                        write!(out, r#","stress_index":{}"#, stress_index.current)
317                            .map_err(fmt_err)?;
318                        if let Some(total) = stress_index.total {
319                            write!(out, r#","stress_total":{total}"#).map_err(fmt_err)?;
320                        }
321                    }
322
323                    write!(out, "}}").map_err(fmt_err)?;
324                }
325
326                out.extend_from_slice(b"}\n");
327
328                e.insert(LibtestSuite {
329                    running,
330                    failed: 0,
331                    succeeded: 0,
332                    ignored,
333                    filtered,
334                    stress_index: *stress_index,
335                    meta: test_instance.suite_info,
336                    total: std::time::Duration::new(0, 0),
337                    ignore_block: None,
338                    output_block: out,
339                })
340            }
341            std::collections::btree_map::Entry::Occupied(e) => e.into_mut(),
342        };
343
344        let out = &mut test_suite.output_block;
345
346        // After all the tests have been started or ignored, put the block of
347        // tests that were ignored just as libtest does
348        if matches!(event.kind, TestEventKind::TestFinished { .. }) {
349            if let Some(ib) = test_suite.ignore_block.take() {
350                out.extend_from_slice(&ib);
351            }
352        }
353
354        // This is one place where we deviate from the behavior of libtest, by
355        // always prefixing the test name with both the crate and the binary name,
356        // as this information is quite important to distinguish tests from each
357        // other when testing inside a large workspace with hundreds or thousands
358        // of tests
359        //
360        // Additionally, a `#<n>` is used as a suffix if the test was retried,
361        // as libtest does not support that functionality
362        write!(
363            out,
364            r#"{{"type":"{kind}","event":"{eve}","name":"{}::{}"#,
365            suite_info.package.name(),
366            suite_info.binary_name,
367        )
368        .map_err(fmt_err)?;
369
370        if let Some(stress_index) = stress_index {
371            write!(out, "@stress-{}", stress_index.current).map_err(fmt_err)?;
372        }
373        write!(out, "${}", test_instance.name).map_err(fmt_err)?;
374        if let Some(retry_count) = retries {
375            write!(out, "#{retry_count}\"").map_err(fmt_err)?;
376        } else {
377            out.extend_from_slice(b"\"");
378        }
379
380        match &event.kind {
381            TestEventKind::TestFinished { run_statuses, .. } => {
382                let last_status = run_statuses.last_status();
383
384                test_suite.total += last_status.time_taken;
385                test_suite.running -= 1;
386
387                // libtest actually requires an additional `--report-time` flag to be
388                // passed for the exec_time information to be written. This doesn't
389                // really make sense when outputting structured output so we emit it
390                // unconditionally
391                write!(
392                    out,
393                    r#","exec_time":{}"#,
394                    last_status.time_taken.as_secs_f64()
395                )
396                .map_err(fmt_err)?;
397
398                match last_status.result {
399                    ExecutionResult::Fail { .. } | ExecutionResult::ExecFail => {
400                        test_suite.failed += 1;
401
402                        // Write the output from the test into the `stdout` (even
403                        // though it could contain stderr output as well).
404                        write!(out, r#","stdout":""#).map_err(fmt_err)?;
405
406                        strip_human_output_from_failed_test(
407                            &last_status.output,
408                            out,
409                            test_instance.name,
410                        )?;
411                        out.extend_from_slice(b"\"");
412                    }
413                    ExecutionResult::Timeout => {
414                        test_suite.failed += 1;
415                        out.extend_from_slice(br#","reason":"time limit exceeded""#);
416                    }
417                    _ => {
418                        test_suite.succeeded += 1;
419                    }
420                }
421            }
422            TestEventKind::TestSkipped { .. } => {
423                test_suite.running -= 1;
424
425                if test_suite.ignore_block.is_none() {
426                    test_suite.ignore_block = Some(bytes::BytesMut::with_capacity(1024));
427                }
428
429                let ib = test_suite
430                    .ignore_block
431                    .get_or_insert_with(|| bytes::BytesMut::with_capacity(1024));
432
433                writeln!(
434                    ib,
435                    r#"{{"type":"{kind}","event":"{EVENT_IGNORED}","name":"{}::{}${}"}}"#,
436                    suite_info.package.name(),
437                    suite_info.binary_name,
438                    test_instance.name,
439                )
440                .map_err(fmt_err)?;
441            }
442            _ => {}
443        };
444
445        out.extend_from_slice(b"}\n");
446
447        if self.emit_nextest_obj {
448            {
449                use std::io::Write as _;
450
451                let mut stdout = std::io::stdout().lock();
452                stdout.write_all(out).map_err(WriteEventError::Io)?;
453                stdout.flush().map_err(WriteEventError::Io)?;
454                out.clear();
455            }
456
457            if test_suite.running == 0 {
458                if let Some(test_suite) = self.test_suites.remove(suite_info.binary_id.as_str()) {
459                    self.finalize(test_suite)?;
460                }
461            }
462        } else {
463            // If this is the last test of the suite, emit the test suite summary
464            // before emitting the entire block
465            if test_suite.running > 0 {
466                return Ok(());
467            }
468
469            if let Some(test_suite) = self.test_suites.remove(suite_info.binary_id.as_str()) {
470                self.finalize(test_suite)?;
471            }
472        }
473
474        Ok(())
475    }
476
477    fn finalize(&self, mut test_suite: LibtestSuite) -> Result<(), WriteEventError> {
478        let event = if test_suite.failed > 0 {
479            EVENT_FAILED
480        } else {
481            EVENT_OK
482        };
483
484        let out = &mut test_suite.output_block;
485        let suite_info = test_suite.meta;
486
487        // It's possible that a test failure etc has cancelled the run, in which
488        // case we might still have tests that are "running", even ones that are
489        // actually skipped, so we just add those to the filtered list
490        if test_suite.running > 0 {
491            test_suite.filtered += test_suite.running;
492        }
493
494        write!(
495            out,
496            r#"{{"type":"{KIND_SUITE}","event":"{event}","passed":{},"failed":{},"ignored":{},"measured":0,"filtered_out":{},"exec_time":{}"#,
497            test_suite.succeeded,
498            test_suite.failed,
499            test_suite.ignored,
500            test_suite.filtered,
501            test_suite.total.as_secs_f64(),
502        )
503        .map_err(fmt_err)?;
504
505        if self.emit_nextest_obj {
506            let crate_name = suite_info.package.name();
507            let binary_name = &suite_info.binary_name;
508            write!(
509                out,
510                r#","nextest":{{"crate":"{crate_name}","test_binary":"{binary_name}","kind":"{}""#,
511                suite_info.kind,
512            )
513            .map_err(fmt_err)?;
514
515            if let Some(stress_index) = test_suite.stress_index {
516                write!(out, r#","stress_index":{}"#, stress_index.current).map_err(fmt_err)?;
517                if let Some(total) = stress_index.total {
518                    write!(out, r#","stress_total":{total}"#).map_err(fmt_err)?;
519                }
520            }
521
522            write!(out, "}}").map_err(fmt_err)?;
523        }
524
525        out.extend_from_slice(b"}\n");
526
527        {
528            use std::io::Write as _;
529
530            let mut stdout = std::io::stdout().lock();
531            stdout.write_all(out).map_err(WriteEventError::Io)?;
532            stdout.flush().map_err(WriteEventError::Io)?;
533        }
534
535        Ok(())
536    }
537}
538
539/// Unfortunately, to replicate the libtest json output, we need to do our own
540/// filtering of the output to strip out the data emitted by libtest in the
541/// human format.
542///
543/// This function relies on the fact that nextest runs every individual test in
544/// isolation.
545fn strip_human_output_from_failed_test(
546    output: &ChildExecutionOutput,
547    out: &mut bytes::BytesMut,
548    test_name: &str,
549) -> Result<(), WriteEventError> {
550    match output {
551        ChildExecutionOutput::Output {
552            result: _,
553            output,
554            errors,
555        } => {
556            match output {
557                ChildOutput::Combined { output } => {
558                    strip_human_stdout_or_combined(output, out, test_name)?;
559                }
560                ChildOutput::Split(split) => {
561                    // This is not a case that we hit because we always set CaptureStrategy to Combined. But
562                    // handle it in a reasonable fashion. (We do have a unit test for this case, so gate the
563                    // assertion with cfg(not(test)).)
564                    #[cfg(not(test))]
565                    {
566                        debug_assert!(false, "libtest output requires CaptureStrategy::Combined");
567                    }
568                    if let Some(stdout) = &split.stdout {
569                        if !stdout.is_empty() {
570                            write!(out, "--- STDOUT ---\\n").map_err(fmt_err)?;
571                            strip_human_stdout_or_combined(stdout, out, test_name)?;
572                        }
573                    } else {
574                        write!(out, "(stdout not captured)").map_err(fmt_err)?;
575                    }
576                    // If stderr is not empty, just write all of it in.
577                    if let Some(stderr) = &split.stderr {
578                        if !stderr.is_empty() {
579                            write!(out, "\\n--- STDERR ---\\n").map_err(fmt_err)?;
580                            write!(out, "{}", EscapedString(stderr.as_str_lossy()))
581                                .map_err(fmt_err)?;
582                        }
583                    } else {
584                        writeln!(out, "\\n(stderr not captured)").map_err(fmt_err)?;
585                    }
586                }
587            }
588
589            if let Some(errors) = errors {
590                write!(out, "\\n--- EXECUTION ERRORS ---\\n").map_err(fmt_err)?;
591                write!(
592                    out,
593                    "{}",
594                    EscapedString(&DisplayErrorChain::new(errors).to_string())
595                )
596                .map_err(fmt_err)?;
597            }
598        }
599        ChildExecutionOutput::StartError(error) => {
600            write!(out, "--- EXECUTION ERROR ---\\n").map_err(fmt_err)?;
601            write!(
602                out,
603                "{}",
604                EscapedString(&DisplayErrorChain::new(error).to_string())
605            )
606            .map_err(fmt_err)?;
607        }
608    }
609    Ok(())
610}
611
612fn strip_human_stdout_or_combined(
613    output: &ChildSingleOutput,
614    out: &mut bytes::BytesMut,
615    test_name: &str,
616) -> Result<(), WriteEventError> {
617    if output.buf.contains_str("running 1 test\n") {
618        // This is most likely the default test harness.
619        let lines = output
620            .lines()
621            .skip_while(|line| line != b"running 1 test")
622            .skip(1)
623            .take_while(|line| {
624                if let Some(name) = line
625                    .strip_prefix(b"test ")
626                    .and_then(|np| np.strip_suffix(b" ... FAILED"))
627                {
628                    if test_name.as_bytes() == name {
629                        return false;
630                    }
631                }
632
633                true
634            })
635            .map(|line| line.to_str_lossy());
636
637        for line in lines {
638            // This will never fail unless we are OOM
639            write!(out, "{}\\n", EscapedString(&line)).map_err(fmt_err)?;
640        }
641    } else {
642        // This is most likely a custom test harness. Just write out the entire
643        // output.
644        write!(out, "{}", EscapedString(output.as_str_lossy())).map_err(fmt_err)?;
645    }
646
647    Ok(())
648}
649
650/// Copy of the same string escaper used in libtest
651///
652/// <https://github.com/rust-lang/rust/blob/f440b5f0ea042cb2087a36631b20878f9847ee28/library/test/src/formatters/json.rs#L222-L285>
653struct EscapedString<'s>(&'s str);
654
655impl std::fmt::Display for EscapedString<'_> {
656    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> ::std::fmt::Result {
657        let mut start = 0;
658        let s = self.0;
659
660        for (i, byte) in s.bytes().enumerate() {
661            let escaped = match byte {
662                b'"' => "\\\"",
663                b'\\' => "\\\\",
664                b'\x00' => "\\u0000",
665                b'\x01' => "\\u0001",
666                b'\x02' => "\\u0002",
667                b'\x03' => "\\u0003",
668                b'\x04' => "\\u0004",
669                b'\x05' => "\\u0005",
670                b'\x06' => "\\u0006",
671                b'\x07' => "\\u0007",
672                b'\x08' => "\\b",
673                b'\t' => "\\t",
674                b'\n' => "\\n",
675                b'\x0b' => "\\u000b",
676                b'\x0c' => "\\f",
677                b'\r' => "\\r",
678                b'\x0e' => "\\u000e",
679                b'\x0f' => "\\u000f",
680                b'\x10' => "\\u0010",
681                b'\x11' => "\\u0011",
682                b'\x12' => "\\u0012",
683                b'\x13' => "\\u0013",
684                b'\x14' => "\\u0014",
685                b'\x15' => "\\u0015",
686                b'\x16' => "\\u0016",
687                b'\x17' => "\\u0017",
688                b'\x18' => "\\u0018",
689                b'\x19' => "\\u0019",
690                b'\x1a' => "\\u001a",
691                b'\x1b' => "\\u001b",
692                b'\x1c' => "\\u001c",
693                b'\x1d' => "\\u001d",
694                b'\x1e' => "\\u001e",
695                b'\x1f' => "\\u001f",
696                b'\x7f' => "\\u007f",
697                _ => {
698                    continue;
699                }
700            };
701
702            if start < i {
703                f.write_str(&s[start..i])?;
704            }
705
706            f.write_str(escaped)?;
707
708            start = i + 1;
709        }
710
711        if start != self.0.len() {
712            f.write_str(&s[start..])?;
713        }
714
715        Ok(())
716    }
717}
718
719#[cfg(test)]
720mod test {
721    use crate::{
722        errors::ChildStartError,
723        reporter::structured::libtest::strip_human_output_from_failed_test,
724        test_output::{ChildExecutionOutput, ChildOutput, ChildSplitOutput},
725    };
726    use bytes::BytesMut;
727    use color_eyre::eyre::eyre;
728    use std::{io, sync::Arc};
729
730    /// Validates that the human output portion from a failed test is stripped
731    /// out when writing a JSON string, as it is not part of the output when
732    /// libtest itself outputs the JSON, so we have 100% identical output to libtest
733    #[test]
734    fn strips_human_output() {
735        const TEST_OUTPUT: &[&str] = &[
736            "\n",
737            "running 1 test\n",
738            "[src/index.rs:185] \"boop\" = \"boop\"\n",
739            "this is stdout\n",
740            "this i stderr\nok?\n",
741            "thread 'index::test::download_url_crates_io'",
742            r" panicked at src/index.rs:206:9:
743oh no
744stack backtrace:
745    0: rust_begin_unwind
746                at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1/library/std/src/panicking.rs:597:5
747    1: core::panicking::panic_fmt
748                at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1/library/core/src/panicking.rs:72:14
749    2: tame_index::index::test::download_url_crates_io
750                at ./src/index.rs:206:9
751    3: tame_index::index::test::download_url_crates_io::{{closure}}
752                at ./src/index.rs:179:33
753    4: core::ops::function::FnOnce::call_once
754                at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1/library/core/src/ops/function.rs:250:5
755    5: core::ops::function::FnOnce::call_once
756                at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1/library/core/src/ops/function.rs:250:5
757note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.
758",
759            "test index::test::download_url_crates_io ... FAILED\n",
760            "\n\nfailures:\n\nfailures:\n    index::test::download_url_crates_io\n\ntest result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 13 filtered out; finished in 0.01s\n",
761        ];
762
763        let output = {
764            let mut acc = BytesMut::new();
765            for line in TEST_OUTPUT {
766                acc.extend_from_slice(line.as_bytes());
767            }
768
769            ChildOutput::Combined {
770                output: acc.freeze().into(),
771            }
772        };
773
774        let mut actual = bytes::BytesMut::new();
775        strip_human_output_from_failed_test(
776            &ChildExecutionOutput::Output {
777                result: None,
778                output,
779                errors: None,
780            },
781            &mut actual,
782            "index::test::download_url_crates_io",
783        )
784        .unwrap();
785
786        insta::assert_snapshot!(std::str::from_utf8(&actual).unwrap());
787    }
788
789    #[test]
790    fn strips_human_output_custom_test_harness() {
791        // For a custom test harness, we don't strip the human output at all.
792        const TEST_OUTPUT: &[&str] = &["\n", "this is a custom test harness!!!\n", "1 test passed"];
793
794        let output = {
795            let mut acc = BytesMut::new();
796            for line in TEST_OUTPUT {
797                acc.extend_from_slice(line.as_bytes());
798            }
799
800            ChildOutput::Combined {
801                output: acc.freeze().into(),
802            }
803        };
804
805        let mut actual = bytes::BytesMut::new();
806        strip_human_output_from_failed_test(
807            &ChildExecutionOutput::Output {
808                result: None,
809                output,
810                errors: None,
811            },
812            &mut actual,
813            "non-existent",
814        )
815        .unwrap();
816
817        insta::assert_snapshot!(std::str::from_utf8(&actual).unwrap());
818    }
819
820    #[test]
821    fn strips_human_output_start_error() {
822        let inner_error = eyre!("inner error");
823        let error = io::Error::other(inner_error);
824
825        let output = ChildExecutionOutput::StartError(ChildStartError::Spawn(Arc::new(error)));
826
827        let mut actual = bytes::BytesMut::new();
828        strip_human_output_from_failed_test(&output, &mut actual, "non-existent").unwrap();
829
830        insta::assert_snapshot!(std::str::from_utf8(&actual).unwrap());
831    }
832
833    #[test]
834    fn strips_human_output_none() {
835        let mut actual = bytes::BytesMut::new();
836        strip_human_output_from_failed_test(
837            &ChildExecutionOutput::Output {
838                result: None,
839                output: ChildOutput::Split(ChildSplitOutput {
840                    stdout: None,
841                    stderr: None,
842                }),
843                errors: None,
844            },
845            &mut actual,
846            "non-existent",
847        )
848        .unwrap();
849
850        insta::assert_snapshot!(std::str::from_utf8(&actual).unwrap());
851    }
852}