1use crate::{
26 config::elements::LeakTimeoutResult,
27 errors::{DisplayErrorChain, FormatVersionError, FormatVersionErrorInner, WriteEventError},
28 list::RustTestSuite,
29 reporter::events::{ExecutionResult, StressIndex, TestEvent, TestEventKind},
30 test_output::{ChildExecutionOutput, ChildOutput, ChildSingleOutput},
31};
32use bstr::ByteSlice;
33use nextest_metadata::MismatchReason;
34use std::{collections::BTreeMap, fmt::Write as _};
35
36#[derive(Copy, Clone)]
39#[repr(u8)]
40enum FormatMinorVersion {
41 First = 1,
57 #[doc(hidden)]
58 _Max,
59}
60
61#[derive(Copy, Clone)]
65#[repr(u8)]
66enum FormatMajorVersion {
67 Unstable = 0,
69 #[doc(hidden)]
70 _Max,
71}
72
73struct LibtestSuite<'cfg> {
75 failed: usize,
77 succeeded: usize,
79 ignored: usize,
81 filtered: usize,
83 running: usize,
85
86 stress_index: Option<StressIndex>,
87 meta: &'cfg RustTestSuite<'cfg>,
88 total: std::time::Duration,
90 ignore_block: Option<bytes::BytesMut>,
94 output_block: bytes::BytesMut,
100}
101
102#[derive(Copy, Clone, Debug)]
105pub enum EmitNextestObject {
106 Yes,
108 No,
110}
111
112const KIND_TEST: &str = "test";
113const KIND_SUITE: &str = "suite";
114
115const EVENT_STARTED: &str = "started";
116const EVENT_IGNORED: &str = "ignored";
117const EVENT_OK: &str = "ok";
118const EVENT_FAILED: &str = "failed";
119
120#[inline]
121fn fmt_err(err: std::fmt::Error) -> WriteEventError {
122 WriteEventError::Io(std::io::Error::new(std::io::ErrorKind::OutOfMemory, err))
123}
124
125pub struct LibtestReporter<'cfg> {
128 _minor: FormatMinorVersion,
129 _major: FormatMajorVersion,
130 test_suites: BTreeMap<&'cfg str, LibtestSuite<'cfg>>,
131 emit_nextest_obj: bool,
134}
135
136impl<'cfg> LibtestReporter<'cfg> {
137 pub fn new(
148 version: Option<&str>,
149 emit_nextest_obj: EmitNextestObject,
150 ) -> Result<Self, FormatVersionError> {
151 let emit_nextest_obj = matches!(emit_nextest_obj, EmitNextestObject::Yes);
152
153 let Some(version) = version else {
154 return Ok(Self {
155 _minor: FormatMinorVersion::First,
156 _major: FormatMajorVersion::Unstable,
157 test_suites: BTreeMap::new(),
158 emit_nextest_obj,
159 });
160 };
161 let Some((major, minor)) = version.split_once('.') else {
162 return Err(FormatVersionError {
163 input: version.into(),
164 error: FormatVersionErrorInner::InvalidFormat {
165 expected: "<major>.<minor>",
166 },
167 });
168 };
169
170 let major: u8 = major.parse().map_err(|err| FormatVersionError {
171 input: version.into(),
172 error: FormatVersionErrorInner::InvalidInteger {
173 which: "major",
174 err,
175 },
176 })?;
177
178 let minor: u8 = minor.parse().map_err(|err| FormatVersionError {
179 input: version.into(),
180 error: FormatVersionErrorInner::InvalidInteger {
181 which: "minor",
182 err,
183 },
184 })?;
185
186 let major = match major {
187 0 => FormatMajorVersion::Unstable,
188 o => {
189 return Err(FormatVersionError {
190 input: version.into(),
191 error: FormatVersionErrorInner::InvalidValue {
192 which: "major",
193 value: o,
194 range: (FormatMajorVersion::Unstable as u8)
195 ..(FormatMajorVersion::_Max as u8),
196 },
197 });
198 }
199 };
200
201 let minor = match minor {
202 1 => FormatMinorVersion::First,
203 o => {
204 return Err(FormatVersionError {
205 input: version.into(),
206 error: FormatVersionErrorInner::InvalidValue {
207 which: "minor",
208 value: o,
209 range: (FormatMinorVersion::First as u8)..(FormatMinorVersion::_Max as u8),
210 },
211 });
212 }
213 };
214
215 Ok(Self {
216 _major: major,
217 _minor: minor,
218 test_suites: BTreeMap::new(),
219 emit_nextest_obj,
220 })
221 }
222
223 pub(crate) fn write_event(&mut self, event: &TestEvent<'cfg>) -> Result<(), WriteEventError> {
224 let mut retries = None;
225
226 let (kind, eve, stress_index, test_instance) = match &event.kind {
228 TestEventKind::TestStarted {
229 stress_index,
230 test_instance,
231 ..
232 } => (KIND_TEST, EVENT_STARTED, stress_index, test_instance),
233 TestEventKind::TestSkipped {
234 stress_index,
235 test_instance,
236 reason: MismatchReason::Ignored,
237 } => {
238 (KIND_TEST, EVENT_STARTED, stress_index, test_instance)
242 }
243 TestEventKind::TestFinished {
244 stress_index,
245 test_instance,
246 run_statuses,
247 ..
248 } => {
249 if run_statuses.len() > 1 {
250 retries = Some(run_statuses.len());
251 }
252
253 (
254 KIND_TEST,
255 match run_statuses.last_status().result {
256 ExecutionResult::Pass
257 | ExecutionResult::Leak {
258 result: LeakTimeoutResult::Pass,
259 } => EVENT_OK,
260 ExecutionResult::Leak {
261 result: LeakTimeoutResult::Fail,
262 }
263 | ExecutionResult::Fail { .. }
264 | ExecutionResult::ExecFail
265 | ExecutionResult::Timeout => EVENT_FAILED,
266 },
267 stress_index,
268 test_instance,
269 )
270 }
271 TestEventKind::StressSubRunFinished { .. } | TestEventKind::RunFinished { .. } => {
272 for test_suite in std::mem::take(&mut self.test_suites).into_values() {
273 self.finalize(test_suite)?;
274 }
275
276 return Ok(());
277 }
278 _ => return Ok(()),
279 };
280
281 let suite_info = test_instance.suite_info;
282 let crate_name = suite_info.package.name();
283 let binary_name = &suite_info.binary_name;
284
285 let test_suite = match self.test_suites.entry(suite_info.binary_id.as_str()) {
287 std::collections::btree_map::Entry::Vacant(e) => {
288 let (running, ignored, filtered) =
289 suite_info.status.test_cases().fold((0, 0, 0), |acc, tc| {
290 if tc.1.ignored {
291 (acc.0, acc.1 + 1, acc.2)
292 } else if tc.1.filter_match.is_match() {
293 (acc.0 + 1, acc.1, acc.2)
294 } else {
295 (acc.0, acc.1, acc.2 + 1)
296 }
297 });
298
299 let mut out = bytes::BytesMut::with_capacity(1024);
300 write!(
301 &mut out,
302 r#"{{"type":"{KIND_SUITE}","event":"{EVENT_STARTED}","test_count":{}"#,
303 running + ignored,
304 )
305 .map_err(fmt_err)?;
306
307 if self.emit_nextest_obj {
308 write!(
309 out,
310 r#","nextest":{{"crate":"{crate_name}","test_binary":"{binary_name}","kind":"{}""#,
311 suite_info.kind,
312 )
313 .map_err(fmt_err)?;
314
315 if let Some(stress_index) = stress_index {
316 write!(out, r#","stress_index":{}"#, stress_index.current)
317 .map_err(fmt_err)?;
318 if let Some(total) = stress_index.total {
319 write!(out, r#","stress_total":{total}"#).map_err(fmt_err)?;
320 }
321 }
322
323 write!(out, "}}").map_err(fmt_err)?;
324 }
325
326 out.extend_from_slice(b"}\n");
327
328 e.insert(LibtestSuite {
329 running,
330 failed: 0,
331 succeeded: 0,
332 ignored,
333 filtered,
334 stress_index: *stress_index,
335 meta: test_instance.suite_info,
336 total: std::time::Duration::new(0, 0),
337 ignore_block: None,
338 output_block: out,
339 })
340 }
341 std::collections::btree_map::Entry::Occupied(e) => e.into_mut(),
342 };
343
344 let out = &mut test_suite.output_block;
345
346 if matches!(event.kind, TestEventKind::TestFinished { .. }) {
349 if let Some(ib) = test_suite.ignore_block.take() {
350 out.extend_from_slice(&ib);
351 }
352 }
353
354 write!(
363 out,
364 r#"{{"type":"{kind}","event":"{eve}","name":"{}::{}"#,
365 suite_info.package.name(),
366 suite_info.binary_name,
367 )
368 .map_err(fmt_err)?;
369
370 if let Some(stress_index) = stress_index {
371 write!(out, "@stress-{}", stress_index.current).map_err(fmt_err)?;
372 }
373 write!(out, "${}", test_instance.name).map_err(fmt_err)?;
374 if let Some(retry_count) = retries {
375 write!(out, "#{retry_count}\"").map_err(fmt_err)?;
376 } else {
377 out.extend_from_slice(b"\"");
378 }
379
380 match &event.kind {
381 TestEventKind::TestFinished { run_statuses, .. } => {
382 let last_status = run_statuses.last_status();
383
384 test_suite.total += last_status.time_taken;
385 test_suite.running -= 1;
386
387 write!(
392 out,
393 r#","exec_time":{}"#,
394 last_status.time_taken.as_secs_f64()
395 )
396 .map_err(fmt_err)?;
397
398 match last_status.result {
399 ExecutionResult::Fail { .. } | ExecutionResult::ExecFail => {
400 test_suite.failed += 1;
401
402 write!(out, r#","stdout":""#).map_err(fmt_err)?;
405
406 strip_human_output_from_failed_test(
407 &last_status.output,
408 out,
409 test_instance.name,
410 )?;
411 out.extend_from_slice(b"\"");
412 }
413 ExecutionResult::Timeout => {
414 test_suite.failed += 1;
415 out.extend_from_slice(br#","reason":"time limit exceeded""#);
416 }
417 _ => {
418 test_suite.succeeded += 1;
419 }
420 }
421 }
422 TestEventKind::TestSkipped { .. } => {
423 test_suite.running -= 1;
424
425 if test_suite.ignore_block.is_none() {
426 test_suite.ignore_block = Some(bytes::BytesMut::with_capacity(1024));
427 }
428
429 let ib = test_suite
430 .ignore_block
431 .get_or_insert_with(|| bytes::BytesMut::with_capacity(1024));
432
433 writeln!(
434 ib,
435 r#"{{"type":"{kind}","event":"{EVENT_IGNORED}","name":"{}::{}${}"}}"#,
436 suite_info.package.name(),
437 suite_info.binary_name,
438 test_instance.name,
439 )
440 .map_err(fmt_err)?;
441 }
442 _ => {}
443 };
444
445 out.extend_from_slice(b"}\n");
446
447 if self.emit_nextest_obj {
448 {
449 use std::io::Write as _;
450
451 let mut stdout = std::io::stdout().lock();
452 stdout.write_all(out).map_err(WriteEventError::Io)?;
453 stdout.flush().map_err(WriteEventError::Io)?;
454 out.clear();
455 }
456
457 if test_suite.running == 0 {
458 if let Some(test_suite) = self.test_suites.remove(suite_info.binary_id.as_str()) {
459 self.finalize(test_suite)?;
460 }
461 }
462 } else {
463 if test_suite.running > 0 {
466 return Ok(());
467 }
468
469 if let Some(test_suite) = self.test_suites.remove(suite_info.binary_id.as_str()) {
470 self.finalize(test_suite)?;
471 }
472 }
473
474 Ok(())
475 }
476
477 fn finalize(&self, mut test_suite: LibtestSuite) -> Result<(), WriteEventError> {
478 let event = if test_suite.failed > 0 {
479 EVENT_FAILED
480 } else {
481 EVENT_OK
482 };
483
484 let out = &mut test_suite.output_block;
485 let suite_info = test_suite.meta;
486
487 if test_suite.running > 0 {
491 test_suite.filtered += test_suite.running;
492 }
493
494 write!(
495 out,
496 r#"{{"type":"{KIND_SUITE}","event":"{event}","passed":{},"failed":{},"ignored":{},"measured":0,"filtered_out":{},"exec_time":{}"#,
497 test_suite.succeeded,
498 test_suite.failed,
499 test_suite.ignored,
500 test_suite.filtered,
501 test_suite.total.as_secs_f64(),
502 )
503 .map_err(fmt_err)?;
504
505 if self.emit_nextest_obj {
506 let crate_name = suite_info.package.name();
507 let binary_name = &suite_info.binary_name;
508 write!(
509 out,
510 r#","nextest":{{"crate":"{crate_name}","test_binary":"{binary_name}","kind":"{}""#,
511 suite_info.kind,
512 )
513 .map_err(fmt_err)?;
514
515 if let Some(stress_index) = test_suite.stress_index {
516 write!(out, r#","stress_index":{}"#, stress_index.current).map_err(fmt_err)?;
517 if let Some(total) = stress_index.total {
518 write!(out, r#","stress_total":{total}"#).map_err(fmt_err)?;
519 }
520 }
521
522 write!(out, "}}").map_err(fmt_err)?;
523 }
524
525 out.extend_from_slice(b"}\n");
526
527 {
528 use std::io::Write as _;
529
530 let mut stdout = std::io::stdout().lock();
531 stdout.write_all(out).map_err(WriteEventError::Io)?;
532 stdout.flush().map_err(WriteEventError::Io)?;
533 }
534
535 Ok(())
536 }
537}
538
539fn strip_human_output_from_failed_test(
546 output: &ChildExecutionOutput,
547 out: &mut bytes::BytesMut,
548 test_name: &str,
549) -> Result<(), WriteEventError> {
550 match output {
551 ChildExecutionOutput::Output {
552 result: _,
553 output,
554 errors,
555 } => {
556 match output {
557 ChildOutput::Combined { output } => {
558 strip_human_stdout_or_combined(output, out, test_name)?;
559 }
560 ChildOutput::Split(split) => {
561 #[cfg(not(test))]
565 {
566 debug_assert!(false, "libtest output requires CaptureStrategy::Combined");
567 }
568 if let Some(stdout) = &split.stdout {
569 if !stdout.is_empty() {
570 write!(out, "--- STDOUT ---\\n").map_err(fmt_err)?;
571 strip_human_stdout_or_combined(stdout, out, test_name)?;
572 }
573 } else {
574 write!(out, "(stdout not captured)").map_err(fmt_err)?;
575 }
576 if let Some(stderr) = &split.stderr {
578 if !stderr.is_empty() {
579 write!(out, "\\n--- STDERR ---\\n").map_err(fmt_err)?;
580 write!(out, "{}", EscapedString(stderr.as_str_lossy()))
581 .map_err(fmt_err)?;
582 }
583 } else {
584 writeln!(out, "\\n(stderr not captured)").map_err(fmt_err)?;
585 }
586 }
587 }
588
589 if let Some(errors) = errors {
590 write!(out, "\\n--- EXECUTION ERRORS ---\\n").map_err(fmt_err)?;
591 write!(
592 out,
593 "{}",
594 EscapedString(&DisplayErrorChain::new(errors).to_string())
595 )
596 .map_err(fmt_err)?;
597 }
598 }
599 ChildExecutionOutput::StartError(error) => {
600 write!(out, "--- EXECUTION ERROR ---\\n").map_err(fmt_err)?;
601 write!(
602 out,
603 "{}",
604 EscapedString(&DisplayErrorChain::new(error).to_string())
605 )
606 .map_err(fmt_err)?;
607 }
608 }
609 Ok(())
610}
611
612fn strip_human_stdout_or_combined(
613 output: &ChildSingleOutput,
614 out: &mut bytes::BytesMut,
615 test_name: &str,
616) -> Result<(), WriteEventError> {
617 if output.buf.contains_str("running 1 test\n") {
618 let lines = output
620 .lines()
621 .skip_while(|line| line != b"running 1 test")
622 .skip(1)
623 .take_while(|line| {
624 if let Some(name) = line
625 .strip_prefix(b"test ")
626 .and_then(|np| np.strip_suffix(b" ... FAILED"))
627 {
628 if test_name.as_bytes() == name {
629 return false;
630 }
631 }
632
633 true
634 })
635 .map(|line| line.to_str_lossy());
636
637 for line in lines {
638 write!(out, "{}\\n", EscapedString(&line)).map_err(fmt_err)?;
640 }
641 } else {
642 write!(out, "{}", EscapedString(output.as_str_lossy())).map_err(fmt_err)?;
645 }
646
647 Ok(())
648}
649
650struct EscapedString<'s>(&'s str);
654
655impl std::fmt::Display for EscapedString<'_> {
656 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> ::std::fmt::Result {
657 let mut start = 0;
658 let s = self.0;
659
660 for (i, byte) in s.bytes().enumerate() {
661 let escaped = match byte {
662 b'"' => "\\\"",
663 b'\\' => "\\\\",
664 b'\x00' => "\\u0000",
665 b'\x01' => "\\u0001",
666 b'\x02' => "\\u0002",
667 b'\x03' => "\\u0003",
668 b'\x04' => "\\u0004",
669 b'\x05' => "\\u0005",
670 b'\x06' => "\\u0006",
671 b'\x07' => "\\u0007",
672 b'\x08' => "\\b",
673 b'\t' => "\\t",
674 b'\n' => "\\n",
675 b'\x0b' => "\\u000b",
676 b'\x0c' => "\\f",
677 b'\r' => "\\r",
678 b'\x0e' => "\\u000e",
679 b'\x0f' => "\\u000f",
680 b'\x10' => "\\u0010",
681 b'\x11' => "\\u0011",
682 b'\x12' => "\\u0012",
683 b'\x13' => "\\u0013",
684 b'\x14' => "\\u0014",
685 b'\x15' => "\\u0015",
686 b'\x16' => "\\u0016",
687 b'\x17' => "\\u0017",
688 b'\x18' => "\\u0018",
689 b'\x19' => "\\u0019",
690 b'\x1a' => "\\u001a",
691 b'\x1b' => "\\u001b",
692 b'\x1c' => "\\u001c",
693 b'\x1d' => "\\u001d",
694 b'\x1e' => "\\u001e",
695 b'\x1f' => "\\u001f",
696 b'\x7f' => "\\u007f",
697 _ => {
698 continue;
699 }
700 };
701
702 if start < i {
703 f.write_str(&s[start..i])?;
704 }
705
706 f.write_str(escaped)?;
707
708 start = i + 1;
709 }
710
711 if start != self.0.len() {
712 f.write_str(&s[start..])?;
713 }
714
715 Ok(())
716 }
717}
718
719#[cfg(test)]
720mod test {
721 use crate::{
722 errors::ChildStartError,
723 reporter::structured::libtest::strip_human_output_from_failed_test,
724 test_output::{ChildExecutionOutput, ChildOutput, ChildSplitOutput},
725 };
726 use bytes::BytesMut;
727 use color_eyre::eyre::eyre;
728 use std::{io, sync::Arc};
729
730 #[test]
734 fn strips_human_output() {
735 const TEST_OUTPUT: &[&str] = &[
736 "\n",
737 "running 1 test\n",
738 "[src/index.rs:185] \"boop\" = \"boop\"\n",
739 "this is stdout\n",
740 "this i stderr\nok?\n",
741 "thread 'index::test::download_url_crates_io'",
742 r" panicked at src/index.rs:206:9:
743oh no
744stack backtrace:
745 0: rust_begin_unwind
746 at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1/library/std/src/panicking.rs:597:5
747 1: core::panicking::panic_fmt
748 at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1/library/core/src/panicking.rs:72:14
749 2: tame_index::index::test::download_url_crates_io
750 at ./src/index.rs:206:9
751 3: tame_index::index::test::download_url_crates_io::{{closure}}
752 at ./src/index.rs:179:33
753 4: core::ops::function::FnOnce::call_once
754 at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1/library/core/src/ops/function.rs:250:5
755 5: core::ops::function::FnOnce::call_once
756 at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1/library/core/src/ops/function.rs:250:5
757note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.
758",
759 "test index::test::download_url_crates_io ... FAILED\n",
760 "\n\nfailures:\n\nfailures:\n index::test::download_url_crates_io\n\ntest result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 13 filtered out; finished in 0.01s\n",
761 ];
762
763 let output = {
764 let mut acc = BytesMut::new();
765 for line in TEST_OUTPUT {
766 acc.extend_from_slice(line.as_bytes());
767 }
768
769 ChildOutput::Combined {
770 output: acc.freeze().into(),
771 }
772 };
773
774 let mut actual = bytes::BytesMut::new();
775 strip_human_output_from_failed_test(
776 &ChildExecutionOutput::Output {
777 result: None,
778 output,
779 errors: None,
780 },
781 &mut actual,
782 "index::test::download_url_crates_io",
783 )
784 .unwrap();
785
786 insta::assert_snapshot!(std::str::from_utf8(&actual).unwrap());
787 }
788
789 #[test]
790 fn strips_human_output_custom_test_harness() {
791 const TEST_OUTPUT: &[&str] = &["\n", "this is a custom test harness!!!\n", "1 test passed"];
793
794 let output = {
795 let mut acc = BytesMut::new();
796 for line in TEST_OUTPUT {
797 acc.extend_from_slice(line.as_bytes());
798 }
799
800 ChildOutput::Combined {
801 output: acc.freeze().into(),
802 }
803 };
804
805 let mut actual = bytes::BytesMut::new();
806 strip_human_output_from_failed_test(
807 &ChildExecutionOutput::Output {
808 result: None,
809 output,
810 errors: None,
811 },
812 &mut actual,
813 "non-existent",
814 )
815 .unwrap();
816
817 insta::assert_snapshot!(std::str::from_utf8(&actual).unwrap());
818 }
819
820 #[test]
821 fn strips_human_output_start_error() {
822 let inner_error = eyre!("inner error");
823 let error = io::Error::other(inner_error);
824
825 let output = ChildExecutionOutput::StartError(ChildStartError::Spawn(Arc::new(error)));
826
827 let mut actual = bytes::BytesMut::new();
828 strip_human_output_from_failed_test(&output, &mut actual, "non-existent").unwrap();
829
830 insta::assert_snapshot!(std::str::from_utf8(&actual).unwrap());
831 }
832
833 #[test]
834 fn strips_human_output_none() {
835 let mut actual = bytes::BytesMut::new();
836 strip_human_output_from_failed_test(
837 &ChildExecutionOutput::Output {
838 result: None,
839 output: ChildOutput::Split(ChildSplitOutput {
840 stdout: None,
841 stderr: None,
842 }),
843 errors: None,
844 },
845 &mut actual,
846 "non-existent",
847 )
848 .unwrap();
849
850 insta::assert_snapshot!(std::str::from_utf8(&actual).unwrap());
851 }
852}