1use crate::{
26 config::elements::LeakTimeoutResult,
27 errors::{DisplayErrorChain, FormatVersionError, FormatVersionErrorInner, WriteEventError},
28 list::RustTestSuite,
29 reporter::events::{ExecutionResult, StressIndex, TestEvent, TestEventKind},
30 test_output::{ChildExecutionOutput, ChildOutput, ChildSingleOutput},
31};
32use bstr::ByteSlice;
33use iddqd::{IdOrdItem, IdOrdMap, id_ord_map, id_upcast};
34use nextest_metadata::{MismatchReason, RustBinaryId};
35use std::fmt::Write as _;
36
37#[derive(Copy, Clone)]
40#[repr(u8)]
41enum FormatMinorVersion {
42 First = 1,
58 #[doc(hidden)]
59 _Max,
60}
61
62#[derive(Copy, Clone)]
66#[repr(u8)]
67enum FormatMajorVersion {
68 Unstable = 0,
70 #[doc(hidden)]
71 _Max,
72}
73
74struct LibtestSuite<'cfg> {
76 failed: usize,
78 succeeded: usize,
80 ignored: usize,
82 filtered: usize,
84 running: usize,
86
87 stress_index: Option<StressIndex>,
88 meta: &'cfg RustTestSuite<'cfg>,
89 total: std::time::Duration,
91 ignore_block: Option<bytes::BytesMut>,
95 output_block: bytes::BytesMut,
101}
102
103impl IdOrdItem for LibtestSuite<'_> {
104 type Key<'a>
105 = &'a RustBinaryId
106 where
107 Self: 'a;
108
109 fn key(&self) -> Self::Key<'_> {
110 &self.meta.binary_id
111 }
112
113 id_upcast!();
114}
115
116#[derive(Copy, Clone, Debug)]
119pub enum EmitNextestObject {
120 Yes,
122 No,
124}
125
126const KIND_TEST: &str = "test";
127const KIND_SUITE: &str = "suite";
128
129const EVENT_STARTED: &str = "started";
130const EVENT_IGNORED: &str = "ignored";
131const EVENT_OK: &str = "ok";
132const EVENT_FAILED: &str = "failed";
133
134#[inline]
135fn fmt_err(err: std::fmt::Error) -> WriteEventError {
136 WriteEventError::Io(std::io::Error::new(std::io::ErrorKind::OutOfMemory, err))
137}
138
139pub struct LibtestReporter<'cfg> {
142 _minor: FormatMinorVersion,
143 _major: FormatMajorVersion,
144 test_suites: IdOrdMap<LibtestSuite<'cfg>>,
145 emit_nextest_obj: bool,
148}
149
150impl<'cfg> LibtestReporter<'cfg> {
151 pub fn new(
162 version: Option<&str>,
163 emit_nextest_obj: EmitNextestObject,
164 ) -> Result<Self, FormatVersionError> {
165 let emit_nextest_obj = matches!(emit_nextest_obj, EmitNextestObject::Yes);
166
167 let Some(version) = version else {
168 return Ok(Self {
169 _minor: FormatMinorVersion::First,
170 _major: FormatMajorVersion::Unstable,
171 test_suites: IdOrdMap::new(),
172 emit_nextest_obj,
173 });
174 };
175 let Some((major, minor)) = version.split_once('.') else {
176 return Err(FormatVersionError {
177 input: version.into(),
178 error: FormatVersionErrorInner::InvalidFormat {
179 expected: "<major>.<minor>",
180 },
181 });
182 };
183
184 let major: u8 = major.parse().map_err(|err| FormatVersionError {
185 input: version.into(),
186 error: FormatVersionErrorInner::InvalidInteger {
187 which: "major",
188 err,
189 },
190 })?;
191
192 let minor: u8 = minor.parse().map_err(|err| FormatVersionError {
193 input: version.into(),
194 error: FormatVersionErrorInner::InvalidInteger {
195 which: "minor",
196 err,
197 },
198 })?;
199
200 let major = match major {
201 0 => FormatMajorVersion::Unstable,
202 o => {
203 return Err(FormatVersionError {
204 input: version.into(),
205 error: FormatVersionErrorInner::InvalidValue {
206 which: "major",
207 value: o,
208 range: (FormatMajorVersion::Unstable as u8)
209 ..(FormatMajorVersion::_Max as u8),
210 },
211 });
212 }
213 };
214
215 let minor = match minor {
216 1 => FormatMinorVersion::First,
217 o => {
218 return Err(FormatVersionError {
219 input: version.into(),
220 error: FormatVersionErrorInner::InvalidValue {
221 which: "minor",
222 value: o,
223 range: (FormatMinorVersion::First as u8)..(FormatMinorVersion::_Max as u8),
224 },
225 });
226 }
227 };
228
229 Ok(Self {
230 _major: major,
231 _minor: minor,
232 test_suites: IdOrdMap::new(),
233 emit_nextest_obj,
234 })
235 }
236
237 pub(crate) fn write_event(&mut self, event: &TestEvent<'cfg>) -> Result<(), WriteEventError> {
238 let mut retries = None;
239
240 let (kind, eve, stress_index, test_instance) = match &event.kind {
242 TestEventKind::TestStarted {
243 stress_index,
244 test_instance,
245 ..
246 } => (KIND_TEST, EVENT_STARTED, stress_index, test_instance),
247 TestEventKind::TestSkipped {
248 stress_index,
249 test_instance,
250 reason: MismatchReason::Ignored,
251 } => {
252 (KIND_TEST, EVENT_STARTED, stress_index, test_instance)
256 }
257 TestEventKind::TestFinished {
258 stress_index,
259 test_instance,
260 run_statuses,
261 ..
262 } => {
263 if run_statuses.len() > 1 {
264 retries = Some(run_statuses.len());
265 }
266
267 (
268 KIND_TEST,
269 match run_statuses.last_status().result {
270 ExecutionResult::Pass
271 | ExecutionResult::Leak {
272 result: LeakTimeoutResult::Pass,
273 } => EVENT_OK,
274 ExecutionResult::Leak {
275 result: LeakTimeoutResult::Fail,
276 }
277 | ExecutionResult::Fail { .. }
278 | ExecutionResult::ExecFail
279 | ExecutionResult::Timeout => EVENT_FAILED,
280 },
281 stress_index,
282 test_instance,
283 )
284 }
285 TestEventKind::StressSubRunFinished { .. } | TestEventKind::RunFinished { .. } => {
286 for test_suite in std::mem::take(&mut self.test_suites) {
287 self.finalize(test_suite)?;
288 }
289
290 return Ok(());
291 }
292 _ => return Ok(()),
293 };
294
295 let suite_info = test_instance.suite_info;
296 let crate_name = suite_info.package.name();
297 let binary_name = &suite_info.binary_name;
298
299 let mut test_suite = match self.test_suites.entry(&suite_info.binary_id) {
301 id_ord_map::Entry::Vacant(e) => {
302 let (running, ignored, filtered) =
303 suite_info.status.test_cases().fold((0, 0, 0), |acc, tc| {
304 if tc.1.ignored {
305 (acc.0, acc.1 + 1, acc.2)
306 } else if tc.1.filter_match.is_match() {
307 (acc.0 + 1, acc.1, acc.2)
308 } else {
309 (acc.0, acc.1, acc.2 + 1)
310 }
311 });
312
313 let mut out = bytes::BytesMut::with_capacity(1024);
314 write!(
315 &mut out,
316 r#"{{"type":"{KIND_SUITE}","event":"{EVENT_STARTED}","test_count":{}"#,
317 running + ignored,
318 )
319 .map_err(fmt_err)?;
320
321 if self.emit_nextest_obj {
322 write!(
323 out,
324 r#","nextest":{{"crate":"{crate_name}","test_binary":"{binary_name}","kind":"{}""#,
325 suite_info.kind,
326 )
327 .map_err(fmt_err)?;
328
329 if let Some(stress_index) = stress_index {
330 write!(out, r#","stress_index":{}"#, stress_index.current)
331 .map_err(fmt_err)?;
332 if let Some(total) = stress_index.total {
333 write!(out, r#","stress_total":{total}"#).map_err(fmt_err)?;
334 }
335 }
336
337 write!(out, "}}").map_err(fmt_err)?;
338 }
339
340 out.extend_from_slice(b"}\n");
341
342 e.insert(LibtestSuite {
343 running,
344 failed: 0,
345 succeeded: 0,
346 ignored,
347 filtered,
348 stress_index: *stress_index,
349 meta: test_instance.suite_info,
350 total: std::time::Duration::new(0, 0),
351 ignore_block: None,
352 output_block: out,
353 })
354 }
355 id_ord_map::Entry::Occupied(e) => e.into_mut(),
356 };
357
358 let test_suite_mut = &mut *test_suite;
359 let out = &mut test_suite_mut.output_block;
360
361 if matches!(event.kind, TestEventKind::TestFinished { .. }) {
364 if let Some(ib) = test_suite_mut.ignore_block.take() {
365 out.extend_from_slice(&ib);
366 }
367 }
368
369 write!(
378 out,
379 r#"{{"type":"{kind}","event":"{eve}","name":"{}::{}"#,
380 suite_info.package.name(),
381 suite_info.binary_name,
382 )
383 .map_err(fmt_err)?;
384
385 if let Some(stress_index) = stress_index {
386 write!(out, "@stress-{}", stress_index.current).map_err(fmt_err)?;
387 }
388 write!(out, "${}", test_instance.name).map_err(fmt_err)?;
389 if let Some(retry_count) = retries {
390 write!(out, "#{retry_count}\"").map_err(fmt_err)?;
391 } else {
392 out.extend_from_slice(b"\"");
393 }
394
395 match &event.kind {
396 TestEventKind::TestFinished { run_statuses, .. } => {
397 let last_status = run_statuses.last_status();
398
399 test_suite_mut.total += last_status.time_taken;
400 test_suite_mut.running -= 1;
401
402 write!(
407 out,
408 r#","exec_time":{}"#,
409 last_status.time_taken.as_secs_f64()
410 )
411 .map_err(fmt_err)?;
412
413 match last_status.result {
414 ExecutionResult::Fail { .. } | ExecutionResult::ExecFail => {
415 test_suite_mut.failed += 1;
416
417 write!(out, r#","stdout":""#).map_err(fmt_err)?;
420
421 strip_human_output_from_failed_test(
422 &last_status.output,
423 out,
424 test_instance.name,
425 )?;
426 out.extend_from_slice(b"\"");
427 }
428 ExecutionResult::Timeout => {
429 test_suite_mut.failed += 1;
430 out.extend_from_slice(br#","reason":"time limit exceeded""#);
431 }
432 _ => {
433 test_suite_mut.succeeded += 1;
434 }
435 }
436 }
437 TestEventKind::TestSkipped { .. } => {
438 test_suite_mut.running -= 1;
439
440 if test_suite_mut.ignore_block.is_none() {
441 test_suite_mut.ignore_block = Some(bytes::BytesMut::with_capacity(1024));
442 }
443
444 let ib = test_suite_mut
445 .ignore_block
446 .get_or_insert_with(|| bytes::BytesMut::with_capacity(1024));
447
448 writeln!(
449 ib,
450 r#"{{"type":"{kind}","event":"{EVENT_IGNORED}","name":"{}::{}${}"}}"#,
451 suite_info.package.name(),
452 suite_info.binary_name,
453 test_instance.name,
454 )
455 .map_err(fmt_err)?;
456 }
457 _ => {}
458 };
459
460 out.extend_from_slice(b"}\n");
461
462 if self.emit_nextest_obj {
463 {
464 use std::io::Write as _;
465
466 let mut stdout = std::io::stdout().lock();
467 stdout.write_all(out).map_err(WriteEventError::Io)?;
468 stdout.flush().map_err(WriteEventError::Io)?;
469 out.clear();
470 }
471
472 if test_suite_mut.running == 0 {
473 std::mem::drop(test_suite);
474
475 if let Some(test_suite) = self.test_suites.remove(&suite_info.binary_id) {
476 self.finalize(test_suite)?;
477 }
478 }
479 } else {
480 if test_suite_mut.running > 0 {
483 return Ok(());
484 }
485
486 std::mem::drop(test_suite);
487
488 if let Some(test_suite) = self.test_suites.remove(&suite_info.binary_id) {
489 self.finalize(test_suite)?;
490 }
491 }
492
493 Ok(())
494 }
495
496 fn finalize(&self, mut test_suite: LibtestSuite) -> Result<(), WriteEventError> {
497 let event = if test_suite.failed > 0 {
498 EVENT_FAILED
499 } else {
500 EVENT_OK
501 };
502
503 let out = &mut test_suite.output_block;
504 let suite_info = test_suite.meta;
505
506 if test_suite.running > 0 {
510 test_suite.filtered += test_suite.running;
511 }
512
513 write!(
514 out,
515 r#"{{"type":"{KIND_SUITE}","event":"{event}","passed":{},"failed":{},"ignored":{},"measured":0,"filtered_out":{},"exec_time":{}"#,
516 test_suite.succeeded,
517 test_suite.failed,
518 test_suite.ignored,
519 test_suite.filtered,
520 test_suite.total.as_secs_f64(),
521 )
522 .map_err(fmt_err)?;
523
524 if self.emit_nextest_obj {
525 let crate_name = suite_info.package.name();
526 let binary_name = &suite_info.binary_name;
527 write!(
528 out,
529 r#","nextest":{{"crate":"{crate_name}","test_binary":"{binary_name}","kind":"{}""#,
530 suite_info.kind,
531 )
532 .map_err(fmt_err)?;
533
534 if let Some(stress_index) = test_suite.stress_index {
535 write!(out, r#","stress_index":{}"#, stress_index.current).map_err(fmt_err)?;
536 if let Some(total) = stress_index.total {
537 write!(out, r#","stress_total":{total}"#).map_err(fmt_err)?;
538 }
539 }
540
541 write!(out, "}}").map_err(fmt_err)?;
542 }
543
544 out.extend_from_slice(b"}\n");
545
546 {
547 use std::io::Write as _;
548
549 let mut stdout = std::io::stdout().lock();
550 stdout.write_all(out).map_err(WriteEventError::Io)?;
551 stdout.flush().map_err(WriteEventError::Io)?;
552 }
553
554 Ok(())
555 }
556}
557
558fn strip_human_output_from_failed_test(
565 output: &ChildExecutionOutput,
566 out: &mut bytes::BytesMut,
567 test_name: &str,
568) -> Result<(), WriteEventError> {
569 match output {
570 ChildExecutionOutput::Output {
571 result: _,
572 output,
573 errors,
574 } => {
575 match output {
576 ChildOutput::Combined { output } => {
577 strip_human_stdout_or_combined(output, out, test_name)?;
578 }
579 ChildOutput::Split(split) => {
580 #[cfg(not(test))]
584 {
585 debug_assert!(false, "libtest output requires CaptureStrategy::Combined");
586 }
587 if let Some(stdout) = &split.stdout {
588 if !stdout.is_empty() {
589 write!(out, "--- STDOUT ---\\n").map_err(fmt_err)?;
590 strip_human_stdout_or_combined(stdout, out, test_name)?;
591 }
592 } else {
593 write!(out, "(stdout not captured)").map_err(fmt_err)?;
594 }
595 if let Some(stderr) = &split.stderr {
597 if !stderr.is_empty() {
598 write!(out, "\\n--- STDERR ---\\n").map_err(fmt_err)?;
599 write!(out, "{}", EscapedString(stderr.as_str_lossy()))
600 .map_err(fmt_err)?;
601 }
602 } else {
603 writeln!(out, "\\n(stderr not captured)").map_err(fmt_err)?;
604 }
605 }
606 }
607
608 if let Some(errors) = errors {
609 write!(out, "\\n--- EXECUTION ERRORS ---\\n").map_err(fmt_err)?;
610 write!(
611 out,
612 "{}",
613 EscapedString(&DisplayErrorChain::new(errors).to_string())
614 )
615 .map_err(fmt_err)?;
616 }
617 }
618 ChildExecutionOutput::StartError(error) => {
619 write!(out, "--- EXECUTION ERROR ---\\n").map_err(fmt_err)?;
620 write!(
621 out,
622 "{}",
623 EscapedString(&DisplayErrorChain::new(error).to_string())
624 )
625 .map_err(fmt_err)?;
626 }
627 }
628 Ok(())
629}
630
631fn strip_human_stdout_or_combined(
632 output: &ChildSingleOutput,
633 out: &mut bytes::BytesMut,
634 test_name: &str,
635) -> Result<(), WriteEventError> {
636 if output.buf.contains_str("running 1 test\n") {
637 let lines = output
639 .lines()
640 .skip_while(|line| line != b"running 1 test")
641 .skip(1)
642 .take_while(|line| {
643 if let Some(name) = line
644 .strip_prefix(b"test ")
645 .and_then(|np| np.strip_suffix(b" ... FAILED"))
646 {
647 if test_name.as_bytes() == name {
648 return false;
649 }
650 }
651
652 true
653 })
654 .map(|line| line.to_str_lossy());
655
656 for line in lines {
657 write!(out, "{}\\n", EscapedString(&line)).map_err(fmt_err)?;
659 }
660 } else {
661 write!(out, "{}", EscapedString(output.as_str_lossy())).map_err(fmt_err)?;
664 }
665
666 Ok(())
667}
668
669struct EscapedString<'s>(&'s str);
673
674impl std::fmt::Display for EscapedString<'_> {
675 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> ::std::fmt::Result {
676 let mut start = 0;
677 let s = self.0;
678
679 for (i, byte) in s.bytes().enumerate() {
680 let escaped = match byte {
681 b'"' => "\\\"",
682 b'\\' => "\\\\",
683 b'\x00' => "\\u0000",
684 b'\x01' => "\\u0001",
685 b'\x02' => "\\u0002",
686 b'\x03' => "\\u0003",
687 b'\x04' => "\\u0004",
688 b'\x05' => "\\u0005",
689 b'\x06' => "\\u0006",
690 b'\x07' => "\\u0007",
691 b'\x08' => "\\b",
692 b'\t' => "\\t",
693 b'\n' => "\\n",
694 b'\x0b' => "\\u000b",
695 b'\x0c' => "\\f",
696 b'\r' => "\\r",
697 b'\x0e' => "\\u000e",
698 b'\x0f' => "\\u000f",
699 b'\x10' => "\\u0010",
700 b'\x11' => "\\u0011",
701 b'\x12' => "\\u0012",
702 b'\x13' => "\\u0013",
703 b'\x14' => "\\u0014",
704 b'\x15' => "\\u0015",
705 b'\x16' => "\\u0016",
706 b'\x17' => "\\u0017",
707 b'\x18' => "\\u0018",
708 b'\x19' => "\\u0019",
709 b'\x1a' => "\\u001a",
710 b'\x1b' => "\\u001b",
711 b'\x1c' => "\\u001c",
712 b'\x1d' => "\\u001d",
713 b'\x1e' => "\\u001e",
714 b'\x1f' => "\\u001f",
715 b'\x7f' => "\\u007f",
716 _ => {
717 continue;
718 }
719 };
720
721 if start < i {
722 f.write_str(&s[start..i])?;
723 }
724
725 f.write_str(escaped)?;
726
727 start = i + 1;
728 }
729
730 if start != self.0.len() {
731 f.write_str(&s[start..])?;
732 }
733
734 Ok(())
735 }
736}
737
738#[cfg(test)]
739mod test {
740 use crate::{
741 errors::ChildStartError,
742 reporter::structured::libtest::strip_human_output_from_failed_test,
743 test_output::{ChildExecutionOutput, ChildOutput, ChildSplitOutput},
744 };
745 use bytes::BytesMut;
746 use color_eyre::eyre::eyre;
747 use std::{io, sync::Arc};
748
749 #[test]
753 fn strips_human_output() {
754 const TEST_OUTPUT: &[&str] = &[
755 "\n",
756 "running 1 test\n",
757 "[src/index.rs:185] \"boop\" = \"boop\"\n",
758 "this is stdout\n",
759 "this i stderr\nok?\n",
760 "thread 'index::test::download_url_crates_io'",
761 r" panicked at src/index.rs:206:9:
762oh no
763stack backtrace:
764 0: rust_begin_unwind
765 at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1/library/std/src/panicking.rs:597:5
766 1: core::panicking::panic_fmt
767 at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1/library/core/src/panicking.rs:72:14
768 2: tame_index::index::test::download_url_crates_io
769 at ./src/index.rs:206:9
770 3: tame_index::index::test::download_url_crates_io::{{closure}}
771 at ./src/index.rs:179:33
772 4: core::ops::function::FnOnce::call_once
773 at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1/library/core/src/ops/function.rs:250:5
774 5: core::ops::function::FnOnce::call_once
775 at /rustc/a28077b28a02b92985b3a3faecf92813155f1ea1/library/core/src/ops/function.rs:250:5
776note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.
777",
778 "test index::test::download_url_crates_io ... FAILED\n",
779 "\n\nfailures:\n\nfailures:\n index::test::download_url_crates_io\n\ntest result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 13 filtered out; finished in 0.01s\n",
780 ];
781
782 let output = {
783 let mut acc = BytesMut::new();
784 for line in TEST_OUTPUT {
785 acc.extend_from_slice(line.as_bytes());
786 }
787
788 ChildOutput::Combined {
789 output: acc.freeze().into(),
790 }
791 };
792
793 let mut actual = bytes::BytesMut::new();
794 strip_human_output_from_failed_test(
795 &ChildExecutionOutput::Output {
796 result: None,
797 output,
798 errors: None,
799 },
800 &mut actual,
801 "index::test::download_url_crates_io",
802 )
803 .unwrap();
804
805 insta::assert_snapshot!(std::str::from_utf8(&actual).unwrap());
806 }
807
808 #[test]
809 fn strips_human_output_custom_test_harness() {
810 const TEST_OUTPUT: &[&str] = &["\n", "this is a custom test harness!!!\n", "1 test passed"];
812
813 let output = {
814 let mut acc = BytesMut::new();
815 for line in TEST_OUTPUT {
816 acc.extend_from_slice(line.as_bytes());
817 }
818
819 ChildOutput::Combined {
820 output: acc.freeze().into(),
821 }
822 };
823
824 let mut actual = bytes::BytesMut::new();
825 strip_human_output_from_failed_test(
826 &ChildExecutionOutput::Output {
827 result: None,
828 output,
829 errors: None,
830 },
831 &mut actual,
832 "non-existent",
833 )
834 .unwrap();
835
836 insta::assert_snapshot!(std::str::from_utf8(&actual).unwrap());
837 }
838
839 #[test]
840 fn strips_human_output_start_error() {
841 let inner_error = eyre!("inner error");
842 let error = io::Error::other(inner_error);
843
844 let output = ChildExecutionOutput::StartError(ChildStartError::Spawn(Arc::new(error)));
845
846 let mut actual = bytes::BytesMut::new();
847 strip_human_output_from_failed_test(&output, &mut actual, "non-existent").unwrap();
848
849 insta::assert_snapshot!(std::str::from_utf8(&actual).unwrap());
850 }
851
852 #[test]
853 fn strips_human_output_none() {
854 let mut actual = bytes::BytesMut::new();
855 strip_human_output_from_failed_test(
856 &ChildExecutionOutput::Output {
857 result: None,
858 output: ChildOutput::Split(ChildSplitOutput {
859 stdout: None,
860 stderr: None,
861 }),
862 errors: None,
863 },
864 &mut actual,
865 "non-existent",
866 )
867 .unwrap();
868
869 insta::assert_snapshot!(std::str::from_utf8(&actual).unwrap());
870 }
871}