1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
// Copyright (c) The nextest Contributors
// SPDX-License-Identifier: MIT OR Apache-2.0

//! Support for double-spawning test processes.
//!
//! Nextest has experimental support on Unix for spawning test processes twice, to enable better
//! isolation and solve some thorny issues.
//!
//! ## Issues this currently solves
//!
//! ### `posix_spawn` SIGTSTP race
//!
//! It's been empirically observed that if nextest receives a `SIGTSTP` (Ctrl-Z) while it's running,
//! it can get completely stuck sometimes. This is due to a race between the child being spawned and it
//! receiving a `SIGTSTP` signal.
//!
//! For more details, see [this
//! message](https://sourceware.org/pipermail/libc-help/2022-August/006263.html) on the glibc-help
//! mailing list.
//!
//! To solve this issue, we do the following:
//!
//! 1. In the main nextest runner process, using `DoubleSpawnContext`, block `SIGTSTP` in the
//!    current thread (using `pthread_sigmask`) before spawning the stub child cargo-nextest
//!    process.
//! 2. In the stub child process, unblock `SIGTSTP`.
//!
//! With this approach, the race condition between posix_spawn and `SIGTSTP` no longer exists.

use std::path::Path;

/// Information about double-spawning processes. This determines whether a process will be
/// double-spawned.
///
/// This is used by the main nextest process.
#[derive(Clone, Debug)]
pub struct DoubleSpawnInfo {
    inner: imp::DoubleSpawnInfo,
}

impl DoubleSpawnInfo {
    /// The name of the double-spawn subcommand, used throughout nextest.
    pub const SUBCOMMAND_NAME: &'static str = "__double-spawn";

    /// Attempts to enable double-spawning and returns a new `DoubleSpawnInfo`.
    ///
    /// If double-spawning is not available, [`current_exe`](Self::current_exe) returns `None`.
    pub fn try_enable() -> Self {
        Self {
            inner: imp::DoubleSpawnInfo::try_enable(),
        }
    }

    /// Returns a `DoubleSpawnInfo` which doesn't perform any double-spawning.
    pub fn disabled() -> Self {
        Self {
            inner: imp::DoubleSpawnInfo::disabled(),
        }
    }

    /// Returns the current executable, if one is available.
    ///
    /// If `None`, double-spawning is not used.
    pub fn current_exe(&self) -> Option<&Path> {
        self.inner.current_exe()
    }

    /// Returns a context that is meant to be obtained before spawning processes and dropped afterwards.
    pub fn spawn_context(&self) -> Option<DoubleSpawnContext> {
        self.current_exe().map(|_| DoubleSpawnContext::new())
    }
}

/// Context to be used before spawning processes and dropped afterwards.
///
/// Returned by [`DoubleSpawnInfo::spawn_context`].
#[derive(Debug)]
pub struct DoubleSpawnContext {
    // Only used for the Drop impl.
    #[allow(dead_code)]
    inner: imp::DoubleSpawnContext,
}

impl DoubleSpawnContext {
    #[inline]
    fn new() -> Self {
        Self {
            inner: imp::DoubleSpawnContext::new(),
        }
    }

    /// Close the double-spawn context, dropping any changes that needed to be done to it.
    pub fn finish(self) {}
}

/// Initialization for the double-spawn child.
pub fn double_spawn_child_init() {
    imp::double_spawn_child_init()
}

#[cfg(unix)]
mod imp {
    use super::*;
    use nix::sys::signal::{SigSet, Signal};
    use std::path::PathBuf;
    use tracing::warn;

    #[derive(Clone, Debug)]
    pub(super) struct DoubleSpawnInfo {
        current_exe: Option<PathBuf>,
    }

    impl DoubleSpawnInfo {
        #[inline]
        pub(super) fn try_enable() -> Self {
            // Attempt to obtain the current exe, and warn if it couldn't be found.
            // TODO: maybe add an option to fail?
            // TODO: Always use /proc/self/exe directly on Linux, just make sure it's always accessible
            let current_exe = get_current_exe().map_or_else(
                |error| {
                    warn!(
                        "unable to determine current exe, will not use double-spawning \
                        for better isolation: {error}"
                    );
                    None
                },
                Some,
            );
            Self { current_exe }
        }

        #[inline]
        pub(super) fn disabled() -> Self {
            Self { current_exe: None }
        }

        #[inline]
        pub(super) fn current_exe(&self) -> Option<&Path> {
            self.current_exe.as_deref()
        }
    }

    #[cfg(target_os = "linux")]
    fn get_current_exe() -> std::io::Result<PathBuf> {
        static PROC_SELF_EXE: &str = "/proc/self/exe";

        // Always use /proc/self/exe directly rather than trying to readlink it. Just make sure it's
        // accessible.
        let path = Path::new(PROC_SELF_EXE);
        match path.symlink_metadata() {
            Ok(_) => Ok(path.to_owned()),
            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Err(std::io::Error::new(
                std::io::ErrorKind::Other,
                "no /proc/self/exe available. Is /proc mounted?",
            )),
            Err(e) => Err(e),
        }
    }

    // TODO: Add other symlinks as well, e.g. /proc/self/path/a.out on solaris/illumos.

    #[cfg(not(target_os = "linux"))]
    #[inline]
    fn get_current_exe() -> std::io::Result<PathBuf> {
        std::env::current_exe()
    }

    #[derive(Debug)]
    pub(super) struct DoubleSpawnContext {
        to_unblock: Option<SigSet>,
    }

    impl DoubleSpawnContext {
        #[inline]
        pub(super) fn new() -> Self {
            // Block SIGTSTP, unblocking it in the child process. This avoids a complex race
            // condition.
            let mut sigset = SigSet::empty();
            sigset.add(Signal::SIGTSTP);
            let to_unblock = sigset.thread_block().ok().map(|()| sigset);
            Self { to_unblock }
        }
    }

    impl Drop for DoubleSpawnContext {
        fn drop(&mut self) {
            if let Some(sigset) = &self.to_unblock {
                _ = sigset.thread_unblock();
            }
        }
    }

    #[inline]
    pub(super) fn double_spawn_child_init() {
        let mut sigset = SigSet::empty();
        sigset.add(Signal::SIGTSTP);
        if sigset.thread_unblock().is_err() {
            warn!("[double-spawn] unable to unblock SIGTSTP in child");
        }
    }
}

#[cfg(not(unix))]
mod imp {
    use super::*;

    #[derive(Clone, Debug)]
    pub(super) struct DoubleSpawnInfo {}

    impl DoubleSpawnInfo {
        #[inline]
        pub(super) fn try_enable() -> Self {
            Self {}
        }

        #[inline]
        pub(super) fn disabled() -> Self {
            Self {}
        }

        #[inline]
        pub(super) fn current_exe(&self) -> Option<&Path> {
            None
        }
    }

    #[derive(Debug)]
    pub(super) struct DoubleSpawnContext {}

    impl DoubleSpawnContext {
        #[inline]
        pub(super) fn new() -> Self {
            Self {}
        }
    }

    #[inline]
    pub(super) fn double_spawn_child_init() {}
}