nextest_runner/reuse_build/
mod.rs

1// Copyright (c) The nextest Contributors
2// SPDX-License-Identifier: MIT OR Apache-2.0
3
4//! Reuse builds performed earlier.
5//!
6//! Nextest allows users to reuse builds done on one machine. This module contains support for that.
7//!
8//! The main data structures here are [`ReuseBuildInfo`] and [`PathMapper`].
9
10use crate::{
11    errors::{
12        ArchiveExtractError, ArchiveReadError, MetadataMaterializeError, PathMapperConstructError,
13        PathMapperConstructKind,
14    },
15    list::BinaryList,
16    platform::PlatformLibdir,
17};
18use camino::{Utf8Path, Utf8PathBuf};
19use camino_tempfile::Utf8TempDir;
20use guppy::graph::PackageGraph;
21use nextest_metadata::{BinaryListSummary, PlatformLibdirUnavailable};
22use std::{fmt, fs, io, sync::Arc};
23
24mod archive_reporter;
25mod archiver;
26mod unarchiver;
27
28pub use archive_reporter::*;
29pub use archiver::*;
30pub use unarchiver::*;
31
32/// The name of the file in which Cargo metadata is stored.
33pub const CARGO_METADATA_FILE_NAME: &str = "target/nextest/cargo-metadata.json";
34
35/// The name of the file in which binaries metadata is stored.
36pub const BINARIES_METADATA_FILE_NAME: &str = "target/nextest/binaries-metadata.json";
37
38/// The name of the directory in which libdirs are stored.
39pub const LIBDIRS_BASE_DIR: &str = "target/nextest/libdirs";
40
41/// Reuse build information.
42#[derive(Debug, Default)]
43pub struct ReuseBuildInfo {
44    /// Cargo metadata and remapping for the target directory.
45    pub cargo_metadata: Option<MetadataWithRemap<ReusedCargoMetadata>>,
46
47    /// Binaries metadata JSON and remapping for the target directory.
48    pub binaries_metadata: Option<MetadataWithRemap<ReusedBinaryList>>,
49
50    /// A remapper for libdirs.
51    pub libdir_mapper: LibdirMapper,
52
53    /// Optional temporary directory used for cleanup.
54    _temp_dir: Option<Utf8TempDir>,
55}
56
57impl ReuseBuildInfo {
58    /// Creates a new [`ReuseBuildInfo`] from the given cargo and binaries metadata information.
59    pub fn new(
60        cargo_metadata: Option<MetadataWithRemap<ReusedCargoMetadata>>,
61        binaries_metadata: Option<MetadataWithRemap<ReusedBinaryList>>,
62        // TODO: accept libdir_mapper as an argument, as well
63    ) -> Self {
64        Self {
65            cargo_metadata,
66            binaries_metadata,
67            libdir_mapper: LibdirMapper::default(),
68            _temp_dir: None,
69        }
70    }
71
72    /// Extracts an archive and constructs a [`ReuseBuildInfo`] from it.
73    pub fn extract_archive<F>(
74        archive_file: &Utf8Path,
75        format: ArchiveFormat,
76        dest: ExtractDestination,
77        callback: F,
78        workspace_remap: Option<&Utf8Path>,
79    ) -> Result<Self, ArchiveExtractError>
80    where
81        F: for<'e> FnMut(ArchiveEvent<'e>) -> io::Result<()>,
82    {
83        let mut file = fs::File::open(archive_file)
84            .map_err(|err| ArchiveExtractError::Read(ArchiveReadError::Io(err)))?;
85
86        let mut unarchiver = Unarchiver::new(&mut file, format);
87        let ExtractInfo {
88            dest_dir,
89            temp_dir,
90            binary_list,
91            cargo_metadata_json,
92            graph,
93            libdir_mapper,
94        } = unarchiver.extract(dest, callback)?;
95
96        let cargo_metadata = MetadataWithRemap {
97            metadata: ReusedCargoMetadata::new((cargo_metadata_json, graph)),
98            remap: workspace_remap.map(|p| p.to_owned()),
99        };
100        let binaries_metadata = MetadataWithRemap {
101            metadata: ReusedBinaryList::new(binary_list),
102            remap: Some(dest_dir.join("target")),
103        };
104
105        Ok(Self {
106            cargo_metadata: Some(cargo_metadata),
107            binaries_metadata: Some(binaries_metadata),
108            libdir_mapper,
109            _temp_dir: temp_dir,
110        })
111    }
112
113    /// Returns the Cargo metadata.
114    pub fn cargo_metadata(&self) -> Option<&ReusedCargoMetadata> {
115        self.cargo_metadata.as_ref().map(|m| &m.metadata)
116    }
117
118    /// Returns the binaries metadata, reading it from disk if necessary.
119    pub fn binaries_metadata(&self) -> Option<&ReusedBinaryList> {
120        self.binaries_metadata.as_ref().map(|m| &m.metadata)
121    }
122
123    /// Returns true if any component of the build is being reused.
124    #[inline]
125    pub fn is_active(&self) -> bool {
126        self.cargo_metadata.is_some() || self.binaries_metadata.is_some()
127    }
128
129    /// Returns the new workspace directory.
130    pub fn workspace_remap(&self) -> Option<&Utf8Path> {
131        self.cargo_metadata
132            .as_ref()
133            .and_then(|m| m.remap.as_deref())
134    }
135
136    /// Returns the new target directory.
137    pub fn target_dir_remap(&self) -> Option<&Utf8Path> {
138        self.binaries_metadata
139            .as_ref()
140            .and_then(|m| m.remap.as_deref())
141    }
142}
143
144/// Metadata as either deserialized contents or a path, along with a possible directory remap.
145#[derive(Clone, Debug)]
146pub struct MetadataWithRemap<T> {
147    /// The metadata.
148    pub metadata: T,
149
150    /// The remapped directory.
151    pub remap: Option<Utf8PathBuf>,
152}
153
154/// Type parameter for [`MetadataWithRemap`].
155pub trait MetadataKind: Clone + fmt::Debug {
156    /// The type of metadata stored.
157    type MetadataType: Sized;
158
159    /// Constructs a new [`MetadataKind`] from the given metadata.
160    fn new(metadata: Self::MetadataType) -> Self;
161
162    /// Reads a path, resolving it into this data type.
163    fn materialize(path: &Utf8Path) -> Result<Self, MetadataMaterializeError>;
164}
165
166/// [`MetadataKind`] for a [`BinaryList`].
167#[derive(Clone, Debug)]
168pub struct ReusedBinaryList {
169    /// The binary list.
170    pub binary_list: Arc<BinaryList>,
171}
172
173impl MetadataKind for ReusedBinaryList {
174    type MetadataType = BinaryList;
175
176    fn new(binary_list: Self::MetadataType) -> Self {
177        Self {
178            binary_list: Arc::new(binary_list),
179        }
180    }
181
182    fn materialize(path: &Utf8Path) -> Result<Self, MetadataMaterializeError> {
183        // Three steps: read the contents, turn it into a summary, and then turn it into a
184        // BinaryList.
185        //
186        // Buffering the contents in memory is generally much faster than trying to read it
187        // using a BufReader.
188        let contents =
189            fs::read_to_string(path).map_err(|error| MetadataMaterializeError::Read {
190                path: path.to_owned(),
191                error,
192            })?;
193
194        let summary: BinaryListSummary = serde_json::from_str(&contents).map_err(|error| {
195            MetadataMaterializeError::Deserialize {
196                path: path.to_owned(),
197                error,
198            }
199        })?;
200
201        let binary_list = BinaryList::from_summary(summary).map_err(|error| {
202            MetadataMaterializeError::RustBuildMeta {
203                path: path.to_owned(),
204                error,
205            }
206        })?;
207
208        Ok(Self::new(binary_list))
209    }
210}
211
212/// [`MetadataKind`] for Cargo metadata.
213#[derive(Clone, Debug)]
214pub struct ReusedCargoMetadata {
215    /// Cargo metadata JSON.
216    pub json: Arc<String>,
217
218    /// The package graph.
219    pub graph: Arc<PackageGraph>,
220}
221
222impl MetadataKind for ReusedCargoMetadata {
223    type MetadataType = (String, PackageGraph);
224
225    fn new((json, graph): Self::MetadataType) -> Self {
226        Self {
227            json: Arc::new(json),
228            graph: Arc::new(graph),
229        }
230    }
231
232    fn materialize(path: &Utf8Path) -> Result<Self, MetadataMaterializeError> {
233        // Read the contents into memory, then parse them as a `PackageGraph`.
234        let json =
235            std::fs::read_to_string(path).map_err(|error| MetadataMaterializeError::Read {
236                path: path.to_owned(),
237                error,
238            })?;
239        let graph = PackageGraph::from_json(&json).map_err(|error| {
240            MetadataMaterializeError::PackageGraphConstruct {
241                path: path.to_owned(),
242                error,
243            }
244        })?;
245
246        Ok(Self::new((json, graph)))
247    }
248}
249
250/// A helper for path remapping.
251///
252/// This is useful when running tests in a different directory, or a different computer, from
253/// building them.
254#[derive(Clone, Debug, Default)]
255pub struct PathMapper {
256    workspace: Option<(Utf8PathBuf, Utf8PathBuf)>,
257    target_dir: Option<(Utf8PathBuf, Utf8PathBuf)>,
258    libdir_mapper: LibdirMapper,
259}
260
261impl PathMapper {
262    /// Constructs the path mapper.
263    pub fn new(
264        orig_workspace_root: impl Into<Utf8PathBuf>,
265        workspace_remap: Option<&Utf8Path>,
266        orig_target_dir: impl Into<Utf8PathBuf>,
267        target_dir_remap: Option<&Utf8Path>,
268        libdir_mapper: LibdirMapper,
269    ) -> Result<Self, PathMapperConstructError> {
270        let workspace_root = workspace_remap
271            .map(|root| Self::canonicalize_dir(root, PathMapperConstructKind::WorkspaceRoot))
272            .transpose()?;
273        let target_dir = target_dir_remap
274            .map(|dir| Self::canonicalize_dir(dir, PathMapperConstructKind::TargetDir))
275            .transpose()?;
276
277        Ok(Self {
278            workspace: workspace_root.map(|w| (orig_workspace_root.into(), w)),
279            target_dir: target_dir.map(|d| (orig_target_dir.into(), d)),
280            libdir_mapper,
281        })
282    }
283
284    /// Constructs a no-op path mapper.
285    pub fn noop() -> Self {
286        Self {
287            workspace: None,
288            target_dir: None,
289            libdir_mapper: LibdirMapper::default(),
290        }
291    }
292
293    /// Returns the libdir mapper.
294    pub fn libdir_mapper(&self) -> &LibdirMapper {
295        &self.libdir_mapper
296    }
297
298    fn canonicalize_dir(
299        input: &Utf8Path,
300        kind: PathMapperConstructKind,
301    ) -> Result<Utf8PathBuf, PathMapperConstructError> {
302        let canonicalized_path =
303            input
304                .canonicalize()
305                .map_err(|err| PathMapperConstructError::Canonicalization {
306                    kind,
307                    input: input.into(),
308                    err,
309                })?;
310        let canonicalized_path: Utf8PathBuf =
311            canonicalized_path
312                .try_into()
313                .map_err(|err| PathMapperConstructError::NonUtf8Path {
314                    kind,
315                    input: input.into(),
316                    err,
317                })?;
318        if !canonicalized_path.is_dir() {
319            return Err(PathMapperConstructError::NotADirectory {
320                kind,
321                input: input.into(),
322                canonicalized_path,
323            });
324        }
325
326        Ok(os_imp::strip_verbatim(canonicalized_path))
327    }
328
329    /// Returns the canonicalized workspace root, if a workspace remap was
330    /// specified.
331    pub fn new_workspace_root(&self) -> Option<&Utf8Path> {
332        self.workspace.as_ref().map(|(_, new)| new.as_path())
333    }
334
335    pub(super) fn new_target_dir(&self) -> Option<&Utf8Path> {
336        self.target_dir.as_ref().map(|(_, new)| new.as_path())
337    }
338
339    pub(crate) fn map_cwd(&self, path: Utf8PathBuf) -> Utf8PathBuf {
340        match &self.workspace {
341            Some((from, to)) => match path.strip_prefix(from) {
342                Ok(p) if !p.as_str().is_empty() => to.join(p),
343                Ok(_) => to.clone(),
344                Err(_) => path,
345            },
346            None => path,
347        }
348    }
349
350    pub(crate) fn map_binary(&self, path: Utf8PathBuf) -> Utf8PathBuf {
351        match &self.target_dir {
352            Some((from, to)) => match path.strip_prefix(from) {
353                Ok(p) if !p.as_str().is_empty() => to.join(p),
354                Ok(_) => to.clone(),
355                Err(_) => path,
356            },
357            None => path,
358        }
359    }
360}
361
362/// A mapper for lib dirs.
363///
364/// Archives store parts of lib dirs, which must be remapped to the new target directory.
365#[derive(Clone, Debug, Default)]
366pub struct LibdirMapper {
367    /// The host libdir mapper.
368    pub(crate) host: PlatformLibdirMapper,
369
370    /// The target libdir mapper.
371    pub(crate) target: PlatformLibdirMapper,
372}
373
374/// A mapper for an individual platform libdir.
375///
376/// Part of [`LibdirMapper`].
377#[derive(Clone, Debug, Default)]
378pub(crate) enum PlatformLibdirMapper {
379    Path(Utf8PathBuf),
380    Unavailable,
381    #[default]
382    NotRequested,
383}
384
385impl PlatformLibdirMapper {
386    pub(crate) fn map(&self, original: &PlatformLibdir) -> PlatformLibdir {
387        match self {
388            PlatformLibdirMapper::Path(new) => {
389                // Just use the new path. (We may have to check original in the future, but it
390                // doesn't seem necessary for now -- if a libdir has been provided to the remapper,
391                // that's that.)
392                PlatformLibdir::Available(new.clone())
393            }
394            PlatformLibdirMapper::Unavailable => {
395                // In this case, the original value is ignored -- we expected a libdir to be
396                // present, but it wasn't.
397                PlatformLibdir::Unavailable(PlatformLibdirUnavailable::NOT_IN_ARCHIVE)
398            }
399            PlatformLibdirMapper::NotRequested => original.clone(),
400        }
401    }
402}
403
404#[cfg(windows)]
405mod os_imp {
406    use camino::Utf8PathBuf;
407    use std::ptr;
408    use windows_sys::Win32::Storage::FileSystem::GetFullPathNameW;
409
410    /// Strips verbatim prefix from a path if possible.
411    pub(super) fn strip_verbatim(path: Utf8PathBuf) -> Utf8PathBuf {
412        let path_str = String::from(path);
413        if path_str.starts_with(r"\\?\UNC") {
414            // In general we don't expect UNC paths, so just return the path as is.
415            path_str.into()
416        } else if path_str.starts_with(r"\\?\") {
417            const START_LEN: usize = r"\\?\".len();
418
419            let is_absolute_exact = {
420                let mut v = path_str[START_LEN..].encode_utf16().collect::<Vec<u16>>();
421                // Ensure null termination.
422                v.push(0);
423                is_absolute_exact(&v)
424            };
425
426            if is_absolute_exact {
427                path_str[START_LEN..].into()
428            } else {
429                path_str.into()
430            }
431        } else {
432            // Not a verbatim path, so return it as is.
433            path_str.into()
434        }
435    }
436
437    /// Test that the path is absolute, fully qualified and unchanged when processed by the Windows API.Add commentMore actions
438    ///
439    /// For example:
440    ///
441    /// - `C:\path\to\file` will return true.
442    /// - `C:\path\to\nul` returns false because the Windows API will convert it to \\.\NUL
443    /// - `C:\path\to\..\file` returns false because it will be resolved to `C:\path\file`.
444    ///
445    /// This is a useful property because it means the path can be converted from and to and verbatim
446    /// path just by changing the prefix.
447    fn is_absolute_exact(path: &[u16]) -> bool {
448        // Adapted from the Rust project: https://github.com/rust-lang/rust/commit/edfc74722556c659de6fa03b23af3b9c8ceb8ac2
449
450        // This is implemented by checking that passing the path through
451        // GetFullPathNameW does not change the path in any way.
452
453        // Windows paths are limited to i16::MAX length
454        // though the API here accepts a u32 for the length.
455        if path.is_empty() || path.len() > u32::MAX as usize || path.last() != Some(&0) {
456            return false;
457        }
458        // The path returned by `GetFullPathNameW` must be the same length as the
459        // given path, otherwise they're not equal.
460        let buffer_len = path.len();
461        let mut new_path = Vec::with_capacity(buffer_len);
462        let result = unsafe {
463            GetFullPathNameW(
464                path.as_ptr(),
465                new_path.capacity() as u32,
466                new_path.as_mut_ptr(),
467                ptr::null_mut(),
468            )
469        };
470        // Note: if non-zero, the returned result is the length of the buffer without the null termination
471        if result == 0 || result as usize != buffer_len - 1 {
472            false
473        } else {
474            // SAFETY: `GetFullPathNameW` initialized `result` bytes and does not exceed `nBufferLength - 1` (capacity).
475            unsafe {
476                new_path.set_len((result as usize) + 1);
477            }
478            path == new_path
479        }
480    }
481}
482
483#[cfg(unix)]
484mod os_imp {
485    use camino::Utf8PathBuf;
486
487    pub(super) fn strip_verbatim(path: Utf8PathBuf) -> Utf8PathBuf {
488        // On Unix, there aren't any verbatin paths, so just return the path as
489        // is.
490        path
491    }
492}
493
494#[cfg(test)]
495mod tests {
496    use super::*;
497
498    /// Ensure that PathMapper turns relative paths into absolute ones.
499    #[test]
500    fn test_path_mapper_relative() {
501        let current_dir: Utf8PathBuf = std::env::current_dir()
502            .expect("current dir obtained")
503            .try_into()
504            .expect("current dir is valid UTF-8");
505
506        let temp_workspace_root = Utf8TempDir::new().expect("new temp dir created");
507        let workspace_root_path: Utf8PathBuf = os_imp::strip_verbatim(
508            temp_workspace_root
509                .path()
510                // On Mac, the temp dir is a symlink, so canonicalize it.
511                .canonicalize()
512                .expect("workspace root canonicalized correctly")
513                .try_into()
514                .expect("workspace root is valid UTF-8"),
515        );
516        let rel_workspace_root = pathdiff::diff_utf8_paths(&workspace_root_path, &current_dir)
517            .expect("abs to abs diff is non-None");
518
519        let temp_target_dir = Utf8TempDir::new().expect("new temp dir created");
520        let target_dir_path: Utf8PathBuf = os_imp::strip_verbatim(
521            temp_target_dir
522                .path()
523                .canonicalize()
524                .expect("target dir canonicalized correctly")
525                .try_into()
526                .expect("target dir is valid UTF-8"),
527        );
528        let rel_target_dir = pathdiff::diff_utf8_paths(&target_dir_path, &current_dir)
529            .expect("abs to abs diff is non-None");
530
531        // These aren't really used other than to do mapping against.
532        let orig_workspace_root = Utf8Path::new(env!("CARGO_MANIFEST_DIR"));
533        let orig_target_dir = orig_workspace_root.join("target");
534
535        let path_mapper = PathMapper::new(
536            orig_workspace_root,
537            Some(&rel_workspace_root),
538            &orig_target_dir,
539            Some(&rel_target_dir),
540            LibdirMapper::default(),
541        )
542        .expect("remapped paths exist");
543
544        assert_eq!(
545            path_mapper.map_cwd(orig_workspace_root.join("foobar")),
546            workspace_root_path.join("foobar")
547        );
548        assert_eq!(
549            path_mapper.map_binary(orig_target_dir.join("foobar")),
550            target_dir_path.join("foobar")
551        );
552    }
553}
nextest_runner/reuse_build/mod.rs

nextest_runner/reuse_build/
mod.rs
