nextest_runner/reuse_build/
mod.rs

1// Copyright (c) The nextest Contributors
2// SPDX-License-Identifier: MIT OR Apache-2.0
3
4//! Reuse builds performed earlier.
5//!
6//! Nextest allows users to reuse builds done on one machine. This module contains support for that.
7//!
8//! The main data structures here are [`ReuseBuildInfo`] and [`PathMapper`].
9
10use crate::{
11    errors::{
12        ArchiveExtractError, ArchiveReadError, MetadataMaterializeError, PathMapperConstructError,
13        PathMapperConstructKind,
14    },
15    list::BinaryList,
16    platform::PlatformLibdir,
17};
18use camino::{Utf8Path, Utf8PathBuf};
19use camino_tempfile::Utf8TempDir;
20use guppy::graph::PackageGraph;
21use nextest_metadata::{BinaryListSummary, PlatformLibdirUnavailable};
22use std::{fmt, fs, io, sync::Arc};
23
24mod archive_reporter;
25mod archiver;
26mod unarchiver;
27
28pub use archive_reporter::*;
29pub use archiver::*;
30pub use unarchiver::*;
31
32/// The name of the file in which Cargo metadata is stored.
33pub const CARGO_METADATA_FILE_NAME: &str = "target/nextest/cargo-metadata.json";
34
35/// The name of the file in which binaries metadata is stored.
36pub const BINARIES_METADATA_FILE_NAME: &str = "target/nextest/binaries-metadata.json";
37
38/// The name of the directory in which libdirs are stored.
39pub const LIBDIRS_BASE_DIR: &str = "target/nextest/libdirs";
40
41/// Reuse build information.
42#[derive(Debug, Default)]
43pub struct ReuseBuildInfo {
44    /// Cargo metadata and remapping for the target directory.
45    pub cargo_metadata: Option<MetadataWithRemap<ReusedCargoMetadata>>,
46
47    /// Binaries metadata JSON and remapping for the target directory.
48    pub binaries_metadata: Option<MetadataWithRemap<ReusedBinaryList>>,
49
50    /// A remapper for libdirs.
51    pub libdir_mapper: LibdirMapper,
52
53    /// Optional temporary directory used for cleanup.
54    _temp_dir: Option<Utf8TempDir>,
55}
56
57impl ReuseBuildInfo {
58    /// Creates a new [`ReuseBuildInfo`] from the given cargo and binaries metadata information.
59    pub fn new(
60        cargo_metadata: Option<MetadataWithRemap<ReusedCargoMetadata>>,
61        binaries_metadata: Option<MetadataWithRemap<ReusedBinaryList>>,
62        // TODO: accept libdir_mapper as an argument, as well
63    ) -> Self {
64        Self {
65            cargo_metadata,
66            binaries_metadata,
67            libdir_mapper: LibdirMapper::default(),
68            _temp_dir: None,
69        }
70    }
71
72    /// Extracts an archive and constructs a [`ReuseBuildInfo`] from it.
73    pub fn extract_archive<F>(
74        archive_file: &Utf8Path,
75        format: ArchiveFormat,
76        dest: ExtractDestination,
77        callback: F,
78        workspace_remap: Option<&Utf8Path>,
79    ) -> Result<Self, ArchiveExtractError>
80    where
81        F: for<'e> FnMut(ArchiveEvent<'e>) -> io::Result<()>,
82    {
83        let mut file = fs::File::open(archive_file)
84            .map_err(|err| ArchiveExtractError::Read(ArchiveReadError::Io(err)))?;
85
86        let mut unarchiver = Unarchiver::new(&mut file, format);
87        let ExtractInfo {
88            dest_dir,
89            temp_dir,
90            binary_list,
91            cargo_metadata_json,
92            graph,
93            libdir_mapper,
94        } = unarchiver.extract(dest, callback)?;
95
96        let cargo_metadata = MetadataWithRemap {
97            metadata: ReusedCargoMetadata::new((cargo_metadata_json, graph)),
98            remap: workspace_remap.map(|p| p.to_owned()),
99        };
100        let binaries_metadata = MetadataWithRemap {
101            metadata: ReusedBinaryList::new(binary_list),
102            remap: Some(dest_dir.join("target")),
103        };
104
105        Ok(Self {
106            cargo_metadata: Some(cargo_metadata),
107            binaries_metadata: Some(binaries_metadata),
108            libdir_mapper,
109            _temp_dir: temp_dir,
110        })
111    }
112
113    /// Returns the Cargo metadata.
114    pub fn cargo_metadata(&self) -> Option<&ReusedCargoMetadata> {
115        self.cargo_metadata.as_ref().map(|m| &m.metadata)
116    }
117
118    /// Returns the binaries metadata, reading it from disk if necessary.
119    pub fn binaries_metadata(&self) -> Option<&ReusedBinaryList> {
120        self.binaries_metadata.as_ref().map(|m| &m.metadata)
121    }
122
123    /// Returns true if any component of the build is being reused.
124    #[inline]
125    pub fn is_active(&self) -> bool {
126        self.cargo_metadata.is_some() || self.binaries_metadata.is_some()
127    }
128
129    /// Returns the new workspace directory.
130    pub fn workspace_remap(&self) -> Option<&Utf8Path> {
131        self.cargo_metadata
132            .as_ref()
133            .and_then(|m| m.remap.as_deref())
134    }
135
136    /// Returns the new target directory.
137    pub fn target_dir_remap(&self) -> Option<&Utf8Path> {
138        self.binaries_metadata
139            .as_ref()
140            .and_then(|m| m.remap.as_deref())
141    }
142}
143
144/// Metadata as either deserialized contents or a path, along with a possible directory remap.
145#[derive(Clone, Debug)]
146pub struct MetadataWithRemap<T> {
147    /// The metadata.
148    pub metadata: T,
149
150    /// The remapped directory.
151    pub remap: Option<Utf8PathBuf>,
152}
153
154/// Type parameter for [`MetadataWithRemap`].
155pub trait MetadataKind: Clone + fmt::Debug {
156    /// The type of metadata stored.
157    type MetadataType: Sized;
158
159    /// Constructs a new [`MetadataKind`] from the given metadata.
160    fn new(metadata: Self::MetadataType) -> Self;
161
162    /// Reads a path, resolving it into this data type.
163    fn materialize(path: &Utf8Path) -> Result<Self, MetadataMaterializeError>;
164}
165
166/// [`MetadataKind`] for a [`BinaryList`].
167#[derive(Clone, Debug)]
168pub struct ReusedBinaryList {
169    /// The binary list.
170    pub binary_list: Arc<BinaryList>,
171}
172
173impl MetadataKind for ReusedBinaryList {
174    type MetadataType = BinaryList;
175
176    fn new(binary_list: Self::MetadataType) -> Self {
177        Self {
178            binary_list: Arc::new(binary_list),
179        }
180    }
181
182    fn materialize(path: &Utf8Path) -> Result<Self, MetadataMaterializeError> {
183        // Three steps: read the contents, turn it into a summary, and then turn it into a
184        // BinaryList.
185        //
186        // Buffering the contents in memory is generally much faster than trying to read it
187        // using a BufReader.
188        let contents =
189            fs::read_to_string(path).map_err(|error| MetadataMaterializeError::Read {
190                path: path.to_owned(),
191                error,
192            })?;
193
194        let summary: BinaryListSummary = serde_json::from_str(&contents).map_err(|error| {
195            MetadataMaterializeError::Deserialize {
196                path: path.to_owned(),
197                error,
198            }
199        })?;
200
201        let binary_list = BinaryList::from_summary(summary).map_err(|error| {
202            MetadataMaterializeError::RustBuildMeta {
203                path: path.to_owned(),
204                error,
205            }
206        })?;
207
208        Ok(Self::new(binary_list))
209    }
210}
211
212/// [`MetadataKind`] for Cargo metadata.
213#[derive(Clone, Debug)]
214pub struct ReusedCargoMetadata {
215    /// Cargo metadata JSON.
216    pub json: Arc<String>,
217
218    /// The package graph.
219    pub graph: Arc<PackageGraph>,
220}
221
222impl MetadataKind for ReusedCargoMetadata {
223    type MetadataType = (String, PackageGraph);
224
225    fn new((json, graph): Self::MetadataType) -> Self {
226        Self {
227            json: Arc::new(json),
228            graph: Arc::new(graph),
229        }
230    }
231
232    fn materialize(path: &Utf8Path) -> Result<Self, MetadataMaterializeError> {
233        // Read the contents into memory, then parse them as a `PackageGraph`.
234        let json =
235            std::fs::read_to_string(path).map_err(|error| MetadataMaterializeError::Read {
236                path: path.to_owned(),
237                error,
238            })?;
239        let graph = PackageGraph::from_json(&json).map_err(|error| {
240            MetadataMaterializeError::PackageGraphConstruct {
241                path: path.to_owned(),
242                error,
243            }
244        })?;
245
246        Ok(Self::new((json, graph)))
247    }
248}
249
250/// A helper for path remapping.
251///
252/// This is useful when running tests in a different directory, or a different computer, from
253/// building them.
254#[derive(Clone, Debug, Default)]
255pub struct PathMapper {
256    workspace: Option<(Utf8PathBuf, Utf8PathBuf)>,
257    target_dir: Option<(Utf8PathBuf, Utf8PathBuf)>,
258    libdir_mapper: LibdirMapper,
259}
260
261impl PathMapper {
262    /// Constructs the path mapper.
263    pub fn new(
264        orig_workspace_root: impl Into<Utf8PathBuf>,
265        workspace_remap: Option<&Utf8Path>,
266        orig_target_dir: impl Into<Utf8PathBuf>,
267        target_dir_remap: Option<&Utf8Path>,
268        libdir_mapper: LibdirMapper,
269    ) -> Result<Self, PathMapperConstructError> {
270        let workspace_root = workspace_remap
271            .map(|root| Self::canonicalize_dir(root, PathMapperConstructKind::WorkspaceRoot))
272            .transpose()?;
273        let target_dir = target_dir_remap
274            .map(|dir| Self::canonicalize_dir(dir, PathMapperConstructKind::TargetDir))
275            .transpose()?;
276
277        Ok(Self {
278            workspace: workspace_root.map(|w| (orig_workspace_root.into(), w)),
279            target_dir: target_dir.map(|d| (orig_target_dir.into(), d)),
280            libdir_mapper,
281        })
282    }
283
284    /// Constructs a no-op path mapper.
285    pub fn noop() -> Self {
286        Self {
287            workspace: None,
288            target_dir: None,
289            libdir_mapper: LibdirMapper::default(),
290        }
291    }
292
293    /// Returns the libdir mapper.
294    pub fn libdir_mapper(&self) -> &LibdirMapper {
295        &self.libdir_mapper
296    }
297
298    fn canonicalize_dir(
299        input: &Utf8Path,
300        kind: PathMapperConstructKind,
301    ) -> Result<Utf8PathBuf, PathMapperConstructError> {
302        let canonicalized_path =
303            input
304                .canonicalize()
305                .map_err(|err| PathMapperConstructError::Canonicalization {
306                    kind,
307                    input: input.into(),
308                    err,
309                })?;
310        let canonicalized_path: Utf8PathBuf =
311            canonicalized_path
312                .try_into()
313                .map_err(|err| PathMapperConstructError::NonUtf8Path {
314                    kind,
315                    input: input.into(),
316                    err,
317                })?;
318        if !canonicalized_path.is_dir() {
319            return Err(PathMapperConstructError::NotADirectory {
320                kind,
321                input: input.into(),
322                canonicalized_path,
323            });
324        }
325
326        Ok(os_imp::strip_verbatim(canonicalized_path))
327    }
328
329    pub(super) fn new_target_dir(&self) -> Option<&Utf8Path> {
330        self.target_dir.as_ref().map(|(_, new)| new.as_path())
331    }
332
333    pub(crate) fn map_cwd(&self, path: Utf8PathBuf) -> Utf8PathBuf {
334        match &self.workspace {
335            Some((from, to)) => match path.strip_prefix(from) {
336                Ok(p) => to.join(p),
337                Err(_) => path,
338            },
339            None => path,
340        }
341    }
342
343    pub(crate) fn map_binary(&self, path: Utf8PathBuf) -> Utf8PathBuf {
344        match &self.target_dir {
345            Some((from, to)) => match path.strip_prefix(from) {
346                Ok(p) => to.join(p),
347                Err(_) => path,
348            },
349            None => path,
350        }
351    }
352}
353
354/// A mapper for lib dirs.
355///
356/// Archives store parts of lib dirs, which must be remapped to the new target directory.
357#[derive(Clone, Debug, Default)]
358pub struct LibdirMapper {
359    /// The host libdir mapper.
360    pub(crate) host: PlatformLibdirMapper,
361
362    /// The target libdir mapper.
363    pub(crate) target: PlatformLibdirMapper,
364}
365
366/// A mapper for an individual platform libdir.
367///
368/// Part of [`LibdirMapper`].
369#[derive(Clone, Debug, Default)]
370pub(crate) enum PlatformLibdirMapper {
371    Path(Utf8PathBuf),
372    Unavailable,
373    #[default]
374    NotRequested,
375}
376
377impl PlatformLibdirMapper {
378    pub(crate) fn map(&self, original: &PlatformLibdir) -> PlatformLibdir {
379        match self {
380            PlatformLibdirMapper::Path(new) => {
381                // Just use the new path. (We may have to check original in the future, but it
382                // doesn't seem necessary for now -- if a libdir has been provided to the remapper,
383                // that's that.)
384                PlatformLibdir::Available(new.clone())
385            }
386            PlatformLibdirMapper::Unavailable => {
387                // In this case, the original value is ignored -- we expected a libdir to be
388                // present, but it wasn't.
389                PlatformLibdir::Unavailable(PlatformLibdirUnavailable::NOT_IN_ARCHIVE)
390            }
391            PlatformLibdirMapper::NotRequested => original.clone(),
392        }
393    }
394}
395
396#[cfg(windows)]
397mod os_imp {
398    use camino::Utf8PathBuf;
399    use std::ptr;
400    use windows_sys::Win32::Storage::FileSystem::GetFullPathNameW;
401
402    /// Strips verbatim prefix from a path if possible.
403    pub(super) fn strip_verbatim(path: Utf8PathBuf) -> Utf8PathBuf {
404        let path_str = String::from(path);
405        if path_str.starts_with(r"\\?\UNC") {
406            // In general we don't expect UNC paths, so just return the path as is.
407            path_str.into()
408        } else if path_str.starts_with(r"\\?\") {
409            const START_LEN: usize = r"\\?\".len();
410
411            let is_absolute_exact = {
412                let mut v = path_str[START_LEN..].encode_utf16().collect::<Vec<u16>>();
413                // Ensure null termination.
414                v.push(0);
415                is_absolute_exact(&v)
416            };
417
418            if is_absolute_exact {
419                path_str[START_LEN..].into()
420            } else {
421                path_str.into()
422            }
423        } else {
424            // Not a verbatim path, so return it as is.
425            path_str.into()
426        }
427    }
428
429    /// Test that the path is absolute, fully qualified and unchanged when processed by the Windows API.Add commentMore actions
430    ///
431    /// For example:
432    ///
433    /// - `C:\path\to\file` will return true.
434    /// - `C:\path\to\nul` returns false because the Windows API will convert it to \\.\NUL
435    /// - `C:\path\to\..\file` returns false because it will be resolved to `C:\path\file`.
436    ///
437    /// This is a useful property because it means the path can be converted from and to and verbatim
438    /// path just by changing the prefix.
439    fn is_absolute_exact(path: &[u16]) -> bool {
440        // Adapted from the Rust project: https://github.com/rust-lang/rust/commit/edfc74722556c659de6fa03b23af3b9c8ceb8ac2
441
442        // This is implemented by checking that passing the path through
443        // GetFullPathNameW does not change the path in any way.
444
445        // Windows paths are limited to i16::MAX length
446        // though the API here accepts a u32 for the length.
447        if path.is_empty() || path.len() > u32::MAX as usize || path.last() != Some(&0) {
448            return false;
449        }
450        // The path returned by `GetFullPathNameW` must be the same length as the
451        // given path, otherwise they're not equal.
452        let buffer_len = path.len();
453        let mut new_path = Vec::with_capacity(buffer_len);
454        let result = unsafe {
455            GetFullPathNameW(
456                path.as_ptr(),
457                new_path.capacity() as u32,
458                new_path.as_mut_ptr(),
459                ptr::null_mut(),
460            )
461        };
462        // Note: if non-zero, the returned result is the length of the buffer without the null termination
463        if result == 0 || result as usize != buffer_len - 1 {
464            false
465        } else {
466            // SAFETY: `GetFullPathNameW` initialized `result` bytes and does not exceed `nBufferLength - 1` (capacity).
467            unsafe {
468                new_path.set_len((result as usize) + 1);
469            }
470            path == new_path
471        }
472    }
473}
474
475#[cfg(unix)]
476mod os_imp {
477    use camino::Utf8PathBuf;
478
479    pub(super) fn strip_verbatim(path: Utf8PathBuf) -> Utf8PathBuf {
480        // On Unix, there aren't any verbatin paths, so just return the path as
481        // is.
482        path
483    }
484}
485
486#[cfg(test)]
487mod tests {
488    use super::*;
489
490    /// Ensure that PathMapper turns relative paths into absolute ones.
491    #[test]
492    fn test_path_mapper_relative() {
493        let current_dir: Utf8PathBuf = std::env::current_dir()
494            .expect("current dir obtained")
495            .try_into()
496            .expect("current dir is valid UTF-8");
497
498        let temp_workspace_root = Utf8TempDir::new().expect("new temp dir created");
499        let workspace_root_path: Utf8PathBuf = os_imp::strip_verbatim(
500            temp_workspace_root
501                .path()
502                // On Mac, the temp dir is a symlink, so canonicalize it.
503                .canonicalize()
504                .expect("workspace root canonicalized correctly")
505                .try_into()
506                .expect("workspace root is valid UTF-8"),
507        );
508        let rel_workspace_root = pathdiff::diff_utf8_paths(&workspace_root_path, &current_dir)
509            .expect("abs to abs diff is non-None");
510
511        let temp_target_dir = Utf8TempDir::new().expect("new temp dir created");
512        let target_dir_path: Utf8PathBuf = os_imp::strip_verbatim(
513            temp_target_dir
514                .path()
515                .canonicalize()
516                .expect("target dir canonicalized correctly")
517                .try_into()
518                .expect("target dir is valid UTF-8"),
519        );
520        let rel_target_dir = pathdiff::diff_utf8_paths(&target_dir_path, &current_dir)
521            .expect("abs to abs diff is non-None");
522
523        // These aren't really used other than to do mapping against.
524        let orig_workspace_root = Utf8Path::new(env!("CARGO_MANIFEST_DIR"));
525        let orig_target_dir = orig_workspace_root.join("target");
526
527        let path_mapper = PathMapper::new(
528            orig_workspace_root,
529            Some(&rel_workspace_root),
530            &orig_target_dir,
531            Some(&rel_target_dir),
532            LibdirMapper::default(),
533        )
534        .expect("remapped paths exist");
535
536        assert_eq!(
537            path_mapper.map_cwd(orig_workspace_root.join("foobar")),
538            workspace_root_path.join("foobar")
539        );
540        assert_eq!(
541            path_mapper.map_binary(orig_target_dir.join("foobar")),
542            target_dir_path.join("foobar")
543        );
544    }
545}
nextest_runner/reuse_build/mod.rs

nextest_runner/reuse_build/
mod.rs
