nextest_runner/reuse_build/
mod.rs

1// Copyright (c) The nextest Contributors
2// SPDX-License-Identifier: MIT OR Apache-2.0
3
4//! Reuse builds performed earlier.
5//!
6//! Nextest allows users to reuse builds done on one machine. This module contains support for that.
7//!
8//! The main data structures here are [`ReuseBuildInfo`] and [`PathMapper`].
9
10use crate::{
11    errors::{
12        ArchiveExtractError, ArchiveReadError, MetadataMaterializeError, PathMapperConstructError,
13        PathMapperConstructKind,
14    },
15    list::BinaryList,
16    platform::PlatformLibdir,
17};
18use camino::{Utf8Path, Utf8PathBuf};
19use camino_tempfile::Utf8TempDir;
20use guppy::graph::PackageGraph;
21use nextest_metadata::{BinaryListSummary, PlatformLibdirUnavailable};
22use std::{fmt, fs, io, sync::Arc};
23
24mod archive_reporter;
25mod archiver;
26mod unarchiver;
27
28pub use archive_reporter::*;
29pub use archiver::*;
30pub use unarchiver::*;
31
32/// The name of the file in which Cargo metadata is stored.
33pub const CARGO_METADATA_FILE_NAME: &str = "target/nextest/cargo-metadata.json";
34
35/// The name of the file in which binaries metadata is stored.
36pub const BINARIES_METADATA_FILE_NAME: &str = "target/nextest/binaries-metadata.json";
37
38/// The name of the directory in which libdirs are stored.
39pub const LIBDIRS_BASE_DIR: &str = "target/nextest/libdirs";
40
41/// Reuse build information.
42#[derive(Debug, Default)]
43pub struct ReuseBuildInfo {
44    /// Cargo metadata and remapping for the target directory.
45    pub cargo_metadata: Option<MetadataWithRemap<ReusedCargoMetadata>>,
46
47    /// Binaries metadata JSON and remapping for the target directory.
48    pub binaries_metadata: Option<MetadataWithRemap<ReusedBinaryList>>,
49
50    /// A remapper for libdirs.
51    pub libdir_mapper: LibdirMapper,
52
53    /// Optional temporary directory used for cleanup.
54    _temp_dir: Option<Utf8TempDir>,
55}
56
57impl ReuseBuildInfo {
58    /// Creates a new [`ReuseBuildInfo`] from the given cargo and binaries metadata information.
59    pub fn new(
60        cargo_metadata: Option<MetadataWithRemap<ReusedCargoMetadata>>,
61        binaries_metadata: Option<MetadataWithRemap<ReusedBinaryList>>,
62        // TODO: accept libdir_mapper as an argument, as well
63    ) -> Self {
64        Self {
65            cargo_metadata,
66            binaries_metadata,
67            libdir_mapper: LibdirMapper::default(),
68            _temp_dir: None,
69        }
70    }
71
72    /// Extracts an archive and constructs a [`ReuseBuildInfo`] from it.
73    pub fn extract_archive<F>(
74        archive_file: &Utf8Path,
75        format: ArchiveFormat,
76        dest: ExtractDestination,
77        callback: F,
78        workspace_remap: Option<&Utf8Path>,
79    ) -> Result<Self, ArchiveExtractError>
80    where
81        F: for<'e> FnMut(ArchiveEvent<'e>) -> io::Result<()>,
82    {
83        let mut file = fs::File::open(archive_file)
84            .map_err(|err| ArchiveExtractError::Read(ArchiveReadError::Io(err)))?;
85
86        let mut unarchiver = Unarchiver::new(&mut file, format);
87        let ExtractInfo {
88            dest_dir,
89            temp_dir,
90            binary_list,
91            cargo_metadata_json,
92            graph,
93            libdir_mapper,
94        } = unarchiver.extract(dest, callback)?;
95
96        let cargo_metadata = MetadataWithRemap {
97            metadata: ReusedCargoMetadata::new((cargo_metadata_json, graph)),
98            remap: workspace_remap.map(|p| p.to_owned()),
99        };
100        let binaries_metadata = MetadataWithRemap {
101            metadata: ReusedBinaryList::new(binary_list),
102            remap: Some(dest_dir.join("target")),
103        };
104
105        Ok(Self {
106            cargo_metadata: Some(cargo_metadata),
107            binaries_metadata: Some(binaries_metadata),
108            libdir_mapper,
109            _temp_dir: temp_dir,
110        })
111    }
112
113    /// Returns the Cargo metadata.
114    pub fn cargo_metadata(&self) -> Option<&ReusedCargoMetadata> {
115        self.cargo_metadata.as_ref().map(|m| &m.metadata)
116    }
117
118    /// Returns the binaries metadata, reading it from disk if necessary.
119    pub fn binaries_metadata(&self) -> Option<&ReusedBinaryList> {
120        self.binaries_metadata.as_ref().map(|m| &m.metadata)
121    }
122
123    /// Returns true if any component of the build is being reused.
124    #[inline]
125    pub fn is_active(&self) -> bool {
126        self.cargo_metadata.is_some() || self.binaries_metadata.is_some()
127    }
128
129    /// Returns the new workspace directory.
130    pub fn workspace_remap(&self) -> Option<&Utf8Path> {
131        self.cargo_metadata
132            .as_ref()
133            .and_then(|m| m.remap.as_deref())
134    }
135
136    /// Returns the new target directory.
137    pub fn target_dir_remap(&self) -> Option<&Utf8Path> {
138        self.binaries_metadata
139            .as_ref()
140            .and_then(|m| m.remap.as_deref())
141    }
142}
143
144/// Metadata as either deserialized contents or a path, along with a possible directory remap.
145#[derive(Clone, Debug)]
146pub struct MetadataWithRemap<T> {
147    /// The metadata.
148    pub metadata: T,
149
150    /// The remapped directory.
151    pub remap: Option<Utf8PathBuf>,
152}
153
154/// Type parameter for [`MetadataWithRemap`].
155pub trait MetadataKind: Clone + fmt::Debug {
156    /// The type of metadata stored.
157    type MetadataType: Sized;
158
159    /// Constructs a new [`MetadataKind`] from the given metadata.
160    fn new(metadata: Self::MetadataType) -> Self;
161
162    /// Reads a path, resolving it into this data type.
163    fn materialize(path: &Utf8Path) -> Result<Self, MetadataMaterializeError>;
164}
165
166/// [`MetadataKind`] for a [`BinaryList`].
167#[derive(Clone, Debug)]
168pub struct ReusedBinaryList {
169    /// The binary list.
170    pub binary_list: Arc<BinaryList>,
171}
172
173impl MetadataKind for ReusedBinaryList {
174    type MetadataType = BinaryList;
175
176    fn new(binary_list: Self::MetadataType) -> Self {
177        Self {
178            binary_list: Arc::new(binary_list),
179        }
180    }
181
182    fn materialize(path: &Utf8Path) -> Result<Self, MetadataMaterializeError> {
183        // Three steps: read the contents, turn it into a summary, and then turn it into a
184        // BinaryList.
185        //
186        // Buffering the contents in memory is generally much faster than trying to read it
187        // using a BufReader.
188        let contents =
189            fs::read_to_string(path).map_err(|error| MetadataMaterializeError::Read {
190                path: path.to_owned(),
191                error,
192            })?;
193
194        let summary: BinaryListSummary = serde_json::from_str(&contents).map_err(|error| {
195            MetadataMaterializeError::Deserialize {
196                path: path.to_owned(),
197                error,
198            }
199        })?;
200
201        let binary_list = BinaryList::from_summary(summary).map_err(|error| {
202            MetadataMaterializeError::RustBuildMeta {
203                path: path.to_owned(),
204                error,
205            }
206        })?;
207
208        Ok(Self::new(binary_list))
209    }
210}
211
212/// [`MetadataKind`] for Cargo metadata.
213#[derive(Clone, Debug)]
214pub struct ReusedCargoMetadata {
215    /// Cargo metadata JSON.
216    pub json: Arc<String>,
217
218    /// The package graph.
219    pub graph: Arc<PackageGraph>,
220}
221
222impl MetadataKind for ReusedCargoMetadata {
223    type MetadataType = (String, PackageGraph);
224
225    fn new((json, graph): Self::MetadataType) -> Self {
226        Self {
227            json: Arc::new(json),
228            graph: Arc::new(graph),
229        }
230    }
231
232    fn materialize(path: &Utf8Path) -> Result<Self, MetadataMaterializeError> {
233        // Read the contents into memory, then parse them as a `PackageGraph`.
234        let json =
235            std::fs::read_to_string(path).map_err(|error| MetadataMaterializeError::Read {
236                path: path.to_owned(),
237                error,
238            })?;
239        let graph = PackageGraph::from_json(&json).map_err(|error| {
240            MetadataMaterializeError::PackageGraphConstruct {
241                path: path.to_owned(),
242                error,
243            }
244        })?;
245
246        Ok(Self::new((json, graph)))
247    }
248}
249
250/// A helper for path remapping.
251///
252/// This is useful when running tests in a different directory, or a different computer, from
253/// building them.
254#[derive(Clone, Debug, Default)]
255pub struct PathMapper {
256    workspace: Option<(Utf8PathBuf, Utf8PathBuf)>,
257    target_dir: Option<(Utf8PathBuf, Utf8PathBuf)>,
258    libdir_mapper: LibdirMapper,
259}
260
261impl PathMapper {
262    /// Constructs the path mapper.
263    pub fn new(
264        orig_workspace_root: impl Into<Utf8PathBuf>,
265        workspace_remap: Option<&Utf8Path>,
266        orig_target_dir: impl Into<Utf8PathBuf>,
267        target_dir_remap: Option<&Utf8Path>,
268        libdir_mapper: LibdirMapper,
269    ) -> Result<Self, PathMapperConstructError> {
270        let workspace_root = workspace_remap
271            .map(|root| Self::canonicalize_dir(root, PathMapperConstructKind::WorkspaceRoot))
272            .transpose()?;
273        let target_dir = target_dir_remap
274            .map(|dir| Self::canonicalize_dir(dir, PathMapperConstructKind::TargetDir))
275            .transpose()?;
276
277        Ok(Self {
278            workspace: workspace_root.map(|w| (orig_workspace_root.into(), w)),
279            target_dir: target_dir.map(|d| (orig_target_dir.into(), d)),
280            libdir_mapper,
281        })
282    }
283
284    /// Constructs a no-op path mapper.
285    pub fn noop() -> Self {
286        Self {
287            workspace: None,
288            target_dir: None,
289            libdir_mapper: LibdirMapper::default(),
290        }
291    }
292
293    /// Returns the libdir mapper.
294    pub fn libdir_mapper(&self) -> &LibdirMapper {
295        &self.libdir_mapper
296    }
297
298    fn canonicalize_dir(
299        input: &Utf8Path,
300        kind: PathMapperConstructKind,
301    ) -> Result<Utf8PathBuf, PathMapperConstructError> {
302        let canonicalized_path =
303            input
304                .canonicalize()
305                .map_err(|err| PathMapperConstructError::Canonicalization {
306                    kind,
307                    input: input.into(),
308                    err,
309                })?;
310        let canonicalized_path: Utf8PathBuf =
311            canonicalized_path
312                .try_into()
313                .map_err(|err| PathMapperConstructError::NonUtf8Path {
314                    kind,
315                    input: input.into(),
316                    err,
317                })?;
318        if !canonicalized_path.is_dir() {
319            return Err(PathMapperConstructError::NotADirectory {
320                kind,
321                input: input.into(),
322                canonicalized_path,
323            });
324        }
325
326        Ok(canonicalized_path)
327    }
328
329    pub(super) fn new_target_dir(&self) -> Option<&Utf8Path> {
330        self.target_dir.as_ref().map(|(_, new)| new.as_path())
331    }
332
333    pub(crate) fn map_cwd(&self, path: Utf8PathBuf) -> Utf8PathBuf {
334        match &self.workspace {
335            Some((from, to)) => match path.strip_prefix(from) {
336                Ok(p) => to.join(p),
337                Err(_) => path,
338            },
339            None => path,
340        }
341    }
342
343    pub(crate) fn map_binary(&self, path: Utf8PathBuf) -> Utf8PathBuf {
344        match &self.target_dir {
345            Some((from, to)) => match path.strip_prefix(from) {
346                Ok(p) => to.join(p),
347                Err(_) => path,
348            },
349            None => path,
350        }
351    }
352}
353
354/// A mapper for lib dirs.
355///
356/// Archives store parts of lib dirs, which must be remapped to the new target directory.
357#[derive(Clone, Debug, Default)]
358pub struct LibdirMapper {
359    /// The host libdir mapper.
360    pub(crate) host: PlatformLibdirMapper,
361
362    /// The target libdir mapper.
363    pub(crate) target: PlatformLibdirMapper,
364}
365
366/// A mapper for an individual platform libdir.
367///
368/// Part of [`LibdirMapper`].
369#[derive(Clone, Debug, Default)]
370pub(crate) enum PlatformLibdirMapper {
371    Path(Utf8PathBuf),
372    Unavailable,
373    #[default]
374    NotRequested,
375}
376
377impl PlatformLibdirMapper {
378    pub(crate) fn map(&self, original: &PlatformLibdir) -> PlatformLibdir {
379        match self {
380            PlatformLibdirMapper::Path(new) => {
381                // Just use the new path. (We may have to check original in the future, but it
382                // doesn't seem necessary for now -- if a libdir has been provided to the remapper,
383                // that's that.)
384                PlatformLibdir::Available(new.clone())
385            }
386            PlatformLibdirMapper::Unavailable => {
387                // In this case, the original value is ignored -- we expected a libdir to be
388                // present, but it wasn't.
389                PlatformLibdir::Unavailable(PlatformLibdirUnavailable::NOT_IN_ARCHIVE)
390            }
391            PlatformLibdirMapper::NotRequested => original.clone(),
392        }
393    }
394}
395
396#[cfg(test)]
397mod tests {
398    use super::*;
399
400    /// Ensure that PathMapper turns relative paths into absolute ones.
401    #[test]
402    fn test_path_mapper_relative() {
403        let current_dir: Utf8PathBuf = std::env::current_dir()
404            .expect("current dir obtained")
405            .try_into()
406            .expect("current dir is valid UTF-8");
407
408        let temp_workspace_root = Utf8TempDir::new().expect("new temp dir created");
409        let workspace_root_path: Utf8PathBuf = temp_workspace_root
410            .path()
411            // On Mac, the temp dir is a symlink, so canonicalize it.
412            .canonicalize()
413            .expect("workspace root canonicalized correctly")
414            .try_into()
415            .expect("workspace root is valid UTF-8");
416        let rel_workspace_root = pathdiff::diff_utf8_paths(&workspace_root_path, &current_dir)
417            .expect("abs to abs diff is non-None");
418
419        let temp_target_dir = Utf8TempDir::new().expect("new temp dir created");
420        let target_dir_path: Utf8PathBuf = temp_target_dir
421            .path()
422            .canonicalize()
423            .expect("target dir canonicalized correctly")
424            .try_into()
425            .expect("target dir is valid UTF-8");
426        let rel_target_dir = pathdiff::diff_utf8_paths(&target_dir_path, &current_dir)
427            .expect("abs to abs diff is non-None");
428
429        // These aren't really used other than to do mapping against.
430        let orig_workspace_root = Utf8Path::new(env!("CARGO_MANIFEST_DIR"));
431        let orig_target_dir = orig_workspace_root.join("target");
432
433        let path_mapper = PathMapper::new(
434            orig_workspace_root,
435            Some(&rel_workspace_root),
436            &orig_target_dir,
437            Some(&rel_target_dir),
438            LibdirMapper::default(),
439        )
440        .expect("remapped paths exist");
441
442        assert_eq!(
443            path_mapper.map_cwd(orig_workspace_root.join("foobar")),
444            workspace_root_path.join("foobar")
445        );
446        assert_eq!(
447            path_mapper.map_binary(orig_target_dir.join("foobar")),
448            target_dir_path.join("foobar")
449        );
450    }
451}
nextest_runner/reuse_build/mod.rs

nextest_runner/reuse_build/
mod.rs
