nextest_runner/reuse_build/
unarchiver.rs

1// Copyright (c) The nextest Contributors
2// SPDX-License-Identifier: MIT OR Apache-2.0
3
4use super::{
5    ArchiveEvent, ArchiveFormat, BINARIES_METADATA_FILE_NAME, CARGO_METADATA_FILE_NAME,
6    LIBDIRS_BASE_DIR, LibdirMapper, PlatformLibdirMapper,
7};
8use crate::{
9    errors::{ArchiveExtractError, ArchiveReadError},
10    helpers::convert_rel_path_to_main_sep,
11    list::BinaryList,
12};
13use camino::{Utf8Component, Utf8Path, Utf8PathBuf};
14use camino_tempfile::Utf8TempDir;
15use guppy::{CargoMetadata, graph::PackageGraph};
16use nextest_metadata::BinaryListSummary;
17use std::{
18    fs,
19    io::{self, Seek},
20    time::Instant,
21};
22use tar::EntryType;
23
24#[derive(Debug)]
25pub(crate) struct Unarchiver<'a> {
26    file: &'a mut fs::File,
27    format: ArchiveFormat,
28}
29
30impl<'a> Unarchiver<'a> {
31    pub(crate) fn new(file: &'a mut fs::File, format: ArchiveFormat) -> Self {
32        Self { file, format }
33    }
34
35    pub(crate) fn extract<F>(
36        &mut self,
37        dest: ExtractDestination,
38        mut callback: F,
39    ) -> Result<ExtractInfo, ArchiveExtractError>
40    where
41        F: for<'e> FnMut(ArchiveEvent<'e>) -> io::Result<()>,
42    {
43        let (dest_dir, temp_dir) = match dest {
44            ExtractDestination::TempDir { persist } => {
45                // Create a new temporary directory and extract contents to it.
46                let temp_dir = camino_tempfile::Builder::new()
47                    .prefix("nextest-archive-")
48                    .tempdir()
49                    .map_err(ArchiveExtractError::TempDirCreate)?;
50
51                let dest_dir: Utf8PathBuf = temp_dir.path().to_path_buf();
52                let dest_dir = temp_dir.path().canonicalize_utf8().map_err(|error| {
53                    ArchiveExtractError::DestDirCanonicalization {
54                        dir: dest_dir,
55                        error,
56                    }
57                })?;
58
59                let temp_dir = if persist {
60                    // Persist the temporary directory.
61                    let _ = temp_dir.keep();
62                    None
63                } else {
64                    Some(temp_dir)
65                };
66
67                (dest_dir, temp_dir)
68            }
69            ExtractDestination::Destination { dir, overwrite } => {
70                // Extract contents to the destination directory.
71                let dest_dir = dir
72                    .canonicalize_utf8()
73                    .map_err(|error| ArchiveExtractError::DestDirCanonicalization { dir, error })?;
74
75                let dest_target = dest_dir.join("target");
76                if dest_target.exists() && !overwrite {
77                    return Err(ArchiveExtractError::DestinationExists(dest_target));
78                }
79
80                (dest_dir, None)
81            }
82        };
83
84        let start_time = Instant::now();
85
86        // Extract the archive.
87        self.file
88            .rewind()
89            .map_err(|error| ArchiveExtractError::Read(ArchiveReadError::Io(error)))?;
90        let mut archive_reader =
91            ArchiveReader::new(self.file, self.format).map_err(ArchiveExtractError::Read)?;
92
93        // Will be filled out by the for loop below.
94        let mut binary_list = None;
95        let mut graph_data = None;
96        let mut host_libdir = PlatformLibdirMapper::Unavailable;
97        let mut target_libdir = PlatformLibdirMapper::Unavailable;
98        let binaries_metadata_path = Utf8Path::new(BINARIES_METADATA_FILE_NAME);
99        let cargo_metadata_path = Utf8Path::new(CARGO_METADATA_FILE_NAME);
100
101        let mut file_count = 0;
102
103        for entry in archive_reader
104            .entries()
105            .map_err(ArchiveExtractError::Read)?
106        {
107            file_count += 1;
108            let (mut entry, path) = entry.map_err(ArchiveExtractError::Read)?;
109
110            entry
111                .unpack_in(&dest_dir)
112                .map_err(|error| ArchiveExtractError::WriteFile {
113                    path: path.clone(),
114                    error,
115                })?;
116
117            // For archives created by nextest, binaries_metadata_path should be towards the beginning
118            // so this should report the ExtractStarted event instantly.
119            if path == binaries_metadata_path {
120                // Try reading the binary list from the file on disk.
121                let mut file = fs::File::open(dest_dir.join(binaries_metadata_path))
122                    .map_err(|error| ArchiveExtractError::WriteFile { path, error })?;
123
124                let summary: BinaryListSummary =
125                    serde_json::from_reader(&mut file).map_err(|error| {
126                        ArchiveExtractError::Read(ArchiveReadError::MetadataDeserializeError {
127                            path: binaries_metadata_path,
128                            error,
129                        })
130                    })?;
131
132                let this_binary_list = BinaryList::from_summary(summary)?;
133                let test_binary_count = this_binary_list.rust_binaries.len();
134                let non_test_binary_count = this_binary_list
135                    .rust_build_meta
136                    .non_test_binaries
137                    .binary_count();
138                let build_script_out_dir_count =
139                    this_binary_list.rust_build_meta.build_script_out_dirs.len();
140                let linked_path_count = this_binary_list.rust_build_meta.linked_paths.len();
141
142                // TODO: also store a manifest of extra paths, and report them here.
143
144                // Report begin extraction.
145                callback(ArchiveEvent::ExtractStarted {
146                    test_binary_count,
147                    non_test_binary_count,
148                    build_script_out_dir_count,
149                    linked_path_count,
150                    dest_dir: &dest_dir,
151                })
152                .map_err(ArchiveExtractError::ReporterIo)?;
153
154                binary_list = Some(this_binary_list);
155            } else if path == cargo_metadata_path {
156                // Parse the input Cargo metadata as a `PackageGraph`.
157                let json = fs::read_to_string(dest_dir.join(cargo_metadata_path))
158                    .map_err(|error| ArchiveExtractError::WriteFile { path, error })?;
159
160                // Doing this in multiple steps results in better error messages.
161                let cargo_metadata: CargoMetadata =
162                    serde_json::from_str(&json).map_err(|error| {
163                        ArchiveExtractError::Read(ArchiveReadError::MetadataDeserializeError {
164                            path: binaries_metadata_path,
165                            error,
166                        })
167                    })?;
168
169                let package_graph = cargo_metadata.build_graph().map_err(|error| {
170                    ArchiveExtractError::Read(ArchiveReadError::PackageGraphConstructError {
171                        path: cargo_metadata_path,
172                        error: Box::new(error),
173                    })
174                })?;
175                graph_data = Some((json, package_graph));
176                continue;
177            } else if let Ok(suffix) = path.strip_prefix(LIBDIRS_BASE_DIR) {
178                if suffix.starts_with("host") {
179                    host_libdir = PlatformLibdirMapper::Path(dest_dir.join(
180                        convert_rel_path_to_main_sep(&Utf8Path::new(LIBDIRS_BASE_DIR).join("host")),
181                    ));
182                } else if suffix.starts_with("target/0") {
183                    // Currently we only support one target, so just check explicitly for target/0.
184                    target_libdir =
185                        PlatformLibdirMapper::Path(dest_dir.join(convert_rel_path_to_main_sep(
186                            &Utf8Path::new(LIBDIRS_BASE_DIR).join("target/0"),
187                        )));
188                }
189            }
190        }
191
192        let binary_list = binary_list.ok_or_else(|| {
193            ArchiveExtractError::Read(ArchiveReadError::MetadataFileNotFound(
194                binaries_metadata_path,
195            ))
196        })?;
197
198        let (cargo_metadata_json, graph) = graph_data.ok_or_else(|| {
199            ArchiveExtractError::Read(ArchiveReadError::MetadataFileNotFound(cargo_metadata_path))
200        })?;
201
202        let elapsed = start_time.elapsed();
203        // Report end extraction.
204        callback(ArchiveEvent::Extracted {
205            file_count,
206            dest_dir: &dest_dir,
207            elapsed,
208        })
209        .map_err(ArchiveExtractError::ReporterIo)?;
210
211        Ok(ExtractInfo {
212            dest_dir,
213            temp_dir,
214            binary_list,
215            cargo_metadata_json,
216            graph,
217            libdir_mapper: LibdirMapper {
218                host: host_libdir,
219                target: target_libdir,
220            },
221        })
222    }
223}
224
225#[derive(Debug)]
226pub(crate) struct ExtractInfo {
227    /// The destination directory.
228    pub dest_dir: Utf8PathBuf,
229
230    /// An optional [`Utf8TempDir`], used for cleanup.
231    pub temp_dir: Option<Utf8TempDir>,
232
233    /// The [`BinaryList`] read from the archive.
234    pub binary_list: BinaryList,
235
236    /// The Cargo metadata JSON.
237    pub cargo_metadata_json: String,
238
239    /// The [`PackageGraph`] read from the archive.
240    pub graph: PackageGraph,
241
242    /// A remapper for the Rust libdir.
243    pub libdir_mapper: LibdirMapper,
244}
245
246struct ArchiveReader<'a> {
247    archive: tar::Archive<zstd::Decoder<'static, io::BufReader<&'a mut fs::File>>>,
248}
249
250impl<'a> ArchiveReader<'a> {
251    fn new(file: &'a mut fs::File, format: ArchiveFormat) -> Result<Self, ArchiveReadError> {
252        let archive = match format {
253            ArchiveFormat::TarZst => {
254                let decoder = zstd::Decoder::new(file).map_err(ArchiveReadError::Io)?;
255                tar::Archive::new(decoder)
256            }
257        };
258        Ok(Self { archive })
259    }
260
261    fn entries<'r>(
262        &'r mut self,
263    ) -> Result<
264        impl Iterator<Item = Result<(ArchiveEntry<'r, 'a>, Utf8PathBuf), ArchiveReadError>>,
265        ArchiveReadError,
266    > {
267        let entries = self.archive.entries().map_err(ArchiveReadError::Io)?;
268        Ok(entries.map(|entry| {
269            let entry = entry.map_err(ArchiveReadError::Io)?;
270
271            // Validation: entry paths must be valid UTF-8.
272            let path = entry_path(&entry)?;
273
274            // Validation: paths start with "target".
275            if !path.starts_with("target") {
276                return Err(ArchiveReadError::NoTargetPrefix(path));
277            }
278
279            // Validation: paths only contain normal components.
280            for component in path.components() {
281                match component {
282                    Utf8Component::Normal(_) => {}
283                    other => {
284                        return Err(ArchiveReadError::InvalidComponent {
285                            path: path.clone(),
286                            component: other.as_str().to_owned(),
287                        });
288                    }
289                }
290            }
291
292            match entry.header().entry_type() {
293                EntryType::Regular | EntryType::GNUSparse | EntryType::Directory => {}
294                entry_type => {
295                    return Err(ArchiveReadError::UnsupportedEntryType { path, entry_type });
296                }
297            }
298
299            // Validation: checksum matches.
300            let mut header = entry.header().clone();
301            let actual_cksum = header
302                .cksum()
303                .map_err(|error| ArchiveReadError::ChecksumRead {
304                    path: path.clone(),
305                    error,
306                })?;
307
308            header.set_cksum();
309            let expected_cksum = header
310                .cksum()
311                .expect("checksum that was just set can't be invalid");
312
313            if expected_cksum != actual_cksum {
314                return Err(ArchiveReadError::InvalidChecksum {
315                    path,
316                    expected: expected_cksum,
317                    actual: actual_cksum,
318                });
319            }
320
321            Ok((entry, path))
322        }))
323    }
324}
325
326/// Given an entry, returns its path as a `Utf8Path`.
327fn entry_path(entry: &ArchiveEntry<'_, '_>) -> Result<Utf8PathBuf, ArchiveReadError> {
328    let path_bytes = entry.path_bytes();
329    let path_str = std::str::from_utf8(&path_bytes)
330        .map_err(|_| ArchiveReadError::NonUtf8Path(path_bytes.to_vec()))?;
331    let utf8_path = Utf8Path::new(path_str);
332    Ok(utf8_path.to_owned())
333}
334
335/// Where to extract a nextest archive to.
336#[derive(Clone, Debug)]
337pub enum ExtractDestination {
338    /// Extract the archive to a new temporary directory.
339    TempDir {
340        /// Whether to persist the temporary directory at the end of execution.
341        persist: bool,
342    },
343    /// Extract the archive to a custom destination.
344    Destination {
345        /// The directory to extract to.
346        dir: Utf8PathBuf,
347        /// Whether to overwrite existing contents.
348        overwrite: bool,
349    },
350}
351
352type ArchiveEntry<'r, 'a> = tar::Entry<'r, zstd::Decoder<'static, io::BufReader<&'a mut fs::File>>>;
353
354#[cfg(test)]
355mod tests {
356    use super::*;
357
358    #[test]
359    fn test_entry_type_validation() {
360        let rejected = [
361            EntryType::Symlink,
362            EntryType::Link,
363            EntryType::Char,
364            EntryType::Block,
365            EntryType::Fifo,
366        ];
367        for entry_type in rejected {
368            let mut header = tar::Header::new_gnu();
369            header.set_entry_type(entry_type);
370            header.set_size(0);
371            header.set_mode(0o644);
372            let mut file = archive_with(|builder| {
373                builder
374                    .append_data(&mut header, "target/entry", io::empty())
375                    .expect("appended entry");
376            });
377
378            let mut reader =
379                ArchiveReader::new(&mut file, ArchiveFormat::TarZst).expect("archive is readable");
380            let first = reader
381                .entries()
382                .expect("entries are readable")
383                .next()
384                .expect("archive has an entry")
385                .map(|(_, path)| path);
386            match first {
387                Err(ArchiveReadError::UnsupportedEntryType {
388                    path,
389                    entry_type: actual,
390                }) => {
391                    assert_eq!(path, "target/entry", "path for {entry_type:?}");
392                    assert_eq!(actual, entry_type, "entry type for {entry_type:?}");
393                }
394                other => {
395                    panic!("{entry_type:?}: expected UnsupportedEntryType, found {other:?}")
396                }
397            }
398        }
399
400        let source_dir = camino_tempfile::tempdir().expect("created source dir");
401        let regular_path = source_dir.path().join("regular");
402        fs::write(&regular_path, b"contents").expect("wrote regular file");
403        let sparse_path = source_dir.path().join("sparse");
404        fs::File::create(&sparse_path)
405            .expect("created sparse file")
406            .set_len(64 * 1024)
407            .expect("extended sparse file");
408
409        let mut file = archive_with(|builder| {
410            builder
411                .append_path_with_name(&regular_path, "target/regular")
412                .expect("appended regular file");
413            builder
414                .append_path_with_name(&sparse_path, "target/sparse")
415                .expect("appended sparse file");
416            builder
417                .append_dir("target/dir", source_dir.path())
418                .expect("appended directory");
419        });
420
421        let mut reader =
422            ArchiveReader::new(&mut file, ArchiveFormat::TarZst).expect("archive is readable");
423        let accepted: Vec<_> = reader
424            .entries()
425            .expect("entries are readable")
426            .map(|entry| {
427                let (entry, path) = entry.expect("entry is accepted");
428                (path, entry.header().entry_type())
429            })
430            .collect();
431        // Note that tar only detects sparse files on Linux, Android, and
432        // FreeBSD, so on other platforms the sparse file is a Regular entry.
433        let expected_sparse_type = if cfg!(any(
434            target_os = "linux",
435            target_os = "android",
436            target_os = "freebsd"
437        )) {
438            EntryType::GNUSparse
439        } else {
440            EntryType::Regular
441        };
442        assert_eq!(
443            accepted,
444            [
445                (Utf8PathBuf::from("target/regular"), EntryType::Regular),
446                (Utf8PathBuf::from("target/sparse"), expected_sparse_type),
447                (Utf8PathBuf::from("target/dir"), EntryType::Directory),
448            ],
449            "accepted entries"
450        );
451    }
452
453    fn archive_with(
454        build: impl FnOnce(&mut tar::Builder<zstd::Encoder<'static, fs::File>>),
455    ) -> fs::File {
456        let file = camino_tempfile::tempfile().expect("created archive file");
457        let encoder = zstd::Encoder::new(file, 0).expect("created zstd encoder");
458        let mut builder = tar::Builder::new(encoder);
459        build(&mut builder);
460        let mut file = builder
461            .into_inner()
462            .expect("finished tar archive")
463            .finish()
464            .expect("finished zstd stream");
465        file.rewind().expect("rewound archive file");
466        file
467    }
468}
nextest_runner/reuse_build/unarchiver.rs

nextest_runner/reuse_build/
unarchiver.rs
