#6990 Implement batched download of small files Issue created by illicitonion on <!date^1545667079^{date_short}|2018-12-24T15:57:59Z> As per https://github.com/bazelbuild/remote-apis/blob/ed4849810292e5fb3c844992133523f01a4ad420/build/bazel/remote/execution/v2/remote_execution.proto#L273-L294 Currently we have two codepaths which may cause multiple files to be downloaded when performing remote execution: pants/src/rust/engine/fs/src/store.rs Lines 518 to 556 in</pantsbuild/pants/commit/c6e5702216ee97ad83f8f80ac27d8cefd6adbcd0|c6e5702> | pub fn materialize_directory( | | ------------------------------------------------------------------------------------- | | &self, | | destination: PathBuf, | | digest: Digest, | | ) -> BoxFuture<(), String> { | | try_future!(super::safe_create_dir_all(&destination)); | | let store = self.clone(); | | self | | .load_directory(digest) | | .and_then(move \|directory_opt| { | | directory_opt.ok_or_else(\|| format!("Directory with digest {:?} not found", digest)) | | }) | | .and_then(move \|directory| { | | let file_futures = directory | | .get_files() | | .iter() | | .map(\|file_node| { | | let store = store.clone(); | | let path = destination.join(file_node.get_name()); | | let digest = try_future!(file_node.get_digest().into()); | | store.materialize_file(path, digest, file_node.is_executable) | | }) | | .collect::<Vec<_>>(); | | let directory_futures = directory | | .get_directories() | | .iter() | | .map(\|directory_node| { | | let store = store.clone(); | | let path = destination.join(directory_node.get_name()); | | let digest = try_future!(directory_node.get_digest().into()); | | store.materialize_directory(path, digest) | | }) | | .collect::<Vec<_>>(); | | future::join_all(file_futures) | | .join(future::join_all(directory_futures)) | | .map(\|_| ()) | | }) | | .to_boxed() | | } | and pants/src/rust/engine/fs/src/store.rs Lines 386 to 418 in</pantsbuild/pants/commit/c6e5702216ee97ad83f8f80ac27d8cefd6adbcd0|c6e5702> | pub fn ensure_local_has_recursive_directory(&self, dir_digest: Digest) -> BoxFuture<(), String> { | | | ------------------------------------------------------------------------------------------------- | ---------- | | let store = self.clone(); | | | self | | | .load_directory(dir_digest) | | | .and_then(move \|directory_opt| { | | | directory_opt.ok_or_else(\|| format!("Could not read dir with digest {:?}", dir_digest)) | | | }) | | | .and_then(move \|directory| { | | | // Traverse the files within directory | | | let file_futures = directory | | | .get_files() | | | .iter() | | | .map(\|file_node| { | | | let file_digest = try_future!(file_node.get_digest().into()); | | | store.load_bytes_with(EntryType::File, file_digest, \|_| Ok(()), | _| Ok(())) | | }) | | | .collect::<Vec<_>>(); | | | | | | // Recursively call with sub-directories | | | let directory_futures = directory | | | .get_directories() | | | .iter() | | | .map(move \|child_dir| { | | | let child_digest = try_future!(child_dir.get_digest().into()); | | | store.ensure_local_has_recursive_directory(child_digest) | | | }) | | | .collect::<Vec<_>>(); | | | future::join_all(file_futures) | | | .join… pantsbuild/pants