diff --git a/Cargo.lock b/Cargo.lock index 0796737b7f2..5de7c42c276 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1397,6 +1397,7 @@ dependencies = [ "gix-revision", "gix-revwalk 0.17.0", "gix-sec 0.10.10", + "gix-shallow", "gix-status", "gix-submodule", "gix-tempfile 15.0.0", @@ -2383,8 +2384,15 @@ dependencies = [ "gix-date 0.9.2", "gix-features 0.39.1", "gix-hash 0.15.1", + "gix-lock 15.0.1", + "gix-negotiate", + "gix-object 0.46.0", "gix-packetline", - "gix-testtools", + "gix-ref 0.49.0", + "gix-refspec", + "gix-revwalk 0.17.0", + "gix-shallow", + "gix-trace 0.1.11", "gix-transport", "gix-utils 0.1.13", "maybe-async", @@ -2572,6 +2580,17 @@ dependencies = [ name = "gix-sequencer" version = "0.0.0" +[[package]] +name = "gix-shallow" +version = "0.1.0" +dependencies = [ + "bstr", + "gix-hash 0.15.1", + "gix-lock 15.0.1", + "serde", + "thiserror 2.0.3", +] + [[package]] name = "gix-status" version = "0.15.0" diff --git a/Cargo.toml b/Cargo.toml index 8323a650cb3..231dacc83a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -304,6 +304,7 @@ members = [ "gix-ref/tests", "gix-config/tests", "gix-traverse/tests", + "gix-shallow" ] [workspace.dependencies] diff --git a/gitoxide-core/src/pack/receive.rs b/gitoxide-core/src/pack/receive.rs index ad89f0c2fe2..8270d244601 100644 --- a/gitoxide-core/src/pack/receive.rs +++ b/gitoxide-core/src/pack/receive.rs @@ -1,28 +1,29 @@ -use std::{ - borrow::Cow, - io, - path::PathBuf, - sync::{atomic::AtomicBool, Arc}, -}; - +use crate::net; +use crate::pack::receive::protocol::fetch::negotiate; +use crate::OutputFormat; +use gix::config::tree::Key; +use gix::protocol::maybe_async; +use gix::DynNestedProgress; pub use gix::{ hash::ObjectId, objs::bstr::{BString, ByteSlice}, odb::pack, protocol, protocol::{ - fetch::{Action, Arguments, Response}, + fetch::{Arguments, Response}, handshake::Ref, transport, transport::client::Capabilities, }, - Progress, + NestedProgress, Progress, +}; +use std::{ + io, + path::PathBuf, + sync::{atomic::AtomicBool, Arc}, }; - -use crate::OutputFormat; pub const PROGRESS_RANGE: std::ops::RangeInclusive = 1..=3; - pub struct Context { pub thread_limit: Option, pub format: OutputFormat, @@ -31,262 +32,141 @@ pub struct Context { pub object_hash: gix::hash::Kind, } -struct CloneDelegate { - ctx: Context, +#[maybe_async::maybe_async] +pub async fn receive( + protocol: Option, + url: &str, directory: Option, refs_directory: Option, - ref_filter: Option<&'static [&'static str]>, - wanted_refs: Vec, -} -static FILTER: &[&str] = &["HEAD", "refs/tags", "refs/heads"]; - -fn remote_supports_ref_in_want(server: &Capabilities) -> bool { - server - .capability("fetch") - .and_then(|cap| cap.supports("ref-in-want")) - .unwrap_or(false) -} - -impl protocol::fetch::DelegateBlocking for CloneDelegate { - fn prepare_ls_refs( - &mut self, - server: &Capabilities, - arguments: &mut Vec, - _features: &mut Vec<(&str, Option>)>, - ) -> io::Result { - if server.contains("ls-refs") { - arguments.extend(FILTER.iter().map(|r| format!("ref-prefix {r}").into())); - } - Ok(if self.wanted_refs.is_empty() { - ls_refs::Action::Continue - } else { - ls_refs::Action::Skip - }) - } - - fn prepare_fetch( - &mut self, - version: transport::Protocol, - server: &Capabilities, - _features: &mut Vec<(&str, Option>)>, - _refs: &[Ref], - ) -> io::Result { - if !self.wanted_refs.is_empty() && !remote_supports_ref_in_want(server) { - return Err(io::Error::new( - io::ErrorKind::Other, - "Want to get specific refs, but remote doesn't support this capability", - )); - } - if version == transport::Protocol::V1 { - self.ref_filter = Some(FILTER); - } - Ok(Action::Continue) - } + mut wanted_refs: Vec, + mut progress: P, + ctx: Context, +) -> anyhow::Result<()> +where + W: std::io::Write, + P: NestedProgress + 'static, + P::SubProgress: 'static, +{ + let mut transport = net::connect( + url, + gix::protocol::transport::client::connect::Options { + version: protocol.unwrap_or_default().into(), + ..Default::default() + }, + ) + .await?; + let trace_packetlines = std::env::var_os( + gix::config::tree::Gitoxide::TRACE_PACKET + .environment_override() + .expect("set"), + ) + .is_some(); - fn negotiate( - &mut self, - refs: &[Ref], - arguments: &mut Arguments, - _previous_response: Option<&Response>, - ) -> io::Result { - if self.wanted_refs.is_empty() { - for r in refs { - let (path, id, _) = r.unpack(); - if let Some(id) = id { - match self.ref_filter { - Some(ref_prefixes) => { - if ref_prefixes.iter().any(|prefix| path.starts_with_str(prefix)) { - arguments.want(id); - } - } - None => arguments.want(id), - } - } - } - } else { - for r in &self.wanted_refs { - arguments.want_ref(r.as_ref()); - } - } - Ok(Action::Cancel) + let agent = gix::protocol::agent(gix::env::agent()); + let mut handshake = gix::protocol::fetch::handshake( + &mut transport, + gix::protocol::credentials::builtin, + vec![("agent".into(), Some(agent.clone()))], + &mut progress, + ) + .await?; + if wanted_refs.is_empty() { + wanted_refs.push("refs/heads/*:refs/remotes/origin/*".into()); } -} - -#[cfg(feature = "blocking-client")] -mod blocking_io { - use std::{io, io::BufRead, path::PathBuf}; - - use gix::{ - bstr::BString, - config::tree::Key, - protocol, - protocol::{fetch::Response, handshake::Ref}, - NestedProgress, - }; - - use super::{receive_pack_blocking, CloneDelegate, Context}; - use crate::net; - - impl protocol::fetch::Delegate for CloneDelegate { - fn receive_pack( - &mut self, - input: impl BufRead, - progress: impl NestedProgress + 'static, - refs: &[Ref], - _previous_response: &Response, - ) -> io::Result<()> { + let fetch_refspecs: Vec<_> = wanted_refs + .into_iter() + .map(|ref_name| { + gix::refspec::parse(ref_name.as_bstr(), gix::refspec::parse::Operation::Fetch).map(|r| r.to_owned()) + }) + .collect::>()?; + let user_agent = ("agent", Some(agent.clone().into())); + let refmap = gix::protocol::fetch::RefMap::new( + &mut progress, + &fetch_refspecs, + gix::protocol::fetch::Context { + handshake: &mut handshake, + transport: &mut transport, + user_agent: user_agent.clone(), + trace_packetlines, + }, + gix::protocol::fetch::refmap::init::Options::default(), + ) + .await?; + let mut negotiate = Negotiate { refmap: &refmap }; + gix::protocol::fetch( + &refmap, + &mut negotiate, + |read_pack, progress, should_interrupt| { receive_pack_blocking( - self.directory.take(), - self.refs_directory.take(), - &mut self.ctx, - input, + directory, + refs_directory, + read_pack, progress, - refs, + &refmap.remote_refs, + should_interrupt, + ctx.out, + ctx.thread_limit, + ctx.object_hash, + ctx.format, ) - } - } - - pub fn receive( - protocol: Option, - url: &str, - directory: Option, - refs_directory: Option, - wanted_refs: Vec, - progress: P, - ctx: Context, - ) -> anyhow::Result<()> - where - W: std::io::Write, - P: NestedProgress + 'static, - P::SubProgress: 'static, - { - let transport = net::connect( - url, - gix::protocol::transport::client::connect::Options { - version: protocol.unwrap_or_default().into(), - ..Default::default() - }, - )?; - let delegate = CloneDelegate { - ctx, - directory, - refs_directory, - ref_filter: None, - wanted_refs, - }; - protocol::fetch( - transport, - delegate, - protocol::credentials::builtin, - progress, - protocol::FetchConnection::TerminateOnSuccessfulCompletion, - gix::env::agent(), - std::env::var_os( - gix::config::tree::Gitoxide::TRACE_PACKET - .environment_override() - .expect("set"), - ) - .is_some(), - )?; - Ok(()) - } + .map(|_| true) + }, + progress, + &ctx.should_interrupt, + gix::protocol::fetch::Context { + handshake: &mut handshake, + transport: &mut transport, + user_agent, + trace_packetlines, + }, + gix::protocol::fetch::Options { + shallow_file: "no shallow file required as we reject it to keep it simple".into(), + shallow: &Default::default(), + tags: Default::default(), + expected_object_hash: Default::default(), + reject_shallow_remote: true, + }, + ) + .await?; + Ok(()) } -#[cfg(feature = "blocking-client")] -pub use blocking_io::receive; -use gix::{protocol::ls_refs, NestedProgress}; - -#[cfg(feature = "async-client")] -mod async_io { - use std::{io, io::BufRead, path::PathBuf}; - - use async_trait::async_trait; - use futures_io::AsyncBufRead; - use gix::{ - bstr::{BString, ByteSlice}, - config::tree::Key, - odb::pack, - protocol, - protocol::{fetch::Response, handshake::Ref}, - Progress, - }; +struct Negotiate<'a> { + refmap: &'a gix::protocol::fetch::RefMap, +} - use super::{print, receive_pack_blocking, write_raw_refs, CloneDelegate, Context}; - use crate::{net, OutputFormat}; +impl gix::protocol::fetch::Negotiate for Negotiate<'_> { + fn mark_complete_and_common_ref(&mut self) -> Result { + Ok(negotiate::Action::MustNegotiate { + remote_ref_target_known: vec![], /* we don't really negotiate */ + }) + } - #[async_trait(?Send)] - impl protocol::fetch::Delegate for CloneDelegate { - async fn receive_pack( - &mut self, - input: impl AsyncBufRead + Unpin + 'async_trait, - progress: impl gix::NestedProgress + 'static, - refs: &[Ref], - _previous_response: &Response, - ) -> io::Result<()> { - receive_pack_blocking( - self.directory.take(), - self.refs_directory.take(), - &mut self.ctx, - futures_lite::io::BlockOn::new(input), - progress, - refs, - ) + fn add_wants(&mut self, arguments: &mut Arguments, _remote_ref_target_known: &[bool]) { + for id in self.refmap.mappings.iter().filter_map(|m| m.remote.as_id()) { + arguments.want(id); } } - pub async fn receive( - protocol: Option, - url: &str, - directory: Option, - refs_directory: Option, - wanted_refs: Vec, - progress: P, - ctx: Context, - ) -> anyhow::Result<()> - where - P: gix::NestedProgress + 'static, - W: io::Write + Send + 'static, - { - let transport = net::connect( - url, - #[allow(clippy::needless_update)] - gix::protocol::transport::client::connect::Options { - version: protocol.unwrap_or_default().into(), - ..Default::default() + fn one_round( + &mut self, + _state: &mut negotiate::one_round::State, + _arguments: &mut Arguments, + _previous_response: Option<&Response>, + ) -> Result<(negotiate::Round, bool), negotiate::Error> { + Ok(( + negotiate::Round { + haves_sent: 0, + in_vain: 0, + haves_to_send: 0, + previous_response_had_at_least_one_in_common: false, }, - ) - .await?; - let mut delegate = CloneDelegate { - ctx, - directory, - refs_directory, - ref_filter: None, - wanted_refs, - }; - blocking::unblock(move || { - futures_lite::future::block_on(protocol::fetch( - transport, - delegate, - protocol::credentials::builtin, - progress, - protocol::FetchConnection::TerminateOnSuccessfulCompletion, - gix::env::agent(), - std::env::var_os( - gix::config::tree::Gitoxide::TRACE_PACKET - .environment_override() - .expect("set"), - ) - .is_some(), - )) - }) - .await?; - Ok(()) + // is done + true, + )) } } -#[cfg(feature = "async-client")] -pub use self::async_io::receive; - #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct JsonBundleWriteOutcome { pub index_version: pack::index::Version, @@ -376,25 +256,30 @@ fn write_raw_refs(refs: &[Ref], directory: PathBuf) -> std::io::Result<()> { Ok(()) } -fn receive_pack_blocking( +#[allow(clippy::too_many_arguments)] +fn receive_pack_blocking( mut directory: Option, mut refs_directory: Option, - ctx: &mut Context, mut input: impl io::BufRead, - mut progress: impl NestedProgress + 'static, + progress: &mut dyn DynNestedProgress, refs: &[Ref], + should_interrupt: &AtomicBool, + mut out: impl std::io::Write, + thread_limit: Option, + object_hash: gix::hash::Kind, + format: OutputFormat, ) -> io::Result<()> { let options = pack::bundle::write::Options { - thread_limit: ctx.thread_limit, + thread_limit, index_version: pack::index::Version::V2, iteration_mode: pack::data::input::Mode::Verify, - object_hash: ctx.object_hash, + object_hash, }; let outcome = pack::Bundle::write_to_directory( &mut input, directory.take().as_deref(), - &mut progress, - &ctx.should_interrupt, + progress, + should_interrupt, None::, options, ) @@ -404,11 +289,11 @@ fn receive_pack_blocking( write_raw_refs(refs, directory)?; } - match ctx.format { - OutputFormat::Human => drop(print(&mut ctx.out, outcome, refs)), + match format { + OutputFormat::Human => drop(print(&mut out, outcome, refs)), #[cfg(feature = "serde")] OutputFormat::Json => { - serde_json::to_writer_pretty(&mut ctx.out, &JsonOutcome::from_outcome_and_refs(outcome, refs))?; + serde_json::to_writer_pretty(&mut out, &JsonOutcome::from_outcome_and_refs(outcome, refs))?; } }; Ok(()) diff --git a/gitoxide-core/src/repository/clone.rs b/gitoxide-core/src/repository/clone.rs index cf810deb389..88daf903e3e 100644 --- a/gitoxide-core/src/repository/clone.rs +++ b/gitoxide-core/src/repository/clone.rs @@ -89,7 +89,7 @@ pub(crate) mod function { if handshake_info { writeln!(out, "Handshake Information")?; - writeln!(out, "\t{:?}", fetch_outcome.ref_map.handshake)?; + writeln!(out, "\t{:?}", fetch_outcome.handshake)?; } match fetch_outcome.status { diff --git a/gitoxide-core/src/repository/fetch.rs b/gitoxide-core/src/repository/fetch.rs index f20525da981..bf1cbcce21d 100644 --- a/gitoxide-core/src/repository/fetch.rs +++ b/gitoxide-core/src/repository/fetch.rs @@ -70,7 +70,7 @@ pub(crate) mod function { if handshake_info { writeln!(out, "Handshake Information")?; - writeln!(out, "\t{:?}", res.ref_map.handshake)?; + writeln!(out, "\t{:?}", res.handshake)?; } let ref_specs = remote.refspecs(gix::remote::Direction::Fetch); @@ -210,7 +210,7 @@ pub(crate) mod function { mut out: impl std::io::Write, mut err: impl std::io::Write, ) -> anyhow::Result<()> { - let mut last_spec_index = gix::remote::fetch::SpecIndex::ExplicitInRemote(usize::MAX); + let mut last_spec_index = gix::remote::fetch::refmap::SpecIndex::ExplicitInRemote(usize::MAX); let mut updates = update_refs .iter_mapping_updates(&map.mappings, refspecs, &map.extra_refspecs) .filter_map(|(update, mapping, spec, edit)| spec.map(|spec| (update, mapping, spec, edit))) @@ -258,10 +258,10 @@ pub(crate) mod function { write!(out, "\t")?; match &mapping.remote { - gix::remote::fetch::Source::ObjectId(id) => { + gix::remote::fetch::refmap::Source::ObjectId(id) => { write!(out, "{}", id.attach(repo).shorten_or_id())?; } - gix::remote::fetch::Source::Ref(r) => { + gix::remote::fetch::refmap::Source::Ref(r) => { crate::repository::remote::refs::print_ref(&mut out, r)?; } }; diff --git a/gitoxide-core/src/repository/remote.rs b/gitoxide-core/src/repository/remote.rs index 01aed6dc291..5a1cb061a34 100644 --- a/gitoxide-core/src/repository/remote.rs +++ b/gitoxide-core/src/repository/remote.rs @@ -4,7 +4,7 @@ mod refs_impl { use gix::{ protocol::handshake, refspec::{match_group::validate::Fix, RefSpec}, - remote::fetch::Source, + remote::fetch::refmap::Source, }; use super::by_name_or_url; @@ -72,7 +72,7 @@ mod refs_impl { .context("Remote didn't have a URL to connect to")? .to_bstring() )); - let map = remote + let (map, handshake) = remote .connect(gix::remote::Direction::Fetch) .await? .ref_map( @@ -86,7 +86,7 @@ mod refs_impl { if handshake_info { writeln!(out, "Handshake Information")?; - writeln!(out, "\t{:?}", map.handshake)?; + writeln!(out, "\t{handshake:?}")?; } match kind { refs::Kind::Tracking { .. } => print_refmap( @@ -119,7 +119,7 @@ mod refs_impl { mut out: impl std::io::Write, mut err: impl std::io::Write, ) -> anyhow::Result<()> { - let mut last_spec_index = gix::remote::fetch::SpecIndex::ExplicitInRemote(usize::MAX); + let mut last_spec_index = gix::remote::fetch::refmap::SpecIndex::ExplicitInRemote(usize::MAX); map.mappings.sort_by_key(|m| m.spec_index); for mapping in &map.mappings { if mapping.spec_index != last_spec_index { @@ -146,11 +146,11 @@ mod refs_impl { write!(out, "\t")?; let target_id = match &mapping.remote { - gix::remote::fetch::Source::ObjectId(id) => { + gix::remote::fetch::refmap::Source::ObjectId(id) => { write!(out, "{id}")?; id } - gix::remote::fetch::Source::Ref(r) => print_ref(&mut out, r)?, + gix::remote::fetch::refmap::Source::Ref(r) => print_ref(&mut out, r)?, }; match &mapping.local { Some(local) => { diff --git a/gix-protocol/Cargo.toml b/gix-protocol/Cargo.toml index b94f0c500d7..fc57b327186 100644 --- a/gix-protocol/Cargo.toml +++ b/gix-protocol/Cargo.toml @@ -23,26 +23,46 @@ doctest = false #! Specifying both causes a compile error, preventing the use of `--all-features`. ## If set, blocking command implementations are available and will use the blocking version of the `gix-transport` crate. -blocking-client = ["gix-transport/blocking-client", "maybe-async/is_sync"] +blocking-client = [ + "gix-transport/blocking-client", + "maybe-async/is_sync", + "handshake", + "fetch" +] ## As above, but provides async implementations instead. async-client = [ "gix-transport/async-client", - "async-trait", - "futures-io", - "futures-lite", + "dep:async-trait", + "dep:futures-io", + "dep:futures-lite", + "handshake", + "fetch" +] + +## Add implementations for performing a `handshake` along with the dependencies needed for it. +handshake = ["dep:gix-credentials"] + +## Add implementations for performing a `fetch` (for packs) along with the dependencies needed for it. +fetch = [ + "dep:gix-negotiate", + "dep:gix-object", + "dep:gix-revwalk", + "dep:gix-lock", + "dep:gix-refspec", + "dep:gix-trace", ] #! ### Other ## Data structures implement `serde::Serialize` and `serde::Deserialize`. -serde = ["dep:serde", "bstr/serde", "gix-transport/serde", "gix-hash/serde"] +serde = ["dep:serde", "bstr/serde", "gix-transport/serde", "gix-hash/serde", "gix-shallow/serde"] [[test]] -name = "blocking-client-protocol" +name = "blocking" path = "tests/blocking-protocol.rs" required-features = ["blocking-client"] [[test]] -name = "async-client-protocol" +name = "async" path = "tests/async-protocol.rs" required-features = ["async-client"] @@ -52,9 +72,18 @@ gix-features = { version = "^0.39.1", path = "../gix-features", features = [ ] } gix-transport = { version = "^0.43.1", path = "../gix-transport" } gix-hash = { version = "^0.15.1", path = "../gix-hash" } +gix-shallow = { version = "^0.1.0", path = "../gix-shallow" } gix-date = { version = "^0.9.2", path = "../gix-date" } -gix-credentials = { version = "^0.25.1", path = "../gix-credentials" } gix-utils = { version = "^0.1.13", path = "../gix-utils" } +gix-ref = { version = "^0.49.0", path = "../gix-ref" } + +gix-trace = { version = "^0.1.11", path = "../gix-trace", optional = true } +gix-negotiate = { version = "^0.17.0", path = "../gix-negotiate", optional = true } +gix-object = { version = "^0.46.0", path = "../gix-object", optional = true } +gix-revwalk = { version = "^0.17.0", path = "../gix-revwalk", optional = true } +gix-credentials = { version = "^0.25.1", path = "../gix-credentials", optional = true } +gix-refspec = { version = "^0.27.0", path = "../gix-refspec", optional = true } +gix-lock = { version = "^15.0.0", path = "../gix-lock", optional = true } thiserror = "2.0.0" serde = { version = "1.0.114", optional = true, default-features = false, features = [ @@ -77,7 +106,6 @@ document-features = { version = "0.2.0", optional = true } [dev-dependencies] async-std = { version = "1.9.0", features = ["attributes"] } gix-packetline = { path = "../gix-packetline", version = "^0.18.1" } -gix-testtools = { path = "../tests/tools" } [package.metadata.docs.rs] features = ["blocking-client", "document-features", "serde"] diff --git a/gix-protocol/src/command/mod.rs b/gix-protocol/src/command.rs similarity index 79% rename from gix-protocol/src/command/mod.rs rename to gix-protocol/src/command.rs index 9fcc48e8720..40f7759f861 100644 --- a/gix-protocol/src/command/mod.rs +++ b/gix-protocol/src/command.rs @@ -90,9 +90,10 @@ mod with_io { } } - /// Compute initial arguments based on the given `features`. They are typically provided by the `default_features(…)` method. - /// Only useful for V2 - pub(crate) fn initial_arguments(&self, features: &[Feature]) -> Vec { + /// Provide the initial arguments based on the given `features`. + /// They are typically provided by the [`Self::default_features`] method. + /// Only useful for V2, and based on heuristics/experimentation. + pub fn initial_v2_arguments(&self, features: &[Feature]) -> Vec { match self { Command::Fetch => ["thin-pack", "ofs-delta"] .iter() @@ -157,20 +158,24 @@ mod with_io { Command::LsRefs => vec![], } } - /// Panics if the given arguments and features don't match what's statically known. It's considered a bug in the delegate. - pub(crate) fn validate_argument_prefixes_or_panic( + /// Return an error if the given `arguments` and `features` don't match what's statically known. + pub fn validate_argument_prefixes( &self, version: gix_transport::Protocol, server: &Capabilities, arguments: &[BString], features: &[Feature], - ) { + ) -> Result<(), validate_argument_prefixes::Error> { + use validate_argument_prefixes::Error; let allowed = self.all_argument_prefixes(); for arg in arguments { if allowed.iter().any(|allowed| arg.starts_with(allowed.as_bytes())) { continue; } - panic!("{}: argument {} is not known or allowed", self.as_str(), arg); + return Err(Error::UnsupportedArgument { + command: self.as_str(), + argument: arg.clone(), + }); } match version { gix_transport::Protocol::V0 | gix_transport::Protocol::V1 => { @@ -181,14 +186,17 @@ mod with_io { { continue; } - panic!("{}: capability {} is not supported", self.as_str(), feature); + return Err(Error::UnsupportedCapability { + command: self.as_str(), + feature: feature.to_string(), + }); } } gix_transport::Protocol::V2 => { let allowed = server .iter() .find_map(|c| { - if c.name() == self.as_str().as_bytes().as_bstr() { + if c.name() == self.as_str() { c.values().map(|v| v.map(ToString::to_string).collect::>()) } else { None @@ -201,14 +209,34 @@ mod with_io { } match *feature { "agent" => {} - _ => panic!("{}: V2 feature/capability {} is not supported", self.as_str(), feature), + _ => { + return Err(Error::UnsupportedCapability { + command: self.as_str(), + feature: feature.to_string(), + }) + } } } } } + Ok(()) } } -} -#[cfg(test)] -mod tests; + /// + pub mod validate_argument_prefixes { + use bstr::BString; + + /// The error returned by [Command::validate_argument_prefixes()](super::Command::validate_argument_prefixes()). + #[derive(Debug, thiserror::Error)] + #[allow(missing_docs)] + pub enum Error { + #[error("{command}: argument {argument} is not known or allowed")] + UnsupportedArgument { command: &'static str, argument: BString }, + #[error("{command}: capability {feature} is not supported")] + UnsupportedCapability { command: &'static str, feature: String }, + } + } +} +#[cfg(any(test, feature = "async-client", feature = "blocking-client"))] +pub use with_io::validate_argument_prefixes; diff --git a/gix-protocol/src/fetch/arguments/mod.rs b/gix-protocol/src/fetch/arguments/mod.rs index ee1f4dc2870..b25f8fb9f03 100644 --- a/gix-protocol/src/fetch/arguments/mod.rs +++ b/gix-protocol/src/fetch/arguments/mod.rs @@ -24,6 +24,7 @@ pub struct Arguments { #[cfg(any(feature = "async-client", feature = "blocking-client"))] version: gix_transport::Protocol, + #[cfg(any(feature = "async-client", feature = "blocking-client"))] trace: bool, } @@ -164,6 +165,7 @@ impl Arguments { /// Permanently allow the server to include tags that point to commits or objects it would return. /// /// Needs to only be called once. + #[cfg(any(feature = "async-client", feature = "blocking-client"))] pub fn use_include_tag(&mut self) { debug_assert!(self.supports_include_tag, "'include-tag' feature required"); if self.supports_include_tag { @@ -176,6 +178,7 @@ impl Arguments { /// Note that sending an unknown or unsupported feature may cause the remote to terminate /// the connection. Use this method if you know what you are doing *and* there is no specialized /// method for this, e.g. [`Self::use_include_tag()`]. + #[cfg(any(feature = "async-client", feature = "blocking-client"))] pub fn add_feature(&mut self, feature: &str) { match self.version { gix_transport::Protocol::V0 | gix_transport::Protocol::V1 => { @@ -228,7 +231,7 @@ impl Arguments { } gix_transport::Protocol::V2 => { supports_include_tag = true; - (Command::Fetch.initial_arguments(&features), None) + (Command::Fetch.initial_v2_arguments(&features), None) } }; diff --git a/gix-protocol/src/fetch/delegate.rs b/gix-protocol/src/fetch/delegate.rs deleted file mode 100644 index e6f18740287..00000000000 --- a/gix-protocol/src/fetch/delegate.rs +++ /dev/null @@ -1,313 +0,0 @@ -use std::{ - borrow::Cow, - io, - ops::{Deref, DerefMut}, -}; - -use bstr::BString; -use gix_transport::client::Capabilities; - -use crate::{ - fetch::{Arguments, Response}, - handshake::Ref, -}; - -/// Defines what to do next after certain [`Delegate`] operations. -#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] -pub enum Action { - /// Continue the typical flow of operations in this flow. - Continue, - /// Return at the next possible opportunity without making further requests, possibly after closing the connection. - Cancel, -} - -/// The non-IO protocol delegate is the bare minimal interface needed to fully control the [`fetch`][crate::fetch()] operation, sparing -/// the IO parts. -/// Async implementations must treat it as blocking and unblock it by evaluating it elsewhere. -/// -/// See [Delegate] for the complete trait. -pub trait DelegateBlocking { - /// Return extra parameters to be provided during the handshake. - /// - /// Note that this method is only called once and the result is reused during subsequent handshakes which may happen - /// if there is an authentication failure. - fn handshake_extra_parameters(&self) -> Vec<(String, Option)> { - Vec::new() - } - /// Called before invoking 'ls-refs' on the server to allow providing it with additional `arguments` and to enable `features`. - /// If the server `capabilities` don't match the requirements abort with an error to abort the entire fetch operation. - /// - /// Note that some arguments are preset based on typical use, and `features` are preset to maximize options. - /// The `server` capabilities can be used to see which additional capabilities the server supports as per the handshake which happened prior. - /// - /// If the delegate returns [`ls_refs::Action::Skip`], no `ls-refs` command is sent to the server. - /// - /// Note that this is called only if we are using protocol version 2. - fn prepare_ls_refs( - &mut self, - _server: &Capabilities, - _arguments: &mut Vec, - _features: &mut Vec<(&str, Option>)>, - ) -> std::io::Result { - Ok(ls_refs::Action::Continue) - } - - /// Called before invoking the 'fetch' interaction with `features` pre-filled for typical use - /// and to maximize capabilities to allow aborting an interaction early. - /// - /// `refs` is a list of known references on the remote based on the handshake or a prior call to `ls_refs`. - /// These can be used to abort early in case the refs are already known here. - /// - /// As there will be another call allowing to post arguments conveniently in the correct format, i.e. `want hex-oid`, - /// there is no way to set arguments at this time. - /// - /// `version` is the actually supported version as reported by the server, which is relevant in case the server requested a downgrade. - /// `server` capabilities is a list of features the server supports for your information, along with enabled `features` that the server knows about. - fn prepare_fetch( - &mut self, - _version: gix_transport::Protocol, - _server: &Capabilities, - _features: &mut Vec<(&str, Option>)>, - _refs: &[Ref], - ) -> std::io::Result { - Ok(Action::Continue) - } - - /// A method called repeatedly to negotiate the objects to receive in [`receive_pack(…)`][Delegate::receive_pack()]. - /// - /// The first call has `previous_response` set to `None` as there was no previous response. Every call that follows `previous_response` - /// will be set to `Some`. - /// - /// ### If `previous_response` is `None`… - /// - /// Given a list of `arguments` to populate with wants, want-refs, shallows, filters and other contextual information to be - /// sent to the server. This method is called once. - /// Send the objects you `have` have afterwards based on the tips of your refs, in preparation to walk down their parents - /// with each call to `negotiate` to find the common base(s). - /// - /// Note that you should not `want` and object that you already have. - /// `refs` are the tips of on the server side, effectively the latest objects _they_ have. - /// - /// Return `Action::Close` if you know that there are no `haves` on your end to allow the server to send all of its objects - /// as is the case during initial clones. - /// - /// ### If `previous_response` is `Some`… - /// - /// Populate `arguments` with the objects you `have` starting from the tips of _your_ refs, taking into consideration - /// the `previous_response` response of the server to see which objects they acknowledged to have. You have to maintain - /// enough state to be able to walk down from your tips on each call, if they are not in common, and keep setting `have` - /// for those which are in common if that helps teaching the server about our state and to acknowledge their existence on _their_ end. - /// This method is called until the other side signals they are ready to send a pack. - /// Return `Action::Close` if you want to give up before finding a common base. This can happen if the remote repository - /// has radically changed so there are no bases, or they are very far in the past, causing all objects to be sent. - fn negotiate( - &mut self, - refs: &[Ref], - arguments: &mut Arguments, - previous_response: Option<&Response>, - ) -> io::Result; -} - -impl DelegateBlocking for Box { - fn handshake_extra_parameters(&self) -> Vec<(String, Option)> { - self.deref().handshake_extra_parameters() - } - - fn prepare_ls_refs( - &mut self, - _server: &Capabilities, - _arguments: &mut Vec, - _features: &mut Vec<(&str, Option>)>, - ) -> io::Result { - self.deref_mut().prepare_ls_refs(_server, _arguments, _features) - } - - fn prepare_fetch( - &mut self, - _version: gix_transport::Protocol, - _server: &Capabilities, - _features: &mut Vec<(&str, Option>)>, - _refs: &[Ref], - ) -> io::Result { - self.deref_mut().prepare_fetch(_version, _server, _features, _refs) - } - - fn negotiate( - &mut self, - refs: &[Ref], - arguments: &mut Arguments, - previous_response: Option<&Response>, - ) -> io::Result { - self.deref_mut().negotiate(refs, arguments, previous_response) - } -} - -impl DelegateBlocking for &mut T { - fn handshake_extra_parameters(&self) -> Vec<(String, Option)> { - self.deref().handshake_extra_parameters() - } - - fn prepare_ls_refs( - &mut self, - _server: &Capabilities, - _arguments: &mut Vec, - _features: &mut Vec<(&str, Option>)>, - ) -> io::Result { - self.deref_mut().prepare_ls_refs(_server, _arguments, _features) - } - - fn prepare_fetch( - &mut self, - _version: gix_transport::Protocol, - _server: &Capabilities, - _features: &mut Vec<(&str, Option>)>, - _refs: &[Ref], - ) -> io::Result { - self.deref_mut().prepare_fetch(_version, _server, _features, _refs) - } - - fn negotiate( - &mut self, - refs: &[Ref], - arguments: &mut Arguments, - previous_response: Option<&Response>, - ) -> io::Result { - self.deref_mut().negotiate(refs, arguments, previous_response) - } -} - -#[cfg(feature = "blocking-client")] -mod blocking_io { - use std::{ - io::{self, BufRead}, - ops::DerefMut, - }; - - use gix_features::progress::NestedProgress; - - use crate::{ - fetch::{DelegateBlocking, Response}, - handshake::Ref, - }; - - /// The protocol delegate is the bare minimal interface needed to fully control the [`fetch`][crate::fetch()] operation. - /// - /// Implementations of this trait are controlled by code with intricate knowledge about how fetching works in protocol version V1 and V2, - /// so you don't have to. - /// Everything is tucked away behind type-safety so 'nothing can go wrong'©. Runtime assertions assure invalid - /// features or arguments don't make it to the server in the first place. - /// Please note that this trait mostly corresponds to what V2 would look like, even though V1 is supported as well. - pub trait Delegate: DelegateBlocking { - /// Receive a pack provided from the given `input`. - /// - /// Use `progress` to emit your own progress messages when decoding the pack. - /// - /// `refs` of the remote side are provided for convenience, along with the parsed `previous_response` response in case you want - /// to check additional acks. - fn receive_pack( - &mut self, - input: impl io::BufRead, - progress: impl NestedProgress + 'static, - refs: &[Ref], - previous_response: &Response, - ) -> io::Result<()>; - } - - impl Delegate for Box { - fn receive_pack( - &mut self, - input: impl BufRead, - progress: impl NestedProgress + 'static, - refs: &[Ref], - previous_response: &Response, - ) -> io::Result<()> { - self.deref_mut().receive_pack(input, progress, refs, previous_response) - } - } - - impl Delegate for &mut T { - fn receive_pack( - &mut self, - input: impl BufRead, - progress: impl NestedProgress + 'static, - refs: &[Ref], - previous_response: &Response, - ) -> io::Result<()> { - self.deref_mut().receive_pack(input, progress, refs, previous_response) - } - } -} -#[cfg(feature = "blocking-client")] -pub use blocking_io::Delegate; - -#[cfg(feature = "async-client")] -mod async_io { - use std::{io, ops::DerefMut}; - - use async_trait::async_trait; - use futures_io::AsyncBufRead; - use gix_features::progress::NestedProgress; - - use crate::{ - fetch::{DelegateBlocking, Response}, - handshake::Ref, - }; - - /// The protocol delegate is the bare minimal interface needed to fully control the [`fetch`][crate::fetch()] operation. - /// - /// Implementations of this trait are controlled by code with intricate knowledge about how fetching works in protocol version V1 and V2, - /// so you don't have to. - /// Everything is tucked away behind type-safety so 'nothing can go wrong'©. Runtime assertions assure invalid - /// features or arguments don't make it to the server in the first place. - /// Please note that this trait mostly corresponds to what V2 would look like, even though V1 is supported as well. - #[async_trait(?Send)] - pub trait Delegate: DelegateBlocking { - /// Receive a pack provided from the given `input`, and the caller should consider it to be blocking as - /// most operations on the received pack are implemented in a blocking fashion. - /// - /// Use `progress` to emit your own progress messages when decoding the pack. - /// - /// `refs` of the remote side are provided for convenience, along with the parsed `previous_response` response in case you want - /// to check additional acks. - async fn receive_pack( - &mut self, - input: impl AsyncBufRead + Unpin + 'async_trait, - progress: impl NestedProgress + 'static, - refs: &[Ref], - previous_response: &Response, - ) -> io::Result<()>; - } - #[async_trait(?Send)] - impl Delegate for Box { - async fn receive_pack( - &mut self, - input: impl AsyncBufRead + Unpin + 'async_trait, - progress: impl NestedProgress + 'static, - refs: &[Ref], - previous_response: &Response, - ) -> io::Result<()> { - self.deref_mut() - .receive_pack(input, progress, refs, previous_response) - .await - } - } - - #[async_trait(?Send)] - impl Delegate for &mut T { - async fn receive_pack( - &mut self, - input: impl AsyncBufRead + Unpin + 'async_trait, - progress: impl NestedProgress + 'static, - refs: &[Ref], - previous_response: &Response, - ) -> io::Result<()> { - self.deref_mut() - .receive_pack(input, progress, refs, previous_response) - .await - } - } -} -#[cfg(feature = "async-client")] -pub use async_io::Delegate; - -use crate::ls_refs; diff --git a/gix-protocol/src/fetch/error.rs b/gix-protocol/src/fetch/error.rs index 5646ce4ecd4..6eab014b9c9 100644 --- a/gix-protocol/src/fetch/error.rs +++ b/gix-protocol/src/fetch/error.rs @@ -1,21 +1,48 @@ -use std::io; - -use gix_transport::client; - -use crate::{fetch::response, handshake, ls_refs}; - -/// The error used in [`fetch()`][crate::fetch()]. +/// The error returned by [`fetch()`](crate::fetch()). #[derive(Debug, thiserror::Error)] #[allow(missing_docs)] pub enum Error { + #[error("Could not decode server reply")] + FetchResponse(#[from] crate::fetch::response::Error), + #[error("Cannot fetch from a remote that uses {remote} while local repository uses {local} for object hashes")] + IncompatibleObjectHash { + local: gix_hash::Kind, + remote: gix_hash::Kind, + }, #[error(transparent)] - Handshake(#[from] handshake::Error), - #[error("Could not access repository or failed to read streaming pack file")] - Io(#[from] io::Error), - #[error(transparent)] - Transport(#[from] client::Error), + Negotiate(#[from] crate::fetch::negotiate::Error), #[error(transparent)] - LsRefs(#[from] ls_refs::Error), - #[error(transparent)] - Response(#[from] response::Error), + Client(#[from] crate::transport::client::Error), + #[error("Server lack feature {feature:?}: {description}")] + MissingServerFeature { + feature: &'static str, + description: &'static str, + }, + #[error("Could not write 'shallow' file to incorporate remote updates after fetching")] + WriteShallowFile(#[from] gix_shallow::write::Error), + #[error("Could not read 'shallow' file to send current shallow boundary")] + ReadShallowFile(#[from] gix_shallow::read::Error), + #[error("'shallow' file could not be locked in preparation for writing changes")] + LockShallowFile(#[from] gix_lock::acquire::Error), + #[error("Receiving objects from shallow remotes is prohibited due to the value of `clone.rejectShallow`")] + RejectShallowRemote, + #[error("None of the refspec(s) {} matched any of the {num_remote_refs} refs on the remote", refspecs.iter().map(|r| r.to_ref().instruction().to_bstring().to_string()).collect::>().join(", "))] + NoMapping { + refspecs: Vec, + num_remote_refs: usize, + }, + #[error("Failed to consume the pack sent by the remove")] + ConsumePack(Box), + #[error("Failed to read remaining bytes in stream")] + ReadRemainingBytes(#[source] std::io::Error), +} + +impl crate::transport::IsSpuriousError for Error { + fn is_spurious(&self) -> bool { + match self { + Error::FetchResponse(err) => err.is_spurious(), + Error::Client(err) => err.is_spurious(), + _ => false, + } + } } diff --git a/gix-protocol/src/fetch/function.rs b/gix-protocol/src/fetch/function.rs new file mode 100644 index 00000000000..e215ffd038c --- /dev/null +++ b/gix-protocol/src/fetch/function.rs @@ -0,0 +1,276 @@ +use crate::fetch::{ + negotiate, Context, Error, Negotiate, NegotiateOutcome, Options, Outcome, ProgressId, RefMap, Shallow, Tags, +}; +use crate::{fetch::Arguments, transport::packetline::read::ProgressAction}; +use gix_features::progress::DynNestedProgress; +use std::path::Path; +use std::sync::atomic::{AtomicBool, Ordering}; + +/// Perform one fetch operation, relying on a `transport`, right after a [`ref_map`](RefMap::new()) was created so +/// it's clear what the remote has. +/// `negotiate` is used to run the negotiation of objects that should be contained in the pack, *if* one is to be received. +/// `progress` and `should_interrupt` is passed to all potentially long-running parts of the operation. +/// +/// `consume_pack(pack_read, progress, interrupt) -> bool` is always called to consume all bytes that are sent by the server, returning `true` if we should assure the pack is read to the end, +/// or `false` to do nothing. Dropping the reader without reading to EOF (i.e. returning `false`) is an offense to the server, and +/// `transport` won't be in the correct state to perform additional operations, or indicate the end of operation. +/// Note that the passed reader blocking as the pack-writing is blocking as well. +/// +/// The `Context` and `Options` further define parts of this `fetch` operation. +/// +/// As opposed to a full `git fetch`, this operation does *not*… +/// +/// * …update local refs +/// * …end the interaction after the fetch +/// +/// Note that the interaction will never be ended, even on error or failure, leaving it up to the caller to do that, maybe +/// with the help of [`SendFlushOnDrop`](crate::SendFlushOnDrop) which can wrap `transport`. +/// Generally, the `transport` is left in a state that allows for more commands to be run. +/// +/// Return `Ok(None)` if there was nothing to do because all remote refs are at the same state as they are locally, or `Ok(Some(outcome))` +/// to inform about all the changes that were made. +#[maybe_async::maybe_async] +pub async fn fetch( + ref_map: &RefMap, + negotiate: &mut impl Negotiate, + consume_pack: impl FnOnce(&mut dyn std::io::BufRead, &mut dyn DynNestedProgress, &AtomicBool) -> Result, + mut progress: P, + should_interrupt: &AtomicBool, + Context { + handshake, + transport, + user_agent, + trace_packetlines, + }: Context<'_, T>, + Options { + shallow_file, + shallow, + tags, + expected_object_hash, + reject_shallow_remote, + }: Options<'_>, +) -> Result, Error> +where + P: gix_features::progress::NestedProgress, + P::SubProgress: 'static, + T: gix_transport::client::Transport, + E: Into>, +{ + let _span = gix_trace::coarse!("gix_protocol::fetch()"); + + if ref_map.mappings.is_empty() && !ref_map.remote_refs.is_empty() { + let mut specs = ref_map.refspecs.clone(); + specs.extend(ref_map.extra_refspecs.clone()); + return Err(Error::NoMapping { + refspecs: specs, + num_remote_refs: ref_map.remote_refs.len(), + }); + } + + let v1_shallow_updates = handshake.v1_shallow_updates.take(); + let protocol_version = handshake.server_protocol_version; + + let fetch = crate::Command::Fetch; + let fetch_features = { + let mut f = fetch.default_features(protocol_version, &handshake.capabilities); + f.push(user_agent); + f + }; + + crate::fetch::Response::check_required_features(protocol_version, &fetch_features)?; + let sideband_all = fetch_features.iter().any(|(n, _)| *n == "sideband-all"); + let mut arguments = Arguments::new(protocol_version, fetch_features, trace_packetlines); + if matches!(tags, Tags::Included) { + if !arguments.can_use_include_tag() { + return Err(Error::MissingServerFeature { + feature: "include-tag", + description: + // NOTE: if this is an issue, we could probably do what's proposed here. + "To make this work we would have to implement another pass to fetch attached tags separately", + }); + } + arguments.use_include_tag(); + } + let (shallow_commits, mut shallow_lock) = add_shallow_args(&mut arguments, shallow, &shallow_file)?; + + if ref_map.object_hash != expected_object_hash { + return Err(Error::IncompatibleObjectHash { + local: expected_object_hash, + remote: ref_map.object_hash, + }); + } + + let negotiate_span = gix_trace::detail!( + "negotiate", + protocol_version = handshake.server_protocol_version as usize + ); + let action = negotiate.mark_complete_and_common_ref()?; + let mut previous_response = None::; + match &action { + negotiate::Action::NoChange | negotiate::Action::SkipToRefUpdate => Ok(None), + negotiate::Action::MustNegotiate { + remote_ref_target_known, + } => { + negotiate.add_wants(&mut arguments, remote_ref_target_known); + let mut rounds = Vec::new(); + let is_stateless = arguments.is_stateless(!transport.connection_persists_across_multiple_requests()); + let mut state = negotiate::one_round::State::new(is_stateless); + let mut reader = 'negotiation: loop { + let _round = gix_trace::detail!("negotiate round", round = rounds.len() + 1); + progress.step(); + progress.set_name(format!("negotiate (round {})", rounds.len() + 1)); + + let is_done = match negotiate.one_round(&mut state, &mut arguments, previous_response.as_ref()) { + Ok((round, is_done)) => { + rounds.push(round); + is_done + } + Err(err) => { + return Err(err.into()); + } + }; + let mut reader = arguments.send(transport, is_done).await?; + if sideband_all { + setup_remote_progress(&mut progress, &mut reader, should_interrupt); + } + let response = + crate::fetch::Response::from_line_reader(protocol_version, &mut reader, is_done, !is_done).await?; + let has_pack = response.has_pack(); + previous_response = Some(response); + if has_pack { + progress.step(); + progress.set_name("receiving pack".into()); + if !sideband_all { + setup_remote_progress(&mut progress, &mut reader, should_interrupt); + } + break 'negotiation reader; + } + }; + drop(negotiate_span); + + let mut previous_response = previous_response.expect("knowledge of a pack means a response was received"); + previous_response.append_v1_shallow_updates(v1_shallow_updates); + if !previous_response.shallow_updates().is_empty() && shallow_lock.is_none() { + if reject_shallow_remote { + return Err(Error::RejectShallowRemote); + } + shallow_lock = acquire_shallow_lock(&shallow_file).map(Some)?; + } + + #[cfg(feature = "async-client")] + let mut rd = crate::futures_lite::io::BlockOn::new(reader); + #[cfg(not(feature = "async-client"))] + let mut rd = reader; + let may_read_to_end = + consume_pack(&mut rd, &mut progress, should_interrupt).map_err(|err| Error::ConsumePack(err.into()))?; + #[cfg(feature = "async-client")] + { + reader = rd.into_inner(); + } + #[cfg(not(feature = "async-client"))] + { + reader = rd; + } + + if may_read_to_end { + // Assure the final flush packet is consumed. + let has_read_to_end = reader.stopped_at().is_some(); + #[cfg(feature = "async-client")] + { + if !has_read_to_end { + futures_lite::io::copy(&mut reader, &mut futures_lite::io::sink()) + .await + .map_err(Error::ReadRemainingBytes)?; + } + } + #[cfg(not(feature = "async-client"))] + { + if !has_read_to_end { + std::io::copy(&mut reader, &mut std::io::sink()).map_err(Error::ReadRemainingBytes)?; + } + } + } + drop(reader); + + if let Some(shallow_lock) = shallow_lock { + if !previous_response.shallow_updates().is_empty() { + gix_shallow::write(shallow_lock, shallow_commits, previous_response.shallow_updates())?; + } + } + Ok(Some(Outcome { + last_response: previous_response, + negotiate: NegotiateOutcome { action, rounds }, + })) + } + } +} + +fn acquire_shallow_lock(shallow_file: &Path) -> Result { + gix_lock::File::acquire_to_update_resource(shallow_file, gix_lock::acquire::Fail::Immediately, None) + .map_err(Into::into) +} + +fn add_shallow_args( + args: &mut Arguments, + shallow: &Shallow, + shallow_file: &std::path::Path, +) -> Result<(Option>, Option), Error> { + let expect_change = *shallow != Shallow::NoChange; + let shallow_lock = expect_change.then(|| acquire_shallow_lock(shallow_file)).transpose()?; + + let shallow_commits = gix_shallow::read(shallow_file)?; + if (shallow_commits.is_some() || expect_change) && !args.can_use_shallow() { + // NOTE: if this is an issue, we can always unshallow the repo ourselves. + return Err(Error::MissingServerFeature { + feature: "shallow", + description: "shallow clones need server support to remain shallow, otherwise bigger than expected packs are sent effectively unshallowing the repository", + }); + } + if let Some(shallow_commits) = &shallow_commits { + for commit in shallow_commits.iter() { + args.shallow(commit); + } + } + match shallow { + Shallow::NoChange => {} + Shallow::DepthAtRemote(commits) => args.deepen(commits.get() as usize), + Shallow::Deepen(commits) => { + args.deepen(*commits as usize); + args.deepen_relative(); + } + Shallow::Since { cutoff } => { + args.deepen_since(cutoff.seconds); + } + Shallow::Exclude { + remote_refs, + since_cutoff, + } => { + if let Some(cutoff) = since_cutoff { + args.deepen_since(cutoff.seconds); + } + for ref_ in remote_refs { + args.deepen_not(ref_.as_ref().as_bstr()); + } + } + } + Ok((shallow_commits, shallow_lock)) +} + +fn setup_remote_progress<'a>( + progress: &mut dyn gix_features::progress::DynNestedProgress, + reader: &mut Box + Unpin + 'a>, + should_interrupt: &'a AtomicBool, +) { + use crate::transport::client::ExtendedBufRead; + reader.set_progress_handler(Some(Box::new({ + let mut remote_progress = progress.add_child_with_id("remote".to_string(), ProgressId::RemoteProgress.into()); + move |is_err: bool, data: &[u8]| { + crate::RemoteProgress::translate_to_progress(is_err, data, &mut remote_progress); + if should_interrupt.load(Ordering::Relaxed) { + ProgressAction::Interrupt + } else { + ProgressAction::Continue + } + } + }) as crate::transport::client::HandleProgress<'a>)); +} diff --git a/gix-protocol/src/fetch/mod.rs b/gix-protocol/src/fetch/mod.rs index 0828ea733a2..99f8df15162 100644 --- a/gix-protocol/src/fetch/mod.rs +++ b/gix-protocol/src/fetch/mod.rs @@ -1,20 +1,52 @@ +/// A module providing low-level primitives to flexibly perform various `fetch` related activities. Note that the typesystem isn't used +/// to assure they are always performed in the right order, the caller has to follow some parts of the protocol itself. +/// +/// ### Order for receiving a pack +/// +/// * [handshake](handshake()) +/// * **ls-refs** +/// * [get available refs by refspecs](RefMap::new()) +/// * **fetch pack** +/// * `negotiate` until a pack can be received (TBD) +/// * [officially terminate the connection](crate::indicate_end_of_interaction()) +/// - Consider wrapping the transport in [`SendFlushOnDrop`](crate::SendFlushOnDrop) to be sure the connection is terminated +/// gracefully even if there is an application error. +/// +/// Note that this flow doesn't involve actually writing the pack, or indexing it. Nor does it contain machinery +/// to write or update references based on the fetched remote references. +/// +/// Also, when the server supports [version 2](crate::transport::Protocol::V2) of the protocol, then each of the listed commands, +/// `ls-refs` and `fetch` can be invoked multiple times in any order. +// Note: for ease of use, this is tested in `gix` itself. The test-suite here uses a legacy implementation. mod arguments; pub use arguments::Arguments; -/// -pub mod delegate; -#[cfg(any(feature = "async-client", feature = "blocking-client"))] -pub use delegate::Delegate; -pub use delegate::{Action, DelegateBlocking}; - +#[cfg(any(feature = "blocking-client", feature = "async-client"))] +#[cfg(feature = "fetch")] mod error; +#[cfg(any(feature = "blocking-client", feature = "async-client"))] +#[cfg(feature = "fetch")] pub use error::Error; /// pub mod response; -pub use response::Response; +#[cfg(any(feature = "blocking-client", feature = "async-client"))] +#[cfg(feature = "fetch")] +pub(crate) mod function; + +#[cfg(any(feature = "blocking-client", feature = "async-client"))] +#[cfg(feature = "handshake")] mod handshake; +#[cfg(any(feature = "blocking-client", feature = "async-client"))] +#[cfg(feature = "handshake")] pub use handshake::upload_pack as handshake; -#[cfg(test)] -mod tests; +#[cfg(feature = "fetch")] +pub mod negotiate; + +/// +#[cfg(feature = "fetch")] +pub mod refmap; + +mod types; +pub use types::*; diff --git a/gix/src/remote/connection/fetch/negotiate.rs b/gix-protocol/src/fetch/negotiate.rs similarity index 55% rename from gix/src/remote/connection/fetch/negotiate.rs rename to gix-protocol/src/fetch/negotiate.rs index b8c65859c9a..9dd4f3222e5 100644 --- a/gix/src/remote/connection/fetch/negotiate.rs +++ b/gix-protocol/src/fetch/negotiate.rs @@ -1,15 +1,20 @@ -use std::borrow::Cow; - +//! A modules with primitives to perform negotiation as part of a fetch operation. +//! +//! The functions provided are called in a certain order: +//! +//! 1. [`mark_complete_and_common_ref()`] - initialize the [`negotiator`](gix_negotiate::Negotiator) with all state known on the remote. +//! 2. [`add_wants()`] is called if the call at 1) returned [`Action::MustNegotiate`]. +//! 3. [`one_round()`] is called for each negotiation round, providing information if the negotiation is done. use gix_date::SecondsSinceUnixEpoch; use gix_negotiate::Flags; -use gix_odb::HeaderExt; -use gix_pack::Find; +use gix_ref::file::ReferenceExt; +use std::borrow::Cow; -use crate::remote::{fetch, fetch::Shallow}; +use crate::fetch::{refmap, RefMap, Shallow, Tags}; type Queue = gix_revwalk::PriorityQueue; -/// The error returned during negotiation. +/// The error returned during [`one_round()`] or [`mark_complete_and_common_ref()`]. #[derive(Debug, thiserror::Error)] #[allow(missing_docs)] pub enum Error { @@ -18,17 +23,21 @@ pub enum Error { #[error(transparent)] LookupCommitInGraph(#[from] gix_revwalk::graph::get_or_insert_default::Error), #[error(transparent)] - InitRefsIterator(#[from] crate::reference::iter::init::Error), + OpenPackedRefsBuffer(#[from] gix_ref::packed::buffer::open::Error), #[error(transparent)] - InitRefsIteratorPlatform(#[from] crate::reference::iter::Error), + IO(#[from] std::io::Error), #[error(transparent)] - ObtainRefDuringIteration(#[from] Box), + InitRefIter(#[from] gix_ref::file::iter::loose_then_packed::Error), #[error(transparent)] - LoadIndex(#[from] gix_odb::store::load_index::Error), + PeelToId(#[from] gix_ref::peel::to_id::Error), + #[error(transparent)] + AlternateRefsAndObjects(Box), } +/// Determines what should be done after [preparing the commit-graph for negotiation](mark_complete_and_common_ref). #[must_use] -pub(crate) enum Action { +#[derive(Debug, Clone)] +pub enum Action { /// None of the remote refs moved compared to our last recorded state (via tracking refs), so there is nothing to do at all, /// not even a ref update. NoChange, @@ -43,7 +52,32 @@ pub(crate) enum Action { }, } -/// This function is modeled after the similarly named one in the git codebase to do the following: +/// Key information about each round in the pack-negotiation, as produced by [`one_round()`]. +#[derive(Debug, Clone, Copy)] +pub struct Round { + /// The amount of `HAVE` lines sent this round. + /// + /// Each `HAVE` is an object that we tell the server about which would acknowledge each one it has as well. + pub haves_sent: usize, + /// A total counter, over all previous rounds, indicating how many `HAVE`s we sent without seeing a single acknowledgement, + /// i.e. the indication of a common object. + /// + /// This number maybe zero or be lower compared to the previous round if we have received at least one acknowledgement. + pub in_vain: usize, + /// The amount of haves we should send in this round. + /// + /// If the value is lower than `haves_sent` (the `HAVE` lines actually sent), the negotiation algorithm has run out of options + /// which typically indicates the end of the negotiation phase. + pub haves_to_send: usize, + /// If `true`, the server reported, as response to our previous `HAVE`s, that at least one of them is in common by acknowledging it. + /// + /// This may also lead to the server responding with a pack. + pub previous_response_had_at_least_one_in_common: bool, +} + +/// This function is modeled after the similarly named one in the git codebase to mark known refs in a commit-graph. +/// +/// It to do the following: /// /// * figure out all advertised refs on the remote *that we already have* and keep track of the oldest one as cutoff date. /// * mark all of our own refs as tips for a traversal. @@ -62,25 +96,53 @@ pub(crate) enum Action { /// our own walk. /// /// Return whether we should negotiate, along with a queue for later use. -pub(crate) fn mark_complete_and_common_ref( - repo: &crate::Repository, +/// +/// # Parameters +/// +/// * `objects` +/// - Access to the object database. *Note* that the `exists()` calls must not trigger a refresh of the ODB packs as plenty of them might fail, i.e. find on object. +/// * `refs` +/// - Access to the git references database. +/// * `alternates` +/// - A function that returns an iterator over `(refs, objects)` for each alternate repository, to assure all known objects are added also according to their tips. +/// * `negotiator` +/// - The implementation that performs the negotiation later, i.e. prepare wants and haves. +/// * `graph` +/// - The commit-graph for use by the `negotiator` - we populate it with tips to initialize the graph traversal. +/// * `ref_map` +/// - The references known on the remote, as previously obtained with [`RefMap::new()`]. +/// * `shallow` +/// - How to deal with shallow repositories. It does affect how negotiations are performed. +/// * `mapping_is_ignored` +/// - `f(mapping) -> bool` returns `true` if the given mapping should not participate in change tracking. +/// - [`make_refmapping_ignore_predicate()`] is a typical implementation for this. +#[allow(clippy::too_many_arguments)] +pub fn mark_complete_and_common_ref( + objects: &(impl gix_object::Find + gix_object::FindHeader + gix_object::Exists), + refs: &gix_ref::file::Store, + alternates: impl FnOnce() -> Result, negotiator: &mut dyn gix_negotiate::Negotiator, graph: &mut gix_negotiate::Graph<'_, '_>, - ref_map: &fetch::RefMap, - shallow: &fetch::Shallow, - mapping_is_ignored: impl Fn(&fetch::Mapping) -> bool, -) -> Result { + ref_map: &RefMap, + shallow: &Shallow, + mapping_is_ignored: impl Fn(&refmap::Mapping) -> bool, +) -> Result +where + E: Into>, + Out: Iterator, + F: gix_object::Find, +{ let _span = gix_trace::detail!("mark_complete_and_common_ref", mappings = ref_map.mappings.len()); if ref_map.mappings.is_empty() { return Ok(Action::NoChange); } - if let fetch::Shallow::Deepen(0) = shallow { + if let Shallow::Deepen(0) = shallow { // Avoid deepening (relative) with zero as it seems to upset the server. Git also doesn't actually // perform the negotiation for some reason (couldn't find it in code). return Ok(Action::NoChange); } - if let Some(fetch::Mapping { - remote: fetch::Source::Ref(gix_protocol::handshake::Ref::Unborn { .. }), + if let Some(refmap::Mapping { + remote: refmap::Source::Ref(crate::handshake::Ref::Unborn { .. }), .. }) = ref_map.mappings.last().filter(|_| ref_map.mappings.len() == 1) { @@ -100,8 +162,8 @@ pub(crate) fn mark_complete_and_common_ref( let want_id = mapping.remote.as_id(); let have_id = mapping.local.as_ref().and_then(|name| { // this is the only time git uses the peer-id. - let r = repo.find_reference(name).ok()?; - r.target().try_id().map(ToOwned::to_owned) + let r = refs.find(name).ok()?; + r.target.try_id().map(ToOwned::to_owned) }); // Even for ignored mappings we want to know if the `want` is already present locally, so skip nothing else. @@ -119,7 +181,7 @@ pub(crate) fn mark_complete_and_common_ref( { remote_ref_target_known[mapping_idx] = true; cutoff_date = cutoff_date.unwrap_or_default().max(commit.commit_time).into(); - } else if want_id.map_or(false, |maybe_annotated_tag| repo.objects.contains(maybe_annotated_tag)) { + } else if want_id.map_or(false, |maybe_annotated_tag| objects.exists(maybe_annotated_tag)) { remote_ref_target_known[mapping_idx] = true; } } @@ -140,8 +202,10 @@ pub(crate) fn mark_complete_and_common_ref( // color our commits as complete as identified by references, unconditionally // (`git` is conditional here based on `deepen`, but it doesn't make sense and it's hard to extract from history when that happened). let mut queue = Queue::new(); - mark_all_refs_in_repo(repo, graph, &mut queue, Flags::COMPLETE)?; - mark_alternate_complete(repo, graph, &mut queue)?; + mark_all_refs_in_repo(refs, objects, graph, &mut queue, Flags::COMPLETE)?; + for (alt_refs, alt_objs) in alternates().map_err(|err| Error::AlternateRefsAndObjects(err.into()))? { + mark_all_refs_in_repo(&alt_refs, &alt_objs, graph, &mut queue, Flags::COMPLETE)?; + } // Keep track of the tips, which happen to be on our queue right, before we traverse the graph with cutoff. let tips = if let Some(cutoff) = cutoff_date { let tips = Cow::Owned(queue.clone()); @@ -191,14 +255,11 @@ pub(crate) fn mark_complete_and_common_ref( /// /// We want to ignore mappings during negotiation if they would be handled implicitly by the server, which is the case /// when tags would be sent implicitly due to `Tags::Included`. -pub(crate) fn make_refmapping_ignore_predicate( - fetch_tags: fetch::Tags, - ref_map: &fetch::RefMap, -) -> impl Fn(&fetch::Mapping) -> bool + '_ { +pub fn make_refmapping_ignore_predicate(fetch_tags: Tags, ref_map: &RefMap) -> impl Fn(&refmap::Mapping) -> bool + '_ { // With included tags, we have to keep mappings of tags to handle them later when updating refs, but we don't want to // explicitly `want` them as the server will determine by itself which tags are pointing to a commit it wants to send. // If we would not exclude implicit tag mappings like this, we would get too much of the graph. - let tag_refspec_to_ignore = matches!(fetch_tags, crate::remote::fetch::Tags::Included) + let tag_refspec_to_ignore = matches!(fetch_tags, Tags::Included) .then(|| fetch_tags.to_refspec()) .flatten(); move |mapping| { @@ -212,27 +273,37 @@ pub(crate) fn make_refmapping_ignore_predicate( } } -/// Add all `wants` to `arguments`, which is the unpeeled direct target that the advertised remote ref points to. -pub(crate) fn add_wants( - repo: &crate::Repository, - arguments: &mut gix_protocol::fetch::Arguments, - ref_map: &fetch::RefMap, - mapping_known: &[bool], - shallow: &fetch::Shallow, - mapping_is_ignored: impl Fn(&fetch::Mapping) -> bool, +/// Add all 'wants' to `arguments` once it's known negotiation is necessary. +/// +/// This is a call to be made when [`mark_complete_and_common_ref()`] returned [`Action::MustNegotiate`]. +/// That variant also contains the `remote_ref_target_known` field which is supposed to be passed here. +/// +/// `objects` are used to see if remote ids are known here and are tags, in which case they are also added as 'haves' as +/// [negotiators](gix_negotiate::Negotiator) don't see tags at all. +/// +/// * `ref_map` is the state of refs as known on the remote. +/// * `shallow` defines if the history should be shallow. +/// * `mapping_is_ignored` is typically initialized with [`make_refmapping_ignore_predicate`]. +pub fn add_wants( + objects: &impl gix_object::FindHeader, + arguments: &mut crate::fetch::Arguments, + ref_map: &RefMap, + remote_ref_target_known: &[bool], + shallow: &Shallow, + mapping_is_ignored: impl Fn(&refmap::Mapping) -> bool, ) { // When using shallow, we can't exclude `wants` as the remote won't send anything then. Thus, we have to resend everything // we have as want instead to get exactly the same graph, but possibly deepened. - let is_shallow = !matches!(shallow, fetch::Shallow::NoChange); + let is_shallow = !matches!(shallow, Shallow::NoChange); let wants = ref_map .mappings .iter() - .zip(mapping_known) + .zip(remote_ref_target_known) .filter_map(|(m, known)| (is_shallow || !*known).then_some(m)) .filter(|m| !mapping_is_ignored(m)); for want in wants { let id_on_remote = want.remote.as_id(); - if !arguments.can_use_ref_in_want() || matches!(want.remote, fetch::Source::ObjectId(_)) { + if !arguments.can_use_ref_in_want() || matches!(want.remote, refmap::Source::ObjectId(_)) { if let Some(id) = id_on_remote { arguments.want(id); } @@ -244,8 +315,8 @@ pub(crate) fn add_wants( ); } let id_is_annotated_tag_we_have = id_on_remote - .and_then(|id| repo.objects.header(id).ok().map(|h| (id, h))) - .filter(|(_, h)| h.kind() == gix_object::Kind::Tag) + .and_then(|id| objects.try_header(id).ok().flatten().map(|h| (id, h))) + .filter(|(_, h)| h.kind == gix_object::Kind::Tag) .map(|(id, _)| id); if let Some(tag_on_remote) = id_is_annotated_tag_we_have { // Annotated tags are not handled at all by negotiators in the commit-graph - they only see commits and thus won't @@ -285,15 +356,20 @@ fn mark_recent_complete_commits( } fn mark_all_refs_in_repo( - repo: &crate::Repository, + store: &gix_ref::file::Store, + objects: &impl gix_object::Find, graph: &mut gix_negotiate::Graph<'_, '_>, queue: &mut Queue, mark: Flags, ) -> Result<(), Error> { let _span = gix_trace::detail!("mark_all_refs"); - for local_ref in repo.references()?.all()?.peeled()? { - let local_ref = local_ref?; - let id = local_ref.id().detach(); + for local_ref in store.iter()?.all()? { + let mut local_ref = local_ref?; + let id = local_ref.peel_to_id_in_place_packed( + store, + objects, + store.cached_packed_buffer()?.as_ref().map(|b| &***b), + )?; let mut is_complete = false; if let Some(commit) = graph .get_or_insert_commit(id, |md| { @@ -308,45 +384,77 @@ fn mark_all_refs_in_repo( Ok(()) } -fn mark_alternate_complete( - repo: &crate::Repository, - graph: &mut gix_negotiate::Graph<'_, '_>, - queue: &mut Queue, -) -> Result<(), Error> { - let alternates = repo.objects.store_ref().alternate_db_paths()?; - let _span = gix_trace::detail!("mark_alternate_refs", num_odb = alternates.len()); +/// +pub mod one_round { + /// State to keep between individual [rounds](super::one_round()). + #[derive(Clone, Debug)] + pub struct State { + /// The amount of haves to send the next round. + /// It's initialized with the standard window size for negotations. + pub haves_to_send: usize, + /// Is turned `true` if the remote as confirmed any common commit so far. + pub(super) seen_ack: bool, + /// The amount of haves we have sent that didn't have a match on the remote. + /// + /// The higher this number, the more time was wasted. + pub(super) in_vain: usize, + /// Commits we have in common. + /// + /// Only set when we are stateless as we have to resend known common commits each round. + pub(super) common_commits: Option>, + } - for alternate_repo in alternates.into_iter().filter_map(|path| { - path.ancestors() - .nth(1) - .and_then(|git_dir| crate::open_opts(git_dir, repo.options.clone()).ok()) - }) { - mark_all_refs_in_repo(&alternate_repo, graph, queue, Flags::ALTERNATE | Flags::COMPLETE)?; + impl State { + /// Create a new instance. + /// + /// setting `connection_is_stateless` accordingly which affects the amount of haves to send. + pub fn new(connection_is_stateless: bool) -> Self { + State { + haves_to_send: gix_negotiate::window_size(connection_is_stateless, None), + seen_ack: false, + in_vain: 0, + common_commits: connection_is_stateless.then(Vec::new), + } + } + } + + impl State { + /// Return `true` if the transports connection is stateless. + fn connection_is_stateless(&self) -> bool { + self.common_commits.is_some() + } + pub(super) fn adjust_window_size(&mut self) { + self.haves_to_send = gix_negotiate::window_size(self.connection_is_stateless(), Some(self.haves_to_send)); + } } - Ok(()) } -/// Negotiate the nth `round` with `negotiator` sending `haves_to_send` after possibly making the known common commits -/// as sent by the remote known to `negotiator` using `previous_response` if this isn't the first round. -/// All `haves` are added to `arguments` accordingly. -/// Returns the amount of haves actually sent. -pub(crate) fn one_round( +/// Prepare to negotiate a single round in the process of letting the remote know what we have, and have in common. +/// +/// Note that this function only configures `arguments`, no IO is performed. +/// +/// The operation is performed with `negotiator` and `graph`, sending the amount of `haves_to_send` after possibly +/// making the common commits (as sent by the remote) known to `negotiator` using `previous_response`, if this isn't the first round. +/// All [commits we have](crate::fetch::Arguments::have()) are added to `arguments` accordingly. +/// +/// Returns information about this round, and `true` if we are done and should stop negotiating *after* the `arguments` have +/// been sent to the remote one last time. +pub fn one_round( negotiator: &mut dyn gix_negotiate::Negotiator, graph: &mut gix_negotiate::Graph<'_, '_>, - haves_to_send: usize, - arguments: &mut gix_protocol::fetch::Arguments, - previous_response: Option<&gix_protocol::fetch::Response>, - mut common: Option<&mut Vec>, -) -> Result<(usize, bool), Error> { + state: &mut one_round::State, + arguments: &mut crate::fetch::Arguments, + previous_response: Option<&crate::fetch::Response>, +) -> Result<(Round, bool), Error> { let mut seen_ack = false; if let Some(response) = previous_response { - use gix_protocol::fetch::response::Acknowledgement; + use crate::fetch::response::Acknowledgement; for ack in response.acknowledgements() { match ack { Acknowledgement::Common(id) => { seen_ack = true; negotiator.in_common_with_remote(*id, graph)?; - if let Some(ref mut common) = common { + if let Some(common) = &mut state.common_commits { common.push(*id); } } @@ -361,19 +469,33 @@ pub(crate) fn one_round( // `common` is set only if this is a stateless transport, and we repeat previously confirmed common commits as HAVE, because // we are not going to repeat them otherwise. - if let Some(common) = common { + if let Some(common) = &mut state.common_commits { for have_id in common { arguments.have(have_id); } } - let mut haves_sent = 0; - for have_id in (0..haves_to_send).map_while(|_| negotiator.next_have(graph)) { + let mut haves_added = 0; + for have_id in (0..state.haves_to_send).map_while(|_| negotiator.next_have(graph)) { arguments.have(have_id?); - haves_sent += 1; + haves_added += 1; } // Note that we are differing from the git implementation, which does an extra-round of with no new haves sent at all. - // For us it seems better to just say we are done when we know we are done, as potentially additional acks won't affect the - // queue of any of our implementation at all (so the negotiator won't come up with more haves next time either). - Ok((haves_sent, seen_ack)) + // For us, it seems better to just say we are done when we know we are done, as potentially additional acks won't affect the + // queue of our implementation at all (so the negotiator won't come up with more haves next time either). + if seen_ack { + state.in_vain = 0; + } + state.seen_ack |= seen_ack; + state.in_vain += haves_added; + let round = Round { + haves_sent: haves_added, + in_vain: state.in_vain, + haves_to_send: state.haves_to_send, + previous_response_had_at_least_one_in_common: seen_ack, + }; + let is_done = haves_added != state.haves_to_send || (state.seen_ack && state.in_vain >= 256); + state.adjust_window_size(); + + Ok((round, is_done)) } diff --git a/gix-protocol/src/fetch/refmap/init.rs b/gix-protocol/src/fetch/refmap/init.rs new file mode 100644 index 00000000000..f8168d50968 --- /dev/null +++ b/gix-protocol/src/fetch/refmap/init.rs @@ -0,0 +1,174 @@ +use std::collections::HashSet; + +use crate::fetch; +use crate::fetch::refmap::{Mapping, Source, SpecIndex}; +use crate::fetch::RefMap; +use crate::transport::client::Transport; +use bstr::{BString, ByteVec}; +use gix_features::progress::Progress; + +/// The error returned by [`RefMap::new()`]. +#[derive(Debug, thiserror::Error)] +#[allow(missing_docs)] +pub enum Error { + #[error("The object format {format:?} as used by the remote is unsupported")] + UnknownObjectFormat { format: BString }, + #[error(transparent)] + MappingValidation(#[from] gix_refspec::match_group::validate::Error), + #[error(transparent)] + ListRefs(#[from] crate::ls_refs::Error), +} + +/// For use in [`RefMap::new()`]. +#[derive(Debug, Clone)] +pub struct Options { + /// Use a two-component prefix derived from the ref-spec's source, like `refs/heads/` to let the server pre-filter refs + /// with great potential for savings in traffic and local CPU time. Defaults to `true`. + pub prefix_from_spec_as_filter_on_remote: bool, + /// A list of refspecs to use as implicit refspecs which won't be saved or otherwise be part of the remote in question. + /// + /// This is useful for handling `remote..tagOpt` for example. + pub extra_refspecs: Vec, +} + +impl Default for Options { + fn default() -> Self { + Options { + prefix_from_spec_as_filter_on_remote: true, + extra_refspecs: Vec::new(), + } + } +} + +impl RefMap { + /// Create a new instance by obtaining all references on the remote that have been filtered through our remote's + /// for _fetching_. + /// + /// A [context](fetch::Context) is provided to bundle what would be additional parameters, + /// and [options](Options) are used to further configure the call. + /// + /// * `progress` is used if `ls-refs` is invoked on the remote. Always the case when V2 is used. + /// * `fetch_refspecs` are all explicit refspecs to identify references on the remote that you are interested in. + /// Note that these are copied to [`RefMap::refspecs`] for convenience, as `RefMap::mappings` refer to them by index. + #[allow(clippy::result_large_err)] + #[maybe_async::maybe_async] + pub async fn new( + mut progress: impl Progress, + fetch_refspecs: &[gix_refspec::RefSpec], + fetch::Context { + handshake, + transport, + user_agent, + trace_packetlines, + }: fetch::Context<'_, T>, + Options { + prefix_from_spec_as_filter_on_remote, + extra_refspecs, + }: Options, + ) -> Result + where + T: Transport, + { + let _span = gix_trace::coarse!("gix_protocol::fetch::RefMap::new()"); + let null = gix_hash::ObjectId::null(gix_hash::Kind::Sha1); // OK to hardcode Sha1, it's not supposed to match, ever. + + let all_refspecs = { + let mut s: Vec<_> = fetch_refspecs.to_vec(); + s.extend(extra_refspecs.clone()); + s + }; + let remote_refs = match handshake.refs.take() { + Some(refs) => refs, + None => { + crate::ls_refs( + transport, + &handshake.capabilities, + |_capabilities, arguments, features| { + features.push(user_agent); + if prefix_from_spec_as_filter_on_remote { + let mut seen = HashSet::new(); + for spec in &all_refspecs { + let spec = spec.to_ref(); + if seen.insert(spec.instruction()) { + let mut prefixes = Vec::with_capacity(1); + spec.expand_prefixes(&mut prefixes); + for mut prefix in prefixes { + prefix.insert_str(0, "ref-prefix "); + arguments.push(prefix); + } + } + } + } + Ok(crate::ls_refs::Action::Continue) + }, + &mut progress, + trace_packetlines, + ) + .await? + } + }; + let num_explicit_specs = fetch_refspecs.len(); + let group = gix_refspec::MatchGroup::from_fetch_specs(all_refspecs.iter().map(gix_refspec::RefSpec::to_ref)); + let (res, fixes) = group + .match_remotes(remote_refs.iter().map(|r| { + let (full_ref_name, target, object) = r.unpack(); + gix_refspec::match_group::Item { + full_ref_name, + target: target.unwrap_or(&null), + object, + } + })) + .validated()?; + + let mappings = res.mappings; + let mappings = mappings + .into_iter() + .map(|m| Mapping { + remote: m.item_index.map_or_else( + || { + Source::ObjectId(match m.lhs { + gix_refspec::match_group::SourceRef::ObjectId(id) => id, + _ => unreachable!("no item index implies having an object id"), + }) + }, + |idx| Source::Ref(remote_refs[idx].clone()), + ), + local: m.rhs.map(std::borrow::Cow::into_owned), + spec_index: if m.spec_index < num_explicit_specs { + SpecIndex::ExplicitInRemote(m.spec_index) + } else { + SpecIndex::Implicit(m.spec_index - num_explicit_specs) + }, + }) + .collect(); + + let object_hash = extract_object_format(handshake)?; + Ok(RefMap { + mappings, + refspecs: fetch_refspecs.to_vec(), + extra_refspecs, + fixes, + remote_refs, + object_hash, + }) + } +} + +/// Assume sha1 if server says nothing, otherwise configure anything beyond sha1 in the local repo configuration +#[allow(clippy::result_large_err)] +fn extract_object_format(outcome: &crate::handshake::Outcome) -> Result { + use bstr::ByteSlice; + let object_hash = + if let Some(object_format) = outcome.capabilities.capability("object-format").and_then(|c| c.value()) { + let object_format = object_format.to_str().map_err(|_| Error::UnknownObjectFormat { + format: object_format.into(), + })?; + match object_format { + "sha1" => gix_hash::Kind::Sha1, + unknown => return Err(Error::UnknownObjectFormat { format: unknown.into() }), + } + } else { + gix_hash::Kind::Sha1 + }; + Ok(object_hash) +} diff --git a/gix-protocol/src/fetch/refmap/mod.rs b/gix-protocol/src/fetch/refmap/mod.rs new file mode 100644 index 00000000000..8fdc3d4eeb3 --- /dev/null +++ b/gix-protocol/src/fetch/refmap/mod.rs @@ -0,0 +1,105 @@ +/// +#[cfg(any(feature = "blocking-client", feature = "async-client"))] +pub mod init; + +/// Either an object id that the remote has or the matched remote ref itself. +#[derive(Debug, Clone)] +pub enum Source { + /// An object id, as the matched ref-spec was an object id itself. + ObjectId(gix_hash::ObjectId), + /// The remote reference that matched the ref-specs name. + Ref(crate::handshake::Ref), +} + +impl Source { + /// Return either the direct object id we refer to or the direct target that a reference refers to. + /// The latter may be a direct or a symbolic reference. + /// If unborn, `None` is returned. + pub fn as_id(&self) -> Option<&gix_hash::oid> { + match self { + Source::ObjectId(id) => Some(id), + Source::Ref(r) => r.unpack().1, + } + } + + /// Return the target that this symbolic ref is pointing to, or `None` if it is no symbolic ref. + pub fn as_target(&self) -> Option<&bstr::BStr> { + match self { + Source::ObjectId(_) => None, + Source::Ref(r) => match r { + crate::handshake::Ref::Peeled { .. } | crate::handshake::Ref::Direct { .. } => None, + crate::handshake::Ref::Symbolic { target, .. } | crate::handshake::Ref::Unborn { target, .. } => { + Some(target.as_ref()) + } + }, + } + } + + /// Returns the peeled id of this instance, that is the object that can't be de-referenced anymore. + pub fn peeled_id(&self) -> Option<&gix_hash::oid> { + match self { + Source::ObjectId(id) => Some(id), + Source::Ref(r) => { + let (_name, target, peeled) = r.unpack(); + peeled.or(target) + } + } + } + + /// Return ourselves as the full name of the reference we represent, or `None` if this source isn't a reference but an object. + pub fn as_name(&self) -> Option<&bstr::BStr> { + match self { + Source::ObjectId(_) => None, + Source::Ref(r) => match r { + crate::handshake::Ref::Unborn { full_ref_name, .. } + | crate::handshake::Ref::Symbolic { full_ref_name, .. } + | crate::handshake::Ref::Direct { full_ref_name, .. } + | crate::handshake::Ref::Peeled { full_ref_name, .. } => Some(full_ref_name.as_ref()), + }, + } + } +} + +/// An index into various lists of refspecs that have been used in a [Mapping] of remote references to local ones. +#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash, Ord, PartialOrd)] +pub enum SpecIndex { + /// An index into the _refspecs of the remote_ that triggered a fetch operation. + /// These refspecs are explicit and visible to the user. + ExplicitInRemote(usize), + /// An index into the list of [extra refspecs](crate::fetch::RefMap::extra_refspecs) that are implicit + /// to a particular fetch operation. + Implicit(usize), +} + +impl SpecIndex { + /// Depending on our index variant, get the index either from `refspecs` or from `extra_refspecs` for `Implicit` variants. + pub fn get<'a>( + self, + refspecs: &'a [gix_refspec::RefSpec], + extra_refspecs: &'a [gix_refspec::RefSpec], + ) -> Option<&'a gix_refspec::RefSpec> { + match self { + SpecIndex::ExplicitInRemote(idx) => refspecs.get(idx), + SpecIndex::Implicit(idx) => extra_refspecs.get(idx), + } + } + + /// If this is an `Implicit` variant, return its index. + pub fn implicit_index(self) -> Option { + match self { + SpecIndex::Implicit(idx) => Some(idx), + SpecIndex::ExplicitInRemote(_) => None, + } + } +} + +/// A mapping between a single remote reference and its advertised objects to a local destination which may or may not exist. +#[derive(Debug, Clone)] +pub struct Mapping { + /// The reference on the remote side, along with information about the objects they point to as advertised by the server. + pub remote: Source, + /// The local tracking reference to update after fetching the object visible via `remote`. + pub local: Option, + /// The index into the fetch ref-specs used to produce the mapping, allowing it to be recovered. + pub spec_index: SpecIndex, +} diff --git a/gix-protocol/src/fetch/response/async_io.rs b/gix-protocol/src/fetch/response/async_io.rs index 547881331eb..aee078d19e1 100644 --- a/gix-protocol/src/fetch/response/async_io.rs +++ b/gix-protocol/src/fetch/response/async_io.rs @@ -4,6 +4,7 @@ use gix_transport::{client, Protocol}; use crate::fetch::{ response, + response::shallow_update_from_line, response::{Acknowledgement, ShallowUpdate, WantedRef}, Response, }; @@ -132,7 +133,7 @@ impl Response { } } "shallow-info" => { - if parse_v2_section(&mut line, reader, &mut shallows, ShallowUpdate::from_line).await? { + if parse_v2_section(&mut line, reader, &mut shallows, shallow_update_from_line).await? { break 'section false; } } diff --git a/gix-protocol/src/fetch/response/blocking_io.rs b/gix-protocol/src/fetch/response/blocking_io.rs index 08460ae39f7..b06bd3a1fd3 100644 --- a/gix-protocol/src/fetch/response/blocking_io.rs +++ b/gix-protocol/src/fetch/response/blocking_io.rs @@ -2,6 +2,7 @@ use std::io; use gix_transport::{client, Protocol}; +use crate::fetch::response::shallow_update_from_line; use crate::fetch::{ response, response::{Acknowledgement, ShallowUpdate, WantedRef}, @@ -128,7 +129,7 @@ impl Response { } } "shallow-info" => { - if parse_v2_section(&mut line, reader, &mut shallows, ShallowUpdate::from_line)? { + if parse_v2_section(&mut line, reader, &mut shallows, shallow_update_from_line)? { break 'section false; } } diff --git a/gix-protocol/src/fetch/response/mod.rs b/gix-protocol/src/fetch/response/mod.rs index 5073cc13e29..941f3fcd9c1 100644 --- a/gix-protocol/src/fetch/response/mod.rs +++ b/gix-protocol/src/fetch/response/mod.rs @@ -2,6 +2,7 @@ use bstr::BString; use gix_transport::{client, Protocol}; use crate::command::Feature; +use crate::fetch::Response; /// The error returned in the [response module][crate::fetch::response]. #[derive(Debug, thiserror::Error)] @@ -59,15 +60,7 @@ pub enum Acknowledgement { Nak, } -/// A shallow line received from the server. -#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub enum ShallowUpdate { - /// Shallow the given `id`. - Shallow(gix_hash::ObjectId), - /// Don't shallow the given `id` anymore. - Unshallow(gix_hash::ObjectId), -} +pub use gix_shallow::Update as ShallowUpdate; /// A wanted-ref line received from the server. #[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] @@ -79,21 +72,19 @@ pub struct WantedRef { pub path: BString, } -impl ShallowUpdate { - /// Parse a `ShallowUpdate` from a `line` as received to the server. - pub fn from_line(line: &str) -> Result { - match line.trim_end().split_once(' ') { - Some((prefix, id)) => { - let id = gix_hash::ObjectId::from_hex(id.as_bytes()) - .map_err(|_| Error::UnknownLineType { line: line.to_owned() })?; - Ok(match prefix { - "shallow" => ShallowUpdate::Shallow(id), - "unshallow" => ShallowUpdate::Unshallow(id), - _ => return Err(Error::UnknownLineType { line: line.to_owned() }), - }) - } - None => Err(Error::UnknownLineType { line: line.to_owned() }), +/// Parse a `ShallowUpdate` from a `line` as received to the server. +pub fn shallow_update_from_line(line: &str) -> Result { + match line.trim_end().split_once(' ') { + Some((prefix, id)) => { + let id = gix_hash::ObjectId::from_hex(id.as_bytes()) + .map_err(|_| Error::UnknownLineType { line: line.to_owned() })?; + Ok(match prefix { + "shallow" => ShallowUpdate::Shallow(id), + "unshallow" => ShallowUpdate::Unshallow(id), + _ => return Err(Error::UnknownLineType { line: line.to_owned() }), + }) } + None => Err(Error::UnknownLineType { line: line.to_owned() }), } } @@ -148,15 +139,6 @@ impl WantedRef { } } -/// A representation of a complete fetch response -#[derive(Debug)] -pub struct Response { - acks: Vec, - shallows: Vec, - wanted_refs: Vec, - has_pack: bool, -} - impl Response { /// Return true if the response has a pack which can be read next. pub fn has_pack(&self) -> bool { @@ -236,7 +218,7 @@ impl Response { } None => acks.push(ack), }, - Err(_) => match ShallowUpdate::from_line(peeked_line) { + Err(_) => match shallow_update_from_line(peeked_line) { Ok(shallow) => { shallows.push(shallow); } diff --git a/gix-protocol/src/fetch/tests.rs b/gix-protocol/src/fetch/tests.rs deleted file mode 100644 index d8f61a426b4..00000000000 --- a/gix-protocol/src/fetch/tests.rs +++ /dev/null @@ -1,416 +0,0 @@ -#[cfg(any(feature = "async-client", feature = "blocking-client"))] -mod arguments { - use bstr::ByteSlice; - use gix_transport::Protocol; - - use crate::fetch; - - fn arguments_v1(features: impl IntoIterator) -> fetch::Arguments { - fetch::Arguments::new(Protocol::V1, features.into_iter().map(|n| (n, None)).collect(), false) - } - - fn arguments_v2(features: impl IntoIterator) -> fetch::Arguments { - fetch::Arguments::new(Protocol::V2, features.into_iter().map(|n| (n, None)).collect(), false) - } - - struct Transport { - inner: T, - stateful: bool, - } - - #[cfg(feature = "blocking-client")] - mod impls { - use std::borrow::Cow; - - use bstr::BStr; - use gix_transport::{ - client, - client::{Error, MessageKind, RequestWriter, SetServiceResponse, WriteMode}, - Protocol, Service, - }; - - use crate::fetch::tests::arguments::Transport; - - impl client::TransportWithoutIO for Transport { - fn set_identity(&mut self, identity: client::Account) -> Result<(), Error> { - self.inner.set_identity(identity) - } - - fn request( - &mut self, - write_mode: WriteMode, - on_into_read: MessageKind, - trace: bool, - ) -> Result, Error> { - self.inner.request(write_mode, on_into_read, trace) - } - - fn to_url(&self) -> Cow<'_, BStr> { - self.inner.to_url() - } - - fn supported_protocol_versions(&self) -> &[Protocol] { - self.inner.supported_protocol_versions() - } - - fn connection_persists_across_multiple_requests(&self) -> bool { - self.stateful - } - - fn configure( - &mut self, - config: &dyn std::any::Any, - ) -> Result<(), Box> { - self.inner.configure(config) - } - } - - impl client::Transport for Transport { - fn handshake<'a>( - &mut self, - service: Service, - extra_parameters: &'a [(&'a str, Option<&'a str>)], - ) -> Result, Error> { - self.inner.handshake(service, extra_parameters) - } - } - } - - #[cfg(feature = "async-client")] - mod impls { - use std::borrow::Cow; - - use async_trait::async_trait; - use bstr::BStr; - use gix_transport::{ - client, - client::{Error, MessageKind, RequestWriter, SetServiceResponse, WriteMode}, - Protocol, Service, - }; - - use crate::fetch::tests::arguments::Transport; - impl client::TransportWithoutIO for Transport { - fn set_identity(&mut self, identity: client::Account) -> Result<(), Error> { - self.inner.set_identity(identity) - } - - fn request( - &mut self, - write_mode: WriteMode, - on_into_read: MessageKind, - trace: bool, - ) -> Result, Error> { - self.inner.request(write_mode, on_into_read, trace) - } - - fn to_url(&self) -> Cow<'_, BStr> { - self.inner.to_url() - } - - fn supported_protocol_versions(&self) -> &[Protocol] { - self.inner.supported_protocol_versions() - } - - fn connection_persists_across_multiple_requests(&self) -> bool { - self.stateful - } - - fn configure( - &mut self, - config: &dyn std::any::Any, - ) -> Result<(), Box> { - self.inner.configure(config) - } - } - - #[async_trait(?Send)] - impl client::Transport for Transport { - async fn handshake<'a>( - &mut self, - service: Service, - extra_parameters: &'a [(&'a str, Option<&'a str>)], - ) -> Result, Error> { - self.inner.handshake(service, extra_parameters).await - } - } - } - - fn transport( - out: &mut Vec, - stateful: bool, - ) -> Transport>> { - Transport { - inner: gix_transport::client::git::Connection::new( - &[], - out, - Protocol::V1, // does not matter - b"does/not/matter".as_bstr().to_owned(), - None::<(&str, _)>, - gix_transport::client::git::ConnectMode::Process, // avoid header to be sent - false, - ), - stateful, - } - } - - fn id(hex: &str) -> gix_hash::ObjectId { - gix_hash::ObjectId::from_hex(hex.as_bytes()).expect("expect valid hex id") - } - - mod v1 { - use bstr::ByteSlice; - - use crate::fetch::tests::arguments::{arguments_v1, id, transport}; - - #[maybe_async::test(feature = "blocking-client", async(feature = "async-client", async_std::test))] - async fn include_tag() { - let mut out = Vec::new(); - let mut t = transport(&mut out, true); - let mut arguments = arguments_v1(["include-tag", "feature-b"].iter().copied()); - assert!(arguments.can_use_include_tag()); - - arguments.use_include_tag(); - arguments.want(id("ff333369de1221f9bfbbe03a3a13e9a09bc1ffff")); - arguments.send(&mut t, true).await.expect("sending to buffer to work"); - assert_eq!( - out.as_bstr(), - b"0048want ff333369de1221f9bfbbe03a3a13e9a09bc1ffff feature-b include-tag -00000009done -" - .as_bstr() - ); - } - - #[maybe_async::test(feature = "blocking-client", async(feature = "async-client", async_std::test))] - async fn no_include_tag() { - let mut out = Vec::new(); - let mut t = transport(&mut out, true); - let mut arguments = arguments_v1(["include-tag", "feature-b"].iter().copied()); - assert!(arguments.can_use_include_tag()); - - arguments.want(id("ff333369de1221f9bfbbe03a3a13e9a09bc1ffff")); - arguments.send(&mut t, true).await.expect("sending to buffer to work"); - assert_eq!( - out.as_bstr(), - b"003cwant ff333369de1221f9bfbbe03a3a13e9a09bc1ffff feature-b -00000009done -" - .as_bstr(), - "it's possible to not have it enabled, even though it's advertised by the server" - ); - } - - #[maybe_async::test(feature = "blocking-client", async(feature = "async-client", async_std::test))] - async fn haves_and_wants_for_clone() { - let mut out = Vec::new(); - let mut t = transport(&mut out, true); - let mut arguments = arguments_v1(["feature-a", "feature-b"].iter().copied()); - assert!( - !arguments.can_use_include_tag(), - "needs to be enabled by features in V1" - ); - - arguments.want(id("7b333369de1221f9bfbbe03a3a13e9a09bc1c907")); - arguments.want(id("ff333369de1221f9bfbbe03a3a13e9a09bc1ffff")); - arguments.send(&mut t, true).await.expect("sending to buffer to work"); - assert_eq!( - out.as_bstr(), - b"0046want 7b333369de1221f9bfbbe03a3a13e9a09bc1c907 feature-a feature-b -0032want ff333369de1221f9bfbbe03a3a13e9a09bc1ffff -00000009done -" - .as_bstr() - ); - } - - #[maybe_async::test(feature = "blocking-client", async(feature = "async-client", async_std::test))] - async fn haves_and_wants_for_fetch_stateless() { - let mut out = Vec::new(); - let mut t = transport(&mut out, false); - let mut arguments = arguments_v1(["feature-a", "shallow", "deepen-since", "deepen-not"].iter().copied()); - - arguments.deepen(1); - arguments.shallow(id("7b333369de1221f9bfbbe03a3a13e9a09bc1c9ff")); - arguments.want(id("7b333369de1221f9bfbbe03a3a13e9a09bc1c907")); - arguments.deepen_since(12345); - arguments.deepen_not("refs/heads/main".into()); - arguments.have(id("0000000000000000000000000000000000000000")); - arguments.send(&mut t, false).await.expect("sending to buffer to work"); - - arguments.have(id("1111111111111111111111111111111111111111")); - arguments.send(&mut t, true).await.expect("sending to buffer to work"); - assert_eq!( - out.as_bstr(), - b"005cwant 7b333369de1221f9bfbbe03a3a13e9a09bc1c907 feature-a shallow deepen-since deepen-not -0035shallow 7b333369de1221f9bfbbe03a3a13e9a09bc1c9ff -000ddeepen 1 -0017deepen-since 12345 -001fdeepen-not refs/heads/main -00000032have 0000000000000000000000000000000000000000 -0000005cwant 7b333369de1221f9bfbbe03a3a13e9a09bc1c907 feature-a shallow deepen-since deepen-not -0035shallow 7b333369de1221f9bfbbe03a3a13e9a09bc1c9ff -000ddeepen 1 -0017deepen-since 12345 -001fdeepen-not refs/heads/main -00000032have 1111111111111111111111111111111111111111 -0009done -" - .as_bstr() - ); - } - - #[maybe_async::test(feature = "blocking-client", async(feature = "async-client", async_std::test))] - async fn haves_and_wants_for_fetch_stateful() { - let mut out = Vec::new(); - let mut t = transport(&mut out, true); - let mut arguments = arguments_v1(["feature-a", "shallow"].iter().copied()); - - arguments.deepen(1); - arguments.want(id("7b333369de1221f9bfbbe03a3a13e9a09bc1c907")); - arguments.have(id("0000000000000000000000000000000000000000")); - arguments.send(&mut t, false).await.expect("sending to buffer to work"); - - arguments.have(id("1111111111111111111111111111111111111111")); - arguments.send(&mut t, true).await.expect("sending to buffer to work"); - assert_eq!( - out.as_bstr(), - b"0044want 7b333369de1221f9bfbbe03a3a13e9a09bc1c907 feature-a shallow -000ddeepen 1 -00000032have 0000000000000000000000000000000000000000 -00000032have 1111111111111111111111111111111111111111 -0009done -" - .as_bstr() - ); - } - } - - mod v2 { - use bstr::ByteSlice; - - use crate::fetch::tests::arguments::{arguments_v2, id, transport}; - - #[maybe_async::test(feature = "blocking-client", async(feature = "async-client", async_std::test))] - async fn include_tag() { - let mut out = Vec::new(); - let mut t = transport(&mut out, true); - let mut arguments = arguments_v2(["does not matter for us here"].iter().copied()); - assert!(arguments.can_use_include_tag(), "always on in V2"); - arguments.use_include_tag(); - - arguments.want(id("ff333369de1221f9bfbbe03a3a13e9a09bc1ffff")); - arguments.send(&mut t, true).await.expect("sending to buffer to work"); - assert_eq!( - out.as_bstr(), - b"0012command=fetch -0001000ethin-pack -000eofs-delta -0010include-tag -0032want ff333369de1221f9bfbbe03a3a13e9a09bc1ffff -0009done -0000" - .as_bstr(), - "we filter features/capabilities without value as these apparently shouldn't be listed (remote dies otherwise)" - ); - } - - #[maybe_async::test(feature = "blocking-client", async(feature = "async-client", async_std::test))] - async fn haves_and_wants_for_clone_stateful() { - let mut out = Vec::new(); - let mut t = transport(&mut out, true); - let mut arguments = arguments_v2(["feature-a", "shallow"].iter().copied()); - assert!(arguments.is_stateless(true), "V2 is stateless…"); - assert!(arguments.is_stateless(false), "…in all cases"); - - arguments.add_feature("no-progress"); - arguments.deepen(1); - arguments.deepen_relative(); - arguments.want(id("7b333369de1221f9bfbbe03a3a13e9a09bc1c907")); - arguments.want(id("ff333369de1221f9bfbbe03a3a13e9a09bc1ffff")); - arguments.send(&mut t, true).await.expect("sending to buffer to work"); - assert_eq!( - out.as_bstr(), - b"0012command=fetch -0001000ethin-pack -000eofs-delta -0010no-progress -000ddeepen 1 -0014deepen-relative -0032want 7b333369de1221f9bfbbe03a3a13e9a09bc1c907 -0032want ff333369de1221f9bfbbe03a3a13e9a09bc1ffff -0009done -0000" - .as_bstr(), - "we filter features/capabilities without value as these apparently shouldn't be listed (remote dies otherwise)" - ); - } - - #[maybe_async::test(feature = "blocking-client", async(feature = "async-client", async_std::test))] - async fn haves_and_wants_for_fetch_stateless_and_stateful() { - for is_stateful in &[false, true] { - let mut out = Vec::new(); - let mut t = transport(&mut out, *is_stateful); - let mut arguments = arguments_v2(Some("shallow")); - - arguments.add_feature("no-progress"); - arguments.deepen(1); - arguments.deepen_since(12345); - arguments.shallow(id("7b333369de1221f9bfbbe03a3a13e9a09bc1c9ff")); - arguments.want(id("7b333369de1221f9bfbbe03a3a13e9a09bc1c907")); - arguments.deepen_not("refs/heads/main".into()); - arguments.have(id("0000000000000000000000000000000000000000")); - arguments.send(&mut t, false).await.expect("sending to buffer to work"); - - arguments.have(id("1111111111111111111111111111111111111111")); - arguments.send(&mut t, true).await.expect("sending to buffer to work"); - assert_eq!( - out.as_bstr(), - b"0012command=fetch -0001000ethin-pack -000eofs-delta -0010no-progress -000ddeepen 1 -0017deepen-since 12345 -0035shallow 7b333369de1221f9bfbbe03a3a13e9a09bc1c9ff -0032want 7b333369de1221f9bfbbe03a3a13e9a09bc1c907 -001fdeepen-not refs/heads/main -0032have 0000000000000000000000000000000000000000 -00000012command=fetch -0001000ethin-pack -000eofs-delta -0010no-progress -000ddeepen 1 -0017deepen-since 12345 -0035shallow 7b333369de1221f9bfbbe03a3a13e9a09bc1c9ff -0032want 7b333369de1221f9bfbbe03a3a13e9a09bc1c907 -001fdeepen-not refs/heads/main -0032have 1111111111111111111111111111111111111111 -0009done -0000" - .as_bstr(), - "V2 is stateless by default, so it repeats all but 'haves' in each request" - ); - } - } - - #[maybe_async::test(feature = "blocking-client", async(feature = "async-client", async_std::test))] - async fn ref_in_want() { - let mut out = Vec::new(); - let mut t = transport(&mut out, false); - let mut arguments = arguments_v2(["ref-in-want"].iter().copied()); - - arguments.want_ref(b"refs/heads/main".as_bstr()); - arguments.send(&mut t, true).await.expect("sending to buffer to work"); - assert_eq!( - out.as_bstr(), - b"0012command=fetch -0001000ethin-pack -000eofs-delta -001dwant-ref refs/heads/main -0009done -0000" - .as_bstr() - ); - } - } -} diff --git a/gix-protocol/src/fetch/types.rs b/gix-protocol/src/fetch/types.rs new file mode 100644 index 00000000000..bd80eb2c3c3 --- /dev/null +++ b/gix-protocol/src/fetch/types.rs @@ -0,0 +1,243 @@ +use crate::fetch::response::{Acknowledgement, ShallowUpdate, WantedRef}; +use std::path::PathBuf; + +/// Options for use in [`fetch()`](`crate::fetch()`) +#[derive(Debug, Clone)] +pub struct Options<'a> { + /// The path to the file containing the shallow commit boundary. + /// + /// When needed, it will be locked in preparation for being modified. + pub shallow_file: PathBuf, + /// How to deal with shallow repositories. It does affect how negotiations are performed. + pub shallow: &'a Shallow, + /// Describe how to handle tags when fetching. + pub tags: Tags, + /// The hash the remote repository is expected to use, as it's what the local repository is initialized as. + pub expected_object_hash: gix_hash::Kind, + /// If `true`, if we fetch from a remote that only offers shallow clones, the operation will fail with an error + /// instead of writing the shallow boundary to the shallow file. + pub reject_shallow_remote: bool, +} + +/// For use in [`RefMap::new()`] and [`fetch`](crate::fetch()). +#[cfg(feature = "handshake")] +pub struct Context<'a, T> { + /// The outcome of the handshake performed with the remote. + /// + /// Note that it's mutable as depending on the protocol, it may contain refs that have been sent unconditionally. + pub handshake: &'a mut crate::handshake::Outcome, + /// The transport to use when making an `ls-refs` or `fetch` call. + /// + /// This is always done if the underlying protocol is V2, which is implied by the absence of refs in the `handshake` outcome. + pub transport: &'a mut T, + /// How to self-identify during the `ls-refs` call in [`RefMap::new()`] or the `fetch` call in [`fetch()`](crate::fetch()). + /// + /// This could be read from the `gitoxide.userAgent` configuration variable. + pub user_agent: (&'static str, Option>), + /// If `true`, output all packetlines using the the `gix-trace` machinery. + pub trace_packetlines: bool, +} + +#[cfg(feature = "fetch")] +mod with_fetch { + use crate::fetch; + use crate::fetch::{negotiate, refmap}; + + /// For use in [`fetch`](crate::fetch()). + pub struct NegotiateContext<'a, 'b, 'c, Objects, Alternates, AlternatesOut, AlternatesErr, Find> + where + Objects: gix_object::Find + gix_object::FindHeader + gix_object::Exists, + Alternates: FnOnce() -> Result, + AlternatesErr: Into>, + AlternatesOut: Iterator, + Find: gix_object::Find, + { + /// Access to the object database. + /// *Note* that the `exists()` calls must not trigger a refresh of the ODB packs as plenty of them might fail, i.e. find on object. + pub objects: &'a Objects, + /// Access to the git references database. + pub refs: &'a gix_ref::file::Store, + /// A function that returns an iterator over `(refs, objects)` for each alternate repository, to assure all known objects are added also according to their tips. + pub alternates: Alternates, + /// The implementation that performs the negotiation later, i.e. prepare wants and haves. + pub negotiator: &'a mut dyn gix_negotiate::Negotiator, + /// The commit-graph for use by the `negotiator` - we populate it with tips to initialize the graph traversal. + pub graph: &'a mut gix_negotiate::Graph<'b, 'c>, + } + + /// A trait to encapsulate steps to negotiate the contents of the pack. + /// + /// Typical implementations use the utilities found in the [`negotiate`] module. + pub trait Negotiate { + /// Typically invokes [`negotiate::mark_complete_and_common_ref()`]. + fn mark_complete_and_common_ref(&mut self) -> Result; + /// Typically invokes [`negotiate::add_wants()`]. + fn add_wants(&mut self, arguments: &mut fetch::Arguments, remote_ref_target_known: &[bool]); + /// Typically invokes [`negotiate::one_round()`]. + fn one_round( + &mut self, + state: &mut negotiate::one_round::State, + arguments: &mut fetch::Arguments, + previous_response: Option<&fetch::Response>, + ) -> Result<(negotiate::Round, bool), negotiate::Error>; + } + + /// The outcome of [`fetch()`](crate::fetch()). + #[derive(Debug, Clone)] + pub struct Outcome { + /// The most recent server response. + /// + /// Useful to obtain information about new shallow boundaries. + pub last_response: fetch::Response, + /// Information about the negotiation to receive the new pack. + pub negotiate: NegotiateOutcome, + } + + /// The negotiation-specific outcome of [`fetch()`](crate::fetch()). + #[derive(Debug, Clone)] + pub struct NegotiateOutcome { + /// The outcome of the negotiation stage of the fetch operation. + /// + /// If it is… + /// + /// * [`negotiate::Action::MustNegotiate`] there will always be a `pack`. + /// * [`negotiate::Action::SkipToRefUpdate`] there is no `pack` but references can be updated right away. + /// + /// Note that this is never [negotiate::Action::NoChange`] as this would mean there is no negotiation information at all + /// so this structure wouldn't be present. + pub action: negotiate::Action, + /// Additional information for each round of negotiation. + pub rounds: Vec, + } + + /// Information about the relationship between our refspecs, and remote references with their local counterparts. + /// + /// It's the first stage that offers connection to the server, and is typically required to perform one or more fetch operations. + #[derive(Default, Debug, Clone)] + pub struct RefMap { + /// A mapping between a remote reference and a local tracking branch. + pub mappings: Vec, + /// The explicit refspecs that were supposed to be used for fetching. + /// + /// Typically, they are configured by the remote and are referred to by + /// [`refmap::SpecIndex::ExplicitInRemote`] in [`refmap::Mapping`]. + pub refspecs: Vec, + /// Refspecs which have been added implicitly due to settings of the `remote`, usually pre-initialized from + /// [`extra_refspecs` in RefMap options](refmap::init::Options). + /// They are referred to by [`refmap::SpecIndex::Implicit`] in [`refmap::Mapping`]. + /// + /// They are never persisted nor are they typically presented to the user. + pub extra_refspecs: Vec, + /// Information about the fixes applied to the `mapping` due to validation and sanitization. + pub fixes: Vec, + /// All refs advertised by the remote. + pub remote_refs: Vec, + /// The kind of hash used for all data sent by the server, if understood by this client implementation. + /// + /// It was extracted from the `handshake` as advertised by the server. + pub object_hash: gix_hash::Kind, + } +} +#[cfg(feature = "fetch")] +pub use with_fetch::*; + +/// Describe how shallow clones are handled when fetching, with variants defining how the *shallow boundary* is handled. +/// +/// The *shallow boundary* is a set of commits whose parents are not present in the repository. +#[derive(Default, Debug, Clone, PartialEq, Eq)] +pub enum Shallow { + /// Fetch all changes from the remote without affecting the shallow boundary at all. + /// + /// This also means that repositories that aren't shallow will remain like that. + #[default] + NoChange, + /// Receive update to `depth` commits in the history of the refs to fetch (from the viewpoint of the remote), + /// with the value of `1` meaning to receive only the commit a ref is pointing to. + /// + /// This may update the shallow boundary to increase or decrease the amount of available history. + DepthAtRemote(std::num::NonZeroU32), + /// Increase the number of commits and thus expand the shallow boundary by `depth` commits as seen from our local + /// shallow boundary, with a value of `0` having no effect. + Deepen(u32), + /// Set the shallow boundary at the `cutoff` time, meaning that there will be no commits beyond that time. + Since { + /// The date beyond which there will be no history. + cutoff: gix_date::Time, + }, + /// Receive all history excluding all commits reachable from `remote_refs`. These can be long or short + /// ref names or tag names. + Exclude { + /// The ref names to exclude, short or long. Note that ambiguous short names will cause the remote to abort + /// without an error message being transferred (because the protocol does not support it) + remote_refs: Vec, + /// If some, this field has the same meaning as [`Shallow::Since`] which can be used in combination + /// with excluded references. + since_cutoff: Option, + }, +} + +impl Shallow { + /// Produce a variant that causes the repository to loose its shallow boundary, effectively by extending it + /// beyond all limits. + pub fn undo() -> Self { + Shallow::DepthAtRemote((i32::MAX as u32).try_into().expect("valid at compile time")) + } +} + +/// Describe how to handle tags when fetching +#[derive(Default, Debug, Copy, Clone, PartialEq, Eq)] +pub enum Tags { + /// Fetch all tags from the remote, even if these are not reachable from objects referred to by our refspecs. + All, + /// Fetch only the tags that point to the objects being sent. + /// That way, annotated tags that point to an object we receive are automatically transmitted and their refs are created. + /// The same goes for lightweight tags. + #[default] + Included, + /// Do not fetch any tags. + None, +} + +impl Tags { + /// Obtain a refspec that determines whether or not to fetch all tags, depending on this variant. + /// + /// The returned refspec is the default refspec for tags, but won't overwrite local tags ever. + #[cfg(feature = "fetch")] + pub fn to_refspec(&self) -> Option> { + match self { + Tags::All | Tags::Included => Some( + gix_refspec::parse("refs/tags/*:refs/tags/*".into(), gix_refspec::parse::Operation::Fetch) + .expect("valid"), + ), + Tags::None => None, + } + } +} + +/// A representation of a complete fetch response +#[derive(Debug, Clone)] +pub struct Response { + pub(crate) acks: Vec, + pub(crate) shallows: Vec, + pub(crate) wanted_refs: Vec, + pub(crate) has_pack: bool, +} + +/// The progress ids used in during various steps of the fetch operation. +/// +/// Note that tagged progress isn't very widely available yet, but support can be improved as needed. +/// +/// Use this information to selectively extract the progress of interest in case the parent application has custom visualization. +#[derive(Debug, Copy, Clone)] +pub enum ProgressId { + /// The progress name is defined by the remote and the progress messages it sets, along with their progress values and limits. + RemoteProgress, +} + +impl From for gix_features::progress::Id { + fn from(v: ProgressId) -> Self { + match v { + ProgressId::RemoteProgress => *b"FERP", + } + } +} diff --git a/gix-protocol/src/fetch_fn.rs b/gix-protocol/src/fetch_fn.rs deleted file mode 100644 index 2cf60fdc4fd..00000000000 --- a/gix-protocol/src/fetch_fn.rs +++ /dev/null @@ -1,183 +0,0 @@ -use std::borrow::Cow; - -use gix_features::progress::NestedProgress; -use gix_transport::client; -use maybe_async::maybe_async; - -use crate::{ - credentials, - fetch::{Action, Arguments, Delegate, Error, Response}, - indicate_end_of_interaction, Command, -}; - -/// A way to indicate how to treat the connection underlying the transport, potentially allowing to reuse it. -#[derive(Default, Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub enum FetchConnection { - /// Use this variant if server should be informed that the operation is completed and no further commands will be issued - /// at the end of the fetch operation or after deciding that no fetch operation should happen after references were listed. - /// - /// When indicating the end-of-fetch, this flag is only relevant in protocol V2. - /// Generally it only applies when using persistent transports. - /// - /// In most explicit client side failure modes the end-of-operation' notification will be sent to the server automatically. - #[default] - TerminateOnSuccessfulCompletion, - - /// Indicate that persistent transport connections can be reused by _not_ sending an 'end-of-operation' notification to the server. - /// This is useful if multiple `fetch(…)` calls are used in succession. - /// - /// Note that this has no effect in case of non-persistent connections, like the ones over HTTP. - /// - /// As an optimization, callers can use `AllowReuse` here as the server will also know the client is done - /// if the connection is closed. - AllowReuse, -} - -/// Perform a 'fetch' operation with the server using `transport`, with `delegate` handling all server interactions. -/// **Note** that `delegate` has blocking operations and thus this entire call should be on an executor which can handle -/// that. This could be the current thread blocking, or another thread. -/// -/// * `authenticate(operation_to_perform)` is used to receive credentials for the connection and potentially store it -/// if the server indicates 'permission denied'. Note that not all transport support authentication or authorization. -/// * `progress` is used to emit progress messages. -/// * `name` is the name of the git client to present as `agent`, like `"my-app (v2.0)"`". -/// * If `trace` is `true`, all packetlines received or sent will be passed to the facilities of the `gix-trace` crate. -/// -/// _Note_ that depending on the `delegate`, the actual action performed can be `ls-refs`, `clone` or `fetch`. -/// -/// # WARNING - Do not use! -/// -/// As it will hang when having multiple negotiation rounds. -#[allow(clippy::result_large_err)] -#[maybe_async] -// TODO: remove this without losing test coverage - we have the same but better in `gix` and it's -// not really worth it to maintain the delegates here. -pub async fn fetch( - mut transport: T, - mut delegate: D, - authenticate: F, - mut progress: P, - fetch_mode: FetchConnection, - agent: impl Into, - trace: bool, -) -> Result<(), Error> -where - F: FnMut(credentials::helper::Action) -> credentials::protocol::Result, - D: Delegate, - T: client::Transport, - P: NestedProgress + 'static, - P::SubProgress: 'static, -{ - let crate::handshake::Outcome { - server_protocol_version: protocol_version, - refs, - v1_shallow_updates: _ignored_shallow_updates_as_it_is_deprecated, - capabilities, - } = crate::fetch::handshake( - &mut transport, - authenticate, - delegate.handshake_extra_parameters(), - &mut progress, - ) - .await?; - - let agent = crate::agent(agent); - let refs = match refs { - Some(refs) => refs, - None => { - crate::ls_refs( - &mut transport, - &capabilities, - |a, b, c| { - let res = delegate.prepare_ls_refs(a, b, c); - c.push(("agent", Some(Cow::Owned(agent.clone())))); - res - }, - &mut progress, - trace, - ) - .await? - } - }; - - let fetch = Command::Fetch; - let mut fetch_features = fetch.default_features(protocol_version, &capabilities); - match delegate.prepare_fetch(protocol_version, &capabilities, &mut fetch_features, &refs) { - Ok(Action::Cancel) => { - return if matches!(protocol_version, gix_transport::Protocol::V1) - || matches!(fetch_mode, FetchConnection::TerminateOnSuccessfulCompletion) - { - indicate_end_of_interaction(transport, trace).await.map_err(Into::into) - } else { - Ok(()) - }; - } - Ok(Action::Continue) => { - fetch.validate_argument_prefixes_or_panic(protocol_version, &capabilities, &[], &fetch_features); - } - Err(err) => { - indicate_end_of_interaction(transport, trace).await?; - return Err(err.into()); - } - } - - Response::check_required_features(protocol_version, &fetch_features)?; - let sideband_all = fetch_features.iter().any(|(n, _)| *n == "sideband-all"); - fetch_features.push(("agent", Some(Cow::Owned(agent)))); - let mut arguments = Arguments::new(protocol_version, fetch_features, trace); - let mut previous_response = None::; - let mut round = 1; - 'negotiation: loop { - progress.step(); - progress.set_name(format!("negotiate (round {round})")); - round += 1; - let action = delegate.negotiate(&refs, &mut arguments, previous_response.as_ref())?; - let mut reader = arguments.send(&mut transport, action == Action::Cancel).await?; - if sideband_all { - setup_remote_progress(&mut progress, &mut reader); - } - let response = Response::from_line_reader( - protocol_version, - &mut reader, - true, /* hack, telling us we don't want this delegate approach anymore */ - false, /* just as much of a hack which causes us to expect a pack immediately */ - ) - .await?; - previous_response = if response.has_pack() { - progress.step(); - progress.set_name("receiving pack".into()); - if !sideband_all { - setup_remote_progress(&mut progress, &mut reader); - } - delegate.receive_pack(reader, progress, &refs, &response).await?; - break 'negotiation; - } else { - match action { - Action::Cancel => break 'negotiation, - Action::Continue => Some(response), - } - } - } - if matches!(protocol_version, gix_transport::Protocol::V2) - && matches!(fetch_mode, FetchConnection::TerminateOnSuccessfulCompletion) - { - indicate_end_of_interaction(transport, trace).await?; - } - Ok(()) -} - -fn setup_remote_progress

( - progress: &mut P, - reader: &mut Box + Unpin + '_>, -) where - P: NestedProgress, - P::SubProgress: 'static, -{ - reader.set_progress_handler(Some(Box::new({ - let mut remote_progress = progress.add_child("remote"); - move |is_err: bool, data: &[u8]| { - crate::RemoteProgress::translate_to_progress(is_err, data, &mut remote_progress); - gix_transport::packetline::read::ProgressAction::Continue - } - }) as gix_transport::client::HandleProgress<'_>)); -} diff --git a/gix-protocol/src/handshake/mod.rs b/gix-protocol/src/handshake/mod.rs index d704385cf4c..64ba19aa0eb 100644 --- a/gix-protocol/src/handshake/mod.rs +++ b/gix-protocol/src/handshake/mod.rs @@ -1,5 +1,7 @@ use bstr::BString; -use gix_transport::client::Capabilities; + +/// +pub mod refs; /// A git reference, commonly referred to as 'ref', as returned by a git server before sending a pack. #[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] @@ -51,6 +53,7 @@ pub enum Ref { /// The result of the [`handshake()`][super::handshake()] function. #[derive(Default, Debug, Clone)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg(feature = "handshake")] pub struct Outcome { /// The protocol version the server responded with. It might have downgraded the desired version. pub server_protocol_version: gix_transport::Protocol, @@ -58,11 +61,12 @@ pub struct Outcome { pub refs: Option>, /// Shallow updates as part of the `Protocol::V1`, to shallow a particular object. /// Note that unshallowing isn't supported here. - pub v1_shallow_updates: Option>, + pub v1_shallow_updates: Option>, /// The server capabilities. - pub capabilities: Capabilities, + pub capabilities: gix_transport::client::Capabilities, } +#[cfg(feature = "handshake")] mod error { use bstr::BString; use gix_transport::client; @@ -96,10 +100,9 @@ mod error { } } } -use crate::fetch::response::ShallowUpdate; +#[cfg(feature = "handshake")] pub use error::Error; +#[cfg(any(feature = "blocking-client", feature = "async-client"))] +#[cfg(feature = "handshake")] pub(crate) mod function; - -/// -pub mod refs; diff --git a/gix-protocol/src/handshake/refs/mod.rs b/gix-protocol/src/handshake/refs/mod.rs index 39c6c85a942..6cc90304e5c 100644 --- a/gix-protocol/src/handshake/refs/mod.rs +++ b/gix-protocol/src/handshake/refs/mod.rs @@ -74,6 +74,3 @@ pub use async_io::{from_v1_refs_received_as_part_of_handshake_and_capabilities, mod blocking_io; #[cfg(feature = "blocking-client")] pub use blocking_io::{from_v1_refs_received_as_part_of_handshake_and_capabilities, from_v2_refs}; - -#[cfg(test)] -mod tests; diff --git a/gix-protocol/src/handshake/refs/shared.rs b/gix-protocol/src/handshake/refs/shared.rs index aa0d94f292d..e321b25dbd8 100644 --- a/gix-protocol/src/handshake/refs/shared.rs +++ b/gix-protocol/src/handshake/refs/shared.rs @@ -285,3 +285,38 @@ pub(in crate::handshake::refs) fn parse_v2(line: &BStr) -> Result { _ => Err(Error::MalformedV2RefLine(trimmed.to_owned().into())), } } + +#[cfg(test)] +mod tests { + use gix_transport::client; + + use crate::handshake::{refs, refs::shared::InternalRef}; + + #[test] + fn extract_symbolic_references_from_capabilities() -> Result<(), client::Error> { + let caps = client::Capabilities::from_bytes( + b"\0unrelated symref=HEAD:refs/heads/main symref=ANOTHER:refs/heads/foo symref=MISSING_NAMESPACE_TARGET:(null) agent=git/2.28.0", + )? + .0; + let out = refs::shared::from_capabilities(caps.iter()).expect("a working example"); + + assert_eq!( + out, + vec![ + InternalRef::SymbolicForLookup { + path: "HEAD".into(), + target: Some("refs/heads/main".into()) + }, + InternalRef::SymbolicForLookup { + path: "ANOTHER".into(), + target: Some("refs/heads/foo".into()) + }, + InternalRef::SymbolicForLookup { + path: "MISSING_NAMESPACE_TARGET".into(), + target: None + } + ] + ); + Ok(()) + } +} diff --git a/gix-protocol/src/lib.rs b/gix-protocol/src/lib.rs index 6b4ee69d2b6..65cd7359b75 100644 --- a/gix-protocol/src/lib.rs +++ b/gix-protocol/src/lib.rs @@ -1,7 +1,14 @@ //! An abstraction over [fetching][fetch()] a pack from the server. //! -//! This implementation hides the transport layer, statefulness and the protocol version to the [fetch delegate][fetch::Delegate], -//! the actual client implementation. +//! Generally, there is the following order of operations. +//! +//! * create a [`Transport`](gix_transport::client::Transport) +//! * perform a [`handshake()`] +//! * execute a [`Command`] +//! - [list references](ls_refs()) +//! - create a mapping between [refspecs and references](fetch::RefMap) +//! - [receive a pack](fetch()) +//! //! ## Feature Flags #![cfg_attr( all(doc, feature = "document-features"), @@ -10,6 +17,12 @@ #![cfg_attr(all(doc, feature = "document-features"), feature(doc_cfg, doc_auto_cfg))] #![deny(missing_docs, rust_2018_idioms, unsafe_code)] +/// A function that performs a given credential action, trying to obtain credentials for an operation that needs it. +/// +/// Useful for both `fetch` and `push`. +#[cfg(feature = "handshake")] +pub type AuthenticateFn<'a> = Box gix_credentials::protocol::Result + 'a>; + /// A selector for V2 commands to invoke on the server for purpose of pre-invocation validation. #[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] pub enum Command { @@ -20,25 +33,22 @@ pub enum Command { } pub mod command; -#[cfg(feature = "async-trait")] +#[cfg(feature = "async-client")] pub use async_trait; -#[cfg(feature = "futures-io")] +#[cfg(feature = "async-client")] pub use futures_io; -#[cfg(feature = "futures-lite")] +#[cfg(feature = "async-client")] pub use futures_lite; +#[cfg(feature = "handshake")] pub use gix_credentials as credentials; /// A convenience export allowing users of gix-protocol to use the transport layer without their own cargo dependency. pub use gix_transport as transport; pub use maybe_async; /// -#[cfg(any(feature = "blocking-client", feature = "async-client"))] pub mod fetch; - -#[cfg(any(feature = "blocking-client", feature = "async-client"))] -mod fetch_fn; #[cfg(any(feature = "blocking-client", feature = "async-client"))] -pub use fetch_fn::{fetch, FetchConnection}; +pub use fetch::function::fetch; mod remote_progress; pub use remote_progress::RemoteProgress; @@ -47,18 +57,15 @@ pub use remote_progress::RemoteProgress; compile_error!("Cannot set both 'blocking-client' and 'async-client' features as they are mutually exclusive"); /// -#[cfg(any(feature = "blocking-client", feature = "async-client"))] pub mod handshake; #[cfg(any(feature = "blocking-client", feature = "async-client"))] +#[cfg(feature = "handshake")] pub use handshake::function::handshake; /// -#[cfg(any(feature = "blocking-client", feature = "async-client"))] pub mod ls_refs; #[cfg(any(feature = "blocking-client", feature = "async-client"))] pub use ls_refs::function::ls_refs; mod util; -pub use util::agent; -#[cfg(any(feature = "blocking-client", feature = "async-client"))] -pub use util::indicate_end_of_interaction; +pub use util::*; diff --git a/gix-protocol/src/ls_refs.rs b/gix-protocol/src/ls_refs.rs index 052576402a7..6bd15108124 100644 --- a/gix-protocol/src/ls_refs.rs +++ b/gix-protocol/src/ls_refs.rs @@ -1,3 +1,4 @@ +#[cfg(any(feature = "blocking-client", feature = "async-client"))] mod error { use crate::handshake::refs::parse; @@ -11,6 +12,8 @@ mod error { Transport(#[from] gix_transport::client::Error), #[error(transparent)] Parse(#[from] parse::Error), + #[error(transparent)] + ArgumentValidation(#[from] crate::command::validate_argument_prefixes::Error), } impl gix_transport::IsSpuriousError for Error { @@ -23,6 +26,7 @@ mod error { } } } +#[cfg(any(feature = "blocking-client", feature = "async-client"))] pub use error::Error; /// What to do after preparing ls-refs in [`ls_refs()`][crate::ls_refs()]. @@ -37,6 +41,7 @@ pub enum Action { Skip, } +#[cfg(any(feature = "blocking-client", feature = "async-client"))] pub(crate) mod function { use std::borrow::Cow; @@ -70,7 +75,7 @@ pub(crate) mod function { let _span = gix_features::trace::detail!("gix_protocol::ls_refs()", capabilities = ?capabilities); let ls_refs = Command::LsRefs; let mut ls_features = ls_refs.default_features(gix_transport::Protocol::V2, capabilities); - let mut ls_args = ls_refs.initial_arguments(&ls_features); + let mut ls_args = ls_refs.initial_v2_arguments(&ls_features); if capabilities .capability("ls-refs") .and_then(|cap| cap.supports("unborn")) @@ -81,12 +86,12 @@ pub(crate) mod function { let refs = match prepare_ls_refs(capabilities, &mut ls_args, &mut ls_features) { Ok(Action::Skip) => Vec::new(), Ok(Action::Continue) => { - ls_refs.validate_argument_prefixes_or_panic( + ls_refs.validate_argument_prefixes( gix_transport::Protocol::V2, capabilities, &ls_args, &ls_features, - ); + )?; progress.step(); progress.set_name("list refs".into()); diff --git a/gix-protocol/src/util.rs b/gix-protocol/src/util.rs index 09636d75200..adba4bb7784 100644 --- a/gix-protocol/src/util.rs +++ b/gix-protocol/src/util.rs @@ -6,25 +6,92 @@ pub fn agent(name: impl Into) -> String { } name } - -/// Send a message to indicate the remote side that there is nothing more to expect from us, indicating a graceful shutdown. -/// If `trace` is `true`, all packetlines received or sent will be passed to the facilities of the `gix-trace` crate. #[cfg(any(feature = "blocking-client", feature = "async-client"))] -#[maybe_async::maybe_async] -pub async fn indicate_end_of_interaction( - mut transport: impl gix_transport::client::Transport, - trace: bool, -) -> Result<(), gix_transport::client::Error> { - // An empty request marks the (early) end of the interaction. Only relevant in stateful transports though. - if transport.connection_persists_across_multiple_requests() { - transport - .request( - gix_transport::client::WriteMode::Binary, - gix_transport::client::MessageKind::Flush, - trace, - )? - .into_read() - .await?; +mod with_transport { + use gix_transport::client::Transport; + + /// Send a message to indicate the remote side that there is nothing more to expect from us, indicating a graceful shutdown. + /// If `trace` is `true`, all packetlines received or sent will be passed to the facilities of the `gix-trace` crate. + #[maybe_async::maybe_async] + pub async fn indicate_end_of_interaction( + mut transport: impl gix_transport::client::Transport, + trace: bool, + ) -> Result<(), gix_transport::client::Error> { + // An empty request marks the (early) end of the interaction. Only relevant in stateful transports though. + if transport.connection_persists_across_multiple_requests() { + transport + .request( + gix_transport::client::WriteMode::Binary, + gix_transport::client::MessageKind::Flush, + trace, + )? + .into_read() + .await?; + } + Ok(()) + } + + /// A utility to automatically send a flush packet when the instance is dropped, assuring a graceful termination of any + /// interaction with the server. + pub struct SendFlushOnDrop + where + T: Transport, + { + /// The actual transport instance. + pub inner: T, + /// If `true`, the packetline used to indicate the end of interaction will be traced using `gix-trace`. + trace_packetlines: bool, + /// If `true`, we should not send another flush packet. + flush_packet_sent: bool, + } + + impl SendFlushOnDrop + where + T: Transport, + { + /// Create a new instance with `transport`, while optionally tracing packetlines with `trace_packetlines`. + pub fn new(transport: T, trace_packetlines: bool) -> Self { + Self { + inner: transport, + trace_packetlines, + flush_packet_sent: false, + } + } + + /// Useful to explicitly invalidate the connection by sending a flush-packet. + /// This will happen exactly once, and it is not considered an error to call it multiple times. + /// + /// For convenience, this is not consuming, but could be to assure the underlying transport isn't used anymore. + #[maybe_async::maybe_async] + pub async fn indicate_end_of_interaction(&mut self) -> Result<(), gix_transport::client::Error> { + if self.flush_packet_sent { + return Ok(()); + } + + self.flush_packet_sent = true; + indicate_end_of_interaction(&mut self.inner, self.trace_packetlines).await + } + } + + impl Drop for SendFlushOnDrop + where + T: Transport, + { + fn drop(&mut self) { + #[cfg(feature = "async-client")] + { + // TODO: this should be an async drop once the feature is available. + // Right now we block the executor by forcing this communication, but that only + // happens if the user didn't actually try to receive a pack, which consumes the + // connection in an async context. + crate::futures_lite::future::block_on(self.indicate_end_of_interaction()).ok(); + } + #[cfg(not(feature = "async-client"))] + { + self.indicate_end_of_interaction().ok(); + } + } } - Ok(()) } +#[cfg(any(feature = "blocking-client", feature = "async-client"))] +pub use with_transport::*; diff --git a/gix-protocol/tests/async-protocol.rs b/gix-protocol/tests/async-protocol.rs index 772e2bd386d..a7794a69bac 100644 --- a/gix-protocol/tests/async-protocol.rs +++ b/gix-protocol/tests/async-protocol.rs @@ -1,9 +1,2 @@ -type Result = std::result::Result<(), Box>; - -pub fn fixture_bytes(path: &str) -> Vec { - std::fs::read(std::path::PathBuf::from("tests").join("fixtures").join(path)) - .expect("fixture to be present and readable") -} - -mod fetch; -mod remote_progress; +mod protocol; +pub use protocol::*; diff --git a/gix-protocol/tests/blocking-protocol.rs b/gix-protocol/tests/blocking-protocol.rs index 772e2bd386d..a7794a69bac 100644 --- a/gix-protocol/tests/blocking-protocol.rs +++ b/gix-protocol/tests/blocking-protocol.rs @@ -1,9 +1,2 @@ -type Result = std::result::Result<(), Box>; - -pub fn fixture_bytes(path: &str) -> Vec { - std::fs::read(std::path::PathBuf::from("tests").join("fixtures").join(path)) - .expect("fixture to be present and readable") -} - -mod fetch; -mod remote_progress; +mod protocol; +pub use protocol::*; diff --git a/gix-protocol/src/command/tests.rs b/gix-protocol/tests/protocol/command.rs similarity index 68% rename from gix-protocol/src/command/tests.rs rename to gix-protocol/tests/protocol/command.rs index ac44a6c036c..9f878188d32 100644 --- a/gix-protocol/src/command/tests.rs +++ b/gix-protocol/tests/protocol/command.rs @@ -8,10 +8,8 @@ mod v1 { const GITHUB_CAPABILITIES: &str = "multi_ack thin-pack side-band ofs-delta shallow deepen-since deepen-not deepen-relative no-progress include-tag allow-tip-sha1-in-want allow-reachable-sha1-in-want no-done symref=HEAD:refs/heads/main filter agent=git/github-gdf51a71f0236"; mod fetch { mod default_features { - use crate::{ - command::tests::v1::{capabilities, GITHUB_CAPABILITIES}, - Command, - }; + use super::super::{capabilities, GITHUB_CAPABILITIES}; + use gix_protocol::Command; #[test] fn it_chooses_the_best_multi_ack_and_sideband() { @@ -60,7 +58,8 @@ mod v2 { mod fetch { mod default_features { - use crate::{command::tests::v2::capabilities, Command}; + use super::super::capabilities; + use gix_protocol::Command; #[test] fn all_features() { @@ -80,12 +79,13 @@ mod v2 { mod initial_arguments { use bstr::ByteSlice; - use crate::{command::tests::v2::capabilities, Command}; + use super::super::capabilities; + use gix_protocol::Command; #[test] fn for_all_features() { assert_eq!( - Command::Fetch.initial_arguments(&Command::Fetch.default_features( + Command::Fetch.initial_v2_arguments(&Command::Fetch.default_features( gix_transport::Protocol::V2, &capabilities("fetch", "shallow filter sideband-all packfile-uris") )), @@ -101,7 +101,8 @@ mod v2 { mod ls_refs { mod default_features { - use crate::{command::tests::v2::capabilities, Command}; + use super::super::capabilities; + use gix_protocol::Command; #[test] fn default_as_there_are_no_features() { @@ -118,37 +119,50 @@ mod v2 { mod validate { use bstr::ByteSlice; - use crate::{command::tests::v2::capabilities, Command}; + use super::super::capabilities; + use gix_protocol::Command; #[test] fn ref_prefixes_can_always_be_used() { - Command::LsRefs.validate_argument_prefixes_or_panic( - gix_transport::Protocol::V2, - &capabilities("something else", "do-not-matter"), - &[b"ref-prefix hello/".as_bstr().into()], - &[], - ); + assert!(Command::LsRefs + .validate_argument_prefixes( + gix_transport::Protocol::V2, + &capabilities("something else", "do-not-matter"), + &[b"ref-prefix hello/".as_bstr().into()], + &[], + ) + .is_ok()); } #[test] - #[should_panic] fn unknown_argument() { - Command::LsRefs.validate_argument_prefixes_or_panic( - gix_transport::Protocol::V2, - &capabilities("other", "do-not-matter"), - &[b"definitely-nothing-we-know".as_bstr().into()], - &[], + assert_eq!( + Command::LsRefs + .validate_argument_prefixes( + gix_transport::Protocol::V2, + &capabilities("other", "do-not-matter"), + &[b"definitely-nothing-we-know".as_bstr().into()], + &[], + ) + .unwrap_err() + .to_string(), + "ls-refs: argument definitely-nothing-we-know is not known or allowed" ); } #[test] - #[should_panic] fn unknown_feature() { - Command::LsRefs.validate_argument_prefixes_or_panic( - gix_transport::Protocol::V2, - &capabilities("other", "do-not-matter"), - &[], - &[("some-feature-that-does-not-exist", None)], + assert_eq!( + Command::LsRefs + .validate_argument_prefixes( + gix_transport::Protocol::V2, + &capabilities("other", "do-not-matter"), + &[], + &[("some-feature-that-does-not-exist", None)], + ) + .unwrap_err() + .to_string(), + "ls-refs: capability some-feature-that-does-not-exist is not supported" ); } } diff --git a/gix-protocol/tests/protocol/fetch/_impl.rs b/gix-protocol/tests/protocol/fetch/_impl.rs new file mode 100644 index 00000000000..8570e1809fd --- /dev/null +++ b/gix-protocol/tests/protocol/fetch/_impl.rs @@ -0,0 +1,504 @@ +mod fetch_fn { + use std::borrow::Cow; + + use gix_features::progress::NestedProgress; + use gix_transport::client; + use maybe_async::maybe_async; + + use super::{Action, Delegate}; + use crate::fetch::Error; + use gix_protocol::{ + credentials, + fetch::{Arguments, Response}, + indicate_end_of_interaction, Command, + }; + + /// A way to indicate how to treat the connection underlying the transport, potentially allowing to reuse it. + #[derive(Default, Debug, Copy, Clone, PartialEq, Eq, Hash)] + pub enum FetchConnection { + /// Use this variant if server should be informed that the operation is completed and no further commands will be issued + /// at the end of the fetch operation or after deciding that no fetch operation should happen after references were listed. + /// + /// When indicating the end-of-fetch, this flag is only relevant in protocol V2. + /// Generally it only applies when using persistent transports. + /// + /// In most explicit client side failure modes the end-of-operation' notification will be sent to the server automatically. + #[default] + TerminateOnSuccessfulCompletion, + + /// Indicate that persistent transport connections can be reused by _not_ sending an 'end-of-operation' notification to the server. + /// This is useful if multiple `fetch(…)` calls are used in succession. + /// + /// Note that this has no effect in case of non-persistent connections, like the ones over HTTP. + /// + /// As an optimization, callers can use `AllowReuse` here as the server will also know the client is done + /// if the connection is closed. + AllowReuse, + } + + /// Perform a 'fetch' operation with the server using `transport`, with `delegate` handling all server interactions. + /// **Note** that `delegate` has blocking operations and thus this entire call should be on an executor which can handle + /// that. This could be the current thread blocking, or another thread. + /// + /// * `authenticate(operation_to_perform)` is used to receive credentials for the connection and potentially store it + /// if the server indicates 'permission denied'. Note that not all transport support authentication or authorization. + /// * `progress` is used to emit progress messages. + /// * `name` is the name of the git client to present as `agent`, like `"my-app (v2.0)"`". + /// * If `trace` is `true`, all packetlines received or sent will be passed to the facilities of the `gix-trace` crate. + /// + /// _Note_ that depending on the `delegate`, the actual action performed can be `ls-refs`, `clone` or `fetch`. + /// + /// # WARNING - Do not use! + /// + /// As it will hang when having multiple negotiation rounds. + #[allow(clippy::result_large_err)] + #[maybe_async] + // TODO: remove this without losing test coverage - we have the same but better in `gix` and it's + // not really worth it to maintain the delegates here. + pub async fn legacy_fetch( + mut transport: T, + mut delegate: D, + authenticate: F, + mut progress: P, + fetch_mode: FetchConnection, + agent: impl Into, + trace: bool, + ) -> Result<(), Error> + where + F: FnMut(credentials::helper::Action) -> credentials::protocol::Result, + D: Delegate, + T: client::Transport, + P: NestedProgress + 'static, + P::SubProgress: 'static, + { + let gix_protocol::handshake::Outcome { + server_protocol_version: protocol_version, + refs, + v1_shallow_updates: _ignored_shallow_updates_as_it_is_deprecated, + capabilities, + } = gix_protocol::fetch::handshake( + &mut transport, + authenticate, + delegate.handshake_extra_parameters(), + &mut progress, + ) + .await?; + + let agent = gix_protocol::agent(agent); + let refs = match refs { + Some(refs) => refs, + None => { + gix_protocol::ls_refs( + &mut transport, + &capabilities, + |a, b, c| { + let res = delegate.prepare_ls_refs(a, b, c); + c.push(("agent", Some(Cow::Owned(agent.clone())))); + res + }, + &mut progress, + trace, + ) + .await? + } + }; + + let fetch = Command::Fetch; + let mut fetch_features = fetch.default_features(protocol_version, &capabilities); + match delegate.prepare_fetch(protocol_version, &capabilities, &mut fetch_features, &refs) { + Ok(Action::Cancel) => { + return if matches!(protocol_version, gix_transport::Protocol::V1) + || matches!(fetch_mode, FetchConnection::TerminateOnSuccessfulCompletion) + { + indicate_end_of_interaction(transport, trace).await.map_err(Into::into) + } else { + Ok(()) + }; + } + Ok(Action::Continue) => { + fetch + .validate_argument_prefixes(protocol_version, &capabilities, &[], &fetch_features) + .expect("BUG: delegates must always produce valid arguments"); + } + Err(err) => { + indicate_end_of_interaction(transport, trace).await?; + return Err(err.into()); + } + } + + Response::check_required_features(protocol_version, &fetch_features)?; + let sideband_all = fetch_features.iter().any(|(n, _)| *n == "sideband-all"); + fetch_features.push(("agent", Some(Cow::Owned(agent)))); + let mut arguments = Arguments::new(protocol_version, fetch_features, trace); + let mut previous_response = None::; + let mut round = 1; + 'negotiation: loop { + progress.step(); + progress.set_name(format!("negotiate (round {round})")); + round += 1; + let action = delegate.negotiate(&refs, &mut arguments, previous_response.as_ref())?; + let mut reader = arguments.send(&mut transport, action == Action::Cancel).await?; + if sideband_all { + setup_remote_progress(&mut progress, &mut reader); + } + let response = Response::from_line_reader( + protocol_version, + &mut reader, + true, /* hack, telling us we don't want this delegate approach anymore */ + false, /* just as much of a hack which causes us to expect a pack immediately */ + ) + .await?; + previous_response = if response.has_pack() { + progress.step(); + progress.set_name("receiving pack".into()); + if !sideband_all { + setup_remote_progress(&mut progress, &mut reader); + } + delegate.receive_pack(reader, progress, &refs, &response).await?; + break 'negotiation; + } else { + match action { + Action::Cancel => break 'negotiation, + Action::Continue => Some(response), + } + } + } + if matches!(protocol_version, gix_transport::Protocol::V2) + && matches!(fetch_mode, FetchConnection::TerminateOnSuccessfulCompletion) + { + indicate_end_of_interaction(transport, trace).await?; + } + Ok(()) + } + + fn setup_remote_progress

( + progress: &mut P, + reader: &mut Box + Unpin + '_>, + ) where + P: NestedProgress, + P::SubProgress: 'static, + { + reader.set_progress_handler(Some(Box::new({ + let mut remote_progress = progress.add_child("remote"); + move |is_err: bool, data: &[u8]| { + gix_protocol::RemoteProgress::translate_to_progress(is_err, data, &mut remote_progress); + gix_transport::packetline::read::ProgressAction::Continue + } + }) as gix_transport::client::HandleProgress<'_>)); + } +} +pub use fetch_fn::{legacy_fetch as fetch, FetchConnection}; + +mod delegate { + use std::{ + borrow::Cow, + io, + ops::{Deref, DerefMut}, + }; + + use bstr::BString; + use gix_transport::client::Capabilities; + + use gix_protocol::ls_refs; + use gix_protocol::{ + fetch::{Arguments, Response}, + handshake::Ref, + }; + + /// Defines what to do next after certain [`Delegate`] operations. + #[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] + pub enum Action { + /// Continue the typical flow of operations in this flow. + Continue, + /// Return at the next possible opportunity without making further requests, possibly after closing the connection. + Cancel, + } + + /// The non-IO protocol delegate is the bare minimal interface needed to fully control the [`fetch`][gix_protocol::fetch()] operation, sparing + /// the IO parts. + /// Async implementations must treat it as blocking and unblock it by evaluating it elsewhere. + /// + /// See [Delegate] for the complete trait. + pub trait DelegateBlocking { + /// Return extra parameters to be provided during the handshake. + /// + /// Note that this method is only called once and the result is reused during subsequent handshakes which may happen + /// if there is an authentication failure. + fn handshake_extra_parameters(&self) -> Vec<(String, Option)> { + Vec::new() + } + /// Called before invoking 'ls-refs' on the server to allow providing it with additional `arguments` and to enable `features`. + /// If the server `capabilities` don't match the requirements abort with an error to abort the entire fetch operation. + /// + /// Note that some arguments are preset based on typical use, and `features` are preset to maximize options. + /// The `server` capabilities can be used to see which additional capabilities the server supports as per the handshake which happened prior. + /// + /// If the delegate returns [`ls_refs::Action::Skip`], no `ls-refs` command is sent to the server. + /// + /// Note that this is called only if we are using protocol version 2. + fn prepare_ls_refs( + &mut self, + _server: &Capabilities, + _arguments: &mut Vec, + _features: &mut Vec<(&str, Option>)>, + ) -> std::io::Result { + Ok(ls_refs::Action::Continue) + } + + /// Called before invoking the 'fetch' interaction with `features` pre-filled for typical use + /// and to maximize capabilities to allow aborting an interaction early. + /// + /// `refs` is a list of known references on the remote based on the handshake or a prior call to `ls_refs`. + /// These can be used to abort early in case the refs are already known here. + /// + /// As there will be another call allowing to post arguments conveniently in the correct format, i.e. `want hex-oid`, + /// there is no way to set arguments at this time. + /// + /// `version` is the actually supported version as reported by the server, which is relevant in case the server requested a downgrade. + /// `server` capabilities is a list of features the server supports for your information, along with enabled `features` that the server knows about. + fn prepare_fetch( + &mut self, + _version: gix_transport::Protocol, + _server: &Capabilities, + _features: &mut Vec<(&str, Option>)>, + _refs: &[Ref], + ) -> std::io::Result { + Ok(Action::Continue) + } + + /// A method called repeatedly to negotiate the objects to receive in [`receive_pack(…)`][Delegate::receive_pack()]. + /// + /// The first call has `previous_response` set to `None` as there was no previous response. Every call that follows `previous_response` + /// will be set to `Some`. + /// + /// ### If `previous_response` is `None`… + /// + /// Given a list of `arguments` to populate with wants, want-refs, shallows, filters and other contextual information to be + /// sent to the server. This method is called once. + /// Send the objects you `have` have afterwards based on the tips of your refs, in preparation to walk down their parents + /// with each call to `negotiate` to find the common base(s). + /// + /// Note that you should not `want` and object that you already have. + /// `refs` are the tips of on the server side, effectively the latest objects _they_ have. + /// + /// Return `Action::Close` if you know that there are no `haves` on your end to allow the server to send all of its objects + /// as is the case during initial clones. + /// + /// ### If `previous_response` is `Some`… + /// + /// Populate `arguments` with the objects you `have` starting from the tips of _your_ refs, taking into consideration + /// the `previous_response` response of the server to see which objects they acknowledged to have. You have to maintain + /// enough state to be able to walk down from your tips on each call, if they are not in common, and keep setting `have` + /// for those which are in common if that helps teaching the server about our state and to acknowledge their existence on _their_ end. + /// This method is called until the other side signals they are ready to send a pack. + /// Return `Action::Close` if you want to give up before finding a common base. This can happen if the remote repository + /// has radically changed so there are no bases, or they are very far in the past, causing all objects to be sent. + fn negotiate( + &mut self, + refs: &[Ref], + arguments: &mut Arguments, + previous_response: Option<&Response>, + ) -> io::Result; + } + + impl DelegateBlocking for Box { + fn handshake_extra_parameters(&self) -> Vec<(String, Option)> { + self.deref().handshake_extra_parameters() + } + + fn prepare_ls_refs( + &mut self, + _server: &Capabilities, + _arguments: &mut Vec, + _features: &mut Vec<(&str, Option>)>, + ) -> io::Result { + self.deref_mut().prepare_ls_refs(_server, _arguments, _features) + } + + fn prepare_fetch( + &mut self, + _version: gix_transport::Protocol, + _server: &Capabilities, + _features: &mut Vec<(&str, Option>)>, + _refs: &[Ref], + ) -> io::Result { + self.deref_mut().prepare_fetch(_version, _server, _features, _refs) + } + + fn negotiate( + &mut self, + refs: &[Ref], + arguments: &mut Arguments, + previous_response: Option<&Response>, + ) -> io::Result { + self.deref_mut().negotiate(refs, arguments, previous_response) + } + } + + impl DelegateBlocking for &mut T { + fn handshake_extra_parameters(&self) -> Vec<(String, Option)> { + self.deref().handshake_extra_parameters() + } + + fn prepare_ls_refs( + &mut self, + _server: &Capabilities, + _arguments: &mut Vec, + _features: &mut Vec<(&str, Option>)>, + ) -> io::Result { + self.deref_mut().prepare_ls_refs(_server, _arguments, _features) + } + + fn prepare_fetch( + &mut self, + _version: gix_transport::Protocol, + _server: &Capabilities, + _features: &mut Vec<(&str, Option>)>, + _refs: &[Ref], + ) -> io::Result { + self.deref_mut().prepare_fetch(_version, _server, _features, _refs) + } + + fn negotiate( + &mut self, + refs: &[Ref], + arguments: &mut Arguments, + previous_response: Option<&Response>, + ) -> io::Result { + self.deref_mut().negotiate(refs, arguments, previous_response) + } + } + + #[cfg(feature = "blocking-client")] + mod blocking_io { + use std::{ + io::{self, BufRead}, + ops::DerefMut, + }; + + use gix_features::progress::NestedProgress; + + use super::DelegateBlocking; + use gix_protocol::{fetch::Response, handshake::Ref}; + + /// The protocol delegate is the bare minimal interface needed to fully control the [`fetch`][gix_protocol::fetch()] operation. + /// + /// Implementations of this trait are controlled by code with intricate knowledge about how fetching works in protocol version V1 and V2, + /// so you don't have to. + /// Everything is tucked away behind type-safety so 'nothing can go wrong'©. Runtime assertions assure invalid + /// features or arguments don't make it to the server in the first place. + /// Please note that this trait mostly corresponds to what V2 would look like, even though V1 is supported as well. + pub trait Delegate: DelegateBlocking { + /// Receive a pack provided from the given `input`. + /// + /// Use `progress` to emit your own progress messages when decoding the pack. + /// + /// `refs` of the remote side are provided for convenience, along with the parsed `previous_response` response in case you want + /// to check additional acks. + fn receive_pack( + &mut self, + input: impl io::BufRead, + progress: impl NestedProgress + 'static, + refs: &[Ref], + previous_response: &Response, + ) -> io::Result<()>; + } + + impl Delegate for Box { + fn receive_pack( + &mut self, + input: impl BufRead, + progress: impl NestedProgress + 'static, + refs: &[Ref], + previous_response: &Response, + ) -> io::Result<()> { + self.deref_mut().receive_pack(input, progress, refs, previous_response) + } + } + + impl Delegate for &mut T { + fn receive_pack( + &mut self, + input: impl BufRead, + progress: impl NestedProgress + 'static, + refs: &[Ref], + previous_response: &Response, + ) -> io::Result<()> { + self.deref_mut().receive_pack(input, progress, refs, previous_response) + } + } + } + #[cfg(feature = "blocking-client")] + pub use blocking_io::Delegate; + + #[cfg(feature = "async-client")] + mod async_io { + use std::{io, ops::DerefMut}; + + use async_trait::async_trait; + use futures_io::AsyncBufRead; + use gix_features::progress::NestedProgress; + + use super::DelegateBlocking; + use gix_protocol::{fetch::Response, handshake::Ref}; + + /// The protocol delegate is the bare minimal interface needed to fully control the [`fetch`][gix_protocol::fetch()] operation. + /// + /// Implementations of this trait are controlled by code with intricate knowledge about how fetching works in protocol version V1 and V2, + /// so you don't have to. + /// Everything is tucked away behind type-safety so 'nothing can go wrong'©. Runtime assertions assure invalid + /// features or arguments don't make it to the server in the first place. + /// Please note that this trait mostly corresponds to what V2 would look like, even though V1 is supported as well. + #[async_trait(?Send)] + pub trait Delegate: DelegateBlocking { + /// Receive a pack provided from the given `input`, and the caller should consider it to be blocking as + /// most operations on the received pack are implemented in a blocking fashion. + /// + /// Use `progress` to emit your own progress messages when decoding the pack. + /// + /// `refs` of the remote side are provided for convenience, along with the parsed `previous_response` response in case you want + /// to check additional acks. + async fn receive_pack( + &mut self, + input: impl AsyncBufRead + Unpin + 'async_trait, + progress: impl NestedProgress + 'static, + refs: &[Ref], + previous_response: &Response, + ) -> io::Result<()>; + } + #[async_trait(?Send)] + impl Delegate for Box { + async fn receive_pack( + &mut self, + input: impl AsyncBufRead + Unpin + 'async_trait, + progress: impl NestedProgress + 'static, + refs: &[Ref], + previous_response: &Response, + ) -> io::Result<()> { + self.deref_mut() + .receive_pack(input, progress, refs, previous_response) + .await + } + } + + #[async_trait(?Send)] + impl Delegate for &mut T { + async fn receive_pack( + &mut self, + input: impl AsyncBufRead + Unpin + 'async_trait, + progress: impl NestedProgress + 'static, + refs: &[Ref], + previous_response: &Response, + ) -> io::Result<()> { + self.deref_mut() + .receive_pack(input, progress, refs, previous_response) + .await + } + } + } + #[cfg(feature = "async-client")] + pub use async_io::Delegate; +} +#[cfg(any(feature = "async-client", feature = "blocking-client"))] +pub use delegate::Delegate; +pub use delegate::{Action, DelegateBlocking}; diff --git a/gix-protocol/tests/protocol/fetch/arguments.rs b/gix-protocol/tests/protocol/fetch/arguments.rs new file mode 100644 index 00000000000..2dd9a982b9e --- /dev/null +++ b/gix-protocol/tests/protocol/fetch/arguments.rs @@ -0,0 +1,413 @@ +use bstr::ByteSlice; +use gix_transport::Protocol; + +use crate::fetch; + +fn arguments_v1(features: impl IntoIterator) -> fetch::Arguments { + fetch::Arguments::new(Protocol::V1, features.into_iter().map(|n| (n, None)).collect(), false) +} + +fn arguments_v2(features: impl IntoIterator) -> fetch::Arguments { + fetch::Arguments::new(Protocol::V2, features.into_iter().map(|n| (n, None)).collect(), false) +} + +struct Transport { + inner: T, + stateful: bool, +} + +#[cfg(feature = "blocking-client")] +mod impls { + use std::borrow::Cow; + + use bstr::BStr; + use gix_transport::{ + client, + client::{Error, MessageKind, RequestWriter, SetServiceResponse, WriteMode}, + Protocol, Service, + }; + + use super::Transport; + + impl client::TransportWithoutIO for Transport { + fn set_identity(&mut self, identity: client::Account) -> Result<(), Error> { + self.inner.set_identity(identity) + } + + fn request( + &mut self, + write_mode: WriteMode, + on_into_read: MessageKind, + trace: bool, + ) -> Result, Error> { + self.inner.request(write_mode, on_into_read, trace) + } + + fn to_url(&self) -> Cow<'_, BStr> { + self.inner.to_url() + } + + fn supported_protocol_versions(&self) -> &[Protocol] { + self.inner.supported_protocol_versions() + } + + fn connection_persists_across_multiple_requests(&self) -> bool { + self.stateful + } + + fn configure( + &mut self, + config: &dyn std::any::Any, + ) -> Result<(), Box> { + self.inner.configure(config) + } + } + + impl client::Transport for Transport { + fn handshake<'a>( + &mut self, + service: Service, + extra_parameters: &'a [(&'a str, Option<&'a str>)], + ) -> Result, Error> { + self.inner.handshake(service, extra_parameters) + } + } +} + +#[cfg(feature = "async-client")] +mod impls { + use std::borrow::Cow; + + use async_trait::async_trait; + use bstr::BStr; + use gix_transport::{ + client, + client::{Error, MessageKind, RequestWriter, SetServiceResponse, WriteMode}, + Protocol, Service, + }; + + use super::Transport; + impl client::TransportWithoutIO for Transport { + fn set_identity(&mut self, identity: client::Account) -> Result<(), Error> { + self.inner.set_identity(identity) + } + + fn request( + &mut self, + write_mode: WriteMode, + on_into_read: MessageKind, + trace: bool, + ) -> Result, Error> { + self.inner.request(write_mode, on_into_read, trace) + } + + fn to_url(&self) -> Cow<'_, BStr> { + self.inner.to_url() + } + + fn supported_protocol_versions(&self) -> &[Protocol] { + self.inner.supported_protocol_versions() + } + + fn connection_persists_across_multiple_requests(&self) -> bool { + self.stateful + } + + fn configure( + &mut self, + config: &dyn std::any::Any, + ) -> Result<(), Box> { + self.inner.configure(config) + } + } + + #[async_trait(?Send)] + impl client::Transport for Transport { + async fn handshake<'a>( + &mut self, + service: Service, + extra_parameters: &'a [(&'a str, Option<&'a str>)], + ) -> Result, Error> { + self.inner.handshake(service, extra_parameters).await + } + } +} + +fn transport( + out: &mut Vec, + stateful: bool, +) -> Transport>> { + Transport { + inner: gix_transport::client::git::Connection::new( + &[], + out, + Protocol::V1, // does not matter + b"does/not/matter".as_bstr().to_owned(), + None::<(&str, _)>, + gix_transport::client::git::ConnectMode::Process, // avoid header to be sent + false, + ), + stateful, + } +} + +fn id(hex: &str) -> gix_hash::ObjectId { + gix_hash::ObjectId::from_hex(hex.as_bytes()).expect("expect valid hex id") +} + +mod v1 { + use bstr::ByteSlice; + + use super::{arguments_v1, id, transport}; + + #[maybe_async::test(feature = "blocking-client", async(feature = "async-client", async_std::test))] + async fn include_tag() { + let mut out = Vec::new(); + let mut t = transport(&mut out, true); + let mut arguments = arguments_v1(["include-tag", "feature-b"].iter().copied()); + assert!(arguments.can_use_include_tag()); + + arguments.use_include_tag(); + arguments.want(id("ff333369de1221f9bfbbe03a3a13e9a09bc1ffff")); + arguments.send(&mut t, true).await.expect("sending to buffer to work"); + assert_eq!( + out.as_bstr(), + b"0048want ff333369de1221f9bfbbe03a3a13e9a09bc1ffff feature-b include-tag +00000009done +" + .as_bstr() + ); + } + + #[maybe_async::test(feature = "blocking-client", async(feature = "async-client", async_std::test))] + async fn no_include_tag() { + let mut out = Vec::new(); + let mut t = transport(&mut out, true); + let mut arguments = arguments_v1(["include-tag", "feature-b"].iter().copied()); + assert!(arguments.can_use_include_tag()); + + arguments.want(id("ff333369de1221f9bfbbe03a3a13e9a09bc1ffff")); + arguments.send(&mut t, true).await.expect("sending to buffer to work"); + assert_eq!( + out.as_bstr(), + b"003cwant ff333369de1221f9bfbbe03a3a13e9a09bc1ffff feature-b +00000009done +" + .as_bstr(), + "it's possible to not have it enabled, even though it's advertised by the server" + ); + } + + #[maybe_async::test(feature = "blocking-client", async(feature = "async-client", async_std::test))] + async fn haves_and_wants_for_clone() { + let mut out = Vec::new(); + let mut t = transport(&mut out, true); + let mut arguments = arguments_v1(["feature-a", "feature-b"].iter().copied()); + assert!( + !arguments.can_use_include_tag(), + "needs to be enabled by features in V1" + ); + + arguments.want(id("7b333369de1221f9bfbbe03a3a13e9a09bc1c907")); + arguments.want(id("ff333369de1221f9bfbbe03a3a13e9a09bc1ffff")); + arguments.send(&mut t, true).await.expect("sending to buffer to work"); + assert_eq!( + out.as_bstr(), + b"0046want 7b333369de1221f9bfbbe03a3a13e9a09bc1c907 feature-a feature-b +0032want ff333369de1221f9bfbbe03a3a13e9a09bc1ffff +00000009done +" + .as_bstr() + ); + } + + #[maybe_async::test(feature = "blocking-client", async(feature = "async-client", async_std::test))] + async fn haves_and_wants_for_fetch_stateless() { + let mut out = Vec::new(); + let mut t = transport(&mut out, false); + let mut arguments = arguments_v1(["feature-a", "shallow", "deepen-since", "deepen-not"].iter().copied()); + + arguments.deepen(1); + arguments.shallow(id("7b333369de1221f9bfbbe03a3a13e9a09bc1c9ff")); + arguments.want(id("7b333369de1221f9bfbbe03a3a13e9a09bc1c907")); + arguments.deepen_since(12345); + arguments.deepen_not("refs/heads/main".into()); + arguments.have(id("0000000000000000000000000000000000000000")); + arguments.send(&mut t, false).await.expect("sending to buffer to work"); + + arguments.have(id("1111111111111111111111111111111111111111")); + arguments.send(&mut t, true).await.expect("sending to buffer to work"); + assert_eq!( + out.as_bstr(), + b"005cwant 7b333369de1221f9bfbbe03a3a13e9a09bc1c907 feature-a shallow deepen-since deepen-not +0035shallow 7b333369de1221f9bfbbe03a3a13e9a09bc1c9ff +000ddeepen 1 +0017deepen-since 12345 +001fdeepen-not refs/heads/main +00000032have 0000000000000000000000000000000000000000 +0000005cwant 7b333369de1221f9bfbbe03a3a13e9a09bc1c907 feature-a shallow deepen-since deepen-not +0035shallow 7b333369de1221f9bfbbe03a3a13e9a09bc1c9ff +000ddeepen 1 +0017deepen-since 12345 +001fdeepen-not refs/heads/main +00000032have 1111111111111111111111111111111111111111 +0009done +" + .as_bstr() + ); + } + + #[maybe_async::test(feature = "blocking-client", async(feature = "async-client", async_std::test))] + async fn haves_and_wants_for_fetch_stateful() { + let mut out = Vec::new(); + let mut t = transport(&mut out, true); + let mut arguments = arguments_v1(["feature-a", "shallow"].iter().copied()); + + arguments.deepen(1); + arguments.want(id("7b333369de1221f9bfbbe03a3a13e9a09bc1c907")); + arguments.have(id("0000000000000000000000000000000000000000")); + arguments.send(&mut t, false).await.expect("sending to buffer to work"); + + arguments.have(id("1111111111111111111111111111111111111111")); + arguments.send(&mut t, true).await.expect("sending to buffer to work"); + assert_eq!( + out.as_bstr(), + b"0044want 7b333369de1221f9bfbbe03a3a13e9a09bc1c907 feature-a shallow +000ddeepen 1 +00000032have 0000000000000000000000000000000000000000 +00000032have 1111111111111111111111111111111111111111 +0009done +" + .as_bstr() + ); + } +} + +mod v2 { + use bstr::ByteSlice; + + use super::{arguments_v2, id, transport}; + + #[maybe_async::test(feature = "blocking-client", async(feature = "async-client", async_std::test))] + async fn include_tag() { + let mut out = Vec::new(); + let mut t = transport(&mut out, true); + let mut arguments = arguments_v2(["does not matter for us here"].iter().copied()); + assert!(arguments.can_use_include_tag(), "always on in V2"); + arguments.use_include_tag(); + + arguments.want(id("ff333369de1221f9bfbbe03a3a13e9a09bc1ffff")); + arguments.send(&mut t, true).await.expect("sending to buffer to work"); + assert_eq!( + out.as_bstr(), + b"0012command=fetch +0001000ethin-pack +000eofs-delta +0010include-tag +0032want ff333369de1221f9bfbbe03a3a13e9a09bc1ffff +0009done +0000" + .as_bstr(), + "we filter features/capabilities without value as these apparently shouldn't be listed (remote dies otherwise)" + ); + } + + #[maybe_async::test(feature = "blocking-client", async(feature = "async-client", async_std::test))] + async fn haves_and_wants_for_clone_stateful() { + let mut out = Vec::new(); + let mut t = transport(&mut out, true); + let mut arguments = arguments_v2(["feature-a", "shallow"].iter().copied()); + assert!(arguments.is_stateless(true), "V2 is stateless…"); + assert!(arguments.is_stateless(false), "…in all cases"); + + arguments.add_feature("no-progress"); + arguments.deepen(1); + arguments.deepen_relative(); + arguments.want(id("7b333369de1221f9bfbbe03a3a13e9a09bc1c907")); + arguments.want(id("ff333369de1221f9bfbbe03a3a13e9a09bc1ffff")); + arguments.send(&mut t, true).await.expect("sending to buffer to work"); + assert_eq!( + out.as_bstr(), + b"0012command=fetch +0001000ethin-pack +000eofs-delta +0010no-progress +000ddeepen 1 +0014deepen-relative +0032want 7b333369de1221f9bfbbe03a3a13e9a09bc1c907 +0032want ff333369de1221f9bfbbe03a3a13e9a09bc1ffff +0009done +0000" + .as_bstr(), + "we filter features/capabilities without value as these apparently shouldn't be listed (remote dies otherwise)" + ); + } + + #[maybe_async::test(feature = "blocking-client", async(feature = "async-client", async_std::test))] + async fn haves_and_wants_for_fetch_stateless_and_stateful() { + for is_stateful in &[false, true] { + let mut out = Vec::new(); + let mut t = transport(&mut out, *is_stateful); + let mut arguments = arguments_v2(Some("shallow")); + + arguments.add_feature("no-progress"); + arguments.deepen(1); + arguments.deepen_since(12345); + arguments.shallow(id("7b333369de1221f9bfbbe03a3a13e9a09bc1c9ff")); + arguments.want(id("7b333369de1221f9bfbbe03a3a13e9a09bc1c907")); + arguments.deepen_not("refs/heads/main".into()); + arguments.have(id("0000000000000000000000000000000000000000")); + arguments.send(&mut t, false).await.expect("sending to buffer to work"); + + arguments.have(id("1111111111111111111111111111111111111111")); + arguments.send(&mut t, true).await.expect("sending to buffer to work"); + assert_eq!( + out.as_bstr(), + b"0012command=fetch +0001000ethin-pack +000eofs-delta +0010no-progress +000ddeepen 1 +0017deepen-since 12345 +0035shallow 7b333369de1221f9bfbbe03a3a13e9a09bc1c9ff +0032want 7b333369de1221f9bfbbe03a3a13e9a09bc1c907 +001fdeepen-not refs/heads/main +0032have 0000000000000000000000000000000000000000 +00000012command=fetch +0001000ethin-pack +000eofs-delta +0010no-progress +000ddeepen 1 +0017deepen-since 12345 +0035shallow 7b333369de1221f9bfbbe03a3a13e9a09bc1c9ff +0032want 7b333369de1221f9bfbbe03a3a13e9a09bc1c907 +001fdeepen-not refs/heads/main +0032have 1111111111111111111111111111111111111111 +0009done +0000" + .as_bstr(), + "V2 is stateless by default, so it repeats all but 'haves' in each request" + ); + } + } + + #[maybe_async::test(feature = "blocking-client", async(feature = "async-client", async_std::test))] + async fn ref_in_want() { + let mut out = Vec::new(); + let mut t = transport(&mut out, false); + let mut arguments = arguments_v2(["ref-in-want"].iter().copied()); + + arguments.want_ref(b"refs/heads/main".as_bstr()); + arguments.send(&mut t, true).await.expect("sending to buffer to work"); + assert_eq!( + out.as_bstr(), + b"0012command=fetch +0001000ethin-pack +000eofs-delta +001dwant-ref refs/heads/main +0009done +0000" + .as_bstr() + ); + } +} diff --git a/gix-protocol/tests/fetch/mod.rs b/gix-protocol/tests/protocol/fetch/mod.rs similarity index 85% rename from gix-protocol/tests/fetch/mod.rs rename to gix-protocol/tests/protocol/fetch/mod.rs index c2c760d97f6..ab6f1cc2a9a 100644 --- a/gix-protocol/tests/fetch/mod.rs +++ b/gix-protocol/tests/protocol/fetch/mod.rs @@ -2,13 +2,44 @@ use std::{borrow::Cow, io}; use bstr::{BString, ByteSlice}; use gix_protocol::{ - fetch::{self, Action, Arguments, Response}, + fetch::{Arguments, Response}, handshake, ls_refs, }; + use gix_transport::client::Capabilities; use crate::fixture_bytes; +pub(super) mod _impl; +use _impl::{Action, DelegateBlocking}; + +mod error { + use std::io; + + use gix_transport::client; + + use gix_protocol::{fetch::response, handshake, ls_refs}; + + /// The error used in [`fetch()`][crate::fetch()]. + #[derive(Debug, thiserror::Error)] + #[allow(missing_docs)] + pub enum Error { + #[error(transparent)] + Handshake(#[from] handshake::Error), + #[error("Could not access repository or failed to read streaming pack file")] + Io(#[from] io::Error), + #[error(transparent)] + Transport(#[from] client::Error), + #[error(transparent)] + LsRefs(#[from] ls_refs::Error), + #[error(transparent)] + Response(#[from] response::Error), + } +} +pub use error::Error; + +mod arguments; + #[cfg(feature = "blocking-client")] type Cursor = std::io::Cursor>; #[cfg(feature = "async-client")] @@ -25,7 +56,7 @@ pub struct CloneDelegate { abort_with: Option, } -impl fetch::DelegateBlocking for CloneDelegate { +impl DelegateBlocking for CloneDelegate { fn prepare_fetch( &mut self, _version: gix_transport::Protocol, @@ -72,7 +103,7 @@ pub struct CloneRefInWantDelegate { wanted_refs: Vec, } -impl fetch::DelegateBlocking for CloneRefInWantDelegate { +impl DelegateBlocking for CloneRefInWantDelegate { fn prepare_ls_refs( &mut self, _server: &Capabilities, @@ -113,7 +144,7 @@ pub struct LsRemoteDelegate { abort_with: Option, } -impl fetch::DelegateBlocking for LsRemoteDelegate { +impl DelegateBlocking for LsRemoteDelegate { fn handshake_extra_parameters(&self) -> Vec<(String, Option)> { vec![("value-only".into(), None), ("key".into(), Some("value".into()))] } @@ -134,9 +165,9 @@ impl fetch::DelegateBlocking for LsRemoteDelegate { _server: &Capabilities, _features: &mut Vec<(&str, Option>)>, refs: &[handshake::Ref], - ) -> io::Result { + ) -> io::Result { refs.clone_into(&mut self.refs); - Ok(fetch::Action::Cancel) + Ok(Action::Cancel) } fn negotiate( &mut self, @@ -153,11 +184,12 @@ mod blocking_io { use std::io; use gix_features::progress::NestedProgress; - use gix_protocol::{fetch, fetch::Response, handshake, handshake::Ref}; + use gix_protocol::{fetch::Response, handshake, handshake::Ref}; + use super::_impl::Delegate; use crate::fetch::{CloneDelegate, CloneRefInWantDelegate, LsRemoteDelegate}; - impl fetch::Delegate for CloneDelegate { + impl Delegate for CloneDelegate { fn receive_pack( &mut self, mut input: impl io::BufRead, @@ -170,7 +202,7 @@ mod blocking_io { } } - impl fetch::Delegate for CloneRefInWantDelegate { + impl Delegate for CloneRefInWantDelegate { fn receive_pack( &mut self, mut input: impl io::BufRead, @@ -189,7 +221,7 @@ mod blocking_io { } } - impl fetch::Delegate for LsRemoteDelegate { + impl Delegate for LsRemoteDelegate { fn receive_pack( &mut self, _input: impl io::BufRead, @@ -209,12 +241,13 @@ mod async_io { use async_trait::async_trait; use futures_io::AsyncBufRead; use gix_features::progress::NestedProgress; - use gix_protocol::{fetch, fetch::Response, handshake, handshake::Ref}; + use gix_protocol::{fetch::Response, handshake, handshake::Ref}; + use super::_impl::Delegate; use crate::fetch::{CloneDelegate, CloneRefInWantDelegate, LsRemoteDelegate}; #[async_trait(?Send)] - impl fetch::Delegate for CloneDelegate { + impl Delegate for CloneDelegate { async fn receive_pack( &mut self, mut input: impl AsyncBufRead + Unpin + 'async_trait, @@ -228,7 +261,7 @@ mod async_io { } #[async_trait(?Send)] - impl fetch::Delegate for CloneRefInWantDelegate { + impl Delegate for CloneRefInWantDelegate { async fn receive_pack( &mut self, mut input: impl AsyncBufRead + Unpin + 'async_trait, @@ -248,7 +281,7 @@ mod async_io { } #[async_trait(?Send)] - impl fetch::Delegate for LsRemoteDelegate { + impl Delegate for LsRemoteDelegate { async fn receive_pack( &mut self, _input: impl AsyncBufRead + Unpin + 'async_trait, diff --git a/gix-protocol/tests/fetch/response.rs b/gix-protocol/tests/protocol/fetch/response.rs similarity index 100% rename from gix-protocol/tests/fetch/response.rs rename to gix-protocol/tests/protocol/fetch/response.rs diff --git a/gix-protocol/tests/fetch/v1.rs b/gix-protocol/tests/protocol/fetch/v1.rs similarity index 96% rename from gix-protocol/tests/fetch/v1.rs rename to gix-protocol/tests/protocol/fetch/v1.rs index 844b8f24485..dede20e15be 100644 --- a/gix-protocol/tests/fetch/v1.rs +++ b/gix-protocol/tests/protocol/fetch/v1.rs @@ -1,6 +1,7 @@ +use crate::fetch::_impl::FetchConnection; use bstr::ByteSlice; use gix_features::progress; -use gix_protocol::{handshake, FetchConnection}; +use gix_protocol::handshake; use gix_transport::Protocol; use crate::fetch::{helper_unused, oid, transport, CloneDelegate, LsRemoteDelegate}; @@ -14,7 +15,7 @@ async fn clone() -> crate::Result { "v1/clone{}.response", with_keepalive.then_some("-with-keepalive").unwrap_or_default() ); - gix_protocol::fetch( + crate::fetch( transport( out, &fixture, @@ -38,7 +39,7 @@ async fn clone() -> crate::Result { async fn clone_empty_with_capabilities() -> crate::Result { let out = Vec::new(); let mut dlg = CloneDelegate::default(); - gix_protocol::fetch( + crate::fetch( transport( out, "v1/clone-empty-with-capabilities.response", @@ -67,7 +68,7 @@ async fn ls_remote() -> crate::Result { Protocol::V1, gix_transport::client::git::ConnectMode::Daemon, ); - gix_protocol::fetch( + crate::fetch( &mut transport, &mut delegate, helper_unused, @@ -106,7 +107,7 @@ async fn ls_remote_handshake_failure_due_to_downgrade() -> crate::Result { let out = Vec::new(); let delegate = LsRemoteDelegate::default(); - gix_protocol::fetch( + crate::fetch( transport( out, "v1/clone.response", diff --git a/gix-protocol/tests/fetch/v2.rs b/gix-protocol/tests/protocol/fetch/v2.rs similarity index 93% rename from gix-protocol/tests/fetch/v2.rs rename to gix-protocol/tests/protocol/fetch/v2.rs index de2c0a9b464..9e0bfecc2fe 100644 --- a/gix-protocol/tests/fetch/v2.rs +++ b/gix-protocol/tests/protocol/fetch/v2.rs @@ -1,9 +1,12 @@ use bstr::ByteSlice; use gix_features::progress; -use gix_protocol::{fetch, handshake, ls_refs, FetchConnection}; +use gix_protocol::{handshake, ls_refs}; use gix_transport::Protocol; -use crate::fetch::{helper_unused, oid, transport, CloneDelegate, CloneRefInWantDelegate, LsRemoteDelegate}; +use crate::fetch::{ + _impl::FetchConnection, helper_unused, oid, transport, CloneDelegate, CloneRefInWantDelegate, Error, + LsRemoteDelegate, +}; #[maybe_async::test(feature = "blocking-client", async(feature = "async-client", async_std::test))] async fn clone_abort_prep() -> crate::Result { @@ -19,7 +22,7 @@ async fn clone_abort_prep() -> crate::Result { gix_transport::client::git::ConnectMode::Daemon, ); let agent = "agent"; - let err = gix_protocol::fetch( + let err = crate::fetch( &mut transport, &mut dlg, helper_unused, @@ -46,7 +49,7 @@ async fn clone_abort_prep() -> crate::Result { .as_bstr() ); match err { - fetch::Error::Io(err) => { + Error::Io(err) => { assert_eq!(err.kind(), std::io::ErrorKind::Other); assert_eq!(err.get_ref().expect("other error").to_string(), "hello world"); } @@ -66,7 +69,7 @@ async fn ls_remote() -> crate::Result { gix_transport::client::git::ConnectMode::Daemon, ); let agent = "agent"; - gix_protocol::fetch( + crate::fetch( &mut transport, &mut delegate, helper_unused, @@ -122,7 +125,7 @@ async fn ls_remote_abort_in_prep_ls_refs() -> crate::Result { Protocol::V2, gix_transport::client::git::ConnectMode::Daemon, ); - let err = gix_protocol::fetch( + let err = crate::fetch( &mut transport, &mut delegate, helper_unused, @@ -140,7 +143,7 @@ async fn ls_remote_abort_in_prep_ls_refs() -> crate::Result { b"0044git-upload-pack does/not/matter\x00\x00version=2\x00value-only\x00key=value\x000000".as_bstr() ); match err { - fetch::Error::LsRefs(ls_refs::Error::Io(err)) => { + Error::LsRefs(ls_refs::Error::Io(err)) => { assert_eq!(err.kind(), std::io::ErrorKind::Other); assert_eq!(err.get_ref().expect("other error").to_string(), "hello world"); } @@ -164,7 +167,7 @@ async fn ref_in_want() -> crate::Result { ); let agent = "agent"; - gix_protocol::fetch( + crate::fetch( &mut transport, &mut delegate, helper_unused, diff --git a/gix-protocol/src/handshake/refs/tests.rs b/gix-protocol/tests/protocol/handshake.rs similarity index 90% rename from gix-protocol/src/handshake/refs/tests.rs rename to gix-protocol/tests/protocol/handshake.rs index 69ea5872790..b0d8e42d0a7 100644 --- a/gix-protocol/src/handshake/refs/tests.rs +++ b/gix-protocol/tests/protocol/handshake.rs @@ -1,11 +1,11 @@ -use gix_transport::{client, client::Capabilities}; +use gix_transport::client::Capabilities; /// Convert a hexadecimal hash into its corresponding `ObjectId` or _panic_. fn oid(hex: &str) -> gix_hash::ObjectId { gix_hash::ObjectId::from_hex(hex.as_bytes()).expect("40 bytes hex") } -use crate::handshake::{refs, refs::shared::InternalRef, Ref}; +use gix_protocol::handshake::{refs, Ref}; #[maybe_async::test(feature = "blocking-client", async(feature = "async-client", async_std::test))] async fn extract_references_from_v2_refs() { @@ -19,7 +19,7 @@ unborn refs/heads/symbolic symref-target:refs/heads/target 7fe1b98b39423b71e14217aa299a03b7c937d6ff refs/tags/blaz 978f927e6397113757dfec6332e7d9c7e356ac25 refs/heads/symbolic symref-target:refs/tags/v1.0 peeled:4d979abcde5cea47b079c38850828956c9382a56 " - .as_bytes(), + .as_bytes(), ); let out = refs::from_v2_refs(input).await.expect("no failure on valid input"); @@ -121,7 +121,7 @@ dce0ea858eef7ff61ad345cc5cdac62203fb3c10 refs/tags/gix-commitgraph-v0.0.0 #[maybe_async::test(feature = "blocking-client", async(feature = "async-client", async_std::test))] async fn extract_references_from_v1_refs_with_shallow() { - use crate::fetch::response::ShallowUpdate; + use gix_protocol::fetch::response::ShallowUpdate; let input = &mut Fixture( "73a6868963993a3328e7d8fe94e5a6ac5078a944 HEAD 21c9b7500cb144b3169a6537961ec2b9e865be81 MISSING_NAMESPACE_TARGET @@ -180,34 +180,6 @@ shallow dce0ea858eef7ff61ad345cc5cdac62203fb3c10" ); } -#[test] -fn extract_symbolic_references_from_capabilities() -> Result<(), client::Error> { - let caps = client::Capabilities::from_bytes( - b"\0unrelated symref=HEAD:refs/heads/main symref=ANOTHER:refs/heads/foo symref=MISSING_NAMESPACE_TARGET:(null) agent=git/2.28.0", - )? - .0; - let out = refs::shared::from_capabilities(caps.iter()).expect("a working example"); - - assert_eq!( - out, - vec![ - InternalRef::SymbolicForLookup { - path: "HEAD".into(), - target: Some("refs/heads/main".into()) - }, - InternalRef::SymbolicForLookup { - path: "ANOTHER".into(), - target: Some("refs/heads/foo".into()) - }, - InternalRef::SymbolicForLookup { - path: "MISSING_NAMESPACE_TARGET".into(), - target: None - } - ] - ); - Ok(()) -} - #[cfg(any(feature = "async-client", feature = "blocking-client"))] struct Fixture<'a>(&'a [u8]); diff --git a/gix-protocol/tests/protocol/mod.rs b/gix-protocol/tests/protocol/mod.rs new file mode 100644 index 00000000000..a22b3b99f99 --- /dev/null +++ b/gix-protocol/tests/protocol/mod.rs @@ -0,0 +1,12 @@ +pub type Result = std::result::Result<(), Box>; + +pub fn fixture_bytes(path: &str) -> Vec { + std::fs::read(std::path::PathBuf::from("tests").join("fixtures").join(path)) + .expect("fixture to be present and readable") +} + +mod command; +pub mod fetch; +mod handshake; +pub use fetch::_impl::{fetch, FetchConnection}; +pub mod remote_progress; diff --git a/gix-protocol/tests/remote_progress/mod.rs b/gix-protocol/tests/protocol/remote_progress.rs similarity index 100% rename from gix-protocol/tests/remote_progress/mod.rs rename to gix-protocol/tests/protocol/remote_progress.rs diff --git a/gix-shallow/Cargo.toml b/gix-shallow/Cargo.toml new file mode 100644 index 00000000000..61438be635e --- /dev/null +++ b/gix-shallow/Cargo.toml @@ -0,0 +1,28 @@ +lints.workspace = true + +[package] +name = "gix-shallow" +version = "0.1.0" +repository = "https://github.com/GitoxideLabs/gitoxide" +authors = ["Sebastian Thiel "] +license = "MIT OR Apache-2.0" +description = "Handle files specifying the shallow boundary" +edition = "2021" +include = ["src/**/*", "LICENSE-*"] +rust-version = "1.65" + +[lib] +doctest = false +test = false + +[features] +## Data structures implement `serde::Serialize` and `serde::Deserialize`. +serde = ["dep:serde", "gix-hash/serde"] + +[dependencies] +gix-hash = { version = "^0.15.1", path = "../gix-hash" } +gix-lock = { version = "^15.0.0", path = "../gix-lock" } + +thiserror = "2.0.0" +bstr = { version = "1.5.0", default-features = false } +serde = { version = "1.0.114", optional = true, default-features = false, features = ["std", "derive"] } diff --git a/gix-shallow/LICENSE-APACHE b/gix-shallow/LICENSE-APACHE new file mode 120000 index 00000000000..965b606f331 --- /dev/null +++ b/gix-shallow/LICENSE-APACHE @@ -0,0 +1 @@ +../LICENSE-APACHE \ No newline at end of file diff --git a/gix-shallow/LICENSE-MIT b/gix-shallow/LICENSE-MIT new file mode 120000 index 00000000000..76219eb72e8 --- /dev/null +++ b/gix-shallow/LICENSE-MIT @@ -0,0 +1 @@ +../LICENSE-MIT \ No newline at end of file diff --git a/gix-shallow/src/lib.rs b/gix-shallow/src/lib.rs new file mode 100644 index 00000000000..414244ad9a9 --- /dev/null +++ b/gix-shallow/src/lib.rs @@ -0,0 +1,122 @@ +//! [Read](read()) and [write](write()) shallow files, while performing typical operations on them. +#![deny(missing_docs, rust_2018_idioms)] + +/// An instruction on how to +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum Update { + /// Shallow the given `id`. + Shallow(gix_hash::ObjectId), + /// Don't shallow the given `id` anymore. + Unshallow(gix_hash::ObjectId), +} + +/// Return a list of shallow commits as unconditionally read from `shallow_file`. +/// +/// The list of shallow commits represents the shallow boundary, beyond which we are lacking all (parent) commits. +/// Note that the list is never empty, as `Ok(None)` is returned in that case indicating the repository +/// isn't a shallow clone. +pub fn read(shallow_file: &std::path::Path) -> Result>, read::Error> { + use bstr::ByteSlice; + let buf = match std::fs::read(shallow_file) { + Ok(buf) => buf, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(err) => return Err(err.into()), + }; + + let mut commits = buf + .lines() + .map(gix_hash::ObjectId::from_hex) + .collect::, _>>()?; + + commits.sort(); + if commits.is_empty() { + Ok(None) + } else { + Ok(Some(commits)) + } +} + +/// +pub mod write { + pub(crate) mod function { + use std::io::Write; + + use super::Error; + use crate::Update; + + /// Write the [previously obtained](crate::read()) (possibly non-existing) `shallow_commits` to the shallow `file` + /// after applying all `updates`. + /// + /// If this leaves the list of shallow commits empty, the file is removed. + /// + /// ### Deviation + /// + /// Git also prunes the set of shallow commits while writing, we don't until we support some sort of pruning. + pub fn write( + mut file: gix_lock::File, + shallow_commits: Option>, + updates: &[Update], + ) -> Result<(), Error> { + let mut shallow_commits = shallow_commits.unwrap_or_default(); + for update in updates { + match update { + Update::Shallow(id) => { + shallow_commits.push(*id); + } + Update::Unshallow(id) => shallow_commits.retain(|oid| oid != id), + } + } + if shallow_commits.is_empty() { + std::fs::remove_file(file.resource_path())?; + drop(file); + return Ok(()); + } + + if shallow_commits.is_empty() { + if let Err(err) = std::fs::remove_file(file.resource_path()) { + if err.kind() != std::io::ErrorKind::NotFound { + return Err(err.into()); + } + } + } else { + shallow_commits.sort(); + let mut buf = Vec::::new(); + for commit in shallow_commits { + commit.write_hex_to(&mut buf).map_err(Error::Io)?; + buf.push(b'\n'); + } + file.write_all(&buf).map_err(Error::Io)?; + file.flush()?; + } + file.commit()?; + Ok(()) + } + } + + /// The error returned by [`write()`](crate::write()). + #[derive(Debug, thiserror::Error)] + #[allow(missing_docs)] + pub enum Error { + #[error(transparent)] + Commit(#[from] gix_lock::commit::Error), + #[error("Could not remove an empty shallow file")] + RemoveEmpty(#[from] std::io::Error), + #[error("Failed to write object id to shallow file")] + Io(std::io::Error), + } +} +pub use write::function::write; + +/// +pub mod read { + /// The error returned by [`read`](crate::read()). + #[derive(Debug, thiserror::Error)] + #[allow(missing_docs)] + pub enum Error { + #[error("Could not open shallow file for reading")] + Io(#[from] std::io::Error), + #[error("Could not decode a line in shallow file as hex-encoded object hash")] + DecodeHash(#[from] gix_hash::decode::Error), + } +} diff --git a/gix-transport/src/client/blocking_io/file.rs b/gix-transport/src/client/blocking_io/file.rs index 8f0186c59aa..a145a305fab 100644 --- a/gix-transport/src/client/blocking_io/file.rs +++ b/gix-transport/src/client/blocking_io/file.rs @@ -109,7 +109,7 @@ impl client::TransportWithoutIO for SpawnProcessOnDemand { ) -> Result, client::Error> { self.connection .as_mut() - .expect("handshake() to have been called first") + .ok_or(client::Error::MissingHandshake)? .request(write_mode, on_into_read, trace) } diff --git a/gix-transport/src/client/blocking_io/http/mod.rs b/gix-transport/src/client/blocking_io/http/mod.rs index 5994c88c76e..503924e9690 100644 --- a/gix-transport/src/client/blocking_io/http/mod.rs +++ b/gix-transport/src/client/blocking_io/http/mod.rs @@ -333,7 +333,7 @@ impl client::TransportWithoutIO for Transport { on_into_read: MessageKind, trace: bool, ) -> Result, client::Error> { - let service = self.service.expect("handshake() must have been called first"); + let service = self.service.ok_or(client::Error::MissingHandshake)?; let url = append_url(&self.url, service.as_str()); let static_headers = &[ Cow::Borrowed(self.user_agent_header), diff --git a/gix-transport/src/client/non_io_types.rs b/gix-transport/src/client/non_io_types.rs index d4d6cc8c19b..24a69a57e21 100644 --- a/gix-transport/src/client/non_io_types.rs +++ b/gix-transport/src/client/non_io_types.rs @@ -112,6 +112,8 @@ mod error { #[derive(thiserror::Error, Debug)] #[allow(missing_docs)] pub enum Error { + #[error("A request was performed without performing the handshake first")] + MissingHandshake, #[error("An IO error occurred when talking to the server")] Io(#[from] std::io::Error), #[error("Capabilities could not be parsed")] diff --git a/gix/Cargo.toml b/gix/Cargo.toml index 20cbb29c1d8..80aaffb1538 100644 --- a/gix/Cargo.toml +++ b/gix/Cargo.toml @@ -160,6 +160,7 @@ worktree-archive = ["gix-archive", "worktree-stream", "attributes"] async-network-client = [ "gix-protocol/async-client", "gix-pack/streaming-input", + "dep:gix-transport", "attributes", "credentials", ] @@ -173,6 +174,7 @@ async-network-client-async-std = [ blocking-network-client = [ "gix-protocol/blocking-client", "gix-pack/streaming-input", + "dep:gix-transport", "attributes", "credentials", ] @@ -288,7 +290,7 @@ serde = [ "dep:serde", "gix-pack/serde", "gix-object/serde", - "gix-protocol?/serde", + "gix-protocol/serde", "gix-transport?/serde", "gix-ref/serde", "gix-odb/serde", @@ -328,6 +330,7 @@ gix-dir = { version = "^0.10.0", path = "../gix-dir", optional = true } gix-config = { version = "^0.42.0", path = "../gix-config" } gix-odb = { version = "^0.65.0", path = "../gix-odb" } gix-hash = { version = "^0.15.1", path = "../gix-hash" } +gix-shallow = { version = "^0.1.0", path = "../gix-shallow" } gix-object = { version = "^0.46.0", path = "../gix-object" } gix-actor = { version = "^0.33.1", path = "../gix-actor" } gix-pack = { version = "^0.55.0", path = "../gix-pack", default-features = false, features = [ @@ -370,7 +373,7 @@ gix-worktree-stream = { version = "^0.17.0", path = "../gix-worktree-stream", op gix-archive = { version = "^0.17.0", path = "../gix-archive", default-features = false, optional = true } # For communication with remotes -gix-protocol = { version = "^0.46.1", path = "../gix-protocol", optional = true } +gix-protocol = { version = "^0.46.1", path = "../gix-protocol" } gix-transport = { version = "^0.43.1", path = "../gix-transport", optional = true } # Just to get the progress-tree feature diff --git a/gix/src/clone/fetch/mod.rs b/gix/src/clone/fetch/mod.rs index f0f1a133ac7..1485871a159 100644 --- a/gix/src/clone/fetch/mod.rs +++ b/gix/src/clone/fetch/mod.rs @@ -72,15 +72,6 @@ impl PrepareFetch { P: crate::NestedProgress, P::SubProgress: 'static, { - self.fetch_only_inner(&mut progress, should_interrupt).await - } - - #[gix_protocol::maybe_async::maybe_async] - async fn fetch_only_inner( - &mut self, - progress: &mut dyn crate::DynNestedProgress, - should_interrupt: &std::sync::atomic::AtomicBool, - ) -> Result<(crate::Repository, crate::remote::fetch::Outcome), Error> { use crate::{bstr::ByteVec, remote, remote::fetch::RefLogMessage}; let repo = self @@ -156,18 +147,19 @@ impl PrepareFetch { } opts }; - match connection.prepare_fetch(&mut *progress, fetch_opts.clone()).await { + match connection.prepare_fetch(&mut progress, fetch_opts.clone()).await { Ok(prepare) => prepare, - Err(remote::fetch::prepare::Error::RefMap(remote::ref_map::Error::MappingValidation(err))) - if err.issues.len() == 1 - && fetch_opts.extra_refspecs.contains(&head_refspec) - && matches!( - err.issues.first(), - Some(gix_refspec::match_group::validate::Issue::Conflict { - destination_full_ref_name, - .. - }) if *destination_full_ref_name == head_local_tracking_branch - ) => + Err(remote::fetch::prepare::Error::RefMap(remote::ref_map::Error::InitRefMap( + gix_protocol::fetch::refmap::init::Error::MappingValidation(err), + ))) if err.issues.len() == 1 + && fetch_opts.extra_refspecs.contains(&head_refspec) + && matches!( + err.issues.first(), + Some(gix_refspec::match_group::validate::Issue::Conflict { + destination_full_ref_name, + .. + }) if *destination_full_ref_name == head_local_tracking_branch + ) => { let head_refspec_idx = fetch_opts .extra_refspecs @@ -180,7 +172,7 @@ impl PrepareFetch { // refspec, as git can do this without connecting twice. let connection = remote.connect(remote::Direction::Fetch).await?; fetch_opts.extra_refspecs.remove(head_refspec_idx); - connection.prepare_fetch(&mut *progress, fetch_opts).await? + connection.prepare_fetch(&mut progress, fetch_opts).await? } Err(err) => return Err(err.into()), } @@ -204,7 +196,7 @@ impl PrepareFetch { message: reflog_message.clone(), }) .with_shallow(self.shallow.clone()) - .receive_inner(progress, should_interrupt) + .receive(&mut progress, should_interrupt) .await?; util::append_config_to_repo_config(repo, config); diff --git a/gix/src/lib.rs b/gix/src/lib.rs index 8764b7163ce..906db6bb3e8 100644 --- a/gix/src/lib.rs +++ b/gix/src/lib.rs @@ -127,7 +127,6 @@ pub use gix_object::bstr; pub use gix_odb as odb; #[cfg(feature = "credentials")] pub use gix_prompt as prompt; -#[cfg(feature = "gix-protocol")] pub use gix_protocol as protocol; pub use gix_ref as refs; pub use gix_refspec as refspec; diff --git a/gix/src/remote/connect.rs b/gix/src/remote/connect.rs index 589c353ae19..1c70928c5ec 100644 --- a/gix/src/remote/connect.rs +++ b/gix/src/remote/connect.rs @@ -52,7 +52,7 @@ pub use error::Error; impl<'repo> Remote<'repo> { /// Create a new connection using `transport` to communicate, with `progress` to indicate changes. /// - /// Note that this method expects the `transport` to be created by the user, which would involve the [`url()`][Self::url()]. + /// Note that this method expects the `transport` to be created by the user, which would involve the [`url()`](Self::url()). /// It's meant to be used when async operation is needed with runtimes of the user's choice. pub fn to_connection_with_transport(&self, transport: T) -> Connection<'_, 'repo, T> where @@ -63,7 +63,8 @@ impl<'repo> Remote<'repo> { remote: self, authenticate: None, transport_options: None, - transport, + handshake: None, + transport: gix_protocol::SendFlushOnDrop::new(transport, trace), trace, } } diff --git a/gix/src/remote/connection/access.rs b/gix/src/remote/connection/access.rs index 2b9c3f1dadf..c97fdde912e 100644 --- a/gix/src/remote/connection/access.rs +++ b/gix/src/remote/connection/access.rs @@ -4,7 +4,10 @@ use crate::{ }; /// Builder -impl<'a, T> Connection<'a, '_, T> { +impl<'a, T> Connection<'a, '_, T> +where + T: gix_transport::client::Transport, +{ /// Set a custom credentials callback to provide credentials if the remotes require authentication. /// /// Otherwise, we will use the git configuration to perform the same task as the `git credential` helper program, @@ -25,8 +28,8 @@ impl<'a, T> Connection<'a, '_, T> { } /// Provide configuration to be used before the first handshake is conducted. - /// It's typically created by initializing it with [`Repository::transport_options()`](crate::Repository::transport_options()), which - /// is also the default if this isn't set explicitly. Note that all of the default configuration is created from `git` + /// It's typically created by initializing it with [`Repository::transport_options()`](crate::Repository::transport_options()), + /// which is also the default if this isn't set explicitly. Note that all the default configuration is created from `git` /// configuration, which can also be manipulated through overrides to affect the default configuration. /// /// Use this method to provide transport configuration with custom backend configuration that is not configurable by other means and @@ -38,7 +41,10 @@ impl<'a, T> Connection<'a, '_, T> { } /// Mutation -impl<'a, T> Connection<'a, '_, T> { +impl<'a, T> Connection<'a, '_, T> +where + T: gix_transport::client::Transport, +{ /// Like [`with_credentials()`](Self::with_credentials()), but without consuming the connection. pub fn set_credentials( &mut self, @@ -56,7 +62,10 @@ impl<'a, T> Connection<'a, '_, T> { } /// Access -impl<'repo, T> Connection<'_, 'repo, T> { +impl<'repo, T> Connection<'_, 'repo, T> +where + T: gix_transport::client::Transport, +{ /// A utility to return a function that will use this repository's configuration to obtain credentials, similar to /// what `git credential` is doing. /// @@ -80,6 +89,6 @@ impl<'repo, T> Connection<'_, 'repo, T> { /// as we will call it automatically before performing the handshake. Instead, to bring in custom configuration, /// call [`with_transport_options()`](Connection::with_transport_options()). pub fn transport_mut(&mut self) -> &mut T { - &mut self.transport + &mut self.transport.inner } } diff --git a/gix/src/remote/connection/fetch/error.rs b/gix/src/remote/connection/fetch/error.rs index 129a2e6d345..2c868e75b04 100644 --- a/gix/src/remote/connection/fetch/error.rs +++ b/gix/src/remote/connection/fetch/error.rs @@ -1,9 +1,12 @@ use crate::config; /// The error returned by [`receive()`](super::Prepare::receive()). +// TODO: remove unused variants #[derive(Debug, thiserror::Error)] #[allow(missing_docs)] pub enum Error { + #[error(transparent)] + Fetch(#[from] gix_protocol::fetch::Error), #[error("The value to configure pack threads should be 0 to auto-configure or the amount of threads to use")] PackThreads(#[from] config::unsigned_integer::Error), #[error("The value to configure the pack index version should be 1 or 2")] @@ -16,7 +19,9 @@ pub enum Error { remote: gix_hash::Kind, }, #[error(transparent)] - Negotiate(#[from] super::negotiate::Error), + LoadAlternates(#[from] gix_odb::store::load_index::Error), + #[error(transparent)] + Negotiate(#[from] gix_protocol::fetch::negotiate::Error), #[error(transparent)] Client(#[from] gix_protocol::transport::client::Error), #[error(transparent)] @@ -29,7 +34,7 @@ pub enum Error { source: std::io::Error, }, #[error(transparent)] - ShallowOpen(#[from] crate::shallow::open::Error), + ShallowOpen(#[from] crate::shallow::read::Error), #[error("Server lack feature {feature:?}: {description}")] MissingServerFeature { feature: &'static str, @@ -47,11 +52,6 @@ pub enum Error { NegotiationAlgorithmConfig(#[from] config::key::GenericErrorWithValue), #[error("Failed to read remaining bytes in stream")] ReadRemainingBytes(#[source] std::io::Error), - #[error("None of the refspec(s) {} matched any of the {num_remote_refs} refs on the remote", refspecs.iter().map(|r| r.to_ref().instruction().to_bstring().to_string()).collect::>().join(", "))] - NoMapping { - refspecs: Vec, - num_remote_refs: usize, - }, } impl gix_protocol::transport::IsSpuriousError for Error { diff --git a/gix/src/remote/connection/fetch/mod.rs b/gix/src/remote/connection/fetch/mod.rs index e0017e6ed39..2e0014b4bea 100644 --- a/gix/src/remote/connection/fetch/mod.rs +++ b/gix/src/remote/connection/fetch/mod.rs @@ -72,6 +72,8 @@ pub enum Status { pub struct Outcome { /// The result of the initial mapping of references, the prerequisite for any fetch. pub ref_map: RefMap, + /// The outcome of the handshake with the server. + pub handshake: gix_protocol::handshake::Outcome, /// The status of the operation to indicate what happened. pub status: Status, } @@ -86,56 +88,11 @@ pub mod outcome { /// The negotiation graph indicating what kind of information 'the algorithm' collected in the end. pub graph: gix_negotiate::IdMap, /// Additional information for each round of negotiation. - pub rounds: Vec, - } - - /// - pub mod negotiate { - /// Key information about each round in the pack-negotiation. - #[derive(Debug, Clone)] - pub struct Round { - /// The amount of `HAVE` lines sent this round. - /// - /// Each `HAVE` is an object that we tell the server about which would acknowledge each one it has as well. - pub haves_sent: usize, - /// A total counter, over all previous rounds, indicating how many `HAVE`s we sent without seeing a single acknowledgement, - /// i.e. the indication of a common object. - /// - /// This number maybe zero or be lower compared to the previous round if we have received at least one acknowledgement. - pub in_vain: usize, - /// The amount of haves we should send in this round. - /// - /// If the value is lower than `haves_sent` (the `HAVE` lines actually sent), the negotiation algorithm has run out of options - /// which typically indicates the end of the negotiation phase. - pub haves_to_send: usize, - /// If `true`, the server reported, as response to our previous `HAVE`s, that at least one of them is in common by acknowledging it. - /// - /// This may also lead to the server responding with a pack. - pub previous_response_had_at_least_one_in_common: bool, - } + pub rounds: Vec, } } -/// The progress ids used in during various steps of the fetch operation. -/// -/// Note that tagged progress isn't very widely available yet, but support can be improved as needed. -/// -/// Use this information to selectively extract the progress of interest in case the parent application has custom visualization. -#[derive(Debug, Copy, Clone)] -pub enum ProgressId { - /// The progress name is defined by the remote and the progress messages it sets, along with their progress values and limits. - RemoteProgress, -} - -impl From for gix_features::progress::Id { - fn from(v: ProgressId) -> Self { - match v { - ProgressId::RemoteProgress => *b"FERP", - } - } -} - -pub(crate) mod negotiate; +pub use gix_protocol::fetch::ProgressId; /// pub mod prepare { @@ -185,7 +142,7 @@ where if self.remote.refspecs(remote::Direction::Fetch).is_empty() && options.extra_refspecs.is_empty() { return Err(prepare::Error::MissingRefSpecs); } - let ref_map = self.ref_map_inner(progress, options).await?; + let ref_map = self.ref_map_by_ref(progress, options).await?; Ok(Prepare { con: Some(self), ref_map, @@ -266,29 +223,3 @@ where self } } - -impl Drop for Prepare<'_, '_, T> -where - T: Transport, -{ - fn drop(&mut self) { - if let Some(mut con) = self.con.take() { - #[cfg(feature = "async-network-client")] - { - // TODO: this should be an async drop once the feature is available. - // Right now we block the executor by forcing this communication, but that only - // happens if the user didn't actually try to receive a pack, which consumes the - // connection in an async context. - gix_protocol::futures_lite::future::block_on(gix_protocol::indicate_end_of_interaction( - &mut con.transport, - con.trace, - )) - .ok(); - } - #[cfg(not(feature = "async-network-client"))] - { - gix_protocol::indicate_end_of_interaction(&mut con.transport, con.trace).ok(); - } - } - } -} diff --git a/gix/src/remote/connection/fetch/receive_pack.rs b/gix/src/remote/connection/fetch/receive_pack.rs index b8caf153114..2b7cd59a3bf 100644 --- a/gix/src/remote/connection/fetch/receive_pack.rs +++ b/gix/src/remote/connection/fetch/receive_pack.rs @@ -1,14 +1,3 @@ -use std::{ - ops::DerefMut, - sync::atomic::{AtomicBool, Ordering}, -}; - -use gix_odb::store::RefreshMode; -use gix_protocol::{ - fetch::Arguments, - transport::{client::Transport, packetline::read::ProgressAction}, -}; - use crate::{ config::{ cache::util::ApplyLeniency, @@ -18,54 +7,60 @@ use crate::{ remote::{ connection::fetch::config, fetch, - fetch::{ - negotiate, negotiate::Algorithm, outcome, refs, Error, Outcome, Prepare, ProgressId, RefLogMessage, - Shallow, Status, - }, + fetch::{negotiate::Algorithm, outcome, refs, Error, Outcome, Prepare, RefLogMessage, Status}, }, - Repository, }; +use gix_odb::store::RefreshMode; +use gix_protocol::fetch::negotiate; +use gix_protocol::{fetch::Arguments, transport::client::Transport}; +use std::ops::DerefMut; +use std::path::PathBuf; +use std::sync::atomic::AtomicBool; impl Prepare<'_, '_, T> where T: Transport, { /// Receive the pack and perform the operation as configured by git via `git-config` or overridden by various builder methods. - /// Return `Ok(None)` if there was nothing to do because all remote refs are at the same state as they are locally, or `Ok(Some(outcome))` - /// to inform about all the changes that were made. + /// Return `Ok(Outcome)` with an [`Outcome::status`] indicating if a change was made or not. + /// + /// Note that when in dry-run mode, we don't read the pack the server prepared, which leads the server to be hung up on unexpectedly. /// /// ### Negotiation /// /// "fetch.negotiationAlgorithm" describes algorithms `git` uses currently, with the default being `consecutive` and `skipping` being - /// experimented with. We currently implement something we could call 'naive' which works for now. + /// experimented with. /// /// ### Pack `.keep` files /// - /// That packs that are freshly written to the object database are vulnerable to garbage collection for the brief time that it takes between - /// them being placed and the respective references to be written to disk which binds their objects to the commit graph, making them reachable. + /// That packs that are freshly written to the object database are vulnerable to garbage collection for the brief time that + /// it takes between them being placed and the respective references to be written to disk which binds their objects to the + /// commit graph, making them reachable. /// - /// To circumvent this issue, a `.keep` file is created before any pack related file (i.e. `.pack` or `.idx`) is written, which indicates the - /// garbage collector (like `git maintenance`, `git gc`) to leave the corresponding pack file alone. + /// To circumvent this issue, a `.keep` file is created before any pack related file (i.e. `.pack` or `.idx`) is written, + /// which indicates the garbage collector (like `git maintenance`, `git gc`) to leave the corresponding pack file alone. /// - /// If there were any ref updates or the received pack was empty, the `.keep` file will be deleted automatically leaving in its place at - /// `write_pack_bundle.keep_path` a `None`. + /// If there were any ref updates or the received pack was empty, the `.keep` file will be deleted automatically leaving + /// in its place at `write_pack_bundle.keep_path` a `None`. /// However, if no ref-update happened the path will still be present in `write_pack_bundle.keep_path` and is expected to be handled by the caller. /// A known application for this behaviour is in `remote-helper` implementations which should send this path via `lock ` to stdout /// to inform git about the file that it will remove once it updated the refs accordingly. /// /// ### Deviation /// - /// When **updating refs**, the `git-fetch` docs state that the following: + /// When **updating refs**, the `git-fetch` docs state the following: /// - /// > Unlike when pushing with git-push, any updates outside of refs/{tags,heads}/* will be accepted without + in the refspec (or --force), whether that’s swapping e.g. a tree object for a blob, or a commit for another commit that’s doesn’t have the previous commit as an ancestor etc. + /// > Unlike when pushing with git-push, any updates outside of refs/{tags,heads}/* will be accepted without + in the refspec (or --force), + /// whether that’s swapping e.g. a tree object for a blob, or a commit for another commit that’s doesn’t have the previous commit + /// as an ancestor etc. /// - /// We explicitly don't special case those refs and expect the user to take control. Note that by its nature, + /// We explicitly don't special case those refs and expect the caller to take control. Note that by its nature, /// force only applies to refs pointing to commits and if they don't, they will be updated either way in our /// implementation as well. /// /// ### Async Mode Shortcoming /// - /// Currently the entire process of resolving a pack is blocking the executor. This can be fixed using the `blocking` crate, but it + /// Currently, the entire process of resolving a pack is blocking the executor. This can be fixed using the `blocking` crate, but it /// didn't seem worth the tradeoff of having more complex code. /// /// ### Configuration @@ -73,73 +68,35 @@ where /// - `gitoxide.userAgent` is read to obtain the application user agent for git servers and for HTTP servers as well. /// #[gix_protocol::maybe_async::maybe_async] - pub async fn receive

(self, mut progress: P, should_interrupt: &AtomicBool) -> Result + pub async fn receive

(mut self, progress: P, should_interrupt: &AtomicBool) -> Result where P: gix_features::progress::NestedProgress, P::SubProgress: 'static, { - self.receive_inner(&mut progress, should_interrupt).await - } - - #[gix_protocol::maybe_async::maybe_async] - #[allow(clippy::drop_non_drop)] - pub(crate) async fn receive_inner( - mut self, - progress: &mut dyn crate::DynNestedProgress, - should_interrupt: &AtomicBool, - ) -> Result { - let _span = gix_trace::coarse!("fetch::Prepare::receive()"); let mut con = self.con.take().expect("receive() can only be called once"); - - if self.ref_map.mappings.is_empty() && !self.ref_map.remote_refs.is_empty() { - let mut specs = con.remote.fetch_specs.clone(); - specs.extend(self.ref_map.extra_refspecs.clone()); - return Err(Error::NoMapping { - refspecs: specs, - num_remote_refs: self.ref_map.remote_refs.len(), - }); - } - - let v1_shallow_updates = self.ref_map.handshake.v1_shallow_updates.take(); - let handshake = &self.ref_map.handshake; - let protocol_version = handshake.server_protocol_version; - - let fetch = gix_protocol::Command::Fetch; + let mut handshake = con.handshake.take().expect("receive() can only be called once"); let repo = con.remote.repo; - let fetch_features = { - let mut f = fetch.default_features(protocol_version, &handshake.capabilities); - f.push(repo.config.user_agent_tuple()); - f + let fetch_options = gix_protocol::fetch::Options { + shallow_file: repo.shallow_file(), + shallow: &self.shallow, + tags: con.remote.fetch_tags, + expected_object_hash: repo.object_hash(), + reject_shallow_remote: repo + .config + .resolved + .boolean_filter("clone.rejectShallow", &mut repo.filter_config_section()) + .map(|val| Clone::REJECT_SHALLOW.enrich_error(val)) + .transpose()? + .unwrap_or(false), }; - - gix_protocol::fetch::Response::check_required_features(protocol_version, &fetch_features)?; - let sideband_all = fetch_features.iter().any(|(n, _)| *n == "sideband-all"); - let mut arguments = gix_protocol::fetch::Arguments::new(protocol_version, fetch_features, con.trace); - if matches!(con.remote.fetch_tags, fetch::Tags::Included) { - if !arguments.can_use_include_tag() { - return Err(Error::MissingServerFeature { - feature: "include-tag", - description: - // NOTE: if this is an issue, we could probably do what's proposed here. - "To make this work we would have to implement another pass to fetch attached tags separately", - }); - } - arguments.use_include_tag(); - } - let (shallow_commits, mut shallow_lock) = add_shallow_args(&mut arguments, &self.shallow, repo)?; - - if self.ref_map.object_hash != repo.object_hash() { - return Err(Error::IncompatibleObjectHash { - local: repo.object_hash(), - remote: self.ref_map.object_hash, - }); - } - - let negotiate_span = gix_trace::detail!( - "negotiate", - protocol_version = self.ref_map.handshake.server_protocol_version as usize - ); - let mut negotiator = repo + let context = gix_protocol::fetch::Context { + handshake: &mut handshake, + transport: &mut con.transport.inner, + user_agent: repo.config.user_agent_tuple(), + trace_packetlines: con.trace, + }; + let ref_map = &self.ref_map; + let negotiator = repo .config .resolved .string(Fetch::NEGOTIATION_ALGORITHM.logical_name().as_str()) @@ -158,133 +115,35 @@ where }; let cache = graph_repo.commit_graph_if_enabled().ok().flatten(); let mut graph = graph_repo.revision_graph(cache.as_ref()); - let action = negotiate::mark_complete_and_common_ref( - &graph_repo, - negotiator.deref_mut(), - &mut graph, - &self.ref_map, - &self.shallow, - negotiate::make_refmapping_ignore_predicate(con.remote.fetch_tags, &self.ref_map), - )?; - let mut previous_response = None::; - let (mut write_pack_bundle, negotiate) = match &action { - negotiate::Action::NoChange | negotiate::Action::SkipToRefUpdate => { - gix_protocol::indicate_end_of_interaction(&mut con.transport, con.trace) - .await - .ok(); - (None, None) - } - negotiate::Action::MustNegotiate { - remote_ref_target_known, - } => { - negotiate::add_wants( - repo, - &mut arguments, - &self.ref_map, - remote_ref_target_known, - &self.shallow, - negotiate::make_refmapping_ignore_predicate(con.remote.fetch_tags, &self.ref_map), - ); - let mut rounds = Vec::new(); - let is_stateless = - arguments.is_stateless(!con.transport.connection_persists_across_multiple_requests()); - let mut haves_to_send = gix_negotiate::window_size(is_stateless, None); - let mut seen_ack = false; - let mut in_vain = 0; - let mut common = is_stateless.then(Vec::new); - let mut reader = 'negotiation: loop { - let _round = gix_trace::detail!("negotiate round", round = rounds.len() + 1); - progress.step(); - progress.set_name(format!("negotiate (round {})", rounds.len() + 1)); - - let is_done = match negotiate::one_round( - negotiator.deref_mut(), - &mut graph, - haves_to_send, - &mut arguments, - previous_response.as_ref(), - common.as_mut(), - ) { - Ok((haves_sent, ack_seen)) => { - if ack_seen { - in_vain = 0; - } - seen_ack |= ack_seen; - in_vain += haves_sent; - rounds.push(outcome::negotiate::Round { - haves_sent, - in_vain, - haves_to_send, - previous_response_had_at_least_one_in_common: ack_seen, - }); - let is_done = haves_sent != haves_to_send || (seen_ack && in_vain >= 256); - haves_to_send = gix_negotiate::window_size(is_stateless, Some(haves_to_send)); - is_done - } - Err(err) => { - gix_protocol::indicate_end_of_interaction(&mut con.transport, con.trace) - .await - .ok(); - return Err(err.into()); - } - }; - let mut reader = arguments.send(&mut con.transport, is_done).await?; - if sideband_all { - setup_remote_progress(progress, &mut reader, should_interrupt); - } - let response = gix_protocol::fetch::Response::from_line_reader( - protocol_version, - &mut reader, - is_done, - !is_done, - ) - .await?; - let has_pack = response.has_pack(); - previous_response = Some(response); - if has_pack { - progress.step(); - progress.set_name("receiving pack".into()); - if !sideband_all { - setup_remote_progress(progress, &mut reader, should_interrupt); - } - break 'negotiation reader; - } - }; - let graph = graph.detach(); - drop(graph_repo); - drop(negotiate_span); - - let mut previous_response = - previous_response.expect("knowledge of a pack means a response was received"); - previous_response.append_v1_shallow_updates(v1_shallow_updates); - if !previous_response.shallow_updates().is_empty() && shallow_lock.is_none() { - let reject_shallow_remote = repo - .config - .resolved - .boolean_filter("clone.rejectShallow", &mut repo.filter_config_section()) - .map(|val| Clone::REJECT_SHALLOW.enrich_error(val)) - .transpose()? - .unwrap_or(false); - if reject_shallow_remote { - return Err(Error::RejectShallowRemote); - } - shallow_lock = acquire_shallow_lock(repo).map(Some)?; - } + let alternates = repo.objects.store_ref().alternate_db_paths()?; + let mut negotiate = Negotiate { + objects: &graph_repo.objects, + refs: &graph_repo.refs, + graph: &mut graph, + alternates, + ref_map, + shallow: &self.shallow, + tags: con.remote.fetch_tags, + negotiator, + open_options: repo.options.clone(), + }; - let options = gix_pack::bundle::write::Options { - thread_limit: config::index_threads(repo)?, - index_version: config::pack_index_version(repo)?, - iteration_mode: gix_pack::data::input::Mode::Verify, - object_hash: con.remote.repo.object_hash(), - }; + let write_pack_options = gix_pack::bundle::write::Options { + thread_limit: config::index_threads(repo)?, + index_version: config::pack_index_version(repo)?, + iteration_mode: gix_pack::data::input::Mode::Verify, + object_hash: con.remote.repo.object_hash(), + }; + let mut write_pack_bundle = None; - let write_pack_bundle = if matches!(self.dry_run, fetch::DryRun::No) { - #[cfg(not(feature = "async-network-client"))] - let mut rd = reader; - #[cfg(feature = "async-network-client")] - let mut rd = gix_protocol::futures_lite::io::BlockOn::new(reader); + let res = gix_protocol::fetch( + ref_map, + &mut negotiate, + |reader, progress, should_interrupt| -> Result { + let mut may_read_to_end = false; + write_pack_bundle = if matches!(self.dry_run, fetch::DryRun::No) { let res = gix_pack::Bundle::write_to_directory( - &mut rd, + reader, Some(&repo.objects.store_ref().path().join("pack")), progress, should_interrupt, @@ -292,45 +151,31 @@ where let repo = repo.clone(); repo.objects })), - options, + write_pack_options, )?; - // Assure the final flush packet is consumed. - #[cfg(feature = "async-network-client")] - let has_read_to_end = { rd.get_ref().stopped_at().is_some() }; - #[cfg(not(feature = "async-network-client"))] - let has_read_to_end = { rd.stopped_at().is_some() }; - if !has_read_to_end { - std::io::copy(&mut rd, &mut std::io::sink()).map_err(Error::ReadRemainingBytes)?; - } - #[cfg(feature = "async-network-client")] - { - reader = rd.into_inner(); - } - - #[cfg(not(feature = "async-network-client"))] - { - reader = rd; - } + may_read_to_end = true; Some(res) } else { None }; - drop(reader); - - if matches!(protocol_version, gix_protocol::transport::Protocol::V2) { - gix_protocol::indicate_end_of_interaction(&mut con.transport, con.trace) - .await - .ok(); - } + Ok(may_read_to_end) + }, + progress, + should_interrupt, + context, + fetch_options, + ) + .await?; + let negotiate = res.map(|v| outcome::Negotiate { + graph: graph.detach(), + rounds: v.negotiate.rounds, + }); - if let Some(shallow_lock) = shallow_lock { - if !previous_response.shallow_updates().is_empty() { - crate::shallow::write(shallow_lock, shallow_commits, previous_response.shallow_updates())?; - } - } - (write_pack_bundle, Some(outcome::Negotiate { graph, rounds })) - } - }; + if matches!(handshake.server_protocol_version, gix_protocol::transport::Protocol::V2) { + gix_protocol::indicate_end_of_interaction(&mut con.transport.inner, con.trace) + .await + .ok(); + } let update_refs = refs::update( repo, @@ -354,6 +199,7 @@ where } let out = Outcome { + handshake, ref_map: std::mem::take(&mut self.ref_map), status: match write_pack_bundle { Some(write_pack_bundle) => Status::Change { @@ -372,72 +218,68 @@ where } } -fn acquire_shallow_lock(repo: &Repository) -> Result { - gix_lock::File::acquire_to_update_resource(repo.shallow_file(), gix_lock::acquire::Fail::Immediately, None) - .map_err(Into::into) +struct Negotiate<'a, 'b, 'c> { + objects: &'a crate::OdbHandle, + refs: &'a gix_ref::file::Store, + graph: &'a mut gix_negotiate::Graph<'b, 'c>, + alternates: Vec, + ref_map: &'a gix_protocol::fetch::RefMap, + shallow: &'a gix_protocol::fetch::Shallow, + tags: gix_protocol::fetch::Tags, + negotiator: Box, + open_options: crate::open::Options, } -fn add_shallow_args( - args: &mut Arguments, - shallow: &Shallow, - repo: &Repository, -) -> Result<(Option, Option), Error> { - let expect_change = *shallow != Shallow::NoChange; - let shallow_lock = expect_change.then(|| acquire_shallow_lock(repo)).transpose()?; - - let shallow_commits = repo.shallow_commits()?; - if (shallow_commits.is_some() || expect_change) && !args.can_use_shallow() { - // NOTE: if this is an issue, we can always unshallow the repo ourselves. - return Err(Error::MissingServerFeature { - feature: "shallow", - description: "shallow clones need server support to remain shallow, otherwise bigger than expected packs are sent effectively unshallowing the repository", - }); - } - if let Some(shallow_commits) = &shallow_commits { - for commit in shallow_commits.iter() { - args.shallow(commit); - } +impl gix_protocol::fetch::Negotiate for Negotiate<'_, '_, '_> { + fn mark_complete_and_common_ref(&mut self) -> Result { + negotiate::mark_complete_and_common_ref( + &self.objects, + self.refs, + { + let alternates = std::mem::take(&mut self.alternates); + let open_options = self.open_options.clone(); + move || -> Result<_, std::convert::Infallible> { + Ok(alternates + .into_iter() + .filter_map(move |path| { + path.ancestors() + .nth(1) + .and_then(|git_dir| crate::open_opts(git_dir, open_options.clone()).ok()) + }) + .map(|repo| (repo.refs, repo.objects))) + } + }, + self.negotiator.deref_mut(), + &mut *self.graph, + self.ref_map, + self.shallow, + negotiate::make_refmapping_ignore_predicate(self.tags, self.ref_map), + ) } - match shallow { - Shallow::NoChange => {} - Shallow::DepthAtRemote(commits) => args.deepen(commits.get() as usize), - Shallow::Deepen(commits) => { - args.deepen(*commits as usize); - args.deepen_relative(); - } - Shallow::Since { cutoff } => { - args.deepen_since(cutoff.seconds); - } - Shallow::Exclude { - remote_refs, - since_cutoff, - } => { - if let Some(cutoff) = since_cutoff { - args.deepen_since(cutoff.seconds); - } - for ref_ in remote_refs { - args.deepen_not(ref_.as_ref().as_bstr()); - } - } + + fn add_wants(&mut self, arguments: &mut Arguments, remote_ref_target_known: &[bool]) { + negotiate::add_wants( + self.objects, + arguments, + self.ref_map, + remote_ref_target_known, + self.shallow, + negotiate::make_refmapping_ignore_predicate(self.tags, self.ref_map), + ); } - Ok((shallow_commits, shallow_lock)) -} -fn setup_remote_progress<'a>( - progress: &mut dyn crate::DynNestedProgress, - reader: &mut Box + Unpin + 'a>, - should_interrupt: &'a AtomicBool, -) { - use gix_protocol::transport::client::ExtendedBufRead; - reader.set_progress_handler(Some(Box::new({ - let mut remote_progress = progress.add_child_with_id("remote".to_string(), ProgressId::RemoteProgress.into()); - move |is_err: bool, data: &[u8]| { - gix_protocol::RemoteProgress::translate_to_progress(is_err, data, &mut remote_progress); - if should_interrupt.load(Ordering::Relaxed) { - ProgressAction::Interrupt - } else { - ProgressAction::Continue - } - } - }) as gix_protocol::transport::client::HandleProgress<'a>)); + fn one_round( + &mut self, + state: &mut negotiate::one_round::State, + arguments: &mut Arguments, + previous_response: Option<&gix_protocol::fetch::Response>, + ) -> Result<(negotiate::Round, bool), negotiate::Error> { + negotiate::one_round( + self.negotiator.deref_mut(), + &mut *self.graph, + state, + arguments, + previous_response, + ) + } } diff --git a/gix/src/remote/connection/fetch/update_refs/mod.rs b/gix/src/remote/connection/fetch/update_refs/mod.rs index 7d7d8b763bb..608b1c1c1ba 100644 --- a/gix/src/remote/connection/fetch/update_refs/mod.rs +++ b/gix/src/remote/connection/fetch/update_refs/mod.rs @@ -12,8 +12,9 @@ use crate::{ remote::{ fetch, fetch::{ + refmap::Source, refs::update::{Mode, TypeChange}, - RefLogMessage, Source, + RefLogMessage, }, }, Repository, @@ -63,7 +64,7 @@ impl From for Update { pub(crate) fn update( repo: &Repository, message: RefLogMessage, - mappings: &[fetch::Mapping], + mappings: &[fetch::refmap::Mapping], refspecs: &[gix_refspec::RefSpec], extra_refspecs: &[gix_refspec::RefSpec], fetch_tags: fetch::Tags, @@ -79,7 +80,7 @@ pub(crate) fn update( .to_refspec() .filter(|_| matches!(fetch_tags, crate::remote::fetch::Tags::Included)); for (remote, local, spec, is_implicit_tag) in mappings.iter().filter_map( - |fetch::Mapping { + |fetch::refmap::Mapping { remote, local, spec_index, @@ -388,7 +389,7 @@ fn update_needs_adjustment_as_edits_symbolic_target_is_missing( fn new_value_by_remote( repo: &Repository, remote: &Source, - mappings: &[fetch::Mapping], + mappings: &[fetch::refmap::Mapping], ) -> Result { let remote_id = remote.as_id(); Ok( diff --git a/gix/src/remote/connection/fetch/update_refs/tests.rs b/gix/src/remote/connection/fetch/update_refs/tests.rs index 88d7784d0e9..980634956ea 100644 --- a/gix/src/remote/connection/fetch/update_refs/tests.rs +++ b/gix/src/remote/connection/fetch/update_refs/tests.rs @@ -56,8 +56,11 @@ mod update { remote::{ fetch, fetch::{ + refmap::Mapping, + refmap::Source, + refmap::SpecIndex, refs::{tests::restricted, update::TypeChange}, - Mapping, RefLogMessage, Source, SpecIndex, + RefLogMessage, }, }, }; @@ -910,7 +913,7 @@ mod update { fn mapping_from_spec( spec: &str, remote_repo: &gix::Repository, - ) -> (Vec, Vec) { + ) -> (Vec, Vec) { let spec = gix_refspec::parse(spec.into(), gix_refspec::parse::Operation::Fetch).unwrap(); let group = gix_refspec::MatchGroup::from_fetch_specs(Some(spec)); let references = remote_repo.references().unwrap(); @@ -920,13 +923,13 @@ mod update { .match_remotes(references.iter().map(remote_ref_to_item)) .mappings .into_iter() - .map(|m| fetch::Mapping { + .map(|m| fetch::refmap::Mapping { remote: m.item_index.map_or_else( || match m.lhs { - gix_refspec::match_group::SourceRef::ObjectId(id) => fetch::Source::ObjectId(id), + gix_refspec::match_group::SourceRef::ObjectId(id) => fetch::refmap::Source::ObjectId(id), _ => unreachable!("not a ref, must be id: {:?}", m), }, - |idx| fetch::Source::Ref(references[idx].clone()), + |idx| fetch::refmap::Source::Ref(references[idx].clone()), ), local: m.rhs.map(std::borrow::Cow::into_owned), spec_index: SpecIndex::ExplicitInRemote(m.spec_index), diff --git a/gix/src/remote/connection/fetch/update_refs/update.rs b/gix/src/remote/connection/fetch/update_refs/update.rs index 41ed3753d68..741fc12f5ba 100644 --- a/gix/src/remote/connection/fetch/update_refs/update.rs +++ b/gix/src/remote/connection/fetch/update_refs/update.rs @@ -130,13 +130,13 @@ impl Outcome { /// This can happen if the `refspecs` passed in aren't the respecs used to create the `mapping`, and it's up to the caller to sort it out. pub fn iter_mapping_updates<'a, 'b>( &self, - mappings: &'a [fetch::Mapping], + mappings: &'a [fetch::refmap::Mapping], refspecs: &'b [gix_refspec::RefSpec], extra_refspecs: &'b [gix_refspec::RefSpec], ) -> impl Iterator< Item = ( &super::Update, - &'a fetch::Mapping, + &'a fetch::refmap::Mapping, Option<&'b gix_refspec::RefSpec>, Option<&gix_ref::transaction::RefEdit>, ), diff --git a/gix/src/remote/connection/mod.rs b/gix/src/remote/connection/mod.rs index f9b8aa7e6d4..fc3a37f5be1 100644 --- a/gix/src/remote/connection/mod.rs +++ b/gix/src/remote/connection/mod.rs @@ -1,10 +1,5 @@ use crate::Remote; -pub(crate) struct HandshakeWithRefs { - outcome: gix_protocol::handshake::Outcome, - refs: Vec, -} - /// A function that performs a given credential action, trying to obtain credentials for an operation that needs it. pub type AuthenticateFn<'a> = Box gix_credentials::protocol::Result + 'a>; @@ -12,11 +7,15 @@ pub type AuthenticateFn<'a> = Box /// /// It can be used to perform a variety of operations with the remote without worrying about protocol details, /// much like a remote procedure call. -pub struct Connection<'a, 'repo, T> { +pub struct Connection<'a, 'repo, T> +where + T: gix_transport::client::Transport, +{ pub(crate) remote: &'a Remote<'repo>, pub(crate) authenticate: Option>, pub(crate) transport_options: Option>, - pub(crate) transport: T, + pub(crate) transport: gix_protocol::SendFlushOnDrop, + pub(crate) handshake: Option, pub(crate) trace: bool, } diff --git a/gix/src/remote/connection/ref_map.rs b/gix/src/remote/connection/ref_map.rs index 4752a54290b..36d5fb27d8e 100644 --- a/gix/src/remote/connection/ref_map.rs +++ b/gix/src/remote/connection/ref_map.rs @@ -1,18 +1,17 @@ -use std::collections::HashSet; - use gix_features::progress::Progress; use gix_protocol::transport::client::Transport; use crate::{ - bstr, - bstr::{BString, ByteVec}, - remote::{connection::HandshakeWithRefs, fetch, fetch::SpecIndex, Connection, Direction}, + bstr::BString, + remote::{fetch, Connection, Direction}, }; /// The error returned by [`Connection::ref_map()`]. #[derive(Debug, thiserror::Error)] #[allow(missing_docs)] pub enum Error { + #[error(transparent)] + InitRefMap(#[from] gix_protocol::fetch::refmap::init::Error), #[error("Failed to configure the transport before connecting to {url:?}")] GatherTransportConfig { url: BString, @@ -22,23 +21,16 @@ pub enum Error { ConfigureTransport(#[from] Box), #[error(transparent)] Handshake(#[from] gix_protocol::handshake::Error), - #[error("The object format {format:?} as used by the remote is unsupported")] - UnknownObjectFormat { format: BString }, - #[error(transparent)] - ListRefs(#[from] gix_protocol::ls_refs::Error), #[error(transparent)] Transport(#[from] gix_protocol::transport::client::Error), #[error(transparent)] ConfigureCredentials(#[from] crate::config::credential_helpers::Error), - #[error(transparent)] - MappingValidation(#[from] gix_refspec::match_group::validate::Error), } impl gix_protocol::transport::IsSpuriousError for Error { fn is_spurious(&self) -> bool { match self { Error::Transport(err) => err.is_spurious(), - Error::ListRefs(err) => err.is_spurious(), Error::Handshake(err) => err.is_spurious(), _ => false, } @@ -93,19 +85,21 @@ where /// - `gitoxide.userAgent` is read to obtain the application user agent for git servers and for HTTP servers as well. #[allow(clippy::result_large_err)] #[gix_protocol::maybe_async::maybe_async] - pub async fn ref_map(mut self, progress: impl Progress, options: Options) -> Result { - let res = self.ref_map_inner(progress, options).await; - gix_protocol::indicate_end_of_interaction(&mut self.transport, self.trace) - .await - .ok(); - res + pub async fn ref_map( + mut self, + progress: impl Progress, + options: Options, + ) -> Result<(fetch::RefMap, gix_protocol::handshake::Outcome), Error> { + let refmap = self.ref_map_by_ref(progress, options).await; + let handshake = self.handshake.expect("refmap always performs handshake"); + refmap.map(|map| (map, handshake)) } #[allow(clippy::result_large_err)] #[gix_protocol::maybe_async::maybe_async] - pub(crate) async fn ref_map_inner( + pub(crate) async fn ref_map_by_ref( &mut self, - progress: impl Progress, + mut progress: impl Progress, Options { prefix_from_spec_as_filter_on_remote, handshake_parameters, @@ -113,84 +107,13 @@ where }: Options, ) -> Result { let _span = gix_trace::coarse!("remote::Connection::ref_map()"); - let null = gix_hash::ObjectId::null(gix_hash::Kind::Sha1); // OK to hardcode Sha1, it's not supposed to match, ever. - if let Some(tag_spec) = self.remote.fetch_tags.to_refspec().map(|spec| spec.to_owned()) { if !extra_refspecs.contains(&tag_spec) { extra_refspecs.push(tag_spec); } }; - let specs = { - let mut s = self.remote.fetch_specs.clone(); - s.extend(extra_refspecs.clone()); - s - }; - let remote = self - .fetch_refs( - prefix_from_spec_as_filter_on_remote, - handshake_parameters, - &specs, - progress, - ) - .await?; - let num_explicit_specs = self.remote.fetch_specs.len(); - let group = gix_refspec::MatchGroup::from_fetch_specs(specs.iter().map(gix_refspec::RefSpec::to_ref)); - let (res, fixes) = group - .match_remotes(remote.refs.iter().map(|r| { - let (full_ref_name, target, object) = r.unpack(); - gix_refspec::match_group::Item { - full_ref_name, - target: target.unwrap_or(&null), - object, - } - })) - .validated()?; - - let mappings = res.mappings; - let mappings = mappings - .into_iter() - .map(|m| fetch::Mapping { - remote: m.item_index.map_or_else( - || { - fetch::Source::ObjectId(match m.lhs { - gix_refspec::match_group::SourceRef::ObjectId(id) => id, - _ => unreachable!("no item index implies having an object id"), - }) - }, - |idx| fetch::Source::Ref(remote.refs[idx].clone()), - ), - local: m.rhs.map(std::borrow::Cow::into_owned), - spec_index: if m.spec_index < num_explicit_specs { - SpecIndex::ExplicitInRemote(m.spec_index) - } else { - SpecIndex::Implicit(m.spec_index - num_explicit_specs) - }, - }) - .collect(); - - let object_hash = extract_object_format(self.remote.repo, &remote.outcome)?; - Ok(fetch::RefMap { - mappings, - extra_refspecs, - fixes, - remote_refs: remote.refs, - handshake: remote.outcome, - object_hash, - }) - } - - #[allow(clippy::result_large_err)] - #[gix_protocol::maybe_async::maybe_async] - async fn fetch_refs( - &mut self, - filter_by_prefix: bool, - extra_parameters: Vec<(String, Option)>, - refspecs: &[gix_refspec::RefSpec], - mut progress: impl Progress, - ) -> Result { - let _span = gix_trace::coarse!("remote::Connection::fetch_refs()"); let mut credentials_storage; - let url = self.transport.to_url(); + let url = self.transport.inner.to_url(); let authenticate = match self.authenticate.as_mut() { Some(f) => f, None => { @@ -203,10 +126,9 @@ where } }; + let repo = self.remote.repo; if self.transport_options.is_none() { - self.transport_options = self - .remote - .repo + self.transport_options = repo .transport_options(url.as_ref(), self.remote.name().map(crate::remote::Name::as_bstr)) .map_err(|err| Error::GatherTransportConfig { source: err, @@ -214,63 +136,31 @@ where })?; } if let Some(config) = self.transport_options.as_ref() { - self.transport.configure(&**config)?; + self.transport.inner.configure(&**config)?; } - let mut outcome = - gix_protocol::fetch::handshake(&mut self.transport, authenticate, extra_parameters, &mut progress).await?; - let refs = match outcome.refs.take() { - Some(refs) => refs, - None => { - let agent_feature = self.remote.repo.config.user_agent_tuple(); - gix_protocol::ls_refs( - &mut self.transport, - &outcome.capabilities, - move |_capabilities, arguments, features| { - features.push(agent_feature); - if filter_by_prefix { - let mut seen = HashSet::new(); - for spec in refspecs { - let spec = spec.to_ref(); - if seen.insert(spec.instruction()) { - let mut prefixes = Vec::with_capacity(1); - spec.expand_prefixes(&mut prefixes); - for mut prefix in prefixes { - prefix.insert_str(0, "ref-prefix "); - arguments.push(prefix); - } - } - } - } - Ok(gix_protocol::ls_refs::Action::Continue) - }, - &mut progress, - self.trace, - ) - .await? - } - }; - Ok(HandshakeWithRefs { outcome, refs }) + let mut handshake = gix_protocol::fetch::handshake( + &mut self.transport.inner, + authenticate, + handshake_parameters, + &mut progress, + ) + .await?; + let refmap = gix_protocol::fetch::RefMap::new( + progress, + &self.remote.fetch_specs, + gix_protocol::fetch::Context { + handshake: &mut handshake, + transport: &mut self.transport.inner, + user_agent: self.remote.repo.config.user_agent_tuple(), + trace_packetlines: self.trace, + }, + gix_protocol::fetch::refmap::init::Options { + prefix_from_spec_as_filter_on_remote, + extra_refspecs, + }, + ) + .await?; + self.handshake = Some(handshake); + Ok(refmap) } } - -/// Assume sha1 if server says nothing, otherwise configure anything beyond sha1 in the local repo configuration -#[allow(clippy::result_large_err)] -fn extract_object_format( - _repo: &crate::Repository, - outcome: &gix_protocol::handshake::Outcome, -) -> Result { - use bstr::ByteSlice; - let object_hash = - if let Some(object_format) = outcome.capabilities.capability("object-format").and_then(|c| c.value()) { - let object_format = object_format.to_str().map_err(|_| Error::UnknownObjectFormat { - format: object_format.into(), - })?; - match object_format { - "sha1" => gix_hash::Kind::Sha1, - unknown => return Err(Error::UnknownObjectFormat { format: unknown.into() }), - } - } else { - gix_hash::Kind::Sha1 - }; - Ok(object_hash) -} diff --git a/gix/src/remote/fetch.rs b/gix/src/remote/fetch.rs index 4700201de51..c822f8f9b09 100644 --- a/gix/src/remote/fetch.rs +++ b/gix/src/remote/fetch.rs @@ -4,11 +4,7 @@ pub mod negotiate { pub use gix_negotiate::Algorithm; #[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))] - pub use super::super::connection::fetch::negotiate::Error; - #[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))] - pub(crate) use super::super::connection::fetch::negotiate::{ - add_wants, make_refmapping_ignore_predicate, mark_complete_and_common_ref, one_round, Action, - }; + pub use gix_protocol::fetch::negotiate::Error; } #[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))] @@ -36,205 +32,6 @@ pub(crate) enum WritePackedRefs { Only, } -/// Describe how to handle tags when fetching -#[derive(Default, Debug, Copy, Clone, PartialEq, Eq)] -pub enum Tags { - /// Fetch all tags from the remote, even if these are not reachable from objects referred to by our refspecs. - All, - /// Fetch only the tags that point to the objects being sent. - /// That way, annotated tags that point to an object we receive are automatically transmitted and their refs are created. - /// The same goes for lightweight tags. - #[default] - Included, - /// Do not fetch any tags. - None, -} - -impl Tags { - /// Obtain a refspec that determines whether or not to fetch all tags, depending on this variant. - /// - /// The returned refspec is the default refspec for tags, but won't overwrite local tags ever. - pub fn to_refspec(&self) -> Option> { - match self { - Tags::All | Tags::Included => Some( - gix_refspec::parse("refs/tags/*:refs/tags/*".into(), gix_refspec::parse::Operation::Fetch) - .expect("valid"), - ), - Tags::None => None, - } - } -} - -/// Describe how shallow clones are handled when fetching, with variants defining how the *shallow boundary* is handled. -/// -/// The *shallow boundary* is a set of commits whose parents are not present in the repository. -#[derive(Default, Debug, Clone, PartialEq, Eq)] -pub enum Shallow { - /// Fetch all changes from the remote without affecting the shallow boundary at all. - /// - /// This also means that repositories that aren't shallow will remain like that. - #[default] - NoChange, - /// Receive update to `depth` commits in the history of the refs to fetch (from the viewpoint of the remote), - /// with the value of `1` meaning to receive only the commit a ref is pointing to. - /// - /// This may update the shallow boundary to increase or decrease the amount of available history. - DepthAtRemote(std::num::NonZeroU32), - /// Increase the number of commits and thus expand the shallow boundary by `depth` commits as seen from our local - /// shallow boundary, with a value of `0` having no effect. - Deepen(u32), - /// Set the shallow boundary at the `cutoff` time, meaning that there will be no commits beyond that time. - Since { - /// The date beyond which there will be no history. - cutoff: gix_date::Time, - }, - /// Receive all history excluding all commits reachable from `remote_refs`. These can be long or short - /// ref names or tag names. - Exclude { - /// The ref names to exclude, short or long. Note that ambiguous short names will cause the remote to abort - /// without an error message being transferred (because the protocol does not support it) - remote_refs: Vec, - /// If some, this field has the same meaning as [`Shallow::Since`] which can be used in combination - /// with excluded references. - since_cutoff: Option, - }, -} - -impl Shallow { - /// Produce a variant that causes the repository to loose its shallow boundary, effectively by extending it - /// beyond all limits. - pub fn undo() -> Self { - Shallow::DepthAtRemote((i32::MAX as u32).try_into().expect("valid at compile time")) - } -} - -/// Information about the relationship between our refspecs, and remote references with their local counterparts. -#[derive(Default, Debug, Clone)] -#[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))] -pub struct RefMap { - /// A mapping between a remote reference and a local tracking branch. - pub mappings: Vec, - /// Refspecs which have been added implicitly due to settings of the `remote`, possibly pre-initialized from - /// [`extra_refspecs` in RefMap options][crate::remote::ref_map::Options::extra_refspecs]. - /// - /// They are never persisted nor are they typically presented to the user. - pub extra_refspecs: Vec, - /// Information about the fixes applied to the `mapping` due to validation and sanitization. - pub fixes: Vec, - /// All refs advertised by the remote. - pub remote_refs: Vec, - /// Additional information provided by the server as part of the handshake. - /// - /// Note that the `refs` field is always `None` as the refs are placed in `remote_refs`. - pub handshake: gix_protocol::handshake::Outcome, - /// The kind of hash used for all data sent by the server, if understood by this client implementation. - /// - /// It was extracted from the `handshake` as advertised by the server. - pub object_hash: gix_hash::Kind, -} - -/// Either an object id that the remote has or the matched remote ref itself. -#[derive(Debug, Clone)] -#[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))] -pub enum Source { - /// An object id, as the matched ref-spec was an object id itself. - ObjectId(gix_hash::ObjectId), - /// The remote reference that matched the ref-specs name. - Ref(gix_protocol::handshake::Ref), -} - -#[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))] -impl Source { - /// Return either the direct object id we refer to or the direct target that a reference refers to. - /// The latter may be a direct or a symbolic reference. - /// If unborn, `None` is returned. - pub fn as_id(&self) -> Option<&gix_hash::oid> { - match self { - Source::ObjectId(id) => Some(id), - Source::Ref(r) => r.unpack().1, - } - } - - /// Return the target that this symbolic ref is pointing to, or `None` if it is no symbolic ref. - pub fn as_target(&self) -> Option<&crate::bstr::BStr> { - match self { - Source::ObjectId(_) => None, - Source::Ref(r) => match r { - gix_protocol::handshake::Ref::Peeled { .. } | gix_protocol::handshake::Ref::Direct { .. } => None, - gix_protocol::handshake::Ref::Symbolic { target, .. } - | gix_protocol::handshake::Ref::Unborn { target, .. } => Some(target.as_ref()), - }, - } - } - - /// Returns the peeled id of this instance, that is the object that can't be de-referenced anymore. - pub fn peeled_id(&self) -> Option<&gix_hash::oid> { - match self { - Source::ObjectId(id) => Some(id), - Source::Ref(r) => { - let (_name, target, peeled) = r.unpack(); - peeled.or(target) - } - } - } - - /// Return ourselves as the full name of the reference we represent, or `None` if this source isn't a reference but an object. - pub fn as_name(&self) -> Option<&crate::bstr::BStr> { - match self { - Source::ObjectId(_) => None, - Source::Ref(r) => match r { - gix_protocol::handshake::Ref::Unborn { full_ref_name, .. } - | gix_protocol::handshake::Ref::Symbolic { full_ref_name, .. } - | gix_protocol::handshake::Ref::Direct { full_ref_name, .. } - | gix_protocol::handshake::Ref::Peeled { full_ref_name, .. } => Some(full_ref_name.as_ref()), - }, - } - } -} - -/// An index into various lists of refspecs that have been used in a [Mapping] of remote references to local ones. #[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))] -#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash, Ord, PartialOrd)] -pub enum SpecIndex { - /// An index into the _refspecs of the remote_ that triggered a fetch operation. - /// These refspecs are explicit and visible to the user. - ExplicitInRemote(usize), - /// An index into the list of [extra refspecs][crate::remote::fetch::RefMap::extra_refspecs] that are implicit - /// to a particular fetch operation. - Implicit(usize), -} - -#[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))] -impl SpecIndex { - /// Depending on our index variant, get the index either from `refspecs` or from `extra_refspecs` for `Implicit` variants. - pub fn get<'a>( - self, - refspecs: &'a [gix_refspec::RefSpec], - extra_refspecs: &'a [gix_refspec::RefSpec], - ) -> Option<&'a gix_refspec::RefSpec> { - match self { - SpecIndex::ExplicitInRemote(idx) => refspecs.get(idx), - SpecIndex::Implicit(idx) => extra_refspecs.get(idx), - } - } - - /// If this is an `Implicit` variant, return its index. - pub fn implicit_index(self) -> Option { - match self { - SpecIndex::Implicit(idx) => Some(idx), - SpecIndex::ExplicitInRemote(_) => None, - } - } -} - -/// A mapping between a single remote reference and its advertised objects to a local destination which may or may not exist. -#[derive(Debug, Clone)] -#[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))] -pub struct Mapping { - /// The reference on the remote side, along with information about the objects they point to as advertised by the server. - pub remote: Source, - /// The local tracking reference to update after fetching the object visible via `remote`. - pub local: Option, - /// The index into the fetch ref-specs used to produce the mapping, allowing it to be recovered. - pub spec_index: SpecIndex, -} +pub use gix_protocol::fetch::{refmap, RefMap}; +pub use gix_protocol::fetch::{Shallow, Tags}; diff --git a/gix/src/repository/shallow.rs b/gix/src/repository/shallow.rs index 267b8101315..322c0c315e8 100644 --- a/gix/src/repository/shallow.rs +++ b/gix/src/repository/shallow.rs @@ -1,7 +1,6 @@ use std::{borrow::Cow, path::PathBuf}; use crate::{ - bstr::ByteSlice, config::tree::{gitoxide, Key}, Repository, }; @@ -22,28 +21,10 @@ impl Repository { /// isn't a shallow clone. /// /// The shared list is shared across all clones of this repository. - pub fn shallow_commits(&self) -> Result, crate::shallow::open::Error> { + pub fn shallow_commits(&self) -> Result, crate::shallow::read::Error> { self.shallow_commits.recent_snapshot( || self.shallow_file().metadata().ok().and_then(|m| m.modified().ok()), - || { - let buf = match std::fs::read(self.shallow_file()) { - Ok(buf) => buf, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(err) => return Err(err.into()), - }; - - let mut commits = buf - .lines() - .map(gix_hash::ObjectId::from_hex) - .collect::, _>>()?; - - commits.sort(); - if commits.is_empty() { - Ok(None) - } else { - Ok(Some(commits)) - } - }, + || gix_shallow::read(&self.shallow_file()), ) } diff --git a/gix/src/revision/walk.rs b/gix/src/revision/walk.rs index 8592d3737f5..bdc233d9e41 100644 --- a/gix/src/revision/walk.rs +++ b/gix/src/revision/walk.rs @@ -11,7 +11,7 @@ pub enum Error { #[error(transparent)] SimpleTraversal(#[from] gix_traverse::commit::simple::Error), #[error(transparent)] - ShallowCommits(#[from] crate::shallow::open::Error), + ShallowCommits(#[from] crate::shallow::read::Error), #[error(transparent)] ConfigBoolean(#[from] crate::config::boolean::Error), } diff --git a/gix/src/shallow.rs b/gix/src/shallow.rs index d49653a6506..f8870ba8334 100644 --- a/gix/src/shallow.rs +++ b/gix/src/shallow.rs @@ -5,88 +5,11 @@ pub(crate) type CommitsStorage = pub type Commits = gix_fs::SharedFileSnapshot>; /// -#[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))] -pub mod write { - pub(crate) mod function { - use std::io::Write; - - use gix_protocol::fetch::response::ShallowUpdate; - - use crate::shallow::{write::Error, Commits}; - - /// Write the previously obtained (possibly non-existing) `shallow_commits` to the shallow `file` - /// after applying all `updates`. - /// - /// If this leaves the list of shallow commits empty, the file is removed. - /// - /// ### Deviation - /// - /// Git also prunes the set of shallow commits while writing, we don't until we support some sort of pruning. - pub fn write( - mut file: gix_lock::File, - shallow_commits: Option, - updates: &[ShallowUpdate], - ) -> Result<(), Error> { - let mut shallow_commits = shallow_commits.map(|sc| (**sc).to_owned()).unwrap_or_default(); - for update in updates { - match update { - ShallowUpdate::Shallow(id) => { - shallow_commits.push(*id); - } - ShallowUpdate::Unshallow(id) => shallow_commits.retain(|oid| oid != id), - } - } - if shallow_commits.is_empty() { - std::fs::remove_file(file.resource_path())?; - drop(file); - return Ok(()); - } - - if shallow_commits.is_empty() { - if let Err(err) = std::fs::remove_file(file.resource_path()) { - if err.kind() != std::io::ErrorKind::NotFound { - return Err(err.into()); - } - } - } else { - shallow_commits.sort(); - let mut buf = Vec::::new(); - for commit in shallow_commits { - commit.write_hex_to(&mut buf).map_err(Error::Io)?; - buf.push(b'\n'); - } - file.write_all(&buf).map_err(Error::Io)?; - file.flush()?; - } - file.commit()?; - Ok(()) - } - } - - /// The error returned by [`write()`][crate::shallow::write()]. - #[derive(Debug, thiserror::Error)] - #[allow(missing_docs)] - pub enum Error { - #[error(transparent)] - Commit(#[from] gix_lock::commit::Error), - #[error("Could not remove an empty shallow file")] - RemoveEmpty(#[from] std::io::Error), - #[error("Failed to write object id to shallow file")] - Io(std::io::Error), - } +pub mod read { + pub use gix_shallow::read::Error; } -#[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))] -pub use write::function::write; /// -pub mod open { - /// The error returned by [`Repository::shallow_commits()`][crate::Repository::shallow_commits()]. - #[derive(Debug, thiserror::Error)] - #[allow(missing_docs)] - pub enum Error { - #[error("Could not open shallow file for reading")] - Io(#[from] std::io::Error), - #[error("Could not decode a line in shallow file as hex-encoded object hash")] - DecodeHash(#[from] gix_hash::decode::Error), - } +pub mod write { + pub use gix_shallow::write::Error; } diff --git a/gix/tests/gix/clone.rs b/gix/tests/gix/clone.rs index b5013d5ee84..1a22716f46b 100644 --- a/gix/tests/gix/clone.rs +++ b/gix/tests/gix/clone.rs @@ -9,7 +9,7 @@ mod blocking_io { bstr::BString, config::tree::{Clone, Core, Init, Key}, remote::{ - fetch::{Shallow, SpecIndex}, + fetch::{refmap::SpecIndex, Shallow}, Direction, }, }; @@ -99,7 +99,9 @@ mod blocking_io { assert!( matches!( err, - gix::clone::fetch::Error::Fetch(gix::remote::fetch::Error::RejectShallowRemote) + gix::clone::fetch::Error::Fetch(gix::remote::fetch::Error::Fetch( + gix_protocol::fetch::Error::RejectShallowRemote + )) ), "we can avoid fetching from remotes with this setting" ); @@ -672,7 +674,6 @@ mod blocking_io { ); let supports_unborn = out - .ref_map .handshake .capabilities .capability("ls-refs") diff --git a/gix/tests/gix/remote/ref_map.rs b/gix/tests/gix/remote/ref_map.rs index 77a9eefd846..5dbdad08cf2 100644 --- a/gix/tests/gix/remote/ref_map.rs +++ b/gix/tests/gix/remote/ref_map.rs @@ -65,7 +65,7 @@ mod blocking_and_async_io { daemon.as_ref(), None, ); - let map = remote + let (map, _handshake) = remote .connect(Fetch) .await? .ref_map(progress::Discard, Default::default()) diff --git a/gix/tests/gix/repository/worktree.rs b/gix/tests/gix/repository/worktree.rs index b39db66b9f7..294ab4a5bc2 100644 --- a/gix/tests/gix/repository/worktree.rs +++ b/gix/tests/gix/repository/worktree.rs @@ -1,8 +1,10 @@ use gix_ref::bstr; #[cfg(target_pointer_width = "64")] +#[cfg(feature = "worktree-stream")] const EXPECTED_BUFFER_LENGTH: usize = 102; #[cfg(target_pointer_width = "32")] +#[cfg(feature = "worktree-stream")] const EXPECTED_BUFFER_LENGTH: usize = 86; #[test] diff --git a/justfile b/justfile index f5a49d82641..c1b27f7658c 100755 --- a/justfile +++ b/justfile @@ -47,7 +47,6 @@ check: if cargo check -p gix-transport --all-features 2>/dev/null; then false; else true; fi if cargo check -p gix-protocol --all-features 2>/dev/null; then false; else true; fi cargo tree -p gix --no-default-features -e normal -i imara-diff 2>&1 | grep warning # warning happens if nothing found, no exit code :/ - if cargo tree -p gix --no-default-features -i gix-protocol 2>/dev/null; then false; else true; fi cargo tree -p gix --no-default-features -e normal -i gix-submodule 2>&1 | grep warning cargo tree -p gix --no-default-features -e normal -i gix-pathspec 2>&1 | grep warning cargo tree -p gix --no-default-features -e normal -i gix-filter 2>&1 | grep warning diff --git a/tests/journey/gix.sh b/tests/journey/gix.sh index 4251e0ec26f..a7acfc8f70e 100644 --- a/tests/journey/gix.sh +++ b/tests/journey/gix.sh @@ -107,7 +107,9 @@ title "gix (with repository)" ) fi - (with "git:// protocol" + # for some reason, on CI the daemon always shuts down before we can connect, + # or isn't actually ready despite having accepted the first connection already. + (not_on_ci with "git:// protocol" launch-git-daemon (with "version 1" it "generates the correct output" && { @@ -249,14 +251,14 @@ title "gix commit-graph" (with "version 2" (with "NO output directory" it "generates the correct output" && { - WITH_SNAPSHOT="$snapshot/file-v-any-no-output" \ + WITH_SNAPSHOT="$snapshot/file-v-any-no-output-p2" \ expect_run $SUCCESSFULLY "$exe_plumbing" --no-verbose free pack receive -p 2 .git } ) (with "output directory" mkdir out/ it "generates the correct output" && { - WITH_SNAPSHOT="$snapshot/file-v-any-with-output" \ + WITH_SNAPSHOT="$snapshot/file-v-any-with-output-p2" \ expect_run $SUCCESSFULLY "$exe_plumbing" --no-verbose free pack receive .git out/ } it "creates an index and a pack in the output directory" && { @@ -268,7 +270,7 @@ title "gix commit-graph" if test "$kind" = "max" || test "$kind" = "max-pure"; then (with "--format json" it "generates the correct output in JSON format" && { - WITH_SNAPSHOT="$snapshot/file-v-any-no-output-json" \ + WITH_SNAPSHOT="$snapshot/file-v-any-no-output-json-p2" \ expect_run $SUCCESSFULLY "$exe_plumbing" --no-verbose --format json free pack receive --protocol 2 .git } ) @@ -276,7 +278,7 @@ title "gix commit-graph" ) ) fi - (with "git:// protocol" + (not_on_ci with "git:// protocol" launch-git-daemon (with "version 1" (with "NO output directory" @@ -305,7 +307,7 @@ title "gix commit-graph" (with "NO output directory" (with "NO wanted refs" it "generates the correct output" && { - WITH_SNAPSHOT="$snapshot/file-v-any-no-output" \ + WITH_SNAPSHOT="$snapshot/file-v-any-no-output-p2" \ expect_run $SUCCESSFULLY "$exe_plumbing" --no-verbose free pack receive -p 2 git://localhost/ } ) @@ -324,7 +326,7 @@ title "gix commit-graph" ) (with "output directory" it "generates the correct output" && { - WITH_SNAPSHOT="$snapshot/file-v-any-with-output" \ + WITH_SNAPSHOT="$snapshot/file-v-any-with-output-p2" \ expect_run $SUCCESSFULLY "$exe_plumbing" --no-verbose free pack receive git://localhost/ out/ } ) diff --git a/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output b/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output index b908a0b0987..6ca12def6af 100644 --- a/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output +++ b/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output @@ -1,5 +1,5 @@ -index: c787de2aafb897417ca8167baeb146eabd18bc5f -pack: 346574b7331dc3a1724da218d622c6e1b6c66a57 +index: 8e48437a86dfb3939de997bb66b4bbedde9c2259 +pack: 24d2f055141373bf1011f1b0fce5dd8929a3a869 3f72b39ad1600e6dac63430c15e0d875e9d3f9d6 HEAD symref-target:refs/heads/main ee3c97678e89db4eab7420b04aef51758359f152 refs/heads/dev diff --git a/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output-json b/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output-json index ab2bb31f86b..e23d6acfa63 100644 --- a/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output-json +++ b/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output-json @@ -1,8 +1,8 @@ { "index": { "index_version": "V2", - "index_hash": "c787de2aafb897417ca8167baeb146eabd18bc5f", - "data_hash": "346574b7331dc3a1724da218d622c6e1b6c66a57", + "index_hash": "8e48437a86dfb3939de997bb66b4bbedde9c2259", + "data_hash": "24d2f055141373bf1011f1b0fce5dd8929a3a869", "num_objects": 9 }, "pack_kind": "V2", diff --git a/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output-json-p2 b/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output-json-p2 new file mode 100644 index 00000000000..64244a25ebf --- /dev/null +++ b/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output-json-p2 @@ -0,0 +1,25 @@ +{ + "index": { + "index_version": "V2", + "index_hash": "8e48437a86dfb3939de997bb66b4bbedde9c2259", + "data_hash": "24d2f055141373bf1011f1b0fce5dd8929a3a869", + "num_objects": 9 + }, + "pack_kind": "V2", + "index_path": null, + "data_path": null, + "refs": [ + { + "Direct": { + "path": "refs/heads/dev", + "object": "ee3c97678e89db4eab7420b04aef51758359f152" + } + }, + { + "Direct": { + "path": "refs/heads/main", + "object": "3f72b39ad1600e6dac63430c15e0d875e9d3f9d6" + } + } + ] +} \ No newline at end of file diff --git a/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output-non-existing-single-ref b/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output-non-existing-single-ref index 62d4a6f5c56..d3d745bbd13 100644 --- a/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output-non-existing-single-ref +++ b/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output-non-existing-single-ref @@ -1 +1 @@ -Error: unknown ref refs/heads/does-not-exist \ No newline at end of file +Error: None of the refspec(s) refs/heads/does-not-exist matched any of the 2 refs on the remote \ No newline at end of file diff --git a/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output-p2 b/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output-p2 new file mode 100644 index 00000000000..c9e1ef4f9e4 --- /dev/null +++ b/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output-p2 @@ -0,0 +1,5 @@ +index: 8e48437a86dfb3939de997bb66b4bbedde9c2259 +pack: 24d2f055141373bf1011f1b0fce5dd8929a3a869 + +ee3c97678e89db4eab7420b04aef51758359f152 refs/heads/dev +3f72b39ad1600e6dac63430c15e0d875e9d3f9d6 refs/heads/main \ No newline at end of file diff --git a/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output-single-ref b/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output-single-ref index f712606d55f..3ef28803768 100644 --- a/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output-single-ref +++ b/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output-single-ref @@ -1,2 +1,5 @@ -index: 3ff97f80d63a1261147ace4cb06191a2fd686ff6 -pack: de6c8d1e0ca3ee9331a3f92da74add15abd03049 \ No newline at end of file +index: c787de2aafb897417ca8167baeb146eabd18bc5f +pack: 346574b7331dc3a1724da218d622c6e1b6c66a57 + +ee3c97678e89db4eab7420b04aef51758359f152 refs/heads/dev +3f72b39ad1600e6dac63430c15e0d875e9d3f9d6 refs/heads/main \ No newline at end of file diff --git a/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output-wanted-ref-p1 b/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output-wanted-ref-p1 index c5203d08e6e..88a2166e807 100644 --- a/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output-wanted-ref-p1 +++ b/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output-wanted-ref-p1 @@ -1,4 +1 @@ -Error: Could not access repository or failed to read streaming pack file - -Caused by: - Want to get specific refs, but remote doesn't support this capability \ No newline at end of file +Error: None of the refspec(s) =refs/heads/main matched any of the 5 refs on the remote \ No newline at end of file diff --git a/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-with-output b/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-with-output index 821e8b0d90b..0002fa4dbc5 100644 --- a/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-with-output +++ b/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-with-output @@ -1,5 +1,5 @@ -index: c787de2aafb897417ca8167baeb146eabd18bc5f (out/pack-346574b7331dc3a1724da218d622c6e1b6c66a57.idx) -pack: 346574b7331dc3a1724da218d622c6e1b6c66a57 (out/pack-346574b7331dc3a1724da218d622c6e1b6c66a57.pack) +index: 8e48437a86dfb3939de997bb66b4bbedde9c2259 (out/pack-24d2f055141373bf1011f1b0fce5dd8929a3a869.idx) +pack: 24d2f055141373bf1011f1b0fce5dd8929a3a869 (out/pack-24d2f055141373bf1011f1b0fce5dd8929a3a869.pack) 3f72b39ad1600e6dac63430c15e0d875e9d3f9d6 HEAD symref-target:refs/heads/main ee3c97678e89db4eab7420b04aef51758359f152 refs/heads/dev diff --git a/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-with-output-p2 b/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-with-output-p2 new file mode 100644 index 00000000000..7c3547de8a0 --- /dev/null +++ b/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-with-output-p2 @@ -0,0 +1,5 @@ +index: 8e48437a86dfb3939de997bb66b4bbedde9c2259 (out/pack-24d2f055141373bf1011f1b0fce5dd8929a3a869.idx) +pack: 24d2f055141373bf1011f1b0fce5dd8929a3a869 (out/pack-24d2f055141373bf1011f1b0fce5dd8929a3a869.pack) + +ee3c97678e89db4eab7420b04aef51758359f152 refs/heads/dev +3f72b39ad1600e6dac63430c15e0d875e9d3f9d6 refs/heads/main \ No newline at end of file diff --git a/tests/snapshots/plumbing/no-repo/pack/receive/ls-in-output-dir b/tests/snapshots/plumbing/no-repo/pack/receive/ls-in-output-dir index 71db4814846..ce0f84643da 100644 --- a/tests/snapshots/plumbing/no-repo/pack/receive/ls-in-output-dir +++ b/tests/snapshots/plumbing/no-repo/pack/receive/ls-in-output-dir @@ -1,3 +1,3 @@ -pack-346574b7331dc3a1724da218d622c6e1b6c66a57.idx -pack-346574b7331dc3a1724da218d622c6e1b6c66a57.keep -pack-346574b7331dc3a1724da218d622c6e1b6c66a57.pack \ No newline at end of file +pack-24d2f055141373bf1011f1b0fce5dd8929a3a869.idx +pack-24d2f055141373bf1011f1b0fce5dd8929a3a869.keep +pack-24d2f055141373bf1011f1b0fce5dd8929a3a869.pack \ No newline at end of file