Skip to content
Open
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 117 additions & 9 deletions lore-revision/src/repository/status.rs
Original file line number Diff line number Diff line change
Expand Up @@ -806,6 +806,55 @@ async fn count_subtrees(roots: Vec<CountWork>) -> Result<(u64, u64), StatusError
Ok((directories, files))
}

const REMOTE_RESOLVE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(3);
Comment thread
lehuan5062 marked this conversation as resolved.
Outdated

/// Time-boxed remote branch resolution that degrades gracefully on timeout.
///
/// Resolves the latest revision and authorization status of a branch on the
/// configured remote, timing out after the given deadline. On timeout or remote
/// unavailability, degrades to (None, false, false) — same shape as `NoRemote` —
/// ensuring an unreachable remote never stalls a local status read.
///
/// # Arguments
///
/// * `repository` — repository context with the configured remote.
/// * `branch_id` — the branch to query on the remote.
/// * `deadline` — maximum time to wait for remote resolution.
///
/// # Returns
///
/// A 3-tuple `(latest, authorized, available)` where:
/// - `latest` is the remote branch's latest revision hash, or None if unresolved.
/// - `authorized` is true iff the remote query returned an authoritative answer.
/// - `available` is true iff the remote connected and responded within the deadline
/// (even if with a non-branch-not-found error; a reachable server that errors is not unavailable).
async fn resolve_remote_latest(
repository: &Arc<RepositoryContext>,
branch_id: BranchId,
deadline: std::time::Duration,
) -> (Option<Hash>, bool, bool) {
async fn inner(
repository: &Arc<RepositoryContext>,
branch_id: BranchId,
) -> (Option<Hash>, bool, bool) {
let remote = match repository.remote().await {
Ok(r) => r,
Err(_) => return (None, false, false),
Comment thread
lehuan5062 marked this conversation as resolved.
Outdated
};

match branch::load_remote(remote, repository.id, branch_id).await {
Ok(status) => (Some(status.latest), true, true),
Err(err) if err.is_branch_not_found() => (None, true, true),
Err(_) => (None, false, true),
Comment thread
lehuan5062 marked this conversation as resolved.
Outdated
}
}

match tokio::time::timeout(deadline, inner(repository, branch_id)).await {
Ok((latest, authorized, available)) => (latest, authorized, available),
Err(_) => (None, false, false),
}
}

pub async fn status(
repository: Arc<RepositoryContext>,
paths: Option<Vec<RelativePath>>,
Expand Down Expand Up @@ -907,14 +956,8 @@ pub async fn status(

// Authorized only on an authoritative answer — a latest revision
// or branch not found; proving the identity is authorized and has access
let (remote_latest, remote_authorized) = match repository.remote().await {
Ok(remote) => match branch::load_remote(remote, repository.id, branch.id).await {
Ok(status) => (Some(status.latest), true),
Err(err) if err.is_branch_not_found() => (None, true),
Err(_) => (None, false),
},
Err(_) => (None, false),
};
let (remote_latest, remote_authorized, remote_available) =
resolve_remote_latest(&repository, branch.id, REMOTE_RESOLVE_TIMEOUT).await;

let remote_state = if let Some(remote_latest) = remote_latest {
state::State::deserialize(repository.clone(), remote_latest)
Expand Down Expand Up @@ -1080,7 +1123,7 @@ pub async fn status(
},
local_ahead,
remote_ahead,
repository.remote().await.is_ok(),
remote_available,
remote_authorized,
remote_latest.is_some(),
);
Expand Down Expand Up @@ -1458,3 +1501,68 @@ pub async fn status(

Ok(())
}

#[cfg(test)]
mod remote_resolve_tests {
use std::future::pending;

use futures::FutureExt;

use super::*;
use crate::lore::BranchId;
use crate::repository::RemoteState;
use crate::repository::RepositoryContext;
use crate::repository::RepositoryFormat;
use crate::repository::create_client_memory_stores;

async fn context_with_state(state: RemoteState) -> Arc<RepositoryContext> {
let (immutable, mutable) = create_client_memory_stores()
.await
.expect("in-memory stores should be creatable");
Arc::new(RepositoryContext::new_with_state(
None,
immutable,
mutable,
crate::lore::RepositoryId::default(),
crate::instance::InstanceId::default(),
state,
Arc::default(),
RepositoryFormat::Lore,
))
}

#[tokio::test]
async fn hung_remote_resolution_times_out_and_degrades() {
let never: futures::future::BoxFuture<'static, _> = pending().boxed();
let shared = never.shared();
let ctx = context_with_state(RemoteState::Pending(shared)).await;

let result = resolve_remote_latest(
&ctx,
BranchId::default(),
std::time::Duration::from_millis(10),
)
.await;

assert_eq!(
result,
(None, false, false),
"timeout should degrade to unavailable"
);
}

#[tokio::test]
async fn offline_remote_resolves_to_unavailable() {
let ctx = context_with_state(RemoteState::Offline).await;

let result =
resolve_remote_latest(&ctx, BranchId::default(), std::time::Duration::from_secs(3))
.await;

assert_eq!(
result,
(None, false, false),
"offline should degrade to unavailable"
);
}
}
3 changes: 3 additions & 0 deletions lore-transport/src/grpc/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ pub const REVISION_LIST_STRATEGY_HEADER: &str = "x-lore-revision-list-strategy";
const RETRY_START_BACKOFF_MS: u64 = 50;
const RETRY_MAX_BACKOFF_MS: u64 = 10_000;
const RETRY_MAX_ATTEMPTS: usize = 60;
const GRPC_CONNECT_TIMEOUT_SECS: u64 = 5;

fn grpc_retry() -> crate::util::Retry {
crate::util::retry(
Expand Down Expand Up @@ -575,6 +576,8 @@ async fn connect_to_endpoint(remote: &str) -> Result<Channel, ProtocolError> {

lore_trace!("Set user agent to {user_agent}");

endpoint = endpoint.connect_timeout(Duration::from_secs(GRPC_CONNECT_TIMEOUT_SECS));

// Silent propagation of connection errors
let channel = endpoint
.connect()
Expand Down
15 changes: 12 additions & 3 deletions lore-transport/src/quic/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ pub struct EndpointConfig {

const IDLE_TIMEOUT_MS: u32 = 30000;
const KEEP_ALIVE_MS: u64 = 500;
const HANDSHAKE_TIMEOUT_SECS: u64 = 5;
pub const DEFAULT_EXPECTED_RTT_MS: u64 = 100;

#[derive(Clone, Debug)]
Expand Down Expand Up @@ -772,14 +773,22 @@ pub async fn connect(
Ok(mut endpoint) => {
endpoint.set_default_client_config(client_config.clone());
match endpoint.connect(remote_addr, server_name) {
Ok(connecting) => match connecting.await {
Ok(connection) => {
Ok(connecting) => match tokio::time::timeout(
Duration::from_secs(HANDSHAKE_TIMEOUT_SECS),
connecting,
)
.await
{
Ok(Ok(connection)) => {
lore_debug!("Success QUIC connecting to {remote_addr}");
return Ok(connection);
}
Err(err) => {
Ok(Err(err)) => {
lore_debug!("Failed QUIC connecting to {remote_addr}: {err}");
}
Err(_) => {
lore_debug!("QUIC handshake timeout to {remote_addr}");
}
},
Err(err) => {
lore_debug!("Failed QUIC connect to {remote_addr}: {err}");
Expand Down