mab122
/
radicle-heartwood-lfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add 'local' and 'remote' URL formats 

To distinguish between working-copy-to-storage (local) and
storage-to-storage (remote) replication, we create two new
URL types: a `remote::Url` (`heartwood://`) and a `local::Url`
(`rad://`).

These schemes are to be used for the remote and local transports,
respectively.

The difference between the two schemes is that remote replication
requires a node id.

Signed-off-by: Alexis Sellier <alexis@radicle.xyz>
This commit is contained in:
Alexis Sellier 2022-10-24 20:19:45 +02:00
parent 7c65f08cd1
commit 66493422c2
No known key found for this signature in database
8 changed files with 371 additions and 227 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
radicle-remote-helper/src/lib.rs
View File

@ -7,7 +7,7 @@ use thiserror::Error;
use radicle::crypto::{PublicKey, Signer};
use radicle::node::Handle;
use radicle::ssh;
use radicle::storage::git::transport::{Url, UrlError};
use radicle::storage::git::transport::local::{Url, UrlError};
use radicle::storage::{ReadRepository, WriteRepository, WriteStorage};
/// The service invoked by git on the remote repository, during a push.
@ -52,11 +52,11 @@ pub fn run(profile: radicle::Profile) -> Result<(), Box<dyn std::error::Error +
}
}?;
// Default to profile key.
let public_key = url
.public_key
let namespace = url
.namespace
.unwrap_or_else(|| *profile.signer.public_key());
let proj = profile.storage.repository(url.id)?;
let proj = profile.storage.repository(url.repo)?;
if proj.is_empty()? {
return Err(Error::RepositoryNotFound(proj.path().to_path_buf()).into());
}
@ -84,14 +84,14 @@ pub fn run(profile: radicle::Profile) -> Result<(), Box<dyn std::error::Error +
// 2. Our key is not the one loaded in the profile, which means that the signed refs
// won't match the remote we're pushing to.
if *service == GIT_RECEIVE_PACK {
if profile.signer.public_key() != &public_key {
return Err(Error::KeyMismatch(public_key).into());
if profile.signer.public_key() != &namespace {
return Err(Error::KeyMismatch(namespace).into());
}
if !ssh::agent::connect()?
.request_identities::<PublicKey>()?
.contains(&public_key)
.contains(&namespace)
{
return Err(Error::KeyNotRegistered(public_key).into());
return Err(Error::KeyNotRegistered(namespace).into());
}
}
println!(); // Empty line signifies connection is established.
@ -99,7 +99,7 @@ pub fn run(profile: radicle::Profile) -> Result<(), Box<dyn std::error::Error +
let mut child = process::Command::new(service)
.arg(proj.path())
.env("GIT_DIR", proj.path())
.env("GIT_NAMESPACE", public_key.to_string())
.env("GIT_NAMESPACE", namespace.to_string())
.stdout(process::Stdio::inherit())
.stderr(process::Stdio::inherit())
.stdin(process::Stdio::inherit())
@ -113,7 +113,7 @@ pub fn run(profile: radicle::Profile) -> Result<(), Box<dyn std::error::Error +
// If our node is not running, we simply skip this step, as the
// refs will be announced eventually, when the node restarts.
if let Ok(conn) = profile.node() {
conn.announce_refs(&url.id)?;
conn.announce_refs(&url.repo)?;
}
}
}

10
radicle/src/storage/git.rs
View File

@ -966,18 +966,18 @@ mod tests {
});
opts.remote_callbacks(callbacks);
// Register the `rad://` transport.
transport::register().unwrap();
// Register the `heartwood://` transport.
transport::remote::register().unwrap();
let target = git2::Repository::init_bare(target_path).unwrap();
let stream = net::TcpStream::connect(addr).unwrap();
let smart = transport::Smart::singleton();
let smart = transport::remote::Smart::singleton();
smart.insert(proj, Box::new(stream.try_clone().unwrap()));
// Fetch with the `rad://` transport.
// Fetch with the `heartwood://` transport.
target
.remote_anonymous(&format!("rad://{}", proj))
.remote_anonymous(&format!("heartwood://{remote}/{proj}"))
.unwrap()
.fetch(
&["refs/namespaces/*:refs/namespaces/*"],

135
radicle/src/storage/git/transport.rs
View File

@ -1,133 +1,2 @@
//! Git sub-transport used for fetching radicle data.
//!
//! To have control over the communication, and to allow git streams to be multiplexed over
//! existing TCP connections, we implement the [`git2::transport::SmartSubtransport`] trait.
//!
//! We choose `rad` as the URL scheme for this custom transport, and include only the identity
//! of the repository we're looking to fetch, eg. `rad://zP1GztjSdYNHK7jpdrXbaJ6Ki2Ke`, since
//! we expect a connection to a host to already be established.
//!
//! We then maintain a map from identifier to stream, for all active streams, ie. streams that
//! are associated with an underlying TCP connection. When a URL is requested, we lookup
//! the stream and return it to the [`git2`] smart-protocol implementation, so that it can carry
//! out the git smart protocol.
//!
//! This module is meant to be used by first registering our transport with [`register`] and then
//! adding or removing streams through [`Smart`], which can be obtained via [`Smart::singleton`].
mod url;
use std::collections::HashMap;
use std::str::FromStr;
use std::sync::atomic;
use std::sync::{Arc, Mutex};
use git2::transport::SmartSubtransportStream;
use once_cell::sync::Lazy;
use crate::git;
use crate::identity::Id;
pub use url::{Url, UrlError};
/// The map of git smart sub-transport streams. We keep a global map because we have
/// no control over how [`git2::transport::register`] instantiates our [`Smart`] transport
/// or its underlying streams.
static STREAMS: Lazy<Arc<Mutex<HashMap<Id, Stream>>>> = Lazy::new(Default::default);
/// The stream associated with a repository.
type Stream = Box<dyn SmartSubtransportStream>;
/// Git transport protocol over an I/O stream.
#[derive(Clone)]
pub struct Smart {
/// The underlying active streams, keyed by repository identifier.
streams: Arc<Mutex<HashMap<Id, Stream>>>,
}
impl Smart {
/// Get access to the radicle smart transport protocol.
/// The returned object has mutable access to the underlying stream map, and is safe to clone.
pub fn singleton() -> Self {
Self {
streams: STREAMS.clone(),
}
}
/// Take a stream from the map.
/// This makes the stream unavailable until it is re-inserted.
pub fn take(&self, id: &Id) -> Option<Stream> {
#[allow(clippy::unwrap_used)]
self.streams.lock().unwrap().remove(id)
}
pub fn insert(&self, id: Id, stream: Stream) {
#[allow(clippy::unwrap_used)]
self.streams.lock().unwrap().insert(id, stream);
}
}
impl git2::transport::SmartSubtransport for Smart {
/// Run a git service on this transport.
///
/// Based on the URL, which must be of the form `rad://zP1GztjSdYNHK7jpdrXbaJ6Ki2Ke`,
/// we retrieve an underlying stream and return it.
///
/// We only support the upload-pack service, since only fetches are authorized by the
/// remote.
fn action(
&self,
url: &str,
action: git2::transport::Service,
) -> Result<Box<dyn git2::transport::SmartSubtransportStream>, git2::Error> {
let url = Url::from_str(url).map_err(|e| git2::Error::from_str(e.to_string().as_str()))?;
if let Some(stream) = self.take(&url.id) {
match action {
git2::transport::Service::UploadPackLs => {}
git2::transport::Service::UploadPack => {}
git2::transport::Service::ReceivePack => {
return Err(git2::Error::from_str(
"git-receive-pack is not supported with the custom transport",
));
}
git2::transport::Service::ReceivePackLs => {
return Err(git2::Error::from_str(
"git-receive-pack is not supported with the custom transport",
));
}
}
Ok(stream)
} else {
Err(git2::Error::from_str(&format!(
"repository {} does not have an associated stream",
url.id
)))
}
}
fn close(&self) -> Result<(), git2::Error> {
Ok(())
}
}
/// Register the radicle transport with `git`.
///
/// Returns an error if called more than once.
///
pub fn register() -> Result<(), git2::Error> {
static REGISTERED: atomic::AtomicBool = atomic::AtomicBool::new(false);
// Registration is not thread-safe, so make sure we prevent re-entrancy.
if !REGISTERED.swap(true, atomic::Ordering::SeqCst) {
unsafe {
let prefix = git::url::Scheme::Radicle.to_string();
git2::transport::register(&prefix, move |remote| {
git2::transport::Transport::smart(remote, false, Smart::singleton())
})
}
} else {
Err(git2::Error::from_str(
"custom git transport is already registered",
))
}
}
pub mod local;
pub mod remote;

2
radicle/src/storage/git/transport/local.rs Normal file
View File

@ -0,0 +1,2 @@
pub mod url;
pub use url::{Url, UrlError};

106
radicle/src/storage/git/transport/local/url.rs Normal file
View File

@ -0,0 +1,106 @@
//! Git local transport URLs.
use std::str::FromStr;
use thiserror::Error;
use crate::{
crypto,
identity::{Id, IdError},
};
/// Repository namespace.
type Namespace = crypto::PublicKey;
#[derive(Debug, Error)]
pub enum UrlError {
/// Invalid format.
#[error("invalid url format: expected `rad://<repo>[/<namespace>]`")]
InvalidFormat,
/// Unsupported URL scheme.
#[error("unsupported scheme: expected `rad://`")]
UnsupportedScheme,
/// Invalid repository identifier.
#[error("repo: {0}")]
InvalidRepository(#[source] IdError),
/// Invalid namespace.
#[error("namespace: {0}")]
InvalidNamespace(#[source] crypto::PublicKeyError),
}
/// A git local transport URL.
///
/// * Used to content-address a repository, eg. when sharing projects.
/// * Used as a remore url in a git working copy.
///
/// `rad://<repo>[/<namespace>]`
///
#[derive(Debug)]
pub struct Url {
/// Repository identifier.
pub repo: Id,
/// Repository sub-tree.
pub namespace: Option<Namespace>,
}
impl Url {
/// URL scheme.
pub const SCHEME: &str = "rad";
}
impl FromStr for Url {
type Err = UrlError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let rest = s
.strip_prefix("rad://")
.ok_or(UrlError::UnsupportedScheme)?;
let components = rest.split('/').collect::<Vec<_>>();
let (resource, namespace) = match components.as_slice() {
[resource] => (resource, None),
[resource, namespace] => (resource, Some(namespace)),
_ => return Err(UrlError::InvalidFormat),
};
let resource = Id::from_str(resource).map_err(UrlError::InvalidRepository)?;
let namespace = namespace
.map(|pk| Namespace::from_str(pk).map_err(UrlError::InvalidNamespace))
.transpose()?;
Ok(Url {
repo: resource,
namespace,
})
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn test_url_parse() {
let repo = Id::from_str("z2w8RArM3gaBXZxXhQUswE3hhLcss").unwrap();
let namespace =
Namespace::from_str("z6Mkifeb5NPS6j7JP72kEQEeuqMTpCAVcHsJi1C86jGTzHRi").unwrap();
let url = format!("rad://{repo}");
let url = Url::from_str(&url).unwrap();
assert_eq!(url.repo, repo);
assert_eq!(url.namespace, None);
let url = format!("rad://{repo}/{namespace}");
let url = Url::from_str(&url).unwrap();
assert_eq!(url.repo, repo);
assert_eq!(url.namespace, Some(namespace));
assert!(format!("heartwood://{repo}").parse::<Url>().is_err());
assert!(format!("git://{repo}").parse::<Url>().is_err());
assert!(format!("rad://{namespace}").parse::<Url>().is_err());
assert!(format!("rad://{repo}/{namespace}/fnord")
.parse::<Url>()
.is_err());
}
}

131
radicle/src/storage/git/transport/remote.rs Normal file
View File

@ -0,0 +1,131 @@
//! Git sub-transport used for fetching radicle data.
//!
//! To have control over the communication, and to allow git streams to be multiplexed over
//! existing TCP connections, we implement the [`git2::transport::SmartSubtransport`] trait.
//!
//! We choose `rad` as the URL scheme for this custom transport, and include only the identity
//! of the repository we're looking to fetch, eg. `rad://zP1GztjSdYNHK7jpdrXbaJ6Ki2Ke`, since
//! we expect a connection to a host to already be established.
//!
//! We then maintain a map from identifier to stream, for all active streams, ie. streams that
//! are associated with an underlying TCP connection. When a URL is requested, we lookup
//! the stream and return it to the [`git2`] smart-protocol implementation, so that it can carry
//! out the git smart protocol.
//!
//! This module is meant to be used by first registering our transport with [`register`] and then
//! adding or removing streams through [`Smart`], which can be obtained via [`Smart::singleton`].
pub mod url;
use std::collections::HashMap;
use std::str::FromStr;
use std::sync::atomic;
use std::sync::{Arc, Mutex};
use git2::transport::SmartSubtransportStream;
use once_cell::sync::Lazy;
use crate::identity::Id;
pub use url::{Url, UrlError};
/// The map of git smart sub-transport streams. We keep a global map because we have
/// no control over how [`git2::transport::register`] instantiates our [`Smart`] transport
/// or its underlying streams.
static STREAMS: Lazy<Arc<Mutex<HashMap<Id, Stream>>>> = Lazy::new(Default::default);
/// The stream associated with a repository.
type Stream = Box<dyn SmartSubtransportStream>;
/// Git transport protocol over an I/O stream.
#[derive(Clone)]
pub struct Smart {
/// The underlying active streams, keyed by repository identifier.
streams: Arc<Mutex<HashMap<Id, Stream>>>,
}
impl Smart {
/// Get access to the radicle smart transport protocol.
/// The returned object has mutable access to the underlying stream map, and is safe to clone.
pub fn singleton() -> Self {
Self {
streams: STREAMS.clone(),
}
}
/// Take a stream from the map.
/// This makes the stream unavailable until it is re-inserted.
pub fn take(&self, id: &Id) -> Option<Stream> {
#[allow(clippy::unwrap_used)]
self.streams.lock().unwrap().remove(id)
}
pub fn insert(&self, id: Id, stream: Stream) {
#[allow(clippy::unwrap_used)]
self.streams.lock().unwrap().insert(id, stream);
}
}
impl git2::transport::SmartSubtransport for Smart {
/// Run a git service on this transport.
///
/// Based on the URL, which must be of the form `rad://zP1GztjSdYNHK7jpdrXbaJ6Ki2Ke`,
/// we retrieve an underlying stream and return it.
///
/// We only support the upload-pack service, since only fetches are authorized by the
/// remote.
fn action(
&self,
url: &str,
action: git2::transport::Service,
) -> Result<Box<dyn git2::transport::SmartSubtransportStream>, git2::Error> {
let url = Url::from_str(url).map_err(|e| git2::Error::from_str(e.to_string().as_str()))?;
if let Some(stream) = self.take(&url.repo) {
match action {
git2::transport::Service::UploadPackLs => {}
git2::transport::Service::UploadPack => {}
git2::transport::Service::ReceivePack => {
return Err(git2::Error::from_str(
"git-receive-pack is not supported with the custom transport",
));
}
git2::transport::Service::ReceivePackLs => {
return Err(git2::Error::from_str(
"git-receive-pack is not supported with the custom transport",
));
}
}
Ok(stream)
} else {
Err(git2::Error::from_str(&format!(
"repository {} does not have an associated stream",
url.repo
)))
}
}
fn close(&self) -> Result<(), git2::Error> {
Ok(())
}
}
/// Register the radicle transport with `git`.
///
/// Returns an error if called more than once.
///
pub fn register() -> Result<(), git2::Error> {
static REGISTERED: atomic::AtomicBool = atomic::AtomicBool::new(false);
// Registration is not thread-safe, so make sure we prevent re-entrancy.
if !REGISTERED.swap(true, atomic::Ordering::SeqCst) {
unsafe {
git2::transport::register(Url::SCHEME, move |remote| {
git2::transport::Transport::smart(remote, false, Smart::singleton())
})
}
} else {
Err(git2::Error::from_str(
"custom git transport is already registered",
))
}
}

115
radicle/src/storage/git/transport/remote/url.rs Normal file
View File

@ -0,0 +1,115 @@
//! Git remote transport URLs.
use std::str::FromStr;
use thiserror::Error;
use crate::{
crypto,
identity::{Id, IdError},
};
type NodeId = crypto::PublicKey;
type Namespace = crypto::PublicKey;
#[derive(Debug, Error)]
pub enum UrlError {
/// Invalid format.
#[error("invalid url format: expected `heartwood://<node>/<repo>[/<namespace>]`")]
InvalidFormat,
/// Unsupported URL scheme.
#[error("unsupported scheme: expected `heartwood://`")]
UnsupportedScheme,
/// Invalid node identifier.
#[error("node: {0}")]
InvalidNode(#[source] crypto::PublicKeyError),
/// Invalid repository identifier.
#[error("repo: {0}")]
InvalidRepository(#[source] IdError),
/// Invalid namespace.
#[error("namespace: {0}")]
InvalidNamespace(#[source] crypto::PublicKeyError),
}
/// A git remote transport URL.
///
/// `heartwood://<node>/<repo>[/<namespace>]`
///
#[derive(Debug)]
pub struct Url {
/// Node identifier.
pub node: NodeId,
/// Repository identifier.
pub repo: Id,
/// Repository sub-tree.
pub namespace: Option<Namespace>,
}
impl Url {
/// URL scheme.
pub const SCHEME: &str = "heartwood";
}
impl FromStr for Url {
type Err = UrlError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let rest = s
.strip_prefix("heartwood://")
.ok_or(UrlError::UnsupportedScheme)?;
let components = rest.split('/').collect::<Vec<_>>();
let (node, resource, namespace) = match components.as_slice() {
[node, resource] => (node, resource, None),
[node, resource, namespace] => (node, resource, Some(namespace)),
_ => return Err(UrlError::InvalidFormat),
};
let node = NodeId::from_str(node).map_err(UrlError::InvalidNode)?;
let resource = Id::from_str(resource).map_err(UrlError::InvalidRepository)?;
let namespace = namespace
.map(|pk| Namespace::from_str(pk).map_err(UrlError::InvalidNamespace))
.transpose()?;
Ok(Url {
node,
repo: resource,
namespace,
})
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn test_url_parse() {
let node = NodeId::from_str("z6MkhaXgBZDvotDkL5257faiztiGiC2QtKLGpbnnEGta2doK").unwrap();
let repo = Id::from_str("z2w8RArM3gaBXZxXhQUswE3hhLcss").unwrap();
let namespace =
Namespace::from_str("z6Mkifeb5NPS6j7JP72kEQEeuqMTpCAVcHsJi1C86jGTzHRi").unwrap();
let url = format!("heartwood://{node}/{repo}");
let url = Url::from_str(&url).unwrap();
assert_eq!(url.node, node);
assert_eq!(url.repo, repo);
assert_eq!(url.namespace, None);
let url = format!("heartwood://{node}/{repo}/{namespace}");
let url = Url::from_str(&url).unwrap();
assert_eq!(url.node, node);
assert_eq!(url.repo, repo);
assert_eq!(url.namespace, Some(namespace));
assert!(format!("heartwood://{node}").parse::<Url>().is_err());
assert!(format!("rad://{node}").parse::<Url>().is_err());
assert!(format!("heartwood://{node}/{namespace}")
.parse::<Url>()
.is_err());
assert!(format!("heartwood://{node}/{repo}/{namespace}/fnord")
.parse::<Url>()
.is_err());
}
}

79
radicle/src/storage/git/transport/url.rs
View File

@ -1,79 +0,0 @@
//! Git transport URLs.
use std::str::FromStr;
use thiserror::Error;
use crate::crypto::PublicKey;
use crate::{crypto, git, identity};
#[derive(Debug, Error)]
pub enum UrlError {
/// Failed to parse.
#[error(transparent)]
Parse(#[from] git::url::parse::Error),
/// Unsupported URL scheme.
#[error("{0}: unsupported scheme: expected `rad://`")]
UnsupportedScheme(git::Url),
/// Missing host.
#[error("{0}: missing id")]
MissingId(git::Url),
/// Invalid remote repository identifier.
#[error("{0}: id: {1}")]
InvalidId(git::Url, identity::IdError),
/// Invalid public key.
#[error("{0}: key: {1}")]
InvalidKey(git::Url, crypto::PublicKeyError),
}
/// A git remote URL.
///
/// `rad://<id>/[<pubkey>]`
///
/// Eg. `rad://zUBDc1UdoEzbpaGcNXqauQkERJ8r` without the public key,
/// and `rad://zUBDc1UdoEzbpaGcNXqauQkERJ8r/zCQTxdZGCzQXWBV3XbY3fgkHM3gfkLGyYMd2nL5R2MxQv` with.
///
#[derive(Debug)]
pub struct Url {
pub id: identity::Id,
pub public_key: Option<PublicKey>,
}
impl FromStr for Url {
type Err = UrlError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let url: git::Url = s.as_bytes().try_into()?;
Url::try_from(url)
}
}
impl TryFrom<git::Url> for Url {
type Error = UrlError;
fn try_from(url: git::Url) -> Result<Self, Self::Error> {
if url.scheme != git::url::Scheme::Radicle {
return Err(Self::Error::UnsupportedScheme(url));
}
let id: identity::Id = url
.host
.as_ref()
.ok_or_else(|| Self::Error::MissingId(url.clone()))?
.parse()
.map_err(|e| Self::Error::InvalidId(url.clone(), e))?;
let public_key: Option<PublicKey> = if url.path.is_empty() {
Ok(None)
} else {
let path = url.path.to_string();
path.strip_prefix('/')
.unwrap_or(&path)
.parse()
.map(Some)
.map_err(|e| Self::Error::InvalidKey(url.clone(), e))
}?;
Ok(Url { id, public_key })
}
}