From f18afb793c52fb5501546fe33a8d335aad8ba7b5 Mon Sep 17 00:00:00 2001 From: Alexis Sellier Date: Mon, 29 Aug 2022 11:16:53 +0200 Subject: [PATCH] node: Change storage layout Projects now have their own repos. Signed-off-by: Alexis Sellier --- node/src/git.rs | 133 ++++++++++++----------------- node/src/identity.rs | 25 +++--- node/src/protocol.rs | 9 +- node/src/storage.rs | 168 ++++++++++--------------------------- node/src/storage/git.rs | 56 ++++--------- node/src/test/arbitrary.rs | 3 +- node/src/test/fixtures.rs | 18 ++-- node/src/test/storage.rs | 17 +++- 8 files changed, 164 insertions(+), 265 deletions(-) diff --git a/node/src/git.rs b/node/src/git.rs index 160a85bc..1097f4b4 100644 --- a/node/src/git.rs +++ b/node/src/git.rs @@ -1,85 +1,62 @@ -use crate::identity::ProjId; -use crate::storage::{Error, WriteStorage}; +use std::str::FromStr; + +use crate::collections::HashMap; +use crate::identity::UserId; +use crate::storage::{Remote, Remotes, Unverified}; +use git_ref_format as format; /// Default port of the `git` transport protocol. pub const PROTOCOL_PORT: u16 = 9418; -/// Fetch all remotes of a project from the given URL. -pub fn fetch(proj: &ProjId, url: &str, mut storage: S) -> Result<(), Error> { - // TODO: Use `Url` type? - // TODO: Have function to fetch specific remotes. - // TODO: Return meaningful info on success. - // - // Repository layout should look like this: - // - // /refs/namespaces/ - // /refs/namespaces/ - // /heads - // /master - // /tags - // ... - // - let repo = storage.repository(); - let refs: &[&str] = &[&format!( - "refs/namespaces/{}/refs/*:refs/namespaces/{}/refs/*", - proj, proj - )]; - let mut remote = repo.remote_anonymous(url)?; - let mut opts = git2::FetchOptions::default(); - - remote.fetch(refs, Some(&mut opts), None)?; - - Ok(()) +#[derive(thiserror::Error, Debug)] +pub enum RefError { + #[error("invalid ref name '{0}'")] + InvalidName(format::RefString), + #[error("invalid ref format: {0}")] + Format(#[from] format::Error), } -#[cfg(test)] -mod tests { - use super::*; - use crate::hash::Digest; - use crate::identity::ProjId; - use crate::storage::Storage; - - /// Create an initial empty commit. - fn initial_commit(repo: &git2::Repository) -> Result { - // First use the config to initialize a commit signature for the user. - let sig = git2::Signature::now("cloudhead", "cloudhead@radicle.xyz")?; - // Now let's create an empty tree for this commit. - let tree_id = repo.index()?.write_tree()?; - let tree = repo.find_tree(tree_id)?; - let oid = repo.commit(Some("HEAD"), &sig, &sig, "Initial commit", &tree, &[])?; - - Ok(oid) - } - - #[test] - fn test_fetch() { - let path = tempfile::tempdir().unwrap().into_path(); - let alice = git2::Repository::init_bare(path.join("alice")).unwrap(); - let bob = git2::Repository::init_bare(path.join("bob")).unwrap(); - let mut bob_storage = Storage::from(bob); - let proj = ProjId::from(Digest::new(&[42])); - let master = format!("refs/namespaces/{}/refs/heads/master", proj); - let alice_oid = initial_commit(&alice).unwrap(); - - alice - .reference(&master, alice_oid, false, "Create master branch") - .unwrap(); - - // Have Bob fetch Alice's refs. - fetch( - &proj, - &format!("file://{}/alice", path.display()), - &mut bob_storage, - ) - .unwrap(); - - let bob_oid = bob_storage - .repository() - .find_reference(&master) - .unwrap() - .target() - .unwrap(); - - assert_eq!(alice_oid, bob_oid); - } +#[derive(thiserror::Error, Debug)] +pub enum ListRefsError { + #[error("git error: {0}")] + Git(#[from] git2::Error), + #[error("invalid ref: {0}")] + InvalidRef(#[from] RefError), +} + +/// List remote refs of a project, given the remote URL. +pub fn list_remotes(url: &str) -> Result, ListRefsError> { + let mut remotes = HashMap::default(); + let mut remote = git2::Remote::create_detached(url)?; + + remote.connect(git2::Direction::Fetch)?; + + let refs = remote.list()?; + for r in refs { + let (id, refname) = parse_ref::(r.name())?; + let entry = remotes + .entry(id.clone()) + .or_insert_with(|| Remote::new(id, HashMap::default())); + + entry.refs.insert(refname.to_string(), r.oid().into()); + } + + Ok(Remotes::new(remotes)) +} + +/// Parse a ref string. +pub fn parse_ref(s: &str) -> Result<(T, format::RefString), RefError> { + let input = format::RefStr::try_from_str(s)?; + let suffix = input + .strip_prefix(format::refname!("refs/namespaces")) + .ok_or_else(|| RefError::InvalidName(input.to_owned()))?; + + let mut components = suffix.components(); + let id = components + .next() + .ok_or_else(|| RefError::InvalidName(input.to_owned()))?; + let id = T::from_str(&id.to_string()).map_err(|_| RefError::InvalidName(input.to_owned()))?; + let refstr = components.collect::(); + + Ok((id, refstr)) } diff --git a/node/src/identity.rs b/node/src/identity.rs index adf9ee2a..e1dede62 100644 --- a/node/src/identity.rs +++ b/node/src/identity.rs @@ -1,4 +1,4 @@ -use std::{fmt, io, ops::Deref, str::FromStr}; +use std::{ffi::OsString, fmt, io, ops::Deref, str::FromStr}; use ed25519_consensus::{VerificationKey, VerificationKeyBytes}; use nonempty::NonEmpty; @@ -10,8 +10,6 @@ use crate::hash; #[derive(Error, Debug)] pub enum ProjIdError { - #[error("invalid ref '{0}'")] - InvalidRef(String), #[error("invalid digest: {0}")] InvalidDigest(#[from] hash::DecodeError), } @@ -35,24 +33,25 @@ impl ProjId { pub fn encode(&self) -> String { multibase::encode(multibase::Base::Base58Btc, &self.0.as_ref()) } - - pub(crate) fn from_ref(s: &str) -> Result { - if let Some(s) = s.split('/').nth(2) { - let id = Self::from_str(s)?; - return Ok(id); - } - Err(ProjIdError::InvalidRef(s.to_owned())) - } } impl FromStr for ProjId { - type Err = hash::DecodeError; + type Err = ProjIdError; fn from_str(s: &str) -> Result { Ok(Self(hash::Digest::from_str(s)?)) } } +impl TryFrom for ProjId { + type Error = ProjIdError; + + fn try_from(value: OsString) -> Result { + let string = value.to_string_lossy(); + Self::from_str(&string) + } +} + impl From for ProjId { fn from(digest: hash::Digest) -> Self { Self(digest) @@ -134,8 +133,6 @@ pub enum UserIdError { InvalidKey(#[from] ed25519_consensus::Error), } -impl UserId {} - #[derive(Error, Debug)] pub enum DocError { #[error("toml: {0}")] diff --git a/node/src/protocol.rs b/node/src/protocol.rs index 34633c8c..75cd6d89 100644 --- a/node/src/protocol.rs +++ b/node/src/protocol.rs @@ -16,9 +16,8 @@ use crate::address_manager::AddressManager; use crate::clock::RefClock; use crate::collections::{HashMap, HashSet}; use crate::decoder::Decoder; -use crate::git; use crate::identity::{ProjId, UserId}; -use crate::storage; +use crate::storage::{self, WriteRepository}; use crate::storage::{Inventory, ReadStorage, Remotes, Unverified, WriteStorage}; /// Network peer identifier. @@ -452,7 +451,11 @@ where match cmd { Command::Connect(addr) => self.context.connect(addr), Command::Fetch(proj, remote) => { - git::fetch(&proj, &format!("git://{}", remote), &mut self.storage).unwrap(); + self.storage + .repository(&proj) + .unwrap() + .fetch(&format!("git://{}", remote)) + .unwrap(); } } } diff --git a/node/src/storage.rs b/node/src/storage.rs index 93af439b..9af8ce48 100644 --- a/node/src/storage.rs +++ b/node/src/storage.rs @@ -1,22 +1,22 @@ +pub mod git; + +use std::collections::hash_map; +use std::io; use std::marker::PhantomData; use std::ops::{Deref, DerefMut}; use std::path::Path; -use std::{fmt, fs, io}; -use git_ref_format::refspec; use once_cell::sync::Lazy; -use radicle_git_ext as git_ext; use serde::{Deserialize, Serialize}; use thiserror::Error; pub use radicle_git_ext::Oid; use crate::collections::HashMap; +use crate::git::RefError; use crate::identity; use crate::identity::{ProjId, ProjIdError, UserId}; -pub static RAD_ID_GLOB: Lazy = - Lazy::new(|| refspec::pattern!("refs/namespaces/*/refs/rad/id")); pub static IDENTITY_PATH: Lazy<&Path> = Lazy::new(|| Path::new(".rad/identity.toml")); pub type BranchName = String; @@ -27,6 +27,8 @@ pub type Inventory = Vec<(ProjId, HashMap>)>; pub enum Error { #[error("invalid git reference")] InvalidRef, + #[error("git reference error: {0}")] + Ref(#[from] RefError), #[error("git: {0}")] Git(#[from] git2::Error), #[error("id: {0}")] @@ -51,7 +53,7 @@ pub struct Verified; pub struct Unverified; /// Project remotes. Tracks the git state of a project. -#[derive(Debug, Default, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct Remotes(HashMap>); impl Remotes { @@ -60,6 +62,21 @@ impl Remotes { } } +impl Default for Remotes { + fn default() -> Self { + Self(HashMap::default()) + } +} + +impl IntoIterator for Remotes { + type Item = (RemoteId, Remote); + type IntoIter = hash_map::IntoIter>; + + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} + #[allow(clippy::from_over_into)] impl Into>> for Remotes { fn into(self) -> HashMap> { @@ -73,20 +90,23 @@ impl Into>> for Remotes { } /// A project remote. -#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, Eq)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] pub struct Remote { + /// ID of remote. + pub id: UserId, /// Git references published under this remote, and their hashes. - refs: HashMap, + pub refs: HashMap, /// Whether this remote is of a project delegate. - delegate: bool, + pub delegate: bool, /// Whether the remote is verified or not, ie. whether its signed refs were checked. #[serde(skip)] verified: PhantomData, } impl Remote { - pub fn new(refs: HashMap) -> Self { + pub fn new(id: UserId, refs: HashMap) -> Self { Self { + id, refs, delegate: false, verified: PhantomData, @@ -100,12 +120,18 @@ pub trait ReadStorage { } pub trait WriteStorage { - fn repository(&mut self) -> &mut git2::Repository; - fn namespace( - &mut self, - proj: &ProjId, - user: &UserId, - ) -> Result<&mut git2::Repository, git2::Error>; + type Repository: WriteRepository; + + fn repository(&self, proj: &ProjId) -> Result; +} + +pub trait ReadRepository { + fn remotes(&self) -> Result, Error>; +} + +pub trait WriteRepository { + fn fetch(&mut self, url: &str) -> Result<(), git2::Error>; + fn namespace(&mut self, user: &UserId) -> Result<&mut git2::Repository, git2::Error>; } impl ReadStorage for T @@ -127,114 +153,10 @@ where T: DerefMut, S: WriteStorage + 'static, { - fn repository(&mut self) -> &mut git2::Repository { - self.deref_mut().repository() - } + type Repository = S::Repository; - fn namespace( - &mut self, - proj: &ProjId, - user: &UserId, - ) -> Result<&mut git2::Repository, git2::Error> { - self.deref_mut().namespace(proj, user) - } -} - -pub struct Storage { - backend: git2::Repository, -} - -impl From for Storage { - fn from(backend: git2::Repository) -> Self { - Self { backend } - } -} - -impl fmt::Debug for Storage { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "Storage(..)") - } -} - -impl ReadStorage for Storage { - fn get(&self, _id: &ProjId) -> Result>, Error> { - todo!() - } - - fn inventory(&self) -> Result { - let glob: String = RAD_ID_GLOB.clone().into(); - let refs = self.backend.references_glob(glob.as_str())?; - - for r in refs { - let r = r?; - let name = r.name().ok_or(Error::InvalidRef)?; - let _id = ProjId::from_ref(name)?; - - todo!(); - } - Ok(vec![]) - } -} - -impl WriteStorage for Storage { - fn repository(&mut self) -> &mut git2::Repository { - &mut self.backend - } - - fn namespace( - &mut self, - proj: &ProjId, - user: &UserId, - ) -> Result<&mut git2::Repository, git2::Error> { - let path = self.backend.path(); - - self.backend = git2::Repository::open_bare(path)?; - self.backend.set_namespace(&format!("{}/{}", proj, user))?; - - Ok(&mut self.backend) - } -} - -impl Storage { - pub fn open>(path: P) -> Result { - let path = path.as_ref(); - let backend = match git2::Repository::open_bare(path) { - Err(e) if git_ext::is_not_found_err(&e) => { - let backend = git2::Repository::init_opts( - path, - git2::RepositoryInitOptions::new() - .bare(true) - .no_reinit(true) - .external_template(false), - )?; - - Ok(backend) - } - Ok(repo) => Ok(repo), - Err(e) => Err(e), - }?; - - Ok(Self { backend }) - } - - pub fn create( - &self, - repo: &git2::Repository, - identity: impl Into, - ) -> Result<(ProjId, git2::Reference), Error> { - let doc = identity.into(); - let file = fs::OpenOptions::new() - .create_new(true) - .write(true) - .open(*IDENTITY_PATH)?; - let id = doc.write(file)?; - let ref_name = RAD_ID_GLOB.replace('*', &id.encode()); - let oid = repo.head()?.target().ok_or(Error::InvalidHead)?; - let reference = self.backend.reference(&ref_name, oid, false, "")?; - - // TODO: Push project to monorepo. - - Ok((id, reference)) + fn repository(&self, proj: &ProjId) -> Result { + self.deref().repository(proj) } } diff --git a/node/src/storage/git.rs b/node/src/storage/git.rs index 9e4a5a48..6779b7be 100644 --- a/node/src/storage/git.rs +++ b/node/src/storage/git.rs @@ -1,29 +1,26 @@ -use std::marker::PhantomData; -use std::ops::{Deref, DerefMut}; use std::path::{Path, PathBuf}; -use std::str::FromStr; -use std::{fmt, fs, io}; +use std::{fmt, fs}; use git_ref_format::refspec; -use git_url::Url; use once_cell::sync::Lazy; use radicle_git_ext as git_ext; -use serde::{Deserialize, Serialize}; pub use radicle_git_ext::Oid; use crate::collections::HashMap; use crate::git; use crate::identity; -use crate::identity::{ProjId, ProjIdError, UserId}; +use crate::identity::{ProjId, UserId}; use super::{ - Error, Inventory, ReadRepository, ReadStorage, Remote, Remotes, Unverified, Verified, - WriteRepository, WriteStorage, + Error, Inventory, ReadRepository, ReadStorage, Remote, Remotes, Unverified, WriteRepository, + WriteStorage, }; pub static RAD_ROOT_GLOB: Lazy = Lazy::new(|| refspec::pattern!("refs/namespaces/*/refs/rad/root")); +pub static NAMESPACES_GLOB: Lazy = + Lazy::new(|| refspec::pattern!("refs/namespaces/*")); pub static IDENTITY_PATH: Lazy<&Path> = Lazy::new(|| Path::new(".rad/identity.toml")); pub struct Storage { @@ -48,7 +45,6 @@ impl ReadStorage for Storage { } fn inventory(&self) -> Result { - let glob: String = RAD_ROOT_GLOB.clone().into(); let projs = self.projects()?; let mut inv = Vec::new(); @@ -57,7 +53,7 @@ impl ReadStorage for Storage { let remotes = repo .remotes()? .into_iter() - .map(|r| (r.id.to_string(), r)) + .map(|(id, r)| (id.to_string(), r)) .collect(); inv.push((proj, remotes)); @@ -157,8 +153,8 @@ impl Repository { } impl ReadRepository for Repository { - fn remotes(&self) -> Result>, Error> { - let refs = self.backend.references_glob(RAD_ROOT_GLOB.as_str())?; + fn remotes(&self) -> Result, Error> { + let refs = self.backend.references_glob(NAMESPACES_GLOB.as_str())?; let mut remotes = HashMap::default(); for r in refs { @@ -172,7 +168,7 @@ impl ReadRepository for Repository { entry.refs.insert(refname.to_string(), oid.into()); } - Ok(remotes.into_values().collect()) + Ok(Remotes::new(remotes)) } } @@ -192,7 +188,7 @@ impl WriteRepository for Repository { // /tags // ... // - let refs: &[&str] = &[&format!("refs/namespaces/*:refs/namespaces/*")]; + let refs: &[&str] = &["refs/namespaces/*:refs/namespaces/*"]; let mut remote = self.backend.remote_anonymous(url)?; let mut opts = git2::FetchOptions::default(); @@ -221,38 +217,22 @@ impl From for Repository { mod tests { use super::*; use crate::git; - use crate::hash::Digest; - use crate::identity::ProjId; use crate::storage::{ReadStorage, WriteRepository}; use crate::test::fixtures; - /// Create an initial empty commit. - fn initial_commit(repo: &git2::Repository) -> Result { - // First use the config to initialize a commit signature for the user. - let sig = git2::Signature::now("cloudhead", "cloudhead@radicle.xyz")?; - // Now let's create an empty tree for this commit. - let tree_id = repo.index()?.write_tree()?; - let tree = repo.find_tree(tree_id)?; - let oid = repo.commit(Some("HEAD"), &sig, &sig, "Initial commit", &tree, &[])?; - - Ok(oid) - } - #[test] - fn test_ls_remote() { - crate::test::logger::init(log::Level::Debug); - + fn test_list_remotes() { let dir = tempfile::tempdir().unwrap(); let storage = fixtures::storage(dir.path()); let inv = storage.inventory().unwrap(); let (proj, _) = inv.first().unwrap(); - let refs = git::list_refs(&format!( + let refs = git::list_remotes(&format!( "file://{}", dir.path().join(&proj.to_string()).display(), )) .unwrap(); - let remotes = storage.repository(&proj).unwrap().remotes().unwrap(); + let remotes = storage.repository(proj).unwrap().remotes().unwrap(); assert_eq!(refs, remotes); } @@ -267,7 +247,7 @@ mod tests { let refname = "refs/heads/master"; // Have Bob fetch Alice's refs. - bob.repository(&proj) + bob.repository(proj) .unwrap() .fetch(&format!( "file://{}", @@ -275,14 +255,14 @@ mod tests { )) .unwrap(); - for (_, remote) in remotes { + for remote in remotes.values() { let alice_oid = alice - .repository(&proj) + .repository(proj) .unwrap() .find_reference(&remote.id, refname) .unwrap(); let bob_oid = bob - .repository(&proj) + .repository(proj) .unwrap() .find_reference(&remote.id, refname) .unwrap(); diff --git a/node/src/test/arbitrary.rs b/node/src/test/arbitrary.rs index c4fa1e94..0989875e 100644 --- a/node/src/test/arbitrary.rs +++ b/node/src/test/arbitrary.rs @@ -26,6 +26,7 @@ impl quickcheck::Arbitrary for storage::Remote { let mut refs: HashMap = HashMap::with_hasher(rng.into()); let mut bytes: [u8; 20] = [0; 20]; let names = &["master", "dev", "feature/1", "feature/2", "feature/3"]; + let id = UserId::arbitrary(g); for _ in 0..g.size().min(2) { if let Some(name) = g.choose(names) { @@ -36,7 +37,7 @@ impl quickcheck::Arbitrary for storage::Remote { refs.insert(name.to_string(), oid); } } - storage::Remote::new(refs) + storage::Remote::new(id, refs) } } diff --git a/node/src/test/fixtures.rs b/node/src/test/fixtures.rs index d20efbeb..8f62a077 100644 --- a/node/src/test/fixtures.rs +++ b/node/src/test/fixtures.rs @@ -4,7 +4,8 @@ use std::str::FromStr; use once_cell::sync::Lazy; use crate::identity::{ProjId, UserId}; -use crate::storage::{Storage, WriteStorage}; +use crate::storage::git::Storage; +use crate::storage::{WriteRepository, WriteStorage}; pub static USER_IDS: Lazy<[UserId; 16]> = Lazy::new(|| { [ @@ -48,16 +49,23 @@ pub static PROJ_IDS: Lazy<[ProjId; 16]> = Lazy::new(|| { ] }); -pub fn storage(path: &Path) -> Storage { - let mut storage = Storage::open(path).unwrap(); +pub fn storage>(path: P) -> Storage { + let path = path.as_ref(); + let storage = Storage::new(path); for proj in PROJ_IDS.iter().take(3) { + log::debug!("creating {}...", proj); + let mut repo = storage.repository(proj).unwrap(); + for user in USER_IDS.iter().take(3) { - let repo = storage.namespace(proj, user).unwrap(); + let repo = repo.namespace(user).unwrap(); let head_oid = initial_commit(repo).unwrap(); let head = repo.find_commit(head_oid).unwrap(); - log::debug!("creating {}...", repo.namespace().unwrap()); + log::debug!("{}: creating {}...", proj, repo.namespace().unwrap()); + + repo.reference("refs/rad/root", head_oid, false, "test") + .unwrap(); // TODO: Different commits. repo.branch("master", &head, false).unwrap(); diff --git a/node/src/test/storage.rs b/node/src/test/storage.rs index 815c1455..ccb5b9bc 100644 --- a/node/src/test/storage.rs +++ b/node/src/test/storage.rs @@ -1,5 +1,7 @@ use crate::identity::ProjId; -use crate::storage::{Error, Inventory, ReadStorage, Remotes, Unverified, WriteStorage}; +use crate::storage::{ + Error, Inventory, ReadStorage, Remotes, Unverified, WriteRepository, WriteStorage, +}; #[derive(Clone, Debug)] pub struct MockStorage { @@ -38,13 +40,22 @@ impl ReadStorage for MockStorage { } impl WriteStorage for MockStorage { - fn repository(&mut self) -> &mut git2::Repository { + type Repository = MockRepository; + + fn repository(&self, _proj: &ProjId) -> Result { + todo!() + } +} + +pub struct MockRepository {} + +impl WriteRepository for MockRepository { + fn fetch(&mut self, _url: &str) -> Result<(), git2::Error> { todo!() } fn namespace( &mut self, - _proj: &ProjId, _user: &crate::identity::UserId, ) -> Result<&mut git2::Repository, git2::Error> { todo!()