mab122
/
radicle-heartwood-lfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

node: Change storage layout 

Projects now have their own repos.

Signed-off-by: Alexis Sellier <self@cloudhead.io>
This commit is contained in:
Alexis Sellier 2022-08-29 11:16:53 +02:00
parent 5c41af060f
commit f18afb793c
No known key found for this signature in database
8 changed files with 164 additions and 265 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

133
node/src/git.rs
View File

@ -1,85 +1,62 @@
use crate::identity::ProjId;
use crate::storage::{Error, WriteStorage};
use std::str::FromStr;
use crate::collections::HashMap;
use crate::identity::UserId;
use crate::storage::{Remote, Remotes, Unverified};
use git_ref_format as format;
/// Default port of the `git` transport protocol.
pub const PROTOCOL_PORT: u16 = 9418;
/// Fetch all remotes of a project from the given URL.
pub fn fetch<S: WriteStorage>(proj: &ProjId, url: &str, mut storage: S) -> Result<(), Error> {
// TODO: Use `Url` type?
// TODO: Have function to fetch specific remotes.
// TODO: Return meaningful info on success.
//
// Repository layout should look like this:
//
// /refs/namespaces/<project>
// /refs/namespaces/<remote>
// /heads
// /master
// /tags
// ...
//
let repo = storage.repository();
let refs: &[&str] = &[&format!(
"refs/namespaces/{}/refs/*:refs/namespaces/{}/refs/*",
proj, proj
)];
let mut remote = repo.remote_anonymous(url)?;
let mut opts = git2::FetchOptions::default();
remote.fetch(refs, Some(&mut opts), None)?;
Ok(())
#[derive(thiserror::Error, Debug)]
pub enum RefError {
#[error("invalid ref name '{0}'")]
InvalidName(format::RefString),
#[error("invalid ref format: {0}")]
Format(#[from] format::Error),
}
#[cfg(test)]
mod tests {
use super::*;
use crate::hash::Digest;
use crate::identity::ProjId;
use crate::storage::Storage;
/// Create an initial empty commit.
fn initial_commit(repo: &git2::Repository) -> Result<git2::Oid, Error> {
// First use the config to initialize a commit signature for the user.
let sig = git2::Signature::now("cloudhead", "cloudhead@radicle.xyz")?;
// Now let's create an empty tree for this commit.
let tree_id = repo.index()?.write_tree()?;
let tree = repo.find_tree(tree_id)?;
let oid = repo.commit(Some("HEAD"), &sig, &sig, "Initial commit", &tree, &[])?;
Ok(oid)
}
#[test]
fn test_fetch() {
let path = tempfile::tempdir().unwrap().into_path();
let alice = git2::Repository::init_bare(path.join("alice")).unwrap();
let bob = git2::Repository::init_bare(path.join("bob")).unwrap();
let mut bob_storage = Storage::from(bob);
let proj = ProjId::from(Digest::new(&[42]));
let master = format!("refs/namespaces/{}/refs/heads/master", proj);
let alice_oid = initial_commit(&alice).unwrap();
alice
.reference(&master, alice_oid, false, "Create master branch")
.unwrap();
// Have Bob fetch Alice's refs.
fetch(
&proj,
&format!("file://{}/alice", path.display()),
&mut bob_storage,
)
.unwrap();
let bob_oid = bob_storage
.repository()
.find_reference(&master)
.unwrap()
.target()
.unwrap();
assert_eq!(alice_oid, bob_oid);
}
#[derive(thiserror::Error, Debug)]
pub enum ListRefsError {
#[error("git error: {0}")]
Git(#[from] git2::Error),
#[error("invalid ref: {0}")]
InvalidRef(#[from] RefError),
}
/// List remote refs of a project, given the remote URL.
pub fn list_remotes(url: &str) -> Result<Remotes<Unverified>, ListRefsError> {
let mut remotes = HashMap::default();
let mut remote = git2::Remote::create_detached(url)?;
remote.connect(git2::Direction::Fetch)?;
let refs = remote.list()?;
for r in refs {
let (id, refname) = parse_ref::<UserId>(r.name())?;
let entry = remotes
.entry(id.clone())
.or_insert_with(|| Remote::new(id, HashMap::default()));
entry.refs.insert(refname.to_string(), r.oid().into());
}
Ok(Remotes::new(remotes))
}
/// Parse a ref string.
pub fn parse_ref<T: FromStr>(s: &str) -> Result<(T, format::RefString), RefError> {
let input = format::RefStr::try_from_str(s)?;
let suffix = input
.strip_prefix(format::refname!("refs/namespaces"))
.ok_or_else(|| RefError::InvalidName(input.to_owned()))?;
let mut components = suffix.components();
let id = components
.next()
.ok_or_else(|| RefError::InvalidName(input.to_owned()))?;
let id = T::from_str(&id.to_string()).map_err(|_| RefError::InvalidName(input.to_owned()))?;
let refstr = components.collect::<format::RefString>();
Ok((id, refstr))
}

25
node/src/identity.rs
View File

@ -1,4 +1,4 @@
use std::{fmt, io, ops::Deref, str::FromStr};
use std::{ffi::OsString, fmt, io, ops::Deref, str::FromStr};
use ed25519_consensus::{VerificationKey, VerificationKeyBytes};
use nonempty::NonEmpty;
@ -10,8 +10,6 @@ use crate::hash;
#[derive(Error, Debug)]
pub enum ProjIdError {
#[error("invalid ref '{0}'")]
InvalidRef(String),
#[error("invalid digest: {0}")]
InvalidDigest(#[from] hash::DecodeError),
}
@ -35,24 +33,25 @@ impl ProjId {
pub fn encode(&self) -> String {
multibase::encode(multibase::Base::Base58Btc, &self.0.as_ref())
}
pub(crate) fn from_ref(s: &str) -> Result<ProjId, ProjIdError> {
if let Some(s) = s.split('/').nth(2) {
let id = Self::from_str(s)?;
return Ok(id);
}
Err(ProjIdError::InvalidRef(s.to_owned()))
}
}
impl FromStr for ProjId {
type Err = hash::DecodeError;
type Err = ProjIdError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Ok(Self(hash::Digest::from_str(s)?))
}
}
impl TryFrom<OsString> for ProjId {
type Error = ProjIdError;
fn try_from(value: OsString) -> Result<Self, Self::Error> {
let string = value.to_string_lossy();
Self::from_str(&string)
}
}
impl From<hash::Digest> for ProjId {
fn from(digest: hash::Digest) -> Self {
Self(digest)
@ -134,8 +133,6 @@ pub enum UserIdError {
InvalidKey(#[from] ed25519_consensus::Error),
}
impl UserId {}
#[derive(Error, Debug)]
pub enum DocError {
#[error("toml: {0}")]

9
node/src/protocol.rs
View File

@ -16,9 +16,8 @@ use crate::address_manager::AddressManager;
use crate::clock::RefClock;
use crate::collections::{HashMap, HashSet};
use crate::decoder::Decoder;
use crate::git;
use crate::identity::{ProjId, UserId};
use crate::storage;
use crate::storage::{self, WriteRepository};
use crate::storage::{Inventory, ReadStorage, Remotes, Unverified, WriteStorage};
/// Network peer identifier.
@ -452,7 +451,11 @@ where
match cmd {
Command::Connect(addr) => self.context.connect(addr),
Command::Fetch(proj, remote) => {
git::fetch(&proj, &format!("git://{}", remote), &mut self.storage).unwrap();
self.storage
.repository(&proj)
.unwrap()
.fetch(&format!("git://{}", remote))
.unwrap();
}
}
}

168
node/src/storage.rs
View File

@ -1,22 +1,22 @@
pub mod git;
use std::collections::hash_map;
use std::io;
use std::marker::PhantomData;
use std::ops::{Deref, DerefMut};
use std::path::Path;
use std::{fmt, fs, io};
use git_ref_format::refspec;
use once_cell::sync::Lazy;
use radicle_git_ext as git_ext;
use serde::{Deserialize, Serialize};
use thiserror::Error;
pub use radicle_git_ext::Oid;
use crate::collections::HashMap;
use crate::git::RefError;
use crate::identity;
use crate::identity::{ProjId, ProjIdError, UserId};
pub static RAD_ID_GLOB: Lazy<refspec::PatternString> =
Lazy::new(|| refspec::pattern!("refs/namespaces/*/refs/rad/id"));
pub static IDENTITY_PATH: Lazy<&Path> = Lazy::new(|| Path::new(".rad/identity.toml"));
pub type BranchName = String;
@ -27,6 +27,8 @@ pub type Inventory = Vec<(ProjId, HashMap<String, Remote<Unverified>>)>;
pub enum Error {
#[error("invalid git reference")]
InvalidRef,
#[error("git reference error: {0}")]
Ref(#[from] RefError),
#[error("git: {0}")]
Git(#[from] git2::Error),
#[error("id: {0}")]
@ -51,7 +53,7 @@ pub struct Verified;
pub struct Unverified;
/// Project remotes. Tracks the git state of a project.
#[derive(Debug, Default, Clone, PartialEq, Eq)]
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Remotes<V>(HashMap<RemoteId, Remote<V>>);
impl Remotes<Unverified> {
@ -60,6 +62,21 @@ impl Remotes<Unverified> {
}
}
impl Default for Remotes<Unverified> {
fn default() -> Self {
Self(HashMap::default())
}
}
impl<V> IntoIterator for Remotes<V> {
type Item = (RemoteId, Remote<V>);
type IntoIter = hash_map::IntoIter<RemoteId, Remote<V>>;
fn into_iter(self) -> Self::IntoIter {
self.0.into_iter()
}
}
#[allow(clippy::from_over_into)]
impl Into<HashMap<String, Remote<Unverified>>> for Remotes<Unverified> {
fn into(self) -> HashMap<String, Remote<Unverified>> {
@ -73,20 +90,23 @@ impl Into<HashMap<String, Remote<Unverified>>> for Remotes<Unverified> {
}
/// A project remote.
#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, Eq)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub struct Remote<V> {
/// ID of remote.
pub id: UserId,
/// Git references published under this remote, and their hashes.
refs: HashMap<RefName, Oid>,
pub refs: HashMap<RefName, Oid>,
/// Whether this remote is of a project delegate.
delegate: bool,
pub delegate: bool,
/// Whether the remote is verified or not, ie. whether its signed refs were checked.
#[serde(skip)]
verified: PhantomData<V>,
}
impl Remote<Unverified> {
pub fn new(refs: HashMap<RefName, Oid>) -> Self {
pub fn new(id: UserId, refs: HashMap<RefName, Oid>) -> Self {
Self {
id,
refs,
delegate: false,
verified: PhantomData,
@ -100,12 +120,18 @@ pub trait ReadStorage {
}
pub trait WriteStorage {
fn repository(&mut self) -> &mut git2::Repository;
fn namespace(
&mut self,
proj: &ProjId,
user: &UserId,
) -> Result<&mut git2::Repository, git2::Error>;
type Repository: WriteRepository;
fn repository(&self, proj: &ProjId) -> Result<Self::Repository, Error>;
}
pub trait ReadRepository {
fn remotes(&self) -> Result<Remotes<Unverified>, Error>;
}
pub trait WriteRepository {
fn fetch(&mut self, url: &str) -> Result<(), git2::Error>;
fn namespace(&mut self, user: &UserId) -> Result<&mut git2::Repository, git2::Error>;
}
impl<T, S> ReadStorage for T
@ -127,114 +153,10 @@ where
T: DerefMut<Target = S>,
S: WriteStorage + 'static,
{
fn repository(&mut self) -> &mut git2::Repository {
self.deref_mut().repository()
}
type Repository = S::Repository;
fn namespace(
&mut self,
proj: &ProjId,
user: &UserId,
) -> Result<&mut git2::Repository, git2::Error> {
self.deref_mut().namespace(proj, user)
}
}
pub struct Storage {
backend: git2::Repository,
}
impl From<git2::Repository> for Storage {
fn from(backend: git2::Repository) -> Self {
Self { backend }
}
}
impl fmt::Debug for Storage {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "Storage(..)")
}
}
impl ReadStorage for Storage {
fn get(&self, _id: &ProjId) -> Result<Option<Remotes<Unverified>>, Error> {
todo!()
}
fn inventory(&self) -> Result<Inventory, Error> {
let glob: String = RAD_ID_GLOB.clone().into();
let refs = self.backend.references_glob(glob.as_str())?;
for r in refs {
let r = r?;
let name = r.name().ok_or(Error::InvalidRef)?;
let _id = ProjId::from_ref(name)?;
todo!();
}
Ok(vec![])
}
}
impl WriteStorage for Storage {
fn repository(&mut self) -> &mut git2::Repository {
&mut self.backend
}
fn namespace(
&mut self,
proj: &ProjId,
user: &UserId,
) -> Result<&mut git2::Repository, git2::Error> {
let path = self.backend.path();
self.backend = git2::Repository::open_bare(path)?;
self.backend.set_namespace(&format!("{}/{}", proj, user))?;
Ok(&mut self.backend)
}
}
impl Storage {
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self, git2::Error> {
let path = path.as_ref();
let backend = match git2::Repository::open_bare(path) {
Err(e) if git_ext::is_not_found_err(&e) => {
let backend = git2::Repository::init_opts(
path,
git2::RepositoryInitOptions::new()
.bare(true)
.no_reinit(true)
.external_template(false),
)?;
Ok(backend)
}
Ok(repo) => Ok(repo),
Err(e) => Err(e),
}?;
Ok(Self { backend })
}
pub fn create(
&self,
repo: &git2::Repository,
identity: impl Into<identity::Doc>,
) -> Result<(ProjId, git2::Reference), Error> {
let doc = identity.into();
let file = fs::OpenOptions::new()
.create_new(true)
.write(true)
.open(*IDENTITY_PATH)?;
let id = doc.write(file)?;
let ref_name = RAD_ID_GLOB.replace('*', &id.encode());
let oid = repo.head()?.target().ok_or(Error::InvalidHead)?;
let reference = self.backend.reference(&ref_name, oid, false, "")?;
// TODO: Push project to monorepo.
Ok((id, reference))
fn repository(&self, proj: &ProjId) -> Result<Self::Repository, Error> {
self.deref().repository(proj)
}
}

56
node/src/storage/git.rs
View File

@ -1,29 +1,26 @@
use std::marker::PhantomData;
use std::ops::{Deref, DerefMut};
use std::path::{Path, PathBuf};
use std::str::FromStr;
use std::{fmt, fs, io};
use std::{fmt, fs};
use git_ref_format::refspec;
use git_url::Url;
use once_cell::sync::Lazy;
use radicle_git_ext as git_ext;
use serde::{Deserialize, Serialize};
pub use radicle_git_ext::Oid;
use crate::collections::HashMap;
use crate::git;
use crate::identity;
use crate::identity::{ProjId, ProjIdError, UserId};
use crate::identity::{ProjId, UserId};
use super::{
Error, Inventory, ReadRepository, ReadStorage, Remote, Remotes, Unverified, Verified,
WriteRepository, WriteStorage,
Error, Inventory, ReadRepository, ReadStorage, Remote, Remotes, Unverified, WriteRepository,
WriteStorage,
};
pub static RAD_ROOT_GLOB: Lazy<refspec::PatternString> =
Lazy::new(|| refspec::pattern!("refs/namespaces/*/refs/rad/root"));
pub static NAMESPACES_GLOB: Lazy<refspec::PatternString> =
Lazy::new(|| refspec::pattern!("refs/namespaces/*"));
pub static IDENTITY_PATH: Lazy<&Path> = Lazy::new(|| Path::new(".rad/identity.toml"));
pub struct Storage {
@ -48,7 +45,6 @@ impl ReadStorage for Storage {
}
fn inventory(&self) -> Result<Inventory, Error> {
let glob: String = RAD_ROOT_GLOB.clone().into();
let projs = self.projects()?;
let mut inv = Vec::new();
@ -57,7 +53,7 @@ impl ReadStorage for Storage {
let remotes = repo
.remotes()?
.into_iter()
.map(|r| (r.id.to_string(), r))
.map(|(id, r)| (id.to_string(), r))
.collect();
inv.push((proj, remotes));
@ -157,8 +153,8 @@ impl Repository {
}
impl ReadRepository for Repository {
fn remotes(&self) -> Result<Vec<Remote<Unverified>>, Error> {
let refs = self.backend.references_glob(RAD_ROOT_GLOB.as_str())?;
fn remotes(&self) -> Result<Remotes<Unverified>, Error> {
let refs = self.backend.references_glob(NAMESPACES_GLOB.as_str())?;
let mut remotes = HashMap::default();
for r in refs {
@ -172,7 +168,7 @@ impl ReadRepository for Repository {
entry.refs.insert(refname.to_string(), oid.into());
}
Ok(remotes.into_values().collect())
Ok(Remotes::new(remotes))
}
}
@ -192,7 +188,7 @@ impl WriteRepository for Repository {
// /tags
// ...
//
let refs: &[&str] = &[&format!("refs/namespaces/*:refs/namespaces/*")];
let refs: &[&str] = &["refs/namespaces/*:refs/namespaces/*"];
let mut remote = self.backend.remote_anonymous(url)?;
let mut opts = git2::FetchOptions::default();
@ -221,38 +217,22 @@ impl From<git2::Repository> for Repository {
mod tests {
use super::*;
use crate::git;
use crate::hash::Digest;
use crate::identity::ProjId;
use crate::storage::{ReadStorage, WriteRepository};
use crate::test::fixtures;
/// Create an initial empty commit.
fn initial_commit(repo: &git2::Repository) -> Result<git2::Oid, Error> {
// First use the config to initialize a commit signature for the user.
let sig = git2::Signature::now("cloudhead", "cloudhead@radicle.xyz")?;
// Now let's create an empty tree for this commit.
let tree_id = repo.index()?.write_tree()?;
let tree = repo.find_tree(tree_id)?;
let oid = repo.commit(Some("HEAD"), &sig, &sig, "Initial commit", &tree, &[])?;
Ok(oid)
}
#[test]
fn test_ls_remote() {
crate::test::logger::init(log::Level::Debug);
fn test_list_remotes() {
let dir = tempfile::tempdir().unwrap();
let storage = fixtures::storage(dir.path());
let inv = storage.inventory().unwrap();
let (proj, _) = inv.first().unwrap();
let refs = git::list_refs(&format!(
let refs = git::list_remotes(&format!(
"file://{}",
dir.path().join(&proj.to_string()).display(),
))
.unwrap();
let remotes = storage.repository(&proj).unwrap().remotes().unwrap();
let remotes = storage.repository(proj).unwrap().remotes().unwrap();
assert_eq!(refs, remotes);
}
@ -267,7 +247,7 @@ mod tests {
let refname = "refs/heads/master";
// Have Bob fetch Alice's refs.
bob.repository(&proj)
bob.repository(proj)
.unwrap()
.fetch(&format!(
"file://{}",
@ -275,14 +255,14 @@ mod tests {
))
.unwrap();
for (_, remote) in remotes {
for remote in remotes.values() {
let alice_oid = alice
.repository(&proj)
.repository(proj)
.unwrap()
.find_reference(&remote.id, refname)
.unwrap();
let bob_oid = bob
.repository(&proj)
.repository(proj)
.unwrap()
.find_reference(&remote.id, refname)
.unwrap();

3
node/src/test/arbitrary.rs
View File

@ -26,6 +26,7 @@ impl quickcheck::Arbitrary for storage::Remote<storage::Unverified> {
let mut refs: HashMap<storage::BranchName, storage::Oid> = HashMap::with_hasher(rng.into());
let mut bytes: [u8; 20] = [0; 20];
let names = &["master", "dev", "feature/1", "feature/2", "feature/3"];
let id = UserId::arbitrary(g);
for _ in 0..g.size().min(2) {
if let Some(name) = g.choose(names) {
@ -36,7 +37,7 @@ impl quickcheck::Arbitrary for storage::Remote<storage::Unverified> {
refs.insert(name.to_string(), oid);
}
}
storage::Remote::new(refs)
storage::Remote::new(id, refs)
}
}

18
node/src/test/fixtures.rs
View File

@ -4,7 +4,8 @@ use std::str::FromStr;
use once_cell::sync::Lazy;
use crate::identity::{ProjId, UserId};
use crate::storage::{Storage, WriteStorage};
use crate::storage::git::Storage;
use crate::storage::{WriteRepository, WriteStorage};
pub static USER_IDS: Lazy<[UserId; 16]> = Lazy::new(|| {
[
@ -48,16 +49,23 @@ pub static PROJ_IDS: Lazy<[ProjId; 16]> = Lazy::new(|| {
]
});
pub fn storage(path: &Path) -> Storage {
let mut storage = Storage::open(path).unwrap();
pub fn storage<P: AsRef<Path>>(path: P) -> Storage {
let path = path.as_ref();
let storage = Storage::new(path);
for proj in PROJ_IDS.iter().take(3) {
log::debug!("creating {}...", proj);
let mut repo = storage.repository(proj).unwrap();
for user in USER_IDS.iter().take(3) {
let repo = storage.namespace(proj, user).unwrap();
let repo = repo.namespace(user).unwrap();
let head_oid = initial_commit(repo).unwrap();
let head = repo.find_commit(head_oid).unwrap();
log::debug!("creating {}...", repo.namespace().unwrap());
log::debug!("{}: creating {}...", proj, repo.namespace().unwrap());
repo.reference("refs/rad/root", head_oid, false, "test")
.unwrap();
// TODO: Different commits.
repo.branch("master", &head, false).unwrap();

17
node/src/test/storage.rs
View File

@ -1,5 +1,7 @@
use crate::identity::ProjId;
use crate::storage::{Error, Inventory, ReadStorage, Remotes, Unverified, WriteStorage};
use crate::storage::{
Error, Inventory, ReadStorage, Remotes, Unverified, WriteRepository, WriteStorage,
};
#[derive(Clone, Debug)]
pub struct MockStorage {
@ -38,13 +40,22 @@ impl ReadStorage for MockStorage {
}
impl WriteStorage for MockStorage {
fn repository(&mut self) -> &mut git2::Repository {
type Repository = MockRepository;
fn repository(&self, _proj: &ProjId) -> Result<Self::Repository, Error> {
todo!()
}
}
pub struct MockRepository {}
impl WriteRepository for MockRepository {
fn fetch(&mut self, _url: &str) -> Result<(), git2::Error> {
todo!()
}
fn namespace(
&mut self,
_proj: &ProjId,
_user: &crate::identity::UserId,
) -> Result<&mut git2::Repository, git2::Error> {
todo!()