mab122
/
radicle-heartwood-lfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

node: Implement private repos 

Private repos are implemented by extending the identity document with a
`visibility` attribute, that can either be `"public"` (default) or
`"private"`.

In case of `private` visibility, only the delegates are allowed to view
the repo, as well as any DIDs added to the allow list.

To implement repo visibility, we simply block fetches from and
announcements to peers for whom the repo should remain invisible.

Private repos are also not announced in the `inventory` message, since
the full list of peers that *may* have the repo is retrievable from the
repo identity. This could cause errors if eg. a peer who is allowed to
view the repo doesn't actually have it. However this is an ok trade-off
for now to keep the complexity low. For repos to truly be private, it's
important that the RIDs don't leak either.

Finally, we modify `radicle-httpd` for now to only list public repos.
Eventually, we would want to change this depending on whether an allowed
peer is authenticated with the service or not.

---

It's also worth mentioning why this approach was taken, vs. end-to-end
encryption. The reasons are as follows:

1. Nodes that do not have access to a private repo will generally not want to
   replicate encrypted data that they cannot examine or use.
2. The chosen solution is trivial, while encrypting git objects isn't.
3. Performance of the chosen solution is much better, there is no
   overhead.
4. Privacy of the chosen solution is better: RIDs are never leaked, and
   neither is the existence of a private repo, nor who has access to it.

There is one downside: Paying for storage of private repos is no better
in terms of privacy than what GitHub offers. Hosting providers will have
access to your private repos, if this solution is used.
This commit is contained in:
cloudhead 2023-08-28 13:04:11 +02:00
parent 47f09e6ff4
commit 27f39514d4
No known key found for this signature in database
20 changed files with 215 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
radicle-cli/src/commands/init.rs
View File

@ -9,6 +9,7 @@ use serde_json as json;
use radicle::crypto::ssh;
use radicle::git::RefString;
use radicle::identity::Visibility;
use radicle::node::tracking::Scope;
use radicle::node::{Handle, NodeId};
use radicle::profile;
@ -172,6 +173,7 @@ pub fn init(options: Options, profile: &profile::Profile) -> anyhow::Result<()>
let path = options.path.unwrap_or_else(|| cwd.clone());
let path = path.as_path().canonicalize()?;
let interactive = options.interactive;
let visibility = Visibility::default();
term::headline(format!(
"Initializing radicle 👾 project in {}",
@ -222,6 +224,7 @@ pub fn init(options: Options, profile: &profile::Profile) -> anyhow::Result<()>
&name,
&description,
branch,
visibility,
&signer,
&profile.storage,
) {

24
radicle-cli/src/commands/ls.rs
View File

@ -1,7 +1,5 @@
use std::ffi::OsString;
use radicle::storage::{ReadRepository, ReadStorage};
use crate::terminal as term;
use crate::terminal::args::{Args, Error, Help};
@ -53,26 +51,8 @@ pub fn run(options: Options, ctx: impl term::Context) -> anyhow::Result<()> {
let storage = &profile.storage;
let mut table = term::Table::default();
for id in storage.repositories()? {
let repo = match storage.repository(id) {
Ok(repo) => repo,
Err(err) => {
if options.verbose {
term::warning(&format!("failed to load project '{id}': {err}"));
}
continue;
}
};
let head = match repo.head() {
Ok((_, head)) => head,
Err(err) => {
if options.verbose {
term::warning(&format!("failed to get head of project '{id}': {err}"));
}
continue;
}
};
let proj = match repo.project() {
for (id, head, doc) in storage.repositories()? {
let proj = match doc.verified()?.project() {
Ok(proj) => proj,
Err(err) => {
if options.verbose {

2
radicle-httpd/src/api/v1/delegates.rs
View File

@ -37,7 +37,7 @@ async fn delegates_projects_handler(
let storage = &ctx.profile.storage;
let routing = &ctx.profile.routing()?;
let projects = storage
.repositories()?
.inventory()?
.into_iter()
.filter_map(|id| {
let Ok(repo) = storage.repository(id) else { return None };

2
radicle-httpd/src/api/v1/projects.rs
View File

@ -81,7 +81,7 @@ async fn project_root_handler(
let storage = &ctx.profile.storage;
let routing = &ctx.profile.routing()?;
let projects = storage
.repositories()?
.inventory()?
.into_iter()
.filter_map(|id| {
let Ok(repo) = storage.repository(id) else { return None };

3
radicle-httpd/src/api/v1/stats.rs
View File

@ -2,6 +2,7 @@ use axum::extract::State;
use axum::response::IntoResponse;
use axum::routing::get;
use axum::{Json, Router};
use radicle::storage::ReadStorage as _;
use serde_json::json;
use crate::api::error::Error;
@ -17,7 +18,7 @@ pub fn router(ctx: Context) -> Router {
/// `GET /stats`
async fn stats_handler(State(ctx): State<Context>) -> impl IntoResponse {
let storage = &ctx.profile.storage;
let projects = storage.repositories()?.len();
let projects = storage.inventory()?.len();
Ok::<_, Error>(Json(
json!({ "projects": { "count": projects }, "users": { "count": 0 } }),

14
radicle-httpd/src/test.rs
View File

@ -16,6 +16,7 @@ use radicle::crypto::ssh::keystore::MemorySigner;
use radicle::crypto::ssh::Keystore;
use radicle::crypto::{KeyPair, Seed, Signer};
use radicle::git::{raw as git2, RefString};
use radicle::identity::Visibility;
use radicle::node;
use radicle::node::address as AddressStore;
use radicle::node::routing as RoutingStore;
@ -175,8 +176,17 @@ fn seed_with_signer<G: Signer>(dir: &Path, profile: radicle::Profile, signer: &G
let name = "hello-world".to_string();
let description = "Rad repository for tests".to_string();
let branch = RefString::try_from("master").unwrap();
let (id, _, _) =
radicle::rad::init(&repo, &name, &description, branch, signer, &profile.storage).unwrap();
let visibility = Visibility::default();
let (id, _, _) = radicle::rad::init(
&repo,
&name,
&description,
branch,
visibility,
signer,
&profile.storage,
)
.unwrap();
let storage = &profile.storage;
let repo = storage.repository(id).unwrap();

34
radicle-node/src/service.rs
View File

@ -20,6 +20,7 @@ use localtime::{LocalDuration, LocalTime};
use log::*;
use nonempty::NonEmpty;
use radicle::identity;
use radicle::node::address;
use radicle::node::address::{AddressBook, KnownAddress};
use radicle::node::config::PeerConfig;
@ -109,9 +110,13 @@ pub enum Error {
#[error(transparent)]
Storage(#[from] storage::Error),
#[error(transparent)]
Refs(#[from] storage::refs::Error),
#[error(transparent)]
Routing(#[from] routing::Error),
#[error(transparent)]
Tracking(#[from] tracking::Error),
#[error(transparent)]
Identity(#[from] identity::IdentityError),
#[error("namespaces error: {0}")]
Namespaces(#[from] NamespacesError),
}
@ -590,7 +595,8 @@ where
}
}
pub fn fetch(&mut self, rid: Id, from: &NodeId) {
/// Initiate an outgoing fetch for some repository.
fn fetch(&mut self, rid: Id, from: &NodeId) {
let Some(session) = self.sessions.get_mut(from) else {
error!(target: "service", "Session {from} does not exist; cannot initiate fetch");
return;
@ -716,6 +722,21 @@ where
}
}
/// Called when a remote requests a repository be uploaded to it.
/// The upload is authorized if this function returns `true`.
pub fn upload(&mut self, rid: Id, remote: &NodeId) -> bool {
let Ok(repo) = self.storage.repository(rid) else {
return false;
};
let Ok((_, doc)) = repo.identity_doc() else {
return false;
};
if !doc.is_visible_to(remote) {
return false;
}
true
}
/// Inbound connection attempt.
pub fn accepted(&mut self, addr: Address) -> bool {
// Always accept trusted connections.
@ -1266,8 +1287,9 @@ where
&mut self,
rid: Id,
remotes: impl IntoIterator<Item = NodeId>,
) -> Result<(), storage::Error> {
) -> Result<(), Error> {
let repo = self.storage.repository(rid)?;
let (_, doc) = repo.identity_doc()?;
let peers = self.sessions.connected().map(|(_, p)| p);
let timestamp = self.time();
let mut refs = BoundedVec::<_, REF_REMOTE_LIMIT>::new();
@ -1293,7 +1315,13 @@ where
});
let ann = msg.signed(&self.signer);
self.outbox.broadcast(ann, peers);
self.outbox.broadcast(
ann,
peers.filter(|p| {
// Only announce to peers who are allowed to view this repo.
doc.is_visible_to(&p.id)
}),
);
Ok(())
}

3
radicle-node/src/test/environment.rs
View File

@ -17,7 +17,7 @@ use radicle::crypto::test::signer::MockSigner;
use radicle::crypto::{KeyPair, Seed, Signer};
use radicle::git;
use radicle::git::refname;
use radicle::identity::Id;
use radicle::identity::{Id, Visibility};
use radicle::node::address::Book;
use radicle::node::routing;
use radicle::node::routing::Store;
@ -402,6 +402,7 @@ impl<G: cyphernet::Ecdh<Pk = NodeId> + Signer + Clone> Node<G> {
name,
description,
refname!("master"),
Visibility::default(),
&self.signer,
&self.storage,
)

2
radicle-node/src/test/peer.rs
View File

@ -6,6 +6,7 @@ use std::str::FromStr;
use log::*;
use radicle::identity::Visibility;
use radicle::node::address::Store;
use radicle::node::{address, Alias, ConnectOptions};
use radicle::rad;
@ -132,6 +133,7 @@ impl<G: Signer> Peer<Storage, G> {
name,
description,
radicle::git::refname!("master"),
Visibility::default(),
self.signer(),
self.storage(),
)

2
radicle-node/src/tests.rs
View File

@ -8,6 +8,7 @@ use std::time;
use crossbeam_channel as chan;
use netservices::Direction as Link;
use radicle::identity::Visibility;
use radicle::node::routing::Store as _;
use radicle::node::ConnectOptions;
use radicle::storage::ReadRepository;
@ -1282,6 +1283,7 @@ fn test_push_and_pull() {
"alice",
"alice's repo",
git::refname!("master"),
Visibility::default(),
alice.signer(),
alice.storage(),
)

15
radicle-node/src/worker.rs
View File

@ -11,7 +11,7 @@ use crossbeam_channel as chan;
use radicle::identity::Id;
use radicle::prelude::NodeId;
use radicle::storage::{Namespaces, ReadRepository, RefUpdate};
use radicle::storage::{Namespaces, ReadRepository, ReadStorage, RefUpdate};
use radicle::{git, storage, Storage};
use crate::runtime::{thread, Handle};
@ -66,6 +66,12 @@ pub enum UploadError {
PacketLine(io::Error),
#[error(transparent)]
Io(#[from] io::Error),
#[error("{0} is not authorized to fetch {1}")]
Unauthorized(NodeId, Id),
#[error(transparent)]
Storage(#[from] radicle::storage::Error),
#[error(transparent)]
Identity(#[from] radicle::identity::IdentityError),
}
impl UploadError {
@ -309,6 +315,13 @@ impl Worker {
};
log::debug!(target: "worker", "Received Git request pktline for {rid}..");
let repo = self.storage.repository(rid)?;
let (_, doc) = repo.identity_doc()?;
if !doc.is_visible_to(&remote) {
return Err(UploadError::Unauthorized(remote, rid));
}
match self._upload_pack(rid, remote, request, stream, stream_r, stream_w) {
Ok(()) => {
log::debug!(target: "worker", "Upload of {rid} to {remote} on stream {stream} exited successfully");

3
radicle-tools/src/rad-init.rs
View File

@ -1,6 +1,6 @@
use std::path::Path;
use radicle::{git, Profile};
use radicle::{git, identity::Visibility, Profile};
fn main() -> anyhow::Result<()> {
let cwd = Path::new(".").canonicalize()?;
@ -13,6 +13,7 @@ fn main() -> anyhow::Result<()> {
&name,
"",
git::refname!("master"),
Visibility::default(),
&signer,
&profile.storage,
)?;

3
radicle/src/identity.rs
View File

@ -15,7 +15,7 @@ use crate::storage::{refs, ReadRepository, RemoteId};
pub use crypto::PublicKey;
pub use did::Did;
pub use doc::{Doc, Id, IdError, PayloadError};
pub use doc::{Doc, Id, IdError, PayloadError, Visibility};
pub use project::Project;
/// Untrusted, well-formed input.
@ -164,6 +164,7 @@ impl Identity<Untrusted> {
})
}
}
#[cfg(test)]
mod test {
use qcheck_macros::quickcheck;

73
radicle/src/identity/doc.rs
View File

@ -135,6 +135,26 @@ impl Deref for DocAt {
}
}
/// Repository visibility.
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", tag = "type")]
pub enum Visibility {
/// Anyone and everyone.
#[default]
Public,
/// Delegates plus the allowed DIDs.
Private {
#[serde(default, skip_serializing_if = "Vec::is_empty")]
allow: Vec<Did>,
},
}
impl Visibility {
pub fn is_public(&self) -> bool {
matches!(self, Self::Public)
}
}
/// An identity document.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
@ -145,12 +165,25 @@ pub struct Doc<V> {
pub delegates: NonEmpty<Did>,
/// The signature threshold.
pub threshold: usize,
/// Repository visibility.
#[serde(default, skip_serializing_if = "Visibility::is_public")]
pub visibility: Visibility,
#[serde(skip)]
verified: PhantomData<V>,
}
impl<V> Doc<V> {
/// Check whether this document and the associated repository is visible to the given peer.
pub fn is_visible_to(&self, peer: &PublicKey) -> bool {
match &self.visibility {
Visibility::Public => true,
Visibility::Private { allow } => {
allow.contains(&Did::from(*peer)) || self.is_delegate(peer)
}
}
}
pub fn canonical_head(repo: &storage::git::Repository) -> Result<Oid, DocError> {
repo.backend
.refname_to_id(storage::git::CANONICAL_IDENTITY.as_str())
@ -327,17 +360,23 @@ impl Doc<Verified> {
payload: self.payload,
delegates: self.delegates,
threshold: self.threshold,
visibility: self.visibility,
verified: PhantomData,
}
}
}
impl Doc<Unverified> {
pub fn initial(project: Project, delegate: Did) -> Self {
Self::new(project, NonEmpty::new(delegate), 1)
pub fn initial(project: Project, delegate: Did, visibility: Visibility) -> Self {
Self::new(project, NonEmpty::new(delegate), 1, visibility)
}
pub fn new(project: Project, delegates: NonEmpty<Did>, threshold: usize) -> Self {
pub fn new(
project: Project,
delegates: NonEmpty<Did>,
threshold: usize,
visibility: Visibility,
) -> Self {
let project =
serde_json::to_value(project).expect("Doc::initial: payload must be serializable");
@ -345,6 +384,7 @@ impl Doc<Unverified> {
payload: BTreeMap::from_iter([(PayloadId::project(), Payload::from(project))]),
delegates,
threshold,
visibility,
verified: PhantomData,
}
}
@ -377,6 +417,7 @@ impl Doc<Unverified> {
payload: self.payload,
delegates: self.delegates,
threshold: self.threshold,
visibility: self.visibility,
verified: PhantomData,
})
}
@ -424,6 +465,7 @@ mod test {
"heartwood",
"Radicle Heartwood Protocol & Stack",
git::refname!("master"),
Visibility::default(),
&delegate,
&storage,
)
@ -470,6 +512,7 @@ mod test {
"heartwood",
"Radicle Heartwood Protocol & Stack",
git::refname!("master"),
Visibility::default(),
&delegate,
&storage,
)
@ -484,4 +527,28 @@ mod test {
let (_, bytes) = doc.encode().unwrap();
assert_eq!(Doc::from_json(&bytes).unwrap().verified().unwrap(), doc);
}
#[test]
fn test_visibility_json() {
use std::str::FromStr;
assert_eq!(
serde_json::to_value(Visibility::Public).unwrap(),
serde_json::json!({ "type": "public" })
);
assert_eq!(
serde_json::to_value(Visibility::Private { allow: vec![] }).unwrap(),
serde_json::json!({ "type": "private" })
);
assert_eq!(
serde_json::to_value(Visibility::Private {
allow: vec![Did::from_str(
"did:key:z6MksFqXN3Yhqk8pTJdUGLwATkRfQvwZXPqR2qMEhbS9wzpT"
)
.unwrap()]
})
.unwrap(),
serde_json::json!({ "type": "private", "allow": ["did:key:z6MksFqXN3Yhqk8pTJdUGLwATkRfQvwZXPqR2qMEhbS9wzpT"] })
);
}
}

1
radicle/src/node/tracking/schema.sql
View File

@ -22,7 +22,6 @@ create table if not exists "repo-policies" (
-- Valid values are:
--
-- "trusted" track repository delegates and remotes in the `node-policies` table.
-- "delegates-only" only track repository delegates.
-- "all" track all remotes.
--
"scope" text default 'trusted',

8
radicle/src/rad.rs
View File

@ -9,7 +9,7 @@ use thiserror::Error;
use crate::cob::ObjectId;
use crate::crypto::{Signer, Verified};
use crate::git;
use crate::identity::doc::{DocError, Id};
use crate::identity::doc::{DocError, Id, Visibility};
use crate::identity::project::Project;
use crate::identity::{doc, IdentityError};
use crate::storage::git::transport;
@ -54,6 +54,7 @@ pub fn init<G: Signer, S: WriteStorage>(
name: &str,
description: &str,
default_branch: BranchName,
visibility: Visibility,
signer: &G,
storage: S,
) -> Result<(Id, identity::Doc<Verified>, SignedRefs<Verified>), InitError> {
@ -73,7 +74,7 @@ pub fn init<G: Signer, S: WriteStorage>(
.join(", "),
)
})?;
let doc = identity::Doc::initial(proj, delegate).verified()?;
let doc = identity::Doc::initial(proj, delegate, visibility).verified()?;
let (project, _) = Repository::init(&doc, pk, storage, signer)?;
let url = git::Url::from(project.id);
@ -388,6 +389,7 @@ mod tests {
"acme",
"Acme's repo",
git::refname!("master"),
Visibility::default(),
&signer,
&storage,
)
@ -442,6 +444,7 @@ mod tests {
"acme",
"Acme's repo",
git::refname!("master"),
Visibility::default(),
&alice,
&storage,
)
@ -477,6 +480,7 @@ mod tests {
"acme",
"Acme's repo",
git::refname!("master"),
Visibility::default(),
&signer,
&storage,
)

1
radicle/src/storage.rs
View File

@ -303,6 +303,7 @@ pub trait ReadStorage {
/// Check whether storage contains a repository.
fn contains(&self, rid: &Id) -> Result<bool, IdentityError>;
/// Get the inventory of repositories hosted under this storage.
/// This function should typically only return public repositories.
fn inventory(&self) -> Result<Inventory, Error>;
/// Open or create a read-only repository.
fn repository(&self, rid: Id) -> Result<Self::Repository, Error>;

45
radicle/src/storage/git.rs
View File

@ -105,7 +105,13 @@ impl ReadStorage for Storage {
}
fn inventory(&self) -> Result<Inventory, Error> {
self.repositories()
let repos = self.repositories()?;
Ok(repos
.into_iter()
.filter(|(_, _, doc)| doc.visibility.is_public())
.map(|(rid, _, _)| rid)
.collect())
}
fn repository(&self, rid: Id) -> Result<Self::Repository, Error> {
@ -143,7 +149,7 @@ impl Storage {
self.path.as_path()
}
pub fn repositories(&self) -> Result<Vec<Id>, Error> {
pub fn repositories(&self) -> Result<Vec<(Id, Oid, Doc<Unverified>)>, Error> {
let mut repos = Vec::new();
for result in fs::read_dir(&self.path)? {
@ -159,28 +165,45 @@ impl Storage {
}
let rid =
Id::try_from(path.file_name()).map_err(|_| Error::InvalidId(path.file_name()))?;
let repo = self.repository(rid)?;
let repo = match self.repository(rid) {
Ok(repo) => repo,
Err(e) => {
log::warn!(target: "storage", "Repository {rid} is invalid: {e}");
continue;
}
};
let doc = match repo.identity_doc() {
Ok((_, doc)) => doc,
Err(e) => {
log::warn!(target: "storage", "Repository {rid} is invalid: looking up doc: {e}");
continue;
}
};
// For performance reasons, we don't do a full repository check here.
if let Err(e) = repo.head() {
log::warn!(target: "storage", "Repository {rid} is invalid: looking up head: {e}");
continue;
}
repos.push(rid);
let head = match repo.head() {
Ok((_, head)) => head,
Err(e) => {
log::warn!(target: "storage", "Repository {rid} is invalid: looking up head: {e}");
continue;
}
};
repos.push((rid, head, doc));
}
Ok(repos)
}
pub fn inspect(&self) -> Result<(), Error> {
for proj in self.repositories()? {
let repo = self.repository(proj)?;
for (rid, _, _) in self.repositories()? {
let repo = self.repository(rid)?;
for r in repo.raw().references()? {
let r = r?;
let name = r.name().ok_or(Error::InvalidRef)?;
let oid = r.target().ok_or(Error::InvalidRef)?;
println!("{} {oid} {name}", proj.urn());
println!("{} {oid} {name}", rid.urn());
}
}
Ok(())

19
radicle/src/test/arbitrary.rs
View File

@ -10,6 +10,7 @@ use nonempty::NonEmpty;
use qcheck::Arbitrary;
use crate::collections::RandomMap;
use crate::identity::doc::Visibility;
use crate::identity::{
doc::{Doc, Id},
project::Project,
@ -115,12 +116,25 @@ impl Arbitrary for Project {
}
}
impl Arbitrary for Visibility {
fn arbitrary(g: &mut qcheck::Gen) -> Self {
if bool::arbitrary(g) {
Visibility::Public
} else {
Visibility::Private {
allow: Vec::arbitrary(g),
}
}
}
}
impl Arbitrary for Doc<Unverified> {
fn arbitrary(g: &mut qcheck::Gen) -> Self {
let proj = Project::arbitrary(g);
let delegate = Did::arbitrary(g);
let visibility = Visibility::arbitrary(g);
Self::initial(proj, delegate)
Self::initial(proj, delegate, visibility)
}
}
@ -134,7 +148,8 @@ impl Arbitrary for Doc<Verified> {
.try_into()
.unwrap();
let threshold = delegates.len() / 2 + 1;
let doc: Doc<Unverified> = Doc::new(project, delegates, threshold);
let visibility = Visibility::arbitrary(g);
let doc: Doc<Unverified> = Doc::new(project, delegates, threshold, visibility);
doc.verified().unwrap()
}

12
radicle/src/test/fixtures.rs
View File

@ -2,6 +2,7 @@ use std::path::Path;
use crate::crypto::{Signer, Verified};
use crate::git;
use crate::identity::doc::Visibility;
use crate::identity::Id;
use crate::rad;
use crate::storage::git::transport;
@ -25,7 +26,15 @@ pub fn storage<P: AsRef<Path>, G: Signer>(path: P, signer: &G) -> Result<Storage
("rx", "A pixel editor"),
] {
let (repo, _) = repository(path.join("workdir").join(name));
rad::init(&repo, name, desc, git::refname!("master"), signer, &storage)?;
rad::init(
&repo,
name,
desc,
git::refname!("master"),
Visibility::default(),
signer,
&storage,
)?;
}
Ok(storage)
@ -45,6 +54,7 @@ pub fn project<P: AsRef<Path>, G: Signer>(
"acme",
"Acme's repository",
git::refname!("master"),
Visibility::default(),
signer,
storage,
)?;