From 6be77ca9c3f830aa28e03b532edd27590bf17c69 Mon Sep 17 00:00:00 2001 From: cloudhead Date: Fri, 5 Apr 2024 21:35:08 +0200 Subject: [PATCH] radicle: Implement `remote_refs_at` Implement this function on `RemoteRepository`, and move `repositories` to the trait level. This function will be used to quickly get the `RefsAt` of all repository remotes. --- radicle-cli/src/commands/ls.rs | 2 +- radicle-fetch/src/state.rs | 4 + radicle-httpd/src/api/v1/stats.rs | 2 + radicle/src/node/refs/store.rs | 87 ++++++++++++++++- radicle/src/storage.rs | 25 ++++- radicle/src/storage/git.rs | 149 +++++++++++++++--------------- radicle/src/storage/git/cob.rs | 4 + radicle/src/storage/refs.rs | 4 + radicle/src/test/storage.rs | 28 +++++- 9 files changed, 221 insertions(+), 84 deletions(-) diff --git a/radicle-cli/src/commands/ls.rs b/radicle-cli/src/commands/ls.rs index 01672a49..2161183a 100644 --- a/radicle-cli/src/commands/ls.rs +++ b/radicle-cli/src/commands/ls.rs @@ -1,6 +1,6 @@ use std::ffi::OsString; -use radicle::storage::git::RepositoryInfo; +use radicle::storage::{ReadStorage, RepositoryInfo}; use crate::terminal as term; use crate::terminal::args::{Args, Error, Help}; diff --git a/radicle-fetch/src/state.rs b/radicle-fetch/src/state.rs index 0799d50e..5c944ecd 100644 --- a/radicle-fetch/src/state.rs +++ b/radicle-fetch/src/state.rs @@ -670,6 +670,10 @@ impl<'a, S> RemoteRepository for Cached<'a, S> { .map(|id| self.remote(id).map(|remote| (*id, remote))) .collect::>() } + + fn remote_refs_at(&self) -> Result, storage::refs::Error> { + self.handle.repo.remote_refs_at() + } } impl<'a, S> ValidateRepository for Cached<'a, S> { diff --git a/radicle-httpd/src/api/v1/stats.rs b/radicle-httpd/src/api/v1/stats.rs index c8f4792b..056f5e2d 100644 --- a/radicle-httpd/src/api/v1/stats.rs +++ b/radicle-httpd/src/api/v1/stats.rs @@ -4,6 +4,8 @@ use axum::routing::get; use axum::{Json, Router}; use serde_json::json; +use radicle::storage::ReadStorage; + use crate::api::error::Error; use crate::api::Context; diff --git a/radicle/src/node/refs/store.rs b/radicle/src/node/refs/store.rs index 3516fcf1..fde561c6 100644 --- a/radicle/src/node/refs/store.rs +++ b/radicle/src/node/refs/store.rs @@ -10,6 +10,8 @@ use crate::git::{Oid, Qualified}; use crate::node::Database; use crate::node::NodeId; use crate::prelude::RepoId; +use crate::storage; +use crate::storage::{ReadRepository, ReadStorage, RemoteRepository, RepositoryError}; #[derive(Error, Debug)] pub enum Error { @@ -19,12 +21,25 @@ pub enum Error { /// Timestamp error. #[error("invalid timestamp: {0}")] Timestamp(#[from] TryFromIntError), + /// Repository error. + #[error("repository error: {0}")] + Repository(#[from] RepositoryError), + /// Storage error. + #[error("storage error: {0}")] + Storage(#[from] storage::Error), + /// Storage refs error. + #[error("storage refs error: {0}")] + Refs(#[from] storage::refs::Error), + /// No rows returned in query result. + #[error("no rows returned")] + NoRows, } /// Refs store. /// /// Used to cache git references. pub trait Store { + /// Set a reference under a remote namespace to the given [`Oid`]. fn set( &mut self, repo: &RepoId, @@ -33,16 +48,28 @@ pub trait Store { oid: Oid, timestamp: LocalTime, ) -> Result; - + /// Get a reference's [`Oid`] and timestamp. fn get( &self, repo: &RepoId, namespace: &NodeId, refname: &Qualified, ) -> Result, Error>; - - fn delete(&self, repo: &RepoId, namespace: &NodeId, refname: &Qualified) - -> Result; + /// Delete a reference. + fn delete( + &mut self, + repo: &RepoId, + namespace: &NodeId, + refname: &Qualified, + ) -> Result; + /// Populate the database from storage. + fn populate(&mut self, storage: &S) -> Result<(), Error>; + /// Return the number of references. + fn count(&self) -> Result; + /// Check if there are any references. + fn is_empty(&self) -> Result { + self.count().map(|l| l == 0) + } } impl Store for Database { @@ -103,7 +130,7 @@ impl Store for Database { } fn delete( - &self, + &mut self, repo: &RepoId, namespace: &NodeId, refname: &Qualified, @@ -119,6 +146,30 @@ impl Store for Database { Ok(self.db.change_count() > 0) } + + fn count(&self) -> Result { + let row = self + .db + .prepare("SELECT COUNT(*) FROM refs")? + .into_iter() + .next() + .ok_or(Error::NoRows)??; + let count = row.read::(0) as usize; + + Ok(count) + } + + fn populate(&mut self, storage: &S) -> Result<(), Error> { + let now = LocalTime::now(); + + for info in storage.repositories()? { + let repo = storage.repository(info.rid)?; + for refs_at in repo.remote_refs_at()? { + self.set(&repo.id(), &refs_at.remote, refs_at.path(), refs_at.at, now)?; + } + } + Ok(()) + } } #[cfg(test)] @@ -128,6 +179,32 @@ mod test { use crate::test::arbitrary; use localtime::{LocalDuration, LocalTime}; + #[test] + fn test_count() { + let mut db = Database::memory().unwrap(); + let oid = arbitrary::oid(); + + let repo = arbitrary::gen::(1); + let namespace = arbitrary::gen::(1); + let refname1 = qualified!("refs/heads/master"); + let refname2 = qualified!("refs/heads/main"); + let timestamp = LocalTime::now(); + + assert!(db.is_empty().unwrap()); + assert_eq!(db.count().unwrap(), 0); + + assert!(db + .set(&repo, &namespace, &refname1, oid, timestamp) + .unwrap()); + assert!(!db.is_empty().unwrap()); + assert_eq!(db.count().unwrap(), 1); + + assert!(db + .set(&repo, &namespace, &refname2, oid, timestamp) + .unwrap()); + assert_eq!(db.count().unwrap(), 2); + } + #[test] fn test_set_and_delete() { let mut db = Database::memory().unwrap(); diff --git a/radicle/src/storage.rs b/radicle/src/storage.rs index d3e87436..0d65bb7e 100644 --- a/radicle/src/storage.rs +++ b/radicle/src/storage.rs @@ -25,11 +25,25 @@ use crate::storage::git::NAMESPACES_GLOB; use crate::storage::refs::Refs; use self::git::UserInfo; -use self::refs::SignedRefs; +use self::refs::{RefsAt, SignedRefs}; pub type BranchName = git::RefString; pub type Inventory = BTreeSet; +/// Basic repository information. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RepositoryInfo { + /// Repository identifier. + pub rid: RepoId, + /// Head of default branch. + pub head: Oid, + /// Identity document. + pub doc: Doc, + /// Local signed refs, if any. + /// Repositories with this set to `None` are ones that are seeded but not forked. + pub refs: Option, +} + /// Describes one or more namespaces. #[derive(Default, Debug, Clone, PartialEq, Eq)] pub enum Namespaces { @@ -400,6 +414,8 @@ pub trait ReadStorage { /// Get the inventory of repositories hosted under this storage. /// This function should typically only return public repositories. fn inventory(&self) -> Result; + /// Return all repositories (public and private). + fn repositories(&self) -> Result>, Error>; /// Insert this repository into the inventory. fn insert(&self, rid: RepoId); /// Open or create a read-only repository. @@ -585,6 +601,9 @@ pub trait RemoteRepository { /// Get all remotes. fn remotes(&self) -> Result, refs::Error>; + + /// Get [`RefsAt`] of all remotes. + fn remote_refs_at(&self) -> Result, refs::Error>; } pub trait ValidateRepository @@ -671,6 +690,10 @@ where fn repository(&self, rid: RepoId) -> Result { self.deref().repository(rid) } + + fn repositories(&self) -> Result>, Error> { + self.deref().repositories() + } } impl WriteStorage for T diff --git a/radicle/src/storage/git.rs b/radicle/src/storage/git.rs index a03a36bb..1fb596f3 100644 --- a/radicle/src/storage/git.rs +++ b/radicle/src/storage/git.rs @@ -20,8 +20,8 @@ use crate::identity::{Identity, Project}; use crate::storage::refs; use crate::storage::refs::{Refs, SignedRefs, SignedRefsAt}; use crate::storage::{ - Inventory, ReadRepository, ReadStorage, Remote, Remotes, RepositoryError, SetHead, - SignRepository, WriteRepository, WriteStorage, + Inventory, ReadRepository, ReadStorage, Remote, Remotes, RepositoryError, RepositoryInfo, + SetHead, SignRepository, WriteRepository, WriteStorage, }; pub use crate::git::{ @@ -29,6 +29,7 @@ pub use crate::git::{ }; pub use crate::storage::Error; +use super::refs::RefsAt; use super::{RemoteId, RemoteRepository, ValidateRepository}; pub static NAMESPACES_GLOB: Lazy = @@ -43,20 +44,6 @@ pub static CANONICAL_IDENTITY: Lazy = Lazy::new(|| { ) }); -/// Basic repository information. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct RepositoryInfo { - /// Repository identifier. - pub rid: RepoId, - /// Head of default branch. - pub head: Oid, - /// Identity document. - pub doc: Doc, - /// Local signed refs, if any. - /// Repositories with this set to `None` are ones that are seeded but not forked. - pub refs: Option, -} - /// A parsed Git reference. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Ref { @@ -148,6 +135,65 @@ impl ReadStorage for Storage { fn repository(&self, rid: RepoId) -> Result { Repository::open(paths::repository(self, &rid), rid) } + + fn repositories(&self) -> Result>, Error> { + let mut repos = Vec::new(); + + for result in fs::read_dir(&self.path)? { + let path = result?; + + // Skip non-directories. + if !path.file_type()?.is_dir() { + continue; + } + // Skip hidden files. + if path.file_name().to_string_lossy().starts_with('.') { + continue; + } + // Skip lock files. + if let Some(ext) = path.path().extension() { + if ext == "lock" { + continue; + } + } + let rid = RepoId::try_from(path.file_name()) + .map_err(|_| Error::InvalidId(path.file_name()))?; + + let repo = match self.repository(rid) { + Ok(repo) => repo, + Err(e) => { + log::warn!(target: "storage", "Repository {rid} is invalid: {e}"); + continue; + } + }; + let doc = match repo.identity_doc() { + Ok(doc) => doc.into(), + Err(e) => { + log::warn!(target: "storage", "Repository {rid} is invalid: looking up doc: {e}"); + continue; + } + }; + + // For performance reasons, we don't do a full repository check here. + let head = match repo.head() { + Ok((_, head)) => head, + Err(e) => { + log::warn!(target: "storage", "Repository {rid} is invalid: looking up head: {e}"); + continue; + } + }; + // Nb. This will be `None` if they were not found. + let refs = refs::SignedRefsAt::load(self.info.key, &repo)?; + + repos.push(RepositoryInfo { + rid, + head, + doc, + refs, + }); + } + Ok(repos) + } } impl WriteStorage for Storage { @@ -218,65 +264,6 @@ impl Storage { self.path.as_path() } - pub fn repositories(&self) -> Result>, Error> { - let mut repos = Vec::new(); - - for result in fs::read_dir(&self.path)? { - let path = result?; - - // Skip non-directories. - if !path.file_type()?.is_dir() { - continue; - } - // Skip hidden files. - if path.file_name().to_string_lossy().starts_with('.') { - continue; - } - // Skip lock files. - if let Some(ext) = path.path().extension() { - if ext == "lock" { - continue; - } - } - let rid = RepoId::try_from(path.file_name()) - .map_err(|_| Error::InvalidId(path.file_name()))?; - - let repo = match self.repository(rid) { - Ok(repo) => repo, - Err(e) => { - log::warn!(target: "storage", "Repository {rid} is invalid: {e}"); - continue; - } - }; - let doc = match repo.identity_doc() { - Ok(doc) => doc.into(), - Err(e) => { - log::warn!(target: "storage", "Repository {rid} is invalid: looking up doc: {e}"); - continue; - } - }; - - // For performance reasons, we don't do a full repository check here. - let head = match repo.head() { - Ok((_, head)) => head, - Err(e) => { - log::warn!(target: "storage", "Repository {rid} is invalid: looking up head: {e}"); - continue; - } - }; - // Nb. This will be `None` if they were not found. - let refs = refs::SignedRefsAt::load(self.info.key, &repo)?; - - repos.push(RepositoryInfo { - rid, - head, - doc, - refs, - }); - } - Ok(repos) - } - pub fn repositories_by_id<'a>( &self, mut rids: impl Iterator, @@ -578,6 +565,18 @@ impl RemoteRepository for Repository { let refs = SignedRefs::load(*remote, self)?; Ok(Remote::::new(refs)) } + + fn remote_refs_at(&self) -> Result, refs::Error> { + let mut all = Vec::new(); + + for remote in self.remote_ids()? { + let remote = remote?; + let refs_at = RefsAt::new(self, remote)?; + + all.push(refs_at); + } + Ok(all) + } } impl ValidateRepository for Repository { diff --git a/radicle/src/storage/git/cob.rs b/radicle/src/storage/git/cob.rs index 2f93865e..115f93d0 100644 --- a/radicle/src/storage/git/cob.rs +++ b/radicle/src/storage/git/cob.rs @@ -241,6 +241,10 @@ impl<'a, R: storage::RemoteRepository> RemoteRepository for DraftStore<'a, R> { fn remotes(&self) -> Result, storage::refs::Error> { RemoteRepository::remotes(self.repo) } + + fn remote_refs_at(&self) -> Result, storage::refs::Error> { + RemoteRepository::remote_refs_at(self.repo) + } } impl<'a, R: storage::ValidateRepository> ValidateRepository for DraftStore<'a, R> { diff --git a/radicle/src/storage/refs.rs b/radicle/src/storage/refs.rs index f5cd33af..bef6c686 100644 --- a/radicle/src/storage/refs.rs +++ b/radicle/src/storage/refs.rs @@ -387,6 +387,10 @@ impl RefsAt { pub fn load(&self, repo: &S) -> Result { SignedRefsAt::load_at(self.at, self.remote, repo) } + + pub fn path(&self) -> &git::Qualified { + &SIGREFS_BRANCH + } } /// Verified [`SignedRefs`] that keeps track of their content address diff --git a/radicle/src/test/storage.rs b/radicle/src/test/storage.rs index dda33ade..d3f84883 100644 --- a/radicle/src/test/storage.rs +++ b/radicle/src/test/storage.rs @@ -12,7 +12,7 @@ use crate::node::NodeId; pub use crate::storage::*; -use super::fixtures; +use super::{arbitrary, fixtures}; #[derive(Clone, Debug)] pub struct MockStorage { @@ -89,6 +89,19 @@ impl ReadStorage for MockStorage { }) .cloned() } + + fn repositories(&self) -> Result>, Error> { + Ok(self + .repos + .iter() + .map(|(rid, r)| RepositoryInfo { + rid: *rid, + head: r.head().unwrap().1, + doc: r.doc.clone().into(), + refs: None, + }) + .collect()) + } } impl WriteStorage for MockStorage { @@ -159,6 +172,17 @@ impl RemoteRepository for MockRepository { }) .collect()) } + + fn remote_refs_at(&self) -> Result, refs::Error> { + Ok(self + .remotes + .values() + .map(|s| refs::RefsAt { + remote: s.id, + at: s.at, + }) + .collect()) + } } impl ValidateRepository for MockRepository { @@ -177,7 +201,7 @@ impl ReadRepository for MockRepository { } fn head(&self) -> Result<(fmt::Qualified, Oid), RepositoryError> { - todo!() + Ok((fmt::qualified!("refs/heads/master"), arbitrary::oid())) } fn canonical_head(&self) -> Result<(fmt::Qualified, Oid), RepositoryError> {