node: Cache node inventory

For large seeds with many repos, it's expensive to traverse the
inventory everytime the node connects to a new peer, for example.

To improve the situation, we cache the inventory per `Storage` instance,
and add a `refresh` method to refresh the inventory.

This method is called when the node is told to synchronize its
inventory.

radicle-node/src/runtime.rs

@@ -20,6 +20,7 @@ use radicle::node::address::Store as _;
 use radicle::node::notifications;
 use radicle::node::Handle as _;
 use radicle::profile::Home;
+use radicle::storage;
 use radicle::Storage;
 use radicle::{cob, git};
 
@@ -48,6 +49,9 @@ pub enum Error {
     /// A node database error.
     #[error("node database error: {0}")]
     Database(#[from] node::db::Error),
+    /// A storage error.
+    #[error("storage error: {0}")]
+    Storage(#[from] storage::Error),
     /// A policies database error.
     #[error("policies database error: {0}")]
     Policy(#[from] policy::Error),

radicle-node/src/service.rs

@@ -1765,6 +1765,8 @@ where
 
     /// Update our routing table with our local node's inventory.
     fn sync_inventory(&mut self) -> Result<SyncedRouting, Error> {
+        self.storage.refresh()?; // Refresh storage inventory cache.
+
         let inventory = self.storage.inventory()?;
         let result = self.sync_routing(&inventory, self.node_id(), self.time())?;
 

radicle-node/src/tests/e2e.rs

@@ -182,6 +182,9 @@ fn test_replication() {
 
     log::debug!(target: "test", "Fetch complete with {}", bob.id);
 
+    alice.storage.refresh().unwrap();
+    bob.storage.refresh().unwrap();
+
     let inventory = alice.storage.inventory().unwrap();
     let alice_repo = alice.storage.repository(acme).unwrap();
     let bob_repo = bob.storage.repository(acme).unwrap();

radicle/src/profile.rs

@@ -109,6 +109,8 @@ pub enum Error {
     Repository(#[from] storage::RepositoryError),
     #[error(transparent)]
     CobsCache(#[from] cob::cache::Error),
+    #[error(transparent)]
+    Storage(#[from] storage::Error),
 }
 
 #[derive(Debug, Error)]

radicle/src/storage.rs

@@ -122,6 +122,8 @@ pub enum Error {
     Ext(#[from] git::ext::Error),
     #[error("invalid repository identifier {0:?}")]
     InvalidId(std::ffi::OsString),
+    #[error("inventory: {0}")]
+    Inventory(io::Error),
     #[error("i/o: {0}")]
     Io(#[from] io::Error),
 }
@@ -372,6 +374,8 @@ pub trait ReadStorage {
     /// Get the inventory of repositories hosted under this storage.
     /// This function should typically only return public repositories.
     fn inventory(&self) -> Result<Inventory, Error>;
+    /// Refresh storage inventory.
+    fn refresh(&self) -> Result<(), Error>;
     /// Open or create a read-only repository.
     fn repository(&self, rid: RepoId) -> Result<Self::Repository, Error>;
     /// Get a repository's identity if it exists.
@@ -630,6 +634,10 @@ where
         self.deref().inventory()
     }
 
+    fn refresh(&self) -> Result<(), Error> {
+        self.deref().refresh()
+    }
+
     fn get(&self, rid: RepoId) -> Result<Option<Doc<Verified>>, RepositoryError> {
         self.deref().get(rid)
     }

radicle/src/storage/git.rs

@@ -5,6 +5,7 @@ pub mod transport;
 use std::collections::{BTreeMap, BTreeSet, HashMap};
 use std::ops::{Deref, DerefMut};
 use std::path::{Path, PathBuf};
+use std::sync::{Arc, Mutex};
 use std::{fs, io};
 
 use crypto::{Signer, Verified};
@@ -88,6 +89,7 @@ impl<'a> TryFrom<git2::Reference<'a>> for Ref {
 pub struct Storage {
     path: PathBuf,
     info: UserInfo,
+    inventory: Arc<Mutex<Vec<RepoId>>>,
 }
 
 impl ReadStorage for Storage {
@@ -114,13 +116,33 @@ impl ReadStorage for Storage {
     }
 
     fn inventory(&self) -> Result<Inventory, Error> {
-        let repos = self.repositories()?;
+        match self.inventory.lock() {
+            Ok(locked) => Ok(locked.clone()),
+            Err(poisoned) => {
+                let inv = poisoned.into_inner();
+                Ok(inv.clone())
+            }
+        }
+    }
 
-        Ok(repos
+    fn refresh(&self) -> Result<(), Error> {
+        let repos = self.repositories()?;
+        let rids = repos
             .into_iter()
             .filter(|r| r.doc.visibility.is_public())
             .map(|r| r.rid)
-            .collect())
+            .collect();
+
+        match self.inventory.lock() {
+            Ok(mut locked) => {
+                *locked = rids;
+            }
+            Err(poisoned) => {
+                let mut inv = poisoned.into_inner();
+                *inv = rids;
+            }
+        }
+        Ok(())
     }
 
     fn repository(&self, rid: RepoId) -> Result<Self::Repository, Error> {
@@ -155,17 +177,23 @@ impl WriteStorage for Storage {
 }
 
 impl Storage {
-    // TODO: Return a better error when not found.
-    pub fn open<P: AsRef<Path>>(path: P, info: UserInfo) -> Result<Self, io::Error> {
+    /// Open a new storage instance and load its inventory.
+    pub fn open<P: AsRef<Path>>(path: P, info: UserInfo) -> Result<Self, Error> {
         let path = path.as_ref().to_path_buf();
 
         match fs::create_dir_all(&path) {
             Err(err) if err.kind() == io::ErrorKind::AlreadyExists => {}
-            Err(err) => return Err(err),
+            Err(err) => return Err(Error::Io(err)),
             Ok(()) => {}
         }
+        let storage = Self {
+            path,
+            info,
+            inventory: Arc::new(Mutex::new(vec![])),
+        };
+        storage.refresh()?;
 
-        Ok(Self { path, info })
+        Ok(storage)
     }
 
     /// Create a [`Repository`] in a temporary directory.
@@ -1248,11 +1276,11 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let signer = MockSigner::default();
         let storage = fixtures::storage(dir.path(), &signer).unwrap();
-        let inv = storage.inventory().unwrap();
+        let inv = storage.repositories().unwrap();
         let proj = inv.first().unwrap();
-        let mut refs = git::remote_refs(&git::Url::from(*proj)).unwrap();
+        let mut refs = git::remote_refs(&git::Url::from(proj.rid)).unwrap();
 
-        let project = storage.repository(*proj).unwrap();
+        let project = storage.repository(proj.rid).unwrap();
         let remotes = project.remotes().unwrap();
 
         // Strip the remote refs of sigrefs so we can compare them.

radicle/src/test/fixtures.rs

@@ -10,6 +10,7 @@ use crate::rad;
 use crate::storage::git::transport;
 use crate::storage::git::Storage;
 use crate::storage::refs::SignedRefs;
+use crate::storage::ReadStorage;
 
 /// The birth of the radicle project, January 1st, 2018.
 pub const RADICLE_EPOCH: i64 = 1514817556;
@@ -46,6 +47,7 @@ pub fn storage<P: AsRef<Path>, G: Signer>(path: P, signer: &G) -> Result<Storage
             &storage,
         )?;
     }
+    storage.refresh()?;
 
     Ok(storage)
 }
@@ -68,6 +70,7 @@ pub fn project<P: AsRef<Path>, G: Signer>(
         signer,
         storage,
     )?;
+    storage.refresh()?;
 
     Ok((id, refs, working, head))
 }

radicle/src/test/storage.rs

@@ -79,6 +79,10 @@ impl ReadStorage for MockStorage {
         Ok(self.repos.keys().cloned().collect::<Vec<_>>())
     }
 
+    fn refresh(&self) -> Result<(), Error> {
+        Ok(())
+    }
+
     fn repository(&self, rid: RepoId) -> Result<Self::Repository, Error> {
         self.repos
             .get(&rid)