node: Use inventory cache for checking missing

Use the cache when checking whether we're missing an inventory.
This is a lot faster.
This commit is contained in:
cloudhead 2024-05-13 14:30:41 +02:00
parent cce46a3d8b
commit 25c6660a59
No known key found for this signature in database
5 changed files with 72 additions and 13 deletions

View File

@ -159,7 +159,7 @@ impl<T, const N: usize> BoundedVec<T, N> {
self.v self.v
} }
/// Calls [`Vec::Drain`]. /// Calls [`std::vec::Drain`].
pub fn drain<R: RangeBounds<usize>>(&mut self, range: R) -> std::vec::Drain<T> { pub fn drain<R: RangeBounds<usize>>(&mut self, range: R) -> std::vec::Drain<T> {
self.v.drain(range) self.v.drain(range)
} }

View File

@ -1431,6 +1431,9 @@ where
} }
} }
let inventory = self.storage.inventory_ref();
let mut missing = Vec::new();
for id in message.inventory.as_slice() { for id in message.inventory.as_slice() {
// TODO: Move this out (good luck with the borrow checker). // TODO: Move this out (good luck with the borrow checker).
if let Some(sess) = self.sessions.get_mut(announcer) { if let Some(sess) = self.sessions.get_mut(announcer) {
@ -1448,22 +1451,18 @@ where
) { ) {
// Only if we do not have the repository locally do we fetch here. // Only if we do not have the repository locally do we fetch here.
// If we do have it, only fetch after receiving a ref announcement. // If we do have it, only fetch after receiving a ref announcement.
match self.storage.contains(id) { if !inventory.contains(id) {
Ok(true) => { missing.push(*id);
// Do nothing.
}
Ok(false) => {
debug!(target: "service", "Missing seeded inventory {id}; initiating fetch..");
self.fetch(*id, *announcer, FETCH_TIMEOUT, None);
}
Err(e) => {
error!(target: "service", "Error checking local inventory for {id}: {e}");
}
} }
} }
} }
} }
drop(inventory);
for rid in missing {
debug!(target: "service", "Missing seeded inventory {rid}; initiating fetch..");
self.fetch(rid, *announcer, FETCH_TIMEOUT, None);
}
return Ok(relay); return Ok(relay);
} }
AnnouncementMessage::Refs(message) => { AnnouncementMessage::Refs(message) => {

View File

@ -402,6 +402,9 @@ impl<V> Deref for Remote<V> {
/// Read-only operations on a storage instance. /// Read-only operations on a storage instance.
pub trait ReadStorage { pub trait ReadStorage {
type Repository: ReadRepository; type Repository: ReadRepository;
type InventoryRef<'a>: Deref<Target = Inventory>
where
Self: 'a;
/// Get user info for this storage. /// Get user info for this storage.
fn info(&self) -> &UserInfo; fn info(&self) -> &UserInfo;
@ -414,6 +417,9 @@ pub trait ReadStorage {
/// Get the inventory of repositories hosted under this storage. /// Get the inventory of repositories hosted under this storage.
/// This function should typically only return public repositories. /// This function should typically only return public repositories.
fn inventory(&self) -> Result<Inventory, Error>; fn inventory(&self) -> Result<Inventory, Error>;
/// Return a reference to the inventory. Note that this function may hold a
/// lock to the inventory.
fn inventory_ref<'a, 's: 'a>(&'s self) -> Self::InventoryRef<'a>;
/// Return all repositories (public and private). /// Return all repositories (public and private).
fn repositories(&self) -> Result<Vec<RepositoryInfo<Verified>>, Error>; fn repositories(&self) -> Result<Vec<RepositoryInfo<Verified>>, Error>;
/// Insert this repository into the inventory. /// Insert this repository into the inventory.
@ -658,6 +664,7 @@ where
S: ReadStorage + 'static, S: ReadStorage + 'static,
{ {
type Repository = S::Repository; type Repository = S::Repository;
type InventoryRef<'a> = S::InventoryRef<'a> where T: 'a;
fn info(&self) -> &UserInfo { fn info(&self) -> &UserInfo {
self.deref().info() self.deref().info()
@ -683,6 +690,13 @@ where
self.deref().inventory() self.deref().inventory()
} }
fn inventory_ref<'a, 's: 'a>(&'s self) -> Self::InventoryRef<'a>
where
T: 'a,
{
self.deref().inventory_ref()
}
fn get(&self, rid: RepoId) -> Result<Option<Doc<Verified>>, RepositoryError> { fn get(&self, rid: RepoId) -> Result<Option<Doc<Verified>>, RepositoryError> {
self.deref().get(rid) self.deref().get(rid)
} }

View File

@ -5,7 +5,7 @@ pub mod transport;
use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::ops::{Deref, DerefMut}; use std::ops::{Deref, DerefMut};
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex, MutexGuard};
use std::{fs, io}; use std::{fs, io};
use crypto::{Signer, Verified}; use crypto::{Signer, Verified};
@ -72,6 +72,23 @@ impl<'a> TryFrom<git2::Reference<'a>> for Ref {
} }
} }
/// A reference to a locked inventory.
pub struct InventoryLock<'a> {
guard: MutexGuard<'a, Option<BTreeSet<RepoId>>>,
}
impl<'a> Deref for InventoryLock<'a> {
type Target = BTreeSet<RepoId>;
#[track_caller]
fn deref(&self) -> &Self::Target {
match *self.guard {
Some(ref cache) => cache,
None => panic!("InventoryLock::deref: inventory cache was not initialized"),
}
}
}
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct Storage { pub struct Storage {
path: PathBuf, path: PathBuf,
@ -82,6 +99,7 @@ pub struct Storage {
impl ReadStorage for Storage { impl ReadStorage for Storage {
type Repository = Repository; type Repository = Repository;
type InventoryRef<'a> = InventoryLock<'a>;
fn info(&self) -> &UserInfo { fn info(&self) -> &UserInfo {
&self.info &self.info
@ -103,6 +121,15 @@ impl ReadStorage for Storage {
Ok(false) Ok(false)
} }
fn inventory_ref<'a, 's: 'a>(&'s self) -> InventoryLock<'a> {
let guard = self
.inventory
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
InventoryLock { guard }
}
fn inventory(&self) -> Result<Inventory, Error> { fn inventory(&self) -> Result<Inventory, Error> {
let mut cache = self let mut cache = self
.inventory .inventory

View File

@ -56,8 +56,21 @@ impl MockStorage {
} }
} }
pub struct InventoryLock {
inner: Inventory,
}
impl std::ops::Deref for InventoryLock {
type Target = BTreeSet<RepoId>;
fn deref(&self) -> &Self::Target {
&self.inner
}
}
impl ReadStorage for MockStorage { impl ReadStorage for MockStorage {
type Repository = MockRepository; type Repository = MockRepository;
type InventoryRef<'a> = InventoryLock;
fn info(&self) -> &git::UserInfo { fn info(&self) -> &git::UserInfo {
&self.info &self.info
@ -79,6 +92,12 @@ impl ReadStorage for MockStorage {
Ok(self.repos.keys().cloned().collect::<BTreeSet<_>>()) Ok(self.repos.keys().cloned().collect::<BTreeSet<_>>())
} }
fn inventory_ref<'a, 's: 'a>(&'s self) -> Self::InventoryRef<'a> {
InventoryLock {
inner: self.inventory().unwrap(),
}
}
fn insert(&self, _rid: RepoId) {} fn insert(&self, _rid: RepoId) {}
fn repository(&self, rid: RepoId) -> Result<Self::Repository, RepositoryError> { fn repository(&self, rid: RepoId) -> Result<Self::Repository, RepositoryError> {