node: Use inventory cache for checking missing
Use the cache when checking whether we're missing an inventory. This is a lot faster.
This commit is contained in:
parent
cce46a3d8b
commit
25c6660a59
|
|
@ -159,7 +159,7 @@ impl<T, const N: usize> BoundedVec<T, N> {
|
||||||
self.v
|
self.v
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Calls [`Vec::Drain`].
|
/// Calls [`std::vec::Drain`].
|
||||||
pub fn drain<R: RangeBounds<usize>>(&mut self, range: R) -> std::vec::Drain<T> {
|
pub fn drain<R: RangeBounds<usize>>(&mut self, range: R) -> std::vec::Drain<T> {
|
||||||
self.v.drain(range)
|
self.v.drain(range)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1431,6 +1431,9 @@ where
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let inventory = self.storage.inventory_ref();
|
||||||
|
let mut missing = Vec::new();
|
||||||
|
|
||||||
for id in message.inventory.as_slice() {
|
for id in message.inventory.as_slice() {
|
||||||
// TODO: Move this out (good luck with the borrow checker).
|
// TODO: Move this out (good luck with the borrow checker).
|
||||||
if let Some(sess) = self.sessions.get_mut(announcer) {
|
if let Some(sess) = self.sessions.get_mut(announcer) {
|
||||||
|
|
@ -1448,22 +1451,18 @@ where
|
||||||
) {
|
) {
|
||||||
// Only if we do not have the repository locally do we fetch here.
|
// Only if we do not have the repository locally do we fetch here.
|
||||||
// If we do have it, only fetch after receiving a ref announcement.
|
// If we do have it, only fetch after receiving a ref announcement.
|
||||||
match self.storage.contains(id) {
|
if !inventory.contains(id) {
|
||||||
Ok(true) => {
|
missing.push(*id);
|
||||||
// Do nothing.
|
|
||||||
}
|
|
||||||
Ok(false) => {
|
|
||||||
debug!(target: "service", "Missing seeded inventory {id}; initiating fetch..");
|
|
||||||
self.fetch(*id, *announcer, FETCH_TIMEOUT, None);
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
error!(target: "service", "Error checking local inventory for {id}: {e}");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
drop(inventory);
|
||||||
|
|
||||||
|
for rid in missing {
|
||||||
|
debug!(target: "service", "Missing seeded inventory {rid}; initiating fetch..");
|
||||||
|
self.fetch(rid, *announcer, FETCH_TIMEOUT, None);
|
||||||
|
}
|
||||||
return Ok(relay);
|
return Ok(relay);
|
||||||
}
|
}
|
||||||
AnnouncementMessage::Refs(message) => {
|
AnnouncementMessage::Refs(message) => {
|
||||||
|
|
|
||||||
|
|
@ -402,6 +402,9 @@ impl<V> Deref for Remote<V> {
|
||||||
/// Read-only operations on a storage instance.
|
/// Read-only operations on a storage instance.
|
||||||
pub trait ReadStorage {
|
pub trait ReadStorage {
|
||||||
type Repository: ReadRepository;
|
type Repository: ReadRepository;
|
||||||
|
type InventoryRef<'a>: Deref<Target = Inventory>
|
||||||
|
where
|
||||||
|
Self: 'a;
|
||||||
|
|
||||||
/// Get user info for this storage.
|
/// Get user info for this storage.
|
||||||
fn info(&self) -> &UserInfo;
|
fn info(&self) -> &UserInfo;
|
||||||
|
|
@ -414,6 +417,9 @@ pub trait ReadStorage {
|
||||||
/// Get the inventory of repositories hosted under this storage.
|
/// Get the inventory of repositories hosted under this storage.
|
||||||
/// This function should typically only return public repositories.
|
/// This function should typically only return public repositories.
|
||||||
fn inventory(&self) -> Result<Inventory, Error>;
|
fn inventory(&self) -> Result<Inventory, Error>;
|
||||||
|
/// Return a reference to the inventory. Note that this function may hold a
|
||||||
|
/// lock to the inventory.
|
||||||
|
fn inventory_ref<'a, 's: 'a>(&'s self) -> Self::InventoryRef<'a>;
|
||||||
/// Return all repositories (public and private).
|
/// Return all repositories (public and private).
|
||||||
fn repositories(&self) -> Result<Vec<RepositoryInfo<Verified>>, Error>;
|
fn repositories(&self) -> Result<Vec<RepositoryInfo<Verified>>, Error>;
|
||||||
/// Insert this repository into the inventory.
|
/// Insert this repository into the inventory.
|
||||||
|
|
@ -658,6 +664,7 @@ where
|
||||||
S: ReadStorage + 'static,
|
S: ReadStorage + 'static,
|
||||||
{
|
{
|
||||||
type Repository = S::Repository;
|
type Repository = S::Repository;
|
||||||
|
type InventoryRef<'a> = S::InventoryRef<'a> where T: 'a;
|
||||||
|
|
||||||
fn info(&self) -> &UserInfo {
|
fn info(&self) -> &UserInfo {
|
||||||
self.deref().info()
|
self.deref().info()
|
||||||
|
|
@ -683,6 +690,13 @@ where
|
||||||
self.deref().inventory()
|
self.deref().inventory()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn inventory_ref<'a, 's: 'a>(&'s self) -> Self::InventoryRef<'a>
|
||||||
|
where
|
||||||
|
T: 'a,
|
||||||
|
{
|
||||||
|
self.deref().inventory_ref()
|
||||||
|
}
|
||||||
|
|
||||||
fn get(&self, rid: RepoId) -> Result<Option<Doc<Verified>>, RepositoryError> {
|
fn get(&self, rid: RepoId) -> Result<Option<Doc<Verified>>, RepositoryError> {
|
||||||
self.deref().get(rid)
|
self.deref().get(rid)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ pub mod transport;
|
||||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||||
use std::ops::{Deref, DerefMut};
|
use std::ops::{Deref, DerefMut};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex, MutexGuard};
|
||||||
use std::{fs, io};
|
use std::{fs, io};
|
||||||
|
|
||||||
use crypto::{Signer, Verified};
|
use crypto::{Signer, Verified};
|
||||||
|
|
@ -72,6 +72,23 @@ impl<'a> TryFrom<git2::Reference<'a>> for Ref {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A reference to a locked inventory.
|
||||||
|
pub struct InventoryLock<'a> {
|
||||||
|
guard: MutexGuard<'a, Option<BTreeSet<RepoId>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a> Deref for InventoryLock<'a> {
|
||||||
|
type Target = BTreeSet<RepoId>;
|
||||||
|
|
||||||
|
#[track_caller]
|
||||||
|
fn deref(&self) -> &Self::Target {
|
||||||
|
match *self.guard {
|
||||||
|
Some(ref cache) => cache,
|
||||||
|
None => panic!("InventoryLock::deref: inventory cache was not initialized"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct Storage {
|
pub struct Storage {
|
||||||
path: PathBuf,
|
path: PathBuf,
|
||||||
|
|
@ -82,6 +99,7 @@ pub struct Storage {
|
||||||
|
|
||||||
impl ReadStorage for Storage {
|
impl ReadStorage for Storage {
|
||||||
type Repository = Repository;
|
type Repository = Repository;
|
||||||
|
type InventoryRef<'a> = InventoryLock<'a>;
|
||||||
|
|
||||||
fn info(&self) -> &UserInfo {
|
fn info(&self) -> &UserInfo {
|
||||||
&self.info
|
&self.info
|
||||||
|
|
@ -103,6 +121,15 @@ impl ReadStorage for Storage {
|
||||||
Ok(false)
|
Ok(false)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn inventory_ref<'a, 's: 'a>(&'s self) -> InventoryLock<'a> {
|
||||||
|
let guard = self
|
||||||
|
.inventory
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||||
|
|
||||||
|
InventoryLock { guard }
|
||||||
|
}
|
||||||
|
|
||||||
fn inventory(&self) -> Result<Inventory, Error> {
|
fn inventory(&self) -> Result<Inventory, Error> {
|
||||||
let mut cache = self
|
let mut cache = self
|
||||||
.inventory
|
.inventory
|
||||||
|
|
|
||||||
|
|
@ -56,8 +56,21 @@ impl MockStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub struct InventoryLock {
|
||||||
|
inner: Inventory,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::ops::Deref for InventoryLock {
|
||||||
|
type Target = BTreeSet<RepoId>;
|
||||||
|
|
||||||
|
fn deref(&self) -> &Self::Target {
|
||||||
|
&self.inner
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl ReadStorage for MockStorage {
|
impl ReadStorage for MockStorage {
|
||||||
type Repository = MockRepository;
|
type Repository = MockRepository;
|
||||||
|
type InventoryRef<'a> = InventoryLock;
|
||||||
|
|
||||||
fn info(&self) -> &git::UserInfo {
|
fn info(&self) -> &git::UserInfo {
|
||||||
&self.info
|
&self.info
|
||||||
|
|
@ -79,6 +92,12 @@ impl ReadStorage for MockStorage {
|
||||||
Ok(self.repos.keys().cloned().collect::<BTreeSet<_>>())
|
Ok(self.repos.keys().cloned().collect::<BTreeSet<_>>())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn inventory_ref<'a, 's: 'a>(&'s self) -> Self::InventoryRef<'a> {
|
||||||
|
InventoryLock {
|
||||||
|
inner: self.inventory().unwrap(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn insert(&self, _rid: RepoId) {}
|
fn insert(&self, _rid: RepoId) {}
|
||||||
|
|
||||||
fn repository(&self, rid: RepoId) -> Result<Self::Repository, RepositoryError> {
|
fn repository(&self, rid: RepoId) -> Result<Self::Repository, RepositoryError> {
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue