radicle: Consolidate databases

Instead of having a routing database, and an addresses database,
consolidate that in a single database called `node.db`.

We also simplify database access, by providing a single type for opening
the database, and use traits to access the various functions.

We create a new `seed::Store` that has seed-related methods split out of
the `address::Store`.

Note that tracking is still in its own database currently, as it isn't
ephemeral state. This makes it easy to delete the new `node.db` without
risk of losing any important information.

To avoid renaming all database methods, we create a `Stores` wrapper
that allows us to access routing, addresses, seeds etc. without type
ambiguity.

This is a breaking change and requires `addresses.db` and `routing.db`
to be deleted.
This commit is contained in:
cloudhead 2023-11-28 18:00:47 +01:00
parent df785aa0bf
commit 6b04eff34c
No known key found for this signature in database
28 changed files with 867 additions and 878 deletions

View File

@ -3,7 +3,7 @@ use std::time;
use anyhow::anyhow; use anyhow::anyhow;
use radicle::node::{Address, Node, NodeId, PeerAddr, ROUTING_DB_FILE}; use radicle::node::{Address, Node, NodeId, PeerAddr};
use radicle::prelude::Id; use radicle::prelude::Id;
use crate::terminal as term; use crate::terminal as term;
@ -249,8 +249,7 @@ pub fn run(options: Options, ctx: impl term::Context) -> anyhow::Result<()> {
events::run(node, count, timeout)?; events::run(node, count, timeout)?;
} }
Operation::Routing { rid, nid, json } => { Operation::Routing { rid, nid, json } => {
let store = let store = profile.database()?;
radicle::node::routing::Table::reader(profile.home.node().join(ROUTING_DB_FILE))?;
routing::run(&store, rid, nid, json)?; routing::run(&store, rid, nid, json)?;
} }
Operation::Logs { lines } => control::logs(lines, Some(time::Duration::MAX), &profile)?, Operation::Logs { lines } => control::logs(lines, Some(time::Duration::MAX), &profile)?,

View File

@ -854,9 +854,10 @@ fn rad_clone_connect() {
let mut bob = bob.spawn(); let mut bob = bob.spawn();
// Let Eve know about Alice and Bob having the repo. // Let Eve know about Alice and Bob having the repo.
eve.routing.insert([&acme], alice.id, now).unwrap(); eve.db.routing_mut().insert([&acme], alice.id, now).unwrap();
eve.routing.insert([&acme], bob.id, now).unwrap(); eve.db.routing_mut().insert([&acme], bob.id, now).unwrap();
eve.addresses eve.db
.addresses_mut()
.insert( .insert(
&alice.id, &alice.id,
node::Features::SEED, node::Features::SEED,
@ -869,7 +870,8 @@ fn rad_clone_connect() {
)], )],
) )
.unwrap(); .unwrap();
eve.addresses eve.db
.addresses_mut()
.insert( .insert(
&bob.id, &bob.id,
node::Features::SEED, node::Features::SEED,

View File

@ -62,8 +62,8 @@ impl Context {
let delegates = doc.delegates; let delegates = doc.delegates;
let issues = issue::Issues::open(&repo)?.counts()?; let issues = issue::Issues::open(&repo)?.counts()?;
let patches = patch::Patches::open(&repo)?.counts()?; let patches = patch::Patches::open(&repo)?.counts()?;
let routing = &self.profile.routing()?; let db = &self.profile.database()?;
let trackings = routing.count(&id).unwrap_or_default(); let trackings = db.count(&id).unwrap_or_default();
Ok(project::Info { Ok(project::Info {
payload, payload,

View File

@ -74,9 +74,9 @@ pub enum Error {
#[error(transparent)] #[error(transparent)]
TrackingStore(#[from] radicle::node::tracking::store::Error), TrackingStore(#[from] radicle::node::tracking::store::Error),
/// Routing store error. /// Node database error.
#[error(transparent)] #[error(transparent)]
RoutingStore(#[from] radicle::node::routing::Error), Database(#[from] radicle::node::db::Error),
/// Node error. /// Node error.
#[error(transparent)] #[error(transparent)]

View File

@ -38,7 +38,7 @@ async fn delegates_projects_handler(
let page = page.unwrap_or(0); let page = page.unwrap_or(0);
let per_page = per_page.unwrap_or(10); let per_page = per_page.unwrap_or(10);
let storage = &ctx.profile.storage; let storage = &ctx.profile.storage;
let routing = &ctx.profile.routing()?; let db = &ctx.profile.database()?;
let mut projects = storage let mut projects = storage
.repositories()? .repositories()?
.into_iter() .into_iter()
@ -78,7 +78,7 @@ async fn delegates_projects_handler(
}; };
let delegates = id.doc.delegates; let delegates = id.doc.delegates;
let trackings = routing.count(&id.rid).unwrap_or_default(); let trackings = db.count(&id.rid).unwrap_or_default();
Some(Info { Some(Info {
payload, payload,

View File

@ -83,7 +83,7 @@ async fn project_root_handler(
let page = page.unwrap_or(0); let page = page.unwrap_or(0);
let per_page = per_page.unwrap_or(10); let per_page = per_page.unwrap_or(10);
let storage = &ctx.profile.storage; let storage = &ctx.profile.storage;
let routing = &ctx.profile.routing()?; let db = &ctx.profile.database()?;
let mut projects = storage let mut projects = storage
.repositories()? .repositories()?
.into_iter() .into_iter()
@ -119,7 +119,7 @@ async fn project_root_handler(
return None; return None;
}; };
let delegates = id.doc.delegates; let delegates = id.doc.delegates;
let trackings = routing.count(&id.rid).unwrap_or_default(); let trackings = db.count(&id.rid).unwrap_or_default();
Some(Info { Some(Info {
payload, payload,

View File

@ -19,9 +19,6 @@ use radicle::crypto::{KeyPair, Seed, Signer};
use radicle::git::{raw as git2, RefString}; use radicle::git::{raw as git2, RefString};
use radicle::identity::Visibility; use radicle::identity::Visibility;
use radicle::node; use radicle::node;
use radicle::node::address as AddressStore;
use radicle::node::routing as RoutingStore;
use radicle::node::tracking::store as TrackingStore;
use radicle::profile; use radicle::profile;
use radicle::profile::Home; use radicle::profile::Home;
use radicle::storage::ReadStorage; use radicle::storage::ReadStorage;
@ -96,15 +93,10 @@ pub fn contributor(dir: &Path) -> Context {
fn seed_with_signer<G: Signer>(dir: &Path, profile: radicle::Profile, signer: &G) -> Context { fn seed_with_signer<G: Signer>(dir: &Path, profile: radicle::Profile, signer: &G) -> Context {
const DEFAULT_BRANCH: &str = "master"; const DEFAULT_BRANCH: &str = "master";
let tracking_db = dir.join("radicle").join("node").join("tracking.db");
let routing_db = dir.join("radicle").join("node").join("routing.db");
let addresses_db = dir.join("radicle").join("node").join("addresses.db");
crate::logger::init().ok(); crate::logger::init().ok();
TrackingStore::Config::open(tracking_db).unwrap(); profile.tracking_mut().unwrap();
RoutingStore::Table::open(routing_db).unwrap(); profile.database_mut().unwrap(); // Create the database.
AddressStore::Book::open(addresses_db).unwrap();
let workdir = dir.join("hello-world-private"); let workdir = dir.join("hello-world-private");
fs::create_dir_all(&workdir).unwrap(); fs::create_dir_all(&workdir).unwrap();

View File

@ -19,7 +19,6 @@ use radicle::node;
use radicle::node::address; use radicle::node::address;
use radicle::node::address::Store as _; use radicle::node::address::Store as _;
use radicle::node::Handle as _; use radicle::node::Handle as _;
use radicle::node::{ADDRESS_DB_FILE, NODE_ANNOUNCEMENT_FILE, ROUTING_DB_FILE, TRACKING_DB_FILE};
use radicle::profile::Home; use radicle::profile::Home;
use radicle::Storage; use radicle::Storage;
@ -42,15 +41,18 @@ pub enum Error {
/// A routing database error. /// A routing database error.
#[error("routing database error: {0}")] #[error("routing database error: {0}")]
Routing(#[from] routing::Error), Routing(#[from] routing::Error),
/// An address database error. /// A node database error.
#[error("address database error: {0}")] #[error("node database error: {0}")]
Addresses(#[from] address::Error), Database(#[from] node::db::Error),
/// A tracking database error. /// A tracking database error.
#[error("tracking database error: {0}")] #[error("tracking database error: {0}")]
Tracking(#[from] tracking::Error), Tracking(#[from] tracking::Error),
/// A gossip database error. /// A gossip database error.
#[error("gossip database error: {0}")] #[error("gossip database error: {0}")]
Gossip(#[from] gossip::Error), Gossip(#[from] gossip::Error),
/// An address database error.
#[error("address database error: {0}")]
Address(#[from] address::Error),
/// An I/O error. /// An I/O error.
#[error("i/o error: {0}")] #[error("i/o error: {0}")]
Io(#[from] io::Error), Io(#[from] io::Error),
@ -137,29 +139,20 @@ impl Runtime {
let rng = fastrand::Rng::new(); let rng = fastrand::Rng::new();
let clock = LocalTime::now(); let clock = LocalTime::now();
let storage = Storage::open(home.storage(), git::UserInfo { alias, key: id })?; let storage = Storage::open(home.storage(), git::UserInfo { alias, key: id })?;
let address_db = node_dir.join(ADDRESS_DB_FILE);
let routing_db = node_dir.join(ROUTING_DB_FILE);
let tracking_db = node_dir.join(TRACKING_DB_FILE);
let scope = config.scope; let scope = config.scope;
let policy = config.policy; let policy = config.policy;
log::info!(target: "node", "Opening address book {}..", address_db.display()); log::info!(target: "node", "Opening node database..");
let mut addresses = address::Book::open(address_db.clone())?; let mut db: service::Stores<_> = home.database_mut()?.into();
log::info!(target: "node", "Opening gossip store from {}..", address_db.display()); log::info!(target: "node", "Opening tracking policy configuration..");
let gossip = gossip::Store::open(address_db)?; // Nb. same database as address book. let tracking = home.tracking_mut()?;
log::info!(target: "node", "Opening routing table {}..", routing_db.display());
let routing = routing::Table::open(routing_db)?;
log::info!(target: "node", "Opening tracking policy table {}..", tracking_db.display());
let tracking = tracking::Store::open(tracking_db.clone())?;
let tracking = tracking::Config::new(policy, scope, tracking); let tracking = tracking::Config::new(policy, scope, tracking);
log::info!(target: "node", "Default tracking policy set to '{}'", &policy); log::info!(target: "node", "Default tracking policy set to '{}'", &policy);
log::info!(target: "node", "Initializing service ({:?})..", network); log::info!(target: "node", "Initializing service ({:?})..", network);
let announcement = if let Some(ann) = fs::read(node_dir.join(NODE_ANNOUNCEMENT_FILE)) let announcement = if let Some(ann) = fs::read(node_dir.join(node::NODE_ANNOUNCEMENT_FILE))
.ok() .ok()
.and_then(|ann| NodeAnnouncement::decode(&mut ann.as_slice()).ok()) .and_then(|ann| NodeAnnouncement::decode(&mut ann.as_slice()).ok())
.and_then(|ann| { .and_then(|ann| {
@ -185,13 +178,13 @@ impl Runtime {
.expect("Runtime::init: unable to solve proof-of-work puzzle") .expect("Runtime::init: unable to solve proof-of-work puzzle")
}; };
if config.connect.is_empty() && addresses.is_empty()? { if config.connect.is_empty() && db.addresses().is_empty()? {
log::info!(target: "node", "Address book is empty. Adding bootstrap nodes.."); log::info!(target: "node", "Address book is empty. Adding bootstrap nodes..");
for (alias, addr) in config.network.bootstrap() { for (alias, addr) in config.network.bootstrap() {
let (id, addr) = addr.into(); let (id, addr) = addr.into();
addresses.insert( db.addresses_mut().insert(
&id, &id,
radicle::node::Features::SEED, radicle::node::Features::SEED,
alias, alias,
@ -200,17 +193,15 @@ impl Runtime {
[node::KnownAddress::new(addr, address::Source::Bootstrap)], [node::KnownAddress::new(addr, address::Source::Bootstrap)],
)?; )?;
} }
log::info!(target: "node", "{} nodes added to address book", addresses.len()?); log::info!(target: "node", "{} nodes added to address book", db.addresses().len()?);
} }
let emitter: Emitter<Event> = Default::default(); let emitter: Emitter<Event> = Default::default();
let service = service::Service::new( let service = service::Service::new(
config, config,
clock, clock,
routing, db,
storage.clone(), storage.clone(),
addresses,
gossip,
tracking, tracking,
signer.clone(), signer.clone(),
rng, rng,
@ -246,7 +237,7 @@ impl Runtime {
let fetch = worker::FetchConfig { let fetch = worker::FetchConfig {
policy, policy,
scope, scope,
tracking_db, tracking_db: home.node().join(node::TRACKING_DB_FILE),
limit: FetchLimit::default(), limit: FetchLimit::default(),
local: nid, local: nid,
expiry: worker::garbage::Expiry::default(), expiry: worker::garbage::Expiry::default(),

View File

@ -21,9 +21,14 @@ use localtime::{LocalDuration, LocalTime};
use log::*; use log::*;
use nonempty::NonEmpty; use nonempty::NonEmpty;
use radicle::node;
use radicle::node::address; use radicle::node::address;
use radicle::node::address::Store as _;
use radicle::node::address::{AddressBook, KnownAddress}; use radicle::node::address::{AddressBook, KnownAddress};
use radicle::node::config::PeerConfig; use radicle::node::config::PeerConfig;
use radicle::node::routing::Store as _;
use radicle::node::seed;
use radicle::node::seed::Store as _;
use radicle::node::ConnectOptions; use radicle::node::ConnectOptions;
use radicle::storage::RepositoryError; use radicle::storage::RepositoryError;
@ -37,6 +42,7 @@ use crate::node::{
}; };
use crate::prelude::*; use crate::prelude::*;
use crate::runtime::Emitter; use crate::runtime::Emitter;
use crate::service::gossip::Store as _;
use crate::service::message::{Announcement, AnnouncementMessage, Info, Ping}; use crate::service::message::{Announcement, AnnouncementMessage, Info, Ping};
use crate::service::message::{NodeAnnouncement, RefsAnnouncement}; use crate::service::message::{NodeAnnouncement, RefsAnnouncement};
use crate::service::tracking::{store::Write, Scope}; use crate::service::tracking::{store::Write, Scope};
@ -128,7 +134,11 @@ pub enum Error {
#[error(transparent)] #[error(transparent)]
Routing(#[from] routing::Error), Routing(#[from] routing::Error),
#[error(transparent)] #[error(transparent)]
Addresses(#[from] address::Error), Address(#[from] address::Error),
#[error(transparent)]
Database(#[from] node::db::Error),
#[error(transparent)]
Seeds(#[from] seed::Error),
#[error(transparent)] #[error(transparent)]
Tracking(#[from] tracking::Error), Tracking(#[from] tracking::Error),
#[error(transparent)] #[error(transparent)]
@ -137,6 +147,11 @@ pub enum Error {
Namespaces(#[from] NamespacesError), Namespaces(#[from] NamespacesError),
} }
/// A store for all node data.
pub trait Store: address::Store + gossip::Store + routing::Store + seed::Store {}
impl Store for node::Database {}
/// Function used to query internal service state. /// Function used to query internal service state.
pub type QueryState = dyn Fn(&dyn ServiceState) -> Result<(), CommandError> + Send + Sync; pub type QueryState = dyn Fn(&dyn ServiceState) -> Result<(), CommandError> + Send + Sync;
@ -223,23 +238,74 @@ struct FetchState {
subscribers: Vec<chan::Sender<FetchResult>>, subscribers: Vec<chan::Sender<FetchResult>>,
} }
/// Holds all node stores.
#[derive(Debug)]
pub struct Stores<D>(D);
impl<D> Stores<D>
where
D: Store,
{
/// Get the database as a routing store.
pub fn routing(&self) -> &impl routing::Store {
&self.0
}
/// Get the database as a routing store, mutably.
pub fn routing_mut(&mut self) -> &mut impl routing::Store {
&mut self.0
}
/// Get the database as an address store.
pub fn addresses(&self) -> &impl address::Store {
&self.0
}
/// Get the database as an address store, mutably.
pub fn addresses_mut(&mut self) -> &mut impl address::Store {
&mut self.0
}
/// Get the database as a gossip store.
pub fn gossip(&self) -> &impl gossip::Store {
&self.0
}
/// Get the database as a gossip store, mutably.
pub fn gossip_mut(&mut self) -> &mut impl gossip::Store {
&mut self.0
}
/// Get the database as a seed store.
pub fn seeds(&self) -> &impl seed::Store {
&self.0
}
/// Get the database as a seed store, mutably.
pub fn seeds_mut(&mut self) -> &mut impl seed::Store {
&mut self.0
}
}
impl<D> From<D> for Stores<D> {
fn from(db: D) -> Self {
Self(db)
}
}
/// The node service. /// The node service.
#[derive(Debug)] #[derive(Debug)]
pub struct Service<R, A, S, G> { pub struct Service<D, S, G> {
/// Service configuration. /// Service configuration.
config: Config, config: Config,
/// Our cryptographic signer and key. /// Our cryptographic signer and key.
signer: G, signer: G,
/// Project storage. /// Project storage.
storage: S, storage: S,
/// Network routing table. Keeps track of where projects are located. /// Node database.
routing: R, db: Stores<D>,
/// Node address manager.
addresses: A,
/// Tracking policy configuration. /// Tracking policy configuration.
tracking: tracking::Config<Write>, tracking: tracking::Config<Write>,
/// State relating to gossip.
gossip: gossip::Store,
/// Peer sessions, currently or recently connected. /// Peer sessions, currently or recently connected.
sessions: Sessions, sessions: Sessions,
/// Clock. Tells the time. /// Clock. Tells the time.
@ -272,7 +338,7 @@ pub struct Service<R, A, S, G> {
emitter: Emitter<Event>, emitter: Emitter<Event>,
} }
impl<R, A, S, G> Service<R, A, S, G> impl<D, S, G> Service<D, S, G>
where where
G: crypto::Signer, G: crypto::Signer,
{ {
@ -287,20 +353,17 @@ where
} }
} }
impl<R, A, S, G> Service<R, A, S, G> impl<D, S, G> Service<D, S, G>
where where
R: routing::Store, D: Store,
A: address::Store,
S: ReadStorage + 'static, S: ReadStorage + 'static,
G: Signer, G: Signer,
{ {
pub fn new( pub fn new(
config: Config, config: Config,
clock: LocalTime, clock: LocalTime,
routing: R, db: Stores<D>,
storage: S, storage: S,
addresses: A,
gossip: gossip::Store,
tracking: tracking::Config<Write>, tracking: tracking::Config<Write>,
signer: G, signer: G,
rng: Rng, rng: Rng,
@ -312,14 +375,12 @@ where
Self { Self {
config, config,
storage, storage,
addresses,
tracking, tracking,
signer, signer,
rng, rng,
node, node,
clock, clock,
routing, db,
gossip,
outbox: Outbox::default(), outbox: Outbox::default(),
limiter: RateLimiter::default(), limiter: RateLimiter::default(),
sessions, sessions,
@ -382,19 +443,14 @@ where
todo!() todo!()
} }
/// Get the address book instance. /// Get the database.
pub fn addresses(&self) -> &A { pub fn database(&self) -> &Stores<D> {
&self.addresses &self.db
} }
/// Get the mutable address book instance. /// Get the mutable database.
pub fn addresses_mut(&mut self) -> &mut A { pub fn database_mut(&mut self) -> &mut Stores<D> {
&mut self.addresses &mut self.db
}
/// Get the routing store.
pub fn routing(&self) -> &R {
&self.routing
} }
/// Get the storage instance. /// Get the storage instance.
@ -427,9 +483,9 @@ where
&mut self.outbox &mut self.outbox
} }
/// Lookup a project, both locally and in the routing table. /// Lookup a repository, both locally and in the routing table.
pub fn lookup(&self, rid: Id) -> Result<Lookup, LookupError> { pub fn lookup(&self, rid: Id) -> Result<Lookup, LookupError> {
let remote = self.routing.get(&rid)?.iter().cloned().collect(); let remote = self.db.routing().get(&rid)?.iter().cloned().collect();
Ok(Lookup { Ok(Lookup {
local: self.storage.get(rid)?, local: self.storage.get(rid)?,
@ -440,12 +496,14 @@ where
pub fn initialize(&mut self, time: LocalTime) -> Result<(), Error> { pub fn initialize(&mut self, time: LocalTime) -> Result<(), Error> {
debug!(target: "service", "Init @{}", time.as_millis()); debug!(target: "service", "Init @{}", time.as_millis());
let nid = self.node_id();
self.start_time = time; self.start_time = time;
// Ensure that our local node is in our address database. // Ensure that our local node is in our address database.
self.addresses self.db
.addresses_mut()
.insert( .insert(
&self.node_id(), &nid,
self.node.features, self.node.features,
self.node.alias.clone(), self.node.alias.clone(),
self.node.work(), self.node.work(),
@ -466,12 +524,11 @@ where
// all of it. It can happen that inventory is not properly tracked if for eg. the // all of it. It can happen that inventory is not properly tracked if for eg. the
// user creates a new repository while the node is stopped. // user creates a new repository while the node is stopped.
let rids = self.storage.inventory()?; let rids = self.storage.inventory()?;
self.routing self.db.routing_mut().insert(&rids, nid, time.as_millis())?;
.insert(&rids, self.node_id(), time.as_millis())?;
let nid = self.node_id();
let announced = self let announced = self
.addresses .db
.seeds()
.seeded_by(&nid)? .seeded_by(&nid)?
.collect::<Result<HashMap<_, _>, _>>()?; .collect::<Result<HashMap<_, _>, _>>()?;
for rid in rids { for rid in rids {
@ -493,15 +550,19 @@ where
} }
// Make sure our local node's sync status is up to date with storage. // Make sure our local node's sync status is up to date with storage.
debug!(target: "service", "Saving local sync status for {rid}.."); debug!(target: "service", "Saving local sync status for {rid}..");
self.addresses self.db.seeds_mut().synced(
.synced(&rid, &nid, updated_at.oid, updated_at.timestamp.as_millis())?; &rid,
&nid,
updated_at.oid,
updated_at.timestamp.as_millis(),
)?;
// If we got here, it likely means a repo was updated while the node was stopped. // If we got here, it likely means a repo was updated while the node was stopped.
// Therefore, we pre-load a refs announcement for this repo, so that it is included in // Therefore, we pre-load a refs announcement for this repo, so that it is included in
// the historical gossip messages when a node connects and subscribes to this repo. // the historical gossip messages when a node connects and subscribes to this repo.
if let Ok((ann, _)) = self.refs_announcement_for(rid, [nid]) { if let Ok((ann, _)) = self.refs_announcement_for(rid, [nid]) {
debug!(target: "service", "Adding refs announcement for {rid} to historical gossip messages.."); debug!(target: "service", "Adding refs announcement for {rid} to historical gossip messages..");
self.gossip.announced(&nid, &ann)?; self.db.gossip_mut().announced(&nid, &ann)?;
} }
} }
@ -566,7 +627,8 @@ where
error!(target: "service", "Error pruning routing entries: {err}"); error!(target: "service", "Error pruning routing entries: {err}");
} }
if let Err(err) = self if let Err(err) = self
.gossip .db
.gossip_mut()
.prune((now - self.config.limits.gossip_max_age).as_millis()) .prune((now - self.config.limits.gossip_max_age).as_millis())
{ {
error!(target: "service", "Error pruning gossip entries: {err}"); error!(target: "service", "Error pruning gossip entries: {err}");
@ -925,7 +987,7 @@ where
peer.to_connected(self.clock); peer.to_connected(self.clock);
self.outbox.write_all(peer, msgs); self.outbox.write_all(peer, msgs);
if let Err(e) = self.addresses.connected(&remote, &peer.addr, now) { if let Err(e) = self.db.addresses_mut().connected(&remote, &peer.addr, now) {
error!(target: "service", "Error updating address book with connection: {e}"); error!(target: "service", "Error updating address book with connection: {e}");
} }
} }
@ -1000,7 +1062,8 @@ where
debug!(target: "service", "Dropping peer {remote}.."); debug!(target: "service", "Dropping peer {remote}..");
if let Err(e) = if let Err(e) =
self.addresses self.db
.addresses_mut()
.disconnected(&remote, &session.addr, reason.is_transient()) .disconnected(&remote, &session.addr, reason.is_transient())
{ {
error!(target: "service", "Error updating address store: {e}"); error!(target: "service", "Error updating address store: {e}");
@ -1068,7 +1131,7 @@ where
// announcement timestamp, but before the other announcements. In that case, we simply // announcement timestamp, but before the other announcements. In that case, we simply
// ignore all announcements of that node until we get a node announcement. // ignore all announcements of that node until we get a node announcement.
if let AnnouncementMessage::Inventory(_) | AnnouncementMessage::Refs(_) = message { if let AnnouncementMessage::Inventory(_) | AnnouncementMessage::Refs(_) = message {
match self.addresses.get(announcer) { match self.db.addresses().get(announcer) {
Ok(node) => { Ok(node) => {
if node.is_none() { if node.is_none() {
debug!(target: "service", "Ignoring announcement from unknown node {announcer}"); debug!(target: "service", "Ignoring announcement from unknown node {announcer}");
@ -1083,7 +1146,7 @@ where
} }
// Discard announcement messages we've already seen, otherwise update our last seen time. // Discard announcement messages we've already seen, otherwise update our last seen time.
match self.gossip.announced(announcer, announcement) { match self.db.gossip_mut().announced(announcer, announcement) {
Ok(fresh) => { Ok(fresh) => {
if !fresh { if !fresh {
trace!(target: "service", "Ignoring stale inventory announcement from {announcer} (t={})", self.time()); trace!(target: "service", "Ignoring stale inventory announcement from {announcer} (t={})", self.time());
@ -1162,7 +1225,8 @@ where
// We update inventories when receiving ref announcements, as these could come // We update inventories when receiving ref announcements, as these could come
// from a new repository being initialized. // from a new repository being initialized.
if let Ok(result) = if let Ok(result) =
self.routing self.db
.routing_mut()
.insert([&message.rid], *announcer, message.timestamp) .insert([&message.rid], *announcer, message.timestamp)
{ {
if let &[(_, InsertResult::SeedAdded)] = result.as_slice() { if let &[(_, InsertResult::SeedAdded)] = result.as_slice() {
@ -1176,10 +1240,12 @@ where
// Update sync status for this repo. // Update sync status for this repo.
if let Some(refs) = message.refs.iter().find(|r| &r.remote == self.nid()) { if let Some(refs) = message.refs.iter().find(|r| &r.remote == self.nid()) {
match self match self.db.seeds_mut().synced(
.addresses &message.rid,
.synced(&message.rid, announcer, refs.at, message.timestamp) announcer,
{ refs.at,
message.timestamp,
) {
Ok(updated) => { Ok(updated) => {
if updated { if updated {
debug!( debug!(
@ -1295,7 +1361,7 @@ where
return Ok(relay); return Ok(relay);
} }
match self.addresses.insert( match self.db.addresses_mut().insert(
announcer, announcer,
*features, *features,
ann.alias.clone(), ann.alias.clone(),
@ -1426,7 +1492,8 @@ where
(session::State::Connected { .. }, Message::Subscribe(subscribe)) => { (session::State::Connected { .. }, Message::Subscribe(subscribe)) => {
// Filter announcements by interest. // Filter announcements by interest.
match self match self
.gossip .db
.gossip()
.filtered(&subscribe.filter, subscribe.since, subscribe.until) .filtered(&subscribe.filter, subscribe.since, subscribe.until)
{ {
Ok(anns) => { Ok(anns) => {
@ -1507,7 +1574,7 @@ where
// //
// If this is our first connection to the network, we just ask for a fixed backlog // If this is our first connection to the network, we just ask for a fixed backlog
// of messages to get us started. // of messages to get us started.
let since = match self.gossip.last() { let since = match self.db.gossip().last() {
Ok(Some(last)) => last - MAX_TIME_DELTA.as_millis() as Timestamp, Ok(Some(last)) => last - MAX_TIME_DELTA.as_millis() as Timestamp,
Ok(None) => (*now - INITIAL_SUBSCRIBE_BACKLOG_DELTA).as_millis() as Timestamp, Ok(None) => (*now - INITIAL_SUBSCRIBE_BACKLOG_DELTA).as_millis() as Timestamp,
Err(e) => { Err(e) => {
@ -1545,7 +1612,7 @@ where
let mut synced = SyncedRouting::default(); let mut synced = SyncedRouting::default();
let included: HashSet<&Id> = HashSet::from_iter(inventory); let included: HashSet<&Id> = HashSet::from_iter(inventory);
for (rid, result) in self.routing.insert(inventory, from, timestamp)? { for (rid, result) in self.db.routing_mut().insert(inventory, from, timestamp)? {
match result { match result {
InsertResult::SeedAdded => { InsertResult::SeedAdded => {
info!(target: "service", "Routing table updated for {rid} with seed {from}"); info!(target: "service", "Routing table updated for {rid} with seed {from}");
@ -1565,9 +1632,9 @@ where
InsertResult::NotUpdated => {} InsertResult::NotUpdated => {}
} }
} }
for rid in self.routing.get_resources(&from)?.into_iter() { for rid in self.db.routing().get_resources(&from)?.into_iter() {
if !included.contains(&rid) { if !included.contains(&rid) {
if self.routing.remove(&rid, &from)? { if self.db.routing_mut().remove(&rid, &from)? {
synced.removed.push(rid); synced.removed.push(rid);
self.emitter.emit(Event::SeedDropped { rid, nid: from }); self.emitter.emit(Event::SeedDropped { rid, nid: from });
} }
@ -1622,7 +1689,8 @@ where
// the node was stopped. // the node was stopped.
if let Some(refs) = refs.iter().find(|r| r.remote == ann.node) { if let Some(refs) = refs.iter().find(|r| r.remote == ann.node) {
if let Err(e) = self if let Err(e) = self
.addresses .db
.seeds_mut()
.synced(&rid, &ann.node, refs.at, ann.timestamp()) .synced(&rid, &ann.node, refs.at, ann.timestamp())
{ {
error!(target: "service", "Error updating sync status for local node: {e}"); error!(target: "service", "Error updating sync status for local node: {e}");
@ -1634,7 +1702,7 @@ where
// Only announce to peers who are allowed to view this repo. // Only announce to peers who are allowed to view this repo.
doc.is_visible_to(&p.id) doc.is_visible_to(&p.id)
}), }),
&mut self.gossip, self.db.gossip_mut(),
); );
Ok(refs) Ok(refs)
@ -1682,8 +1750,9 @@ where
return false; return false;
} }
let persistent = self.config.is_persistent(&nid); let persistent = self.config.is_persistent(&nid);
let time = self.time();
if let Err(e) = self.addresses.attempted(&nid, &addr, self.time()) { if let Err(e) = self.db.addresses_mut().attempted(&nid, &addr, time) {
error!(target: "service", "Error updating address book with connection attempt: {e}"); error!(target: "service", "Error updating address book with connection attempt: {e}");
} }
self.sessions.insert( self.sessions.insert(
@ -1709,7 +1778,7 @@ where
// our own refs. // our own refs.
if let Ok(repo) = self.storage.repository(*rid) { if let Ok(repo) = self.storage.repository(*rid) {
if let Ok(local) = RefsAt::new(&repo, self.node_id()) { if let Ok(local) = RefsAt::new(&repo, self.node_id()) {
for seed in self.addresses.seeds(rid)? { for seed in self.db.seeds().seeds_for(rid)? {
let seed = seed?; let seed = seed?;
let state = self.sessions.get(&seed.nid).map(|s| s.state.clone()); let state = self.sessions.get(&seed.nid).map(|s| s.state.clone());
let synced = if local.at == seed.synced_at.oid { let synced = if local.at == seed.synced_at.oid {
@ -1730,7 +1799,7 @@ where
// Then, add peers we know about but have no information about the sync status. // Then, add peers we know about but have no information about the sync status.
// These peers have announced that they track the repository via an inventory // These peers have announced that they track the repository via an inventory
// announcement, but we haven't received any ref announcements from them. // announcement, but we haven't received any ref announcements from them.
for nid in self.routing.get(rid)? { for nid in self.db.routing().get(rid)? {
if nid == self.node_id() { if nid == self.node_id() {
continue; continue;
} }
@ -1738,7 +1807,7 @@ where
// We already have a richer entry for this node. // We already have a richer entry for this node.
continue; continue;
} }
let addrs = self.addresses.addresses(&nid)?; let addrs = self.db.addresses().addresses_of(&nid)?;
let state = self.sessions.get(&nid).map(|s| s.state.clone()); let state = self.sessions.get(&nid).map(|s| s.state.clone());
seeds.insert(Seed::new(nid, addrs, state, None)); seeds.insert(Seed::new(nid, addrs, state, None));
@ -1773,19 +1842,19 @@ where
self.outbox.announce( self.outbox.announce(
msg.signed(&self.signer), msg.signed(&self.signer),
self.sessions.connected().map(|(_, p)| p), self.sessions.connected().map(|(_, p)| p),
&mut self.gossip, self.db.gossip_mut(),
); );
Ok(()) Ok(())
} }
fn prune_routing_entries(&mut self, now: &LocalTime) -> Result<(), routing::Error> { fn prune_routing_entries(&mut self, now: &LocalTime) -> Result<(), routing::Error> {
let count = self.routing.len()?; let count = self.db.routing().len()?;
if count <= self.config.limits.routing_max_size { if count <= self.config.limits.routing_max_size {
return Ok(()); return Ok(());
} }
let delta = count - self.config.limits.routing_max_size; let delta = count - self.config.limits.routing_max_size;
self.routing.prune( self.db.routing_mut().prune(
(*now - self.config.limits.routing_max_age).as_millis(), (*now - self.config.limits.routing_max_age).as_millis(),
Some(delta), Some(delta),
)?; )?;
@ -1825,7 +1894,7 @@ where
/// Get a list of peers available to connect to. /// Get a list of peers available to connect to.
fn available_peers(&mut self) -> HashMap<NodeId, Vec<KnownAddress>> { fn available_peers(&mut self) -> HashMap<NodeId, Vec<KnownAddress>> {
match self.addresses.entries() { match self.db.addresses().entries() {
Ok(entries) => { Ok(entries) => {
// Nb. we don't want to connect to any peers that already have a session with us, // Nb. we don't want to connect to any peers that already have a session with us,
// even if it's in a disconnected state. Those sessions are re-attempted automatically. // even if it's in a disconnected state. Those sessions are re-attempted automatically.
@ -1969,9 +2038,9 @@ pub trait ServiceState {
fn config(&self) -> &Config; fn config(&self) -> &Config;
} }
impl<R, A, S, G> ServiceState for Service<R, A, S, G> impl<D, S, G> ServiceState for Service<D, S, G>
where where
R: routing::Store, D: routing::Store,
G: Signer, G: Signer,
S: ReadStorage, S: ReadStorage,
{ {

View File

@ -3,7 +3,7 @@ pub mod store;
use super::*; use super::*;
pub use store::Error; pub use store::Error;
pub use store::GossipStore as Store; pub use store::Store;
pub fn node(config: &Config, timestamp: Timestamp) -> NodeAnnouncement { pub fn node(config: &Config, timestamp: Timestamp) -> NodeAnnouncement {
let features = config.features(); let features = config.features();

View File

@ -1,10 +1,10 @@
use std::{fmt, io, path::Path}; use std::{fmt, io};
use radicle::crypto::Signature; use radicle::crypto::Signature;
use sqlite as sql; use sqlite as sql;
use thiserror::Error; use thiserror::Error;
use crate::node::NodeId; use crate::node::{Database, NodeId};
use crate::prelude::{Filter, Timestamp}; use crate::prelude::{Filter, Timestamp};
use crate::service::message::{ use crate::service::message::{
Announcement, AnnouncementMessage, InventoryAnnouncement, NodeAnnouncement, RefsAnnouncement, Announcement, AnnouncementMessage, InventoryAnnouncement, NodeAnnouncement, RefsAnnouncement,
@ -14,39 +14,42 @@ use crate::wire::Decode;
#[derive(Error, Debug)] #[derive(Error, Debug)]
pub enum Error { pub enum Error {
/// I/O error.
#[error("i/o error: {0}")]
Io(#[from] io::Error),
/// An Internal error. /// An Internal error.
#[error("internal error: {0}")] #[error("internal error: {0}")]
Internal(#[from] sql::Error), Internal(#[from] sql::Error),
} }
/// A database that has access to historical gossip messages.
/// Keeps track of the latest received gossip messages for each node. /// Keeps track of the latest received gossip messages for each node.
/// Grows linearly with the number of nodes on the network. /// Grows linearly with the number of nodes on the network.
pub struct GossipStore { pub trait Store {
db: sql::Connection,
}
impl fmt::Debug for GossipStore {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("GossipStore").finish()
}
}
impl GossipStore {
/// Open a gossip store at the given path. Creates a new store if it doesn't exist.
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self, Error> {
let db = sql::Connection::open_with_flags(
path,
sqlite::OpenFlags::new().with_read_write().with_full_mutex(),
)?;
Ok(Self { db })
}
/// Prune announcements older than the cutoff time. /// Prune announcements older than the cutoff time.
pub fn prune(&mut self, cutoff: Timestamp) -> Result<usize, Error> { fn prune(&mut self, cutoff: Timestamp) -> Result<usize, Error>;
/// Get the timestamp of the last announcement in the store.
fn last(&self) -> Result<Option<Timestamp>, Error>;
/// Process an announcement for the given node.
/// Returns `true` if the timestamp was updated or the announcement wasn't there before.
fn announced(&mut self, nid: &NodeId, ann: &Announcement) -> Result<bool, Error>;
/// Get all the latest gossip messages of all nodes, filtered by inventory filter and
/// announcement timestamps.
///
/// # Panics
///
/// Panics if `from` > `to`.
///
fn filtered<'a>(
&'a self,
filter: &'a Filter,
from: Timestamp,
to: Timestamp,
) -> Result<Box<dyn Iterator<Item = Result<Announcement, Error>> + 'a>, Error>;
}
impl Store for Database {
fn prune(&mut self, cutoff: Timestamp) -> Result<usize, Error> {
let mut stmt = self let mut stmt = self
.db .db
.prepare("DELETE FROM `announcements` WHERE timestamp < ?1")?; .prepare("DELETE FROM `announcements` WHERE timestamp < ?1")?;
@ -57,8 +60,7 @@ impl GossipStore {
Ok(self.db.change_count()) Ok(self.db.change_count())
} }
/// Get the timestamp of the last announcement in the store. fn last(&self) -> Result<Option<Timestamp>, Error> {
pub fn last(&self) -> Result<Option<Timestamp>, Error> {
let stmt = self let stmt = self
.db .db
.prepare("SELECT MAX(timestamp) AS latest FROM `announcements`")?; .prepare("SELECT MAX(timestamp) AS latest FROM `announcements`")?;
@ -71,9 +73,7 @@ impl GossipStore {
Ok(None) Ok(None)
} }
/// Process an announcement for the given node. fn announced(&mut self, nid: &NodeId, ann: &Announcement) -> Result<bool, Error> {
/// Returns `true` if the timestamp was updated or the announcement wasn't there before.
pub fn announced(&mut self, nid: &NodeId, ann: &Announcement) -> Result<bool, Error> {
let mut stmt = self.db.prepare( let mut stmt = self.db.prepare(
"INSERT INTO `announcements` (node, repo, type, message, signature, timestamp) "INSERT INTO `announcements` (node, repo, type, message, signature, timestamp)
VALUES (?1, ?2, ?3, ?4, ?5, ?6) VALUES (?1, ?2, ?3, ?4, ?5, ?6)
@ -107,19 +107,12 @@ impl GossipStore {
Ok(self.db.change_count() > 0) Ok(self.db.change_count() > 0)
} }
/// Get all the latest gossip messages of all nodes, filtered by inventory filter and fn filtered<'a>(
/// announcement timestamps.
///
/// # Panics
///
/// Panics if `from` > `to`.
///
pub fn filtered<'a>(
&'a self, &'a self,
filter: &'a Filter, filter: &'a Filter,
from: Timestamp, from: Timestamp,
to: Timestamp, to: Timestamp,
) -> Result<impl Iterator<Item = Result<Announcement, Error>> + 'a, Error> { ) -> Result<Box<dyn Iterator<Item = Result<Announcement, Error>> + 'a>, Error> {
let mut stmt = self.db.prepare( let mut stmt = self.db.prepare(
"SELECT node, type, message, signature, timestamp "SELECT node, type, message, signature, timestamp
FROM announcements FROM announcements
@ -131,41 +124,42 @@ impl GossipStore {
stmt.bind((1, i64::try_from(from).unwrap_or(i64::MAX)))?; stmt.bind((1, i64::try_from(from).unwrap_or(i64::MAX)))?;
stmt.bind((2, i64::try_from(to).unwrap_or(i64::MAX)))?; stmt.bind((2, i64::try_from(to).unwrap_or(i64::MAX)))?;
Ok(stmt Ok(Box::new(
.into_iter() stmt.into_iter()
.map(|row| { .map(|row| {
let row = row?; let row = row?;
let node = row.read::<NodeId, _>("node"); let node = row.read::<NodeId, _>("node");
let gt = row.read::<GossipType, _>("type"); let gt = row.read::<GossipType, _>("type");
let message = match gt { let message = match gt {
GossipType::Refs => { GossipType::Refs => {
let ann = row.read::<RefsAnnouncement, _>("message"); let ann = row.read::<RefsAnnouncement, _>("message");
AnnouncementMessage::Refs(ann) AnnouncementMessage::Refs(ann)
} }
GossipType::Inventory => { GossipType::Inventory => {
let ann = row.read::<InventoryAnnouncement, _>("message"); let ann = row.read::<InventoryAnnouncement, _>("message");
AnnouncementMessage::Inventory(ann) AnnouncementMessage::Inventory(ann)
} }
GossipType::Node => { GossipType::Node => {
let ann = row.read::<NodeAnnouncement, _>("message"); let ann = row.read::<NodeAnnouncement, _>("message");
AnnouncementMessage::Node(ann) AnnouncementMessage::Node(ann)
} }
}; };
let signature = row.read::<Signature, _>("signature"); let signature = row.read::<Signature, _>("signature");
let timestamp = row.read::<i64, _>("timestamp"); let timestamp = row.read::<i64, _>("timestamp");
debug_assert_eq!(timestamp, message.timestamp() as i64); debug_assert_eq!(timestamp, message.timestamp() as i64);
Ok(Announcement { Ok(Announcement {
node, node,
message, message,
signature, signature,
})
}) })
}) .filter(|ann| match ann {
.filter(|ann| match ann { Ok(a) => a.matches(filter),
Ok(a) => a.matches(filter), Err(_) => true,
Err(_) => true, }),
})) ))
} }
} }

View File

@ -68,7 +68,7 @@ impl Outbox {
&mut self, &mut self,
ann: Announcement, ann: Announcement,
peers: impl Iterator<Item = &'a Session>, peers: impl Iterator<Item = &'a Session>,
gossip: &mut gossip::Store, gossip: &mut impl gossip::Store,
) { ) {
// Store our announcement so that it can be retrieved from us later, just like // Store our announcement so that it can be retrieved from us later, just like
// announcements we receive from peers. // announcements we receive from peers.

View File

@ -18,11 +18,10 @@ use radicle::crypto::{KeyPair, Seed, Signer};
use radicle::git; use radicle::git;
use radicle::git::refname; use radicle::git::refname;
use radicle::identity::{Id, Visibility}; use radicle::identity::{Id, Visibility};
use radicle::node::address::Book;
use radicle::node::routing;
use radicle::node::routing::Store; use radicle::node::routing::Store;
use radicle::node::tracking::store as tracking; use radicle::node::tracking::store as tracking;
use radicle::node::{Alias, ADDRESS_DB_FILE, ROUTING_DB_FILE, TRACKING_DB_FILE}; use radicle::node::Database;
use radicle::node::{Alias, TRACKING_DB_FILE};
use radicle::node::{ConnectOptions, Handle as _}; use radicle::node::{ConnectOptions, Handle as _};
use radicle::profile; use radicle::profile;
use radicle::profile::Home; use radicle::profile::Home;
@ -84,20 +83,15 @@ impl Environment {
let tracking_db = profile.home.node().join(TRACKING_DB_FILE); let tracking_db = profile.home.node().join(TRACKING_DB_FILE);
let tracking = tracking::Config::open(tracking_db).unwrap(); let tracking = tracking::Config::open(tracking_db).unwrap();
let db = profile.database_mut().unwrap();
let routing_db = profile.home.node().join(ROUTING_DB_FILE); let db = service::Stores::from(db);
let routing = routing::Table::open(routing_db).unwrap();
let addresses_db = profile.home.node().join(ADDRESS_DB_FILE);
let addresses = Book::open(addresses_db).unwrap();
Node { Node {
id: *profile.id(), id: *profile.id(),
home: profile.home, home: profile.home,
config, config,
signer, signer,
addresses, db,
routing,
tracking, tracking,
storage: profile.storage, storage: profile.storage,
} }
@ -129,8 +123,7 @@ impl Environment {
.unwrap(); .unwrap();
tracking::Config::open(tracking_db).unwrap(); tracking::Config::open(tracking_db).unwrap();
let addresses_db = home.node().join(ADDRESS_DB_FILE); home.database_mut().unwrap(); // Just create the database.
Book::open(addresses_db).unwrap();
transport::local::register(storage.clone()); transport::local::register(storage.clone());
keystore.store(keypair.clone(), "radicle", None).unwrap(); keystore.store(keypair.clone(), "radicle", None).unwrap();
@ -155,8 +148,7 @@ pub struct Node<G> {
pub signer: G, pub signer: G,
pub storage: Storage, pub storage: Storage,
pub config: Config, pub config: Config,
pub addresses: Book, pub db: service::Stores<Database>,
pub routing: routing::Table,
pub tracking: tracking::Config<tracking::Write>, pub tracking: tracking::Config<tracking::Write>,
} }
@ -217,7 +209,7 @@ impl<G: Signer + cyphernet::Ecdh> NodeHandle<G> {
/// Get routing table entries. /// Get routing table entries.
pub fn routing(&self) -> impl Iterator<Item = (Id, NodeId)> { pub fn routing(&self) -> impl Iterator<Item = (Id, NodeId)> {
radicle::node::routing::Table::reader(self.home.node().join(radicle::node::ROUTING_DB_FILE)) Database::reader(self.home.node().join(node::NODE_DB_FILE))
.unwrap() .unwrap()
.entries() .entries()
.unwrap() .unwrap()
@ -356,9 +348,9 @@ impl Node<MockSigner> {
}, },
) )
.unwrap(); .unwrap();
let addresses = home.addresses_mut().unwrap();
let tracking = home.tracking_mut().unwrap(); let tracking = home.tracking_mut().unwrap();
let routing = home.routing_mut().unwrap(); let db = home.database_mut().unwrap();
let db = service::Stores::from(db);
log::debug!(target: "test", "Node::init {}: {}", config.alias, signer.public_key()); log::debug!(target: "test", "Node::init {}: {}", config.alias, signer.public_key());
Self { Self {
@ -367,9 +359,8 @@ impl Node<MockSigner> {
signer, signer,
storage, storage,
config, config,
addresses, db,
tracking, tracking,
routing,
} }
} }
} }

View File

@ -7,7 +7,8 @@ use std::str::FromStr;
use log::*; use log::*;
use radicle::identity::Visibility; use radicle::identity::Visibility;
use radicle::node::address::Store; use radicle::node::address::Store as _;
use radicle::node::Database;
use radicle::node::{address, Alias, ConnectOptions}; use radicle::node::{address, Alias, ConnectOptions};
use radicle::rad; use radicle::rad;
use radicle::storage::refs::RefsAt; use radicle::storage::refs::RefsAt;
@ -18,7 +19,6 @@ use crate::crypto::test::signer::MockSigner;
use crate::crypto::Signer; use crate::crypto::Signer;
use crate::identity::Id; use crate::identity::Id;
use crate::node; use crate::node;
use crate::node::routing;
use crate::prelude::*; use crate::prelude::*;
use crate::runtime::Emitter; use crate::runtime::Emitter;
use crate::service; use crate::service;
@ -36,7 +36,7 @@ use crate::Link;
use crate::{LocalDuration, LocalTime}; use crate::{LocalDuration, LocalTime};
/// Service instantiation used for testing. /// Service instantiation used for testing.
pub type Service<S, G> = service::Service<routing::Table, address::Book, S, G>; pub type Service<S, G> = service::Service<Database, S, G>;
#[derive(Debug)] #[derive(Debug)]
pub struct Peer<S, G> { pub struct Peer<S, G> {
@ -100,8 +100,7 @@ where
pub struct Config<G: Signer + 'static> { pub struct Config<G: Signer + 'static> {
pub config: service::Config, pub config: service::Config,
pub addrs: address::Book, pub db: Stores<node::Database>,
pub gossip: gossip::Store,
pub local_time: LocalTime, pub local_time: LocalTime,
pub policy: Policy, pub policy: Policy,
pub scope: Scope, pub scope: Scope,
@ -115,13 +114,13 @@ impl Default for Config<MockSigner> {
let mut rng = fastrand::Rng::new(); let mut rng = fastrand::Rng::new();
let signer = MockSigner::new(&mut rng); let signer = MockSigner::new(&mut rng);
let tmp = tempfile::TempDir::new().unwrap(); let tmp = tempfile::TempDir::new().unwrap();
let addrs = address::Book::open(tmp.path().join("addresses.db")).unwrap(); let db = Database::open(tmp.path().join(node::NODE_DB_FILE))
let gossip = gossip::Store::open(tmp.path().join("addresses.db")).unwrap(); .unwrap()
.into();
Config { Config {
config: service::Config::test(Alias::from_str("mocky").unwrap()), config: service::Config::test(Alias::from_str("mocky").unwrap()),
addrs, db,
gossip,
local_time: LocalTime::now(), local_time: LocalTime::now(),
policy: Policy::default(), policy: Policy::default(),
scope: Scope::default(), scope: Scope::default(),
@ -163,7 +162,6 @@ where
storage: S, storage: S,
mut config: Config<G>, mut config: Config<G>,
) -> Self { ) -> Self {
let routing = routing::Table::memory().unwrap();
let tracking = tracking::Store::<tracking::store::Write>::memory().unwrap(); let tracking = tracking::Store::<tracking::store::Write>::memory().unwrap();
let mut tracking = tracking::Config::new(config.policy, config.scope, tracking); let mut tracking = tracking::Config::new(config.policy, config.scope, tracking);
let id = *config.signer.public_key(); let id = *config.signer.public_key();
@ -181,10 +179,8 @@ where
let service = Service::new( let service = Service::new(
config.config, config.config,
config.local_time, config.local_time,
routing, config.db,
storage, storage,
config.addrs,
config.gossip,
tracking, tracking,
config.signer, config.signer,
config.rng.clone(), config.rng.clone(),
@ -236,8 +232,9 @@ where
pub fn import_addresses<'a>(&mut self, peers: impl IntoIterator<Item = &'a Self>) { pub fn import_addresses<'a>(&mut self, peers: impl IntoIterator<Item = &'a Self>) {
let timestamp = self.timestamp(); let timestamp = self.timestamp();
for peer in peers.into_iter() { for peer in peers.into_iter() {
let known_address = address::KnownAddress::new(peer.address(), address::Source::Peer); let known_address = node::KnownAddress::new(peer.address(), address::Source::Peer);
self.service self.service
.database_mut()
.addresses_mut() .addresses_mut()
.insert( .insert(
&peer.node_id(), &peer.node_id(),

View File

@ -273,7 +273,7 @@ fn test_inventory_sync() {
); );
for proj in &projs { for proj in &projs {
let seeds = alice.routing().get(proj).unwrap(); let seeds = alice.database().routing().get(proj).unwrap();
assert!(seeds.contains(&bob.node_id())); assert!(seeds.contains(&bob.node_id()));
} }
} }
@ -382,7 +382,7 @@ fn test_inventory_pruning() {
assert_eq!( assert_eq!(
test.expected_routing_table_size, test.expected_routing_table_size,
alice.routing().len().unwrap() alice.database().routing().len().unwrap()
); );
} }
} }

View File

@ -22,7 +22,7 @@ use netservices::{NetConnection, NetProtocol, NetReader, NetWriter};
use reactor::Timestamp; use reactor::Timestamp;
use radicle::collections::RandomMap; use radicle::collections::RandomMap;
use radicle::node::{address, routing, NodeId}; use radicle::node::NodeId;
use radicle::storage::WriteStorage; use radicle::storage::WriteStorage;
use crate::crypto::Signer; use crate::crypto::Signer;
@ -316,9 +316,9 @@ impl Peers {
} }
/// Wire protocol implementation for a set of peers. /// Wire protocol implementation for a set of peers.
pub struct Wire<R, S, W, G: Signer + Ecdh> { pub struct Wire<D, S, G: Signer + Ecdh> {
/// Backing service instance. /// Backing service instance.
service: Service<R, S, W, G>, service: Service<D, S, G>,
/// Worker pool interface. /// Worker pool interface.
worker: chan::Sender<Task>, worker: chan::Sender<Task>,
/// Used for authentication. /// Used for authentication.
@ -331,15 +331,14 @@ pub struct Wire<R, S, W, G: Signer + Ecdh> {
proxy: net::SocketAddr, proxy: net::SocketAddr,
} }
impl<R, S, W, G> Wire<R, S, W, G> impl<D, S, G> Wire<D, S, G>
where where
R: routing::Store, D: service::Store,
S: address::Store, S: WriteStorage + 'static,
W: WriteStorage + 'static,
G: Signer + Ecdh<Pk = NodeId>, G: Signer + Ecdh<Pk = NodeId>,
{ {
pub fn new( pub fn new(
mut service: Service<R, S, W, G>, mut service: Service<D, S, G>,
worker: chan::Sender<Task>, worker: chan::Sender<Task>,
signer: G, signer: G,
proxy: net::SocketAddr, proxy: net::SocketAddr,
@ -450,11 +449,10 @@ where
} }
} }
impl<R, S, W, G> reactor::Handler for Wire<R, S, W, G> impl<D, S, G> reactor::Handler for Wire<D, S, G>
where where
R: routing::Store + Send, D: service::Store + Send,
S: address::Store + Send, S: WriteStorage + Send + 'static,
W: WriteStorage + Send + 'static,
G: Signer + Ecdh<Pk = NodeId> + Clone + Send, G: Signer + Ecdh<Pk = NodeId> + Clone + Send,
{ {
type Listener = NetAccept<WireSession<G>>; type Listener = NetAccept<WireSession<G>>;
@ -742,11 +740,10 @@ where
} }
} }
impl<R, S, W, G> Iterator for Wire<R, S, W, G> impl<D, S, G> Iterator for Wire<D, S, G>
where where
R: routing::Store, D: service::Store,
S: address::Store, S: WriteStorage + 'static,
W: WriteStorage + 'static,
G: Signer + Ecdh<Pk = NodeId>, G: Signer + Ecdh<Pk = NodeId>,
{ {
type Item = Action<G>; type Item = Action<G>;

View File

@ -9,6 +9,7 @@ extern crate amplify;
extern crate radicle_git_ext as git_ext; extern crate radicle_git_ext as git_ext;
mod canonical; mod canonical;
pub mod cli; pub mod cli;
pub mod cob; pub mod cob;
pub mod collections; pub mod collections;

View File

@ -2,8 +2,10 @@ mod features;
pub mod address; pub mod address;
pub mod config; pub mod config;
pub mod db;
pub mod events; pub mod events;
pub mod routing; pub mod routing;
pub mod seed;
pub mod tracking; pub mod tracking;
use std::collections::{BTreeSet, HashMap, HashSet}; use std::collections::{BTreeSet, HashMap, HashSet};
@ -27,11 +29,13 @@ use crate::profile;
use crate::storage::refs::RefsAt; use crate::storage::refs::RefsAt;
use crate::storage::RefUpdate; use crate::storage::RefUpdate;
pub use address::{KnownAddress, SyncedAt}; pub use address::KnownAddress;
pub use config::Config; pub use config::Config;
pub use cyphernet::addr::{HostName, PeerAddr}; pub use cyphernet::addr::{HostName, PeerAddr};
pub use db::Database;
pub use events::{Event, Events}; pub use events::{Event, Events};
pub use features::Features; pub use features::Features;
pub use seed::SyncedAt;
/// Default name for control socket file. /// Default name for control socket file.
pub const DEFAULT_SOCKET_NAME: &str = "control.sock"; pub const DEFAULT_SOCKET_NAME: &str = "control.sock";
@ -41,10 +45,8 @@ pub const DEFAULT_PORT: u16 = 8776;
pub const DEFAULT_TIMEOUT: time::Duration = time::Duration::from_secs(9); pub const DEFAULT_TIMEOUT: time::Duration = time::Duration::from_secs(9);
/// Maximum length in bytes of a node alias. /// Maximum length in bytes of a node alias.
pub const MAX_ALIAS_LENGTH: usize = 32; pub const MAX_ALIAS_LENGTH: usize = 32;
/// Filename of routing table database under the node directory. /// Filename of node database under the node directory.
pub const ROUTING_DB_FILE: &str = "routing.db"; pub const NODE_DB_FILE: &str = "node.db";
/// Filename of address database under the node directory.
pub const ADDRESS_DB_FILE: &str = "addresses.db";
/// Filename of tracking table database under the node directory. /// Filename of tracking table database under the node directory.
pub const TRACKING_DB_FILE: &str = "tracking.db"; pub const TRACKING_DB_FILE: &str = "tracking.db";
/// Filename of last node announcement, when running in debug mode. /// Filename of last node announcement, when running in debug mode.
@ -462,13 +464,18 @@ impl Session {
} }
} }
/// A seed for some repository, with metadata about its status.
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub struct Seed { pub struct Seed {
/// The Node ID.
pub nid: NodeId, pub nid: NodeId,
/// Known addresses for this seed.
pub addrs: Vec<KnownAddress>, pub addrs: Vec<KnownAddress>,
/// The seed's session state, if any.
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub state: Option<State>, pub state: Option<State>,
/// The seed's sync status, if any.
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub sync: Option<SyncStatus>, pub sync: Option<SyncStatus>,
} }
@ -1069,18 +1076,6 @@ pub trait AliasStore {
fn alias(&self, nid: &NodeId) -> Option<Alias>; fn alias(&self, nid: &NodeId) -> Option<Alias>;
} }
impl<T: AliasStore + ?Sized> AliasStore for &T {
fn alias(&self, nid: &NodeId) -> Option<Alias> {
(*self).alias(nid)
}
}
impl<T: AliasStore + ?Sized> AliasStore for Box<T> {
fn alias(&self, nid: &NodeId) -> Option<Alias> {
self.deref().alias(nid)
}
}
impl AliasStore for HashMap<NodeId, Alias> { impl AliasStore for HashMap<NodeId, Alias> {
fn alias(&self, nid: &NodeId) -> Option<Alias> { fn alias(&self, nid: &NodeId) -> Option<Alias> {
self.get(nid).map(ToOwned::to_owned) self.get(nid).map(ToOwned::to_owned)

View File

@ -1,13 +1,190 @@
mod store; pub mod store;
mod types; pub use store::{Error, Store};
pub use store::*; use std::cell::RefCell;
pub use types::*; use std::ops::{Deref, DerefMut};
use std::{hash, net};
use cyphernet::addr::HostName; use cyphernet::addr::HostName;
use std::net; use localtime::LocalTime;
use nonempty::NonEmpty;
use crate::node::Address; use crate::collections::RandomMap;
use crate::node::{Address, Alias};
use crate::prelude::Timestamp;
use crate::{node, profile};
/// A map with the ability to randomly select values.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
#[serde(transparent)]
pub struct AddressBook<K: hash::Hash + Eq, V> {
inner: RandomMap<K, V>,
#[serde(skip)]
rng: RefCell<fastrand::Rng>,
}
impl<K: hash::Hash + Eq, V> AddressBook<K, V> {
/// Create a new address book.
pub fn new(rng: fastrand::Rng) -> Self {
Self {
inner: RandomMap::with_hasher(rng.clone().into()),
rng: RefCell::new(rng),
}
}
/// Pick a random value in the book.
pub fn sample(&self) -> Option<(&K, &V)> {
self.sample_with(|_, _| true)
}
/// Pick a random value in the book matching a predicate.
pub fn sample_with(&self, mut predicate: impl FnMut(&K, &V) -> bool) -> Option<(&K, &V)> {
if let Some(pairs) = NonEmpty::from_vec(
self.inner
.iter()
.filter(|(k, v)| predicate(*k, *v))
.collect(),
) {
let ix = self.rng.borrow_mut().usize(..pairs.len());
let pair = pairs[ix]; // Can't fail.
Some(pair)
} else {
None
}
}
/// Return a new address book with the given RNG.
pub fn with(self, rng: fastrand::Rng) -> Self {
Self {
inner: self.inner,
rng: RefCell::new(rng),
}
}
}
impl<K: hash::Hash + Eq + Ord + Copy, V> AddressBook<K, V> {
/// Return a shuffled iterator.
pub fn shuffled(&self) -> std::vec::IntoIter<(&K, &V)> {
let mut items = self.inner.iter().collect::<Vec<_>>();
items.sort_by_key(|(k, _)| *k);
self.rng.borrow_mut().shuffle(&mut items);
items.into_iter()
}
/// Turn this object into a shuffled iterator.
pub fn into_shuffled(self) -> impl Iterator<Item = (K, V)> {
let mut items = self.inner.into_iter().collect::<Vec<_>>();
items.sort_by_key(|(k, _)| *k);
self.rng.borrow_mut().shuffle(&mut items);
items.into_iter()
}
/// Cycle through the keys at random. The random cycle repeats ad-infintum.
pub fn cycle(&self) -> impl Iterator<Item = &K> {
self.shuffled().map(|(k, _)| k).cycle()
}
}
impl<K: hash::Hash + Eq, V> FromIterator<(K, V)> for AddressBook<K, V> {
fn from_iter<T: IntoIterator<Item = (K, V)>>(iter: T) -> Self {
let rng = profile::env::rng();
let mut inner = RandomMap::with_hasher(rng.clone().into());
for (k, v) in iter {
inner.insert(k, v);
}
Self {
inner,
rng: RefCell::new(rng),
}
}
}
impl<K: hash::Hash + Eq, V> Deref for AddressBook<K, V> {
type Target = RandomMap<K, V>;
fn deref(&self) -> &Self::Target {
&self.inner
}
}
impl<K: hash::Hash + Eq, V> DerefMut for AddressBook<K, V> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.inner
}
}
/// Node public data.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Node {
/// Advertized alias.
pub alias: Alias,
/// Advertized features.
pub features: node::Features,
/// Advertized addresses
pub addrs: Vec<KnownAddress>,
/// Proof-of-work included in node announcement.
pub pow: u32,
/// When this data was published.
pub timestamp: Timestamp,
}
/// A known address.
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct KnownAddress {
/// Network address.
pub addr: Address,
/// Address of the peer who sent us this address.
pub source: Source,
/// Last time this address was used to successfully connect to a peer.
#[serde(with = "crate::serde_ext::localtime::option::time")]
pub last_success: Option<LocalTime>,
/// Last time this address was tried.
#[serde(with = "crate::serde_ext::localtime::option::time")]
pub last_attempt: Option<LocalTime>,
/// Whether this address has been banned.
pub banned: bool,
}
impl KnownAddress {
/// Create a new known address.
pub fn new(addr: Address, source: Source) -> Self {
Self {
addr,
source,
last_success: None,
last_attempt: None,
banned: false,
}
}
}
/// Address source. Specifies where an address originated from.
#[derive(Debug, Copy, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum Source {
/// An address that was shared by another peer.
Peer,
/// An bootstrap node address.
Bootstrap,
/// An address that came from some source external to the system, eg.
/// specified by the user or added directly to the address manager.
Imported,
}
impl std::fmt::Display for Source {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Peer => write!(f, "Peer"),
Self::Bootstrap => write!(f, "Bootstrap"),
Self::Imported => write!(f, "Imported"),
}
}
}
/// Address type. /// Address type.
#[repr(u8)] #[repr(u8)]

View File

@ -1,93 +1,64 @@
#![allow(clippy::type_complexity)]
use std::path::Path;
use std::str::FromStr; use std::str::FromStr;
use std::{fmt, io};
use localtime::LocalTime; use localtime::LocalTime;
use sqlite as sql; use sqlite as sql;
use thiserror::Error; use thiserror::Error;
use crate::git::Oid;
use crate::node; use crate::node;
use crate::node::address::{KnownAddress, Source}; use crate::node::address::{AddressType, KnownAddress, Node, Source};
use crate::node::{Address, Alias, AliasError, AliasStore, NodeId}; use crate::node::{Address, Alias, AliasError, AliasStore, Database, NodeId};
use crate::prelude::{Id, Timestamp}; use crate::prelude::Timestamp;
use crate::sql::transaction; use crate::sql::transaction;
use super::types;
use super::{AddressType, SyncedAt};
#[derive(Error, Debug)] #[derive(Error, Debug)]
pub enum Error { pub enum Error {
/// I/O error.
#[error("i/o error: {0}")]
Io(#[from] io::Error),
#[error("alias error: {0}")]
InvalidAlias(#[from] AliasError),
/// An Internal error. /// An Internal error.
#[error("internal error: {0}")] #[error("internal error: {0}")]
Internal(#[from] sql::Error), Internal(#[from] sql::Error),
#[error("alias error: {0}")]
InvalidAlias(#[from] AliasError),
} }
/// A file-backed address book. /// Address store.
pub struct Book { ///
db: sql::Connection, /// Used to store node addresses and metadata.
pub trait Store {
/// Get the information we have about a node.
fn get(&self, id: &NodeId) -> Result<Option<Node>, Error>;
/// Get the addresses of a node.
fn addresses_of(&self, node: &NodeId) -> Result<Vec<KnownAddress>, Error>;
/// Insert a node with associated addresses into the store.
///
/// Returns `true` if the node or addresses were updated, and `false` otherwise.
fn insert(
&mut self,
node: &NodeId,
features: node::Features,
alias: Alias,
pow: u32,
timestamp: Timestamp,
addrs: impl IntoIterator<Item = KnownAddress>,
) -> Result<bool, Error>;
/// Remove a node from the store.
fn remove(&mut self, id: &NodeId) -> Result<bool, Error>;
/// Returns the number of addresses.
fn len(&self) -> Result<usize, Error>;
/// Returns true if there are no addresses.
fn is_empty(&self) -> Result<bool, Error> {
self.len().map(|l| l == 0)
}
/// Get the address entries in the store.
fn entries(&self) -> Result<Box<dyn Iterator<Item = (NodeId, KnownAddress)>>, Error>;
/// Mark a node as attempted at a certain time.
fn attempted(&self, nid: &NodeId, addr: &Address, time: Timestamp) -> Result<(), Error>;
/// Mark a node as successfully connected at a certain time.
fn connected(&self, nid: &NodeId, addr: &Address, time: Timestamp) -> Result<(), Error>;
/// Mark a node as disconnected.
fn disconnected(&mut self, nid: &NodeId, addr: &Address, transient: bool) -> Result<(), Error>;
} }
impl fmt::Debug for Book { impl Store for Database {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn get(&self, node: &NodeId) -> Result<Option<Node>, Error> {
write!(f, "Book(..)")
}
}
impl From<sql::Connection> for Book {
fn from(db: sql::Connection) -> Self {
Self { db }
}
}
impl Book {
const SCHEMA: &'static str = include_str!("schema.sql");
const PRAGMA: &'static str = "PRAGMA foreign_keys = ON";
/// Open an address book at the given path. Creates a new address book if it
/// doesn't exist.
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self, Error> {
let db = sql::Connection::open_with_flags(
path,
sqlite::OpenFlags::new()
.with_create()
.with_read_write()
.with_full_mutex(),
)?;
db.execute(Self::PRAGMA)?;
db.execute(Self::SCHEMA)?;
Ok(Self { db })
}
/// Same as [`Self::open`], but in read-only mode. This is useful to have multiple
/// open databases, as no locking is required.
pub fn reader<P: AsRef<Path>>(path: P) -> Result<Self, Error> {
let db = sql::Connection::open_with_flags(path, sqlite::OpenFlags::new().with_read_only())?;
db.execute(Self::PRAGMA)?;
db.execute(Self::SCHEMA)?;
Ok(Self { db })
}
/// Create a new in-memory address book.
pub fn memory() -> Result<Self, Error> {
let db = sql::Connection::open(":memory:")?;
db.execute(Self::PRAGMA)?;
db.execute(Self::SCHEMA)?;
Ok(Self { db })
}
}
impl Store for Book {
fn get(&self, node: &NodeId) -> Result<Option<types::Node>, Error> {
let mut stmt = self let mut stmt = self
.db .db
.prepare("SELECT features, alias, pow, timestamp FROM nodes WHERE id = ?")?; .prepare("SELECT features, alias, pow, timestamp FROM nodes WHERE id = ?")?;
@ -99,9 +70,9 @@ impl Store for Book {
let alias = Alias::from_str(row.read::<&str, _>("alias"))?; let alias = Alias::from_str(row.read::<&str, _>("alias"))?;
let timestamp = row.read::<i64, _>("timestamp") as Timestamp; let timestamp = row.read::<i64, _>("timestamp") as Timestamp;
let pow = row.read::<i64, _>("pow") as u32; let pow = row.read::<i64, _>("pow") as u32;
let addrs = self.addresses(node)?; let addrs = self.addresses_of(node)?;
Ok(Some(types::Node { Ok(Some(Node {
features, features,
alias, alias,
pow, pow,
@ -113,7 +84,7 @@ impl Store for Book {
} }
} }
fn addresses(&self, node: &NodeId) -> Result<Vec<KnownAddress>, Error> { fn addresses_of(&self, node: &NodeId) -> Result<Vec<KnownAddress>, Error> {
let mut addrs = Vec::new(); let mut addrs = Vec::new();
let mut stmt = self.db.prepare( let mut stmt = self.db.prepare(
"SELECT type, value, source, last_attempt, last_success, banned FROM addresses WHERE node = ?", "SELECT type, value, source, last_attempt, last_success, banned FROM addresses WHERE node = ?",
@ -211,90 +182,6 @@ impl Store for Book {
Ok(self.db.change_count() > 0) Ok(self.db.change_count() > 0)
} }
fn synced(
&mut self,
rid: &Id,
nid: &NodeId,
at: Oid,
timestamp: Timestamp,
) -> Result<bool, Error> {
let mut stmt = self.db.prepare(
"INSERT INTO `repo-sync-status` (repo, node, head, timestamp)
VALUES (?1, ?2, ?3, ?4)
ON CONFLICT DO UPDATE
SET head = ?3, timestamp = ?4
WHERE timestamp < ?4",
)?;
stmt.bind((1, rid))?;
stmt.bind((2, nid))?;
stmt.bind((3, at.to_string().as_str()))?;
stmt.bind((4, timestamp as i64))?;
stmt.next()?;
Ok(self.db.change_count() > 0)
}
fn seeds(
&self,
rid: &Id,
) -> Result<Box<dyn Iterator<Item = Result<types::Seed, Error>> + '_>, Error> {
let mut stmt = self.db.prepare(
"SELECT node, head, timestamp
FROM `repo-sync-status`
WHERE repo = ?",
)?;
stmt.bind((1, rid))?;
Ok(Box::new(stmt.into_iter().map(|row| {
let row = row?;
let nid = row.try_read::<NodeId, _>("node")?;
let oid = row.try_read::<&str, _>("head")?;
let oid = Oid::from_str(oid).map_err(|e| {
Error::Internal(sql::Error {
code: None,
message: Some(format!("sql: invalid oid '{oid}': {e}")),
})
})?;
let timestamp = row.try_read::<i64, _>("timestamp")?;
let timestamp = LocalTime::from_millis(timestamp as u128);
let addresses = self.addresses(&nid)?;
Ok(types::Seed {
nid,
addresses,
synced_at: SyncedAt { oid, timestamp },
})
})))
}
fn seeded_by(
&self,
nid: &NodeId,
) -> Result<Box<dyn Iterator<Item = Result<(Id, SyncedAt), Error>> + '_>, Error> {
let mut stmt = self.db.prepare(
"SELECT repo, head, timestamp
FROM `repo-sync-status`
WHERE node = ?",
)?;
stmt.bind((1, nid))?;
Ok(Box::new(stmt.into_iter().map(|row| {
let row = row?;
let rid = row.try_read::<Id, _>("repo")?;
let oid = row.try_read::<&str, _>("head")?;
let oid = Oid::from_str(oid).map_err(|e| {
Error::Internal(sql::Error {
code: None,
message: Some(format!("sql: invalid oid '{oid}': {e}")),
})
})?;
let timestamp = row.try_read::<i64, _>("timestamp")?;
let timestamp = LocalTime::from_millis(timestamp as u128);
Ok((rid, SyncedAt { oid, timestamp }))
})))
}
fn entries(&self) -> Result<Box<dyn Iterator<Item = (NodeId, KnownAddress)>>, Error> { fn entries(&self) -> Result<Box<dyn Iterator<Item = (NodeId, KnownAddress)>>, Error> {
let mut stmt = self let mut stmt = self
.db .db
@ -381,7 +268,10 @@ impl Store for Book {
} }
} }
impl AliasStore for Book { impl<T> AliasStore for T
where
T: Store,
{
/// Retrieve `alias` of given node. /// Retrieve `alias` of given node.
/// Calls `Self::get` under the hood. /// Calls `Self::get` under the hood.
fn alias(&self, nid: &NodeId) -> Option<Alias> { fn alias(&self, nid: &NodeId) -> Option<Alias> {
@ -391,62 +281,6 @@ impl AliasStore for Book {
} }
} }
/// Address store.
///
/// Used to store node addresses and metadata.
pub trait Store {
/// Get the information we have about a node.
fn get(&self, id: &NodeId) -> Result<Option<types::Node>, Error>;
/// Get the addresses of a node.
fn addresses(&self, node: &NodeId) -> Result<Vec<KnownAddress>, Error>;
/// Insert a node with associated addresses into the store.
///
/// Returns `true` if the node or addresses were updated, and `false` otherwise.
fn insert(
&mut self,
node: &NodeId,
features: node::Features,
alias: Alias,
pow: u32,
timestamp: Timestamp,
addrs: impl IntoIterator<Item = KnownAddress>,
) -> Result<bool, Error>;
/// Remove a node from the store.
fn remove(&mut self, id: &NodeId) -> Result<bool, Error>;
/// Mark a repo as synced on the given node.
fn synced(
&mut self,
rid: &Id,
nid: &NodeId,
at: Oid,
timestamp: Timestamp,
) -> Result<bool, Error>;
/// Get nodes that have synced the given repo.
fn seeds(
&self,
rid: &Id,
) -> Result<Box<dyn Iterator<Item = Result<types::Seed, Error>> + '_>, Error>;
/// Get the repos seeded by the given node.
fn seeded_by(
&self,
nid: &NodeId,
) -> Result<Box<dyn Iterator<Item = Result<(Id, SyncedAt), Error>> + '_>, Error>;
/// Returns the number of addresses.
fn len(&self) -> Result<usize, Error>;
/// Returns true if there are no addresses.
fn is_empty(&self) -> Result<bool, Error> {
self.len().map(|l| l == 0)
}
/// Get the address entries in the store.
fn entries(&self) -> Result<Box<dyn Iterator<Item = (NodeId, KnownAddress)>>, Error>;
/// Mark a node as attempted at a certain time.
fn attempted(&self, nid: &NodeId, addr: &Address, time: Timestamp) -> Result<(), Error>;
/// Mark a node as successfully connected at a certain time.
fn connected(&self, nid: &NodeId, addr: &Address, time: Timestamp) -> Result<(), Error>;
/// Mark a node as disconnected.
fn disconnected(&mut self, nid: &NodeId, addr: &Address, transient: bool) -> Result<(), Error>;
}
impl TryFrom<&sql::Value> for Source { impl TryFrom<&sql::Value> for Source {
type Error = sql::Error; type Error = sql::Error;
@ -521,7 +355,7 @@ mod test {
fn test_empty() { fn test_empty() {
let tmp = tempfile::tempdir().unwrap(); let tmp = tempfile::tempdir().unwrap();
let path = tmp.path().join("cache"); let path = tmp.path().join("cache");
let cache = Book::open(path).unwrap(); let cache = Database::open(path).unwrap();
assert!(cache.is_empty().unwrap()); assert!(cache.is_empty().unwrap());
} }
@ -529,7 +363,7 @@ mod test {
#[test] #[test]
fn test_get_none() { fn test_get_none() {
let alice = arbitrary::gen::<NodeId>(1); let alice = arbitrary::gen::<NodeId>(1);
let cache = Book::memory().unwrap(); let cache = Database::memory().unwrap();
let result = cache.get(&alice).unwrap(); let result = cache.get(&alice).unwrap();
assert!(result.is_none()); assert!(result.is_none());
@ -538,7 +372,7 @@ mod test {
#[test] #[test]
fn test_remove_nothing() { fn test_remove_nothing() {
let alice = arbitrary::gen::<NodeId>(1); let alice = arbitrary::gen::<NodeId>(1);
let mut cache = Book::memory().unwrap(); let mut cache = Database::memory().unwrap();
let removed = cache.remove(&alice).unwrap(); let removed = cache.remove(&alice).unwrap();
assert!(!removed); assert!(!removed);
@ -547,7 +381,7 @@ mod test {
#[test] #[test]
fn test_alias() { fn test_alias() {
let alice = arbitrary::gen::<NodeId>(1); let alice = arbitrary::gen::<NodeId>(1);
let mut cache = Book::memory().unwrap(); let mut cache = Database::memory().unwrap();
let features = node::Features::SEED; let features = node::Features::SEED;
let timestamp = LocalTime::now().as_millis(); let timestamp = LocalTime::now().as_millis();
@ -567,7 +401,7 @@ mod test {
#[test] #[test]
fn test_insert_and_get() { fn test_insert_and_get() {
let alice = arbitrary::gen::<NodeId>(1); let alice = arbitrary::gen::<NodeId>(1);
let mut cache = Book::memory().unwrap(); let mut cache = Database::memory().unwrap();
let features = node::Features::SEED; let features = node::Features::SEED;
let timestamp = LocalTime::now().as_millis(); let timestamp = LocalTime::now().as_millis();
@ -602,7 +436,7 @@ mod test {
#[test] #[test]
fn test_insert_duplicate() { fn test_insert_duplicate() {
let alice = arbitrary::gen::<NodeId>(1); let alice = arbitrary::gen::<NodeId>(1);
let mut cache = Book::memory().unwrap(); let mut cache = Database::memory().unwrap();
let features = node::Features::SEED; let features = node::Features::SEED;
let timestamp = LocalTime::now().as_millis(); let timestamp = LocalTime::now().as_millis();
let alias = Alias::new("alice"); let alias = Alias::new("alice");
@ -630,7 +464,7 @@ mod test {
#[test] #[test]
fn test_insert_and_update() { fn test_insert_and_update() {
let alice = arbitrary::gen::<NodeId>(1); let alice = arbitrary::gen::<NodeId>(1);
let mut cache = Book::memory().unwrap(); let mut cache = Database::memory().unwrap();
let timestamp = LocalTime::now().as_millis(); let timestamp = LocalTime::now().as_millis();
let features = node::Features::SEED; let features = node::Features::SEED;
let alias1 = Alias::new("alice"); let alias1 = Alias::new("alice");
@ -685,7 +519,7 @@ mod test {
fn test_insert_and_remove() { fn test_insert_and_remove() {
let alice = arbitrary::gen::<NodeId>(1); let alice = arbitrary::gen::<NodeId>(1);
let bob = arbitrary::gen::<NodeId>(1); let bob = arbitrary::gen::<NodeId>(1);
let mut cache = Book::memory().unwrap(); let mut cache = Database::memory().unwrap();
let timestamp = LocalTime::now().as_millis(); let timestamp = LocalTime::now().as_millis();
let features = node::Features::SEED; let features = node::Features::SEED;
let alice_alias = Alias::new("alice"); let alice_alias = Alias::new("alice");
@ -732,7 +566,7 @@ mod test {
fn test_entries() { fn test_entries() {
let ids = arbitrary::vec::<NodeId>(16); let ids = arbitrary::vec::<NodeId>(16);
let mut rng = fastrand::Rng::new(); let mut rng = fastrand::Rng::new();
let mut cache = Book::memory().unwrap(); let mut cache = Database::memory().unwrap();
let mut expected = Vec::new(); let mut expected = Vec::new();
let timestamp = LocalTime::now().as_millis(); let timestamp = LocalTime::now().as_millis();
let features = node::Features::SEED; let features = node::Features::SEED;

View File

@ -1,238 +0,0 @@
use std::cell::RefCell;
use std::hash;
use std::ops::{Deref, DerefMut};
use localtime::LocalTime;
use nonempty::NonEmpty;
use crate::collections::RandomMap;
use crate::git;
use crate::node::{Address, Alias};
use crate::prelude::{NodeId, Timestamp};
use crate::storage::{refs::RefsAt, ReadRepository, RemoteId};
use crate::{node, profile};
/// A map with the ability to randomly select values.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
#[serde(transparent)]
pub struct AddressBook<K: hash::Hash + Eq, V> {
inner: RandomMap<K, V>,
#[serde(skip)]
rng: RefCell<fastrand::Rng>,
}
impl<K: hash::Hash + Eq, V> AddressBook<K, V> {
/// Create a new address book.
pub fn new(rng: fastrand::Rng) -> Self {
Self {
inner: RandomMap::with_hasher(rng.clone().into()),
rng: RefCell::new(rng),
}
}
/// Pick a random value in the book.
pub fn sample(&self) -> Option<(&K, &V)> {
self.sample_with(|_, _| true)
}
/// Pick a random value in the book matching a predicate.
pub fn sample_with(&self, mut predicate: impl FnMut(&K, &V) -> bool) -> Option<(&K, &V)> {
if let Some(pairs) = NonEmpty::from_vec(
self.inner
.iter()
.filter(|(k, v)| predicate(*k, *v))
.collect(),
) {
let ix = self.rng.borrow_mut().usize(..pairs.len());
let pair = pairs[ix]; // Can't fail.
Some(pair)
} else {
None
}
}
/// Return a new address book with the given RNG.
pub fn with(self, rng: fastrand::Rng) -> Self {
Self {
inner: self.inner,
rng: RefCell::new(rng),
}
}
}
impl<K: hash::Hash + Eq + Ord + Copy, V> AddressBook<K, V> {
/// Return a shuffled iterator.
pub fn shuffled(&self) -> std::vec::IntoIter<(&K, &V)> {
let mut items = self.inner.iter().collect::<Vec<_>>();
items.sort_by_key(|(k, _)| *k);
self.rng.borrow_mut().shuffle(&mut items);
items.into_iter()
}
/// Turn this object into a shuffled iterator.
pub fn into_shuffled(self) -> impl Iterator<Item = (K, V)> {
let mut items = self.inner.into_iter().collect::<Vec<_>>();
items.sort_by_key(|(k, _)| *k);
self.rng.borrow_mut().shuffle(&mut items);
items.into_iter()
}
/// Cycle through the keys at random. The random cycle repeats ad-infintum.
pub fn cycle(&self) -> impl Iterator<Item = &K> {
self.shuffled().map(|(k, _)| k).cycle()
}
}
impl<K: hash::Hash + Eq, V> FromIterator<(K, V)> for AddressBook<K, V> {
fn from_iter<T: IntoIterator<Item = (K, V)>>(iter: T) -> Self {
let rng = profile::env::rng();
let mut inner = RandomMap::with_hasher(rng.clone().into());
for (k, v) in iter {
inner.insert(k, v);
}
Self {
inner,
rng: RefCell::new(rng),
}
}
}
impl<K: hash::Hash + Eq, V> Deref for AddressBook<K, V> {
type Target = RandomMap<K, V>;
fn deref(&self) -> &Self::Target {
&self.inner
}
}
impl<K: hash::Hash + Eq, V> DerefMut for AddressBook<K, V> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.inner
}
}
/// Node public data.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Node {
/// Advertized alias.
pub alias: Alias,
/// Advertized features.
pub features: node::Features,
/// Advertized addresses
pub addrs: Vec<KnownAddress>,
/// Proof-of-work included in node announcement.
pub pow: u32,
/// When this data was published.
pub timestamp: Timestamp,
}
/// A known address.
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct KnownAddress {
/// Network address.
pub addr: Address,
/// Address of the peer who sent us this address.
pub source: Source,
/// Last time this address was used to successfully connect to a peer.
#[serde(with = "crate::serde_ext::localtime::option::time")]
pub last_success: Option<LocalTime>,
/// Last time this address was tried.
#[serde(with = "crate::serde_ext::localtime::option::time")]
pub last_attempt: Option<LocalTime>,
/// Whether this address has been banned.
pub banned: bool,
}
impl KnownAddress {
/// Create a new known address.
pub fn new(addr: Address, source: Source) -> Self {
Self {
addr,
source,
last_success: None,
last_attempt: None,
banned: false,
}
}
}
/// Address source. Specifies where an address originated from.
#[derive(Debug, Copy, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum Source {
/// An address that was shared by another peer.
Peer,
/// An bootstrap node address.
Bootstrap,
/// An address that came from some source external to the system, eg.
/// specified by the user or added directly to the address manager.
Imported,
}
impl std::fmt::Display for Source {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Peer => write!(f, "Peer"),
Self::Bootstrap => write!(f, "Bootstrap"),
Self::Imported => write!(f, "Imported"),
}
}
}
/// Holds an oid and timestamp.
#[derive(Debug, Copy, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct SyncedAt {
/// Head of `rad/sigrefs`.
pub oid: git_ext::Oid,
/// When these refs were synced.
#[serde(with = "crate::serde_ext::localtime::time")]
pub timestamp: LocalTime,
}
impl SyncedAt {
/// Load a new [`SyncedAt`] for the given remote.
pub fn load<S: ReadRepository>(repo: &S, remote: RemoteId) -> Result<Self, git::ext::Error> {
let refs = RefsAt::new(repo, remote)?;
let oid = refs.at;
Self::new(oid, repo)
}
/// Create a new [`SyncedAt`] given an OID, by looking up the timestamp in the repo.
pub fn new<S: ReadRepository>(oid: git::ext::Oid, repo: &S) -> Result<Self, git::ext::Error> {
let timestamp = repo.commit(oid)?.time();
let timestamp = LocalTime::from_secs(timestamp.seconds() as u64);
Ok(Self { oid, timestamp })
}
}
impl Ord for SyncedAt {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.timestamp.cmp(&other.timestamp)
}
}
impl PartialOrd for SyncedAt {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
/// Seed of a specific repository.
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Seed {
/// The Node ID.
pub nid: NodeId,
/// Known addresses for this node.
pub addresses: Vec<KnownAddress>,
/// Sync information for a given repo.
pub synced_at: SyncedAt,
}

77
radicle/src/node/db.rs Normal file
View File

@ -0,0 +1,77 @@
use std::path::Path;
use std::{fmt, time};
use sqlite as sql;
use thiserror::Error;
/// How long to wait for the database lock to be released before failing a read.
const DB_READ_TIMEOUT: time::Duration = time::Duration::from_secs(3);
/// How long to wait for the database lock to be released before failing a write.
const DB_WRITE_TIMEOUT: time::Duration = time::Duration::from_secs(6);
#[derive(Error, Debug)]
pub enum Error {
/// An Internal error.
#[error("internal error: {0}")]
Internal(#[from] sql::Error),
}
/// A file-backed database storing information about the network.
pub struct Database {
pub db: sql::Connection,
}
impl fmt::Debug for Database {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Database").finish()
}
}
impl From<sql::Connection> for Database {
fn from(db: sql::Connection) -> Self {
Self { db }
}
}
impl Database {
const SCHEMA: &'static str = include_str!("db/schema.sql");
const PRAGMA: &'static str = "PRAGMA foreign_keys = ON";
/// Open an address book at the given path. Creates a new address book if it
/// doesn't exist.
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self, Error> {
let mut db = sql::Connection::open_with_flags(
path,
sqlite::OpenFlags::new()
.with_create()
.with_read_write()
.with_full_mutex(),
)?;
db.set_busy_timeout(DB_WRITE_TIMEOUT.as_millis() as usize)?;
db.execute(Self::PRAGMA)?;
db.execute(Self::SCHEMA)?;
Ok(Self { db })
}
/// Same as [`Self::open`], but in read-only mode. This is useful to have multiple
/// open databases, as no locking is required.
pub fn reader<P: AsRef<Path>>(path: P) -> Result<Self, Error> {
let mut db =
sql::Connection::open_with_flags(path, sqlite::OpenFlags::new().with_read_only())?;
db.set_busy_timeout(DB_READ_TIMEOUT.as_millis() as usize)?;
db.execute(Self::PRAGMA)?;
db.execute(Self::SCHEMA)?;
Ok(Self { db })
}
/// Create a new in-memory address book.
pub fn memory() -> Result<Self, Error> {
let db = sql::Connection::open(":memory:")?;
db.execute(Self::PRAGMA)?;
db.execute(Self::SCHEMA)?;
Ok(Self { db })
}
}

View File

@ -1,6 +1,4 @@
-- -- Discovered nodes.
-- Address book SQL schema.
--
create table if not exists "nodes" ( create table if not exists "nodes" (
-- Node ID. -- Node ID.
"id" text primary key not null, "id" text primary key not null,
@ -17,6 +15,7 @@ create table if not exists "nodes" (
-- --
) strict; ) strict;
-- Node addresses.
create table if not exists "addresses" ( create table if not exists "addresses" (
-- Node ID. -- Node ID.
"node" text not null references "nodes" ("id") on delete cascade, "node" text not null references "nodes" ("id") on delete cascade,
@ -42,9 +41,24 @@ create table if not exists "addresses" (
-- --
) strict; ) strict;
-- Routing table. Tracks inventories.
create table if not exists "routing" (
-- Repository being seeded.
"repo" text not null,
-- Node ID.
-- TODO: Add foreign-key constraint.
"node" text not null,
-- UNIX time at which this entry was added or refreshed.
"timestamp" integer not null,
primary key ("repo", "node")
);
-- Gossip message store.
create table if not exists "announcements" ( create table if not exists "announcements" (
-- Node ID. -- Node ID.
"node" text not null references "nodes" ("id") on delete cascade, -- TODO: Add foreign-key constraint.
"node" text not null,
-- Repo ID, if any, for example in ref announcements. -- Repo ID, if any, for example in ref announcements.
-- For other announcement types, this should be an empty string. -- For other announcement types, this should be an empty string.
"repo" text not null, "repo" text not null,
@ -64,6 +78,7 @@ create table if not exists "announcements" (
"timestamp" integer not null, "timestamp" integer not null,
-- --
unique ("node", "repo", "type") unique ("node", "repo", "type")
--
) strict; ) strict;
-- Repository sync status. -- Repository sync status.
@ -78,4 +93,5 @@ create table if not exists "repo-sync-status" (
"timestamp" integer not null, "timestamp" integer not null,
-- --
unique ("repo", "node") unique ("repo", "node")
--
) strict; ) strict;

View File

@ -1,21 +1,15 @@
use std::collections::HashSet; use std::collections::HashSet;
use std::path::Path;
use std::{fmt, time};
use sqlite as sql; use sqlite as sql;
use thiserror::Error; use thiserror::Error;
use crate::node::Database;
use crate::{ use crate::{
prelude::Timestamp, prelude::Timestamp,
prelude::{Id, NodeId}, prelude::{Id, NodeId},
sql::transaction, sql::transaction,
}; };
/// How long to wait for the database lock to be released before failing a read.
const DB_READ_TIMEOUT: time::Duration = time::Duration::from_secs(3);
/// How long to wait for the database lock to be released before failing a write.
const DB_WRITE_TIMEOUT: time::Duration = time::Duration::from_secs(6);
/// Result of inserting into the routing table. /// Result of inserting into the routing table.
#[derive(Debug, Copy, Clone, PartialEq, Eq)] #[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum InsertResult { pub enum InsertResult {
@ -38,50 +32,6 @@ pub enum Error {
UnitOverflow, UnitOverflow,
} }
/// Persistent file storage for a routing table.
pub struct Table {
db: sql::Connection,
}
impl fmt::Debug for Table {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "Table(..)")
}
}
impl Table {
const SCHEMA: &'static str = include_str!("routing/schema.sql");
/// Open a routing file store at the given path. Creates a new empty store
/// if an existing store isn't found.
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self, Error> {
let mut db = sql::Connection::open(path)?;
db.set_busy_timeout(DB_WRITE_TIMEOUT.as_millis() as usize)?;
db.execute(Self::SCHEMA)?;
Ok(Self { db })
}
/// Same as [`Self::open`], but in read-only mode. This is useful to have multiple
/// open databases, as no locking is required.
pub fn reader<P: AsRef<Path>>(path: P) -> Result<Self, Error> {
let mut db =
sql::Connection::open_with_flags(path, sqlite::OpenFlags::new().with_read_only())?;
db.set_busy_timeout(DB_READ_TIMEOUT.as_millis() as usize)?;
db.execute(Self::SCHEMA)?;
Ok(Self { db })
}
/// Create a new in-memory routing table.
pub fn memory() -> Result<Self, Error> {
let db = sql::Connection::open(":memory:")?;
db.execute(Self::SCHEMA)?;
Ok(Self { db })
}
}
/// Backing store for a routing table. /// Backing store for a routing table.
pub trait Store { pub trait Store {
/// Get the nodes seeding the given id. /// Get the nodes seeding the given id.
@ -113,11 +63,11 @@ pub trait Store {
fn count(&self, id: &Id) -> Result<usize, Error>; fn count(&self, id: &Id) -> Result<usize, Error>;
} }
impl Store for Table { impl Store for Database {
fn get(&self, id: &Id) -> Result<HashSet<NodeId>, Error> { fn get(&self, id: &Id) -> Result<HashSet<NodeId>, Error> {
let mut stmt = self let mut stmt = self
.db .db
.prepare("SELECT (node) FROM routing WHERE resource = ?")?; .prepare("SELECT (node) FROM routing WHERE repo = ?")?;
stmt.bind((1, id))?; stmt.bind((1, id))?;
let mut nodes = HashSet::new(); let mut nodes = HashSet::new();
@ -128,14 +78,12 @@ impl Store for Table {
} }
fn get_resources(&self, node: &NodeId) -> Result<HashSet<Id>, Error> { fn get_resources(&self, node: &NodeId) -> Result<HashSet<Id>, Error> {
let mut stmt = self let mut stmt = self.db.prepare("SELECT repo FROM routing WHERE node = ?")?;
.db
.prepare("SELECT resource FROM routing WHERE node = ?")?;
stmt.bind((1, node))?; stmt.bind((1, node))?;
let mut resources = HashSet::new(); let mut resources = HashSet::new();
for row in stmt.into_iter() { for row in stmt.into_iter() {
resources.insert(row?.read::<Id, _>("resource")); resources.insert(row?.read::<Id, _>("repo"));
} }
Ok(resources) Ok(resources)
} }
@ -143,13 +91,13 @@ impl Store for Table {
fn entry(&self, id: &Id, node: &NodeId) -> Result<Option<Timestamp>, Error> { fn entry(&self, id: &Id, node: &NodeId) -> Result<Option<Timestamp>, Error> {
let mut stmt = self let mut stmt = self
.db .db
.prepare("SELECT (time) FROM routing WHERE resource = ? AND node = ?")?; .prepare("SELECT (timestamp) FROM routing WHERE repo = ? AND node = ?")?;
stmt.bind((1, id))?; stmt.bind((1, id))?;
stmt.bind((2, node))?; stmt.bind((2, node))?;
if let Some(Ok(row)) = stmt.into_iter().next() { if let Some(Ok(row)) = stmt.into_iter().next() {
return Ok(Some(row.read::<i64, _>("time") as Timestamp)); return Ok(Some(row.read::<i64, _>("timestamp") as Timestamp));
} }
Ok(None) Ok(None)
} }
@ -166,18 +114,18 @@ impl Store for Table {
transaction(&self.db, |db| { transaction(&self.db, |db| {
for id in ids.into_iter() { for id in ids.into_iter() {
let mut stmt = let mut stmt =
db.prepare("SELECT (time) FROM routing WHERE resource = ? AND node = ?")?; db.prepare("SELECT (timestamp) FROM routing WHERE repo = ? AND node = ?")?;
stmt.bind((1, id))?; stmt.bind((1, id))?;
stmt.bind((2, &node))?; stmt.bind((2, &node))?;
let existed = stmt.into_iter().next().is_some(); let existed = stmt.into_iter().next().is_some();
let mut stmt = db.prepare( let mut stmt = db.prepare(
"INSERT INTO routing (resource, node, time) "INSERT INTO routing (repo, node, timestamp)
VALUES (?, ?, ?) VALUES (?, ?, ?)
ON CONFLICT DO UPDATE ON CONFLICT DO UPDATE
SET time = ?3 SET timestamp = ?3
WHERE time < ?3", WHERE timestamp < ?3",
)?; )?;
stmt.bind((1, id))?; stmt.bind((1, id))?;
@ -200,12 +148,12 @@ impl Store for Table {
fn entries(&self) -> Result<Box<dyn Iterator<Item = (Id, NodeId)>>, Error> { fn entries(&self) -> Result<Box<dyn Iterator<Item = (Id, NodeId)>>, Error> {
let mut stmt = self let mut stmt = self
.db .db
.prepare("SELECT resource, node FROM routing ORDER BY resource")? .prepare("SELECT repo, node FROM routing ORDER BY repo")?
.into_iter(); .into_iter();
let mut entries = Vec::new(); let mut entries = Vec::new();
while let Some(Ok(row)) = stmt.next() { while let Some(Ok(row)) = stmt.next() {
let id = row.read("resource"); let id = row.read("repo");
let node = row.read("node"); let node = row.read("node");
entries.push((id, node)); entries.push((id, node));
@ -216,7 +164,7 @@ impl Store for Table {
fn remove(&mut self, id: &Id, node: &NodeId) -> Result<bool, Error> { fn remove(&mut self, id: &Id, node: &NodeId) -> Result<bool, Error> {
let mut stmt = self let mut stmt = self
.db .db
.prepare("DELETE FROM routing WHERE resource = ? AND node = ?")?; .prepare("DELETE FROM routing WHERE repo = ? AND node = ?")?;
stmt.bind((1, id))?; stmt.bind((1, id))?;
stmt.bind((2, node))?; stmt.bind((2, node))?;
@ -245,7 +193,7 @@ impl Store for Table {
let mut stmt = self.db.prepare( let mut stmt = self.db.prepare(
"DELETE FROM routing WHERE rowid IN "DELETE FROM routing WHERE rowid IN
(SELECT rowid FROM routing WHERE time < ? LIMIT ?)", (SELECT rowid FROM routing WHERE timestamp < ? LIMIT ?)",
)?; )?;
stmt.bind((1, oldest))?; stmt.bind((1, oldest))?;
stmt.bind((2, limit))?; stmt.bind((2, limit))?;
@ -257,7 +205,7 @@ impl Store for Table {
fn count(&self, id: &Id) -> Result<usize, Error> { fn count(&self, id: &Id) -> Result<usize, Error> {
let mut stmt = self let mut stmt = self
.db .db
.prepare("SELECT COUNT(*) FROM routing WHERE resource = ?")?; .prepare("SELECT COUNT(*) FROM routing WHERE repo = ?")?;
stmt.bind((1, id))?; stmt.bind((1, id))?;
@ -284,7 +232,7 @@ mod test {
fn test_insert_and_get() { fn test_insert_and_get() {
let ids = arbitrary::set::<Id>(5..10); let ids = arbitrary::set::<Id>(5..10);
let nodes = arbitrary::set::<NodeId>(5..10); let nodes = arbitrary::set::<NodeId>(5..10);
let mut db = Table::open(":memory:").unwrap(); let mut db = Database::open(":memory:").unwrap();
for node in &nodes { for node in &nodes {
assert_eq!( assert_eq!(
@ -307,7 +255,7 @@ mod test {
fn test_insert_and_get_resources() { fn test_insert_and_get_resources() {
let ids = arbitrary::set::<Id>(5..10); let ids = arbitrary::set::<Id>(5..10);
let nodes = arbitrary::set::<NodeId>(5..10); let nodes = arbitrary::set::<NodeId>(5..10);
let mut db = Table::open(":memory:").unwrap(); let mut db = Database::open(":memory:").unwrap();
for node in &nodes { for node in &nodes {
db.insert(&ids, *node, 0).unwrap(); db.insert(&ids, *node, 0).unwrap();
@ -325,7 +273,7 @@ mod test {
fn test_entries() { fn test_entries() {
let ids = arbitrary::set::<Id>(6..9); let ids = arbitrary::set::<Id>(6..9);
let nodes = arbitrary::set::<NodeId>(6..9); let nodes = arbitrary::set::<NodeId>(6..9);
let mut db = Table::open(":memory:").unwrap(); let mut db = Database::open(":memory:").unwrap();
for node in &nodes { for node in &nodes {
assert!(db assert!(db
@ -348,7 +296,7 @@ mod test {
fn test_insert_and_remove() { fn test_insert_and_remove() {
let ids = arbitrary::set::<Id>(5..10); let ids = arbitrary::set::<Id>(5..10);
let nodes = arbitrary::set::<NodeId>(5..10); let nodes = arbitrary::set::<NodeId>(5..10);
let mut db = Table::open(":memory:").unwrap(); let mut db = Database::open(":memory:").unwrap();
for node in &nodes { for node in &nodes {
db.insert(&ids, *node, 0).unwrap(); db.insert(&ids, *node, 0).unwrap();
@ -367,7 +315,7 @@ mod test {
fn test_insert_duplicate() { fn test_insert_duplicate() {
let id = arbitrary::gen::<Id>(1); let id = arbitrary::gen::<Id>(1);
let node = arbitrary::gen::<NodeId>(1); let node = arbitrary::gen::<NodeId>(1);
let mut db = Table::open(":memory:").unwrap(); let mut db = Database::open(":memory:").unwrap();
assert_eq!( assert_eq!(
db.insert([&id], node, 0).unwrap(), db.insert([&id], node, 0).unwrap(),
@ -387,7 +335,7 @@ mod test {
fn test_insert_existing_updated_time() { fn test_insert_existing_updated_time() {
let id = arbitrary::gen::<Id>(1); let id = arbitrary::gen::<Id>(1);
let node = arbitrary::gen::<NodeId>(1); let node = arbitrary::gen::<NodeId>(1);
let mut db = Table::open(":memory:").unwrap(); let mut db = Database::open(":memory:").unwrap();
assert_eq!( assert_eq!(
db.insert([&id], node, 0).unwrap(), db.insert([&id], node, 0).unwrap(),
@ -405,7 +353,7 @@ mod test {
let id1 = arbitrary::gen::<Id>(1); let id1 = arbitrary::gen::<Id>(1);
let id2 = arbitrary::gen::<Id>(1); let id2 = arbitrary::gen::<Id>(1);
let node = arbitrary::gen::<NodeId>(1); let node = arbitrary::gen::<NodeId>(1);
let mut db = Table::open(":memory:").unwrap(); let mut db = Database::open(":memory:").unwrap();
assert_eq!( assert_eq!(
db.insert([&id1], node, 0).unwrap(), db.insert([&id1], node, 0).unwrap(),
@ -431,7 +379,7 @@ mod test {
fn test_remove_redundant() { fn test_remove_redundant() {
let id = arbitrary::gen::<Id>(1); let id = arbitrary::gen::<Id>(1);
let node = arbitrary::gen::<NodeId>(1); let node = arbitrary::gen::<NodeId>(1);
let mut db = Table::open(":memory:").unwrap(); let mut db = Database::open(":memory:").unwrap();
assert_eq!( assert_eq!(
db.insert([&id], node, 0).unwrap(), db.insert([&id], node, 0).unwrap(),
@ -443,7 +391,7 @@ mod test {
#[test] #[test]
fn test_len() { fn test_len() {
let mut db = Table::open(":memory:").unwrap(); let mut db = Database::open(":memory:").unwrap();
let ids = arbitrary::vec::<Id>(10); let ids = arbitrary::vec::<Id>(10);
let node = arbitrary::gen(1); let node = arbitrary::gen(1);
@ -458,7 +406,7 @@ mod test {
let now = LocalTime::now(); let now = LocalTime::now();
let ids = arbitrary::vec::<Id>(10); let ids = arbitrary::vec::<Id>(10);
let nodes = arbitrary::vec::<NodeId>(10); let nodes = arbitrary::vec::<NodeId>(10);
let mut db = Table::open(":memory:").unwrap(); let mut db = Database::open(":memory:").unwrap();
for node in &nodes { for node in &nodes {
let time = rng.u64(..now.as_millis()); let time = rng.u64(..now.as_millis());
@ -488,7 +436,7 @@ mod test {
fn test_count() { fn test_count() {
let id = arbitrary::gen::<Id>(1); let id = arbitrary::gen::<Id>(1);
let nodes = arbitrary::set::<NodeId>(5..10); let nodes = arbitrary::set::<NodeId>(5..10);
let mut db = Table::open(":memory:").unwrap(); let mut db = Database::open(":memory:").unwrap();
for node in &nodes { for node in &nodes {
db.insert([&id], *node, 0).unwrap(); db.insert([&id], *node, 0).unwrap();

View File

@ -1,13 +0,0 @@
--
-- Routing table SQL schema.
--
create table if not exists "routing" (
-- Resource being seeded.
"resource" text not null,
-- Node ID.
"node" text not null,
-- UNIX time at which this entry was added or refreshed.
"time" integer not null,
primary key ("resource", "node")
);

62
radicle/src/node/seed.rs Normal file
View File

@ -0,0 +1,62 @@
pub mod store;
pub use store::{Error, Store};
use localtime::LocalTime;
use crate::git;
use crate::node::KnownAddress;
use crate::prelude::NodeId;
use crate::storage::{refs::RefsAt, ReadRepository, RemoteId};
/// Holds an oid and timestamp.
#[derive(Debug, Copy, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct SyncedAt {
/// Head of `rad/sigrefs`.
pub oid: git_ext::Oid,
/// When these refs were synced.
#[serde(with = "crate::serde_ext::localtime::time")]
pub timestamp: LocalTime,
}
impl SyncedAt {
/// Load a new [`SyncedAt`] for the given remote.
pub fn load<S: ReadRepository>(repo: &S, remote: RemoteId) -> Result<Self, git::ext::Error> {
let refs = RefsAt::new(repo, remote)?;
let oid = refs.at;
Self::new(oid, repo)
}
/// Create a new [`SyncedAt`] given an OID, by looking up the timestamp in the repo.
pub fn new<S: ReadRepository>(oid: git::ext::Oid, repo: &S) -> Result<Self, git::ext::Error> {
let timestamp = repo.commit(oid)?.time();
let timestamp = LocalTime::from_secs(timestamp.seconds() as u64);
Ok(Self { oid, timestamp })
}
}
impl Ord for SyncedAt {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.timestamp.cmp(&other.timestamp)
}
}
impl PartialOrd for SyncedAt {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
/// Seed of a specific repository that has been synced at least once.
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct SyncedSeed {
/// The Node ID.
pub nid: NodeId,
/// Known addresses for this node.
pub addresses: Vec<KnownAddress>,
/// Sync information for a given repo.
pub synced_at: SyncedAt,
}

View File

@ -0,0 +1,133 @@
#![allow(clippy::type_complexity)]
use std::str::FromStr;
use localtime::LocalTime;
use sqlite as sql;
use thiserror::Error;
use crate::git::Oid;
use crate::node::address;
use crate::node::address::Store as _;
use crate::node::NodeId;
use crate::node::{seed::SyncedSeed, Database, SyncedAt};
use crate::prelude::{Id, Timestamp};
#[derive(Error, Debug)]
pub enum Error {
/// An Internal error.
#[error("internal error: {0}")]
Internal(#[from] sql::Error),
/// An address store error.
#[error("address store error: {0}")]
Addresses(#[from] address::Error),
}
/// Seed store.
///
/// Used to store seed sync statuses.
pub trait Store: address::Store {
/// Mark a repo as synced on the given node.
fn synced(
&mut self,
rid: &Id,
nid: &NodeId,
at: Oid,
timestamp: Timestamp,
) -> Result<bool, Error>;
/// Get the repos seeded by the given node.
fn seeded_by(
&self,
nid: &NodeId,
) -> Result<Box<dyn Iterator<Item = Result<(Id, SyncedAt), Error>> + '_>, Error>;
/// Get nodes that have synced the given repo.
fn seeds_for(
&self,
rid: &Id,
) -> Result<Box<dyn Iterator<Item = Result<SyncedSeed, Error>> + '_>, Error>;
}
impl Store for Database {
fn synced(
&mut self,
rid: &Id,
nid: &NodeId,
at: Oid,
timestamp: Timestamp,
) -> Result<bool, Error> {
let mut stmt = self.db.prepare(
"INSERT INTO `repo-sync-status` (repo, node, head, timestamp)
VALUES (?1, ?2, ?3, ?4)
ON CONFLICT DO UPDATE
SET head = ?3, timestamp = ?4
WHERE timestamp < ?4",
)?;
stmt.bind((1, rid))?;
stmt.bind((2, nid))?;
stmt.bind((3, at.to_string().as_str()))?;
stmt.bind((4, timestamp as i64))?;
stmt.next()?;
Ok(self.db.change_count() > 0)
}
fn seeds_for(
&self,
rid: &Id,
) -> Result<Box<dyn Iterator<Item = Result<SyncedSeed, Error>> + '_>, Error> {
let mut stmt = self.db.prepare(
"SELECT node, head, timestamp
FROM `repo-sync-status`
WHERE repo = ?",
)?;
stmt.bind((1, rid))?;
Ok(Box::new(stmt.into_iter().map(|row| {
let row = row?;
let nid = row.try_read::<NodeId, _>("node")?;
let oid = row.try_read::<&str, _>("head")?;
let oid = Oid::from_str(oid).map_err(|e| {
Error::Internal(sql::Error {
code: None,
message: Some(format!("sql: invalid oid '{oid}': {e}")),
})
})?;
let timestamp = row.try_read::<i64, _>("timestamp")?;
let timestamp = LocalTime::from_millis(timestamp as u128);
let addresses = self.addresses_of(&nid)?;
Ok(SyncedSeed {
nid,
addresses,
synced_at: SyncedAt { oid, timestamp },
})
})))
}
fn seeded_by(
&self,
nid: &NodeId,
) -> Result<Box<dyn Iterator<Item = Result<(Id, SyncedAt), Error>> + '_>, Error> {
let mut stmt = self.db.prepare(
"SELECT repo, head, timestamp
FROM `repo-sync-status`
WHERE node = ?",
)?;
stmt.bind((1, nid))?;
Ok(Box::new(stmt.into_iter().map(|row| {
let row = row?;
let rid = row.try_read::<Id, _>("repo")?;
let oid = row.try_read::<&str, _>("head")?;
let oid = Oid::from_str(oid).map_err(|e| {
Error::Internal(sql::Error {
code: None,
message: Some(format!("sql: invalid oid '{oid}': {e}")),
})
})?;
let timestamp = row.try_read::<i64, _>("timestamp")?;
let timestamp = LocalTime::from_millis(timestamp as u128);
Ok((rid, SyncedAt { oid, timestamp }))
})))
}
}

View File

@ -20,7 +20,7 @@ use thiserror::Error;
use crate::crypto::ssh::agent::Agent; use crate::crypto::ssh::agent::Agent;
use crate::crypto::ssh::{keystore, Keystore, Passphrase}; use crate::crypto::ssh::{keystore, Keystore, Passphrase};
use crate::crypto::{PublicKey, Signer}; use crate::crypto::{PublicKey, Signer};
use crate::node::{address, routing, tracking, Alias, AliasStore}; use crate::node::{tracking, Alias, AliasStore};
use crate::prelude::Did; use crate::prelude::Did;
use crate::prelude::{Id, NodeId}; use crate::prelude::{Id, NodeId};
use crate::storage::git::transport; use crate::storage::git::transport;
@ -148,7 +148,7 @@ pub enum Error {
#[error(transparent)] #[error(transparent)]
TrackingStore(#[from] node::tracking::store::Error), TrackingStore(#[from] node::tracking::store::Error),
#[error(transparent)] #[error(transparent)]
AddressStore(#[from] node::address::Error), DatabaseStore(#[from] node::db::Error),
} }
#[derive(Debug, Error)] #[derive(Debug, Error)]
@ -357,22 +357,6 @@ impl Profile {
Ok(config) Ok(config)
} }
/// Return a read-only handle to the routing database of the node.
pub fn routing(&self) -> Result<routing::Table, routing::Error> {
let path = self.home.node().join(node::ROUTING_DB_FILE);
let router = routing::Table::reader(path)?;
Ok(router)
}
/// Return a handle to the addresses database of the node.
pub fn addresses(&self) -> Result<address::Book, address::Error> {
let path = self.home.node().join(node::ADDRESS_DB_FILE);
let addresses = address::Book::reader(path)?;
Ok(addresses)
}
/// Get radicle home. /// Get radicle home.
pub fn home(&self) -> &Home { pub fn home(&self) -> &Home {
&self.home &self.home
@ -381,12 +365,9 @@ impl Profile {
/// Return a multi-source store for aliases. /// Return a multi-source store for aliases.
pub fn aliases(&self) -> Aliases { pub fn aliases(&self) -> Aliases {
let tracking = self.home.tracking().ok(); let tracking = self.home.tracking().ok();
let addresses = self.home.addresses().ok(); let db = self.home.database().ok();
Aliases { Aliases { tracking, db }
tracking,
addresses,
}
} }
} }
@ -408,7 +389,7 @@ impl std::ops::DerefMut for Profile {
/// them one by one when asking for an alias. /// them one by one when asking for an alias.
pub struct Aliases { pub struct Aliases {
tracking: Option<tracking::store::ConfigReader>, tracking: Option<tracking::store::ConfigReader>,
addresses: Option<address::Book>, db: Option<node::Database>,
} }
impl AliasStore for Aliases { impl AliasStore for Aliases {
@ -418,7 +399,7 @@ impl AliasStore for Aliases {
self.tracking self.tracking
.as_ref() .as_ref()
.and_then(|db| db.alias(nid)) .and_then(|db| db.alias(nid))
.or_else(|| self.addresses.as_ref().and_then(|db| db.alias(nid))) .or_else(|| self.db.as_ref().and_then(|db| db.alias(nid)))
} }
} }
@ -522,36 +503,20 @@ impl Home {
Ok(config) Ok(config)
} }
/// Return a read-only handle to the routing database of the node. /// Return a handle to a read-only database of the node.
pub fn routing(&self) -> Result<routing::Table, routing::Error> { pub fn database(&self) -> Result<node::Database, node::db::Error> {
let path = self.node().join(node::ROUTING_DB_FILE); let path = self.node().join(node::NODE_DB_FILE);
let router = routing::Table::reader(path)?; let db = node::Database::reader(path)?;
Ok(router) Ok(db)
} }
/// Return a read-write handle to the routing database of the node. /// Return a handle to the database of the node.
pub fn routing_mut(&self) -> Result<routing::Table, routing::Error> { pub fn database_mut(&self) -> Result<node::Database, node::db::Error> {
let path = self.node().join(node::ROUTING_DB_FILE); let path = self.node().join(node::NODE_DB_FILE);
let router = routing::Table::open(path)?; let db = node::Database::open(path)?;
Ok(router) Ok(db)
}
/// Return a handle to a read-only addresses database of the node.
pub fn addresses(&self) -> Result<address::Book, address::Error> {
let path = self.node().join(node::ADDRESS_DB_FILE);
let addresses = address::Book::reader(path)?;
Ok(addresses)
}
/// Return a handle to the addresses database of the node.
pub fn addresses_mut(&self) -> Result<address::Book, address::Error> {
let path = self.node().join(node::ADDRESS_DB_FILE);
let addresses = address::Book::open(path)?;
Ok(addresses)
} }
} }