From 2f07b76fcbec54e1fb691d00c8858750726c6ddd Mon Sep 17 00:00:00 2001 From: Alexis Sellier Date: Wed, 28 Dec 2022 11:48:08 +0100 Subject: [PATCH] Switch to our own DAG representation for COBs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We're using petgraph to encode an operation based CRDT (each graph node is an operation) as a DAG, and one of the important things to ensure is that the final state is not influenced by the order of concurrent operations. In this case a concurrent operation would mean two potential graph traversal orders, as there's no direct edge between the two concurrent ops. This is essential a "partial order" of operations. To be able to test this, we'd want some kind of control over how neighbors are iterated over I guess, when there's more than one possible sort order. Say we have a graph like this: ┌────────b◄─────┐ ▼ │ a d ▲ │ └────────c◄─────┘ There are two possible topological traverse orders: [a, b, c, d] and [a, c, b, d]. Having a way to go through these different orders would be super handy. One option would be to allow random order traversal. This would allow us to test all orders by running the test enough times to likely test all permutations. Since petgraph doesn't support this, we implement our own simple DAG in `radicle-dag`, which implements random-order topological orders. For now, we don't make explicit use of these improvements, we simply replace the underlying graph with our own. --- Cargo.lock | 25 +- Cargo.toml | 1 + radicle-cob/Cargo.toml | 6 +- radicle-cob/src/change_graph.rs | 77 ++--- radicle-cob/src/change_graph/evaluation.rs | 16 +- radicle-cob/src/history.rs | 70 ++--- radicle-cob/src/object/collaboration/info.rs | 10 +- radicle-dag/Cargo.toml | 7 + radicle-dag/src/lib.rs | 301 +++++++++++++++++++ 9 files changed, 392 insertions(+), 121 deletions(-) create mode 100644 radicle-dag/Cargo.toml create mode 100644 radicle-dag/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index fe635db5..d22cbfd1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -906,12 +906,6 @@ dependencies = [ "static_assertions", ] -[[package]] -name = "fixedbitset" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" - [[package]] name = "flate2" version = "1.0.25" @@ -1861,16 +1855,6 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" -[[package]] -name = "petgraph" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "467d164a6de56270bd7c4d070df81d07beace25012d5103ced4e9ff08d6afdb7" -dependencies = [ - "fixedbitset", - "indexmap", -] - [[package]] name = "pin-project" version = "1.0.12" @@ -2130,10 +2114,10 @@ dependencies = [ "git2", "log", "nonempty 0.8.1", - "petgraph", "qcheck", "qcheck-macros", "radicle-crypto", + "radicle-dag", "radicle-git-ext", "serde", "serde_json", @@ -2177,6 +2161,13 @@ dependencies = [ "zeroize", ] +[[package]] +name = "radicle-dag" +version = "0.1.0" +dependencies = [ + "fastrand", +] + [[package]] name = "radicle-git-ext" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 0a51a3b8..6dd142df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "radicle-cli", "radicle-crdt", "radicle-crypto", + "radicle-dag", "radicle-httpd", "radicle-node", "radicle-remote-helper", diff --git a/radicle-cob/Cargo.toml b/radicle-cob/Cargo.toml index 7d2d5526..dd0f4a49 100644 --- a/radicle-cob/Cargo.toml +++ b/radicle-cob/Cargo.toml @@ -13,12 +13,12 @@ keywords = ["radicle", "collaborative objects", "cob", "cobs"] [lib] [dependencies] +fastrand = { version = "1.8.0" } git-commit = { version = "0.2" } git-ref-format = { version = "0.1" } git-trailers = { version = "0.1" } log = { version = "0.4.17" } nonempty = { version = "0.8.1", features = ["serialize"] } -petgraph = { version = "0.5" } radicle-git-ext = { version = "0" } serde_json = { version = "1.0" } thiserror = { version = "1.0" } @@ -33,6 +33,10 @@ path = "../radicle-crypto" version = "0.1" features = ["ssh"] +[dependencies.radicle-dag] +path = "../radicle-dag" +version = "0.1" + [dependencies.serde] version = "1.0" features = ["derive"] diff --git a/radicle-cob/src/change_graph.rs b/radicle-cob/src/change_graph.rs index e518b2f7..b1c20466 100644 --- a/radicle-cob/src/change_graph.rs +++ b/radicle-cob/src/change_graph.rs @@ -9,10 +9,7 @@ use std::{ }; use git_ext::Oid; -use petgraph::{ - visit::{EdgeRef, Topo, Walker}, - EdgeDirection, -}; +use radicle_dag::{Dag, Node}; use crate::{ change, object, signatures::Signature, Change, CollaborativeObject, ObjectId, TypeName, @@ -24,7 +21,7 @@ use evaluation::evaluate; /// The graph of changes for a particular collaborative object pub(super) struct ChangeGraph { object_id: ObjectId, - graph: petgraph::Graph, + graph: Dag, } impl ChangeGraph { @@ -94,27 +91,23 @@ impl ChangeGraph { /// or which do not have permission to make a change, or which make a /// change which invalidates the schema of the object pub(crate) fn evaluate(&self) -> CollaborativeObject { - let mut roots: Vec> = self - .graph - .externals(petgraph::Direction::Incoming) - .collect(); - roots.sort(); + let mut roots: Vec<(&Oid, &Node<_, _>)> = self.graph.roots().collect(); + roots.sort_by_key(|(k, _)| *k); // This is okay because we check that the graph has a root node in // GraphBuilder::build - let root = roots.first().unwrap(); - let manifest = { - let first_node = &self.graph[*root]; - first_node.manifest.clone() - }; - let topo = Topo::new(&self.graph); - let items = topo.iter(&self.graph).map(|idx| { - let node = &self.graph[idx]; - let outgoing_edges = self.graph.edges_directed(idx, EdgeDirection::Outgoing); - let child_commits = outgoing_edges - .map(|e| *self.graph[e.target()].id()) + let (root, root_node) = roots.first().unwrap(); + let manifest = root_node.manifest.clone(); + let rng = fastrand::Rng::new(); + let sorted = self.graph.sorted(rng); + let items = sorted.iter().map(|oid| { + let node = &self.graph[oid]; + let child_commits = node + .dependents + .iter() + .map(|e| *self.graph[e].id()) .collect::>(); - (node, idx, child_commits) + (&node.value, *oid, child_commits) }); let history = { let root_change = &self.graph[*root]; @@ -129,35 +122,24 @@ impl ChangeGraph { /// Get the tips of the collaborative object pub(crate) fn tips(&self) -> BTreeSet { - self.graph - .externals(petgraph::Direction::Outgoing) - .map(|n| { - let change = &self.graph[n]; - *change.id() - }) - .collect() + self.graph.tips().map(|(_, change)| *change.id()).collect() } pub(crate) fn number_of_nodes(&self) -> u64 { - self.graph.node_count().try_into().unwrap() - } - - pub(crate) fn graphviz(&self) -> String { - let for_display = self.graph.map(|_ix, n| n.to_string(), |_ix, _e| ""); - petgraph::dot::Dot::new(&for_display).to_string() + self.graph.len().try_into().unwrap() } } struct GraphBuilder { - node_indices: HashMap>, - graph: petgraph::Graph, + node_indices: HashMap, + graph: Dag, } impl Default for GraphBuilder { fn default() -> Self { GraphBuilder { node_indices: HashMap::new(), - graph: petgraph::graph::Graph::new(), + graph: Dag::new(), } } } @@ -173,11 +155,11 @@ impl GraphBuilder { let resource_commit = *change.resource(); let commit_id = commit.id; if let Entry::Vacant(e) = self.node_indices.entry(commit_id) { - let ix = self.graph.add_node(change); - e.insert(ix); + self.graph.node(commit_id, change); + e.insert(commit_id); } commit.parents.into_iter().filter_map(move |parent| { - if parent.id != resource_commit && !self.has_edge(parent.id, commit_id) { + if parent.id != resource_commit && !self.has_dependency(commit_id, parent.id) { Some((parent, commit_id)) } else { None @@ -185,11 +167,11 @@ impl GraphBuilder { }) } - fn has_edge(&mut self, parent_id: Oid, child_id: Oid) -> bool { + fn has_dependency(&mut self, child_id: Oid, parent_id: Oid) -> bool { let parent_ix = self.node_indices.get(&parent_id); let child_ix = self.node_indices.get(&child_id); match (parent_ix, child_ix) { - (Some(parent_ix), Some(child_ix)) => self.graph.contains_edge(*parent_ix, *child_ix), + (Some(parent_ix), Some(child_ix)) => self.graph.has_dependency(child_ix, parent_ix), _ => false, } } @@ -204,16 +186,11 @@ impl GraphBuilder { .node_indices .get(&parent) .expect("BUG: parent id expected to in graph"); - self.graph.update_edge(*parent_id, *child_id, ()); + self.graph.dependency(*child_id, *parent_id); } fn build(self, object_id: ObjectId) -> Option { - if self - .graph - .externals(petgraph::Direction::Incoming) - .next() - .is_some() - { + if self.graph.roots().next().is_some() { Some(ChangeGraph { object_id, graph: self.graph, diff --git a/radicle-cob/src/change_graph/evaluation.rs b/radicle-cob/src/change_graph/evaluation.rs index ad96d3c8..9de460c2 100644 --- a/radicle-cob/src/change_graph/evaluation.rs +++ b/radicle-cob/src/change_graph/evaluation.rs @@ -6,7 +6,7 @@ use std::{collections::HashMap, ops::ControlFlow}; use git_ext::Oid; -use petgraph::{visit::EdgeRef, EdgeDirection}; +use radicle_dag::Dag; use crate::history::entry::{EntryId, EntryWithClock}; use crate::history::Clock; @@ -17,8 +17,8 @@ use crate::{change::Change, history, pruning_fold}; /// If the change corresponding to the root OID is not in `items` pub fn evaluate<'b>( root: Oid, - graph: &petgraph::Graph, - items: impl Iterator, Vec)>, + graph: &Dag, + items: impl Iterator)>, ) -> history::History { let entries = pruning_fold::pruning_fold( HashMap::::new(), @@ -37,11 +37,11 @@ pub fn evaluate<'b>( } Ok(entry) => { // Get parent commits and calculate this node's clock based on theirs. - let incoming = graph.edges_directed(c.idx, EdgeDirection::Incoming); - let clock = incoming - .into_iter() + let clock = graph[&c.idx] + .dependencies + .iter() .map(|e| { - let entry = &entries[&graph[e.source()].id.into()]; + let entry = &entries[&graph[e].id.into()]; let clock = entry.clock(); clock + entry.contents().len() as Clock - 1 @@ -81,7 +81,7 @@ fn evaluate_change( } struct ChangeWithChildren<'a> { - idx: petgraph::graph::NodeIndex, + idx: Oid, change: &'a Change, child_commits: Vec, } diff --git a/radicle-cob/src/history.rs b/radicle-cob/src/history.rs index 04cb92d6..7545a242 100644 --- a/radicle-cob/src/history.rs +++ b/radicle-cob/src/history.rs @@ -9,8 +9,8 @@ use std::{ }; use git_ext::Oid; -use petgraph::visit::Walker as _; use radicle_crypto::PublicKey; +use radicle_dag::Dag; use crate::pruning_fold; @@ -20,8 +20,8 @@ pub use entry::{Clock, Contents, Entry, EntryId, EntryWithClock, Timestamp}; /// The DAG of changes making up the history of a collaborative object. #[derive(Clone, Debug)] pub struct History { - graph: petgraph::Graph, - indices: HashMap>, + graph: Dag, + indices: HashMap, } impl PartialEq for History { @@ -61,7 +61,7 @@ impl History { let mut entries = HashMap::new(); entries.insert(id, EntryWithClock::from(root_entry)); - create_petgraph(&id, &entries) + create_dag(&id, &entries) } pub fn new(root: Id, entries: HashMap) -> Result @@ -72,7 +72,7 @@ impl History { if !entries.contains_key(&root) { Err(CreateError::MissingRoot) } else { - Ok(create_petgraph(&root, &entries)) + Ok(create_dag(&root, &entries)) } } @@ -80,11 +80,8 @@ impl History { /// This is the maximum value of all tips. pub fn clock(&self) -> Clock { self.graph - .externals(petgraph::Direction::Outgoing) - .map(|n| { - let node = &self.graph[n]; - node.clock + node.entry.contents.len() as Clock - 1 - }) + .tips() + .map(|(_, node)| node.clock + node.entry.contents.len() as Clock - 1) .max() .unwrap_or_default() } @@ -93,8 +90,8 @@ impl History { /// This is the latest timestamp of any tip. pub fn timestamp(&self) -> Timestamp { self.graph - .externals(petgraph::Direction::Outgoing) - .map(|n| self.graph[n].timestamp()) + .tips() + .map(|(_, n)| n.timestamp()) .max() .unwrap_or_default() } @@ -109,9 +106,9 @@ impl History { where F: for<'r> FnMut(A, &'r EntryWithClock) -> ControlFlow, { - let topo = petgraph::visit::Topo::new(&self.graph); + let sorted = self.graph.sorted(fastrand::Rng::new()); #[allow(clippy::let_and_return)] - let items = topo.iter(&self.graph).map(|idx| { + let items = sorted.iter().map(|idx| { let entry = &self.graph[idx]; entry }); @@ -120,11 +117,8 @@ impl History { pub(crate) fn tips(&self) -> BTreeSet { self.graph - .externals(petgraph::Direction::Outgoing) - .map(|n| { - let entry = &self.graph[n]; - (*entry.id()).into() - }) + .tips() + .map(|(_, entry)| (*entry.id()).into()) .collect() } @@ -148,34 +142,36 @@ impl History { new_contents, new_timestamp, ); - let new_ix = self.graph.add_node(EntryWithClock { - entry: new_entry, - clock: self.clock() + 1, - }); + self.graph.node( + new_id, + EntryWithClock { + entry: new_entry, + clock: self.clock() + 1, + }, + ); for tip in tips { let tip_ix = self.indices.get(&tip.into()).unwrap(); - self.graph.update_edge(*tip_ix, new_ix, ()); + self.graph.dependency(new_id, (*tip_ix).into()); } } } -fn create_petgraph<'a>( - root: &'a EntryId, - entries: &'a HashMap, -) -> History { - let mut graph = petgraph::Graph::new(); - let mut indices = HashMap::>::new(); - let root = entries.get(root).unwrap().clone(); - let root_ix = graph.add_node(root.clone()); - indices.insert(root.id, root_ix); - let mut to_process = vec![root]; +fn create_dag<'a>(root: &'a EntryId, entries: &'a HashMap) -> History { + let mut graph: Dag = Dag::new(); + let mut indices = HashMap::::new(); + let root_entry = entries.get(root).unwrap().clone(); + graph.node(*root, root_entry.clone()); + indices.insert(root_entry.id, (*root).into()); + let mut to_process = vec![root_entry]; + while let Some(entry) = to_process.pop() { let entry_ix = indices[&entry.id]; + for child_id in entry.children() { let child = entries[child_id].clone(); - let child_ix = graph.add_node(child.clone()); - indices.insert(child.id, child_ix); - graph.update_edge(entry_ix, child_ix, ()); + graph.node(*child_id, child.clone()); + indices.insert(child.id, (*child_id).into()); + graph.dependency(*child_id, entry_ix.into()); to_process.push(child.clone()); } } diff --git a/radicle-cob/src/object/collaboration/info.rs b/radicle-cob/src/object/collaboration/info.rs index a134b881..e3fb34bd 100644 --- a/radicle-cob/src/object/collaboration/info.rs +++ b/radicle-cob/src/object/collaboration/info.rs @@ -4,11 +4,8 @@ // Linking Exception. For full terms see the included LICENSE file. //! [`ChangeGraphInfo`] provides a useful debugging structure for -//! represnting a single [`crate::CollaborativeObject`]'s underlying -//! change graph. This includes a [`ChangeGraphInfo::dotviz`] for -//! describing the graph via [graphviz]. -//! -//! [graphviz]: https://graphviz.org/ +//! representing a single [`crate::CollaborativeObject`]'s underlying +//! change graph. use std::collections::BTreeSet; @@ -22,8 +19,6 @@ use super::error; pub struct ChangeGraphInfo { /// The ID of the object pub object_id: ObjectId, - /// A graphviz description of the changegraph of the object - pub dotviz: String, /// The number of nodes in the change graph of the object pub number_of_nodes: u64, /// The "tips" of the change graph, i.e the object IDs pointed to by @@ -54,7 +49,6 @@ where Ok( ChangeGraph::load(storage, tip_refs.iter(), typename, oid).map(|graph| ChangeGraphInfo { object_id: *oid, - dotviz: graph.graphviz(), number_of_nodes: graph.number_of_nodes(), tips: graph.tips(), }), diff --git a/radicle-dag/Cargo.toml b/radicle-dag/Cargo.toml new file mode 100644 index 00000000..76a22d99 --- /dev/null +++ b/radicle-dag/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "radicle-dag" +version = "0.1.0" +edition = "2021" + +[dependencies] +fastrand = { version = "1.8.0" } diff --git a/radicle-dag/src/lib.rs b/radicle-dag/src/lib.rs new file mode 100644 index 00000000..0fce13e4 --- /dev/null +++ b/radicle-dag/src/lib.rs @@ -0,0 +1,301 @@ +use std::{ + borrow::Borrow, + collections::{HashMap, HashSet}, + fmt, + hash::Hash, + ops::{Deref, Index}, +}; + +/// A node in the graph. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Node { + /// The node value, stored by the user. + pub value: V, + /// Nodes depended on. + pub dependencies: HashSet, + /// Nodes depending on this node. + pub dependents: HashSet, +} + +impl Borrow for &Node { + fn borrow(&self) -> &V { + &self.value + } +} + +impl Deref for Node { + type Target = V; + + fn deref(&self) -> &Self::Target { + &self.value + } +} + +/// A directed acyclic graph. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Dag { + graph: HashMap>, + tips: HashSet, + roots: HashSet, +} + +impl Dag { + /// Create a new empty DAG. + pub fn new() -> Self { + Self { + graph: HashMap::new(), + tips: HashSet::new(), + roots: HashSet::new(), + } + } + + /// Check whether there are any nodes in the graph. + pub fn is_empty(&self) -> bool { + self.graph.is_empty() + } + + /// Return the number of nodes in the graph. + pub fn len(&self) -> usize { + self.graph.len() + } + + /// Add a node to the graph. + pub fn node(&mut self, key: K, value: V) -> Option> { + self.tips.insert(key); + self.roots.insert(key); + self.graph.insert( + key, + Node { + value, + dependencies: HashSet::new(), + dependents: HashSet::new(), + }, + ) + } + + /// Add a dependency from one node to the other. + pub fn dependency(&mut self, from: K, to: K) { + if let Some(node) = self.graph.get_mut(&from) { + node.dependencies.insert(to); + self.roots.remove(&from); + } + if let Some(node) = self.graph.get_mut(&to) { + node.dependents.insert(from); + self.tips.remove(&to); + } + } + + /// Get a node. + pub fn get(&self, key: &K) -> Option<&Node> { + self.graph.get(key) + } + + /// Check whether there is a dependency between two nodes. + pub fn has_dependency(&self, from: &K, to: &K) -> bool { + self.graph + .get(from) + .map(|n| n.dependencies.contains(to)) + .unwrap_or_default() + } + + /// Get the graph's root nodes, ie. nodes which don't depend on other nodes. + pub fn roots(&self) -> impl Iterator)> + '_ { + self.roots + .iter() + .filter_map(|k| self.graph.get(k).map(|n| (k, n))) + } + + /// Get the graph's tip nodes, ie. nodes which aren't depended on by other nodes. + pub fn tips(&self) -> impl Iterator)> + '_ { + self.tips + .iter() + .filter_map(|k| self.graph.get(k).map(|n| (k, n))) + } + + /// Return a topological ordering of the graph's nodes, using the given RNG. + /// Graphs with more than one partial order will return an arbitrary topological ordering. + /// + /// Calling this function over and over will eventually yield all possible orderings. + pub fn sorted(&self, rng: fastrand::Rng) -> Vec { + let mut order = Vec::new(); // Stores the topological order. + let mut visited = HashSet::new(); // Nodes that have been visited. + let mut keys = self.graph.keys().collect::>(); + + rng.shuffle(&mut keys); + + for node in keys { + self.visit(node, &mut visited, &mut order); + } + order + } + + /// Add nodes recursively to the topological order, starting from the given node. + fn visit(&self, key: &K, visited: &mut HashSet, order: &mut Vec) { + if visited.contains(key) { + return; + } + visited.insert(*key); + + // Recursively visit all of the node's dependencies. + if let Some(node) = self.graph.get(key) { + for dependency in &node.dependencies { + self.visit(dependency, visited, order); + } + } + // Add the node to the topological order. + order.push(*key); + } +} + +impl Index<&K> for Dag { + type Output = Node; + + fn index(&self, key: &K) -> &Self::Output { + self.get(key) + .unwrap_or_else(|| panic!("Dag::index: node {:?} not found in graph", key)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_len() { + let mut dag = Dag::new(); + + dag.node(0, ()); + dag.node(1, ()); + dag.node(2, ()); + + assert_eq!(dag.len(), 3); + } + + #[test] + fn test_is_empty() { + let mut dag = Dag::new(); + assert!(dag.is_empty()); + + dag.node(0, ()); + assert!(!dag.is_empty()); + } + + #[test] + fn test_dependencies() { + let mut dag = Dag::new(); + + dag.node(0, ()); + dag.node(1, ()); + dag.dependency(0, 1); + + assert!(dag.has_dependency(&0, &1)); + assert!(!dag.has_dependency(&1, &0)); + } + + #[test] + fn test_get() { + let mut dag = Dag::new(); + + dag.node(0, "rad"); + dag.node(1, "dar"); + + assert_eq!(dag[&0].value, "rad"); + assert_eq!(dag[&1].value, "dar"); + assert!(dag.get(&2).is_none()); + } + + #[test] + fn test_cycle() { + let mut dag = Dag::new(); + + dag.node(0, ()); + dag.node(1, ()); + + dag.dependency(0, 1); + dag.dependency(1, 0); + + let sorted = dag.sorted(fastrand::Rng::new()); + let expected: &[&[i32]] = &[&[0, 1], &[1, 0]]; + + assert!(expected.contains(&sorted.as_slice())); + } + + #[test] + fn test_diamond() { + let mut dag = Dag::new(); + + dag.node(0, ()); + dag.node(1, ()); + dag.node(2, ()); + dag.node(3, ()); + + dag.dependency(1, 0); + dag.dependency(2, 0); + dag.dependency(3, 1); + dag.dependency(3, 2); + + assert_eq!(dag.tips().map(|(k, _)| *k).collect::>(), vec![3]); + assert_eq!(dag.roots().map(|(k, _)| *k).collect::>(), vec![0]); + + // All of the possible sort orders for the above graph. + let expected: &[&[i32]] = &[&[0, 1, 2, 3], &[0, 2, 1, 3]]; + let actual = dag.sorted(fastrand::Rng::new()); + + assert!(expected.contains(&actual.as_slice()), "{:?}", actual); + } + + #[test] + fn test_complex() { + let mut dag = Dag::new(); + + dag.node(0, ()); + dag.node(1, ()); + dag.node(2, ()); + dag.node(3, ()); + dag.node(4, ()); + dag.node(5, ()); + + dag.dependency(3, 2); + dag.dependency(1, 3); + dag.dependency(2, 5); + dag.dependency(0, 5); + dag.dependency(0, 4); + dag.dependency(1, 4); + + assert_eq!( + dag.tips().map(|(k, _)| *k).collect::>(), + HashSet::from_iter([1, 0]) + ); + assert_eq!( + dag.roots().map(|(k, _)| *k).collect::>(), + HashSet::from_iter([4, 5]) + ); + + // All of the possible sort orders for the above graph. + let expected = &[ + [4, 5, 0, 2, 3, 1], + [4, 5, 2, 0, 3, 1], + [4, 5, 2, 3, 0, 1], + [4, 5, 2, 3, 1, 0], + [5, 2, 3, 4, 0, 1], + [5, 2, 3, 4, 1, 0], + [5, 2, 4, 0, 3, 1], + [5, 2, 4, 3, 0, 1], + [5, 2, 4, 3, 1, 0], + [5, 4, 0, 2, 3, 1], + [5, 4, 2, 0, 3, 1], + [5, 4, 2, 3, 0, 1], + [5, 4, 2, 3, 1, 0], + ]; + let rng = fastrand::Rng::new(); + let mut sorts = HashSet::new(); + + while sorts.len() < expected.len() { + sorts.insert(dag.sorted(rng.clone())); + } + for e in expected { + assert!(sorts.remove(e.to_vec().as_slice())); + } + assert!(sorts.is_empty()); + } +}