Test code (#31)

* better testing

* remove unused code

* remove unused code. Increase the number of tests

* Add necessary include

* Less duplication - but more complexity.

* Can test on all .json files too

* Replace with macros as suggested by @oflatt

* Fixed the random egraph generator - previously it was only producing acyclic egraphs. Disabled the faster-greedy-dag extractor - the update fuzzer has generated more egraphs that trigger failure

* extra test cases from fuzzing

* * Use the new ILP timeout class, * Shift to nextest

* Revert. nextest is not currently installed by github actions

* Fix crash if some optimal extractors aren't provided. Generate egraphs that are more likely to be problematic

* oops. remove debug code

* Use enum so that optimal tree/dag are distinct

* Move test files so they don't impact the benchmarking

* move test code to a separate module

* Move into separate files as requested

Cargo.toml

@@ -14,6 +14,8 @@ indexmap = "2.0.0"
 log = "0.4.19"
 ordered-float = "3"
 pico-args = { version = "0.5.0", features = ["eq-separator"] }
+rand = "0.8.5"
+walkdir = "2.4.0"
 
 anyhow = "1.0.71"
 coin_cbc = { version = "0.1.6", optional = true }

src/extract/mod.rs

@@ -12,6 +12,9 @@ pub mod greedy_dag;
 #[cfg(feature = "ilp-cbc")]
 pub mod ilp_cbc;
 
+// Allowance for floating point values to be considered equal
+pub const EPSILON_ALLOWANCE: f64 = 0.00001;
+
 pub trait Extractor: Sync {
     fn extract(&self, egraph: &EGraph, roots: &[ClassId]) -> ExtractionResult;
 

src/main.rs

@@ -15,26 +15,71 @@ use std::path::PathBuf;
 pub type Cost = NotNan<f64>;
 pub const INFINITY: Cost = unsafe { NotNan::new_unchecked(std::f64::INFINITY) };
 
-fn main() {
-    env_logger::init();
+#[derive(PartialEq, Eq)]
+enum Optimal {
+    Tree,
+    DAG,
+    Neither,
+}
+
+struct ExtractorDetail {
+    extractor: Box<dyn Extractor>,
+    optimal: Optimal,
+    use_for_bench: bool,
+}
 
-    let extractors: IndexMap<&str, Box<dyn Extractor>> = [
-        ("bottom-up", extract::bottom_up::BottomUpExtractor.boxed()),
+fn extractors() -> IndexMap<&'static str, ExtractorDetail> {
+    let extractors: IndexMap<&'static str, ExtractorDetail> = [
         (
-            "faster-bottom-up",
-            extract::faster_bottom_up::FasterBottomUpExtractor.boxed(),
+            "bottom-up",
+            ExtractorDetail {
+                extractor: extract::bottom_up::BottomUpExtractor.boxed(),
+                optimal: Optimal::Tree,
+                use_for_bench: true,
+            },
         ),
         (
-            "greedy-dag",
-            extract::greedy_dag::GreedyDagExtractor.boxed(),
+            "faster-bottom-up",
+            ExtractorDetail {
+                extractor: extract::faster_bottom_up::FasterBottomUpExtractor.boxed(),
+                optimal: Optimal::Tree,
+                use_for_bench: true,
+            },
         ),
-        (
+        /*(
             "faster-greedy-dag",
-            extract::faster_greedy_dag::FasterGreedyDagExtractor.boxed(),
+            ExtractorDetail {
+                extractor: extract::faster_greedy_dag::FasterGreedyDagExtractor.boxed(),
+                optimal: Optimal::Neither,
+                use_for_bench: true,
+            },
+        ),*/
+
+        /*(
+            "global-greedy-dag",
+            ExtractorDetail {
+                extractor: extract::global_greedy_dag::GlobalGreedyDagExtractor.boxed(),
+                optimal: Optimal::Neither,
+                use_for_bench: true,
+            },
+        ),*/
+        #[cfg(feature = "ilp-cbc")]
+        (
+            "ilp-cbc-timeout",
+            ExtractorDetail {
+                extractor: extract::ilp_cbc::CbcExtractorWithTimeout::<10>.boxed(),
+                optimal: Optimal::DAG,
+                use_for_bench: true,
+            },
         ),
+        #[cfg(feature = "ilp-cbc")]
         (
-            "global-greedy-dag",
-            extract::global_greedy_dag::GlobalGreedyDagExtractor.boxed(),
+            "ilp-cbc",
+            ExtractorDetail {
+                extractor: extract::ilp_cbc::CbcExtractor.boxed(),
+                optimal: Optimal::DAG,
+                use_for_bench: false, // takes >10 hours sometimes
+            },
         ),
         #[cfg(feature = "ilp-cbc")]
         (
@@ -44,6 +89,14 @@ fn main() {
     ]
     .into_iter()
     .collect();
+    return extractors;
+}
+
+fn main() {
+    env_logger::init();
+
+    let mut extractors = extractors();
+    extractors.retain(|_, ed| ed.use_for_bench);
 
     let mut args = pico_args::Arguments::from_env();
 
@@ -76,13 +129,13 @@ fn main() {
         .with_context(|| format!("Failed to parse {filename}"))
         .unwrap();
 
-    let extractor = extractors
+    let ed = extractors
         .get(extractor_name.as_str())
         .with_context(|| format!("Unknown extractor: {extractor_name}"))
         .unwrap();
 
     let start_time = std::time::Instant::now();
-    let result = extractor.extract(&egraph, &egraph.root_eclasses);
+    let result = ed.extractor.extract(&egraph, &egraph.root_eclasses);
     let us = start_time.elapsed().as_micros();
 
     result.check(&egraph);
@@ -103,3 +156,6 @@ fn main() {
     )
     .unwrap();
 }
+
+#[cfg(test)]
+mod test;

src/test.rs

@@ -0,0 +1,211 @@
+/*
+ * Checks that no extractors produce better results than the extractors that produce optimal results.
+ * Checks that the extractions are valid.
+ */
+
+use super::*;
+
+use crate::{extractors, Extractor, Optimal, EPSILON_ALLOWANCE};
+pub type Cost = NotNan<f64>;
+use egraph_serialize::{EGraph, Node, NodeId};
+use ordered_float::NotNan;
+use rand::Rng;
+
+// generates a float between 0 and 1
+fn generate_random_not_nan() -> NotNan<f64> {
+    let mut rng: rand::prelude::ThreadRng = rand::thread_rng();
+    let random_float: f64 = rng.gen();
+    NotNan::new(random_float).unwrap()
+}
+
+//make a random egraph that has a loop-free extraction.
+pub fn generate_random_egraph() -> EGraph {
+    let mut rng = rand::thread_rng();
+    let core_node_count = rng.gen_range(1..100) as usize;
+    let extra_node_count = rng.gen_range(1..100);
+    let mut nodes: Vec<Node> = Vec::with_capacity(core_node_count + extra_node_count);
+    let mut eclass = 0;
+
+    let id2nid = |id: usize| -> NodeId { format!("node_{}", id).into() };
+
+    // Unless we do it explicitly, the costs are almost never equal to others' costs or zero:
+    let get_semi_random_cost = |nodes: &Vec<Node>| -> Cost {
+        let mut rng = rand::thread_rng();
+
+        if nodes.len() > 0 && rng.gen_bool(0.1) {
+            return nodes[rng.gen_range(0..nodes.len())].cost;
+        } else if rng.gen_bool(0.05) {
+            return Cost::default();
+        } else {
+            return generate_random_not_nan() * 100.0;
+        }
+    };
+
+    for i in 0..core_node_count {
+        let children: Vec<NodeId> = (0..i).filter(|_| rng.gen_bool(0.1)).map(id2nid).collect();
+
+        if rng.gen_bool(0.2) {
+            eclass += 1;
+        }
+
+        nodes.push(Node {
+            op: "operation".to_string(),
+            children: children,
+            eclass: eclass.to_string().clone().into(),
+            cost: get_semi_random_cost(&nodes),
+        });
+    }
+
+    // So far we have the nodes for a feasible egraph. Now we add some
+    // cycles to extra nodes - nodes that aren't required in the extraction.
+    for _ in 0..extra_node_count {
+        nodes.push(Node {
+            op: "operation".to_string(),
+            children: vec![],
+            eclass: rng.gen_range(0..eclass * 2 + 1).to_string().clone().into(),
+            cost: get_semi_random_cost(&nodes),
+        });
+    }
+
+    for i in core_node_count..nodes.len() {
+        for j in 0..nodes.len() {
+            if rng.gen_bool(0.05) {
+                nodes.get_mut(i).unwrap().children.push(id2nid(j));
+            }
+        }
+    }
+
+    let mut egraph = EGraph::default();
+
+    for i in 0..nodes.len() {
+        egraph.add_node(id2nid(i), nodes[i].clone());
+    }
+
+    // Set roots
+    for _ in 1..rng.gen_range(2..6) {
+        egraph.root_eclasses.push(
+            nodes
+                .get(rng.gen_range(0..core_node_count))
+                .unwrap()
+                .eclass
+                .clone(),
+        );
+    }
+
+    egraph
+}
+
+fn check_optimal_results<I: Iterator<Item = EGraph>>(egraphs: I) {
+    let mut optimal_dag: Vec<Box<dyn Extractor>> = Default::default();
+    let mut optimal_tree: Vec<Box<dyn Extractor>> = Default::default();
+    let mut others: Vec<Box<dyn Extractor>> = Default::default();
+
+    for (_, ed) in extractors().into_iter() {
+        match ed.optimal {
+            Optimal::DAG => optimal_dag.push(ed.extractor),
+            Optimal::Tree => optimal_tree.push(ed.extractor),
+            Optimal::Neither => others.push(ed.extractor),
+        }
+    }
+
+    for egraph in egraphs {
+        let mut optimal_dag_cost: Option<Cost> = None;
+
+        for e in &optimal_dag {
+            let extract = e.extract(&egraph, &egraph.root_eclasses);
+            extract.check(&egraph);
+            let dag_cost = extract.dag_cost(&egraph, &egraph.root_eclasses);
+            let tree_cost = extract.tree_cost(&egraph, &egraph.root_eclasses);
+            if optimal_dag_cost.is_none() {
+                optimal_dag_cost = Some(dag_cost);
+                continue;
+            }
+
+            assert!(
+                (dag_cost.into_inner() - optimal_dag_cost.unwrap().into_inner()).abs()
+                    < EPSILON_ALLOWANCE
+            );
+
+            assert!(
+                tree_cost.into_inner() + EPSILON_ALLOWANCE > optimal_dag_cost.unwrap().into_inner()
+            );
+        }
+
+        let mut optimal_tree_cost: Option<Cost> = None;
+
+        for e in &optimal_tree {
+            let extract = e.extract(&egraph, &egraph.root_eclasses);
+            extract.check(&egraph);
+            let tree_cost = extract.tree_cost(&egraph, &egraph.root_eclasses);
+            if optimal_tree_cost.is_none() {
+                optimal_tree_cost = Some(tree_cost);
+                continue;
+            }
+
+            assert!(
+                (tree_cost.into_inner() - optimal_tree_cost.unwrap().into_inner()).abs()
+                    < EPSILON_ALLOWANCE
+            );
+        }
+
+        if optimal_dag_cost.is_some() && optimal_tree_cost.is_some() {
+            assert!(optimal_dag_cost.unwrap() < optimal_tree_cost.unwrap() + EPSILON_ALLOWANCE);
+        }
+
+        for e in &others {
+            let extract = e.extract(&egraph, &egraph.root_eclasses);
+            extract.check(&egraph);
+            let tree_cost = extract.tree_cost(&egraph, &egraph.root_eclasses);
+            let dag_cost = extract.dag_cost(&egraph, &egraph.root_eclasses);
+
+            // The optimal tree cost should be <= any extractor's tree cost.
+            if optimal_tree_cost.is_some() {
+                assert!(optimal_tree_cost.unwrap() <= tree_cost + EPSILON_ALLOWANCE);
+            }
+
+            if optimal_dag_cost.is_some() {
+                // The optimal dag should be less <= any extractor's dag cost
+                assert!(optimal_dag_cost.unwrap() <= dag_cost + EPSILON_ALLOWANCE);
+            }
+        }
+    }
+}
+
+// Run on all the .json test files
+#[test]
+fn run_on_test_egraphs() {
+    use walkdir::WalkDir;
+
+    let egraphs = WalkDir::new("./test_data/")
+        .into_iter()
+        .filter_map(Result::ok)
+        .filter(|e| {
+            e.file_type().is_file()
+                && e.path().extension().and_then(std::ffi::OsStr::to_str) == Some("json")
+        })
+        .map(|e| e.path().to_string_lossy().into_owned())
+        .map(|e| EGraph::from_json_file(e).unwrap());
+    check_optimal_results(egraphs);
+}
+
+#[test]
+#[should_panic]
+fn check_assert_enabled() {
+    assert!(false);
+}
+
+macro_rules! create_optimal_check_tests {
+    ($($name:ident),*) => {
+        $(
+            #[test]
+            fn $name() {
+                let optimal_dag_found = extractors().into_iter().any(|(_, ed)| ed.optimal == Optimal::DAG);
+                let iterations = if optimal_dag_found { 100 } else { 10000 };
+                let egraphs = (0..iterations).map(|_| generate_random_egraph());
+                check_optimal_results(egraphs);
+            }
+        )*
+    }
+}
+
+create_optimal_check_tests!(check0, check1, check2, check3, check4, check5, check6, check7);

test_data/fuzz/19.json

@@ -0,0 +1 @@
+{"nodes":{"node_0":{"op":"operation","children":[],"eclass":"0","cost":88.33907914523702},"node_1":{"op":"operation","children":[],"eclass":"0","cost":31.380094022709294},"node_2":{"op":"operation","children":[],"eclass":"0","cost":85.46008208334565},"node_3":{"op":"operation","children":[],"eclass":"1","cost":51.03798035678284},"node_4":{"op":"operation","children":[],"eclass":"1","cost":88.57609102193999},"node_5":{"op":"operation","children":["node_3","node_4"],"eclass":"1","cost":29.920743409811124},"node_6":{"op":"operation","children":[],"eclass":"1","cost":83.84156235361831},"node_7":{"op":"operation","children":[],"eclass":"1","cost":15.961085173271005},"node_8":{"op":"operation","children":["node_0","node_4"],"eclass":"2","cost":95.02774306310243},"node_9":{"op":"operation","children":[],"eclass":"2","cost":59.71792436293144},"node_10":{"op":"operation","children":["node_5"],"eclass":"2","cost":68.45922247445554},"node_11":{"op":"operation","children":["node_5","node_6","node_9"],"eclass":"2","cost":57.01097446465635},"node_12":{"op":"operation","children":["node_9"],"eclass":"2","cost":81.34398176692994},"node_13":{"op":"operation","children":["node_0","node_10"],"eclass":"2","cost":88.8278857769136},"node_14":{"op":"operation","children":["node_6"],"eclass":"2","cost":29.12956833073791},"node_15":{"op":"operation","children":["node_7"],"eclass":"2","cost":60.913906070481275},"node_16":{"op":"operation","children":["node_3","node_14"],"eclass":"2","cost":38.327931493858856},"node_17":{"op":"operation","children":[],"eclass":"2","cost":3.0666168877196975},"node_18":{"op":"operation","children":["node_3","node_8","node_13"],"eclass":"2","cost":90.71871650194228},"node_19":{"op":"operation","children":["node_17"],"eclass":"2","cost":36.90315552724963},"node_20":{"op":"operation","children":[],"eclass":"3","cost":68.74450717533955},"node_21":{"op":"operation","children":["node_15"],"eclass":"4","cost":86.4333128929997},"node_22":{"op":"operation","children":[],"eclass":"4","cost":39.135523596243125},"node_23":{"op":"operation","children":["node_0","node_2","node_22"],"eclass":"4","cost":68.33455667461268},"node_24":{"op":"operation","children":["node_2","node_4","node_5","node_7","node_10"],"eclass":"4","cost":48.63662876945013},"node_25":{"op":"operation","children":[],"eclass":"4","cost":5.275616481149159},"node_26":{"op":"operation","children":["node_22"],"eclass":"4","cost":48.82599038198296},"node_27":{"op":"operation","children":["node_7"],"eclass":"4","cost":49.69001156482923},"node_28":{"op":"operation","children":["node_5","node_12","node_20","node_24"],"eclass":"4","cost":7.036804972594057},"node_29":{"op":"operation","children":["node_17","node_28"],"eclass":"4","cost":70.54109393760133},"node_30":{"op":"operation","children":["node_6","node_7","node_14"],"eclass":"5","cost":95.93578126220126},"node_31":{"op":"operation","children":["node_5","node_13","node_25"],"eclass":"5","cost":54.28803013353224},"node_32":{"op":"operation","children":["node_7","node_10","node_30"],"eclass":"5","cost":3.572346212908928},"node_33":{"op":"operation","children":["node_0","node_10","node_15","node_24","node_32"],"eclass":"6","cost":84.31285228381701},"node_34":{"op":"operation","children":["node_1","node_3","node_16","node_22","node_23"],"eclass":"6","cost":6.515351494494381},"node_35":{"op":"operation","children":["node_1","node_13","node_22","node_32"],"eclass":"6","cost":33.036709109796256},"node_36":{"op":"operation","children":["node_24"],"eclass":"7","cost":90.6873521656599},"node_37":{"op":"operation","children":["node_1","node_2","node_8","node_29","node_31"],"eclass":"7","cost":1.8364613848407596},"node_38":{"op":"operation","children":["node_8","node_25","node_30","node_32","node_36","node_37"],"eclass":"7","cost":94.45614868004778},"node_39":{"op":"operation","children":["node_11","node_31","node_33"],"eclass":"7","cost":87.30408798371053},"node_40":{"op":"operation","children":["node_2","node_4","node_8","node_15"],"eclass":"7","cost":99.33533866173661},"node_41":{"op":"operation","children":["node_0","node_3","node_6","node_8","node_11","node_23","node_33","node_38"],"eclass":"7","cost":1.017173923360426},"node_42":{"op":"operation","children":["node_11"],"eclass":"7","cost":67.35391642646766},"node_43":{"op":"operation","children":["node_4","node_6","node_14","node_35","node_37","node_39"],"eclass":"7","cost":25.188737716223553},"node_44":{"op":"operation","children":["node_16","node_21","node_24"],"eclass":"7","cost":6.621741488453536},"node_45":{"op":"operation","children":["node_16","node_18","node_38"],"eclass":"7","cost":37.226773804762125},"node_46":{"op":"operation","children":["node_3","node_25","node_27"],"eclass":"7","cost":98.0170455983861},"node_47":{"op":"operation","children":["node_5","node_10","node_12","node_22","node_25","node_26","node_32","node_38","node_39","node_43"],"eclass":"8","cost":8.286875513283443},"node_48":{"op":"operation","children":["node_11","node_21","node_29"],"eclass":"9","cost":57.43141619732109},"node_49":{"op":"operation","children":["node_14","node_17","node_21","node_22","node_28","node_30"],"eclass":"9","cost":6.31613518850469},"node_50":{"op":"operation","children":["node_20","node_33","node_40"],"eclass":"9","cost":51.70111022535549},"node_51":{"op":"operation","children":["node_26","node_47"],"eclass":"9","cost":50.74398959092199},"node_52":{"op":"operation","children":["node_10","node_13","node_23","node_45","node_51"],"eclass":"9","cost":92.18725389709668},"node_53":{"op":"operation","children":["node_35","node_37","node_44"],"eclass":"9","cost":72.97669763885398},"node_54":{"op":"operation","children":["node_2","node_11","node_12","node_22","node_25","node_32","node_33","node_34"],"eclass":"9","cost":89.03357592761667},"node_55":{"op":"operation","children":["node_3","node_4","node_14","node_34","node_49"],"eclass":"9","cost":28.60041721917057}},"root_eclasses":["7","7","7"]}