test links fail over

1 files changed, 105 insertions, 0 deletions

diff --git a/src/simtest/links.rs b/src/simtest/links.rs
index 17c7ced..e8c2a36 100644
--- a/src/simtest/links.rs
+++ b/src/simtest/links.rs
@@ -151,3 +151,108 @@ async fn links_are_relayed_to_members_that_dont_answer() {
     }
     assert!(!c.links.subscribers().contains(&a.local_id().to_owned()));
 }
+
+/// a and c can't talk; b and d both could relay. Once the relay a asks through is cut
+/// off from c (and from the other relay, so it can't pass the ask on), it brings
+/// nothing: after `FAILED_AFTER` a's renewal goes through the other one, and the
+/// subscription at the first runs out by itself.
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn links_fail_over_to_another_relay() {
+    let sim = Sim::with_identity(67);
+    let cut = crate::transport::sim::LinkParams {
+        loss: 1.0,
+        ..Default::default()
+    };
+    let cut_between = |x: &str, y: &str| {
+        sim.net.set_link(x, y, cut.clone());
+        sim.net.set_link(y, x, cut.clone());
+    };
+    cut_between("a", "c");
+    let b = sim.start_on("b", &["sim"], &[], Start::Genesis);
+    let d = sim.start_on("d", &["sim"], &["b"], Start::Join);
+    let a = sim.start_on("a", &["sim"], &["b"], Start::Join);
+    let c = sim.start_on("c", &["sim"], &["b"], Start::Join);
+    sim.wait_ready(&[&a, &b, &c, &d], SOON).await;
+    let (a_id, c_id) = (a.local_id().to_owned(), c.local_id().to_owned());
+    {
+        let (a, c_id) = (&a, c_id.clone());
+        sim.wait_until(
+            "a knows c, and that it doesn't answer",
+            SOON * 3,
+            move || {
+                let c_id = c_id.clone();
+                async move {
+                    let listed = members::members(&a.doc.read().await.doc).contains(&c_id);
+                    listed && !topology::reaches(a, &c_id).await
+                }
+            },
+        )
+        .await;
+    }
+
+    let lease = Duration::from_secs(3);
+    links::ask_all(&a, vec![c_id.clone()], lease).await;
+    {
+        let (b, d, c_id, a_id) = (&b, &d, c_id.clone(), a_id.clone());
+        sim.wait_until("one of b and d relays c's links to a", SOON, move || {
+            let relayed = (c_id.clone(), vec![a_id.clone()]);
+            async move {
+                b.links.relaying().contains(&relayed) || d.links.relaying().contains(&relayed)
+            }
+        })
+        .await;
+    }
+    let relayed = (c_id.clone(), vec![a_id.clone()]);
+    let (first, other) = if b.links.relaying().contains(&relayed) {
+        (&b, &d)
+    } else {
+        (&d, &b)
+    };
+
+    // The relay in use reaches c no more, nor the other relay; a still reaches it.
+    cut_between(first.local_id(), &c_id);
+    cut_between(first.local_id(), other.local_id());
+    let cut_at = tokio::time::Instant::now();
+
+    // a renews as a dashboard does, and gets c's links through the other relay.
+    let renewing = {
+        let (a, c_id) = (a.clone(), c_id.clone());
+        tokio::spawn(async move {
+            loop {
+                tokio::time::sleep(lease / 2).await;
+                links::ask_all(&a, vec![c_id.clone()], lease).await;
+            }
+        })
+    };
+    {
+        let (a, other, relayed, c_id) = (&a, other, relayed.clone(), c_id.clone());
+        sim.wait_until(
+            "the other relay relays c's links to a, and they arrive",
+            SOON * 3,
+            move || {
+                let (relayed, c_id) = (relayed.clone(), c_id.clone());
+                async move {
+                    other.links.relaying().contains(&relayed)
+                        && a.links
+                            .replies()
+                            .iter()
+                            .any(|(from, age, _)| from == &c_id && *age < cut_at.elapsed())
+                }
+            },
+        )
+        .await;
+    }
+    {
+        let (first, relayed) = (first, relayed.clone());
+        sim.wait_until(
+            "the first relay's subscription ran out",
+            lease * 2,
+            move || {
+                let relayed = relayed.clone();
+                async move { !first.links.relaying().contains(&relayed) }
+            },
+        )
+        .await;
+    }
+    renewing.abort();
+}