diff options
121 files changed, 32183 insertions, 6238 deletions
diff --git a/.cargo/config.toml b/.cargo/config.toml index 8660776..7544206 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,6 +1,9 @@ # Shortcuts for the development loop (docs/development.md). [alias] # Everything but the simulations: a few seconds. -quick = "test --features libp2p,tor -- --skip simtest" +quick = "test --features mycelium,libp2p,tor,dashboard -- --skip simtest" # The simulations: whole swarms in one process, in real time. -sim = "test --features libp2p,tor simtest" +sim = "test --features mycelium,libp2p,tor,dashboard simtest" +# The benchmarks: ignored tests named bench_*, in release builds, one at a time, with +# what they print (`cargo bench` itself takes no aliases). +benchmark = "test --release --features mycelium,libp2p,tor,dashboard bench_ -- --ignored --nocapture --test-threads=1" diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..45e6148 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,8467 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common 0.1.7", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher 0.4.4", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f0f96ce78e38c3dc6d8948aa8163d06385be74000f3c7a95bf1eef35d3ea32" +dependencies = [ + "cipher 0.5.2", + "cpubits", + "cpufeatures 0.3.1", + "zeroize", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes 0.8.4", + "cipher 0.4.4", + "ctr 0.9.2", + "ghash", + "subtle", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "amplify" +version = "4.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f7fb4ac7c881e54a8e7015e399b6112a2a5bc958b6c89ac510840ff20273b31" +dependencies = [ + "amplify_derive", + "amplify_num", + "ascii", + "getrandom 0.2.17", + "getrandom 0.3.4", + "wasm-bindgen", +] + +[[package]] +name = "amplify_derive" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a6309e6b8d89b36b9f959b7a8fa093583b94922a0f6438a24fb08936de4d428" +dependencies = [ + "amplify_syn", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "amplify_num" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afed304556696656d2d71495e1e5f2c4b524a3fb6eb0f2f3778ffc482a40b8a8" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "amplify_syn" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7736fb8d473c0d83098b5bac44df6a561e20470375cd8bcae30516dc889fd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "argon2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "134c52ddac6d63c576bef8168db10c83c49c26444ecbc68060fef078925a901c" +dependencies = [ + "base64ct", + "blake2 0.11.0", + "cpufeatures 0.3.1", + "password-hash", +] + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "arti-client" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43e6531f26fe6f44d71dabdc63b7845bae81547cc738aa942f0e3e482f776388" +dependencies = [ + "async-trait", + "cfg-if", + "derive-deftly", + "derive_builder_fork_arti", + "derive_more", + "educe", + "fs-mistrust", + "futures", + "hostname-validator", + "humantime", + "humantime-serde", + "libc", + "once_cell", + "postage", + "rand 0.10.3", + "safelog", + "serde", + "tempfile", + "thiserror 2.0.21", + "time", + "tor-async-utils", + "tor-basic-utils", + "tor-chanmgr", + "tor-circmgr", + "tor-config", + "tor-config-path", + "tor-dircommon", + "tor-dirmgr", + "tor-error", + "tor-guardmgr", + "tor-hsclient", + "tor-hscrypto", + "tor-hsservice", + "tor-keymgr", + "tor-linkspec", + "tor-llcrypto", + "tor-memquota", + "tor-netdir", + "tor-netdoc", + "tor-persist", + "tor-proto", + "tor-protover", + "tor-rtcompat", + "tracing", + "void", + "web-time-compat", +] + +[[package]] +name = "ascii" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" + +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom 7.1.3", + "num-traits", + "rusticata-macros", + "thiserror 2.0.21", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure 0.13.2", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "assert_matches" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" + +[[package]] +name = "async-compression" +version = "0.4.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5755640120b24961f8231e05892e4a4f7e0d04e8aeca161508c9c397946970" +dependencies = [ + "compression-codecs", + "compression-core", + "futures-io", + "pin-project-lite", +] + +[[package]] +name = "async-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" +dependencies = [ + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-native-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37dd6b179962fe4048a6f81d4c0d7ed419a21fdf49204b4c6b04971693358e79" +dependencies = [ + "futures-util", + "native-tls", + "thiserror 2.0.21", + "url", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "async_executors" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a982d2f86de6137cc05c9db9a915a19886c97911f9790d04f174cede74be01a5" +dependencies = [ + "blanket", + "futures-core", + "futures-task", + "futures-util", + "pin-project", + "rustc_version", + "tokio", +] + +[[package]] +name = "asynchronous-codec" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a860072022177f903e59730004fb5dc13db9275b79bb2aef7ba8ce831956c233" +dependencies = [ + "bytes", + "futures-sink", + "futures-util", + "memchr", + "pin-project-lite", +] + +[[package]] +name = "atomic" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba" + +[[package]] +name = "atomic" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89cbf775b137e9b968e67227ef7f775587cde3fd31b0d8599dbd0f598a48340" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "attohttpc" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e2cdb6d5ed835199484bb92bb8b3edd526effe995c61732580439c1a67e2e9" +dependencies = [ + "base64 0.22.1", + "http", + "log", + "url", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "automerge" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fb849130973a7808a167887f56b1e8bdc36b9b541542602dcb337f34aecf47b" +dependencies = [ + "cfg-if", + "flate2", + "getrandom 0.4.3", + "hex", + "hexane", + "itertools 0.15.0", + "leb128", + "rand 0.10.3", + "rustc-hash", + "serde", + "sha2 0.11.0", + "smol_str", + "thiserror 2.0.21", + "tinyvec", + "tracing", + "unicode-segmentation", +] + +[[package]] +name = "aws-lc-rs" +version = "1.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faac5829c2b74c28f830747e7818ccfb684261b5f48a1118b1e2a13d36dfab13" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1622d8446a2d4b2ce0c7eefc73dd43a99779028d5ee5c2dd8073a658ba8a2bc" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "base-x" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" + +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + +[[package]] +name = "base256emoji" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e9430d9a245a77c92176e649af6e275f20839a48389859d1661e9a128d077c" +dependencies = [ + "const-str", + "match-lookup", +] + +[[package]] +name = "base45" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240e56f4d3c453c36faacb695c535a4d5f8c7d23dac175014f32eb0a71012a03" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bincode" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" +dependencies = [ + "serde", + "unty", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "bitvec" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "blake2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b5d4d889834ee8ecfc0f8426ad30faf7cdcb10f741a8e6d7224d95325479f6f" +dependencies = [ + "digest 0.11.3", +] + +[[package]] +name = "blake3" +version = "1.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" +dependencies = [ + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.1", +] + +[[package]] +name = "blanket" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0b121a9fe0df916e362fb3271088d071159cdf11db0e4182d02152850756eff" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", + "zeroize", +] + +[[package]] +name = "borsh" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a" +dependencies = [ + "bytes", + "cfg_aliases", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", + "regex-automata", + "serde_core", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "by_address" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64fa3c856b712db6612c019f14756e64e4bcea13337a6b33b696333a9eaa2d06" + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "caret" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3135adcc2ec978dbad556b41b7c71e0be7625d70b79159a8a19b5879a761115" + +[[package]] +name = "cc" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" +dependencies = [ + "cfg-if", + "cipher 0.4.4", + "cpufeatures 0.2.17", +] + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core 0.10.1", +] + +[[package]] +name = "chacha20poly1305" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" +dependencies = [ + "aead", + "chacha20 0.9.1", + "cipher 0.4.4", + "poly1305", + "zeroize", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "serde", + "windows-link", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout 0.1.4", + "zeroize", +] + +[[package]] +name = "cipher" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", + "inout 0.2.5", + "zeroize", +] + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim 0.11.1", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "coarsetime" +version = "0.1.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae63702c5627c75addbfb1ea9d1b3842205a6ffd43e4cce884a2a9eb1828fd7" +dependencies = [ + "libc", + "wasix", + "wasm-bindgen", +] + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "combine" +version = "4.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "compression-codecs" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cae36683101b2899f48bd31f8db9839437356188e3b1ff23096e73dab09ba8fe" +dependencies = [ + "compression-core", + "flate2", + "liblzma", + "zstd", + "zstd-safe", +] + +[[package]] +name = "compression-core" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ebdda89edc534bf8d5dd2857562633b6495557e5846552ade994b99ed13c36a" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "console-api" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8599749b6667e2f0c910c1d0dff6901163ff698a52d5a39720f61b5be4b20d3" +dependencies = [ + "futures-core", + "prost", + "prost-types", + "tonic", + "tonic-prost", + "tracing-core", +] + +[[package]] +name = "console-subscriber" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb4915b7d8dd960457a1b6c380114c2944f728e7c65294ab247ae6b6f1f37592" +dependencies = [ + "console-api", + "crossbeam-channel", + "crossbeam-utils", + "futures-task", + "hdrhistogram", + "humantime", + "hyper-util", + "prost", + "prost-types", + "serde", + "serde_json", + "thread_local", + "tokio", + "tokio-stream", + "tonic", + "tracing", + "tracing-core", + "tracing-subscriber", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "const-str" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3" + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "continuum" +version = "1.0.0" +dependencies = [ + "anyhow", + "argon2", + "arti-client", + "async-stream", + "async-trait", + "automerge", + "axum", + "base64 0.23.1", + "blake3", + "bytes", + "clap", + "console-subscriber", + "dashmap", + "ed25519-dalek 3.0.0", + "futures", + "getrandom 0.4.3", + "jiff", + "libc", + "libp2p", + "libp2p-stream", + "password-hash", + "reqwest", + "rmp-serde", + "rustls", + "safelog", + "serde", + "serde_bytes", + "serde_json", + "sha2 0.11.0", + "thiserror 2.0.21", + "tokio", + "toml 1.1.8+spec-1.1.0", + "tor-cell", + "tor-hsservice", + "tor-proto", + "tor-rtcompat", + "tracing", + "tracing-subscriber", + "url", + "utoipa-swagger-ui", + "uuid", +] + +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "cookie-factory" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9885fa71e26b8ab7855e2ec7cae6e9b380edff76cd052e07c683a0319d51b3a2" +dependencies = [ + "futures", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "crossbeam-channel" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher 0.4.4", +] + +[[package]] +name = "ctr" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" +dependencies = [ + "cipher 0.5.2", +] + +[[package]] +name = "ctutils" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03bb0e1cc970d482d121d9a1744999169b69a07470b3d644a7894e53fcaf4574" +dependencies = [ + "cmov", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto 0.2.9", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5eed333089e2e1c1ac8c6c0398e5e2497b4c9926ca6d0365ed1e099afa5bc23" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "curve25519-dalek-derive", + "digest 0.11.3", + "fiat-crypto 0.3.0", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core 0.14.4", + "darling_macro 0.14.4", +] + +[[package]] +name = "darling" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +dependencies = [ + "darling_core 0.21.3", + "darling_macro 0.21.3", +] + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core 0.24.1", + "darling_macro 0.24.1", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn 1.0.109", +] + +[[package]] +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim 0.11.1", + "syn 3.0.6", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core 0.14.4", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core 0.21.3", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core 0.24.1", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + +[[package]] +name = "data-encoding-macro" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6a127ecbb3c4632e1525380e04c0c3fcf8dcb44d32a79ea290d8a36906edcd8" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c54e03a951783e8b327515db3f2a2fd0e3bed362a96b066f341ce66ed49b4ead" +dependencies = [ + "data-encoding", + "syn 3.0.6", +] + +[[package]] +name = "defmt" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "defmt-parser" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" +dependencies = [ + "thiserror 2.0.21", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "cookie-factory", + "displaydoc", + "nom 7.1.3", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "deranged" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "derive-deftly" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e64c6955def7f25b8c3becad9e1762ff6abc3ea62704b0883284c771149b45a6" +dependencies = [ + "derive-deftly-macros", + "heck", +] + +[[package]] +name = "derive-deftly-macros" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36e10ac4bd47bf8d24a4852f001df23b7acbbca6e59d0adec0c2e2c4409dfe5b" +dependencies = [ + "heck", + "indexmap 2.14.2", + "itertools 0.15.0", + "proc-macro-crate", + "proc-macro2", + "quote", + "sha3 0.12.0", + "strum", + "syn 3.0.6", + "unicode-ident", + "void", +] + +[[package]] +name = "derive_builder_core_fork_arti" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24c1b715c79be6328caa9a5e1a387a196ea503740f0722ec3dd8f67a9e72314d" +dependencies = [ + "darling 0.14.4", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derive_builder_fork_arti" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3eae24d595f4d0ecc90a9a5a6d11c2bd8dafe2375ec4a1ec63250e5ade7d228" +dependencies = [ + "derive_builder_macro_fork_arti", +] + +[[package]] +name = "derive_builder_macro_fork_arti" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69887769a2489cd946bf782eb2b1bb2cb7bc88551440c94a765d4f040c08ebf3" +dependencies = [ + "derive_builder_core_fork_arti", + "syn 1.0.109", +] + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.119", + "unicode-xid", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", +] + +[[package]] +name = "directories" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f5094c54661b38d03bd7e50df373292118db60b585c08a411c6d840017fe7d" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "downcast-rs" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" + +[[package]] +name = "dtoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest 0.10.7", + "elliptic-curve", + "rfc6979", + "signature 2.2.0", + "spki", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature 2.2.0", +] + +[[package]] +name = "ed25519" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29fcf32e6c73d1079f83ab4d782de2d81620346a5f38c6237a86a22f8368980a" +dependencies = [ + "signature 3.0.0", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek 4.1.3", + "ed25519 2.2.3", + "merlin", + "rand_core 0.6.4", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + +[[package]] +name = "ed25519-dalek" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ebaa1a2bf1290ab3bfe5a7b771d050ebffab2711c19a81691c683a5144a25de" +dependencies = [ + "curve25519-dalek 5.0.0", + "ed25519 3.0.0", + "sha2 0.11.0", + "subtle", + "zeroize", +] + +[[package]] +name = "educe" +version = "0.4.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f0042ff8246a363dbe77d2ceedb073339e85a804b9a47636c6e016a9a32c05f" +dependencies = [ + "enum-ordinalize", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + +[[package]] +name = "encoding_rs" +version = "0.8.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679" +dependencies = [ + "cfg-if", + "core_detect", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", +] + +[[package]] +name = "enum-ordinalize" +version = "3.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bf1fa3f06bbff1ea5b1a9c7b14aa992a39657db60a2759457328d7e058f49ee" +dependencies = [ + "num-bigint", + "num-traits", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "enum_dispatch" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" +dependencies = [ + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "enumset" +version = "1.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e6bd2d7c76d078cd7a460fa169f778c42c026fe16d90e37d7a45d37181bdce5" +dependencies = [ + "enumset_derive", +] + +[[package]] +name = "enumset_derive" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60587c2044cabd91c4d4c73e3912e9f1ea7d76905f31c7eec51869305dae044" +dependencies = [ + "darling 0.21.3", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "extend" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "311a6d2f1f9d60bff73d2c78a0af97ed27f79672f15c238192a5bbb64db56d00" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fastbloom" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef975e30683b2d965054bb0a836f8973857c4ebf6acf274fe46617cd285060d8" +dependencies = [ + "foldhash 0.2.0", + "libm", + "portable-atomic", + "siphasher", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "fiat-crypto" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" + +[[package]] +name = "figment" +version = "0.10.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cb01cd46b0cf372153850f4c6c272d9cbea2da513e07538405148f95bd789f3" +dependencies = [ + "atomic 0.6.1", + "serde", + "toml 0.8.23", + "uncased", + "version_check", +] + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fluid-let" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "749cff877dc1af878a0b31a41dd221a753634401ea0ef2f87b62d3171522485a" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs-mistrust" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7f210768dd30e30895a6024be8472615bd7b6caec6df4f4105fc03e7608bdd" +dependencies = [ + "derive_builder_fork_arti", + "dirs", + "extend", + "libc", + "pwd-grp", + "serde", + "thiserror 2.0.21", + "void", + "walkdir", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "fslock-guard" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8082ded4af6b2e8e75d0ff71848ad7649d1ddff369a718e8e1db918795967bce" +dependencies = [ + "libc", + "thiserror 2.0.21", + "windows-sys 0.61.2", +] + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-bounded" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b604752cefc5aa3ab98992a107a8bd99465d2825c1584e0b60cb6957b21e19d7" +dependencies = [ + "futures-timer", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "futures-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f2f12607f92c69b12ed746fabf9ca4f5c482cba46679c1a75b874ed7c26adb" +dependencies = [ + "futures-io", + "rustls", + "rustls-pki-types", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" +dependencies = [ + "gloo-timers", + "send_wrapper", +] + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", + "zeroize", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", +] + +[[package]] +name = "getset" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cf442baaabe4213ce7d1239afc26c039180b6456da2cededa316ae2c8a77a77" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval 0.6.2", +] + +[[package]] +name = "glob-match" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9985c9503b412198aa4197559e9a318524ebc4519c229bfa05a535828c950b9d" + +[[package]] +name = "gloo-timers" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "growable-bloom-filter" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d174ccb4ba660d431329e7f0797870d0a4281e36353ec4b4a3c5eab6c2cfb6f1" +dependencies = [ + "serde", + "serde_bytes", + "serde_derive", + "xxhash-rust", +] + +[[package]] +name = "h2" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap 2.14.2", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash 0.2.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash 0.2.0", +] + +[[package]] +name = "hashlink" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" +dependencies = [ + "hashbrown 0.17.1", +] + +[[package]] +name = "hdrhistogram" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49d1053f4708f0af3cf9fc5bffc7e68a914a3c45becb231c80068c9c3f78bea" +dependencies = [ + "base64 0.22.1", + "byteorder", + "flate2", + "nom 8.0.0", + "num-traits", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hexane" +version = "1.0.0-alpha.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81df830f11babb0eee9a082e09135ef0a7d8791f3fe9b279bb9485bf7eaf23e0" +dependencies = [ + "leb128", + "thiserror 2.0.21", +] + +[[package]] +name = "hickory-net" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c480823ed7c2c5d0f09c41020cb6b7c28029ce60ec42dc942158dcf22f8e0a4d" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "futures-channel", + "futures-io", + "futures-util", + "hickory-proto", + "idna", + "ipnet", + "jni", + "rand 0.10.3", + "thiserror 2.0.21", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "hickory-proto" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12b92608f679a6fa515dd1d15c1ff89443026e391200a2c840c7afcba482893d" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni", + "once_cell", + "prefix-trie", + "rand 0.10.3", + "ring", + "thiserror 2.0.21", + "tinyvec", + "tracing", + "url", +] + +[[package]] +name = "hickory-resolver" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3da5255c95d5a716857d54b5b8f4e8d67c3484d3beaaaae2ce25063b3ba981" +dependencies = [ + "cfg-if", + "futures-util", + "hickory-net", + "hickory-proto", + "ipconfig", + "ipnet", + "jni", + "moka", + "ndk-context", + "once_cell", + "parking_lot", + "rand 0.10.3", + "resolv-conf", + "smallvec", + "system-configuration", + "thiserror 2.0.21", + "tokio", + "tracing", +] + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac 0.12.1", +] + +[[package]] +name = "hkdf" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018" +dependencies = [ + "hmac 0.13.0", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + +[[package]] +name = "hostname-validator" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f558a64ac9af88b5ba400d99b579451af0d39c6d360980045b91aac966d705e2" + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" + +[[package]] +name = "humantime-serde" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a3db5ea5923d99402c94e9feb261dc5ee9b4efa158b0315f788cf549cc200c" +dependencies = [ + "humantime", + "serde", +] + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", + "zeroize", +] + +[[package]] +name = "hyper" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "httparse", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "if-addrs" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0a05c691e1fae256cf7013d99dad472dc52d5543322761f83ec8d47eab40d2b" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "if-watch" +version = "3.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71c02a5161c313f0cbdbadc511611893584a10a7b6153cb554bdf83ddce99ec2" +dependencies = [ + "async-io", + "core-foundation 0.9.4", + "fnv", + "futures", + "if-addrs", + "ipnet", + "log", + "netlink-packet-core", + "netlink-packet-route", + "netlink-proto", + "netlink-sys", + "rtnetlink", + "system-configuration", + "tokio", + "windows", +] + +[[package]] +name = "igd-next" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de7238d487a9aff61f81b5ab41c0a841532a115a398b5fa92a2fadd0885e2581" +dependencies = [ + "attohttpc", + "bytes", + "futures", + "http", + "http-body-util", + "hyper", + "hyper-util", + "log", + "rand 0.10.3", + "tokio", + "url", + "xmltree", +] + +[[package]] +name = "imara-diff" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f01d462f766df78ab820dd06f5eb700233c51f0f4c2e846520eaf4ba6aa5c5c" +dependencies = [ + "hashbrown 0.15.5", + "memchr", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "inotify" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cc00ea907cab49550b7da656f80ebb97be1b997d931fbcd28d39734e17ce592" +dependencies = [ + "bitflags 2.13.2", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c033f80b2c113cdf91ab7a33faa9cbc014726dcad99880c8609af2a370edf37d" +dependencies = [ + "libc", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "inout" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b5cf4372d022092674a83b8febf5865f7a573cc0c5baec5e49c879b6a20978f" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "inventory" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6928282826c822ad91bf1c9a1cb90a30ba1c26770749929b4656cd6be829cd7c" +dependencies = [ + "rustversion", +] + +[[package]] +name = "ipconfig" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" +dependencies = [ + "socket2", + "widestring", + "windows-registry", + "windows-result", + "windows-sys 0.61.2", +] + +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" +dependencies = [ + "serde", +] + +[[package]] +name = "iprange" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37209be0ad225457e63814401415e748e2453a5297f9b637338f5fb8afa4ec00" +dependencies = [ + "ipnet", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" +dependencies = [ + "defmt", + "log", +] + +[[package]] +name = "jiff-static" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8377070c6bae868759445e5a77f66d84f0b72f3a054bfb00e6d038b8282da7" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.21", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "k12" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4dc5fdb62af2f520116927304f15d25b3c2667b4817b90efdc045194c912c54" +dependencies = [ + "digest 0.10.7", + "sha3 0.10.9", +] + +[[package]] +name = "keccak" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb26cec98cce3a3d96cbb7bced3c4b16e3d13f27ec56dbd62cbc8f39cfb9d653" +dependencies = [ + "cpufeatures 0.2.17", +] + +[[package]] +name = "keccak" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8f198d1db720e4940b5a493201d199d9f24f568f8f746bd13706243a2f71598" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", +] + +[[package]] +name = "kqueue" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d763e5b24120b4ddf50de6c92308156765aabfbbccebf401da7cff2d70a41ea" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07293a4e297ac234359b510362495713f75ea345d5307140414f20c69ffeb087" +dependencies = [ + "bitflags 2.13.2", + "libc", +] + +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" +dependencies = [ + "spin", +] + +[[package]] +name = "leb128" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c83bff1d572d6b9aeef67ddfc8448e4a3737909cb28e81f97c791b9018703e52" + +[[package]] +name = "libc" +version = "0.2.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" + +[[package]] +name = "liblzma" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa980ea6744074a5b5e0ac6b164aca94fc970121869b3e78ae12a2b0097dbea7" +dependencies = [ + "liblzma-sys", +] + +[[package]] +name = "liblzma-sys" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c7e3581f367a78d7b7e7ae948d023310556f0cfc13156c2e4e00e25616492b9" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libp2p" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bd752f66522780daa3acfd0103a2dccb3921e4c09db5f777bfca98227c8e74d" +dependencies = [ + "bytes", + "either", + "futures", + "futures-timer", + "getrandom 0.4.3", + "libp2p-allow-block-list", + "libp2p-autonat", + "libp2p-connection-limits", + "libp2p-core", + "libp2p-dcutr", + "libp2p-dns", + "libp2p-identify", + "libp2p-identity", + "libp2p-kad", + "libp2p-mdns", + "libp2p-metrics", + "libp2p-noise", + "libp2p-ping", + "libp2p-quic", + "libp2p-relay", + "libp2p-swarm", + "libp2p-tcp", + "libp2p-upnp", + "libp2p-yamux", + "multiaddr", + "pin-project", + "rw-stream-sink", + "thiserror 2.0.21", +] + +[[package]] +name = "libp2p-allow-block-list" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3074eaa93c889a9f56e5e4ecbbb8a5213652b22af0a1d880504345f2bd8768c4" +dependencies = [ + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", +] + +[[package]] +name = "libp2p-autonat" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "651fcbe0470d52db88ace805e94fd074aa18537312d43cbb531f761fa6916b43" +dependencies = [ + "asynchronous-codec", + "either", + "futures", + "futures-bounded", + "futures-timer", + "libp2p-core", + "libp2p-identity", + "libp2p-request-response", + "libp2p-swarm", + "prost", + "prost-codec", + "rand 0.10.3", + "rand_core 0.10.1", + "thiserror 2.0.21", + "tracing", + "web-time", +] + +[[package]] +name = "libp2p-connection-limits" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11b98b22c89fc70113a8988db523b0715e299c804e08da7eb20b8440b8249833" +dependencies = [ + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", +] + +[[package]] +name = "libp2p-core" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2b1a55b9d5e51389db344a609d8c2ff35eabd870a8400067d66d2b1b6f8df68" +dependencies = [ + "either", + "fnv", + "futures", + "futures-timer", + "libp2p-identity", + "multiaddr", + "multihash", + "multistream-select", + "parking_lot", + "pin-project", + "prost", + "rand 0.10.3", + "rw-stream-sink", + "thiserror 2.0.21", + "tracing", + "unsigned-varint", + "web-time", +] + +[[package]] +name = "libp2p-dcutr" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78e2108e262a7a1c4d547bbed120c3b095864f3f093ad19cf510b1785d472c9" +dependencies = [ + "asynchronous-codec", + "either", + "futures", + "futures-bounded", + "futures-timer", + "hashlink", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", + "prost", + "prost-codec", + "thiserror 2.0.21", + "tracing", + "web-time", +] + +[[package]] +name = "libp2p-dns" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18694695e98ff9ce26ee245fdede7cd0b4186cb793b25da3258d9c053400a291" +dependencies = [ + "futures", + "hickory-resolver", + "libp2p-core", + "libp2p-identity", + "parking_lot", + "smallvec", + "tracing", +] + +[[package]] +name = "libp2p-identify" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2dd130b2cbbcecb55e923c5ba4183e6b9af542b4251680017300c693a6e183c" +dependencies = [ + "asynchronous-codec", + "either", + "futures", + "futures-bounded", + "futures-timer", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", + "prost", + "prost-codec", + "smallvec", + "thiserror 2.0.21", + "tracing", +] + +[[package]] +name = "libp2p-identity" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4af6ad75465f6bf26a1001ddbf6f2b6b84a4673bf6de289f68cdbb1430098627" +dependencies = [ + "bs58", + "ed25519-dalek 3.0.0", + "hkdf 0.13.0", + "multihash", + "prost", + "rand 0.10.3", + "sha2 0.11.0", + "thiserror 2.0.21", + "tracing", + "zeroize", +] + +[[package]] +name = "libp2p-kad" +version = "0.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "973caa45045e53f3cf1cf3d596888b82602b30640fd75485836c81aa66e7c38a" +dependencies = [ + "asynchronous-codec", + "bytes", + "either", + "fnv", + "futures", + "futures-bounded", + "futures-timer", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", + "prost", + "prost-codec", + "rand 0.10.3", + "sha2 0.11.0", + "smallvec", + "thiserror 2.0.21", + "tracing", + "uint", + "web-time", +] + +[[package]] +name = "libp2p-mdns" +version = "0.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e82fb6db53f8ba1642bc0695ee589f7bcc1ff9028a9d068a88ee0db5b94667f" +dependencies = [ + "futures", + "hickory-proto", + "if-watch", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", + "rand 0.10.3", + "smallvec", + "socket2", + "tokio", + "tracing", +] + +[[package]] +name = "libp2p-metrics" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3106bb2e4116c9cfa6b87ece6632d9582606fe923aabde27a42a001f3061ff82" +dependencies = [ + "futures", + "libp2p-core", + "libp2p-dcutr", + "libp2p-identify", + "libp2p-identity", + "libp2p-kad", + "libp2p-ping", + "libp2p-relay", + "libp2p-swarm", + "pin-project", + "prometheus-client", + "web-time", +] + +[[package]] +name = "libp2p-noise" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7895a7bf90479291bff16ffddca81f539fc59d62c17b3cce17620cf75abaa6fd" +dependencies = [ + "asynchronous-codec", + "bytes", + "futures", + "getrandom 0.3.4", + "libp2p-core", + "libp2p-identity", + "multiaddr", + "multihash", + "prost", + "rand 0.10.3", + "snow", + "static_assertions", + "thiserror 2.0.21", + "tracing", + "x25519-dalek 3.0.0", + "zeroize", +] + +[[package]] +name = "libp2p-ping" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f4d4bdcf67cf2c71d092b9ffe851d1769e69b97091f37cffd1e596f3763ee9d" +dependencies = [ + "futures", + "futures-timer", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", + "rand 0.10.3", + "tracing", + "web-time", +] + +[[package]] +name = "libp2p-quic" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f78ca359466657b380e469fe8c04df2f4447d1430838e6c3cef4a1c51ccb2ee" +dependencies = [ + "futures", + "futures-timer", + "if-watch", + "libp2p-core", + "libp2p-identity", + "libp2p-tls", + "quinn", + "quinn-proto", + "rand 0.10.3", + "ring", + "socket2", + "thiserror 2.0.21", + "tokio", + "tracing", +] + +[[package]] +name = "libp2p-relay" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5227b053db429996242c00f6c756fd9b5382b942a03eec34b465431df29697a" +dependencies = [ + "asynchronous-codec", + "bytes", + "either", + "futures", + "futures-bounded", + "futures-timer", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", + "prost", + "prost-codec", + "rand 0.10.3", + "static_assertions", + "thiserror 2.0.21", + "tracing", + "web-time", +] + +[[package]] +name = "libp2p-request-response" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca1d1f3a4d7b7080eb959098f276f57ecbc3102f7d3d46f109ccdaa00d2842d9" +dependencies = [ + "futures", + "futures-bounded", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", + "rand 0.10.3", + "smallvec", + "tracing", +] + +[[package]] +name = "libp2p-stream" +version = "0.5.0-alpha" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f601608210b097bfd4433de5fd262fdfddc89b70ebdb00fb7f2f10c2f5ffb677" +dependencies = [ + "futures", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", + "rand 0.10.3", + "tracing", +] + +[[package]] +name = "libp2p-swarm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57ccbe1baeaef036ffde4b265871e11c64d29464036bba635378f356bcdca854" +dependencies = [ + "either", + "fnv", + "futures", + "futures-timer", + "getrandom 0.4.3", + "hashlink", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm-derive", + "multistream-select", + "rand 0.10.3", + "smallvec", + "tokio", + "tracing", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "libp2p-swarm-derive" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbfd25141698cdf6878f26a839f28caa54a7906fbb3e6d0988b9eeb34dea3103" +dependencies = [ + "heck", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "libp2p-tcp" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e753c9e69fec40b3687e38a25e34fc5dfa9c6d0e29614c9b4d28ee7dc8c32a7a" +dependencies = [ + "futures", + "futures-timer", + "if-watch", + "libc", + "libp2p-core", + "socket2", + "tokio", + "tracing", +] + +[[package]] +name = "libp2p-tls" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01dc85ad2bbc3506502f430a00dd913ec87f31e1894291cb9b923168babb6485" +dependencies = [ + "futures", + "futures-rustls", + "libp2p-core", + "libp2p-identity", + "rcgen", + "ring", + "rustls", + "rustls-webpki", + "thiserror 2.0.21", + "x509-parser", + "yasna 0.6.0", +] + +[[package]] +name = "libp2p-upnp" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "298e2b41af943bc4e429f03a72b9991c1353a4c5bbb0a45ae06b393002965b70" +dependencies = [ + "futures", + "futures-timer", + "igd-next", + "libp2p-core", + "libp2p-swarm", + "tokio", + "tracing", +] + +[[package]] +name = "libp2p-yamux" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "308e4033fd1ac9312c827405d3bf568ae335a7d8248071dc7c75c8c0b64977f9" +dependencies = [ + "futures", + "libp2p-core", + "thiserror 2.0.21", + "tracing", + "yamux", +] + +[[package]] +name = "libredox" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" +dependencies = [ + "libc", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.38.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" + +[[package]] +name = "match-lookup" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "549e39695cc0b640f3cb378053832db3d2133422d49e8dcae5c866a2aaf1f730" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memmap2" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b0ca1155a9ecc25aa12e8313a119f32edf0255fea2f44839c4ac0a12b2ac157" +dependencies = [ + "libc", +] + +[[package]] +name = "merlin" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58c38e2799fc0978b65dfff8023ec7843e2330bb462f19198840b34b6582397d" +dependencies = [ + "byteorder", + "keccak 0.1.6", + "rand_core 0.6.4", + "zeroize", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "moka" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9" +dependencies = [ + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "multiaddr" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd707c05eadfb37577f01cbec206726feeecf0b9edea259aee6fc9bc541ee253" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "libp2p-identity", + "multibase", + "multihash", + "percent-encoding", + "serde", + "static_assertions", + "unsigned-varint", + "url", +] + +[[package]] +name = "multibase" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e0e4a371cbf1dfd666b658ba137763edb23c45beb43cfe369b5593cd6b437b6" +dependencies = [ + "base-x", + "base256emoji", + "base45", + "data-encoding", + "data-encoding-macro", +] + +[[package]] +name = "multihash" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "577c63b00ad74d57e8c9aa870b5fccebf2fd64a308a5aee9f1bb88e4aea19447" +dependencies = [ + "unsigned-varint", +] + +[[package]] +name = "multistream-select" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1365a5fcbeb5c688f3c849229d06692e27c9af0c16c3126bd1589b680aa39f88" +dependencies = [ + "bytes", + "futures", + "pin-project", + "smallvec", + "tracing", + "unsigned-varint", +] + +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + +[[package]] +name = "netlink-packet-core" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b897d7bd4f0af82e68d40d0344cf37e97f9c97ddf74a098de3e4da05e96ca395" +dependencies = [ + "paste", +] + +[[package]] +name = "netlink-packet-route" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ce3636fa715e988114552619582b530481fd5ef176a1e5c1bf024077c2c9445" +dependencies = [ + "bitflags 2.13.2", + "libc", + "log", + "netlink-packet-core", +] + +[[package]] +name = "netlink-proto" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93af8261786086024cd5e96e0a991dd65ced07bbf7c233a487bbc96b971d5539" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "log", + "netlink-packet-core", + "netlink-sys", + "thiserror 2.0.21", +] + +[[package]] +name = "netlink-sys" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd6c30ed10fa69cc491d491b85cc971f6bdeb8e7367b7cde2ee6cc878d583fae" +dependencies = [ + "bytes", + "futures-util", + "libc", + "log", + "tokio", +] + +[[package]] +name = "nix" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "nohash-hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "nonany" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b8866ec53810a9a4b3d434a29801e78c707430a9ae11c2db4b8b62bb9675a0" + +[[package]] +name = "notify" +version = "8.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f812702fbc75d35416d3f884e04925be7e9498f4eceea2e24ba34559284674" +dependencies = [ + "bitflags 2.13.2", + "inotify", + "kqueue", + "libc", + "log", + "mio", + "notify-types", + "walkdir", + "windows-sys 0.61.2", +] + +[[package]] +name = "notify-types" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a437a9308232ce794b1e7fdf540d38ea37efc3c73492640c13b4c8568b8f108" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.8", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "critical-section", + "portable-atomic", +] + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "oneshot-fused-workaround" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40e2e88d3d699c52530078d715a6b5b10b3010afe570cfdc0240fee1c9d764ba" +dependencies = [ + "futures", +] + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "openssl" +version = "0.10.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "foreign-types", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "openssl-sys" +version = "0.9.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" +dependencies = [ + "memchr", +] + +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2 0.10.9", +] + +[[package]] +name = "p384" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2 0.10.9", +] + +[[package]] +name = "p521" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fc9e2161f1f215afdfce23677034ae137bbd45016a880c2eb3ba8eb95f085b2" +dependencies = [ + "base16ct", + "ecdsa", + "elliptic-curve", + "primeorder", + "rand_core 0.6.4", + "sha2 0.10.9", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "password-hash" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aab41826031698d6ffcd9cff78ef56ef998e39dc7e5067cdfebe373842d4723b" +dependencies = [ + "getrandom 0.4.3", + "phc", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64 0.22.1", + "serde_core", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phc" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44dc769b75f93afdddd8c7fa12d685292ddeff1e66f7f0f3a234cf1818afe892" +dependencies = [ + "base64ct", + "ctutils", + "getrandom 0.4.3", +] + +[[package]] +name = "phf" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6" +dependencies = [ + "phf_macros", + "phf_shared", + "serde", +] + +[[package]] +name = "phf_generator" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeb62e0959d5a1bebc965f4d15d9e2b7cea002b6b0f5ba8cde6cc26738467100" +dependencies = [ + "fastrand", + "phf_shared", +] + +[[package]] +name = "phf_macros" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fa8d0ca26d424d27630da600c6624696e7dec8bf7b3b492b383c5dc49e5e085" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "phf_shared" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6fd9027e2d9319be6349febd1db4e8d02aa544921200c9b777720ac34a3aa89" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "poly1305" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" +dependencies = [ + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash 0.5.1", +] + +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash 0.5.1", +] + +[[package]] +name = "polyval" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" +dependencies = [ + "cpubits", + "cpufeatures 0.3.1", + "universal-hash 0.6.1", +] + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "postage" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af3fb618632874fb76937c2361a7f22afd393c982a2165595407edc75b06d3c1" +dependencies = [ + "atomic 0.5.3", + "crossbeam-queue", + "futures", + "parking_lot", + "pin-project", + "static_assertions", + "thiserror 1.0.69", +] + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prefix-trie" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf6e3177f0684016a5c209b00882e15f8bdd3f3bb48f0491df10cd102d0c6e7" +dependencies = [ + "either", + "ipnet", + "num-traits", +] + +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + +[[package]] +name = "priority-queue" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93980406f12d9f8140ed5abe7155acb10bb1e69ea55c88960b9c2f117445ef96" +dependencies = [ + "equivalent", + "indexmap 2.14.2", + "serde", +] + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit 0.25.17+spec-1.1.0", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prometheus-client" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1784dc11a05f5a8f57c4a62915686be140f24f70301290b997e317241fa9ffc2" +dependencies = [ + "dtoa", + "itoa", + "parking_lot", + "prometheus-client-derive-encode", +] + +[[package]] +name = "prometheus-client-derive-encode" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01e34894696ff94f64a20c2c373a6440903e9c2789a303d68ec6e6f953f890e4" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-codec" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1da63327bb7a8c30a2227df7f43379d0a4731a2eb517b0f61416da6c2a3c739" +dependencies = [ + "asynchronous-codec", + "bytes", + "prost", + "thiserror 2.0.21", + "unsigned-varint", +] + +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +dependencies = [ + "prost", +] + +[[package]] +name = "pwd-grp" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e2023f41b5fcb7c30eb5300a5733edfaa9e0e0d502d51b586f65633fd39e40c" +dependencies = [ + "derive-deftly", + "libc", + "paste", + "thiserror 2.0.21", +] + +[[package]] +name = "quinn" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" +dependencies = [ + "bytes", + "cfg_aliases", + "futures-io", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.21", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" +dependencies = [ + "aws-lc-rs", + "bytes", + "fastbloom", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.3", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.21", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20 0.10.2", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e6af7f3e25ded52c41df4e0b1af2d047e45896c2f3281792ed68a1c243daedb" +dependencies = [ + "ppv-lite86", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_jitter" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdcd80e68f0a8f9ca5ec7cfd02fd5fbb8fbe6ef4e9b90ea2f48bb929b74f88e" +dependencies = [ + "libc", + "rand_core 0.10.1", + "windows-sys 0.59.0", +] + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + +[[package]] +name = "rangemap" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a611d15b50743feb4c76b7d03edcb0e64f399c26961e4efe6975bc398be6aa3d" + +[[package]] +name = "rcgen" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "yasna 0.5.2", +] + +[[package]] +name = "rdrand" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84448986e59427c795b929d8dbe12d176275c7d0887ee48098c35b7206f51bae" +dependencies = [ + "libc", + "rand_core 0.10.1", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "redox_users" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60dc65c0ff1a7ae1294b0c67b9f14baf70b644404010370171787bfac1038fc0" +dependencies = [ + "libredox", + "thiserror 2.0.21", +] + +[[package]] +name = "ref-cast" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" +dependencies = [ + "base64 0.23.1", + "bytes", + "encoding_rs", + "futures-core", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "reseeding_rng" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecfc8e855ab517235426ec1290bfb0745e73fa478b29619eb66a2a81be9d8716" +dependencies = [ + "rand_core 0.10.1", +] + +[[package]] +name = "resolv-conf" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" + +[[package]] +name = "retry-error" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90a0dea62e6084ede74beea58ee263fd2d70bdb59260a3bf3ede338dfb76f5cc" +dependencies = [ + "humantime", + "web-time", +] + +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac 0.12.1", + "subtle", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rmp" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "rmp-serde" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155" +dependencies = [ + "rmp", + "serde", +] + +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid 0.9.6", + "digest 0.10.7", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "sha2 0.10.9", + "signature 2.2.0", + "spki", + "subtle", + "zeroize", +] + +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror 2.0.21", +] + +[[package]] +name = "rtnetlink" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b960d5d873a75b5be9761b1e73b146f52dddcd27bac75263f40fba686d4d7b5" +dependencies = [ + "futures-channel", + "futures-util", + "log", + "netlink-packet-core", + "netlink-packet-route", + "netlink-proto", + "netlink-sys", + "nix", + "thiserror 1.0.69", + "tokio", +] + +[[package]] +name = "rusqlite" +version = "0.40.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" +dependencies = [ + "bitflags 2.13.2", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", + "sqlite-wasm-rs", + "time", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom 7.1.3", +] + +[[package]] +name = "rustix" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +dependencies = [ + "aws-lc-rs", + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "rw-stream-sink" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57736c2b47f266c94699b56dc7b4f0a5a669ee87e5ef62bc4a54d8e05945353d" +dependencies = [ + "futures", + "pin-project", + "static_assertions", +] + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "safelog" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e091498d7a0a629f344befcf15b1f823a24d5a83d584feeacacc7c764e67c11" +dependencies = [ + "derive_more", + "educe", + "either", + "fluid-let", + "thiserror 2.0.21", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "sanitize-filename" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc984f4f9ceb736a7bb755c3e3bd17dc56370af2600c9780dcc48c66453da34d" +dependencies = [ + "regex", +] + +[[package]] +name = "saturating-time" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a586db154f85f3faf78398c37b6792d71f99d2253d3277571e14f8a8c6e49a65" +dependencies = [ + "web-time-compat", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab508826f74a77ca9d5aba6ff19b522583ee3eaf28a19384ff3d0e5835fadf6e" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.13.2", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "send_wrapper" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde-value" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" +dependencies = [ + "ordered-float", + "serde", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "serde_ignored" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115dffd5f3853e06e746965a20dcbae6ee747ae30b543d91b0e089668bb07798" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_spanned" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7523beb55eece201a2356bee0bbca0d1ab466c14c07703b2e0ee6d42cb0c2c" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9adc193c780ef8f159aee8b61e2d5801aaa555e6eb0947fe45530ec506296f" +dependencies = [ + "base64 0.23.1", + "bs58", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.14.2", + "jiff", + "schemars 0.9.0", + "schemars 1.2.3", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e17bbc68e28663bbbb90df47e058aa7eda4fb445b89fe70457bb94fbccf6e49" +dependencies = [ + "darling 0.24.1", + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + +[[package]] +name = "sha3" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77fd7028345d415a4034cf8777cd4f8ab1851274233b45f84e3d955502d93874" +dependencies = [ + "digest 0.10.7", + "keccak 0.1.6", +] + +[[package]] +name = "sha3" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc9bad02c26382724b2d2692c6f179285e4b54eeecd7968f52a50059c3c11759" +dependencies = [ + "digest 0.11.3", + "keccak 0.2.2", + "sponge-cursor", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shellexpand" +version = "3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32824fab5e16e6c4d86dc1ba84489390419a39f97699852b66480bb87d297ed8" +dependencies = [ + "bstr", + "dirs", + "os_str_bytes", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + +[[package]] +name = "signature" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "siphasher" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "slotmap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038" +dependencies = [ + "serde", + "version_check", +] + +[[package]] +name = "slotmap-careful" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dae6673c710ab011fc8d2ff604b16201df0a59f943976d911d83bbac2a1854f" +dependencies = [ + "paste", + "serde", + "slotmap", + "thiserror 2.0.21", +] + +[[package]] +name = "smallvec" +version = "1.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" + +[[package]] +name = "smol_str" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa7368fcf4852a4c2dd92df0cace6a71f2091ca0a23391ce7f3a31833f1523" +dependencies = [ + "borsh", + "serde_core", +] + +[[package]] +name = "snow" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "599b506ccc4aff8cf7844bc42cf783009a434c1e26c964432560fb6d6ad02d82" +dependencies = [ + "aes-gcm", + "blake2 0.10.6", + "chacha20poly1305", + "curve25519-dalek 4.1.3", + "getrandom 0.3.4", + "ring", + "rustc_version", + "sha2 0.10.9", + "subtle", +] + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sponge-cursor" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a0219bd7d979d58245a4f41f695e1ac9f8befdffadd7f61f1bae9e39abc6620" + +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + +[[package]] +name = "ssh-cipher-fork-arti" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "125c5795103fc93fced42d123c8044180afc55469caa1ab56487c3c5543c898d" +dependencies = [ + "cipher 0.4.4", + "ssh-encoding-fork-arti", +] + +[[package]] +name = "ssh-encoding-fork-arti" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0cf03c3a7ea88451ff83a129a79451fd9891d44fc76c25e916a11848b81814c" +dependencies = [ + "base64ct", + "pem-rfc7468", + "sha2 0.10.9", +] + +[[package]] +name = "ssh-key-fork-arti" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "433782176b73ea7907763dc314c4a17438a231864d4aa683ba47c07c1cf0a388" +dependencies = [ + "num-bigint-dig", + "p256", + "p384", + "p521", + "rand_core 0.6.4", + "rsa", + "sec1", + "sha2 0.10.9", + "signature 2.2.0", + "ssh-cipher-fork-arti", + "ssh-encoding-fork-arti", + "subtle", + "zeroize", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "sysinfo" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags 2.13.2", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl 2.0.21", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "js-sys", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "serde_core", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + +[[package]] +name = "tokio" +version = "1.53.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e95f91fcc7a621e8b030f6aa23c71fe9838ae2fb4d8118b75602a328f5144044" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-io", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "toml_edit 0.22.27", +] + +[[package]] +name = "toml" +version = "1.1.8+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20489e00e4d8741d6be680764cc12e270655e375a20d1011e844a9c3379e678d" +dependencies = [ + "indexmap 2.14.2", + "serde_core", + "serde_spanned 1.1.2", + "toml_datetime 1.1.2+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 1.0.4", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_datetime" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap 2.14.2", + "serde", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "toml_write", + "winnow 0.7.15", +] + +[[package]] +name = "toml_edit" +version = "0.25.17+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3641d5bbb5349a79e1020a242d251efbc546ad8048d133958323ce9c40a9c9c" +dependencies = [ + "indexmap 2.14.2", + "toml_datetime 1.1.2+spec-1.1.0", + "toml_parser", + "winnow 1.0.4", +] + +[[package]] +name = "toml_parser" +version = "1.1.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" +dependencies = [ + "winnow 1.0.4", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "toml_writer" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06bdbd8cfc056b8d2e2e85f29b56a3bdbecb527cef81eb39e3e7b98af4652770" + +[[package]] +name = "tonic" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" +dependencies = [ + "async-trait", + "axum", + "base64 0.22.1", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "socket2", + "sync_wrapper", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-prost" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" +dependencies = [ + "bytes", + "prost", + "tonic", +] + +[[package]] +name = "tor-async-utils" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64e39ddd4c352b8e3166ea0ec49b82a33ad9eeae8bdbbf76e1d2f1e856a6316b" +dependencies = [ + "cfg-if", + "derive-deftly", + "educe", + "extend", + "futures", + "oneshot-fused-workaround", + "pin-project", + "postage", + "slotmap-careful", + "sync_wrapper", + "thiserror 2.0.21", + "tokio", + "tokio-util", + "tor-basic-utils", + "tor-rtcompat", + "tracing", + "void", + "web-time-compat", +] + +[[package]] +name = "tor-basic-utils" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e44c7b006f22966d44a2a0a6c1ee71e472c62182c7f0460077df3294d52f23c4" +dependencies = [ + "derive-deftly", + "derive_more", + "educe", + "extend", + "getrandom 0.4.3", + "hex", + "itertools 0.15.0", + "libc", + "paste", + "rand 0.10.3", + "rand_chacha 0.10.0", + "serde", + "slab", + "smallvec", + "thiserror 2.0.21", + "tracing", + "weak-table", + "web-time-compat", +] + +[[package]] +name = "tor-bytes" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccfed3adc9db61334445e921b95c26c896994db64e031c31e5cc48bd73e57ac7" +dependencies = [ + "bytes", + "derive-deftly", + "digest 0.10.7", + "educe", + "getrandom 0.4.3", + "safelog", + "thiserror 2.0.21", + "tor-error", + "tor-llcrypto", + "zeroize", +] + +[[package]] +name = "tor-cell" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2d6398fd085306de5af739899d0390674b8646d4ff6cfb69d65f40a5f9d88da" +dependencies = [ + "amplify", + "bitflags 2.13.2", + "bytes", + "caret", + "derive-deftly", + "derive_more", + "educe", + "itertools 0.15.0", + "paste", + "rand 0.10.3", + "smallvec", + "thiserror 2.0.21", + "tor-basic-utils", + "tor-bytes", + "tor-cert", + "tor-error", + "tor-hscrypto", + "tor-linkspec", + "tor-llcrypto", + "tor-memquota", + "tor-protover", + "tor-units", + "void", +] + +[[package]] +name = "tor-cert" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98dec924caf6e9548b4860878373132bf89536eb0d1658666df12dff72b855cd" +dependencies = [ + "caret", + "derive_builder_fork_arti", + "derive_more", + "digest 0.10.7", + "thiserror 2.0.21", + "tor-bytes", + "tor-checkable", + "tor-error", + "tor-llcrypto", + "web-time-compat", +] + +[[package]] +name = "tor-chanmgr" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "431ba53bf020ea8681ef0c2fbf0f0f854997fca9afe1b2c081f41f14ec00cf5d" +dependencies = [ + "async-trait", + "base64ct", + "caret", + "cfg-if", + "derive-deftly", + "derive_more", + "educe", + "futures", + "httparse", + "oneshot-fused-workaround", + "percent-encoding", + "postage", + "rand 0.10.3", + "safelog", + "serde", + "serde_with", + "thiserror 2.0.21", + "tor-async-utils", + "tor-basic-utils", + "tor-cell", + "tor-config", + "tor-error", + "tor-keymgr", + "tor-linkspec", + "tor-llcrypto", + "tor-memquota", + "tor-netdir", + "tor-proto", + "tor-rtcompat", + "tor-socksproto", + "tor-units", + "tracing", + "url", + "void", + "web-time-compat", +] + +[[package]] +name = "tor-checkable" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e58a50a2dbf5d21758c9e2dbaaf085403670b8bf52475c815dd1185698b7af9a" +dependencies = [ + "humantime", + "itertools 0.15.0", + "signature 2.2.0", + "thiserror 2.0.21", + "tor-llcrypto", + "web-time-compat", +] + +[[package]] +name = "tor-circmgr" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6b55747c690e0c931c5a081640ffdca6d1003c01b57b8102afcbdbaf3ec3fd" +dependencies = [ + "amplify", + "async-trait", + "cfg-if", + "derive-deftly", + "derive_builder_fork_arti", + "derive_more", + "downcast-rs", + "dyn-clone", + "educe", + "futures", + "humantime-serde", + "itertools 0.15.0", + "once_cell", + "oneshot-fused-workaround", + "pin-project", + "rand 0.10.3", + "retry-error", + "safelog", + "serde", + "thiserror 2.0.21", + "tor-async-utils", + "tor-basic-utils", + "tor-cell", + "tor-chanmgr", + "tor-config", + "tor-dircommon", + "tor-error", + "tor-guardmgr", + "tor-linkspec", + "tor-memquota", + "tor-netdir", + "tor-netdoc", + "tor-persist", + "tor-proto", + "tor-protover", + "tor-relay-selection", + "tor-rtcompat", + "tor-units", + "tracing", + "void", + "weak-table", + "web-time-compat", +] + +[[package]] +name = "tor-config" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b849c69c695b9534cf945c0422b85eaf5760bb6141a73c47b72fd6f95ea4d719" +dependencies = [ + "amplify", + "cfg-if", + "derive-deftly", + "derive_builder_fork_arti", + "educe", + "either", + "extend", + "figment", + "fs-mistrust", + "futures", + "humantime-serde", + "itertools 0.15.0", + "notify", + "paste", + "postage", + "regex", + "serde", + "serde-value", + "serde_ignored", + "strum", + "thiserror 2.0.21", + "toml 1.1.8+spec-1.1.0", + "tor-basic-utils", + "tor-error", + "tor-rtcompat", + "tracing", + "void", +] + +[[package]] +name = "tor-config-path" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b7f10bfd0d0fb159a8abc13d2593625bfd9f9a394715605262236d01bc5500f" +dependencies = [ + "directories", + "serde", + "shellexpand", + "thiserror 2.0.21", + "tor-error", + "tor-general-addr", +] + +[[package]] +name = "tor-consdiff" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ed097865bfb6d453d9258620cbc952cf7481e9dcfc8901071ee1b8506cea86" +dependencies = [ + "derive_more", + "digest 0.10.7", + "hex", + "imara-diff", + "static_assertions", + "thiserror 2.0.21", + "tor-error", + "tor-llcrypto", + "tor-netdoc", +] + +[[package]] +name = "tor-dirclient" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f942f4ebaf2ac5275e48d02d77abe1616ce2e8854f032a1b0698ede0566dddfc" +dependencies = [ + "async-compression", + "base64ct", + "derive_more", + "futures", + "hex", + "http", + "httparse", + "httpdate", + "itertools 0.15.0", + "thiserror 2.0.21", + "tor-circmgr", + "tor-error", + "tor-hscrypto", + "tor-linkspec", + "tor-llcrypto", + "tor-netdoc", + "tor-proto", + "tor-rtcompat", + "tracing", + "web-time-compat", +] + +[[package]] +name = "tor-dircommon" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c16065e5166e529492d1be856e1e40316e2532053027c6b7ab5a4a53c14d8c51" +dependencies = [ + "base64ct", + "derive-deftly", + "getset", + "humantime", + "humantime-serde", + "serde", + "tor-basic-utils", + "tor-checkable", + "tor-config", + "tor-linkspec", + "tor-llcrypto", + "tor-netdoc", + "tracing", +] + +[[package]] +name = "tor-dirmgr" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34ea5e77b43bf1ef5fa213a89cb67080f218df6b418c82527e6bb9862a39aef5" +dependencies = [ + "async-trait", + "base64ct", + "derive_builder_fork_arti", + "derive_more", + "digest 0.10.7", + "educe", + "event-listener", + "fs-mistrust", + "fslock-guard", + "futures", + "hex", + "humantime", + "humantime-serde", + "itertools 0.15.0", + "memmap2", + "oneshot-fused-workaround", + "paste", + "postage", + "rand 0.10.3", + "rusqlite", + "safelog", + "scopeguard", + "serde", + "serde_json", + "signature 2.2.0", + "static_assertions", + "strum", + "thiserror 2.0.21", + "time", + "tor-async-utils", + "tor-basic-utils", + "tor-checkable", + "tor-circmgr", + "tor-config", + "tor-consdiff", + "tor-dirclient", + "tor-dircommon", + "tor-error", + "tor-guardmgr", + "tor-llcrypto", + "tor-netdir", + "tor-netdoc", + "tor-persist", + "tor-proto", + "tor-protover", + "tor-rtcompat", + "tracing", + "void", + "web-time-compat", +] + +[[package]] +name = "tor-error" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca01de513501849b2298e4f16f8760832ef43344b10fe17600daed9a35128b24" +dependencies = [ + "derive_more", + "futures", + "paste", + "retry-error", + "static_assertions", + "strum", + "thiserror 2.0.21", + "tracing", + "void", + "web-time-compat", +] + +[[package]] +name = "tor-general-addr" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72965a7044f573bb121467404d423d296592dd3cef87ad7aca4c96cc2b65e75a" +dependencies = [ + "derive_more", + "thiserror 2.0.21", + "void", +] + +[[package]] +name = "tor-guardmgr" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87694603d6dcee5e0b050f0e7ac8be86348eb78ecb90c36ee8b70bd5822f3eff" +dependencies = [ + "amplify", + "base64ct", + "derive-deftly", + "derive_builder_fork_arti", + "derive_more", + "dyn-clone", + "educe", + "futures", + "humantime", + "humantime-serde", + "itertools 0.15.0", + "num_enum", + "oneshot-fused-workaround", + "pin-project", + "postage", + "rand 0.10.3", + "safelog", + "serde", + "strum", + "thiserror 2.0.21", + "tor-async-utils", + "tor-basic-utils", + "tor-config", + "tor-dircommon", + "tor-error", + "tor-linkspec", + "tor-llcrypto", + "tor-netdir", + "tor-netdoc", + "tor-persist", + "tor-proto", + "tor-relay-selection", + "tor-rtcompat", + "tor-units", + "tracing", + "web-time-compat", +] + +[[package]] +name = "tor-hsclient" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42addb7f2a7a86e813df223ed4485f2fb61b5736aff040cb5e8f551f83dfade0" +dependencies = [ + "async-trait", + "cfg-if", + "derive-deftly", + "derive_more", + "educe", + "either", + "futures", + "itertools 0.15.0", + "oneshot-fused-workaround", + "postage", + "rand 0.10.3", + "retry-error", + "safelog", + "slotmap-careful", + "strum", + "thiserror 2.0.21", + "tor-async-utils", + "tor-basic-utils", + "tor-bytes", + "tor-cell", + "tor-checkable", + "tor-circmgr", + "tor-config", + "tor-dirclient", + "tor-error", + "tor-hscrypto", + "tor-keymgr", + "tor-linkspec", + "tor-llcrypto", + "tor-memquota", + "tor-netdir", + "tor-netdoc", + "tor-persist", + "tor-proto", + "tor-protover", + "tor-rtcompat", + "tracing", + "web-time-compat", +] + +[[package]] +name = "tor-hscrypto" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6613c5336b71ebd34b8d90c233a7b6b06a0b81899d0545c656b0cac6f24e54a2" +dependencies = [ + "cipher 0.5.2", + "data-encoding", + "derive-deftly", + "derive_more", + "digest 0.10.7", + "hex", + "humantime", + "itertools 0.15.0", + "paste", + "rand 0.10.3", + "safelog", + "serde", + "signature 2.2.0", + "subtle", + "thiserror 2.0.21", + "tor-basic-utils", + "tor-bytes", + "tor-error", + "tor-key-forge", + "tor-llcrypto", + "tor-memquota", + "tor-units", + "void", + "web-time-compat", + "zeroize", +] + +[[package]] +name = "tor-hsservice" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55809560d9295dcc2ee77002c5a4275d1f5f5a54203de7cd6acb52215f5344bd" +dependencies = [ + "amplify", + "async-trait", + "base64ct", + "cfg-if", + "derive-deftly", + "derive_builder_fork_arti", + "derive_more", + "digest 0.10.7", + "educe", + "fs-mistrust", + "futures", + "growable-bloom-filter", + "hex", + "humantime", + "itertools 0.15.0", + "k12", + "once_cell", + "oneshot-fused-workaround", + "postage", + "rand 0.10.3", + "rand_core 0.10.1", + "retry-error", + "safelog", + "serde", + "serde_with", + "strum", + "thiserror 2.0.21", + "tor-async-utils", + "tor-basic-utils", + "tor-bytes", + "tor-cell", + "tor-circmgr", + "tor-config", + "tor-config-path", + "tor-dirclient", + "tor-error", + "tor-hscrypto", + "tor-keymgr", + "tor-linkspec", + "tor-llcrypto", + "tor-log-ratelim", + "tor-netdir", + "tor-netdoc", + "tor-persist", + "tor-proto", + "tor-protover", + "tor-relay-selection", + "tor-rtcompat", + "tracing", + "void", + "web-time-compat", +] + +[[package]] +name = "tor-key-forge" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5534581a7f347ffcade7ea32ecccdbc035d36468bc8c03b0dce83363b6d41801" +dependencies = [ + "derive-deftly", + "derive_more", + "downcast-rs", + "paste", + "rand 0.10.3", + "rsa", + "signature 2.2.0", + "ssh-key-fork-arti", + "thiserror 2.0.21", + "tor-bytes", + "tor-cert", + "tor-checkable", + "tor-error", + "tor-llcrypto", +] + +[[package]] +name = "tor-keymgr" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e269210a2ba305bf6524990a3c37aab4529b92a01f3d15ef6af23c067c8eb2c4" +dependencies = [ + "amplify", + "arrayvec", + "cfg-if", + "data-encoding", + "derive-deftly", + "derive_builder_fork_arti", + "derive_more", + "downcast-rs", + "dyn-clone", + "fs-mistrust", + "glob-match", + "humantime", + "inventory", + "itertools 0.15.0", + "rand 0.10.3", + "safelog", + "serde", + "signature 2.2.0", + "ssh-key-fork-arti", + "thiserror 2.0.21", + "tor-basic-utils", + "tor-bytes", + "tor-config", + "tor-config-path", + "tor-error", + "tor-hscrypto", + "tor-key-forge", + "tor-llcrypto", + "tor-persist", + "tracing", + "visibility", + "walkdir", + "web-time-compat", + "zeroize", +] + +[[package]] +name = "tor-linkspec" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d287978bd5e1ffbe7ca2eb8daadb4e06e42d084bdad26bea4d4bbe1ff877e10f" +dependencies = [ + "base64ct", + "by_address", + "caret", + "derive-deftly", + "derive_builder_fork_arti", + "derive_more", + "hex", + "itertools 0.15.0", + "safelog", + "serde", + "serde_with", + "strum", + "thiserror 2.0.21", + "tor-basic-utils", + "tor-bytes", + "tor-config", + "tor-llcrypto", + "tor-memquota", + "tor-protover", +] + +[[package]] +name = "tor-llcrypto" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95f921a609ff73736e8e21d11dbee6617cd7568841790f2e3327295627a51b75" +dependencies = [ + "aes 0.9.3", + "base64ct", + "ctr 0.10.1", + "curve25519-dalek 4.1.3", + "der-parser", + "derive-deftly", + "derive_more", + "digest 0.10.7", + "ed25519-dalek 2.2.0", + "educe", + "getrandom 0.2.17", + "getrandom 0.3.4", + "getrandom 0.4.3", + "hex", + "rand 0.10.3", + "rand_chacha 0.10.0", + "rand_core 0.10.1", + "rand_core 0.6.4", + "rand_jitter", + "rdrand", + "reseeding_rng", + "rsa", + "safelog", + "serde", + "sha1", + "sha2 0.10.9", + "sha3 0.10.9", + "signature 2.2.0", + "subtle", + "thiserror 2.0.21", + "tor-error", + "tor-memquota-cost", + "visibility", + "x25519-dalek 2.0.1", + "zeroize", +] + +[[package]] +name = "tor-log-ratelim" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2096a87f0691f0781bf564249c524d62b5338c3954ba1c6ef10ec90df1d97733" +dependencies = [ + "futures", + "humantime", + "thiserror 2.0.21", + "tor-error", + "tor-rtcompat", + "tracing", + "weak-table", + "web-time-compat", +] + +[[package]] +name = "tor-memquota" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96853fc516264fe2ebefbe40d276b6a3b84e15fc08b74fd7a85e2c5df9f11c37" +dependencies = [ + "cfg-if", + "derive-deftly", + "derive_more", + "dyn-clone", + "educe", + "extend", + "futures", + "itertools 0.15.0", + "paste", + "pin-project", + "serde", + "slotmap-careful", + "static_assertions", + "sysinfo", + "thiserror 2.0.21", + "tor-async-utils", + "tor-basic-utils", + "tor-config", + "tor-error", + "tor-log-ratelim", + "tor-memquota-cost", + "tor-rtcompat", + "tracing", + "void", +] + +[[package]] +name = "tor-memquota-cost" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5e5712bfe2995a60eddcc32d07e2e38a42120318a8f5411d621bf0a8fa77219" +dependencies = [ + "derive-deftly", + "itertools 0.15.0", + "paste", + "void", +] + +[[package]] +name = "tor-netdir" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd22ffd4d6aa9389ec8e20e9b0d579d61101682c5ec8379115a388e473e83297" +dependencies = [ + "async-trait", + "bitflags 2.13.2", + "derive_more", + "digest 0.10.7", + "futures", + "hex", + "humantime", + "itertools 0.15.0", + "num_enum", + "rand 0.10.3", + "serde", + "strum", + "thiserror 2.0.21", + "time", + "tor-basic-utils", + "tor-error", + "tor-hscrypto", + "tor-linkspec", + "tor-llcrypto", + "tor-netdoc", + "tor-protover", + "tor-units", + "tracing", + "typed-index-collections", + "web-time-compat", +] + +[[package]] +name = "tor-netdoc" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78dc8a397a86fd00b17607941b204281dd8a77a6018ca8eedc257bdccb4f3aa7" +dependencies = [ + "amplify", + "base64ct", + "cipher 0.5.2", + "derive-deftly", + "derive_builder_fork_arti", + "derive_more", + "digest 0.10.7", + "educe", + "enumset", + "extend", + "hex", + "hostname-validator", + "humantime", + "ipnet", + "iprange", + "itertools 0.15.0", + "memchr", + "paste", + "phf", + "rand 0.10.3", + "rangemap", + "saturating-time", + "serde", + "serde_with", + "signature 2.2.0", + "smallvec", + "strum", + "subtle", + "thiserror 2.0.21", + "time", + "tinystr", + "tor-basic-utils", + "tor-bytes", + "tor-cell", + "tor-cert", + "tor-checkable", + "tor-error", + "tor-hscrypto", + "tor-linkspec", + "tor-llcrypto", + "tor-protover", + "tor-units", + "void", + "web-time-compat", + "zeroize", +] + +[[package]] +name = "tor-persist" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a3bc60cbe74bf41cd63e2506a9a16bd52047668771578b12ff2453ec4c7522" +dependencies = [ + "amplify", + "derive-deftly", + "derive_more", + "extend", + "filetime", + "fs-mistrust", + "fslock-guard", + "futures", + "itertools 0.15.0", + "oneshot-fused-workaround", + "paste", + "sanitize-filename", + "serde", + "serde_json", + "thiserror 2.0.21", + "time", + "tor-basic-utils", + "tor-error", + "tracing", + "void", + "web-time-compat", +] + +[[package]] +name = "tor-proto" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fceb9d4017849440584e541054b6a26517655a3ae48942b6e67476c5376262c3" +dependencies = [ + "aes 0.9.3", + "amplify", + "async-trait", + "asynchronous-codec", + "bitvec", + "bytes", + "caret", + "cfg-if", + "cipher 0.5.2", + "coarsetime", + "ctr 0.10.1", + "derive-deftly", + "derive_builder_fork_arti", + "derive_more", + "digest 0.10.7", + "educe", + "enum_dispatch", + "futures", + "futures-util", + "hkdf 0.12.4", + "hmac 0.12.1", + "itertools 0.15.0", + "nonany", + "oneshot-fused-workaround", + "pin-project", + "polyval 0.7.3", + "postage", + "rand 0.10.3", + "rand_core 0.10.1", + "safelog", + "slotmap-careful", + "smallvec", + "static_assertions", + "strum", + "subtle", + "thiserror 2.0.21", + "tokio", + "tokio-util", + "tor-async-utils", + "tor-basic-utils", + "tor-bytes", + "tor-cell", + "tor-cert", + "tor-checkable", + "tor-config", + "tor-error", + "tor-hscrypto", + "tor-linkspec", + "tor-llcrypto", + "tor-log-ratelim", + "tor-memquota", + "tor-protover", + "tor-rtcompat", + "tor-rtmock", + "tor-units", + "tracing", + "typenum", + "visibility", + "void", + "web-time-compat", + "zeroize", +] + +[[package]] +name = "tor-protover" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e69fbc196208a9d6232d2c38e32806f3bb064792fd84ac6ee12b5ba1e482240" +dependencies = [ + "caret", + "derive-deftly", + "paste", + "serde_with", + "thiserror 2.0.21", + "tor-basic-utils", + "tor-bytes", +] + +[[package]] +name = "tor-relay-selection" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e420365ef1e89ee93b34b5ab71a6aa840cadcbe04e2ac4b72043846fb2f9e002" +dependencies = [ + "rand 0.10.3", + "serde", + "tor-basic-utils", + "tor-linkspec", + "tor-netdir", + "tor-netdoc", +] + +[[package]] +name = "tor-rtcompat" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99036f7a4fa9735e0beac7f241592da1a072c94a018e4fa702001700f5922291" +dependencies = [ + "amplify", + "async-native-tls", + "async-trait", + "async_executors", + "asynchronous-codec", + "cfg-if", + "coarsetime", + "derive_builder_fork_arti", + "derive_more", + "dyn-clone", + "educe", + "futures", + "hex", + "libc", + "native-tls", + "paste", + "pin-project", + "socket2", + "thiserror 2.0.21", + "tokio", + "tokio-util", + "tor-error", + "tor-general-addr", + "tracing", + "void", + "web-time-compat", + "zeroize", +] + +[[package]] +name = "tor-rtmock" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9140515718465f1ede62fa33e96cc929acf075a3893cb349d1c606976c846a21" +dependencies = [ + "amplify", + "assert_matches", + "async-trait", + "derive-deftly", + "derive_more", + "educe", + "futures", + "humantime", + "itertools 0.15.0", + "oneshot-fused-workaround", + "pin-project", + "priority-queue", + "slotmap-careful", + "strum", + "thiserror 2.0.21", + "tor-error", + "tor-general-addr", + "tor-rtcompat", + "tracing", + "tracing-test", + "void", + "web-time-compat", +] + +[[package]] +name = "tor-socksproto" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62d1f5c19f2339b2634556ed35865844276ab889708b482dc351fa18184c5b24" +dependencies = [ + "caret", + "derive-deftly", + "educe", + "safelog", + "subtle", + "thiserror 2.0.21", + "tor-bytes", + "tor-error", +] + +[[package]] +name = "tor-units" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa976f4c84404c9aaf7e8690fe30904a69084d0069ff7b2dfc0c58919d65eebd" +dependencies = [ + "derive-deftly", + "derive_more", + "serde", + "thiserror 2.0.21", + "tor-memquota", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 2.14.2", + "pin-project-lite", + "slab", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags 2.13.2", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "tracing-test" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19a4c448db514d4f24c5ddb9f73f2ee71bfb24c526cf0c570ba142d1119e0051" +dependencies = [ + "tracing-core", + "tracing-subscriber", + "tracing-test-macro", +] + +[[package]] +name = "tracing-test-macro" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad06847b7afb65c7866a36664b75c40b895e318cea4f71299f013fb22965329d" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typed-index-collections" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "898160f1dfd383b4e92e17f0512a7d62f3c51c44937b23b6ffc3a1614a8eaccd" +dependencies = [ + "bincode", + "serde", +] + +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "uint" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2833f28e3adbd1b867292ee4053252ea89ab85e44e5e4701ffead8cada0539fa" +dependencies = [ + "byteorder", + "crunchy", + "hex", + "static_assertions", +] + +[[package]] +name = "uncased" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicase" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357cc3acc6a036009fd6c973ed009037c732d60d0b4f6c673e9041497482a28f" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common 0.1.7", + "subtle", +] + +[[package]] +name = "universal-hash" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4987bdc12753382e0bec4a65c50738ffaabc998b9cdd1f952fb5f39b0048a96" +dependencies = [ + "crypto-common 0.2.2", + "ctutils", +] + +[[package]] +name = "unsigned-varint" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "unty" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "utoipa" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8765fe27aeff71012a3f90fa474cf1df863a7d0dd81ed25de4e63fff2544994f" +dependencies = [ + "indexmap 2.14.2", + "serde", + "serde_json", + "utoipa-gen", +] + +[[package]] +name = "utoipa-gen" +version = "6.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d935f1c83fdf8b88f09bbe050d0cea78ea4afee6d7b0317403228c1200b2c8ab" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "utoipa-swagger-ui" +version = "10.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f5ec9d7816e4ce8ccb7e0bd7276e090cad0fcb1def4f3540e5b0f7485d7583e" +dependencies = [ + "axum", + "base64 0.23.1", + "mime_guess", + "regex", + "serde", + "serde_json", + "url", + "utoipa", + "zip", +] + +[[package]] +name = "uuid" +version = "1.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cc1186384beb7dd8eedea376413fd654937285ea6c9cfbb928dc3043ea4b606" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "visibility" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasix" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae86f02046da16a333a9129d31451423e1657737ecdafed4193838a5f54c5cfe" +dependencies = [ + "wasi", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" +dependencies = [ + "cfg-if", + "futures-util", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "weak-table" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eba2ec791141b0bf4db542f6b038a8f9310cee33e076cb181245780b09a24ff" +dependencies = [ + "cfg-if", + "hashbrown 0.16.1", +] + +[[package]] +name = "web-sys" +version = "0.3.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time-compat" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c38a07cb4a2705c91feaedc9013a65481ec1b38d818b885d642b9f06ec6edd" +dependencies = [ + "web-time", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "x25519-dalek" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" +dependencies = [ + "curve25519-dalek 4.1.3", + "rand_core 0.6.4", + "serde", + "zeroize", +] + +[[package]] +name = "x25519-dalek" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e8131a03190127fb2263afc72b322ecadae46b6ff8c6f399ff5d02f5559af6" +dependencies = [ + "curve25519-dalek 5.0.0", + "rand_core 0.10.1", + "zeroize", +] + +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom 7.1.3", + "oid-registry", + "rusticata-macros", + "thiserror 2.0.21", + "time", +] + +[[package]] +name = "xml-rs" +version = "0.8.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e450f9b2ed1dff33c94c12589a87338689467b9c4f5d8a5710bd09a847d2c8a7" + +[[package]] +name = "xmltree" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7d8a75eaf6557bb84a65ace8609883db44a29951042ada9b393151532e41fcb" +dependencies = [ + "xml-rs", +] + +[[package]] +name = "xxhash-rust" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "550a2b930b62486a393c52d5c3b84bff264b28aa437ed64694d31e93b1757af7" + +[[package]] +name = "yamux" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f4e661d6ba6f09a768e31c6f3d88f9fac596ad5d376515d598a37b62d220b34" +dependencies = [ + "futures", + "log", + "nohash-hasher", + "parking_lot", + "pin-project", + "static_assertions", + "web-time", +] + +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + +[[package]] +name = "yasna" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", + "synstructure 0.14.0", +] + +[[package]] +name = "zerocopy" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", + "synstructure 0.14.0", +] + +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "serde", + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "zip" +version = "8.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +dependencies = [ + "crc32fast", + "flate2", + "indexmap 2.14.2", + "memchr", + "typed-path", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zopfli" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaf7fc5d30c28483d93805c4a5e12b05bbb52407fa67c5f8bd552374cd01fb11" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + +[[package]] +name = "zstd" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded520f16b29e1a9b771c722221b2b0a0398c2316d9bb3c42b458a5414d7434b" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.1.2+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" +dependencies = [ + "cc", + "pkg-config", +] @@ -3,11 +3,18 @@ name = "continuum" version = "1.0.0" edition = "2024" +# The transports are features, none of them default: a build has those it names, at +# least one (`--features mycelium`, `--features libp2p,tor`, ...). [features] tokio-console = ["dep:console-subscriber"] -# The libp2p transport (`--transports libp2p`). +# A dashboard of the node and a map of the swarm in the HTTP API (`/admin/dashboard`). +dashboard = [] +# The Mycelium transport (`--mycelium-enable`): through a Mycelium daemon's API, one it +# supervises or one running elsewhere. +mycelium = ["dep:reqwest", "dep:rustls", "dep:libc"] +# The libp2p transport (`--libp2p-enable`). libp2p = ["dep:libp2p", "dep:libp2p-stream", "dep:futures"] -# The Tor transport (`--transports tor`): onion services, with arti. +# The Tor transport (`--tor-enable`): onion services, with arti. tor = [ "dep:arti-client", "dep:tor-hsservice", @@ -40,13 +47,18 @@ bytes = "1.12" clap = "4.6" dashmap = "6.2" futures = { version = "0.3", optional = true } -libp2p = { version = "0.57", optional = true, features = ["tokio", "tcp", "quic", "dns", "noise", "yamux", "ed25519", "identify", "macros", "relay", "dcutr", "autonat", "mdns", "kad"] } +libp2p = { version = "0.57", optional = true, features = ["tokio", "tcp", "quic", "dns", "noise", "yamux", "ed25519", "identify", "ping", "macros", "relay", "dcutr", "upnp", "autonat", "mdns", "kad"] } libp2p-stream = { version = "0.5.0-alpha", optional = true } ed25519-dalek = "3" getrandom = "0.4" -libc = "0.2" +# Timestamps written and read as text (RFC 3339, RFC 9110 dates): no time zones needed. +jiff = { version = "0.2", default-features = false, features = ["std"] } +libc = { version = "0.2", optional = true } password-hash = "0.6" -reqwest = { version = "0.13", features = ["json", "query"] } +# Its TLS (only for a Mycelium daemon elsewhere, over https) is rustls with ring, which +# libp2p's QUIC uses too, rather than reqwest's default aws-lc (a C library, cmake). +reqwest = { version = "0.13", optional = true, default-features = false, features = ["json", "query", "http2", "charset", "system-proxy", "rustls-no-provider"] } +rustls = { version = "0.23", optional = true, default-features = false, features = ["ring", "std", "tls12", "logging"] } rmp-serde = "1.3" serde = { version = "1.0", features = ["derive"] } serde_bytes = "0.11" diff --git a/README.md b/README.md new file mode 100644 index 0000000..e626f69 --- /dev/null +++ b/README.md @@ -0,0 +1,108 @@ +# continuum + +A replicated [Automerge](https://automerge.org) document for fleets of machines that +aren't always online, read and written as JSON by path (`settings/theme`, +`servers/0/name`) at any depth. Every node holds the whole document, takes reads and +writes on its own, and syncs with its peers whenever it can reach them. Think +Syncthing, but for structured data behind an HTTP API. + +There is no leader and no quorum for writes: a node keeps working through a network +partition, on a laptop on a train, or as the only one left. Concurrent writes are +merged, and conflicts are reported rather than hidden. + +A node is its key, not its address. Nodes find and reach each other over any of three +transports, one or several at once, each a cargo feature: + +* **[Mycelium](https://github.com/threefoldtech/mycelium)**: an encrypted overlay + network addressed by keys, so NAT, changing IP addresses and firewalls don't need + configuring. +* **[libp2p](https://libp2p.io)**: direct connections over TCP and QUIC, with relays + (members of the swarm, or a relay-only continuum), hole punching, UPnP and LAN + discovery for nodes behind NAT. +* **Tor**: an onion service per node (arti, embedded), reachable from anywhere without + a forwarded port. For reachability, not anonymity. + +A node on several transports reaches each peer over the fastest path they share, falls +back to the next when one fails, and carries the traffic between nodes that share +none. + +The one thing that takes a quorum is a remake: compacting the history, or another move +to a new document. A majority of the members must vote for it, so it waits, and in the +end aborts, while too many of them are out of reach; reads and writes go on meanwhile +([`docs/remakes.md`](docs/remakes.md)). + +## What it isn't + +Continuum is eventually consistent. Reads are local and may be behind; there are no +compare-and-set, transactions, leases, locks or leader election. Don't build +coordination on it. + +## Quick start + +```sh +# The transports are cargo features: mycelium, libp2p, tor (any, at least one); +# dashboard adds the live page (`/admin/dashboard`). +cargo build --release --features mycelium,dashboard + +# An HTTP API user (Argon2 hash; e.g. with the `argon2` command-line tool). +printf '%s' 'secret' | argon2 "$(openssl rand -hex 16)" -id -e | sed 's/^/alice:/' > htpasswd + +# The first node starts a new swarm. --node-key-create makes the node's key on its first +# start (a new key is a new node; keep the file). +./target/release/continuum --genesis --htpasswd htpasswd --state-file state.json \ + --node-key-file node.key --node-key-create --mycelium-enable + +# Further nodes, on other machines, join through any node of it: one of its addresses +# (`addresses` in `GET /admin/stats`: `mycelium:<public key>`, `libp2p:<multiaddr>`, +# `tor:<onion>.onion:<port>`), or for Mycelium its public key or IPv6 address, or a +# host name with an AAAA record for one. +./target/release/continuum --seeds <address of the first node> --htpasswd htpasswd \ + --state-file state.json --node-key-file node.key --node-key-create --mycelium-enable +``` + +A node uses the transports enabled, one at least (`--mycelium-enable`, +`--libp2p-enable`, `--tor-enable`). With Mycelium, each node runs its own Mycelium +daemon, unless pointed at one with `--mycelium-rest-url`; `--mycelium-peers` names +Mycelium nodes to connect through. With libp2p, `--libp2p-listen` says where to listen +(`/ip4/0.0.0.0/tcp/9661`), and seeds are multiaddrs +(`libp2p:/ip4/192.0.2.7/tcp/9661`); with Tor (`--tor-enable`), `--tor-state-dir` +keeps the onion service's key, and seeds are onion addresses. [`docs/transports.md`](docs/transports.md) +has the rest. The HTTP API listens on `127.0.0.1:3000`: + +```sh +curl -u alice:secret -X PUT localhost:3000/store/config/feature \ + -H 'Content-Type: application/json' -d '{"enabled": true}' +curl -u alice:secret localhost:3000/store/config/feature +curl -u alice:secret -N localhost:3000/stream/config # follow changes +curl -u alice:secret localhost:3000/peers # who is out there +``` + +## Documentation + +**Running nodes** + +* [`docs/configuration.md`](docs/configuration.md): options, files and imports (`continuum --help` lists them); + [`docs/example.toml`](docs/example.toml) has every option, explained. +* [`docs/transports.md`](docs/transports.md): Mycelium, libp2p and Tor: what each is for, how they combine, + relays, and their options. +* [`docs/lifecycle.md`](docs/lifecycle.md): starting, joining, rejoining and stopping a node. +* [`docs/operations.md`](docs/operations.md): adding and removing nodes, keys, backups, upgrades, what to + alert on. +* [`docs/remakes.md`](docs/remakes.md): compacting history and other moves to a new document. +* [`docs/trust.md`](docs/trust.md): who can do what. +* [`docs/measurements.md`](docs/measurements.md): `/admin/stats` and its Prometheus metrics. +* [`docs/dashboard.md`](docs/dashboard.md): the dashboard, a live page of a node's view of the swarm. + +**Writing applications** + +* [`docs/api.md`](docs/api.md): the store API for applications: consistency, concurrent writes, + streams. +* [`docs/openapi.yaml`](docs/openapi.yaml): the HTTP API (also served at `/openapi.yaml`, with a Swagger + UI at `/swagger-ui`). + +**Working on continuum** + +* [`docs/architecture.md`](docs/architecture.md): the map of the code, and where to read on. +* [`docs/development.md`](docs/development.md): building and testing quickly. +* [`docs/peers.md`](docs/peers.md): the protocol between nodes. +* [`docs/glossary.md`](docs/glossary.md): the words used in the code, logs, API and docs. diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..5b4dac4 --- /dev/null +++ b/docs/api.md @@ -0,0 +1,247 @@ +# The store API, for applications + +What an application can rely on when it reads and writes through a node's HTTP API. The +endpoints and their answers, one by one, are in [`openapi.yaml`](openapi.yaml) (served by every node at +`/openapi.yaml`, with a Swagger UI at `/swagger-ui`); this page is about what they mean +together. + +## The model + +The store is one document, the same on every node once they have synced, read and +written as JSON. A path names a place in it: `settings/theme` is the key `theme` in the +map `settings`, and within a list a segment is an index (`servers/0/name`; `-1` the last +item). Every node holds all of it and answers from its own copy: there is no leader to +ask, and no request goes to another node. + +```sh +curl -u alice:secret -X PUT localhost:3000/store/settings \ + -H 'Content-Type: application/json' -d '{"theme": "dark", "fontSize": 14}' +curl -u alice:secret localhost:3000/store/settings/theme # "dark" +curl -u alice:secret -H 'Accept: text/plain' localhost:3000/store/settings/theme # dark +curl -u alice:secret -X DELETE localhost:3000/store/settings/fontSize +curl -u alice:secret -X POST localhost:3000/store/queue/-1 -d '"job 7"' # append +curl -u alice:secret -X POST 'localhost:3000/store/stats/visits?increment=1' # count +``` + +* `GET /store/<path>` reads, `PUT` writes (what is there then reads as the value + written; the maps on the way are created), `DELETE` deletes (`?prune=true` also + removes the maps it leaves empty), and `POST` inserts into a list at an index, the + items from there moving up, or with `?increment=n` adds `n` to a counter (Concurrent + writes), or with `?cast=<type>` casts what is there to another type (Types). +* Bodies are JSON (`application/json`, the default), text (`text/plain`, stored as a + string), or a value of a type of its own (Types); answers are JSON, or as `Accept` + asks. Numbers in JSON are stored as integers where they fit, as floats otherwise. +* Every request needs a user from the node's `--htpasswd` (basic authentication). Users + are all alike: each can read and write everything but protocol data (below). +* A key with `/` in it, or an empty key, can be written inside a value at its parent, + but not named in a path: read it through the parent. + +## Types + +Every value is stored as one of Automerge's types: a map, a list, a text, or a scalar: +a string, bytes, a signed or unsigned integer, a float, a boolean, null, a counter or a +timestamp (milliseconds since 1970). Every read says which in `x-continuum-type` +(`map`, `list`, `text`, `string`, `bytes`, `int`, `uint`, `float`, `boolean`, `null`, +`counter`, `timestamp`), and so does `?shallow=true` for each entry. + +Each has a JSON form, so any value reads as JSON: bytes as base64, counters and the +integers as numbers, timestamps as RFC 3339 strings (`2026-11-01T21:50:20.345Z`), or as +`Accept: application/json; timestamp-format=…` says: `rfc3339`, `unix` (seconds, with +the milliseconds as a fraction), `unix-ms` or `http-date` (RFC 9110). JSON alone +doesn't tell them apart: written as JSON, values are maps, lists, strings, signed +integers, floats, booleans and nulls. + +A scalar is also read and written as its own media type, its value in the body: + +| Type | Media type | Body | +|---|---|---| +| string, text | `text/plain` | the characters | +| bytes | `application/octet-stream` | the bytes | +| int | `application/vnd.continuum.int` | decimal | +| uint | `application/vnd.continuum.uint` | decimal | +| float | `application/vnd.continuum.float` | decimal | +| boolean | `application/vnd.continuum.boolean` | `true` or `false` | +| counter | `application/vnd.continuum.counter` | decimal (a `PUT` sets it) | +| timestamp | `application/vnd.continuum.timestamp; format=…` | as the format says (`rfc3339` by default) | + +```sh +curl -u alice:secret -X PUT localhost:3000/store/logo -H 'Content-Type: application/octet-stream' --data-binary @logo.png +curl -u alice:secret -H 'Accept: application/octet-stream' localhost:3000/store/logo > logo.png +curl -u alice:secret -X PUT localhost:3000/store/deploy/at \ + -H 'Content-Type: application/vnd.continuum.timestamp; format=unix' -d 1793569820 +curl -u alice:secret -H 'Accept: application/vnd.continuum.timestamp; format=http-date' \ + localhost:3000/store/deploy/at # Sun, 01 Nov 2026 21:50:20 GMT +``` + +Typed JSON (`application/vnd.continuum.typed+json`, for reads and writes) keeps every +type: it is JSON, but for what JSON would show as another type, which is a single-key +object whose key is a tag: `{"$uint": 5}`, `{"$counter": 5}`, `{"$bytes": "<base64>"}`, +`{"$timestamp": "2026-11-01T21:50:20.345Z"}` (in its `timestamp-format`) and +`{"$text": "…"}` (a text); a float is a number, as in plain JSON. A map whose only key +starts with `$` is `{"$map": {…}}`, so nothing else is taken for a tag; any other +single `$` key is `400`. A whole map with typed values in it is written at once this +way, and reads back as written. A text written over a text is changed where it differs +(a diff, as splices), so concurrent edits elsewhere in it stay. + +Types are sticky: plain JSON (or text) written over a value takes its type where it is +that type's JSON form. A string over a timestamp that reads as one (RFC 3339, or as the +body's `Content-Type: application/json; timestamp-format=…` says, where numbers are +read as timestamps too), over bytes that is base64, over a text (then changed where it +differs); an integer over a float, an unsigned integer (if it isn't negative) or a +counter (which it sets). So what a client read as JSON writes back as it was stored, +whole maps included. Typed JSON and a type's own media type say the type themselves, +but for an integer over a float, which stays a float in typed JSON too. + +A write that changes the type of what it writes over (a string over a timestamp that +doesn't read as one, a map over a list) says which in `x-continuum-retyped` +(`deploy/at: timestamp -> string`, comma-separated); with `?retype=false` it is refused +instead (`409`, naming the first). A `PUT` answers with what is there after it, read +back, so its `x-continuum-type` is what the value was stored as. + +`POST <path>?cast=<type>` casts what is there to a type (as `x-continuum-type` names +it), and answers as a `PUT` does: numbers (integers, floats, counters, timestamps as +milliseconds since 1970) to numbers where they are exactly that, any scalar but bytes +to a string or a text as its media type shows it (a timestamp in `?format=`, +`rfc3339` by default) and back, and bytes to a string or a text and back as UTF-8. +Anything else, or a value that isn't one of the type (`"soon"` to a timestamp, `2.5` +to an integer), is `409`, saying why; maps, lists and nulls aren't cast. + +```sh +curl -u alice:secret -X POST 'localhost:3000/store/deploy/at?cast=timestamp&format=unix' +``` + +`Accept` is taken in its order (quality values aren't weighed): the first media type the +value can be shown as, JSON for `*/*` or without an `Accept`, and `406` (naming those it +can be shown as) if none. `text/plain` shows a string as it is and anything else as +JSON. + +## Consistency + +**Reads are local.** A node answers from its copy, which may lack writes other nodes +made a moment ago: they arrive by sync, as fast as messages get from the writer to +this node (a hop between neighbours, a few in a large swarm; [`measurements.md`](measurements.md), +Propagation), and after a partition once it heals. There is +no read that waits for the rest of the swarm. + +**Writes are local too, and never refused for want of others.** A write is applied to +the node's copy at once (the answer holds the value written) and spreads from there. +Two nodes cut off from each other both go on taking writes; when they meet again, the +writes merge. So a write that was answered `200` is durable on that node (with a +`--document-file`, within `--document-append`, 5 s by default) and reaches the others +eventually, but it isn't confirmed by anyone else when answered. + +**Read your own writes** on the node you wrote to: its later reads include the write. +On another node, they include it once synced. `?heads=true` on a read gives the +document's heads (`x-continuum-heads`, the hashes of its latest changes): two reads with +the same heads read the same document. Every write answers with the heads after it, in +the same header. `?at=<heads>` reads the document as it was at those heads, on any node +that has them (`404` on one that doesn't, yet), however it changed since: a client that +wrote through one node and reads its write through another reads with the write's +heads, and retries on `404`. + +**One request is one change.** What a `PUT` writes is applied whole or not at all, also +on the other nodes: a map written in one `PUT` never shows half its keys. There are no +transactions across requests: to change several values together, write their common +parent. + +## Concurrent writes + +Writes made on different nodes before they synced are concurrent. They merge, the same +way on every node: + +* **Different keys** both stay: two nodes writing `settings/theme` and + `settings/fontSize` get both. +* **The same key**: one write wins, the same one everywhere. "Newest" means by + Automerge's operation order (the write made after seeing more changes; between equals, + by the writer's actor ID), not by any clock: of two truly concurrent writes, which wins + is arbitrary, but it is the same on every node. +* **Maps written at the same key** merge into one: their keys are united, each key as + above. +* **A write and a delete** of the same key: the write stays (a delete removes only what + the deleting node had seen). +* **List inserts** both stay, in an order every node agrees on. Writing a list item + (`PUT` at its index) overwrites it: two nodes writing the same item concurrently are + two writes of the same key. +* **Counters** (`POST <path>?increment=n`, made by the first increment) add up: two + nodes incrementing one counter by 1 concurrently leave it 2 higher. Two nodes making + the same counter concurrently make two, and one wins with its increments. +* **Strings** are values, not texts: concurrent writes to one string don't merge + character by character; one wins. + +A write changes only what differs from what is there: a map written over a map keeps +the keys that stay (and what is inside them), gains those it lacks and loses those the +value doesn't have; a list written over a list does the same item by item, by +position; a value equal to the one there isn't written again. So a `PUT` of `settings` +as a whole, concurrent with a `PUT` of `settings/theme` elsewhere, keeps the theme, +unless the whole write changed the theme too (then one wins) or left it out (then the +theme's write stays, as against a delete). Where the path holds concurrent values, or +one of another kind (a map written over a list), it is written anew. + +`?conflicts=true` on a read answers `{"value": …, "conflicts": [...]}`: every place on +the way where concurrent values were resolved, with all candidates and which was +selected (`mergedMaps` or `newestWins`). A later write at that path ends the conflict. + +## Following changes + +`GET /stream/<path>` (or `/stream` for everything) is newline-delimited JSON: one event +with the current value, then one whenever the value there changes, by a write on this +node or by sync: + +``` +{"key":"settings/theme","value":"dark"} +{"key":"settings/theme","value":"light"} +{"key":"settings/theme","deleted":true} +``` + +An event carries the value as a read would return it then. Changes in quick succession +may come as one event with the latest value: a stream tells the value, not every write. +A change that leaves the value as it was sends nothing. The connection stays open; after +it breaks, reconnect, and the first event is the current value again. + +## While a node isn't ready + +A node serves the store once it holds the swarm's document ([`lifecycle.md`](lifecycle.md)): + +* **Joining** (a new node, or one restarted without its document file): `503` for + reads, writes and streams, but for `GET /store`, the whole store, which answers with + what has arrived so far. `GET /ready` (no authentication) says `200` once it is ready. +* **Rejoining** a newer epoch after a remake it missed: reads answer from the document + before, with `x-continuum-stale` naming that epoch; writes are answered as usual and + go into that document (so reads show them), carried over into the new epoch when the + rejoin ends, unless newer writes touched their paths (Concurrent writes); streams are + `503`. +* **Remakes** ([`remakes.md`](remakes.md)) don't interrupt anything else: a node switches to the new + document between two requests, and writes made meanwhile are carried over (at + their path, or where the remake's rules put them; [`remakes.md`](remakes.md), Writes around a + remake). + +Load balancers can send requests to any ready node: `/ready` is the check. + +## Who wrote what + +Every change records the node that committed it (as its Automerge author) and, for a +write through the API, the user who wrote it (in its message, as a trailer like Git's +`Signed-off-by:`: `User: alice`), with when (by the writer's clock). User names are +each node's own (`--htpasswd`), so who wrote it is that user at that node: `alice` on +two nodes is one person only where their operators make it so, which nothing checks. +`GET /admin/changes` lists them, +newest first, with the paths each touched; `GET /admin/changes/settings` those that +touched what is at `settings`. +Every node holds every change, so user names are known to the whole swarm, and so is +who wrote what, until a remake drops the history ([`remakes.md`](remakes.md)). + +## Protocol data + +`_continuum` holds what the nodes coordinate through: the swarm ID, the member registry, +remakes and votes ([`architecture.md`](architecture.md), The document). It is part of the store and shows +in `GET /store`, but isn't application data: only users in `--protocol-writers` may +write it, for repairs. Keep application data out of it, and leave it out when reading +the whole store if it's in the way. + +## Large stores + +`?shallow=true` on a read answers what is there one level deep (a map's keys with their +types and sizes, short scalars' values) instead of the whole value, to browse a large +store a level at a time. Requests are limited to 2 MiB. Every write keeps its history +until a remake compacts it; a document of many small writes grows faster than its +content ([`remakes.md`](remakes.md)). diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..e0e2c94 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,306 @@ +# Architecture + +Continuum is a replicated Automerge document, read and written by path: every node +holds the same document, exposes it over an HTTP API, and syncs it with its peers over one or more +transports (Mycelium, libp2p, Tor; [`transports.md`](transports.md)). There +is no leader and no central service; nodes find each other through seeds and a member +registry kept in the document itself. Everything the swarm agrees on, it agrees on +through that document. + +This page is the map: what the parts are, how they fit, and where the complexity sits. +The details live in the other pages ([`lifecycle.md`](lifecycle.md), [`operations.md`](operations.md), [`peers.md`](peers.md), [`transports.md`](transports.md), +[`remakes.md`](remakes.md), [`api.md`](api.md), [`configuration.md`](configuration.md), [`measurements.md`](measurements.md), [`dashboard.md`](dashboard.md), [`trust.md`](trust.md), +[`glossary.md`](glossary.md)) and +in the module docs at the top of each source file. + +## Layers + +``` + operators, applications + │ HTTP (basic auth) + ┌────────▼────────────────────────────────────────────────────────────────┐ + │ api/ /store, /stream, /peers, /admin/*, /debug/* │ + ├─────────────────────────────────────────────────────────────────────────┤ + │ state/ AppState: the replica, lifecycle, peer states, timings, │ + │ shared tables; the Host impl for remakes │ + ├──────────────────────────────┬──────────────────────────────────────────┤ + │ swarm/ │ remake/ │ + │ peers, sync, relay, topology,│ epochs: proposal, votes, decision, │ + │ members, blobs, catch-up, │ snapshot, switch, rejoin, quarantine │ + │ stability, bootstrap, files │ (reaches the node only through `Host`) │ + ├──────────────────────────────┴──────────────────────────────────────────┤ + │ document/ Replica (document + epoch), the store view, the document │ + │ file │ + ├─────────────────────────────────────────────────────────────────────────┤ + │ transport/ message types and wire format, traffic counting; the │ + │ identity layer (node IDs) over the paths layer (addresses, │ + │ PING/PONG, the debug firewall) over the transports: │ + │ Mycelium (REST client, daemon), libp2p, Tor │ + └─────────────────────────────────────────────────────────────────────────┘ + Mycelium daemon · libp2p peers and relays · the Tor network +``` + +Dependencies point downwards, with two deliberate exceptions: [`state/`](../src/state/) is the hub +everything else reaches (it holds the shared state and is reached by most modules), and +[`remake/`](../src/remake/) never touches `AppState`, peers or sync directly; it sees the node through +the `Host` trait ([`remake/mod.rs`](../src/remake/mod.rs)), which `AppState` implements. That boundary is what +keeps the epoch machinery testable with plain documents. + +## The document + +One Automerge document per node (`document::replica::Replica`: the document, its epoch, +cached heads). Automerge is used directly only in `document` and in what remakes do to +documents ([`remake/`](../src/remake/): `snapshot`, `delta`, `catchup`, `kind`, `quarantine`); the rest of +the node names its types as `document` re-exports them (`Document`, `ChangeHash`, +`Change`, `ActorId`) and works through `document`'s functions. Two views of it: + +* **The store** (`document::store`): JSON over slash-separated paths, with Automerge's + concurrent values merged into one view (maps union, scalars newest-wins) and + reported on request (`?conflicts=true`). The HTTP API and the remake kinds work + through it; nothing else reads Automerge objects directly. Within a list, a path + segment is an index (`servers/0/name`); a negative one counts from the end. A key with `/` in it, or + an empty one, can be written inside a value but not named in a path: it is read and + written through its parent. +* **Protocol data** under `_continuum`: the swarm ID, the member registry, forget + records, remake records, the votes register and epoch metadata. The nodes coordinate + through these keys and nothing else: there is no control channel besides the + document and the handshake. `--protocol-writers` may edit them for repairs; the + swarm ID and epoch metadata never. + +**Epochs.** The document's history can't be dropped in place, so a *remake* moves the +whole swarm to a new document (same content, no history; or new content), the next +epoch. Nodes on different epochs don't sync. The epoch is `{generation, id}`; the ID +is the hash of the new document's first change, which every node builds itself, byte +for byte the same ([`remake/snapshot.rs`](../src/remake/snapshot.rs)). That determinism is what lets a node verify +it switched to the right document without trusting anyone. + +## Modules + +**[`transport/`](../src/transport/)** — [`mod.rs`](../src/transport/mod.rs) defines the message types (HELLO, HELLO_REPLY, sync +envelope, STATUS/STATUS_REPLY, PING/PONG, blob messages, LINKS/LINKS_REPLY) and the `Transport` trait; +[`wire.rs`](../src/transport/wire.rs) is the wire format (a version byte, bumped with every change to the format, +and a kind byte, then the message in MessagePack, derived from the types in [`mod.rs`](../src/transport/mod.rs) +with codecs for hashes and IDs); [`mycelium/`](../src/transport/mycelium/) (with the `mycelium` feature) is the REST +client (polling inbox, outbound POST, one topic, `continuum`, for every message), with +[`mycelium/daemon.rs`](../src/transport/mycelium/daemon.rs) running the daemon (an external one by its URL, or one it starts +and supervises); [`libp2p/`](../src/transport/libp2p/) (with the `libp2p` feature) is libp2p as a transport, +direct connections with one message per stream, relays and hole punching ([`driver.rs`](../src/transport/libp2p/driver.rs), +[`behaviour.rs`](../src/transport/libp2p/behaviour.rs)), the addresses known per peer ([`book.rs`](../src/transport/libp2p/book.rs)), and which of them are +ignored or unroutable ([`ignore.rs`](../src/transport/libp2p/ignore.rs), [`routes.rs`](../src/transport/libp2p/routes.rs)); [`tor.rs`](../src/transport/tor.rs) (with the `tor` feature) is Tor as a +transport, an onion service per node and one connection per pair, authenticated by the +challenge handshake of [`challenge.rs`](../src/transport/challenge.rs); [`paths.rs`](../src/transport/paths.rs) makes one address space +(`<scheme>:<target>`) of the node's transports (one or more), routing each send to the +transport of its address, measuring the paths to peers on several (PING, PONG) and +sending over the best, the next when one fails; [`identity.rs`](../src/transport/identity.rs) is the identity layer above it, so the node +deals in node IDs only (WHO and ME prove which node is at an address; [`peers.md`](peers.md)); [`traffic.rs`](../src/transport/traffic.rs) counts messages and +bytes per peer; [`stats.rs`](../src/transport/stats.rs) the transports' own figures; [`firewall.rs`](../src/transport/firewall.rs) is the debug firewall (`/debug/firewall`), which the paths layer applies to +everything, by the node at each address; [`sim.rs`](../src/transport/sim.rs) (tests only) is a +simulated network: many nodes in one process, with per-link delay, loss and +duplication from a seed, nodes taken down and brought up, and messages injected as if +a node had sent them. It sends the wire encoding, so tests exercise the codec, or a +transport's envelope around it ([`development.md`](development.md)). + +**[`simtest/`](../src/simtest/)** (tests only) — whole nodes on the simulated network, built as `main` +builds them, with a 250 ms keepalive and a 1 s sync reply timeout: joining and +converging, dormancy and return, late messages, a stalled catch-up source; and an +ignored 25-node soak test with a propagation bound (`cargo test soak -- --ignored`). How to build and test quickly: [`development.md`](development.md). + +**[`swarm/`](../src/swarm/)** — the live protocol between nodes: + +| Module | Does | Documented in | +|---|---|---| +| `bootstrap` | genesis, joining through seeds (forever), starting on a document | [`lifecycle.md`](lifecycle.md) | +| [`peer/`](../src/swarm/peer/) | handshake, keepalives and healing, peers' epochs, admission with a swarm key, forget records, inbound dispatch, the peer worker (a file each) | [`peers.md`](peers.md) | +| `sync` | the sync worker, Automerge sync messages with retries and reply deadlines | [`peers.md`](peers.md) | +| `changes` | the change publisher: the store paths each change touched (`change_events`) | module docs | +| `document_file` | the worker keeping the document file (`document::file`) up to date | module docs | +| `source` | the one peer a catching-up node syncs with, and when to give it up | [`peers.md`](peers.md) | +| `relay` | announcing heads to neighbours and pulling what's missing (partial mesh) | module docs | +| `links` | answering LINKS with this node's links, for the dashboard's map | [`dashboard.md`](dashboard.md) | +| [`topology/`](../src/swarm/topology/) | which members are neighbours: ring plus low-score links, stable members preferred | module docs | +| `members` | the member registry: aliases, alive stamps, tiers, the electorate | module docs | +| `stability` | a node's own connectivity record and its stable/transient tier | module docs | +| `auth` | the pre-shared swarm key and handshake tags | [`trust.md`](trust.md) | +| `blob` | chunked content-addressed transfers from several peers | module docs | +| `rtt`, `throughput` | round-trip times (of the paths, from HELLOs until measured) and sync throughput, for the node's own choices | [`measurements.md`](measurements.md) | +| `skew` | this node's clock against its peers' (from their PONGs), and a warning while it is off | [`measurements.md`](measurements.md) | +| `pair` | which side of a pair leads (sends the first keepalive) | module docs | +| `stats` | node-wide figures for `/admin/stats` | [`measurements.md`](measurements.md) | +| `state_file` | what a node remembers across restarts | [`lifecycle.md`](lifecycle.md) | + +**[`remake/`](../src/remake/)** — epochs and the move between them ([`remakes.md`](remakes.md) for operators): + +| Module | Does | +|---|---| +| `records` | remake records and the votes register in the document | +| `api` | `/admin/remakes`: proposing, reports, aborting, superseding | +| `proposal` | the checks before a proposal, superseding, the electorate and quorum | +| `vote` | who votes for what, tallies, the one-vote-per-epoch rule | +| `coordinator` | counts votes, writes the decision, builds the snapshot on commit | +| `kind` | `compact`, `replace`, `import`: how the new document is built and where later writes go | +| `snapshot` | the deterministic build and its verification; epoch metadata | +| `switch` | the remake worker: switching this node to a committed epoch, replaying writes since the base | +| `catchup` | rejoining a newer epoch from peers (download, verify, replay own writes) | +| `delta` | replaying changes between two sets of heads into another document | +| `quarantine` | writes a remake had no place for | +| `content` | the new document of a `replace`/`import`, fetched by voters | +| `local` | in-memory per-node remake state: prebuilt snapshots, archive, own actors | + +**[`document/`](../src/document/)** — [`replica.rs`](../src/document/replica.rs) (the document with its epoch), [`store.rs`](../src/document/store.rs) (the JSON +view), [`file.rs`](../src/document/file.rs) (the document file: full save plus appended increments), [`changes.rs`](../src/document/changes.rs) +(changes as bytes: one by its hash, and deltas between heads), [`sync.rs`](../src/document/sync.rs) (one sync with +a peer, `SyncSession`: Automerge's sync state with our workarounds for unreliable +delivery), [`meta.rs`](../src/document/meta.rs) (the protocol's paths under `_continuum`: the only place their +names are spelled). + +**[`api/`](../src/api/)** — axum handlers: `store` (`/store`, `/stream`), `peers`, [`stats/`](../src/api/stats/) (JSON and +Prometheus), `document` (backup export), `log` (filter at runtime), `config` +(`/admin/config`), `health` (`/health`, `/ready`), `debug`, `auth` (htpasswd with +Argon2), [`dashboard/`](../src/api/dashboard/) (with the `dashboard` feature: the page and its events; +[`dashboard.md`](dashboard.md)), and [`mod.rs`](../src/api/mod.rs) (router, protocol-path rules). The remake routes +come from [`remake/api.rs`](../src/remake/api.rs). [`docs/openapi.yaml`](openapi.yaml) describes it all; nodes serve it at +`/openapi.yaml`, with a Swagger UI at `/swagger-ui`. + +**Top level** — [`main.rs`](../src/main.rs) reads the configuration, picks the starting document and +runs the shutdown sequence; [`node.rs`](../src/node.rs) builds a node's state and starts its workers +(also used by the simulator); +[`config/`](../src/config/) loads options from defaults, files and arguments ([`configuration.md`](configuration.md)); +[`force.rs`](../src/force.rs) is the one format for overriding checks; [`files.rs`](../src/files.rs) reads and writes the +node's own files and counts them; [`clock.rs`](../src/clock.rs) is the node's wall clock (every Unix +timestamp it stamps or judges by goes through it, so tests can skew one node's); +[`locks.rs`](../src/locks.rs) takes std locks without poisoning (`lock_or_recover` and the like); +[`backoff.rs`](../src/backoff.rs), [`logging.rs`](../src/logging.rs); [`testrand.rs`](../src/testrand.rs) (tests only) is a seeded generator and a +runner for property tests over many seeds (topology, heads summaries, replay). + +## Runtime + +**Tasks.** Started by [`node.rs`](../src/node.rs), running for the life of the process: + +* the **consumer**: every transport's inbound messages, through the paths and the + identity layer (Mycelium polls its inbox, two polls at once so HTTP round trips + overlap; libp2p and Tor read their connections), handed one at a time to + `peer::handle_message`; +* the **sync lanes**: four tasks that apply inbound sync messages, each sender's on one + lane in arrival order, so a slow apply doesn't hold up the consumer + (`peer::lanes`); +* the **sync worker**: one queue of dirty peers, coalesced, one Automerge sync + message generated per peer per pass and sent by a task of its own, so a slow send + holds up no other peer (`swarm::sync`); +* the **peer worker**: when the earliest is due, keepalives, dormancy, HELLO retries, + overdue sync replies; every 5 s (and at once when the node becomes ready), forget records + (also when they changed), registry renewal, stability, measurements, the state file + (`swarm::peer::run_peer_worker`); +* the **topology worker**: recomputes the neighbours when the member list or a peer's + reachability changes (`swarm::topology::run_worker`); +* the **remake worker**: every second, on record changes and when a vote is due, + voting, switching, rejoining (`remake::switch::run_worker`); +* the **change publisher**: diffs the document after each change for `/stream`; +* the **document file** writer, if configured; +* **bootstrap**, until the node is ready, then the HTTP server. + +Short-lived tasks: a coordinator per remake this node proposed, blob serving and +fetching, the snapshot download of a rejoin, announcement rounds. + +**Locks.** The document is an `RwLock<Replica>`; what changes it (writes, applying +sync messages and fetched changes, the epoch switch) takes the write lock, and what +only reads it (generating sync messages, looking changes up, saving) the read lock, +both on the blocking pool. Peers are a `RwLock<HashMap<_, Arc<Mutex<PeerState>>>>`: the map lock +is held briefly, each peer's mutex for one operation. Lock order is document, then +peer, wherever both are held (sync generation, the epoch switch). Whether a write may +go in is checked under the write lock too: a rejoin's fresh document takes none until +the rejoin is done (`Replica::writable`), so a write checked as ready just before the +rejoin can't land in it; it goes into the document before the rejoin (`StaleView`), +which is replayed when it ends. A rejoin begins and ends under the write lock, with +the lifecycle (taken after the document's) and the stale view. The small shared +tables (RTT, throughput, neighbours, stability, catch-up source, remake state) sit +behind std mutexes and are never held across an await. + +**Lifecycle.** `Joining` → `Ready`, and back to `Joining` (keeping the swarm ID) while +a rejoin runs. Only a ready node registers, votes, serves writes and is followed as a +catch-up source. The phases, what each serves and how the files come in: [`lifecycle.md`](lifecycle.md). + +## The paths that matter + +**A write.** `PUT /store/{path}` → `AppState::mutate_doc` (one Automerge change under +the write lock, stamped with the clock) → `publish_change` (bumps the version, emits +`DocumentChanged`) → the sync worker generates a message per neighbour → each +neighbour applies it, relays its new heads to *its* neighbours, which pull what they +lack. In a full mesh (up to `k + 1` members) the writer reaches everyone directly. + +**An inbound message.** Consumer → `handle_message`: drop our own and stale-run +replies, admit (swarm key), mark the peer seen, record its epoch (ignoring late ones), +then by type: HELLO answered, HELLO_REPLY establishes membership and discovers peers, +sync queued on its lane if on our epoch (held back from everyone but the source while catching +up), STATUS answered with a STATUS_REPLY and compared for divergence, blobs served or collected, +LINKS answered with the node's links if the asker is a member, and again on each change while it subscribes (`swarm::links`, for the dashboard); +PING and PONG (the paths layer answered the PING already) only mark the peer seen, and +after a PONG the peer's round-trip time is its path's again. + +**A remake.** Proposal written by the coordinator → every electorate member votes once +it holds the content → the coordinator commits on a majority (checking the deadline +first, writing the decision atomically with a re-check) → after the switch delay +every node builds the snapshot, verifies the epoch ID, replays writes since the base +into it and swaps documents → nodes that missed it rejoin: download the snapshot in +chunks, sync, replay their own writes. [`remakes.md`](remakes.md) has the operator's view, +[`remake/*.rs`](../src/remake/) the rules. + +## What is kept, and where + +Nothing in the document is durable unless `--document-file` says so: a node that +restarts without one starts empty and syncs. The optional files ([`files.rs`](../src/files.rs)): + +| File | Holds | Written | +|---|---|---| +| state file | swarm ID, members to rejoin through | once ready, when the selection changes, at shutdown | +| document file | the document, this node's actor IDs | appended every few seconds, rewritten on epoch change and growth | +| backup | `GET /admin/document`, restored with `--recover-from` or imported | by the operator | + +Everything else (archive of old epochs, quarantine, prebuilt snapshots, measurements, +peer states) is in memory and gone at restart. + +## Design rules + +The decisions that recur throughout the code; changing one means revisiting the +places that rely on it. + +* **The document is the only shared state.** Membership, forgetting, remakes and votes + are all keys under `_continuum`. Nodes never agree on anything out of band. +* **One writer per key.** Each part of a remake record, each registry entry, each + vote-register entry has exactly one node that writes it, so concurrent writes never + fight. Where two writers are possible anyway (decisions after expiry), the record + is marked contested and nobody acts on it alone. +* **Deterministic builds, verified by hash.** A snapshot's bytes depend only on the + base document and the proposal, so the epoch ID doubles as proof of the content. +* **Messages may be late, lost or duplicated.** Every message carries epoch and + epoch-change count; sync is idempotent and repaired by deadlines; keepalives only + inform; a HELLO resets the session on both sides. +* **Peers on different epochs don't sync.** Ever. A node moves epochs by switching + (building) or rejoining (downloading), never by merging. +* **Writes since the base are replayed as state, not history.** Per map key, whole + lists and texts; by every switching node, and only by their author on a rejoin. +* **Members are trusted with everything** ([`trust.md`](trust.md)). The swarm key is the + boundary; nothing inside it is signed. +* **Conflicts are errors; overrides are named.** A contradiction in the configuration + or at the API refuses with a message that names the `force` value that would + override it, in one camelCase vocabulary everywhere. +* **Wire formats grow by appending.** New fields default when absent, so a swarm can + be upgraded one node at a time (until the first release lifts that constraint). + +## Where the complexity sits + +Roughly by size and by how many rules interact there: + +| Area | Lines (tests included) | Why it is complex | +|---|---|---| +| [`remake/`](../src/remake/) | 6,900 | Thirteen modules, each simple, but the sequence spans the whole node: proposal, votes, decision, snapshot, switch, rejoin, replay, quarantine, and every failure in between. Most of the project's rules live here. | +| [`swarm/peer/`](../src/swarm/peer/) + [`sync.rs`](../src/swarm/sync.rs) + [`source.rs`](../src/swarm/source.rs) | 2,300 | The protocol state machine: handshake, liveness, sync sessions with deadlines, who syncs with whom while catching up. Timing-dependent; tested on the simulated network ([`simtest/`](../src/simtest/)). | +| [`state/`](../src/state/) | 1,600 | `AppState` and its write helpers ([`mod.rs`](../src/state/mod.rs)), `PeerState` ([`peer.rs`](../src/state/peer.rs)), `Timing`, reads ([`read.rs`](../src/state/read.rs)) and the `Host` impl ([`host.rs`](../src/state/host.rs)). A hub, not a layer. | +| [`transport/`](../src/transport/) | 11,500 | libp2p ([`libp2p/`](../src/transport/libp2p/), 4,400: addresses, relays, hole punching, which paths to keep), the paths and identity layers (2,300: measuring, failing over, proving node IDs), Mycelium and its daemon ([`mycelium/`](../src/transport/mycelium/), 1,200), Tor (700), the wire format ([`wire.rs`](../src/transport/wire.rs), derived), the simulator. The transports are self-contained behind the `Transport` trait; libp2p's choice of addresses is the subtle part. | +| [`config/`](../src/config/) | 2,800 | Layered sources, importers and placeholders, conflicts, the report. Self-contained. | +| [`document/store.rs`](../src/document/store.rs) | 1,200 | Conflict-aware JSON over Automerge. Subtle (merged maps, canonical writes), well covered by the rest relying on it. | +| [`swarm/topology/`](../src/swarm/topology/), [`blob.rs`](../src/swarm/blob.rs) | 1,500 | Pure algorithms with tests; the topology hashes each member once, then scores the N² pairs by cheap mixing. | +| [`api/`](../src/api/) | 3,600 | Handlers; the Prometheus rendering is the longest file. The dashboard's page ([`dashboard.html`](../src/api/dashboard/dashboard.html), 2,000 more) is one file of HTML, CSS and script. | + +The two places where a change needs the most care are [`peer/`](../src/swarm/peer/)/[`sync.rs`](../src/swarm/sync.rs) (every +rule about when to send what interacts with every other) and [`remake/catchup.rs`](../src/remake/catchup.rs) +(a rejoin touches the lifecycle, the document swap, blobs, sync and replay at once). diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..d3f7e56 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,240 @@ +# Configuration + +Every option can be given on the command line, in configuration files, or both. +Precedence, lowest first: + +1. built-in defaults; +2. configuration files, in the order of `--config` (repeatable; later files win); +3. command-line arguments. + +An argument that replaces a value from a file is logged (info) with both values and +the file. Imported values are never logged, only where they come from; text with +placeholders is logged as it ends up. Secrets (`mycelium-rest-url`, which may carry +credentials) are never logged, however they are given. + +`continuum --help` lists all options with their defaults. + +## Durations and sizes + +Durations (`--keepalive`, `--dormant-probe-max`, `--document-append`, +`--document-rewrite`, `--electorate-window`, `--clock-skew-warn`, and the API's +`settle`, `switchDelay` and retention `age`) are a bare number of seconds (`30`, `0.5`) +or components with units in descending order: `ms`, `s`, `m`, `h`, `d`, e.g. `500ms`, +`500s`, `1h30m`, `1d2h3s`, `1.5h`. Precision is a millisecond; anything finer is an +error. Sizes (`--blob-chunk`) are a bare number of bytes or a number with `B`, `KiB`, +`MiB` or `GiB`, e.g. `64KiB`, `1.5MiB`; they must come to whole bytes. In configuration +files both may also be numbers (seconds, bytes). + +What the node reports (stats, `/peers`, remake reports) stays numeric, with the unit +in the name: `uptimeSecs`, `dormantMs`, `savedBytes`. Timestamps are Unix +milliseconds, named `…At` or `…Until`. + +## Files + +TOML (`.toml`) for people, JSON (`.json`) for generated configuration. Keys are the +long option names. Nested tables join their keys with `-`, so these are the same: + +```toml +mycelium-peers = ["tls://203.0.113.5:9651"] + +[mycelium] +peers = ["tls://203.0.113.5:9651"] +``` + +Unknown keys are an error. + +An example ([`example.toml`](example.toml) has every option, explained): + +```toml +include = ["/run/secrets/continuum.json"] + +bind-addr = "[::1]:3000" +htpasswd = "/etc/continuum/htpasswd" +node-key-file = "/var/lib/continuum/node.key" +seeds = ["node1.example.org", "node2.example.org"] +alias = { command = ["hostname", "-s"] } +keepalive = "1m" +neighbours = 12 + +[mycelium] +enable = true +key-file = "/var/lib/continuum/mycelium.key" +peers = ["tls://203.0.113.5:9651", "quic://203.0.113.6:9651"] +``` + +The same as JSON, as a system like NixOS would generate it: + +```json +{ + "include": ["/run/secrets/continuum.json"], + "bind-addr": "[::1]:3000", + "htpasswd": "/etc/continuum/htpasswd", + "node-key-file": "/var/lib/continuum/node.key", + "seeds": ["node1.example.org", "node2.example.org"], + "alias": { "command": ["hostname", "-s"] }, + "mycelium": { + "enable": true, + "key-file": "/var/lib/continuum/mycelium.key", + "peers": ["tls://203.0.113.5:9651"] + } +} +``` + +### Includes + +`include` (a path or a list) loads other files first, in the order listed; an entry +`{ path = "…", optional = true }` is skipped where the file is missing. Relative paths +are relative to the including file. The including file's own values win over its +includes, later includes over earlier ones (with a warning, since two includes setting +the same option is usually a mistake). A cycle is an error. + +This keeps public settings in version control and secrets elsewhere: + +```toml +# continuum.toml, committed: the secrets, then local settings over them +include = ["/run/secrets/continuum.toml", { path = "local.toml", optional = true }] +seeds = ["node1.example.org"] +``` + +## Transports + +A node uses every transport enabled: `--mycelium-enable`, `--libp2p-enable` and +`--tor-enable`, each in builds with its cargo feature (`--features mycelium,libp2p`, +none by default). None is on by default: a node needs one at least, and without, the +error says which this build has. As each is its own option, a file that configures a +transport can enable it too, and including it is all it takes: + +```toml +# continuum.toml +include = ["libp2p.toml"] + +# libp2p.toml +[libp2p] +enable = true +listen = ["/ip4/0.0.0.0/tcp/9661", "/ip4/0.0.0.0/udp/9661/quic-v1"] +``` + +`--transport-order` is their order of preference (`mycelium,libp2p,tor` by default), +whether enabled or not; those it leaves out come after, in the default order. How a node +uses several: [`transports.md`](transports.md), Several transports. Options +of a transport the node doesn't use (not enabled), or this build doesn't have, are +ignored, and the node says which at start (`Options of a transport this node doesn't use are ignored`, +with the transport, whether it's built, and the options); a configuration shared +between nodes may carry them on purpose. A transport's options are checked when the +configuration is loaded if the node uses it (Mycelium's always, where it is built), so +`--print-config` catches a wrong one too. + +What each transport is for and its options: [`transports.md`](transports.md). + +## Unsetting and conflicting options + +A later source can set an option back to its default: `--unset OPTION` on the command +line (repeatable), `unset = ["option", …]` in a file (overriding its includes and +earlier files), or `null` for the option in a JSON file. + +Some options can't be combined: `mycelium-rest-url` (use a Mycelium daemon running +elsewhere) excludes the options of a supervised daemon (`mycelium-key-file`, +`mycelium-key-create`, `mycelium-network-key-file`, `mycelium-network-name`, +`mycelium-tcp-listen-port`). +When a later source sets one side, values for the other side from earlier sources are +dropped (and logged), so `--mycelium-rest-url` on the command line just works with a +file that configures a supervised daemon. Setting both sides in the same source is an +error. + +Other combinations contradict each other and are an error wherever their values come +from: `genesis` with `seeds` (a new swarm has nobody to join) or with `recover-from`. +To start a node with `--genesis` whose configuration file lists seeds, unset them: +`--genesis --unset seeds`. + +## Forcing + +Some checks refuse to start the node because something looks wrong, but can be +overridden when you know better: `--force` lists them by name, or is `true` for all. +An unknown name is an error. The API's `?force=` parameters use the same format. +`--force` is accepted on the command line only: it is meant for the one start it was +given for, and in a configuration file it would apply to every start. + +| Name | Check | +|---|---| +| `swarmId` | the document to start on (`--recover-from`, `--document-file`) is of another swarm than the state file remembers; with it, the document's swarm wins | +| `badDocumentFile` | the document file can't be read; with it, it is moved aside and the node starts without it | +| `replaceDocumentFile` | `--recover-from` while the document file holds a document; with it, the backup wins | + +Names are camelCase everywhere, on the command line too, like the API's +(`switchDelay`, `supersede`, `alive`). + +## Importing values + +A value can be taken from elsewhere instead of written out: + +| In a file | On the command line | Value | +|---|---|---| +| `{ env = "NAME" }` | `env:NAME` | environment variable | +| `{ file = "PATH" }` | `file:PATH` | file contents | +| `{ command = "COMMAND" }` | `cmd:COMMAND` | output of `sh -c COMMAND` | +| `{ command = ["prog", "arg"] }` | | output of the program, no shell | + +Commands run as the node's user, at startup: whoever can write the configuration can run +them ([`trust.md`](trust.md)). + +The content is text with trailing line breaks removed, read like a command-line +value: numbers for numeric options, `true`/`false` for switches, comma- or +line-separated items for lists. With `format = "json"` in a file, or `+json` on the +command line (`env+json:SEEDS`, `file+json:PATH`, `cmd+json:COMMAND`), it is parsed +as JSON instead (e.g. a list as `["a", "b"]`). + +In files, relative paths are relative to the file, and commands run in its directory. + +Any other text value can mix fixed text with imported values: each `${env:NAME}`, +`${file:PATH}` or `${cmd:COMMAND}` in it is replaced by that content, as text (no +`+json` in a placeholder). `$$` is a `$`, and any other `$` outside a placeholder is +an error (a shell-style `$NAME` would otherwise silently stay as it is). A placeholder +ends at the `}` that matches its `{`: braces inside it balance, and in a command, braces +in shell quotes (`'...'`, `"..."`) or after a backslash don't count, so a command may +hold `$`, quotes and braces of its own. Placeholders don't nest. `literal:` on the +command line turns this off too. + +```sh +continuum --alias 'node-${env:HOSTNAME}' +``` + +```toml +alias = "${env:SITE}-${cmd:hostname -s}" +# A command with both kinds of quotes: a TOML literal string ('''...''') needs no escapes. +libp2p-listen = ['''/ip6/${cmd:ip -j addr show wireguard | jq -r '.[].addr_info[]|select(.family=="inet6" and .scope=="global").local'}/tcp/9661'''] +``` + +A command that prints several lines (above: several addresses) doesn't give one item +per line: as a list element the text is one item, line breaks and all, and as a whole +list value it is split at commas and line breaks, the fixed text going with the first +and last item only. Make it print one (`| head -n 1`). +List elements can be imported one by one: + +```toml +seeds = ["node1.example.org", { env = "EXTRA_SEED" }] +``` + +On the command line, `literal:` passes a value that happens to start with one of the +prefixes: `--alias literal:env:prod`. + +```sh +continuum --config /etc/continuum.toml --alias env:HOSTNAME --seeds cmd+json:'my-inventory --seeds' +``` + +Imports are read once, at start. + +## Checking the configuration + +`--print-config` prints the effective configuration as JSON and exits: every option +with its value and where it came from (`default`, `file` with the file, or +`argument`; `unset` if it was set back to its default there). Imported values are left +out and their importers listed instead (`imported`), and secrets are left out however +they were given (`secret`), so the output can be shared. Text with placeholders is +shown as it ends up, with the text as written under `templates`: keep secrets of +other options out of placeholders, and import them whole instead. A running node +answers the same at `GET /admin/config`; with `?values=original`, text with +placeholders is shown as written instead. + +```sh +continuum --config /etc/continuum.toml --print-config | jq .options.seeds +``` diff --git a/docs/dashboard.md b/docs/dashboard.md new file mode 100644 index 0000000..84d19c8 --- /dev/null +++ b/docs/dashboard.md @@ -0,0 +1,334 @@ +# The dashboard + +A page showing one node's view of the swarm, live: a map of the members and the links +between them, the node's peers, its figures, and the document, which it can also edit. +This page is about using it; how it works underneath (events, LINKS, what is sent +when) is at the end, [Underneath](#underneath). + +## Opening it + +The dashboard is in builds with the `dashboard` cargo feature: + +```sh +cargo build --release --features mycelium,dashboard +``` + +It is at `/admin/dashboard` on the node's HTTP API, by default +`http://127.0.0.1:3000/admin/dashboard`, and asks for a user from `--htpasswd`, as the +rest of the API does. The API listens on localhost unless `--bind-addr` says otherwise; +to look at a node on another machine, forward the port (`ssh -L 3000:localhost:3000 +<machine>`) or put a TLS proxy in front ([`trust.md`](trust.md)). + +Every node has its own dashboard, and shows the swarm as that node sees it: what it +knows of the others, what they report to it, and its own figures. + +## The page + +At the top, the node's alias and node ID; in orange, when this node's clock is off +from its peers' ([`measurements.md`](measurements.md), Clocks), by how much; and the connection's status: + +| Status | Meaning | +|---|---| +| `connecting…` | Opening the page's event stream. | +| `live` | Up to date. The dot beside it breathes with every update from the node. | +| `reconnecting…` | The stream broke; the browser reconnects by itself. | +| `disconnected; retrying…` | The node answered with an error (it may be up but not serving the page yet); again in 5 s. | +| `node stopped; reconnecting…` | The node said it is stopping; again in 3 s, until it is back. | +| `paused while hidden` | The page was hidden (another tab, a minimised window) for 30 s: it closed its stream, so the node does no work for it. Shown again, it reconnects. | +| `new version; reloading…` | The node now serves another version of the page; it is loaded. | + +A click on the status refreshes everything at once while live, and reconnects at once +otherwise. While not live, the page is dimmed and shows what it had last: ages stop +counting. + +Below, five boxes: the swarm's map on the left; the peers, the figures, the document +and the console on the right. In a narrow window they are stacked. + +Dragging the space between two boxes resizes them: between the map and the right +column, below the peers, between the figures and the document, and above the +console. Dragged all the way to the right, the right column goes, and the map takes +the whole width; drag the space right of the map back to bring it again. A +double-click there sets that space back to its default. The sizes are kept in the +browser, for the next time the page is opened. + +The `▾` before a box's title collapses it to its title (`▸` opens it again). A +collapsed box isn't drawn at all, so it costs the browser nothing while the node +updates; opened, it shows what's current. The open boxes take the room the collapsed +ones leave: in the right column they share it as before (the peers take all of it +when they're the only one open); the map collapsed, the right column takes the width. +The right column all collapsed keeps its width: its titles stay at the top, with a +hint that dragging the space beside them to the right hides the column. Which boxes +are collapsed is kept in the browser too. + +## The map + +### Nodes + +Every member of the swarm, by alias (or a shortened node ID): + +* **This node**: filled blue, slightly larger. +* **Filled**: the member reports its links to this node. +* **Hollow**: it reports none: it doesn't answer this node's LINKS, or its last answer + is more than 90 s old (it is unreachable from here, stopped, or doesn't yet take this + node for a member). +* **Red outline**: this node holds it dormant (it has been silent, or said farewell). +* **Dashed orange ring**: its clock is off the swarm's: it differs from the median of + the members' clocks by more than `--clock-skew-warn`, beyond what the measurements + allow. The members report their peers' clocks with their links, so this works for + members this node doesn't reach itself; it needs three members compared. + +### Links + +Two kinds, drawn over each other: + +* **Intended** (dashed grey): the neighbours the member list gives each member, as this + node computes them. Shown only where no measured link backs it. +* **Measured** (coloured): the links members report. A link is drawn if either end + reports it. + +A measured link is coloured by transport (blue Mycelium, purple libp2p, teal Tor). The +transport in use is the thick line in the middle, labelled with its round trip in +milliseconds; the other transports the peers can reach each other on are thin lines +beside it, from node to node, bowing out a little (furthest at the middle), with their +round trips smaller. The round trips are drawn near the end that +measured them: of the two ends' reports, the one drawn is the one with a round trip, +else the fresher. Then: + +* **Solid**: direct. +* **Dashed**: through a relay. +* **Dotted**: one end reports no links: what is drawn may be out of date. +* **Narrow, over a red band**: either end holds the other dormant, or the path in use + is down (no path to the peer answers; `/peers` shows its connection as `null`). + +Messages this node exchanges run along its links as dots, from sender to receiver. +They come from the per-peer message counts, which arrive every 2 s: what was sent and +received since is spread over the next 2 s, at most 6 dots per link and direction, +shared out by message kind, each kind in its colour (named in the legend; the console +colours them alike). A dot is as large as its kind's messages were on average +(by the size's logarithm: a ping small, a large sync message large); the slider right +after their legend makes all dots larger or smaller, kept in the browser. A picture +of the traffic, not a record of it; links between other members show none. +`messages: on` after the slider hides them (`messages: off` shows them again; the +legend and the slider greyed meanwhile); the choice is kept in the browser. + +A label on the map that changes, a round trip or a node's name, breathes once: it +glows around its new text and fades (not where the system asks for reduced motion). + +`links: live` beside the box's title says the members tell this node of their links' +changes as they happen. Clicked, it's `links: every 30 s`: the members are only asked +every 30 s, which costs the swarm fewer messages and leaves the map up to 30 s behind +(this node's own links stay current). It's the node's choice, for every page open on +it, until clicked again or the node restarts. + +The legend above the map names the transports that appear; `hide legend` beside the +box's title gives its room to the map (`show legend` brings it back). + +`layout: static`, beside it, is the default: nodes move only when the map is laid out +(below). A click switches to `layout: dynamic`: each measured link is as +long as its round trip is slow, against the median link's, by the square root, from +0.4 to 2.5 times the usual length; links without a round trip keep the usual length. +In either layout, a link only intended (dashed grey) gives way to the measured ones: it +pulls its ends together only a little when stretched, and doesn't push them apart. As +round trips change, the map moves to the new lengths smoothly. Clicked again, it's +static, the nodes staying where they are. The choice is kept in the browser. + +Hovering over a node or a link highlights it and shows a tooltip: for a link, each of +its lines with transport, direct or relayed, round trip, and how long ago it was +measured. + +### Moving around + +* **Wheel**, or pinching on a touch screen: zoom, around the pointer (or between the + fingers, the map moving with them). +* **Drag beside the nodes, or along a link**: pan. +* **Two fingers turning**, on a touch screen: the map turns with them; labels stay + upright. +* **Drag a node**: move it. On a dynamic map, the others follow at once, and once + let go of, it moves back among them; on a static map it stays where it is put until + the map is laid out again. +* **Double-click a node**: pin it where it is (a dot in its middle): the layout leaves + it there. A double-click whose second press drags the node pins it where it's + dropped. Double-clicked again, it's let go, and moves to its place among the others + (on a static map, only it moves). + +The map is laid out when the page opens, and again when the window or the box is +resized. A member that comes later is placed among the others, which stay where they +are, as they do when links appear or disappear; pinned nodes always stay put. The view and pins last as long as +the page is open. + +## Focus + +What the Figures box shows is what is in focus: + +* **Nothing** (the default): this node's figures. +* **A node** (click it on the map, or its row in Peers): this node's entry for that + peer (as `/peers` has it), and the links that member reports. Ringed on the map, + highlighted in Peers. A second click on its row, a click beside the nodes or a click + on this node goes back. +* **A link** (click it): whether the member list intends it, each end's report of it + (transport, relayed, round trip, dormant, its other transports, and how old the + report is), and, where this node is an end, the link as this node has it in `/peers`: + where messages go, its paths (each with its connection and round trip), and the + addresses known of the other end. Both ends are ringed. + +The focus is in the page's address (`#<node ID>`, or the two IDs of a link), so a +reload keeps it and a link to the page can name it. + +## Peers + +Every peer this node knows, with: + +| Column | Meaning | +|---|---| +| peer | Alias, or a shortened node ID (the full one in the tooltip). | +| status | `active`, `dormant` (for how long: in the tooltip) or `unknown` (not heard from since this node started). | +| seen | How long ago this node last heard from it. | +| via | The address this node sends to it at, and whether `direct` or `relayed`. | +| rtt | The round trip on that path. | +| clock | How far its clock is ahead of this node's (behind: `−`), give or take (in the tooltip); in orange beyond `--clock-skew-warn`, for sure ([`measurements.md`](measurements.md), Clocks). | +| neighbour | Whether this node keeps contact with it (HELLOs, keepalives, sync); others it only answers. | +| same epoch | Whether it is on this node's epoch ([`remakes.md`](remakes.md)); blank while unknown. | + +The box is as tall as its rows, up to eight; then it scrolls. + +## Figures + +The figures of what is in focus, as a tree: this node's `/admin/stats`, or a peer's +`/peers` entry and links. Branches stay as opened across updates, and ages count on +between them. What each figure means: [`measurements.md`](measurements.md). + +## Document + +The swarm's document, as this node holds it, its first level open. A branch is read +when opened, and read again when the document changes there; what changed or came +breathes once, as the map's labels do: its row glows and fades. + +Editing writes to the document, as the user the page was opened as, and the write goes +to the whole swarm like any other: + +* **Double-click an entry** to edit it: a string as text, anything else as JSON, a map + or list whole. Saved, it is shown open. The red `Delete` beside `Save` deletes the + entry instead, on a second click (`Delete: sure?`) from half a second to 3 s later: a + double-click doesn't delete. +* **`+` on a map's row** adds a key and its value. +* **`+` on a list's row** inserts an item at an index (`-1`: at the end); the items + from there on move up. + +Protocol data (`_continuum`) is written only by users in `--protocol-writers`, and epoch +metadata and the swarm ID by nobody ([`trust.md`](trust.md)). A key containing `/`, or an empty +key, is listed but can't be opened. + +## Console + +The messages this node exchanged with its peers, one by one: when (on the 24-hour +clock, to the millisecond), which way (`→` sent, `←` received, `↛` refused: the +transport wouldn't send it), with whom, the type (`HELLO`, `SYNC`, `STATUS_REPLY`, +`LINKS`, …, in its kind's colour, as on the map) and the size. The latest 500, the +newest at the bottom; scrolled along while at the bottom. They arrive with every update +(every 2 s). The PINGs the paths layer sends on its own, and the PONGs it answers +PINGs with, are listed too: under the node at the address they went to, where it is +known, else under the address. Not listed: WHO and ME, which stay in the layers +below the node (they are counted, [`measurements.md`](measurements.md), Traffic). + +A click on a message shows it in the figures box: when, with whom, its type and size, +and the message written out (its bytes as sizes). A click on it again, or on anything on +the map or in the peers, goes back. + +Pointing at a message marks the others of its exchange (a lighter grey, and a bar on +their left); while one is shown, its exchange stays marked. An exchange is a PING and +its PONG (by their nonce, wherever they were listed), a HELLO and its replies (by the +run of the node that sent the HELLO), a LINKS and its replies (by their nonce), a blob's +requests and answers, from whichever peers (by the blob), and a delta offered in a SYNC +with the blob it is fetched as. A STATUS and its reply carry nothing that ties them, nor +does an ordinary SYNC. + +The map marks the message's two nodes and the link between them in the grey its line +has in the console, each in its own shape: a circle around each node, and the link's +lines as they bow. The message shown and the one pointed at are both marked, each in +its line's grey; where they mark the same node or link, the shown one's wins. + +The other way round, the messages exchanged with a node, or over one of this node's +links, are tinted in the colour of its halo on the map: lightly while it is pointed at +there, stronger while it is in focus (clicked there or in the peers); where both tint a +message, in focus wins. + +## While you read + +Updates don't pull things from under you: while text is selected in a box, that box +waits, and while an entry is edited, the branch it is in waits; both catch up once you +are done. The map isn't redrawn under the pointer: what you point at stays put. + +## When something looks odd + +* **The status stays at `connecting…`, or the page doesn't load**: the node isn't + built with `dashboard` (404), or the user isn't in `--htpasswd` (401). +* **A hollow node**: no links report from it. If it is also dormant here, it is + probably down or unreachable; if not, it may not take this node for a member yet + (it hasn't registered it), or its answers don't arrive. +* **Only dashed grey between two nodes**: the member list wants them as neighbours, but + neither reports a link to the other: they haven't reached each other (yet). +* **A dotted link**: one end has stopped reporting; the link is as it was last + reported. +* **A red band**: one end holds the other dormant. If the other end still reports the + link, the trouble is likely in one direction only. + +## Underneath + +The page is `GET /admin/dashboard`, authenticated as the rest of `/admin`. It is kept up +to date through Server-Sent Events (`/admin/dashboard/events`); +`/admin/dashboard/state` has the latest state at once. + +**Events.** One producer per node serves every client, and runs only while one is +connected (or for a minute after a `/state` request); a page hidden for 30 s +disconnects until it is shown again. Every 2 s it builds the state; when anything but +ages changed, it sends every client a JSON merge patch onto the state before. Nothing +changed, nothing is sent. Ages are as of the event, and the page counts them on. A click +on the status refreshes at once (`POST /admin/dashboard/refresh`). When the node stops, +every client gets an `offline` event and its stream ends, so stopping doesn't wait for +them; the page reconnects once the node is back, and reloads if the node now serves +another version of it (the state's `page`). + +**The map's graphs.** The intended one is the neighbours the member list gives every +member (`swarm::topology`), as this node computes them. The measured one is every +member's links as it reports them in reply to a LINKS (sent to every member, at most +512, while the producer runs). The LINKS subscribes: the member replies again whenever +its links change (a peer, a transport, relayed, dormant, or a round trip moved by a +fifth; it looks every 2 s), for 90 s, renewed every 30 s (each renewal answered), and +ended when the producer stops; a node that is gone stops it by not renewing. A reply +older than 90 s is left out. A member keeps 64 subscribers at most. With `links: every +30 s` (`POST /admin/dashboard/links?live=false`; `true` back), the subscriptions end +and the members are only asked every 30 s, for every page open on this node, until +asked otherwise or the node restarts. + +The links a member reports: its neighbours and the peers it holds dormant (a neighbour +that goes dormant is one no more, but stays listed until it is heard from again), by +which transport, whether only through a relay, the round trip (as the paths layer +measures it), whether it holds the peer dormant or the path is down, the peer's other +transports with a path up (each with its fastest path's round trip, and whether it goes +only through a relay), and the peer's clock against the member's ([`measurements.md`](measurements.md), +Per peer, `clock`). A member answers a LINKS from members only, at most every 5 s, and +every build answers, with or without the feature (`swarm::links`). + +**Clocks.** The clocks in the links chain every member's clock to this node's, along +the narrowest bounds, and the map takes each against their median: a member more than +`--clock-skew-warn` off it, beyond the chained bound, is ringed. With fewer than three +members compared, none is. + +**Messages on the map** come from the per-peer message counts in the state (`/peers`, +`traffic`), not from messages of their own. **The console's** come from a log the node +keeps only while the producer runs (`traffic::MessageLog`, the latest 1000), sent as a +`messages` event every 2 s with those since the last. + +**The document** is read through the store API, a level at a time as it is opened +(`GET /store?shallow=true`, then `/store/<path>?shallow=true` for each map opened, and +`/store/<path>` for a long value). Where it changes, the node says so (a `document` +event with the paths changed, at most every 2 s), and the page reads again what is open +there. A key with `/` in it (or an empty key) can't be opened: store paths are split at +`/`. + +Edits go through the store API too, as the user the page was opened as: `PUT +/store/<path>` (a text saved becomes a string; the root isn't written whole), a list's +items each on its own (`PUT /store/<path>/<index>`), a new key with `PUT +/store/<path>/<key>`, an inserted item with `POST /store/<path>/<index>` (`-1` the end; +an index past the end is refused), and `DELETE /store/<path>`. The dashboard itself +writes nothing to the document: only edits do. diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..93726b1 --- /dev/null +++ b/docs/development.md @@ -0,0 +1,52 @@ +# Development + +## The loop + +```sh +cargo check --features mycelium,libp2p,tor,dashboard # does it compile: seconds +cargo quick # every test but the simulations: seconds +cargo sim # the simulations (src/simtest/): under a minute +cargo test --features mycelium,libp2p,tor,dashboard # both +cargo benchmarks # the benchmarks, in release builds +``` + +`cargo quick`, `cargo sim` and `cargo benchmarks` are aliases ([`.cargo/config.toml`](../.cargo/config.toml)), +with every transport and the dashboard (`dashboard`, `GET /admin/dashboard`; +[`docs/dashboard.md`](dashboard.md)). Each transport is a feature, none of them default, and a build +needs at least one: a transport left out takes its tests and dependencies with it +(`--features libp2p` builds neither `reqwest` and the daemon supervision of Mycelium, +nor arti; among Mycelium's tests is a swarm simulated on its envelope). + +The simulations send the wire encoding alone, as libp2p and Tor do; a simulated net +can put messages into a transport's envelope instead (`SimNet::with_codec`), and every +message kind is round-tripped through each codec (`wire::samples`). + +The dev profile ([`Cargo.toml`](../Cargo.toml)) keeps the loop short: the crate gets line tables only +(backtraces still name files and lines), and the dependencies are built optimised and +without debug info. Automerge's work dominates the tests, so they run several times +faster; the binaries are a quarter of the size, so they link faster. The dependencies +build once (a few minutes, longer on few cores); after that, a change rebuilds in +seconds. To step through a dependency in a debugger, override it locally (`debug = +true` for that package). + +Measured on 2 cores, `--features libp2p,tor` (before Mycelium became a feature), after +touching one file, with Cargo's default dev profile and with this one: + +| | Default profile | This profile | +|---|---|---| +| `cargo check` | — | 3 s | +| test build | 8 s | 6 s | +| binary build | 13 s | 6 s | +| tests but the simulations | 6 s | 1.4 s | +| all tests | 40 s | 31–35 s | +| test binary | 550–670 MB | 100–155 MB | + +## The simulations + +The simulations ([`src/simtest/`](../src/simtest/)) run whole nodes on a simulated network +([`transport/sim.rs`](../src/transport/sim.rs)), in real time: tokio's clock isn't paused, because a node's +document work runs on the blocking pool, which a paused clock doesn't wait for. So they +take as long as the protocol's timeouts make them, however fast the machine, and on a +busy machine some fail now and then (a node's timer fires before the work it waits for +is done). One failing alone (`cargo sim <name>`) and passing then is that, not a +regression; one failing again and again is a regression. diff --git a/docs/example.toml b/docs/example.toml new file mode 100644 index 0000000..95b6f52 --- /dev/null +++ b/docs/example.toml @@ -0,0 +1,280 @@ +# An example configuration with every option, explained (`configuration.md` has the +# rules). Use it with `continuum --config example.toml`; `continuum --print-config` +# shows what a configuration amounts to, and `continuum --help` lists the options. +# +# Keys are the long option names. Nested tables join their keys with `-`: `key-file` +# under `[mycelium]` is `mycelium-key-file`. Options left out (or commented out, as most +# are here) take their defaults, shown in the comments. Command-line arguments win over +# files. +# +# Any value can come from elsewhere instead of being written out: +# { env = "NAME" }, { file = "PATH" }, { command = "COMMAND" } or +# { command = ["program", "argument"] }, with format = "json" to parse it as JSON; +# and text can mix in such values: "node-${env:HOSTNAME}" (`$$` for a `$`). +# Imported values are never logged or shown by --print-config. + +# --- Other files ----------------------------------------------------------------------- + +# Load these first (a path or a list), in this order; relative to this file. This file's +# own values win over its includes, later includes over earlier ones. An entry +# { path = "...", optional = true } is skipped where the file is missing. +# include = ["/run/secrets/continuum.toml", { path = "local.toml", optional = true }] +# Options to set back to their defaults here (e.g. one an include sets). +# unset = ["alias"] + +# --- Who this node is ------------------------------------------------------------------ + +# This node's key (Ed25519, 32 raw bytes): who the node is, whatever transport reaches +# it. Keep it with the node: a new key is a new node. Required (but with relay-only). +node-key-file = "/var/lib/continuum/node.key" + +# Create a new node key at node-key-file if there is no file there (a new node). Without +# it, a missing key file is an error, so a mistyped path never makes a new node. Best +# given once on the command line (--node-key-create) rather than kept here. +# node-key-create = false + +# A name for this node, shown next to its node ID (/peers, metrics). Published in the +# member list; not unique, never an identity. +# alias = "node-${env:HOSTNAME}" + +# Only relay (libp2p circuit relay), without being a node: no document, no swarm, no +# node key. Needs libp2p alone (libp2p's enable, no other), its listen and +# key-file; options about a swarm or a document (node-key-file, seeds, genesis, +# state-file, ...) are errors with it. The HTTP API, with htpasswd, has /health, +# /admin/stats and /admin/log only. List such relays in the document +# (_continuum/relays) or in libp2p's relays. +# relay-only = false + +# --- The HTTP API ------------------------------------------------------------------------ + +# Address of the HTTP API. Plain HTTP: to expose it, put a TLS proxy in front. +# bind-addr = "127.0.0.1:3000" + +# Users of the HTTP API (an htpasswd file with Argon2 hashes). Required (but with +# relay-only: without it, no HTTP API). +htpasswd = "/etc/continuum/htpasswd" + +# HTTP API users (from htpasswd) who may write protocol data under /store/_continuum +# (remake records, members, forget records), e.g. to repair a stuck remake. Such writes +# act on the whole swarm and are logged. Epoch metadata and the swarm ID are never +# writable. +# protocol-writers = ["alice"] + +# How log lines look: text (timestamped), journal (for systemd's journal: no timestamp or +# colours, the level as each line's priority, for journalctl -p), or auto: journal when +# stdout goes to the journal, else text. +# log-format = "auto" + +# Log node IDs and transport identities as they are. Without it, a member's read +# `<scheme>:<alias>` wherever they appear (`continuum:<alias>`, `libp2p:<alias>`), as the +# member list names it. +# log-raw-ids = false + +# Enable the /debug endpoints (tests only). +# debug-api = false + +# --- The swarm --------------------------------------------------------------------------- + +# How to reach a node of the swarm to join: an address (`mycelium:<public key>` or +# `libp2p:<multiaddr>`, as GET /admin/stats lists them), a Mycelium public key or IPv6 +# address, a host name with an AAAA record for one, or a libp2p multiaddr (with or +# without /p2p/<peer ID>). A node keeps trying until it has joined. +# seeds = ["node1.example.org", "libp2p:/ip6/fd00::1/tcp/9661"] + +# Start a new swarm, as its genesis node, instead of joining one through seeds. Every +# start with it starts another new swarm: drop it once the swarm exists, and give the +# node seeds instead. Can't be combined with seeds. With a state file that remembers a +# swarm, it starts that swarm again instead (empty; nodes of it still running merge with +# this one). Best given once on the command line (--genesis). +# genesis = false + +# With genesis: make the new swarm's document as a remake of this kind would, instead of +# starting it empty: import:PATH (an Automerge document, e.g. a backup, without its +# history and protocol data) or replace:PATH (a JSON document, the store root); compact +# changes nothing. +# remake = "import:/var/backups/continuum.automerge" + +# Start the swarm again from a backup (as GET /admin/document saves it): this node loads +# it, on the swarm and epoch it names, and is ready at once; other nodes join it. +# Members of the swarm still running merge with it, or it rejoins them if they moved on +# to a newer epoch (the backup is then dropped). For one start: command line. +# recover-from = "/var/backups/continuum.automerge" + +# Pre-shared swarm key: only nodes that know it may join (raw bytes, at least 16; e.g. +# `head -c 32 /dev/urandom`). Without it, any node that can reach a member may join. +# swarm-key-file = "/etc/continuum/swarm.key" + +# Create a new swarm key at swarm-key-file if there is no file there: on the swarm's +# first node, as the others need a copy of it (a key of their own keeps them out). +# Without it, a missing key file is an error. +# swarm-key-create = false + +# Not in files: `force` (override checks that refuse to start: swarmId, +# badDocumentFile, replaceDocumentFile) is for one start, on the command line only. + +# --- Files of this node ------------------------------------------------------------------ + +# Where this node remembers its swarm across restarts: the swarm ID and some members to +# rejoin through (not the document). Written once ready, then rarely. Without it, a +# restarted node knows only its configured seeds. +# state-file = "/var/lib/continuum/state.json" + +# This node's document on disk. Read at start: the node resumes with it, ready at once. +# Kept up to date while ready: changes are appended every document-append, and the file +# is rewritten after a switch, a rejoin or a restart, and when the appended changes +# outgrew it. +# document-file = "/var/lib/continuum/document.automerge" + +# How often changes are appended to the document file. +# document-append = "5s" + +# Shortest time between rewrites of the document file because its appended changes +# outgrew its full save (a new epoch or a restart rewrites it at once). +# document-rewrite = "1h" + +# --- Timings and sizes ------------------------------------------------------------------- +# Durations: seconds (30, 0.5) or with units (500ms, 30s, 1m, 1h30m, 7d). +# Sizes: bytes or with units (64KiB, 4MiB). + +# Keepalive interval; dormancy, resync and probe timings derive from it. 15–30s on a +# LAN or between servers, 1–2m over WAN, mobile or metered links. +# keepalive = "1m" + +# Longest wait between HELLOs to a member that doesn't answer: they start at 10 +# keepalives and double up to this, so it bounds how long a returning member, or the +# other side of a partition, takes to be noticed. Default: 60 keepalives (1h at 1m). +# dormant-probe-max = "15m" + +# Neighbours each node keeps (at least 2). Swarms of up to this many + 1 nodes are a +# full mesh; larger ones sync over several hops. With several transports, about this +# many per transport. +# neighbours = 12 + +# Members that renewed their alive stamp within this time vote on remakes (a majority of +# them must agree). At least 7h. Members renew their stamps every seventh of the +# shortest window any of them publishes, hourly at most, so nodes may differ. Matters +# on the node that coordinates a remake. +# electorate-window = "7d" + +# What becomes of a write a remake or rejoin would carry over into the new epoch though +# a newer one touched its path (last writer wins, by the writers' clocks): "drop" leaves +# it out (logged), "quarantine" keeps it for an operator (GET /admin/quarantine). +# superseded-writes = "drop" + +# Warn in the log when this node's clock differs from most of its peers' by more than +# this, for sure (beyond half the round trip it was measured over, from their PONGs); 0: +# never. +# clock-skew-warn = "2s" + +# Chunk size when serving large transfers to other nodes (e.g. an epoch's snapshot to a +# rejoining node), 1KiB to 4MiB. +# blob-chunk = "64KiB" + +# --- Transports -------------------------------------------------------------------------- + +# The transports' order of preference, whether enabled or not (each is enabled in its +# own table: enable = true). A peer is reached on the first enabled one both have until +# the paths are measured, then on the fastest; nodes on several carry the traffic +# between nodes that share none. Options of a transport not enabled are ignored. +# transport-order = ["mycelium", "libp2p", "tor"] + +[mycelium] +# Use this transport (in builds with the mycelium feature). A node needs one at least; +# none is on by default. +# enable = false + +# Peers of the supervised Mycelium daemon (overlay links), not continuum peers. Public +# Mycelium nodes are used only if listed here. +# peers = ["tls://127.0.0.1:9651"] + +# Private key of the supervised Mycelium daemon (its identity, and so its address); a +# new one each start if absent. +# key-file = "/var/lib/continuum/mycelium.key" + +# Create a new key at key-file if there is no file there (a new Mycelium address). +# Without it, a missing key file is an error. +# key-create = false + +# TCP port the supervised daemon listens on for overlay peers; a free port if absent. +# tcp-listen-port = 9651 + +# A private Mycelium network for the supervised daemon: its key and its name, together. +# They keep other Mycelium nodes from reaching ours at all. +# network-key-file = "/etc/continuum/mycelium-network.key" +# network-name = "example" + +# Use a Mycelium daemon running elsewhere (its REST API) instead of supervising one. +# Excludes the options above except peers. Secret: a URL may carry credentials. +# rest-url = "http://[::1]:8989" + +[libp2p] +# Use this transport (in builds with the libp2p feature). +# enable = false + +# Where to listen. Without, the node only connects out (peers answer over its +# connections, and a relay makes it reachable). Its listen addresses are published in +# the member list, loopback and link-local ones left out. +# listen = ["/ip4/0.0.0.0/tcp/9661", "/ip4/0.0.0.0/udp/9661/quic-v1", "/ip6/::/tcp/9661"] + +# The addresses to publish instead of those listened on, e.g. a public address forwarded +# to this node's listen port. +# announce = ["/ip4/203.0.113.7/tcp/9661"] + +# Its key (Ed25519, 32 raw bytes), which makes its peer ID; a new one each start if +# absent (the node ID stays the same either way). Worth keeping on a relay, so others' +# addresses through it stay the same. +# key-file = "/var/lib/continuum/libp2p.key" + +# Create a new libp2p key at key-file if there is no file there (a new peer ID). Without +# it, a missing key file is an error. +# key-create = false + +# Relay for nodes of the swarm that can't be dialled (bounded: 1024 circuits, 64 per +# peer, each up to an hour and 1 GiB), and say so in the member list, where nodes that +# need a relay find it. Needs a listen address others can dial. +# relay = false + +# Relays to use when this node can't be dialled (no listen address others could dial, +# or AutoNAT says so): multiaddrs, with or without the relay's peer ID. It picks up to +# two relays, these first, then members that relay. Any libp2p node serving circuit +# relay v2 will do; most keep circuits short (2 minutes, 128 KiB), enough to punch holes. +# relays = ["/ip4/203.0.113.7/tcp/9661"] + +# Find nodes on the LAN (multicast DNS, IPv4). Those that speak continuum are seeds +# while joining, so nodes on one LAN find each other without seeds; a swarm key keeps a +# new node from joining another swarm on the same LAN. +# mdns = false + +# Ask the router to forward this node's listen ports (UPnP), and publish where, so that +# nodes elsewhere can dial it without a relay or hole punching. Opens those ports on the +# router for as long as the node runs. +# upnp = false + +# Address ranges libp2p leaves out (VPNs, say): neither published nor offered for hole +# punching nor dialled, and connections there are refused. Announced addresses are +# published anyway. +# ignore = ["10.244.0.0/16", "42::/16"] + +# Hosts on the internet (names or addresses), only to tell whether this node has a +# route to the internet, per IPv4 and IPv6 (the routing table is asked; nothing is sent +# to them). Where it has none, its own public addresses of that family aren't +# published. Without, it isn't asked. +# internet-hosts = ["one.one.one.one"] + +# Use public libp2p infrastructure: join the IPFS DHT (through the IPFS bootstrap nodes) +# to find peers' addresses and public relays, picked after members that relay. Public +# relays only keep circuits for minutes and some KiB: enough to punch holes through +# NATs. Makes this node's peer ID and addresses known to the DHT. Without it, nothing +# public is used. +# public = false + +[tor] +# Use this transport (in builds with the tor feature). +# enable = false + +# Where arti (Tor, embedded) keeps its state and cache, among them the onion service's +# key, which makes its address: keep it with the node. Required with tor. +# state-dir = "/var/lib/continuum/tor" + +# The port of this node's onion service. +# port = 9662 diff --git a/docs/glossary.md b/docs/glossary.md new file mode 100644 index 0000000..71d3014 --- /dev/null +++ b/docs/glossary.md @@ -0,0 +1,154 @@ +# Glossary + +The words used in the code, logs, API and docs, and what they mean here. + +## Nodes and the swarm + +**node**: a running continuum instance, together with its transports (and a Mycelium +daemon it supervises), document, lifecycle (Joining, Ready, Rejoining), API and log. The word for what a single instance +does: "the node becomes ready", "a new node joins". Rule of thumb: say *node* when +nobody else is involved. + +**peer**: another node, seen from one node's side. Always relative to someone, which +is why `PeerState`, `/peers` and dormancy are per peer. "A node forgets a dormant peer" +reads right; "a peer becomes ready" doesn't. Rule of thumb: say *peer* when you mean +"the other side" or "the others". + +**swarm**: all the nodes together, sharing one document. Identified by the **swarm +ID**, which the genesis node chooses and records in the document (`_continuum/swarm`). An optional **swarm key** (`--swarm-key-file`) +is a shared secret every node must prove in the handshake. + +**node ID**: who a node is, `continuum:<hex>`, from its **node key** +(`--node-key-file`); the same whatever transport reaches it. An **address** says where +a node is on one transport (`mycelium:<public key>`); a node proves its node ID at an +address with WHO and ME ([`peers.md`](peers.md)). + +**seed**: something that can be resolved to reach a node (`--seeds`): an address on +any transport, or a transport's own short form (for Mycelium a public key, an IPv6 +address, or a host name with an AAAA record for one; [`transports.md`](transports.md)). + +**state file**: what a node remembers across restarts (`--state-file`): its swarm ID +and some members to rejoin through. Not the document. + +**document file**: a node's document on disk (`--document-file`), with the actor IDs +it wrote with; the node resumes with it after a restart. + +**standing** of a peer, from one node's side (`Standing`): +* **stranger**: has sent us something, nothing more. +* **admitted**: proved the swarm key (or there is none). +* **member**: completed the handshake and belongs to our swarm. Only members sync. + +**registered member**: a node listed in the swarm's member registry +(`_continuum/members/<node ID>`), which every node writes for itself once ready. +The registry, not the handshake, decides the topology and who votes on remakes. When +the context doesn't make clear which is meant, say *registered member*. + +**neighbour**: a member this node keeps alive and syncs with on its own (keepalives, +sync openings, relay). Up to `--neighbours` (k) of them, chosen from the registry +(`swarm::topology`); with at most k + 1 reachable members, everyone is everyone's +neighbour (**full mesh**). Other members are only answered. + +**alias**: the name a node gives itself (`--alias`), published in the registry. Not +unique, never an identity; the node ID is. + +**dormant**: a peer we haven't heard from for 4 × keepalive. It is probed rarely and +left out of the topology until it answers again. + +**stability tier**: what a node judges about itself and publishes in the registry. It +becomes **stable** once it was connected at least 95 % of a full day, and **transient** +again below 80 %. Stable members are preferred as neighbours, forming a backbone. + +**alive stamp**: the time a node last renewed its registry entry (about daily). Nodes +renewed within `--electorate-window` form the electorate of a remake. + +## The document and sync + +**document**: the replicated Automerge document. The store API reads and writes paths +in it. + +**protocol data**: what the nodes keep under `_continuum` in the document: the swarm +ID, the member registry, forget records, remake records, the votes register and epoch +metadata. Only users listed in `--protocol-writers` may change it through the store +API; the swarm ID and epoch metadata never. + +**heads**: the hashes of the document's latest changes, those no other change depends +on. Several heads mean concurrent changes that no later change has merged yet. Two +nodes with the same heads have the same document. + +**opening**: the first sync message to a peer after a reset, before anything has come +back from it. After `reset_sync()` the Automerge `sync::State` is new, so +`their_heads`, `their_have` and `their_need` are all unset; an opening can only say +"these are my heads, here is a Bloom filter of what we share", and carries no changes +since we don't know yet what the peer has. The code doesn't track openings +separately: `their_heads.is_none()` means nothing has come from the peer since the +reset. Who may open towards whom: [`peers.md`](peers.md), Handshake and join. + +**path**: an address of a peer on one of this node's transports (`<scheme>:<target>`). A +peer with several (on several transports) is reached on the best: paths are measured +with a **PING** each answered by a **PONG**, and a path that fails, or stops answering, +is passed over for the next. + +**keepalive**: what a pair exchanges when it has been quiet for a keepalive interval +(`--keepalive`): a PING and its PONG, and every third time a **STATUS** and its +STATUS_REPLY instead, carrying the sender's heads (or a digest of them and some of +them), epoch and readiness. The two sides of a pair take turns; the one that **leads** +the pair sends the first. + +**initial sync**: a joining node's first complete sync. It's **caught up** once a ready +peer with non-empty heads has nothing left that it lacks. + +**catch-up source**: the one peer a joining node syncs with while catching up, the +closest by round-trip time first. + +**relay**: in a partial mesh, a node announces new heads to its neighbours, which pull +the changes they lack. + +**blob**: bytes too large for one message, fetched in chunks from several peers at once: +an epoch's snapshot change (for a rejoining node) or a remake's content (for voters). + +## Epochs and remakes + +**epoch**: one stretch of the document's life, from one remake to the next. Peers on +different epochs don't sync. Identified by its **generation** and its ID (the hash of its +snapshot change), shown as `generation:id`. + +**generation**: an epoch's counter: the number of remakes since genesis. It orders +epochs, so a node can tell whether a peer is behind or ahead. Two epochs of the same +generation (different IDs) can only come from concurrent remakes. + +**genesis**: the first epoch (generation 0), and the **genesis node** that started the +swarm (started with `--genesis`, empty or with content from `--remake`). + +**remake**: a coordinated move of the whole swarm to a new epoch. Kinds: +**compact** (same content, no history), **replace** (new content as JSON), +**import** (new content as an Automerge document). + +**base**: the heads the new epoch's content accounts for. Writes made after them are +carried over into the new epoch (**replayed**), at their path or where the remake's +**rules** say: by every node that switches, and by their author when it rejoins. A compaction's base is the coordinator's +heads when it commits; other kinds fix it when proposed. + +**snapshot**: the new epoch's first change, which every node builds itself, byte for +byte the same; its hash is the epoch ID. + +**coordinator**: the node a remake was proposed to. It counts the votes and writes the +decision. If it is gone for good, another node can **adopt** the remake and become +its coordinator. + +**electorate**: the registered members whose votes count for a remake, fixed when it is +proposed. The remake commits once a **majority** (the **quorum**: more than half) voted +for it. Each member has one vote per epoch, kept in the **votes register** +(`_continuum/votes`); it is free again only if that remake is aborted. + +**switch**: a node moving to a committed epoch by building its snapshot itself. + +**rejoin**: a node moving to a newer epoch by fetching it from peers instead (it was away, +or its own build failed). + +**replay window**: how far behind a rejoining node may be and still have its own writes +carried over automatically. + +**quarantine**: a node's own writes a remake couldn't place, kept on that node for an +operator to replay or discard. + +**archive**: the documents of previous epochs a node keeps for a while. diff --git a/docs/lifecycle.md b/docs/lifecycle.md new file mode 100644 index 0000000..9fe9442 --- /dev/null +++ b/docs/lifecycle.md @@ -0,0 +1,238 @@ +# The life of a node + +A node passes through a few phases between starting and stopping. This page says what +each phase is for, how a node enters and leaves it, and what it does and doesn't do +meanwhile. The protocol between peers is in [`peers.md`](peers.md), remakes in [`remakes.md`](remakes.md); this +page refers to them rather than repeating them. + +```mermaid +stateDiagram-v2 + direction LR + [*] --> Starting + Starting --> Ready: --genesis, --recover-from,<br/>or a document file holding a document + Starting --> Joining: otherwise + Joining --> Ready: swarm ID known, initial sync done,<br/>no rejoin running + Ready --> Rejoining: enough ready members<br/>advertise a newer epoch + Rejoining --> Ready: epoch verified, own writes<br/>replayed or quarantined + Rejoining --> Ready: abandoned - old document restored + Ready --> Stopping: SIGINT, SIGTERM,<br/>or a transport stops + Joining --> Stopping + Stopping --> [*] +``` + +In the code there are two lifecycle states, `Joining` and `Ready` (`NodeLifecycle`); +rejoining is `Joining` with the swarm ID kept, and starting and stopping are what +[`main.rs`](../src/main.rs) does before and after. The phases below are the operator's view. + +| Phase | Reads | Writes | Registers, votes, syncs as a source | Files written | +|---|---|---|---|---| +| Starting | — | — | no | — | +| Joining | `503` (`GET /store` root: the live, partial document) | `503` | no | — | +| Ready | live | yes | yes | state file, document file | +| Rejoining | the pre-rejoin document, with `x-continuum-stale` | into the pre-rejoin document, carried over at the end | no | document file (the pre-rejoin document) | +| Stopping | draining | draining | no | state file, document file | + +`/health`, `/peers`, `/admin/stats`, `/admin/config` and `/admin/log` answer in every +phase, and `/ready` says `200` only when ready (both without authentication); the +remake endpoints and `GET /admin/document` need a ready node. + +## Starting + +What happens before the node talks to anyone, in order: + +1. **Configuration** is loaded from defaults, files and arguments ([`configuration.md`](configuration.md)). + Contradictions are errors; `--print-config` shows the result and exits. +2. **The node key** (`--node-key-file`) and **the swarm key** (`--swarm-key-file`) are + read, so a bad file fails at once. A missing key file is an error, unless + `--node-key-create` asks for a new node key (a new node), or `--swarm-key-create` + for a new swarm key (on the swarm's first node; the others need a copy). +3. **The state file** (`--state-file`) is read: the swarm ID the node belongs to and + members to reach it through. +4. **A document to start on**, if any: the document file (`--document-file`) or a + backup (`--recover-from`). It names the swarm and the epoch the node is on. A new + swarm's document is made here too (`--genesis --remake KIND:PATH`). +5. **The transports** (those enabled), in order: for Mycelium, + the node connects to a running daemon (`--mycelium-rest-url`) or starts and + supervises one; Tor bootstraps arti. A signal stops the node while they start. +6. **The workers** start: message consumer, sync, peers, remakes. +7. **The HTTP API** listens (`--bind-addr`), also while the node joins (the table + above says what it answers then). + +Then the node is either ready at once or joining: + +| Start | Without a remembered swarm | With one (state file) | +|---|---|---| +| `--genesis` | a new swarm, ready at once | that swarm again, empty; nodes of it still running merge with this one | +| document file holding a document | ready at once, on the document's swarm and epoch | the same; the two must name the same swarm | +| `--recover-from` | ready at once, on the backup's swarm and epoch | the same; the two must name the same swarm | +| seeds, or none | joining through the seeds | joining that swarm, through the seeds and the remembered members | + +Conflicts between these inputs are errors unless overridden with `--force` +([`configuration.md`](configuration.md), "Forcing"): `swarmId` (the document names another swarm than the +state file remembers), `badDocumentFile` (the file can't be read: moved aside, start +without it), `replaceDocumentFile` (`--recover-from` while the document file holds a +document: the backup wins). `--genesis` with seeds, with `--recover-from`, or with a +document file holding a document is an error with no override. + +### What the files are for + +* **The state file** holds the swarm ID and the addresses of up to 16 members alive + within the electorate window (stable ones first), and the members' aliases, so the + logs name members by them from the start (perhaps stale; a change is logged once the + member list says so; `--log-raw-ids` leaves IDs as they are). With it, a restarted node joins only its own swarm (its + HELLOs claim the remembered ID) and finds it without configured seeds. Without it, + a node remembers nothing, and `--genesis` on every start makes another new swarm + while the old one's nodes go on without it. The file is written when the node becomes + ready, when the remembered members change (checked every 5 minutes, written at most + hourly), and at shutdown; atomically; an unreadable one is moved aside (`.bad`). +* **The document file** holds the document itself and the actor IDs this node wrote + with, so a restart resumes with the data and a later rejoin still replays the node's + own writes. It is appended to every `--document-append` (default 5s) and + rewritten after an epoch change, a restart, or when the appended part outgrew the + full save (at most every `--document-rewrite`, default 1h); a record cut short + by a crash is dropped when reading. A node starting from it is ready at once; sync + merges what it missed, or it rejoins a newer epoch. +* **A backup** (`GET /admin/document`) is the whole document with its history, naming + its swarm and epoch. `--recover-from` starts a node on it; `--genesis --remake + import:FILE` starts a *new* swarm with its content; an `import` remake brings its + content into a running swarm ([`remakes.md`](remakes.md)). + +### A whole swarm restarting + +If every node restarts without document files, none has any data: they all join, wait +for each other, and none becomes ready. Start one with `--genesis` (with a state file: +the same swarm again, empty) or with `--recover-from` and a backup; the others join +it. Nodes with document files are ready at once and merge with each other. + +## Joining + +The node knows no swarm (or only the remembered one) and has no document. It resolves +its seeds and says HELLO; the first HELLO_REPLY tells it the swarm ID and other peers +to greet. It then catches up: it syncs with one peer only, the *catch-up source*, so +that not every peer sends it the whole document at once; the others wait and are +answered once the initial sync is done ([`peers.md`](peers.md), "Handshake and join", says how +the source is chosen and given up). + +A joining node never gives up: seeds that don't resolve are retried with backoff, +unanswered HELLOs are resent, and every five minutes it logs why it is still joining. +A node without seeds and without `--genesis` has nothing to join and waits (seeds may +be added later). + +If the swarm has moved on to a newer epoch than the node's (a new node starts at +genesis), the node rejoins while still joining: it downloads that epoch's snapshot +first, then catches up. Bootstrap waits for the rejoin, so the node becomes ready only +after the epoch check. + +The node becomes **ready** when it knows the swarm ID, its initial sync is done (a +ready peer with a non-empty document has nothing left that the node lacks) and no +rejoin is running. + +## Ready + +The normal phase. The node: + +* serves reads and writes, and syncs with its neighbours ([`peers.md`](peers.md), "Member + registry and neighbours"); its own writes go out to them, others' changes arrive by + sync and relay; +* **registers** in the member registry, renews its alive stamp daily, and publishes + its stability tier; +* takes part in **remakes**: votes once it holds the content, coordinates those + proposed to it, and **switches** when one commits ([`remakes.md`](remakes.md)). A switch is a + step inside this phase, not a phase of its own: after the switch delay the node + builds the new document, verifies the epoch, replays the writes since the base into + it and swaps documents under the lock. Sync with every peer restarts; peers still + on the old epoch are told by a STATUS; +* judges its peers' liveness (dormancy), heals diverged sessions, and answers anyone; +* keeps its files up to date (above). + +## Rejoining + +Entered when enough active, ready members (two if this node was ready, one if it +never was) advertise the same newer epoch, at most 1000 generations ahead: the node +missed a switch (it was away, or its own build didn't match). With several epochs of +the newest generation advertised (two remakes committed for one epoch; [`remakes.md`](remakes.md), +Limits), it follows the one most of them are on (of those, the lowest ID). A ready node +also rejoins **sideways**, to another epoch of its own generation, when more ready +members are on it than on its own (counting itself; two at least; on a tie, the lower +ID): its own writes on the epoch it leaves all go to quarantine (step 5). What happens: + +1. at once (under the document's lock), the node swaps in an empty document labelled + with the target epoch, which takes no writes; sync with every peer resets; and a + ready node **suspends** (`Joining` again, swarm ID kept) and keeps its document as + the **stale view**: reads are answered from it, marked `x-continuum-stale`, and + writes go into it, answered as usual (and appended to the document file); a node + that wasn't ready refuses writes (`503`) as before; +2. it doesn't sync the stale view with anyone: what it takes is carried over at the + end (step 5); +3. it **downloads the epoch's snapshot** in chunks from ready members on that epoch + ([`swarm/blob.rs`](../src/swarm/blob.rs)), then ordinary sync adds the changes made since, from one source + as when joining; +4. once caught up, it **verifies** that the document starts with the target epoch's + snapshot change and that the metadata matches; +5. it decides about **its own writes** the swarm never saw: those made before the + rejoin are replayed (at their path, or where the remake's rules say) if within the + replay window of the first remake it missed, the rest go to quarantine for an + operator ([`remakes.md`](remakes.md), "Quarantine"); those made during the rejoin are replayed + whatever the window says. Either way, a write is left out where newer writes + touched its path (last writer wins, [`remakes.md`](remakes.md), "Writes around a remake"); +6. at once (under the document's lock, with step 5's replay), the node **resumes**, + drops the stale view and takes writes in the live document again; the old document + goes to the **archive** (kept as the remake's retention says). + +If no peer completes a sync within `max(dormant_after, 60 s)`, or the check in step 4 +fails, the rejoin is **abandoned**: the old document, with the writes it took +meanwhile, and its epoch are restored, the node resumes (at once, under the document's +lock), and tries again 30 s later. `/admin/epochs` shows the attempt and any error. + +```mermaid +flowchart TD + A[Active, ready members advertise a newer epoch<br/>at most 1000 generations ahead] --> A2{Enough of them for one epoch?<br/>2 if we are ready, or all our active members if fewer;<br/>else 1} + A2 -- no --> S{Ready, and more ready members on another<br/>epoch of our generation than on ours?} + S -- no --> X[Log and wait] + S -- yes --> C + A2 -- yes --> B[Target: the newest such epoch;<br/>of several in one generation, the one most of them are on] + B --> C{Retry pause over?<br/>30 s after a failure} + C -- no --> X + C -- yes --> F[At once: swap in an empty document labelled with the target epoch;<br/>if ready: suspend, the old document becomes the stale view, taking writes;<br/>reset sync with all peers, announce epoch:<br/>sync opening to ready peers on it, STATUS to the rest] + F --> F2[Download the snapshot change in chunks<br/>from ready members on the epoch; sync waits] + F2 --> G[Ordinary sync adds the changes since<br/>or the whole epoch if the download failed] + G --> H{Initial sync done?} + H -- no --> T{Longer than max of dormant_after and 60 s?} + T -- no --> G + T -- yes --> Z + H -- yes --> I{Snapshot change of the target present?} + I -- no --> G + I -- yes --> J{Snapshot metadata valid?} + J -- no --> Z[Abandon: restore old document and epoch,<br/>resume, retry later] + J -- yes --> K{Own changes missing from<br/>the epoch after ours?} + K -- none --> W + K -- "some, after a sideways rejoin" --> Q + K -- some --> L{Within that epoch's replay window?<br/>gap in epochs or time since proposal} + L -- yes --> R[Replay them, last writer wins] + L -- no, or unknown --> Q[Quarantine: /admin/quarantine] + R --> W + Q --> W[At once: replay the writes made during the rejoin,<br/>resume, drop the stale view, take writes again] + W --> O[Archive old document per retention,<br/>unless the quarantine keeps it] +``` + +## Stopping + +On SIGINT or SIGTERM (or when a transport stops: a supervised Mycelium daemon exits, +or the libp2p or Tor transport ends; `A transport stopped`, error), the node stops +accepting HTTP connections and drains the open ones (the dashboard's event streams end +with an `offline` event; from then on its keepalives say it is leaving), sends its neighbours a farewell: a last STATUS saying it isn't +ready and is leaving (one catching up from it picks another source, and each holds it +dormant at once rather than after `dormant_after`; at most 2 s), writes the state file +if the +remembered members changed, appends what the document file lacks, and exits. A second +signal exits at once. Nothing else is saved: the archive, the quarantine and all +measurements are gone. A node that stops in the middle of a rejoin loses nothing its +peers don't have, and starts the rejoin over when it returns. + +## Other phase machines + +* **A remake** goes `proposed` → `committed` or `aborted`, or `expired` when its + coordinator is gone: [`remakes.md`](remakes.md). +* **A peer**, seen from one node, has a standing (stranger, admitted, member), a + liveness (unknown, active, dormant) and a handshake state: [`peers.md`](peers.md), "Liveness, dormancy + and HELLO state". diff --git a/docs/measurements.md b/docs/measurements.md new file mode 100644 index 0000000..fb01de0 --- /dev/null +++ b/docs/measurements.md @@ -0,0 +1,299 @@ +# Measurements + +What a node measures about itself and its peers, where to read it, and what the +numbers mean. Everything here comes from traffic and work the node has anyway; it +sends no probes of its own and nothing is measured by anyone else, but for the +dashboard's map (LINKS, only while someone looks at it). + +Four places to read: + +* `GET /admin/stats`: the node's figures as JSON, or as Prometheus text when asked + for it (`Accept: text/plain` or `application/openmetrics-text`). Prometheus names + are `continuum_…`; per-peer series carry a `peer` label (the node ID) and can be + joined with `continuum_peer_info` for the alias. +* `GET /peers`: one entry per known peer, keyed by its node ID, with the per-peer + figures. +* The log: incidents as they happen (a peer going dormant, a heal, a stalled catch-up), + at warn or info; `PUT /admin/log` changes the filter at runtime. +* `GET /admin/dashboard` (builds with the `dashboard` cargo feature): a page with the + two above and a map of the swarm, live ([`dashboard.md`](dashboard.md)). + +All counters count since the process started; a restart resets them. Durations are +whole seconds in JSON unless named `…Ms`; Prometheus has seconds throughout. + +## About the node + +| JSON | Prometheus | Meaning | +|---|---|---| +| `uptimeSecs` | `uptime_seconds` | Since the process started. | +| `ready` | `ready` | The node holds the swarm's document and serves writes ([`lifecycle.md`](lifecycle.md)). | +| `epoch` | `epoch_generation` | The local epoch; the generation counts remakes since genesis. | +| `staleHelloReplies` | `stale_hello_replies_total` | HELLO_REPLYs to a HELLO from an earlier run of this node, ignored. A few after every restart are normal (a transport queued them, Mycelium's above all); a steady stream means a peer keeps answering an old HELLO. | + +## The document + +| JSON | Prometheus | Meaning | +|---|---|---| +| `document.heads` | `document_heads` | Heads of the local document now. 1 means every change has been merged into one line of history; more means concurrent writes not yet followed by a merging write. | +| `document.maxHeads` | `document_max_heads` | The most heads seen right after applying a sync message. A high value shows how concurrent the swarm's writing gets; it also bounds keepalive size (above 32 heads, keepalives carry a digest and 32 of them). | +| `document.savedBytes` | `document_saved_bytes` | The document as Automerge saves it (compressed): roughly what a joining node downloads and what a document file rewrite writes. Measured in the background, only after a change, at least a minute apart and at most 1% of the time (a document that takes 2 s to save is measured every 200 s at most); an export (`GET /admin/document`) counts as a measurement. `savedBytesSecsAgo` and `savedBytesTookMs` say how fresh and how costly. | +| `document.lockWait.read`, `.write`: `count`, `sumUs`, `maxUs`, `buckets[]` (`leUs`) | `document_lock_wait_seconds{mode}` (histogram), `document_lock_wait_max_seconds{mode}` | How long takers of the document's lock waited for it, since start, by mode. Readers (the API's reads, topology, stats, generating sync messages, serving changes and deltas, saves) wait only for a writer; writers (applying sync messages and fetched changes, writes, epoch switches) wait for everyone. Buckets from 10 µs to 10 s, the last unbounded; not cumulative in JSON. | + +**Interpretation.** `savedBytes` growing without bound is the reason to compact +([`remakes.md`](remakes.md)); `savedBytesTookMs` is a direct measure of what a switch, a rejoin and a +document file rewrite cost in CPU. + +The node has one document, and nearly all work on it takes its lock, so `lockWait` says +whether more cores would help: waits of a few microseconds mean the lock is free when +asked for; write waits in the milliseconds (`sumUs` / `count`), growing with the number +of neighbours, mean applying one peer's sync messages queues behind the others'. A +high `maxUs` alone is a single long hold (a large sync message, a remake), not +contention. + +## Catching up + +| JSON | Prometheus | Meaning | +|---|---|---| +| `catchUp.sources` | `catch_up_sources_total` | Catch-up sources chosen. 1 is normal; more means the first source was given up (quiet, not ready, refused, or stalled) and another took over. | +| `catchUp.doneAfterSecs` | `catch_up_duration_seconds` | From start until the initial sync was done; absent until then, and on a node that was ready at once (genesis, document file, recovery). | +| `catchUp.syncBytes` | `catch_up_sync_bytes` | Sync payload received until then, from every peer: what joining cost. | + +Compare `syncBytes` with `savedBytes`: much more received than the document is large +means the node got the document from several peers before the source rule took hold, +or caught up over many small messages. + +## Remakes + +| JSON | Prometheus | Meaning | +|---|---|---| +| `remakes.undecided` | `remakes_undecided` | Remakes of the local epoch not decided yet, expired ones included (`GET /admin/remakes`). | +| `remakes.committed` | `remakes_committed` | Committed remakes of the local epoch: 1 for the moment between a commit and the switch. More than one is a double commit, on which no node switches by itself: alert on it. | +| `remakes.rejoining` | `rejoining` | The node syncs a newer epoch from scratch ([`lifecycle.md`](lifecycle.md)). | +| `remakes.catchUpAttempts` | `catch_up_attempts` | Attempts at catching up with a newer epoch; 0 when not catching up. A growing count means the node can't get the epoch its peers advertise. | +| `remakes.quarantined`, `remakes.oldestQuarantineSecs` | `quarantined`, `oldest_quarantine_seconds` | Change sets set aside on a rejoin (`GET /admin/quarantine`). They live in memory only, so a restart loses them: alerting is how to learn of them. | +| `remakes.archivedEpochs` | `archived_epochs` | Old epochs' documents kept in memory (`archiveRetention`). | + +## Sync and relay + +| JSON | Prometheus | Meaning | +|---|---|---| +| `sync.lastFromNeighbourSecs` | `last_sync_from_neighbour_seconds` | Since a sync message from any neighbour was applied. Growing without bound while others write: the node is cut off from the swarm's changes. | +| `sync.divergedPeers` | `diverged_peers` | Peers whose heads differ from ours with no sync message in between. | +| `sync.largestReceivedBytes` | `largest_sync_received_bytes` | The largest sync message received since start. Larger transfers come as deltas (blobs of 16 chunks or more, [`peers.md`](peers.md)), so this stays below 16 × `--blob-chunk` from nodes of this version. | +| `relay.announceRounds` | `announce_rounds_total` | Announcement rounds: our new heads to the neighbours after applying their changes (partial mesh only). | +| `relay.pulls` | `pulls_total` | Pulls of changes we lacked from a neighbour that announced them. | +| `relay.pullsQueued` | `pulls_queued_total` | Announcements of heads we lacked that waited for the pull in progress. Many compared with `pulls`: changes arrive from several sides at once. | +| `relay.blobRequestsDropped` | `blob_requests_dropped_total` | Blob requests not answered because too many were being answered; the requester asks again or asks another peer. Also logged as a warning, at most once a minute with the count. | + +## Topology + +| JSON | Prometheus | Meaning | +|---|---|---| +| `topology.k` | `neighbours_target` | `--neighbours`. | +| `topology.view.members` | `members` | Registered members, this node included. | +| `topology.view.reachable` | `reachable_members` | Members other than this node not judged unreachable (three unanswered HELLOs, or dormant). | +| `topology.view.stable` | `stable_members` | Members whose registry entry says `stable`. | +| `topology.view.neighbours` | `neighbours` | Current neighbours: peers this node keeps alive and syncs with on its own. At least `k` (links chosen by either end count); well above `k` on a stable member that many others chose, bounded at about 8 × k. | +| `topology.view.stableNeighbours` | `stable_neighbours` | Of those, stable members. | +| `topology.view.fullMesh` | `full_mesh` | Every reachable member is a neighbour (at most `k + 1` reachable members). | + +`view` is absent until the node is ready and knows the member list; until then +everyone counts as a neighbour. + +**Interpretation.** `members − reachable − 1` is how many members are offline or +unreachable from here; if it is half or more of the electorate, no remake can commit. +`neighbours` far above `k` on one node means it is a hub (a stable member among few), +which is by design but costs it bandwidth. + +## Stability + +| JSON | Prometheus | Meaning | +|---|---|---| +| `stability.tier` | `stable` | What this node published about itself: `stable` once connected at least 95% of a full day, `transient` again below 80%. | +| `stability.connectedFraction` | `connected_ratio` | Fraction of the observed time (up to 24 h) with at least one live peer. | +| `stability.observedSecs` | `connectivity_observed_seconds` | How much time the fraction is based on; a day at most. | + +A node that restarted keeps a stable tier only if a peer says it was away less than an +hour; otherwise it starts over with no history, and `observedSecs` says so. + +## Propagation + +How long changes written elsewhere took to reach this node, from the writer's commit +stamp to the apply here, for every change in every applied sync message once caught up. +The apply is timed by the peers' clocks: this node's own offset from them +(`clock.offsetMs`, Clocks) is taken off. +A histogram with buckets at 100 ms, 250 ms, 500 ms, 1 s, 2.5 s, 5 s, 10 s, 30 s, 1 min, +5 min, 1 h, and the rest. + +| JSON | Prometheus | Meaning | +|---|---|---| +| `propagation.count`, `.sumMs` | `propagation_delay_seconds_count`, `_sum` | For the mean. | +| `propagation.buckets[]` | `propagation_delay_seconds_bucket{le}` | Per bucket (not cumulative in JSON; cumulative in Prometheus, as histograms are). | +| `propagation.ahead` | `propagation_clock_ahead_total` | Changes stamped later than their arrival (by the peers' clocks): the writer's clock is ahead. Counted in the first bucket. | + +**Interpretation.** The delay includes the writer's clock skew (this node's own is taken +off), so changes from a writer with a badly set clock show nonsense here; `ahead` +growing is the sign, and that writer's own `clock.off` ([`trust.md`](trust.md), "What depends on +clocks"). Writers aren't known by node (Automerge's actors are per run), so their skew +can't be taken off too. With sane clocks: in a full mesh most +changes land in the first buckets; in a partial mesh each hop adds about a round trip +plus the 200 ms announcement delay. Mass in the minutes buckets means changes arrive by +keepalive-triggered pulls rather than relay, i.e. announcements are being lost. + +## Clocks + +Each PONG says when its sender answered, by its clock, so every peer's clock is known +to within half a round trip (`clock` per peer, in Per peer). This node's own clock is +the odd one out when it differs from most of its active peers' by more than +`--clock-skew-warn` (2 s by default), for sure: beyond their bounds. + +| JSON | Prometheus | Meaning | +|---|---|---| +| `clock.peers` | `clock_peers_compared` | Active peers whose clocks were compared (a PONG from them lately). | +| `clock.offsetMs` | `clock_offset_seconds` | How far this node's clock is ahead of its peers' (behind if negative): the median over them. Absent with none compared. | +| `clock.behind`, `clock.ahead` | — | Peers whose clocks are behind ours, and ahead, by more than `--clock-skew-warn` for sure. | +| `clock.off` | `clock_off` | Whether this node's clock is off: more than half of them differ from it beyond the limit, the same way. | +| `clock.limitMs` | — | The limit (`--clock-skew-warn`). | + +**Interpretation.** A peer or two `ahead` or `behind` are their clocks, which they warn +about themselves; `off` is ours. While it is, the log warns every 10 minutes, and says +when it agrees again, and `/health` says so in a second line (still `200`). A clock set +anew (by NTP, a resumed machine, by hand) by more than the limit shows as a jump in the +log: this node's (a warning) when most peers' clocks moved alike, else the peers' whose +did. Two nodes alone can't tell whose clock is off: both warn. Clocks matter for alive +stamps, remake deadlines and the propagation figures ([`trust.md`](trust.md), "What depends on +clocks"); setting them is the operating system's job (NTP). + +## Traffic + +Messages and payload bytes by direction (`sent`, `received`, `refused`) and kind +(`hello`, `helloReply`, `sync`, `status`, `statusReply`, `blob`, `identity` for WHO and +ME, and `paths` for PING and PONG). The layers below the node count WHO and ME, and the +paths layer's own PINGs and PONGs, under the address they went to or came from; a peer +(`/peers`) has those of its addresses too, where it is known to be at them. Payload bytes are what a transport's message carries (the wire +encoding): not the transport's framing (for Mycelium, its topic and the base64 of its +HTTP API). `refused` is what the transports wouldn't accept to send (no path to the +peer), not counted as sent. + +| JSON | Prometheus | Scope | +|---|---|---| +| `traffic` in `/admin/stats` | `messages_total{direction,kind}`, `payload_bytes_total{direction,kind}` | All peers, including ones no longer listed. | +| `traffic` per entry in `/peers` | `peer_messages_total{peer,direction,kind}`, `peer_payload_bytes_total{peer,…}` | One peer. | + +**Interpretation.** Keepalives (`paths` for PINGs and PONGs, every third one `status` + `statusReply`) should be about two per neighbour per +keepalive interval; much more means announcements (relay) or heals. `sync` bytes +dominate while catching up and on big writes; a steady `sync` stream on an idle swarm +means two peers disagree and keep healing. `refused` growing for one peer means no +transport has a path to it while we still consider it a neighbour. + +## Transports + +How the transports find and reach peers, under `transports` in `/admin/stats`; each +part is absent (`null`) where it doesn't apply. + +The identity layer (`transport::identity`, [`peers.md`](peers.md)), under `transports.identity`: + +| JSON | Prometheus | Meaning | +|---|---|---| +| `proven` | `identity_proven_nodes{transport}` | Nodes proven at an address now, by transport; a node on several counts on each. | +| `relayed` | `identity_relayed_nodes{transport}` | Of those, the nodes the transport reaches only through a relay (libp2p), by transport; in JSON absent where none is. Many means hole punching doesn't get through (`libp2p.holePunches`). | +| `hinted` | `identity_hinted_nodes` | Nodes known only from the member list or HELLO_REPLYs, proven nowhere yet. | +| `proofsRejected` | `identity_proofs_rejected_total` | WHO and ME that didn't hold: a bad signature, or made for other addresses. Normally 0; anything else is someone replaying or forging proofs. | +| `addressChanges` | `identity_address_changes_total` | Nodes that proved themselves at another address on a transport than before: a restart with a new libp2p key or Mycelium key. Frequent ones mean a node doesn't keep its transport key (`--libp2p-key-file`, `--mycelium-key-file`). | +| `droppedUnproven` | `identity_dropped_unproven_total` | Messages dropped because no node proved itself at their address (the address is asked WHO). Some after a restart are normal; a steady stream means something keeps sending from an address that never proves itself. | +| `refusedUnproven` | `identity_refused_unproven_total` | Sends refused because the peer hadn't proven where it is yet; retried once it has. | +| `seedsUnanswered` | `identity_seeds_unanswered_total` | Seeds at which no node answered WHO in time. | + +The paths layer (`transport::paths`, [`peers.md`](peers.md)), under `transports.paths`: the +paths to peers, measured by PINGs. + +| JSON | Prometheus | Meaning | +|---|---|---| +| `paths[]` (`address`, `rttMs`, `down`, `connection`) | — | Every path measured: those of peers this node chose among or sent to in the last 5 minutes, and every path its keepalive PINGs went over (a peer's only path too). | +| `probesSent`, `probesAnswered`, `probesUnanswered` | `paths_probes_total{result}` | PINGs, the keepalive's and every 30 s (or every keepalive interval, if that is longer) per path in use that no keepalive PING went over in that time (a busy pair needs no keepalives, so its paths would go unmeasured), and their PONGs. Unanswered ones (no PONG within 15 s) on one path mean it doesn't deliver; three in a row make it down. | +| `sendFailures` | `paths_send_failures_total` | Sends that failed on a path; each is tried at once at the peer's next path. | +| `failovers` | `paths_failovers_total` | Sends that went over another path after one failed. | +| `switches` | `paths_switches_total` | Changes of the path in use for one faster by a quarter (and 5 ms). Frequent ones mean two paths about as fast, with changing delays. | + +Per peer, in `/peers` (`destination.paths`), and as `peer_path_rtt_seconds{peer,transport}` +and `peer_path_down{peer,transport}`: each path to it, best first. A path's `address` is +where it goes now: for libp2p, the remote address of its connection, ending in the peer ID +(just the peer ID while there is none); a Mycelium public key; an onion service. + +libp2p (`--libp2p-enable`), under `transports.libp2p`: + +| JSON | Prometheus | Meaning | +|---|---|---| +| `reachability`, `publicAddress` | `libp2p_reachability{status}` | What AutoNAT says: `unknown`, `private` (others can't dial this node: it needs a relay) or `public`, with the address it confirmed. | +| `connections.direct`, `.relayed` | `libp2p_connections{kind}` | Open connections now, direct or through a relay. | +| `relaySlots.held` | `libp2p_relay_slots` | Slots held on relays now, to be reachable through them. | +| `relaySlots.paused` | `libp2p_relays_paused` | Relays not picked again for a while: 10 minutes after refusing a slot, 30 seconds after not granting it otherwise. | +| `relaySlots.granted`, `.lost`, `.notGranted` | `libp2p_relay_slot_events_total{event}` | Slots granted (renewals not counted), lost after being granted (not when one moves: reserved again at once over the connection kept, as a duplicate connection to the relay closed), and asked for but never granted. `notGranted` growing: a relay is unreachable, or another peer answers at its address (a relay restarted without its libp2p key). | +| `holePunches.succeeded`, `.failed` | `libp2p_hole_punches_total{result}` | Direct connections tried over a relayed one (DCUtR). A failure leaves the relay carrying the traffic. | +| `dialFailures` | `libp2p_dial_failures_total{cause}` | Connections that couldn't be made: `unreachable` (refused, timed out, no route), `wrongPeerId` (another peer at the address: one restarted without its libp2p key, or a NATed peer's LAN address in a range ours uses too; that address isn't taken for the peer again), `noAddresses`, `other`. | + +A relay that is no node (`--relay-only`) has `/admin/stats` too: its `peerId`, its +`addresses` (as `libp2p:` addresses, ready for `_continuum/relays`), `uptimeSecs`, and +`transports.libp2p` as above (Prometheus: `relay_info{peer_id}`, `uptime_seconds` and +the `libp2p_…` series). + +**Interpretation.** A node that should be dialable but shows `private` has a port +closed or not forwarded. `relayed` connections that stay while `holePunches.failed` +grows means the relays carry the traffic for good: fine within their limits, but it +costs the relay. `wrongPeerId` and `addressChanges` together point at a node restarted +without its keys. + +## Incidents + +Counted per peer, summed in `/admin/stats` over the peers currently listed (a +forgotten peer's are gone). + +| JSON | Prometheus | Meaning | +|---|---|---| +| `incidents.replyTimeouts` | `peer_reply_timeouts_total` | Our sync messages the peer didn't answer in time (10 s, 30 s with Tor enabled, doubling to 30 times that); resent. Lost messages, or a slow peer. | +| `incidents.heals` | `peer_heals_total` | Re-handshakes because our heads and the peer's differed for `resync_after` with no sync between ([`peers.md`](peers.md), "Keepalive and self-healing"). Rare in a healthy swarm; frequent means sync state keeps getting out of step. | +| `incidents.dormancies` | `peer_dormancies_total` | Times the peer went dormant (nothing received for `dormant_after`). | +| `incidents.dormantMs` | `peer_dormant_seconds_total` | Time it spent dormant, up to its last return. | + +## Per peer + +In `/peers`, and as `peer_…` series with a `peer` label: + +| JSON | Prometheus | Meaning | +|---|---|---| +| `status`, `dormantForSecs` | `peer_unknown`, `peer_dormant` | Unknown (nothing heard from it since this node started: known from the member list or another peer), active, or dormant, and for how long. | +| `lastSeenSecsAgo` | `peer_last_seen_seconds` | Since any message from it. | +| `neighbour` | `peer_neighbour` | Whether we initiate towards it. | +| `destination` | `peer_destination_info{transport,proven,connection}` | Where messages to it go: the address (its transport), whether it proved itself there (else sends wait for its proof), and the connection there: libp2p's `direct` or `relayed`, Tor's `direct` (an onion circuit); none on Mycelium; while `relayed` after a failed hole punch, `punching`: punches failed in a row (`failed`) and when it's tried again (`nextInSecs`). | +| `destination.paths[].rttMs`, `.down`, `.connection` | `peer_path_rtt_seconds{transport}`, `peer_path_down{transport}` | Every path to it, best first: its round-trip time (smoothed, from PINGs), whether it is down, and the transport's connection there (`direct` or `relayed`, where it has connections). The first is the one in use: the peer's round-trip time. | +| `lastSyncSecsAgo` | `peer_last_sync_seconds` | Since a sync message from it was applied (or the epoch changed). | +| `divergedForSecs` | `peer_diverged_seconds` | How long its heads have differed from ours with no sync message from it in between; absent while they match. Long values precede a healing re-handshake ([`peers.md`](peers.md)). | +| `syncThroughput` | `peer_sync_throughput_bytes_per_second` | Sync bytes per second over exchanges of at least 32 KiB, round trip taken off, smoothed. Absent before the first large exchange; comes with its age (`secsAgo`) since large exchanges are rare. This is the rate the peer delivers at in practice (its apply time included), not the path's capacity. | +| `clock.offsetMs`, `.withinMs` | `peer_clock_offset_seconds`, `peer_clock_offset_within_seconds` | How far its clock is ahead of ours (behind if negative), give or take. Each PONG says when the peer answered, by its clock, which lies between our PING going and the PONG coming: within half the round trip of its middle, however unevenly the two ways took. Of its paths and the last 5 minutes, the narrowest bound is kept; a sample it contradicts (a clock set anew) replaces it at once. Absent before a PONG from it. | +| `epoch`, `sameEpoch`, `heads` | — | What it last advertised; `sameEpoch: false` means we don't sync with it. | +| `handshakeComplete`, `hasPendingDelivery` | — | Session state, for debugging. | + +## Disk + +Per kind of the node's own files (`state`, `document`, `backup`, `remake`), since +start: `reads`, `readBytes`, `writes` (whole files), `writtenBytes`, `appends`, +`appendedBytes`, `errors`. Prometheus: `disk_…_total{file}`. The document file is the +only one written steadily: `appendedBytes` over time is the write rate it costs the +device; a rising `writes` count there means rewrites because the appended part keeps +outgrowing the full save (busy swarm, or a `--document-rewrite` set low). + +## The dashboard + +In builds with the `dashboard` cargo feature, `GET /admin/dashboard` is a page showing +`/admin/stats`, `/peers` and a map of the swarm, live: [`dashboard.md`](dashboard.md), and how it works +underneath (events, LINKS) at its end. Its LINKS are the one thing a node sends only to +measure: the links every member reports, while someone looks. + +## What is not measured + +* Nothing about other nodes but what they send us: no probes, no remote figures (but + the dashboard's LINKS). +* Clock skew between nodes that aren't each other's peers: only peers' clocks are + compared (`clock`, per peer). +* Remake durations and outcomes: the remake reports (`GET /admin/remakes`) hold them. +* Mycelium's own routing, retransmissions and queueing: the daemon's API has them. diff --git a/docs/openapi.yaml b/docs/openapi.yaml index acd2ce0..78912fa 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -3,8 +3,8 @@ info: title: continuum version: 0.1.0 description: | - HTTP API of a continuum node: a replicated Automerge key-value store that syncs - over Mycelium. + HTTP API of a continuum node: a replicated Automerge document, read and written + as JSON by path, that syncs over Mycelium, libp2p or Tor. Every endpoint requires HTTP basic authentication (users from `--htpasswd`, Argon2 hashes), except `/health`, `/ready`, the Swagger UI (`/swagger-ui`) and @@ -54,11 +54,17 @@ paths: parameters: - $ref: '#/components/parameters/conflicts' - $ref: '#/components/parameters/heads' + - $ref: '#/components/parameters/shallow' + - $ref: '#/components/parameters/at' responses: '200': $ref: '#/components/responses/Value' + '400': + description: '`at` doesn''t name heads.' '401': $ref: '#/components/responses/Unauthorized' + '404': + description: This node lacks the heads `at` names. '500': description: Reading or serializing failed. '503': @@ -71,12 +77,20 @@ paths: tags: [Store] summary: Read a value description: | - The value at the path, as JSON, or with `Accept: text/plain` as text (strings - unquoted, anything else as JSON). Concurrent writes resolve as follows: - concurrent maps merge, anything else keeps the newest write. + The value at the path, as JSON, or as `Accept` asks (`api.md`, Types): its + type's own media type for a scalar (`application/octet-stream` for bytes, + `application/vnd.continuum.<type>`), typed JSON + (`application/vnd.continuum.typed+json`, which tags what plain JSON would show + as another type), or `text/plain` (strings unquoted, anything else as JSON). `Accept` is taken in its order; `406` if it names + nothing the value can be shown as. `application/json; timestamp-format=…` shows + timestamps as `rfc3339` (the default), `unix`, `unix-ms` or `http-date`. + Concurrent writes resolve as follows: concurrent maps merge, anything else + keeps the newest write. parameters: - $ref: '#/components/parameters/conflicts' - $ref: '#/components/parameters/heads' + - $ref: '#/components/parameters/shallow' + - $ref: '#/components/parameters/at' responses: '200': $ref: '#/components/responses/Value' @@ -85,7 +99,9 @@ paths: '401': $ref: '#/components/responses/Unauthorized' '404': - description: No value at this path. + description: No value at this path (with `at`, then), or this node lacks the heads `at` names. + '406': + description: '`Accept` names nothing the value can be shown as; the message says what it can.' '500': description: Reading or serializing failed. '503': @@ -94,12 +110,37 @@ paths: tags: [Store] summary: Write a value description: | - Writes the value, creating intermediate maps. One request is one Automerge - change. Answers with the value written (JSON, or text with - `Accept: text/plain`). + Writes the value, creating intermediate maps (not lists): the path then reads as + the value, what was there changed only where it differs (`api.md`, Concurrent + writes). In a list, an index overwrites the item there, the list's length + appends, and past it is `404`. One request is one Automerge change. Answers + with the value written, shown as `Accept` asks (as a read). JSON maps and arrays become Automerge maps and lists; numbers become integers when they fit, floats otherwise; strings are plain strings (not text objects). + Types are sticky: plain JSON or text written over a value of a type JSON + doesn't have is stored as that type where it is its JSON form (a timestamp's + string, bytes' base64, a number over a float, uint or counter, a string over a + text; `api.md`, Types). A change of type is listed in `x-continuum-retyped`; + with `retype=false` it is `409` instead. Answers with what is there after the + write, read back. + + Typed JSON (`application/vnd.continuum.typed+json`) writes values of every + type, tagged (`{"$timestamp": …}`, `{"$bytes": …}`, `{"$uint": …}`, + `{"$counter": …}`, `{"$text": …}`, `{"$map": …}`); a float is a number, and + a number without a fraction written over a float stays one. + A scalar of another type is also written as its media type (`api.md`, Types): + `application/octet-stream` (bytes), `application/vnd.continuum.int`, `.uint`, + `.float`, `.boolean`, `.counter` (decimal or `true`/`false`), or + `application/vnd.continuum.timestamp; format=rfc3339|unix|unix-ms|http-date`; + a body that isn't one is `400`, saying why. + parameters: + - name: retype + in: query + description: Whether the write may change the type of what it writes over (default true); if not, a write that would is `409`. + schema: + type: boolean + default: true requestBody: required: true content: @@ -110,6 +151,14 @@ paths: schema: type: string example: dark + application/octet-stream: + schema: + type: string + format: binary + application/vnd.continuum.timestamp: + schema: + type: string + example: '2026-11-01T21:50:20.345Z' responses: '200': $ref: '#/components/responses/Value' @@ -120,26 +169,121 @@ paths: '403': description: Protocol data (`_continuum`) and the user isn't in `protocol-writers`, or epoch metadata or the swarm ID (`_continuum/epochs`, `_continuum/swarm`, never writable). '409': - description: The path crosses an existing non-map value. + description: The path crosses an existing non-map value, or with `retype=false` the write would change a type (the message says where). '415': - description: Content-Type is neither `application/json` nor `text/plain` (absent means JSON). + description: Content-Type is none the API takes (`application/json`, the default; `text/plain`; a type's own, `api.md`, Types). '500': description: Writing failed. '503': - description: Node is not ready. + description: 'Node is not ready (and isn''t rejoining: a rejoining node takes writes into its document before the rejoin, carried over when it ends).' + post: + tags: [Store] + summary: Insert into a list, increment a counter, or cast a value + description: | + Inserts the value (as `PUT` takes it) into a list: the path ends in the index + it goes to (`queue/2`), the items from there on moving up one; the list's + length, or `-1` (a `307` to it), appends. One request is one Automerge change. + To add a key to a map, `PUT` it; to overwrite an item, `PUT` its index. + + With `increment`, adds it to the counter at the path instead (no body): one + is made where there is nothing (in a map; a list's items are only + incremented, `-1` naming the last). Increments of a counter on several nodes + add up. Answers `200` with the counter's value then. Reads show a counter as + a number; a `PUT` there makes it a plain number again. + + With `cast`, casts what is at the path to that type instead (no body; `-1` + names the last item): numbers (integers, floats, counters, timestamps as + milliseconds since 1970) to numbers where they are exactly that, any scalar + but bytes to a string or a text as its media type shows it (timestamps in + `format`) and back, bytes to a string or a text and back as UTF-8. Answers + `200` with what is there then, as `PUT` does (`x-continuum-retyped` saying + the change). + parameters: + - name: increment + in: query + description: How much to add (negative to subtract). + schema: + type: integer + format: int64 + - name: cast + in: query + description: The type to cast to. + schema: + type: string + enum: [text, string, bytes, int, uint, float, boolean, counter, timestamp] + - name: format + in: query + description: With `cast`, how timestamps are read from and written as strings. + schema: + type: string + enum: [rfc3339, unix, unix-ms, http-date] + default: rfc3339 + requestBody: + required: false + content: + application/json: + schema: {} + text/plain: + schema: + type: string + responses: + '200': + description: With `increment`, the counter's value after it; with `cast`, what is there after it (as `GET` shows it, with `x-continuum-type` and `x-continuum-retyped`). + headers: + x-continuum-heads: + $ref: '#/components/headers/WrittenHeads' + content: + application/json: + schema: + type: integer + '201': + description: Inserted, and where. + headers: + x-continuum-heads: + $ref: '#/components/headers/WrittenHeads' + content: + application/json: + schema: + type: object + required: [index] + properties: + index: + type: integer + '400': + description: Invalid path, index, malformed JSON or invalid UTF-8; an unknown `cast` type or `format`, or both `increment` and `cast`. + '401': + $ref: '#/components/responses/Unauthorized' + '403': + description: As for `PUT`. + '404': + description: Nothing at this path. + '307': + description: A negative index, made positive (`Location`). + '409': + description: Not a list there, or the index is past its end (the message says which, and the length); with `increment`, not a counter there; with `cast`, no cast possible (the message says why). + '415': + description: Content-Type is none the API takes (`application/json`, the default; `text/plain`; a type's own, `api.md`, Types). + '503': + description: 'Node is not ready (and isn''t rejoining: a rejoining node takes writes into its document before the rejoin, carried over when it ends).' delete: tags: [Store] summary: Delete a value + description: | + Deletes the value; in a list, the item at the index (the items after it move + down one). parameters: - name: prune in: query - description: Also remove ancestor maps left empty. + description: Also remove ancestor maps left empty (not through a list). schema: type: boolean default: false responses: '204': description: Deleted. + headers: + x-continuum-heads: + $ref: '#/components/headers/WrittenHeads' '400': description: Invalid path. '401': @@ -151,7 +295,7 @@ paths: '500': description: Deleting failed. '503': - description: Node is not ready. + description: 'Node is not ready (and isn''t rejoining: a rejoining node takes writes into its document before the rejoin, carried over when it ends).' /stream: get: @@ -299,7 +443,10 @@ paths: get: tags: [Admin] summary: Liveness - description: The HTTP API is up. Unauthenticated. + description: | + The HTTP API is up. Unauthenticated. A second line while this node's clock is + off from most of its peers' (`measurements.md`, Clocks), still `200`: + `clock off: 40s behind most of its peers'`. security: [] responses: '200': @@ -342,13 +489,18 @@ paths: `text/plain` (as Prometheus does), the Prometheus text format (0.0.4) instead, with per-peer series labelled `peer` (the node ID). Aliases are only in `continuum_info` and `continuum_peer_info`, for joining. + + A node started with `--relay-only` answers `RelayOnlyStats` instead (and has + only `/health`, `/admin/stats` and `/admin/log`). responses: '200': description: Statistics. content: application/json: schema: - $ref: '#/components/schemas/Stats' + oneOf: + - $ref: '#/components/schemas/Stats' + - $ref: '#/components/schemas/RelayOnlyStats' text/plain; version=0.0.4: schema: type: string @@ -400,6 +552,114 @@ paths: '503': description: Node is not ready. + /admin/changes: + get: + tags: [Admin] + summary: Who wrote what + description: | + The document's changes, newest first: each with when (its writer's clock, Unix + ms; 0 for a remake's snapshot), the node that committed it (`node`, its node + ID) and, for a write through the API, the user at that node who wrote it + (`user`, from the message's `User:` trailer), and the store + paths it touched (a list's items by index). Every node holds every change, so + any node answers for the whole swarm, as far as it has synced; history before + the last remake is gone with it (`remakes.md`). Changes made before nodes + recorded their writers have neither `node` nor `user`. + parameters: + - name: limit + in: query + description: How many at most (default 100, at most 1000). + schema: + type: integer + minimum: 1 + responses: + '200': + description: The changes. + content: + application/json: + schema: + type: object + required: [changes, more] + properties: + changes: + type: array + items: + type: object + required: [hash, at, paths] + properties: + hash: + type: string + at: + type: integer + node: + type: string + user: + type: string + paths: + type: array + items: + type: string + more: + type: boolean + description: Older changes weren't looked at. + '401': + $ref: '#/components/responses/Unauthorized' + '503': + description: Node is not ready. + + /admin/changes/{key}: + get: + tags: [Admin] + summary: Who wrote what, at a path + description: | + As `/admin/changes`, but only the changes that touched what is at the path (as + `/store/{key}` names it), inside it or above it. At most 10000 changes are + looked at for it. + parameters: + - $ref: '#/components/parameters/key' + - name: limit + in: query + description: How many at most (default 100, at most 1000). + schema: + type: integer + minimum: 1 + responses: + '200': + description: The changes. + content: + application/json: + schema: + type: object + required: [changes, more] + properties: + changes: + type: array + items: + type: object + required: [hash, at, paths] + properties: + hash: + type: string + at: + type: integer + node: + type: string + user: + type: string + paths: + type: array + items: + type: string + more: + type: boolean + description: Older changes weren't looked at. + '400': + description: A path with an empty segment. + '401': + $ref: '#/components/responses/Unauthorized' + '503': + description: Node is not ready. + /admin/config: get: tags: [Admin] @@ -465,6 +725,110 @@ paths: '401': $ref: '#/components/responses/Unauthorized' + /admin/dashboard: + get: + tags: [Admin] + summary: Dashboard page + description: | + A page showing this node's figures (`/admin/stats`), its peers (`/peers`) and a + map of the swarm, live through `/admin/dashboard/events`. Only in builds with + the `dashboard` cargo feature (else `404`). + responses: + '200': + description: The page. + content: + text/html: + schema: + type: string + /admin/dashboard/state: + get: + tags: [Admin] + summary: Dashboard state + description: | + The dashboard's latest snapshot: `node` (`id`, `alias`), `page` (the page's + version, a hash: a page showing another reloads), `stats` (as + `/admin/stats`), `peers` (as `/peers`) and `graph`: `members` (by node ID: + `alias`, `self`), `intended` (by node ID: the neighbours the member list gives + it, as this node computes them), `measured` (by node ID: `secsAgo` and + `links`, each with `peer`, `transport`, `relayed`, `down`, `rttMs`, `dormant` and + `others` (the peer's other transports: `transport`, `rttMs`, `relayed`), as the member + answered a LINKS; ours always, the others' while fresh) and `live` (whether the + members' links are subscribed to, `/admin/dashboard/links`). One producer + per node makes it every 2 s while anyone looks (and for a minute after this + request), subscribed to the members' links (each tells of a change as it + happens; renewed every 30 s); every client gets the same. Only with the `dashboard` cargo feature. + responses: + '200': + description: The snapshot. + content: + application/json: + schema: + type: object + required: [node, stats, peers, graph] + additionalProperties: true + /admin/dashboard/refresh: + post: + tags: [Admin] + summary: Dashboard refresh + description: | + The dashboard's producer builds the state now rather than at its next tick, and + asks the members for their links again; what changed comes as events. Only with + the `dashboard` cargo feature. + responses: + '204': + description: Done. + /admin/dashboard/links: + post: + tags: [Admin] + summary: Dashboard links, live or asked + description: | + Whether the dashboard's producer subscribes to the members' links (`live=true`, + the default: each member tells of a change as it happens) or only asks for them + every 30 s (`live=false`: fewer messages in the swarm, a map up to 30 s + behind). For every client of this node, until set again or the node restarts; + the state's `graph.live` says which. Only with the `dashboard` cargo feature. + parameters: + - name: live + in: query + required: true + schema: + type: boolean + responses: + '204': + description: Done. + /admin/dashboard/events: + get: + tags: [Admin] + summary: Dashboard events + description: | + Server-Sent Events: a `snapshot` event (the state as + `/admin/dashboard/state`) first, then a `patch` event (a JSON merge patch, + RFC 7386, onto the state before) whenever something but ages changed, at most + every 2 s. Each event's ID is its number. A client that falls behind gets a + `snapshot` again. A `document` event says where the document changed since + the last (at most every 2 s): `{"paths": [[segment, ...], ...]}`, or + `{"all": true}` when too many changed or the node lost count. A `messages` + event lists the messages the node exchanged since the last (at most every + 2 s, 500 at most): `seq`, `at` (Unix ms), `way` (`sent`, `received`, + `refused`), `peer` (its node ID; the address for the paths layer's own PINGs + and PONGs to an address no known node is at), `address` (for those to an + address a known node is at: which), `type` (`SYNC`, …), `counted` (its traffic kind), `bytes`, + `text` (the message written out) and, where it belongs to an exchange, + `related` (the same for every message of it). When the node + stops, an `offline` event ends the stream (a client reconnects once the node + is back). Nothing changed, nothing is sent but keep-alive comments. + Ages (`…SecsAgo`, `secsAgo`, `uptimeSecs`, `dormantForSecs`, + `divergedForSecs`, `observedSecs`, `lastFromNeighbourSecs`, + `oldestQuarantineSecs`) are as of the event: they alone changing sends + nothing, the client counts them on. Only with the + `dashboard` cargo feature. + responses: + '200': + description: The event stream. + content: + text/event-stream: + schema: + type: string /admin/remakes: get: tags: [Remakes] @@ -584,7 +948,8 @@ paths: each with `id`, `coordinator`, `coordinatorActive`, `expiredAt`, `abortableAt` and `tally`; `force` names the override, if any); or this node can't vote for a new one: it voted for another remake of this epoch, - or may have before it restarted. + or may have before it restarted; or this node's clock is off from most of + its peers' (`force` is `clock`). content: application/json: schema: @@ -669,7 +1034,7 @@ paths: - $ref: '#/components/parameters/wait' - name: force in: query - description: '`true`, or a comma-separated list of `coordinatorActive` (abort an expired remake whose coordinator this node sees as an active peer) and `coordinatorInactive` (request the abort of an undecided remake whose coordinator this node doesn''t reach).' + description: '`true`, or a comma-separated list of `coordinatorActive` (abort an expired remake whose coordinator this node sees as an active peer), `coordinatorInactive` (request the abort of an undecided remake whose coordinator this node doesn''t reach) and `clock` (abort an expired remake although this node''s clock is off from most of its peers''; it judges whether the remake expired).' schema: type: string responses: @@ -915,7 +1280,16 @@ components: name: key in: path required: true - description: Slash-separated store path, e.g. `settings/theme`. + description: | + Slash-separated store path, e.g. `settings/theme`. Within a list a segment is + an index (`servers/0/name`): decimal, from 0, without a sign or leading zeros; + anything else names nothing there (`404`). A negative index counts from the end + (`-1` the last item; for `POST`, the end): answered with a `307` to the path + with it made positive. In a map every segment is a key, `0` and `-1` too. A key + with `/` in it, or an + empty key, can't be named here (neither `%2F` nor `//` helps: the path is + split after decoding, and empty segments are `400`); such keys exist only + inside a value written at a parent path, and are read through that parent. schema: type: string conflicts: @@ -934,6 +1308,30 @@ components: schema: type: boolean default: false + at: + name: at + in: query + description: | + Read as the document was at these heads (comma-separated hex, as + `x-continuum-heads` gives them), not as it is now; a negative index counts from + the list's end then. `404` if this node lacks one of them (not synced yet, or + of another epoch). Combines with `conflicts` and `shallow`. + schema: + type: string + shallow: + name: shallow + in: query + description: | + Instead of the value, what is there one level deep, as JSON, to browse a large + store a level at a time: `type` (`map`, `list`, `text`, `string`, `bytes`, + `int`, `uint`, `float`, `boolean`, `null`, `counter`, `timestamp`, `unknown`), `size` (a map's keys, a list's items, + a text's characters, a string's or bytes' bytes), `value` for a scalar up to + 256 bytes, and for a map `entries`: each key's summary (without entries of its + own). Concurrent values resolved as a read resolves them. Not with `conflicts` + (`400`). + schema: + type: boolean + default: false wait: name: wait in: query @@ -948,8 +1346,9 @@ components: Checks to override: `true` for all, or a comma-separated list of `switchDelay` (accept one below a keepalive interval; tests only) and `supersede` (abort expired remakes of this epoch that still hold votes, and - propose in their place) and `coordinatorActive` (supersede even though this - node sees their coordinator as an active peer). + propose in their place), `coordinatorActive` (supersede even though this + node sees their coordinator as an active peer) and `clock` (propose although + this node's clock is off from most of its peers'). schema: type: string examples: @@ -972,6 +1371,14 @@ components: type: string format: uuid + headers: + WrittenHeads: + description: >- + The document's heads after the write (sorted hex, comma-separated): a read with + them (`at`) on any node that has them includes the write. + schema: + type: string + responses: Unauthorized: description: Missing or wrong credentials. @@ -987,7 +1394,7 @@ components: description: The value. headers: x-continuum-heads: - description: With `heads=true`, the heads at the time of the read (sorted hex, comma-separated). + description: With `heads=true`, the heads at the time of the read (sorted hex, comma-separated); on a write, those after it (`WrittenHeads`). schema: type: string x-continuum-stale: @@ -998,6 +1405,14 @@ components: description: SHA-256 of the body. schema: type: string + x-continuum-retyped: + description: 'On a write that changed the type of what it wrote over: where, and from what to what (`<path>: <from> -> <to>`, comma-separated, 16 at most, then `…`).' + schema: + type: string + x-continuum-type: + description: What the value is stored as (`map`, `list`, `text`, `string`, `bytes`, `int`, `uint`, `float`, `boolean`, `null`, `counter`, `timestamp`). + schema: + type: string content: application/json: schema: @@ -1007,6 +1422,13 @@ components: text/plain: schema: type: string + application/octet-stream: + schema: + type: string + format: binary + application/vnd.continuum.timestamp: + schema: + type: string Error: description: Error. content: @@ -1143,7 +1565,16 @@ components: connection: type: [string, 'null'] enum: [direct, relayed, null] - description: The libp2p connection to it; null if none, or on a transport without connections (Mycelium). + description: 'The connection to it: libp2p''s direct or through a relay, Tor''s always direct (an onion circuit, no relay of ours); null if none, or on a transport without connections (Mycelium).' + punching: + type: object + description: 'While it is connected only through a relay and a hole punch failed: how often in a row, and when the next try is (after 1, 5, then every 30 minutes). Absent otherwise.' + required: [failed, nextInSecs] + properties: + failed: + type: integer + nextInSecs: + type: integer paths: type: array description: Every address it proved itself at, best first, as the paths layer measures them. @@ -1160,6 +1591,10 @@ components: down: type: boolean description: Its PINGs went unanswered three times in a row, or a send there failed within a minute. + connection: + type: string + enum: [direct, relayed] + description: The transport's connection there, if it has connections (libp2p, Tor); absent otherwise. validatedSwarm: type: [string, 'null'] status: @@ -1171,6 +1606,12 @@ components: type: [integer, 'null'] dormantForSecs: type: [integer, 'null'] + lastSyncSecsAgo: + type: [integer, 'null'] + description: Since a sync message from it was applied (or the epoch changed). + divergedForSecs: + type: [integer, 'null'] + description: How long its heads have differed from ours with no sync message from it in between; null while they match. hasPendingDelivery: type: boolean epoch: @@ -1193,6 +1634,10 @@ components: oneOf: - $ref: '#/components/schemas/Throughput' - type: 'null' + clock: + oneOf: + - $ref: '#/components/schemas/ClockOffset' + - type: 'null' Counter: type: object @@ -1202,11 +1647,11 @@ components: type: integer bytes: type: integer - description: Payload bytes (without Mycelium framing). + description: Payload bytes (without the transport's framing). ByKind: type: object - required: [hello, helloReply, sync, status, statusReply, blob, identity, paths] + required: [hello, helloReply, sync, status, statusReply, blob, identity, paths, links] properties: hello: $ref: '#/components/schemas/Counter' @@ -1227,6 +1672,9 @@ components: paths: $ref: '#/components/schemas/Counter' description: PING and PONG; the paths layer's own PINGs counted under the address (so only in the totals). + links: + $ref: '#/components/schemas/Counter' + description: LINKS and LINKS_REPLY, asked for and answered for a map of the swarm (the dashboard). Traffic: type: object @@ -1239,7 +1687,7 @@ components: $ref: '#/components/schemas/ByKind' refused: $ref: '#/components/schemas/ByKind' - description: Sends Mycelium refused (not counted as sent). + description: Sends the transports refused (no path; not counted as sent). Incidents: type: object @@ -1257,6 +1705,18 @@ components: type: integer description: Time spent dormant up to the last return (the current dormancy isn't included). + ClockOffset: + type: object + description: How far a peer's clock is ahead of ours, from its PONGs (`measurements.md`, Per peer). + required: [offsetMs, withinMs] + properties: + offsetMs: + type: integer + description: Ahead of ours; behind if negative. + withinMs: + type: integer + description: Give or take this much (half the round trip). + Throughput: type: object required: [bytesPerSec, samples, secsAgo] @@ -1270,6 +1730,150 @@ components: type: integer description: Age of the last sample. + TransportStats: + type: object + description: How the transports find and reach peers (`measurements.md`, Transports). + properties: + paths: + description: The paths layer's; null without one. + oneOf: + - type: 'null' + - type: object + required: [paths, probesSent, probesAnswered, probesUnanswered, sendFailures, failovers, switches] + properties: + paths: + type: array + description: Every path measured (of peers with several addresses used lately, and every path keepalives went over). + items: + type: object + required: [address, down] + properties: + address: + type: string + rttMs: + type: [number, 'null'] + down: + type: boolean + connection: + type: string + enum: [direct, relayed] + probesSent: + type: integer + probesAnswered: + type: integer + probesUnanswered: + type: integer + sendFailures: + type: integer + failovers: + type: integer + switches: + type: integer + identity: + description: The identity layer's; null without one. + oneOf: + - type: 'null' + - type: object + required: [proven, relayed, hinted, proofsRejected, addressChanges, droppedUnproven, refusedUnproven, seedsUnanswered] + properties: + proven: + type: object + description: Nodes proven at an address, by transport (scheme). + additionalProperties: + type: integer + relayed: + type: object + description: Of those, the nodes the transport reaches only through a relay, by transport; absent where none is. + additionalProperties: + type: integer + hinted: + type: integer + description: Nodes known only from hints, proven nowhere. + proofsRejected: + type: integer + addressChanges: + type: integer + droppedUnproven: + type: integer + refusedUnproven: + type: integer + seedsUnanswered: + type: integer + libp2p: + description: The libp2p transport's; null if it isn't one of this node's. + oneOf: + - type: 'null' + - type: object + required: [reachability, connections, relaySlots, holePunches, dialFailures] + properties: + reachability: + type: string + enum: [unknown, private, public] + description: What AutoNAT says. + publicAddress: + type: [string, 'null'] + description: The address AutoNAT confirmed, while public. + connections: + type: object + required: [direct, relayed] + properties: + direct: + type: integer + relayed: + type: integer + relaySlots: + type: object + required: [held, paused, granted, lost, notGranted] + properties: + held: + type: integer + paused: + type: integer + description: Relays not picked again for a while, after refusing a slot or not granting it. + granted: + type: integer + lost: + type: integer + notGranted: + type: integer + holePunches: + type: object + required: [succeeded, failed] + properties: + succeeded: + type: integer + failed: + type: integer + dialFailures: + type: object + required: [unreachable, wrongPeerId, noAddresses, other] + properties: + unreachable: + type: integer + wrongPeerId: + type: integer + noAddresses: + type: integer + other: + type: integer + RelayOnlyStats: + type: object + description: '`/admin/stats` of a node started with `--relay-only`.' + required: [peerId, addresses, uptimeSecs, transports] + properties: + peerId: + type: string + description: The relay's libp2p peer ID. + addresses: + type: array + items: + type: string + description: Where nodes reach it, as `libp2p:` addresses to list in `_continuum/relays`. + uptimeSecs: + type: integer + transports: + $ref: '#/components/schemas/TransportStats' + Stats: type: object required: [nodeId, addresses, uptimeSecs, ready, epoch, document, catchUp, remakes, sync, relay, topology, stability, staleHelloReplies, traffic, incidents, propagation, disk, transports] @@ -1439,129 +2043,13 @@ components: $ref: '#/components/schemas/Traffic' description: All peers, including ones no longer listed. transports: - type: object - description: How the transports find and reach peers (`measurements.md`, Transports). - properties: - paths: - description: The paths layer's; null without one. - oneOf: - - type: 'null' - - type: object - required: [paths, probesSent, probesAnswered, probesUnanswered, sendFailures, failovers, switches] - properties: - paths: - type: array - description: Every path measured (of peers with several addresses used lately, and every path keepalives went over). - items: - type: object - required: [address, down] - properties: - address: - type: string - rttMs: - type: [number, 'null'] - down: - type: boolean - probesSent: - type: integer - probesAnswered: - type: integer - probesUnanswered: - type: integer - sendFailures: - type: integer - failovers: - type: integer - switches: - type: integer - identity: - description: The identity layer's; null without one. - oneOf: - - type: 'null' - - type: object - required: [proven, hinted, proofsRejected, addressChanges, droppedUnproven, refusedUnproven, seedsUnanswered] - properties: - proven: - type: object - description: Nodes proven at an address, by transport (scheme). - additionalProperties: - type: integer - hinted: - type: integer - description: Nodes known only from hints, proven nowhere. - proofsRejected: - type: integer - addressChanges: - type: integer - droppedUnproven: - type: integer - refusedUnproven: - type: integer - seedsUnanswered: - type: integer - libp2p: - description: The libp2p transport's; null if it isn't one of this node's. - oneOf: - - type: 'null' - - type: object - required: [reachability, connections, relaySlots, holePunches, dialFailures] - properties: - reachability: - type: string - enum: [unknown, private, public] - description: What AutoNAT says. - publicAddress: - type: [string, 'null'] - description: The address AutoNAT confirmed, while public. - connections: - type: object - required: [direct, relayed] - properties: - direct: - type: integer - relayed: - type: integer - relaySlots: - type: object - required: [held, paused, granted, lost, notGranted] - properties: - held: - type: integer - paused: - type: integer - description: Relays picked that never granted a slot, not picked again for a while. - granted: - type: integer - lost: - type: integer - notGranted: - type: integer - holePunches: - type: object - required: [succeeded, failed] - properties: - succeeded: - type: integer - failed: - type: integer - dialFailures: - type: object - required: [unreachable, wrongPeerId, noAddresses, other] - properties: - unreachable: - type: integer - wrongPeerId: - type: integer - noAddresses: - type: integer - other: - type: integer + $ref: '#/components/schemas/TransportStats' incidents: $ref: '#/components/schemas/Incidents' description: The peers currently listed. propagation: type: object - description: Time from a change's commit elsewhere until applied here, once caught up (includes clock skew). + description: Time from a change's commit elsewhere until applied here, once caught up, by the peers' clocks (includes the writer's clock skew). required: [count, sumMs, ahead, buckets] properties: count: @@ -1570,7 +2058,7 @@ components: type: integer ahead: type: integer - description: Changes stamped later than our clock on arrival. + description: Changes stamped later than their arrival (writer's clock ahead). buckets: type: array description: Not cumulative; the last bucket has no bound. @@ -1582,6 +2070,29 @@ components: type: [integer, 'null'] count: type: integer + clock: + type: object + description: This node's clock against its active peers' (`measurements.md`, Clocks). + required: [peers, offsetMs, behind, ahead, off, limitMs] + properties: + peers: + type: integer + description: Active peers whose clocks were compared. + offsetMs: + type: [integer, 'null'] + description: How far ours is ahead of theirs (behind if negative), the median. + behind: + type: integer + description: Peers behind ours by more than --clock-skew-warn, for sure. + ahead: + type: integer + description: Peers ahead of ours by more than --clock-skew-warn, for sure. + off: + type: boolean + description: Ours differs from more than half of theirs that way. + limitMs: + type: integer + description: --clock-skew-warn. LockWaits: type: object @@ -1733,6 +2244,9 @@ components: type: integer switchDelayMs: type: integer + graceMs: + type: integer + description: The coordinator's grace period (four of its keepalives), by which every node judges when the remake expired and when others may abort it. Absent in records from before it was written down (each node's own then). electorate: type: array description: Members whose votes count, fixed at proposal time. diff --git a/docs/operations.md b/docs/operations.md new file mode 100644 index 0000000..fe09537 --- /dev/null +++ b/docs/operations.md @@ -0,0 +1,149 @@ +# Operations + +Running a swarm from day to day: adding and removing nodes, keys, backups, upgrades, +what to alert on. How each part works is in the pages linked; this one says what to do. + +## Before the first node + +Settle on these, as they are hard to change later: + +* **A swarm key** (`--swarm-key-file`, made with `--swarm-key-create` on the first node, + copied to the others). Without one, any node that can reach a member may join + ([`trust.md`](trust.md)). +* **A node key per node** (`--node-key-file`, `--node-key-create` on its first start). + The node key is the node: a new one is a new member. +* **A state file and a document file per node** (`--state-file`, `--document-file`). + With them a node restarts on its swarm, with its data, ready at once; without them it + starts empty and has to join again ([`lifecycle.md`](lifecycle.md), What the files are for). +* **The transports' keys** (`--mycelium-key-file`, `--libp2p-key-file`, + `--tor-state-dir`), so a node's addresses stay the same across restarts + ([`transports.md`](transports.md), Keys). +* **A supervisor** that restarts the node: it stops when a transport stops (`A + transport stopped`). + +## Adding a node + +Give it the swarm key, a node key to create, its files, and a seed: any member's +address (`addresses` in `GET /admin/stats` there), or nothing on a LAN with libp2p's +`--libp2p-mdns`. It joins: `503` from the store until it is ready (`/ready` says +`200` then), with `Still joining the swarm; trying on` in the log every five minutes +while it isn't, saying why. Once ready it registers itself in the member list, and the +others make it a neighbour where the topology says so ([`peers.md`](peers.md)). + +## Removing a node + +Stop it, then forget it from any node: `DELETE /peers/{node ID}`. That writes a +swarm-wide forget record and removes its registry entry; every node drops it. A node +that is forgotten but still alive cancels the record with its next HELLO and registers +again, so stop it first. + +A node stopped but not forgotten stays a member: it is probed now and then, and it +counts in the electorate of remakes until its alive stamp is older than +`--electorate-window` (7 days by default; [`remakes.md`](remakes.md), Electorate and majority). +Several of them can keep a remake from reaching a majority: forget those that won't +come back. + +## Keys + +**The swarm key** can't be changed while the swarm runs: a node with another key is a +stranger to the others (`Peer no longer proves the swarm key; dropping its +membership`). Stop every node, give each the new key, and start them; with document +files they resume with their data. Nodes started one by one with the new key form a +second group that the old ones don't talk to, until all have it; they merge then, as +both sides kept writing. + +**A node key** is the node's identity: a new one makes a new member. Start the node +with the new key (its document file can stay: its data is the swarm's), and forget the +old node ID. + +**A transport key** (Mycelium's, libp2p's, Tor's) only moves the node's address on +that transport: peers learn the new one from its registry entry and the identity layer +(`addressChanges` in the transports' figures, [`measurements.md`](measurements.md)). Seeds given as the old +address go stale. + +## Backups + +`GET /admin/document` on a ready node exports the whole document with its history, +naming its swarm and epoch ([`openapi.yaml`](openapi.yaml)). Every node has all the data, so one +node's export is a backup of the swarm. A document file is a backup too, of its node, +but it is being written: export instead. + +To restore: + +* **into a running swarm**: an `import` remake with the backup ([`remakes.md`](remakes.md), Import + and restoring a backup), which replaces the content for everyone; +* **a swarm that is gone**: one node with `--recover-from FILE` (ready at once, on the + backup's swarm and epoch), the others joining it; +* **its content as a new swarm**: `--genesis --remake import:FILE`. + +Quarantined writes (`GET /admin/quarantine`) and the archive of old epochs live in +memory only, and aren't in any backup. + +## A whole swarm down + +Nodes with document files are ready at once on restart and merge with each other. If +none has its data any more, they all join and wait for each other, and none becomes +ready: start one with `--recover-from` and a backup, or with `--genesis` (with its state +file: the same swarm again, empty), and the others join it ([`lifecycle.md`](lifecycle.md), A whole +swarm restarting). + +## Upgrading + +One node at a time, restarting each with its files. What to know: + +* **The wire format.** Each change to it bumps a version byte, and nodes ignore + messages of another version. Across such a change, upgraded nodes and the others + don't talk: the swarm is split until every node is upgraded, both sides go on + writing, and they merge then. Mycelium logs it (`Peer speaks another wire version + (another build); ignoring its messages`, warn, once per peer); libp2p and Tor at debug + only (`Dropping a message that doesn't decode`). Until the first release, assume + every upgrade is such a change, and upgrade all nodes close together. +* **Automerge.** A remake needs every node to build the same snapshot, byte for byte, + so the same Automerge version everywhere; a node with another one rejoins instead of + switching ([`remakes.md`](remakes.md), Limits). Don't remake while the swarm runs two versions. +* **The dashboard** reloads by itself once its node serves another version of it. + +## What to alert on + +From the metrics (`GET /admin/stats` as Prometheus, [`measurements.md`](measurements.md)): + +| Metric | Alert when | Means | +|---|---|---| +| `continuum_ready` | 0 for longer than a join takes | the node can't join or rejoin (its log says why) | +| `continuum_clock_off` | 1 | its clock is off from its peers' ([`trust.md`](trust.md), What depends on clocks) | +| `continuum_remakes_committed` | above 1 | a double commit; no node switches on its own | +| `continuum_quarantined` | above 0 | writes set aside by a rejoin, lost at the next restart | +| `continuum_catch_up_attempts` | growing | the node can't get the epoch its peers advertise | +| `continuum_reachable_members` | at most half of `continuum_members` | no remake can commit | +| `continuum_peer_heals_total` | growing steadily | sync keeps getting out of step with a peer | + +And from the log, at warn or error: + +* `A transport stopped`: the node stops; +* `Still joining the swarm; trying on`: with why (seeds unresolved, peers' states); +* `Several remakes committed for this epoch; not switching on my own`, `Decisions on + this remake disagree; not switching on my own`, `Peers disagree about the newest + epoch; following the one most of them are on`, `More peers are on another epoch of + this generation than on ours`: two remakes committed for one epoch; look at the + quarantines that follow ([`remakes.md`](remakes.md), Troubleshooting); +* `Quarantined writes the remake has no place for`; +* `Couldn't write the state file`, `Couldn't rewrite the document file`: a full or + read-only disk; +* `The … file is readable by others than its owner; chmod 600 it`; +* `Peer speaks another wire version (another build); ignoring its messages`: a node on + another build. + +## What a node uses + +* **Memory**: the document with its history (Automerge holds more in memory than + `savedBytes` on disk), the archive of old epochs (`archiveRetention`, 2 epochs by + default), quarantined writes and per-peer state. Remaking (`compact`) drops the + history ([`remakes.md`](remakes.md)). +* **Disk**: the document file, between the full save and that plus what was appended + since its last rewrite (`--document-rewrite`); the state file (small); Tor's state + directory. +* **Network**: per neighbour, about two keepalives per `--keepalive` interval when + quiet ([`measurements.md`](measurements.md), Traffic), plus sync for every write, and LINKS while a + dashboard is open; a joining node downloads the document once (`savedBytes`, about). +* **Processes and ports**: the HTTP API (`--bind-addr`); a supervised Mycelium daemon + and its overlay port; libp2p's listen addresses. diff --git a/docs/peers.md b/docs/peers.md new file mode 100644 index 0000000..95b4dab --- /dev/null +++ b/docs/peers.md @@ -0,0 +1,358 @@ +# Peers: handshake, keepalive, dormancy and topology + +The timings named below (`dormant_after` and the like) derive from `--keepalive` +([Timings](#timings), at the end). + +The phases a node goes through (starting, joining, ready, rejoining, stopping) are in +[`lifecycle.md`](lifecycle.md); this page is the protocol between peers. + +## Node IDs and addresses + +A node is its node ID (`continuum:<hex>`, from its node key); a transport address only +says where it is (`mycelium:<public key>`). The identity layer (`transport::identity`) +between the node and its transports keeps the two apart, so everything below speaks of +node IDs. A node may have several transports, and reaches a peer on one both have +(module `transport::paths`). With several, it measures them: every 30 s (or every +keepalive interval, if that is longer) each gets a PING (the keepalive's, or one of its own), answered by a PONG over the same path, which +gives its round-trip time. A peer's only path is measured the same way once messages +go over it (a busy pair sends no keepalives). The +configured order decides until both are measured; then the node changes to another +path when it is a quarter faster (and at least 5 ms), so it doesn't flap. A send that +fails goes to the next path at once, and a path whose sends fail, or whose PINGs go +unanswered three times in a row, is down: used only when no other works. A node on +several transports bridges nodes that share none ([`transports.md`](transports.md), Several transports): +the topology links it on each of its transports (below). + +```mermaid +sequenceDiagram + participant N as New node + participant S as Seed + N->>S: WHO(N's node key, signature over N's and S's address) + S-->>N: ME(S's node key, signature over S's and N's address) + Note over N,S: each knows which node is at the other's address +``` + +A WHO asks who is at an address and says who is asking; a ME answers. Each carries the +sender's public node key and its signature over the sender's address and the +recipient's. The transport authenticates the sender's address, so a proof holds for +that pair of addresses only. Once a node proved itself at an address, messages from +there are that node's; messages from an address no node proved itself at are dropped, +and the address is asked WHO (at most every 5 s), so after a restart the first message +from a peer brings its proof back. + +A seed is an address (or something the transport resolves to one: a Mycelium public +key, IPv6 address or host name); it is asked WHO, and its ME names the node. Where other +nodes are comes from HELLO_REPLYs and the member list: hints, which are asked WHO before +anything is sent there; a send to a node that hasn't proven itself anywhere yet is +refused, and retried like any refused send. A node proven on some transports is also +asked WHO at its hinted addresses on the others, at most once a minute each: its ME +there adds a path to it (a second transport, Tor say), which is then measured and failed +over to. No seed is needed for that: the member list carries every member's addresses. + +## Handshake and join + +```mermaid +sequenceDiagram + autonumber + participant N as New node N + participant S as Node S (reached via a seed) + participant P as Peer P (known to S) + + N->>S: HELLO(swarm ID if known, epoch, key proof) + Note over S: claimed swarm ID matches ours (or none claimed)<br/>N is member, Established, sync reset + S->>N: HELLO_REPLY(swarm ID, active established neighbours [P], epoch, key proof) + Note over N: swarm ID matches or is adopted<br/>S is member, Established, sync reset + N->>P: HELLO(swarm ID, epoch, key proof) - discovered via S + P->>N: HELLO_REPLY(...) + par sync with one ready peer on the same epoch, the catch-up source (here S) + S-->>N: sync messages + N-->>S: sync messages + end + Note over N: all of the source's heads applied: initial sync done<br/>then sync with every neighbour on the same epoch + alt S is on a newer epoch than N + Note over N: no sync (epochs differ)<br/>remake worker starts a rejoin + N->>S: BLOB request (snapshot change) + S-->>N: manifest, then chunks (also from other peers on the epoch) + Note over N: snapshot applied, then sync as above + end +``` + +One round trip makes both sides members of each other: the HELLO proves the sender's +key, the reply the swarm ID and the responder's key. A new handshake resets the sync +session on both sides; HELLOs carry the handshake's number, so a resent HELLO (its +reply was lost) or a late copy is answered without resetting again. A node that doesn't know its swarm +yet (it is joining itself) doesn't answer a HELLO; the sender retries. + +The reply lists at most 64 of the responder's active neighbours, each with the addresses +it reaches them at (at most 8): if it has more, those the recipient would most likely +pick itself (lowest pair score with it). The recipient takes at most 64 from a reply, +only valid identities (node IDs), and, once it knows the member list, only +its own neighbours; their addresses are hints for its transport, and each gets a +HELLO. + +Every HELLO carries a random id of the sender's current run, and the HELLO_REPLY echoes +it. A reply with another id answers a HELLO from an earlier run (delivered late, or +queued in Mycelium while the node was down) and is ignored. + +**Stale messages.** Mycelium may deliver a message up to 5 minutes late, and an external +Mycelium queues messages while our node is down (a restart drains that queue for up to +2 s, but only to cut noise). The protocol therefore must not depend on where in a +peer's message stream a node starts listening: unanswered HELLOs are retried, a HELLO +resets the session on both sides, sync messages are idempotent and checked against the +epoch, and lost or stale sync state is repaired by the reply timeout, the `need` rule and +the heal. Keepalives only inform. + +Every message carries the sender's epoch, and with it how many times the sender changed +epoch in its current run. A node records a peer's epoch only from a message whose count +is at least the newest it has seen from that peer, so a message delayed from before +the peer switched (or rejoined, or gave up a rejoin) can't flip its recorded epoch +back. The count starts over when the peer restarts, which its HELLO or HELLO_REPLY +shows by a new run ID. + +With a swarm key (`--swarm-key-file`), every HELLO and HELLO_REPLY carries +`HMAC-SHA256(key, label, swarm ID, from, to)`. A peer is admitted by its first valid one; +until then everything else it sends is ignored, and a handshake without a valid proof +gets no answer at all. + +Retries: an unanswered HELLO is resent after 1 s, doubling up to 30 s. To an +unreachable peer (dormant, or three unanswered HELLOs) it is a probe: after +`dormant_probe`, doubling up to 6 × `dormant_probe` (or `--dormant-probe-max`, which is +every wait if shorter); members get probes even when they +aren't neighbours. A member a HELLO_REPLY lists among the sender's active neighbours is +probed at once: the sender reaches it, so after a partition the first pair to find each +other again brings the others' probes forward, and the sides reconnect within seconds +rather than pair by pair. So are all unreachable members once this node becomes +reachable where it wasn't (libp2p: a relay grants it a slot, or AutoNAT finds it +public): two nodes behind NATs that restart together can't reach each other until they +hold slots, and the second to get one reaches the first at once instead of at its next +probe. Sync messages from a peer that isn't a member yet are +dropped (the handshake resets the session anyway) and trigger our HELLO. + +After a sync reset, both sides may open the sync; crossed openings cost nothing between +peers that share history. Only a ready node never opens towards a peer that isn't ready +(HELLO, HELLO_REPLY, sync messages and keepalives all say whether the sender is): that +peer's answer would carry empty heads, and the whole document would go out twice. The +peer that isn't ready opens itself. + +While a node catches up (joining, or rejoining a newer epoch, until its initial sync is +done), it syncs with one peer only, the first it tries: otherwise every peer would send it +the whole document (or everything it lacks) at once. Messages from the others are applied +but not answered until the initial sync is done; then they only exchange heads. The +source counts as alive while any message from it arrives; after `catch_up_probe` of +silence (keepalive / 30, between 1 and 5 s) it gets a STATUS. After three unanswered +probes, or when it says it isn't ready, or twice that it wouldn't sync with us, the next +peer takes over. Being alive isn't progress, though: the next peer also takes over when +three of our sync messages in a row went unanswered and the source has sent no sync +message for the transport's delivery limit (five minutes for Mycelium: by then it has +delivered or dropped anything sent, however large); the source given up then isn't +chosen again for as long, +so a stalled source doesn't simply take the role back, and gets a fresh HELLO, whose +reply brings its current peer list (the first one may have listed nobody: a peer lists +only its neighbours, and a new member isn't one until its registration has spread). Every minute without a sync +message from the source is logged as a warning. + +A sync message larger than 16 chunks (16 × `--blob-chunk`, 1 MiB by default) isn't +sent: it would be all or nothing, sent again whole after a lost copy, and on a slow link +may not arrive within the transport's delivery limit (Mycelium's five minutes). The sender takes it back and offers a +*delta* instead: the changes in the history of its heads but not of the peer's (as far +as it knows them). The peer fetches them as a blob, in chunks, from the sender and any +other peer holding those heads (every such peer produces the same bytes), loads them, +and syncs on; the rest goes in ordinary messages. Until the peer answers, the sender +sends it nothing; if the answer is overdue, it offers again. This is how a join without a +recent remake, or a return after much was written, gets the bulk of its changes. + +Towards a peer on a slow link the limit is lower: a message is offered as a delta if it +would take more than 5 s at the throughput measured to that peer (`syncThroughput`, +[`measurements.md`](measurements.md)), or if it is at least half as large as one that didn't make it to +the peer in the last hour (its send failed, or it went unanswered; only those of more +than two chunks count), but never below one chunk. A message too large for the link +then goes chunk by chunk instead of failing and being sent again whole. + +Every sync message says whether its sender is ready. Only a ready peer can be the source, +and only a message from one completes the initial sync: a peer that is itself still +catching up has empty heads, which any document trivially contains. + +The two sides of a pair take turns sending keepalives; the one that leads the pair sends +the first. The leader is chosen by one bit of `SHA-256(label, smaller key, larger +key)`, so both sides agree without talking and each node leads about half of its pairs +([`swarm/pair.rs`](../src/swarm/pair.rs)). + +## Liveness, dormancy and HELLO state (per peer) + +```mermaid +stateDiagram-v2 + direction LR + state Liveness { + [*] --> Unknown + Unknown --> Active: any message received + Unknown --> Dormant: node ready, a neighbour, nothing received for dormant_after + Active --> Dormant: node ready, a neighbour, nothing received for dormant_after + Active --> Dormant: its farewell (a STATUS saying it is leaving) + Dormant --> Active: any message received (after its farewell: from its next run) + } + state Handshake { + [*] --> NotSent + NotSent --> Sending: HELLO goes out + Sending --> AwaitingReply: sent (or failed) + AwaitingReply --> Sending: retry due + AwaitingReply --> Established: HELLO_REPLY + AwaitingReply --> NotSent: no longer a neighbour (unless probed while unreachable) + Sending --> Established: HELLO_REPLY + Established --> NotSent: heal, stall reset, peer reactivated,<br/>or it no longer proves the swarm key + } +``` + +- **Unknown** is a peer nothing was heard from since this node started (it is known + from the member list, or another peer named it). It is treated as active; it only + shows that nobody answered yet. +- **Becoming dormant** resets sync with the peer, stops syncing and pinging it, and + probes it with a HELLO (after `dormant_probe`, doubling). A dormant member stays in + the electorate of remakes while its alive stamp is recent ([`remakes.md`](remakes.md)), so enough + of them make a majority impossible; an operator can forget them (`DELETE /peers/{peer}`, + a swarm-wide record that is cancelled if the peer turns out to be alive). +- **Any message** from a dormant peer makes it active again and forces a fresh + handshake, whose reply resets sync on both sides. +- Dormancy is only judged while the node is ready, so joining nodes keep fast retries. + A peer's farewell (it is shutting down: a STATUS with `leaving`) makes it dormant at + once, whatever our phase; it isn't counted as an incident (`dormancies`). What that + run still sends while it stops (sync messages, PINGs, a LINKS_REPLY) doesn't make it + active again: only a HELLO or HELLO_REPLY from its next run does. +- **Strangers** (peers that never became members: listed to us but never answering, or + failing the handshake) are forgotten locally once silent for a day, so peers named + to a node can't grow its list without bound. + +## Member registry and neighbours + +`_continuum/members/<node ID>` lists every node that has joined, replicated with the +document and carried into new epochs. An entry says where its node can be reached +(`addresses`, kept up to date by the node), so members find each other without having +met. It is a pure registry, not access control (that +is the swarm key). A node registers itself as soon as it is ready and again whenever its +entry is missing while it runs (checked every 5 s). Forgetting a peer (`DELETE /peers/{peer}`) +removes its entry along with writing the forget record; a forgotten node that is alive +cancels the record and registers again. Each node acts on a record as soon as it has +it (and looks at the records again every 5 s), once: a peer it hears of again before +the cancel arrives is kept. + +From the member list every node computes the graph ([`swarm/topology/`](../src/swarm/topology/)), leaving out +members it can't reach (three unanswered HELLOs, or dormant, until they answer a probe; +or with no address on any of its transports), and those whose alive stamp is older than +the electorate window (`electorate-window`) that it hasn't heard from since it started +(a member coming back sends its HELLO, and counts again from then): +a ring by key hash, each member linked to the nearest reachable member +in both directions, plus, per member, the `k − 2` reachable others with the lowest pair +score; a link exists if either end chose it. With several transports, each has such a +graph among the members on it (by the addresses in their entries), and a node's +neighbours are its neighbours on each transport it is on: every transport's graph is +connected, and a member on two transports is in both, so as long as some members are +on several, the swarm is connected, and those have about `k` neighbours per +transport. With 100 members of which 10 are online, +the 10 form a full mesh; nodes that briefly disagree on who is reachable only add links. `k` is `--neighbours` (default 12), so swarms of up to 13 nodes are a +full mesh. A node *initiates* only towards its neighbours (HELLO, keepalive, dormancy +checks, sync openings, pushing its changes, epoch announcements, the heal, the peer list +in HELLO_REPLY) and *answers* anyone. Changes travel over several hops; heads in +keepalives tell a node when a neighbour has something it lacks. Until a node is ready +and knows the member list, everyone counts as a neighbour, so joining works as before. +A peer that proved itself a member of the swarm (its handshake) but isn't in the member +list yet is a neighbour too, until it is: otherwise a node that joined before the +genesis node registered itself, and registered itself in a document without the genesis +node, would never sync with it, and neither registration would pass. +So is a member that lists no address on the node's transports but reaches it anyway, as +long as it says it syncs with the node (STATUS): it only connects out (a libp2p node +without a listen address), so it is in no graph but its own, and the nodes it picked +would otherwise only answer it, never send it their changes. +The neighbours are recomputed as soon as the member list changes or a peer becomes +unreachable or reachable again, and every 5 s for what has no event of its own +(unreachability expiring); new ones get a HELLO and a sync opening, former ones a session +reset. + +Each node judges its own stability ([`swarm/stability.rs`](../src/swarm/stability.rs)): the fraction of the last 24 h +it had at least one live peer. It writes only a coarse tier into its registry entry +(`tier`: `stable` after a day at ≥ 95%, back to `transient` below 80%), and only when it +changes. After a restart, a stable node stays stable if a peer's HELLO_REPLY says it was +away for less than an hour; otherwise it starts over as transient. In the neighbour graph, +half of each member's scored slots prefer stable members, so long-running nodes form the +backbone (with 20 stable of 250 members and k = 12: about 65 links per stable member). + +Changes travel through a partial mesh by announcement and pull ([`swarm/relay.rs`](../src/swarm/relay.rs)): a node +pushes its own writes to its neighbours; after applying changes from one neighbour it +sends its other neighbours a STATUS with its new heads, and a neighbour that lacks those +heads pulls them (it sets them as the peer's heads in its sync state, so Automerge asks +for what's missing). It pulls from one neighbour at a time (15 s timeout); neighbours +that announce heads it lacks meanwhile wait their turn and are pulled from as soon as +the current pull is answered, if they still have something it lacks. The regular +STATUS (every third keepalive) triggers the same pull, so gaps close within a few +keepalive intervals. A node +that just caught up announces its heads once. In a full mesh nothing is announced: the +writer reaches everyone (while a node doesn't know its neighbours yet, it announces). + +## Keepalive and self-healing + +```mermaid +sequenceDiagram + participant A as Node A (its turn) + participant B as Node B + + Note over A: B established and active, quiet for keepalive,<br/>no keepalive to B for keepalive, known for keepalive + alt no STATUS between them for 3 × keepalive + A->>B: STATUS(heads, epoch, ready, syncs with you) + Note over B: mark A seen (reactivate if dormant)<br/>record A's epoch, reset sync if it changed<br/>re-handshake if A was dormant or unknown + B->>A: STATUS_REPLY(heads, epoch, ready, syncs with you) + Note over A: mark B seen, record B's epoch + else otherwise + A->>B: PING(nonce) + Note over B: mark A seen (reactivate if dormant) + B->>A: PONG(nonce, B's time) + Note over A: mark B seen, round-trip sample + end + alt after a STATUS: same epoch, heads have differed for resync_after with no sync from it since,<br/>no answer due to our own sync message, last heal at least resync_after ago + A->>B: HELLO + B->>A: HELLO_REPLY + Note over A,B: sync reset on both sides, sync restarts + end + Note over A,B: after an epoch switch a node announces its epoch at once:<br/>ready peers already on it get its sync opening,<br/>all others a STATUS#59; a node that becomes ready<br/>sends every neighbour a STATUS +``` + +A quiet pair's keepalive is a PING, which the peer's paths layer answers at once with a +PONG over the same path (`transport::paths`): both sides learn the other is alive, and +the leader gets a round-trip time. Every third one is a STATUS instead, answered with a STATUS_REPLY: each +side's state, for the heal and against lost announcements. A node's state also goes out +when it changes: its heads (announcements, above), its epoch after a switch, and its +readiness when it becomes ready (or ready again after a rejoin) and at shutdown. + +A STATUS lists the sender's heads (up to 32; more as a digest of them with 32 of +them, another 32 each time, so a recipient pulls the listed ones it lacks), its epoch, +whether it is ready, and whether it would sync with the recipient. + +The two sides take turns: after a keepalive from the peer, the next one is ours (the +leader of the pair sends the first). On its turn a node sends once the pair has been +quiet for a keepalive interval; off its turn only after 1.5 × keepalive, in case the +other one doesn't, e.g. because a keepalive or its answer was lost (any message counts +as heard, so sync traffic suppresses keepalives on both sides). If two keepalives +cross anyway (the peer's arrives within half an interval after ours), the next turn is +the leader's, so they don't keep crossing. One keepalive and its answer per pair and interval, each side sending +every other one, so both measure the round trip, instead of keepalives from both sides +that often cross. + +Inside sync, a lost message is resent after the reply timeout (10 s, 30 s with Tor +enabled, doubling up to 30 times that). If the peer asks for a change we believe it has (the message carrying it was +lost while later ones arrived), we forget which changes were sent, and the next message +carries everything it lacks. A peer that got a change without the ones it builds on asks +only for those, without saying what it has; Automerge would send just them, and the peer +ask again for theirs, one round trip per missing change. So we answer as if it had +nothing since what we share: everything since then goes at once (a blob, if large). + +## Timings + +Derived from `--keepalive` (`Timing::from_keepalive`, [`state/timing.rs`](../src/state/timing.rs)): + +| Name | Value | What for | +|---|---|---| +| `status_refresh` | 3 × keepalive | every third keepalive is a STATUS | +| `dormant_after` | 4 × keepalive | silent this long, a peer is dormant (three keepalives lost in a row are tolerated) | +| `resync_after` | 2 × keepalive, at least twice the first sync reply timeout (20 s; 60 s with Tor) | heads differing this long make a heal | +| `dormant_probe` | 10 × keepalive | the first probe of an unreachable peer, doubling up to 6 × (or `--dormant-probe-max`) | +| `catch_up_probe` | keepalive / 30, from 1 to 5 s | how soon a silent catch-up source is probed | +| sync reply timeout | 10 s (30 s with Tor enabled: the longest of the transports'), doubling up to 30 × | resending a sync message nothing answered | + +The peer worker runs when the earliest of these is due for some peer (and when something may have made one due sooner: a HELLO sent, a sync message sent, a keepalive received, a new peer or new neighbours), at least every 5 s; the remake worker every second. diff --git a/docs/remakes.md b/docs/remakes.md new file mode 100644 index 0000000..efcc8e3 --- /dev/null +++ b/docs/remakes.md @@ -0,0 +1,410 @@ +# Remakes: an operator's guide + +A remake moves the whole swarm to a new document, the next *epoch*. Use one to: + +* **compact** (`compact`): same content, no history. The document shrinks to what it + holds now, and joining nodes download less. +* **replace** (`replace`): new content you supply as JSON (the shape `GET /store` + returns), with rules for where writes made in the meantime go. +* **import** (`import`): new content from an Automerge document, e.g. a backup made + with `GET /admin/document`. Unlike `replace`, Automerge types survive (text, + counters, bytes, timestamps, floats). + +Everything else is the same for all three kinds. All endpoints need the node's basic +auth; examples leave it out. + +## How a remake runs + +1. **Proposal.** You `POST /admin/remakes` to any node; it becomes the *coordinator*. + It fixes the *electorate* and, for `replace` and `import`, the *base* (the + document's heads now, or `baseHeads`), writes the proposal into the document and + votes for it. +2. **Vote.** Every member in the electorate votes for it once the proposal reaches it, + after a random wait of up to half a keepalive interval, unless it already voted for + a competing remake of the same epoch or the remake has its majority already (then a + vote would only be one more change for everyone). For + `replace` and `import`, a member first downloads the new document from the + coordinator and the members that have it, so a committed remake has its document on + a majority. +3. **Decision.** The coordinator commits as soon as a majority of the electorate voted + for the remake. It aborts if that becomes impossible, if the voting period + (`settle`) ends without it, or on your request. A majority that arrives after + the voting period doesn't count, and an abort request or a competitor's commit that + arrives while the coordinator builds the new document still wins. Once the remake + has expired (below), the coordinator doesn't commit it any more. A compaction's + base is fixed now: the coordinator's heads as it starts building, recorded in the + decision. +4. **Switch.** After the switch delay, every node builds the new document on its own, + checks that it matches the committed epoch, and carries over the writes made since + the base that it has, its own and other nodes'. Its old document goes into its archive. +5. **Stragglers.** Nodes that were away, or whose build didn't match, notice peers on + the newer epoch and *rejoin*: they download the epoch's snapshot, sync the changes + made since, and carry over their own writes within the replay window + ([`lifecycle.md`](lifecycle.md), "Rejoining"). + +Peers on different epochs don't sync with each other, so a node stays on its epoch +until it switches or rejoins. + +### Electorate and majority + +The electorate is the set of members whose alive stamp in the member registry is +recent: the window is the coordinator's `--electorate-window` (default 7d, at least +7h). Every node publishes its window in its registry entry (`window`), and renews its +stamp every seventh of the shortest window any member publishes (once a day by +default), at most hourly: so a coordinator with a shorter window than the others still +counts every live member. A stamp more than a renewal interval in the future (its +node's clock is that far ahead) doesn't count. The coordinator is always included. A remake +needs ⌊n/2⌋+1 of the n electorate members. + +Each member votes for at most one remake of its epoch, and votes again only once that +remake is aborted, so two competing remakes, even on two sides of a network partition, +can't both get a majority. Votes are kept in the votes register +(`_continuum/votes/<node ID>`), one entry per member. + +What this means in practice: + +* Members that have been gone for longer than the window don't count, and don't block + anything. +* Members that are offline but within the window count. While more than half of the + electorate is reachable, they don't block anything either. +* If half or more of the electorate is offline, no remake can commit. Wait for nodes to + return, or forget members that are gone for good (`DELETE /peers/{peer}`), then + propose again. +* With two nodes, both must vote; with three, one can be offline. + +Only electorate members vote. A node that joined after the proposal doesn't. + +## Proposing + +All fields are optional: + +```sh +curl -X POST localhost:3000/admin/remakes -H 'Content-Type: application/json' -d '{ + "kind": "compact", + "settle": "8m", + "switchDelay": "1m", + "replayWindow": { "epochs": 1, "age": "7d" }, + "archiveRetention": { "epochs": 2, "age": "30d" } +}' +``` + +| Field | Default | Meaning | +|---|---|---| +| `kind` | `compact` | `compact`, `replace` or `import`. | +| `spec` | none | The kind's input (below). | +| `baseHeads` | current heads | `replace` and `import` only: what the new document accounts for; writes after it are carried over. | +| `settle` | 8 × keepalive | Voting period, a duration ([`configuration.md`](configuration.md), "Durations and sizes"). A short one only risks an abort for lack of votes. | +| `switchDelay` | 1 × keepalive | Wait between commit and switch, so the commit reaches every node first. Shorter only with `?force=switchDelay`. | +| `replayWindow` | 1 epoch, 7 days | How far behind a returning node may be and still have its writes carried over automatically. | +| `archiveRetention` | 2 epochs, 30 days | How long nodes keep the old document. | + +A retention is kept until *every* limit it gives has passed. + +`?force=` overrides checks: `true` all of them, or a comma-separated list of +`switchDelay` (accept a shorter switch delay), `supersede` (abort expired remakes of +this epoch and propose in their place, see below) and `clock` (propose although this +node's clock is off from its peers'). + +Answers: + +* `202` with the report and a `Location` header. With `?wait=true`, the final report + once decided (`200`). +* `409` if another remake of this epoch is still open on this node's view, if an + expired one still holds votes (unless `?force=supersede`), or if this node can't + vote for a new one (it voted for another remake of this epoch, or may have before it + restarted), or if this node's clock is off from most of its peers' ([`measurements.md`](measurements.md), + Clocks): the coordinator stamps and judges the deadlines by it (unless + `?force=clock`). +* `400` for an invalid request; `503` if the node isn't ready. + +Dropping the HTTP connection doesn't abort the remake. + +### Replace + +```json +{ + "kind": "replace", + "spec": { + "document": { "accounts": { "alice": { "name": "Alice" } } }, + "rules": [ + { "from": "users", "to": "accounts" }, + { "from": "users/admin", "drop": true }, + { "from": "settings", "to": "settings" } + ] + } +} +``` + +The rules decide where writes made after the base go: from the old epoch, and from +nodes that rejoin later. Paths are slash-separated strings, or arrays of segments for +keys that contain `/`. The longest matching `from` wins, and the rest of the path is +appended to `to`. `"drop": true` discards matching writes. A write no rule covers is +quarantined on the node that made it. `""` is the root: `{"from": "", "to": ""}` keeps +every path. + +`_continuum` (protocol data) can't be replaced or targeted by rules. + +To replace with something derived from the current content, read it with +`GET /store?heads=true`, transform it, and pass the heads from `x-continuum-heads` as +`baseHeads`: then exactly the writes you didn't see are carried over. + +### Import and restoring a backup + +Take backups with: + +```sh +curl -o backup.automerge localhost:3000/admin/document +``` + +The file is the whole document (history included). Response headers name the swarm, +the epoch and the heads. + +Restore it, keeping writes made since at their paths: + +```sh +curl -X POST localhost:3000/admin/remakes \ + -H 'Content-Type: application/octet-stream' \ + --data-binary @backup.automerge \ + --url-query 'rules=[{"from":"","to":""}]' +``` + +With `application/octet-stream`, `rules`, `baseHeads`, `settle` and +`switchDelay` go in the query; for the retention settings use the JSON form, with +the document base64-encoded: + +```sh +base64 -w0 backup.automerge | + jq -Rs '{kind: "import", spec: {document: ., rules: [{from: "", to: ""}]}}' | + curl -X POST localhost:3000/admin/remakes -H 'Content-Type: application/json' --data-binary @- +``` + +The coordinator keeps only the content: the backup's history and its `_continuum` +(protocol data of whatever swarm it came from) are dropped. A backup of one swarm can +be imported into another. + +If no node of the swarm is left, start one from the backup instead: +`continuum --recover-from backup.automerge …` ([`lifecycle.md`](lifecycle.md), "Starting"). + +### A new swarm with content + +A new swarm can start with content instead of empty: `--genesis --remake KIND:PATH` +makes the genesis document the way a remake of that kind would make a new epoch. + +```sh +continuum --genesis --remake import:backup.automerge … # a backup's content, of any swarm +continuum --genesis --remake replace:seed.json … # a JSON document, the store root +``` + +As with an `import` remake, only the content is kept: the backup's history and +protocol data (its swarm ID included) are dropped, so it starts a new swarm, not the +old one. `compact` is accepted and changes nothing. + +Without rules, every write made after the base is quarantined. Leaving rules out is +right when the backup is meant to win completely. + +## Watching and aborting + +```sh +curl localhost:3000/admin/remakes/<id> # one report +curl 'localhost:3000/admin/remakes/<id>?wait=true' # until decided +curl localhost:3000/admin/remakes # all of them +curl localhost:3000/admin/epochs # this node's epoch, archive, rejoin +``` + +A report has a `phase`: + +* `proposed`: voting. +* `committed`: decided; nodes switch after the switch delay. It can't be undone. +* `aborted`: the `decision.reason` says why; for missing votes it includes the tally, + e.g. `2 of 5 voted for it, 2 for a competing remake, 1 not yet; 3 needed`. +* `expired`: no decision long after the voting period (`settle` plus the grace + of four of the coordinator's keepalives, written in the proposal as `graceMs`, so + every node judges it alike). The coordinator is gone, and won't commit it any more. Its + votes still count: the members that cast them can't vote for another remake of this + epoch until this one is aborted, which any node can do now (below). + +`votes` lists who voted, `proposal.electorate` and `proposal.quorum` who counts and +how many are needed. `local` reports what happened on the node you asked: when it +switched, how many paths it carried over, and the quarantine entry for writes it +couldn't place. + +Abort while undecided: + +```sh +curl -X DELETE localhost:3000/admin/remakes/<id> # this one +curl -X DELETE localhost:3000/admin/remakes # whichever is in progress +``` + +The coordinator applies it within about a second (`202`, or the final report with +`?wait=true`). If the node you ask doesn't reach the coordinator (it is in another +partition, or gone but not yet expired), the request answers `409`: the coordinator +could commit before the request reaches it. Ask a node that reaches it, or wait until +the remake expires (`abortableAt`); `?force=coordinatorInactive` writes the request +anyway. An expired remake has no coordinator to apply it, so the node you ask +aborts it itself, under three conditions that keep it from crossing a late commit: + +* a whole grace period (the remake's, `graceMs`) must have passed since it expired; until + then only its coordinator may decide it, and the `409` says when (`abortableAt`); +* if the node you ask sees the coordinator as an active peer, the coordinator is + stuck rather than gone (a different version, a bug: look at its log). Aborting then + needs `?force=coordinatorActive`; +* the node you ask must have its clock in line with its peers' ([`measurements.md`](measurements.md), + Clocks): whether the remake expired is judged by it, and a node ahead would abort a + remake its coordinator may still commit. Otherwise `409`, unless `?force=clock`. + +A member that saw a remake expire before it voted for it says so in its log, once +(`Didn't vote for a remake`, warn if its clock is off: it then misses remakes others +still vote on). + +A committed or aborted remake answers `409`. + +### Superseding an expired remake + +Proposing while an expired remake of the epoch still holds votes answers `409` with +what you need to decide: for each such remake its `id`, its `coordinator`, whether +the node you asked sees that coordinator as an active peer (`coordinatorActive`), +when it expired (`expiredAt`), from when it can be aborted (`abortableAt`) and its +`tally`. Either abort it and propose again, or do both at once: + +```sh +curl -X POST 'localhost:3000/admin/remakes?force=supersede' -H 'Content-Type: application/json' -d '{}' +``` + +The expired remakes are aborted ("superseded by remake …") in the same change that +writes the new proposal, and their voters can vote for the new one. The same two +conditions as for aborting apply; with an active coordinator, add `coordinatorActive`: +`?force=supersede,coordinatorActive`. + +### Adopting a remake + +If a remake's coordinator is gone for good, another node can take over: + +```sh +curl -X POST localhost:3000/admin/remakes/<id>/adopt +``` + +Only for an expired remake (or one adopted before, whose adopter is gone too). The +node you ask becomes its coordinator (`adoptedBy` in the report). An adopted remake +has no voting period: it commits once a majority voted for it, members that haven't +voted yet can still do so, and it aborts only once a majority is out of reach or on +your request. + +Adopt on one node only, and only when the coordinator is really gone: two nodes +coordinating at once could decide differently. + +## Writes around a remake + +The swarm stays writable throughout. + +* **Writes on nodes that switch** are carried over into the new epoch, at their path + (compaction) or wherever the rules say. +* **Writes on nodes that were away** are carried over when they rejoin, if the node + is within the replay window of the first remake it missed: at most `epochs` remakes + behind, or that remake was proposed at most `age` ago. Otherwise all of them are + quarantined, as are writes whose fate can't be determined. +* **Last writer wins**: a write is carried over only if nothing newer touched its path, + a path above it or one below it, in the new epoch; newer by the times the writers' + clocks gave their changes. A carried-over write keeps the time it was made, so a + later carry-over judges by that. A write that loses is left out, logged (`Writes not + replayed: newer ones touched their paths`), or with `--superseded-writes quarantine` + kept in the quarantine. Clocks that disagree (beyond `--clock-skew-warn`) can make + the older write win. +* **Granularity**: map entries are carried over per key. Lists and text are carried + over whole, so concurrent edits of the same list on two nodes around a remake end as + last-writer-wins. +* **Who carries what**: a node that switches carries over every write made since the + base that it has, its own and other nodes'; a node that rejoins carries only its own + (the others' reached the new epoch through the nodes that switched). So a write is + lost only if no node that has it switches and its author can't rejoin with it: say, + it was made by a node cut off during the remake, synced only to nodes cut off too, + and its author restarted (keeping no document) before they rejoined. + +A rejoining node answers reads from its old document, marked with the +`x-continuum-stale` header, and writes go into it; they are carried over into the new +epoch when the rejoin ends, like the node's writes from before ([`lifecycle.md`](lifecycle.md), +Rejoining). + +## Quarantine + +Writes a remake couldn't place stay on the node that made them, in memory: + +```sh +curl localhost:3000/admin/quarantine # entries on this node +curl localhost:3000/admin/quarantine/<id> # paths and values +curl -o old.automerge localhost:3000/admin/quarantine/<id>/document +curl -X POST localhost:3000/admin/quarantine/<id>/replay # write them anyway +curl -X DELETE localhost:3000/admin/quarantine/<id> # discard +``` + +Replaying writes the values at their old paths as fresh writes (they win over what is +there now). Quarantine doesn't survive a restart of that node, so look at it soon; the +remake report's `local.quarantine` on that node names the entry. + +## Troubleshooting + +* **Aborted, "no majority in the voting period".** Too few electorate members voted in + time. Check which ones are missing (`proposal.electorate` minus `votes`). Retry with + a longer `settle` if they are only slow or briefly away; forget them if they are + gone for good. +* **Aborted, "a majority is out of reach".** Members voted for a competing remake. + Wait until that one is decided, then propose again if still needed. +* **Expired.** The coordinator stopped before deciding. If it restarts, it takes the + remake up again once ready and, the voting period being over, aborts it. If it is + gone for good, abort the remake, or supersede it with a new proposal + (`?force=supersede`). +* **`409` on proposing.** A remake of this epoch is still open: wait for it, or abort + it. Or this node voted for another remake of this epoch: propose on a node that + didn't. +* **A node stays on the old epoch.** It rejoins once enough ready members advertise + the new one ([`lifecycle.md`](lifecycle.md), "Rejoining"); `GET /admin/epochs` on it shows the + attempt and any error. Nodes keep trying. +* **Several remakes committed for one epoch, or a contested remake** (`contested: + true`: a commit and an abort written concurrently, possible only if clocks disagree; + Limits, Clocks). No node switches to one on its own records (`not switching on my + own`, error), but it follows peers that did: the epoch most of them are on. Nodes on + the other epoch of that generation rejoin it sideways once they see themselves + outnumbered (`More peers are on another epoch of this generation than on ours`, + warn), and what was written there goes to quarantine ([`lifecycle.md`](lifecycle.md), Rejoining). + Look at those quarantines; report how it came about (Limits). +* **Repairing records by hand.** Remake records live in the document under + `_continuum/remakes/<id>`. Old ones need no cleaning up: an epoch's document holds + the remakes proposed in it and the record of the remake that made it, and earlier + ones go with the previous epoch's document. Users listed in `protocol-writers` may change or delete + them through the store API, e.g. `curl -X DELETE …/store/_continuum/remakes/<id>` + to remove a broken one. Deleting a record doesn't free the votes cast for it (the + votes register, `_continuum/votes`, keeps them): abort it instead where you can. Every node acts on what it finds there, so such a write acts + on the whole swarm; it is logged. Other users get `403`, and epoch metadata + (`_continuum/epochs`) and the swarm ID (`_continuum/swarm`) are never writable. + +## Limits + +* **Clocks.** Voting periods, expiry and the replay window are judged by each node's + own clock ([`trust.md`](trust.md), What depends on clocks). With clocks further apart than the + grace period (four of the coordinator's keepalives), a remake aborted from another node after expiry + could also have been committed by its coordinator: a contested remake. + +* **Document size.** Large transfers go in chunks (`--blob-chunk`, default 64KiB) + from up to four peers at once: the new document of a `replace`/`import` remake (it + stays on the coordinator, and every member downloads it before voting), and the + epoch's snapshot for a rejoining node. One transfer is still a single message: the + changes made since the last remake, which reach a rejoining node in one sync + message. A transport drops messages it can't deliver within its delivery limit + (Mycelium: five minutes), which limits + that one to somewhere in the MiB range; remaking regularly keeps it small. Requests + are limited to 64 MiB. +* **Two remakes committed for one epoch.** Majority voting rules it out while every + node sees the same registry and keeps its records, but three things get past it: + a coordinator cut off right after it committed, while the others aborted the remake + as expired and committed another (it switches to its own); a partition that outlasts + `--electorate-window`, after which each side counts only itself as the electorate; + and a node without `--document-file` that voted, restarted, and votes again on the + other side of a partition (its vote is forgotten with the records). Two epochs of + the next generation result; the swarm follows the one more nodes are on, and the + others' writes there are quarantined (Troubleshooting). +* **Same Automerge version.** Every node must build byte-identical snapshots, so all + nodes need the same Automerge version. A node whose build differs rejoins instead. +* **Lost votes.** Votes aren't stored outside the document. A node that voted, then + restarted before its vote reached anyone but the coordinator, doesn't see its vote. + So after a restart a node votes only for the one undecided remake older than its + start, if there is just one, and not at all while there are several. Should it still + vote twice, it counts for neither (the tally reports it as `for two`). diff --git a/docs/transports.md b/docs/transports.md new file mode 100644 index 0000000..8fb81d3 --- /dev/null +++ b/docs/transports.md @@ -0,0 +1,296 @@ +# Transports + +How nodes reach each other. A node is its node ID, from its node key; a transport +only says where it is ([`peers.md`](peers.md), node IDs and addresses). Continuum has three +transports, each a cargo feature (`--features mycelium,libp2p,tor`), any of them at +once. Which a node uses is `--<transport>-enable`, and in which order +`--transport-order` ([`configuration.md`](configuration.md), Transports); the options below are each +transport's. + +| | Mycelium | libp2p | Tor | +|---|---|---|---| +| What it is | An encrypted overlay network, addressed by keys; a daemon per node (supervised, or one running elsewhere) | Direct connections, TCP (Noise, Yamux) and QUIC | An onion service per node (arti, embedded) | +| Through NAT and firewalls | Yes: the overlay routes around them, through its peers | With help: a forwarded or UPnP-mapped port, a relay, hole punching | Yes: onion services need no open port | +| Addresses | `mycelium:<public key>` | `libp2p:<multiaddr>/p2p/<peer ID>` | `tor:<onion>.onion:<port>` | +| Encryption | End to end (the overlay) | Per connection, between the two nodes, through a relay too (Noise, QUIC's TLS) | End to end (Tor) | +| Round trips | Those of the overlay's routes | The network's: the fastest on a LAN or VPN | A second or more | +| Needs | Overlay peers to connect through (`--mycelium-peers`), or a daemon elsewhere | A listen address on at least one side of each pair, or a relay | Time to bootstrap; arti's onion services are still experimental | + +## Several transports + +A node's addresses, one or more per transport, are `addresses` in `GET /admin/stats`. +A node on several transports reaches a peer on the first one in its order that both +have, until the paths are measured, then on the fastest; a send that fails goes over +the next at once ([`peers.md`](peers.md), node IDs and addresses). A node on several also carries +the traffic between nodes that share none, through the document: it syncs with its +neighbours on each side, and the changes travel through its copy. No node forwards +another's messages: a node that bridges two transports holds the document. + +Within one transport, networks are bridged by that transport: a Mycelium daemon (one +without continuum will do) routes for the overlay, and a libp2p relay (a node with +`--libp2p-relay`, or continuum with `--relay-only`, which holds no document) carries +connections between nodes that can't dial each other. + +## Mycelium + +Each node runs its own Mycelium daemon, supervised: started with the node, and +stopped with it. `--mycelium-peers` names the overlay peers it connects through +(`tls://host:9651`, not continuum peers; public Mycelium nodes are used only if listed +there), and `--mycelium-tcp-listen-port` the port it listens on for overlay peers (a +free one without). Its key is its identity on the overlay, and so its address: +`--mycelium-key-file` keeps it across restarts (a new one each start without), and +`--mycelium-key-create` makes it on the first start. `--mycelium-network-key-file` and +`--mycelium-network-name` join a private Mycelium network instead of the public one. + +`--mycelium-rest-url` uses a daemon running elsewhere, through its REST API, instead +of supervising one; then the options above don't apply: those of a supervised daemon +can't be combined with it ([`configuration.md`](configuration.md), unsetting and conflicting options), and +`--mycelium-peers` is left as it is but unused (that daemon has peers of its own). Seeds are a node's address, +its Mycelium public key or IPv6 address, or a host name with an AAAA record for one. + +### When Mycelium doesn't work + +The supervised daemon's own output goes where the node's does, so its complaints are in +the node's log, between the node's lines. + +* **`Starting the supervised Mycelium daemon`** with **``Running `mycelium` ``**: the + `mycelium` program isn't on the node's `PATH`. Supervising needs Linux (the keys are + handed over in memory, and the daemon dies with the node); elsewhere, run the daemon + yourself and give `--mycelium-rest-url`. +* **`Querying local Mycelium identity (no answer within 30s)`**: the daemon didn't answer + its API: it failed to start (its output says why: a port taken, a bad key file), or + `--mycelium-rest-url` points elsewhere. +* **Nobody reaches the node, nor it anyone** (seeds stay unanswered, `refused` grows in + the traffic, `Mycelium rejected outbound message`): the daemons aren't connected on + the overlay. A supervised daemon connects only to `--mycelium-peers` (by default + `tls://127.0.0.1:9651`, a daemon on the same machine) and to who connects to it + (`--mycelium-tcp-listen-port`, which must then be reachable); it doesn't discover + peers on the LAN. Nodes of a private network need the same name and key, or they + don't see each other. +* **A peer's address changes at every restart** (`addressChanges` in the transports' + figures, [`measurements.md`](measurements.md)): it has no `--mycelium-key-file`, so its daemon gets a new + key each start. The node ID stays, so this costs a WHO round, but seeds given as its + Mycelium address go stale. +* **`Discarded messages Mycelium queued for us before we started`** (info), and a few + `staleHelloReplies`: normal after a restart; Mycelium keeps messages for a while. +* **The daemon exits**: the node stops (`A transport stopped`, error), to be restarted + by its supervisor. + +## libp2p + +libp2p connects directly, over TCP (Noise, Yamux) and QUIC: `--libp2p-listen` says +where to listen (e.g. `/ip4/0.0.0.0/tcp/9661`), and the addresses it listens on go +into the member list as `libp2p:<multiaddr>/p2p/<peer ID>`, loopback and link-local +ones left out (unless there are no others). `--libp2p-announce` publishes other +addresses instead, e.g. a public address forwarded to the listen port. A port forwarded +on the router needs no announce: once AutoNAT finds the node dialable at the router's +address (peers tell it where they see it, and dial it back there), that address is +published next to the listen addresses. A node's entry in the member list takes its +transports' addresses in turns (each one's best first), 16 at most, so many libp2p +addresses don't crowd out its other transports'. Connected peers +tell each other where they listen (libp2p's identify), so a node also learns where +peers that dialled it can be reached. A node with no reachable listen address can still +take part: it connects out, and its peers answer over that connection (each pair of +peers needs one side that can be dialled). Two peers connected only through a relay +(below) dial each other directly where they know an address for the other; that fails +behind firewalls, so it's tried again after 30 seconds, then less often, up to every 10 +minutes. A directly connected peer's other addresses (from identify) are dialled too, +for a better path: of several connections to a peer, one is kept 20 s after the last +opened, a direct one before a relayed one, then the one with the shortest round trip +measured since then (so a new ethernet path can win over a WLAN connection hours old). + +### Relays + +A node nobody can dial (behind NAT, or with its port closed) can be reached through a +relay instead: a node of the swarm that others can dial, started with `--libp2p-relay`. +A relay says so in the member list (each of its addresses again, ending in +`/p2p-circuit`), and a node that needs a relay picks up to two of those by itself. It +needs one when it has no address others could dial, or when AutoNAT (connected peers +dialling it back) says it can't be dialled; once AutoNAT says it can, it lets them go. +A slot lost is reserved again at once. A relay that refuses a slot (it has none left, +doesn't allow us, isn't a relay, or another peer answers at its address) isn't picked +again for 10 minutes; one that doesn't grant it otherwise (it doesn't answer, or the +connection closed) isn't for 30 seconds. Give relays a `--libp2p-key-file` (with +`--libp2p-key-create`, made on the first start): without +one, a relay's peer ID changes with every start, and the addresses others have for it, +and for the nodes behind it, are wrong until the member list catches up. +`--libp2p-relays /ip4/203.0.113.7/tcp/9661` (with or without the relay's +`/p2p/<peer ID>`) names relays to pick first, before members: used the same way, only +while the node needs a relay. Any libp2p node that serves circuit relay v2 will do, +but most keep circuits short (2 minutes and 128 KiB by default): enough to punch holes, +not to carry traffic where that fails. A relay configured without its peer ID is +connected to, to find it out, and again when it doesn't grant a slot. + +Relays for the whole swarm can be listed in the document instead, so no node needs +them configured: a user in `--protocol-writers` writes their addresses (as `libp2p:` +addresses, with or without `/p2p/<peer ID>`, at most 32) to `_continuum/relays`: + +```sh +curl -u admin:secret -X PUT localhost:3000/store/_continuum/relays \ + -H 'Content-Type: application/json' \ + -d '["libp2p:/ip6/fd00::10/tcp/9661", "libp2p:/ip6/fd00::11/tcp/9661"]' +``` + +Every member picks among them as among configured relays, after those, and before +members that relay; the list goes into new epochs with the rest of `_continuum`. + +A relay need not be a node: `--relay-only` runs continuum as a libp2p relay alone, for +machines that should relay but hold no document (stable hosts on a VPN, say). It has +no node key, joins no swarm and keeps no files but its libp2p key; nodes don't take it +for one of them (it doesn't speak continuum's protocol), nor for a seed: a node given it +as one says so (`Initial HELLO failed`, "a relay, not a node"); its seed is a node behind +the relay (`<relay>/p2p-circuit/p2p/<peer ID>`, or `/p2p/<peer ID>` with the relay in +`--libp2p-relays`). It needs `transports = +["libp2p"]`, `libp2p-listen` and `libp2p-key-file` (a relay's peer ID must stay the +same); options about a swarm or a document are errors with it. With `htpasswd`, its +HTTP API has `/health`, `/admin/stats` (its addresses, ready to list, and its +transport figures) and `/admin/log`; without, it has none. + +```toml +relay-only = true +[libp2p] +enable = true +listen = ["/ip6/::/tcp/9661", "/ip6/::/udp/9661/quic-v1"] +key-file = "/var/lib/continuum/libp2p.key" +``` + +Then list it for the swarm (`_continuum/relays`, above), or configure it on the nodes +that need it (`--libp2p-relays`). A node +publishes its address through a relay (`…/p2p/<relay>/p2p-circuit/p2p/<peer ID>`) with +the others. Peers that reach +it through the relay then try to connect directly (hole punching, libp2p's DCUtR); where +that fails, the relay carries the traffic. Hole punching needs both to listen, though +the firewall may stay closed (`/ip6/::/udp/0/quic-v1`: QUIC sends from where it +listens). Each offers where the relay saw it and where it listens, so nodes on one LAN +meet there, rather than at their router's public address. A punch that fails is tried +again after a minute, then after 5, then every 30 minutes, while the peers stay +connected through the relay: what failed may have been passing. A relay takes at most 1024 circuits (64 per +peer), each for up to an hour and 1 GiB, and doesn't limit how often they're asked for; a circuit that ends is made again on the next +message. Only relays of the swarm are used, unless the node is told to use public +infrastructure (below). How it goes (AutoNAT's verdict, connections direct or relayed, +relay slots, hole punches, failed dials) is in `/admin/stats` under +`transports.libp2p`, and each peer's connection in `/peers` ([`measurements.md`](measurements.md), +Transports). + +### Address ranges + +With `--libp2p-ignore`, libp2p leaves address ranges out, e.g. `10.244.0.0/16` and +`42::/16` for VPNs it should neither depend on nor take a detour through: addresses +there are neither published (announced ones are, whatever they are) nor told peers +that connect nor offered for hole punching nor dialled, and connections there are +refused, whoever dials. Peers connected to the node forget its addresses there; others +when they next connect. Addresses of peers there aren't kept either (nor shown, nor +passed on), nor those through a relay reached there, as they're never dialled. libp2p still listens there, if told to listen +on every interface: the log (at debug level) says which addresses are left out. A node +listening on every interface (`0.0.0.0`, `::`) has many addresses, and a node +publishes 4 direct ones at most (those the router or AutoNAT found first, then those in +`--libp2p-prefer`), +and 2 through relays: leaving VPN ranges out keeps the ones that matter. A relay reached +only through a left-out range (or one without a route) can't be used then, and isn't +tried. + +Ranges this node has no route to are left out alike, without being named: the +operating system is asked whether it has a route to a peer's address (per /24 or /64, +again every 30 s), and where it has none, such addresses aren't kept or dialled. A +node without IPv6 to the internet thus leaves peers' public IPv6 addresses out; once +it has a route again, they're taken again as they're heard of next. Each range that +loses or gets its route is logged (at debug level). + +With `--libp2p-internet-hosts`, e.g. `one.one.one.one` (names or addresses; names are +resolved every 30 s, so both families are covered), the node also asks whether it has a +route to the internet at all, per IPv4 and IPv6: where it has none, it doesn't publish +its own public addresses of that family, which nobody could get answers from; nor any +public address of its own before the first answer (a few seconds after starting). +Nothing is sent to those hosts; only the routing table is asked. Without it, or for a family +none of them resolves to, this isn't asked, and every address is published. + +With `--libp2p-prefer`, the ranges that matter most come first, best first, e.g. the +LAN's `10.23.0.0/16`: of the addresses a node listens on, those there are published +first, and of the addresses a connected peer tells (identify), they are kept first +(a node keeps 8 per peer). Without it, and after the preferred ones, public addresses +come first (IPv4 outside the private, shared and reserved ranges; IPv6 in global +unicast, `2000::/3`), then those through relays, then all others (container bridges, +other LANs, overlays such as Mycelium's `400::/7`). Nodes on one LAN that are connected only through a relay find each +other's LAN address this way, and dial it. A preferred range all inside an ignored one +is refused (nothing there would be used); one with an ignored range inside it is +taken, with a warning: the ignored part stays left out. + +### Through the router: UPnP + +With `--libp2p-upnp`, a node asks its router to forward its listen ports (UPnP, which +most home routers offer, IPv4), and publishes where: the router's public address, next +to its listen addresses. Nodes elsewhere then dial it directly, with neither a relay +nor hole punching, which matters where the other side's NAT defeats punching (mobile +networks' carrier-grade NAT). It needs a fixed port to be useful across restarts +(`/ip4/0.0.0.0/udp/9661/quic-v1`), and different ones for nodes behind one router. The +log says what the router did (`The router forwards to us`), or that no router answered; +a router behind another NAT maps nothing useful, and the log says so too. The ports are +open on the router for as long as the node runs; `--libp2p-announce` takes precedence. + +### Public infrastructure + +With `--libp2p-public`, a node uses public libp2p infrastructure: it joins the IPFS DHT +(Kademlia, through the IPFS bootstrap nodes at `/dnsaddr/bootstrap.libp2p.io`), where it +finds addresses of peers the member list doesn't have, and public peers that relay; +those are picked after members that relay. Public relays keep circuits short (2 minutes +and 128 KiB by default): enough to punch holes through NATs, not to carry a swarm's +traffic for long. It also makes the node's peer ID and addresses known to the DHT, so +to anyone. Without it, nothing public is used. (For Mycelium, public peers are in +`--mycelium-peers`: none unless given.) + +### On the LAN + +With `--libp2p-mdns`, a node finds libp2p peers on its LAN (multicast DNS, IPv4): it +notes their LAN addresses, and those that speak continuum are seeds while it joins, so +nodes on one LAN find each other without `--seeds`. On a LAN with several swarms, a new +node without a state file joins whichever answers first, unless a swarm key +(`--swarm-key-file`) keeps it out of the others. Seeds for it are such +multiaddrs, with or without the peer ID and with or without the `libp2p:` scheme +(`/ip6/fd00::1/tcp/9661`, `libp2p:/ip6/fd00::1/tcp/9661/p2p/12D3…`); without the peer ID, +the node connects to find out who is there. A peer ID alone (`libp2p:/p2p/12D3…`) is +reached where the node knows it, or else through the relays it knows (`--libp2p-relays`, +above): a seed behind a relay, without spelling out the circuit. It doesn't get through NAT yet, and never uses +public libp2p infrastructure (bootstrap nodes, relays). + +## Tor + +Tor (`--tor-enable`, in builds with the `tor` cargo feature) is for reachability: +every node runs an onion service (with arti, embedded), which other nodes reach from +anywhere, without a forwarded port or a relay. It isn't for anonymity: a node's other +addresses are in the member list next to its onion address. `--tor-state-dir` is where +arti keeps its state and cache, the onion service's key among them, which makes its +address: keep it with the node, like the node key. `--tor-port` is the onion service's +port (9662). A node's address is its onion service, `tor:<onion>.onion:<port>`, which is +also what a seed gives (one ending in `/<node key>`, as nodes published them before, +still works). Arti bootstraps in the background, which takes a while: the node starts +without waiting, sends over Tor wait for it, and a new onion service may take a minute +or two to be reachable. + +Onion services prove themselves to who connects, but not the other way: at the start of +every connection, both sides prove their node key in a challenge handshake bound to the +onion address dialed ([`transport/challenge.rs`](../src/transport/challenge.rs)), and the dialer says which onion service +is its own. The node dialed believes that once a dial there found the same node key: it +dials back to check if it hasn't dialed there yet, once, and closes that connection +again; a claim that failed that check is refused without dialing for 10 minutes. Nodes +keep one connection per pair. Round trips over Tor take a second or more, so a peer on a +faster transport too is reached there while it works (the paths layer picks by +round-trip time). Arti's onion services are still experimental: expect them to be less +reliable than the other transports. + +## Keys + +Each key file is read when the node starts; a missing one is an error, unless the +option creating it is given (a new key, in a new file only its owner may read): + +| Key | File | Created with | A new one means | +|---|---|---|---| +| Node | `--node-key-file` | `--node-key-create` | A new node | +| Swarm | `--swarm-key-file` | `--swarm-key-create` | Only on the swarm's first node: the others need a copy | +| Mycelium | `--mycelium-key-file` | `--mycelium-key-create` | A new Mycelium address | +| libp2p | `--libp2p-key-file` | `--libp2p-key-create` | A new peer ID | + +Tor's onion service key is in `--tor-state-dir`, made by arti. + +## Figures + +How the transports find and reach peers: [`measurements.md`](measurements.md), Transports. diff --git a/docs/trust.md b/docs/trust.md new file mode 100644 index 0000000..521a175 --- /dev/null +++ b/docs/trust.md @@ -0,0 +1,114 @@ +# Trust model + +Who can do what to a swarm, and what stands in the way. In short: the swarm key +decides who is a member, and members are trusted with everything. + +## The boundaries + +**The network.** Every transport authenticates the sender of every message, each by +its own key: Mycelium by the public key of the sending daemon, libp2p by the peer ID +of the connection, Tor by a challenge handshake on each connection, in which both sides +prove their node key ([`transports.md`](transports.md)). So a node always knows which address sent it something, though not +who wrote what that message carries (changes are relayed). On top, a node proves its +node ID at each address with a signature by its node key (WHO and ME, [`peers.md`](peers.md)), so a +node is known by its node ID whichever transport and address it uses. A private Mycelium +network (`--mycelium-network-key-file`, `--mycelium-network-name`) additionally keeps +other Mycelium nodes from reaching ours at all; on libp2p and Tor, anyone who knows an +address can connect, and the swarm key is what keeps them out. + +**The swarm key.** With `--swarm-key-file`, a node proves the key in the handshake +before anything else it sends is acted on; until then it is a *stranger*. Without a +swarm key, every node that can reach a member can join: a joining node doesn't need to +know the swarm ID, and the swarm ID is no secret anyway. The key file should be readable +by its owner only; a node warns at startup if others may read it. + +**Public infrastructure.** Nodes use none unless told to: Mycelium peers are those in +`--mycelium-peers`, and libp2p reaches only seeds, members and the relays of the swarm. +With `--libp2p-public`, a node joins the IPFS DHT and may use public relays: they see +whom it talks to and how much (the traffic itself is encrypted end to end), and the DHT +makes its peer ID and addresses public. + +**The node key.** `--node-key-file` holds the node's own key, from which its node ID +(`continuum:<hex>`) derives: who the node is, whatever transport reaches it. It is a +secret like the swarm key: readable by its owner only (a node warns otherwise; a key +it creates is). + +**The HTTP API.** Every endpoint but the API description needs a user from +`--htpasswd`. It listens on `127.0.0.1:3000` by default (`--bind-addr`) and speaks +plain HTTP: to expose it, put a TLS proxy in front. All users may read and write user +data and use the admin endpoints (remakes, forgetting peers, exports, log filters). +There are no read-only users: whoever may open the dashboard may also edit and delete +the document's data from it, and its writes reach the whole swarm. +Only the users in `--protocol-writers` may write protocol data through the store; epoch +metadata and the swarm ID are never writable. The `/debug` endpoints exist only with +`--debug-api`. + +**The configuration.** Whoever can write a node's configuration file or its options can +run commands as the node's user: a `cmd:` (or `command`) value runs at startup, and a +`file:` value reads any file the node can ([`configuration.md`](configuration.md), "Importing values"). +Protect the configuration like the swarm key. + +## What members are trusted with + +Everything in the document. A member syncs changes to and from its peers, and every +node acts on what the document says: + +* **User data**: any member can write or delete any path. +* **Who wrote what** (`GET /admin/changes`): each change names its node and API user + as the writing node says; nothing signs that, so a member can name another. The + names of the users who write are in the document, and so on every node. +* **The member registry**: each node writes its own entry, but nothing stops a member + from writing others'. The registry decides the topology and, through the alive + stamps, the electorate of a remake. +* **Forget records**: any member can make the swarm forget a peer (until it says HELLO + again). +* **Remakes**: any member can propose one, vote, and, as coordinator, decide. Votes + are not signed: the votes register says who voted, but any member could write any + entry. Every node switches to a committed epoch without counting the votes again. +* **Epochs it advertises**: a rejoining node follows the epoch ready members + advertise, corroborated by two of them once it was ready (one if only one is active, + and one before it was ever ready), and downloads the epoch's snapshot from them, + checked against its hash. +* **What it says about itself**: its keepalives and handshake replies say whether it is + ready, whether it would sync with the recipient, and how long it last heard from it + (which counts towards the recipient's stability tier). Nodes believe that: a member + claiming to be ready can become a joining node's catch-up source (though one with an + empty document can't make it ready: empty heads prove nothing), and one misreporting + absences can move a node between tiers. + +So a member that misbehaves, by a bug or on purpose, can rewrite or delete data, +manipulate the registry and the electorate, forge votes, commit a remake without a +majority, and make the swarm forget peers. Nothing but the swarm key protects against +that; keep it to nodes you trust as much as your data. + +## What depends on clocks + +Nodes don't agree on time; each judges by its own clock: voting periods, when a remake +expires and may be aborted by others, the replay window, alive stamps (and so the +electorate). Keep the clocks within a grace period (four keepalives) of each other, +e.g. with NTP. A node whose clock is further off can, together with an operator's +abort, make a remake contested ([`remakes.md`](remakes.md), Limits). A node compares its clock with its +peers' and warns when its own is off (`--clock-skew-warn`; [`measurements.md`](measurements.md), Clocks). + +## What a stranger can do + +Send messages. Before it proves the swarm key, a node ignores everything from it but +the handshake, and doesn't sync with it. A stranger can still cost a node the work of +reading and dropping its messages. + +## What protects against mistakes + +Not against a hostile member, but against honest ones getting in each other's way: + +* Majority voting with one vote per member and epoch: two remakes can't both commit + ([`remakes.md`](remakes.md)). +* A snapshot every node builds itself and checks against the committed epoch, and + downloads checked against their hashes. +* Protocol data writable through the API only by `--protocol-writers`, and logged. + +## Not yet + +Revoking a single member, or limiting what a member may do, needs per-node keys: a +member list in the document, with keys that sign what each node writes about itself +(its registry entry, its votes). See [`review-rejections.md`](review-rejections.md), "Author binding for remake +votes and decisions". diff --git a/src/api/changes.rs b/src/api/changes.rs new file mode 100644 index 0000000..419f5ed --- /dev/null +++ b/src/api/changes.rs @@ -0,0 +1,135 @@ +//! `GET /admin/changes[/<path>]`: who wrote what, newest first, of the whole document +//! or of what is at a path (as `/store/<path>` names it). Every change names the node that +//! committed it (its author) and, for a write through the API, the user who wrote it (in +//! its message, `User:`), with when (its writer's clock) and the store paths it touched +//! (`document::replica::Writer`). Every node holds every change, so any node answers +//! for the whole swarm, as far as it has synced. + +use axum::{ + Json, + extract::{Path, Query, State}, + http::StatusCode, +}; +use serde::{Deserialize, Serialize}; + +use crate::document::store::{self, InLists}; +use crate::state::AppState; + +/// Changes listed at most. +const MAX_LIMIT: usize = 1000; +/// Changes looked at at most, newest first, for those touching `path`: each costs +/// decoding it, under the document's read lock (`store::changed_by`). +const MAX_SCANNED: usize = 10_000; + +#[derive(Debug, Deserialize)] +pub(crate) struct ChangesQuery { + /// How many, at most (100 by default, up to `MAX_LIMIT`). + limit: Option<usize>, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ChangeReport { + hash: String, + /// When, by its writer's clock (Unix milliseconds); 0 for a remake's snapshot. + at: i64, + /// The node that made it (its node ID), if it says. + #[serde(skip_serializing_if = "Option::is_none")] + node: Option<String>, + /// The user it was written for through the API, if any. + #[serde(skip_serializing_if = "Option::is_none")] + user: Option<String>, + /// The store paths it touched (a list's items by index). + paths: Vec<String>, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ChangesReport { + changes: Vec<ChangeReport>, + /// Older ones weren't looked at (`MAX_SCANNED`, or `limit` reached). + more: bool, +} + +/// `GET /admin/changes`: all of them. +pub(crate) async fn get_changes( + State(state): State<AppState>, + Query(query): Query<ChangesQuery>, +) -> Result<Json<ChangesReport>, StatusCode> { + report(state, query, None).await +} + +/// `GET /admin/changes/{key}`: those that touched what is at the path, inside it, or +/// above it. +pub(crate) async fn get_changes_at( + State(state): State<AppState>, + Path(key): Path<String>, + Query(query): Query<ChangesQuery>, +) -> Result<Json<ChangesReport>, StatusCode> { + let segments: Vec<String> = key.split('/').map(str::to_owned).collect(); + if segments.iter().any(String::is_empty) { + return Err(StatusCode::BAD_REQUEST); + } + report(state, query, Some(segments)).await +} + +async fn report( + state: AppState, + query: ChangesQuery, + path: Option<Vec<String>>, +) -> Result<Json<ChangesReport>, StatusCode> { + if !state.is_ready().await { + return Err(StatusCode::SERVICE_UNAVAILABLE); + } + let limit = query.limit.unwrap_or(100).clamp(1, MAX_LIMIT); + Ok(Json( + state + .render( + move |doc| changes(doc, limit, path.as_deref()), + |report| report, + ) + .await, + )) +} + +fn changes( + doc: &crate::document::Document, + limit: usize, + path: Option<&[String]>, +) -> ChangesReport { + let all = doc.get_changes_meta(&[]); + let mut changes = Vec::new(); + let mut scanned = 0; + let mut more = false; + for meta in all.iter().rev() { + if changes.len() == limit || scanned == MAX_SCANNED { + more = true; + break; + } + scanned += 1; + // What this change did, from its operations. + let paths = store::changed_by(doc, &meta.hash, InLists::Index).unwrap_or_default(); + if let Some(path) = path + && !paths + .iter() + .any(|touched| touched.starts_with(path) || path.starts_with(touched)) + { + continue; + } + changes.push(ChangeReport { + hash: meta.hash.to_string(), + at: meta.timestamp, + node: meta + .author + .as_ref() + .and_then(|author| String::from_utf8(author.as_bytes().to_vec()).ok()), + user: meta + .message + .as_deref() + .and_then(crate::document::replica::user_of) + .map(str::to_owned), + paths: paths.iter().map(|path| path.join("/")).collect(), + }); + } + ChangesReport { changes, more } +} diff --git a/src/api/config.rs b/src/api/config.rs new file mode 100644 index 0000000..10a9f80 --- /dev/null +++ b/src/api/config.rs @@ -0,0 +1,25 @@ +//! `GET /admin/config`: the effective configuration, as `--print-config` prints it. + +use axum::{ + Json, + extract::{Query, State}, +}; +use serde_json::Value as JsonValue; + +use crate::state::AppState; + +#[derive(Debug, Default, serde::Deserialize)] +pub(crate) struct ConfigQuery { + /// Text with placeholders as it ends up, or as written. + #[serde(default)] + values: crate::config::Values, +} + +/// `GET /admin/config`: the effective configuration with each value's source; +/// imported values and secrets are left out. +pub(crate) async fn get_config( + State(state): State<AppState>, + Query(query): Query<ConfigQuery>, +) -> Json<JsonValue> { + Json(state.config.get(query.values).clone()) +} diff --git a/src/api/dashboard/clocks.rs b/src/api/dashboard/clocks.rs new file mode 100644 index 0000000..9f069be --- /dev/null +++ b/src/api/dashboard/clocks.rs @@ -0,0 +1,127 @@ +//! Every member's clock against the swarm's, for the map: from the clocks each member +//! reports of its peers (`Link::clock`), chained from this node along the narrowest +//! bounds, then taken against their median. A member is off when it differs from the +//! median beyond its chained bound and the limit (`--clock-skew-warn`); with fewer than +//! three members compared, none is (two can't tell whose clock is off). An estimate for +//! the eye: each node judges its own clock by its peers' (`swarm::skew`). + +use std::cmp::Reverse; +use std::collections::{BTreeMap, BinaryHeap, HashMap}; + +use serde::Serialize; + +use crate::transport::Link; + +/// A member's clock against the swarm's. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct MemberClock { + /// How far it is ahead of the median (behind if negative). + pub(super) offset_ms: i64, + /// Give or take: the bounds chained to it from this node. + pub(super) within_ms: u64, + pub(super) off: bool, +} + +/// From `local` and `reports` (each member's links), with `limit_ms`; by member. +pub(super) fn swarm<'a>( + local: &str, + reports: impl IntoIterator<Item = (&'a str, &'a [Link])>, + limit_ms: u64, +) -> BTreeMap<String, MemberClock> { + // Each way between two members: how far the far end's clock is ahead, give or take. + let mut edges: HashMap<&str, Vec<(&str, i64, u64)>> = HashMap::new(); + for (reporter, links) in reports { + for link in links { + if let Some(clock) = link.clock { + edges.entry(reporter).or_default().push(( + link.peer.as_str(), + clock.offset_ms, + clock.within_ms, + )); + edges.entry(link.peer.as_str()).or_default().push(( + reporter, + -clock.offset_ms, + clock.within_ms, + )); + } + } + } + // Ahead of this node's, along the narrowest chained bounds (Dijkstra). + let mut reached: HashMap<&str, (i64, u64)> = HashMap::new(); + let mut queue = BinaryHeap::from([Reverse((0_u64, local, 0_i64))]); + while let Some(Reverse((within, member, offset))) = queue.pop() { + if reached.contains_key(member) { + continue; + } + reached.insert(member, (offset, within)); + for &(next, step, step_within) in edges.get(member).into_iter().flatten() { + if !reached.contains_key(next) { + queue.push(Reverse(( + within.saturating_add(step_within), + next, + offset.saturating_add(step), + ))); + } + } + } + let mut offsets: Vec<i64> = reached.values().map(|(offset, _)| *offset).collect(); + offsets.sort_unstable(); + let median = offsets.get(offsets.len() / 2).copied().unwrap_or(0); + let judged = reached.len() >= 3 && limit_ms > 0; + reached + .into_iter() + .map(|(member, (offset, within))| { + let offset_ms = offset - median; + let off = judged && offset_ms.unsigned_abs() > within.saturating_add(limit_ms); + ( + member.to_owned(), + MemberClock { + offset_ms, + within_ms: within, + off, + }, + ) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::transport::paths::ClockOffset; + + fn link(peer: &str, offset_ms: i64) -> Link { + Link { + peer: peer.into(), + transport: None, + relayed: false, + down: false, + rtt_us: None, + dormant: false, + others: Vec::new(), + clock: Some(ClockOffset { + offset_ms, + within_ms: 20, + }), + } + } + + /// Chained through the reports: the one clock 10 s ahead is off, wherever it is. + #[test] + fn the_odd_clock_out() { + // a – b – c – d, d's clock 10 s ahead; a only knows b. + let a = [link("b", 0)]; + let b = [link("a", 0), link("c", 5)]; + let c = [link("d", 10_000)]; + let clocks = swarm("a", [("a", &a[..]), ("b", &b[..]), ("c", &c[..])], 2_000); + assert_eq!(clocks.len(), 4); + assert!(clocks["d"].off); + assert!(clocks["d"].offset_ms > 9_900); + assert!(!clocks["a"].off && !clocks["b"].off && !clocks["c"].off); + assert_eq!(clocks["d"].within_ms, 60); + // Two alone: none. + let two = swarm("a", [("a", &[link("b", 10_000)][..])], 2_000); + assert!(two.values().all(|clock| !clock.off)); + } +} diff --git a/src/api/dashboard/dashboard.html b/src/api/dashboard/dashboard.html new file mode 100644 index 0000000..1b9a8cb --- /dev/null +++ b/src/api/dashboard/dashboard.html @@ -0,0 +1,2198 @@ +<!doctype html> +<html lang="en"> +<head> +<meta charset="utf-8"> +<meta name="viewport" content="width=device-width, initial-scale=1"> +<title>continuum</title> +<style> + :root { + --bg: #fafaf9; --fg: #1c1917; --muted: #78716c; --line: #e7e5e4; --panel: #ffffff; + --ok: #15803d; --warn: #c2410c; --bad: #b91c1c; --intended: #a8a29e; --self: #1d4ed8; + /* How strong the grey under a console message is, pointed at or shown, and under + its nodes and link on the map. */ + --pointed: .12; --shown: .22; + /* How strong the halo of a node or link is, pointed at or in focus, and the tint of + the console messages exchanged with it. */ + --halo-hovered: .15; --halo-focused: .35; + } + @media (prefers-color-scheme: dark) { + :root { + --bg: #1c1917; --fg: #f5f5f4; --muted: #a8a29e; --line: #44403c; --panel: #292524; + --ok: #4ade80; --warn: #fb923c; --bad: #f87171; --intended: #57534e; --self: #60a5fa; + } + } + * { box-sizing: border-box; } + html { background: var(--bg); } + /* The window's height, unless it is way too small: then the page scrolls. */ + body { margin: 0; background: var(--bg); color: var(--fg); + font: 14px/1.4 system-ui, sans-serif; + height: 100vh; min-height: 480px; display: flex; flex-direction: column; } + header { display: flex; gap: 1em; align-items: baseline; padding: 12px 16px; + border-bottom: 1px solid var(--line); flex-wrap: wrap; } + header h1 { font-size: 16px; margin: 0; } + #connection { margin-left: auto; color: var(--muted); } + #clock-warning, #peers .clock-off { color: var(--warn); } + #connection.live { color: var(--ok); } + #connection.down { color: var(--bad); } + /* A dot in the status's colour that breathes once with every event. */ + #dot { display: inline-block; width: 8px; height: 8px; border-radius: 50%; + background: currentColor; margin-right: 6px; vertical-align: 1px; opacity: .45; } + #dot.pulse { animation: breath .9s ease-in-out; } + @keyframes breath { + 0% { opacity: .45; transform: scale(1); } + 30% { opacity: 1; transform: scale(1.35); box-shadow: 0 0 6px currentColor; } + 100% { opacity: .45; transform: scale(1); } + } + @media (prefers-reduced-motion: reduce) { #dot.pulse { animation-duration: .01s; } } + body.stale main { opacity: .55; } + /* Swarm on the left, the full height; peers (as tall as their rows, up to 8) above + the figures (the rest) on the right. */ + /* The columns and rows as dragged (`--columns`, `--rows`, set by `sizes`). */ + main { flex: 1; min-height: 0; display: grid; gap: 16px; padding: 16px; + grid-template-columns: var(--columns, minmax(0, 3fr) minmax(0, 2fr)); + grid-template-rows: var(--rows, auto minmax(0, .4fr) minmax(0, .4fr) minmax(0, .2fr)); + grid-template-areas: "map peers" "map figures" "map document" "map console"; } + @media (max-width: 900px) { + body { height: auto; } + /* Stacked: each box as tall as set here, a collapsed one as its title. */ + main { grid-template-columns: minmax(0, 1fr); grid-template-rows: none; + grid-auto-rows: auto; grid-template-areas: "map" "peers" "figures" "document" "console"; } + #map, #figures-box, #document-box { height: 70vh; } + #console-box { height: 40vh; } + main > section.collapsed { height: auto; } + } + section { background: var(--panel); border: 1px solid var(--line); border-radius: 6px; + padding: 12px; min-width: 0; min-height: 0; display: flex; flex-direction: column; } + section h2 { font-size: 13px; text-transform: uppercase; letter-spacing: .04em; + color: var(--muted); margin: 0 0 8px; flex: none; } + section h2 .what { text-transform: none; letter-spacing: 0; color: var(--fg); } + /* Collapsed: the title only, and nothing drawn in it (`busy`). */ + button.collapse { font: inherit; padding: 0 4px 0 0; color: var(--muted); background: none; + border: none; cursor: pointer; } + button.collapse:hover { color: var(--fg); } + section.collapsed > :not(h2) { display: none; } + section.collapsed { align-self: start; } + section.collapsed h2 { margin: 0; } + section.collapsed h2 .toggle { display: none; } + /* The right column dragged off to the side: the map takes the width. */ + @media (min-width: 901px) { + main.right-hidden > section:not(#map) { display: none; } + /* The right column all collapsed: its titles at the top, a hint below them. */ + main.right-shut { grid-template-rows: auto auto auto auto minmax(0, 1fr); + grid-template-areas: "map peers" "map figures" "map document" "map console" "map hint"; } + main.right-shut #column-hint { display: block; } + } + #column-hint { grid-area: hint; display: none; font-size: 12px; } + section h2 .what .id { color: var(--muted); font-weight: normal; word-break: break-all; } + /* The space between boxes, where they are resized. */ + main.resize-columns, main.resize-columns * { cursor: col-resize; } + main.resize-rows, main.resize-rows * { cursor: row-resize; } + /* The peers dragged to a height: the table fills it, and scrolls. */ + main.peers-sized #peers-scroll { flex: 1; max-height: none !important; } + #map { grid-area: map; } + #peers-box { grid-area: peers; } + #figures-box { grid-area: figures; } + #document-box { grid-area: document; } + #console-box { grid-area: console; } + #console-box .scroll { flex: 1; } + .console { font-family: ui-monospace, monospace; font-size: 12px; white-space: pre-wrap; } + .console .when { color: var(--muted); } + /* Room on the left for the bar of a related message, so it covers no text. */ + .console > div { cursor: pointer; border-left: 3px solid transparent; padding-left: 4px; } + /* Of one exchange with the message pointed at or shown (`related`): a lighter grey, and + the bar. */ + /* Exchanged with the node or over the link pointed at on the map, or in focus: the + colour of its halo; in focus wins. */ + .console > div.peer-hovered { + background: color-mix(in srgb, var(--self) calc(var(--halo-hovered) * 100%), transparent); } + .console > div.peer-focused { + background: color-mix(in srgb, var(--self) calc(var(--halo-focused) * 100%), transparent); } + .console > div.related { background: rgba(127, 127, 127, .08); border-left-color: var(--self); } + .console > div:hover { background: rgb(127 127 127 / var(--pointed)); } + .console > div.focused { background: rgb(127 127 127 / var(--shown)); } + /* The message takes the box's width and scrolls with it, not in a box of its own. */ + #stats pre.message { margin: 4px 0 0; font-size: 12px; white-space: pre-wrap; + max-height: none; overflow: visible; } + svg { flex: 1; min-height: 0; width: 100%; display: block; cursor: default; } + /* Dragged to pan or to move a node, also by touch. */ + /* Dragged, not selected: neither its labels' text nor the picture itself. */ + #graph { touch-action: none; user-select: none; -webkit-user-select: none; } + #graph.dragging, #graph.dragging * { cursor: grabbing; } + svg circle { cursor: pointer; } + /* Labels on an underlay of their own shape, so they read where they lie on a line. */ + #graph text { paint-order: stroke; stroke: var(--panel); stroke-width: 4px; + stroke-linejoin: round; } + .legend { color: var(--muted); font-size: 12px; display: flex; gap: 1em; flex-wrap: wrap; + flex: none; } + .legend span::before { content: ""; display: inline-block; width: 18px; height: 0; + vertical-align: middle; margin-right: 4px; border-top: 2px solid var(--colour, currentColor); } + .legend .intended::before { border-top-style: dashed; --colour: var(--intended); } + .legend .used::before { border-top-width: 3px; --colour: var(--muted); } + .legend .other::before { border-top-width: 1px; --colour: var(--muted); } + .legend .direct::before { --colour: var(--muted); } + .legend .relayed::before { border-top-style: dashed; --colour: var(--muted); } + .legend .dormant::before { border-top-width: 1.5px; --colour: var(--muted); + box-shadow: 0 0 0 2.5px color-mix(in srgb, var(--bad) 50%, transparent); } + .legend .silent::before { border-top-style: dotted; --colour: var(--muted); } + .legend .hint::before { display: none; } + /* A kind of message: a dot in its colour, as on the map. */ + .legend .message::before { width: 8px; height: 8px; border: none; border-radius: 50%; + background: var(--colour); } + /* The dots' size, right after their legend, no taller than its text. */ + .legend .dot-size { display: inline-flex; align-items: center; gap: 4px; } + .legend .dot-size::before { display: none; } + .legend .dot-size input { width: 6em; height: 1em; margin: 0; } + /* After the slider, as tall; the kinds and the slider greyed while the dots are off. */ + .legend button.toggle { float: none; margin-left: 0; line-height: 1em; } + .legend.off .message, .legend.off .dot-size { opacity: .45; } + #legends { flex: none; } + /* Hides the legend, and shows it again. */ + button.toggle { font: inherit; font-size: 12px; text-transform: none; letter-spacing: 0; + float: right; margin-left: 6px; padding: 0 6px; color: var(--muted); background: none; + border: 1px solid var(--line); border-radius: 3px; cursor: pointer; } + button.toggle:hover { color: var(--fg); border-color: var(--muted); } + /* Nodes: circles, as on the map. */ + .legend .node::before { width: 10px; height: 10px; border: 1.5px solid var(--fg); + border-radius: 50%; background: var(--fg); } + .legend .node.silent::before { background: var(--panel); } + .legend .node.self::before { border-color: var(--self); background: var(--self); } + .legend .node.dormant-node::before { border: 2.5px solid var(--bad); } + .legend .node.clock-off::before { width: 14px; height: 14px; background: none; + border: 1.5px dashed var(--warn); } + .legend .node.focus::before { width: 14px; height: 14px; background: var(--self); + border: none; opacity: .35; } + .scroll { overflow: auto; min-height: 0; } + #figures-box .scroll, #document-box .scroll { flex: 1; } + .tree pre { margin: 2px 0 4px 1.1em; white-space: pre-wrap; word-break: break-all; + max-height: 20em; overflow: auto; } + .note { color: var(--muted); font-style: italic; } + .error { color: var(--bad); } + .tree button.add { font: inherit; line-height: 1; padding: 0 4px; margin-left: 4px; + color: var(--muted); background: none; border: 1px solid var(--line); border-radius: 3px; + cursor: pointer; } + .tree button.add:hover { color: var(--fg); border-color: var(--muted); } + .tree .nested { padding-left: 1.1em; } + .editor input.extra[type=text] { width: 12em; margin-bottom: 2px; display: block; } + .editor { margin: 2px 0 4px; } + .tree details > .editor { margin-left: 1.1em; } + .editor input[type=text], .editor textarea { width: 100%; font: inherit; color: var(--fg); + background: var(--bg); border: 1px solid var(--line); border-radius: 4px; padding: 2px 4px; } + .editor button { font: inherit; } + .editor button.delete { color: #fff; background: var(--bad); border: 1px solid var(--bad); + border-radius: 3px; } + .tree details.editing > :not(summary):not(.editor) { display: none; } + .tree .leaf[title] { cursor: text; } + table { border-collapse: collapse; width: 100%; font-variant-numeric: tabular-nums; } + th, td { text-align: left; padding: 3px 8px 3px 0; border-bottom: 1px solid var(--line); + white-space: nowrap; } + th { color: var(--muted); font-weight: 500; position: sticky; top: 0; + background: var(--panel); } + #peers tr[data-peer] { cursor: pointer; } + #peers tr.focused td { background: var(--line); } + .id { font-family: ui-monospace, monospace; font-size: 12px; } + /* A peer's status (not the legend's "dormant"). */ + #peers .active { color: var(--ok); } #peers .dormant { color: var(--bad); } + #peers .unknown { color: var(--muted); } + .tree { font-family: ui-monospace, monospace; font-size: 12px; } + /* Every entry indented alike: a branch's triangle sits in the indentation, left of + where the text of every entry begins. */ + .tree details, .tree .leaf { position: relative; padding-left: 1.1em; } + .tree summary { cursor: pointer; list-style: none; } + .tree summary::-webkit-details-marker { display: none; } + .tree summary::before { content: "▸"; position: absolute; left: .1em; color: var(--muted); } + .tree details[open] > summary::before { content: "▾"; } + .tree details > .note { padding-left: 1.1em; } + .key { color: var(--muted); } +</style> +</head> +<body> +<header> + <h1>continuum</h1> + <span id="node" class="id"></span> + <span id="clock-warning"></span> + <span id="connection"><span id="dot"></span><span id="connection-text">connecting…</span></span> +</header> +<main> + <section id="map"> + <h2>Swarm <button id="legend-toggle" class="toggle">hide legend</button> + <button id="rtt-toggle" class="toggle">layout: static</button> + <button id="links-toggle" class="toggle">links: live</button></h2> + <div id="legends"> + <div class="legend"> + <span class="node self">this node</span> + <span class="node">reports its links</span> + <span class="node silent">reports none (no LINKS_REPLY)</span> + <span class="node dormant-node">dormant (to this node)</span> + <span class="node clock-off">clock off the swarm's</span> + <span class="node focus">in focus (click a node)</span> + </div> + <div class="legend" id="legend"></div> + <div class="legend" id="messages-legend"><button id="flows-toggle" + class="toggle">messages: on</button></div> + <div class="legend"> + <span class="hint">drag to pan, wheel or pinch to zoom, turn with two fingers; + drag a node to move it, double-click it to pin it there or let it go</span> + </div> + </div> + <svg id="graph"><g id="view"><g id="flows" pointer-events="none"></g></g></svg> + </section> + <section id="peers-box"> + <h2>Peers</h2> + <div class="scroll" id="peers-scroll"><table id="peers"></table></div> + </section> + <section id="figures-box"> + <h2><span id="figures-title">Figures</span></h2> + <div class="scroll"><div id="stats" class="tree"></div></div> + </section> + <section id="document-box"> + <h2>Document</h2> + <div class="scroll"><div id="document" class="tree"></div></div> + </section> + <div id="column-hint" class="note">Drag the space left of these boxes to the right to + hide the column.</div> + <section id="console-box"> + <h2>Console</h2> + <div class="scroll" id="console-scroll"><div id="console" class="console"></div></div> + </section> +</main> +<script> +"use strict"; +// Which page this is (the node fills it in); a snapshot naming another came from +// another build: reloaded then, to get its page. +const PAGE_VERSION = "__PAGE_VERSION__"; +// The state as the node published it, and when it did (the last event). +let state = null; +let stateAt = 0; + +// Fields that grow by themselves: the node's list (`GROWING` and `is_age` in its +// dashboard), filled in when it serves the page. +const GROWING = new Set(__GROWING__); +const isAge = key => key === "secsAgo" || key.endsWith("SecsAgo") || GROWING.has(key); +const elapsed = () => Math.floor((Date.now() - stateAt) / 1000); +const aged = (key, value) => + isAge(key) && typeof value === "number" ? value + elapsed() : value; + +// RFC 7386. +function mergePatch(target, patch) { + if (patch === null || typeof patch !== "object" || Array.isArray(patch)) return patch; + const result = target !== null && typeof target === "object" && !Array.isArray(target) + ? target : {}; + for (const [key, value] of Object.entries(patch)) { + if (value === null) delete result[key]; + else result[key] = mergePatch(result[key], value); + } + return result; +} + +function el(tag, attrs, ...children) { + const node = document.createElement(tag); + for (const [key, value] of Object.entries(attrs || {})) node.setAttribute(key, value); + for (const child of children) { + node.append(child instanceof Node ? child : document.createTextNode(String(child))); + } + return node; +} +const SVG = "http://www.w3.org/2000/svg"; +function svg(tag, attrs) { + const node = document.createElementNS(SVG, tag); + for (const [key, value] of Object.entries(attrs || {})) node.setAttribute(key, value); + return node; +} + +const short = id => id.length > 14 ? id.slice(0, 6) + "…" + id.slice(-6) : id; +function name(id) { + const member = state.graph.members[id]; + return (member && member.alias) || (state.peers.peers[id] || {}).alias || short(id); +} +// Milliseconds as signed seconds: "+5.0 s". +const seconds = ms => (ms < 0 ? "−" : "+") + (Math.abs(ms) / 1000).toFixed(1) + " s"; +// Whether `clock` (a peer's, `/peers`) differs from this node's beyond the limit for sure. +function clockOff(clock) { + const limit = (state.stats.clock || {}).limitMs || 0; + return !!clock && limit > 0 && Math.abs(clock.offsetMs) > clock.withinMs + limit; +} +function age(secs) { + if (secs === null || secs === undefined) return ""; + if (secs < 120) return secs + " s"; + if (secs < 7200) return Math.floor(secs / 60) + " min"; + if (secs < 172800) return Math.floor(secs / 3600) + " h"; + return Math.floor(secs / 86400) + " d"; +} + +// What is in focus, its figures shown: a node's ID, or a link's pair key (`pairKey`, +// two IDs). null: this node's (`/admin/stats`). +let focused = (() => { + try { return decodeURIComponent(location.hash.slice(1)) || null; } catch { return null; } +})(); +const pairKey = (a, b) => a < b ? a + " " + b : b + " " + a; +const isPair = key => key !== null && key.includes(" "); +function focus(id) { + focused = id === null || id === state.node.id ? null : id; + if (shownMessage !== null) { + shownMessage = null; + document.querySelectorAll("#console > div.focused").forEach(line => line.classList.remove("focused")); + markRelated(null); + } + // In the URL too, so a reload (a new version) keeps it. + try { + history.replaceState(null, "", focused === null ? location.pathname + : "#" + encodeURIComponent(focused)); + } catch { /* not where history works */ } + render(); + markPeerMessages(); +} + +// --- Peers --- +function renderPeers() { + const table = document.getElementById("peers"); + const rows = [el("tr", {}, ...["peer", "status", "seen", "via", "rtt", "clock", "neighbour", + "same epoch"] + .map(h => el("th", {}, h)))]; + for (const [id, peer] of Object.entries(state.peers.peers)) { + const destination = peer.destination; + const path = destination && destination.paths && destination.paths[0]; + const rtt = path && typeof path.rttMs === "number" ? path.rttMs.toFixed(1) + " ms" : ""; + const via = destination + ? destination.address + (destination.connection ? " (" + destination.connection + ")" : "") + : ""; + const row = el("tr", { "data-peer": id, class: id === focused ? "focused" : "" }, + el("td", { title: id }, peer.alias || el("span", { class: "id" }, short(id))), + // How long dormant is about how long unseen ("seen"): in the tooltip only. + el("td", { class: peer.status, title: peer.status === "dormant" + ? "dormant for " + age(aged("dormantForSecs", peer.dormantForSecs)) : "" }, peer.status), + el("td", {}, age(aged("lastSeenSecsAgo", peer.lastSeenSecsAgo))), + el("td", { class: "id" }, via), + el("td", {}, rtt), + // Its clock against this node's, flagged beyond the limit (`--clock-skew-warn`). + el("td", { class: clockOff(peer.clock) ? "clock-off" : "", title: peer.clock + ? "ahead of this node's by " + peer.clock.offsetMs + " ms, give or take " + + peer.clock.withinMs + " ms" : "" }, peer.clock ? seconds(peer.clock.offsetMs) : ""), + el("td", {}, peer.neighbour ? "yes" : ""), + el("td", {}, peer.sameEpoch === null || peer.sameEpoch === undefined + ? "" : peer.sameEpoch ? "yes" : "no")); + // On press: the table may be drawn anew (ages) before a click would complete. + row.addEventListener("pointerdown", () => focus(id === focused ? null : id)); + rows.push(row); + } + table.replaceChildren(...rows); + // As tall as the rows, up to 8 of them (and the header); then it scrolls. + const scroll = document.getElementById("peers-scroll"); + scroll.style.maxHeight = rows.length > 9 + ? (rows[8].offsetTop + rows[8].offsetHeight) + "px" : ""; +} + +// --- Figures: a tree, its open branches kept across renders --- +const open = new Set(["stats", "peer", "links", "from", "to", "paths", "paths.destination"]); +function tree(key, value, path) { + // Empty: nothing to open. + if (value !== null && typeof value === "object" && Object.keys(value).length === 0) { + return el("div", { class: "leaf" }, el("span", { class: "key" }, key + ": "), + Array.isArray(value) ? "[]" : "{}"); + } + if (value !== null && typeof value === "object") { + const details = el("details", {}); + if (open.has(path)) details.open = true; + details.addEventListener("toggle", () => { + if (details.open) open.add(path); else open.delete(path); + }); + const entries = Array.isArray(value) ? value.map((v, i) => [String(i), v]) : Object.entries(value); + const summary = el("summary", {}, el("span", { class: "key" }, key), + Array.isArray(value) ? " [" + value.length + "]" : ""); + // Opened or closed on press, as the peers' rows are focused: the tree is drawn anew + // every second (ages), and a click whose press and release land on two drawings is + // lost. Keys (Enter, space) still click. + summary.addEventListener("pointerdown", event => { + if (event.button !== 0) return; + event.preventDefault(); + details.open = !details.open; + }); + summary.addEventListener("click", event => { + if (event.detail !== 0) event.preventDefault(); + }); + details.append(summary); + for (const [k, v] of entries) details.append(tree(k, v, path + "." + k)); + return details; + } + return el("div", { class: "leaf" }, el("span", { class: "key" }, key + ": "), + JSON.stringify(aged(key, value))); +} +function renderStats() { + const title = document.getElementById("figures-title"); + const what = el("span", { class: "what" }); + let shown; + const stats = document.getElementById("stats"); + if (shownMessage !== null) { + const message = shownMessage; + // Drawn once: a message doesn't change, and drawing it anew would scroll it back up. + if (stats.dataset.message === String(message.seq)) return; + stats.dataset.message = String(message.seq); + what.textContent = message.type + " " + WAYS[message.way] + " " + name(message.peer); + const field = (key, value) => el("div", { class: "leaf" }, + el("span", { class: "key" }, key + ": "), value); + title.replaceChildren("Message: ", what); + document.getElementById("stats").replaceChildren( + field("at", clockTime(new Date(message.at))), + field(message.way === "received" ? "from" : "to", message.peer), + field("way", message.way), + field("type", message.type), + field("bytes", String(message.bytes)), + el("pre", { class: "message" }, message.text)); + return; + } + delete stats.dataset.message; + if (isPair(focused)) { + const [a, b] = focused.split(" "); + what.textContent = name(a) + " – " + name(b); + what.title = focused; + const listed = (x, y) => (state.graph.intended[x] || []).includes(y); + shown = [el("div", { class: "leaf" }, el("span", { class: "key" }, "intended: "), + listed(a, b) || listed(b, a) ? "yes (member list)" : "no")]; + // Each end's view of it, as it reported it. + for (const [x, y, path] of [[a, b, "from"], [b, a, "to"]]) { + const report = state.graph.measured[x]; + const link = report && report.links.find(link => link.peer === y); + if (link) { + shown.push(tree(name(x) + " reports", { secsAgo: report.secsAgo, ...link }, path)); + } else { + shown.push(el("div", { class: "key leaf" }, name(x) + (report + ? " reports no link to " + name(y) + "." : " reports no links."))); + } + } + // Where this node is an end: the link itself, as `/peers` has the other end (where + // messages go, the paths, the addresses). The whole entry is the node's focus. + const self = state.node.id; + const other = a === self ? b : b === self ? a : null; + const peer = other !== null && state.peers.peers[other]; + if (peer) { + shown.push(tree("this node's paths", { + destination: peer.destination, addresses: peer.addresses }, "paths")); + } + title.replaceChildren("Link: ", what); + document.getElementById("stats").replaceChildren(...shown); + return; + } + if (focused === null) { + what.textContent = name(state.node.id) + " (this node)"; + shown = [tree("stats", state.stats, "stats")]; + } else { + // Its alias (if it has one) and its node ID, in full. + const alias = (state.graph.members[focused] || {}).alias || + (state.peers.peers[focused] || {}).alias; + what.replaceChildren(...(alias ? [alias, " "] : []), el("span", { class: "id" }, focused)); + const peer = state.peers.peers[focused]; + const report = state.graph.measured[focused]; + shown = []; + if (peer) shown.push(tree("peer", peer, "peer")); + else shown.push(el("div", { class: "key leaf" }, "Not a peer of this node.")); + if (report) shown.push(tree("links", report, "links")); + } + title.replaceChildren(focused === null ? "Figures: " : "Peer: ", what); + document.getElementById("stats").replaceChildren(...shown); +} + +// --- Swarm map: a small force layout, positions kept across renders --- +const positions = new Map(); +// Nodes put somewhere by hand (dragged): the layout leaves them there. +const pinned = new Set(); +const transportColour = new Map([["mycelium", "#2563eb"], ["libp2p", "#9333ea"], + ["tor", "#0891b2"]]); +const SPARE = ["#ca8a04", "#db2777", "#65a30d"]; +function colourOf(transport) { + if (!transport) return "var(--muted)"; + if (!transportColour.has(transport)) { + transportColour.set(transport, SPARE[(transportColour.size - 3) % SPARE.length]); + } + return transportColour.get(transport); +} + +function renderLegend(measured) { + const transports = new Set(); + for (const { link } of measured.values()) { + transports.add(link.transport || ""); + for (const other of link.others || []) transports.add(other.transport); + } + const items = [el("span", { class: "intended" }, "intended (member list)")]; + for (const transport of [...transports].sort()) { + const item = el("span", {}, transport || "unknown transport"); + item.style.setProperty("--colour", colourOf(transport || null)); + items.push(item); + } + items.push(el("span", { class: "used" }, "in use"), + el("span", { class: "other" }, "other transport"), + el("span", { class: "direct" }, "direct"), + el("span", { class: "relayed" }, "relayed"), + el("span", { class: "dormant" }, "dormant or down"), + el("span", { class: "silent" }, "an end reports no links")); + document.getElementById("legend").replaceChildren(...items); +} + +function edges() { + const intended = new Map(); + for (const [a, neighbours] of Object.entries(state.graph.intended)) { + for (const b of neighbours) { + intended.set(pairKey(a, b), [a, b]); + } + } + // One measured edge per pair: a report with an RTT, then the fresher. + const measured = new Map(); + for (const [a, report] of Object.entries(state.graph.measured)) { + for (const link of report.links) { + const key = pairKey(a, link.peer); + const known = measured.get(key); + const timed = typeof link.rttMs === "number"; + const knownTimed = known && typeof known.link.rttMs === "number"; + if (!known || (timed && !knownTimed) || + (timed === knownTimed && report.secsAgo < known.secsAgo)) { + measured.set(key, { a, b: link.peer, link, secsAgo: report.secsAgo, + dormant: link.dormant || (known && known.dormant), + down: link.down || (known && known.down) }); + } else { + if (link.dormant) known.dormant = true; + if (link.down) known.down = true; + } + } + } + // Dormant, or down, if either end says so, whichever report is drawn. + for (const edge of measured.values()) { + if ((edge.dormant && !edge.link.dormant) || (edge.down && !edge.link.down)) { + edge.link = { ...edge.link, dormant: !!edge.dormant, down: !!edge.down }; + } + } + return { intended, measured }; +} + +// How long a link is, ideally, for `count` nodes on a map of that size. +function idealLength(count, width, height) { + return Math.min(width, height) / Math.max(2, Math.sqrt(count) * 1.5); +} +// `pairs`: [a, b, length, loose], the length the link pulls towards (else +// `idealLength`), and whether it is loose (`lengths`). +// How hard a loose link pulls, against a measured one. +const LOOSE = .1; +// `only`, if given: the nodes laid out; the others stay where they are. +function layout(nodes, pairs, width, height, only) { + for (const id of nodes) { + if (!positions.has(id)) { + positions.set(id, { x: width / 2 + (Math.random() - .5) * width / 2, + y: height / 2 + (Math.random() - .5) * height / 2 }); + } + } + for (let round = 0; round < 120; round++) relax(nodes, pairs, width, height, 1, 10, only); +} +// One round of the layout: every node moved by `gain` of the forces on it, at most +// `most` px. Returns how far the furthest moved. A gain of 1 overshoots where a node +// has several links (the pulls add up): it gets close fast, then swings about its +// place; well below 1, it comes to rest. `only`, if given: the nodes that move. +function relax(nodes, pairs, width, height, gain, most, only) { + const ideal = idealLength(nodes.length, width, height); + { + const force = new Map(nodes.map(id => [id, { x: 0, y: 0 }])); + for (let i = 0; i < nodes.length; i++) { + for (let j = i + 1; j < nodes.length; j++) { + const p = positions.get(nodes[i]), q = positions.get(nodes[j]); + const dx = p.x - q.x, dy = p.y - q.y, d2 = Math.max(dx * dx + dy * dy, 1); + const push = ideal * ideal / d2; + force.get(nodes[i]).x += dx * push / 10; force.get(nodes[i]).y += dy * push / 10; + force.get(nodes[j]).x -= dx * push / 10; force.get(nodes[j]).y -= dy * push / 10; + } + } + for (const [a, b, length, loose] of pairs) { + const p = positions.get(a), q = positions.get(b); + if (!p || !q) continue; + const dx = q.x - p.x, dy = q.y - p.y, d = Math.max(Math.hypot(dx, dy), 1); + const stretch = d - (length || ideal); + // A loose link pulls a little, and never pushes. + const pull = loose ? Math.max(0, stretch) / d / 4 * LOOSE : stretch / d / 4; + force.get(a).x += dx * pull; force.get(a).y += dy * pull; + force.get(b).x -= dx * pull; force.get(b).y -= dy * pull; + } + let moved = 0; + for (const id of nodes) { + if (pinned.has(id) || id === grabbed || (only && !only.has(id))) continue; + const p = positions.get(id), f = force.get(id); + const fx = f.x * gain, fy = f.y * gain; + const x = Math.min(width - 40, Math.max(40, p.x + Math.max(-most, Math.min(most, fx)))); + const y = Math.min(height - 20, Math.max(20, p.y + Math.max(-most, Math.min(most, fy)))); + moved = Math.max(moved, Math.abs(x - p.x), Math.abs(y - p.y)); + p.x = x; p.y = y; + } + return moved; + } +} + +// The layout (`rtt-toggle`, kept in this browser): static by default, nodes moving only +// when laid out anew (members or links come or go); or dynamic, resized by round trip: +// each measured link's length grows with the square root of its round trip against +// the median one's, within bounds. Then, when the lengths change, the map moves to +// them a little every frame (`settle`), not at once. +let byRtt = false; +try { byRtt = localStorage.getItem("continuum-by-rtt") === "1"; } catch (_) {} +// One spring per pair of nodes, intended or measured or both: [a, b, length, loose]. +// Measured links shape the map; a link only intended (nothing measured backs it) is +// `loose`: it gives way to them, pulling only a little where stretched and never pushing +// where they hold its ends closer (`LOOSE`). +function lengths(intended, measured, count, width, height) { + const ideal = idealLength(count, width, height); + const rtts = [...measured.values()].map(edge => edge.link.rttMs) + .filter(rtt => typeof rtt === "number" && rtt > 0).sort((x, y) => x - y); + const median = rtts[Math.floor(rtts.length / 2)]; + const pairs = new Map([...intended].map(([key, [a, b]]) => [key, [a, b, ideal, true]])); + for (const [key, { a, b, link }] of measured) { + const rtt = link.rttMs; + const length = byRtt && median && typeof rtt === "number" && rtt > 0 + ? ideal * Math.min(2.5, Math.max(.4, Math.sqrt(rtt / median))) : ideal; + pairs.set(key, [a, b, Math.round(length), false]); + } + return [...pairs.values()]; +} +// What `settle` moves the map towards: the latest render's nodes and lengths. +let settling = false, settleTo = null, lastLengths = ""; +// Nodes let go (`unpin`) on a static map: the only ones `settle` moves there, to +// their places among the others. +const freed = new Set(); +function settle() { + if (settling) return; + settling = true; + let frames = 0; + const step = () => { + // Static again: where they are now (but those let go). + if (!byRtt && freed.size === 0) { settling = false; return; } + const { nodes, pairs, width, height } = settleTo; + // Damped: the swings of the layout's rounds die down instead of going on. + const moved = relax(nodes, pairs, width, height, .15, 2, byRtt ? null : freed); + renderGraph(); + if (moved > .1 && ++frames < 600 && !document.hidden) requestAnimationFrame(step); + else { + settling = false; + freed.clear(); + // Hidden: on again with the next render. + if (document.hidden) lastLengths = ""; + } + }; + requestAnimationFrame(step); +} + +// The node or link (its pair key) under the pointer, if any. Kept here rather than +// left to :hover: its halo is an element of its own, drawn behind it. +let hovered = null; +function hover(id) { + if (hovered === id) return; + hovered = id; + if (state) renderGraph(); + markPeerMessages(); +} +function hoverable(element, id) { + element.addEventListener("mouseenter", () => hover(id)); + element.addEventListener("mouseleave", () => hover(null)); +} + +// The map's elements by what they show (a key), kept across renders and updated in +// place: the one under the pointer stays itself. +const drawn = new Map(); +// The element `key` names, with exactly `attrs`; `made` sets up a new one (its +// listeners). +function drawing(key, tag, attrs, made) { + let node = drawn.get(key); + if (!node) { + node = svg(tag); + if (made) made(node); + drawn.set(key, node); + } + for (const attr of node.getAttributeNames()) if (!(attr in attrs)) node.removeAttribute(attr); + for (const [attr, value] of Object.entries(attrs)) { + if (node.getAttribute(attr) !== String(value)) node.setAttribute(attr, value); + } + return node; +} +// A label's `attrs`, turned back where the map is turned (`view`): upright, about where +// it is anchored. +function upright(attrs) { + if (!view.turn) return attrs; + return { ...attrs, transform: "rotate(" + -degrees(view.turn) + " " + attrs.x + " " + + attrs.y + ")" }; +} +function text(node, content) { + if (node.textContent !== content) node.textContent = content; +} +// A label on the map: its text, breathing once when it changed (not when new), as the +// connection's dot does: its underlay, in the shape of the new text, glows and fades. +// An animation of its own, not a class: every render sets the label's attributes. +const calm = matchMedia("(prefers-reduced-motion: reduce)"); +function label(node, content) { + if (node.textContent === content) return; + const before = node.textContent; + node.textContent = content; + if (!before || calm.matches || !node.animate) return; + const colours = getComputedStyle(document.documentElement); + const panel = colours.getPropertyValue("--panel").trim(); + const glow = "color-mix(in srgb, " + colours.getPropertyValue("--self").trim() + " 45%, " + + panel + ")"; + node.animate([ + { stroke: panel, strokeWidth: "4px" }, + { stroke: glow, strokeWidth: "7px", offset: .3 }, + { stroke: panel, strokeWidth: "4px" }, + ], { duration: 900, easing: "ease-in-out" }); +} +// A tooltip whose text changes under the pointer goes and comes back (the ages in it +// change on every render): its new text waits until the pointer leaves. +const tooltipsDue = new WeakMap(); +function tooltip(node, content) { + let title = node.firstElementChild; + if (!title) node.append(title = svg("title")); + if (title.textContent === content || !title.textContent || !node.matches(":hover")) { + tooltipsDue.delete(node); + text(title, content); + return; + } + if (!tooltipsDue.has(node)) { + node.addEventListener("mouseleave", () => { + const due = tooltipsDue.get(node); + tooltipsDue.delete(node); + if (due !== undefined) text(title, due); + }, { once: true }); + } + tooltipsDue.set(node, content); +} +// `root` showing `children`, in order: the others go, and only what is out of place +// moves. +function show(root, children) { + const shown = new Set(children); + for (const [key, node] of drawn) { + if (!shown.has(node)) { node.remove(); drawn.delete(key); } + } + let at = root.firstChild; + for (const node of children) { + if (node === at) at = at.nextSibling; else root.insertBefore(node, at); + } +} + +let lastShape = ""; +// The whole map laid out again with the next render (the window or the box resized), +// not only nodes new to it. +let relayout = true; +function layoutAgain() { + relayout = true; + lastShape = ""; +} +function renderGraph() { + if (document.getElementById("map").classList.contains("collapsed")) return; + const root = document.getElementById("graph"); + const width = root.clientWidth || 800, height = root.clientHeight || 560; + root.setAttribute("viewBox", "0 0 " + width + " " + height); + const nodes = Object.keys(state.graph.members).sort(); + const { intended, measured } = edges(); + renderLegend(measured); + const pairs = lengths(intended, measured, nodes.length, width, height); + // Laid out when who's there or who links to whom changed: only the nodes new to the + // map, the others staying where they are (links that came, after the first reports, + // don't move them); all of them at first, and when the map was resized. + const shape = nodes.join(",") + "|" + [...intended.keys(), ...measured.keys()].sort().join(","); + if (shape !== lastShape) { + for (const id of [...positions.keys()]) { + if (!nodes.includes(id)) { positions.delete(id); pinned.delete(id); } + } + const fresh = new Set(nodes.filter(id => !positions.has(id))); + if (relayout) { + layout(nodes, pairs, width, height); + // Dynamic: settled at once, not moving into place before one's eyes. + if (byRtt) { + for (let round = 0; round < 3000; round++) { + if (relax(nodes, pairs, width, height, .15, 2) <= .1) break; + } + lastLengths = pairs.map(pair => pair.join(" ")).join(","); + } + } else if (fresh.size > 0) { + layout(nodes, pairs, width, height, fresh); + } + relayout = false; + lastShape = shape; + } + settleTo = { nodes, pairs, width, height }; + const wanted = pairs.map(pair => pair.join(" ")).join(","); + if (byRtt && wanted !== lastLengths) { + lastLengths = wanted; + settle(); + } + const children = []; + // Under everything: the nodes and the link of the message shown in the console, and + // of the one pointed at, each in the grey its line has there (the shown one's where + // both mark the same), in its own shape (the lines of a link as they bow, a node's + // circle). A group per grey, so where its shapes overlap it is no darker. + const marking = new Map(); + for (const [message, how] of [[pointedMessage, "pointed"], [shownMessage, "shown"]]) { + if (message === null) continue; + for (const key of [state.node.id, message.peer, pairKey(state.node.id, message.peer)]) { + marking.set(key, how); + } + } + const marks = Object.fromEntries(["pointed", "shown"].map(how => [how, + drawing("marks " + how, "g", { "pointer-events": "none", + style: "opacity: var(--" + how + ")" })])); + const marked = { pointed: [], shown: [] }; + // A shape under `key` (a node's ID, a link's pair key), if it is marked. + const mark = (key, tag, attrs) => { + const how = marking.get(key); + if (how) marked[how].push(svg(tag, attrs)); + }; + const markLine = (key, d, width) => mark(key, "path", { d, fill: "none", + stroke: "rgb(127 127 127)", "stroke-width": width + 8, "stroke-linecap": "round" }); + children.push(marks.pointed, marks.shown); + // Behind a link: a halo while it's hovered or in focus (`width`: of its lines). + const halo = (key, p, q, width) => { + if (key !== hovered && key !== focused) return; + children.push(drawing("halo " + key, "line", { x1: p.x, y1: p.y, x2: q.x, y2: q.y, + stroke: "var(--self)", + style: "stroke-opacity: var(" + (key === focused ? "--halo-focused" : "--halo-hovered") + ")", + "stroke-width": width + 8, "stroke-linecap": "round" })); + }; + // Over a link: an invisible, wider line to point at and click. + const hits = []; + // The links' round trips. + const labels = []; + const hit = (key, p, q, width) => { + const line = drawing("hit " + key, "line", { x1: p.x, y1: p.y, x2: q.x, y2: q.y, + stroke: "transparent", "stroke-width": width + 8, cursor: "pointer" }, line => { + // Dragged, it pans, as beside the nodes; a click, it is in focus. + line.addEventListener("pointerdown", event => { event.stopPropagation(); pan(event, () => focus(key)); }); + hoverable(line, key); + }); + hits.push(line); + return line; + }; + for (const [key, [a, b]] of intended) { + if (measured.has(key)) continue; + const p = positions.get(a), q = positions.get(b); + halo(key, p, q, 1); + markLine(key, "M " + p.x + " " + p.y + " L " + q.x + " " + q.y, 1); + children.push(drawing("intended " + key, "line", { x1: p.x, y1: p.y, x2: q.x, y2: q.y, + stroke: "var(--intended)", "stroke-dasharray": "4 4", "stroke-width": 1 })); + tooltip(hit(key, p, q, 1), name(a) + " – " + name(b) + ": intended (member list), not measured"); + } + const ms = rtt => rtt < 10 ? rtt.toFixed(1) : String(Math.round(rtt)); + for (const [key, { a, b, link, secsAgo }] of measured) { + const p = positions.get(a), q = positions.get(b); + if (!p || !q) continue; + // The transport in use thick, in the middle; the others thin, beside it. + const lines = [{ transport: link.transport, rttMs: link.rttMs, used: true }]; + for (const other of link.others || []) lines.push({ ...other, used: false }); + // As wide as the lines side by side. + const width = 3 + 12 * Math.ceil((lines.length - 1) / 2); + halo(key, p, q, width); + // It covers the lines: their descriptions are its tooltip. + const covering = hit(key, p, q, width); + const descriptions = []; + const length = Math.max(Math.hypot(q.x - p.x, q.y - p.y), 1); + const nx = -(q.y - p.y) / length, ny = (q.x - p.x) / length; + // An end that reports no links (any more): what's drawn may be out of date. + const silent = !(a in state.graph.measured) || !(b in state.graph.measured); + // Dormant, or down (no path answers, whatever its connection was): a red underlay + // as wide as the lines, as a dormant node's red outline. + const broken = link.dormant || link.down; + if (broken) { + children.push(drawing("underlay " + key, "line", { x1: p.x, y1: p.y, x2: q.x, y2: q.y, + stroke: "var(--bad)", "stroke-opacity": .5, "stroke-width": width + 4, + "stroke-linecap": "round" })); + } + // The one in use drawn last: on top of the others where they meet, at the nodes. + const drawnLines = []; + lines.forEach((line, i) => { + // 0, then +1, -1, +2, -2 … steps of 6 px to the side, at the middle: from node to + // node, each bowing out that far (a quadratic curve, its control point twice as + // far out); `aside(t)`, how far out at `t` along. + const side = i === 0 ? 0 : (i % 2 ? 1 : -1) * Math.ceil(i / 2) * 6; + const aside = t => 4 * t * (1 - t) * side; + const cx = (p.x + q.x) / 2 + nx * side * 2, cy = (p.y + q.y) / 2 + ny * side * 2; + const attrs = { d: "M " + p.x + " " + p.y + " Q " + cx + " " + cy + " " + q.x + " " + q.y, + fill: "none", stroke: colourOf(line.transport), + // Narrow where dormant or down (over its underlay). + "stroke-width": broken ? (line.used ? 1.5 : 0.8) : line.used ? 3 : 1.2 }; + // Dotted where an end reports nothing, dashed where relayed, solid where direct. + if (silent) { + attrs["stroke-dasharray"] = line.used ? "1 5" : "1 3"; + attrs["stroke-linecap"] = "round"; + } else if (line.used ? link.relayed : line.relayed) { + attrs["stroke-dasharray"] = "7 4"; + } + markLine(key, attrs.d, attrs["stroke-width"]); + const drawnLine = drawing("path " + key + " " + i, "path", attrs); + const description = name(a) + " – " + name(b) + + (line.transport ? " via " + line.transport : "") + (line.used ? " (in use)" : "") + + ((line.used ? link.relayed : line.relayed) ? ", relayed" : "") + (link.dormant ? ", dormant" : "") + + (line.used && link.down ? ", down" : "") + + (silent ? ", one end reports no links" : "") + + (typeof line.rttMs === "number" ? ", " + line.rttMs.toFixed(1) + " ms" : "") + + ", measured " + age(secsAgo + elapsed()) + " ago"; + tooltip(drawnLine, description); + descriptions.push(description); + drawnLines.push(drawnLine); + if (typeof line.rttMs === "number") { + // Near the end that measured it (`a`, whose report is drawn), clear of its + // node and name, and short of the middle: the one in use larger, the others + // closer still, beside theirs. + const along = Math.min(length / 2, Math.max(36, length * .3)); + const t = (line.used ? along : Math.max(24, along * .6)) / length; + const out = line.used ? -4 : Math.sign(side) * 9; + const rtt = drawing("rtt " + key + " " + i, "text", upright({ + x: p.x + (q.x - p.x) * t + nx * (aside(t) + out), + y: p.y + (q.y - p.y) * t + ny * (aside(t) + out) + (line.used ? 0 : 3), + "text-anchor": "middle", "font-size": line.used ? 11 : 9, + fill: line.used ? "var(--fg)" : "var(--muted)" })); + label(rtt, ms(line.rttMs)); + labels.push(rtt); + } + }); + children.push(...drawnLines.reverse()); + tooltip(covering, descriptions.join("\n")); + } + // Above every line, so none crosses them. + children.push(...labels); + // Above the lines and their labels, below the nodes. + children.push(...hits); + for (const id of nodes) { + const p = positions.get(id), member = state.graph.members[id]; + const reported = id in state.graph.measured; + const shown = focused === null ? member.self : id === focused; + // A link in focus: both ends ringed. + // A halo as links have: light under the pointer, stronger in focus. + if (shown || id === hovered) { + children.push(drawing("ring " + id, "circle", { cx: p.x, cy: p.y, r: 13, + fill: "var(--self)", stroke: "none", + style: "fill-opacity: var(" + (shown ? "--halo-focused" : "--halo-hovered") + ")" })); + } + const pressed = element => { + element.addEventListener("pointerdown", event => { event.stopPropagation(); drag(event, id); }); + hoverable(element, id); + }; + // A clock off the swarm's: a dashed orange ring. + if (((state.graph.clocks || {})[id] || {}).off) { + children.push(drawing("clock " + id, "circle", { cx: p.x, cy: p.y, r: 11, fill: "none", + stroke: "var(--warn)", "stroke-width": 1.5, "stroke-dasharray": "3 2" })); + } + // Dormant as this node sees it (`/peers`): a red outline. + const dormant = (state.peers.peers[id] || {}).status === "dormant"; + const radius = member.self ? 8 : 6; + mark(id, "circle", { cx: p.x, cy: p.y, r: radius + 6, fill: "rgb(127 127 127)" }); + const circle = drawing("node " + id, "circle", { cx: p.x, cy: p.y, r: radius, + fill: member.self ? "var(--self)" : reported ? "var(--fg)" : "var(--panel)", + stroke: member.self ? "var(--self)" : dormant ? "var(--bad)" : "var(--fg)", + "stroke-width": dormant ? 2.5 : 1.5 }, pressed); + // Its clock against the swarm's, as the members' reports chain it (the node's). + const clock = ((state.graph.clocks || {})[id]); + tooltip(circle, id + (reported ? "" : " (no links reported)") + + (dormant ? " (dormant)" : "") + + (clock && clock.off ? " (clock " + seconds(clock.offsetMs) + " off the swarm's, ±" + + clock.withinMs + " ms)" : "")); + // Pinned (double-clicked): a dot in the middle. + const pin = pinned.has(id) ? [drawing("pin " + id, "circle", { cx: p.x, cy: p.y, r: 2, + fill: member.self || reported ? "var(--panel)" : "var(--fg)", + "pointer-events": "none" })] : []; + const nameLabel = drawing("name " + id, "text", upright({ x: p.x + 10, y: p.y + 4, + "font-size": 12, fill: "var(--fg)", cursor: "pointer" }), pressed); + label(nameLabel, name(id)); + children.push(circle, ...pin, nameLabel); + } + marks.pointed.replaceChildren(...marked.pointed); + marks.shown.replaceChildren(...marked.shown); + show(document.getElementById("view"), children); +} + +// --- Document: read a level at a time (`/store?shallow=true`) as it is opened, and +// read again where the node says it changed (`document` events) --- +// What is open, by path (as JSON): the first level to begin with. +const documentOpen = new Set([JSON.stringify([])]); +// The open branches read, by path (as JSON): to read again on a change. +const documentBranches = new Map(); +function storeUrl(path, shallow) { + const where = path.length ? "../store/" + path.map(encodeURIComponent).join("/") : "../store"; + return where + (shallow ? "?shallow=true" : ""); +} +// Store paths split at "/": a key with one (or an empty key) can't be read on its own. +const addressable = path => path.every(segment => segment !== "" && !segment.includes("/")); +// Either is within the other. +const related = (a, b) => a.length <= b.length + ? a.every((segment, i) => segment === b[i]) : related(b, a); +function describe(summary) { + switch (summary.type) { + case "map": return "{" + summary.size + (summary.size === 1 ? " key}" : " keys}"); + case "list": return "[" + summary.size + (summary.size === 1 ? " item]" : " items]"); + case "text": return "text, " + summary.size + " characters"; + default: return summary.type + ("size" in summary ? ", " + summary.size + " bytes" : ""); + } +} +async function readStore(path, shallow) { + const response = await fetch(storeUrl(path, shallow), { headers: { Accept: "application/json" } }); + if (!response.ok) throw new Error("HTTP " + response.status); + return response.json(); +} +// An empty map, list or text: shown as it is, nothing to open. +const EMPTY = { map: "{}", list: "[]", text: "\"\"" }; +const EMPTY_VALUE = { map: () => ({}), list: () => [], text: () => "" }; + +// --- Editing: a double click on an entry turns it into a field holding its contents; +// saved with PUT /store/<path> (the whole value, objects too) --- +// A string is edited as text (sent as text/plain), anything else as JSON; "JSON" +// switches. `done(saved, response)` when saved or cancelled. +// `options` for adding rather than replacing: `extra`, a short field before the value +// (`{ label, initial, hint }`: the key, or the index); `check(extra)`, an error text if +// it won't do; `send(body, type, extra)`, the request (PUT to `path` without). +function editor(path, value, done, options = {}) { + const message = el("span", { class: "note" }); + const fail = text => { message.className = "error"; message.textContent = text; }; + const asText = typeof value === "string"; + const text = asText ? value : JSON.stringify(value, null, 2); + const field = el("textarea", { spellcheck: "false", + rows: String(Math.max(2, Math.min(20, text.split("\n").length + 1))) }); + field.value = text; + const json = el("input", { type: "checkbox" }); + json.checked = !asText; + json.addEventListener("change", () => { + if (json.checked) { field.value = JSON.stringify(field.value); return; } + try { + const parsed = JSON.parse(field.value); + if (typeof parsed !== "string") throw new Error("not a string"); + field.value = parsed; + } catch { + json.checked = true; + fail("Only a JSON string can be edited as text."); + } + }); + const extra = options.extra + ? el("input", { type: "text", spellcheck: "false", class: "extra", + placeholder: options.extra.label, title: options.extra.hint || options.extra.label }) + : null; + if (extra) extra.value = options.extra.initial || ""; + const send = options.send || ((body, type) => fetch(storeUrl(path, false), + { method: "PUT", body, headers: { "Content-Type": type } })); + const save = el("button", { type: "button" }, options.extra ? "Add" : "Save"); + const cancel = el("button", { type: "button" }, "Cancel"); + const submit = async () => { + const body = field.value; + const problem = extra && options.check ? options.check(extra.value) : null; + if (problem) { fail(problem); return; } + if (json.checked) { + try { JSON.parse(body); } catch (error) { + fail("Not JSON: " + error.message); return; + } + } + save.disabled = true; + message.className = "note"; + message.textContent = "Saving…"; + try { + const response = await send(body, json.checked ? "application/json" : "text/plain", + extra ? extra.value : null); + if (!response.ok) { + // The node says why (`{"error": …}`), where it does. + let why = response.status === 403 ? "not allowed" : ""; + try { why = (await response.json()).error || why; } catch { /* no body */ } + throw new Error("HTTP " + response.status + (why ? ": " + why : "")); + } + done(true, response); + } catch (error) { + save.disabled = false; + fail("Not saved: " + error.message); + } + }; + save.addEventListener("click", submit); + cancel.addEventListener("click", () => done(false)); + // Editing what's there: it can be deleted too (DELETE /store/<path>), after a second + // click to be sure, not too soon after the first: a double-click doesn't delete. + const remove = options.extra ? null : el("button", { type: "button", class: "delete" }, + "Delete"); + let sure = null, askedAt = 0; + remove?.addEventListener("click", async () => { + if (sure === null) { + remove.textContent = "Delete: sure?"; + askedAt = performance.now(); + sure = setTimeout(() => { sure = null; remove.textContent = "Delete"; }, 3000); + return; + } + if (performance.now() - askedAt < 500) return; + clearTimeout(sure); + remove.disabled = save.disabled = true; + message.className = "note"; + message.textContent = "Deleting…"; + try { + const response = await fetch(storeUrl(path, false), { method: "DELETE" }); + if (!response.ok) { + let why = response.status === 403 ? "not allowed" : ""; + try { why = (await response.json()).error || why; } catch { /* no body */ } + throw new Error("HTTP " + response.status + (why ? ": " + why : "")); + } + done("deleted", response); + } catch (error) { + sure = null; + remove.textContent = "Delete"; + remove.disabled = save.disabled = false; + fail("Not deleted: " + error.message); + } + }); + // Enter in the short field saves; in the area, Ctrl or ⌘ and Enter does. + const keys = (input, line) => input.addEventListener("keydown", event => { + if (event.key === "Escape") { event.preventDefault(); done(false); } + else if (event.key === "Enter" && (line || event.ctrlKey || event.metaKey)) { + event.preventDefault(); submit(); + } + }); + keys(field, false); + if (extra) keys(extra, true); + setTimeout(() => (extra && !extra.value ? extra : field).focus()); + return el("div", { class: "editor" }, ...(extra ? [extra] : []), field, + el("div", {}, save, " ", cancel, " ", ...(remove ? [remove, " "] : []), + el("label", {}, json, " JSON"), " ", + el("span", { class: "note" }, "Ctrl+Enter saves, Esc cancels"), " ", message)); +} + +// Store paths name keys below the root; the root itself isn't written whole. +const editable = path => path.length > 0 && addressable(path); +// Branches whose reading again waited for an edit in them to end, by path (as JSON). +const documentDeferred = new Set(); +// After an edit: what changed is read again (the node's `document` event follows), +// and what was waiting for the edit; `opened`, what it made, shown open. `saved`: +// true, false (cancelled), or "deleted" (then its parent is read again). +function edited(path, saved, opened = path) { + if (saved === "deleted") path = path.slice(0, -1); + else if (saved) documentOpen.add(JSON.stringify(opened)); + release(); + const waiting = [...documentDeferred].map(id => JSON.parse(id)); + documentDeferred.clear(); + if (saved || waiting.length) documentChanged({ paths: saved ? [path, ...waiting] : waiting }); +} +// Whether `element` holds a field being edited: left as it is until it's done. +const editing = element => element.querySelector(".editor") !== null; + +// Adding to a map (a key and its value, PUT /store/<path>/<key>) or a list (an index +// and the value, POST /store/<path>/<index>: the items from there on move up; -1 is the +// end), without writing it whole: a "+" on the row, the field below it. `known()`: a +// map's keys as last read, so an existing one isn't overwritten. +function addButton(kind, path, known, place) { + const map = kind === "map"; + const button = el("button", { type: "button", class: "add", + title: map ? "Add a key" : "Insert an item" }, "+"); + button.addEventListener("click", event => { + // Not a click on the row: it would open or close it. + event.preventDefault(); + event.stopPropagation(); + const area = place(); + if (editing(area)) return; + const form = editor(path, "", async (saved, response) => { + form.remove(); + let added = path; + if (saved) { + if (map) added = [...path, form.dataset.extra]; + else { + try { added = [...path, String((await response.json()).index)]; } catch { /* path */ } + } + } + edited(path, saved, added); + }, { + extra: map + ? { label: "key", initial: "" } + : { label: "index", initial: "-1", hint: "where: 0 is the first; -1 the end" }, + check: extra => { + form.dataset.extra = extra; + return map + ? (extra === "" ? "A key, please." + : extra.includes("/") ? "A key with \"/\" can't be named in a path." + : known().includes(extra) ? "\"" + extra + "\" exists already: edit it instead." + : null) + : (/^(-1|\d+)$/.test(extra.trim()) ? null : "The index: 0 or more, or -1 for the end."); + }, + send: (body, type, extra) => fetch(storeUrl([...path, map ? extra : extra.trim()], false), + { method: map ? "PUT" : "POST", body, headers: { "Content-Type": type } }), + }); + area.prepend(form); + }); + return button; +} + +function editableLeaf(key, shown, value, path) { + const leaf = el("div", { class: "leaf" }, el("span", { class: "key" }, key + ": "), shown); + if (!editable(path)) return leaf; + leaf.title = "Double-click to edit"; + leaf.addEventListener("dblclick", event => { + if (editing(leaf)) return; + event.preventDefault(); + window.getSelection().removeAllRanges(); + const before = [...leaf.childNodes]; + leaf.replaceChildren(before[0], editor(path, value, saved => { + leaf.replaceChildren(...before); + edited(path, saved); + })); + }); + return leaf; +} + +// An entry that changed in the document breathes once, as the map's labels do: its +// row glows and fades. +function glow(entry) { + if (calm.matches || !entry.animate) return; + const row = entry.tagName === "DETAILS" ? entry.querySelector(":scope > summary") + : entry.classList.contains("leaf") ? entry : entry.firstElementChild || entry; + const colour = getComputedStyle(document.documentElement).getPropertyValue("--self").trim(); + row.animate([ + { backgroundColor: "transparent" }, + { backgroundColor: "color-mix(in srgb, " + colour + " 30%, transparent)", offset: .3 }, + { backgroundColor: "transparent" }, + ], { duration: 900, easing: "ease-in-out" }); +} + +// One entry: a short scalar as it is; anything else a branch read when opened. A +// branch's entries are kept across reads, so what's open inside stays open. +function documentEntry(key, summary, path) { + if (summary.size === 0 && summary.type in EMPTY) { + const leaf = editableLeaf(key, EMPTY[summary.type], EMPTY_VALUE[summary.type](), path); + // An empty map or list takes its first entry here: "+" on its row, the field below. + if ((summary.type === "map" || summary.type === "list") && editable(path)) { + const below = el("div", { class: "nested" }); + leaf.append(" ", addButton(summary.type, path, () => [], () => below)); + return el("div", {}, leaf, below); + } + return leaf; + } + if ("value" in summary) { + return editableLeaf(key, JSON.stringify(summary.value), summary.value, path); + } + const details = el("details", {}); + const described = document.createTextNode(describe(summary)); + const head = el("summary", {}, el("span", { class: "key" }, key), " ", described); + details.append(head); + // A double click edits the whole of it: read, then shown in a field. + if (editable(path)) { + head.title = "Double-click to edit"; + head.addEventListener("mousedown", event => { if (event.detail > 1) event.preventDefault(); }); + head.addEventListener("dblclick", async event => { + event.preventDefault(); + if (details.querySelector(":scope > .editor")) return; + let value; + try { value = await readStore(path, false); } catch (error) { + described.textContent = describe(summary) + " (not read: " + error.message + ")"; + return; + } + details.open = true; + details.classList.add("editing"); + head.after(editor(path, value, saved => { + details.querySelector(":scope > .editor").remove(); + details.classList.remove("editing"); + edited(path, saved); + })); + }); + } + // A newer summary from the parent: a branch of the same kind stays as it is. + details.updated = newer => { + summary = newer; + described.textContent = describe(newer); + }; + details.sameKind = newer => !("value" in newer) && newer.type === summary.type; + if (!addressable(path)) { + details.append(el("div", { class: "note" }, "Its path can't be read on its own (a key with \"/\").")); + return details; + } + const id = JSON.stringify(path); + const children = new Map(); + const browsed = summary.type === "map" || summary.type === "list"; + // Adding: on the row, the field just below it (the branch opened for it). + if (browsed && (editable(path) || path.length === 0)) { + head.append(" ", addButton(summary.type, path, () => [...children.keys()], () => { + details.open = true; + return { prepend: form => head.after(form), querySelector: q => details.querySelector(":scope > " + q) }; + })); + } + let read = false; + // Read once at least: what's new or changed from then on breathes (`glow`). + let filled = false; + const fill = async () => { + // Being edited as a whole: read again once that's done. + if (details.classList.contains("editing")) { documentDeferred.add(id); return; } + try { + if (browsed) { + const shallow = await readStore(path, true); + details.updated(shallow); + const entries = shallow.entries || {}; + // A list's items by index (the node sends them sorted as text). + const keys = Object.keys(entries); + if (summary.type === "list") keys.sort((a, b) => Number(a) - Number(b)); + for (const [k, child] of children) { + if (k in entries) continue; + // Being edited: kept until that's done. + if (editing(child.element)) { documentDeferred.add(id); continue; } + child.element.remove(); + children.delete(k); + } + for (const k of keys) { + const newer = entries[k]; + const known = children.get(k); + if (known && JSON.stringify(known.summary) === JSON.stringify(newer)) continue; + if (known && editing(known.element)) { documentDeferred.add(id); continue; } + if (known && known.element.sameKind && known.element.sameKind(newer)) { + known.element.updated(newer); + known.summary = newer; + glow(known.element); + continue; + } + const element = documentEntry(k, newer, [...path, k]); + if (known) known.element.replaceWith(element); + children.set(k, { element, summary: newer }); + if (filled) glow(element); + } + // In order; moving keeps their state. + details.querySelectorAll(":scope > .note").forEach(note => note.remove()); + if (!children.size) details.append(el("div", { class: "note" }, "empty")); + for (const k of keys) if (children.has(k)) details.append(children.get(k).element); + } else { + const value = await readStore(path, false); + const shown = el("pre", {}, typeof value === "string" ? value : JSON.stringify(value, null, 2)); + const before = details.querySelector(":scope > pre"); + if (before) before.replaceWith(shown); else details.append(shown); + if (before && before.textContent !== shown.textContent) glow(shown); + } + filled = true; + } catch (error) { + details.querySelectorAll(":scope > .note").forEach(note => note.remove()); + details.append(el("div", { class: "note error" }, "Not read: " + error.message)); + } + }; + const open = () => { + documentOpen.add(id); + documentBranches.set(id, { path, element: details, refresh: fill }); + if (!read) { read = true; fill(); } + }; + details.addEventListener("toggle", () => { + if (details.open) open(); + else { documentOpen.delete(id); documentBranches.delete(id); } + }); + if (documentOpen.has(id)) { details.open = true; open(); } + return details; +} +// The node says what changed (paths), or that anything may have (`all`). +// Changes waiting while text in the document box is selected (`busy`): merged, applied +// after. A field being edited holds only what it is in (`editing`, `documentDeferred`). +let documentPending = null; +function documentChanged(change) { + if (busy(document.getElementById("document-box"))) { + documentPending = documentPending === null ? change + : documentPending.all || change.all ? { all: true } + : { paths: [...documentPending.paths, ...change.paths] }; + return; + } + for (const [id, branch] of documentBranches) { + if (!branch.element.isConnected) { documentBranches.delete(id); continue; } + if (change.all || change.paths.some(path => related(path, branch.path))) branch.refresh(); + } +} +document.getElementById("document").replaceChildren( + documentEntry("store", { type: "map", size: "…" }, [])); + +// A box is busy while text in it is selected: drawing it anew would lose the selection, +// so it waits (`held`) until it's gone. +// Collapsed, or not shown at all (the right column dragged off): drawn when opened +// (`release`). +function hidden(box) { + return box.classList.contains("collapsed") || box.getClientRects().length === 0; +} +function busy(box) { + if (hidden(box)) return true; + const selection = window.getSelection(); + if (!selection || selection.isCollapsed || !selection.rangeCount) return false; + return box.contains(selection.getRangeAt(0).commonAncestorContainer); +} +let held = false; +function render(ages = false) { + if (!state) return; + document.getElementById("node").textContent = + (state.node.alias ? state.node.alias + " · " : "") + state.node.id; + const skew = state.stats.clock; + document.getElementById("clock-warning").textContent = skew && skew.off + ? "clock " + seconds(skew.offsetMs) + " off its peers'" : ""; + showLinksToggle(); + held = false; + const boxes = [["peers-box", renderPeers], ["figures-box", renderStats]]; + // Its ages are in tooltips only: drawn on events, not every second. + if (!ages) boxes.push(["map", renderGraph]); + for (const [box, draw] of boxes) { + if (busy(document.getElementById(box))) held = true; else draw(); + } +} +// What waited for a box to be free: drawn, or applied, now. +function release() { + if (held) render(); + if (documentPending !== null && !busy(document.getElementById("document-box"))) { + const change = documentPending; + documentPending = null; + documentChanged(change); + } +} +document.addEventListener("selectionchange", release); + +// The legend, hidden and shown again; the map takes the room. +const legendToggle = document.getElementById("legend-toggle"); +legendToggle.addEventListener("click", () => { + const legends = document.getElementById("legends"); + legends.hidden = !legends.hidden; + legendToggle.textContent = legends.hidden ? "show legend" : "hide legend"; + if (state) renderGraph(); +}); +// The members' links subscribed to (`live`: each tells of a change as it happens), or +// only asked for every 30 s; the node's choice, for every page (`POST +// /admin/dashboard/links`), shown as the state has it. +const linksToggle = document.getElementById("links-toggle"); +function showLinksToggle() { + const live = !state || state.graph.live !== false; + linksToggle.textContent = live ? "links: live" : "links: every 30 s"; + linksToggle.title = live + ? "The members tell of their links' changes as they happen. Click to only ask them every 30 s (fewer messages; for every page on this node)" + : "The members are asked for their links every 30 s. Click to have them tell of changes as they happen (for every page on this node)"; +} +linksToggle.addEventListener("click", () => { + const live = !state || state.graph.live !== false; + fetch("dashboard/links?live=" + !live, { method: "POST" }).catch(() => {}); +}); +// Links as long as their round trips, and back. +const rttToggle = document.getElementById("rtt-toggle"); +function showRttToggle() { + rttToggle.textContent = byRtt ? "layout: dynamic" : "layout: static"; + rttToggle.title = "Click for " + (byRtt ? "a static layout" + : "a dynamic one: links as long as their round trips"); +} +showRttToggle(); +rttToggle.addEventListener("click", () => { + byRtt = !byRtt; + try { localStorage.setItem("continuum-by-rtt", byRtt ? "1" : "0"); } catch (_) {} + showRttToggle(); + // To the lengths afresh, when resized again. + lastLengths = ""; + if (state) renderGraph(); +}); + +// --- Pan, zoom and turn: the map's group moved, scaled and turned (`turn`, in +// radians). Lines, labels and nodes scale with it, as on a map; labels stay upright +// (`upright`). --- +const graph = document.getElementById("graph"); +const view = { x: 0, y: 0, k: 1, turn: 0 }; +const degrees = radians => radians * 180 / Math.PI; +function moveView() { + document.getElementById("view").setAttribute("transform", + "translate(" + view.x + " " + view.y + ") rotate(" + degrees(view.turn) + ") scale(" + + view.k + ")"); +} +// `[x, y]` turned by `angle`. +function turned(x, y, angle) { + const c = Math.cos(angle), s = Math.sin(angle); + return [x * c - y * s, x * s + y * c]; +} +// Where the pointer is, in the map's own units (its `viewBox` is its size). +function pointer(event) { + const box = graph.getBoundingClientRect(); + return { x: event.clientX - box.left, y: event.clientY - box.top }; +} +// How far it zooms: from a quarter to eight times. +const zoomed = k => Math.min(8, Math.max(.25, k)); +// The wheel zooms around the pointer. +graph.addEventListener("wheel", event => { + event.preventDefault(); + const at = pointer(event); + const k = zoomed(view.k * Math.exp(-event.deltaY * .002)); + view.x = at.x - (at.x - view.x) * k / view.k; + view.y = at.y - (at.y - view.y) * k / view.k; + view.k = k; + moveView(); +}, { passive: false }); +// A press on the map: dragged (more than a few pixels), it `move`s by as much as the +// pointer did; let go where pressed, it's a `click`. +let press = null; +function pressing(event, move, click) { + press = { from: pointer(event), moved: false, move, click }; + graph.setPointerCapture(event.pointerId); +} +// Fingers on the map, by pointer. Two pinch: the map zooms around the point between +// them, moves with it, and turns as they do. A press (`pressing`) is given up when the +// second comes. +const touching = new Map(); +let pinch = null; +const between = (a, b) => ({ x: (a.x + b.x) / 2, y: (a.y + b.y) / 2 }); +// From the first two fingers as they are now. +function pinchFrom() { + const [one, two] = touching.values(); + pinch = { view: { ...view }, at: between(one, two), + distance: Math.max(Math.hypot(two.x - one.x, two.y - one.y), 1), + angle: Math.atan2(two.y - one.y, two.x - one.x) }; +} +function pinched() { + const [one, two] = touching.values(); + const at = between(one, two); + const k = zoomed(pinch.view.k * Math.hypot(two.x - one.x, two.y - one.y) / pinch.distance); + const turn = pinch.view.turn + Math.atan2(two.y - one.y, two.x - one.x) - pinch.angle; + // What was between the fingers (on the map, unturned and unscaled) stays between them. + const [x, y] = turned(pinch.at.x - pinch.view.x, pinch.at.y - pinch.view.y, -pinch.view.turn); + const [dx, dy] = turned(x * k / pinch.view.k, y * k / pinch.view.k, turn); + Object.assign(view, { x: at.x - dx, y: at.y - dy, k, turn }); + moveView(); + // Its labels upright again. + if (state) renderGraph(); +} +// Before the map's own handlers (a node's included): a second finger takes over. +graph.addEventListener("pointerdown", event => { + if (event.pointerType !== "touch") return; + touching.set(event.pointerId, pointer(event)); + graph.setPointerCapture(event.pointerId); + if (touching.size < 2) return; + press = null; + graph.classList.remove("dragging"); + pinchFrom(); + event.stopImmediatePropagation(); +}, true); +// A finger lifted: the others pinch on from where they are, or (one left) do nothing +// until lifted too. +function untouched(event) { + if (!touching.delete(event.pointerId)) return; + if (touching.size >= 2) pinchFrom(); else pinch = null; +} +graph.addEventListener("pointermove", event => { + if (touching.has(event.pointerId)) touching.set(event.pointerId, pointer(event)); + if (pinch) { pinched(); return; } + if (!press) return; + const at = pointer(event); + const dx = at.x - press.from.x, dy = at.y - press.from.y; + if (!press.moved && Math.hypot(dx, dy) < 4) return; + press.moved = true; + graph.classList.add("dragging"); + press.move(dx, dy); +}); +graph.addEventListener("pointerup", event => { + untouched(event); + const click = press && !press.moved ? press.click : null; + press = null; + graph.classList.remove("dragging"); + dropped(); + if (click) click(); +}); +graph.addEventListener("pointercancel", event => { + untouched(event); + press = null; + graph.classList.remove("dragging"); + dropped(); +}); +// The map dragged: it pans; a click, `click`. +function pan(event, click) { + const from = { ...view }; + pressing(event, (dx, dy) => { + view.x = from.x + dx; + view.y = from.y + dy; + moveView(); + }, click); +} +// Beside the nodes: dragged, it pans; a click, back to this node. +graph.addEventListener("pointerdown", event => { + if (event.target !== graph) return; + pan(event, () => focus(null)); +}); +// Not the browser's drag and drop of the picture. +graph.addEventListener("dragstart", event => event.preventDefault()); +// A node: dragged, it moves, held where the pointer is (`grabbed`) until let go; a click, +// it is in focus; a second one soon after pins it where it is (`pinned`: the layout +// leaves it there), or lets it go if pinned (`unpin`), and so does a second press soon +// after that drags it (pinned where it's dropped). Told apart here: the map holds the +// pointer while pressed (`pressing`), so the browser's clicks and double-clicks go to +// the map, not the node. +let lastNodeClick = { id: null, at: 0 }; +const DOUBLE_CLICK_MS = 400; +let grabbed = null; +// Of the press under way on a node: whether it's the second of a double-click, and +// whether it moved the node. +let secondPress = false, dragMoved = false; +function drag(event, id) { + const p = positions.get(id), from = { x: p.x, y: p.y }; + grabbed = id; + secondPress = lastNodeClick.id === id && performance.now() - lastNodeClick.at < DOUBLE_CLICK_MS; + dragMoved = false; + pressing(event, (dx, dy) => { + dragMoved = true; + // The pointer's move, on the map (unturned and unscaled). + const [x, y] = turned(dx, dy, -view.turn); + p.x = from.x + x / view.k; + p.y = from.y + y / view.k; + renderGraph(); + // Dynamic: the others follow it at once. + if (byRtt) settle(); + }, () => { + const now = performance.now(); + if (lastNodeClick.id === id && now - lastNodeClick.at < DOUBLE_CLICK_MS) { + lastNodeClick = { id: null, at: now }; + if (pinned.has(id)) unpin(id); else pin(id); + } else { + lastNodeClick = { id, at: now }; + focus(id); + } + }); +} +// A dragged node let go of: dynamic, it moves to its place among the others again; +// dragged by a double-click's second press, its pin toggled. +function dropped() { + if (grabbed === null) return; + const id = grabbed; + grabbed = null; + if (dragMoved && secondPress) { + lastNodeClick = { id: null, at: 0 }; + if (pinned.has(id)) unpin(id); else pin(id); + return; + } + if (byRtt) settle(); +} +// A node pinned where it is: the layout leaves it there. +function pin(id) { + pinned.add(id); + renderGraph(); +} +// A node let go: it moves to its place among the others, smoothly (`settle`); on a +// static map, only it moves. +function unpin(id) { + if (!pinned.delete(id)) return; + freed.add(id); + renderGraph(); + settle(); +} +// Beside the nodes, or off the map: nothing hovered (the element left may be gone). +graph.addEventListener("mousemove", event => { if (event.target === graph) hover(null); }); +graph.addEventListener("mouseleave", () => hover(null)); + +// --- Messages on the map: this node's traffic with each peer (`/peers`, `traffic`), +// as it grew since the last update, drawn as dots running along the link the way the +// messages went. Counts come with each update (every 2 s), so the dots are spread +// over the next 2 s, a few at most per link and way: a picture of the traffic, not a +// record of it. Kept above everything else on the map (`flows`, after what `show` +// places). --- +const FLOW_MS = 700, FLOW_SPREAD_MS = 2000, FLOWS_AT_MOST = 6; +// Shown or not (`flows-toggle`, kept in this browser; shown at first). +let flowsShown = true; +try { flowsShown = localStorage.getItem("continuum-flows") !== "0"; } catch (_) {} +const flowsToggle = document.getElementById("flows-toggle"); +function showFlowsToggle() { + flowsToggle.textContent = flowsShown ? "messages: on" : "messages: off"; + flowsToggle.title = "Click to " + (flowsShown ? "hide" : "show") + + " the messages running along this node's links"; +} +showFlowsToggle(); +flowsToggle.addEventListener("click", () => { + flowsShown = !flowsShown; + try { localStorage.setItem("continuum-flows", flowsShown ? "1" : "0"); } catch (_) {} + showFlowsToggle(); + showMessagesLegend(); + if (!flowsShown) { + flows.length = 0; + document.getElementById("flows").replaceChildren(); + } +}); +let lastTraffic = null; +const flows = []; +let flowFrame = null; +// Messages and bytes by kind, of a peer's `traffic.sent` or `.received`. +function messages(byKind) { + return Object.fromEntries(Object.entries(byKind || {}) + .map(([kind, counter]) => [kind, { messages: counter.messages, bytes: counter.bytes }])); +} +// The colour of a kind of message (`traffic`'s kinds), on the map and in the console; +// the replies as their requests. +const KIND_COLOURS = [["sync", "#ea580c"], ["blob", "#a16207"], ["hello", "#16a34a"], + ["status", "#e11d48"], ["identity", "#0d9488"], ["paths", "var(--muted)"], + ["links", "#db2777"]]; +const kindColour = kind => (KIND_COLOURS.find(([name]) => kind === name || + kind === name + "Reply") || [null, "var(--fg)"])[1]; +// How large the dots are drawn, times `dotRadius` (the slider after their legend, kept +// in this browser). +let dotScale = 2; +try { dotScale = Number(localStorage.getItem("continuum-dot-scale")) || 2; } catch (_) {} +dotScale = Math.min(4, Math.max(1.25, dotScale)); +let dotSlider = null; +function showMessagesLegend() { + const legend = document.getElementById("messages-legend"); + legend.classList.toggle("off", !flowsShown); + if (dotSlider === null) { + const slider = dotSlider = el("input", { type: "range", min: "1.25", max: "4", step: ".25", + title: "How large the dots are" }); + slider.value = String(dotScale); + slider.addEventListener("input", () => { + dotScale = Number(slider.value); + try { localStorage.setItem("continuum-dot-scale", String(dotScale)); } catch (_) {} + }); + // Before the toggle, which ends the line. + legend.prepend(...KIND_COLOURS.map(([kind, colour]) => + el("span", { class: "message", style: "--colour: " + colour }, + kind === "paths" ? "paths (pings)" : kind)), + el("label", { class: "dot-size" }, "size", slider)); + } + dotSlider.disabled = !flowsShown; +} +showMessagesLegend(); +// A dot's radius for a message of `bytes`: grows with the size's logarithm, from 1.5 +// (a ping) to 8. +const dotRadius = bytes => Math.min(8, Math.max(1.5, 1.5 + 1.1 * Math.log10(1 + bytes / 32))); +// The dots for messages that grew by `kinds` (kind: messages, bytes): `FLOWS_AT_MOST` +// at most, shared out by kind (one each at least, the busiest first), each as large as +// its kind's messages on average; interleaved. +function dotsFor(kinds, total) { + const byKind = Object.entries(kinds).sort(([, a], [, b]) => b.messages - a.messages) + .map(([kind, grown]) => ({ kind, bytes: grown.bytes / grown.messages, + count: Math.min(grown.messages, Math.max(1, Math.round(FLOWS_AT_MOST * grown.messages / total))) })); + const dots = []; + for (let round = 0; dots.length < FLOWS_AT_MOST && byKind.some(k => k.count > round); round++) { + for (const kind of byKind) { + if (kind.count > round && dots.length < FLOWS_AT_MOST) dots.push(kind); + } + } + return dots; +} +// The messages exchanged with each peer since the last update, on the map (`flows`, if +// shown). +function trafficArrived() { + if (!state) return; + const self = state.node.id, now = performance.now(); + const dots = flowsShown && !calm.matches && !document.hidden && + !document.getElementById("map").classList.contains("collapsed"); + const current = new Map(); + for (const [id, peer] of Object.entries(state.peers.peers)) { + if (peer.traffic) { + current.set(id, { sent: messages(peer.traffic.sent), + received: messages(peer.traffic.received) }); + } + } + if (lastTraffic) { + for (const [id, counts] of current) { + const before = lastTraffic.get(id); + if (!before) continue; + for (const [way, from, to] of [["sent", self, id], ["received", id, self]]) { + const kinds = {}; + let total = 0; + for (const [kind, count] of Object.entries(counts[way])) { + const was = before[way][kind] || { messages: 0, bytes: 0 }; + const grown = Math.max(0, count.messages - was.messages); + if (grown > 0) { + kinds[kind] = { messages: grown, bytes: Math.max(0, count.bytes - was.bytes) }; + total += grown; + } + } + if (total === 0) continue; + if (!dots) continue; + const shown = dotsFor(kinds, total); + shown.forEach((dot, i) => { + flows.push({ from, to, start: now + i * FLOW_SPREAD_MS / shown.length, + kind: dot.kind, r: dotRadius(dot.bytes) }); + }); + } + } + } + lastTraffic = current; + if (flows.length > 0 && flowFrame === null) flowFrame = requestAnimationFrame(drawFlows); +} + +// --- Console: the messages this node exchanged with its peers, one by one, as the +// node logs them (`messages` events, `traffic::MessageLog`), the latest at the bottom, +// `CONSOLE_LINES` at most. Scrolled along while at the bottom. A click on one shows it +// in the figures box (`shownMessage`). --- +const CONSOLE_LINES = 500; +// A time of day on the 24-hour clock, whatever the browser's locale: 18:04:05.123. +const clockTime = date => [date.getHours(), date.getMinutes(), date.getSeconds()] + .map(part => String(part).padStart(2, "0")).join(":") + "." + + String(date.getMilliseconds()).padStart(3, "0"); +const size = bytes => bytes < 1024 ? bytes + " B" : bytes < 1048576 + ? (bytes / 1024).toFixed(1) + " KiB" : (bytes / 1048576).toFixed(1) + " MiB"; +// The messages, kept while the box is collapsed and drawn when it's opened +// (`showConsole`). +const consoleMessages = []; +// The message shown in the figures box instead of what is in focus; null: none. +let shownMessage = null; +const WAYS = { sent: "→", received: "←", refused: "↛" }; +// The exchange marked in the console (a message's `related`): the one of the message +// pointed at, else of the one shown; null: none. +let relatedMarked = null; +// The message pointed at in the console, if any: its nodes and link are marked on the +// map (`renderGraph`), else the shown one's. +let pointedMessage = null; +function pointAt(message) { + pointedMessage = message; + markRelated(message ? message.related : shownRelated()); + if (state) renderGraph(); +} +function markRelated(key) { + relatedMarked = key || null; + for (const line of document.getElementById("console").children) { + line.classList.toggle("related", relatedMarked !== null && line.dataset.related === relatedMarked); + } +} +const shownRelated = () => shownMessage && shownMessage.related; +// The peer whose messages `key` (a node, or a link's pair key) stands for: the node, +// or the other end of one of this node's links; null: none (this node itself, whose +// every message it would be, or a link between others). +function peerOf(key) { + if (key === null || !state) return null; + const self = state.node.id; + if (!isPair(key)) return key === self ? null : key; + const [a, b] = key.split(" "); + return a === self ? b : b === self ? a : null; +} +// A console line's tint for the node or link pointed at on the map, or in focus. +function peerClasses(peer) { + return (peer === peerOf(focused) ? " peer-focused" : "") + + (peer === peerOf(hovered) ? " peer-hovered" : ""); +} +function markPeerMessages() { + const focusedPeer = peerOf(focused), hoveredPeer = peerOf(hovered); + for (const line of document.getElementById("console").children) { + line.classList.toggle("peer-focused", focusedPeer !== null && line.dataset.peer === focusedPeer); + line.classList.toggle("peer-hovered", hoveredPeer !== null && line.dataset.peer === hoveredPeer); + } +} +// A line: when, which way, with whom, the type (in its kind's colour) and the size. +function consoleLine(message) { + const line = el("div", { "data-seq": String(message.seq), + "data-related": message.related || "", + "data-peer": message.peer, + class: (shownMessage && shownMessage.seq === message.seq ? "focused" : "") + + (relatedMarked !== null && message.related === relatedMarked ? " related" : "") + + peerClasses(message.peer), + title: message.way === "refused" ? "refused: the transport wouldn't send it" : "" }, + el("span", { class: "when" }, clockTime(new Date(message.at)) + " "), + WAYS[message.way] + " " + name(message.peer) + " ", + el("span", { style: "color: " + kindColour(message.counted) }, message.type), + " " + size(message.bytes)); + // On press, as the peers' rows: lines come and go under the pointer. + line.addEventListener("pointerdown", event => { + if (event.button !== 0) return; + showMessage(shownMessage && shownMessage.seq === message.seq ? null : message); + }); + // The messages of its exchange marked while pointed at; then the shown one's again. + line.addEventListener("pointerenter", () => pointAt(message)); + line.addEventListener("pointerleave", () => pointAt(null)); + return line; +} +function showMessage(message) { + shownMessage = message; + const log = document.getElementById("console"); + for (const line of log.children) { + line.classList.toggle("focused", message !== null && line.dataset.seq === String(message.seq)); + } + markRelated(message !== null ? message.related : null); + render(); +} +function messagesArrived(messages) { + consoleMessages.push(...messages); + consoleMessages.splice(0, Math.max(0, consoleMessages.length - CONSOLE_LINES)); + if (hidden(document.getElementById("console-box"))) return; + const log = document.getElementById("console"); + const scroll = document.getElementById("console-scroll"); + const following = scroll.scrollTop + scroll.clientHeight >= scroll.scrollHeight - 4; + log.append(...messages.slice(-CONSOLE_LINES).map(consoleLine)); + while (log.childElementCount > CONSOLE_LINES) log.firstElementChild.remove(); + if (following) scroll.scrollTop = scroll.scrollHeight; +} +function showConsole() { + document.getElementById("console").replaceChildren(...consoleMessages.map(consoleLine)); + const scroll = document.getElementById("console-scroll"); + scroll.scrollTop = scroll.scrollHeight; +} +function drawFlows() { + flowFrame = null; + const layer = document.getElementById("flows"); + const now = performance.now(); + for (let i = flows.length - 1; i >= 0; i--) { + if (now > flows[i].start + FLOW_MS) flows.splice(i, 1); + } + const dots = []; + for (const flow of flows) { + const p = positions.get(flow.from), q = positions.get(flow.to); + if (now < flow.start || !p || !q) continue; + const t = (now - flow.start) / FLOW_MS; + dots.push({ x: p.x + (q.x - p.x) * t, y: p.y + (q.y - p.y) * t, r: flow.r, kind: flow.kind, + opacity: Math.sin(Math.PI * t) }); + } + while (layer.children.length < dots.length) layer.append(svg("circle")); + while (layer.children.length > dots.length) layer.lastChild.remove(); + dots.forEach((dot, i) => { + const circle = layer.children[i]; + circle.setAttribute("cx", dot.x); + circle.setAttribute("cy", dot.y); + circle.setAttribute("r", (dot.r * dotScale).toFixed(1)); + circle.setAttribute("fill", kindColour(dot.kind)); + circle.setAttribute("opacity", dot.opacity.toFixed(2)); + }); + if (flows.length > 0) flowFrame = requestAnimationFrame(drawFlows); +} + +// --- Events --- +const connection = document.getElementById("connection"); +const dot = document.getElementById("dot"); +// One breath per event from the node. +function pulse() { + dot.classList.remove("pulse"); + void dot.offsetWidth; // restarts the animation + dot.classList.add("pulse"); +} +function status(text, live) { + document.getElementById("connection-text").textContent = text; + connection.className = live ? "live" : "down"; + document.body.classList.toggle("stale", !live && state !== null); +} +// While not live, what is shown stays as it was last (ages don't count on). +let live = false; +let events = null; +// The next attempt, while one is waited for: a connect now replaces it. +let retry = null; +function connect() { + clearTimeout(retry); + retry = null; + events = new EventSource("dashboard/events"); + events.addEventListener("snapshot", event => { + const snapshot = JSON.parse(event.data); + if (snapshot.page !== PAGE_VERSION) { + events.close(); + status("new version; reloading…", false); + location.reload(); + return; + } + // Changes while not connected went unseen: what is open is read again. + if (state !== null) documentChanged({ all: true }); + state = snapshot; stateAt = Date.now(); live = true; + status("live", true); pulse(); render(); trafficArrived(); + }); + events.addEventListener("patch", event => { + if (!state) return; + state = mergePatch(state, JSON.parse(event.data)); stateAt = Date.now(); pulse(); render(); + trafficArrived(); + }); + events.addEventListener("document", event => { pulse(); documentChanged(JSON.parse(event.data)); }); + events.addEventListener("messages", event => messagesArrived(JSON.parse(event.data))); + // The node stops: it says so, and ends the stream. Back once it is. + events.addEventListener("offline", () => { + events.close(); live = false; + status("node stopped; reconnecting…", false); + retry = setTimeout(connect, 3000); + }); + events.addEventListener("error", () => { + // Each attempt that failed: a breath too, so it shows it keeps trying. + pulse(); + live = false; + if (events.readyState === EventSource.CLOSED) { + // Given up (an HTTP error, e.g. the node is up but not serving the page yet). + status("disconnected; retrying…", false); + retry = setTimeout(connect, 5000); + } else { + // EventSource reconnects by itself; the node sends a snapshot first again. + status("reconnecting…", false); + } + }); +} +connect(); +// A page not shown (another tab, a minimised window) is paused: hidden for a while, it +// closes its stream, so the node stops building updates and asking members for their +// links for it; shown again, it reconnects. Briefly hidden, nothing changes. +const PAUSE_AFTER = 30000; +let paused = false; +let pausing = null; +document.addEventListener("visibilitychange", () => { + if (document.hidden) { + pausing = setTimeout(() => { + pausing = null; + paused = true; + clearTimeout(retry); + retry = null; + if (events) events.close(); + live = false; + status("paused while hidden", false); + }, PAUSE_AFTER); + } else { + clearTimeout(pausing); + pausing = null; + if (paused) { + paused = false; + status("reconnecting…", false); + connect(); + } + } +}); +// A click on the status: live, everything now (a tick on the node, the members asked +// again, what's open of the document read again); otherwise, reconnecting now. +connection.title = "Click to refresh now"; +connection.style.cursor = "pointer"; +connection.addEventListener("click", () => { + pulse(); + if (live) { + fetch("dashboard/refresh", { method: "POST" }).catch(() => {}); + documentChanged({ all: true }); + } else { + if (events) events.close(); + status("reconnecting…", false); + connect(); + } +}); +// Ages count on between events; nothing else changes without one. +setInterval(() => { if (live) render(true); }, 1000); +window.addEventListener("resize", () => { layoutAgain(); render(); }); + +// --- Sizes: the space between boxes dragged to resize them, double-clicked to reset; +// kept in this browser --- +const main = document.querySelector("main"); +// The map's share of the width (1: the right column dragged off to the side, hidden); +// the peers' height in px; the console's share of what the peers leave, and the +// figures' share of the rest. Null: as laid out by default. +let sizes = { columns: null, peers: null, split: null, console: null }; +try { sizes = { ...sizes, ...JSON.parse(localStorage.getItem("continuum-sizes") || "{}") }; } +catch (_) {} +function applySizes() { + const style = main.style; + const shut = id => document.getElementById(id).classList.contains("collapsed"); + const right = ["peers-box", "figures-box", "document-box", "console-box"]; + const wasHidden = main.classList.contains("right-hidden"); + main.classList.toggle("right-hidden", sizes.columns === 1); + main.classList.toggle("right-shut", sizes.columns !== 1 && right.every(shut)); + // What's collapsed gives its room to what isn't: the map collapsed, the right column + // takes the width but the map's title. The right column all collapsed keeps its + // width, resized as ever (dragged to the side, it goes). + const mapShut = shut("map"), rightShut = right.every(shut); + if (sizes.columns === 1) style.setProperty("--columns", "minmax(0, 1fr) 0"); + else if (mapShut && !rightShut) style.setProperty("--columns", "max-content minmax(0, 1fr)"); + else if (sizes.columns === null) style.removeProperty("--columns"); + else style.setProperty("--columns", "minmax(0, " + sizes.columns + "fr) minmax(0, " + + (1 - sizes.columns) + "fr)"); + // Back: what waited is drawn. + if (wasHidden && sizes.columns !== 1) { + if (!hidden(document.getElementById("console-box"))) showConsole(); + release(); + } + // A collapsed box's row as tall as its title; the open ones share the rest, as they + // did, and the peers take it all when they're the only one open. Their fractions + // add up to 1: fewer would leave room unused (CSS grid gives fractions adding up to + // less than 1 only that part of the room). + const split = sizes.split === null ? .5 : sizes.split; + const consoleShare = sizes.console === null ? .2 : sizes.console; + const rest = 1 - consoleShare; + const fractions = { "figures-box": split * rest, "document-box": (1 - split) * rest, + "console-box": consoleShare }; + const openShare = Object.keys(fractions).filter(id => !shut(id)) + .reduce((sum, id) => sum + fractions[id], 0); + const share = id => shut(id) ? "auto" : "minmax(0, " + fractions[id] / openShare + "fr)"; + const peersOnly = !shut("peers-box") && right.slice(1).every(shut); + const peers = shut("peers-box") ? "auto" : peersOnly ? "minmax(0, 1fr)" + : sizes.peers === null ? "auto" : sizes.peers + "px"; + style.setProperty("--rows", peers + " " + share("figures-box") + " " + + share("document-box") + " " + share("console-box")); + main.classList.toggle("peers-sized", sizes.peers !== null || peersOnly); + try { localStorage.setItem("continuum-sizes", JSON.stringify(sizes)); } catch (_) {} +} +const box = id => document.getElementById(id).getBoundingClientRect(); +// Which space between boxes `event` is over, if any: "columns", "peers" (below the +// peers), "split" (between figures and document) or "console" (above the console). +// Only side by side; with the right column hidden, right of the map: "columns". +function gapAt(event) { + if (event.target !== main) return null; + if (sizes.columns === 1 && main.classList.contains("right-hidden")) { + return event.clientX >= box("map").right ? "columns" : null; + } + const map = box("map"), peers = box("peers-box"), figures = box("figures-box"), + documentBox = box("document-box"), consoleBox = box("console-box"); + if (map.right >= peers.left) return null; + if (event.clientX >= map.right && event.clientX <= peers.left) return "columns"; + if (event.clientX < peers.left || event.clientX > peers.right) return null; + if (event.clientY >= peers.bottom && event.clientY <= figures.top) return "peers"; + if (event.clientY >= figures.bottom && event.clientY <= documentBox.top) return "split"; + if (event.clientY >= documentBox.bottom && event.clientY <= consoleBox.top) return "console"; + return null; +} +function resizeTo(gap, event) { + const clamp = (value, low, high) => Math.min(high, Math.max(low, value)); + // The space between boxes (`main`'s gap), half of it each side. + const half = parseFloat(getComputedStyle(main).columnGap) / 2; + if (gap === "columns") { + // Across the whole width: the right column's right edge is `main`'s, inside. + const map = box("map"); + const right = main.getBoundingClientRect().right - + parseFloat(getComputedStyle(main).paddingRight); + const share = (event.clientX - half - map.left) / (right - map.left - 2 * half); + // Dragged to the side: hidden. + sizes.columns = share > .9 ? 1 : clamp(share, .2, .8); + } else if (gap === "peers") { + const peers = box("peers-box"), consoleBox = box("console-box"); + sizes.peers = Math.round(clamp(event.clientY - half - peers.top, 60, + consoleBox.bottom - peers.top - 6 * half - 200)); + } else if (gap === "console") { + const figures = box("figures-box"), consoleBox = box("console-box"); + sizes.console = clamp((consoleBox.bottom - event.clientY - half) / + (consoleBox.bottom - figures.top - 4 * half), .05, .8); + } else { + const figures = box("figures-box"), documentBox = box("document-box"); + sizes.split = clamp((event.clientY - half - figures.top) / + (documentBox.bottom - figures.top - 2 * half), .1, .9); + } + applySizes(); +} +let resizing = null; +let resizeFrame = null; +main.addEventListener("pointermove", event => { + if (resizing) { + resizeTo(resizing, event); + // The map's view box follows its size: drawn once a frame. + if (resizeFrame === null) resizeFrame = requestAnimationFrame(() => { + resizeFrame = null; + render(); + }); + return; + } + const gap = gapAt(event); + main.classList.toggle("resize-columns", gap === "columns"); + main.classList.toggle("resize-rows", gap === "peers" || gap === "split" || gap === "console"); +}); +main.addEventListener("pointerdown", event => { + resizing = gapAt(event); + if (!resizing) return; + event.preventDefault(); + main.setPointerCapture(event.pointerId); +}); +const resized = () => { + if (!resizing) return; + resizing = null; + main.classList.remove("resize-columns", "resize-rows"); + layoutAgain(); + render(); +}; +main.addEventListener("pointerup", resized); +main.addEventListener("pointercancel", resized); +// --- Boxes collapsed to their titles, and opened again; kept in this browser. A +// collapsed box isn't drawn (`busy`), so it costs nothing while the node updates. --- +const collapsed = new Set(); +try { + for (const id of JSON.parse(localStorage.getItem("continuum-collapsed") || "[]")) { + collapsed.add(id); + } +} catch (_) {} +for (const section of document.querySelectorAll("main > section")) { + const button = el("button", { class: "collapse" }); + section.querySelector("h2").prepend(button); + const show = () => { + const shut = collapsed.has(section.id); + section.classList.toggle("collapsed", shut); + button.textContent = shut ? "▸" : "▾"; + button.title = shut ? "Open" : "Collapse"; + }; + show(); + button.addEventListener("click", () => { + if (collapsed.has(section.id)) collapsed.delete(section.id); else collapsed.add(section.id); + try { localStorage.setItem("continuum-collapsed", JSON.stringify([...collapsed])); } + catch (_) {} + show(); + applySizes(); + if (collapsed.has(section.id)) return; + if (section.id === "console-box") showConsole(); + if (section.id === "map") layoutAgain(); + release(); + render(); + }); +} +applySizes(); +main.addEventListener("dblclick", event => { + const gap = gapAt(event); + if (!gap) return; + sizes[gap] = null; + applySizes(); + layoutAgain(); + render(); +}); +</script> +</body> +</html> diff --git a/src/api/dashboard/mod.rs b/src/api/dashboard/mod.rs new file mode 100644 index 0000000..2557906 --- /dev/null +++ b/src/api/dashboard/mod.rs @@ -0,0 +1,667 @@ +//! The dashboard (`--features dashboard`): a page showing the node's figures +//! (`/admin/stats`), its peers (`/peers`) and a map of the swarm, kept up to date +//! through Server-Sent Events. +//! +//! One producer per node serves every client: while at least one is connected (or +//! asked for the state within `LINGER`), it builds the snapshot every `TICK` and, if +//! something changed, publishes it, and a JSON merge patch (RFC 7386) from the one +//! before to every client connected. Nothing changed, nothing is sent (but SSE's +//! keep-alive comments). Ages (`…SecsAgo`, `uptimeSecs` and the like, `GROWING`) grow by +//! themselves: they alone changing is no change, and the page counts them on. +//! +//! The map: the graph the member list makes (`topology::intended_graph`, as this node +//! sees it), and the links every member measures: subscribed to with LINKS while anyone +//! looks (renewed every `LINKS_EVERY`, ended when nobody does; `swarm::links`), so a +//! member tells of a change as it happens; the same answers for all clients. Or, if +//! asked (`POST /admin/dashboard/links?live=false`, for every client, until asked +//! otherwise or the node restarts), only asked every `LINKS_EVERY`, without +//! subscribing: fewer messages in the swarm, a map up to `LINKS_EVERY` behind. With +//! the clocks the links carry, every member's clock against the swarm's (`clocks`). + +mod clocks; + +use std::collections::{BTreeMap, BTreeSet}; +use std::convert::Infallible; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; +use std::time::Duration; + +use axum::extract::Query; +use axum::http::{HeaderValue, StatusCode, header::CONTENT_TYPE}; +use axum::response::sse::{Event, KeepAlive, Sse}; +use axum::response::{Html, IntoResponse, Response}; +use axum::routing::{get, post}; +use axum::{Extension, Router}; +use serde_json::{Map, Value as JsonValue, json}; +use tokio::sync::{Notify, broadcast, watch}; +use tokio::time::Instant; + +use crate::locks::MutexExt; +use crate::state::AppState; +use crate::swarm::{links, members, topology}; +use crate::transport::{Hex, Link}; + +/// How often the snapshot is built and compared, while anyone looks. +const TICK: Duration = Duration::from_secs(2); +/// How often the subscriptions to the members' links are renewed (each renewal is +/// answered: a reply at least this often), and the graph recomputed; changes come in +/// between, as they happen (`swarm::links`). +const LINKS_EVERY: Duration = Duration::from_secs(30); +/// How long a subscription lasts unless renewed: a node that is gone stops it. +const LINKS_LEASE: Duration = Duration::from_secs(90); +/// A member's links older than this are left out (it didn't answer since). +const LINKS_KEPT_FOR: Duration = Duration::from_secs(90); +/// The producer runs on this long after the last request for the state. +const LINGER: Duration = Duration::from_secs(60); +/// Members asked for their links at most, each round. +const MAX_ASKED: usize = 512; + +/// A snapshot or a patch as published: its number, and its JSON. +struct Published { + seq: u64, + json: String, +} + +/// The producer and what it published, shared by all clients. +struct Hub { + state: AppState, + /// The latest snapshot; None while the producer hasn't made one (since it started). + latest: watch::Sender<Option<Arc<Published>>>, + /// Each patch, to the clients connected. + patches: broadcast::Sender<Arc<Published>>, + /// Where the document changed since the last tick, to the clients connected: + /// `{"paths": [...]}`, or `{"all": true}` if too many or not known. + documents: broadcast::Sender<Arc<String>>, + /// The messages exchanged since the last tick, one by one (`traffic::MessageLog`), + /// to the clients connected: a JSON list. + messages: broadcast::Sender<Arc<String>>, + control: Mutex<Control>, + /// Someone asked for it now (`refresh`): a tick at once, members asked again. + poke: Notify, + /// The members' links subscribed to (true, at first), or only asked for every + /// `LINKS_EVERY` (`set_links`). + live: AtomicBool, + /// True once the node stops: every client gets an `offline` event, and its stream + /// ends (so the HTTP server needn't wait for clients to go). + closing: watch::Receiver<bool>, +} + +struct Control { + running: bool, + /// The producer stops once no client is connected and this is past. + wanted_until: Instant, +} + +impl Hub { + fn new(state: AppState, closing: watch::Receiver<bool>) -> Arc<Self> { + let (latest, _) = watch::channel(None); + let (patches, _) = broadcast::channel(64); + let (documents, _) = broadcast::channel(64); + let (messages, _) = broadcast::channel(64); + Arc::new(Self { + state, + latest, + patches, + documents, + messages, + poke: Notify::new(), + live: AtomicBool::new(true), + control: Mutex::new(Control { + running: false, + wanted_until: Instant::now(), + }), + closing, + }) + } + + /// Someone looks: the producer runs (started if it wasn't), for `LINGER` at least. + fn want(self: &Arc<Self>) { + let mut control = self.control.lock_or_recover(); + control.wanted_until = Instant::now() + LINGER; + if !control.running && !*self.closing.borrow() { + control.running = true; + // What it published before it stopped is stale. + self.latest.send_replace(None); + tokio::spawn(Arc::clone(self).run()); + } + } + + /// The latest snapshot, once there is one; None if the node stops first. + async fn current(&self) -> Option<Arc<Published>> { + let mut latest = self.latest.subscribe(); + let mut closing = self.closing.clone(); + // The borrows (lock guards, not Send) end within the arms. + tokio::select! { + published = latest.wait_for(Option::is_some) => { + published.ok().and_then(|current| current.clone()) + } + _ = closing.wait_for(|closing| *closing) => None, + } + } + + /// Resolves once the node stops (or its signal is gone). + async fn closed(&self) { + let mut closing = self.closing.clone(); + let _ = closing.wait_for(|closing| *closing).await; + } + + async fn run(self: Arc<Self>) { + let mut last: Option<JsonValue> = None; + let mut seq = 0u64; + let mut graph = Graph::default(); + let mut graph_due = Instant::now(); + let mut changes = self.state.change_events.subscribe(); + // The messages one by one, from now on, while it runs. + let log = &self.state.traffic.log; + log.watch(true); + let mut logged = 0u64; + loop { + let live = self.live.load(Ordering::Relaxed); + if Instant::now() >= graph_due { + graph.refresh(&self.state, live).await; + graph_due = Instant::now() + LINKS_EVERY; + } + let snapshot = build(&self.state, &graph, live).await; + let changed = last + .as_ref() + .is_none_or(|old| without_ages(old) != without_ages(&snapshot)); + if changed { + seq += 1; + if let Some(old) = &last + && let Some(patch) = merge_patch(old, &snapshot) + { + let _ = self.patches.send(Arc::new(Published { + seq, + json: patch.to_string(), + })); + } + self.latest.send_replace(Some(Arc::new(Published { + seq, + json: snapshot.to_string(), + }))); + last = Some(snapshot); + } + let poked = tokio::select! { + () = tokio::time::sleep(TICK) => false, + () = self.poke.notified() => true, + }; + if poked { + graph_due = Instant::now(); + } + if let Some(changed) = document_changes(&mut changes) { + let _ = self.documents.send(Arc::new(changed.to_string())); + } + let mut messages = log.after(logged, MESSAGES_AT_MOST); + by_node(&self.state, &mut messages).await; + if let Some(last) = messages.last() { + logged = last.seq; + if let Ok(json) = serde_json::to_string(&messages) { + let _ = self.messages.send(Arc::new(json)); + } + } + let unwanted = || { + self.patches.receiver_count() == 0 + && Instant::now() >= self.control.lock_or_recover().wanted_until + }; + if *self.closing.borrow() { + log.watch(false); + self.control.lock_or_recover().running = false; + return; + } + if unwanted() { + // Nobody looks: the members needn't tell us any more. Unless someone + // came meanwhile, then it goes on (and subscribes again). + for member in self.state.links.subscribed() { + links::unsubscribe(&self.state, &member).await; + } + let mut control = self.control.lock_or_recover(); + if self.patches.receiver_count() == 0 && Instant::now() >= control.wanted_until { + log.watch(false); + control.running = false; + return; + } + drop(control); + graph_due = Instant::now(); + } + } + } +} + +/// Paths changed in a tick listed at most; more: `all`. +/// The messages the paths layer logged under an address (its own PINGs and PONGs) under +/// the node there instead, where one is known: the address it proved itself at, or is +/// said to be at (`Transport::identities_of`), named in `address`. +async fn by_node(state: &AppState, messages: &mut [crate::transport::traffic::Logged]) { + let peers = state.peers_snapshot().await; + let known = |peer: &str| peers.iter().any(|(known, _)| known == peer); + if messages.iter().all(|message| known(&message.peer)) { + return; + } + let mut nodes: std::collections::HashMap<String, String> = std::collections::HashMap::new(); + for (peer, _) in &peers { + for address in state.transport.identities_of(peer) { + nodes.entry(address).or_insert_with(|| peer.clone()); + } + } + for message in messages { + if let Some(node) = nodes.get(&message.peer) { + message.address = Some(std::mem::replace(&mut message.peer, node.clone())); + } + } +} + +const MAX_CHANGED: usize = 256; +/// Messages sent to the clients per tick at most, the latest (the console keeps fewer). +const MESSAGES_AT_MOST: usize = 500; + +/// Where the document changed since last asked (`Hub::documents`), if anywhere. +fn document_changes( + changes: &mut broadcast::Receiver<Arc<crate::document::store::ChangeSet>>, +) -> Option<JsonValue> { + let mut paths = BTreeSet::new(); + let mut all = false; + loop { + match changes.try_recv() { + Ok(set) => { + paths.extend(set.paths.iter().cloned()); + all |= paths.len() > MAX_CHANGED; + } + Err(broadcast::error::TryRecvError::Lagged(_)) => all = true, + Err(_) => break, + } + } + if all { + Some(json!({ "all": true })) + } else if paths.is_empty() { + None + } else { + Some(json!({ "paths": paths })) + } +} + +/// What a client's stream waits for. +enum Next { + Patch(Result<Arc<Published>, broadcast::error::RecvError>), + Document(Result<Arc<String>, broadcast::error::RecvError>), + Messages(Result<Arc<String>, broadcast::error::RecvError>), + Closing, +} + +/// The intended graph, recomputed every `LINKS_EVERY` (it costs about `members²`), when +/// members are asked for their links too: subscribed to, if `live`. +#[derive(Default)] +struct Graph { + intended: BTreeMap<String, BTreeSet<String>>, +} + +impl Graph { + async fn refresh(&mut self, state: &AppState, live: bool) { + let lease = if live { LINKS_LEASE } else { Duration::ZERO }; + self.intended = topology::intended_graph(state).unwrap_or_default(); + let members: Vec<String> = { + let replica = state.doc.read().await; + members::members(&replica.doc) + }; + for member in members + .into_iter() + .filter(|member| member != state.local_id()) + .take(MAX_ASKED) + { + let state = state.clone(); + tokio::spawn(async move { links::ask(&state, &member, lease).await }); + } + } +} + +/// The snapshot: the node, its figures, its peers, and the map (`live`: its links +/// subscribed to). +async fn build(state: &AppState, graph: &Graph, live: bool) -> JsonValue { + // The peers once, for their list and the figures' totals over them. + let list = super::peers::peer_list(state).await; + let stats = serde_json::to_value(super::stats::collect(state, &list).await).unwrap_or_default(); + let peers = serde_json::to_value(&list).unwrap_or_default(); + let aliases = { + let replica = state.doc.read().await; + members::aliases(&replica.doc) + }; + let local = state.local_id(); + let intended: Map<String, JsonValue> = graph + .intended + .iter() + .map(|(member, neighbours)| (member.clone(), json!(neighbours))) + .collect(); + let own = links::own_links(state).await; + let replies: Vec<_> = state + .links + .replies() + .into_iter() + .filter(|(_, age, _)| *age <= LINKS_KEPT_FOR) + .collect(); + let mut measured = Map::new(); + measured.insert( + local.to_owned(), + json!({ "secsAgo": 0, "links": links_json(&own) }), + ); + for (member, age, member_links) in &replies { + measured.insert( + member.clone(), + json!({ "secsAgo": age.as_secs(), "links": links_json(member_links) }), + ); + } + let clocks = clocks::swarm( + local, + std::iter::once((local, &own[..])).chain( + replies + .iter() + .map(|(member, _, member_links)| (member.as_str(), &member_links[..])), + ), + u64::try_from(state.clock_skew_warn.as_millis()).unwrap_or(u64::MAX), + ); + // Everyone on the map: in the member list's graph, or measured. + let mut members_json = Map::new(); + for member in graph.intended.keys().chain(measured.keys()) { + members_json.insert( + member.clone(), + json!({ "alias": aliases.get(member), "self": member == local }), + ); + } + json!({ + "node": { "id": local, "alias": state.alias }, + "page": page().0, + "stats": stats, + "peers": peers, + "graph": { + "members": members_json, + "intended": intended, + "measured": measured, + "clocks": clocks, + "live": live, + }, + }) +} + +fn links_json(links: &[Link]) -> JsonValue { + links + .iter() + .map(|link| { + json!({ + "peer": link.peer, + "transport": link.transport, + "relayed": link.relayed, + "down": link.down, + "rttMs": link.rtt_us.map(|us| f64::from(us) / 1000.0), + "dormant": link.dormant, + "clock": link.clock, + "others": link.others.iter().map(|other| json!({ + "transport": other.transport, + "rttMs": other.rtt_us.map(|us| f64::from(us) / 1000.0), + "relayed": other.relayed, + })).collect::<Vec<_>>(), + }) + }) + .collect() +} + +/// Fields that grow by themselves besides those named `…SecsAgo`; the page gets the +/// list when served (`GROWING_MARK`). +const GROWING: &[&str] = &[ + "uptimeSecs", + "dormantForSecs", + "divergedForSecs", + "observedSecs", + "lastFromNeighbourSecs", + "oldestQuarantineSecs", +]; + +/// Whether the field `key` is an age, which grows by itself. +fn is_age(key: &str) -> bool { + key == "secsAgo" || key.ends_with("SecsAgo") || GROWING.contains(&key) +} + +/// `value` without its ages, to tell whether anything else changed. +fn without_ages(value: &JsonValue) -> JsonValue { + match value { + JsonValue::Object(object) => JsonValue::Object( + object + .iter() + .filter(|(key, _)| !is_age(key)) + .map(|(key, value)| (key.clone(), without_ages(value))) + .collect(), + ), + JsonValue::Array(items) => JsonValue::Array(items.iter().map(without_ages).collect()), + other => other.clone(), + } +} + +/// The JSON merge patch (RFC 7386) from `old` to `new`, if they differ: objects field +/// by field (a field gone is null), anything else replaced whole. A null in `new` reads +/// as the field gone, as a merge patch can't tell them apart. +fn merge_patch(old: &JsonValue, new: &JsonValue) -> Option<JsonValue> { + match (old, new) { + (JsonValue::Object(old), JsonValue::Object(new)) => { + let mut patch = Map::new(); + for key in old.keys() { + if !new.contains_key(key) { + patch.insert(key.clone(), JsonValue::Null); + } + } + for (key, value) in new { + match old.get(key) { + Some(previous) => { + if let Some(change) = merge_patch(previous, value) { + patch.insert(key.clone(), change); + } + } + None => { + patch.insert(key.clone(), value.clone()); + } + } + } + (!patch.is_empty()).then_some(JsonValue::Object(patch)) + } + (old, new) if old == new => None, + (_, new) => Some(new.clone()), + } +} + +/// The dashboard's routes (authenticated, as the rest of `/admin`), with their hub. +pub(crate) fn router(state: AppState) -> Router<AppState> { + // Its streams end when the node shuts down (`AppState::closing`). + let closing = state.closing.subscribe(); + Router::new() + .route("/admin/dashboard", get(page_html)) + .route("/admin/dashboard/state", get(current_state)) + .route("/admin/dashboard/events", get(events)) + .route("/admin/dashboard/refresh", post(refresh)) + .route("/admin/dashboard/links", post(set_links)) + .layer(Extension(Hub::new(state, closing))) +} + +/// The page as built into this binary. +const PAGE: &str = include_str!("dashboard.html"); +/// Where the page says which version of it it is. +const PAGE_VERSION_MARK: &str = "__PAGE_VERSION__"; +/// Where the page gets the age fields (`GROWING`), so the list is kept once. +const GROWING_MARK: &str = "__GROWING__"; + +/// The page's version (a hash of it), and the page saying so: a page whose version +/// differs from the snapshot's `page` was served by another build, and reloads. +fn page() -> &'static (String, String) { + static PAGE_SERVED: OnceLock<(String, String)> = OnceLock::new(); + PAGE_SERVED.get_or_init(|| { + use sha2::{Digest, Sha256}; + let growing = serde_json::to_string(GROWING).unwrap_or_else(|_| "[]".to_owned()); + let page = PAGE.replace(GROWING_MARK, &growing); + // Of the page with its list: a new list is a new version too. + let version = Hex(&Sha256::digest(page.as_bytes())[..8]).to_string(); + let served = page.replace(PAGE_VERSION_MARK, &version); + (version, served) + }) +} + +/// `GET /admin/dashboard`: the page. +async fn page_html() -> Html<&'static str> { + Html(page().1.as_str()) +} + +/// `POST /admin/dashboard/refresh`: a tick now rather than at the next `TICK`, and the +/// members asked for their links again; what changed comes as events, as ever. +async fn refresh(Extension(hub): Extension<Arc<Hub>>) -> StatusCode { + hub.want(); + hub.poke.notify_one(); + StatusCode::NO_CONTENT +} + +#[derive(serde::Deserialize)] +struct LinksQuery { + live: bool, +} + +/// `POST /admin/dashboard/links?live=…`: the members' links subscribed to (`true`), or +/// only asked for every `LINKS_EVERY` (`false`), for every client; the subscriptions +/// ended at once, and the members asked again now. +async fn set_links( + Extension(hub): Extension<Arc<Hub>>, + Query(query): Query<LinksQuery>, +) -> StatusCode { + hub.live.store(query.live, Ordering::Relaxed); + if !query.live { + for member in hub.state.links.subscribed() { + links::unsubscribe(&hub.state, &member).await; + } + } + hub.want(); + hub.poke.notify_one(); + StatusCode::NO_CONTENT +} + +/// `GET /admin/dashboard/state`: the latest snapshot, for a client that wants it whole. +async fn current_state(Extension(hub): Extension<Arc<Hub>>) -> Response { + hub.want(); + let Some(current) = hub.current().await else { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + }; + ( + [(CONTENT_TYPE, HeaderValue::from_static("application/json"))], + current.json.clone(), + ) + .into_response() +} + +/// `GET /admin/dashboard/events`: Server-Sent Events: a `snapshot` first, then a +/// `patch` whenever something changed (each with its number as the event's ID; a +/// client that fell behind gets a `snapshot` again), a `document` where the document +/// changed (`Hub::documents`, at most every tick), `messages` with the messages since +/// the last tick (`Hub::messages`), and `offline` when the node stops, which ends the +/// stream. +async fn events(Extension(hub): Extension<Arc<Hub>>) -> impl IntoResponse { + // Subscribed before the snapshot is read: no patch after it is missed. + let mut patches = hub.patches.subscribe(); + let mut documents = hub.documents.subscribe(); + let mut messages = hub.messages.subscribe(); + hub.want(); + let first = hub.current().await; + let stream = async_stream::stream! { + let Some(first) = first else { + yield Ok::<_, Infallible>(offline_event()); + return; + }; + let mut seen = first.seq; + yield Ok(snapshot_event(&first)); + loop { + let next = tokio::select! { + received = patches.recv() => Next::Patch(received), + changed = documents.recv() => Next::Document(changed), + logged = messages.recv() => Next::Messages(logged), + () = hub.closed() => Next::Closing, + }; + let received = match next { + Next::Patch(received) => received, + Next::Document(Ok(changed)) => { + yield Ok(Event::default().event("document").data(changed.as_str())); + continue; + } + Next::Document(Err(broadcast::error::RecvError::Lagged(_))) => { + yield Ok(Event::default().event("document").data(r#"{"all":true}"#)); + continue; + } + Next::Document(Err(broadcast::error::RecvError::Closed)) => break, + Next::Messages(Ok(logged)) => { + yield Ok(Event::default().event("messages").data(logged.as_str())); + continue; + } + // Missed ones are only missed in the console. + Next::Messages(Err(broadcast::error::RecvError::Lagged(_))) => continue, + Next::Messages(Err(broadcast::error::RecvError::Closed)) => break, + Next::Closing => { + yield Ok(offline_event()); + break; + } + }; + match received { + Ok(patch) if patch.seq <= seen => {} + Ok(patch) => { + seen = patch.seq; + yield Ok(Event::default() + .event("patch") + .id(patch.seq.to_string()) + .data(patch.json.clone())); + } + Err(broadcast::error::RecvError::Lagged(_)) => { + let Some(current) = hub.current().await else { + yield Ok(offline_event()); + break; + }; + seen = current.seq; + yield Ok(snapshot_event(¤t)); + } + Err(broadcast::error::RecvError::Closed) => break, + } + } + }; + Sse::new(stream).keep_alive(KeepAlive::default()) +} + +/// The node stops: the client may reconnect once it is back. +fn offline_event() -> Event { + Event::default().event("offline").data("{}") +} + +fn snapshot_event(published: &Published) -> Event { + Event::default() + .event("snapshot") + .id(published.seq.to_string()) + .data(published.json.clone()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn merge_patches() { + let old = json!({ "a": 1, "b": { "c": 2, "d": [1, 2] }, "gone": true }); + let new = json!({ "a": 1, "b": { "c": 3, "d": [1, 2] }, "added": "x" }); + assert_eq!( + merge_patch(&old, &new), + Some(json!({ "b": { "c": 3 }, "gone": null, "added": "x" })) + ); + assert_eq!(merge_patch(&old, &old), None); + assert_eq!( + merge_patch(&json!([1]), &json!([1, 2])), + Some(json!([1, 2])) + ); + } + + #[test] + fn ages_alone_are_no_change() { + let old = + json!({ "uptimeSecs": 1, "peers": { "p": { "lastSeenSecsAgo": 3, "rttMs": 2 } } }); + let older = + json!({ "uptimeSecs": 9, "peers": { "p": { "lastSeenSecsAgo": 7, "rttMs": 2 } } }); + assert_eq!(without_ages(&old), without_ages(&older)); + let moved = + json!({ "uptimeSecs": 9, "peers": { "p": { "lastSeenSecsAgo": 7, "rttMs": 5 } } }); + assert_ne!(without_ages(&old), without_ages(&moved)); + } +} diff --git a/src/api/document.rs b/src/api/document.rs index 20bbc7b..bc8c47e 100644 --- a/src/api/document.rs +++ b/src/api/document.rs @@ -50,12 +50,11 @@ pub(crate) async fn get_document(State(state): State<AppState>) -> Response { // Both steps take time proportional to the document: off the async threads, and // the lock only for the copy. let saved = tokio::task::spawn_blocking(move || { - let (mut copy, epoch) = { + let (copy, epoch, heads) = { let replica = doc.blocking_read(); - (replica.doc.clone(), replica.epoch) + (replica.doc.clone(), replica.epoch, replica.heads().to_vec()) }; - let heads = copy.get_heads(); - (copy.save(), heads, epoch) + (crate::document::save(©), heads, epoch) }) .await; let (bytes, heads, epoch) = match saved { diff --git a/src/api/health.rs b/src/api/health.rs index b1e4523..3bd374e 100644 --- a/src/api/health.rs +++ b/src/api/health.rs @@ -1,15 +1,32 @@ //! `GET /health` and `GET /ready`: unauthenticated, for supervisors and load balancers. -//! They say whether the node is up and whether it serves the store, nothing more. +//! They say whether the node is up and whether it serves the store, and `/health` on a +//! node whether its clock is off, nothing more. + +use std::time::Duration; use axum::{extract::State, http::StatusCode}; use crate::state::AppState; -/// The HTTP API is up. +/// The HTTP API is up (a relay's). pub(crate) async fn health() -> &'static str { "ok\n" } +/// The HTTP API is up (a node's); a second line while its clock is off from its peers' +/// (`swarm::skew`), which doesn't make it less up. +pub(crate) async fn node_health(State(state): State<AppState>) -> String { + let skew = crate::swarm::skew::measure(&state).await; + match skew.offset_ms { + Some(offset) if skew.off => format!( + "ok\nclock off: {} {} most of its peers'\n", + crate::units::format_duration(Duration::from_millis(offset.unsigned_abs())), + if offset < 0 { "behind" } else { "ahead of" } + ), + _ => "ok\n".to_owned(), + } +} + /// The node serves reads and writes: ready, not joining or rejoining. pub(crate) async fn ready(State(state): State<AppState>) -> (StatusCode, &'static str) { if state.is_ready().await { diff --git a/src/api/mod.rs b/src/api/mod.rs index 2cb0415..9bb2bed 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -1,6 +1,11 @@ -//! HTTP API: router, the API description and shared response helpers. +//! HTTP API: router, the API description and shared response helpers (the store's +//! own are in `store`). pub(crate) mod auth; +mod changes; +mod config; +#[cfg(feature = "dashboard")] +mod dashboard; mod debug; mod document; mod health; @@ -12,22 +17,20 @@ mod store; use axum::{ Json, Router, body::Body, - extract::{Request, State}, + extract::Request, http::{ HeaderValue, StatusCode, - header::{CONTENT_LENGTH, CONTENT_TYPE, ETAG}, + header::{CONTENT_LENGTH, CONTENT_TYPE}, }, middleware::{self, Next}, response::{IntoResponse, Response}, routing::{delete, get}, }; use serde_json::{Value as JsonValue, json}; -use sha2::{Digest, Sha256}; use tracing::{error, warn}; use utoipa_swagger_ui::SwaggerUi; -use crate::api::auth::{AuthState, AuthenticatedUser, basic_auth}; -use crate::document::meta; +use crate::api::auth::{AuthState, basic_auth}; use crate::document::store::StoreError; use crate::state::AppState; @@ -40,6 +43,7 @@ pub(crate) fn router(state: AppState, auth_state: AuthState, debug_api: bool) -> "/store/{*key}", get(store::get_store_entry) .put(store::put_store_entry) + .post(store::post_store_entry) .delete(store::delete_store_entry), ) .route("/stream", get(store::stream_store_root)) @@ -52,9 +56,16 @@ pub(crate) fn router(state: AppState, auth_state: AuthState, debug_api: bool) -> ) .route("/admin/stats", get(stats::get_stats)) .route("/admin/document", get(document::get_document)) - .route("/admin/config", get(get_config)) + .route("/admin/changes", get(changes::get_changes)) + .route("/admin/changes/{*key}", get(changes::get_changes_at)) + .route("/admin/config", get(config::get_config)) .merge(crate::remake::router::<AppState>()); + #[cfg(feature = "dashboard")] + { + api = api.merge(dashboard::router(state.clone())); + } + if debug_api { warn!("Debug API enabled"); api = api.route( @@ -69,7 +80,7 @@ pub(crate) fn router(state: AppState, auth_state: AuthState, debug_api: bool) -> SwaggerUi::new("/swagger-ui").config(utoipa_swagger_ui::Config::from("/openapi.yaml")), ) .route("/openapi.yaml", get(openapi_spec)) - .route("/health", get(health::health)) + .route("/health", get(health::node_health)) .route("/ready", get(health::ready)) // Outermost, so authentication failures get the error body too. .layer(middleware::from_fn(json_errors)) @@ -159,119 +170,62 @@ async fn openapi_spec() -> impl IntoResponse { ) } -#[derive(Debug, Default, serde::Deserialize)] -struct ConfigQuery { - /// Text with placeholders as it ends up, or as written. - #[serde(default)] - values: crate::config::Values, -} - -/// `GET /admin/config`: the effective configuration with each value's source; -/// imported values and secrets are left out. -async fn get_config( - State(state): State<AppState>, - axum::extract::Query(query): axum::extract::Query<ConfigQuery>, -) -> Json<JsonValue> { - Json(state.config.get(query.values).clone()) -} - // --- shared helpers ---------------------------------------------------------- -pub(crate) fn store_status(error: StoreError, context: &str) -> StatusCode { - match error { - StoreError::Conflict => StatusCode::CONFLICT, - StoreError::NotFound => StatusCode::NOT_FOUND, - other => { - error!(error = %format!("{other:#}"), "{context}"); - StatusCode::INTERNAL_SERVER_ERROR - } +/// An error answer: its status, and why where it is known (`error_response`); a bare +/// status otherwise (`json_errors` gives it the body). +pub(crate) struct ApiError(StatusCode, Option<String>); + +impl ApiError { + pub(crate) fn status(&self) -> StatusCode { + self.0 } } -fn etag_for(bytes: &[u8]) -> String { - let digest = Sha256::digest(bytes); - let hex = digest - .iter() - .map(|byte| format!("{byte:02x}")) - .collect::<String>(); - - format!("\"{hex}\"") +impl From<StatusCode> for ApiError { + fn from(status: StatusCode) -> Self { + Self(status, None) + } } -fn parse_key_path(key: &str) -> Result<Vec<String>, StatusCode> { - let segments: Vec<String> = key.split('/').map(str::to_owned).collect(); - - if segments.is_empty() || segments.iter().any(|segment| segment.is_empty()) { - return Err(StatusCode::BAD_REQUEST); +impl IntoResponse for ApiError { + fn into_response(self) -> Response { + match self.1 { + Some(message) => error_response(self.0, message, None), + None => self.0.into_response(), + } } - - Ok(segments) } -/// A path for a write. Protocol data (`_continuum`: remake records, members, forget -/// records, epoch metadata) is what every node acts on, so a write there acts on the -/// whole swarm: only users listed in `--protocol-writers` may, and each such write is -/// logged. Epoch metadata (and `_continuum` as a whole, which contains it) is never -/// writable: the protocol reads it at the snapshot change, so an edit only misleads -/// whoever reads it later. `403` otherwise. -fn parse_write_path( - state: &AppState, - user: &AuthenticatedUser, - key: &str, - method: &str, -) -> Result<Vec<String>, StatusCode> { - let path = parse_key_path(key)?; - if !meta::is_protocol(&path) { - return Ok(path); - } - // `_continuum` itself, epoch metadata and the swarm ID: nobody. - let fixed = path.len() < 2 || path[1] == meta::EPOCHS || path[1] == meta::SWARM; - if fixed || !state.protocol_writers.contains(&user.0) { - warn!(user = %user.0, method, path = %key, "Refused a write to protocol data"); - return Err(StatusCode::FORBIDDEN); +/// A store error as an answer, saying why (`StoreError`'s text) where it is the +/// client's; others are logged with `context`, and are a bare `500`. +/// A read's (`store_status`). +impl From<StoreError> for ApiError { + fn from(error: StoreError) -> Self { + store_status(error, "Failed to read store path") } - warn!(user = %user.0, method, path = %key, "Protocol data written through the store API; this acts on the whole swarm"); - Ok(path) } -fn response_for_value( - value: &JsonValue, - accept: Option<&HeaderValue>, -) -> Result<Response, StatusCode> { - let wants_text = accept - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| { - value - .split(',') - .any(|item| item.trim().starts_with("text/plain")) - }); - - let (content_type, body) = if wants_text { - let body = match value { - JsonValue::String(value) => value.clone(), - other => serde_json::to_string(other).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?, - }; - - ("text/plain; charset=utf-8", body) - } else { - ( - "application/json", - serde_json::to_string(value).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?, - ) +pub(crate) fn store_status(error: StoreError, context: &str) -> ApiError { + let status = match error { + StoreError::Conflict + | StoreError::NotAList + | StoreError::PastTheEnd { .. } + | StoreError::NotACounter + | StoreError::Retyping(_) + | StoreError::NotCastable { .. } => StatusCode::CONFLICT, + StoreError::NotFound | StoreError::UnknownHeads => StatusCode::NOT_FOUND, + StoreError::Rejoining => StatusCode::SERVICE_UNAVAILABLE, + other => { + error!(error = %format!("{other:#}"), "{context}"); + return StatusCode::INTERNAL_SERVER_ERROR.into(); + } }; + ApiError(status, Some(error.to_string())) +} - let etag = etag_for(body.as_bytes()); - - let mut response = body.into_response(); - - response - .headers_mut() - .insert(CONTENT_TYPE, HeaderValue::from_static(content_type)); - - response.headers_mut().insert( - ETAG, - HeaderValue::from_str(&etag).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?, - ); - - Ok(response) +/// A media type without its parameters: `text/plain; charset=utf-8` is `text/plain`. +/// For `Content-Type`, and each item of an `Accept` (split at `,`). +pub(crate) fn media_type(value: &str) -> &str { + value.split(';').next().unwrap_or("").trim() } diff --git a/src/api/peers.rs b/src/api/peers.rs index eeb3fb2..537ad98 100644 --- a/src/api/peers.rs +++ b/src/api/peers.rs @@ -12,12 +12,13 @@ use tokio::time::Instant; use tracing::warn; use super::store_status; -use crate::document::replica::Epoch; +use crate::document::replica::EpochRecord; use crate::document::store; use crate::locks::MutexExt; use crate::state::{AppState, HelloState, Incidents, Liveness}; use crate::swarm::peer::forgotten_path; use crate::transport::Destination; +use crate::transport::paths::ClockOffset; use crate::transport::traffic::PeerTraffic; #[derive(Debug, Serialize)] @@ -31,77 +32,68 @@ pub(crate) enum PeerLivenessStatus { #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] -pub(crate) struct EpochInfo { - generation: u64, - /// Hex-encoded snapshot hash; all zeros for the genesis epoch. - id: String, -} - -impl From<Epoch> for EpochInfo { - fn from(epoch: Epoch) -> Self { - Self { - generation: epoch.generation, - id: epoch.id_hex(), - } - } -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] pub(crate) struct PeerStatus { /// The name the peer gave itself in the member list, if any (not unique). - alias: Option<String>, + pub(crate) alias: Option<String>, /// Where this node reaches it, or would (`Transport::addresses_of`). - addresses: Vec<String>, + pub(crate) addresses: Vec<String>, /// Where messages to it go now: the address, whether the peer proved itself there, /// and the connection there (libp2p: direct or relayed). Absent if nowhere. - destination: Option<Destination>, - validated_swarm: Option<String>, - status: PeerLivenessStatus, - handshake_complete: bool, - last_seen_secs_ago: Option<u64>, - dormant_for_secs: Option<u64>, - has_pending_delivery: bool, + pub(crate) destination: Option<Destination>, + pub(crate) validated_swarm: Option<String>, + pub(crate) status: PeerLivenessStatus, + pub(crate) handshake_complete: bool, + pub(crate) last_seen_secs_ago: Option<u64>, + pub(crate) dormant_for_secs: Option<u64>, + /// Since a sync message from it was applied (or the epoch changed). + pub(crate) last_sync_secs_ago: Option<u64>, + /// How long its heads have differed from ours with no sync message from it in + /// between; absent while they match. + pub(crate) diverged_for_secs: Option<u64>, + pub(crate) has_pending_delivery: bool, /// Epoch the peer last advertised; absent until it has sent a message. - epoch: Option<EpochInfo>, + pub(crate) epoch: Option<EpochRecord>, /// Whether that epoch equals ours, i.e. whether we sync with this peer. - same_epoch: Option<bool>, + pub(crate) same_epoch: Option<bool>, /// Number of heads in its last keepalive; absent before one, or when it had more /// than 32 (then only their digest and some of them are sent). - heads: Option<usize>, + pub(crate) heads: Option<usize>, /// We keep it alive and sync with it on our own; other peers we only answer. - neighbour: bool, + pub(crate) neighbour: bool, /// Messages and payload bytes exchanged with it, by kind, since we started (or /// since it was last listed). - traffic: PeerTraffic, - incidents: Incidents, + pub(crate) traffic: PeerTraffic, + pub(crate) incidents: Incidents, /// Smoothed sync throughput (`swarm::throughput`); absent before the first large /// exchange. - sync_throughput: Option<ThroughputInfo>, + pub(crate) sync_throughput: Option<ThroughputInfo>, + /// How far its clock is ahead of ours, give or take (`Transport::clock_offset`); + /// absent before a PONG from it. + pub(crate) clock: Option<ClockOffset>, } #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] pub(crate) struct ThroughputInfo { - bytes_per_sec: f64, - samples: u32, + pub(crate) bytes_per_sec: f64, + pub(crate) samples: u32, /// Age of the last sample. - secs_ago: u64, + pub(crate) secs_ago: u64, } #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] pub(crate) struct PeerListResponse { /// Our own alias (`--alias`). - local_alias: Option<String>, - local_epoch: EpochInfo, + pub(crate) local_alias: Option<String>, + pub(crate) local_epoch: EpochRecord, /// Number of heads of our document. Several heads are concurrent changes no later /// change has merged yet. - local_heads: usize, + pub(crate) local_heads: usize, /// Where this node can be reached (`Transport::local_addresses`). - local_addresses: Vec<String>, + pub(crate) local_addresses: Vec<String>, /// By node ID. - peers: BTreeMap<String, PeerStatus>, + pub(crate) peers: BTreeMap<String, PeerStatus>, } #[derive(Debug, Deserialize)] @@ -114,6 +106,12 @@ pub(crate) struct DeletePeerQuery { pub(crate) async fn list_peers( State(state): State<AppState>, ) -> Result<Json<PeerListResponse>, StatusCode> { + Ok(Json(peer_list(&state).await)) +} + +/// What `GET /peers` answers, for others to show too (the dashboard). +pub(crate) async fn peer_list(state: &AppState) -> PeerListResponse { + let state = state.clone(); let snapshot = state.peers_snapshot().await; let now = Instant::now(); @@ -123,7 +121,9 @@ pub(crate) async fn list_peers( for (peer, peer_state_mutex) in snapshot { let neighbour = crate::swarm::topology::is_neighbour(&state, &peer); - let traffic = state.traffic.peer(&peer).unwrap_or_default(); + let traffic = state + .traffic + .node(&peer, &state.transport.identities_of(&peer)); let sync_throughput = state .throughput .lock_or_recover() @@ -147,6 +147,7 @@ pub(crate) async fn list_peers( let alias = aliases.remove(&peer); let addresses = state.transport.addresses_of(&peer); let destination = state.transport.destination(&peer); + let clock = state.transport.clock_offset(&peer); peers.insert( peer, PeerStatus { @@ -160,26 +161,33 @@ pub(crate) async fn list_peers( .last_seen .map(|t| now.saturating_duration_since(t).as_secs()), dormant_for_secs, + last_sync_secs_ago: ps + .last_sync_rx + .map(|t| now.saturating_duration_since(t).as_secs()), + diverged_for_secs: ps + .diverged_since + .map(|t| now.saturating_duration_since(t).as_secs()), has_pending_delivery: ps.sync.sending.is_some() || ps.sync.retry_at.is_some(), - epoch: ps.epoch.map(EpochInfo::from), + epoch: ps.epoch.map(EpochRecord::from), same_epoch: ps.epoch.map(|epoch| epoch == local_epoch), heads: ps.reported_heads, neighbour, traffic, incidents: ps.incidents, sync_throughput, + clock, }, ); } let local_heads = state.doc.read().await.heads().len(); - Ok(Json(PeerListResponse { + PeerListResponse { local_alias: state.alias.clone(), local_addresses: state.transport.local_addresses(), local_epoch: local_epoch.into(), local_heads, peers, - })) + } } /// `DELETE /peers/{pk}`: forgets a peer swarm-wide. @@ -218,7 +226,7 @@ pub(crate) async fn delete_peer( crate::swarm::members::unregister(doc, &peer_clone) }) .await - .map_err(|e| store_status(e, "Failed to write forget record"))?; + .map_err(|e| store_status(e, "Failed to write forget record").status())?; // Unless forced, a peer that came alive since the check above stays: it cancels // the record. diff --git a/src/api/stats.rs b/src/api/stats.rs index 889b675..558507a 100644 --- a/src/api/stats.rs +++ b/src/api/stats.rs @@ -13,8 +13,10 @@ use std::collections::BTreeMap; use serde::Serialize; -use super::peers::EpochInfo; +use super::peers::{PeerListResponse, PeerLivenessStatus, peer_list}; +use crate::document::replica::EpochRecord; use crate::state::{AppState, Incidents}; +use crate::swarm::skew::{self, Skew}; use crate::swarm::{members, stability, topology}; use crate::transport::stats::TransportStats; use crate::transport::traffic::PeerTraffic; @@ -28,7 +30,7 @@ pub(crate) struct StatsResponse { alias: Option<String>, uptime_secs: u64, ready: bool, - epoch: EpochInfo, + epoch: EpochRecord, document: DocumentStats, catch_up: CatchUpStats, remakes: RemakeStats, @@ -43,6 +45,8 @@ pub(crate) struct StatsResponse { /// The peers listed in /peers together (a forgotten peer's are gone). incidents: Incidents, propagation: PropagationStats, + /// This node's clock against its active peers' (`swarm::skew`). + clock: Skew, /// This node's own files (`files`), by kind. disk: BTreeMap<&'static str, DiskStats>, /// How the transports find and reach peers (`Transport::stats`). @@ -63,13 +67,14 @@ struct DiskStats { } /// How long changes written elsewhere took from their commit there until applied here, -/// once caught up. Includes the clock skew between the writer and us. +/// once caught up, by the peers' clocks (`swarm::skew`). Includes the writer's clock +/// skew. #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct PropagationStats { count: u64, sum_ms: u64, - /// Changes stamped later than our clock on arrival (writer's clock ahead). + /// Changes stamped later than their arrival (writer's clock ahead). ahead: u64, /// Not cumulative; the last bucket has no bound. buckets: Vec<PropagationBucket>, @@ -237,19 +242,22 @@ pub(crate) async fn get_relay_only_stats( /// `GET /admin/stats`. pub(crate) async fn get_stats(State(state): State<AppState>, headers: HeaderMap) -> Response { - let stats = collect(&state).await; + let list = peer_list(&state).await; + let stats = collect(&state, &list).await; let accept = headers .get(header::ACCEPT) .and_then(|value| value.to_str().ok()); if prometheus::wanted(accept) { - let body = prometheus::render(&state, &stats).await; + let body = prometheus::render(&stats, &list); ([(header::CONTENT_TYPE, prometheus::CONTENT_TYPE)], body).into_response() } else { Json(stats).into_response() } } -async fn collect(state: &AppState) -> StatsResponse { +/// What `GET /admin/stats` answers as JSON, for others to show too (the dashboard); its +/// totals over the peers from their list (`peer_list`), read once for both. +pub(crate) async fn collect(state: &AppState, list: &PeerListResponse) -> StatsResponse { let node = state.stats.snapshot(); let (heads, tier, remakes) = { @@ -267,17 +275,28 @@ async fn collect(state: &AppState) -> StatsResponse { let (connected_fraction, observed) = stability::measured(state); + let clock = skew::of( + &list + .peers + .values() + .filter(|peer| !matches!(peer.status, PeerLivenessStatus::Dormant)) + .filter_map(|peer| peer.clock) + .collect::<Vec<_>>(), + state.clock_skew_warn, + ); + let mut incidents = Incidents::default(); - let mut last_from_neighbour: Option<tokio::time::Instant> = None; + let mut last_from_neighbour_secs: Option<u64> = None; let mut diverged_peers = 0; - for (peer, peer_state) in state.peers_snapshot().await { - let peer_state = peer_state.lock().await; - incidents.add(&peer_state.incidents); - if peer_state.diverged_since.is_some() { + for peer in list.peers.values() { + incidents.add(&peer.incidents); + if peer.diverged_for_secs.is_some() { diverged_peers += 1; } - if topology::is_neighbour(state, &peer) { - last_from_neighbour = last_from_neighbour.max(peer_state.last_sync_rx); + if peer.neighbour + && let Some(secs) = peer.last_sync_secs_ago + { + last_from_neighbour_secs = Some(last_from_neighbour_secs.map_or(secs, |s| s.min(secs))); } } @@ -313,7 +332,7 @@ async fn collect(state: &AppState) -> StatsResponse { archived_epochs: remakes.archived_epochs, }, sync: SyncStats { - last_from_neighbour_secs: last_from_neighbour.map(|at| at.elapsed().as_secs()), + last_from_neighbour_secs, diverged_peers, largest_received_bytes: node.largest_sync_received, }, @@ -342,6 +361,7 @@ async fn collect(state: &AppState) -> StatsResponse { stale_hello_replies: node.stale_hello_replies, traffic: state.traffic.total(), incidents, + clock, propagation: PropagationStats { count: node.propagation.count, sum_ms: node.propagation.sum_ms, diff --git a/src/api/stats/prometheus.rs b/src/api/stats/prometheus.rs index 690effa..0a1d75d 100644 --- a/src/api/stats/prometheus.rs +++ b/src/api/stats/prometheus.rs @@ -4,12 +4,12 @@ use std::fmt::Write; -use tokio::time::Instant; - use super::StatsResponse; -use crate::locks::MutexExt; -use crate::state::{AppState, Incidents, Liveness}; +use crate::api::media_type; +use crate::api::peers::{PeerListResponse, PeerLivenessStatus}; +use crate::state::Incidents; use crate::transport::Destination; +use crate::transport::paths::ClockOffset; use crate::transport::stats::{Reachability, TransportStats}; use crate::transport::traffic::{ByKind, Counter, PeerTraffic}; @@ -19,10 +19,10 @@ pub(super) const CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8" /// plain text; browsers and curl don't name either. pub(super) fn wanted(accept: Option<&str>) -> bool { accept.is_some_and(|accept| { - accept.split(',').any(|item| { - let media = item.split(';').next().unwrap_or("").trim(); - media == "application/openmetrics-text" || media == "text/plain" - }) + accept + .split(',') + .map(media_type) + .any(|media| media == "application/openmetrics-text" || media == "text/plain") }) } @@ -50,6 +50,32 @@ impl Out { let _ = writeln!(self.0, " {value}"); } + /// One histogram's series (its family declared before): each bucket counting what + /// fell into it and every one before (upper bounds in seconds, the last unbounded), + /// the sum (seconds) and the count. + fn histogram( + &mut self, + name: &str, + labels: &[(&str, &str)], + buckets: impl Iterator<Item = (Option<f64>, u64)>, + sum: f64, + count: u64, + ) { + let mut cumulative = 0; + for (le, in_bucket) in buckets { + cumulative += in_bucket; + let le = match le { + Some(seconds) => seconds.to_string(), + None => "+Inf".to_owned(), + }; + let mut with_le = labels.to_vec(); + with_le.push(("le", &le)); + self.sample(&format!("{name}_bucket"), &with_le, cumulative as f64); + } + self.sample(&format!("{name}_sum"), labels, sum); + self.sample(&format!("{name}_count"), labels, count as f64); + } + /// A family with one unlabelled sample. fn single(&mut self, name: &str, kind: &str, help: &str, value: f64) { self.family(name, kind, help); @@ -126,45 +152,30 @@ struct Peer { incidents: Incidents, throughput: Option<f64>, destination: Option<Destination>, + clock: Option<ClockOffset>, } -async fn peers(state: &AppState) -> Vec<Peer> { - let now = Instant::now(); - let snapshot = state.peers_snapshot().await; - let mut aliases = crate::swarm::members::aliases(&state.doc.read().await.doc); - let mut peers = Vec::with_capacity(snapshot.len()); - for (key, peer_state) in snapshot { - let neighbour = crate::swarm::topology::is_neighbour(state, &key); - let throughput = state - .throughput - .lock_or_recover() - .get(&key) - .map(|estimate| estimate.bytes_per_sec); - let traffic = state.traffic.peer(&key).unwrap_or_default(); - let peer_state = peer_state.lock().await; - peers.push(Peer { - neighbour, - unknown: peer_state.liveness == Liveness::Unknown, - dormant: matches!(peer_state.liveness, Liveness::Dormant { .. }), - last_seen_secs: peer_state - .last_seen - .map(|t| now.saturating_duration_since(t).as_secs_f64()), - last_sync_secs: peer_state - .last_sync_rx - .map(|t| now.saturating_duration_since(t).as_secs_f64()), - diverged_secs: peer_state - .diverged_since - .map(|t| now.saturating_duration_since(t).as_secs_f64()), - traffic, - incidents: peer_state.incidents, - throughput, - alias: aliases.remove(&key), - destination: state.transport.destination(&key), - key, - }); - } - peers.sort_by(|a, b| a.key.cmp(&b.key)); - peers +/// The rows of `/peers` (`peers::peer_list`), as the series need them. +fn peers(list: &PeerListResponse) -> Vec<Peer> { + let secs = |secs: Option<u64>| secs.map(|secs| secs as f64); + list.peers + .iter() + .map(|(key, peer)| Peer { + key: key.clone(), + alias: peer.alias.clone(), + neighbour: peer.neighbour, + unknown: matches!(peer.status, PeerLivenessStatus::Unknown), + dormant: matches!(peer.status, PeerLivenessStatus::Dormant), + last_seen_secs: secs(peer.last_seen_secs_ago), + last_sync_secs: secs(peer.last_sync_secs_ago), + diverged_secs: secs(peer.diverged_for_secs), + traffic: peer.traffic, + incidents: peer.incidents, + throughput: peer.sync_throughput.as_ref().map(|t| t.bytes_per_sec), + destination: peer.destination.clone(), + clock: peer.clock, + }) + .collect() } /// Every (direction, kind, counter) of `traffic`. @@ -179,6 +190,7 @@ fn traffic_series(traffic: &PeerTraffic) -> Vec<(&'static str, &'static str, Cou (direction, "blob", by.blob), (direction, "identity", by.identity), (direction, "paths", by.paths), + (direction, "links", by.links), ] }; let mut series = Vec::new(); @@ -203,6 +215,19 @@ fn transports(out: &mut Out, stats: &TransportStats) { *nodes as f64, ); } + out.family( + "identity_relayed_nodes", + "gauge", + "Proven nodes a transport reaches only through a relay, by transport.", + ); + for transport in identity.proven.keys() { + let relayed = identity.relayed.get(transport).copied().unwrap_or(0); + out.sample( + "identity_relayed_nodes", + &[("transport", transport)], + relayed as f64, + ); + } out.single( "identity_hinted_nodes", "gauge", @@ -245,7 +270,7 @@ fn transports(out: &mut Out, stats: &TransportStats) { out.family( "paths_probes_total", "counter", - "PINGs to paths of peers with several, by result.", + "PINGs to paths in use, by result.", ); for (result, count) in [ ("sent", counts.probes_sent), @@ -385,7 +410,8 @@ pub(super) fn render_relay_only(stats: &super::RelayOnlyStats) -> String { /// A per-peer incident counter: its name, its help and how to read it. type IncidentSeries = (&'static str, &'static str, fn(&Incidents) -> f64); -pub(super) async fn render(state: &AppState, stats: &StatsResponse) -> String { +/// `stats` and `list` (`/peers`) as series: nothing read anew. +pub(super) fn render(stats: &StatsResponse, list: &PeerListResponse) -> String { let mut out = Out::default(); // Names for joining (`* on(peer) group_left(alias) continuum_peer_info`), kept out @@ -416,7 +442,7 @@ pub(super) async fn render(state: &AppState, stats: &StatsResponse) -> String { "epoch_generation", "gauge", "Generation of the local epoch (remakes so far).", - state.local_epoch().generation as f64, + stats.epoch.generation as f64, ); let document = &stats.document; @@ -448,28 +474,14 @@ pub(super) async fn render(state: &AppState, stats: &StatsResponse) -> String { ); let lock_wait = &document.lock_wait; for (mode, waits) in [("read", &lock_wait.read), ("write", &lock_wait.write)] { - let mut cumulative = 0; - for (le_us, count) in waits.buckets() { - cumulative += count; - let le = match le_us { - Some(us) => (us as f64 / 1e6).to_string(), - None => "+Inf".to_owned(), - }; - out.sample( - "document_lock_wait_seconds_bucket", - &[("mode", mode), ("le", &le)], - cumulative as f64, - ); - } - out.sample( - "document_lock_wait_seconds_sum", + out.histogram( + "document_lock_wait_seconds", &[("mode", mode)], + waits + .buckets() + .map(|(le_us, count)| (le_us.map(|us| us as f64 / 1e6), count)), waits.sum_us as f64 / 1e6, - ); - out.sample( - "document_lock_wait_seconds_count", - &[("mode", mode)], - waits.count as f64, + waits.count, ); } out.family( @@ -732,38 +744,47 @@ pub(super) async fn render(state: &AppState, stats: &StatsResponse) -> String { out.family( "propagation_delay_seconds", "histogram", - "Time from a change's commit elsewhere until applied here (includes clock skew).", + "Time from a change's commit elsewhere until applied here, by the peers' clocks (includes the writer's clock skew).", ); - let mut cumulative = 0; - for bucket in &propagation.buckets { - cumulative += bucket.count; - let le = match bucket.le_ms { - Some(ms) => (ms as f64 / 1000.0).to_string(), - None => "+Inf".to_owned(), - }; - out.sample( - "propagation_delay_seconds_bucket", - &[("le", &le)], - cumulative as f64, - ); - } - out.sample( - "propagation_delay_seconds_sum", + out.histogram( + "propagation_delay_seconds", &[], + propagation + .buckets + .iter() + .map(|bucket| (bucket.le_ms.map(|ms| ms as f64 / 1000.0), bucket.count)), propagation.sum_ms as f64 / 1000.0, - ); - out.sample( - "propagation_delay_seconds_count", - &[], - propagation.count as f64, + propagation.count, ); out.single( "propagation_clock_ahead_total", "counter", - "Changes stamped later than this node's clock on arrival.", + "Changes stamped later than their arrival, by the peers' clocks.", propagation.ahead as f64, ); + let clock = &stats.clock; + out.single( + "clock_peers_compared", + "gauge", + "Active peers whose clocks were compared with ours (from their PONGs).", + clock.peers as f64, + ); + if let Some(offset_ms) = clock.offset_ms { + out.single( + "clock_offset_seconds", + "gauge", + "How far this node's clock is ahead of its peers' (behind: negative), the median.", + offset_ms as f64 / 1000.0, + ); + } + out.single( + "clock_off", + "gauge", + "1 while this node's clock differs from most of its peers' by more than --clock-skew-warn.", + flag(clock.off), + ); + // Node totals, including peers no longer listed. let totals = traffic_series(&stats.traffic); out.family( @@ -793,7 +814,7 @@ pub(super) async fn render(state: &AppState, stats: &StatsResponse) -> String { transports(&mut out, &stats.transports); - let peers = peers(state).await; + let peers = peers(list); out.family( "peer_info", @@ -838,7 +859,7 @@ pub(super) async fn render(state: &AppState, stats: &StatsResponse) -> String { out.family( "peer_path_rtt_seconds", "gauge", - "Smoothed round-trip time of the PINGs on each path to a peer with several.", + "Smoothed round-trip time of the PINGs on each path to a peer.", ); for peer in &peers { for path in peer.destination.iter().flat_map(|d| &d.paths) { @@ -932,6 +953,35 @@ pub(super) async fn render(state: &AppState, stats: &StatsResponse) -> String { } out.family( + "peer_clock_offset_seconds", + "gauge", + "How far the peer's clock is ahead of ours (behind: negative), from its PONGs.", + ); + for peer in &peers { + if let Some(clock) = peer.clock { + out.sample( + "peer_clock_offset_seconds", + &[("peer", &peer.key)], + clock.offset_ms as f64 / 1000.0, + ); + } + } + out.family( + "peer_clock_offset_within_seconds", + "gauge", + "Give or take this much: half the round trip it was measured over.", + ); + for peer in &peers { + if let Some(clock) = peer.clock { + out.sample( + "peer_clock_offset_within_seconds", + &[("peer", &peer.key)], + clock.within_ms as f64 / 1000.0, + ); + } + } + + out.family( "peer_messages_total", "counter", "Messages by direction and kind.", diff --git a/src/api/store.rs b/src/api/store.rs index d0fc807..2b21c83 100644 --- a/src/api/store.rs +++ b/src/api/store.rs @@ -2,27 +2,30 @@ use std::io::Error as IoError; -use automerge::ChangeHash; use axum::{ Extension, body::{Body, Bytes}, - extract::{Path, Query, State}, + extract::{Path, Query, RawQuery, State}, http::{ HeaderMap, HeaderValue, StatusCode, - header::{ACCEPT, CONTENT_TYPE}, + header::{ACCEPT, CONTENT_TYPE, ETAG, LOCATION, VARY}, }, response::{IntoResponse, Response}, }; use serde::Deserialize; use serde_json::Value as JsonValue; +use sha2::{Digest, Sha256}; use tokio::sync::broadcast; -use tracing::{debug, error, instrument}; +use tracing::{debug, error, instrument, warn}; use super::auth::AuthenticatedUser; -use super::{parse_key_path, parse_write_path, response_for_value, store_status}; -use crate::document::store::{self, StoreError}; +use super::{ApiError, store_status}; +use crate::document::ChangeHash; +use crate::document::meta; +use crate::document::store::{self, StoreError, paths_related}; +use crate::document::typed::{self, JsonForm, Kind, TimestampFormat, Val}; use crate::state::{AppState, ReadSource}; -use crate::swarm::sync::paths_related; +use crate::transport::Hex; #[derive(Debug, Default, Deserialize)] pub(crate) struct ReadQuery { @@ -32,6 +35,60 @@ pub(crate) struct ReadQuery { /// to base a replacing remake on exactly what was read. #[serde(default)] heads: bool, + /// What's there, one level deep (`store::read_shallow`), to browse a large store a + /// level at a time. + #[serde(default)] + shallow: bool, + /// Read as the document was at these heads (comma-separated hex, as + /// `x-continuum-heads` gives them), not as it is now (`document::at`). + at: Option<String>, +} + +impl ReadQuery { + /// The heads `at` names, if any; `400` if it doesn't name them. + fn at(&self) -> Result<Option<Vec<ChangeHash>>, ApiError> { + let Some(at) = &self.at else { + return Ok(None); + }; + at.split(',') + .map(|head| head.trim().parse::<ChangeHash>()) + .collect::<Result<Vec<_>, _>>() + .map(Some) + .map_err(|_| ApiError(StatusCode::BAD_REQUEST, Some("at: not heads".to_owned()))) + } +} + +/// Reads `doc` with `read`, as it is or, with `at`, as it was at those heads +/// (`UnknownHeads` if it lacks one). +macro_rules! read_at { + ($doc:expr, $at:expr, |$d:ident| $read:expr) => { + match $at { + None => { + let $d = $doc; + $read + } + Some(heads) => match crate::document::at::At::new($doc, heads) { + Some(at) => { + let $d = &at; + $read + } + None => Err(From::from(StoreError::UnknownHeads)), + }, + } + }; +} + +#[derive(Debug, Deserialize)] +pub(crate) struct PostQuery { + /// Adds this to the counter at the path instead of inserting the body + /// (`store::increment_path`). + increment: Option<i64>, + /// Casts what is at the path to this type instead of inserting the body + /// (`store::cast_path`; `typed::Kind::name` names it). + cast: Option<String>, + /// With `cast`, how timestamps are read from and written as strings (`rfc3339` by + /// default; `typed::TimestampFormat`). + format: Option<String>, } #[derive(Debug, Deserialize)] @@ -40,81 +97,163 @@ pub(crate) struct DeleteQuery { prune: bool, } -fn conflicts_response( - value: JsonValue, - conflicts: Vec<store::ConflictReport>, -) -> Result<Response, StatusCode> { - let body = serde_json::json!({ "value": value, "conflicts": conflicts }); - response_for_value(&body, None) // always JSON -} - /// `GET /store`: the whole store (`docs/openapi.yaml`). pub(crate) async fn get_store_root( State(state): State<AppState>, headers: HeaderMap, Query(query): Query<ReadQuery>, -) -> Result<Response, StatusCode> { +) -> Result<Response, ApiError> { + read(&state, Vec::new(), &headers, &query).await +} + +/// `GET /store/{key}`. +pub(crate) async fn get_store_entry( + State(state): State<AppState>, + Path(key): Path<String>, + headers: HeaderMap, + Query(query): Query<ReadQuery>, + RawQuery(raw_query): RawQuery, +) -> Result<Response, ApiError> { + let path = parse_key_path(&key)?; + let at = query.at()?; + if let Some(redirect) = + to_positive(&state, &path, false, at.as_deref(), raw_query.as_deref()).await? + { + return Ok(redirect); + } + read(&state, path, &headers, &query).await +} + +/// A read of `path` as `query` asks (the empty path: the whole store). The root is read +/// while not ready too (`render_read`'s `unready_live`), and is never missing: an empty +/// store is an empty map. +async fn read( + state: &AppState, + path: Vec<String>, + headers: &HeaderMap, + query: &ReadQuery, +) -> Result<Response, ApiError> { + let root = path.is_empty(); + if query.shallow { + return shallow(state, path, root, query).await; + } let accept = headers.get(ACCEPT).cloned(); let want_conflicts = query.conflicts; + let at = query.at()?; let (response, source, heads) = state .render_read( - true, + root, query.heads, move |doc| { - if want_conflicts { - store::read_path_with_conflicts(doc, &Vec::new()) - .map(|found| found.map(|(value, conflicts)| (value, Some(conflicts)))) - } else { - store::read_path(doc, &Vec::new()).map(|found| found.map(|value| (value, None))) - } + read_at!(doc, at.as_deref(), |doc| read_shown( + doc, + &path, + accept.as_ref(), + want_conflicts + )) }, - move |result| { - let (value, conflicts) = result - .map_err(|e| store_status(e, "Failed to read store path"))? - .unwrap_or_else(|| (JsonValue::Object(Default::default()), Some(Vec::new()))); - match conflicts { - Some(conflicts) => conflicts_response(value, conflicts), - None => response_for_value(&value, accept.as_ref()), + move |result| match result { + Ok(Some(read)) => read.respond(), + Ok(None) if root => Read { + held: store::Held { + kind: Kind::Map, + value: None, + }, + shown: Shown::Json(JsonForm::default()), + json: Some(JsonValue::Object(Default::default())), + conflicts: want_conflicts.then(Vec::new), } + .respond(), + Ok(None) => Err(StatusCode::NOT_FOUND.into()), + Err(error) => Err(error), }, ) .await?; mark_source(response, source, heads) } -/// `GET /store/{key}`. -pub(crate) async fn get_store_entry( - State(state): State<AppState>, - Path(key): Path<String>, - headers: HeaderMap, - Query(query): Query<ReadQuery>, -) -> Result<Response, StatusCode> { - let path = parse_key_path(&key)?; +/// A value read, to show as asked (`Shown`). +struct Read { + held: store::Held, + shown: Shown, + /// Its JSON, of the form asked, unless shown natively. + json: Option<JsonValue>, + /// With `?conflicts=true`: those on the way (then always JSON). + conflicts: Option<Vec<store::ConflictReport>>, +} - let accept = headers.get(ACCEPT).cloned(); - let want_conflicts = query.conflicts; +impl Read { + fn respond(self) -> Result<Response, ApiError> { + if let Some(conflicts) = self.conflicts { + let value = self.json.unwrap_or(JsonValue::Null); + let body = serde_json::json!({ "value": value, "conflicts": conflicts }); + return show(&self.held, Shown::Json(JsonForm::default()), Some(&body)); + } + show(&self.held, self.shown, self.json.as_ref()) + } +} + +/// What `path` holds, read to show as `accept` asks; `406` if it can't be. +fn read_shown( + doc: &impl crate::document::ReadDoc, + path: &[String], + accept: Option<&HeaderValue>, + conflicts: bool, +) -> Result<Option<Read>, ApiError> { + let failed = |e| store_status(e, "Failed to read store path"); + let Some(held) = store::read_held(doc, path).map_err(failed)? else { + return Ok(None); + }; + let shown = if conflicts { + Shown::Json(json_form(accept)?) + } else { + shown(accept, held.kind)? + }; + let (json, conflicts) = match (shown, conflicts) { + (Shown::Native(_), _) => (None, None), + (Shown::Json(form), true) => { + match store::read_path_with_conflicts_as(doc, path, form).map_err(failed)? { + Some((json, conflicts)) => (Some(json), Some(conflicts)), + None => return Ok(None), + } + } + (Shown::Json(form), false) => (store::read_path_as(doc, path, form).map_err(failed)?, None), + (Shown::Text, _) => ( + store::read_path_as(doc, path, JsonForm::default()).map_err(failed)?, + None, + ), + }; + Ok(Some(Read { + held, + shown, + json, + conflicts, + })) +} +/// `?shallow=true`: what's at `path`, one level deep, as JSON (`store::read_shallow`); +/// with conflicts asked for too, `400`. `unready_live` as `render_read` has it. +async fn shallow( + state: &AppState, + path: Vec<String>, + unready_live: bool, + query: &ReadQuery, +) -> Result<Response, ApiError> { + if query.conflicts { + return Err(StatusCode::BAD_REQUEST.into()); + } + let at = query.at()?; let (response, source, heads) = state .render_read( - false, + unready_live, query.heads, - move |doc| { - if want_conflicts { - store::read_path_with_conflicts(doc, &path) - .map(|found| found.map(|(value, conflicts)| (value, Some(conflicts)))) - } else { - store::read_path(doc, &path).map(|found| found.map(|value| (value, None))) - } - }, + move |doc| read_at!(doc, at.as_deref(), |doc| store::read_shallow(doc, &path)), move |result| { - let (value, conflicts) = result + let value = result .map_err(|e| store_status(e, "Failed to read store path"))? .ok_or(StatusCode::NOT_FOUND)?; - match conflicts { - Some(conflicts) => conflicts_response(value, conflicts), - None => response_for_value(&value, accept.as_ref()), - } + respond(None, "application/json", json_bytes(&value)?) }, ) .await?; @@ -125,24 +264,30 @@ pub(crate) async fn get_store_entry( const STALE_HEADER: &str = "x-continuum-stale"; /// Header with the document's heads at the time of the read (sorted, comma-separated -/// hex), on request (`?heads=true`). +/// hex), on request (`?heads=true`); and on every write, after it: a read with these +/// (`?at=`) includes the write. const HEADS_HEADER: &str = "x-continuum-heads"; +fn with_heads(mut response: Response, heads: &[ChangeHash]) -> Response { + let joined = heads + .iter() + .map(ToString::to_string) + .collect::<Vec<_>>() + .join(","); + if let Ok(value) = HeaderValue::from_str(&joined) { + response.headers_mut().insert(HEADS_HEADER, value); + } + response +} + fn mark_source( - response: Result<Response, StatusCode>, + response: Result<Response, ApiError>, source: ReadSource, heads: Option<Vec<ChangeHash>>, -) -> Result<Response, StatusCode> { +) -> Result<Response, ApiError> { let mut response = response?; if let Some(heads) = heads { - let joined = heads - .iter() - .map(ToString::to_string) - .collect::<Vec<_>>() - .join(","); - if let Ok(value) = HeaderValue::from_str(&joined) { - response.headers_mut().insert(HEADS_HEADER, value); - } + response = with_heads(response, &heads); } if let ReadSource::Stale(epoch) = source && let Ok(value) = HeaderValue::from_str(&epoch.to_string()) @@ -152,46 +297,431 @@ fn mark_source( Ok(response) } -/// `PUT /store/{key}`. +#[derive(Debug, Deserialize)] +pub(crate) struct PutQuery { + /// Whether the write may change the type of what it writes over (`true`, the + /// default); if not, one that would is `409` (`store::Writing::retype`). + #[serde(default = "yes")] + retype: bool, +} + +fn yes() -> bool { + true +} + +/// Header listing the places whose type a write changed (`store::Retyped`), as +/// `<path>: <from> -> <to>`, comma-separated, `MAX_RETYPED_LISTED` at most (then `…`). +const RETYPED_HEADER: &str = "x-continuum-retyped"; +const MAX_RETYPED_LISTED: usize = 16; + +/// `PUT /store/{key}`: answered with what is there after the write, shown as `Accept` +/// asks (read back: with sticky types, a JSON value may be stored as another type). pub(crate) async fn put_store_entry( State(state): State<AppState>, Extension(user): Extension<AuthenticatedUser>, Path(key): Path<String>, + Query(query): Query<PutQuery>, + RawQuery(raw_query): RawQuery, headers: HeaderMap, body: Bytes, -) -> Result<Response, StatusCode> { +) -> Result<Response, ApiError> { if !state.is_ready().await { - return Err(StatusCode::SERVICE_UNAVAILABLE); + return Err(StatusCode::SERVICE_UNAVAILABLE.into()); } let path = parse_write_path(&state, &user, &key, "PUT")?; + if let Some(redirect) = to_positive(&state, &path, false, None, raw_query.as_deref()).await? { + return Ok(redirect); + } + let body = body_value(&headers, &body)?; + let accept = headers.get(ACCEPT).cloned(); + let writing = store::Writing::new(body.sticky, body.timestamps, query.retype); + + // A value that can't be shown as asked is refused, and the write with it. + let ((retyped, read), heads) = state + .mutate_doc_for(Some(user.0.clone()), move |doc| { + let retyped = store::write_val_path(doc, &path, &body.value, writing) + .map_err(|e| store_status(e, "Failed to write store value"))?; + let read = read_shown(&*doc, &path, accept.as_ref(), false)?; + Ok::<_, ApiError>((retyped, read)) + }) + .await?; + let read = read.ok_or(ApiError(StatusCode::INTERNAL_SERVER_ERROR, None))?; + + Ok(with_retyped(with_heads(read.respond()?, &heads), &retyped)) +} + +/// `response` saying where a write changed types (`RETYPED_HEADER`), if it did. +fn with_retyped(mut response: Response, retyped: &[store::Retyped]) -> Response { + if !retyped.is_empty() { + let mut listed: Vec<String> = retyped + .iter() + .take(MAX_RETYPED_LISTED) + .map(ToString::to_string) + .collect(); + if retyped.len() > MAX_RETYPED_LISTED { + listed.push("…".to_owned()); + } + if let Ok(value) = HeaderValue::from_str(&listed.join(", ")) { + response.headers_mut().insert(RETYPED_HEADER, value); + } + } + response +} + +fn etag_for(bytes: &[u8]) -> String { + format!("\"{}\"", Hex(&Sha256::digest(bytes))) +} + +fn parse_key_path(key: &str) -> Result<Vec<String>, StatusCode> { + let segments: Vec<String> = key.split('/').map(str::to_owned).collect(); + if segments.is_empty() || segments.iter().any(|segment| segment.is_empty()) { + return Err(StatusCode::BAD_REQUEST); + } + + Ok(segments) +} + +/// A path for a write. Protocol data (`_continuum`: remake records, members, forget +/// records, epoch metadata) is what every node acts on, so a write there acts on the +/// whole swarm: only users listed in `--protocol-writers` may, and each such write is +/// logged. Epoch metadata (and `_continuum` as a whole, which contains it) is never +/// writable: the protocol reads it at the snapshot change, so an edit only misleads +/// whoever reads it later. `403` otherwise. +fn parse_write_path( + state: &AppState, + user: &AuthenticatedUser, + key: &str, + method: &str, +) -> Result<Vec<String>, StatusCode> { + let path = parse_key_path(key)?; + if !meta::is_protocol(&path) { + return Ok(path); + } + // `_continuum` itself, epoch metadata and the swarm ID: nobody. + let fixed = path.len() < 2 || path[1] == meta::EPOCHS || path[1] == meta::SWARM; + if fixed || !state.protocol_writers.contains(&user.0) { + warn!(user = %user.0, method, path = %key, "Refused a write to protocol data"); + return Err(StatusCode::FORBIDDEN); + } + warn!(user = %user.0, method, path = %key, "Protocol data written through the store API; this acts on the whole swarm"); + Ok(path) +} + +/// How a value is shown, as the request's `Accept` asks. +#[derive(Clone, Copy, Debug)] +enum Shown { + Json(JsonForm), + /// `text/plain`: a string as it is, anything else as JSON. + Text, + /// Natively, as its type's own media type (`typed::native`). + Native(TimestampFormat), +} + +/// The JSON form `accept` asks for (`application/json; timestamp-format=…`): `400` for +/// a timestamp format there is none of. +fn json_form(accept: Option<&HeaderValue>) -> Result<JsonForm, ApiError> { + for (media, params) in accepted(accept) { + match media.as_str() { + "application/json" => return json_form_of(params), + typed::TYPED_JSON => { + return Ok(JsonForm { + typed: true, + ..json_form_of(params)? + }); + } + _ => {} + } + } + Ok(JsonForm::default()) +} + +fn json_form_of(params: &str) -> Result<JsonForm, ApiError> { + let timestamps = match typed::param(params, "timestamp-format") { + None => TimestampFormat::default(), + Some(name) => TimestampFormat::named(&name).ok_or_else(|| { + ApiError( + StatusCode::BAD_REQUEST, + Some(format!("no timestamp format {name:?}")), + ) + })?, + }; + Ok(JsonForm { + timestamps, + typed: false, + }) +} + +/// The media types of `accept`, in its order, each with its parameters (q-values are not +/// weighed). Without an `Accept`, none. +fn accepted(accept: Option<&HeaderValue>) -> Vec<(String, &str)> { + let Some(accept) = accept.and_then(|value| value.to_str().ok()) else { + return Vec::new(); + }; + accept + .split(',') + .map(|item| { + let (media, params) = item.split_once(';').unwrap_or((item, "")); + (media.trim().to_ascii_lowercase(), params) + }) + .collect() +} + +/// How a value of `kind` is shown, as `accept` asks: the first it names that the value +/// can be shown as. JSON without an `Accept`, or for `*/*`; `406` naming what it can be +/// shown as if `accept` names none of them. +fn shown(accept: Option<&HeaderValue>, kind: Kind) -> Result<Shown, ApiError> { + let items = accepted(accept); + if items.is_empty() { + return Ok(Shown::Json(JsonForm::default())); + } + let native = kind.media_type(); + for (media, params) in &items { + match media.as_str() { + "application/json" => return Ok(Shown::Json(json_form_of(params)?)), + typed::TYPED_JSON => { + return Ok(Shown::Json(JsonForm { + typed: true, + ..json_form_of(params)? + })); + } + typed::TEXT => return Ok(Shown::Text), + "*/*" | "application/*" => return Ok(Shown::Json(JsonForm::default())), + media if Some(media) == native => { + let format = match typed::param(params, "format") { + None => TimestampFormat::default(), + Some(name) => TimestampFormat::named(&name).ok_or_else(|| { + ApiError( + StatusCode::BAD_REQUEST, + Some(format!("no timestamp format {name:?}")), + ) + })?, + }; + return Ok(Shown::Native(format)); + } + _ => {} + } + } + let mut offered = vec!["application/json", typed::TYPED_JSON, typed::TEXT]; + if let Some(native) = native.filter(|native| *native != typed::TEXT) { + offered.push(native); + } + Err(ApiError( + StatusCode::NOT_ACCEPTABLE, + Some(format!( + "a {} is shown as {}", + kind.name(), + offered.join(", ") + )), + )) +} + +/// The response showing `held` as `shown`: `json` is its JSON, of the form `shown` asks, +/// where it isn't shown natively. +fn show(held: &store::Held, shown: Shown, json: Option<&JsonValue>) -> Result<Response, ApiError> { + let json = || json.ok_or(ApiError(StatusCode::INTERNAL_SERVER_ERROR, None)); + match shown { + Shown::Json(form) => { + let media = if form.typed { + typed::TYPED_JSON + } else { + "application/json" + }; + respond(Some(held.kind), media, json_bytes(json()?)?) + } + Shown::Text => { + let body = match json()? { + JsonValue::String(value) => value.clone().into_bytes(), + other => json_bytes(other)?, + }; + respond(Some(held.kind), "text/plain; charset=utf-8", body) + } + Shown::Native(format) => { + let (content_type, body) = held + .value + .as_ref() + .and_then(|value| typed::native(value, format)) + .ok_or(ApiError(StatusCode::INTERNAL_SERVER_ERROR, None))?; + respond(Some(held.kind), &content_type, body) + } + } +} + +fn json_bytes(value: &JsonValue) -> Result<Vec<u8>, ApiError> { + serde_json::to_vec(value).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR.into()) +} + +/// A response of `body` as `content_type`, with its `ETag`, and what the value is +/// stored as (`x-continuum-type`), if it is one. +fn respond(kind: Option<Kind>, content_type: &str, body: Vec<u8>) -> Result<Response, ApiError> { + let etag = etag_for(&body); + let mut response = body.into_response(); + let headers = response.headers_mut(); + headers.insert( + CONTENT_TYPE, + HeaderValue::from_str(content_type).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?, + ); + headers.insert( + ETAG, + HeaderValue::from_str(&etag).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?, + ); + if let Some(kind) = kind { + headers.insert(TYPE_HEADER, HeaderValue::from_static(kind.name())); + headers.insert(VARY, HeaderValue::from_static("Accept")); + } + Ok(response) +} + +/// Header naming what a value read or written is stored as (`typed::Kind::name`). +const TYPE_HEADER: &str = "x-continuum-type"; + +/// A write's body: the value to write (`typed::Val`), from JSON (the default), typed +/// JSON, text as a string, or a scalar of a type of its own (`typed::from_native`). +struct WriteBody { + value: Val, + /// Which of its values keep the types of what they are written over + /// (`store::Sticky`): plain JSON's and text's where they can, typed JSON's numbers + /// over floats. + sticky: store::Sticky, + /// How timestamps are written in it (`timestamp-format`). + timestamps: TimestampFormat, +} + +fn body_value(headers: &HeaderMap, body: &Bytes) -> Result<WriteBody, ApiError> { let content_type = headers .get(CONTENT_TYPE) .and_then(|value| value.to_str().ok()) - .unwrap_or("application/json") - .split(';') - .next() - .unwrap_or("") - .trim(); - - let value = match content_type { - "application/json" => { - serde_json::from_slice::<JsonValue>(&body).map_err(|_| StatusCode::BAD_REQUEST)? - } - "text/plain" => JsonValue::String( - String::from_utf8(body.to_vec()).map_err(|_| StatusCode::BAD_REQUEST)?, + .unwrap_or("application/json"); + let (media, params) = content_type.split_once(';').unwrap_or((content_type, "")); + let media = media.trim().to_ascii_lowercase(); + let bad = |error: String| ApiError(StatusCode::BAD_REQUEST, Some(error)); + let json = || { + serde_json::from_slice::<JsonValue>(body).map_err(|error| bad(format!("not JSON: {error}"))) + }; + let timestamps = json_form_of(params)?.timestamps; + let (value, sticky) = match media.as_str() { + "application/json" => (Val::from_json(&json()?).map_err(bad)?, store::Sticky::All), + typed::TYPED_JSON => ( + Val::from_typed_json(&json()?, timestamps).map_err(bad)?, + store::Sticky::Floats, + ), + typed::TEXT => ( + String::from_utf8(body.to_vec()) + .map(|text| Val::Scalar(automerge::ScalarValue::Str(text.into()))) + .map_err(|_| bad("not UTF-8".to_owned()))?, + store::Sticky::All, ), - _ => return Err(StatusCode::UNSUPPORTED_MEDIA_TYPE), + media => match typed::from_native(media, params, body) { + Some(Ok(scalar)) => (Val::Scalar(scalar), store::Sticky::No), + Some(Err(error)) => return Err(bad(error)), + None => return Err(StatusCode::UNSUPPORTED_MEDIA_TYPE.into()), + }, }; + Ok(WriteBody { + value, + sticky, + timestamps, + }) +} - let written = value.clone(); - state - .mutate_doc(move |doc| store::write_path(doc, &path, &written)) - .await - .map_err(|e| store_status(e, "Failed to write store value"))?; - - response_for_value(&value, headers.get(ACCEPT)) +/// `POST /store/{key}`: inserts the body into a list, `key` ending in the index it +/// goes to (`store::insert_path`: the items from there on move up; the length, or `-1`, +/// appends); `201` with where it went (`{"index": n}`). Not a list there, or an index +/// past its end: `409`, saying which. With `?increment=n`, adds `n` to the counter at +/// `key` instead (`store::increment_path`), answering with its value then; not a +/// counter there: `409`. With `?cast=<type>`, casts what is at `key` to that type +/// instead (`store::cast_path`), answering with it then, as `PUT` does; no cast +/// possible: `409`. +pub(crate) async fn post_store_entry( + State(state): State<AppState>, + Extension(user): Extension<AuthenticatedUser>, + Path(key): Path<String>, + Query(query): Query<PostQuery>, + RawQuery(raw_query): RawQuery, + headers: HeaderMap, + body: Bytes, +) -> Result<Response, ApiError> { + if !state.is_ready().await { + return Err(StatusCode::SERVICE_UNAVAILABLE.into()); + } + let path = parse_write_path(&state, &user, &key, "POST")?; + if query.increment.is_some() && query.cast.is_some() { + return Err(ApiError( + StatusCode::BAD_REQUEST, + Some("increment or cast, not both".to_owned()), + )); + } + // An increment or a cast names an item (`-1`: the last); an insert where it goes + // (`-1`: the end). + let insertion = query.increment.is_none() && query.cast.is_none(); + if let Some(redirect) = + to_positive(&state, &path, insertion, None, raw_query.as_deref()).await? + { + return Ok(redirect); + } + if let Some(by) = query.increment { + let (value, heads) = state + .mutate_doc_for(Some(user.0.clone()), move |doc| { + store::increment_path(doc, &path, by) + }) + .await + .map_err(|e| store_status(e, "Failed to increment a counter"))?; + return Ok(with_heads( + { + let counter = automerge::ScalarValue::counter(value); + let held = store::Held { + kind: Kind::Counter, + value: Some(counter), + }; + show( + &held, + shown(headers.get(ACCEPT), Kind::Counter)?, + Some(&JsonValue::from(value)), + )? + }, + &heads, + )); + } + if let Some(name) = &query.cast { + let bad = |message: String| ApiError(StatusCode::BAD_REQUEST, Some(message)); + let to = Kind::named(name).ok_or_else(|| bad(format!("no type {name:?}")))?; + let timestamps = match &query.format { + None => TimestampFormat::default(), + Some(format) => TimestampFormat::named(format) + .ok_or_else(|| bad(format!("no timestamp format {format:?}")))?, + }; + let accept = headers.get(ACCEPT).cloned(); + // A value that can't be shown as asked is refused, and the cast with it. + let ((retyped, read), heads) = state + .mutate_doc_for(Some(user.0.clone()), move |doc| { + let retyped = store::cast_path(doc, &path, to, timestamps) + .map_err(|e| store_status(e, "Failed to cast a store value"))?; + let read = read_shown(&*doc, &path, accept.as_ref(), false)?; + Ok::<_, ApiError>((retyped, read)) + }) + .await?; + let read = read.ok_or(ApiError(StatusCode::INTERNAL_SERVER_ERROR, None))?; + return Ok(with_retyped(with_heads(read.respond()?, &heads), &retyped)); + } + let index = path.last().and_then(|leaf| store::list_index(leaf)); + let value = body_value(&headers, &body)?; + let inserted = state + .mutate_doc_for(Some(user.0.clone()), move |doc| { + store::insert_val_path(doc, &path, &value.value) + }) + .await; + match inserted { + Ok(((), heads)) => Ok(with_heads( + ( + StatusCode::CREATED, + axum::Json(serde_json::json!({ "index": index })), + ) + .into_response(), + &heads, + )), + Err(error) => Err(store_status(error, "Failed to insert into a list")), + } } /// `DELETE /store/{key}`. @@ -201,26 +731,89 @@ pub(crate) async fn delete_store_entry( Extension(user): Extension<AuthenticatedUser>, Path(key): Path<String>, Query(query): Query<DeleteQuery>, -) -> Result<StatusCode, StatusCode> { + RawQuery(raw_query): RawQuery, +) -> Result<Response, ApiError> { let path = parse_write_path(&state, &user, &key, "DELETE")?; tracing::Span::current().record("path", path.join("/").as_str()); if !state.is_ready().await { - return Err(StatusCode::SERVICE_UNAVAILABLE); + return Err(StatusCode::SERVICE_UNAVAILABLE.into()); } if path.is_empty() { - return Err(StatusCode::BAD_REQUEST); + return Err(StatusCode::BAD_REQUEST.into()); + } + + if let Some(redirect) = to_positive(&state, &path, false, None, raw_query.as_deref()).await? { + return Ok(redirect); } let prune = query.prune; - state - .mutate_doc(move |doc| store::delete_path(doc, &path, prune)) + let ((), heads) = state + .mutate_doc_for(Some(user.0.clone()), move |doc| { + store::delete_path(doc, &path, prune) + }) .await .map_err(|e| store_status(e, "Failed to delete store value"))?; - Ok(StatusCode::NO_CONTENT) + Ok(with_heads(StatusCode::NO_CONTENT.into_response(), &heads)) +} + +/// A path with negative list indexes (`-1` the last item; for an insert, the end): a +/// `307` to it with them made positive (`store::positive_path`), the query kept, so a +/// client reads, writes or deletes what the index names now. None without any. +async fn to_positive( + state: &AppState, + path: &[String], + insertion: bool, + at: Option<&[ChangeHash]>, + query: Option<&str>, +) -> Result<Option<Response>, ApiError> { + if !path.iter().any(|segment| segment.starts_with('-')) { + return Ok(None); + } + let positive = { + let replica = state.doc.read().await; + // Counted from the end of the list as it was then, for a read at heads. + read_at!(&replica.doc, at, |doc| store::positive_path( + doc, path, insertion + )) + } + .map_err(|e| store_status(e, "Failed to resolve a negative index"))?; + let Some(positive) = positive else { + return Ok(None); + }; + let mut location = format!( + "/store/{}", + positive + .iter() + .map(|segment| encode_segment(segment)) + .collect::<Vec<_>>() + .join("/") + ); + if let Some(query) = query.filter(|query| !query.is_empty()) { + location.push('?'); + location.push_str(query); + } + let location = HeaderValue::from_str(&location).map_err(|_| StatusCode::BAD_REQUEST)?; + Ok(Some( + (StatusCode::TEMPORARY_REDIRECT, [(LOCATION, location)]).into_response(), + )) +} + +/// `segment` as a URL path segment: unreserved characters as they are, the rest +/// percent-encoded (UTF-8). +fn encode_segment(segment: &str) -> String { + let mut encoded = String::with_capacity(segment.len()); + for byte in segment.bytes() { + if byte.is_ascii_alphanumeric() || b"-._~".contains(&byte) { + encoded.push(char::from(byte)); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + encoded } // --- streams ----------------------------------------------------------------- @@ -239,9 +832,7 @@ async fn render_traced( /// `GET /stream`: follows the whole store, as `GET /store` reads it (events have an /// empty `key`). -pub(crate) async fn stream_store_root( - State(state): State<AppState>, -) -> Result<Response, StatusCode> { +pub(crate) async fn stream_store_root(State(state): State<AppState>) -> Result<Response, ApiError> { stream(state, String::new(), Vec::new()).await } @@ -249,16 +840,16 @@ pub(crate) async fn stream_store_root( pub(crate) async fn stream_store_entry( State(state): State<AppState>, Path(key): Path<String>, -) -> Result<Response, StatusCode> { +) -> Result<Response, ApiError> { let path = parse_key_path(&key)?; stream(state, key, path).await } /// Newline-delimited JSON events for the value at `path` (`key` as the client wrote it): /// the current value, then one whenever the resolved value changes. -async fn stream(state: AppState, key: String, path: Vec<String>) -> Result<Response, StatusCode> { +async fn stream(state: AppState, key: String, path: Vec<String>) -> Result<Response, ApiError> { if !state.is_ready().await { - return Err(StatusCode::SERVICE_UNAVAILABLE); + return Err(StatusCode::SERVICE_UNAVAILABLE.into()); } // Subscribe first, so nothing between the initial read and the loop is missed. diff --git a/src/config/file.rs b/src/config/file.rs index b30d147..bd6a3d5 100644 --- a/src/config/file.rs +++ b/src/config/file.rs @@ -2,9 +2,10 @@ //! configuration, with the same keys: the long option names. Nested tables join their //! keys with `-`, so `[mycelium] key-file = …` is `mycelium-key-file`. //! -//! `include = [paths]` and `include-optional = [paths]` load other files first -//! (relative to this one); this file's own values win over theirs, and later includes -//! over earlier ones. A missing optional include is skipped; a cycle is an error. +//! `include = [paths]` loads other files first (relative to this one), in the order +//! listed; an entry `{ path = "…", optional = true }` is skipped where missing. This +//! file's own values win over theirs, and later includes over earlier ones. A cycle is +//! an error. //! //! `unset = [options]` sets options back to their defaults, overriding includes and //! earlier files; in JSON, `null` does the same for one option. @@ -16,11 +17,11 @@ use serde_json::{Map as JsonMap, Value as JsonValue}; use tracing::debug; use super::import; -use super::{Entry, Layer, Overlay, Provenance, overlay, spec, unset_entry}; +use super::{Entry, Layer, Overlay, Provenance, UNSET, overlay, spec, unset_entry}; const INCLUDE: &str = "include"; +/// Gone: optional includes are entries of `include`, so the order is the list's. const INCLUDE_OPTIONAL: &str = "include-optional"; -const UNSET: &str = "unset"; /// The settings `path` makes, its includes resolved. pub(super) fn load(path: &Path) -> Result<Layer> { @@ -40,17 +41,18 @@ fn load_inner(path: &Path, loading: &mut Vec<PathBuf>) -> Result<Layer> { .unwrap_or_else(|| PathBuf::from("/")); let mut root = parse(&canonical)?; - let includes = take_paths(&mut root, INCLUDE, &canonical)?; - let optional = take_paths(&mut root, INCLUDE_OPTIONAL, &canonical)?; + if root.contains_key(INCLUDE_OPTIONAL) { + bail!( + "{}: include-optional is gone; list the file in include as {{ path = \"…\", optional = true }}, where it goes in the order", + canonical.display() + ); + } + let includes = take_includes(&mut root, &canonical)?; let unsets = take_paths(&mut root, UNSET, &canonical)?; loading.push(canonical.clone()); let mut layer = Layer::new(); - for (include, required) in includes - .into_iter() - .map(|p| (p, true)) - .chain(optional.into_iter().map(|p| (p, false))) - { + for (include, required) in includes { let include = dir.join(include); if !required && !include.exists() { debug!(file = %include.display(), "Optional configuration file is missing; skipped"); @@ -118,6 +120,50 @@ fn parse(path: &Path) -> Result<JsonMap<String, JsonValue>> { } } +/// The files `include` names, in its order, and whether each is required: a path, or +/// `{ path = "…", optional = true }`. +fn take_includes( + root: &mut JsonMap<String, JsonValue>, + path: &Path, +) -> Result<Vec<(String, bool)>> { + let items = match root.remove(INCLUDE) { + None => return Ok(Vec::new()), + Some(JsonValue::Array(items)) => items, + Some(single) => vec![single], + }; + items + .into_iter() + .map(|item| match item { + JsonValue::String(file) => Ok((file, true)), + JsonValue::Object(mut entry) => { + let file = match entry.remove("path") { + Some(JsonValue::String(file)) => file, + _ => bail!("{}: an include entry needs a path", path.display()), + }; + let optional = match entry.remove("optional") { + None => false, + Some(JsonValue::Bool(optional)) => optional, + Some(other) => bail!( + "{}: include's optional is true or false, not {other}", + path.display() + ), + }; + if let Some(key) = entry.keys().next() { + bail!( + "{}: unknown key {key:?} in an include entry", + path.display() + ); + } + Ok((file, !optional)) + } + other => Err(anyhow!( + "{}: include entries are paths or {{ path, optional }}, not {other}", + path.display() + )), + }) + .collect() +} + fn take_paths( root: &mut JsonMap<String, JsonValue>, key: &str, @@ -170,6 +216,7 @@ fn flatten( #[cfg(test)] mod tests { use super::*; + use crate::files::scratch_dir; use serde_json::json; fn write(dir: &Path, name: &str, text: &str) -> PathBuf { @@ -178,17 +225,9 @@ mod tests { path } - fn scratch(name: &str) -> PathBuf { - let dir = - std::env::temp_dir().join(format!("continuum-config-{name}-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&dir); - std::fs::create_dir_all(&dir).unwrap(); - dir - } - #[test] fn includes_tables_and_importers() { - let dir = scratch("includes"); + let dir = scratch_dir("config-includes"); write(&dir, "key-path", "/run/secrets/swarm-key\n"); write( &dir, @@ -203,7 +242,7 @@ mod tests { let main = write( &dir, "main.toml", - "include = [\"public.toml\", \"secrets.json\"]\ninclude-optional = [\"missing.toml\"]\nalias = \"main\"\n", + "include = [\"public.toml\", \"secrets.json\", { path = \"missing.toml\", optional = true }]\nalias = \"main\"\n", ); let layer = load(&main).unwrap(); assert_eq!(layer["neighbours"].value, json!(8)); @@ -219,7 +258,7 @@ mod tests { #[test] fn unset() { - let dir = scratch("unset"); + let dir = scratch_dir("config-unset"); write( &dir, "base.toml", @@ -247,7 +286,7 @@ mod tests { #[test] fn errors() { - let dir = scratch("errors"); + let dir = scratch_dir("config-errors"); let unknown = write(&dir, "unknown.toml", "nieghbours = 3\n"); assert!(load(&unknown).is_err()); let cycle = write(&dir, "cycle.toml", "include = [\"cycle.toml\"]\n"); @@ -256,5 +295,30 @@ mod tests { assert!(load(&missing).is_err()); let format = write(&dir, "config.yaml", "alias: x\n"); assert!(load(&format).is_err()); + let gone = write(&dir, "gone.toml", "include-optional = [\"x.toml\"]\n"); + assert!(load(&gone).is_err()); + } + + /// Includes are loaded in the order listed, optional ones too: the later wins. + #[test] + fn includes_in_order() { + let dir = scratch_dir("config-include-order"); + write(&dir, "local.toml", "alias = \"local\"\n"); + write(&dir, "secrets.toml", "alias = \"secrets\"\n"); + let local_last = write( + &dir, + "a.toml", + "include = [\"secrets.toml\", { path = \"local.toml\", optional = true }]\n", + ); + assert_eq!(load(&local_last).unwrap()["alias"].value, json!("local")); + let secrets_last = write( + &dir, + "b.toml", + "include = [{ path = \"local.toml\", optional = true }, \"secrets.toml\"]\n", + ); + assert_eq!( + load(&secrets_last).unwrap()["alias"].value, + json!("secrets") + ); } } diff --git a/src/config/mod.rs b/src/config/mod.rs index 9210922..c7f96a3 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -110,6 +110,13 @@ pub(crate) const OPTIONS: &[OptionSpec] = &[ "HTTP API users (from htpasswd) who may write protocol data under /store/_continuum (remake records, members, forget records), e.g. to repair a stuck remake. Such writes act on the whole swarm and are logged. Epoch metadata and the swarm ID are never writable.", ), option( + "log-format", + Kind::Text, + "auto|text|journal", + Some("auto"), + "How log lines look: `text` (timestamped), `journal` (for systemd's journal: no timestamp or colours, the level as each line's priority, for journalctl -p), or `auto`: journal when stdout goes to the journal, else text.", + ), + option( "log-raw-ids", Kind::Flag, "BOOL", @@ -128,7 +135,7 @@ pub(crate) const OPTIONS: &[OptionSpec] = &[ Kind::List, "STRING,[STRING...]", None, - "How to reach a node of the swarm to join: its address (`mycelium:<public key>`, as GET /admin/stats lists it), a Mycelium public key, a Mycelium IPv6 address, or a host name with an AAAA record for one. A node keeps trying until it has joined.", + "How to reach a node of the swarm to join: its address, as GET /admin/stats lists it (`mycelium:<public key>`, `libp2p:<multiaddr>`, `libp2p:/p2p/<peer ID>` (where it is known, or through a relay), `tor:<name>.onion:<port>`); or, for Mycelium, a public key, an IPv6 address, or a host name with an AAAA record for one. A node keeps trying until it has joined.", ), option( "genesis", @@ -194,7 +201,7 @@ pub(crate) const OPTIONS: &[OptionSpec] = &[ "This node's key (Ed25519, 32 raw bytes): who the node is, whatever transport reaches it. Keep it with the node: a new key is a new node. Required, but with relay-only (then not allowed).", ), option( - "create-node-key", + "node-key-create", Kind::Flag, "BOOL", Some("false"), @@ -208,6 +215,13 @@ pub(crate) const OPTIONS: &[OptionSpec] = &[ "Pre-shared swarm key: only nodes that know it may join (raw bytes, at least 16; e.g. `head -c 32 /dev/urandom`). Without it, any node that can reach a member may join.", ), option( + "swarm-key-create", + Kind::Flag, + "BOOL", + Some("false"), + "Create a new swarm key at swarm-key-file if there is no file there: on the swarm's first node, as the others need a copy of it (a key of their own keeps them out). Without it, a missing key file is an error.", + ), + option( "alias", Kind::Text, "NAME", @@ -222,6 +236,13 @@ pub(crate) const OPTIONS: &[OptionSpec] = &[ "Keepalive interval; dormancy, resync and probe timings derive from it.", ), option( + "dormant-probe-max", + Kind::Duration, + "DURATION", + None, + "Longest wait between HELLOs to a member that doesn't answer (dormant, or unanswered HELLOs): they start at 10 keepalives and double up to this. It bounds how long a returning member, or the other side of a partition, takes to be noticed. Default: 60 keepalives (1h at 1m). Shorter than 10 keepalives, it is every wait.", + ), + option( "neighbours", Kind::Integer, "COUNT", @@ -233,7 +254,21 @@ pub(crate) const OPTIONS: &[OptionSpec] = &[ Kind::Duration, "DURATION", Some("7d"), - "Members that renewed their alive stamp within this time vote on remakes (a majority of them must agree). Matters on the node that coordinates a remake.", + "Members that renewed their alive stamp within this time vote on remakes (a majority of them must agree). At least 7h. Members renew their stamps every seventh of the shortest window any of them publishes, hourly at most, so nodes may differ. Matters on the node that coordinates a remake.", + ), + option( + "superseded-writes", + Kind::Text, + "drop|quarantine", + Some("drop"), + "What becomes of a write a remake or rejoin would carry over though a newer one touched its path (last writer wins, by the writers' clocks): drop leaves it out (logged), quarantine keeps it for an operator (/admin/quarantine).", + ), + option( + "clock-skew-warn", + Kind::Duration, + "DURATION", + Some("2s"), + "Warn in the log when this node's clock differs from most of its peers' by more than this, for sure (beyond half the round trip it was measured over, from their PONGs); 0: never.", ), option( "blob-chunk", @@ -243,11 +278,32 @@ pub(crate) const OPTIONS: &[OptionSpec] = &[ "Chunk size when serving large transfers to other nodes (e.g. an epoch's snapshot to a rejoining node), 1KiB to 4MiB.", ), option( - "transports", + "mycelium-enable", + Kind::Flag, + "BOOL", + Some("false"), + "Use the Mycelium transport (in builds with the mycelium feature). A node uses every transport enabled, and needs one at least (none is on by default); nodes on several carry the traffic between nodes that share none.", + ), + option( + "libp2p-enable", + Kind::Flag, + "BOOL", + Some("false"), + "Use the libp2p transport (in builds with the libp2p feature).", + ), + option( + "tor-enable", + Kind::Flag, + "BOOL", + Some("false"), + "Use the Tor transport (in builds with the tor feature).", + ), + option( + "transport-order", Kind::List, "NAME,[NAME...]", - Some("mycelium"), - "The transports this node uses, in order of preference: mycelium, libp2p and tor (in builds with the libp2p and tor features). A peer is reached on the first one both have, until the paths are measured (then on the fastest); nodes on several carry the traffic between nodes that share none.", + Some("mycelium,libp2p,tor"), + "The transports' order of preference, whether enabled or not: a peer is reached on the first enabled one both have, until the paths are measured (then on the fastest). Transports left out come after, in the default order.", ), option( "libp2p-listen", @@ -264,11 +320,32 @@ pub(crate) const OPTIONS: &[OptionSpec] = &[ "With the libp2p transport: the addresses to publish in the member list instead of those listened on, e.g. a public address forwarded to this node's listen port. Without, the listen addresses are published, loopback and link-local ones left out.", ), option( + "libp2p-ignore", + Kind::List, + "RANGE,[RANGE...]", + None, + "With the libp2p transport: address ranges libp2p leaves out, e.g. 10.244.0.0/16 and 42::/16 (VPNs): addresses there are neither published (but announced ones) nor offered for hole punching nor dialled, and connections there are refused, whoever dials.", + ), + option( + "libp2p-prefer", + Kind::List, + "RANGE,[RANGE...]", + None, + "With the libp2p transport: address ranges that matter most, best first, e.g. the LAN's 10.23.0.0/16: of the many addresses a node listening everywhere has, those there are published first, and of a peer's, kept first and dialled with the others. Without, public addresses come first.", + ), + option( + "libp2p-internet-hosts", + Kind::List, + "HOST,[HOST...]", + None, + "With the libp2p transport: hosts on the internet (names or addresses, e.g. one.one.one.one), only to tell whether this node has a route to the internet, per IPv4 and IPv6 (asked of the routing table every 30 s; nothing is sent to them). Where it has none, its own public addresses of that family aren't published. Without, or for a family none of them has, it isn't asked.", + ), + option( "libp2p-relay", Kind::Flag, "BOOL", Some("false"), - "With the libp2p transport: relay for other nodes of the swarm that can't be dialled (bounded: 64 circuits, 8 per peer, each up to an hour and 1 GiB), and say so in the member list, where nodes that need a relay find it. Needs a listen address others can dial.", + "With the libp2p transport: relay for other nodes of the swarm that can't be dialled (bounded: 1024 circuits, 64 per peer, each up to an hour and 1 GiB), and say so in the member list, where nodes that need a relay find it. Needs a listen address others can dial.", ), option( "libp2p-relays", @@ -285,6 +362,13 @@ pub(crate) const OPTIONS: &[OptionSpec] = &[ "With the libp2p transport: find nodes on the LAN (multicast DNS, IPv4). Those that speak continuum are seeds while joining, so nodes on one LAN find each other without configured seeds; a swarm key keeps a new node from joining another swarm on the same LAN.", ), option( + "libp2p-upnp", + Kind::Flag, + "BOOL", + Some("false"), + "With the libp2p transport: ask the router to forward this node's listen ports (UPnP), and publish where (the router's public address), so that nodes elsewhere can dial it: behind a home router, no relay or hole punching is needed then. Opens those ports on the router for as long as the node runs.", + ), + option( "libp2p-public", Kind::Flag, "BOOL", @@ -310,7 +394,7 @@ pub(crate) const OPTIONS: &[OptionSpec] = &[ Kind::Flag, "BOOL", Some("false"), - "Only relay for nodes of swarms (libp2p circuit relay), without being a node: no document, no swarm, no node key. Needs transports = libp2p, libp2p-listen and libp2p-key-file; options about a swarm or a document are errors with it. List it for a swarm in the document (_continuum/relays), or configure it (libp2p-relays). The HTTP API (with htpasswd) has /health, /admin/stats and /admin/log only.", + "Only relay for nodes of swarms (libp2p circuit relay), without being a node: no document, no swarm, no node key. Needs libp2p alone (libp2p-enable, no other transport), libp2p-listen and libp2p-key-file; options about a swarm or a document are errors with it. List it for a swarm in the document (_continuum/relays), or configure it (libp2p-relays). The HTTP API (with htpasswd) has /health, /admin/stats and /admin/log only.", ), option( "libp2p-key-file", @@ -320,6 +404,13 @@ pub(crate) const OPTIONS: &[OptionSpec] = &[ "With the libp2p transport: its key (Ed25519, 32 raw bytes), which makes its peer ID; a new one each start if absent (the node ID stays the same either way).", ), option( + "libp2p-key-create", + Kind::Flag, + "BOOL", + Some("false"), + "With the libp2p transport: create a new libp2p key at libp2p-key-file if there is no file there (a new peer ID). Without it, a missing key file is an error.", + ), + option( "mycelium-rest-url", Kind::Text, "SCHEME://HOSTNAME:PORT", @@ -336,6 +427,13 @@ pub(crate) const OPTIONS: &[OptionSpec] = &[ "Private key of the supervised Mycelium daemon (its identity); a new one each start if absent.", ), option( + "mycelium-key-create", + Kind::Flag, + "BOOL", + Some("false"), + "Create a new key for the supervised Mycelium daemon at mycelium-key-file if there is no file there (a new Mycelium address). Without it, a missing key file is an error.", + ), + option( "mycelium-network-key-file", Kind::Text, "PATH", @@ -390,6 +488,8 @@ pub(crate) struct Settings { pub(crate) protocol_writers: Vec<String>, #[serde(default)] pub(crate) debug_api: bool, + #[serde(default = "default_log_format")] + pub(crate) log_format: String, #[serde(default)] pub(crate) log_raw_ids: bool, #[serde(default)] @@ -415,11 +515,19 @@ pub(crate) struct Settings { /// Required (`check`), but not allowed with `relay_only`. pub(crate) node_key_file: Option<PathBuf>, #[serde(default)] - pub(crate) create_node_key: bool, + pub(crate) node_key_create: bool, pub(crate) swarm_key_file: Option<PathBuf>, + #[serde(default)] + pub(crate) swarm_key_create: bool, pub(crate) alias: Option<String>, #[serde(default = "default_keepalive", with = "crate::units::duration_option")] pub(crate) keepalive: Duration, + #[serde( + default, + deserialize_with = "crate::units::optional_duration", + serialize_with = "crate::units::serialize_optional_duration" + )] + pub(crate) dormant_probe_max: Option<Duration>, #[serde(default = "default_neighbours")] pub(crate) neighbours: usize, #[serde( @@ -427,10 +535,27 @@ pub(crate) struct Settings { with = "crate::units::duration_option" )] pub(crate) electorate_window: Duration, + #[serde(default)] + pub(crate) superseded_writes: crate::remake::Superseded, + #[serde( + default = "default_clock_skew_warn", + with = "crate::units::duration_option" + )] + pub(crate) clock_skew_warn: Duration, /// In bytes. #[serde(default = "default_blob_chunk", with = "crate::units::size_option")] pub(crate) blob_chunk: u64, - #[serde(default = "default_transports")] + #[serde(default)] + pub(crate) mycelium_enable: bool, + #[serde(default)] + pub(crate) libp2p_enable: bool, + #[serde(default)] + pub(crate) tor_enable: bool, + #[serde(default = "default_transport_order")] + pub(crate) transport_order: Vec<String>, + /// The transports this node uses, in order of preference: those enabled, in + /// `transport_order` (`resolve`). + #[serde(skip)] pub(crate) transports: Vec<String>, #[serde(default)] #[cfg_attr(not(feature = "libp2p"), allow(dead_code))] @@ -449,9 +574,24 @@ pub(crate) struct Settings { pub(crate) libp2p_mdns: bool, #[serde(default)] #[cfg_attr(not(feature = "libp2p"), allow(dead_code))] + pub(crate) libp2p_upnp: bool, + #[serde(default)] + #[cfg_attr(not(feature = "libp2p"), allow(dead_code))] + pub(crate) libp2p_ignore: Vec<String>, + #[serde(default)] + #[cfg_attr(not(feature = "libp2p"), allow(dead_code))] + pub(crate) libp2p_prefer: Vec<String>, + #[serde(default)] + #[cfg_attr(not(feature = "libp2p"), allow(dead_code))] + pub(crate) libp2p_internet_hosts: Vec<String>, + #[serde(default)] + #[cfg_attr(not(feature = "libp2p"), allow(dead_code))] pub(crate) libp2p_public: bool, #[cfg_attr(not(feature = "libp2p"), allow(dead_code))] pub(crate) libp2p_key_file: Option<PathBuf>, + #[serde(default)] + #[cfg_attr(not(feature = "libp2p"), allow(dead_code))] + pub(crate) libp2p_key_create: bool, #[cfg_attr(not(feature = "tor"), allow(dead_code))] pub(crate) tor_state_dir: Option<PathBuf>, #[serde(default = "default_tor_port")] @@ -459,12 +599,21 @@ pub(crate) struct Settings { pub(crate) tor_port: u16, #[serde(default)] pub(crate) relay_only: bool, + #[cfg_attr(not(feature = "mycelium"), allow(dead_code))] pub(crate) mycelium_rest_url: Option<String>, + #[cfg_attr(not(feature = "mycelium"), allow(dead_code))] pub(crate) mycelium_key_file: Option<PathBuf>, + #[serde(default)] + #[cfg_attr(not(feature = "mycelium"), allow(dead_code))] + pub(crate) mycelium_key_create: bool, + #[cfg_attr(not(feature = "mycelium"), allow(dead_code))] pub(crate) mycelium_network_key_file: Option<PathBuf>, + #[cfg_attr(not(feature = "mycelium"), allow(dead_code))] pub(crate) mycelium_network_name: Option<String>, + #[cfg_attr(not(feature = "mycelium"), allow(dead_code))] pub(crate) mycelium_tcp_listen_port: Option<u16>, #[serde(default = "default_mycelium_peers")] + #[cfg_attr(not(feature = "mycelium"), allow(dead_code))] pub(crate) mycelium_peers: Vec<String>, pub(crate) debug_epoch: Option<u64>, } @@ -487,18 +636,27 @@ fn default_neighbours() -> usize { fn default_electorate_window() -> Duration { Duration::from_secs(7 * 24 * 3600) } +fn default_clock_skew_warn() -> Duration { + Duration::from_secs(2) +} fn default_blob_chunk() -> u64 { 64 * 1024 } -fn default_transports() -> Vec<String> { - vec!["mycelium".to_owned()] +fn default_transport_order() -> Vec<String> { + TRANSPORTS + .iter() + .map(|(name, _)| (*name).to_owned()) + .collect() } -/// The transports `transports` may name, and whether this build has each. +/// The transports there are, in the default order, and whether this build has each. pub(crate) const TRANSPORTS: &[(&str, bool)] = &[ - ("mycelium", true), + ("mycelium", cfg!(feature = "mycelium")), ("libp2p", cfg!(feature = "libp2p")), ("tor", cfg!(feature = "tor")), ]; +fn default_log_format() -> String { + "auto".to_owned() +} fn default_tor_port() -> u16 { 9662 } @@ -507,48 +665,14 @@ fn default_mycelium_peers() -> Vec<String> { } impl Settings { - /// `mycelium-rest-url`, parsed (checked when loading). - pub(crate) fn mycelium_rest_url(&self) -> Option<url::Url> { - self.mycelium_rest_url - .as_deref() - .and_then(|url| url::Url::parse(url).ok()) - } - - /// The options only a supervised Mycelium daemon takes, and whether each is set: the - /// other side of `CONFLICTS`' `mycelium-rest-url` (a test keeps them the same). - fn supervised_only(&self) -> [(&'static str, bool); 4] { - [ - ("mycelium-key-file", self.mycelium_key_file.is_some()), - ( - "mycelium-network-key-file", - self.mycelium_network_key_file.is_some(), - ), - ( - "mycelium-network-name", - self.mycelium_network_name.is_some(), - ), - ( - "mycelium-tcp-listen-port", - self.mycelium_tcp_listen_port.is_some(), - ), - ] - } - fn check(&self) -> Result<()> { - if let Some(url) = &self.mycelium_rest_url { - url::Url::parse(url).with_context(|| format!("mycelium-rest-url {url:?}"))?; - } - if self.mycelium_rest_url.is_some() - && let Some((name, _)) = self.supervised_only().iter().find(|(_, set)| *set) - { + if crate::logging::LogFormat::parse(&self.log_format).is_none() { bail!( - "{name} is for a supervised Mycelium daemon and can't be combined with mycelium-rest-url" + "log-format {:?}: one of {}", + self.log_format, + crate::logging::LogFormat::NAMES.join(", ") ); } - // A private network has both; one alone would be ignored or invented. - if self.mycelium_network_key_file.is_some() != self.mycelium_network_name.is_some() { - bail!("mycelium-network-key-file and mycelium-network-name go together"); - } crate::force::Force::parse(self.force.iter().map(String::as_str), FORCE) .map_err(|reason| anyhow!("force: {reason}"))?; // Contradictions, not overrides: an error wherever the options come from. @@ -563,6 +687,12 @@ impl Settings { if self.genesis && self.recover_from.is_some() { bail!("recover-from starts the swarm from a backup; it can't be combined with genesis"); } + if self.node_key_create && self.node_key_file.is_none() { + bail!("node-key-create creates the key at node-key-file; set that too"); + } + if self.swarm_key_create && self.swarm_key_file.is_none() { + bail!("swarm-key-create creates the key at swarm-key-file; set that too"); + } if self.relay_only { self.check_relay_only()?; } else { @@ -571,37 +701,47 @@ impl Settings { } if self.node_key_file.is_none() { bail!( - "node-key-file is required: who this node is (create one with create-node-key)" + "node-key-file is required: who this node is (create one with node-key-create)" ); } } - if self.transports.is_empty() { - bail!("transports: a node needs at least one"); - } - if self.transports.iter().any(|name| name == "tor") && self.tor_state_dir.is_none() { - bail!( - "tor needs tor-state-dir: where arti keeps its state, the onion service's key among it" - ); - } - for (index, name) in self.transports.iter().enumerate() { - match TRANSPORTS.iter().find(|(known, _)| known == name) { - None => bail!( - "transports: unknown transport {name:?} (known: {})", + for (index, name) in self.transport_order.iter().enumerate() { + if !TRANSPORTS.iter().any(|(known, _)| known == name) { + bail!( + "transport-order: unknown transport {name:?} (known: {})", TRANSPORTS .iter() .map(|(known, _)| *known) .collect::<Vec<_>>() .join(", ") - ), - Some((_, false)) => { - bail!("transports: this build has no {name} (built without the {name} feature)") - } - Some(_) if self.transports[..index].contains(name) => { - bail!("transports: {name} is named twice") - } - Some(_) => {} + ); + } + if self.transport_order[..index].contains(name) { + bail!("transport-order: {name} is named twice"); + } + } + for (name, built) in TRANSPORTS { + if self.enabled(name) && !built { + bail!("{name}-enable: this build has no {name} (built without the {name} feature)"); } } + if self.transports.is_empty() { + bail!( + "no transport enabled: a node needs at least one ({}-enable; this build has: {})", + TRANSPORTS + .iter() + .find(|(_, built)| *built) + .map_or("libp2p", |(name, _)| *name), + TRANSPORTS + .iter() + .filter(|(_, built)| *built) + .map(|(name, _)| *name) + .collect::<Vec<_>>() + .join(", ") + ); + } + // Each transport's own options, now that the names are good. + crate::transport::check(self)?; if self.debug_epoch.is_some() && !self.debug_api { bail!("debug-epoch needs debug-api"); } @@ -617,8 +757,11 @@ impl Settings { if !(1024..=4 << 20).contains(&self.blob_chunk) { bail!("blob-chunk must be between 1KiB and 4MiB"); } - if self.electorate_window.is_zero() { - bail!("electorate-window must be longer than 0"); + // Members renew their alive stamp hourly at most (`members::alive_renewal`): in + // a shorter window, live members would drop out between renewals, down to an + // electorate of the coordinator alone. + if self.electorate_window < crate::swarm::members::MIN_ELECTORATE_WINDOW { + bail!("electorate-window must be at least 7h (seven hourly renewals)"); } if let Some(alias) = &self.alias { check_alias(alias)?; @@ -628,6 +771,32 @@ impl Settings { } impl Settings { + /// Whether transport `name` is enabled (`<name>-enable`). + fn enabled(&self, name: &str) -> bool { + match name { + "mycelium" => self.mycelium_enable, + "libp2p" => self.libp2p_enable, + "tor" => self.tor_enable, + _ => false, + } + } + + /// The transports used, in order: the enabled ones, as `transport_order` puts + /// them, then those it leaves out, in the default order. + fn ordered_transports(&self) -> Vec<String> { + self.transport_order + .iter() + .map(String::as_str) + .chain(TRANSPORTS.iter().map(|(name, _)| *name)) + .filter(|name| self.enabled(name)) + .fold(Vec::new(), |mut used, name| { + if !used.iter().any(|known: &String| known == name) { + used.push(name.to_owned()); + } + used + }) + } + /// A relay that is no node: libp2p alone, dialable, with a key kept, and nothing /// about a swarm or a document. fn check_relay_only(&self) -> Result<()> { @@ -635,19 +804,12 @@ impl Settings { bail!("relay-only relays over libp2p; this build has no libp2p"); } if self.transports != ["libp2p"] { - bail!("relay-only relays over libp2p alone: set transports to libp2p"); - } - if self.libp2p_listen.is_empty() { - bail!("relay-only needs libp2p-listen: where nodes reach the relay"); - } - if self.libp2p_key_file.is_none() { - bail!( - "relay-only needs libp2p-key-file: without it, the relay's peer ID changes with every start and nodes lose it" - ); + bail!("relay-only relays over libp2p alone: set libp2p-enable, and no other transport"); } + // Its listen address and key: libp2p's checks (`transport::libp2p::check`). let swarm_options = [ ("node-key-file", self.node_key_file.is_some()), - ("create-node-key", self.create_node_key), + ("node-key-create", self.node_key_create), ("seeds", !self.seeds.is_empty()), ("genesis", self.genesis), ("state-file", self.state_file.is_some()), @@ -655,6 +817,7 @@ impl Settings { ("recover-from", self.recover_from.is_some()), ("remake", self.remake.is_some()), ("swarm-key-file", self.swarm_key_file.is_some()), + ("swarm-key-create", self.swarm_key_create), ("protocol-writers", !self.protocol_writers.is_empty()), ("alias", self.alias.is_some()), ("debug-api", self.debug_api), @@ -776,11 +939,12 @@ fn unset_entry(provenance: Provenance) -> Entry { /// Options that can't be combined. A source that sets one side drops the other side's /// values from earlier sources (lower precedence), so e.g. `--mycelium-rest-url` on the /// command line overrides `mycelium-key-file` from a file. Both set in one source is an -/// error (`Settings::check`). +/// error (`transport::mycelium::check`, where Mycelium is built). const CONFLICTS: &[(&[&str], &[&str])] = &[( &["mycelium-rest-url"], &[ "mycelium-key-file", + "mycelium-key-create", "mycelium-network-key-file", "mycelium-network-name", "mycelium-tcp-listen-port", @@ -1028,12 +1192,13 @@ fn culprit(merged: &Layer) -> Option<(&String, &Entry)> { const CONFIG: &str = "config"; const PRINT_CONFIG: &str = "print-config"; -const UNSET: &str = "unset"; +/// Undoing a value set before: the argument `--unset`, and the key `unset` in a file. +pub(super) const UNSET: &str = "unset"; fn command() -> Command { let mut command = Command::new("continuum") .version(env!("CARGO_PKG_VERSION")) - .about("continuum: a replicated key-value store over Mycelium") + .about("continuum: a replicated Automerge document over Mycelium, libp2p or Tor") .after_help( "Every option can also be set in a configuration file (--config); arguments win.\n\ A value can be imported instead of written out: env:NAME, file:PATH or cmd:COMMAND \ @@ -1146,6 +1311,34 @@ pub(crate) struct Loaded { pub(crate) print_only: bool, } +impl Loaded { + /// The options given for transports this node doesn't use (not enabled), which it + /// ignores: per transport, whether this build has it, and the options (but its + /// `-enable`). A shared configuration may well carry them; still worth a warning, + /// as it may as well be a mistake (a build without that transport, or a transport + /// not enabled). + pub(crate) fn unused_transport_options(&self) -> Vec<(&'static str, bool, Vec<String>)> { + TRANSPORTS + .iter() + .filter(|(name, _)| !self.settings.transports.iter().any(|used| used == name)) + .filter_map(|(name, built)| { + let prefix = format!("{name}-"); + let options: Vec<String> = self + .origins + .iter() + .filter(|(option, origin)| { + option.starts_with(&prefix) + && *option != &format!("{name}-enable") + && !origin.unset + }) + .map(|(option, _)| option.clone()) + .collect(); + (!options.is_empty()).then_some((*name, *built, options)) + }) + .collect() + } +} + /// How `GET /admin/config` shows text with placeholders (`?values=`). #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize)] #[serde(rename_all = "camelCase")] @@ -1301,6 +1494,7 @@ fn resolve(merged: Layer, files: Vec<PathBuf>) -> Result<Loaded> { } })?; settings.alias = settings.alias.map(|alias| alias.trim().to_owned()); + settings.transports = settings.ordered_transports(); settings.check().context("invalid configuration")?; Ok(Loaded { settings, @@ -1318,27 +1512,54 @@ mod tests { use super::*; use serde_json::json; + /// The first transport this build has: what the tests run on when they don't name + /// one, so they pass with any of the transport features. + fn built_transport() -> &'static str { + TRANSPORTS + .iter() + .find(|(_, built)| *built) + .map(|(name, _)| *name) + .expect("a build has a transport") + } + + fn entry(value: JsonValue) -> Entry { + Entry { + value, + origin: Origin { + provenance: Provenance::Argument, + unset: false, + imports: Vec::new(), + templates: Vec::new(), + written: None, + }, + } + } + + /// `values` as arguments, with this build's first transport unless they name some + /// (`built_transport`). fn layer(values: JsonValue) -> Layer { - values + let mut layer: Layer = values .as_object() .unwrap() .iter() - .map(|(name, value)| { - ( - name.clone(), - Entry { - value: value.clone(), - origin: Origin { - provenance: Provenance::Argument, - unset: false, - imports: Vec::new(), - templates: Vec::new(), - written: None, - }, - }, - ) - }) - .collect() + .map(|(name, value)| (name.clone(), entry(value.clone()))) + .collect(); + with_built_transport(&mut layer); + layer + } + + /// Unless `layer` enables transports: this build's first, with what it needs to start + /// (Tor its state directory). + fn with_built_transport(layer: &mut Layer) { + if layer.keys().any(|name| name.ends_with("-enable")) { + return; + } + layer.insert(format!("{}-enable", built_transport()), entry(json!(true))); + if built_transport() == "tor" { + layer + .entry("tor-state-dir".to_owned()) + .or_insert_with(|| entry(json!("tor"))); + } } #[test] @@ -1364,6 +1585,22 @@ mod tests { } #[test] + fn electorate_window_of_seven_renewals_at_least() { + let with = |window: &str| { + resolve( + layer(json!({ + "htpasswd": "u", + "node-key-file": "node.key", + "electorate-window": window, + })), + Vec::new(), + ) + }; + assert!(with("6h59m").is_err()); + assert!(with("7h").is_ok()); + } + + #[test] fn options_and_settings_agree() { // Every option is a Settings field, and every field an option. let loaded = resolve( @@ -1380,8 +1617,10 @@ mod tests { options.sort_unstable(); assert_eq!(names, options); - // Help defaults match the real ones. - for spec in OPTIONS { + // Help defaults match the real ones (but for what `layer` sets to start a + // transport: `with_built_transport`). + let set = format!("{}-enable", built_transport()); + for spec in OPTIONS.iter().filter(|spec| spec.name != set) { if let Some(default) = spec.default { assert_eq!( from_text(spec.kind, default).unwrap(), @@ -1435,6 +1674,7 @@ mod tests { file(json!("a")), Overlay::FileOverFile, ); + with_built_transport(&mut merged); // Both in one source: an error. let mut same = merged.clone(); @@ -1444,7 +1684,11 @@ mod tests { file(url.clone()), Overlay::FileOverFile, ); - assert!(resolve(same, Vec::new()).is_err()); + // (Mycelium's check, where it's built.) + assert_eq!( + resolve(same, Vec::new()).is_err(), + cfg!(feature = "mycelium") + ); // The command line wins over the conflicting file value, and unsets. overlay( @@ -1505,7 +1749,54 @@ mod tests { assert_eq!(from_text(Kind::Flag, "yes").unwrap(), json!(true)); } - /// `Settings::check` and the override rule name the same supervised-only options. + /// Options of transports not in `transports` are named, per transport; those of + /// the transports used aren't, nor unset ones. + #[test] + fn unused_transport_options_are_named() { + let mut values = layer(json!({ + "htpasswd": "u", + "node-key-file": "node.key", + "libp2p-listen": ["/ip4/0.0.0.0/udp/0/quic-v1"], + "libp2p-mdns": true, + "tor-port": 9000, + "mycelium-peers": ["tcp://a:9651"], + })); + values.get_mut("tor-port").unwrap().origin.unset = true; + let loaded = resolve(values, Vec::new()).unwrap(); + let unused = loaded.unused_transport_options(); + for (name, built, options) in &unused { + assert_ne!(*name, built_transport()); + assert_eq!( + *built, + TRANSPORTS.iter().any(|(known, b)| known == name && *b) + ); + assert!( + options + .iter() + .all(|option| option.starts_with(&format!("{name}-"))) + ); + } + let named = |name: &str| { + unused + .iter() + .find(|(n, _, _)| *n == name) + .map(|(_, _, o)| o.clone()) + }; + if built_transport() != "libp2p" { + assert_eq!( + named("libp2p"), + Some(vec!["libp2p-listen".to_owned(), "libp2p-mdns".to_owned()]) + ); + } + if built_transport() != "mycelium" { + assert_eq!(named("mycelium"), Some(vec!["mycelium-peers".to_owned()])); + } + // Unset: not given. + assert_eq!(named("tor"), None); + } + + /// Mycelium's check and the override rule name the same supervised-only options. + #[cfg(feature = "mycelium")] #[test] fn supervised_only_options_are_the_conflicts() { let settings = resolve( @@ -1514,8 +1805,7 @@ mod tests { ) .unwrap() .settings; - let names: Vec<&str> = settings - .supervised_only() + let names: Vec<&str> = crate::transport::mycelium::supervised_only(&settings) .iter() .map(|(name, _)| *name) .collect(); @@ -1541,7 +1831,7 @@ mod tests { ) .is_err() ); - assert!( + assert_eq!( resolve( layer(json!({ "htpasswd": "u", @@ -1551,7 +1841,8 @@ mod tests { })), Vec::new() ) - .is_err() + .is_err(), + cfg!(feature = "mycelium") ); assert!( resolve( @@ -1567,19 +1858,24 @@ mod tests { ) .is_err() ); - let transports = |names: JsonValue| { - resolve( - layer(json!({ "htpasswd": "u", "node-key-file": "node.key", "transports": names })), - Vec::new(), - ) + // The transports enabled (`names`), the others not, in `order` if given. + let transports = |names: &[&str], order: Option<JsonValue>| { + let mut settings = json!({ "htpasswd": "u", "node-key-file": "node.key" }); + for (name, _) in TRANSPORTS { + settings[format!("{name}-enable")] = json!(names.contains(name)); + } + if let Some(order) = order { + settings["transport-order"] = order; + } + resolve(layer(settings), Vec::new()) }; - assert!(transports(json!([])).is_err()); + assert!(transports(&[], None).is_err()); // Without the node key: an error, but for a relay. assert!(resolve(layer(json!({ "htpasswd": "u" })), Vec::new()).is_err()); let relay = |extra: JsonValue| { let mut settings = json!({ "relay-only": true, - "transports": ["libp2p"], + "libp2p-enable": true, "libp2p-listen": ["/ip6/::/tcp/9661"], "libp2p-key-file": "libp2p.key", }); @@ -1596,15 +1892,26 @@ mod tests { ); assert!(relay(json!({ "node-key-file": "node.key" })).is_err()); assert!(relay(json!({ "seeds": ["a"] })).is_err()); - assert!(relay(json!({ "transports": ["mycelium"] })).is_err()); + assert!(relay(json!({ "mycelium-enable": true })).is_err()); + assert!(relay(json!({ "tor-enable": true })).is_err()); assert!(relay(json!({ "libp2p-listen": [] })).is_err()); - assert!(transports(json!(["pigeon"])).is_err()); - assert!(transports(json!(["mycelium", "mycelium"])).is_err()); - assert!(transports(json!(["mycelium"])).is_ok()); + assert!(transports(&["mycelium"], Some(json!(["pigeon"]))).is_err()); + assert!(transports(&["mycelium"], Some(json!(["mycelium", "mycelium"]))).is_err()); assert_eq!( - transports(json!(["libp2p", "mycelium"])).is_ok(), - cfg!(feature = "libp2p") + transports(&["mycelium"], None).is_ok(), + cfg!(feature = "mycelium") ); + if cfg!(feature = "libp2p") && cfg!(feature = "mycelium") { + // In the order given, the ones left out after; whether enabled or not. + let used = |order| { + transports(&["libp2p", "mycelium"], order) + .unwrap() + .settings + .transports + }; + assert_eq!(used(None), ["mycelium", "libp2p"]); + assert_eq!(used(Some(json!(["tor", "libp2p"]))), ["libp2p", "mycelium"]); + } } #[test] @@ -1653,9 +1960,10 @@ mod tests { "c-${env:CONTINUUM_TEST_SITE}", ]) .unwrap(); - let layer = arguments(&matches).unwrap(); + let mut layer = arguments(&matches).unwrap(); assert!(!layer["alias"].origin.is_imported()); assert_eq!(layer["alias"].shown("alias"), "\"node-berlin\""); + with_built_transport(&mut layer); let loaded = resolve(layer, Vec::new()).unwrap(); let report = loaded.report(Values::Interpolated); diff --git a/src/document/at.rs b/src/document/at.rs new file mode 100644 index 0000000..d318f17 --- /dev/null +++ b/src/document/at.rs @@ -0,0 +1,249 @@ +//! The document as it was at given heads: every read of `ReadDoc` made at them (its +//! `*_at` variant), so whatever reads a document (`store`) reads it as it was then. + +use std::ops::RangeBounds; + +use automerge::iter::{DocIter, Keys, ListRange, MapRange, Spans, Values}; +use automerge::marks::{Mark, MarkSet}; +use automerge::{ + AutomergeError, Change, ChangeHash, Cursor, CursorPosition, MoveCursor, ObjId, ObjType, + Parents, Prop, ReadDoc, Stats, TextEncoding, Value, hydrate, +}; + +use super::Document; + +pub(crate) struct At<'a> { + doc: &'a Document, + heads: Vec<ChangeHash>, +} + +impl<'a> At<'a> { + /// `doc` at `heads`; None if it lacks one of them (it hasn't synced it yet, or + /// never will: of another epoch, or no change at all). + pub(crate) fn new(doc: &'a Document, heads: &[ChangeHash]) -> Option<Self> { + heads + .iter() + .all(|head| doc.get_change_by_hash(head).is_some()) + .then(|| Self { + doc, + heads: heads.to_vec(), + }) + } + + fn or_ours<'h>(&'h self, heads: Option<&'h [ChangeHash]>) -> Option<&'h [ChangeHash]> { + Some(heads.unwrap_or(&self.heads)) + } +} + +impl ReadDoc for At<'_> { + fn parents<O: AsRef<ObjId>>(&self, obj: O) -> Result<Parents<'_>, AutomergeError> { + self.doc.parents_at(obj, &self.heads) + } + + fn parents_at<O: AsRef<ObjId>>( + &self, + obj: O, + heads: &[ChangeHash], + ) -> Result<Parents<'_>, AutomergeError> { + self.doc.parents_at(obj, heads) + } + + fn keys<O: AsRef<ObjId>>(&self, obj: O) -> Keys<'_> { + self.doc.keys_at(obj, &self.heads) + } + + fn keys_at<O: AsRef<ObjId>>(&self, obj: O, heads: &[ChangeHash]) -> Keys<'_> { + self.doc.keys_at(obj, heads) + } + + fn iter_at<O: AsRef<ObjId>>(&self, obj: O, heads: Option<&[ChangeHash]>) -> DocIter<'_> { + self.doc.iter_at(obj, self.or_ours(heads)) + } + + fn map_range<'b, O: AsRef<ObjId>, R: RangeBounds<String> + 'b>( + &'b self, + obj: O, + range: R, + ) -> MapRange<'b> { + self.doc.map_range_at(obj, range, &self.heads) + } + + fn map_range_at<'b, O: AsRef<ObjId>, R: RangeBounds<String> + 'b>( + &'b self, + obj: O, + range: R, + heads: &[ChangeHash], + ) -> MapRange<'b> { + self.doc.map_range_at(obj, range, heads) + } + + fn list_range<O: AsRef<ObjId>, R: RangeBounds<usize>>( + &self, + obj: O, + range: R, + ) -> ListRange<'_> { + self.doc.list_range_at(obj, range, &self.heads) + } + + fn list_range_at<O: AsRef<ObjId>, R: RangeBounds<usize>>( + &self, + obj: O, + range: R, + heads: &[ChangeHash], + ) -> ListRange<'_> { + self.doc.list_range_at(obj, range, heads) + } + + fn values<O: AsRef<ObjId>>(&self, obj: O) -> Values<'_> { + self.doc.values_at(obj, &self.heads) + } + + fn values_at<O: AsRef<ObjId>>(&self, obj: O, heads: &[ChangeHash]) -> Values<'_> { + self.doc.values_at(obj, heads) + } + + fn length<O: AsRef<ObjId>>(&self, obj: O) -> usize { + self.doc.length_at(obj, &self.heads) + } + + fn length_at<O: AsRef<ObjId>>(&self, obj: O, heads: &[ChangeHash]) -> usize { + self.doc.length_at(obj, heads) + } + + fn object_type<O: AsRef<ObjId>>(&self, obj: O) -> Result<ObjType, AutomergeError> { + // An object's type is fixed when it is made. + self.doc.object_type(obj) + } + + fn marks<O: AsRef<ObjId>>(&self, obj: O) -> Result<Vec<Mark>, AutomergeError> { + self.doc.marks_at(obj, &self.heads) + } + + fn marks_at<O: AsRef<ObjId>>( + &self, + obj: O, + heads: &[ChangeHash], + ) -> Result<Vec<Mark>, AutomergeError> { + self.doc.marks_at(obj, heads) + } + + fn get_marks<O: AsRef<ObjId>>( + &self, + obj: O, + index: usize, + heads: Option<&[ChangeHash]>, + ) -> Result<MarkSet, AutomergeError> { + self.doc.get_marks(obj, index, self.or_ours(heads)) + } + + fn text<O: AsRef<ObjId>>(&self, obj: O) -> Result<String, AutomergeError> { + self.doc.text_at(obj, &self.heads) + } + + fn text_at<O: AsRef<ObjId>>( + &self, + obj: O, + heads: &[ChangeHash], + ) -> Result<String, AutomergeError> { + self.doc.text_at(obj, heads) + } + + fn spans<O: AsRef<ObjId>>(&self, obj: O) -> Result<Spans<'_>, AutomergeError> { + self.doc.spans_at(obj, &self.heads) + } + + fn spans_at<O: AsRef<ObjId>>( + &self, + obj: O, + heads: &[ChangeHash], + ) -> Result<Spans<'_>, AutomergeError> { + self.doc.spans_at(obj, heads) + } + + fn get_cursor<O: AsRef<ObjId>, I: Into<CursorPosition>>( + &self, + obj: O, + position: I, + at: Option<&[ChangeHash]>, + ) -> Result<Cursor, AutomergeError> { + self.doc.get_cursor(obj, position, self.or_ours(at)) + } + + fn get_cursor_moving<O: AsRef<ObjId>, I: Into<CursorPosition>>( + &self, + obj: O, + position: I, + at: Option<&[ChangeHash]>, + move_cursor: MoveCursor, + ) -> Result<Cursor, AutomergeError> { + self.doc + .get_cursor_moving(obj, position, self.or_ours(at), move_cursor) + } + + fn get_cursor_position<O: AsRef<ObjId>>( + &self, + obj: O, + cursor: &Cursor, + at: Option<&[ChangeHash]>, + ) -> Result<usize, AutomergeError> { + self.doc.get_cursor_position(obj, cursor, self.or_ours(at)) + } + + fn get<O: AsRef<ObjId>, P: Into<Prop>>( + &self, + obj: O, + prop: P, + ) -> Result<Option<(Value<'_>, ObjId)>, AutomergeError> { + self.doc.get_at(obj, prop, &self.heads) + } + + fn get_at<O: AsRef<ObjId>, P: Into<Prop>>( + &self, + obj: O, + prop: P, + heads: &[ChangeHash], + ) -> Result<Option<(Value<'_>, ObjId)>, AutomergeError> { + self.doc.get_at(obj, prop, heads) + } + + fn hydrate<O: AsRef<ObjId>>( + &self, + obj: O, + heads: Option<&[ChangeHash]>, + ) -> Result<hydrate::Value, AutomergeError> { + ReadDoc::hydrate(self.doc, obj, self.or_ours(heads)) + } + + fn get_all<O: AsRef<ObjId>, P: Into<Prop>>( + &self, + obj: O, + prop: P, + ) -> Result<Vec<(Value<'_>, ObjId)>, AutomergeError> { + self.doc.get_all_at(obj, prop, &self.heads) + } + + fn get_all_at<O: AsRef<ObjId>, P: Into<Prop>>( + &self, + obj: O, + prop: P, + heads: &[ChangeHash], + ) -> Result<Vec<(Value<'_>, ObjId)>, AutomergeError> { + self.doc.get_all_at(obj, prop, heads) + } + + fn get_missing_deps(&self, heads: &[ChangeHash]) -> Vec<ChangeHash> { + self.doc.get_missing_deps(heads) + } + + fn get_change_by_hash(&self, hash: &ChangeHash) -> Option<Change> { + self.doc.get_change_by_hash(hash) + } + + fn stats(&self) -> Stats { + self.doc.stats() + } + + fn text_encoding(&self) -> TextEncoding { + self.doc.text_encoding() + } +} diff --git a/src/document/bench.rs b/src/document/bench.rs new file mode 100644 index 0000000..02b530d --- /dev/null +++ b/src/document/bench.rs @@ -0,0 +1,170 @@ +//! Benchmarks, as ignored tests (`cargo benchmarks`, `docs/development.md`): each times +//! what it measures in release builds and prints it; none asserts anything. +//! +//! `change_publishing`: what a change costs until the change publisher knows the paths +//! it touched (`swarm::changes`). Recorded as it is applied (`Replica::logged`, the +//! patch log), against diffing after it (`store::changed`, what the publisher did +//! before, and still does where the log doesn't cover it). Each workload is timed both +//! ways on the same input. + +use std::time::{Duration, Instant}; + +use automerge::sync::{self, SyncDoc}; +use automerge::transaction::Transactable; +use automerge::{ROOT, ReadDoc, ScalarValue}; + +use super::replica::{Replica, Writer, transact}; +use super::store::{self, InLists}; +use super::{ChangeHash, Document}; +use crate::clock::UnixMillis; +use crate::document::replica::Epoch; + +const WRITER: Writer = Writer { + node: "continuum:bench", + user: None, +}; + +fn sorted(mut heads: Vec<ChangeHash>) -> Vec<ChangeHash> { + heads.sort_unstable(); + heads +} + +/// A document with `keys` keys under `data`, each written by a change of its own. +pub(crate) fn document(keys: usize) -> Document { + let mut doc = Document::new(); + let mut map = None; + for key in 0..keys { + transact(&mut doc, UnixMillis(1), &WRITER, |doc| { + let data = match &map { + Some(data) => data, + None => map.insert(doc.put_object(ROOT, "data", automerge::ObjType::Map)?), + }; + doc.put(data, format!("k{key}"), ScalarValue::Int(key as i64)) + }) + .expect("write"); + } + doc +} + +/// The median of `runs` timings of `f`. +pub(crate) fn median(runs: usize, mut f: impl FnMut() -> Duration) -> Duration { + let mut times: Vec<Duration> = (0..runs).map(|_| f()).collect(); + times.sort_unstable(); + times[runs / 2] +} + +fn report(workload: &str, logged: Duration, diffed: Duration) { + println!( + "{workload:<44} logged {:>10.3} ms diffed {:>10.3} ms ({:.2}×)", + logged.as_secs_f64() * 1e3, + diffed.as_secs_f64() * 1e3, + diffed.as_secs_f64() / logged.as_secs_f64().max(f64::MIN_POSITIVE), + ); +} + +/// `writes` local writes of one key each into a document of `keys` keys, the changed +/// paths taken after each (as the publisher does after every change it is told of). +fn local_writes(keys: usize, writes: usize) { + let base = document(keys); + let write = |doc: &mut automerge::transaction::Transaction<'_>, i: usize| { + let data = doc.get(ROOT, "data")?.map(|(_, id)| id).expect("data"); + doc.put( + &data, + format!("k{}", i % keys), + ScalarValue::Int(-(i as i64)), + ) + }; + + let logged = median(5, || { + let mut replica = Replica::new(Epoch::GENESIS, base.clone()); + let started = Instant::now(); + for i in 0..writes { + let before = sorted(replica.doc.get_heads()); + replica + .transact(UnixMillis(2), &WRITER, |doc| write(doc, i)) + .expect("write"); + std::hint::black_box(replica.changed_since(&before, InLists::Index)); + } + started.elapsed() + }); + let diffed = median(5, || { + let mut doc = base.clone(); + let started = Instant::now(); + for i in 0..writes { + let before = sorted(doc.get_heads()); + transact(&mut doc, UnixMillis(2), &WRITER, |doc| write(doc, i)).expect("write"); + let after = sorted(doc.get_heads()); + std::hint::black_box(store::changed(&doc, &before, &after, InLists::Index)); + } + started.elapsed() + }); + report( + &format!("{writes} local writes, {keys} keys"), + logged, + diffed, + ); +} + +/// One sync message carrying `changes` changes to a node that has none of them, and +/// the changed paths after it. +fn sync_message(changes: usize) { + let sender = document(changes); + // The receiver opens; the sender answers with everything it lacks. + let mut receiver_state = sync::State::new(); + let opening = Document::new() + .generate_sync_message(&mut receiver_state) + .expect("an opening"); + let mut sender_doc = sender.clone(); + let mut sender_state = sync::State::new(); + sender_doc + .receive_sync_message(&mut sender_state, opening) + .expect("opening"); + let message = sender_doc + .generate_sync_message(&mut sender_state) + .expect("the changes") + .encode(); + + let logged = median(5, || { + let mut replica = Replica::new(Epoch::GENESIS, Document::new()); + let mut state = receiver_state.clone(); + let message = sync::Message::decode(&message).expect("decode"); + let started = Instant::now(); + let before = sorted(replica.doc.get_heads()); + replica.logged(|doc, log| { + ( + doc.receive_sync_message_log_patches(&mut state, message, log) + .expect("receive"), + true, + ) + }); + std::hint::black_box(replica.changed_since(&before, InLists::Index)); + started.elapsed() + }); + let diffed = median(5, || { + let mut doc = Document::new(); + let mut state = receiver_state.clone(); + let message = sync::Message::decode(&message).expect("decode"); + let started = Instant::now(); + let before = sorted(doc.get_heads()); + doc.receive_sync_message(&mut state, message) + .expect("receive"); + let after = sorted(doc.get_heads()); + std::hint::black_box(store::changed(&doc, &before, &after, InLists::Index)); + started.elapsed() + }); + report( + &format!("one sync message of {changes} changes"), + logged, + diffed, + ); +} + +#[test] +#[ignore = "a benchmark: cargo benchmarks"] +fn bench_change_publishing() { + println!(""); // put reports on new line + local_writes(1_000, 1_000); + local_writes(100_000, 1_000); + sync_message(1_000); + sync_message(10_000); +} diff --git a/src/document/changes.rs b/src/document/changes.rs new file mode 100644 index 0000000..92b7744 --- /dev/null +++ b/src/document/changes.rs @@ -0,0 +1,176 @@ +//! Changes as bytes, to send and to load (`swarm::blob` moves them): one change by its +//! hash, checked against it when it arrives, and a delta, the changes between two sets +//! of heads, the same bytes on every node holding them. + +use std::collections::{HashMap, HashSet}; + +use anyhow::{Context, Result, anyhow, bail}; + +use super::{Change, ChangeHash, Document, ReadDoc}; + +/// Those of `heads` that `doc` doesn't have. +pub(crate) fn missing(doc: &impl ReadDoc, heads: &[ChangeHash]) -> Vec<ChangeHash> { + heads + .iter() + .copied() + .filter(|head| doc.get_change_by_hash(head).is_none()) + .collect() +} + +/// When the changes after `since` were committed, in Unix milliseconds by their +/// writers' clocks (`Replica::transact`); changes without a time (snapshots) left out. +pub(crate) fn commit_times(doc: &Document, since: &[ChangeHash]) -> Vec<i64> { + doc.get_changes(since) + .iter() + .map(|change| change.timestamp()) + .filter(|&time| time > 0) + .collect() +} + +/// The change `hash` as the document encodes it; None if `doc` doesn't have it. +pub(crate) fn change_bytes(doc: &Document, hash: &ChangeHash) -> Option<Vec<u8>> { + doc.get_change_by_hash(hash) + .map(|change| change.raw_bytes().to_vec()) +} + +/// The change in `bytes`, if it is the change `hash`. +pub(crate) fn change_from_bytes(bytes: Vec<u8>, hash: ChangeHash) -> Result<Change> { + let change = Change::from_bytes(bytes) + .map_err(|error| anyhow!("change {hash} doesn't load: {error}"))?; + if change.hash() != hash { + bail!("change has hash {}, not {hash}", change.hash()); + } + Ok(change) +} + +/// The bytes of the delta from `from` to `to` in `doc`: the changes in the history of +/// `to` but not of `from`, each as encoded, sorted by hash and concatenated +/// (`load_delta` reads that). The same on every node holding `to`, whatever else it +/// has. None if `doc` lacks part of `to`. +pub(crate) fn delta_bytes( + doc: &Document, + from: &[ChangeHash], + to: &[ChangeHash], +) -> Option<Vec<u8>> { + if !missing(doc, to).is_empty() { + return None; + } + // Heads of `from` we don't have can't hide anything we have. + let from: Vec<ChangeHash> = from + .iter() + .copied() + .filter(|head| doc.get_change_by_hash(head).is_some()) + .collect(); + let since: HashMap<ChangeHash, (Vec<ChangeHash>, Vec<u8>)> = doc + .get_changes(&from) + .iter() + .map(|change| { + ( + change.hash(), + (change.deps().to_vec(), change.raw_bytes().to_vec()), + ) + }) + .collect(); + // Of those, the ones `to` depends on: what we have beyond `to` stays out. + let mut included: HashSet<ChangeHash> = HashSet::new(); + let mut pending: Vec<ChangeHash> = to.to_vec(); + while let Some(hash) = pending.pop() { + if let Some((deps, _)) = since.get(&hash) + && included.insert(hash) + { + pending.extend(deps.iter().copied()); + } + } + let mut hashes: Vec<ChangeHash> = included.into_iter().collect(); + hashes.sort_unstable(); + let mut bytes = Vec::new(); + for hash in hashes { + bytes.extend_from_slice(&since[&hash].1); + } + Some(bytes) +} + +/// Loads a delta (`delta_bytes`) into `doc`. +pub(crate) fn load_delta(doc: &mut Document, bytes: &[u8]) -> Result<()> { + doc.load_incremental(bytes) + .context("Loading the downloaded changes")?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::document::testing::change; + use automerge::{ActorId, ROOT, transaction::Transactable}; + + fn random_write(rng: &mut crate::testrand::Rng, doc: &mut Document) { + let key = format!("k{}", rng.below(6)); + let value = rng.below(100) as i64; + change(doc, |doc| doc.put(ROOT, key, value).unwrap()); + } + + fn sorted(mut heads: Vec<ChangeHash>) -> Vec<ChangeHash> { + heads.sort_unstable(); + heads + } + + /// Over random histories: two nodes that reached the same heads through different + /// merge orders, one with more on top, produce the same bytes for a delta to those + /// heads; a document at `from` that loads them is at `to`. + #[test] + fn delta_bytes_are_the_same_everywhere_and_complete() { + crate::testrand::for_seeds(200, |rng| { + let actor = |i: u8| ActorId::from(&[i; 16][..]); + let mut base = Document::new().with_actor(actor(0)); + for _ in 0..rng.between(0, 5) { + random_write(rng, &mut base); + } + let from = base.get_heads(); + let mut writers: Vec<Document> = (1..=rng.between(1, 3)) + .map(|i| base.fork().with_actor(actor(i as u8))) + .collect(); + for _ in 0..rng.between(1, 15) { + let writer = rng.below(writers.len()); + random_write(rng, &mut writers[writer]); + } + let mut one = base.fork().with_actor(actor(10)); + let mut two = base.fork().with_actor(actor(11)); + for writer in writers.iter_mut() { + one.merge(writer).unwrap(); + } + for writer in writers.iter_mut().rev() { + two.merge(writer).unwrap(); + } + let to = one.get_heads(); + assert_eq!(sorted(to.clone()), sorted(two.get_heads())); + // Beyond `to`: must stay out of the delta. + random_write(rng, &mut two); + + let bytes = delta_bytes(&one, &from, &to).unwrap(); + assert_eq!(delta_bytes(&two, &from, &to).unwrap(), bytes); + + let mut behind = base.fork(); + load_delta(&mut behind, &bytes).unwrap(); + assert_eq!(sorted(behind.get_heads()), sorted(to)); + }); + } + + #[test] + fn no_delta_without_its_heads() { + let mut doc = Document::new(); + change(&mut doc, |doc| doc.put(ROOT, "k", 1_i64).unwrap()); + assert!(delta_bytes(&doc, &[], &[ChangeHash([7; 32])]).is_none()); + } + + /// A change loads from its bytes only as the change it was asked for. + #[test] + fn a_change_is_checked_against_its_hash() { + let mut doc = Document::new(); + change(&mut doc, |doc| doc.put(ROOT, "k", 1_i64).unwrap()); + let hash = doc.get_heads()[0]; + let bytes = change_bytes(&doc, &hash).unwrap(); + assert_eq!(change_from_bytes(bytes.clone(), hash).unwrap().hash(), hash); + assert!(change_from_bytes(bytes, ChangeHash([7; 32])).is_err()); + assert!(change_bytes(&doc, &ChangeHash([7; 32])).is_none()); + } +} diff --git a/src/document/file.rs b/src/document/file.rs index ed29e9b..3ea3afd 100644 --- a/src/document/file.rs +++ b/src/document/file.rs @@ -7,8 +7,8 @@ //! document, then increments (the changes after the heads written before). A record cut //! short by a crash is dropped when reading. //! -//! Written by a worker (`run`), only while the node is ready and not rejoining (a rejoin -//! keeps the old file until it is done): +//! Written by a worker (`swarm::document_file`), only while the node is ready and not +//! rejoining (a rejoin keeps the old file until it is done): //! * every `--document-append`, what changed is appended; //! * the whole file is rewritten, atomically, when the epoch or this node's actors //! changed (a switch, a rejoin, a restart), and when the appended increments outgrew @@ -20,8 +20,8 @@ use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex, MutexGuard}; use std::time::Duration; +use super::{ActorId, ChangeHash, Document}; use anyhow::{Context, Result, anyhow, bail}; -use automerge::{ActorId, AutoCommit, ChangeHash}; use serde::{Deserialize, Serialize}; use tokio::time::Instant; use tracing::{debug, error, warn}; @@ -29,7 +29,6 @@ use tracing::{debug, error, warn}; use crate::document::replica::Epoch; use crate::files::{FileKind, Files}; use crate::locks::MutexExt; -use crate::state::AppState; const MAGIC: &[u8] = b"continuum-document/1\n"; @@ -43,7 +42,7 @@ struct Header { /// A document file as read at start. pub(crate) struct Loaded { /// Under a fresh actor. - pub(crate) doc: AutoCommit, + pub(crate) doc: Document, /// The actors this node wrote with before. pub(crate) actors: Vec<ActorId>, } @@ -89,7 +88,7 @@ fn parse(bytes: &[u8]) -> Result<(Loaded, usize)> { .collect::<Result<Vec<_>>>()?; let mut records = &rest[line_end + 1..]; - let mut doc: Option<AutoCommit> = None; + let mut doc: Option<Document> = None; while !records.is_empty() { let Some((length, after)) = records.split_first_chunk::<4>() else { break; @@ -100,7 +99,7 @@ fn parse(bytes: &[u8]) -> Result<(Loaded, usize)> { } let (record, after) = after.split_at(length); match &mut doc { - None => doc = Some(AutoCommit::load(record).context("the full save")?), + None => doc = Some(super::load(record).context("the full save")?), Some(doc) => { doc.load_incremental(record).context("an increment")?; } @@ -110,10 +109,7 @@ fn parse(bytes: &[u8]) -> Result<(Loaded, usize)> { let Some(doc) = doc else { bail!("no full save"); }; - let loaded = Loaded { - doc: doc.with_actor(ActorId::random()), - actors, - }; + let loaded = Loaded { doc, actors }; Ok((loaded, records.len())) } @@ -180,166 +176,169 @@ impl DocumentFile { } } -/// Keeps the document file up to date. Runs for as long as the node does. -pub(crate) async fn run(state: AppState) { - let Some(file) = state.document_file.clone() else { - return; - }; - let mut tick = tokio::time::interval(file.append_every); - tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - loop { - tick.tick().await; - flush(&state).await; - } +/// What the document file needs now (`DocumentFile::due`). +pub(crate) enum Due { + /// Written whole: for this reason. + Rewrite(&'static str), + /// What changed since `since` appended, as long as the document is still of `epoch` + /// (an increment of another epoch's document must never be appended). + Append { + epoch: Epoch, + since: Vec<ChangeHash>, + }, + Nothing, } -/// Writes what is due now (see the module docs). Also called at shutdown. -pub(crate) async fn flush(state: &AppState) { - let Some(file) = state.document_file.clone() else { - return; - }; - if !state.is_ready().await || state.remake.is_rejoining() { - return; +impl DocumentFile { + /// How often what changed is appended (`--document-append`). + pub(crate) fn append_every(&self) -> Duration { + self.append_every } - let own = state.remake.own_actors(); - let (epoch, heads) = { - let replica = state.doc.read().await; - (replica.epoch, replica.heads().to_vec()) - }; - let written = file.lock().clone(); - let reason = match &written { - None => Some("first write"), - Some(w) if w.epoch != epoch => Some("new epoch"), - Some(w) if w.actors != own.len() => Some("new actor"), - Some(w) - if w.tail_bytes > w.base_bytes && w.rewritten_at.elapsed() >= file.rewrite_every => - { - Some("increments outgrew the full save") - } - Some(_) => None, - }; - match (reason, written) { - (Some(reason), _) => rewrite(state, &file, own, reason).await, - (None, Some(written)) if written.heads != heads => append(state, &file, written).await, - (None, _) => {} + + /// What is due for a document of `epoch` at `heads`, written by this node's `actors` + /// (see the module docs). + /// How many of this node's actors the file holds; None before it was written. + pub(crate) fn actors(&self) -> Option<usize> { + self.lock().as_ref().map(|written| written.actors) } -} -async fn rewrite(state: &AppState, file: &Arc<DocumentFile>, own: Vec<ActorId>, reason: &str) { - let start = Instant::now(); - let (epoch, mut copy) = { - let replica = state.doc.read().await; - (replica.epoch, replica.doc.clone()) - }; - let own_count = own.len(); - let writer = Arc::clone(file); - let written = tokio::task::spawn_blocking(move || -> Result<(Vec<ChangeHash>, u64)> { - let heads = copy.get_heads(); - // Only the actors with changes in this document matter for a later rejoin; - // the current one in any case. - let present: HashSet<ActorId> = copy - .get_changes(&[]) - .iter() - .map(|change| change.actor_id().clone()) - .collect(); - let current = copy.get_actor().clone(); - let actors: Vec<ActorId> = own - .into_iter() - .filter(|actor| *actor == current || present.contains(actor)) - .collect(); - let save = copy.save(); - let bytes = whole(&actors, &save)?; - writer - .files - .write_atomic(FileKind::Document, &writer.path, &bytes)?; - Ok((heads, save.len() as u64)) - }) - .await; - match written { - Ok(Ok((heads, base_bytes))) => { - debug!( - reason, - bytes = base_bytes, - took_ms = start.elapsed().as_millis() as u64, - "Rewrote the document file" - ); - *file.lock() = Some(Written { - epoch, - actors: own_count, - heads, - base_bytes, - tail_bytes: 0, - rewritten_at: Instant::now(), - }); - } - Ok(Err(error)) => { - error!(path = %file.path.display(), error = %format!("{error:#}"), "Couldn't rewrite the document file"); + pub(crate) fn due(&self, epoch: Epoch, actors: usize, heads: &[ChangeHash]) -> Due { + let written = self.lock().clone(); + let reason = match &written { + None => Some("first write"), + Some(w) if w.epoch != epoch => Some("new epoch"), + Some(w) if w.actors != actors => Some("new actor"), + Some(w) + if w.tail_bytes > w.base_bytes + && w.rewritten_at.elapsed() >= self.rewrite_every => + { + Some("increments outgrew the full save") + } + Some(_) => None, + }; + match (reason, written) { + (Some(reason), _) => Due::Rewrite(reason), + (None, Some(written)) if written.heads != heads => Due::Append { + epoch: written.epoch, + since: written.heads, + }, + (None, _) => Due::Nothing, } - Err(error) => error!(error = %error, "Document file rewrite task failed"), } -} -async fn append(state: &AppState, file: &Arc<DocumentFile>, written: Written) { - let (heads, increment) = { - let mut replica = state.doc.write().await; - // Switched since `flush` looked: the next one rewrites the file instead. An - // increment of another epoch's document must never be appended. - if replica.epoch != written.epoch { - return; - } - let heads = replica.heads().to_vec(); - (heads, replica.doc.save_after(&written.heads)) - }; - let bytes = record(&increment); - let writer = Arc::clone(file); - let appended = tokio::task::spawn_blocking(move || { - writer - .files - .append(FileKind::Document, &writer.path, &bytes) - }) - .await; - match appended { - Ok(Ok(())) => { - let mut guard = file.lock(); - if let Some(current) = guard - .as_mut() - .filter(|current| current.heads == written.heads) - { - current.heads = heads; - current.tail_bytes += increment.len() as u64 + 4; + /// Writes the whole file, atomically: `copy` (a copy of the document, of `epoch`) + /// and this node's actors `own` that wrote in it. + pub(crate) async fn rewrite( + self: &Arc<Self>, + epoch: Epoch, + copy: Document, + own: Vec<ActorId>, + reason: &str, + ) { + let start = Instant::now(); + let own_count = own.len(); + let writer = Arc::clone(self); + let written = tokio::task::spawn_blocking(move || -> Result<(Vec<ChangeHash>, u64)> { + let heads = copy.get_heads(); + // Only the actors with changes in this document matter for a later rejoin; + // the current one in any case. + let present: HashSet<ActorId> = copy + .get_changes(&[]) + .iter() + .map(|change| change.actor_id().clone()) + .collect(); + let current = copy.get_actor().clone(); + let actors: Vec<ActorId> = own + .into_iter() + .filter(|actor| *actor == current || present.contains(actor)) + .collect(); + let save = super::save(©); + let bytes = whole(&actors, &save)?; + writer + .files + .write_atomic(FileKind::Document, &writer.path, &bytes)?; + Ok((heads, save.len() as u64)) + }) + .await; + match written { + Ok(Ok((heads, base_bytes))) => { debug!( - bytes = increment.len(), - tail_bytes = current.tail_bytes, - base_bytes = current.base_bytes, - "Appended to the document file" + reason, + bytes = base_bytes, + took_ms = start.elapsed().as_millis() as u64, + "Rewrote the document file" ); + *self.lock() = Some(Written { + epoch, + actors: own_count, + heads, + base_bytes, + tail_bytes: 0, + rewritten_at: Instant::now(), + }); } + Ok(Err(error)) => { + error!(path = %self.path.display(), error = %format!("{error:#}"), "Couldn't rewrite the document file"); + } + Err(error) => error!(error = %error, "Document file rewrite task failed"), } - Ok(Err(error)) => { - // The file may end in a partial record now; the next rewrite replaces it. - error!(path = %file.path.display(), error = %error, "Couldn't append to the document file; rewriting it next"); - *file.lock() = None; + } + + /// Appends `increment`: what changed from `since` (the heads written last) to + /// `heads`. + pub(crate) async fn append( + self: &Arc<Self>, + since: Vec<ChangeHash>, + heads: Vec<ChangeHash>, + increment: Vec<u8>, + ) { + let bytes = record(&increment); + let writer = Arc::clone(self); + let appended = tokio::task::spawn_blocking(move || { + writer + .files + .append(FileKind::Document, &writer.path, &bytes) + }) + .await; + match appended { + Ok(Ok(())) => { + let mut guard = self.lock(); + if let Some(current) = guard.as_mut().filter(|current| current.heads == since) { + current.heads = heads; + current.tail_bytes += increment.len() as u64 + 4; + debug!( + bytes = increment.len(), + tail_bytes = current.tail_bytes, + base_bytes = current.base_bytes, + "Appended to the document file" + ); + } + } + Ok(Err(error)) => { + // The file may end in a partial record now; the next rewrite replaces it. + error!(path = %self.path.display(), error = %error, "Couldn't append to the document file; rewriting it next"); + *self.lock() = None; + } + Err(error) => error!(error = %error, "Document file append task failed"), } - Err(error) => error!(error = %error, "Document file append task failed"), } } #[cfg(test)] mod tests { use super::*; + use crate::document::testing::change; use automerge::{ROOT, ReadDoc, transaction::Transactable}; #[test] fn whole_then_increments() { - let mut doc = AutoCommit::new(); - doc.put(ROOT, "a", 1_i64).unwrap(); - doc.commit(); + let mut doc = Document::new(); + change(&mut doc, |doc| doc.put(ROOT, "a", 1_i64).unwrap()); let actor = doc.get_actor().clone(); let heads = doc.get_heads(); let mut bytes = whole(std::slice::from_ref(&actor), &doc.save()).unwrap(); - doc.put(ROOT, "b", 2_i64).unwrap(); - doc.commit(); + change(&mut doc, |doc| doc.put(ROOT, "b", 2_i64).unwrap()); bytes.extend(record(&doc.save_after(&heads))); let (loaded, dropped) = parse(&bytes).unwrap(); diff --git a/src/document/lock.rs b/src/document/lock.rs index a394b52..f05c5e3 100644 --- a/src/document/lock.rs +++ b/src/document/lock.rs @@ -27,8 +27,8 @@ pub(crate) struct DocumentLock { pub(crate) struct LockWaits { /// Readers: wait for a writer only. pub(crate) read: Waits, - /// Writers (applying sync messages, generating them, writes, saves): wait for - /// everyone. + /// Writers (applying sync messages and fetched changes, writes, epoch switches): + /// wait for everyone. pub(crate) write: Waits, } @@ -125,13 +125,6 @@ impl DocumentLock { guard } - pub(crate) async fn write(&self) -> RwLockWriteGuard<'_, Replica> { - let since = Instant::now(); - let guard = self.replica.write().await; - self.waited(Mode::Write, since); - guard - } - /// Outside the runtime only (`spawn_blocking`), like `RwLock::blocking_read`. pub(crate) fn blocking_read(&self) -> RwLockReadGuard<'_, Replica> { let since = Instant::now(); diff --git a/src/document/mod.rs b/src/document/mod.rs index b86d1cc..21050ea 100644 --- a/src/document/mod.rs +++ b/src/document/mod.rs @@ -1,8 +1,104 @@ //! The replicated document: the replica with its epoch, and the store semantics //! (merged-view paths over JSON) the HTTP API and remakes work with. +/// The engine's types, under the names the rest of the crate uses: the document +/// (Automerge's `Automerge`), its changes and their hashes, and its writers (actors); +/// a write to it (`Transaction`: one change, committed or dropped whole), and what can +/// be read like a document (`ReadDoc`: the document, or a write in progress) and written +/// like one (`Transactable`). +/// Only `document` and `remake` use Automerge itself (`architecture.md`, The document). +pub(crate) use automerge::transaction::{Transactable, Transaction}; +pub(crate) use automerge::{ActorId, Automerge as Document, Change, ChangeHash, ReadDoc}; + +/// The whole document as bytes, as the engine saves it: a backup, the document +/// file's full save, the size a joining node downloads. +pub(crate) fn save(doc: &Document) -> Vec<u8> { + doc.save() +} + +/// The changes after `since` as bytes that, appended to a save, load with it (the +/// document file's appends). +pub(crate) fn save_since(doc: &Document, since: &[ChangeHash]) -> Vec<u8> { + doc.save_after(since) +} + +/// A saved document (`save`), loaded to write to under a writer ID of its own: one ID +/// must never write in two places, and a save may be loaded on several nodes, or again +/// after the node that saved it wrote on. +pub(crate) fn load(bytes: &[u8]) -> anyhow::Result<Document> { + Ok(Document::load(bytes)?.with_actor(ActorId::random())) +} + +/// The actor (writer ID) `doc` writes under. +pub(crate) fn actor(doc: &Document) -> ActorId { + doc.get_actor().clone() +} + +pub(crate) mod at; +#[cfg(test)] +pub(crate) mod bench; +pub(crate) mod changes; pub(crate) mod file; pub(crate) mod lock; pub(crate) mod meta; pub(crate) mod replica; pub(crate) mod store; +pub(crate) mod sync; +#[cfg(test)] +pub(crate) mod testing; +pub(crate) mod typed; + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + /// Every `.rs` file under `dir`. + fn sources(dir: &Path, out: &mut Vec<PathBuf>) { + for entry in std::fs::read_dir(dir).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + sources(&path, out); + } else if path.extension().is_some_and(|extension| extension == "rs") { + out.push(path); + } + } + } + + /// Automerge is used only in `document` and in what remakes do to documents + /// (snapshots, replaying late writes, catching up, the kinds, quarantined writes); + /// the rest of the node, remakes' coordination included, names its types as + /// `document` re-exports them and works through `document` (`architecture.md`, The + /// document). + #[test] + fn automerge_stays_behind_document() { + let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); + let mut allowed = vec![src.join("document")]; + for engine in ["snapshot", "delta", "catchup", "kind", "quarantine"] { + allowed.push(src.join("remake").join(format!("{engine}.rs"))); + } + let mut files = Vec::new(); + sources(&src, &mut files); + let mut found = Vec::new(); + for file in files + .iter() + .filter(|file| !allowed.iter().any(|allowed| file.starts_with(allowed))) + { + let text = std::fs::read_to_string(file).unwrap(); + for (number, line) in text.lines().enumerate() { + if line.contains("automerge::") || line.trim_start().starts_with("use automerge") { + found.push(format!( + "{}:{}: {}", + file.display(), + number + 1, + line.trim() + )); + } + } + } + assert!( + found.is_empty(), + "Automerge used outside `document`:\n{}", + found.join("\n") + ); + } +} diff --git a/src/document/replica.rs b/src/document/replica.rs index fe76c12..03325eb 100644 --- a/src/document/replica.rs +++ b/src/document/replica.rs @@ -10,7 +10,7 @@ use std::fmt; use std::ops::{Deref, DerefMut}; use std::sync::Arc; -use automerge::{AutoCommit, ChangeHash}; +use super::{ChangeHash, Document, Transaction}; use tokio::sync::watch; /// One generation of the swarm document. @@ -55,7 +55,25 @@ impl Epoch { } pub(crate) fn id_hex(&self) -> String { - self.id.iter().map(|byte| format!("{byte:02x}")).collect() + crate::transport::Hex(&self.id).to_string() + } +} + +/// An epoch as records and the API show it: its generation and its snapshot hash in +/// hex (all zeros for the genesis epoch). +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct EpochRecord { + pub(crate) generation: u64, + pub(crate) id: String, +} + +impl From<Epoch> for EpochRecord { + fn from(epoch: Epoch) -> Self { + Self { + generation: epoch.generation, + id: epoch.id_hex(), + } } } @@ -77,13 +95,12 @@ impl fmt::Debug for Epoch { /// The document and its epoch, always locked and swapped together. /// -/// Derefs to the document, so code that only needs the `AutoCommit` keeps working -/// with a guard of `Replica`. +/// Derefs to the document, so code that only needs the document keeps working with a +/// guard of `Replica`. /// -/// The heads are cached: `AutoCommit::get_heads` takes `&mut self` (it closes a pending -/// transaction), so without the cache every keepalive and status read would need the -/// write lock. Every path that changes the document under the write lock calls -/// [`Replica::refresh_heads`] before releasing it (`AppState::write_doc`, `mutate_doc`, +/// The heads are cached, sorted, so keepalives and status reads (and the stamp below) +/// don't recompute them. Every path that changes the document under the write lock calls +/// [`Replica::refresh_heads`] before releasing it (`AppState::mutate_doc`, /// `switch_epoch`, the sync apply); readers take the read lock and use /// [`Replica::heads`]. /// @@ -95,9 +112,46 @@ pub(crate) struct Replica { /// How many times this node changed epoch (switch, rejoin, abandoned rejoin) in this /// run: messages carry it, so a late one can't flip a peer's view of our epoch back. pub(crate) epoch_seq: u32, - pub(crate) doc: AutoCommit, + pub(crate) doc: Document, + /// Whether writes go in (`AppState::mutate_doc`). False while a rejoin syncs a fresh + /// document: a write would land in a document the epoch's snapshot hasn't reached + /// yet, or be thrown away with it if the rejoin is abandoned. Cleared and set again + /// in the same lock as the swap (`Host::switch_epoch`), so no write checked + /// before a swap lands after it. + pub(crate) writable: bool, heads: Vec<ChangeHash>, stamp: watch::Sender<Stamp>, + /// What local writes and sync changed since the change publisher last looked + /// (`changed_since`), so it needn't diff the document for it. + log: std::sync::Mutex<Log>, +} + +/// The patches of the changes from `from` to `to` (sorted heads) of the document of +/// `epoch_seq`, as they were made. Changes made otherwise (applied from a delta, a +/// merge) aren't in it: the heads then aren't `to` any more, and whoever looks diffs +/// instead. Another document (an epoch switch) starts a log of its own. +struct Log { + epoch_seq: u32, + from: Vec<ChangeHash>, + to: Vec<ChangeHash>, + patches: automerge::PatchLog, +} + +impl Log { + fn at(epoch_seq: u32, heads: Vec<ChangeHash>) -> Self { + Self { + epoch_seq, + from: heads.clone(), + to: heads, + patches: automerge::PatchLog::active(), + } + } +} + +fn sorted_heads(doc: &Document) -> Vec<ChangeHash> { + let mut heads = doc.get_heads(); + heads.sort_unstable(); + heads } /// The epoch label and the heads of the replica, as of its last refresh, published @@ -111,18 +165,21 @@ pub(crate) struct Stamp { } impl Replica { - pub(crate) fn new(epoch: Epoch, doc: AutoCommit) -> Self { + pub(crate) fn new(epoch: Epoch, doc: Document) -> Self { let (stamp, _) = watch::channel(Stamp { epoch, epoch_seq: 0, heads: Arc::from([]), }); + let log = std::sync::Mutex::new(Log::at(0, sorted_heads(&doc))); let mut replica = Self { epoch, epoch_seq: 0, doc, + writable: true, heads: Vec::new(), stamp, + log, }; replica.refresh_heads(); replica @@ -149,18 +206,77 @@ impl Replica { pub(crate) fn stamps(&self) -> watch::Receiver<Stamp> { self.stamp.subscribe() } + + /// The writes `f` makes, as one change stamped `at` if it succeeds (receivers tell + /// by the stamp how long the change took to reach them, `swarm::stats`), by + /// `writer`, dropped if not. Doesn't refresh the heads: the caller does, under the + /// same lock. + pub(crate) fn transact<T, E>( + &mut self, + at: crate::clock::UnixMillis, + writer: &Writer, + f: impl FnOnce(&mut Transaction<'_>) -> Result<T, E>, + ) -> Result<T, E> { + self.logged(|doc, log| transact_logged(doc, at, writer, log, f)) + } + + /// Runs `f`, which changes the document and records what it changed in the log it + /// is given; it returns whether the log still holds everything (a rolled-back + /// transaction takes the log with it). + pub(crate) fn logged<T>( + &mut self, + f: impl FnOnce(&mut Document, &mut automerge::PatchLog) -> (T, bool), + ) -> T { + use crate::locks::MutexExt; + let before = sorted_heads(&self.doc); + let mut log = self.log.lock_or_recover(); + if log.epoch_seq != self.epoch_seq || log.to != before { + // Another document, or changed otherwise since: what the log holds doesn't + // lead here. + *log = Log::at(self.epoch_seq, before); + } + let (out, whole) = f(&mut self.doc, &mut log.patches); + let after = sorted_heads(&self.doc); + if whole { + log.to = after; + } else { + *log = Log::at(self.epoch_seq, after); + } + out + } + + /// The store paths changed since `before` (sorted heads, of this document) to + /// now (`store::changed`): from the log where it covers exactly that, else by a + /// diff. Starts the log again from now. + pub(crate) fn changed_since( + &self, + before: &[ChangeHash], + lists: super::store::InLists, + ) -> Vec<super::store::Changed> { + use crate::locks::MutexExt; + let now = sorted_heads(&self.doc); + let mut log = self.log.lock_or_recover(); + let changed = if log.epoch_seq == self.epoch_seq && log.from == before && log.to == now { + let patches = self.doc.make_patches(&mut log.patches); + super::store::changed_in(&patches, lists) + } else { + super::store::changed(&self.doc, before, &now, lists) + }; + *log = Log::at(self.epoch_seq, now); + changed + } } impl Deref for Replica { - type Target = AutoCommit; + type Target = Document; - fn deref(&self) -> &AutoCommit { + fn deref(&self) -> &Document { &self.doc } } impl DerefMut for Replica { - fn deref_mut(&mut self) -> &mut AutoCommit { + fn deref_mut(&mut self) -> &mut Document { &mut self.doc } } @@ -185,9 +301,191 @@ mod tests { assert_eq!(local.relation_to(&epoch(3, 2)), EpochRelation::Diverged); } + /// A rolled-back write leaves its writer unused in the document; saving must not + /// trip over it (Automerge refuses to save a document with unused writers). + #[test] + fn saves_after_a_rolled_back_write() { + use crate::clock::UnixMillis; + use crate::document::store; + use serde_json::json; + + let path = vec!["a".to_owned()]; + let mut replica = Replica::new(Epoch::GENESIS, Document::new()); + + let writer = Writer { + node: "node", + user: None, + }; + let refused: Result<(), &str> = replica.transact(UnixMillis(1), &writer, |doc| { + store::write_path(doc, &path, &json!(1)).unwrap(); + Err("refused") + }); + assert!(refused.is_err()); + let loaded = super::super::load(&super::super::save(&replica.doc)).unwrap(); + assert!(loaded.get_heads().is_empty()); + assert_eq!(store::read_path(&loaded, &path).unwrap(), None); + + // The same writer commits afterwards, and that saves too. + replica + .transact(UnixMillis(2), &writer, |doc| { + store::write_path(doc, &path, &json!(2)) + }) + .unwrap(); + replica.refresh_heads(); + let loaded = super::super::load(&super::super::save(&replica.doc)).unwrap(); + let mut heads = loaded.get_heads(); + heads.sort_unstable(); + assert_eq!(heads, replica.heads()); + assert_eq!(store::read_path(&loaded, &path).unwrap(), Some(json!(2))); + } + + #[test] + fn messages_name_their_user_in_a_trailer() { + let writer = Writer { + node: "node", + user: Some("tv"), + }; + let message = writer.message().unwrap(); + assert_eq!(message, "User: tv"); + assert_eq!(user_of(&message), Some("tv")); + assert_eq!( + user_of("Moved the settings.\n\nUser: tv\nSigned-off-by: someone"), + Some("tv") + ); + // Before trailers: the user itself. + assert_eq!(user_of("tv"), Some("tv")); + assert_eq!(user_of("Moved the settings."), None); + assert_eq!(user_of("Moved: the settings\nand more"), None); + assert_eq!(user_of("Signed-off-by: someone"), None); + } + #[test] fn display() { assert_eq!(Epoch::GENESIS.to_string(), "0:genesis"); assert_eq!(epoch(2, 0xab).to_string(), "2:abababababab"); } } + +/// Who makes a change, as it records: the node that commits it, as its Automerge author +/// (its node ID), and the user it wrote for through the API, if any, in its message +/// (`User:`, `USER_TRAILER`). That user, at that node, is who wrote it: user names are +/// each node's own (`--htpasswd`). Every node holds every change, so who wrote what is +/// known swarm-wide (`GET /admin/changes`). +pub(crate) struct Writer<'a> { + pub(crate) node: &'a str, + pub(crate) user: Option<&'a str>, +} + +/// The trailer of a change's message naming its user (`Writer`), as Git's +/// `Signed-off-by:` names who signed off. +pub(crate) const USER_TRAILER: &str = "User"; + +impl Writer<'_> { + /// The change's message: its trailers (`trailers`), if it has any. + fn message(&self) -> Option<String> { + self.user.map(|user| format!("{USER_TRAILER}: {user}")) + } +} + +/// The trailers of a change's message, as in Git: the `Key: value` lines of its last +/// paragraph, if each of its lines is one. +pub(crate) fn trailers(message: &str) -> Vec<(&str, &str)> { + let last = message.trim_end().rsplit("\n\n").next().unwrap_or(""); + let trailers: Option<Vec<(&str, &str)>> = last + .lines() + .map(|line| { + let (key, value) = line.split_once(": ")?; + (!key.is_empty() && key.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-')) + .then(|| (key, value.trim())) + }) + .collect(); + trailers.unwrap_or_default() +} + +/// The user a change's message names (`USER_TRAILER`). Changes made before messages had +/// trailers have the user as their whole message: one word, without a `:` +/// (`--htpasswd` names have none). +pub(crate) fn user_of(message: &str) -> Option<&str> { + let trailers = trailers(message); + if trailers.is_empty() { + let message = message.trim(); + let word = + !message.is_empty() && !message.contains(|c: char| c == ':' || c.is_whitespace()); + return word.then_some(message); + } + trailers + .into_iter() + .find(|(key, _)| key.eq_ignore_ascii_case(USER_TRAILER)) + .map(|(_, user)| user) +} + +/// The writes `f` makes to `doc`, as one change stamped `at` by `writer` if it succeeds, +/// dropped if not ([`Replica::transact`], and writes into the document before a rejoin). +pub(crate) fn transact<T, E>( + doc: &mut Document, + at: crate::clock::UnixMillis, + writer: &Writer, + f: impl FnOnce(&mut Transaction<'_>) -> Result<T, E>, +) -> Result<T, E> { + transact_logged(doc, at, writer, &mut automerge::PatchLog::inactive(), f).0 +} + +/// [`transact`], what it changes recorded in `log`; with whether `log` still holds +/// everything (`Replica::logged`). +fn transact_logged<T, E>( + doc: &mut Document, + at: crate::clock::UnixMillis, + writer: &Writer, + log: &mut automerge::PatchLog, + f: impl FnOnce(&mut Transaction<'_>) -> Result<T, E>, +) -> (Result<T, E>, bool) { + authored(doc, writer.node); + let taken = std::mem::replace(log, automerge::PatchLog::inactive()); + if let Ok(transaction) = doc.transaction_log_patches(taken) { + let (out, logged) = run(transaction, at, writer, f); + let whole = logged.is_some(); + if let Some(logged) = logged { + *log = logged; + } + return (out, whole); + } + // Another document's log (the epoch switched): this change goes unlogged. + (run(doc.transaction(), at, writer, f).0, false) +} + +/// Runs `f` in `transaction`, committing what it writes if it succeeds (stamped `at`, +/// by `writer`) with the transaction's log, rolling it back if not. +fn run<T, E>( + mut transaction: Transaction<'_>, + at: crate::clock::UnixMillis, + writer: &Writer, + f: impl FnOnce(&mut Transaction<'_>) -> Result<T, E>, +) -> (Result<T, E>, Option<automerge::PatchLog>) { + let out = f(&mut transaction); + if out.is_err() { + transaction.rollback(); + return (out, None); + } + let mut options = + automerge::transaction::CommitOptions::default().with_time(at.as_u64() as i64); + if let Some(message) = writer.message() { + options.set_message(message); + } + let (_, logged) = transaction.commit_with(options); + (out, Some(logged)) +} + +/// Makes `node` the author of `doc`'s changes from now on. Setting an author gives the +/// document a new actor; this node keeps its own (`with_actor`), which it tracks +/// (`RemakeState::own_actors`, the document file), as only it writes with it. +fn authored(doc: &mut Document, node: &str) { + if doc + .get_author() + .is_some_and(|author| author.as_bytes() == node.as_bytes()) + { + return; + } + let actor = doc.get_actor().clone(); + doc.set_author(Some(automerge::Author::from(node.as_bytes().to_vec()))); + doc.set_actor(actor); +} diff --git a/src/document/store.rs b/src/document/store.rs index e742edb..f18f2fa 100644 --- a/src/document/store.rs +++ b/src/document/store.rs @@ -2,19 +2,168 @@ use anyhow::anyhow; use automerge::{ - ActorId, AutoCommit, ObjId, ObjType, ReadDoc, ScalarValue, Value, transaction::Transactable, + ActorId, Automerge, ObjId, ObjType, Prop, ReadDoc, ScalarValue, Value, + transaction::Transactable, }; -use base64::{Engine, engine::general_purpose::STANDARD as BASE64}; use serde::Serialize; use serde_json::Value as JsonValue; use std::collections::BTreeSet; +use super::typed::{self, JsonForm, Kind, TimestampFormat, Val}; + +/// Store paths touched by one batch of document changes (`AppState::change_events`). +#[derive(Debug)] +pub(crate) struct ChangeSet { + pub(crate) paths: Vec<Vec<String>>, +} + +/// How `changed` names a change inside a list. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum InLists { + /// By the item's index: `servers/0/name` (the change publisher). + Index, + /// By the list: `servers` (remakes replay a list whole). + Whole, +} + +/// A store path changes touched, and whether concurrent writes left a conflict there. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Changed { + pub(crate) path: Vec<String>, + pub(crate) conflict: bool, +} + +/// The store paths the changes from `before` to `after` touched, as the document +/// reports them (a path may come more than once). A map entry by its key, a counter +/// incremented in a map by its key, and list and text edits by the containing object; +/// inside a list as `lists` says. +pub(crate) fn changed( + doc: &Automerge, + before: &[automerge::ChangeHash], + after: &[automerge::ChangeHash], + lists: InLists, +) -> Vec<Changed> { + changed_in(&doc.diff(before, after), lists) +} + +/// The store paths `patches` touched, as [`changed`] has them. +pub(crate) fn changed_in(patches: &[automerge::Patch], lists: InLists) -> Vec<Changed> { + patches + .iter() + .map(|patch| Changed { + path: patch_path(patch, lists), + conflict: matches!( + patch.action, + automerge::PatchAction::PutMap { conflict: true, .. } + ), + }) + .collect() +} + +fn patch_path(patch: &automerge::Patch, lists: InLists) -> Vec<String> { + use automerge::PatchAction; + let mut path = Vec::new(); + for (_, prop) in &patch.path { + match (prop, lists) { + (Prop::Map(key), _) => path.push(key.clone()), + (Prop::Seq(index), InLists::Index) => path.push(index.to_string()), + (Prop::Seq(_), InLists::Whole) => return path, + } + } + match &patch.action { + PatchAction::PutMap { key, .. } + | PatchAction::DeleteMap { key, .. } + | PatchAction::Increment { + prop: Prop::Map(key), + .. + } => path.push(key.clone()), + // List and text edits: the containing object. + _ => {} + } + path +} + +/// The store paths the change `hash` touched (each once, in order), as [`changed`] has +/// them, but from its operations rather than a diff: what a diff costs grows with the +/// objects it looks into, what this costs with the change. An operation on an object +/// that isn't reachable after the change (made and deleted in it) touches nothing. +/// None if `doc` lacks the change. +pub(crate) fn changed_by( + doc: &Automerge, + hash: &automerge::ChangeHash, + lists: InLists, +) -> Option<Vec<Vec<String>>> { + use automerge::legacy::{Key, ObjectId}; + let change = doc.get_change_by_hash(hash)?.decode(); + let heads = [*hash]; + // Each object's path as it was after the change, and whether that is a list's (as + // `InLists::Whole` names what is inside one). + let mut objects: std::collections::HashMap<ObjectId, Option<(Vec<String>, bool)>> = + std::collections::HashMap::new(); + let mut paths = BTreeSet::new(); + for op in change.operations { + let object = objects.entry(op.obj.clone()).or_insert_with(|| { + let id = match &op.obj { + ObjectId::Root => automerge::ROOT, + ObjectId::Id(id) => doc.import_obj(&id.to_string()).ok()?, + }; + let mut path = Vec::new(); + for (_, prop) in doc.parents_at(&id, &heads).ok()?.visible_path()? { + match (prop, lists) { + (Prop::Map(key), _) => path.push(key), + (Prop::Seq(index), InLists::Index) => path.push(index.to_string()), + (Prop::Seq(_), InLists::Whole) => return Some((path, true)), + } + } + Some((path, false)) + }); + let Some((path, in_list)) = object else { + continue; + }; + let mut path = path.clone(); + // A map entry by its key; list and text edits by the containing object. + if let (Key::Map(key), false) = (op.key, *in_list) { + path.push(key.to_string()); + } + paths.insert(path); + } + Some(paths.into_iter().collect()) +} + +/// Whether one path is within the other (a change at either touches both). +pub(crate) fn paths_related(a: &[String], b: &[String]) -> bool { + a.starts_with(b) || b.starts_with(a) +} + #[derive(Debug, thiserror::Error)] pub(crate) enum StoreError { #[error("path crosses a non-map value")] Conflict, #[error("value does not exist")] NotFound, + /// An insert where there is no list. + #[error("not a list")] + NotAList, + /// An index past a list's end (`length` items). + #[error("index {index} is past the list's end ({length} items)")] + PastTheEnd { index: usize, length: usize }, + /// An increment of something that isn't a counter (`increment_path`). + #[error("not a counter")] + NotACounter, + /// A write that would change the type of what it writes over, where that isn't + /// wanted (`Writing::retype`). + #[error("would change {0}")] + Retyping(Retyped), + /// A cast that isn't possible (`cast_path`, `typed::cast`), and why. + #[error("can't cast {} to {}: {why}", .from.name(), .to.name())] + NotCastable { from: Kind, to: Kind, why: String }, + /// A read at heads the document doesn't have (`at::At`). + #[error("this node doesn't have these heads (yet)")] + UnknownHeads, + /// The node is rejoining a newer epoch: no writes until it is done + /// (`Replica::writable`). + #[error("the node is rejoining a newer epoch; writes are refused until it is done")] + Rejoining, #[error(transparent)] Doc(#[from] automerge::AutomergeError), #[error(transparent)] @@ -39,13 +188,13 @@ enum Resolved { /// All concurrent values of `key` across a merged set of parent maps, newest first. fn candidates( - doc: &AutoCommit, + doc: &impl ReadDoc, parents: &[ObjId], - key: &str, + prop: impl Into<Prop> + Clone, ) -> Result<Vec<(Value<'static>, ObjId)>, StoreError> { let mut all = Vec::new(); for parent in parents { - for (value, id) in doc.get_all(parent, key)? { + for (value, id) in doc.get_all(parent, prop.clone())? { all.push((value.into_owned(), id)); } } @@ -54,12 +203,109 @@ fn candidates( } fn resolve( - doc: &AutoCommit, + doc: &impl ReadDoc, parents: &[ObjId], key: &str, log: Option<&mut ConflictLog>, ) -> Result<Option<Resolved>, StoreError> { - let mut all = candidates(doc, parents, key)?; + resolve_prop(doc, parents, key, key, log) +} + +/// The item at `index` of `list` (concurrent writes there resolved as for a key). +fn resolve_item( + doc: &impl ReadDoc, + list: &ObjId, + index: usize, + log: Option<&mut ConflictLog>, +) -> Result<Option<Resolved>, StoreError> { + if index >= doc.length(list) { + return Ok(None); + } + resolve_prop( + doc, + std::slice::from_ref(list), + index, + &index.to_string(), + log, + ) +} + +/// What `segment` names within `current`: a key of merged maps, or the item of a list +/// at the index it spells (`list_index`); None for anything else, and in a scalar. +fn step( + doc: &impl ReadDoc, + current: &Resolved, + segment: &str, + log: Option<&mut ConflictLog>, +) -> Result<Option<Resolved>, StoreError> { + match current { + Resolved::Maps(maps) => resolve(doc, maps, segment, log), + Resolved::Other(Value::Object(ObjType::List), list) => match list_index(segment) { + Some(index) => resolve_item(doc, list, index, log), + None => Ok(None), + }, + Resolved::Other(..) => Ok(None), + } +} + +/// A list index as a path names it: decimal digits, no sign, no leading zeros. +pub(crate) fn list_index(segment: &str) -> Option<usize> { + let digits = !segment.is_empty() && segment.bytes().all(|byte| byte.is_ascii_digit()); + if !digits || (segment.len() > 1 && segment.starts_with('0')) { + return None; + } + segment.parse().ok() +} + +/// A negative list index (`-1` the last item): how far from the end. +fn from_the_end(segment: &str) -> Option<usize> { + list_index(segment.strip_prefix('-')?).filter(|back| *back > 0) +} + +/// `path` with its negative list indexes made positive (`-1` the last item; or, as the +/// last segment with `insertion`, the end: where an insert appends), if it has any; +/// None if it has none. An index before the start names nothing (`NotFound`). Only +/// segments within a list count: in a map, `-1` is a key like any other. +pub(crate) fn positive_path( + doc: &impl ReadDoc, + path: &[String], + insertion: bool, +) -> Result<Option<Vec<String>>, StoreError> { + let mut current = Resolved::Maps(vec![automerge::ROOT]); + let mut positive = path.to_vec(); + let mut changed = false; + for (depth, segment) in path.iter().enumerate() { + let mut segment = segment.clone(); + if let Resolved::Other(Value::Object(ObjType::List), list) = ¤t + && let Some(back) = from_the_end(&segment) + { + let last = depth + 1 == path.len(); + let length = doc.length(list) + usize::from(insertion && last); + let index = length.checked_sub(back).ok_or(StoreError::NotFound)?; + segment = index.to_string(); + positive[depth] = segment.clone(); + changed = true; + } + match step(doc, ¤t, &segment, None)? { + Some(next) => current = next, + // Nothing further to make positive. + None => break, + } + } + Ok(changed.then_some(positive)) +} + +/// Resolves `key` (a map key or a list index) in `parents`; `name` is how conflict +/// reports name it. +fn resolve_prop( + doc: &impl ReadDoc, + parents: &[ObjId], + prop: impl Into<Prop> + Clone, + name: &str, + log: Option<&mut ConflictLog>, +) -> Result<Option<Resolved>, StoreError> { + let key = name; + let mut all = candidates(doc, parents, prop)?; if all.is_empty() { return Ok(None); } @@ -85,21 +331,22 @@ fn resolve( } } -fn merged_keys(doc: &AutoCommit, maps: &[ObjId]) -> BTreeSet<String> { +fn merged_keys(doc: &impl ReadDoc, maps: &[ObjId]) -> BTreeSet<String> { maps.iter().flat_map(|map| doc.keys(map)).collect() } fn resolved_to_json( - doc: &AutoCommit, + doc: &impl ReadDoc, resolved: &Resolved, + form: JsonForm, mut log: Option<&mut ConflictLog>, ) -> Result<JsonValue, StoreError> { let maps = match resolved { Resolved::Maps(maps) => maps, Resolved::Other(Value::Scalar(scalar), _) => { - return scalar_to_json(scalar.clone().into_owned()); + return scalar_to_json(scalar.clone().into_owned(), form); } - Resolved::Other(Value::Object(_), id) => return object_to_json(doc, id, log), + Resolved::Other(Value::Object(_), id) => return object_to_json(doc, id, form, log), }; let mut out = serde_json::Map::new(); @@ -122,7 +369,9 @@ fn resolved_to_json( } }; - let json = nested(&mut log, &key, |log| resolved_to_json(doc, &child, log))?; + let json = nested(&mut log, &key, |log| { + resolved_to_json(doc, &child, form, log) + })?; out.insert(key, json); } } else { @@ -131,40 +380,43 @@ fn resolved_to_json( let Some(child) = resolve(doc, maps, &key, log.as_deref_mut())? else { continue; }; - let json = nested(&mut log, &key, |log| resolved_to_json(doc, &child, log))?; + let json = nested(&mut log, &key, |log| { + resolved_to_json(doc, &child, form, log) + })?; out.insert(key, json); } } - Ok(JsonValue::Object(out)) + Ok(typed::map_to_json(out, form)) } fn object_to_json( - doc: &AutoCommit, + doc: &impl ReadDoc, id: &ObjId, + form: JsonForm, mut log: Option<&mut ConflictLog>, ) -> Result<JsonValue, StoreError> { match doc.object_type(id)? { - ObjType::Map => resolved_to_json(doc, &Resolved::Maps(vec![id.clone()]), log), + ObjType::Map => resolved_to_json(doc, &Resolved::Maps(vec![id.clone()]), form, log), ObjType::List => { let mut out = Vec::new(); for (index, item) in doc.list_range(id, ..).enumerate() { let child_id = item.id(); out.push(match Value::from(item.value) { Value::Object(_) => nested(&mut log, &index.to_string(), |log| { - object_to_json(doc, &child_id, log) + object_to_json(doc, &child_id, form, log) })?, - Value::Scalar(scalar) => scalar_to_json(scalar.into_owned())?, + Value::Scalar(scalar) => scalar_to_json(scalar.into_owned(), form)?, }); } Ok(JsonValue::Array(out)) } - ObjType::Text => Ok(JsonValue::String(doc.text(id)?)), + ObjType::Text => Ok(typed::text_to_json(doc.text(id)?, form)), ObjType::Table => Err(anyhow!("Automerge tables are not supported").into()), } } -fn delete_in_all(doc: &mut AutoCommit, maps: &[ObjId], key: &str) -> Result<(), StoreError> { +fn delete_in_all(doc: &mut impl Transactable, maps: &[ObjId], key: &str) -> Result<(), StoreError> { for map in maps { if doc.get(map, key)?.is_some() { doc.delete(map, key)?; @@ -174,152 +426,779 @@ fn delete_in_all(doc: &mut AutoCommit, maps: &[ObjId], key: &str) -> Result<(), } fn read_path_inner( - doc: &AutoCommit, + doc: &impl ReadDoc, path: &[String], + form: JsonForm, mut log: Option<&mut ConflictLog>, ) -> Result<Option<JsonValue>, StoreError> { - let mut current = Resolved::Maps(vec![automerge::ROOT]); - - for (index, segment) in path.iter().enumerate() { - let Resolved::Maps(maps) = ¤t else { - return Ok(None); - }; - if let Some(log) = log.as_deref_mut() { - // The last segment is the requested value itself, not an ancestor. - log.in_ancestors = index + 1 < path.len(); - } - match resolve(doc, maps, segment, log.as_deref_mut())? { - Some(next) => current = next, - None => return Ok(None), - } - if let Some(log) = log.as_deref_mut() { - log.path.push(segment.clone()); - } - } - - if let Some(log) = log.as_deref_mut() { - log.in_ancestors = false; - } - Ok(Some(resolved_to_json(doc, ¤t, log)?)) + let Some(current) = resolve_logged(doc, path, log.as_deref_mut())? else { + return Ok(None); + }; + Ok(Some(resolved_to_json(doc, ¤t, form, log)?)) } /// Reads a path; the empty path is the whole store. pub(crate) fn read_path( - doc: &AutoCommit, + doc: &impl ReadDoc, path: &[String], ) -> Result<Option<JsonValue>, StoreError> { - read_path_inner(doc, path, None) + read_path_as(doc, path, JsonForm::default()) } +/// [`read_path`], in JSON of `form`. +pub(crate) fn read_path_as( + doc: &impl ReadDoc, + path: &[String], + form: JsonForm, +) -> Result<Option<JsonValue>, StoreError> { + read_path_inner(doc, path, form, None) +} + +#[cfg(test)] pub(crate) fn read_path_with_conflicts( - doc: &AutoCommit, + doc: &impl ReadDoc, + path: &[String], +) -> Result<Option<(JsonValue, Vec<ConflictReport>)>, StoreError> { + read_path_with_conflicts_as(doc, path, JsonForm::default()) +} + +/// [`read_path_with_conflicts`], in JSON of `form`. +pub(crate) fn read_path_with_conflicts_as( + doc: &impl ReadDoc, path: &[String], + form: JsonForm, ) -> Result<Option<(JsonValue, Vec<ConflictReport>)>, StoreError> { let mut log = ConflictLog { want_values: true, + form, ..Default::default() }; - let value = read_path_inner(doc, path, Some(&mut log))?; + let value = read_path_inner(doc, path, form, Some(&mut log))?; Ok(value.map(|value| (value, log.reports))) } +/// What is stored at `path` (the empty path: the root map), and, where it is a scalar +/// or a text, the value itself, to show natively (`typed::native`); None if nothing is +/// there. +pub(crate) fn read_held(doc: &impl ReadDoc, path: &[String]) -> Result<Option<Held>, StoreError> { + let Some(resolved) = resolve_through(doc, path)? else { + return Ok(None); + }; + Ok(Some(match resolved { + Resolved::Maps(_) | Resolved::Other(Value::Object(ObjType::Map), _) => Held { + kind: Kind::Map, + value: None, + }, + Resolved::Other(Value::Object(ObjType::List), _) => Held { + kind: Kind::List, + value: None, + }, + Resolved::Other(Value::Object(ObjType::Text), id) => Held { + kind: Kind::Text, + value: Some(ScalarValue::Str(doc.text(&id)?.into())), + }, + Resolved::Other(Value::Object(ObjType::Table), _) => { + return Err(anyhow!("Automerge tables are not supported").into()); + } + Resolved::Other(Value::Scalar(scalar), _) => { + let scalar = scalar.into_owned(); + Held { + kind: Kind::of(&scalar), + value: Some(scalar), + } + } + })) +} + +/// What a path holds (`read_held`). +#[derive(Debug, Clone)] +pub(crate) struct Held { + pub(crate) kind: Kind, + /// A scalar, or a text's characters as a string. + pub(crate) value: Option<ScalarValue>, +} + /// For streams: the value, plus whether the path or subtree involves any conflict. /// Reports are collected without values, so this costs little more than read_path. pub(crate) fn read_path_traced( - doc: &AutoCommit, + doc: &impl ReadDoc, path: &[String], ) -> Result<(Option<JsonValue>, bool), StoreError> { let mut log = ConflictLog::default(); - let value = read_path_inner(doc, path, Some(&mut log))?; + let value = read_path_inner(doc, path, JsonForm::default(), Some(&mut log))?; Ok((value, !log.reports.is_empty())) } -/// Keys of the merged map at `path`; empty if it doesn't exist or isn't a map. -pub(crate) fn keys_at(doc: &AutoCommit, path: &[String]) -> Result<Vec<String>, StoreError> { - let mut maps = vec![automerge::ROOT]; - for segment in path { - maps = match resolve(doc, &maps, segment, None)? { - Some(Resolved::Maps(next)) => next, - _ => return Ok(Vec::new()), - }; +/// Strings and bytes up to this long are in a shallow read's summaries. +const SHALLOW_INLINE: usize = 256; + +/// What's at `path` (the empty path is the whole store), one level deep: a summary of +/// it, `type` (`map`, `list`, `text`, `string`, `bytes`, `number`, `boolean`, `null`, +/// or `unknown`, an Automerge scalar of a later version) +/// and `size` (keys, items, characters or bytes), and its `value` where it's a short +/// scalar; a map's has its keys' summaries too, under `entries`. Concurrent values +/// resolved as a read resolves them; conflicts not reported. +pub(crate) fn read_shallow( + doc: &impl ReadDoc, + path: &[String], +) -> Result<Option<JsonValue>, StoreError> { + let Some(resolved) = resolve_through(doc, path)? else { + return Ok(None); + }; + summary(doc, &resolved, true).map(Some) +} + +/// The resolved value at `path` (the empty path is the root map), through maps and +/// lists (`step`). +fn resolve_through(doc: &impl ReadDoc, path: &[String]) -> Result<Option<Resolved>, StoreError> { + resolve_logged(doc, path, None) +} + +/// `resolve_through`, noting the conflicts on the way in `log` (as ancestors, but the +/// last segment's: the requested value's), its path left at `path`. +fn resolve_logged( + doc: &impl ReadDoc, + path: &[String], + mut log: Option<&mut ConflictLog>, +) -> Result<Option<Resolved>, StoreError> { + let mut current = Resolved::Maps(vec![automerge::ROOT]); + for (index, segment) in path.iter().enumerate() { + if let Some(log) = log.as_deref_mut() { + // The last segment is the requested value itself, not an ancestor. + log.in_ancestors = index + 1 < path.len(); + } + match step(doc, ¤t, segment, log.as_deref_mut())? { + Some(next) => current = next, + None => return Ok(None), + } + if let Some(log) = log.as_deref_mut() { + log.path.push(segment.clone()); + } + } + if let Some(log) = log { + log.in_ancestors = false; } - Ok(merged_keys(doc, &maps).into_iter().collect()) + Ok(Some(current)) +} + +/// A summary of `resolved` (`read_shallow`); with `entries`, a map's keys' too, or a +/// list's items' (by index). +fn summary( + doc: &impl ReadDoc, + resolved: &Resolved, + entries: bool, +) -> Result<JsonValue, StoreError> { + let mut out = serde_json::Map::new(); + match resolved { + Resolved::Maps(maps) => { + let keys = merged_keys(doc, maps); + out.insert("type".into(), "map".into()); + out.insert("size".into(), keys.len().into()); + if entries { + let mut listed = serde_json::Map::new(); + for key in keys { + if let Some(child) = resolve(doc, maps, &key, None)? { + listed.insert(key, summary(doc, &child, false)?); + } + } + out.insert("entries".into(), JsonValue::Object(listed)); + } + } + Resolved::Other(Value::Object(ObjType::Map), id) => { + return summary(doc, &Resolved::Maps(vec![id.clone()]), entries); + } + Resolved::Other(Value::Object(ObjType::List), id) => { + let length = doc.length(id); + out.insert("type".into(), "list".into()); + out.insert("size".into(), length.into()); + if entries { + let mut listed = serde_json::Map::new(); + for index in 0..length { + if let Some(item) = resolve_item(doc, id, index, None)? { + listed.insert(index.to_string(), summary(doc, &item, false)?); + } + } + out.insert("entries".into(), JsonValue::Object(listed)); + } + } + Resolved::Other(Value::Object(ObjType::Text), id) => { + out.insert("type".into(), "text".into()); + out.insert("size".into(), doc.length(id).into()); + } + Resolved::Other(Value::Object(ObjType::Table), _) => { + return Err(anyhow!("Automerge tables are not supported").into()); + } + Resolved::Other(Value::Scalar(scalar), _) => { + let kind = Kind::of(scalar); + let size = match &**scalar { + ScalarValue::Str(text) => Some(text.len()), + ScalarValue::Bytes(bytes) => Some(bytes.len()), + _ => None, + }; + out.insert("type".into(), kind.name().into()); + if let Some(size) = size { + out.insert("size".into(), size.into()); + } + if kind != Kind::Unknown && size.is_none_or(|size| size <= SHALLOW_INLINE) { + out.insert( + "value".into(), + scalar_to_json(scalar.clone().into_owned(), JsonForm::default())?, + ); + } + } + } + Ok(JsonValue::Object(out)) +} + +/// Keys of the merged map at `path`; empty if it doesn't exist or isn't a map. +pub(crate) fn keys_at(doc: &impl ReadDoc, path: &[String]) -> Result<Vec<String>, StoreError> { + Ok(match resolve_through(doc, path)? { + Some(Resolved::Maps(maps)) => merged_keys(doc, &maps).into_iter().collect(), + _ => Vec::new(), + }) } /// Every concurrent value at `path`, newest first; empty if there is none. Where /// `read_path` picks one (or merges concurrent maps into one), this keeps them apart. pub(crate) fn read_candidates( - doc: &AutoCommit, + doc: &impl ReadDoc, path: &[String], ) -> Result<Vec<JsonValue>, StoreError> { let (leaf, parents) = path.split_last().ok_or(StoreError::NotFound)?; - let mut maps = vec![automerge::ROOT]; - for segment in parents { - maps = match resolve(doc, &maps, segment, None)? { - Some(Resolved::Maps(next)) => next, - _ => return Ok(Vec::new()), - }; - } - candidates(doc, &maps, leaf)? + let Some(Resolved::Maps(maps)) = resolve_through(doc, parents)? else { + return Ok(Vec::new()); + }; + candidates(doc, &maps, leaf.as_str())? .into_iter() .map(|(value, id)| match value { - Value::Scalar(scalar) => scalar_to_json(scalar.into_owned()), - Value::Object(_) => object_to_json(doc, &id, None), + Value::Scalar(scalar) => scalar_to_json(scalar.into_owned(), JsonForm::default()), + Value::Object(_) => object_to_json(doc, &id, JsonForm::default(), None), }) .collect() } -/// Writes (replaces) the value at `path`, creating intermediate maps. +/// Writes the value at `path`, creating intermediate maps: afterwards it reads as +/// `value`, what was there changed only where it differs (`update_key`). Within a list, +/// the segment is an index (`list_index`): an item there is overwritten, one at the end +/// appended; past the end, or anything but an index, names nothing (`NotFound`). pub(crate) fn write_path( - doc: &mut AutoCommit, + doc: &mut impl Transactable, path: &[String], value: &JsonValue, ) -> Result<(), StoreError> { + write_val_path( + doc, + path, + &from_json(value)?, + Writing::sticky(TimestampFormat::default()), + ) + .map(|_| ()) +} + +/// [`write_path`] for a value of any type (`typed::Val`): from typed JSON, or a scalar +/// of a type JSON doesn't have. Returns the places whose type the write changed. +pub(crate) fn write_val_path( + doc: &mut impl Transactable, + path: &[String], + value: &Val, + mut writing: Writing, +) -> Result<Vec<Retyped>, StoreError> { let (leaf, parents) = path.split_last().ok_or(StoreError::NotFound)?; - let maps = ensure_maps(doc, parents)?; + writing.path = parents.to_vec(); + match parent_for_write(doc, parents)? { + Parent::Maps(maps) => update_key(doc, &maps, leaf, value, &mut writing)?, + Parent::List(list) => { + let index = list_index(leaf).ok_or(StoreError::NotFound)?; + let length = doc.length(&list); + if index > length { + return Err(StoreError::NotFound); + } + if index < length { + update_item(doc, &list, index, value, &mut writing)?; + } else { + write_value(doc, &list, Slot::Index(index), value)?; + } + } + } + Ok(writing.retyped) +} - // Replace semantics: clear the key in every merged parent, or an old map - // would merge into the new value. - delete_in_all(doc, &maps, leaf)?; - json_to_automerge(doc, &maps[0], leaf, value)?; - Ok(()) +/// Which values a write writes as the type of what they are written over (`stuck`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Sticky { + /// None: the value says its type (a type's own media type, a cast). + No, + /// A number without a fraction over a float: typed JSON says every other type + /// itself, but not a float's, as JSON numbers don't keep `2.0` apart from `2` in + /// every language. + Floats, + /// Plain JSON, which has fewer types than Automerge: where it is the JSON form of a + /// value of the type there (a string that reads as a timestamp, as base64, a number + /// without a fraction over a float, an unsigned integer or a counter, a string over + /// a text). + All, +} + +/// How a write treats the types of what it writes over. +pub(crate) struct Writing { + /// Types are sticky: what the value is written as where it is the JSON form of a + /// value of the type there. + pub(crate) sticky: Sticky, + /// How sticky timestamps are written in JSON (`typed::TimestampFormat::json`). + pub(crate) timestamps: TimestampFormat, + /// Whether the write may change the type of what it writes over; if not, one that + /// would is refused (`Retyping`). + pub(crate) retype: bool, + path: Vec<String>, + retyped: Vec<Retyped>, } -/// The merged maps at `path`, creating missing maps on the way. -fn ensure_maps(doc: &mut AutoCommit, path: &[String]) -> Result<Vec<ObjId>, StoreError> { - let mut maps = vec![automerge::ROOT]; - for segment in path { - maps = match resolve(doc, &maps, segment, None)? { - Some(Resolved::Maps(next)) => next, - Some(Resolved::Other(..)) => return Err(StoreError::Conflict), - // A concurrent creation elsewhere is merged on read. - None => vec![doc.put_object(&maps[0], segment.as_str(), ObjType::Map)?], +impl Writing { + pub(crate) fn sticky(timestamps: TimestampFormat) -> Self { + Self::new(Sticky::All, timestamps, true) + } + + pub(crate) fn new(sticky: Sticky, timestamps: TimestampFormat, retype: bool) -> Self { + Self { + sticky, + timestamps, + retype, + path: Vec::new(), + retyped: Vec::new(), + } + } + + /// `kind` is written over `held` at the place at `segment`, if `held` is another + /// kind: noted, or with `retype` off, refused. + fn retyping(&mut self, segment: &str, held: Kind, kind: Kind) -> Result<(), StoreError> { + if held == kind { + return Ok(()); + } + let mut path = self.path.clone(); + path.push(segment.to_owned()); + let retyped = Retyped { + path, + from: held, + to: kind, }; + if !self.retype { + return Err(StoreError::Retyping(retyped)); + } + self.retyped.push(retyped); + Ok(()) } - Ok(maps) } -/// The resolved value at `path` (the empty path is the root map). -fn resolve_at(doc: &AutoCommit, path: &[String]) -> Result<Option<Resolved>, StoreError> { - let mut current = Resolved::Maps(vec![automerge::ROOT]); - for segment in path { - let Resolved::Maps(maps) = ¤t else { - return Ok(None); +/// A place whose type a write changed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Retyped { + pub(crate) path: Vec<String>, + pub(crate) from: Kind, + pub(crate) to: Kind, +} + +impl std::fmt::Display for Retyped { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}: {} -> {}", + self.path.join("/"), + self.from.name(), + self.to.name() + ) + } +} + +/// What a value held is stored as. +fn kind_of(held: &Value<'_>) -> Kind { + match held { + Value::Object(ObjType::Map | ObjType::Table) => Kind::Map, + Value::Object(ObjType::List) => Kind::List, + Value::Object(ObjType::Text) => Kind::Text, + Value::Scalar(scalar) => Kind::of(scalar), + } +} + +/// `value` written as the type of `held`, where it is a JSON form of a value of it +/// as `sticky` says (`Writing::sticky`); None where it isn't (it is written as it is). +fn stuck( + held: &Value<'_>, + value: &Val, + sticky: Sticky, + timestamps: TimestampFormat, +) -> Option<Val> { + let Val::Scalar(scalar) = value else { + return None; + }; + let over_float = matches!(held, Value::Scalar(held) if matches!(held.as_ref(), ScalarValue::F64(_))) + && matches!(scalar, ScalarValue::Int(_) | ScalarValue::Uint(_)); + match sticky { + Sticky::No => return None, + Sticky::Floats if !over_float => return None, + Sticky::Floats | Sticky::All => {} + } + let held = match held { + Value::Object(ObjType::Text) => { + return match scalar { + ScalarValue::Str(text) => Some(Val::Text(text.to_string())), + _ => None, + }; + } + Value::Object(_) => return None, + Value::Scalar(held) => held.as_ref(), + }; + let stuck = match (held, scalar) { + (ScalarValue::Timestamp(_), ScalarValue::Str(text)) => { + ScalarValue::Timestamp(timestamps.read(text)?) + } + (ScalarValue::Timestamp(_), ScalarValue::Int(_) | ScalarValue::F64(_)) + if matches!(timestamps, TimestampFormat::Unix | TimestampFormat::UnixMs) => + { + let number = match scalar { + ScalarValue::Int(i) => i.to_string(), + ScalarValue::F64(f) => f.to_string(), + _ => return None, + }; + ScalarValue::Timestamp(timestamps.read(&number)?) + } + (ScalarValue::Bytes(_), ScalarValue::Str(text)) => { + use base64::{Engine, engine::general_purpose::STANDARD as BASE64}; + ScalarValue::Bytes(BASE64.decode(text.as_bytes()).ok()?) + } + (ScalarValue::Uint(_), ScalarValue::Int(i)) => ScalarValue::Uint(u64::try_from(*i).ok()?), + (ScalarValue::F64(_), ScalarValue::Int(i)) => ScalarValue::F64(*i as f64), + (ScalarValue::F64(_), ScalarValue::Uint(u)) => ScalarValue::F64(*u as f64), + (ScalarValue::Counter(_), ScalarValue::Int(i)) => ScalarValue::counter(*i), + _ => return None, + }; + Some(Val::Scalar(stuck)) +} + +fn from_json(value: &JsonValue) -> Result<Val, StoreError> { + Val::from_json(value).map_err(|error| anyhow!(error).into()) +} + +// A write makes its path read as the value written, changing only what differs: a map +// written over a map keeps its keys that stay (and what is inside them), and gets +// those it lacks and loses those the value doesn't have; a list written over a list +// does the same item by item, by position; a text written over a text is changed +// where it differs (a diff, as splices); a scalar equal to the one there isn't written +// again. So a write of a whole map, concurrent with one inside it elsewhere, keeps the +// other unless it changes the same key. Where the path holds concurrent values (a +// conflict), or one of another kind, it is written anew: the write ends the conflict. + +/// What updating a place to a value comes to, given what it holds (all its concurrent +/// values). +enum Update { + /// Nothing: the same scalar is there. + Same, + /// A map there, the only value: its keys updated. + Map(ObjId), + /// A list there, the only value: its items updated. + List(ObjId), + /// A text there, the only value: changed where it differs. + Text(ObjId), + /// Written anew. + Anew, +} + +/// What updating the place at `segment`, holding `holds`, to `value` comes to, with the +/// value to write there: `value`, or as the type there (`stuck`). A change of type is +/// noted (`Writing::retyping`). +fn update_of<'v>( + holds: &[(Value<'static>, ObjId)], + value: &'v Val, + segment: &str, + writing: &mut Writing, +) -> Result<(Update, std::borrow::Cow<'v, Val>), StoreError> { + use std::borrow::Cow; + let Some((held, _)) = holds.first() else { + return Ok((Update::Anew, Cow::Borrowed(value))); + }; + let value = match stuck(held, value, writing.sticky, writing.timestamps) { + Some(stuck) => Cow::Owned(stuck), + None => Cow::Borrowed(value), + }; + let update = match (holds, &*value) { + ([(Value::Object(ObjType::Map), id)], Val::Map(_)) => Update::Map(id.clone()), + ([(Value::Object(ObjType::List), id)], Val::List(_)) => Update::List(id.clone()), + ([(Value::Object(ObjType::Text), id)], Val::Text(_)) => Update::Text(id.clone()), + ([(Value::Scalar(held), _)], Val::Scalar(scalar)) if held.as_ref() == scalar => { + Update::Same + } + _ => { + writing.retyping(segment, kind_of(held), value.kind())?; + Update::Anew + } + }; + Ok((update, value)) +} + +/// Updates `key` of the merged maps `maps` to `value`. +fn update_key( + doc: &mut impl Transactable, + maps: &[ObjId], + key: &str, + value: &Val, + writing: &mut Writing, +) -> Result<(), StoreError> { + let (update, value) = update_of(&candidates(doc, maps, key)?, value, key, writing)?; + writing.path.push(key.to_owned()); + let done = match update { + Update::Same => Ok(()), + Update::Map(map) => update_map(doc, &map, &value, writing), + Update::List(list) => update_list(doc, &list, &value, writing), + Update::Text(text) => update_text(doc, &text, &value), + Update::Anew => { + // Cleared in every merged map, or an old map would merge into the new value. + delete_in_all(doc, maps, key)?; + write_value(doc, &maps[0], Slot::Key(key), &value) + } + }; + writing.path.pop(); + done +} + +/// Updates the item at `index` of `list` (which has it) to `value`. +fn update_item( + doc: &mut impl Transactable, + list: &ObjId, + index: usize, + value: &Val, + writing: &mut Writing, +) -> Result<(), StoreError> { + let segment = index.to_string(); + let holds = candidates(doc, std::slice::from_ref(list), index)?; + let (update, value) = update_of(&holds, value, &segment, writing)?; + writing.path.push(segment); + let done = match update { + Update::Same => Ok(()), + Update::Map(map) => update_map(doc, &map, &value, writing), + Update::List(inner) => update_list(doc, &inner, &value, writing), + Update::Text(text) => update_text(doc, &text, &value), + Update::Anew => write_value(doc, list, Slot::At(index), &value), + }; + writing.path.pop(); + done +} + +fn update_map( + doc: &mut impl Transactable, + map: &ObjId, + value: &Val, + writing: &mut Writing, +) -> Result<(), StoreError> { + let Val::Map(entries) = value else { + unreachable!("a map is updated to a map"); + }; + let maps = std::slice::from_ref(map); + // The keys it loses first, then those it keeps or gets, each in key order. + for key in doc.keys(map).collect::<Vec<_>>() { + if !entries.iter().any(|(kept, _)| *kept == key) { + doc.delete(map, key.as_str())?; + } + } + for (key, value) in entries { + update_key(doc, maps, key, value, writing)?; + } + Ok(()) +} + +fn update_list( + doc: &mut impl Transactable, + list: &ObjId, + value: &Val, + writing: &mut Writing, +) -> Result<(), StoreError> { + let Val::List(items) = value else { + unreachable!("a list is updated to a list"); + }; + let length = doc.length(list); + for (index, item) in items.iter().enumerate() { + if index < length { + update_item(doc, list, index, item, writing)?; + } else { + write_value(doc, list, Slot::Index(index), item)?; + } + } + for index in (items.len()..length).rev() { + doc.delete(list, index)?; + } + Ok(()) +} + +fn update_text(doc: &mut impl Transactable, text: &ObjId, value: &Val) -> Result<(), StoreError> { + let Val::Text(new) = value else { + unreachable!("a text is updated to a text"); + }; + doc.update_text(text, new)?; + Ok(()) +} + +/// What a write's last segment goes into. +enum Parent { + /// Merged maps (`[0]` takes new keys). + Maps(Vec<ObjId>), + List(ObjId), +} + +/// The parent of a write at `parents` + one more segment: missing maps are created on +/// the way; through a list only by an index of an item it has. A scalar or text on the +/// way is a `Conflict`. +fn parent_for_write(doc: &mut impl Transactable, parents: &[String]) -> Result<Parent, StoreError> { + let mut current = Parent::Maps(vec![automerge::ROOT]); + for segment in parents { + let next = match ¤t { + Parent::Maps(maps) => match resolve(doc, maps, segment, None)? { + // A concurrent creation elsewhere is merged on read. + None => Resolved::Maps(vec![doc.put_object( + &maps[0], + segment.as_str(), + ObjType::Map, + )?]), + Some(next) => next, + }, + Parent::List(list) => { + let index = list_index(segment).ok_or(StoreError::NotFound)?; + resolve_item(doc, list, index, None)?.ok_or(StoreError::NotFound)? + } }; - match resolve(doc, maps, segment, None)? { - Some(next) => current = next, - None => return Ok(None), + current = match next { + Resolved::Maps(maps) => Parent::Maps(maps), + Resolved::Other(Value::Object(ObjType::List), list) => Parent::List(list), + Resolved::Other(..) => return Err(StoreError::Conflict), + }; + } + Ok(current) +} + +/// Adds `by` to the counter at `path`, creating one with `by` where there is nothing +/// (in a map: a list's items are only incremented); returns its value then. Increments +/// of one counter on several nodes add up, where writes of numbers would leave one. +/// Anything else there (the newest of concurrent values, as a read resolves them) is +/// `NotACounter`. +pub(crate) fn increment_path( + doc: &mut impl Transactable, + path: &[String], + by: i64, +) -> Result<i64, StoreError> { + let (leaf, parents) = path.split_last().ok_or(StoreError::NotFound)?; + let (parent, prop): (ObjId, Prop) = match parent_for_write(doc, parents)? { + Parent::Maps(maps) => { + let holds = candidates(doc, &maps, leaf)?; + if holds.is_empty() { + doc.put(&maps[0], leaf.as_str(), ScalarValue::counter(by))?; + return Ok(by); + } + // The map that holds the newest value. + let newest = &holds[0].1; + let map = maps + .iter() + .find(|map| { + doc.get_all(*map, leaf.as_str()) + .is_ok_and(|all| all.iter().any(|(_, id)| id == newest)) + }) + .cloned() + .ok_or(StoreError::NotFound)?; + (map, Prop::Map(leaf.clone())) } + Parent::List(list) => { + let index = list_index(leaf).ok_or(StoreError::NotFound)?; + if index >= doc.length(&list) { + return Err(StoreError::NotFound); + } + (list, Prop::Seq(index)) + } + }; + counter_at(doc, &parent, prop.clone())?; + doc.increment(&parent, prop.clone(), by)?; + counter_at(doc, &parent, prop) +} + +/// The value of the counter `prop` of `parent` holds (the newest of its values). +fn counter_at(doc: &impl ReadDoc, parent: &ObjId, prop: Prop) -> Result<i64, StoreError> { + match doc.get(parent, prop)? { + Some((Value::Scalar(scalar), _)) => match scalar.as_ref() { + ScalarValue::Counter(counter) => Ok(i64::from(counter)), + _ => Err(StoreError::NotACounter), + }, + _ => Err(StoreError::NotACounter), } - Ok(Some(current)) +} + +/// Casts what is at `path` (the newest of concurrent values, as a read resolves them) to +/// `to` (`typed::cast`, timestamps to and from strings in `timestamps`), written as a +/// write of the cast value would be (`write_val_path`); returns where that changed the +/// type. Already `to`: nothing changes. Nothing there: `NotFound`; no cast possible: +/// `NotCastable`. +pub(crate) fn cast_path( + doc: &mut impl Transactable, + path: &[String], + to: Kind, + timestamps: TimestampFormat, +) -> Result<Vec<Retyped>, StoreError> { + let held = read_held(doc, path)?.ok_or(StoreError::NotFound)?; + if held.kind == to { + return Ok(Vec::new()); + } + let not_castable = |why: String| StoreError::NotCastable { + from: held.kind, + to, + why, + }; + let value = held + .value + .as_ref() + .ok_or_else(|| not_castable(format!("{} is cast to nothing", held.kind.name())))?; + let cast = typed::cast(held.kind, value, to, timestamps).map_err(not_castable)?; + write_val_path(doc, path, &cast, Writing::new(Sticky::No, timestamps, true)) +} + +/// Inserts `value` into a list: `path` ends in the index it goes to (the items from +/// there on move up one; the list's length appends). An index past the end is +/// `PastTheEnd`; a parent that isn't a list `NotAList`; anything but an index there, or +/// no parent, `NotFound`. +#[cfg(test)] +pub(crate) fn insert_path( + doc: &mut impl Transactable, + path: &[String], + value: &JsonValue, +) -> Result<(), StoreError> { + insert_val_path(doc, path, &from_json(value)?) +} + +/// [`insert_path`] for a value of any type (`typed::Val`). +pub(crate) fn insert_val_path( + doc: &mut impl Transactable, + path: &[String], + value: &Val, +) -> Result<(), StoreError> { + let (list, index) = insertion(doc, path)?; + write_value(doc, &list, Slot::Index(index), value) +} + +/// The list an insert at `path` goes into, and where (`insert_path`). +fn insertion(doc: &impl ReadDoc, path: &[String]) -> Result<(ObjId, usize), StoreError> { + let (leaf, parents) = path.split_last().ok_or(StoreError::NotFound)?; + let list = match resolve_through(doc, parents)? { + None => return Err(StoreError::NotFound), + Some(Resolved::Other(Value::Object(ObjType::List), id)) => id, + Some(_) => return Err(StoreError::NotAList), + }; + let index = list_index(leaf).ok_or(StoreError::NotFound)?; + let length = doc.length(&list); + if index > length { + return Err(StoreError::PastTheEnd { index, length }); + } + Ok((list, index)) } /// Makes `dst` hold at `path` what `src`'s merged view holds there: a type-preserving /// copy (see `materialize_into`) if it exists, a delete otherwise. Replaces the whole /// subtree at `path`; the path must not be empty. pub(crate) fn copy_path( - src: &AutoCommit, - dst: &mut AutoCommit, + src: &impl ReadDoc, + dst: &mut impl Transactable, path: &[String], ) -> Result<(), StoreError> { copy_path_to(src, path, dst, path) @@ -327,21 +1206,25 @@ pub(crate) fn copy_path( /// Like [`copy_path`], but puts what `src` holds at `from` at `to` in `dst`. pub(crate) fn copy_path_to( - src: &AutoCommit, + src: &impl ReadDoc, from: &[String], - dst: &mut AutoCommit, + dst: &mut impl Transactable, to: &[String], ) -> Result<(), StoreError> { let (leaf, parents) = to.split_last().ok_or(StoreError::NotFound)?; - let Some(resolved) = resolve_at(src, from)? else { + let Some(resolved) = resolve_through(src, from)? else { return match delete_path(dst, to, false) { Ok(()) | Err(StoreError::NotFound) => Ok(()), Err(error) => Err(error), }; }; - let maps = ensure_maps(dst, parents)?; + // Into a map: what is copied are map entries (delta replay names nothing inside a + // list: a list is copied whole). + let Parent::Maps(maps) = parent_for_write(dst, parents)? else { + return Err(StoreError::Conflict); + }; delete_in_all(dst, &maps, leaf)?; let mut trace = from.to_vec(); copy_resolved( @@ -355,37 +1238,54 @@ pub(crate) fn copy_path_to( ) } -/// Deletes the value at `path`, optionally pruning ancestors that become empty. +/// Deletes the value at `path`, through lists too, where a segment is an index +/// (`list_index`; the items after it move down one). With `prune`, the maps above it +/// that it leaves empty go too, up to (not including) the root or a list. pub(crate) fn delete_path( - doc: &mut AutoCommit, + doc: &mut impl Transactable, path: &[String], prune: bool, ) -> Result<(), StoreError> { let (leaf, parents) = path.split_last().ok_or(StoreError::NotFound)?; - // levels[d] = merged maps at path[..d] - let mut levels = vec![vec![automerge::ROOT]]; + // levels[d] = what path[..d] resolves to + let mut levels = vec![Resolved::Maps(vec![automerge::ROOT])]; for segment in parents { - let next = match resolve(doc, levels.last().expect("non-empty"), segment, None)? { - Some(Resolved::Maps(maps)) => maps, - _ => return Err(StoreError::NotFound), - }; + let next = step(doc, levels.last().expect("non-empty"), segment, None)? + .ok_or(StoreError::NotFound)?; levels.push(next); } - let maps = levels.last().expect("non-empty"); - if resolve(doc, maps, leaf, None)?.is_none() { - return Err(StoreError::NotFound); + match levels.last().expect("non-empty") { + Resolved::Maps(maps) => { + if resolve(doc, maps, leaf, None)?.is_none() { + return Err(StoreError::NotFound); + } + delete_in_all(doc, maps, leaf)?; + } + Resolved::Other(Value::Object(ObjType::List), list) => { + let index = list_index(leaf).ok_or(StoreError::NotFound)?; + if index >= doc.length(list) { + return Err(StoreError::NotFound); + } + doc.delete(list, index)?; + return Ok(()); + } + Resolved::Other(..) => return Err(StoreError::NotFound), } - delete_in_all(doc, maps, leaf)?; if prune { - // Never prune the root (depth 0). + // Never prune the root (depth 0), nor through a list. for depth in (1..levels.len()).rev() { - if !merged_keys(doc, &levels[depth]).is_empty() { + let (Resolved::Maps(above), Resolved::Maps(maps)) = + (&levels[depth - 1], &levels[depth]) + else { + break; + }; + if !merged_keys(doc, maps).is_empty() { break; } - delete_in_all(doc, &levels[depth - 1], &path[depth - 1])?; + delete_in_all(doc, above, &path[depth - 1])?; } } @@ -406,8 +1306,8 @@ pub(crate) fn delete_path( /// `skip` sees the path of every map entry and list element (list indices as decimal /// strings); returning true leaves that subtree out. pub(crate) fn materialize_into( - src: &AutoCommit, - dst: &mut AutoCommit, + src: &impl ReadDoc, + dst: &mut impl Transactable, skip: &dyn Fn(&[String]) -> bool, ) -> Result<(), StoreError> { let mut path = Vec::new(); @@ -422,9 +1322,9 @@ pub(crate) fn materialize_into( } fn copy_maps( - src: &AutoCommit, + src: &impl ReadDoc, maps: &[ObjId], - dst: &mut AutoCommit, + dst: &mut impl Transactable, target: &ObjId, path: &mut Vec<String>, skip: &dyn Fn(&[String]) -> bool, @@ -443,9 +1343,9 @@ fn copy_maps( } fn copy_resolved( - src: &AutoCommit, + src: &impl ReadDoc, resolved: &Resolved, - dst: &mut AutoCommit, + dst: &mut impl Transactable, parent: &ObjId, slot: Slot, path: &mut Vec<String>, @@ -473,10 +1373,10 @@ fn copy_resolved( #[allow(clippy::too_many_arguments)] fn copy_object( - src: &AutoCommit, + src: &impl ReadDoc, obj_type: ObjType, id: &ObjId, - dst: &mut AutoCommit, + dst: &mut impl Transactable, parent: &ObjId, slot: Slot, path: &mut Vec<String>, @@ -527,10 +1427,12 @@ enum Slot<'a> { Key(&'a str), /// An index in a list. The value is inserted here, not overwritten. Index(usize), + /// An index in a list whose item is overwritten. + At(usize), } fn put_object( - doc: &mut AutoCommit, + doc: &mut impl Transactable, parent: &ObjId, slot: Slot, obj_type: ObjType, @@ -538,11 +1440,12 @@ fn put_object( Ok(match slot { Slot::Key(key) => doc.put_object(parent, key, obj_type)?, Slot::Index(index) => doc.insert_object(parent, index, obj_type)?, + Slot::At(index) => doc.put_object(parent, index, obj_type)?, }) } fn put_scalar( - doc: &mut AutoCommit, + doc: &mut impl Transactable, parent: &ObjId, slot: Slot, value: impl Into<ScalarValue>, @@ -550,72 +1453,59 @@ fn put_scalar( match slot { Slot::Key(key) => doc.put(parent, key, value)?, Slot::Index(index) => doc.insert(parent, index, value)?, + Slot::At(index) => doc.put(parent, index, value)?, } Ok(()) } fn write_value( - doc: &mut AutoCommit, + doc: &mut impl Transactable, parent: &ObjId, slot: Slot, - value: &JsonValue, + value: &Val, ) -> Result<(), StoreError> { match value { - JsonValue::Object(object) => { - let id = put_object(doc, parent, slot, ObjType::Map)?; - for (child_key, child_value) in object { - write_value(doc, &id, Slot::Key(child_key), child_value)?; - } - } - JsonValue::Array(items) => { - let id = put_object(doc, parent, slot, ObjType::List)?; - for (index, item) in items.iter().enumerate() { - write_value(doc, &id, Slot::Index(index), item)?; - } - } - JsonValue::String(value) => put_scalar(doc, parent, slot, value.as_str())?, - JsonValue::Bool(value) => put_scalar(doc, parent, slot, *value)?, - JsonValue::Number(value) => { - if let Some(value) = value.as_i64() { - put_scalar(doc, parent, slot, value)?; - } else if let Some(value) = value.as_u64() { - put_scalar(doc, parent, slot, value)?; - } else if let Some(value) = value.as_f64() { - put_scalar(doc, parent, slot, value)?; - } else { - return Err(anyhow!("Unsupported JSON number").into()); - } + Val::Scalar(scalar) => put_scalar(doc, parent, slot, scalar.clone()), + tree => { + // A new tree in one go, not an operation at a time. + let tree = hydrated(tree, doc.text_encoding()); + let (prop, insert): (Prop, bool) = match slot { + Slot::Key(key) => (key.into(), false), + Slot::Index(index) => (index.into(), true), + Slot::At(index) => (index.into(), false), + }; + doc.batch_create_object(parent, prop, &tree, insert)?; + Ok(()) } - JsonValue::Null => put_scalar(doc, parent, slot, ScalarValue::Null)?, } - - Ok(()) } -fn json_to_automerge( - doc: &mut AutoCommit, - parent: &ObjId, - key: &str, - value: &JsonValue, -) -> Result<(), StoreError> { - write_value(doc, parent, Slot::Key(key), value) +/// `value` as Automerge holds it. +fn hydrated(value: &Val, encoding: automerge::TextEncoding) -> automerge::hydrate::Value { + use automerge::hydrate; + match value { + Val::Map(entries) => hydrate::Value::Map(hydrate::Map::from( + entries + .iter() + .map(|(key, value)| (key.clone(), hydrated(value, encoding))) + .collect::<std::collections::HashMap<_, _>>(), + )), + Val::List(items) => hydrate::Value::from( + items + .iter() + .map(|item| hydrated(item, encoding)) + .collect::<Vec<_>>(), + ), + Val::Text(text) => hydrate::Value::text(encoding, text), + Val::Scalar(scalar) => hydrate::Value::Scalar(scalar.clone()), + } } -fn scalar_to_json(scalar: ScalarValue) -> Result<JsonValue, StoreError> { - match scalar { - ScalarValue::Null => Ok(JsonValue::Null), - ScalarValue::Boolean(b) => Ok(JsonValue::Bool(b)), - ScalarValue::Int(i) => Ok(JsonValue::Number(i.into())), - ScalarValue::Uint(u) => Ok(JsonValue::Number(u.into())), - ScalarValue::F64(f) => serde_json::Number::from_f64(f) - .map(JsonValue::Number) - .ok_or(anyhow!("Non-finite float cannot be represented as JSON").into()), - ScalarValue::Str(s) => Ok(JsonValue::String(s.to_string())), - ScalarValue::Bytes(b) => Ok(JsonValue::String(BASE64.encode(b))), - ScalarValue::Counter(c) => Ok(JsonValue::Number(i64::from(c).into())), - ScalarValue::Timestamp(t) => Ok(JsonValue::Number(t.into())), - ScalarValue::Unknown { .. } => Err(anyhow!("Unsupported Automerge scalar type").into()), - } +fn scalar_to_json(scalar: ScalarValue, form: JsonForm) -> Result<JsonValue, StoreError> { + typed::scalar_to_json(&scalar, form).ok_or_else(|| match scalar { + ScalarValue::F64(_) => anyhow!("Non-finite float cannot be represented as JSON").into(), + _ => anyhow!("Unsupported Automerge scalar type").into(), + }) } // --- conflict log ------------------------------------------------------------ @@ -655,6 +1545,8 @@ struct ConflictLog { want_values: bool, /// Currently walking the ancestors of the requested path. in_ancestors: bool, + /// How candidate values are shown. + form: JsonForm, } fn op_id_string(id: &ObjId) -> String { @@ -664,17 +1556,22 @@ fn op_id_string(id: &ObjId) -> String { } } -fn value_to_json(doc: &AutoCommit, value: &Value<'_>, id: &ObjId) -> Result<JsonValue, StoreError> { +fn value_to_json( + doc: &impl ReadDoc, + value: &Value<'_>, + id: &ObjId, + form: JsonForm, +) -> Result<JsonValue, StoreError> { match value { - Value::Scalar(scalar) => Ok(scalar_to_json(scalar.clone().into_owned())?), - Value::Object(_) => object_to_json(doc, id, None), + Value::Scalar(scalar) => Ok(scalar_to_json(scalar.clone().into_owned(), form)?), + Value::Object(_) => object_to_json(doc, id, form, None), } } impl ConflictLog { fn record( &mut self, - doc: &AutoCommit, + doc: &impl ReadDoc, key: &str, all: &[(Value<'static>, ObjId)], merged: bool, @@ -694,7 +1591,7 @@ impl ConflictLog { op_id: op_id_string(id), selected, value: if with_values { - Some(value_to_json(doc, value, id)?) + Some(value_to_json(doc, value, id, self.form)?) } else { None }, @@ -734,3 +1631,174 @@ fn nested<T>( } out } + +#[cfg(test)] +mod tests { + use super::*; + use crate::document::testing::try_change; + use serde_json::json; + + fn path(at: &str) -> Vec<String> { + at.split('/').map(str::to_owned).collect() + } + + #[test] + fn changes_name_the_paths_their_operations_touch() { + fn touched( + doc: &mut Automerge, + lists: InLists, + write: impl FnOnce(&mut automerge::transaction::Transaction<'_>) -> Result<(), StoreError>, + ) -> Vec<Vec<String>> { + try_change(doc, write).unwrap(); + let head = doc.get_heads()[0]; + changed_by(doc, &head, lists).unwrap() + } + let mut doc = Automerge::new(); + // An item by its list; what is inside it by its index. + assert_eq!( + touched(&mut doc, InLists::Index, |doc| { + write_path(doc, &path("a"), &json!({"b": 1, "list": [{"k": 1}]})) + }), + [path("a"), path("a/b"), path("a/list"), path("a/list/0/k")] + ); + assert_eq!( + touched(&mut doc, InLists::Index, |doc| { + write_path(doc, &path("a/list/0/k"), &json!(2)) + }), + [path("a/list/0/k")] + ); + assert_eq!( + touched(&mut doc, InLists::Whole, |doc| { + write_path(doc, &path("a/list/0/k"), &json!(3)) + }), + [path("a/list")] + ); + assert_eq!( + touched(&mut doc, InLists::Index, |doc| { + delete_path(doc, &path("a/b"), false) + }), + [path("a/b")] + ); + } + + #[test] + fn paths_go_through_lists() { + let mut doc = Automerge::new(); + try_change(&mut doc, |doc| { + write_path(doc, &path("list"), &json!(["b", {"k": 1}, "d"])) + }) + .unwrap(); + assert_eq!(read_path(&doc, &path("list/0")).unwrap(), Some(json!("b"))); + assert_eq!(read_path(&doc, &path("list/1/k")).unwrap(), Some(json!(1))); + // Not an index, or past the end: nothing. + for missing in ["list/3", "list/x", "list/01", "list/-1", "list/1/k/0"] { + assert_eq!(read_path(&doc, &path(missing)).unwrap(), None, "{missing}"); + } + // Replaced, and appended at the end; past it, nothing. + try_change(&mut doc, |doc| { + write_path(doc, &path("list/0"), &json!("a")) + }) + .unwrap(); + try_change(&mut doc, |doc| { + write_path(doc, &path("list/3"), &json!("e")) + }) + .unwrap(); + try_change(&mut doc, |doc| { + write_path(doc, &path("list/1/k"), &json!(2)) + }) + .unwrap(); + assert!(matches!( + try_change(&mut doc, |doc| write_path(doc, &path("list/9"), &json!(0))), + Err(StoreError::NotFound) + )); + assert_eq!( + read_path(&doc, &path("list")).unwrap(), + Some(json!(["a", {"k": 2}, "d", "e"])) + ); + try_change(&mut doc, |doc| delete_path(doc, &path("list/2"), false)).unwrap(); + assert_eq!( + read_path(&doc, &path("list")).unwrap(), + Some(json!(["a", {"k": 2}, "e"])) + ); + assert_eq!( + read_shallow(&doc, &path("list")).unwrap().unwrap()["entries"]["1"]["type"], + json!("map") + ); + } + + #[test] + fn negative_indexes_count_from_the_end() { + let mut doc = Automerge::new(); + try_change(&mut doc, |doc| { + write_path(doc, &path("list"), &json!(["a", "b"])) + }) + .unwrap(); + try_change(&mut doc, |doc| { + write_path(doc, &path("map"), &json!({"-1": "a key"})) + }) + .unwrap(); + assert_eq!( + positive_path(&doc, &path("list/-1"), false).unwrap(), + Some(path("list/1")) + ); + // Inserting: -1 is the end. + assert_eq!( + positive_path(&doc, &path("list/-1"), true).unwrap(), + Some(path("list/2")) + ); + assert!(matches!( + positive_path(&doc, &path("list/-3"), false), + Err(StoreError::NotFound) + )); + assert_eq!(positive_path(&doc, &path("map/-1"), false).unwrap(), None); + assert_eq!(positive_path(&doc, &path("list/0"), false).unwrap(), None); + } + + #[test] + fn inserts_into_lists() { + let mut doc = Automerge::new(); + try_change(&mut doc, |doc| { + write_path(doc, &path("list"), &json!(["b", "d"])) + }) + .unwrap(); + try_change(&mut doc, |doc| { + insert_path(doc, &path("list/0"), &json!("a")) + }) + .unwrap(); + try_change(&mut doc, |doc| { + insert_path(doc, &path("list/2"), &json!("c")) + }) + .unwrap(); + try_change(&mut doc, |doc| { + insert_path(doc, &path("list/4"), &json!({"e": 5})) + }) + .unwrap(); + assert_eq!( + read_path(&doc, &path("list")).unwrap(), + Some(json!(["a", "b", "c", "d", {"e": 5}])) + ); + assert!(matches!( + try_change(&mut doc, |doc| insert_path(doc, &path("list/6"), &json!(1))), + Err(StoreError::PastTheEnd { + index: 6, + length: 5 + }) + )); + try_change(&mut doc, |doc| { + write_path(doc, &path("map"), &json!({"k": 1})) + }) + .unwrap(); + assert!(matches!( + try_change(&mut doc, |doc| insert_path(doc, &path("map/0"), &json!(1))), + Err(StoreError::NotAList) + )); + assert!(matches!( + try_change(&mut doc, |doc| insert_path( + doc, + &path("nothing/0"), + &json!(1) + )), + Err(StoreError::NotFound) + )); + } +} diff --git a/src/state/session.rs b/src/document/sync.rs index 8885699..44ad833 100644 --- a/src/state/session.rs +++ b/src/document/sync.rs @@ -9,20 +9,33 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::time::Duration; +use automerge::AutomergeError; use automerge::sync::{self, SyncDoc}; -use automerge::{AutoCommit, AutomergeError}; use tokio::time::Instant; use tracing::debug; +use super::{Document, ReadDoc}; use crate::backoff::Backoff; use crate::swarm::throughput::SyncBytes; +use crate::transport::SyncPayload; + +/// A sync message from a peer, decoded (`decode`), to apply (`SyncSession::receive`). +pub(crate) struct Received(sync::Message); + +/// `payload` decoded. Before taking the document's lock: decoding a large message takes +/// a while. +pub(crate) fn decode(payload: &SyncPayload) -> anyhow::Result<Received> { + sync::Message::decode(&payload.0) + .map(Received) + .map_err(|error| anyhow::anyhow!("Automerge decode failed: {error:?}")) +} /// A sync message being handed to our transport (for Mycelium: the HTTP request is in /// progress). #[derive(Clone)] pub(crate) struct Sending { /// Changes it carries for the first time; taken back if the send fails. - pub(crate) sent: Vec<automerge::ChangeHash>, + pub(crate) sent: Vec<crate::document::ChangeHash>, /// A sync pass was requested meanwhile. Once this one is handed over, the peer must /// be enqueued again so the deferred changes go out. pub(crate) regenerate: bool, @@ -63,7 +76,7 @@ pub(crate) struct SyncSession { /// once the peer has the delta and says its heads are ours, never answers it, so the /// peer never learns our heads (a joining node then never finishes its initial /// sync). - before_generate: (Vec<automerge::ChangeHash>, bool), + before_generate: (Vec<crate::document::ChangeHash>, bool), /// The answer we expect to our last sync message(s), if any. pub(crate) reply: Option<SyncReplyWait>, @@ -81,7 +94,10 @@ pub(crate) struct SyncReplyWait { /// exchanged with the peer by then (`swarm::throughput`). pub(crate) started: (Instant, SyncBytes), /// Changes first sent in the unanswered messages. - pub(crate) sent: Vec<automerge::ChangeHash>, + pub(crate) sent: Vec<crate::document::ChangeHash>, + /// The largest of them, in bytes: if they go unanswered, messages that large may + /// not make it to the peer whole (`PeerState::whole_failed`). + pub(crate) largest: usize, } impl SyncSession { @@ -105,7 +121,7 @@ impl SyncSession { self.state.their_heads.is_some() } - /// The next sync message for the peer from `doc`, if there is anything to say. A + /// The next sync message for the peer from `doc`, encoded, if there is anything to say. A /// message is marked as being handed over (`sending`), and an answer to it is /// expected, unless the peer's heads are ours already: it has everything, and /// Automerge answers only when it has something to say, so waiting for an answer @@ -113,16 +129,16 @@ impl SyncSession { /// peer so far (`swarm::throughput`). pub(crate) fn generate( &mut self, - doc: &mut AutoCommit, + doc: &Document, now: Instant, bytes: SyncBytes, - ) -> Option<sync::Message> { + ) -> Option<SyncPayload> { let sent_before = self.state.sent_hashes.clone(); self.before_generate = ( self.state.last_sent_heads.clone(), self.state.have_responded, ); - let message = doc.sync().generate_sync_message(&mut self.state); + let message = doc.generate_sync_message(&mut self.state); let in_step = self.state.their_heads.as_ref() == Some(&doc.get_heads()); let sent: Vec<_> = self .state @@ -130,12 +146,13 @@ impl SyncSession { .difference(&sent_before) .copied() .collect(); + let payload = message.map(|message| SyncPayload(message.encode().into())); if in_step { self.reply_settled(); - } else if message.is_some() { - self.expect_reply(sent.clone(), now, bytes); + } else if let Some(payload) = &payload { + self.expect_reply(sent.clone(), now, bytes, payload.0.len()); } - if message.is_some() { + if payload.is_some() { // Marked together with the generation, so no second message is generated // while this one is handed over. self.sending = Some(Sending { @@ -143,18 +160,31 @@ impl SyncSession { regenerate: false, }); } - message + payload } /// Applies a sync message from the peer to `doc`. Doesn't settle the awaited /// answer: the caller measures the exchange first (`reply`). + #[cfg(test)] pub(crate) fn receive( &mut self, - doc: &mut AutoCommit, - message: sync::Message, + doc: &mut Document, + message: Received, + ) -> Result<(), AutomergeError> { + self.receive_logged(doc, message, &mut automerge::PatchLog::inactive()) + } + + /// Applies a sync message from the peer to `doc`, what it changes recorded in `log` + /// (`Replica::logged`). Doesn't settle the awaited answer: the caller measures the + /// exchange first (`reply`). + pub(crate) fn receive_logged( + &mut self, + doc: &mut Document, + message: Received, + log: &mut automerge::PatchLog, ) -> Result<(), AutomergeError> { self.delta_offered = false; - doc.sync().receive_sync_message(&mut self.state, message) + doc.receive_sync_message_log_patches(&mut self.state, message.0, log) } /// The message just generated is too large to send in one piece: take back what it @@ -162,7 +192,7 @@ impl SyncSession { /// message being handed over becomes the offer (it carries nothing); the answer is /// still awaited, and if it doesn't come, the offer is made again. Returns the /// peer's heads as far as we know them: where the delta starts. - pub(crate) fn withdraw_for_delta(&mut self) -> Vec<automerge::ChangeHash> { + pub(crate) fn withdraw_for_delta(&mut self) -> Vec<crate::document::ChangeHash> { if let Some(sending) = &mut self.sending { let sent = std::mem::take(&mut sending.sent); for hash in &sent { @@ -181,7 +211,7 @@ impl SyncSession { /// Whether `doc` holds everything the peer had when it last told us its heads: they /// are known, not empty (empty heads are "contained" in every document, so they /// prove nothing), and all in `doc`. - pub(crate) fn have_all_theirs(&self, doc: &mut AutoCommit) -> bool { + pub(crate) fn have_all_theirs(&self, doc: &Document) -> bool { self.state.their_heads.as_ref().is_some_and(|heads| { !heads.is_empty() && heads @@ -192,7 +222,7 @@ impl SyncSession { /// The peer has heads we lack (its keepalive said so): ask for what they need, /// even though Automerge thinks a message is still in flight. - pub(crate) fn pull_from(&mut self, heads: Vec<automerge::ChangeHash>) { + pub(crate) fn pull_from(&mut self, heads: Vec<crate::document::ChangeHash>) { self.state.their_heads = Some(heads); self.state.in_flight = false; } @@ -232,7 +262,7 @@ impl SyncSession { /// For the simulator's dump: the peer's heads as Automerge knows them, and whether /// it thinks a message is in flight. #[cfg(test)] - pub(crate) fn automerge_view(&self) -> (Option<&[automerge::ChangeHash]>, bool) { + pub(crate) fn sync_view(&self) -> (Option<&[crate::document::ChangeHash]>, bool) { (self.state.their_heads.as_deref(), self.state.in_flight) } @@ -290,22 +320,27 @@ impl SyncSession { Some(delay) } - /// A sync message went out; `sent` are the changes it carried for the first time. - /// `bytes` are the sync bytes exchanged with the peer before it. + /// A sync message of `len` bytes went out; `sent` are the changes it carried for the + /// first time. `bytes` are the sync bytes exchanged with the peer before it. pub(crate) fn expect_reply( &mut self, - sent: Vec<automerge::ChangeHash>, + sent: Vec<crate::document::ChangeHash>, now: Instant, bytes: SyncBytes, + len: usize, ) { match &mut self.reply { // Still waiting for an earlier one: keep its deadline, remember these too. - Some(wait) => wait.sent.extend(sent), + Some(wait) => { + wait.sent.extend(sent); + wait.largest = wait.largest.max(len); + } None => { self.reply = Some(SyncReplyWait { deadline: now + self.reply_backoff.current(), started: (now, bytes), sent, + largest: len, }); } } @@ -321,7 +356,7 @@ impl SyncSession { /// If the answer is overdue: forgets that the unanswered changes were sent and that /// a message is in flight, so the next sync pass sends them again. Returns how many /// changes that concerns. - pub(super) fn take_overdue_reply(&mut self, now: Instant) -> Option<usize> { + pub(crate) fn take_overdue_reply(&mut self, now: Instant) -> Option<usize> { if self.reply.as_ref().is_none_or(|wait| now < wait.deadline) { return None; } @@ -371,47 +406,48 @@ mod tests { /// trip (`answer_for_everything_since_shared`). #[test] fn missing_predecessors_go_at_once() { + use crate::document::testing::change; use automerge::transaction::Transactable; - let mut a = AutoCommit::new(); - let mut b = AutoCommit::new(); + let mut a = Document::new(); + let mut b = Document::new(); let (mut at_a, mut at_b) = (sync::State::new(), sync::State::new()); - a.put(automerge::ROOT, "early", 1).unwrap(); + change(&mut a, |a| a.put(automerge::ROOT, "early", 1).unwrap()); // Until in step. for _ in 0..10 { - let to_b = a.sync().generate_sync_message(&mut at_a); - let to_a = b.sync().generate_sync_message(&mut at_b); + let to_b = a.generate_sync_message(&mut at_a); + let to_a = b.generate_sync_message(&mut at_b); if to_b.is_none() && to_a.is_none() { break; } if let Some(m) = to_b { - b.sync().receive_sync_message(&mut at_b, m).unwrap(); + b.receive_sync_message(&mut at_b, m).unwrap(); } if let Some(m) = to_a { - a.sync().receive_sync_message(&mut at_a, m).unwrap(); + a.receive_sync_message(&mut at_a, m).unwrap(); } } assert_eq!(a.get_heads(), b.get_heads()); // Five changes, whose message is lost; then a sixth, which arrives alone. for i in 0..5 { - a.put(automerge::ROOT, format!("lost{i}"), i).unwrap(); - a.commit(); + change(&mut a, |a| { + a.put(automerge::ROOT, format!("lost{i}"), i).unwrap() + }); } - let _lost = a.sync().generate_sync_message(&mut at_a).unwrap(); - a.put(automerge::ROOT, "last", 6).unwrap(); - a.commit(); + let _lost = a.generate_sync_message(&mut at_a).unwrap(); + change(&mut a, |a| a.put(automerge::ROOT, "last", 6).unwrap()); at_a.in_flight = false; // As after our reply timeout. - let last = a.sync().generate_sync_message(&mut at_a).unwrap(); + let last = a.generate_sync_message(&mut at_a).unwrap(); assert_eq!(last.changes.len(), 1); - b.sync().receive_sync_message(&mut at_b, last).unwrap(); + b.receive_sync_message(&mut at_b, last).unwrap(); // b asks for the fifth, saying nothing of what it has. - let ask = b.sync().generate_sync_message(&mut at_b).unwrap(); + let ask = b.generate_sync_message(&mut at_b).unwrap(); assert!(ask.have.is_empty() && ask.need.len() == 1); - a.sync().receive_sync_message(&mut at_a, ask).unwrap(); + a.receive_sync_message(&mut at_a, ask).unwrap(); assert!(answer_for_everything_since_shared(&mut at_a)); // All since the shared heads, in one message (here the whole document: more // than a third of it is missing). - let answer = a.sync().generate_sync_message(&mut at_a).unwrap(); - b.sync().receive_sync_message(&mut at_b, answer).unwrap(); + let answer = a.generate_sync_message(&mut at_a).unwrap(); + b.receive_sync_message(&mut at_b, answer).unwrap(); assert_eq!(b.get_heads(), a.get_heads()); // A message with what it has is left alone. assert!(!answer_for_everything_since_shared(&mut at_b)); @@ -454,6 +490,7 @@ mod tests { /// overdue answer, gets the same changes in the next message. #[test] fn withdrawing_for_a_delta_keeps_the_changes_for_later() { + use crate::document::testing::change; use automerge::{ROOT, transaction::Transactable}; // 16 KiB of hex, about 8 KiB of information: Automerge compresses what it sends @@ -462,23 +499,22 @@ mod tests { let big: String = (0..1024) .map(|_| format!("{:016x}", rng.next_u64())) .collect(); - let mut ours = AutoCommit::new(); - ours.put(ROOT, "big", big).unwrap(); - ours.commit(); - let mut theirs = AutoCommit::new(); + let mut ours = Document::new(); + change(&mut ours, |ours| ours.put(ROOT, "big", big).unwrap()); + let mut theirs = Document::new(); let (mut a, mut b) = (session(), session()); let now = Instant::now(); let bytes = SyncBytes::default(); // a opens, b answers with its (empty) heads, then a sends the changes. - let opening = a.generate(&mut ours, now, bytes).unwrap(); + let opening = a.generate(&ours, now, bytes).unwrap(); a.send_succeeded(a.id()); - b.receive(&mut theirs, opening).unwrap(); - let answer = b.generate(&mut theirs, now, bytes).unwrap(); + b.receive(&mut theirs, decode(&opening).unwrap()).unwrap(); + let answer = b.generate(&theirs, now, bytes).unwrap(); b.send_succeeded(b.id()); - a.receive(&mut ours, answer).unwrap(); - let carrying = a.generate(&mut ours, now, bytes).unwrap(); - let carried = carrying.clone().encode().len(); + a.receive(&mut ours, decode(&answer).unwrap()).unwrap(); + let carrying = a.generate(&ours, now, bytes).unwrap(); + let carried = carrying.0.len(); assert!(carried > 4_000, "the changes took {carried} bytes"); let from = a.withdraw_for_delta(); @@ -492,8 +528,8 @@ mod tests { .is_some() ); assert!(!a.delta_offered); - let again = a.generate(&mut ours, now, bytes).unwrap(); - let carried_again = again.encode().len(); + let again = a.generate(&ours, now, bytes).unwrap(); + let carried_again = again.0.len(); assert!(carried_again > 4_000, "sent again in {carried_again} bytes"); } @@ -502,42 +538,39 @@ mod tests { /// has everything (`have_all_theirs`). #[test] fn after_a_delta_the_peer_learns_our_heads() { + use crate::document::testing::change; use automerge::{ROOT, transaction::Transactable}; let mut rng = crate::testrand::Rng::new(2); let big: String = (0..1024) .map(|_| format!("{:016x}", rng.next_u64())) .collect(); - let mut ours = AutoCommit::new(); - ours.put(ROOT, "big", big).unwrap(); - ours.commit(); - let mut theirs = AutoCommit::new(); + let mut ours = Document::new(); + change(&mut ours, |ours| ours.put(ROOT, "big", big).unwrap()); + let mut theirs = Document::new(); let (mut a, mut b) = (session(), session()); let now = Instant::now(); let bytes = SyncBytes::default(); // b (joining) opens; a's answer would carry everything: withdrawn for a delta. - let opening = b.generate(&mut theirs, now, bytes).unwrap(); + let opening = b.generate(&theirs, now, bytes).unwrap(); b.send_succeeded(b.id()); - a.receive(&mut ours, opening).unwrap(); - assert!(a.generate(&mut ours, now, bytes).is_some()); + a.receive(&mut ours, decode(&opening).unwrap()).unwrap(); + assert!(a.generate(&ours, now, bytes).is_some()); a.withdraw_for_delta(); a.send_succeeded(a.id()); // b loads the delta and tells a its heads, which are a's now. theirs.merge(&mut ours.clone()).unwrap(); - assert!( - !b.have_all_theirs(&mut theirs), - "b hasn't heard a's heads yet" - ); - let after_delta = b.generate(&mut theirs, now, bytes).unwrap(); + assert!(!b.have_all_theirs(&theirs), "b hasn't heard a's heads yet"); + let after_delta = b.generate(&theirs, now, bytes).unwrap(); b.send_succeeded(b.id()); - a.receive(&mut ours, after_delta).unwrap(); + a.receive(&mut ours, decode(&after_delta).unwrap()).unwrap(); let answer = a - .generate(&mut ours, now, bytes) + .generate(&ours, now, bytes) .expect("a answers with its heads"); - b.receive(&mut theirs, answer).unwrap(); - assert!(b.have_all_theirs(&mut theirs)); + b.receive(&mut theirs, decode(&answer).unwrap()).unwrap(); + assert!(b.have_all_theirs(&theirs)); } } diff --git a/src/document/testing.rs b/src/document/testing.rs new file mode 100644 index 0000000..7b75a96 --- /dev/null +++ b/src/document/testing.rs @@ -0,0 +1,31 @@ +//! For tests: writing to documents the way the node does, one change at a time, so +//! that tests don't depend on how the engine groups writes into changes. + +use super::{Document, Transaction}; + +/// The writes `f` makes to `doc`, as one change; what `f` returns. +pub(crate) fn change<T>(doc: &mut Document, f: impl FnOnce(&mut Transaction<'_>) -> T) -> T { + let mut transaction = doc.transaction(); + let out = f(&mut transaction); + transaction.commit(); + out +} + +/// The writes `f` makes to `doc`, as one change if it succeeds, dropped if not; what +/// `f` returns. Committed at time 0: the same writes make the same change. +pub(crate) fn try_change<T, E>( + doc: &mut Document, + f: impl FnOnce(&mut Transaction<'_>) -> Result<T, E>, +) -> Result<T, E> { + let mut transaction = doc.transaction(); + let out = f(&mut transaction); + match &out { + Ok(_) => { + transaction.commit_with(automerge::transaction::CommitOptions::default().with_time(0)); + } + Err(_) => { + transaction.rollback(); + } + } + out +} diff --git a/src/document/typed.rs b/src/document/typed.rs new file mode 100644 index 0000000..def06df --- /dev/null +++ b/src/document/typed.rs @@ -0,0 +1,679 @@ +//! The types a value is stored as, and how each is shown: as JSON (every type has a +//! JSON form), and natively, as a media type of its own, where it is a scalar or a text +//! (`docs/api.md`, Types). + +use base64::{Engine, engine::general_purpose::STANDARD as BASE64}; +use serde_json::Value as JsonValue; + +use automerge::ScalarValue; + +/// What a value is stored as. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Kind { + Map, + List, + /// An Automerge text (characters that merge one by one), shown as a string. + Text, + String, + Bytes, + Int, + Uint, + Float, + Boolean, + Null, + Counter, + Timestamp, + /// A scalar of a later Automerge version. + Unknown, +} + +pub(crate) const OCTET_STREAM: &str = "application/octet-stream"; +pub(crate) const TEXT: &str = "text/plain"; +pub(crate) const INT: &str = "application/vnd.continuum.int"; +pub(crate) const UINT: &str = "application/vnd.continuum.uint"; +pub(crate) const FLOAT: &str = "application/vnd.continuum.float"; +pub(crate) const BOOLEAN: &str = "application/vnd.continuum.boolean"; +pub(crate) const COUNTER: &str = "application/vnd.continuum.counter"; +pub(crate) const TIMESTAMP: &str = "application/vnd.continuum.timestamp"; +/// JSON whose scalars keep their types (`Val::from_typed_json`, `scalar_to_json`). +pub(crate) const TYPED_JSON: &str = "application/vnd.continuum.typed+json"; + +impl Kind { + pub(crate) fn of(scalar: &ScalarValue) -> Self { + match scalar { + ScalarValue::Str(_) => Kind::String, + ScalarValue::Bytes(_) => Kind::Bytes, + ScalarValue::Int(_) => Kind::Int, + ScalarValue::Uint(_) => Kind::Uint, + ScalarValue::F64(_) => Kind::Float, + ScalarValue::Boolean(_) => Kind::Boolean, + ScalarValue::Null => Kind::Null, + ScalarValue::Counter(_) => Kind::Counter, + ScalarValue::Timestamp(_) => Kind::Timestamp, + ScalarValue::Unknown { .. } => Kind::Unknown, + } + } + + /// As `x-continuum-type` and shallow reads name it. + pub(crate) fn name(self) -> &'static str { + match self { + Kind::Map => "map", + Kind::List => "list", + Kind::Text => "text", + Kind::String => "string", + Kind::Bytes => "bytes", + Kind::Int => "int", + Kind::Uint => "uint", + Kind::Float => "float", + Kind::Boolean => "boolean", + Kind::Null => "null", + Kind::Counter => "counter", + Kind::Timestamp => "timestamp", + Kind::Unknown => "unknown", + } + } + + /// The kind `name` names (`name`). + pub(crate) fn named(name: &str) -> Option<Self> { + [ + Kind::Map, + Kind::List, + Kind::Text, + Kind::String, + Kind::Bytes, + Kind::Int, + Kind::Uint, + Kind::Float, + Kind::Boolean, + Kind::Null, + Kind::Counter, + Kind::Timestamp, + ] + .into_iter() + .find(|kind| kind.name() == name) + } + + /// The media type a value of this kind is shown as natively, if any. + pub(crate) fn media_type(self) -> Option<&'static str> { + Some(match self { + Kind::Text | Kind::String => TEXT, + Kind::Bytes => OCTET_STREAM, + Kind::Int => INT, + Kind::Uint => UINT, + Kind::Float => FLOAT, + Kind::Boolean => BOOLEAN, + Kind::Counter => COUNTER, + Kind::Timestamp => TIMESTAMP, + Kind::Map | Kind::List | Kind::Null | Kind::Unknown => return None, + }) + } +} + +/// How a timestamp is written out: as text natively (`format=`), and in JSON +/// (`timestamp-format=`). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) enum TimestampFormat { + /// `2026-11-01T21:50:20.345Z` (a string in JSON). + #[default] + Rfc3339, + /// Seconds since 1970, with milliseconds as a fraction (a number in JSON). + Unix, + /// Milliseconds since 1970 (an integer in JSON). + UnixMs, + /// `Sun, 01 Nov 2026 21:50:20 GMT` (RFC 9110; whole seconds; a string in JSON). + HttpDate, +} + +impl TimestampFormat { + pub(crate) fn named(name: &str) -> Option<Self> { + Some(match name { + "rfc3339" => Self::Rfc3339, + "unix" => Self::Unix, + "unix-ms" => Self::UnixMs, + "http-date" => Self::HttpDate, + _ => return None, + }) + } + + pub(crate) fn name(self) -> &'static str { + match self { + Self::Rfc3339 => "rfc3339", + Self::Unix => "unix", + Self::UnixMs => "unix-ms", + Self::HttpDate => "http-date", + } + } + + /// `ms` (milliseconds since 1970) as text. + pub(crate) fn text(self, ms: i64) -> String { + match self { + Self::UnixMs => ms.to_string(), + Self::Unix => { + let sign = if ms < 0 { "-" } else { "" }; + let (seconds, millis) = (ms.unsigned_abs() / 1000, ms.unsigned_abs() % 1000); + if millis == 0 { + format!("{sign}{seconds}") + } else { + format!("{sign}{seconds}.{millis:03}") + .trim_end_matches('0') + .to_owned() + } + } + Self::Rfc3339 | Self::HttpDate => match jiff::Timestamp::from_millisecond(ms) { + Ok(at) if self == Self::Rfc3339 => at.to_string(), + Ok(at) => jiff::fmt::rfc2822::DateTimePrinter::new() + .timestamp_to_rfc9110_string(&at) + .unwrap_or_else(|_| ms.to_string()), + // Beyond what a date shows: the number itself. + Err(_) => ms.to_string(), + }, + } + } + + /// `ms` in JSON: a string, or for the Unix formats a number. + pub(crate) fn json(self, ms: i64) -> JsonValue { + match self { + Self::UnixMs => JsonValue::from(ms), + Self::Unix => serde_json::Number::from_f64(ms as f64 / 1000.0) + .map_or(JsonValue::from(ms / 1000), JsonValue::Number), + Self::Rfc3339 | Self::HttpDate => JsonValue::String(self.text(ms)), + } + } + + /// Milliseconds since 1970 from `text` in this format; None if it isn't one, or is + /// finer than a millisecond. + pub(crate) fn read(self, text: &str) -> Option<i64> { + let text = text.trim(); + match self { + Self::UnixMs => text.parse().ok(), + Self::Unix => { + let (seconds, fraction) = text.split_once('.').unwrap_or((text, "")); + if fraction.len() > 3 || !fraction.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + let seconds: i64 = seconds.parse().ok()?; + let millis: i64 = format!("{fraction:0<3}").parse().ok()?; + let millis = if text.starts_with('-') { + -millis + } else { + millis + }; + seconds.checked_mul(1000)?.checked_add(millis) + } + Self::Rfc3339 => { + let at: jiff::Timestamp = text.parse().ok()?; + (at.subsec_nanosecond() % 1_000_000 == 0).then(|| at.as_millisecond()) + } + Self::HttpDate => jiff::fmt::rfc2822::DateTimeParser::new() + .parse_timestamp(text) + .ok() + .map(|at| at.as_millisecond()), + } + } +} + +/// How values are shown in JSON. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct JsonForm { + pub(crate) timestamps: TimestampFormat, + /// Typed JSON (`TYPED_JSON`): what plain JSON doesn't tell apart is tagged, so it + /// reads back as it is stored. + pub(crate) typed: bool, +} + +// Typed JSON is plain JSON but for what plain JSON shows as another type: a single-key +// object whose key is a tag names it. `{"$uint": 5}`, `{"$counter": 5}`, +// `{"$timestamp": "2026-11-01T21:50:20.345Z"}` (in the timestamp format asked), +// `{"$bytes": "<base64>"}`, `{"$text": "…"}` (an Automerge text). A float is a number, +// as in plain JSON; a number without a fraction written over a float stays one +// (`store::Sticky::Floats`). A map whose only key starts with `$` is `{"$map": {…}}`, so that +// nothing else reads as a tag. + +/// A map's single key that is a tag, or would be taken for one. +pub(crate) fn tag_like(map: &serde_json::Map<String, JsonValue>) -> bool { + map.len() == 1 && map.keys().all(|key| key.starts_with('$')) +} + +fn tagged(tag: &str, value: JsonValue) -> JsonValue { + let mut map = serde_json::Map::new(); + map.insert(tag.to_owned(), value); + JsonValue::Object(map) +} + +/// A map in `form` (`$map` around one that would read as a tag in typed JSON). +pub(crate) fn map_to_json(map: serde_json::Map<String, JsonValue>, form: JsonForm) -> JsonValue { + if form.typed && tag_like(&map) { + tagged("$map", JsonValue::Object(map)) + } else { + JsonValue::Object(map) + } +} + +/// A text in `form`. +pub(crate) fn text_to_json(text: String, form: JsonForm) -> JsonValue { + if form.typed { + tagged("$text", JsonValue::String(text)) + } else { + JsonValue::String(text) + } +} + +/// `scalar` in JSON: integers, floats and counters as numbers, bytes as base64, +/// timestamps as `form` says; in typed JSON, tagged where plain JSON would show it as +/// another type. +pub(crate) fn scalar_to_json(scalar: &ScalarValue, form: JsonForm) -> Option<JsonValue> { + if form.typed { + match scalar { + ScalarValue::Uint(u) => return Some(tagged("$uint", (*u).into())), + ScalarValue::Counter(c) => return Some(tagged("$counter", i64::from(c).into())), + ScalarValue::Timestamp(t) => { + return Some(tagged("$timestamp", form.timestamps.json(*t))); + } + ScalarValue::Bytes(b) => return Some(tagged("$bytes", BASE64.encode(b).into())), + _ => {} + } + } + Some(match scalar { + ScalarValue::Null => JsonValue::Null, + ScalarValue::Boolean(b) => JsonValue::Bool(*b), + ScalarValue::Int(i) => JsonValue::Number((*i).into()), + ScalarValue::Uint(u) => JsonValue::Number((*u).into()), + ScalarValue::F64(f) => JsonValue::Number(serde_json::Number::from_f64(*f)?), + ScalarValue::Str(s) => JsonValue::String(s.to_string()), + ScalarValue::Bytes(b) => JsonValue::String(BASE64.encode(b)), + ScalarValue::Counter(c) => JsonValue::Number(i64::from(c).into()), + ScalarValue::Timestamp(t) => form.timestamps.json(*t), + ScalarValue::Unknown { .. } => return None, + }) +} + +/// `scalar` shown natively: its media type (with the timestamp format) and body; None +/// for a null or an unknown scalar, which have none. +pub(crate) fn native( + scalar: &ScalarValue, + timestamps: TimestampFormat, +) -> Option<(String, Vec<u8>)> { + let media = Kind::of(scalar).media_type()?; + let body = match scalar { + ScalarValue::Str(text) => { + return Some((format!("{TEXT}; charset=utf-8"), text.as_bytes().to_vec())); + } + ScalarValue::Bytes(bytes) => bytes.clone(), + ScalarValue::Int(i) => i.to_string().into_bytes(), + ScalarValue::Uint(u) => u.to_string().into_bytes(), + ScalarValue::F64(f) => f.to_string().into_bytes(), + ScalarValue::Boolean(b) => b.to_string().into_bytes(), + ScalarValue::Counter(c) => i64::from(c).to_string().into_bytes(), + ScalarValue::Timestamp(t) => { + return Some(( + format!("{TIMESTAMP}; format={}", timestamps.name()), + timestamps.text(*t).into_bytes(), + )); + } + ScalarValue::Null | ScalarValue::Unknown { .. } => return None, + }; + Some((media.to_owned(), body)) +} + +/// The value of media type `media` (with its `params`) that `body` holds, for a write; +/// None if `media` isn't one of the native types (`text/plain` is a JSON string's, and +/// read as one by the caller). An error says what's wrong with the body. +pub(crate) fn from_native( + media: &str, + params: &str, + body: &[u8], +) -> Option<Result<ScalarValue, String>> { + let text = || { + std::str::from_utf8(body) + .map(str::trim) + .map_err(|_| "not UTF-8".to_owned()) + }; + Some(match media { + OCTET_STREAM => Ok(ScalarValue::Bytes(body.to_vec())), + INT => text().and_then(|text| { + text.parse() + .map(ScalarValue::Int) + .map_err(|_| format!("not an integer: {text}")) + }), + UINT => text().and_then(|text| { + text.parse() + .map(ScalarValue::Uint) + .map_err(|_| format!("not an unsigned integer: {text}")) + }), + FLOAT => text().and_then(|text| { + text.parse::<f64>() + .ok() + .filter(|f| f.is_finite()) + .map(ScalarValue::F64) + .ok_or_else(|| format!("not a finite number: {text}")) + }), + BOOLEAN => text().and_then(|text| match text { + "true" => Ok(ScalarValue::Boolean(true)), + "false" => Ok(ScalarValue::Boolean(false)), + other => Err(format!("not true or false: {other}")), + }), + COUNTER => text().and_then(|text| { + text.parse::<i64>() + .map(ScalarValue::counter) + .map_err(|_| format!("not an integer: {text}")) + }), + TIMESTAMP => { + let format = match param(params, "format") { + None => TimestampFormat::default(), + Some(name) => match TimestampFormat::named(&name) { + Some(format) => format, + None => return Some(Err(format!("unknown timestamp format: {name}"))), + }, + }; + text().and_then(|text| { + format + .read(text) + .map(ScalarValue::Timestamp) + .ok_or_else(|| { + format!( + "not a {} timestamp (to the millisecond): {text}", + format.name() + ) + }) + }) + } + _ => return None, + }) +} + +/// `value` (a scalar, or a text's characters as a string), held as `from`, cast to +/// `to` (`store::cast_path`): numbers (integers, floats, counters, timestamps as +/// milliseconds since 1970) to numbers where they are exactly that; anything but bytes +/// to a string or a text as its type's own media type shows it (`native`, timestamps +/// in `timestamps`), and back (`from_native`); bytes to a string or a text and back as +/// UTF-8. An error says why not (maps, lists and nulls are cast to nothing, and nothing +/// to them). +pub(crate) fn cast( + from: Kind, + value: &ScalarValue, + to: Kind, + timestamps: TimestampFormat, +) -> Result<Val, String> { + let string = |value: &ScalarValue| -> Result<String, String> { + match value { + ScalarValue::Str(text) => Ok(text.to_string()), + ScalarValue::Bytes(bytes) => { + String::from_utf8(bytes.clone()).map_err(|_| "not UTF-8".to_owned()) + } + other => native(other, timestamps) + .and_then(|(_, body)| String::from_utf8(body).ok()) + .ok_or_else(|| "it has no text form".to_owned()), + } + }; + match to { + Kind::String => { + return string(value).map(|text| Val::Scalar(ScalarValue::Str(text.into()))); + } + Kind::Text => return string(value).map(Val::Text), + Kind::Map | Kind::List | Kind::Null | Kind::Unknown => { + return Err(format!("nothing is cast to {}", to.name())); + } + _ => {} + } + if let ScalarValue::Str(text) = value { + if to == Kind::Bytes { + return Ok(Val::Scalar(ScalarValue::Bytes(text.as_bytes().to_vec()))); + } + let media = to.media_type().ok_or("it has no text form")?; + let params = format!("; format={}", timestamps.name()); + return from_native(media, ¶ms, text.as_bytes()) + .unwrap_or_else(|| Err("it has no text form".to_owned())) + .map(Val::Scalar); + } + // Numbers: exactly, as an integer if they are one, else as a float. + let number = match value { + ScalarValue::Int(i) => Ok(i128::from(*i)), + ScalarValue::Uint(u) => Ok(i128::from(*u)), + ScalarValue::Counter(c) => Ok(i128::from(i64::from(c))), + ScalarValue::Timestamp(t) => Ok(i128::from(*t)), + ScalarValue::F64(f) => Err(*f), + _ => return Err(format!("{} isn't a number", from.name())), + }; + let integer = || -> Result<i128, String> { + match number { + Ok(integer) => Ok(integer), + // Integral floats up to 2^64 convert exactly (`as` saturates beyond). + Err(float) if float.fract() == 0.0 && float.abs() < 2f64.powi(64) => Ok(float as i128), + Err(float) => Err(format!("{float} isn't an integer that fits")), + } + }; + let signed = || -> Result<i64, String> { + let integer = integer()?; + i64::try_from(integer).map_err(|_| format!("{integer} doesn't fit a 64-bit integer")) + }; + let scalar = match to { + Kind::Int => ScalarValue::Int(signed()?), + Kind::Counter => ScalarValue::counter(signed()?), + Kind::Timestamp => ScalarValue::Timestamp(signed()?), + Kind::Uint => { + let integer = integer()?; + ScalarValue::Uint( + u64::try_from(integer) + .map_err(|_| format!("{integer} doesn't fit an unsigned 64-bit integer"))?, + ) + } + Kind::Float => match number { + Err(float) => ScalarValue::F64(float), + // Exactly: up to 2^53 every integer is a float. + Ok(integer) if integer.unsigned_abs() <= 1 << 53 => ScalarValue::F64(integer as f64), + Ok(integer) => return Err(format!("{integer} isn't exactly a float")), + }, + _ => return Err(format!("numbers aren't cast to {}", to.name())), + }; + Ok(Val::Scalar(scalar)) +} + +/// The value of parameter `name` among a media type's `params` (`; a=b; c="d"`). +pub(crate) fn param(params: &str, name: &str) -> Option<String> { + params.split(';').find_map(|param| { + let (key, value) = param.split_once('=')?; + key.trim() + .eq_ignore_ascii_case(name) + .then(|| value.trim().trim_matches('"').to_owned()) + }) +} + +/// A value to write (`store::write_val_path`), as Automerge stores it. +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum Val { + /// Its keys in order. + Map(Vec<(String, Val)>), + List(Vec<Val>), + /// An Automerge text. + Text(String), + Scalar(ScalarValue), +} + +/// What a plain JSON `value` is stored as, if it is a scalar: an integer where it fits +/// (signed, else unsigned), a float otherwise. +pub(crate) fn json_scalar(value: &JsonValue) -> Result<Option<ScalarValue>, String> { + Ok(Some(match value { + JsonValue::String(value) => ScalarValue::Str(value.as_str().into()), + JsonValue::Bool(value) => ScalarValue::Boolean(*value), + JsonValue::Number(value) => { + if let Some(value) = value.as_i64() { + ScalarValue::Int(value) + } else if let Some(value) = value.as_u64() { + ScalarValue::Uint(value) + } else if let Some(value) = value.as_f64() { + ScalarValue::F64(value) + } else { + return Err(format!("unsupported JSON number {value}")); + } + } + JsonValue::Null => ScalarValue::Null, + JsonValue::Object(_) | JsonValue::Array(_) => return Ok(None), + })) +} + +impl Val { + /// Plain JSON, as it is stored: maps, lists and scalars (`json_scalar`). + pub(crate) fn from_json(value: &JsonValue) -> Result<Val, String> { + Ok(match value { + JsonValue::Object(map) => Val::Map( + map.iter() + .map(|(key, value)| Ok((key.clone(), Val::from_json(value)?))) + .collect::<Result<_, String>>()?, + ), + JsonValue::Array(items) => { + Val::List(items.iter().map(Val::from_json).collect::<Result<_, _>>()?) + } + scalar => Val::Scalar(json_scalar(scalar)?.expect("a scalar")), + }) + } + + /// Typed JSON (`TYPED_JSON`): plain JSON, but tagged values as their tags say, + /// timestamps in `timestamps`. An object of one key that starts with `$` and is no + /// tag is an error. + pub(crate) fn from_typed_json( + value: &JsonValue, + timestamps: TimestampFormat, + ) -> Result<Val, String> { + let JsonValue::Object(map) = value else { + return match value { + JsonValue::Array(items) => Ok(Val::List( + items + .iter() + .map(|item| Val::from_typed_json(item, timestamps)) + .collect::<Result<_, _>>()?, + )), + scalar => Val::from_json(scalar), + }; + }; + if !tag_like(map) { + return Ok(Val::Map( + map.iter() + .map(|(key, value)| Ok((key.clone(), Val::from_typed_json(value, timestamps)?))) + .collect::<Result<_, String>>()?, + )); + } + let (tag, value) = map.iter().next().expect("one key"); + let integer = |value: &JsonValue| { + value + .as_i64() + .ok_or_else(|| format!("{tag}: not an integer: {value}")) + }; + Ok(match tag.as_str() { + "$map" => match value { + JsonValue::Object(map) => Val::Map( + map.iter() + .map(|(key, value)| { + Ok((key.clone(), Val::from_typed_json(value, timestamps)?)) + }) + .collect::<Result<_, String>>()?, + ), + other => return Err(format!("$map: not an object: {other}")), + }, + "$text" => Val::Text( + value + .as_str() + .ok_or_else(|| format!("$text: not a string: {value}"))? + .to_owned(), + ), + "$bytes" => Val::Scalar(ScalarValue::Bytes( + value + .as_str() + .and_then(|text| BASE64.decode(text).ok()) + .ok_or_else(|| format!("$bytes: not base64: {value}"))?, + )), + "$uint" => { + Val::Scalar(ScalarValue::Uint(value.as_u64().ok_or_else(|| { + format!("$uint: not an unsigned integer: {value}") + })?)) + } + "$counter" => Val::Scalar(ScalarValue::counter(integer(value)?)), + "$timestamp" => { + let text = match value { + JsonValue::String(text) => text.clone(), + JsonValue::Number(number) => number.to_string(), + other => { + return Err(format!( + "$timestamp: neither a string nor a number: {other}" + )); + } + }; + Val::Scalar(ScalarValue::Timestamp(timestamps.read(&text).ok_or_else( + || format!("$timestamp: not a {} timestamp: {text}", timestamps.name()), + )?)) + } + other => { + return Err(format!( + "no tag {other} (a map whose only key starts with $ is written as {{\"$map\": …}})" + )); + } + }) + } + + /// What it is stored as. + pub(crate) fn kind(&self) -> Kind { + match self { + Val::Map(_) => Kind::Map, + Val::List(_) => Kind::List, + Val::Text(_) => Kind::Text, + Val::Scalar(scalar) => Kind::of(scalar), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn cast_to(from: ScalarValue, to: Kind) -> Result<Val, String> { + cast(Kind::of(&from), &from, to, TimestampFormat::default()) + } + + #[test] + fn numbers_cast_where_exact() { + assert_eq!( + cast_to(ScalarValue::F64(2.0), Kind::Int), + Ok(Val::Scalar(ScalarValue::Int(2))) + ); + assert!(cast_to(ScalarValue::F64(2.5), Kind::Int).is_err()); + assert!(cast_to(ScalarValue::Int(-1), Kind::Uint).is_err()); + assert_eq!( + cast_to(ScalarValue::Int(5), Kind::Counter), + Ok(Val::Scalar(ScalarValue::counter(5))) + ); + assert!(cast_to(ScalarValue::Int(i64::MAX), Kind::Float).is_err()); + assert!(cast_to(ScalarValue::Boolean(true), Kind::Int).is_err()); + } + + #[test] + fn strings_cast_as_media_types_show_them() { + assert_eq!( + cast_to( + ScalarValue::Str("2026-11-01T21:50:20.345Z".into()), + Kind::Timestamp + ), + Ok(Val::Scalar(ScalarValue::Timestamp(1_793_569_820_345))) + ); + assert!(cast_to(ScalarValue::Str("soon".into()), Kind::Timestamp).is_err()); + assert_eq!( + cast_to(ScalarValue::Timestamp(1_793_569_820_345), Kind::String), + Ok(Val::Scalar(ScalarValue::Str( + "2026-11-01T21:50:20.345Z".into() + ))) + ); + assert_eq!( + cast_to(ScalarValue::Str("true".into()), Kind::Boolean), + Ok(Val::Scalar(ScalarValue::Boolean(true))) + ); + assert_eq!( + cast_to(ScalarValue::Str("hé".into()), Kind::Text), + Ok(Val::Text("hé".to_owned())) + ); + assert_eq!( + cast_to(ScalarValue::Bytes(b"hi".to_vec()), Kind::String), + Ok(Val::Scalar(ScalarValue::Str("hi".into()))) + ); + assert!(cast_to(ScalarValue::Bytes(vec![0xff]), Kind::String).is_err()); + assert!(cast_to(ScalarValue::Null, Kind::String).is_err()); + assert!(cast_to(ScalarValue::Int(1), Kind::Map).is_err()); + } +} diff --git a/src/files.rs b/src/files.rs index 0ec741e..bc8fb8e 100644 --- a/src/files.rs +++ b/src/files.rs @@ -167,20 +167,101 @@ fn sync_parent(path: &Path) { } } +/// A key file's bytes: read from `path`; or with `create`, if there is no file there, +/// `len` random bytes in a new one only its owner may read (then also `true`). Without +/// `create`, a missing file is an error saying to give `create_option` (`what`: which +/// key; `hint`: what a new one means). Never created where a file is: a mistyped path +/// stays an error unless asked. +pub(crate) fn key_file( + path: &Path, + what: &str, + create: bool, + create_option: &str, + hint: &str, + len: usize, +) -> anyhow::Result<(Vec<u8>, bool)> { + use anyhow::Context; + use std::io::Write; + match std::fs::read(path) { + Ok(bytes) => { + warn_if_readable_by_others(path, what); + Ok((bytes, false)) + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + if !create { + anyhow::bail!( + "The {what} file {} doesn't exist; add --{create_option} to create a new {what} there ({hint})", + path.display() + ); + } + let mut bytes = vec![0u8; len]; + getrandom::fill(&mut bytes) + .map_err(|error| anyhow::anyhow!("No randomness: {error}"))?; + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let written = (|| { + let mut file = options.open(path)?; + file.write_all(&bytes)?; + file.sync_all() + })(); + written.with_context(|| format!("Creating the {what} file {}", path.display()))?; + Ok((bytes, true)) + } + Err(error) => { + Err(error).with_context(|| format!("Reading the {what} from {}", path.display())) + } + } +} + +/// A key is a secret: a file others may read is worth a warning (not an error: a +/// group may be deliberate, e.g. shared with a deployment tool). `what`: which key. +pub(crate) fn warn_if_readable_by_others(path: &Path, what: &str) { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + if let Ok(metadata) = std::fs::metadata(path) { + let mode = metadata.permissions().mode(); + if mode & 0o077 != 0 { + tracing::warn!( + path = %path.display(), + mode = format!("{:o}", mode & 0o777), + "The {what} file is readable by others than its owner; chmod 600 it" + ); + } + } + } + #[cfg(not(unix))] + let _ = (path, what); +} + fn with_suffix(path: &Path, suffix: &str) -> PathBuf { let mut name = path.as_os_str().to_owned(); name.push(suffix); PathBuf::from(name) } +/// An empty directory for the test `name`: `continuum-<name>-<process ID>` in the +/// temporary directory, emptied first. +#[cfg(test)] +pub(crate) fn scratch_dir(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("continuum-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + #[cfg(test)] mod tests { use super::*; #[test] fn writes_reads_and_counts() { - let dir = std::env::temp_dir().join(format!("continuum-files-{}", std::process::id())); - std::fs::create_dir_all(&dir).unwrap(); + let dir = scratch_dir("files"); let path = dir.join("state.json"); let files = Files::default(); diff --git a/src/identity.rs b/src/identity.rs index 93d8739..04b7e6f 100644 --- a/src/identity.rs +++ b/src/identity.rs @@ -3,13 +3,12 @@ //! //! The node ID names the node whatever transport reaches it: a transport address only //! says where it is (`docs/peers.md`, node IDs and addresses). A new key is a new -//! node, so the key is created only when asked to (`--create-node-key`), never because +//! node, so the key is created only when asked to (`--node-key-create`), never because //! a path was mistyped. -use std::io::Write; use std::path::Path; -use anyhow::{Context, Result, bail}; +use anyhow::{Result, bail}; use ed25519_dalek::SigningKey; use tracing::info; @@ -30,25 +29,14 @@ impl NodeKey { /// Reads the key from `path`; with `create`, makes a new one there if the file /// doesn't exist. pub(crate) fn load(path: &Path, create: bool) -> Result<Self> { - let bytes = match std::fs::read(path) { - Ok(bytes) => bytes, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - if !create { - bail!( - "The node key file {} doesn't exist; add --create-node-key to create a new node key there (a new key is a new node)", - path.display() - ); - } - let key = Self::create(path)?; - info!(path = %path.display(), node_id = %key.id(), "Created a new node key: this is a new node"); - return Ok(key); - } - Err(error) => { - return Err(error) - .with_context(|| format!("Reading the node key from {}", path.display())); - } - }; - crate::swarm::auth::warn_if_readable_by_others(path, "node key"); + let (bytes, created) = crate::files::key_file( + path, + "node key", + create, + "node-key-create", + "a new key is a new node", + KEY_BYTES, + )?; let Ok(seed) = <[u8; KEY_BYTES]>::try_from(bytes.as_slice()) else { bail!( "The node key in {} has {} bytes; a node key has {KEY_BYTES}", @@ -56,7 +44,11 @@ impl NodeKey { bytes.len() ); }; - Ok(Self::from_seed(seed)) + let key = Self::from_seed(seed); + if created { + info!(path = %path.display(), node_id = %key.id(), "Created a new node key: this is a new node"); + } + Ok(key) } /// A key from its 32 secret bytes: for simulated nodes, which have no key files. @@ -66,26 +58,6 @@ impl NodeKey { } } - /// A new key in a new file at `path` (only its owner may read it). - fn create(path: &Path) -> Result<Self> { - let mut seed = [0u8; KEY_BYTES]; - getrandom::fill(&mut seed).map_err(|error| anyhow::anyhow!("No randomness: {error}"))?; - let mut options = std::fs::OpenOptions::new(); - options.write(true).create_new(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - options.mode(0o600); - } - let written = (|| { - let mut file = options.open(path)?; - file.write_all(&seed)?; - file.sync_all() - })(); - written.with_context(|| format!("Creating the node key file {}", path.display()))?; - Ok(Self::from_seed(seed)) - } - /// This node's ID. pub(crate) fn id(&self) -> String { node_id_of(&self.public()) @@ -129,18 +101,11 @@ pub(crate) fn is_node_id(id: &str) -> bool { #[cfg(test)] mod tests { use super::*; - - fn scratch(name: &str) -> std::path::PathBuf { - let dir = - std::env::temp_dir().join(format!("continuum-identity-{name}-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&dir); - std::fs::create_dir_all(&dir).unwrap(); - dir - } + use crate::files::scratch_dir; #[test] fn created_only_when_asked_and_then_kept() { - let path = scratch("create").join("node.key"); + let path = scratch_dir("identity-create").join("node.key"); assert!(NodeKey::load(&path, false).is_err()); assert!(!path.exists()); @@ -160,7 +125,7 @@ mod tests { #[test] fn a_key_of_the_wrong_size_is_refused() { - let path = scratch("size").join("node.key"); + let path = scratch_dir("identity-size").join("node.key"); std::fs::write(&path, [7u8; 16]).unwrap(); assert!(NodeKey::load(&path, true).is_err()); } diff --git a/src/logging.rs b/src/logging.rs index 093d49e..c6d5fb7 100644 --- a/src/logging.rs +++ b/src/logging.rs @@ -12,15 +12,24 @@ //! the member list); at start, as the state file remembers them, so they show from the //! first line (stale, perhaps: a member's alias that changed meanwhile is logged as such //! once the member list says so). `--log-raw-ids` leaves IDs as they are. +//! +//! Under systemd's journal (`--log-format journal`, or `auto` when stdout goes to the +//! journal: `JOURNAL_STREAM`), lines go without a timestamp or colours, each with its +//! level as a syslog priority prefix (`<3>` for errors … `<7>` for debug), which the +//! journal takes for the entry's priority (`journalctl -p warning`). use std::borrow::Cow; use std::collections::{BTreeMap, HashMap}; +use std::fmt; use std::io::Write; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{OnceLock, RwLock}; use anyhow::{Result, anyhow}; +use tracing::{Level, Metadata}; use tracing_subscriber::fmt::MakeWriter; +use tracing_subscriber::fmt::format::Writer; +use tracing_subscriber::fmt::time::{FormatTime, SystemTime}; use tracing_subscriber::{EnvFilter, Registry, prelude::*, reload}; use crate::locks::RwLockExt; @@ -37,6 +46,111 @@ static ALIASES: RwLock<Aliases> = RwLock::new(Aliases { }); /// `--log-raw-ids`. static RAW_IDS: AtomicBool = AtomicBool::new(false); +/// Lines for the journal (`set_format`). +static JOURNAL: AtomicBool = AtomicBool::new(false); + +/// `--log-format`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum LogFormat { + /// The journal's if stdout goes to it, else text. + Auto, + /// Timestamped lines, coloured on a terminal. + Text, + /// For systemd's journal (module docs). + Journal, +} + +impl LogFormat { + pub(crate) const NAMES: [&str; 3] = ["auto", "text", "journal"]; + + pub(crate) fn parse(name: &str) -> Option<Self> { + match name { + "auto" => Some(Self::Auto), + "text" => Some(Self::Text), + "journal" => Some(Self::Journal), + _ => None, + } + } +} + +/// Writes lines in `format` from now on. +pub(crate) fn set_format(format: LogFormat) { + let journal = match format { + LogFormat::Auto => to_journal(), + LogFormat::Text => false, + LogFormat::Journal => true, + }; + JOURNAL.store(journal, Ordering::Relaxed); +} + +/// Whether stdout is the journal's stream: systemd says which (`JOURNAL_STREAM`, its +/// device and inode) when it connects one. +fn to_journal() -> bool { + let Some(stream) = std::env::var_os("JOURNAL_STREAM") else { + return false; + }; + let Some((device, inode)) = stream.to_str().and_then(|stream| stream.split_once(':')) else { + return false; + }; + use std::os::fd::AsFd; + use std::os::unix::fs::MetadataExt; + // A duplicate of stdout, to stat it (`dev` and `ino` are u64 on every platform). + let Ok(stdout) = std::io::stdout().as_fd().try_clone_to_owned() else { + return false; + }; + let Ok(metadata) = std::fs::File::from(stdout).metadata() else { + return false; + }; + device.parse::<u64>().ok() == Some(metadata.dev()) + && inode.parse::<u64>().ok() == Some(metadata.ino()) +} + +/// The fmt logger's timestamps, but none for the journal (it has its own). +struct Timestamps; + +impl FormatTime for Timestamps { + fn format_time(&self, writer: &mut Writer<'_>) -> fmt::Result { + if JOURNAL.load(Ordering::Relaxed) { + return Ok(()); + } + SystemTime.format_time(writer) + } +} + +/// The syslog priority of `level`, as the journal reads it at the start of a line. +fn priority(level: &Level) -> &'static str { + match *level { + Level::ERROR => "<3>", + Level::WARN => "<4>", + Level::INFO => "<6>", + _ => "<7>", + } +} + +/// `line` for the journal: without colours or leading blanks (where the timestamp went), +/// each of its lines with `priority` in front. +fn for_journal(line: &str, priority: &str) -> String { + let mut plain = String::with_capacity(line.len()); + let mut chars = line.chars(); + while let Some(c) = chars.next() { + if c == '\x1b' { + // A colour: ESC [ ... m. + for c in chars.by_ref() { + if c == 'm' { + break; + } + } + } else { + plain.push(c); + } + } + let mut out = String::with_capacity(plain.len() + 4); + for part in plain.trim_start().split_inclusive('\n') { + out.push_str(priority); + out.push_str(part); + } + out +} struct Aliases { given: Option<HashMap<String, String>>, @@ -133,12 +247,26 @@ impl<'a> MakeWriter<'a> for Renaming { type Writer = RenamingLine; fn make_writer(&'a self) -> Self::Writer { - RenamingLine(Vec::new()) + RenamingLine(Vec::new(), &Level::INFO) + } + + fn make_writer_for(&'a self, meta: &Metadata<'_>) -> Self::Writer { + RenamingLine(Vec::new(), priority_level(meta.level())) } } -/// One event's text, written renamed when the logger is done with it. -struct RenamingLine(Vec<u8>); +fn priority_level(level: &Level) -> &'static Level { + match *level { + Level::ERROR => &Level::ERROR, + Level::WARN => &Level::WARN, + Level::INFO => &Level::INFO, + Level::DEBUG => &Level::DEBUG, + _ => &Level::TRACE, + } +} + +/// One event's text, and its level, written renamed when the logger is done with it. +struct RenamingLine(Vec<u8>, &'static Level); impl Write for RenamingLine { fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> { @@ -154,7 +282,13 @@ impl Write for RenamingLine { impl Drop for RenamingLine { fn drop(&mut self) { let text = String::from_utf8_lossy(&self.0); - let _ = std::io::stdout().write_all(rename(&text).as_bytes()); + let renamed = rename(&text); + let line = if JOURNAL.load(Ordering::Relaxed) { + Cow::Owned(for_journal(&renamed, priority(self.1))) + } else { + renamed + }; + let _ = std::io::stdout().write_all(line.as_bytes()); } } @@ -163,9 +297,14 @@ pub(crate) fn init() { let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); let (filter, handle) = reload::Layer::new(filter); + // Until the configuration says otherwise (`set_format`). + set_format(LogFormat::Auto); let registry = tracing_subscriber::registry().with( tracing_subscriber::fmt::layer() .compact() + .with_timer(Timestamps) + // The journal gets none either way (`for_journal`). + .with_ansi(!to_journal()) .with_writer(Renaming) .with_filter(filter), ); @@ -208,6 +347,24 @@ mod tests { use super::*; #[test] + fn journal_lines() { + assert_eq!( + for_journal( + " \x1b[32m INFO\x1b[0m continuum: Ready peer=a\n", + priority(&Level::INFO) + ), + "<6>INFO continuum: Ready peer=a\n" + ); + assert_eq!( + for_journal("ERROR x: one\ntwo\n", priority(&Level::ERROR)), + "<3>ERROR x: one\n<3>two\n" + ); + assert_eq!(priority(&Level::TRACE), "<7>"); + assert_eq!(LogFormat::parse("journal"), Some(LogFormat::Journal)); + assert_eq!(LogFormat::parse("syslog"), None); + } + + #[test] fn members_are_named_by_their_alias() { let node = crate::identity::node_id_of(&[1; 32]); let renames = renames(&HashMap::from([ diff --git a/src/main.rs b/src/main.rs index f16138e..a10eb73 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,5 @@ -//! continuum: a replicated Automerge key-value store synchronized over Mycelium. +//! continuum: a replicated Automerge document, read and written by path, synchronized +//! over Mycelium, libp2p or Tor. //! //! main.rs only wires things together: arguments, the starting document, the node //! (`node`), bootstrap and the HTTP server. @@ -8,7 +9,6 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use anyhow::{Context, Result}; -use automerge::AutoCommit; use tokio::sync::watch; use tracing::{debug, error, info, warn}; @@ -34,10 +34,10 @@ mod transport; mod units; use api::auth::AuthState; +use document::Document; use document::replica::Epoch; use swarm::bootstrap; use transport::Transport; -use transport::mycelium::{self, MyceliumSetup, MyceliumSpawnConfig}; #[tokio::main] async fn main() -> Result<()> { @@ -52,6 +52,19 @@ async fn main() -> Result<()> { return Ok(()); } let args = &config.settings; + if let Some(format) = logging::LogFormat::parse(&args.log_format) { + logging::set_format(format); + } + for (transport, built, options) in config.unused_transport_options() { + warn!( + transport, + built, + options = ?options, + "Options of a transport this node doesn't use are ignored" + ); + } + // Only with libp2p: without, `config` refuses `relay-only` already. + #[cfg(feature = "libp2p")] if args.relay_only { return relay_only(args).await; } @@ -63,11 +76,14 @@ async fn main() -> Result<()> { .context("node-key-file is required")?; let node_key = Arc::new(identity::NodeKey::load( node_key_file, - args.create_node_key, + args.node_key_create, )?); info!(node_id = %node_key.id(), "Node identity"); let swarm_key = match &args.swarm_key_file { - Some(path) => Some(Arc::new(swarm::auth::SwarmKey::from_file(path)?)), + Some(path) => Some(Arc::new(swarm::auth::SwarmKey::from_file( + path, + args.swarm_key_create, + )?)), None => { warn!("No swarm key: any node that can reach a member may join"); None @@ -75,34 +91,11 @@ async fn main() -> Result<()> { }; // First thing: until a handler is installed, SIGINT and SIGTERM kill the process on - // the spot, and startup (spawning Mycelium, bootstrap) can take minutes. + // the spot, and startup (starting transports, bootstrap) can take minutes. let stop = watch_for_shutdown().context("Installing signal handlers")?; // A broken htpasswd will abort startup. let auth_state = AuthState::new(args.htpasswd.as_deref().context("htpasswd is required")?)?; - let uses = |name: &str| args.transports.iter().any(|transport| transport == name); - let mut mycelium_setup = if uses("mycelium") { - let spawn_config = MyceliumSpawnConfig { - rest_url_opt: args.mycelium_rest_url(), - key_file_opt: args.mycelium_key_file.clone(), - network_key_file: args.mycelium_network_key_file.clone(), - network_name: args.mycelium_network_name.clone(), - tcp_port: args.mycelium_tcp_listen_port, - peers: args.mycelium_peers.clone(), - }; - let setup = tokio::select! { - setup = MyceliumSetup::bootstrap(spawn_config) => setup?, - _ = stopped(stop.clone()) => return Ok(()), - }; - // Without a password it may carry (the option is secret). - let mut shown_url = setup.rest_url.clone(); - let _ = shown_url.set_password(None); - debug!(url = shown_url.to_string(), "Using Mycelium REST API"); - Some(setup) - } else { - None - }; - info!( bind_addr = args.bind_addr.to_string(), transports = ?args.transports, @@ -158,7 +151,7 @@ async fn main() -> Result<()> { Some((starting.swarm_id, starting.origin)), starting.earlier_actors, ), - None => (AutoCommit::new(), None, None, Vec::new()), + None => (Document::new(), None, None, Vec::new()), }; let mut initial_doc = initial_doc; if let Some(spec) = &args.remake { @@ -176,46 +169,12 @@ async fn main() -> Result<()> { }; debug!(epoch = %initial_epoch, "Local epoch"); - // In the order of preference (`--transports`, checked when loading). + // The enabled, in the order of preference (`--transport-order`, checked when + // loading). let mut transports: Vec<(&'static str, Arc<dyn Transport>)> = Vec::new(); for name in &args.transports { - let transport: (&'static str, Arc<dyn Transport>) = match name.as_str() { - "mycelium" => { - let setup = mycelium_setup - .take() - .context("Mycelium named twice in transports")?; - ( - transport::mycelium::SCHEME, - Arc::new(mycelium::MyceliumTransport::connect(setup).await?), - ) - } - #[cfg(feature = "libp2p")] - "libp2p" => ( - transport::libp2p::SCHEME, - Arc::new(transport::libp2p::Libp2pTransport::start(libp2p_config( - args, false, - )?)?), - ), - #[cfg(feature = "tor")] - "tor" => { - let state_dir = args - .tor_state_dir - .clone() - .context("tor needs tor-state-dir")?; - let transport = tokio::select! { - transport = transport::tor::TorTransport::start(transport::tor::TorConfig { - key: Arc::clone(&node_key), - state_dir, - port: args.tor_port, - }) => transport?, - _ = stopped(stop.clone()) => return Ok(()), - }; - ( - transport::tor::SCHEME, - Arc::new(transport) as Arc<dyn Transport>, - ) - } - other => anyhow::bail!("transports: no {other} in this build"), + let Some(transport) = transport::start(name, args, &node_key, stop.clone()).await? else { + return Ok(()); }; info!( transport = transport.0, @@ -229,6 +188,12 @@ async fn main() -> Result<()> { .map(|(_, transport)| transport.delivery_limit()) .max() .unwrap_or_default(); + // An answer may come over any of them. + let sync_reply_timeout = transports + .iter() + .map(|(_, transport)| transport.sync_reply_timeout()) + .max() + .unwrap_or(state::SYNC_REPLY_TIMEOUT); let mut transports = transports.into_iter(); let (scheme, transport) = transports.next().context("No transports")?; @@ -250,10 +215,13 @@ async fn main() -> Result<()> { initial_epoch, initial_doc, earlier_actors, - timing: state::Timing::from_keepalive(args.keepalive, delivery_limit), + timing: state::Timing::new(args.keepalive, sync_reply_timeout, delivery_limit) + .with_probe_max(args.dormant_probe_max), neighbours_k: args.neighbours, alias: args.alias.clone(), electorate_window: args.electorate_window, + clock_skew_warn: args.clock_skew_warn, + superseded_writes: args.superseded_writes, // At most 4MiB (`Settings::check`). blob_chunk_bytes: args.blob_chunk as u32, swarm_key, @@ -298,15 +266,14 @@ async fn main() -> Result<()> { info!(address = %listener.local_addr()?, "HTTP API listening"); - let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>(); - + // The node shutting down (`AppState::closing`): the server drains. + let closing = state.closing.subscribe(); let server_builder = axum::serve(listener, app).with_graceful_shutdown(async move { - let _ = shutdown_rx.await; + stopped(closing).await; info!("Shutdown signal received. Draining API connections...") }); let server_fut = server_builder.into_future(); - let stopping = state.clone(); tokio::pin!(server_fut); // Joins, then runs until stopped; returns only if joining fails. @@ -328,14 +295,14 @@ async fn main() -> Result<()> { tokio::select! { failed = joining.as_mut() => { - let _ = shutdown_tx.send(()); + state.closing.send_replace(true); let _ = server_fut.as_mut().await; return failed; } exit_status = state.transport.wait_for_exit() => { error!(status = ?exit_status, "A transport stopped"); - let _ = shutdown_tx.send(()); + state.closing.send_replace(true); let _ = server_fut.as_mut().await; } @@ -348,7 +315,7 @@ async fn main() -> Result<()> { } _ = stopped(stop.clone()) => { - let _ = shutdown_tx.send(()); + state.closing.send_replace(true); let _ = server_fut.as_mut().await; } } @@ -358,41 +325,12 @@ async fn main() -> Result<()> { info!("Stopped while joining the swarm"); return Ok(()); } - swarm::peer::farewell(&stopping).await; - swarm::state_file::update(&stopping, true).await; - document::file::flush(&stopping).await; + swarm::peer::farewell(&state).await; + swarm::state_file::update(&state, true).await; + swarm::document_file::flush(&state).await; Ok(()) } -/// The libp2p transport's configuration; `relay_only`: a relay that is no node. -#[cfg(feature = "libp2p")] -fn libp2p_config( - args: &config::Settings, - relay_only: bool, -) -> Result<transport::libp2p::Libp2pConfig> { - let parse = |option: &str, addresses: &[String]| { - addresses - .iter() - .map(|address| { - address - .parse::<libp2p::Multiaddr>() - .with_context(|| format!("{option} {address:?}")) - }) - .collect::<Result<Vec<_>>>() - }; - Ok(transport::libp2p::Libp2pConfig { - key: transport::libp2p::Libp2pConfig::load_key(args.libp2p_key_file.as_deref())?, - listen: parse("libp2p-listen", &args.libp2p_listen)?, - announce: parse("libp2p-announce", &args.libp2p_announce)?, - relays: parse("libp2p-relays", &args.libp2p_relays)?, - relay: args.libp2p_relay || relay_only, - continuum: !relay_only, - mdns: args.libp2p_mdns, - public: args.libp2p_public, - memory: false, - }) -} - /// `--relay-only`: libp2p relaying for nodes of swarms, and the HTTP API's relay part /// (with `--htpasswd`), until stopped. #[cfg(feature = "libp2p")] @@ -400,7 +338,7 @@ async fn relay_only(args: &config::Settings) -> Result<()> { let stop = watch_for_shutdown().context("Installing signal handlers")?; let auth_state = args.htpasswd.as_deref().map(AuthState::new).transpose()?; let transport: Arc<dyn Transport> = Arc::new(transport::libp2p::Libp2pTransport::start( - libp2p_config(args, true)?, + transport::libp2p::Libp2pConfig::from_settings(args, true)?, )?); info!(peer_id = transport.local_id(), "Relaying only (no node)"); @@ -435,11 +373,6 @@ async fn relay_only(args: &config::Settings) -> Result<()> { Ok(()) } -#[cfg(not(feature = "libp2p"))] -async fn relay_only(_args: &config::Settings) -> Result<()> { - anyhow::bail!("relay-only relays over libp2p; this build has no libp2p") -} - /// Turns SIGINT and SIGTERM into a shutdown request. A second signal exits at once (the /// supervised Mycelium still gets its parent-death signal then). fn watch_for_shutdown() -> Result<watch::Receiver<bool>> { diff --git a/src/node.rs b/src/node.rs index 8aab601..4597e66 100644 --- a/src/node.rs +++ b/src/node.rs @@ -8,20 +8,20 @@ use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::time::Duration; -use automerge::{ActorId, AutoCommit}; use tokio::sync::{RwLock, broadcast, mpsc, watch}; use tracing::trace; use crate::clock::Clock; use crate::document::file::DocumentFile; use crate::document::replica::{Epoch, Replica}; +use crate::document::{ActorId, Document}; use crate::files::Files; use crate::remake; use crate::state::{AppState, NodeLifecycle, Timing}; use crate::swarm::auth::SwarmKey; use crate::swarm::state_file::StateFile; use crate::swarm::sync::SyncEvent; -use crate::swarm::{self, peer, sync}; +use crate::swarm::{self, changes, peer, sync}; use crate::transport::Transport; use crate::transport::firewall::Firewall; use crate::transport::identity::{IdentityConfig, IdentityTransport}; @@ -37,14 +37,18 @@ pub(crate) struct NodeConfig { /// (simulations only), the node is its transport identity. pub(crate) identity: Option<IdentityConfig>, pub(crate) initial_epoch: Epoch, - pub(crate) initial_doc: AutoCommit, + pub(crate) initial_doc: Document, /// Actors this node wrote with in earlier runs (`--document-file`). pub(crate) earlier_actors: Vec<ActorId>, - /// `Timing::from_keepalive(--keepalive, transport.delivery_limit())`. + /// `Timing::new(--keepalive, the transports' sync reply timeout and delivery + /// limit)`. pub(crate) timing: Timing, pub(crate) neighbours_k: usize, pub(crate) alias: Option<String>, pub(crate) electorate_window: Duration, + pub(crate) clock_skew_warn: Duration, + /// `--superseded-writes`. + pub(crate) superseded_writes: crate::remake::Superseded, pub(crate) blob_chunk_bytes: u32, pub(crate) swarm_key: Option<Arc<SwarmKey>>, pub(crate) protocol_writers: HashSet<String>, @@ -68,16 +72,17 @@ impl Node { let (sync_tx, sync_rx) = mpsc::unbounded_channel(); let (change_tx, _) = watch::channel(0_u64); let (change_events, _) = broadcast::channel(256); - let (epoch_tx, _) = watch::channel(config.initial_epoch); let firewall = Arc::new(Firewall::default()); let traffic = Arc::new(Traffic::default()); - let initial_actor = config.initial_doc.get_actor().clone(); + let initial_actor = crate::document::actor(&config.initial_doc); let transport = match config.identity { Some(identity) => Arc::new(IdentityTransport::new( config.transport, identity, Arc::clone(&traffic), Some(Arc::clone(&firewall)), + Arc::clone(&config.clock), + config.timing.keepalive, )) as Arc<dyn Transport>, None => config.transport, }; @@ -86,7 +91,6 @@ impl Node { let stamp = replica.stamps(); let state = AppState { doc: Arc::new(crate::document::lock::DocumentLock::new(replica)), - epoch_tx, stamp, lifecycle: Arc::new(RwLock::new(NodeLifecycle::Joining { discovered_swarm_id: None, @@ -103,7 +107,7 @@ impl Node { sync_events: sync_tx, change_tx, initial_sync_done: Arc::new(AtomicBool::new(false)), - stopping: Arc::new(AtomicBool::new(false)), + closing: Arc::new(watch::channel(false).0), change_events, timing: config.timing, remake: Arc::new(remake::RemakeState::new(config.clock.now_millis())), @@ -113,7 +117,12 @@ impl Node { neighbours_k: config.neighbours_k, alias: config.alias, electorate_window: config.electorate_window, + clock_skew_warn: config.clock_skew_warn, + superseded_writes: config.superseded_writes, + clock_ahead_ms: Default::default(), + clock_skew: Default::default(), blobs: Arc::new(swarm::blob::Blobs::new(config.blob_chunk_bytes)), + links: Default::default(), protocol_writers: Arc::new(config.protocol_writers), state_file: config.state_file, document_file: config.document_file, @@ -121,6 +130,9 @@ impl Node { config: Arc::new(config.config_reports), neighbours: Default::default(), topology_changed: Default::default(), + join_progress: Default::default(), + forget_records_changed: Default::default(), + peers_due: Default::default(), pulling: Default::default(), announce_pending: Default::default(), rtt: Default::default(), @@ -170,12 +182,19 @@ impl Node { }); trace!("Sync worker task spawned"); + // Subscribers to our links (`swarm::links`; any build has them). + tokio::spawn(crate::swarm::links::watch(state.clone())); + let peer_worker_state = state.clone(); tokio::spawn(async move { peer::run_peer_worker(peer_worker_state).await; }); trace!("Peer worker task spawned"); + let reachable = Arc::new(tokio::sync::Notify::new()); + state.transport.on_reachable(Arc::clone(&reachable)); + tokio::spawn(peer::probe_when_reachable(state.clone(), reachable)); + tokio::spawn(remake::run_worker(state.clone())); trace!("Remake worker task spawned"); @@ -190,6 +209,6 @@ impl Node { /// from then on, and it avoids diffing the initial sync) and the document file writer. pub(crate) fn spawn_ready_workers(state: &AppState) { let publisher_state = state.clone(); - tokio::spawn(async move { sync::run_change_publisher(publisher_state).await }); - tokio::spawn(crate::document::file::run(state.clone())); + tokio::spawn(async move { changes::run_change_publisher(publisher_state).await }); + tokio::spawn(crate::swarm::document_file::run(state.clone())); } diff --git a/src/remake/api.rs b/src/remake/api.rs index 52338dd..8fd0c5b 100644 --- a/src/remake/api.rs +++ b/src/remake/api.rs @@ -32,6 +32,8 @@ //! not for real swarms); //! * `supersede`: abort expired remakes of this epoch and propose in their place. //! Without it, an expired remake still holding votes answers `409`; +//! * `clock`: propose although this node's clock is off from its peers' +//! (`swarm::skew`): it would judge the remake's deadlines wrongly. Without it, `409`; //! * for `DELETE`: `coordinatorActive` (abort an expired remake whose coordinator this //! node sees as active) and `coordinatorInactive` (ask for an abort of an undecided //! one whose coordinator this node doesn't reach). @@ -50,10 +52,8 @@ //! their paths: //! `curl --data-binary @backup.automerge -H 'Content-Type: application/octet-stream' --url-query 'rules=[{"from":"","to":""}]' …/admin/remakes` -use std::collections::HashSet; use std::time::Duration; -use automerge::ChangeHash; use axum::{ Json, body::Bytes, @@ -71,15 +71,15 @@ use tokio::sync::broadcast; use tracing::{error, info, warn}; use uuid::Uuid; -use super::coordinator; use super::local::LocalStatus; +use super::proposal::{StartError, Supersede, abort_obstacle, abortable_at, active_peers, propose}; use super::records::{ self, AbortRequest, ContentRef, Decision, EpochRecord, Phase, Policy, Proposal, Record, - Retention, Vote, + Retention, }; -use super::{Host, kind, vote}; +use super::{Host, kind}; use crate::api::error_response; -use crate::clock::UnixMillis; +use crate::document::ChangeHash; use crate::document::store::StoreError; use crate::force::Force; @@ -202,81 +202,28 @@ impl From<RetentionRequest> for Retention { } /// What `?force=` may override on a proposal (`force`). -const FORCE: &[&str] = &["switchDelay", "supersede", "coordinatorActive"]; +const FORCE: &[&str] = &["switchDelay", "supersede", "coordinatorActive", "clock"]; /// What `?force=` may override on an abort. -const ABORT_FORCE: &[&str] = &["coordinatorActive", "coordinatorInactive"]; +const ABORT_FORCE: &[&str] = &["coordinatorActive", "coordinatorInactive", "clock"]; -/// When a node other than the coordinator may abort an expired remake: a whole grace -/// period after it expired. Its coordinator stops committing at expiry -/// (`coordinator::decide`), so an abort from elsewhere and a commit can only cross if -/// their clocks are more than a grace period apart. -fn abortable_at(record: &Record, grace: u64) -> UnixMillis { - record - .proposal - .settle_until - .plus_ms(grace.saturating_mul(2)) -} - -/// Why this node may not abort expired `record` now, and the `force` name that would -/// override it (none for one that only time cures). -fn abort_obstacle( - record: &Record, - me: &str, - active_peers: &HashSet<String>, - force: &Force, - now: UnixMillis, - grace: u64, -) -> Option<(String, Option<&'static str>)> { - let at = abortable_at(record, grace); - if now < at { - return Some(( - format!( - "only its coordinator may abort it until a grace period after it expired ({}s from now)", - at.ms_since(now).div_ceil(1000) - ), - None, - )); +/// `409` while this node's clock is off from most of its peers' (and `clock` isn't +/// forced): it would judge a remake's deadlines wrongly, proposing one or aborting one +/// as expired (`trust.md`, "What depends on clocks"). `what` is what it would do. +async fn clock_obstacle<H: Host>(host: &H, force: &Force, what: &str) -> Option<Response> { + let skew = host.clock_skew().await; + if !skew.off || force.allows("clock") { + return None; } - let coordinator = &record.proposal.coordinator; - if coordinator != me && active_peers.contains(coordinator) && !force.allows("coordinatorActive") - { - return Some(( - format!( - "its coordinator {coordinator} is an active peer, so it is stuck rather than gone (look at its log); add force=coordinatorActive to abort it anyway" - ), - Some("coordinatorActive"), - )); - } - None -} - -/// The active peers this node sees. -async fn active_peers<H: Host>(host: &H) -> HashSet<String> { - host.peer_summaries() - .await - .into_iter() - .filter(|peer| peer.active) - .map(|peer| peer.id) - .collect() -} - -/// An expired remake of the epoch a proposal would supersede. -#[derive(Debug)] -struct ExpiredRemake { - id: String, - coordinator: String, - coordinator_active: bool, - /// When it expired. - since: UnixMillis, - /// When another node may abort it (`abortable_at`). - abortable_at: UnixMillis, - tally: vote::Tally, -} - -/// Superseding expired remakes, as the proposal's `force` asks. -struct Supersede { - force: Force, - active_peers: HashSet<String>, + let offset = skew.offset_ms.unwrap_or(0); + Some(error_response( + StatusCode::CONFLICT, + format!( + "this node's clock is {} {} most of its peers' (more than --clock-skew-warn), so it would judge {what} wrongly; fix its time synchronisation, or add ?force=clock", + crate::units::format_duration(Duration::from_millis(offset.unsigned_abs())), + if offset < 0 { "behind" } else { "ahead of" } + ), + Some(json!({ "force": "clock" })), + )) } #[derive(Debug, Serialize)] @@ -291,28 +238,6 @@ struct Report { local: Option<LocalStatus>, } -#[derive(Debug, thiserror::Error)] -enum StartError { - #[error("remake {0} is already open for this epoch")] - Open(String), - /// Expired remakes still hold votes, and aren't superseded: not asked to - /// (`force=supersede`), or not possible yet (`abort_obstacle`). - #[error("{message}")] - Expired { - message: String, - remakes: Vec<ExpiredRemake>, - /// The `force` name that would override it, if any. - force: Option<&'static str>, - }, - /// The proposer votes for its own remake, so it must be free to. - #[error("this node can't vote for a new remake: {0}")] - VoteHeld(String), - #[error("base head {0} is not in this node's document")] - UnknownBase(String), - #[error(transparent)] - Store(#[from] StoreError), -} - fn grace<H: Host>(host: &H) -> u64 { host.dormant_after().as_millis() as u64 } @@ -392,6 +317,11 @@ async fn start_kind<H: Host>( Ok(force) => force, Err(reason) => return error_response(StatusCode::BAD_REQUEST, reason, None), }; + // The coordinator stamps and judges the voting period and the deadlines by its + // clock. + if let Some(refused) = clock_obstacle(&host, &force, "the remake's deadlines").await { + return refused; + } let request: StartRequest = if is_octet_stream(headers) { match query.import_request(&body) { @@ -523,6 +453,7 @@ async fn start_kind<H: Host>( proposed_at: now, settle_until: now.plus(settle), switch_delay_ms: switch_delay.map(|delay| delay.as_millis() as u64), + grace_ms: Some(grace), // Filled in when the proposal is written, from the document at that moment. electorate: Vec::new(), quorum: 0, @@ -637,157 +568,6 @@ async fn start_kind<H: Host>( .into_response() } -/// Writes the proposal (unless another one is open for the epoch) and starts its -/// coordinator. Runs as its own task, independent of the HTTP request. -async fn propose<H: Host>( - host: H, - id: String, - mut proposal: Proposal, - base: Option<Vec<ChangeHash>>, - supersede: Supersede, - now: UnixMillis, - grace: u64, -) -> Result<Record, StartError> { - let record_id = id.clone(); - let electorate_host = host.clone(); - let me = host.node_id(); - let started = host.remake().started_at().plus_ms(grace); - let base_at_commit = kind::by_name(&proposal.kind).is_some_and(|kind| kind.base_at_commit()); - let record = host - .commit_doc(move |doc| -> Result<Record, StartError> { - let mut all = records::load_all(doc); - if let Some((open, _)) = all.iter().find(|(_, record)| { - record.proposal.from_epoch == proposal.from_epoch - && record.phase(now, grace).is_open() - }) { - return Err(StartError::Open(open.clone())); - } - let mut register = records::load_register(doc); - - // Expired remakes hold their votes; their coordinators can't commit them - // any more, so they may be aborted here. - let expired: Vec<&String> = all - .iter() - .filter(|(_, record)| { - record.proposal.from_epoch == proposal.from_epoch - && record.phase(now, grace) == Phase::Expired - }) - .map(|(id, _)| id) - .collect(); - if !expired.is_empty() { - let remakes = || { - all.iter() - .filter(|(id, _)| expired.contains(&id)) - .map(|(id, record)| ExpiredRemake { - id: id.clone(), - coordinator: record.proposal.coordinator.clone(), - coordinator_active: supersede - .active_peers - .contains(&record.proposal.coordinator), - since: record.proposal.settle_until.plus_ms(grace), - abortable_at: abortable_at(record, grace), - tally: vote::tally(id, record, &all, ®ister), - }) - .collect() - }; - if !supersede.force.allows("supersede") { - return Err(StartError::Expired { - message: "an expired remake of this epoch still holds votes: abort it, \ - or add ?force=supersede to abort it and propose in its place" - .to_owned(), - remakes: remakes(), - force: Some("supersede"), - }); - } - let obstacle = - all.iter() - .filter(|(id, _)| expired.contains(&id)) - .find_map(|(id, record)| { - let (why, force) = abort_obstacle( - record, - &me, - &supersede.active_peers, - &supersede.force, - now, - grace, - )?; - Some((format!("can't supersede remake {id}: {why}"), force)) - }); - if let Some((message, force)) = obstacle { - return Err(StartError::Expired { - message, - remakes: remakes(), - force, - }); - } - let decision = Decision::Aborted { - reason: format!("superseded by remake {record_id}"), - decided_at: now, - }; - for id in expired { - records::write_decision(doc, id, &decision)?; - info!(remake = %id, by = %record_id, "Superseded expired remake"); - } - all = records::load_all(doc); - register = records::load_register(doc); - } - - match vote::freedom(&me, &proposal.from_epoch, &all, ®ister, started) { - vote::Freedom::Any => {} - vote::Freedom::Only(id) => { - return Err(StartError::VoteHeld(format!( - "it may have voted for remake {id} before its start" - ))); - } - vote::Freedom::Held(reason) => return Err(StartError::VoteHeld(reason)), - } - - // The base is the requested heads, or everything before the proposal (the - // proposal itself is never in it); or, for a compaction, left to the commit. - if !base_at_commit { - let base = match base { - Some(base) => { - if let Some(unknown) = base - .iter() - .find(|head| doc.get_change_by_hash(head).is_none()) - { - return Err(StartError::UnknownBase(unknown.to_string())); - } - base - } - None => doc.get_heads(), - }; - proposal.base_heads = Some(records::encode_heads(&base)); - } - - // We vote for our own remake, so we are always in the electorate. - let mut electorate = electorate_host.electorate(doc, now); - if let Err(position) = electorate.binary_search(&me) { - electorate.insert(position, me.clone()); - } - proposal.quorum = vote::quorum(electorate.len()); - proposal.electorate = electorate; - - records::write_proposal(doc, &record_id, &proposal)?; - let own_vote = Vote { - remake: record_id.clone(), - at: now, - }; - records::write_vote(doc, &me, &own_vote)?; - Ok(Record { - proposal, - decision: None, - abort_requested: None, - contested: false, - votes: [(me, own_vote)].into(), - }) - }) - .await?; - - tokio::spawn(coordinator::run(host, id)); - Ok(record) -} - pub(crate) async fn list<H: Host>(State(host): State<H>) -> Response { let all = host.read_doc(records::load_all).await; let mut reports: Vec<Report> = all @@ -937,6 +717,11 @@ async fn abort_by_id<H: Host>(host: H, id: String, query: AbortQuery) -> Respons /// commit it any more (`coordinator`), unless `abort_obstacle` says otherwise. Answers /// with the final report. async fn abort_expired<H: Host>(host: &H, current: Report, force: &Force) -> Response { + // Whether it expired is judged by this node's clock: one far ahead would abort a + // remake its coordinator may still commit (a contested remake). + if let Some(refused) = clock_obstacle(host, force, "whether the remake expired").await { + return refused; + } let grace = grace(host); if let Some((why, force)) = abort_obstacle( ¤t.record, @@ -1015,6 +800,8 @@ pub(crate) async fn epochs<H: Host>(State(host): State<H>) -> Response { #[cfg(test)] pub(crate) mod testing { use super::*; + use crate::clock::UnixMillis; + use crate::remake::vote; async fn into_json(response: Response) -> (StatusCode, JsonValue) { let status = response.status(); diff --git a/src/remake/bench.rs b/src/remake/bench.rs new file mode 100644 index 0000000..f262601 --- /dev/null +++ b/src/remake/bench.rs @@ -0,0 +1,91 @@ +//! Benchmarks, as ignored tests (`cargo benchmarks`, `docs/development.md`). +//! +//! `replay`: carrying writes over into a new epoch's document, last writer wins +//! (`delta::replay_newest`): what a rejoin does with the writes made during it, and a +//! switch with those made after the remake's base. It finds the paths of every write +//! carried, and of every newer change in the new document, from its operations +//! (`store::changed_by`). + +use std::time::Instant; + +use automerge::transaction::Transactable; +use automerge::{ObjType, ROOT, ReadDoc, ScalarValue}; + +use super::delta::{Superseded, replay_newest}; +use super::kind::Translator; +use crate::document::Document; +use crate::document::bench::{document, median}; +use crate::document::store; + +/// `count` changes to `doc`, each writing one key under `data` (from `first` on), made +/// at `at` and after. +fn writes(doc: &mut Document, first: usize, count: usize, at: i64) { + for i in 0..count { + let mut transaction = doc.transaction(); + let data = transaction + .get(ROOT, "data") + .expect("read") + .map(|(_, id)| id) + .unwrap_or_else(|| { + transaction + .put_object(ROOT, "data", ObjType::Map) + .expect("map") + }); + transaction + .put( + &data, + format!("late{}", first + i), + ScalarValue::Int(i as i64), + ) + .expect("write"); + transaction + .commit_with(automerge::transaction::CommitOptions::default().with_time(at + i as i64)); + } +} + +/// `late` writes made to the old document after `keys` keys, carried into a new one +/// holding those keys (as a snapshot, at time 0) and `newer` writes of its own made +/// after them. +fn replay(keys: usize, late: usize, newer: usize) { + let old_base = document(keys); + let base = old_base.get_heads(); + let mut old = old_base.clone(); + writes(&mut old, 0, late, 1_000); + let after = old.get_heads(); + + let mut new = Document::new(); + { + let mut transaction = new.transaction(); + store::materialize_into(&old_base, &mut transaction, &|_| false).expect("snapshot"); + transaction.commit_with(automerge::transaction::CommitOptions::default().with_time(0)); + } + writes(&mut new, late, newer, 2_000); + + let took = median(5, || { + let mut target = new.clone(); + let started = Instant::now(); + std::hint::black_box(replay_newest( + &old, + &base, + &after, + &mut target, + &Translator::default(), + Superseded::Drop, + )); + started.elapsed() + }); + println!( + "{:<60} {:>10.3} ms", + format!("replay of {late} writes, {keys} keys, {newer} newer changes"), + took.as_secs_f64() * 1e3 + ); +} + +#[test] +#[ignore = "a benchmark: cargo benchmarks"] +fn bench_replay() { + println!(""); // put reports on new line + replay(1_000, 100, 100); + replay(100_000, 100, 100); + replay(100_000, 1_000, 1_000); +} diff --git a/src/remake/catchup.rs b/src/remake/catchup.rs index 0be7dfd..1ee9508 100644 --- a/src/remake/catchup.rs +++ b/src/remake/catchup.rs @@ -26,12 +26,16 @@ //! //! If no peer completes a sync in time, or the check fails, the node restores its old //! document and epoch and tries again later. +//! +//! Two remakes committed for one epoch make two epochs of the next generation, which +//! never sync. A node behind follows the one more peers advertise (`most_followed`); +//! a ready node on the other rejoins *sideways* once more ready peers are on it than on +//! its own (`sideways`), its writes on the epoch it leaves going to the quarantine. use std::collections::{BTreeMap, HashMap}; -use std::convert::Infallible; use std::time::Duration; -use automerge::{ActorId, AutoCommit, ChangeHash}; +use crate::document::{ActorId, ChangeHash, Document, ReadDoc}; use tokio::time::Instant; use tracing::{debug, error, info, warn}; @@ -87,13 +91,17 @@ pub(super) async fn check<H: Host>(host: &H) { .count(); // Only ready members count: a rejoining peer advertises an epoch it doesn't hold // yet, and a node of another swarm (or a stranger) advertises one that isn't ours. - let mut ahead: Vec<(String, Epoch)> = peers + let mut serving: Vec<(String, Epoch)> = peers .into_iter() .filter(|peer| peer.serves()) .filter_map(|peer| peer.epoch.map(|epoch| (peer.id, epoch))) + .collect(); + serving.sort_by(|a, b| a.0.cmp(&b.0)); + let mut ahead: Vec<(String, Epoch)> = serving + .iter() .filter(|(_, epoch)| epoch.generation > local.generation) + .cloned() .collect(); - ahead.sort_by(|a, b| a.0.cmp(&b.0)); ahead.retain(|(peer, epoch)| { let plausible = epoch.generation - local.generation <= MAX_GENERATION_GAP; if !plausible { @@ -105,7 +113,8 @@ pub(super) async fn check<H: Host>(host: &H) { // One peer's word is not enough for a ready node to give up its document: a buggy // or hostile member could otherwise keep the swarm rejoining an epoch nobody // serves. A newer epoch is followed once enough distinct peers advertise it. - let required = if host.is_node_ready().await { + let ready = host.is_node_ready().await; + let required = if ready { active_members.clamp(1, CORROBORATION) } else { 1 @@ -114,38 +123,47 @@ pub(super) async fn check<H: Host>(host: &H) { for (_, epoch) in &ahead { *advertised.entry(*epoch).or_default() += 1; } - let corroborated: Vec<Epoch> = advertised + let corroborated: Vec<(Epoch, usize)> = advertised .into_iter() .filter(|(_, count)| *count >= required) - .map(|(epoch, _)| epoch) .collect(); - let Some(generation) = corroborated.iter().map(|epoch| epoch.generation).max() else { - if host.remake().catch_up().is_some() { - host.remake().set_catch_up(None); + let mut aside = false; + let target = match corroborated.iter().map(|(epoch, _)| epoch.generation).max() { + Some(generation) => { + let targets: Vec<(Epoch, usize)> = corroborated + .into_iter() + .filter(|(epoch, _)| epoch.generation == generation) + .collect(); + // Several epochs of the newest generation (two remakes committed for one + // epoch): the one most peers are on, as `sideways` would pick. + let target = most_followed(&targets).expect("corroborated has this generation"); + if targets.len() > 1 { + let listed: Vec<String> = + targets.iter().map(|(epoch, _)| epoch.to_string()).collect(); + warn!(epochs = ?listed, chosen = %target, "Peers disagree about the newest epoch; following the one most of them are on"); + } + target } - if !ahead.is_empty() { - debug!( - peers = ahead.len(), - required, "A newer epoch is advertised, but not by enough peers yet" - ); + None => { + if !ahead.is_empty() { + debug!( + peers = ahead.len(), + required, "A newer epoch is advertised, but not by enough peers yet" + ); + } + let sideways = ready.then(|| sideways(local, &serving, required)).flatten(); + let Some(target) = sideways else { + if host.remake().catch_up().is_some() { + host.remake().set_catch_up(None); + } + return; + }; + aside = true; + target } - return; }; - let mut targets: Vec<Epoch> = corroborated - .into_iter() - .filter(|epoch| epoch.generation == generation) - .collect(); - targets.sort_by_key(|epoch| epoch.id); - targets.dedup(); - let [target] = targets.as_slice() else { - let listed: Vec<String> = targets.iter().map(ToString::to_string).collect(); - warn!(epochs = ?listed, "Peers disagree about the newest epoch; not catching up"); - return; - }; - let target = *target; - // Back off between attempts for the same target. let previous = host .remake() @@ -159,9 +177,50 @@ pub(super) async fn check<H: Host>(host: &H) { } let attempts = previous.map_or(0, |catch_up| catch_up.attempts) + 1; + if aside { + warn!( + local = %local, + theirs = %target, + "More peers are on another epoch of this generation than on ours; rejoining theirs (writes made on ours since go to the quarantine)" + ); + } start(host, local, target, attempts).await; } +/// For a ready node, another epoch of its own generation to rejoin, if any. Two remakes +/// committed for one epoch (`remakes.md`, Limits: a coordinator cut off after it +/// committed, electorates of an old partition, a vote lost with a restart) leave two +/// epochs of a generation that never sync. The one more ready members are on wins: +/// a node on the other rejoins it, with `required` of them at least; on a tie, the +/// epoch with the lower ID. Each node judges by the peers it sees, so the side that +/// moves is the one that sees itself outnumbered. +fn sideways(local: Epoch, serving: &[(String, Epoch)], required: usize) -> Option<Epoch> { + let mut counts: HashMap<Epoch, usize> = HashMap::new(); + for (_, epoch) in serving { + if epoch.generation == local.generation { + *counts.entry(*epoch).or_default() += 1; + } + } + // This node is on its own epoch too. + let ours = counts.remove(&local).unwrap_or(0) + 1; + let others: Vec<(Epoch, usize)> = counts + .into_iter() + .filter(|(epoch, count)| { + *count >= required && (*count > ours || (*count == ours && epoch.id < local.id)) + }) + .collect(); + most_followed(&others) +} + +/// Of epochs with how many peers are on each, the one most are on; of those, the one +/// with the lowest ID, so every node picks the same. +fn most_followed(epochs: &[(Epoch, usize)]) -> Option<Epoch> { + epochs + .iter() + .max_by(|a, b| a.1.cmp(&b.1).then(b.0.id.cmp(&a.0.id))) + .map(|(epoch, _)| *epoch) +} + async fn start<H: Host>(host: &H, local: Epoch, target: Epoch, attempts: u32) { host.remake().set_catch_up(Some(CatchUp { target, @@ -170,24 +229,12 @@ async fn start<H: Host>(host: &H, local: Epoch, target: Epoch, attempts: u32) { error: None, })); - // Keep answering reads, from a copy of the current document marked stale, before - // the node stops being ready: no window in which reads get 503. - if host.is_node_ready().await { - let copy = host.read_doc(|doc| doc.clone()).await; - host.set_stale_view(Some((local, copy))); - } - let was_ready = host.suspend().await; let actor = ActorId::random(); host.remake().add_own_actor(actor.clone()); - let old = match host - .switch_epoch(target, move |_live| { - Ok::<_, Infallible>(AutoCommit::new().with_actor(actor)) - }) - .await - { - Ok(old) => old, - Err(never) => match never {}, - }; + // At once: the fresh document takes no writes until the rejoin is done; a ready + // node's document so far answers reads, marked stale, and takes its writes + // meanwhile (`Host::begin_rejoin`). + let begun = host.begin_rejoin(target, actor).await; // After the swap: messages of the old epoch are dropped from now on, so nothing // can mark the new, empty document as caught up by mistake. A stale "done" from // just before is caught in `finish` (the snapshot change is missing then). @@ -197,9 +244,10 @@ async fn start<H: Host>(host: &H, local: Epoch, target: Epoch, attempts: u32) { host.remake().start_rejoin(Rejoin { previous: local, target, - old, + old: begun.old, + heads: begun.heads, started: Instant::now(), - was_ready, + was_ready: begun.was_ready, }); // Sync waits until the snapshot is here (or its download failed). @@ -218,9 +266,7 @@ async fn fetch_snapshot<H: Host>(host: H, target: Epoch) { if still_rejoining { match outcome { Ok(change) => { - let applied = host - .commit_doc(move |doc| doc.apply_changes([change])) - .await; + let applied = host.apply_change(change).await; match applied { Ok(()) => { info!(epoch = %target, "Applied the epoch's snapshot; syncing the changes made since") @@ -242,30 +288,35 @@ async fn fetch_snapshot<H: Host>(host: H, target: Epoch) { host.request_sync(); } -/// What to do with this node's own changes the new epoch lacks. +/// What to do with this node's own changes the new epoch lacks, those made before the +/// rejoin. enum Plan { Nothing, - /// Replay these paths, carried through every remake since the node's epoch. - Replay(Vec<Vec<String>>, Translator), + /// Replay what changed from `before` to `after` in the old document, carried + /// through every remake since the node's epoch. + Replay { + before: Vec<ChangeHash>, + after: Vec<ChangeHash>, + translator: Translator, + }, Quarantine { paths: Vec<Vec<String>>, reason: String, }, } -// Every variant holds a whole document; one value lives for one step of a rejoin, so -// boxing would only add an indirection. -#[allow(clippy::large_enum_variant)] enum Prepared { /// Caught up with someone, but not with the target epoch yet: keep waiting. - NotYet(AutoCommit), - Failed(AutoCommit, String), + NotYet, + Failed(String), Ready { - old: AutoCommit, metadata: EpochMetadata, plan: Plan, + /// Where the writes the old document took during the rejoin go: fresh, they + /// are replayed whatever the replay window says, unless there is no telling + /// where (the reason). + during: Result<Translator, String>, own_missing: usize, - had_history: bool, }, } @@ -277,99 +328,139 @@ async fn finish<H: Host>(host: &H) { previous, target, old, + heads: began, started, was_ready, } = rejoin; - // Everything heavy runs on a copy, without a lock. + // Everything heavy runs on copies, without a lock; the old document takes writes + // meanwhile, and those after `began` are replayed under the lock in the end. let copy = host.read_doc(|doc| doc.clone()).await; + let kept = host.rejoin_document().await; let own = host.remake().own_actors(); let now = host.now_millis(); - let prepared = - tokio::task::spawn_blocking(move || prepare(copy, old, previous, target, own, now)).await; + let prepared = { + let began = began.clone(); + tokio::task::spawn_blocking(move || { + let empty = Document::new(); + let reference = kept.as_ref().or(old.as_ref()).unwrap_or(&empty); + let prepared = prepare(©, reference, &began, previous, target, own, now); + (prepared, old) + }) + .await + }; - let (old, metadata, plan, own_missing, had_history) = match prepared { - Ok(Prepared::Ready { + let (metadata, plan, during, own_missing, old) = match prepared { + Ok(( + Prepared::Ready { + metadata, + plan, + during, + own_missing, + }, old, - metadata, - plan, - own_missing, - had_history, - }) => (old, metadata, plan, own_missing, had_history), - Ok(Prepared::NotYet(old)) => { + )) => (metadata, plan, during, own_missing, old), + Ok((Prepared::NotYet, old)) => { host.reset_initial_sync(); host.remake().start_rejoin(Rejoin { previous, target, old, + heads: began, started, was_ready, }); return; } - Ok(Prepared::Failed(old, reason)) => { + Ok((Prepared::Failed(reason), old)) => { let rejoin = Rejoin { previous, target, old, + heads: began, started, was_ready, }; return abandon(host, rejoin, reason).await; } Err(join_error) => { - // The old document is gone with the panicked task; nothing to restore. + // A document of a node that wasn't ready is gone with the panicked task; a + // ready node's writes during the rejoin wait in the quarantine. error!(error = %join_error, "Rejoin task failed; staying on the new epoch"); - if was_ready { - host.resume().await; - } - host.set_stale_view(None); + let (unplaced, old) = host + .finish_rejoin(None, move |_live, old| { + let reason = "written during a rejoin whose check failed".to_owned(); + delta::changed_paths(old, &began, &old.get_heads()) + .into_iter() + .map(|path| (path, reason.clone())) + .collect::<Vec<_>>() + }) + .await; + quarantine::add(host, previous, target, &old, unplaced); return; } }; - let mut stats = ReplayStats::default(); - let mut quarantine_id = None; - let mut keep_archive = true; - let mut old = Some(old); - - match plan { - Plan::Nothing => {} - Plan::Replay(paths, translator) => { - let document = old.take().expect("old document"); - let (replayed, document) = host - .commit_doc(move |doc| { - let stats = delta::replay_paths(&document, doc, &paths, &translator); - Ok::<_, Infallible>((stats, document)) - }) - .await - .unwrap_or_else(|never| match never {}); - stats = replayed; + // The writes, under the lock, with the switch back to serving requests. + let quarantined_whole = matches!(plan, Plan::Quarantine { .. }); + let superseded = host.superseded(); + let ((stats, unplaced), old) = host + .finish_rejoin(old, move |live, old| { + let mut stats = ReplayStats::default(); + let mut unplaced = Vec::new(); + match plan { + Plan::Nothing => {} + Plan::Replay { + before, + after, + translator, + } => { + stats.absorb(delta::replay_newest( + old, + &before, + &after, + live, + &translator, + superseded, + )); + } + Plan::Quarantine { paths, reason } => { + unplaced.extend(paths.into_iter().map(|path| (path, reason.clone()))); + } + } + let heads = old.get_heads(); + match during { + Ok(translator) => { + stats.absorb(delta::replay_newest( + old, + &began, + &heads, + live, + &translator, + superseded, + )); + } + Err(reason) => { + let reason = format!("written during the rejoin, but {reason}"); + unplaced.extend( + delta::changed_paths(old, &began, &heads) + .into_iter() + .map(|path| (path, reason.clone())), + ); + } + } // Writes the remakes since had no place for. - quarantine_id = quarantine::add( - host, - previous, - target, - &document, - std::mem::take(&mut stats.quarantined), - ); - old = Some(document); - } - Plan::Quarantine { paths, reason } => { - let document = old.take().expect("old document"); - let entries = paths - .into_iter() - .map(|path| (path, reason.clone())) - .collect(); - quarantine_id = quarantine::add(host, previous, target, &document, entries); - keep_archive = false; // the quarantine holds the document - } - } + unplaced.append(&mut stats.quarantined); + (stats, unplaced) + }) + .await; + let quarantine_id = quarantine::add(host, previous, target, &old, unplaced); - if let (true, true, Some(document)) = (had_history, keep_archive, old) { + // Unless the quarantine holds the document. + if !old.get_heads().is_empty() && !quarantined_whole { host.remake().archive( previous, - document, + old, metadata.policy.archive_retention.clone(), target.generation, host.now_millis(), @@ -389,11 +480,6 @@ async fn finish<H: Host>(host: &H) { host.remake().set_catch_up(None); let actor = host.read_doc(|doc| doc.get_actor().clone()).await; host.remake().reset_own_actors(actor); - if was_ready { - host.resume().await; - } - // After resuming, so reads move from the stale view to the live document directly. - host.set_stale_view(None); info!( from = %previous, @@ -401,71 +487,47 @@ async fn finish<H: Host>(host: &H) { own_changes = own_missing, replayed = stats.replayed, failed = stats.failed, + superseded = stats.superseded, quarantine = ?quarantine_id, secs = started.elapsed().as_secs(), "Rejoined newer epoch" ); } -/// Verifies the synced document and decides about the old document's own changes. -/// `own`: this node's actors (`RemakeState::own_actors`); the old document's own actor -/// counts in any case. `now`: Unix milliseconds, by the node's clock. +/// Verifies the synced document `copy` and decides about the old document's own +/// changes: those up to `began` (when the rejoin began) by the replay window, those +/// after it (made during the rejoin) whatever it says. `own`: this node's actors +/// (`RemakeState::own_actors`); the old document's own actor counts in any case. +/// `now`: Unix milliseconds, by the node's clock. fn prepare( - mut copy: AutoCommit, - mut old: AutoCommit, + copy: &Document, + old: &Document, + began: &[ChangeHash], previous: Epoch, target: Epoch, mut own: Vec<ActorId>, now: UnixMillis, ) -> Prepared { if copy.get_change_by_hash(&ChangeHash(target.id)).is_none() { - return Prepared::NotYet(old); + return Prepared::NotYet; } - let metadata = match snapshot::verify_document(&mut copy, target) { + let metadata = match snapshot::verify_document(copy, target) { Ok(metadata) => metadata, - Err(reason) => return Prepared::Failed(old, reason), + Err(reason) => return Prepared::Failed(reason), }; - let old_heads = old.get_heads(); - if old_heads.is_empty() { - return Prepared::Ready { - old, - metadata, - plan: Plan::Nothing, - own_missing: 0, - had_history: false, - }; - } - // The epoch that followed ours records (per actor) which of our changes it took. // Snapshots carry earlier epochs' metadata forward, so this works across any gap. - let next = snapshot::read_epoch_metadata(©, previous.generation + 1) + let next = snapshot::read_epoch_metadata(copy, previous.generation + 1) .ok() .filter(|next| next.previous == previous); - - own.push(old.get_actor().clone()); - let clock = next - .as_ref() - .map_or_else(BTreeMap::new, |next| next.clock.clone()); - let missing = delta::missing_from_snapshot(&mut old, &clock, &own); - if missing.own == 0 { - return Prepared::Ready { - old, - metadata, - plan: Plan::Nothing, - own_missing: 0, - had_history: true, - }; - } - - let paths = delta::changed_paths(&mut old, &missing.before, &old_heads); let gap = target.generation - previous.generation; // Writes made in our epoch go through every remake since, in order. let translator = (previous.generation + 1..=target.generation).try_fold( Translator::default(), |mut translator, generation| { - let epoch = snapshot::read_epoch_metadata(©, generation)?; + let epoch = snapshot::read_epoch_metadata(copy, generation)?; let kind = epoch.kind().ok_or_else(|| { format!( "epoch generation {generation} was made by unknown remake kind {:?}", @@ -476,8 +538,51 @@ fn prepare( Ok::<_, String>(translator) }, ); + let during = match (&next, &translator) { + // Sideways, to another epoch of our generation: the same paths. + _ if gap == 0 => Ok(Translator::default()), + (Some(_), Ok(translator)) => Ok(translator.clone()), + (Some(_), Err(reason)) => Err(format!("cannot be translated: {reason}")), + (None, _) => Err(format!( + "the metadata of the epoch after {previous} is missing" + )), + }; + + if began.is_empty() { + return Prepared::Ready { + metadata, + plan: Plan::Nothing, + during, + own_missing: 0, + }; + } + + own.push(old.get_actor().clone()); + let clock = next + .as_ref() + .map_or_else(BTreeMap::new, |next| next.clock.clone()); + let missing = delta::missing_from_snapshot(old, &clock, &own); + if missing.own == 0 { + return Prepared::Ready { + metadata, + plan: Plan::Nothing, + during, + own_missing: 0, + }; + } + // Up to when the rejoin began; what came after is `during`'s. + let paths = delta::changed_paths(old, &missing.before, began); let plan = match (&next, translator) { + // Rejoined sideways (`sideways`): whatever was written on our epoch, the swarm + // went another way; nothing says where it would go. + _ if gap == 0 => Plan::Quarantine { + paths, + reason: format!( + "{} local changes were made on {previous}, which the swarm left for {target}, another epoch of the same generation", + missing.own + ), + }, (Some(_), Err(reason)) => Plan::Quarantine { paths, reason: format!("cannot translate {} local changes: {reason}", missing.own), @@ -492,7 +597,11 @@ fn prepare( (Some(next), Ok(translator)) => { let away_ms = now.ms_since(next.proposed_at); if next.policy.replay_window.covers(gap, away_ms) { - Plan::Replay(paths, translator) + Plan::Replay { + before: missing.before.clone(), + after: began.to_vec(), + translator, + } } else { Plan::Quarantine { paths, @@ -507,37 +616,27 @@ fn prepare( }; Prepared::Ready { - old, metadata, plan, + during, own_missing: missing.own, - had_history: true, } } -/// Restores the old document and epoch, and retries later. +/// Restores the old document (with the writes it took meanwhile) and epoch, and +/// retries later. async fn abandon<H: Host>(host: &H, rejoin: Rejoin, reason: String) { let Rejoin { previous, target, old, - was_ready, .. } = rejoin; warn!(target = %target, reason = %reason, "Rejoin failed; restoring the previous epoch"); - if let Err(never) = host - .switch_epoch(previous, move |_live| Ok::<_, Infallible>(old)) - .await - { - match never {} - } + host.abandon_rejoin(previous, old).await; // A node that was never ready must sync again before bootstrap may finish. host.reset_initial_sync(); - if was_ready { - host.resume().await; - } - host.set_stale_view(None); let attempts = host .remake() @@ -555,16 +654,59 @@ async fn abandon<H: Host>(host: &H, rejoin: Rejoin, reason: String) { #[cfg(test)] mod tests { use super::*; + use crate::document::testing::change; use crate::remake::snapshot::{SnapshotParams, build}; use automerge::{ActorId, ROOT, transaction::Transactable}; use serde_json::{Value as JsonValue, json}; + #[test] + fn sideways_follows_the_epoch_more_peers_are_on() { + let epoch = |id: u8| Epoch { + generation: 3, + id: [id; 32], + }; + let on = |peers: &[(&str, Epoch)]| -> Vec<(String, Epoch)> { + peers + .iter() + .map(|(peer, epoch)| ((*peer).to_owned(), *epoch)) + .collect() + }; + let (ours, theirs) = (epoch(5), epoch(9)); + // Alone against two: theirs. + let serving = on(&[("b", theirs), ("c", theirs)]); + assert_eq!(sideways(ours, &serving, 2), Some(theirs)); + // Two (with us) against two: the lower ID, ours; they come to us. + let serving = on(&[("b", theirs), ("c", theirs), ("d", ours)]); + assert_eq!(sideways(ours, &serving, 2), None); + assert_eq!( + sideways(theirs, &on(&[("a", ours), ("d", ours), ("b", theirs)]), 2), + Some(ours) + ); + // Fewer than required: nothing, even outnumbered. + assert_eq!(sideways(ours, &on(&[("b", theirs)]), 2), None); + // Another generation isn't sideways. + let newer = Epoch { + generation: 4, + id: [1; 32], + }; + assert_eq!(sideways(ours, &on(&[("b", newer), ("c", newer)]), 2), None); + // Several: the one most are on. + let other = epoch(2); + let serving = on(&[ + ("b", theirs), + ("c", theirs), + ("d", theirs), + ("e", other), + ("f", other), + ]); + assert_eq!(sideways(ours, &serving, 2), Some(theirs)); + } + /// An old-epoch document with one own change after the base, and the swarm's /// document of the next epoch (snapshot plus one foreign change). - fn scenario(policy: &JsonValue) -> (AutoCommit, AutoCommit, Epoch) { - let mut old = AutoCommit::new().with_actor(ActorId::from(&b"late-node"[..])); - old.put(ROOT, "base", 1_i64).unwrap(); - old.commit(); + fn scenario(policy: &JsonValue) -> (Document, Document, Epoch) { + let mut old = Document::new().with_actor(ActorId::from(&b"late-node"[..])); + change(&mut old, |old| old.put(ROOT, "base", 1_i64).unwrap()); let base = old.get_heads(); let snapshot = build( @@ -584,43 +726,76 @@ mod tests { let target = snapshot.epoch; let mut swarm = snapshot.doc.with_actor(ActorId::from(&b"swarm-node"[..])); - swarm.put(ROOT, "theirs", 1_i64).unwrap(); - swarm.commit(); + change(&mut swarm, |swarm| { + swarm.put(ROOT, "theirs", 1_i64).unwrap() + }); - old.put(ROOT, "mine", 1_i64).unwrap(); - old.commit(); + change(&mut old, |old| old.put(ROOT, "mine", 1_i64).unwrap()); (old, swarm, target) } #[test] fn replays_within_the_window() { let (old, swarm, target) = scenario(&json!({})); + let began = old.get_heads(); match prepare( - swarm, - old, + &swarm, + &old, + &began, Epoch::GENESIS, target, Vec::new(), UnixMillis(0), ) { Prepared::Ready { - plan: Plan::Replay(paths, _), + plan: Plan::Replay { before, after, .. }, own_missing, .. } => { assert_eq!(own_missing, 1); - assert_eq!(paths, vec![vec!["mine".to_owned()]]); + assert_eq!( + delta::changed_paths(&old, &before, &after), + vec![vec!["mine".to_owned()]] + ); } _ => panic!("expected a replay"), } } + /// Writes made during the rejoin aren't the window's: those after `began` go + /// wherever the remakes since put them, even outside it. + #[test] + fn writes_during_the_rejoin_are_replayed_outside_the_window() { + let (old, swarm, target) = scenario(&json!({ "replayWindow": { "epochs": 0 } })); + let began = old.get_heads(); + let mut during = old.clone(); + change(&mut during, |doc| doc.put(ROOT, "later", 1_i64).unwrap()); + match prepare( + &swarm, + &during, + &began, + Epoch::GENESIS, + target, + Vec::new(), + UnixMillis(0), + ) { + Prepared::Ready { + plan: Plan::Quarantine { paths, .. }, + during: Ok(_), + .. + } => assert_eq!(paths, vec![vec!["mine".to_owned()]]), + _ => panic!("expected a quarantine before, and a replay during the rejoin"), + } + } + #[test] fn quarantines_outside_the_window() { let (old, swarm, target) = scenario(&json!({ "replayWindow": { "epochs": 0 } })); + let began = old.get_heads(); match prepare( - swarm, - old, + &swarm, + &old, + &began, Epoch::GENESIS, target, Vec::new(), @@ -639,8 +814,9 @@ mod tests { let (_, swarm, target) = scenario(&json!({})); assert!(matches!( prepare( - swarm, - AutoCommit::new(), + &swarm, + &Document::new(), + &[], Epoch::GENESIS, target, Vec::new(), @@ -648,7 +824,7 @@ mod tests { ), Prepared::Ready { plan: Plan::Nothing, - had_history: false, + own_missing: 0, .. } )); @@ -659,14 +835,15 @@ mod tests { let (old, _, target) = scenario(&json!({})); assert!(matches!( prepare( - AutoCommit::new(), - old, + &Document::new(), + &old, + &old.get_heads(), Epoch::GENESIS, target, Vec::new(), UnixMillis(0) ), - Prepared::NotYet(_) + Prepared::NotYet )); } } diff --git a/src/remake/content.rs b/src/remake/content.rs index d036b87..467d083 100644 --- a/src/remake/content.rs +++ b/src/remake/content.rs @@ -11,7 +11,6 @@ use std::collections::HashSet; use std::sync::Arc; -use std::time::Duration; use tracing::{info, warn}; @@ -59,9 +58,7 @@ pub(super) async fn ensure<H: Host>( .ok_or_else(|| format!("couldn't download content {}", reference.blake3)); } // Someone else is downloading it. - while host.remake().content_fetch_running(&hash) { - tokio::time::sleep(Duration::from_millis(500)).await; - } + host.remake().content_fetch_ended(&hash).await; } } diff --git a/src/remake/coordinator.rs b/src/remake/coordinator.rs index 50ed4a3..314cd87 100644 --- a/src/remake/coordinator.rs +++ b/src/remake/coordinator.rs @@ -38,7 +38,7 @@ use super::snapshot; use super::vote; use crate::document::meta::{self, REMAKES, VOTES}; use crate::document::store::StoreError; -use crate::swarm::sync::paths_related; +use crate::document::store::paths_related; const TICK: Duration = Duration::from_secs(1); @@ -178,7 +178,7 @@ async fn commit<H: Host>(host: &H, id: &str, record: &Record) -> Result<(), Stor }; // Clone under the read lock, build without any lock. - let mut copy = host.read_doc(|doc| doc.clone()).await; + let copy = host.read_doc(|doc| doc.clone()).await; // A base left to the commit is what this node has now. Every node that sees the // decision has it too: the decision change depends on it. let (base, fixed_now) = match &record.proposal.base_heads { @@ -195,7 +195,7 @@ async fn commit<H: Host>(host: &H, id: &str, record: &Record) -> Result<(), Stor }; let base_heads = fixed_now.then(|| records::encode_heads(&request.base_heads)); request.content = content; - let mut snapshot = match snapshot::build_detached(copy, request).await { + let snapshot = match snapshot::build_detached(copy, request).await { Ok((snapshot, _copy)) => snapshot, Err(error) => { return decide_abort(host, id, format!("snapshot build failed: {error}")).await; diff --git a/src/remake/delta.rs b/src/remake/delta.rs index 80e5443..638231c 100644 --- a/src/remake/delta.rs +++ b/src/remake/delta.rs @@ -20,15 +20,23 @@ //! //! The protocol's own metadata (`_continuum/remakes`, `_continuum/epochs`) is never //! replayed: the snapshot owns it. +//! +//! Last writer wins, by the writers' clocks ([`replay_newest`]): a replayed value is a +//! fresh operation, which by Automerge's order beats everything the document it goes +//! into has, newer writes included. So a path is replayed only if no change in that +//! document touched it (or a path above or below it) later than the write being +//! replayed, as the changes' timestamps say; and the replayed changes carry the times +//! of the writes they replay, not of the replay, so later replays judge by those. -use std::collections::{BTreeMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; -use automerge::{ActorId, AutoCommit, ChangeHash, PatchAction, Prop}; -use tracing::warn; +use crate::document::{ActorId, ChangeHash, Document, ReadDoc, Transactable}; +use serde::{Deserialize, Serialize}; +use tracing::{info, warn}; use super::kind::{Translation, Translator}; use crate::document::meta::{self, EPOCHS, REMAKES, VOTES}; -use crate::document::store::{self, StoreError}; +use crate::document::store::{self, InLists, StoreError}; #[derive(Debug, Default, Clone)] pub(crate) struct ReplayStats { @@ -36,6 +44,10 @@ pub(crate) struct ReplayStats { pub(crate) failed: usize, /// Paths the remake declared obsolete. pub(crate) dropped: usize, + /// Paths not replayed because a newer write touched them ([`replay_newest`]), and + /// left out (`Superseded::Drop`; with `Superseded::Quarantine`, they are among + /// `quarantined`). + pub(crate) superseded: usize, /// Paths the remake had no place for, with the reason. pub(crate) quarantined: Vec<(Vec<String>, String)>, } @@ -46,17 +58,148 @@ impl ReplayStats { self.replayed += other.replayed; self.failed += other.failed; self.dropped += other.dropped; + self.superseded += other.superseded; self.quarantined.extend(other.quarantined); } } +/// What a replay does with a write that a newer one superseded (`--superseded-writes`): +/// leave it out, logged (last writer wins), or put it in the quarantine, where an +/// operator can still replay it. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) enum Superseded { + #[default] + Drop, + Quarantine, +} + +/// Replays into `dst` what changed in `src` between `before` and `after` (both in +/// `src`), last writer wins: a path whose newest write there (by its change's +/// timestamp) is older than a change in `dst` touching it, or a path above or below it, +/// stays as `dst` has it (`superseded` says what becomes of the write). Ties go to the +/// replay. The replayed paths are committed grouped by the time of the writes they +/// replay, each change stamped with it. +pub(crate) fn replay_newest( + src: &Document, + before: &[ChangeHash], + after: &[ChangeHash], + dst: &mut Document, + translator: &Translator, + superseded: Superseded, +) -> ReplayStats { + let mut stats = ReplayStats::default(); + if before == after { + return stats; + } + let paths = changed_paths(src, before, after); + let written = touched(src, &between(src, before, after)); + let ours: Vec<i64> = paths + .iter() + .map(|path| newest(&written, path).unwrap_or(0)) + .collect(); + // Only changes newer than the oldest write replayed can supersede one; the + // snapshot (time 0) never does. + let oldest = ours.iter().copied().min().unwrap_or(0); + let newer: Vec<ChangeHash> = dst + .get_changes(&[]) + .iter() + .filter(|change| change.timestamp() > oldest) + .map(|change| change.hash()) + .collect(); + let theirs = touched(dst, &newer); + + let mut by_time: BTreeMap<i64, Vec<Vec<String>>> = BTreeMap::new(); + for (path, at) in paths.into_iter().zip(ours) { + match newest(&theirs, &path) { + Some(newer) if newer > at => { + stats.superseded += 1; + if superseded == Superseded::Quarantine { + stats.quarantined.push(( + path, + format!("a newer write (at {newer}, this one at {at}) superseded it"), + )); + } + } + _ => by_time.entry(at).or_default().push(path), + } + } + for (at, paths) in by_time { + let mut transaction = dst.transaction(); + stats.absorb(replay_paths(src, &mut transaction, &paths, translator)); + transaction.commit_with(automerge::transaction::CommitOptions::default().with_time(at)); + } + if stats.superseded > 0 { + info!( + superseded = stats.superseded, + quarantined = superseded == Superseded::Quarantine, + "Writes not replayed: newer ones touched their paths" + ); + } + if superseded == Superseded::Quarantine { + // Counted as quarantined, not as left out. + stats.superseded = 0; + } + stats +} + +/// The changes of `doc` after `before` up to `after`: those `after` builds on that +/// `before` doesn't. +fn between(doc: &Document, before: &[ChangeHash], after: &[ChangeHash]) -> Vec<ChangeHash> { + let deps: HashMap<ChangeHash, Vec<ChangeHash>> = doc + .get_changes(before) + .iter() + .map(|change| (change.hash(), change.deps().to_vec())) + .collect(); + let mut found = HashSet::new(); + let mut todo: Vec<ChangeHash> = after.to_vec(); + while let Some(hash) = todo.pop() { + if let Some(parents) = deps.get(&hash) + && found.insert(hash) + { + todo.extend(parents.iter().copied()); + } + } + found.into_iter().collect() +} + +/// For each of `changes` (in `doc`), its time and the store paths it touched, from its +/// operations (`store::changed_by`: a diff per change would cost what the maps it +/// touched hold, `remake::bench`). +fn touched(doc: &Document, changes: &[ChangeHash]) -> Vec<(i64, Vec<Vec<String>>)> { + changes + .iter() + .filter_map(|hash| { + let change = doc.get_change_by_hash(hash)?; + let paths = store::changed_by(doc, hash, InLists::Whole)?; + Some((change.timestamp(), paths)) + }) + .collect() +} + +/// The time of the newest of `changes` that touched `path`, a path above it or one +/// below it. +fn newest(changes: &[(i64, Vec<Vec<String>>)], path: &[String]) -> Option<i64> { + changes + .iter() + .filter(|(_, paths)| { + paths + .iter() + .any(|other| other.starts_with(path) || path.starts_with(other)) + }) + .map(|(at, _)| *at) + .max() +} + /// Replays into `dst` everything that changed in `src` between `before` and `after`, -/// as one commit. Both head sets must be in `src`. +/// as one commit, whatever `dst` has (no last writer wins): what a replay copies, for +/// tests. Both head sets must be in `src`. +#[cfg(test)] pub(crate) fn replay( - src: &mut AutoCommit, + src: &Document, before: &[ChangeHash], after: &[ChangeHash], - dst: &mut AutoCommit, + dst: &mut Document, translator: &Translator, ) -> ReplayStats { if before == after { @@ -64,14 +207,17 @@ pub(crate) fn replay( } let paths = changed_paths(src, before, after); - replay_paths(src, dst, &paths, translator) + let mut transaction = dst.transaction(); + let stats = replay_paths(src, &mut transaction, &paths, translator); + transaction.commit(); + stats } -/// Replays the state of `paths` (from [`changed_paths`]) from `src` into `dst`, as one -/// commit. Lets callers find the paths outside a lock and copy them inside it. +/// Replays the state of `paths` (from [`changed_paths`]) from `src` into `dst` (the +/// caller commits). Lets callers find the paths outside a lock and copy them inside it. pub(crate) fn replay_paths( - src: &AutoCommit, - dst: &mut AutoCommit, + src: &Document, + dst: &mut impl Transactable, paths: &[Vec<String>], translator: &Translator, ) -> ReplayStats { @@ -79,13 +225,12 @@ pub(crate) fn replay_paths( for path in paths { replay_path(src, dst, path, translator, &mut stats); } - let _ = dst.commit(); stats } fn replay_path( - src: &AutoCommit, - dst: &mut AutoCommit, + src: &Document, + dst: &mut impl Transactable, path: &[String], translator: &Translator, stats: &mut ReplayStats, @@ -164,7 +309,7 @@ pub(crate) struct Missing { /// replayed by the nodes that switched, and replaying them again would overwrite newer /// writes in the new epoch with stale ones. pub(crate) fn missing_from_snapshot( - doc: &mut AutoCommit, + doc: &Document, clock: &BTreeMap<String, u64>, own: &[ActorId], ) -> Missing { @@ -225,11 +370,14 @@ pub(crate) fn missing_from_snapshot( /// The minimal set of store paths covering every change between `before` and `after`: /// sorted, and no path is a prefix of another. pub(crate) fn changed_paths( - doc: &mut AutoCommit, + doc: &Document, before: &[ChangeHash], after: &[ChangeHash], ) -> Vec<Vec<String>> { - let mut paths: Vec<Vec<String>> = doc.diff(before, after).iter().map(patch_path).collect(); + let mut paths: Vec<Vec<String>> = store::changed(doc, before, after, InLists::Whole) + .into_iter() + .map(|changed| changed.path) + .collect(); paths.sort(); paths.dedup(); @@ -245,49 +393,26 @@ pub(crate) fn changed_paths( minimal } -fn patch_path(patch: &automerge::Patch) -> Vec<String> { - let mut path = Vec::new(); - for (_, prop) in &patch.path { - match prop { - Prop::Map(key) => path.push(key.clone()), - // Inside a list: the list is replayed as a whole. - Prop::Seq(_) => return path, - } - } - - match &patch.action { - PatchAction::PutMap { key, .. } | PatchAction::DeleteMap { key, .. } => { - path.push(key.clone()) - } - PatchAction::Increment { - prop: Prop::Map(key), - .. - } => path.push(key.clone()), - // List and text edits: the containing object. - _ => {} - } - path -} - #[cfg(test)] mod tests { use super::*; + use crate::document::testing::{change, try_change}; use automerge::{ObjType, ROOT, ReadDoc, ScalarValue, transaction::Transactable}; use serde_json::json; - fn view(doc: &AutoCommit) -> serde_json::Value { + fn view(doc: &Document) -> serde_json::Value { store::read_path(doc, &[]).unwrap().unwrap() } /// A random store write as the API makes it (one change): a scalar or a small map /// at a path up to three keys deep, or a delete. Writes the store refuses (e.g. into /// a scalar) are rolled back. - fn random_write(rng: &mut crate::testrand::Rng, doc: &mut AutoCommit) { + fn random_write(rng: &mut crate::testrand::Rng, doc: &mut Document) { const KEYS: [&str; 3] = ["a", "b", "c"]; let depth = rng.between(1, 3); let path: Vec<String> = (0..depth).map(|_| rng.pick(&KEYS).to_string()).collect(); - let result = if rng.chance(0.2) { - store::delete_path(doc, &path, false) + let _ = if rng.chance(0.2) { + try_change(doc, |doc| store::delete_path(doc, &path, false)) } else { let value = match rng.below(4) { 0 => json!(rng.below(100)), @@ -299,16 +424,8 @@ mod tests { serde_json::Value::Object(map) } }; - store::write_path(doc, &path, &value) + try_change(doc, |doc| store::write_path(doc, &path, &value)) }; - match result { - Ok(()) => { - doc.commit_with(automerge::transaction::CommitOptions::default().with_time(0)); - } - Err(_) => { - doc.rollback(); - } - } } /// Compaction's promise: a snapshot of the base plus everything replayed since gives @@ -317,16 +434,17 @@ mod tests { fn replay_reproduces_the_document() { crate::testrand::for_seeds(200, |rng| { let actor = |i: u8| ActorId::from(&[i; 16][..]); - let mut old = AutoCommit::new().with_actor(actor(0)); + let mut old = Document::new().with_actor(actor(0)); for _ in 0..rng.between(0, 8) { random_write(rng, &mut old); } let base = old.get_heads(); - let mut new = AutoCommit::new().with_actor(actor(100)); - store::materialize_into(&old, &mut new, &|_| false).unwrap(); - new.commit(); + let mut new = Document::new().with_actor(actor(100)); + change(&mut new, |new| { + store::materialize_into(&old, new, &|_| false).unwrap() + }); - let mut nodes: Vec<AutoCommit> = (1..=rng.between(1, 3)) + let mut nodes: Vec<Document> = (1..=rng.between(1, 3)) .map(|i| old.fork().with_actor(actor(i as u8))) .collect(); for _ in 0..rng.between(1, 20) { @@ -338,7 +456,7 @@ mod tests { } let after = old.get_heads(); - let stats = replay(&mut old, &base, &after, &mut new, &Translator::default()); + let stats = replay(&old, &base, &after, &mut new, &Translator::default()); assert_eq!(stats.failed, 0); assert_eq!(view(&new), view(&old)); }); @@ -369,12 +487,12 @@ mod tests { fn changed_paths_cover_every_change_minimally() { crate::testrand::for_seeds(300, |rng| { let actor = |i: u8| ActorId::from(&[i; 16][..]); - let mut doc = AutoCommit::new().with_actor(actor(0)); + let mut doc = Document::new().with_actor(actor(0)); for _ in 0..rng.between(0, 8) { random_write(rng, &mut doc); } let before = doc.get_heads(); - let mut forks: Vec<AutoCommit> = (1..=rng.between(1, 3)) + let mut forks: Vec<Document> = (1..=rng.between(1, 3)) .map(|i| doc.fork().with_actor(actor(i as u8))) .collect(); for _ in 0..rng.between(0, 12) { @@ -386,7 +504,7 @@ mod tests { } let after = doc.get_heads(); - let paths = changed_paths(&mut doc, &before, &after); + let paths = changed_paths(&doc, &before, &after); for pair in paths.windows(2) { assert!(pair[0] < pair[1], "not sorted: {paths:?}"); assert!(!pair[1].starts_with(&pair[0]), "not minimal: {paths:?}"); @@ -415,23 +533,25 @@ mod tests { #[test] fn minimal_paths() { - let mut doc = AutoCommit::new(); - doc.put(ROOT, "untouched", 1_i64).unwrap(); + let mut doc = Document::new(); + change(&mut doc, |doc| doc.put(ROOT, "untouched", 1_i64).unwrap()); let before = doc.get_heads(); - let m = doc.put_object(ROOT, "m", ObjType::Map).unwrap(); - doc.put(&m, "a", 1_i64).unwrap(); - doc.put(ROOT, "x", 1_i64).unwrap(); - doc.put(ROOT, "c", ScalarValue::counter(0)).unwrap(); - doc.commit(); + change(&mut doc, |doc| { + let m = doc.put_object(ROOT, "m", ObjType::Map).unwrap(); + doc.put(&m, "a", 1_i64).unwrap(); + doc.put(ROOT, "x", 1_i64).unwrap(); + doc.put(ROOT, "c", ScalarValue::counter(0)).unwrap(); + }); let middle = doc.get_heads(); - doc.increment(ROOT, "c", 2).unwrap(); - let list = doc.put_object(ROOT, "l", ObjType::List).unwrap(); - doc.insert(&list, 0, "e").unwrap(); - doc.commit(); + change(&mut doc, |doc| { + doc.increment(ROOT, "c", 2).unwrap(); + let list = doc.put_object(ROOT, "l", ObjType::List).unwrap(); + doc.insert(&list, 0, "e").unwrap(); + }); let after = doc.get_heads(); - let all = changed_paths(&mut doc, &before, &after); + let all = changed_paths(&doc, &before, &after); assert_eq!( all, vec![ @@ -442,37 +562,176 @@ mod tests { ] ); - let later = changed_paths(&mut doc, &middle, &after); + let later = changed_paths(&doc, &middle, &after); assert_eq!(later, vec![vec!["c".to_owned()], vec!["l".to_owned()]]); } + /// Writes to `doc` as one change made at `at` (Unix ms). + fn write_at( + doc: &mut Document, + at: i64, + f: impl FnOnce(&mut automerge::transaction::Transaction<'_>), + ) { + let mut transaction = doc.transaction(); + f(&mut transaction); + transaction.commit_with(automerge::transaction::CommitOptions::default().with_time(at)); + } + + /// An old document with `x`, `m/a` and `m/b` written at 10 after the base, and a new + /// one holding the base; `newer` writes to the new one first. + fn superseding( + newer: impl FnOnce(&mut Document), + superseded: Superseded, + ) -> (Document, ReplayStats) { + let mut old = Document::new(); + write_at(&mut old, 1, |old| { + let m = old.put_object(ROOT, "m", ObjType::Map).unwrap(); + old.put(&m, "a", "base").unwrap(); + }); + let base = old.get_heads(); + let mut new = Document::new(); + write_at(&mut new, 0, |new| { + store::materialize_into(&old, new, &|_| false).unwrap(); + }); + newer(&mut new); + write_at(&mut old, 10, |old| { + old.put(ROOT, "x", "old").unwrap(); + let (_, m) = old.get(ROOT, "m").unwrap().unwrap(); + old.put(&m, "a", "old").unwrap(); + old.put(&m, "b", "old").unwrap(); + }); + let after = old.get_heads(); + let stats = replay_newest( + &old, + &base, + &after, + &mut new, + &Translator::default(), + superseded, + ); + (new, stats) + } + + #[test] + fn replay_newest_keeps_newer_writes() { + // Nothing newer: everything replayed, at the time it was written. + let (new, stats) = superseding(|_| {}, Superseded::Drop); + assert_eq!((stats.replayed, stats.superseded), (3, 0)); + assert_eq!( + view(&new), + json!({ "x": "old", "m": { "a": "old", "b": "old" } }) + ); + let times: Vec<i64> = new.get_changes(&[]).iter().map(|c| c.timestamp()).collect(); + assert_eq!(times, [0, 10]); + + // A newer write at the path wins; an older one doesn't. + let (new, stats) = superseding( + |new| write_at(new, 20, |new| new.put(ROOT, "x", "newer").unwrap()), + Superseded::Drop, + ); + assert_eq!((stats.replayed, stats.superseded), (2, 1)); + assert_eq!(view(&new)["x"], json!("newer")); + assert_eq!(view(&new)["m"], json!({ "a": "old", "b": "old" })); + let (new, _) = superseding( + |new| write_at(new, 5, |new| new.put(ROOT, "x", "older").unwrap()), + Superseded::Drop, + ); + assert_eq!(view(&new)["x"], json!("old")); + + // Per key: a newer `m/a` wins over the replayed `m/a`, not over `m/b`. + let (new, stats) = superseding( + |new| { + write_at(new, 20, |new| { + let (_, m) = new.get(ROOT, "m").unwrap().unwrap(); + new.put(&m, "a", "newer").unwrap(); + }) + }, + Superseded::Drop, + ); + assert_eq!(stats.superseded, 1); + assert_eq!(view(&new)["m"], json!({ "a": "newer", "b": "old" })); + // A newer write above the paths (`m` replaced) wins over both. + let (new, stats) = superseding( + |new| write_at(new, 20, |new| new.put(ROOT, "m", "a scalar now").unwrap()), + Superseded::Drop, + ); + assert_eq!(stats.superseded, 2); + assert_eq!(view(&new)["m"], json!("a scalar now")); + + // Quarantined instead, with the time of each. + let (_, stats) = superseding( + |new| write_at(new, 20, |new| new.put(ROOT, "x", "newer").unwrap()), + Superseded::Quarantine, + ); + assert_eq!(stats.superseded, 0); + assert_eq!(stats.quarantined.len(), 1); + assert_eq!(stats.quarantined[0].0, ["x"]); + } + + /// A newer write below a replayed path wins too: the replay would replace it. + #[test] + fn replay_newest_keeps_newer_writes_below() { + let mut old = Document::new(); + write_at(&mut old, 1, |old| old.put(ROOT, "base", true).unwrap()); + let base = old.get_heads(); + let mut new = Document::new(); + write_at(&mut new, 0, |new| { + store::materialize_into(&old, new, &|_| false).unwrap(); + new.put_object(ROOT, "m", ObjType::Map).unwrap(); + }); + write_at(&mut new, 20, |new| { + let (_, m) = new.get(ROOT, "m").unwrap().unwrap(); + new.put(&m, "a", "newer").unwrap(); + }); + // The whole map, older. + write_at(&mut old, 10, |old| { + let m = old.put_object(ROOT, "m", ObjType::Map).unwrap(); + old.put(&m, "a", "old").unwrap(); + }); + let after = old.get_heads(); + let stats = replay_newest( + &old, + &base, + &after, + &mut new, + &Translator::default(), + Superseded::Drop, + ); + assert_eq!(stats.superseded, 1); + assert_eq!(view(&new)["m"], json!({ "a": "newer" })); + } + #[test] fn replays_state_per_key() { // Base shared by the old document and the snapshot. - let mut old = AutoCommit::new(); - let m = old.put_object(ROOT, "m", ObjType::Map).unwrap(); - old.put(&m, "keep", "base").unwrap(); - old.put(&m, "change", "base").unwrap(); - old.put(ROOT, "gone", true).unwrap(); - old.commit(); + let mut old = Document::new(); + let m = change(&mut old, |old| { + let m = old.put_object(ROOT, "m", ObjType::Map).unwrap(); + old.put(&m, "keep", "base").unwrap(); + old.put(&m, "change", "base").unwrap(); + old.put(ROOT, "gone", true).unwrap(); + m + }); let base = old.get_heads(); - let mut new = AutoCommit::new(); - store::materialize_into(&old, &mut new, &|_| false).unwrap(); - // Something the new epoch already holds that the delta must not clobber. - let Some((_, m_new)) = new.get(ROOT, "m").unwrap() else { - panic!("m missing"); - }; - new.put(&m_new, "keep", "new epoch").unwrap(); - new.commit(); + let mut new = Document::new(); + change(&mut new, |new| { + store::materialize_into(&old, new, &|_| false).unwrap(); + // Something the new epoch already holds that the delta must not clobber. + let Some((_, m_new)) = new.get(ROOT, "m").unwrap() else { + panic!("m missing"); + }; + new.put(&m_new, "keep", "new epoch").unwrap(); + }); // The delta: one key changed, one key deleted. - old.put(&m, "change", "delta").unwrap(); - old.delete(ROOT, "gone").unwrap(); - old.commit(); + change(&mut old, |old| { + old.put(&m, "change", "delta").unwrap(); + old.delete(ROOT, "gone").unwrap(); + }); let after = old.get_heads(); - let stats = replay(&mut old, &base, &after, &mut new, &Translator::default()); + let stats = replay(&old, &base, &after, &mut new, &Translator::default()); assert_eq!(stats.failed, 0); assert_eq!( view(&new), @@ -485,9 +744,8 @@ mod tests { let own = ActorId::from(&b"own-actor"[..]); let other = ActorId::from(&b"other-actor"[..]); - let mut doc = AutoCommit::new().with_actor(own.clone()); - doc.put(ROOT, "base", 1_i64).unwrap(); - doc.commit(); + let mut doc = Document::new().with_actor(own.clone()); + change(&mut doc, |doc| doc.put(ROOT, "base", 1_i64).unwrap()); let base = doc.get_heads(); // The snapshot contains the base only. @@ -495,28 +753,28 @@ mod tests { // A foreign change after the base (replayed by its own node) ... let mut foreign = doc.fork().with_actor(other); - foreign.put(ROOT, "theirs", 1_i64).unwrap(); - foreign.commit(); + change(&mut foreign, |foreign| { + foreign.put(ROOT, "theirs", 1_i64).unwrap() + }); doc.merge(&mut foreign).unwrap(); let with_foreign = doc.get_heads(); // ... and one of ours. - doc.put(ROOT, "mine", 1_i64).unwrap(); - doc.commit(); + change(&mut doc, |doc| doc.put(ROOT, "mine", 1_i64).unwrap()); let after = doc.get_heads(); - let missing = missing_from_snapshot(&mut doc, &clock, std::slice::from_ref(&own)); + let missing = missing_from_snapshot(&doc, &clock, std::slice::from_ref(&own)); assert_eq!(missing.own, 1); assert_eq!(missing.before, with_foreign); assert_ne!(missing.before, base); assert_eq!( - changed_paths(&mut doc, &missing.before, &after), + changed_paths(&doc, &missing.before, &after), vec![vec!["mine".to_owned()]] ); // Nothing of ours missing: nothing to replay. let full: BTreeMap<String, u64> = [(own.to_string(), 2)].into_iter().collect(); - let none = missing_from_snapshot(&mut doc, &full, std::slice::from_ref(&own)); + let none = missing_from_snapshot(&doc, &full, std::slice::from_ref(&own)); assert_eq!(none.own, 0); let mut heads = after.clone(); heads.sort(); @@ -525,32 +783,28 @@ mod tests { #[test] fn protocol_metadata_is_never_replayed() { - let mut old = AutoCommit::new(); - old.put(ROOT, "x", 1_i64).unwrap(); - old.commit(); + let mut old = Document::new(); + change(&mut old, |old| old.put(ROOT, "x", 1_i64).unwrap()); let base = old.get_heads(); - let mut new = AutoCommit::new(); - store::write_path( - &mut new, - &meta::path(&[EPOCHS, "1"]), - &json!({ "generation": 1 }), - ) - .unwrap(); - new.commit(); + let mut new = Document::new(); + change(&mut new, |new| { + store::write_path( + new, + &meta::path(&[EPOCHS, "1"]), + &json!({ "generation": 1 }), + ) + .unwrap() + }); // The old epoch replaces `_continuum` wholesale and writes a remake record. - store::write_path( - &mut old, - &meta::path(&[]), - &json!({ "forgotten": { "pk": 1 } }), - ) - .unwrap(); - store::write_path(&mut old, &meta::path(&[REMAKES, "c"]), &json!({ "x": 1 })).unwrap(); - old.commit(); + change(&mut old, |old| { + store::write_path(old, &meta::path(&[]), &json!({ "forgotten": { "pk": 1 } })).unwrap(); + store::write_path(old, &meta::path(&[REMAKES, "c"]), &json!({ "x": 1 })).unwrap(); + }); let after = old.get_heads(); - replay(&mut old, &base, &after, &mut new, &Translator::default()); + replay(&old, &base, &after, &mut new, &Translator::default()); let meta = &view(&new)[meta::ROOT]; assert_eq!(meta[EPOCHS]["1"]["generation"], json!(1)); assert_eq!(meta["forgotten"]["pk"], json!(1)); diff --git a/src/remake/kind.rs b/src/remake/kind.rs index aa9e978..03f2ef7 100644 --- a/src/remake/kind.rs +++ b/src/remake/kind.rs @@ -19,8 +19,8 @@ use std::path::Path; +use crate::document::{Document, Transaction}; use anyhow::{Context, anyhow, bail}; -use automerge::AutoCommit; use base64::engine::{Engine, general_purpose::STANDARD as BASE64}; use serde::{Deserialize, Serialize}; use serde_json::Value as JsonValue; @@ -69,9 +69,9 @@ pub(crate) trait RemakeKind: Send + Sync + 'static { /// must not write under `_continuum` (anything there is replaced). fn build( &self, - old_at_base: &AutoCommit, + old_at_base: &Document, content: Option<&[u8]>, - target: &mut AutoCommit, + target: &mut Transaction<'_>, ) -> Result<(), StoreError>; /// Every node: where a user path written after the base goes. @@ -105,7 +105,7 @@ pub(crate) fn by_name(name: &str) -> Option<&'static dyn RemakeKind> { pub(crate) fn remake_genesis( spec: &str, files: &Files, - target: &mut AutoCommit, + target: &mut Document, ) -> anyhow::Result<()> { let (name, path) = match spec.split_once(':') { Some((name, path)) => (name, Some(Path::new(path))), @@ -135,16 +135,16 @@ pub(crate) fn remake_genesis( let prepared = kind .propose(&request) .map_err(|reason| anyhow!("{name}: {reason}"))?; - kind.build(&AutoCommit::new(), prepared.content.as_deref(), target)?; - target.commit(); + let mut transaction = target.transaction(); + kind.build( + &Document::new(), + prepared.content.as_deref(), + &mut transaction, + )?; + transaction.commit(); Ok(()) } -/// Paths owned by the protocol rather than by any kind. -pub(crate) fn is_reserved(path: &[String]) -> bool { - meta::is_protocol(path) -} - /// Compaction: the new document has the old one's content at the base, without its /// history. Writes keep their paths. pub(crate) struct Compact; @@ -168,11 +168,11 @@ impl RemakeKind for Compact { fn build( &self, - old_at_base: &AutoCommit, + old_at_base: &Document, _content: Option<&[u8]>, - target: &mut AutoCommit, + target: &mut Transaction<'_>, ) -> Result<(), StoreError> { - store::materialize_into(old_at_base, target, &is_reserved) + store::materialize_into(old_at_base, target, &meta::is_protocol) } fn translate(&self, _path: &[String], _translation: &JsonValue) -> Translation { @@ -272,9 +272,9 @@ impl RemakeKind for Replace { fn build( &self, - _old_at_base: &AutoCommit, + _old_at_base: &Document, content: Option<&[u8]>, - target: &mut AutoCommit, + target: &mut Transaction<'_>, ) -> Result<(), StoreError> { let content = content.ok_or_else(|| anyhow::anyhow!("replace without content"))?; let document: JsonValue = serde_json::from_slice(content) @@ -311,7 +311,7 @@ fn prepare_rules(specs: Vec<RuleSpec>) -> Result<JsonValue, String> { )); } }; - if is_reserved(&from) || to.as_deref().is_some_and(is_reserved) { + if meta::is_protocol(&from) || to.as_deref().is_some_and(meta::is_protocol) { return Err(format!( "rules must not touch the reserved key {:?}", meta::ROOT @@ -370,11 +370,11 @@ struct ImportSpec { } /// The document in a request (base64 of a saved Automerge document). -fn load_encoded(encoded: &str) -> Result<AutoCommit, String> { +fn load_encoded(encoded: &str) -> Result<Document, String> { let bytes = BASE64 .decode(encoded.trim()) .map_err(|error| format!("the document is not valid base64: {error}"))?; - AutoCommit::load(&bytes) + Document::load(&bytes) .map_err(|error| format!("the document is not an Automerge document: {error}")) } @@ -391,9 +391,11 @@ impl RemakeKind for Import { // Keep only what goes into the new epoch: the merged view without protocol data // and history. Usually much smaller than what was uploaded, and it is what every // voter downloads. - let mut content = AutoCommit::new(); - store::materialize_into(&imported, &mut content, &is_reserved) + let mut content = Document::new(); + let mut transaction = content.transaction(); + store::materialize_into(&imported, &mut transaction, &meta::is_protocol) .map_err(|error| format!("the document can't be read: {error}"))?; + transaction.commit(); Ok(Prepared { content: Some(content.save()), @@ -403,14 +405,14 @@ impl RemakeKind for Import { fn build( &self, - _old_at_base: &AutoCommit, + _old_at_base: &Document, content: Option<&[u8]>, - target: &mut AutoCommit, + target: &mut Transaction<'_>, ) -> Result<(), StoreError> { let content = content.ok_or_else(|| anyhow::anyhow!("import without content"))?; - let imported = AutoCommit::load(content)?; + let imported = Document::load(content)?; // Deterministic: the same bytes give the same view, copied in a fixed order. - store::materialize_into(&imported, target, &is_reserved) + store::materialize_into(&imported, target, &meta::is_protocol) } fn translate(&self, path: &[String], translation: &JsonValue) -> Translation { @@ -439,7 +441,7 @@ impl Translator { /// Where `path` ends up after all steps. Reserved paths always stay put. pub(crate) fn translate(&self, path: &[String]) -> Translation { - if is_reserved(path) { + if meta::is_protocol(path) { return Translation::Same; } let mut current = path.to_vec(); @@ -478,9 +480,9 @@ mod tests { } fn build( &self, - _old: &AutoCommit, + _old: &Document, _content: Option<&[u8]>, - _target: &mut AutoCommit, + _target: &mut Transaction<'_>, ) -> Result<(), StoreError> { Ok(()) } @@ -596,17 +598,20 @@ mod tests { #[test] fn import_keeps_types_and_drops_protocol_data() { + use crate::document::testing::{change, try_change}; use automerge::{ObjType, ROOT, ReadDoc, transaction::Transactable}; use serde_json::json; - let mut source = AutoCommit::new(); - let text = source.put_object(ROOT, "notes", ObjType::Text).unwrap(); - source.splice_text(&text, 0, 0, "hello").unwrap(); - source - .put(ROOT, "hits", automerge::ScalarValue::counter(3)) - .unwrap(); - let meta = source.put_object(ROOT, meta::ROOT, ObjType::Map).unwrap(); - source.put(&meta, "members", "elsewhere").unwrap(); + let mut source = Document::new(); + change(&mut source, |source| { + let text = source.put_object(ROOT, "notes", ObjType::Text).unwrap(); + source.splice_text(&text, 0, 0, "hello").unwrap(); + source + .put(ROOT, "hits", automerge::ScalarValue::counter(3)) + .unwrap(); + let meta = source.put_object(ROOT, meta::ROOT, ObjType::Map).unwrap(); + source.put(&meta, "members", "elsewhere").unwrap(); + }); let encoded = BASE64.encode(source.save()); assert!( @@ -624,15 +629,15 @@ mod tests { .propose(&json!({ "document": encoded, "rules": [{ "from": "", "to": "" }] })) .unwrap(); let build = || { - let mut target = AutoCommit::new().with_actor(automerge::ActorId::from(&b"fixed"[..])); - Import - .build(&AutoCommit::new(), prepared.content.as_deref(), &mut target) - .unwrap(); - target.commit_with(automerge::transaction::CommitOptions::default().with_time(0)); + let mut target = Document::new().with_actor(automerge::ActorId::from(&b"fixed"[..])); + try_change(&mut target, |target| { + Import.build(&Document::new(), prepared.content.as_deref(), target) + }) + .unwrap(); target }; - let mut a = build(); - let mut b = build(); + let a = build(); + let b = build(); assert_eq!(a.get_heads(), b.get_heads()); assert!(a.get(ROOT, meta::ROOT).unwrap().is_none()); @@ -653,14 +658,15 @@ mod tests { #[test] fn replace_builds_the_document_deterministically() { + use crate::document::testing::try_change; use serde_json::json; let build = |document: serde_json::Value| { let prepared = replace(json!({ "document": document })).unwrap(); - let mut target = AutoCommit::new().with_actor(automerge::ActorId::from(&b"fixed"[..])); - Replace - .build(&AutoCommit::new(), prepared.content.as_deref(), &mut target) - .unwrap(); - target.commit_with(automerge::transaction::CommitOptions::default().with_time(0)); + let mut target = Document::new().with_actor(automerge::ActorId::from(&b"fixed"[..])); + try_change(&mut target, |target| { + Replace.build(&Document::new(), prepared.content.as_deref(), target) + }) + .unwrap(); ( target.get_heads(), store::read_path(&target, &[]).unwrap().unwrap(), diff --git a/src/remake/local.rs b/src/remake/local.rs index 7810690..c80a5a5 100644 --- a/src/remake/local.rs +++ b/src/remake/local.rs @@ -9,7 +9,6 @@ use std::collections::{HashMap, HashSet}; use std::sync::{Arc, Mutex, MutexGuard}; use std::time::Duration; -use automerge::{ActorId, AutoCommit}; use serde::Serialize; use tokio::time::Instant; @@ -17,6 +16,7 @@ use super::records::Retention; use super::snapshot::Snapshot; use crate::clock::UnixMillis; use crate::document::replica::Epoch; +use crate::document::{ActorId, ChangeHash, Document}; use crate::locks::MutexExt; pub(crate) struct RemakeState { @@ -25,6 +25,8 @@ pub(crate) struct RemakeState { started_at: UnixMillis, // A std mutex: every access is short and never spans an await. inner: Mutex<Inner>, + /// A download of content ended (`end_content_fetch`). + content_fetch_ended: tokio::sync::Notify, } impl RemakeState { @@ -33,6 +35,7 @@ impl RemakeState { Self { started_at, inner: Mutex::default(), + content_fetch_ended: tokio::sync::Notify::new(), } } } @@ -60,6 +63,9 @@ struct Inner { coordinating: HashSet<String>, /// The remake this node is about to vote for, and when (`vote_due`). vote_due: Option<(String, UnixMillis)>, + /// Remakes this node could have voted for but saw expire first (`vote::cast`), each + /// logged once. + missed: HashSet<String>, /// The actors this node writes or wrote with in the current document: this run's, /// and those of earlier runs it knows of (`--document-file`). Its own writes, for a /// rejoin. A switch or rejoin starts the list over with the new document's actor, so @@ -84,8 +90,13 @@ pub(crate) struct CatchUp { pub(crate) struct Rejoin { pub(crate) previous: Epoch, pub(crate) target: Epoch, - /// The document of `previous`, until it is replayed, quarantined or restored. - pub(crate) old: AutoCommit, + /// The document of `previous`, until it is replayed, quarantined or restored; of a + /// node that was ready, kept by the node instead, where it takes its writes + /// meanwhile (`Host::begin_rejoin`). + pub(crate) old: Option<Document>, + /// That document's heads as the rejoin began: the changes after them it took + /// during the rejoin. + pub(crate) heads: Vec<ChangeHash>, pub(crate) started: Instant, /// The node served requests before; it does again once the rejoin is over. pub(crate) was_ready: bool, @@ -102,7 +113,7 @@ pub(crate) struct Quarantined { /// Store paths the changes touched (minimal, prefix-free). pub(crate) paths: Vec<Vec<String>>, /// The old document; `paths` are read from its merged view. - pub(crate) document: AutoCommit, + pub(crate) document: Document, } #[derive(Clone, Debug, Serialize)] @@ -138,7 +149,7 @@ struct Archived { epoch: Epoch, // Kept for auditing and recovery by hand; not read by the protocol. #[allow(dead_code)] - doc: AutoCommit, + doc: Document, archived_at: UnixMillis, retention: Retention, } @@ -215,6 +226,12 @@ impl RemakeState { } } + /// Notes that this node saw `remake` expire without having voted for it: true the + /// first time. + pub(crate) fn missed(&self, remake: &str) -> bool { + self.lock().missed.insert(remake.to_owned()) + } + fn lock(&self) -> MutexGuard<'_, Inner> { self.inner.lock_or_recover() } @@ -249,7 +266,7 @@ impl RemakeState { pub(crate) fn archive( &self, epoch: Epoch, - doc: AutoCommit, + doc: Document, retention: Retention, current_generation: u64, now: UnixMillis, @@ -325,10 +342,19 @@ impl RemakeState { pub(crate) fn end_content_fetch(&self, hash: &[u8; 32]) { self.lock().content_fetches.remove(hash); + self.content_fetch_ended.notify_waiters(); } - pub(crate) fn content_fetch_running(&self, hash: &[u8; 32]) -> bool { - self.lock().content_fetches.contains(hash) + /// Waits until no download of `hash` runs. + pub(crate) async fn content_fetch_ended(&self, hash: &[u8; 32]) { + loop { + // Before looking, so that an end in between isn't missed. + let ended = self.content_fetch_ended.notified(); + if !self.lock().content_fetches.contains(hash) { + return; + } + ended.await; + } } /// Drops content that no remake in `referenced` needs, once its grace has passed. @@ -431,7 +457,7 @@ mod tests { // Kept by epoch count only. remake.archive( epoch(0), - AutoCommit::new(), + Document::new(), Retention { epochs: Some(1), age_ms: None, @@ -446,7 +472,7 @@ mod tests { // Kept by time while the epoch count has passed. remake.archive( epoch(1), - AutoCommit::new(), + Document::new(), Retention { epochs: Some(0), age_ms: Some(3_600_000), @@ -459,7 +485,7 @@ mod tests { // No limits at all: not kept. remake.archive( epoch(2), - AutoCommit::new(), + Document::new(), Retention { epochs: None, age_ms: None, diff --git a/src/remake/mod.rs b/src/remake/mod.rs index ca7f5a0..9351cc5 100644 --- a/src/remake/mod.rs +++ b/src/remake/mod.rs @@ -5,7 +5,9 @@ //! where writes made after the base go): `compact`, `replace` and `import`. //! * `snapshot`: deterministic build of the next epoch's initial document. //! * `records`: the replicated remake records under `_continuum/remakes/<id>`. -//! * `coordinator`: proposal, vote count and decision on the node that was asked. +//! * `proposal`: writing a proposal (its checks, superseding expired remakes), and when +//! a node other than the coordinator may abort an expired one. +//! * `coordinator`: vote count and decision on the node that was asked. //! * `vote`: who votes, for which remake, and when a remake has its majority. //! * `content`: the new document of a `replace`/`import` remake, fetched by every voter. //! * `switch`: switching this node to a committed epoch (worker). @@ -20,12 +22,15 @@ //! depends on `AppState`, peers or sync internals. mod api; +#[cfg(test)] +mod bench; mod catchup; mod content; mod coordinator; mod delta; mod kind; mod local; +mod proposal; mod quarantine; mod records; pub(crate) mod snapshot; @@ -34,6 +39,7 @@ mod vote; #[cfg(test)] pub(crate) use api::testing; +pub(crate) use delta::Superseded; pub(crate) use kind::remake_genesis; pub(crate) use local::RemakeState; pub(crate) use switch::run_worker; @@ -42,7 +48,6 @@ use std::future::Future; use std::sync::Arc; use std::time::Duration; -use automerge::AutoCommit; use axum::{ Router, extract::DefaultBodyLimit, @@ -51,7 +56,8 @@ use axum::{ use tokio::sync::broadcast; use crate::document::replica::Epoch; -use crate::swarm::sync::ChangeSet; +use crate::document::store::{ChangeSet, StoreError}; +use crate::document::{Document, Transaction}; /// Remakes and their aftermath on this node, in numbers (`/admin/stats`). #[derive(Debug, Clone, Copy, Default)] @@ -71,7 +77,7 @@ pub(crate) struct Summary { } /// The `Summary` of `remake` with the records of `doc`, whose epoch is `epoch`. -pub(crate) fn summary(remake: &RemakeState, doc: &AutoCommit, epoch: Epoch) -> Summary { +pub(crate) fn summary(remake: &RemakeState, doc: &Document, epoch: Epoch) -> Summary { let local = records::EpochRecord::from(epoch); let mut summary = Summary { rejoining: remake.is_rejoining(), @@ -120,16 +126,24 @@ pub(crate) trait Host: Clone + Send + Sync + 'static { /// Default basis for the voting period, and the grace margin after it. fn dormant_after(&self) -> Duration; + /// This node's clock against its peers' (`swarm::skew`): a coordinator judges the + /// remake's deadlines by it. + fn clock_skew(&self) -> impl Future<Output = crate::swarm::skew::Skew> + Send; + /// The electorate of a remake proposed now (`vote`): the members that said they are /// alive recently enough, read from `doc`. - fn electorate(&self, doc: &AutoCommit, now: crate::clock::UnixMillis) -> Vec<String>; + fn electorate( + &self, + doc: &impl crate::document::ReadDoc, + now: crate::clock::UnixMillis, + ) -> Vec<String>; /// Downloads a change from peers on the local epoch, in chunks from several at once, /// checked against its hash. fn fetch_change( &self, - hash: automerge::ChangeHash, - ) -> impl Future<Output = anyhow::Result<automerge::Change>> + Send; + hash: crate::document::ChangeHash, + ) -> impl Future<Output = anyhow::Result<crate::document::Change>> + Send; /// Downloads content of `size` bytes by its BLAKE3 hash from peers on the local /// epoch, checked against it. @@ -147,16 +161,44 @@ pub(crate) trait Host: Clone + Send + Sync + 'static { /// Known (not forgotten) peers. fn peer_summaries(&self) -> impl Future<Output = Vec<PeerSummary>> + Send; - /// Stops serving requests (back to `Joining`, keeping the swarm ID), as while a - /// rejoin syncs a fresh document. Returns whether the node was serving. - fn suspend(&self) -> impl Future<Output = bool> + Send; + /// Starts a rejoin of `target`, under the document's write lock: the live document + /// becomes an empty one of `target`, written by `actor`, that takes no writes until + /// the rejoin ends. A ready node stops being ready (back to `Joining`, keeping the + /// swarm ID), and its document so far answers reads (marked stale) and takes its + /// writes meanwhile; a node that wasn't ready gets its document back. Afterwards + /// the node resets every peer's sync state and publishes the change. + fn begin_rejoin( + &self, + target: Epoch, + actor: crate::document::ActorId, + ) -> impl Future<Output = BegunRejoin> + Send; + + /// A copy of the document before the rejoin, with the writes it took since, if the + /// node keeps it ([`Host::begin_rejoin`]: it was ready). + fn rejoin_document(&self) -> impl Future<Output = Option<Document>> + Send; - /// Serves requests again after [`Host::suspend`]. - fn resume(&self) -> impl Future<Output = ()> + Send; + /// Ends a rejoin on its target, under the write lock: `f` writes into the live + /// document from the one before the rejoin (the node's, or `old` where it keeps + /// none), as many changes as it likes; then writes go to the live document again, + /// and a node that was ready is again. Returns what `f` returned and the document + /// before the rejoin. + fn finish_rejoin<T, F>( + &self, + old: Option<Document>, + f: F, + ) -> impl Future<Output = (T, Document)> + Send + where + F: FnOnce(&mut Document, &Document) -> T + Send + 'static, + T: Send + 'static; - /// Serves reads from this document (read-only, marked stale) while the node is not - /// ready; `None` ends it. - fn set_stale_view(&self, view: Option<(Epoch, AutoCommit)>); + /// Ends a rejoin by going back, under the write lock: the document before it (the + /// node's, with the writes it took meanwhile, or `old`) and `previous` are live + /// again, and a node that was ready is again. Afterwards as [`Host::begin_rejoin`]. + fn abandon_rejoin( + &self, + previous: Epoch, + old: Option<Document>, + ) -> impl Future<Output = ()> + Send; /// Whether the document caught up with a peer since the last reset. fn initial_sync_done(&self) -> bool; @@ -170,15 +212,27 @@ pub(crate) trait Host: Clone + Send + Sync + 'static { /// Reads the document on the blocking pool. fn read_doc<T, F>(&self, f: F) -> impl Future<Output = T> + Send where - F: FnOnce(&AutoCommit) -> T + Send + 'static, + F: FnOnce(&Document) -> T + Send + 'static, T: Send + 'static; - /// One committed change (rolled back on error), published and replicated. + /// One committed change (rolled back on error), published and replicated. While a + /// rejoin runs, into the document before it, or refused (`StoreError::Rejoining`) + /// by a node that keeps none (`AppState::mutate_doc`). fn commit_doc<T, E, F>(&self, f: F) -> impl Future<Output = Result<T, E>> + Send where - F: FnOnce(&mut AutoCommit) -> Result<T, E> + Send + 'static, + F: FnOnce(&mut Transaction<'_>) -> Result<T, E> + Send + 'static, T: Send + 'static, - E: Send + 'static; + E: From<StoreError> + Send + 'static; + + /// What a replay does with writes a newer one superseded (`--superseded-writes`). + fn superseded(&self) -> Superseded; + + /// Applies a change downloaded from peers (`fetch_change`), then publishes it like + /// a change received by sync. + fn apply_change( + &self, + change: crate::document::Change, + ) -> impl Future<Output = Result<(), String>> + Send; /// Replaces the document and its epoch atomically. Under the write lock, `f` gets /// the live document and returns the new one. Afterwards the node resets every @@ -187,12 +241,23 @@ pub(crate) trait Host: Clone + Send + Sync + 'static { &self, epoch: Epoch, f: F, - ) -> impl Future<Output = Result<AutoCommit, E>> + Send + ) -> impl Future<Output = Result<Document, E>> + Send where - F: FnOnce(&mut AutoCommit) -> Result<AutoCommit, E> + Send + 'static, + F: FnOnce(&Document) -> Result<Document, E> + Send + 'static, E: Send + 'static; } +/// What [`Host::begin_rejoin`] left the rejoin with. +pub(crate) struct BegunRejoin { + /// The node was ready, and is again once the rejoin ends. + pub(crate) was_ready: bool, + /// The heads of the document before the rejoin as it began: what changes after + /// them, it took during the rejoin. + pub(crate) heads: Vec<crate::document::ChangeHash>, + /// The document before the rejoin, where the node doesn't keep it (it wasn't ready). + pub(crate) old: Option<Document>, +} + #[derive(Clone, Debug)] pub(crate) struct PeerSummary { pub(crate) id: String, @@ -222,12 +287,12 @@ impl PeerSummary { } } -/// Admin routes. Merged into the authenticated part of the node's router. -/// Largest remake request (bytes). The proposal carries the new document to every -/// node in one sync message, which the transport must deliver within its delivery -/// limit, so documents near this size are already a stretch. +/// Largest remake request (bytes). The coordinator holds the new document in memory +/// and every member downloads it in chunks before voting (`content`), so documents near +/// this size cost every node that much memory, and the swarm that many bytes per node. const REMAKE_REQUEST_LIMIT: usize = 64 * 1024 * 1024; +/// Admin routes. Merged into the authenticated part of the node's router. pub(crate) fn router<H: Host>() -> Router<H> { Router::new() .route( diff --git a/src/remake/proposal.rs b/src/remake/proposal.rs new file mode 100644 index 0000000..7695b4c --- /dev/null +++ b/src/remake/proposal.rs @@ -0,0 +1,260 @@ +//! Proposing a remake: the checks (one open remake per epoch, the proposer free to +//! vote, a known base), superseding expired remakes, the electorate and quorum, and the +//! proposal and own vote written in one change; then its coordinator. And when a node +//! other than the coordinator may abort an expired remake. + +use std::collections::HashSet; + +use tracing::info; + +use super::coordinator; +use super::records::{self, Decision, Phase, Proposal, Record, Vote}; +use super::{Host, kind, vote}; +use crate::clock::UnixMillis; +use crate::document::ChangeHash; +use crate::document::store::StoreError; +use crate::force::Force; + +/// When a node other than the coordinator may abort an expired remake: a whole grace +/// period after it expired. Its coordinator stops committing at expiry +/// (`coordinator::decide`), so an abort from elsewhere and a commit can only cross if +/// their clocks are more than a grace period apart. +pub(super) fn abortable_at(record: &Record, grace: u64) -> UnixMillis { + record + .proposal + .settle_until + .plus_ms(record.grace(grace).saturating_mul(2)) +} + +/// Why this node may not abort expired `record` now, and the `force` name that would +/// override it (none for one that only time cures). +pub(super) fn abort_obstacle( + record: &Record, + me: &str, + active_peers: &HashSet<String>, + force: &Force, + now: UnixMillis, + grace: u64, +) -> Option<(String, Option<&'static str>)> { + let at = abortable_at(record, grace); + if now < at { + return Some(( + format!( + "only its coordinator may abort it until a grace period after it expired ({}s from now)", + at.ms_since(now).div_ceil(1000) + ), + None, + )); + } + let coordinator = &record.proposal.coordinator; + if coordinator != me && active_peers.contains(coordinator) && !force.allows("coordinatorActive") + { + return Some(( + format!( + "its coordinator {coordinator} is an active peer, so it is stuck rather than gone (look at its log); add force=coordinatorActive to abort it anyway" + ), + Some("coordinatorActive"), + )); + } + None +} + +/// The active peers this node sees. +pub(super) async fn active_peers<H: Host>(host: &H) -> HashSet<String> { + host.peer_summaries() + .await + .into_iter() + .filter(|peer| peer.active) + .map(|peer| peer.id) + .collect() +} + +/// An expired remake of the epoch a proposal would supersede. +#[derive(Debug)] +pub(super) struct ExpiredRemake { + pub(super) id: String, + pub(super) coordinator: String, + pub(super) coordinator_active: bool, + /// When it expired. + pub(super) since: UnixMillis, + /// When another node may abort it (`abortable_at`). + pub(super) abortable_at: UnixMillis, + pub(super) tally: vote::Tally, +} + +/// Superseding expired remakes, as the proposal's `force` asks. +pub(super) struct Supersede { + pub(super) force: Force, + pub(super) active_peers: HashSet<String>, +} + +#[derive(Debug, thiserror::Error)] +pub(super) enum StartError { + #[error("remake {0} is already open for this epoch")] + Open(String), + /// Expired remakes still hold votes, and aren't superseded: not asked to + /// (`force=supersede`), or not possible yet (`abort_obstacle`). + #[error("{message}")] + Expired { + message: String, + remakes: Vec<ExpiredRemake>, + /// The `force` name that would override it, if any. + force: Option<&'static str>, + }, + /// The proposer votes for its own remake, so it must be free to. + #[error("this node can't vote for a new remake: {0}")] + VoteHeld(String), + #[error("base head {0} is not in this node's document")] + UnknownBase(String), + #[error(transparent)] + Store(#[from] StoreError), +} + +/// Writes the proposal (unless another one is open for the epoch) and starts its +/// coordinator. Runs as its own task, independent of the HTTP request. +pub(super) async fn propose<H: Host>( + host: H, + id: String, + mut proposal: Proposal, + base: Option<Vec<ChangeHash>>, + supersede: Supersede, + now: UnixMillis, + grace: u64, +) -> Result<Record, StartError> { + let record_id = id.clone(); + let electorate_host = host.clone(); + let me = host.node_id(); + let started = host.remake().started_at().plus_ms(grace); + let base_at_commit = kind::by_name(&proposal.kind).is_some_and(|kind| kind.base_at_commit()); + let record = host + .commit_doc(move |doc| -> Result<Record, StartError> { + let mut all = records::load_all(doc); + if let Some((open, _)) = all.iter().find(|(_, record)| { + record.proposal.from_epoch == proposal.from_epoch + && record.phase(now, grace).is_open() + }) { + return Err(StartError::Open(open.clone())); + } + let mut register = records::load_register(doc); + + // Expired remakes hold their votes; their coordinators can't commit them + // any more, so they may be aborted here. + let expired: Vec<&String> = all + .iter() + .filter(|(_, record)| { + record.proposal.from_epoch == proposal.from_epoch + && record.phase(now, grace) == Phase::Expired + }) + .map(|(id, _)| id) + .collect(); + if !expired.is_empty() { + let remakes = || { + all.iter() + .filter(|(id, _)| expired.contains(&id)) + .map(|(id, record)| ExpiredRemake { + id: id.clone(), + coordinator: record.proposal.coordinator.clone(), + coordinator_active: supersede + .active_peers + .contains(&record.proposal.coordinator), + since: record.proposal.settle_until.plus_ms(record.grace(grace)), + abortable_at: abortable_at(record, grace), + tally: vote::tally(id, record, &all, ®ister), + }) + .collect() + }; + if !supersede.force.allows("supersede") { + return Err(StartError::Expired { + message: "an expired remake of this epoch still holds votes: abort it, \ + or add ?force=supersede to abort it and propose in its place" + .to_owned(), + remakes: remakes(), + force: Some("supersede"), + }); + } + let obstacle = + all.iter() + .filter(|(id, _)| expired.contains(&id)) + .find_map(|(id, record)| { + let (why, force) = abort_obstacle( + record, + &me, + &supersede.active_peers, + &supersede.force, + now, + grace, + )?; + Some((format!("can't supersede remake {id}: {why}"), force)) + }); + if let Some((message, force)) = obstacle { + return Err(StartError::Expired { + message, + remakes: remakes(), + force, + }); + } + let decision = Decision::Aborted { + reason: format!("superseded by remake {record_id}"), + decided_at: now, + }; + for id in expired { + records::write_decision(doc, id, &decision)?; + info!(remake = %id, by = %record_id, "Superseded expired remake"); + } + all = records::load_all(doc); + register = records::load_register(doc); + } + + match vote::freedom(&me, &proposal.from_epoch, &all, ®ister, started) { + vote::Freedom::Any => {} + vote::Freedom::Only(id) => { + return Err(StartError::VoteHeld(format!( + "it may have voted for remake {id} before its start" + ))); + } + vote::Freedom::Held(reason) => return Err(StartError::VoteHeld(reason)), + } + + // The base is the requested heads, or everything before the proposal (the + // proposal itself is never in it); or, for a compaction, left to the commit. + if !base_at_commit { + let base = match base { + Some(base) => { + if let Some(unknown) = crate::document::changes::missing(doc, &base).first() + { + return Err(StartError::UnknownBase(unknown.to_string())); + } + base + } + None => doc.get_heads(), + }; + proposal.base_heads = Some(records::encode_heads(&base)); + } + + // We vote for our own remake, so we are always in the electorate. + let mut electorate = electorate_host.electorate(doc, now); + if let Err(position) = electorate.binary_search(&me) { + electorate.insert(position, me.clone()); + } + proposal.quorum = vote::quorum(electorate.len()); + proposal.electorate = electorate; + + records::write_proposal(doc, &record_id, &proposal)?; + let own_vote = Vote { + remake: record_id.clone(), + at: now, + }; + records::write_vote(doc, &me, &own_vote)?; + Ok(Record { + proposal, + decision: None, + abort_requested: None, + contested: false, + votes: [(me, own_vote)].into(), + }) + }) + .await?; + + tokio::spawn(coordinator::run(host, id)); + Ok(record) +} diff --git a/src/remake/quarantine.rs b/src/remake/quarantine.rs index 5300f54..bb34d73 100644 --- a/src/remake/quarantine.rs +++ b/src/remake/quarantine.rs @@ -5,16 +5,16 @@ //! * `GET /admin/quarantine/{id}` shows the writes: every changed path with the value //! the old document's merged view holds there (`deleted` if none). //! * `GET /admin/quarantine/{id}/document` downloads the old document -//! (`AutoCommit::save()`), for inspection with other tools. +//! (`Document::save()`), for inspection with other tools. //! * `POST /admin/quarantine/{id}/replay` writes them into the current epoch anyway (as //! fresh writes, so they win over what is there) and removes the entry. //! * `DELETE /admin/quarantine/{id}` discards the entry. //! //! Quarantine lives in memory on the node that holds it. -use std::convert::Infallible; +use std::sync::Arc; -use automerge::AutoCommit; +use crate::document::Document; use axum::{ Json, extract::{Path, State}, @@ -33,7 +33,7 @@ use super::local::Quarantined; use crate::api::error_response; use crate::clock::UnixMillis; use crate::document::replica::Epoch; -use crate::document::store; +use crate::document::store::{self, StoreError}; #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] @@ -69,7 +69,7 @@ pub(super) fn add<H: Host>( host: &H, epoch: Epoch, target: Epoch, - document: &AutoCommit, + document: &Document, quarantined: Vec<(Vec<String>, String)>, ) -> Option<String> { if quarantined.is_empty() { @@ -132,7 +132,7 @@ pub(crate) async fn get<H: Host>(State(host): State<H>, Path(id): Path<String>) } pub(crate) async fn document<H: Host>(State(host): State<H>, Path(id): Path<String>) -> Response { - let Some(mut document) = host + let Some(document) = host .remake() .with_quarantined(&id, |entry| entry.document.clone()) else { @@ -167,15 +167,30 @@ pub(crate) async fn replay<H: Host>(State(host): State<H>, Path(id): Path<String return not_found(); }; - let (stats, entry) = host - .commit_doc(move |doc| { + let entry = Arc::new(entry); + let replayed = { + let entry = Arc::clone(&entry); + host.commit_doc(move |doc| { // As they are: the operator decided they belong at their old paths. - let stats = - delta::replay_paths(&entry.document, doc, &entry.paths, &Translator::default()); - Ok::<_, Infallible>((stats, entry)) + Ok::<_, StoreError>(delta::replay_paths( + &entry.document, + doc, + &entry.paths, + &Translator::default(), + )) }) .await - .unwrap_or_else(|never| match never {}); + }; + // The commit's task held the only other reference, and it is done. + let entry = Arc::into_inner(entry).expect("the replay's reference is gone"); + let stats = match replayed { + Ok(stats) => stats, + Err(error) => { + // Rejoining since the check above: keep the entry for later. + host.remake().restore_quarantined(entry); + return error_response(StatusCode::SERVICE_UNAVAILABLE, &error.to_string(), None); + } + }; if stats.failed > 0 && stats.replayed == 0 { // Nothing went in; keep the entry for another look. diff --git a/src/remake/records.rs b/src/remake/records.rs index 720aea1..0c4c444 100644 --- a/src/remake/records.rs +++ b/src/remake/records.rs @@ -28,7 +28,6 @@ use std::collections::BTreeMap; -use automerge::{AutoCommit, ChangeHash}; use serde::{Deserialize, Serialize}; use serde_json::Value as JsonValue; use tracing::warn; @@ -39,28 +38,14 @@ use crate::clock::UnixMillis; use crate::document::meta::{self, REMAKES, VOTES}; use crate::document::replica::Epoch; use crate::document::store::{self, StoreError}; +use crate::document::{ChangeHash, ReadDoc, Transactable}; /// One day, for retention defaults. const DAY_MS: u64 = 24 * 60 * 60 * 1000; // --- record types ------------------------------------------------------------ -/// An epoch in a record: generation plus hex-encoded snapshot hash. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct EpochRecord { - pub(crate) generation: u64, - pub(crate) id: String, -} - -impl From<Epoch> for EpochRecord { - fn from(epoch: Epoch) -> Self { - Self { - generation: epoch.generation, - id: epoch.id_hex(), - } - } -} +pub(crate) use crate::document::replica::EpochRecord; impl EpochRecord { pub(crate) fn to_epoch(&self) -> Option<Epoch> { @@ -137,6 +122,11 @@ pub(crate) struct Proposal { /// interval if absent. #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) switch_delay_ms: Option<u64>, + /// The coordinator's grace period (ms, its `dormant_after`): every node judges this + /// remake's expiry, and when others may abort it, by it, whatever its own keepalive + /// (`Record::grace`). Absent in records from before: each node's own. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) grace_ms: Option<u64>, /// Members whose votes count (sorted), fixed at proposal time (`vote`). pub(crate) electorate: Vec<String>, /// Votes needed to commit: a majority of the electorate. @@ -326,14 +316,23 @@ impl Record { }) } - pub(crate) fn phase(&self, now: UnixMillis, grace_ms: u64) -> Phase { + /// The phase at `now`, expiry judged by the record's grace period (`grace`; `own`, + /// this node's, for a record without one). + pub(crate) fn phase(&self, now: UnixMillis, own: u64) -> Phase { match &self.decision { Some(Decision::Committed { .. }) => Phase::Committed, Some(Decision::Aborted { .. }) => Phase::Aborted, - None if now > self.proposal.settle_until.plus_ms(grace_ms) => Phase::Expired, + None if now > self.proposal.settle_until.plus_ms(self.grace(own)) => Phase::Expired, None => Phase::Proposed, } } + + /// The grace period this remake is judged by (ms): its coordinator's, the same on + /// every node, so nodes with different keepalives agree on when it expired; `own` + /// for a record from before it was written down. + pub(crate) fn grace(&self, own: u64) -> u64 { + self.proposal.grace_ms.unwrap_or(own) + } } // --- document access --------------------------------------------------------- @@ -352,13 +351,13 @@ pub(crate) fn record_path(id: &str, part: &str) -> Vec<String> { } /// The record with this ID, if it exists and is readable, with its votes. -pub(crate) fn load(doc: &AutoCommit, id: &str) -> Option<Record> { +pub(crate) fn load(doc: &impl ReadDoc, id: &str) -> Option<Record> { let record = load_record(doc, id)?; Some(with_votes(id, record, &load_register(doc))) } /// All readable records, with their votes. -pub(crate) fn load_all(doc: &AutoCommit) -> Vec<(String, Record)> { +pub(crate) fn load_all(doc: &impl ReadDoc) -> Vec<(String, Record)> { let register = load_register(doc); store::keys_at(doc, &records_dir()) .unwrap_or_default() @@ -381,7 +380,7 @@ fn with_votes(id: &str, mut record: Record, register: &Register) -> Record { record } -fn load_record(doc: &AutoCommit, id: &str) -> Option<Record> { +fn load_record(doc: &impl ReadDoc, id: &str) -> Option<Record> { let value = match store::read_path(doc, &record_path(id, "")) { Ok(Some(value)) => value, Ok(None) => return None, @@ -405,7 +404,7 @@ fn load_record(doc: &AutoCommit, id: &str) -> Option<Record> { } /// A commit among concurrent decisions of remake `id` that disagree, if they do. -fn contested_decision(doc: &AutoCommit, id: &str) -> Option<Decision> { +fn contested_decision(doc: &impl ReadDoc, id: &str) -> Option<Decision> { let decisions: Vec<Decision> = store::read_candidates(doc, &record_path(id, "decision")) .ok()? .into_iter() @@ -433,7 +432,7 @@ fn register_path(voter: &str) -> Vec<String> { } /// The votes register; unreadable entries are skipped with a warning. -pub(crate) fn load_register(doc: &AutoCommit) -> Register { +pub(crate) fn load_register(doc: &impl ReadDoc) -> Register { store::keys_at(doc, &meta::path(&[VOTES])) .unwrap_or_default() .into_iter() @@ -461,7 +460,7 @@ pub(crate) fn load_register(doc: &AutoCommit) -> Register { } fn write_part<T: Serialize>( - doc: &mut AutoCommit, + doc: &mut impl Transactable, id: &str, part: &str, value: &T, @@ -471,7 +470,7 @@ fn write_part<T: Serialize>( } pub(crate) fn write_proposal( - doc: &mut AutoCommit, + doc: &mut impl Transactable, id: &str, proposal: &Proposal, ) -> Result<(), StoreError> { @@ -479,7 +478,7 @@ pub(crate) fn write_proposal( } pub(crate) fn write_decision( - doc: &mut AutoCommit, + doc: &mut impl Transactable, id: &str, decision: &Decision, ) -> Result<(), StoreError> { @@ -487,7 +486,7 @@ pub(crate) fn write_decision( } pub(crate) fn write_abort_request( - doc: &mut AutoCommit, + doc: &mut impl Transactable, id: &str, request: &AbortRequest, ) -> Result<(), StoreError> { @@ -495,7 +494,11 @@ pub(crate) fn write_abort_request( } /// Writes `voter`'s entry in the votes register (replacing its earlier one). -pub(crate) fn write_vote(doc: &mut AutoCommit, voter: &str, vote: &Vote) -> Result<(), StoreError> { +pub(crate) fn write_vote( + doc: &mut impl Transactable, + voter: &str, + vote: &Vote, +) -> Result<(), StoreError> { let json = serde_json::to_value(vote).map_err(anyhow::Error::from)?; store::write_path(doc, ®ister_path(voter), &json) } @@ -514,6 +517,8 @@ pub(crate) fn decode_hex32(hex: &str) -> Option<[u8; 32]> { #[cfg(test)] mod tests { use super::*; + use crate::document::Document; + use crate::document::testing::change; #[test] fn retention_covers_until_every_limit_passed() { @@ -573,6 +578,7 @@ mod tests { proposed_at: UnixMillis(settle_until.saturating_sub(1_000)), settle_until: UnixMillis(settle_until), switch_delay_ms: None, + grace_ms: None, electorate: vec!["node-a".into(), "node-b".into()], quorum: 2, kind: DEFAULT_KIND.to_owned(), @@ -583,18 +589,8 @@ mod tests { #[test] fn round_trip_through_the_document() { - let mut doc = AutoCommit::new(); + let mut doc = Document::new(); let p = proposal(10_000); - write_proposal(&mut doc, "c-1", &p).unwrap(); - write_abort_request( - &mut doc, - "c-1", - &AbortRequest { - by: "x".into(), - at: UnixMillis(5), - }, - ) - .unwrap(); let decision = Decision::Committed { to_epoch: EpochRecord { generation: 1, @@ -604,14 +600,25 @@ mod tests { snapshot_bytes: 123, decided_at: UnixMillis(11_000), }; - write_decision(&mut doc, "c-1", &decision).unwrap(); let vote = |remake: &str| Vote { remake: remake.into(), at: UnixMillis(7), }; - write_vote(&mut doc, "node-b", &vote("c-1")).unwrap(); - write_vote(&mut doc, "node-c", &vote("other")).unwrap(); - doc.commit(); + change(&mut doc, |doc| { + write_proposal(doc, "c-1", &p).unwrap(); + write_abort_request( + doc, + "c-1", + &AbortRequest { + by: "x".into(), + at: UnixMillis(5), + }, + ) + .unwrap(); + write_decision(doc, "c-1", &decision).unwrap(); + write_vote(doc, "node-b", &vote("c-1")).unwrap(); + write_vote(doc, "node-c", &vote("other")).unwrap(); + }); let all = load_all(&doc); assert_eq!(all.len(), 1); @@ -627,19 +634,16 @@ mod tests { #[test] fn concurrent_votes_are_kept_apart() { - let mut a = AutoCommit::new(); + let mut a = Document::new(); let vote = |remake: &str, at: u64| Vote { remake: remake.into(), at: UnixMillis(at), }; - write_vote(&mut a, "node-a", &vote("x", 1)).unwrap(); - a.commit(); + change(&mut a, |a| write_vote(a, "node-a", &vote("x", 1)).unwrap()); let mut b = a.fork(); // node-b votes in both, never seeing its other vote (a lost vote). - write_vote(&mut a, "node-b", &vote("x", 2)).unwrap(); - write_vote(&mut b, "node-b", &vote("y", 3)).unwrap(); - a.commit(); - b.commit(); + change(&mut a, |a| write_vote(a, "node-b", &vote("x", 2)).unwrap()); + change(&mut b, |b| write_vote(b, "node-b", &vote("y", 3)).unwrap()); a.merge(&mut b).unwrap(); let register = load_register(&a); @@ -649,8 +653,7 @@ mod tests { assert_eq!(twice, [vote("x", 2), vote("y", 3)]); // Seen, a vote is replaced. - write_vote(&mut a, "node-b", &vote("z", 4)).unwrap(); - a.commit(); + change(&mut a, |a| write_vote(a, "node-b", &vote("z", 4)).unwrap()); assert_eq!(load_register(&a)["node-b"], [vote("z", 4)]); } @@ -666,6 +669,10 @@ mod tests { assert_eq!(record.phase(UnixMillis(10_000), 500), Phase::Proposed); assert_eq!(record.phase(UnixMillis(10_500), 500), Phase::Proposed); assert_eq!(record.phase(UnixMillis(10_501), 500), Phase::Expired); + // The coordinator's grace period, written down, wins over this node's. + record.proposal.grace_ms = Some(2_000); + assert_eq!(record.phase(UnixMillis(10_501), 500), Phase::Proposed); + assert_eq!(record.phase(UnixMillis(12_001), 500), Phase::Expired); record.decision = Some(Decision::Aborted { reason: "test".into(), decided_at: UnixMillis(1), @@ -700,9 +707,10 @@ mod tests { #[test] fn disagreeing_decisions_are_contested() { - let mut base = AutoCommit::new(); - write_proposal(&mut base, "c-1", &proposal(10_000)).unwrap(); - base.commit(); + let mut base = Document::new(); + change(&mut base, |base| { + write_proposal(base, "c-1", &proposal(10_000)).unwrap() + }); let (mut a, mut b, mut c) = (base.fork(), base.fork(), base.fork()); let committed = Decision::Committed { to_epoch: EpochRecord { @@ -713,21 +721,20 @@ mod tests { snapshot_bytes: 1, decided_at: UnixMillis(2), }; - write_decision(&mut a, "c-1", &committed).unwrap(); - write_decision( - &mut b, - "c-1", - &Decision::Aborted { - reason: "test".into(), - decided_at: UnixMillis(3), - }, - ) - .unwrap(); + change(&mut a, |a| write_decision(a, "c-1", &committed).unwrap()); + change(&mut b, |b| { + write_decision( + b, + "c-1", + &Decision::Aborted { + reason: "test".into(), + decided_at: UnixMillis(3), + }, + ) + .unwrap() + }); // The same decision, concurrently: they agree. - write_decision(&mut c, "c-1", &committed).unwrap(); - a.commit(); - b.commit(); - c.commit(); + change(&mut c, |c| write_decision(c, "c-1", &committed).unwrap()); let mut agreeing = a.fork(); agreeing.merge(&mut c).unwrap(); assert!(!load(&agreeing, "c-1").unwrap().contested); @@ -740,15 +747,16 @@ mod tests { #[test] fn unreadable_records_are_skipped() { - let mut doc = AutoCommit::new(); - store::write_path( - &mut doc, - &record_path("broken", ""), - &serde_json::json!({ "proposal": "not an object" }), - ) - .unwrap(); - write_proposal(&mut doc, "fine", &proposal(1)).unwrap(); - doc.commit(); + let mut doc = Document::new(); + change(&mut doc, |doc| { + store::write_path( + doc, + &record_path("broken", ""), + &serde_json::json!({ "proposal": "not an object" }), + ) + .unwrap(); + write_proposal(doc, "fine", &proposal(1)).unwrap(); + }); let all = load_all(&doc); assert_eq!(all.len(), 1); assert_eq!(all[0].0, "fine"); diff --git a/src/remake/snapshot.rs b/src/remake/snapshot.rs index b8c784e..157a28d 100644 --- a/src/remake/snapshot.rs +++ b/src/remake/snapshot.rs @@ -20,7 +20,8 @@ use std::collections::BTreeMap; -use automerge::{ActorId, AutoCommit, ChangeHash, transaction::CommitOptions}; +use crate::document::{ActorId, ChangeHash, Document, ReadDoc}; +use automerge::transaction::CommitOptions; use serde_json::{Map as JsonMap, Value as JsonValue, json}; use sha2::{Digest, Sha256}; @@ -55,7 +56,7 @@ pub(crate) struct Snapshot { /// The new epoch: next generation, ID = hash of the snapshot change. pub(crate) epoch: Epoch, /// A document consisting of exactly that one change. - pub(crate) doc: AutoCommit, + pub(crate) doc: Document, /// Highest sequence number per actor (hex) contained in the base. A change /// (actor, seq) is part of the snapshot iff seq <= clock[actor]. Read by tests only: /// nodes read the clock from the epoch metadata (`EpochMetadata::clock`). @@ -116,11 +117,11 @@ impl SnapshotRequest { /// No lock is held while building; the caller only needs a lock for the clone. /// Returns the copy as well, for callers that continue working with it. pub(crate) async fn build_detached( - mut copy: AutoCommit, + copy: Document, request: SnapshotRequest, -) -> Result<(Snapshot, AutoCommit), SnapshotError> { +) -> Result<(Snapshot, Document), SnapshotError> { tokio::task::spawn_blocking(move || { - let snapshot = build(&mut copy, &request.params())?; + let snapshot = build(©, &request.params())?; Ok((snapshot, copy)) }) .await @@ -169,7 +170,13 @@ struct MetadataRecord { translation: JsonValue, } -fn parse_metadata(doc: &AutoCommit, generation: u64) -> Result<EpochMetadata, String> { +/// Metadata of an earlier generation, as the current document holds it. Snapshots +/// carry the metadata of all earlier epochs forward (only remake records stay +/// behind), so a node many epochs behind can still find the epoch that followed its own. +pub(crate) fn read_epoch_metadata( + doc: &Document, + generation: u64, +) -> Result<EpochMetadata, String> { let value = store::read_path(doc, &epoch_metadata_path(generation)) .map_err(|error| error.to_string())? .ok_or_else(|| format!("no epoch metadata for generation {generation}"))?; @@ -203,7 +210,7 @@ fn parse_metadata(doc: &AutoCommit, generation: u64) -> Result<EpochMetadata, St /// /// This does not prove the snapshot's content matches the old epoch; the hash commits /// to it, and the peer is a swarm member trusted as much as for ordinary sync. -pub(crate) fn verify_document(doc: &mut AutoCommit, epoch: Epoch) -> Result<EpochMetadata, String> { +pub(crate) fn verify_document(doc: &Document, epoch: Epoch) -> Result<EpochMetadata, String> { let root = ChangeHash(epoch.id); let Some(change) = doc.get_change_by_hash(&root) else { return Err(format!( @@ -215,13 +222,13 @@ pub(crate) fn verify_document(doc: &mut AutoCommit, epoch: Epoch) -> Result<Epoc } let at_snapshot = doc.fork_at(&[root]).map_err(|error| error.to_string())?; - parse_metadata(&at_snapshot, epoch.generation) + read_epoch_metadata(&at_snapshot, epoch.generation) } /// The epoch `doc` is on: genesis if it holds no epoch metadata; otherwise the newest /// generation it has metadata for, whose snapshot change is the document's only change /// without dependencies (verified). For a document loaded from a backup. -pub(crate) fn epoch_of(doc: &mut AutoCommit) -> Result<Epoch, String> { +pub(crate) fn epoch_of(doc: &Document) -> Result<Epoch, String> { let generation = store::keys_at(doc, &meta::path(&[EPOCHS])) .map_err(|error| error.to_string())? .iter() @@ -250,16 +257,6 @@ pub(crate) fn epoch_of(doc: &mut AutoCommit) -> Result<Epoch, String> { Ok(epoch) } -/// Metadata of an earlier generation, as the current document holds it. Snapshots -/// carry the metadata of all earlier epochs forward (only remake records stay -/// behind), so a node many epochs behind can still find the epoch that followed its own. -pub(crate) fn read_epoch_metadata( - doc: &AutoCommit, - generation: u64, -) -> Result<EpochMetadata, String> { - parse_metadata(doc, generation) -} - /// Path of the metadata record for an epoch generation. pub(crate) fn epoch_metadata_path(generation: u64) -> Vec<String> { meta::path(&[EPOCHS, generation.to_string().as_str()]) @@ -267,7 +264,7 @@ pub(crate) fn epoch_metadata_path(generation: u64) -> Vec<String> { /// Builds the snapshot of `source` at `params.base_heads`. pub(crate) fn build( - source: &mut AutoCommit, + source: &Document, params: &SnapshotParams<'_>, ) -> Result<Snapshot, SnapshotError> { // fork_at with unknown heads would quietly build something else. @@ -277,21 +274,24 @@ pub(crate) fn build( } } - let mut base = source.fork_at(params.base_heads)?; - let clock = clock_of(&mut base); + let base = source.fork_at(params.base_heads)?; + let clock = clock_of(&base); let generation = params.previous.generation + 1; let kind = by_name(params.kind).ok_or_else(|| SnapshotError::UnknownKind(params.kind.to_owned()))?; - let mut snapshot = AutoCommit::new().with_actor(snapshot_actor(params.remake_id, generation)); + let mut snapshot = Document::new().with_actor(snapshot_actor(params.remake_id, generation)); + + // One transaction, so the snapshot is a single change. + let mut target = snapshot.transaction(); // The user content is the kind's. - kind.build(&base, params.content, &mut snapshot)?; + kind.build(&base, params.content, &mut target)?; // Protocol data carries forward, whatever the kind: forget records, earlier epochs' // metadata (rejoining nodes need it). Remake records and the votes register describe // the old epoch's protocol and stay behind. - match store::delete_path(&mut snapshot, &meta::path(&[]), false) { + match store::delete_path(&mut target, &meta::path(&[]), false) { Ok(()) | Err(StoreError::NotFound) => {} Err(error) => return Err(error.into()), } @@ -301,7 +301,7 @@ pub(crate) fn build( .into_iter() .filter(|key| key != REMAKES && key != VOTES) { - store::copy_path(&base, &mut snapshot, &meta::path(&[key.as_str()]))?; + store::copy_path(&base, &mut target, &meta::path(&[key.as_str()]))?; } let mut base_heads: Vec<String> = params.base_heads.iter().map(|h| h.to_string()).collect(); @@ -321,9 +321,9 @@ pub(crate) fn build( "kind": kind.name(), "translation": params.translation, })); - store::write_path(&mut snapshot, &epoch_metadata_path(generation), &metadata)?; + store::write_path(&mut target, &epoch_metadata_path(generation), &metadata)?; - let _ = snapshot.commit_with(CommitOptions::default().with_time(0)); + target.commit_with(CommitOptions::default().with_time(0)); let heads = snapshot.get_heads(); let [id] = heads.as_slice() else { @@ -341,7 +341,7 @@ pub(crate) fn build( } /// Highest sequence number per actor in `doc`'s history. -fn clock_of(doc: &mut AutoCommit) -> BTreeMap<String, u64> { +fn clock_of(doc: &Document) -> BTreeMap<String, u64> { let mut clock = BTreeMap::new(); for change in doc.get_changes(&[]) { let seq = clock.entry(change.actor_id().to_string()).or_insert(0); @@ -380,10 +380,11 @@ pub(super) fn canonical(value: JsonValue) -> JsonValue { #[cfg(test)] mod tests { use super::*; + use crate::document::testing::change; use automerge::{ObjType, ROOT, ReadDoc, ScalarValue, Value, transaction::Transactable}; - fn doc(actor: &[u8]) -> AutoCommit { - AutoCommit::new().with_actor(ActorId::from(actor)) + fn doc(actor: &[u8]) -> Document { + Document::new().with_actor(ActorId::from(actor)) } fn params<'a>(heads: &'a [ChangeHash], policy: &'a JsonValue) -> SnapshotParams<'a> { @@ -401,14 +402,14 @@ mod tests { static NULL: JsonValue = JsonValue::Null; - fn snapshot_of(source: &mut AutoCommit) -> Snapshot { + fn snapshot_of(source: &Document) -> Snapshot { let heads = source.get_heads(); let policy = json!({}); build(source, ¶ms(&heads, &policy)).expect("snapshot") } /// The user-visible view, without our own metadata. - fn user_view(doc: &AutoCommit) -> JsonValue { + fn user_view(doc: &Document) -> JsonValue { let mut view = store::read_path(doc, &[]).unwrap().unwrap(); view.as_object_mut().unwrap().remove(meta::ROOT); view @@ -418,33 +419,36 @@ mod tests { fn same_view_different_history_same_snapshot() { // One history: overwrites, increments, one-shot text. let mut a = doc(b"actor-a"); - a.put(ROOT, "x", 1_i64).unwrap(); - a.commit(); - a.put(ROOT, "x", 2_i64).unwrap(); - a.put(ROOT, "c", ScalarValue::counter(0)).unwrap(); - a.increment(ROOT, "c", 5).unwrap(); - let text = a.put_object(ROOT, "t", ObjType::Text).unwrap(); - a.splice_text(&text, 0, 0, "hello").unwrap(); - let m = a.put_object(ROOT, "m", ObjType::Map).unwrap(); - a.put(&m, "k", "v").unwrap(); - a.commit(); + change(&mut a, |a| a.put(ROOT, "x", 1_i64).unwrap()); + change(&mut a, |a| { + a.put(ROOT, "x", 2_i64).unwrap(); + a.put(ROOT, "c", ScalarValue::counter(0)).unwrap(); + a.increment(ROOT, "c", 5).unwrap(); + let text = a.put_object(ROOT, "t", ObjType::Text).unwrap(); + a.splice_text(&text, 0, 0, "hello").unwrap(); + let m = a.put_object(ROOT, "m", ObjType::Map).unwrap(); + a.put(&m, "k", "v").unwrap(); + }); // Another actor, other order, other steps, same result. let mut b = doc(b"actor-b"); - let m = b.put_object(ROOT, "m", ObjType::Map).unwrap(); - b.put(&m, "k", "v").unwrap(); - let text = b.put_object(ROOT, "t", ObjType::Text).unwrap(); - b.splice_text(&text, 0, 0, "hel").unwrap(); - b.commit(); - b.splice_text(&text, 3, 0, "lo").unwrap(); - b.put(ROOT, "c", ScalarValue::counter(5)).unwrap(); - b.put(ROOT, "x", 2_i64).unwrap(); - b.commit(); + let text = change(&mut b, |b| { + let m = b.put_object(ROOT, "m", ObjType::Map).unwrap(); + b.put(&m, "k", "v").unwrap(); + let text = b.put_object(ROOT, "t", ObjType::Text).unwrap(); + b.splice_text(&text, 0, 0, "hel").unwrap(); + text + }); + change(&mut b, |b| { + b.splice_text(&text, 3, 0, "lo").unwrap(); + b.put(ROOT, "c", ScalarValue::counter(5)).unwrap(); + b.put(ROOT, "x", 2_i64).unwrap(); + }); assert_eq!(user_view(&a), user_view(&b)); - let snap_a = snapshot_of(&mut a); - let snap_b = snapshot_of(&mut b); + let snap_a = snapshot_of(&a); + let snap_b = snapshot_of(&b); // Different histories mean different clocks, and the clock is part of the // metadata. Equal views alone therefore give equal content but not equal IDs; @@ -456,20 +460,21 @@ mod tests { #[test] fn rebuild_is_deterministic() { let mut a = doc(b"actor-a"); - a.put(ROOT, "x", 1_i64).unwrap(); - let list = a.put_object(ROOT, "l", ObjType::List).unwrap(); - a.insert(&list, 0, "first").unwrap(); - let inner = a.insert_object(&list, 1, ObjType::Map).unwrap(); - a.put(&inner, "k", true).unwrap(); - a.commit(); + change(&mut a, |a| { + a.put(ROOT, "x", 1_i64).unwrap(); + let list = a.put_object(ROOT, "l", ObjType::List).unwrap(); + a.insert(&list, 0, "first").unwrap(); + let inner = a.insert_object(&list, 1, ObjType::Map).unwrap(); + a.put(&inner, "k", true).unwrap(); + }); // A second node holding the same changes, received in a different way. let mut b = doc(b"actor-b"); b.merge(&mut a.clone()).unwrap(); - let first = snapshot_of(&mut a); - let again = snapshot_of(&mut a); - let elsewhere = snapshot_of(&mut b); + let first = snapshot_of(&a); + let again = snapshot_of(&a); + let elsewhere = snapshot_of(&b); assert_eq!(first.epoch, again.epoch); assert_eq!(first.epoch, elsewhere.epoch); @@ -479,14 +484,16 @@ mod tests { #[test] fn conflicts_collapse_into_the_merged_view() { let mut a = doc(b"actor-a"); - let cfg = a.put_object(ROOT, "cfg", ObjType::Map).unwrap(); - a.put(&cfg, "x", 1_i64).unwrap(); - a.commit(); + change(&mut a, |a| { + let cfg = a.put_object(ROOT, "cfg", ObjType::Map).unwrap(); + a.put(&cfg, "x", 1_i64).unwrap(); + }); let mut b = doc(b"actor-b"); - let cfg = b.put_object(ROOT, "cfg", ObjType::Map).unwrap(); - b.put(&cfg, "y", 2_i64).unwrap(); - b.commit(); + change(&mut b, |b| { + let cfg = b.put_object(ROOT, "cfg", ObjType::Map).unwrap(); + b.put(&cfg, "y", 2_i64).unwrap(); + }); a.merge(&mut b).unwrap(); let (_, conflicts) = store::read_path_with_conflicts(&a, &[]).unwrap().unwrap(); @@ -495,7 +502,7 @@ mod tests { "test setup should produce a conflict" ); - let snap = snapshot_of(&mut a); + let snap = snapshot_of(&a); assert_eq!(user_view(&snap.doc), json!({ "cfg": { "x": 1, "y": 2 } })); let (_, conflicts) = store::read_path_with_conflicts(&snap.doc, &[]) @@ -511,25 +518,22 @@ mod tests { fn base_heads_bound_content_and_clock() { let mut a = doc(b"actor-a"); for i in 0..3_i64 { - a.put(ROOT, format!("a{i}"), i).unwrap(); - a.commit(); + change(&mut a, |a| a.put(ROOT, format!("a{i}"), i).unwrap()); } let mut b = doc(b"actor-b"); for i in 0..2_i64 { - b.put(ROOT, format!("b{i}"), i).unwrap(); - b.commit(); + change(&mut b, |b| b.put(ROOT, format!("b{i}"), i).unwrap()); } a.merge(&mut b).unwrap(); let base = a.get_heads(); // After the base: must not appear in the snapshot or its clock. - a.put(ROOT, "late", true).unwrap(); - a.commit(); + change(&mut a, |a| a.put(ROOT, "late", true).unwrap()); let policy = json!({}); - let snap = build(&mut a, ¶ms(&base, &policy)).unwrap(); + let snap = build(&a, ¶ms(&base, &policy)).unwrap(); let view = user_view(&snap.doc); assert!(view.get("late").is_none()); @@ -545,50 +549,53 @@ mod tests { #[test] fn epoch_of_a_backup() { let mut a = doc(b"actor-a"); - store::write_path(&mut a, &["k".into()], &json!(1)).unwrap(); - a.commit(); - assert_eq!(epoch_of(&mut a), Ok(Epoch::GENESIS)); + change(&mut a, |a| { + store::write_path(a, &["k".into()], &json!(1)).unwrap() + }); + assert_eq!(epoch_of(&a), Ok(Epoch::GENESIS)); let heads = a.get_heads(); let policy = json!({}); - let mut snap = build(&mut a, ¶ms(&heads, &policy)).unwrap(); + let snap = build(&a, ¶ms(&heads, &policy)).unwrap(); // Saved and loaded, with a write since, as a backup would be. let mut later = snap .doc .fork() .with_actor(ActorId::from(b"actor-b".as_slice())); - store::write_path(&mut later, &["k".into()], &json!(2)).unwrap(); - later.commit(); - let mut backup = AutoCommit::load(&later.save()).unwrap(); - assert_eq!(epoch_of(&mut backup), Ok(snap.epoch)); + change(&mut later, |later| { + store::write_path(later, &["k".into()], &json!(2)).unwrap() + }); + let backup = Document::load(&later.save()).unwrap(); + assert_eq!(epoch_of(&backup), Ok(snap.epoch)); } #[test] fn metadata_written_and_remake_records_dropped() { let mut a = doc(b"actor-a"); - store::write_path( - &mut a, - &meta::path(&[REMAKES, "c-1"]), - &json!({ "state": "committed" }), - ) - .unwrap(); - store::write_path( - &mut a, - &meta::path(&[VOTES, "pk"]), - &json!({ "remake": "c-1", "at": 1 }), - ) - .unwrap(); - store::write_path( - &mut a, - &meta::path(&[meta::FORGOTTEN, "pk"]), - &json!({ "at": 1 }), - ) - .unwrap(); - a.commit(); + change(&mut a, |a| { + store::write_path( + a, + &meta::path(&[REMAKES, "c-1"]), + &json!({ "state": "committed" }), + ) + .unwrap(); + store::write_path( + a, + &meta::path(&[VOTES, "pk"]), + &json!({ "remake": "c-1", "at": 1 }), + ) + .unwrap(); + store::write_path( + a, + &meta::path(&[meta::FORGOTTEN, "pk"]), + &json!({ "at": 1 }), + ) + .unwrap(); + }); let heads = a.get_heads(); let policy = json!({ "replayWindow": { "epochs": 1 } }); - let snap = build(&mut a, ¶ms(&heads, &policy)).unwrap(); + let snap = build(&a, ¶ms(&heads, &policy)).unwrap(); let meta = store::read_path(&snap.doc, &meta::path(&[])) .unwrap() @@ -609,18 +616,19 @@ mod tests { #[test] fn automerge_types_survive() { let mut a = doc(b"actor-a"); - a.put(ROOT, "c", ScalarValue::counter(3)).unwrap(); - a.increment(ROOT, "c", 4).unwrap(); - let text = a.put_object(ROOT, "t", ObjType::Text).unwrap(); - a.splice_text(&text, 0, 0, "abc").unwrap(); - a.put(ROOT, "bytes", ScalarValue::Bytes(vec![0, 1, 255])) - .unwrap(); - a.put(ROOT, "ts", ScalarValue::Timestamp(1_700_000_000_000)) - .unwrap(); - a.put(ROOT, "neg_zero", -0.0_f64).unwrap(); - a.commit(); + change(&mut a, |a| { + a.put(ROOT, "c", ScalarValue::counter(3)).unwrap(); + a.increment(ROOT, "c", 4).unwrap(); + let text = a.put_object(ROOT, "t", ObjType::Text).unwrap(); + a.splice_text(&text, 0, 0, "abc").unwrap(); + a.put(ROOT, "bytes", ScalarValue::Bytes(vec![0, 1, 255])) + .unwrap(); + a.put(ROOT, "ts", ScalarValue::Timestamp(1_700_000_000_000)) + .unwrap(); + a.put(ROOT, "neg_zero", -0.0_f64).unwrap(); + }); - let snap = snapshot_of(&mut a); + let snap = snapshot_of(&a); let d = &snap.doc; match d.get(ROOT, "c").unwrap() { @@ -659,10 +667,9 @@ mod tests { #[test] fn snapshot_is_a_single_change_with_the_epoch_id() { let mut a = doc(b"actor-a"); - a.put(ROOT, "x", 1_i64).unwrap(); - a.commit(); + change(&mut a, |a| a.put(ROOT, "x", 1_i64).unwrap()); - let mut snap = snapshot_of(&mut a); + let snap = snapshot_of(&a); let heads = snap.doc.get_heads(); assert_eq!(heads.len(), 1); assert_eq!(heads[0].0, snap.epoch.id); @@ -672,25 +679,23 @@ mod tests { #[test] fn missing_base_is_rejected() { let mut a = doc(b"actor-a"); - a.put(ROOT, "x", 1_i64).unwrap(); - a.commit(); + change(&mut a, |a| a.put(ROOT, "x", 1_i64).unwrap()); let mut other = doc(b"actor-b"); - other.put(ROOT, "y", 1_i64).unwrap(); - other.commit(); + change(&mut other, |other| other.put(ROOT, "y", 1_i64).unwrap()); let foreign = other.get_heads(); let policy = json!({}); assert!(matches!( - build(&mut a, ¶ms(&foreign, &policy)), + build(&a, ¶ms(&foreign, &policy)), Err(SnapshotError::MissingBase(_)) )); } #[test] fn empty_document() { - let mut a = doc(b"actor-a"); - let snap = snapshot_of(&mut a); + let a = doc(b"actor-a"); + let snap = snapshot_of(&a); assert_eq!(snap.epoch.generation, 1); assert!(snap.clock.is_empty()); assert_eq!(user_view(&snap.doc), json!({})); @@ -702,26 +707,23 @@ mod tests { // Old epoch: the base, then a late write by this node. let mut late = doc(b"late-node"); - late.put(ROOT, "shared", "base").unwrap(); - late.commit(); + change(&mut late, |late| late.put(ROOT, "shared", "base").unwrap()); let base = late.get_heads(); let policy = json!({}); - let mut snap = build(&mut late, ¶ms(&base, &policy)).unwrap(); + let snap = build(&late, ¶ms(&base, &policy)).unwrap(); // Meanwhile the swarm moved on in the new epoch. let mut swarm = snap .doc .fork() .with_actor(ActorId::from(&b"swarm-node"[..])); - swarm.put(ROOT, "new", 1_i64).unwrap(); - swarm.commit(); + change(&mut swarm, |swarm| swarm.put(ROOT, "new", 1_i64).unwrap()); let bytes = swarm.save(); - late.put(ROOT, "mine", "late").unwrap(); - late.commit(); + change(&mut late, |late| late.put(ROOT, "mine", "late").unwrap()); - let mut fetched = AutoCommit::load(&bytes).unwrap(); - let metadata = verify_document(&mut fetched, snap.epoch).unwrap(); + let fetched = Document::load(&bytes).unwrap(); + let metadata = verify_document(&fetched, snap.epoch).unwrap(); assert_eq!(metadata.previous, Epoch::GENESIS); assert_eq!(metadata.remake_id, "c-1"); assert_eq!( @@ -732,14 +734,14 @@ mod tests { // Documents of other epochs are refused. let mut wrong = snap.epoch; wrong.generation = 2; - assert!(verify_document(&mut fetched, wrong).is_err()); + assert!(verify_document(&fetched, wrong).is_err()); let mut wrong = snap.epoch; wrong.id = [0; 32]; - assert!(verify_document(&mut fetched, wrong).is_err()); + assert!(verify_document(&fetched, wrong).is_err()); let own = late.get_actor().clone(); let missing = - delta::missing_from_snapshot(&mut late, &metadata.clock, std::slice::from_ref(&own)); + delta::missing_from_snapshot(&late, &metadata.clock, std::slice::from_ref(&own)); assert_eq!(missing.own, 1); let heads = late.get_heads(); let mut adopted = fetched.with_actor(ActorId::from(&b"late-node-next"[..])); diff --git a/src/remake/switch.rs b/src/remake/switch.rs index 6c90f44..4c72b39 100644 --- a/src/remake/switch.rs +++ b/src/remake/switch.rs @@ -20,7 +20,6 @@ use std::time::Duration; -use automerge::{ActorId, AutoCommit, ChangeHash}; use tokio::sync::broadcast; use tracing::{error, info, warn}; @@ -38,8 +37,9 @@ use super::vote; use crate::document::meta::{self, REMAKES}; use crate::document::replica::Epoch; use crate::document::store; +use crate::document::store::paths_related; +use crate::document::{ActorId, ChangeHash, Document}; use crate::locks::MutexExt; -use crate::swarm::sync::paths_related; const TICK: Duration = Duration::from_secs(1); @@ -193,6 +193,7 @@ async fn switch<H: Host>( .translator() .ok_or_else(|| format!("unknown remake kind {:?}", request.kind))?; let late_translator = translator.clone(); + let superseded = host.superseded(); let previous = request.previous; let base = request.base_heads.clone(); let retention = record.proposal.policy.archive_retention.clone(); @@ -218,11 +219,11 @@ async fn switch<H: Host>( // 2. Build (or reuse), verify and replay without holding any lock. let (new_doc, copy_heads, mut stats) = tokio::task::spawn_blocking( - move || -> Result<(AutoCommit, Vec<ChangeHash>, ReplayStats), String> { - let mut copy = copy; + move || -> Result<(Document, Vec<ChangeHash>, ReplayStats), String> { + let copy = copy; let snapshot = match prebuilt { Some(snapshot) => snapshot, - None => snapshot::build(&mut copy, &request.params()) + None => snapshot::build(©, &request.params()) .map_err(|error| format!("snapshot build failed: {error}"))?, }; if snapshot.epoch != expected { @@ -235,13 +236,20 @@ async fn switch<H: Host>( // A fresh actor: this node's writes in the new epoch. let mut doc = snapshot.doc.with_actor(actor); + let heads = copy.get_heads(); + // Keep this remake's record visible in the new epoch, for its history. - if carry_record && let Err(error) = store::copy_path(©, &mut doc, &record_path) { - warn!(error = %error, "Could not carry the remake record into the new epoch"); + if carry_record { + let mut transaction = doc.transaction(); + if let Err(error) = store::copy_path(©, &mut transaction, &record_path) { + warn!(error = %error, "Could not carry the remake record into the new epoch"); + } + transaction.commit(); } - let heads = copy.get_heads(); - let stats = delta::replay(&mut copy, &base, &heads, &mut doc, &translator); + // Each write at the time it was made, so later replays judge by it. + let stats = + delta::replay_newest(©, &base, &heads, &mut doc, &translator, superseded); Ok((doc, heads, stats)) }, ) @@ -276,6 +284,7 @@ async fn switch<H: Host>( replayed = stats.replayed, failed = stats.failed, dropped = stats.dropped, + superseded = stats.superseded, quarantine = ?quarantine, "Switched to the new epoch" ); @@ -288,25 +297,30 @@ async fn switch<H: Host>( pub(super) async fn swap_in<H: Host>( host: &H, target: Epoch, - new_doc: AutoCommit, + new_doc: Document, copy_heads: Vec<ChangeHash>, translator: Translator, -) -> Result<(AutoCommit, ReplayStats), String> { +) -> Result<(Document, ReplayStats), String> { + let superseded = host.superseded(); // The closure runs on the blocking pool; its replay numbers come back through this. let late_stats = std::sync::Arc::new(std::sync::Mutex::new(ReplayStats::default())); let late_out = std::sync::Arc::clone(&late_stats); let old = host .switch_epoch(target, move |live| { // Only one worker switches, but be sure the copy still describes the live document. - if !copy_heads - .iter() - .all(|head| live.get_change_by_hash(head).is_some()) - { + if !crate::document::changes::missing(live, ©_heads).is_empty() { return Err("the document was replaced while preparing the switch".to_owned()); } let mut new_doc = new_doc; let live_heads = live.get_heads(); - let late = delta::replay(live, ©_heads, &live_heads, &mut new_doc, &translator); + let late = delta::replay_newest( + live, + ©_heads, + &live_heads, + &mut new_doc, + &translator, + superseded, + ); if late.replayed + late.failed > 0 { info!( replayed = late.replayed, diff --git a/src/remake/vote.rs b/src/remake/vote.rs index e5f8dbe..12c1333 100644 --- a/src/remake/vote.rs +++ b/src/remake/vote.rs @@ -240,6 +240,33 @@ pub(crate) fn choice( .map(|(id, _)| id.clone()) } +/// Undecided remakes of `epoch` with `me` in their electorate that expired (by `now`) +/// before `me` voted for them. +fn missed( + me: &str, + epoch: &EpochRecord, + all: &[(String, Record)], + register: &Register, + now: UnixMillis, + grace: u64, +) -> Vec<String> { + let voted = |id: &str| { + register + .get(me) + .is_some_and(|votes| votes.iter().any(|vote| vote.remake == id)) + }; + all.iter() + .filter(|(id, record)| { + &record.proposal.from_epoch == epoch + && record.decision.is_none() + && record.phase(now, grace) == Phase::Expired + && record.proposal.electorate.iter().any(|member| member == me) + && !voted(id) + }) + .map(|(id, _)| id.clone()) + .collect() +} + /// Votes for a remake of the local epoch if this node should. Called by the remake /// worker whenever the records change (and on its timer). pub(super) async fn cast<H: Host>(host: &H) { @@ -254,20 +281,36 @@ pub(super) async fn cast<H: Host>(host: &H) { // Usually there is nothing to vote on: look first, without writing. let (voter, wanted) = (me.clone(), epoch.clone()); let now = host.now_millis(); - let due = host + let (due, missed) = host .read_doc(move |doc| { let all = records::load_all(doc); let register = records::load_register(doc); - let id = choice(&voter, &wanted, &all, ®ister, started, now, grace)?; - let (_, record) = all.iter().find(|(other, _)| *other == id)?; - // Enough voted already: ours would only be one more change for everyone. - if tally(&id, record, &all, ®ister).reached() { - return None; - } - let content = record.proposal.content.clone(); - Some((id, content)) + let missed = missed(&voter, &wanted, &all, ®ister, now, grace); + let due = + choice(&voter, &wanted, &all, ®ister, started, now, grace).and_then(|id| { + let (_, record) = all.iter().find(|(other, _)| *other == id)?; + // Enough voted already: ours would only be one more change for everyone. + if tally(&id, record, &all, ®ister).reached() { + return None; + } + let content = record.proposal.content.clone(); + Some((id, content)) + }); + (due, missed) }) .await; + for id in missed { + if host.remake().missed(&id) { + // By this node's clock: one ahead sees remakes expire that others still vote + // on, and on a small swarm, its vote missing keeps them from a majority. + let skew = host.clock_skew().await; + if skew.off { + warn!(remake = %id, offset_ms = skew.offset_ms, "Didn't vote for a remake: it expired by this node's clock, which is off from its peers'"); + } else { + info!(remake = %id, "Didn't vote for a remake: it expired before this node voted"); + } + } + } let Some((chosen, content)) = due else { return; }; @@ -346,6 +389,7 @@ mod tests { proposed_at, settle_until: proposed_at.plus_ms(1_000), switch_delay_ms: None, + grace_ms: None, electorate: electorate.iter().map(|m| (*m).to_owned()).collect(), quorum: quorum(electorate.len()), kind: "compact".to_owned(), diff --git a/src/simtest/deltas.rs b/src/simtest/deltas.rs index ce74607..84497ac 100644 --- a/src/simtest/deltas.rs +++ b/src/simtest/deltas.rs @@ -9,12 +9,11 @@ use std::time::Duration; use serde_json::json; -use super::{Sim, read, write}; +use super::{SOON, Sim, read, write}; use crate::node::NodeConfig; use crate::state::AppState; use crate::swarm::bootstrap::Start; -const SOON: Duration = Duration::from_secs(10); /// Catching up a few hundred KiB in 1 KiB chunks. const CATCH_UP: Duration = Duration::from_secs(60); const CHUNK: u32 = 1024; diff --git a/src/simtest/identity.rs b/src/simtest/identity.rs index 9f2bff3..7dbb722 100644 --- a/src/simtest/identity.rs +++ b/src/simtest/identity.rs @@ -109,6 +109,7 @@ fn status(node: &AppState) -> ParsedMessage { epoch_seq: 0, ready: false, syncs_with_you: false, + leaving: false, }) } @@ -221,8 +222,13 @@ async fn nodes_over_libp2p() { relay: false, continuum: true, mdns: false, + upnp: false, + ignore: Vec::new(), + prefer: Vec::new(), + internet_hosts: Vec::new(), public: false, memory: true, + keepalive: super::KEEPALIVE, }) .unwrap(); config.transport = Arc::new(transport); diff --git a/src/simtest/mod.rs b/src/simtest/mod.rs index be2dc7b..61c81d1 100644 --- a/src/simtest/mod.rs +++ b/src/simtest/mod.rs @@ -20,12 +20,12 @@ use std::future::Future; use std::sync::Arc; use std::time::Duration; -use automerge::{AutoCommit, ChangeHash}; use serde_json::Value as JsonValue; use tokio::time::Instant; use crate::document::replica::Epoch; use crate::document::store; +use crate::document::{ChangeHash, Document}; use crate::files::Files; use crate::identity::NodeKey; use crate::locks::RwLockExt; @@ -33,7 +33,7 @@ use crate::node::{self, Node, NodeConfig}; use crate::state::{AppState, PeerState, Timing}; use crate::swarm::bootstrap::{self, Start}; use crate::transport::identity::IdentityConfig; -use crate::transport::sim::SimNet; +use crate::transport::sim::{DELIVERY_LIMIT, SimNet}; /// Keepalive of simulated nodes: dormancy after 1 s, probes of dormant peers every /// 2.5 s (`Timing::new`). @@ -41,9 +41,9 @@ pub(crate) const KEEPALIVE: Duration = Duration::from_millis(250); /// First wait for a sync answer before sending again (10 s in production, where large /// messages take a while): a lost sync message costs a test 1 s, a heal 2 s. pub(crate) const SYNC_REPLY_TIMEOUT: Duration = Duration::from_secs(1); -/// How long the simulated net may take to deliver a message: well above its delays. -/// A catch-up source that stops sending sync messages is given up after this. -pub(crate) const DELIVERY_LIMIT: Duration = Duration::from_secs(3); +/// How long tests wait for what should happen within a few keepalives: joining, +/// registering, converging. +pub(crate) const SOON: Duration = Duration::from_secs(10); /// A simulated swarm: the net and the nodes started on it. pub(crate) struct Sim { @@ -65,6 +65,16 @@ impl Sim { } } + /// [`new`](Self::new), with messages put into bytes by `codec` (a transport's + /// envelope) instead of the wire encoding alone. + #[cfg_attr(not(feature = "mycelium"), allow(dead_code))] + pub(crate) fn with_codec(seed: u64, codec: crate::transport::sim::Codec) -> Self { + Self { + net: SimNet::with_codec(seed, codec), + ..Self::new(seed) + } + } + /// [`new`](Self::new), with every node under the identity layer: its key derived /// from its name, its node ID as `local_id`. pub(crate) fn with_identity(seed: u64) -> Self { @@ -97,12 +107,14 @@ impl Sim { more: Vec::new(), }), initial_epoch: Epoch::GENESIS, - initial_doc: AutoCommit::new(), + initial_doc: Document::new(), earlier_actors: Vec::new(), timing: Timing::new(KEEPALIVE, SYNC_REPLY_TIMEOUT, DELIVERY_LIMIT), neighbours_k: 8, alias: Some(id.to_owned()), electorate_window: Duration::from_secs(30 * 24 * 3600), + clock_skew_warn: Duration::from_secs(2), + superseded_writes: Default::default(), blob_chunk_bytes: 64 * 1024, swarm_key: None, protocol_writers: HashSet::new(), @@ -132,6 +144,30 @@ impl Sim { } } +impl Sim { + /// Restarts node `id`: its process is gone, as if it crashed (`SimNet::restart`: + /// what it still sends is lost), and a new one starts on the same ID, as + /// [`start_with`](Self::start_with) starts one. Nothing carries over but what + /// `adjust` gives it (its node key, under the identity layer, by its name). + pub(crate) fn restart_with( + &self, + id: &str, + seeds: &[&str], + start: Start, + adjust: impl FnOnce(&mut NodeConfig), + ) -> AppState { + { + let mut nodes = self.nodes.lock().unwrap(); + for (_, old) in nodes.iter().filter(|(name, _)| name == id) { + old.closing.send_replace(true); + } + nodes.retain(|(name, _)| name != id); + } + self.net.restart(id); + self.start_with(id, seeds, start, adjust) + } +} + /// Logs to the test output if `RUST_LOG` is set. fn init_logging() { if std::env::var_os("RUST_LOG").is_some() { @@ -242,11 +278,11 @@ impl Sim { ago(p.last_seen), ago(p.last_sync_rx), p.is_unreachable(), - p.sync.automerge_view().0.map(|heads| heads + p.sync.sync_view().0.map(|heads| heads .iter() .map(|head| head.to_string()[..8].to_owned()) .collect::<Vec<_>>()), - p.sync.automerge_view().1, + p.sync.sync_view().1, p.sync.sending.is_some(), p.sync .reply diff --git a/src/simtest/remakes.rs b/src/simtest/remakes.rs index 28f3768..6a1f0a9 100644 --- a/src/simtest/remakes.rs +++ b/src/simtest/remakes.rs @@ -1,6 +1,10 @@ //! Remakes on whole nodes (`docs/remakes.md`): a compaction everywhere, competing //! proposals across a partition, a node that misses the switch, a coordinator that is -//! gone, and a skewed clock. +//! gone, a skewed clock, writes while a node starts to rejoin, and two remakes committed +//! for one epoch (a coordinator cut off after it committed; electorates of a partition +//! older than the electorate window; a vote lost with a restart), and a node with its +//! clock ahead that would abort +//! a remake as expired while its coordinator still runs it. //! //! The simulated nodes' timings make a remake quick: settle 2 s (two `dormant_after`), //! switch delay a keepalive, votes within half a keepalive. @@ -10,15 +14,15 @@ use std::time::Duration; use axum::http::StatusCode; use serde_json::{Value as JsonValue, json}; -use super::{Sim, read, write}; +use super::{SOON, Sim, read, write}; +use crate::document::ChangeHash; use crate::document::replica::Epoch; +use crate::document::store; use crate::remake::testing; use crate::state::AppState; use crate::swarm::bootstrap::Start; use crate::transport::sim::LinkParams; -/// Joining, registering and converging. -const SOON: Duration = Duration::from_secs(10); /// A remake from proposal to every node switched, rejoins included. const REMAKE: Duration = Duration::from_secs(30); @@ -227,3 +231,346 @@ async fn compaction_with_a_skewed_clock() { same_epoch(&[&a, &b, &c]); assert_eq!(read(&b, "data").await, Some(json!(1))); } + +/// Writes on a node through its rejoin: every one is answered and kept, those during +/// the rejoin in the document before it (`StaleView`) and carried over at the end; +/// none lands in the fresh document before the epoch's snapshot, which every change of +/// the new epoch descends from. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn writes_through_a_rejoin_are_kept() { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Mutex}; + + let sim = Sim::new(36); + let (a, b, c) = three(&sim).await; + sim.net.take_down("c"); + propose(&a, json!({ "kind": "compact" })).await; + on_generation(&sim, &[&a, &b], 1).await; + + // c writes on and on, through its return and rejoin. + let answered = Arc::new(Mutex::new(Vec::<u64>::new())); + let stop = Arc::new(AtomicBool::new(false)); + let writer = { + let (c, answered, stop) = (c.clone(), Arc::clone(&answered), Arc::clone(&stop)); + tokio::spawn(async move { + let mut n = 0_u64; + while !stop.load(Ordering::Relaxed) { + let at = vec!["w".to_owned(), n.to_string()]; + let written = c + .mutate_doc(move |doc| store::write_path(doc, &at, &json!(n))) + .await; + assert!(written.is_ok(), "write {n} refused: {written:?}"); + answered.lock().unwrap().push(n); + n += 1; + tokio::time::sleep(Duration::from_millis(2)).await; + } + }) + }; + tokio::time::sleep(Duration::from_millis(200)).await; + sim.net.bring_up("c"); + on_generation(&sim, &[&a, &b, &c], 1).await; + tokio::time::sleep(Duration::from_millis(200)).await; + stop.store(true, Ordering::Relaxed); + writer.await.unwrap(); + + // Every change of c's document descends from the epoch's snapshot. + let epoch = same_epoch(&[&a, &b, &c]); + { + let replica = c.doc.read().await; + let snapshot = ChangeHash(epoch.id); + let deps: std::collections::HashMap<ChangeHash, Vec<ChangeHash>> = replica + .doc + .get_changes(&[]) + .iter() + .map(|change| (change.hash(), change.deps().to_vec())) + .collect(); + for start in deps.keys() { + let (mut todo, mut seen) = (vec![*start], std::collections::HashSet::new()); + let mut reached = false; + while let Some(hash) = todo.pop() { + if hash == snapshot { + reached = true; + break; + } + if seen.insert(hash) { + todo.extend(deps.get(&hash).into_iter().flatten().copied()); + } + } + assert!(reached, "change {start} doesn't descend from the snapshot"); + } + } + // Every write is there (those before and during the rejoin replayed into it). + let answered = answered.lock().unwrap().clone(); + assert!(!answered.is_empty()); + let c = &c; + sim.wait_until("c has every write it answered", SOON, move || { + let answered = answered.clone(); + async move { + for n in &answered { + if read(c, &format!("w/{n}")).await != Some(json!(n)) { + return false; + } + } + true + } + }) + .await; +} + +/// A coordinator cut off right after it committed: the others never hear of the commit, +/// abort the remake once it expired and commit another. The coordinator switches to its +/// own; once back, it sees more peers on the other epoch of that generation, rejoins +/// that one sideways (`catchup::sideways`), and the swarm is on one epoch again. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn coordinator_cut_off_after_committing_rejoins_the_others() { + let sim = Sim::new(37); + let (a, b, c) = three(&sim).await; + write(&b, "data", json!(1)).await; + sim.wait_converged(&[&a, &b, &c], SOON).await; + + let cut = LinkParams { + loss: 1.0, + ..LinkParams::default() + }; + let lost = propose(&a, json!({ "kind": "compact" })).await; + let (a_ref, b_ref, c_ref, lost_ref) = (&a, &b, &c, lost.as_str()); + sim.wait_until("b and c have the proposal", SOON, move || async move { + testing::report(b_ref, lost_ref).await.is_some() + && testing::report(c_ref, lost_ref).await.is_some() + }) + .await; + // Their votes still reach a; nothing from a reaches them. + for other in ["b", "c"] { + sim.net.set_link("a", other, cut.clone()); + } + sim.wait_until("a committed", SOON, move || async move { + testing::report(a_ref, lost_ref) + .await + .is_some_and(|report| report["phase"] == "committed") + }) + .await; + sim.net.take_down("a"); + on_generation(&sim, &[&a], 1).await; + + // For b and c, the remake expired undecided. + sim.wait_until("b can abort the lost remake", REMAKE, move || async move { + let (status, _) = testing::abort(b_ref.clone(), lost_ref, None).await; + status == StatusCode::OK + }) + .await; + propose(&b, json!({ "kind": "compact" })).await; + on_generation(&sim, &[&b, &c], 1).await; + let theirs = same_epoch(&[&b, &c]); + assert_ne!(a.local_epoch(), theirs, "two epochs of generation 1"); + + for other in ["b", "c"] { + sim.net.set_link("a", other, LinkParams::default()); + } + sim.net.bring_up("a"); + sim.wait_until("a is on b's and c's epoch", REMAKE, move || async move { + a_ref.local_epoch() == theirs && a_ref.is_ready().await + }) + .await; + assert_eq!(read(&a, "data").await, Some(json!(1))); +} + +/// Electorates that don't overlap: on one side of a partition the other side's alive +/// stamps look older than `--electorate-window` (here by the clocks of d and e, a month +/// ahead, as a partition that long would make them), so each side's remake gets a +/// majority of its own electorate and both commit. Once the partition heals, the two +/// nodes of the smaller side rejoin the larger side's epoch. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn both_sides_of_an_old_partition_commit_and_converge() { + use crate::remake::Host; + + let sim = Sim::new(38); + let a = sim.start("a", &[], Start::Genesis); + let [b, c, d, e] = ["b", "c", "d", "e"].map(|id| sim.start(id, &["a"], Start::Join)); + let all = [&a, &b, &c, &d, &e]; + sim.wait_ready(&all, SOON).await; + let nodes = [a.clone(), b.clone(), c.clone(), d.clone(), e.clone()]; + sim.wait_until("every node is registered", SOON, || { + let nodes = nodes.clone(); + async move { + for node in &nodes { + if crate::swarm::members::members(&node.doc.read().await.doc).len() < 5 { + return false; + } + } + true + } + }) + .await; + write(&a, "data", json!(1)).await; + sim.wait_converged(&all, SOON).await; + + let cut = LinkParams { + loss: 1.0, + ..LinkParams::default() + }; + let link = |params: &LinkParams| { + for one in ["a", "b", "c"] { + for other in ["d", "e"] { + sim.net.set_link(one, other, params.clone()); + sim.net.set_link(other, one, params.clone()); + } + } + }; + link(&cut); + let month = 31 * 24 * 3600 * 1000; + d.clock.set_offset_ms(month); + e.clock.set_offset_ms(month); + // d and e renew their stamps (by their clocks, theirs were a month old), and count + // only themselves. + let d_ref = &d; + sim.wait_until("d's electorate is d and e", SOON, move || async move { + let doc = d_ref.doc.read().await; + d_ref.electorate(&doc.doc, d_ref.now_millis()).len() == 2 + }) + .await; + + propose(&a, json!({ "kind": "compact" })).await; + propose(&d, json!({ "kind": "compact" })).await; + on_generation(&sim, &[&a, &b, &c], 1).await; + on_generation(&sim, &[&d, &e], 1).await; + let larger = same_epoch(&[&a, &b, &c]); + assert_ne!(same_epoch(&[&d, &e]), larger, "two epochs of generation 1"); + + link(&LinkParams::default()); + let (d_ref, e_ref) = (&d, &e); + sim.wait_until("d and e are on a's epoch", REMAKE, move || async move { + d_ref.local_epoch() == larger + && e_ref.local_epoch() == larger + && d_ref.is_ready().await + && e_ref.is_ready().await + }) + .await; + assert_eq!(read(&e, "data").await, Some(json!(1))); +} + +/// A node whose clock is ahead sees a remake expire early: aborting it from there is +/// refused while the node knows its clock is off (`?force=clock` overrides), as its +/// coordinator, alive, may still commit it. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn abort_from_a_node_with_its_clock_off_is_refused() { + use crate::remake::Host; + + // Under the identity layer: its paths layer compares the peers' clocks (PONGs). + let sim = Sim::with_identity(39); + let (a, b, c) = three(&sim).await; + // Far beyond the grace period (1 s) and --clock-skew-warn (2 s), and the settle + // period below: by c's clock, the remake is expired once proposed. + c.clock.set_offset_ms(60_000); + let c_ref = &c; + sim.wait_until("c knows its clock is off", SOON, move || async move { + c_ref.clock_skew().await.off + }) + .await; + + // No vote reaches a, so its remake stays undecided for its settle period. + let cut = LinkParams { + loss: 1.0, + ..LinkParams::default() + }; + sim.net.set_link("b", "a", cut.clone()); + sim.net.set_link("c", "a", cut); + let id = propose(&a, json!({ "kind": "compact", "settle": "30s" })).await; + let id_ref = id.as_str(); + sim.wait_until( + "c has the proposal, expired by its clock", + SOON, + move || async move { + testing::report(c_ref, id_ref) + .await + .is_some_and(|report| report["phase"] == "expired") + }, + ) + .await; + let (status, refused) = testing::abort(c.clone(), &id, None).await; + assert_eq!(status, StatusCode::CONFLICT, "{refused}"); + assert_eq!(refused["force"], "clock"); + assert!( + testing::report(&b, &id) + .await + .is_some_and(|report| report["phase"] != "aborted") + ); +} + +/// A vote lost with a restart: v votes on one side of a partition, restarts without a +/// document file, joins the other side, which never saw the remake, and votes there +/// for another. Each side reaches its majority with v, and both commit. Once the +/// partition heals, the side fewer nodes are on rejoins the other's epoch sideways. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_vote_lost_with_a_restart_and_the_swarm_converges() { + let sim = Sim::new(40); + let a = sim.start("a", &[], Start::Genesis); + let [b, v, d, e] = ["b", "v", "d", "e"].map(|id| sim.start(id, &["a"], Start::Join)); + sim.wait_ready(&[&a, &b, &v, &d, &e], SOON).await; + let nodes = [a.clone(), b.clone(), v.clone(), d.clone(), e.clone()]; + sim.wait_until("every node is registered", SOON, || { + let nodes = nodes.clone(); + async move { + for node in &nodes { + if crate::swarm::members::members(&node.doc.read().await.doc).len() < 5 { + return false; + } + } + true + } + }) + .await; + write(&a, "data", json!(1)).await; + sim.wait_converged(&[&a, &b, &v, &d, &e], SOON).await; + + let cut = LinkParams { + loss: 1.0, + ..LinkParams::default() + }; + let link = |side: &[&str], other: &[&str], params: &LinkParams| { + for one in side { + for two in other { + sim.net.set_link(one, two, params.clone()); + sim.net.set_link(two, one, params.clone()); + } + } + }; + link(&["a", "b", "v"], &["d", "e"], &cut); + + // a, b and v: three of five, X commits. + let x = propose(&a, json!({ "kind": "compact" })).await; + let (a_ref, x_ref) = (&a, x.as_str()); + sim.wait_until("X committed", REMAKE, move || async move { + testing::report(a_ref, x_ref) + .await + .is_some_and(|report| report["phase"] == "committed") + }) + .await; + + // v restarts on the other side, with nothing of what it had: no record of X. + link(&["v"], &["a", "b"], &cut); + link(&["v"], &["d", "e"], &LinkParams::default()); + let v = sim.restart_with("v", &["d"], Start::Join, |_| {}); + sim.wait_ready(&[&v], SOON).await; + + // d, e and v: three of five, Y commits too. + propose(&d, json!({ "kind": "compact" })).await; + on_generation(&sim, &[&a, &b], 1).await; + on_generation(&sim, &[&d, &e, &v], 1).await; + let more = same_epoch(&[&d, &e, &v]); + assert_ne!(same_epoch(&[&a, &b]), more, "two epochs of generation 1"); + + link(&["a", "b"], &["v", "d", "e"], &LinkParams::default()); + let (b_ref, more_ref) = (&b, more); + sim.wait_until( + "a and b are on the epoch more nodes are on", + REMAKE, + move || async move { + a_ref.local_epoch() == more_ref + && b_ref.local_epoch() == more_ref + && a_ref.is_ready().await + && b_ref.is_ready().await + }, + ) + .await; + assert_eq!(read(&b, "data").await, Some(json!(1))); +} diff --git a/src/simtest/timing.rs b/src/simtest/timing.rs index 20ad15c..daa588f 100644 --- a/src/simtest/timing.rs +++ b/src/simtest/timing.rs @@ -1,19 +1,17 @@ //! The timing rules of `peers.md` on whole nodes: joining and converging, dormancy and -//! return, and late messages that must not undo what newer ones said. +//! return, a restart, and late messages that must not undo what newer ones said. use std::time::Duration; use serde_json::json; -use super::{Sim, has_neighbour, heads, peer, read, write}; +use super::{SOON, Sim, has_neighbour, heads, peer, read, write}; use crate::document::replica::Epoch; -use crate::state::{AppState, Liveness}; +use crate::state::AppState; use crate::swarm::bootstrap::Start; use crate::transport::sim::LinkParams; use crate::transport::{HeadsSummary, HelloRequest, MessageKind, ParsedMessage, RunId, Status}; -/// Long enough for anything that should happen within a few keepalives. -const SOON: Duration = Duration::from_secs(10); /// With lost messages: several reply timeouts (1 s, doubling) and heals in a row. const LOSSY: Duration = Duration::from_secs(30); @@ -48,6 +46,28 @@ async fn three_nodes_join_and_converge() { } } +/// A node that restarts without a document file (its process gone, a new one on the +/// same ID) joins again and gets the document; what its earlier run sends on is lost. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn restarted_node_joins_again() { + let sim = Sim::new(9); + let a = sim.start("a", &[], Start::Genesis); + let b = sim.start("b", &["a"], Start::Join); + sim.wait_ready(&[&a, &b], SOON).await; + write(&a, "before", json!(1)).await; + sim.wait_converged(&[&a, &b], SOON).await; + + let b = sim.restart_with("b", &["a"], Start::Join, |_| {}); + sim.wait_ready(&[&b], SOON).await; + let b = &b; + sim.wait_until("b has the document again", SOON, move || async move { + read(b, "before").await == Some(json!(1)) + }) + .await; + write(b, "after", json!(2)).await; + sim.wait_converged(&[&a, b], SOON).await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn converges_with_duplicated_and_reordered_messages() { let sim = Sim::new(2); @@ -232,11 +252,7 @@ async fn peer_goes_dormant_and_returns() { } async fn is_dormant(node: &AppState, other: &str) -> bool { - peer(node, other, |p| { - matches!(p.liveness, Liveness::Dormant { .. }) - }) - .await - == Some(true) + peer(node, other, |p| p.liveness.is_dormant()).await == Some(true) } /// R27: a resent HELLO (its reply was lost) or a late copy of one doesn't reset the @@ -318,6 +334,7 @@ async fn late_epoch_does_not_flip_a_peers_epoch() { epoch_seq, ready: true, syncs_with_you: false, + leaving: false, }) }; let a = &a; @@ -414,3 +431,77 @@ async fn forgotten_node_that_is_alive_cancels_its_record() { } sim.wait_converged(&[a, b, c], SOON).await; } + +/// A transport that only connects out: it publishes no address, as a libp2p node +/// without a listen address (`topology::refresh`, `outbound`). +struct OutboundOnly(std::sync::Arc<dyn crate::transport::Transport>); + +#[async_trait::async_trait] +impl crate::transport::Transport for OutboundOnly { + fn local_id(&self) -> &str { + self.0.local_id() + } + fn is_peer_id(&self, id: &str) -> bool { + self.0.is_peer_id(id) + } + fn delivery_limit(&self) -> Duration { + self.0.delivery_limit() + } + fn local_addresses(&self) -> Vec<String> { + Vec::new() + } + async fn resolve_seed(&self, seed: &str) -> anyhow::Result<String> { + self.0.resolve_seed(seed).await + } + fn encode( + &self, + peer: &str, + message: &ParsedMessage, + ) -> anyhow::Result<crate::transport::EncodedMessage> { + self.0.encode(peer, message) + } + async fn send_encoded(&self, message: &crate::transport::EncodedMessage) -> anyhow::Result<()> { + self.0.send_encoded(message).await + } + async fn start_consumer( + &self, + handler: crate::transport::MessageHandler, + ) -> anyhow::Result<()> { + self.0.start_consumer(handler).await + } + async fn wait_for_exit(&self) -> std::io::Result<std::process::ExitStatus> { + self.0.wait_for_exit().await + } +} + +/// A member with no address (it only connects out) is in nobody's graph but its own: +/// those it picked keep it as a neighbour, so their changes reach it too, not only +/// answers to its own. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_member_that_only_connects_out_gets_changes() { + let sim = Sim::new(7); + let a = sim.start("a", &[], Start::Genesis); + let b = sim.start_with("b", &["a"], Start::Join, |config| { + config.transport = std::sync::Arc::new(OutboundOnly(config.transport.clone())); + }); + let c = sim.start("c", &["a"], Start::Join); + let (a, b, c) = (&a, &b, &c); + sim.wait_ready(&[a, b, c], SOON).await; + // Once b is listed (without an address), the member list no longer makes it a's + // or c's neighbour. + sim.wait_until("b is listed", SOON, move || async move { + read(a, "_continuum/members/b").await.is_some() + && read(c, "_continuum/members/b").await.is_some() + }) + .await; + sim.wait_until("a and c keep b", SOON, move || async move { + has_neighbour(a, "b") && has_neighbour(c, "b") + }) + .await; + write(a, "from/a", json!(1)).await; + write(c, "from/c", json!(3)).await; + sim.wait_until("b has a's and c's writes", SOON, move || async move { + read(b, "from/a").await == Some(json!(1)) && read(b, "from/c").await == Some(json!(3)) + }) + .await; +} diff --git a/src/state/host.rs b/src/state/host.rs index b55239d..5751998 100644 --- a/src/state/host.rs +++ b/src/state/host.rs @@ -6,18 +6,83 @@ use std::sync::Arc; use std::time::Duration; use anyhow::Result; -use automerge::{AutoCommit, ChangeHash}; use tokio::sync::broadcast; use tokio::time::Instant; use tracing::debug; use super::{AppState, HelloState, NodeLifecycle, StaleView, Standing}; -use crate::document::replica::Epoch; -use crate::locks::MutexExt; -use crate::remake::{Host, PeerSummary, RemakeState}; -use crate::swarm::sync::{ChangeSet, SyncEvent}; +use crate::document::replica::{Epoch, Replica}; +use crate::document::store::{ChangeSet, StoreError}; +use crate::document::{ChangeHash, Document, ReadDoc, Transaction}; +use crate::locks::RwLockExt; +use crate::remake::{BegunRejoin, Host, PeerSummary, RemakeState}; impl AppState { + /// Under the document's write lock: `doc` of `epoch` replaces the live document, + /// which is returned. Every peer's sync state is reset before the lock is released, + /// so no message is generated from or applied to a state that describes the old + /// document. (Lock order doc -> peer, as in sync generation.) + fn swap_document(&self, replica: &mut Replica, epoch: Epoch, doc: Document) -> Document { + let old = std::mem::replace(&mut replica.doc, doc); + replica.epoch = epoch; + replica.epoch_seq += 1; + // Publishes the epoch with the heads (`Replica::stamps`), under the same lock, + // so it never disagrees with the document for anyone holding the lock. + replica.refresh_heads(); + let now = Instant::now(); + for peer in self.peer_states.blocking_read().values() { + let mut peer = peer.blocking_lock(); + peer.reset_sync(); + // A fresh start, not an idle sync: no heal re-handshake for it. + peer.last_sync_rx = Some(now); + } + old + } + + /// Stops serving requests (back to `Joining`, keeping the swarm ID), as while a + /// rejoin syncs a fresh document; on the blocking pool. Whether the node served. + fn suspend(&self) -> bool { + let mut lifecycle = self.lifecycle.blocking_write(); + let NodeLifecycle::Ready { swarm_id } = &*lifecycle else { + return false; + }; + let swarm_id = swarm_id.clone(); + debug!(swarm_id = %swarm_id, "Node suspended (rejoining)"); + *lifecycle = NodeLifecycle::Joining { + discovered_swarm_id: Some(swarm_id), + }; + true + } + + /// Serves requests again after [`suspend`](Self::suspend); on the blocking pool. + fn resume(&self) { + let mut lifecycle = self.lifecycle.blocking_write(); + if let NodeLifecycle::Joining { + discovered_swarm_id: Some(swarm_id), + } = &*lifecycle + { + let swarm_id = swarm_id.clone(); + debug!(swarm_id = %swarm_id, "Node resumed"); + *lifecycle = NodeLifecycle::Ready { swarm_id }; + } + } + + /// The document before a rejoin, out of the stale view (which ends) if the node + /// keeps it there, else `old`; and whether it was kept there (the node was ready). + fn take_stale(&self, old: Option<Document>) -> (Document, bool) { + let Some(view) = self.stale.write_or_recover().take() else { + return (old.unwrap_or_default(), false); + }; + // A read holding it a moment longer gets a copy of its own. + let doc = Arc::try_unwrap(view.doc) + .map(|doc| { + doc.into_inner() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + }) + .unwrap_or_else(|shared| shared.read_or_recover().clone()); + (doc, true) + } + /// After the document was replaced: every peer's sync state refers to the old /// document, so start over with all of them. Peers still on the old epoch are /// skipped by the sync worker until they switch too. @@ -42,10 +107,8 @@ impl AppState { let mut peer_state = peer_state_mutex.lock().await; // What it said about syncing with us was about our old epoch. peer_state.syncs_with_us = None; - let gets_our_opening = !peer_state.liveness.is_dormant() - && matches!(peer_state.standing, Standing::Member { .. }) - && peer_state.epoch == Some(local) - && peer_state.reports_ready == Some(true); + let gets_our_opening = + peer_state.syncs_on(local) && peer_state.reports_ready == Some(true); if !gets_our_opening { announce.push(id); } @@ -58,7 +121,7 @@ impl AppState { } for (id, _) in peers { - let _ = self.sync_events.send(SyncEvent::PeerNeedsSync(id)); + self.request_sync(&id); } } } @@ -85,14 +148,18 @@ impl Host for AppState { self.timing.dormant_after } - fn electorate(&self, doc: &AutoCommit, now: crate::clock::UnixMillis) -> Vec<String> { + async fn clock_skew(&self) -> crate::swarm::skew::Skew { + crate::swarm::skew::measure(self).await + } + + fn electorate(&self, doc: &impl ReadDoc, now: crate::clock::UnixMillis) -> Vec<String> { crate::swarm::members::electorate(doc, now.as_u64(), self.electorate_window) } fn fetch_change( &self, hash: ChangeHash, - ) -> impl Future<Output = Result<automerge::Change>> + Send { + ) -> impl Future<Output = Result<crate::document::Change>> + Send { let state = self.clone(); async move { let peers = state.active_peers_on_local_epoch().await; @@ -142,54 +209,6 @@ impl Host for AppState { } } - fn suspend(&self) -> impl Future<Output = bool> + Send { - let state = self.clone(); - async move { - let mut lifecycle = state.lifecycle.write().await; - let NodeLifecycle::Ready { swarm_id } = &*lifecycle else { - return false; - }; - let swarm_id = swarm_id.clone(); - debug!(swarm_id = %swarm_id, "Node suspended (rejoining)"); - *lifecycle = NodeLifecycle::Joining { - discovered_swarm_id: Some(swarm_id), - }; - true - } - } - - fn resume(&self) -> impl Future<Output = ()> + Send { - let state = self.clone(); - async move { - let resumed = { - let mut lifecycle = state.lifecycle.write().await; - let swarm_id = match &*lifecycle { - NodeLifecycle::Joining { - discovered_swarm_id: Some(swarm_id), - } => Some(swarm_id.clone()), - _ => None, - }; - if let Some(swarm_id) = &swarm_id { - debug!(swarm_id = %swarm_id, "Node resumed"); - *lifecycle = NodeLifecycle::Ready { - swarm_id: swarm_id.clone(), - }; - } - swarm_id.is_some() - }; - if resumed { - crate::swarm::peer::announce(&state); - } - } - } - - fn set_stale_view(&self, view: Option<(Epoch, AutoCommit)>) { - self.set_stale(view.map(|(epoch, doc)| StaleView { - epoch, - doc: Arc::new(doc), - })); - } - fn initial_sync_done(&self) -> bool { self.initial_sync_done .load(std::sync::atomic::Ordering::Acquire) @@ -198,8 +217,8 @@ impl Host for AppState { fn reset_initial_sync(&self) { self.initial_sync_done .store(false, std::sync::atomic::Ordering::Release); - *self.catch_up_source.lock_or_recover() = None; - *self.catch_up_preference.lock_or_recover() = None; + // The source chosen, and the one avoided, were for the catch-up before. + crate::swarm::source::reset(self); } fn subscribe_changes(&self) -> broadcast::Receiver<Arc<ChangeSet>> { @@ -208,7 +227,7 @@ impl Host for AppState { fn read_doc<T, F>(&self, f: F) -> impl Future<Output = T> + Send where - F: FnOnce(&AutoCommit) -> T + Send + 'static, + F: FnOnce(&Document) -> T + Send + 'static, T: Send + 'static, { let state = self.clone(); @@ -217,48 +236,181 @@ impl Host for AppState { fn commit_doc<T, E, F>(&self, f: F) -> impl Future<Output = Result<T, E>> + Send where - F: FnOnce(&mut AutoCommit) -> Result<T, E> + Send + 'static, + F: FnOnce(&mut Transaction<'_>) -> Result<T, E> + Send + 'static, T: Send + 'static, - E: Send + 'static, + E: From<StoreError> + Send + 'static, { let state = self.clone(); async move { state.mutate_doc(f).await } } + fn superseded(&self) -> crate::remake::Superseded { + self.superseded_writes + } + + fn begin_rejoin( + &self, + target: Epoch, + actor: crate::document::ActorId, + ) -> impl Future<Output = BegunRejoin> + Send { + let state = self.clone(); + async move { + let begun = { + let state = state.clone(); + tokio::task::spawn_blocking(move || { + let mut replica = state.doc.blocking_write(); + let previous = replica.epoch; + let fresh = Document::new().with_actor(actor); + let old = state.swap_document(&mut replica, target, fresh); + replica.writable = false; + let heads = old.get_heads(); + // Under the document's lock: no read or write in between finds the + // node ready on the fresh document. + let was_ready = state.suspend(); + let old = if was_ready { + state.set_stale(Some(StaleView { + epoch: previous, + doc: Arc::new(std::sync::RwLock::new(old)), + })); + None + } else { + Some(old) + }; + BegunRejoin { + was_ready, + heads, + old, + } + }) + .await + .expect("rejoin task panicked") + }; + state.after_epoch_switch().await; + begun + } + } + + fn rejoin_document(&self) -> impl Future<Output = Option<Document>> + Send { + let state = self.clone(); + async move { + let (_, stale) = state.stale_document()?; + tokio::task::spawn_blocking(move || stale.read_or_recover().clone()) + .await + .ok() + } + } + + fn finish_rejoin<T, F>( + &self, + old: Option<Document>, + f: F, + ) -> impl Future<Output = (T, Document)> + Send + where + F: FnOnce(&mut Document, &Document) -> T + Send + 'static, + T: Send + 'static, + { + let state = self.clone(); + async move { + let (out, old, resumed) = { + let state = state.clone(); + tokio::task::spawn_blocking(move || { + let mut replica = state.doc.blocking_write(); + // No write goes into the document before the rejoin any more: they + // take the document's lock first (`mutate_doc`). + let (old, resumed) = state.take_stale(old); + let out = f(&mut replica.doc, &old); + replica.writable = true; + replica.refresh_heads(); + if resumed { + state.resume(); + } + (out, old, resumed) + }) + .await + .expect("rejoin task panicked") + }; + state.publish_change(); + state.join_progress.notify_one(); + if resumed { + crate::swarm::peer::announce(&state); + } + (out, old) + } + } + + fn abandon_rejoin( + &self, + previous: Epoch, + old: Option<Document>, + ) -> impl Future<Output = ()> + Send { + let state = self.clone(); + async move { + let resumed = { + let state = state.clone(); + tokio::task::spawn_blocking(move || { + let mut replica = state.doc.blocking_write(); + let (old, resumed) = state.take_stale(old); + state.swap_document(&mut replica, previous, old); + replica.writable = true; + if resumed { + state.resume(); + } + resumed + }) + .await + .expect("rejoin task panicked") + }; + state.after_epoch_switch().await; + state.join_progress.notify_one(); + if resumed { + crate::swarm::peer::announce(&state); + } + } + } + + fn apply_change( + &self, + change: crate::document::Change, + ) -> impl Future<Output = Result<(), String>> + Send { + let state = self.clone(); + async move { + tokio::task::spawn_blocking(move || { + let result = { + let mut replica = state.doc.blocking_write(); + let result = replica + .apply_changes([change]) + .map_err(|error| error.to_string()); + replica.refresh_heads(); + result + }; // lock released before notifying + + if result.is_ok() { + state.publish_change(); + } + result + }) + .await + .expect("document write task panicked") + } + } + fn switch_epoch<E, F>( &self, epoch: Epoch, f: F, - ) -> impl Future<Output = Result<AutoCommit, E>> + Send + ) -> impl Future<Output = Result<Document, E>> + Send where - F: FnOnce(&mut AutoCommit) -> Result<AutoCommit, E> + Send + 'static, + F: FnOnce(&Document) -> Result<Document, E> + Send + 'static, E: Send + 'static, { let state = self.clone(); async move { let swapped = { let state = state.clone(); - tokio::task::spawn_blocking(move || -> Result<AutoCommit, E> { + tokio::task::spawn_blocking(move || -> Result<Document, E> { let mut replica = state.doc.blocking_write(); - let new_doc = f(&mut replica.doc)?; - let old_doc = std::mem::replace(&mut replica.doc, new_doc); - replica.epoch = epoch; - replica.epoch_seq += 1; - replica.refresh_heads(); - // Updated under the same lock, so the cached epoch never disagrees - // with the document for anyone holding the lock. - state.epoch_tx.send_replace(epoch); - // Reset every peer's sync state before the lock is released, so no - // message is generated from or applied to a state that describes the - // old document. (Lock order doc -> peer, as in sync generation.) - let now = Instant::now(); - for peer in state.peer_states.blocking_read().values() { - let mut peer = peer.blocking_lock(); - peer.reset_sync(); - // A fresh start, not an idle sync: no heal re-handshake for it. - peer.last_sync_rx = Some(now); - } - Ok(old_doc) + let new_doc = f(&replica.doc)?; + Ok(state.swap_document(&mut replica, epoch, new_doc)) }) .await .expect("epoch switch task panicked") diff --git a/src/state/mod.rs b/src/state/mod.rs index fec164d..aebc59a 100644 --- a/src/state/mod.rs +++ b/src/state/mod.rs @@ -2,7 +2,6 @@ //! //! * `peer`: what a node knows about each peer (`PeerState`): standing, liveness, //! handshake, incidents. -//! * `session`: one Automerge sync with a peer (`SyncSession`). //! * `timing`: the intervals the protocol runs on (`Timing`). //! * `read`: serving reads, live or from the pre-rejoin copy. //! * `host`: the node as the remake module sees it (`remake::Host`). @@ -21,13 +20,11 @@ mod host; mod peer; mod read; -mod session; mod timing; pub(crate) use peer::{HelloState, Incidents, Keepalive, Liveness, PeerState, Standing}; pub(crate) use read::{ReadSource, StaleView}; -pub(crate) use session::SyncSession; -pub(crate) use timing::Timing; +pub(crate) use timing::{SYNC_REPLY_TIMEOUT, Timing}; use std::collections::HashMap; use std::sync::Arc; @@ -35,16 +32,17 @@ use std::sync::atomic::AtomicBool; use std::time::Duration; use anyhow::{Result, bail}; -use automerge::AutoCommit; use tokio::sync::{Mutex, RwLock, broadcast, mpsc, watch}; use tokio::time::Instant; use tracing::{debug, error, warn}; use crate::document::lock::DocumentLock; use crate::document::replica::{Epoch, Stamp}; -use crate::locks::MutexExt; +use crate::document::store::{ChangeSet, StoreError}; +use crate::document::{Document, Transaction}; +use crate::locks::{MutexExt, RwLockExt}; use crate::remake::{Host, RemakeState}; -use crate::swarm::sync::{ChangeSet, SyncEvent}; +use crate::swarm::sync::SyncEvent; use crate::swarm::throughput::ThroughputTable; use crate::transport; use crate::transport::firewall::Firewall; @@ -58,13 +56,12 @@ pub(crate) enum NodeLifecycle { #[derive(Clone)] pub(crate) struct AppState { - /// The document and its epoch. Derefs to `AutoCommit`. + /// The document and its epoch. Derefs to `Document`. pub(crate) doc: Arc<DocumentLock>, - /// Copy of `doc.epoch` for cheap, lock-free reads (handshakes, keepalives, /peers). - /// Only ever changed while holding `doc`'s write lock, together with `doc.epoch`. - pub(crate) epoch_tx: watch::Sender<Epoch>, - /// `doc`'s epoch label and heads, readable without its lock (`Replica::stamps`). + /// `doc`'s epoch label and heads, readable without its lock (`Replica::stamps`): + /// for cheap reads (handshakes, keepalives, /peers). Only ever changed while holding + /// `doc`'s write lock, together with `doc.epoch`. pub(crate) stamp: watch::Receiver<Stamp>, pub(crate) lifecycle: Arc<RwLock<NodeLifecycle>>, @@ -84,8 +81,10 @@ pub(crate) struct AppState { pub(crate) change_tx: watch::Sender<u64>, pub(crate) initial_sync_done: Arc<AtomicBool>, - /// The node is shutting down: its keepalives say it isn't ready (`peer::farewell`). - pub(crate) stopping: Arc<AtomicBool>, + /// True once the node shuts down (a signal, a failed join, a transport gone): the + /// HTTP API drains and its long-lived responses end, and keepalives say the node + /// isn't ready and is leaving (`peer::farewell`). + pub(crate) closing: Arc<watch::Sender<bool>>, pub(crate) change_events: broadcast::Sender<Arc<ChangeSet>>, @@ -110,8 +109,20 @@ pub(crate) struct AppState { pub(crate) alias: Option<String>, /// Members alive within this window vote on remakes (`--electorate-window`). pub(crate) electorate_window: Duration, + /// This node's clock is off when it differs from most of its peers' by more than + /// this (`--clock-skew-warn`, `swarm::skew`); 0: never. + pub(crate) clock_skew_warn: Duration, + /// What a replay does with writes a newer one superseded (`--superseded-writes`). + pub(crate) superseded_writes: crate::remake::Superseded, + /// How far this node's clock is ahead of its peers' (their median, `swarm::skew`), as + /// last measured; 0 before. The propagation figures take it off. + pub(crate) clock_ahead_ms: Arc<std::sync::atomic::AtomicI64>, + /// What the clock checks remember between them (`swarm::skew`). + pub(crate) clock_skew: Arc<std::sync::Mutex<crate::swarm::skew::Watch>>, /// Chunked transfers served and fetched (`swarm::blob`). pub(crate) blobs: Arc<crate::swarm::blob::Blobs>, + /// LINKS asked and answered, for a map of the swarm (`swarm::links`). + pub(crate) links: Arc<crate::swarm::links::Links>, /// HTTP API users who may write protocol data through the store API /// (`--protocol-writers`). pub(crate) protocol_writers: Arc<std::collections::HashSet<String>>, @@ -138,8 +149,16 @@ pub(crate) struct AppState { /// Our neighbours, once known (`swarm::topology`). pub(crate) neighbours: Arc<std::sync::RwLock<Option<crate::swarm::topology::NeighbourView>>>, /// Something the neighbours are computed from may have changed: the member list or a - /// peer's reachability (`swarm::topology::run_worker`). + /// peer's state (`swarm::topology::run_worker`, `swarm::topology::change_peer`). pub(crate) topology_changed: Arc<tokio::sync::Notify>, + /// Something a joining node waits for may have happened: the swarm ID learned, the + /// initial sync done, a rejoin ended, a seed found (`swarm::bootstrap`). + pub(crate) join_progress: Arc<tokio::sync::Notify>, + /// The forget records may have changed (`swarm::changes`): the peer worker acts on + /// them at once instead of at its next periodic check. + pub(crate) forget_records_changed: Arc<AtomicBool>, + /// When the peer worker is next due (`swarm::peer::Due`). + pub(crate) peers_due: Arc<crate::swarm::peer::Due>, /// While rejoining a newer epoch, the previous document, served read-only (and /// marked stale) instead of answering 503. @@ -175,6 +194,29 @@ pub(crate) struct CatchUpSource { } impl AppState { + /// Asks the sync worker to look at `peer` (`SyncEvent::PeerNeedsSync`): whether to + /// send it a sync message now. Nothing if the worker is gone (the node stops). + pub(crate) fn request_sync(&self, peer: &str) { + let _ = self + .sync_events + .send(SyncEvent::PeerNeedsSync(peer.to_owned())); + } + + /// `request_sync`, after `delay`. + pub(crate) fn request_sync_after(&self, peer: &str, delay: std::time::Duration) { + let events = self.sync_events.clone(); + let peer = peer.to_owned(); + tokio::spawn(async move { + tokio::time::sleep(delay).await; + let _ = events.send(SyncEvent::PeerNeedsSync(peer)); + }); + } + + /// Whether the node shuts down (`closing`). + pub(crate) fn is_closing(&self) -> bool { + *self.closing.borrow() + } + pub(crate) fn local_id(&self) -> &str { self.transport.local_id() } @@ -198,7 +240,7 @@ impl AppState { /// The local epoch. Code that labels or applies sync messages must use the epoch /// of the locked `Replica` instead, so label and document can't disagree. pub(crate) fn local_epoch(&self) -> Epoch { - *self.epoch_tx.borrow() + self.stamp.borrow().epoch } /// The local epoch and `Replica::epoch_seq`, read together, without waiting for the @@ -242,6 +284,7 @@ impl AppState { discovered_swarm_id: Some(swarm_id), }; debug!("Node transitioned from {old_lifecycle:?} {lifecycle:?}"); + self.join_progress.notify_one(); Ok(()) } NodeLifecycle::Joining { @@ -281,6 +324,8 @@ impl AppState { let old_lifecycle = lifecycle.clone(); *lifecycle = NodeLifecycle::Ready { swarm_id }; debug!("Node transitioned from {old_lifecycle:?} {lifecycle:?}"); + // Dormancy is judged once ready, and the periodic checks run at once. + self.peers_due.now(); } NodeLifecycle::Ready { swarm_id: existing } if existing == &swarm_id => { debug!("Node already ready with the same identity"); @@ -328,6 +373,7 @@ impl AppState { /// State for a peer we haven't known so far (not inserted). pub(crate) fn new_peer_state(&self, pk: &str) -> Arc<Mutex<PeerState>> { let leads = crate::swarm::pair::leads(self.local_id(), pk); + self.peers_due.now(); Arc::new(Mutex::new(PeerState::new(leads, &self.timing))) } @@ -340,7 +386,7 @@ impl AppState { /// Reads the document on the blocking pool; `finish` runs after the lock is released. pub(crate) async fn render<T, R>( &self, - read: impl FnOnce(&AutoCommit) -> T + Send + 'static, + read: impl FnOnce(&Document) -> T + Send + 'static, finish: impl FnOnce(T) -> R + Send + 'static, ) -> R where @@ -358,59 +404,73 @@ impl AppState { .expect("document read task panicked") } - /// Mutates the document on the blocking pool. - pub(crate) async fn write_doc<T>( + /// Mutates the document on the blocking pool as one change, then + /// publishes it. Write and notification run in the same blocking task, so a + /// cancelled request can't leave a committed write unannounced. While a rejoin runs, + /// the write goes into the document before it (`StaleView`), replayed when the + /// rejoin ends; on a node that keeps none, it is refused (`StoreError::Rejoining`, + /// `f` not called). + pub(crate) async fn mutate_doc<T, E>( &self, - f: impl FnOnce(&mut AutoCommit) -> T + Send + 'static, - ) -> T + f: impl FnOnce(&mut Transaction<'_>) -> Result<T, E> + Send + 'static, + ) -> Result<T, E> where T: Send + 'static, + E: From<StoreError> + Send + 'static, { - let doc = Arc::clone(&self.doc); - tokio::task::spawn_blocking(move || { - let mut replica = doc.blocking_write(); - let out = f(&mut replica.doc); - replica.refresh_heads(); - out - }) - .await - .expect("document write task panicked") + self.mutate_doc_for(None, f).await.map(|(out, _)| out) } - /// Mutates the document on the blocking pool as one Automerge change, then - /// publishes it. Write and notification run in the same blocking task, so a - /// cancelled request can't leave a committed write unannounced. - pub(crate) async fn mutate_doc<T, E>( + /// [`mutate_doc`](Self::mutate_doc) for `user`, who wrote through the API: the + /// change names them (`replica::Writer`). With the heads of the document written to + /// after the write (sorted): what a read must have to include it. + pub(crate) async fn mutate_doc_for<T, E>( &self, - f: impl FnOnce(&mut AutoCommit) -> Result<T, E> + Send + 'static, - ) -> Result<T, E> + user: Option<String>, + f: impl FnOnce(&mut Transaction<'_>) -> Result<T, E> + Send + 'static, + ) -> Result<(T, Vec<crate::document::ChangeHash>), E> where T: Send + 'static, - E: Send + 'static, + E: From<StoreError> + Send + 'static, { let state = self.clone(); tokio::task::spawn_blocking(move || { let result = { let mut replica = state.doc.blocking_write(); - let result = f(&mut replica.doc); - match &result { - // One API write = one Automerge change. - Ok(_) => { - // Stamped with our clock (milliseconds), so receivers can tell - // how long the change took to reach them (`swarm::stats`). - replica.doc.commit_with( - automerge::transaction::CommitOptions::default() - .with_time(state.clock.now_millis().as_u64() as i64), - ); - } - // Discard partial ops (e.g. old value deleted, new one not written). - Err(_) => { - replica.doc.rollback(); - } + // Checked here, under the lock: the caller's readiness check (if any) + // was before a rejoin could swap in a fresh document. + if !replica.writable { + // Rejoining: into the document before the rejoin, which the rejoin + // replays when it ends (`StaleView`); refused by a node that keeps + // none (it wasn't ready). + let Some((_, stale)) = state.stale_document() else { + return Err(E::from(StoreError::Rejoining)); + }; + let mut doc = stale.write_or_recover(); + let writer = crate::document::replica::Writer { + node: state.local_id(), + user: user.as_deref(), + }; + let out = crate::document::replica::transact( + &mut doc, + state.clock.now_millis(), + &writer, + f, + )?; + let mut heads = doc.get_heads(); + heads.sort_unstable(); + return Ok((out, heads)); } + // One API write = one change; on error, its partial writes (e.g. the old + // value deleted, the new one not written) are dropped. + let writer = crate::document::replica::Writer { + node: state.local_id(), + user: user.as_deref(), + }; + let result = replica.transact(state.clock.now_millis(), &writer, f); replica.refresh_heads(); - result + result.map(|out| (out, replica.heads().to_vec())) }; // lock released before notifying if result.is_ok() { diff --git a/src/state/peer.rs b/src/state/peer.rs index 923c6d2..063b72b 100644 --- a/src/state/peer.rs +++ b/src/state/peer.rs @@ -4,9 +4,10 @@ use std::time::Duration; use tokio::time::Instant; -use super::{SyncSession, Timing}; +use super::Timing; use crate::backoff::Backoff; use crate::document::replica::Epoch; +use crate::document::sync::SyncSession; use crate::transport; /// How far a peer got into the swarm, as far as we're concerned. @@ -137,6 +138,10 @@ pub(crate) struct PeerState { /// How many heads its most recent keepalive listed (None if it sent a digest). pub(crate) reported_heads: Option<usize>, + /// It said farewell (`say_farewell`): what else that run of it still sends while it + /// stops doesn't make it active again; a message from another run does (`returned`). + pub(crate) left: bool, + /// HELLOs sent in the current handshake attempt without hearing anything back. pub(crate) hello_attempts: u32, /// Since when we consider the peer unreachable (three unanswered HELLOs, or @@ -146,8 +151,17 @@ pub(crate) struct PeerState { /// What went wrong with this peer so far (counted only, for /peers and /admin/stats). pub(crate) incidents: Incidents, + + /// The largest sync message to it that didn't make it (its send failed, or it went + /// unanswered), in bytes, and when: messages about that large go as deltas for a + /// while (`swarm::blob::Blobs::delta_threshold_for`), which a slow link takes chunk + /// by chunk. + whole_failed: Option<(usize, Instant)>, } +/// How long a sync message that didn't make it keeps others that large from going whole. +const WHOLE_FAILED_KEPT: Duration = Duration::from_secs(3600); + /// Trouble with a peer, counted since it was first listed. Lost keepalives and refused /// sends are in the traffic counters (`transport::traffic`): every PONG answers a PING, /// every STATUS_REPLY a STATUS. @@ -212,9 +226,11 @@ impl PeerState { reports_ready: None, syncs_with_us: None, reported_heads: None, + left: false, hello_attempts: 0, unreachable_since: None, incidents: Incidents::default(), + whole_failed: None, } } @@ -233,20 +249,43 @@ impl PeerState { if self.liveness.is_dormant() || self.hello_state != HelloState::Established { return None; } + let largest = self.sync.reply.as_ref().map(|wait| wait.largest); let overdue = self.sync.take_overdue_reply(now); if overdue.is_some() { self.incidents.reply_timeouts += 1; + if let Some(largest) = largest { + self.whole_failed(largest, now); + } } overdue } + /// A sync message of `bytes` to it didn't make it (`whole_failed`). + pub(crate) fn whole_failed(&mut self, bytes: usize, now: Instant) { + let kept = self + .whole_failed + .filter(|(failed, at)| *failed >= bytes && now < *at + WHOLE_FAILED_KEPT); + self.whole_failed = kept.or(Some((bytes, now))); + } + + /// The largest sync message to it that didn't make it lately, in bytes. + pub(crate) fn failed_whole(&self, now: Instant) -> Option<usize> { + self.whole_failed + .filter(|(_, at)| now < *at + WHOLE_FAILED_KEPT) + .map(|(bytes, _)| bytes) + } + fn quiet_for(&self, now: Instant) -> Duration { now.saturating_duration_since(self.last_seen.unwrap_or(self.added_at)) } - /// Records inbound activity. Returns true if the peer was dormant. + /// Records inbound activity. Returns true if the peer was dormant. From a peer that + /// said farewell (`left`), only that it was heard from. pub(crate) fn mark_seen(&mut self, now: Instant) -> bool { self.last_seen = Some(now); + if self.left { + return false; + } self.hello_attempts = 0; self.unreachable_since = None; self.probe_backoff.reset(); @@ -275,11 +314,32 @@ impl PeerState { } } + /// Whether we sync with it on `epoch`: an active member on that epoch (what our + /// STATUS tells it as `syncs_with_you`). + pub(crate) fn syncs_on(&self, epoch: Epoch) -> bool { + !self.liveness.is_dormant() + && matches!(self.standing, Standing::Member { .. }) + && self.epoch == Some(epoch) + } + /// Whether to leave the peer out when choosing neighbours (`swarm::topology`). pub(crate) fn is_unreachable(&self) -> bool { self.unreachable_since.is_some() } + /// What choosing neighbours reads of this peer (`swarm::topology::refresh`): whether + /// it is unreachable, whether it was ever heard from, whether it is a member, and + /// whether it syncs with us. A change to any may change the neighbours + /// (`swarm::topology::change_peer`). + pub(crate) fn neighbour_inputs(&self) -> impl PartialEq + use<> { + ( + self.is_unreachable(), + self.last_seen.is_some(), + matches!(self.standing, Standing::Member { .. }), + self.syncs_with_us, + ) + } + /// The wait before the next HELLO: a probe's if unreachable. pub(crate) fn next_hello_wait(&mut self) -> Duration { if self.is_unreachable() { @@ -304,9 +364,32 @@ impl PeerState { if self.liveness.is_dormant() || self.quiet_for(now) < timing.dormant_after { return false; } + self.incidents.dormancies += 1; + self.go_dormant(now); + true + } + + /// The peer said farewell (it is shutting down): Dormant now, without waiting for + /// its silence, and not counted as an incident. Returns true on the transition. + pub(crate) fn say_farewell(&mut self, now: Instant) -> bool { + self.left = true; + if self.liveness.is_dormant() { + return false; + } + self.go_dormant(now); + true + } + + /// A message from `run` of the peer (a HELLO's or HELLO_REPLY's): a run other than + /// the one that said farewell is back (`left`). + pub(crate) fn returned(&mut self, run: crate::transport::RunId) { + if self.run_id != Some(run) { + self.left = false; + } + } + fn go_dormant(&mut self, now: Instant) { self.liveness = Liveness::Dormant { since: now }; - self.incidents.dormancies += 1; self.unreachable_since.get_or_insert(now); self.reset_sync(); @@ -315,7 +398,6 @@ impl PeerState { next_attempt: now + self.probe_backoff.next(), }; } - true } /// The keepalive an established, quiet peer needs, if any (and records it): a PING, @@ -339,6 +421,26 @@ impl PeerState { timing: &Timing, now: Instant, ) -> Option<Keepalive> { + if self.keepalive_due_at(timing).is_none_or(|at| now < at) { + return None; + } + self.last_keepalive_at = Some(now); + let status = self + .last_status_at + .is_none_or(|t| now.saturating_duration_since(t) >= timing.status_refresh); + Some(if status { + Keepalive::Status + } else { + Keepalive::Ping + }) + } + + /// When the next keepalive is due (`take_keepalive_due`), if any is: none to a + /// dormant peer, nor before the handshake. + pub(crate) fn keepalive_due_at(&self, timing: &Timing) -> Option<Instant> { + if self.liveness.is_dormant() || self.hello_state != HelloState::Established { + return None; + } let our_turn = match (self.last_keepalive_at, self.keepalive_received_at) { (None, None) => self.leads, (None, Some(_)) => true, @@ -354,27 +456,44 @@ impl PeerState { } else { timing.keepalive + timing.keepalive / 2 }; - let due = !self.liveness.is_dormant() - && self.hello_state == HelloState::Established - && now.saturating_duration_since(self.added_at) >= timing.keepalive - && self.quiet_for(now) >= quiet_enough - && self - .last_keepalive_at - .is_none_or(|t| now.saturating_duration_since(t) >= timing.keepalive); - if !due { - return None; - } - self.last_keepalive_at = Some(now); - let status = self - .last_status_at - .is_none_or(|t| now.saturating_duration_since(t) >= timing.status_refresh); - Some(if status { - Keepalive::Status - } else { - Keepalive::Ping + let quiet_since = self.last_seen.unwrap_or(self.added_at); + let at = (self.added_at + timing.keepalive).max(quiet_since + quiet_enough); + Some(match self.last_keepalive_at { + Some(sent) => at.max(sent + timing.keepalive), + None => at, }) } + /// When the peer worker next has something to do for this peer + /// (`swarm::peer::run_peer_worker`): a keepalive (to a `neighbour`), dormancy (of a + /// neighbour, once `ready`), a HELLO retry, an overdue sync answer. None if nothing + /// is due until something changes. + pub(crate) fn next_due( + &self, + timing: &Timing, + neighbour: bool, + ready: bool, + ) -> Option<Instant> { + let dormancy = (ready && neighbour && !self.liveness.is_dormant()) + .then(|| self.last_seen.unwrap_or(self.added_at) + timing.dormant_after); + let keepalive = if neighbour { + self.keepalive_due_at(timing) + } else { + None + }; + let hello = match self.hello_state { + HelloState::AwaitingReply { next_attempt } => Some(next_attempt), + _ => None, + }; + let reply = (!self.liveness.is_dormant() && self.hello_state == HelloState::Established) + .then(|| self.sync.reply.as_ref().map(|wait| wait.deadline)) + .flatten(); + [dormancy, keepalive, hello, reply] + .into_iter() + .flatten() + .min() + } + /// Self-healing check on a STATUS or STATUS_REPLY. Returns true if a re-handshake /// should start. pub(crate) fn needs_heal( @@ -454,6 +573,25 @@ mod tests { assert!(silent.liveness.is_dormant()); } + /// After its farewell, a peer stays dormant whatever its stopping run still sends; + /// its next run makes it active again. + #[test] + fn dormant_after_farewell_until_back() { + let timing = Timing::from_keepalive(Duration::from_secs(60), Duration::from_secs(300)); + let mut peer = PeerState::new(true, &timing); + let now = peer.added_at; + peer.run_id = Some([1; 16]); + peer.mark_seen(now); + assert!(peer.say_farewell(now)); + assert!(!peer.mark_seen(now), "a straggler from the same run"); + assert!(peer.liveness.is_dormant() && peer.is_unreachable()); + peer.returned([1; 16]); + assert!(!peer.mark_seen(now), "the same run again"); + peer.returned([2; 16]); + assert!(peer.mark_seen(now), "its next run"); + assert!(!peer.liveness.is_dormant() && !peer.is_unreachable()); + } + /// The two sides of a pair take turns: each sends every other interval. #[test] fn keepalives_take_turns() { diff --git a/src/state/read.rs b/src/state/read.rs index f8323a2..e325e61 100644 --- a/src/state/read.rs +++ b/src/state/read.rs @@ -1,17 +1,20 @@ -//! Serving reads: from the live document, or from the pre-rejoin copy while rejoining. +//! Serving reads: from the live document, or from the pre-rejoin document while +//! rejoining. use std::sync::Arc; -use automerge::{AutoCommit, ChangeHash}; - use super::AppState; use crate::document::replica::Epoch; +use crate::document::{ChangeHash, Document}; use crate::locks::RwLockExt; -/// A read-only view of the document the node served before it started rejoining. +/// The document the node served before it started rejoining, of the epoch it left: +/// reads are answered from it (marked stale), and writes go into it +/// (`AppState::mutate_doc`), until the rejoin replays them into the new epoch or +/// restores this document (`Host::finish_rejoin`, `Host::abandon_rejoin`). pub(crate) struct StaleView { pub(crate) epoch: Epoch, - pub(crate) doc: Arc<AutoCommit>, + pub(crate) doc: Arc<std::sync::RwLock<Document>>, } /// Where a read was served from. @@ -33,7 +36,7 @@ impl AppState { &self, unready_live: bool, with_heads: bool, - read: impl FnOnce(&AutoCommit) -> T + Send + 'static, + read: impl FnOnce(&Document) -> T + Send + 'static, finish: impl FnOnce(T) -> R + Send + 'static, ) -> Result<(R, ReadSource, Option<Vec<ChangeHash>>), axum::http::StatusCode> where @@ -58,16 +61,19 @@ impl AppState { return Err(axum::http::StatusCode::SERVICE_UNAVAILABLE); }; - let rendered = tokio::task::spawn_blocking(move || finish(read(&doc))) - .await - .expect("document read task panicked"); + let rendered = tokio::task::spawn_blocking(move || { + let data = read(&doc.read_or_recover()); + finish(data) + }) + .await + .expect("document read task panicked"); Ok((rendered, ReadSource::Stale(epoch), None)) } async fn render_live<T, R>( &self, with_heads: bool, - read: impl FnOnce(&AutoCommit) -> T + Send + 'static, + read: impl FnOnce(&Document) -> T + Send + 'static, finish: impl FnOnce(T) -> R + Send + 'static, ) -> (R, Option<Vec<ChangeHash>>) where @@ -94,4 +100,12 @@ impl AppState { pub(super) fn set_stale(&self, view: Option<StaleView>) { *self.stale.write_or_recover() = view; } + + /// The stale view's document and epoch, if there is one. + pub(crate) fn stale_document(&self) -> Option<(Epoch, Arc<std::sync::RwLock<Document>>)> { + self.stale + .read_or_recover() + .as_ref() + .map(|view| (view.epoch, Arc::clone(&view.doc))) + } } diff --git a/src/state/timing.rs b/src/state/timing.rs index 9055ed6..af5c0c0 100644 --- a/src/state/timing.rs +++ b/src/state/timing.rs @@ -4,15 +4,18 @@ use std::time::Duration; use crate::backoff::Backoff; -/// First wait for a sync answer; generous, since a transport may take a while with large -/// messages (Mycelium sends them in many small packets). -const SYNC_REPLY_TIMEOUT: Duration = Duration::from_secs(10); +/// First wait for a sync answer, unless a transport needs longer +/// (`Transport::sync_reply_timeout`); generous, since a transport may take a while with +/// large messages (Mycelium sends them in many small packets). +pub(crate) const SYNC_REPLY_TIMEOUT: Duration = Duration::from_secs(10); /// Upper bound, as a multiple of the first wait: 5 minutes, about as long as a transport /// tries to deliver a message (`Transport::delivery_limit`). const SYNC_REPLY_TIMEOUT_MAX_FACTOR: u32 = 30; /// Upper bound on the wait between probes of an unreachable peer, as a multiple of -/// `dormant_probe`: an hour at the default keepalive (1m). A returning node HELLOs its own -/// neighbours, so this only bounds how long one we never hear from stays out. +/// `dormant_probe`, unless `--dormant-probe-max` says otherwise: an hour at the default +/// keepalive (1m). A returning node HELLOs its own neighbours, and once one pair across +/// a partition meets, the rest are probed at once (`handshake::probe_now`), so this +/// bounds how long one we never hear from stays out. const PROBE_MAX_FACTOR: u32 = 6; #[derive(Clone, Copy, Debug)] @@ -25,6 +28,8 @@ pub(crate) struct Timing { pub(crate) dormant_after: Duration, pub(crate) resync_after: Duration, pub(crate) dormant_probe: Duration, + /// The longest wait between probes (`probe_backoff`). + pub(crate) probe_max: Duration, /// While catching up: send the source a STATUS after this much silence (`swarm::source`). pub(crate) catch_up_probe: Duration, /// First wait for an answer to a sync message before sending again; doubles while @@ -43,6 +48,12 @@ impl Timing { self.keepalive * 5 } + /// [`new`](Self::new) with the default first sync reply timeout. + #[cfg(test)] + pub(crate) fn from_keepalive(keepalive: Duration, delivery_limit: Duration) -> Self { + Self::new(keepalive, SYNC_REPLY_TIMEOUT, delivery_limit) + } + /// The timings for `--keepalive`. /// /// Each node keeps about `k` neighbours (`--neighbours`) whatever the swarm's size, @@ -59,13 +70,9 @@ impl Timing { /// | LAN, servers | 15–30 s | a returning or silent peer is noticed within a minute or two | /// | WAN, mobile, metered (the default, 1m) | 60–120 s | about 0.4 messages per second per node at `k` = 12 and 1m | /// - /// `delivery_limit`: the transport's (`Transport::delivery_limit`). - pub(crate) fn from_keepalive(keepalive: Duration, delivery_limit: Duration) -> Self { - Self::new(keepalive, SYNC_REPLY_TIMEOUT, delivery_limit) - } - - /// [`from_keepalive`](Self::from_keepalive) with another first sync reply timeout: - /// for tests, whose messages don't travel for seconds. + /// `sync_reply_timeout`: the transports' (`Transport::sync_reply_timeout`), or for + /// tests, whose messages don't travel for seconds, a shorter one. `delivery_limit`: + /// the transports' (`Transport::delivery_limit`). pub(crate) fn new( keepalive: Duration, sync_reply_timeout: Duration, @@ -81,6 +88,7 @@ impl Timing { // the first sync reply timeout, so a lost message's sender resends it first. resync_after: (keepalive * 2).max(sync_reply_timeout * 2), dormant_probe: keepalive * 10, // >= 10*: dormant peers cost almost nothing, but still rejoin within minutes + probe_max: keepalive * 10 * PROBE_MAX_FACTOR, // Seconds, not a keepalive interval: a dead source should cost little time. // Only the source gets these, and only while catching up. catch_up_probe: (keepalive / 30).clamp(Duration::from_secs(1), Duration::from_secs(5)), @@ -89,10 +97,18 @@ impl Timing { } } + /// `--dormant-probe-max`, if given: the longest wait between probes. + pub(crate) fn with_probe_max(mut self, max: Option<Duration>) -> Self { + if let Some(max) = max { + self.probe_max = max; + } + self + } + /// Waits between probes of an unreachable peer: `dormant_probe`, doubling up to - /// `PROBE_MAX_FACTOR` times as much. + /// `probe_max` (all of them `probe_max` if that is shorter). pub(crate) fn probe_backoff(&self) -> Backoff { - Backoff::new(self.dormant_probe, self.dormant_probe * PROBE_MAX_FACTOR) + Backoff::new(self.dormant_probe.min(self.probe_max), self.probe_max) } /// Waits for sync answers: `sync_reply_timeout`, doubling up to 30 times as much. diff --git a/src/swarm/auth.rs b/src/swarm/auth.rs index 088f615..98a004c 100644 --- a/src/swarm/auth.rs +++ b/src/swarm/auth.rs @@ -21,7 +21,7 @@ use std::sync::Mutex; use std::time::{Duration, Instant}; use crate::locks::MutexExt; -use anyhow::{Context, Result, bail}; +use anyhow::{Result, bail}; use sha2::{Digest, Sha256}; const HELLO_LABEL: &[u8] = b"continuum/hello/1"; @@ -40,37 +40,25 @@ struct Rejections { suppressed: u64, } -/// A key is a secret: a file others may read is worth a warning (not an error: a -/// group may be deliberate, e.g. shared with a deployment tool). `what`: which key. -pub(crate) fn warn_if_readable_by_others(path: &Path, what: &str) { - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - if let Ok(metadata) = std::fs::metadata(path) { - let mode = metadata.permissions().mode(); - if mode & 0o077 != 0 { - tracing::warn!( - path = %path.display(), - mode = format!("{:o}", mode & 0o777), - "The {what} file is readable by others than its owner; chmod 600 it" - ); - } - } - } - #[cfg(not(unix))] - let _ = (path, what); -} - /// At most one rejection log line per this interval. const REJECTION_LOG_INTERVAL: Duration = Duration::from_secs(10); impl SwarmKey { /// Reads the key from a file; its bytes are the key, as they are. Generate one with /// e.g. `head -c 32 /dev/urandom > swarm.key`. - pub(crate) fn from_file(path: &Path) -> Result<Self> { - let key = std::fs::read(path) - .with_context(|| format!("Reading the swarm key from {}", path.display()))?; - warn_if_readable_by_others(path, "swarm key"); + /// With `create`, a new one (32 random bytes) if there is no file at `path`. + pub(crate) fn from_file(path: &Path, create: bool) -> Result<Self> { + let (key, created) = crate::files::key_file( + path, + "swarm key", + create, + "swarm-key-create", + "on the swarm's first node; the others need a copy of it", + 32, + )?; + if created { + tracing::info!(path = %path.display(), "Created a new swarm key: copy it to the swarm's other nodes"); + } if key.len() < 16 { bail!( "The swarm key in {} has {} bytes; use at least 16 (32 random bytes recommended)", @@ -172,7 +160,18 @@ mod tests { use super::*; fn hex(bytes: &[u8]) -> String { - bytes.iter().map(|byte| format!("{byte:02x}")).collect() + crate::transport::Hex(bytes).to_string() + } + + /// A swarm key file is made only when asked, then read as any other. + #[test] + fn swarm_key_created_only_when_asked() { + let path = crate::files::scratch_dir("swarm-key-create").join("swarm.key"); + assert!(SwarmKey::from_file(&path, false).is_err()); + assert!(!path.exists()); + let created = SwarmKey::from_file(&path, true).unwrap(); + assert_eq!(created.key.len(), 32); + assert_eq!(SwarmKey::from_file(&path, false).unwrap().key, created.key); } #[test] diff --git a/src/swarm/blob.rs b/src/swarm/blob.rs index ec99c93..ce9a2bd 100644 --- a/src/swarm/blob.rs +++ b/src/swarm/blob.rs @@ -27,12 +27,12 @@ use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::{Arc, Mutex, MutexGuard}; use std::time::Duration; -use anyhow::{Result, anyhow, bail}; -use automerge::{AutoCommit, ChangeHash}; +use anyhow::{Context, Result, bail}; use tokio::sync::{Semaphore, mpsc}; use tokio::time::Instant; use tracing::{debug, info, warn}; +use crate::document::{ChangeHash, changes}; use crate::locks::MutexExt; use crate::state::AppState; use crate::transport::{BlobBody, BlobId, BlobMessage, DeltaSpec, ParsedMessage}; @@ -87,6 +87,10 @@ pub(crate) struct Blobs { /// lost message costs too much to send again whole, and a slow link may not deliver it /// within the transport's delivery limit. const DELTA_THRESHOLD_CHUNKS: usize = 16; +/// The longest a sync message should take whole at the throughput measured to its peer +/// (`delta_threshold_for`): well within what a transport allows a message before its send +/// fails (10 s, plus its size at 1 MiB/s, on libp2p). +const WHOLE_WITHIN: Duration = Duration::from_secs(5); /// The largest sync message accepted: the threshold of a sender with the largest chunks /// it may have configured. Anything larger comes from a node that doesn't keep to it. @@ -115,6 +119,25 @@ impl Blobs { DELTA_THRESHOLD_CHUNKS * self.chunk_size as usize } + /// [`delta_threshold`](Self::delta_threshold) for one peer: lower where a message + /// that large wouldn't make it whole. `rate`: the measured throughput to it, bytes + /// per second (`swarm::throughput`), at which a message whole should take at most + /// `WHOLE_WITHIN`; `failed`: the largest message to it that didn't make it lately + /// (`PeerState::failed_whole`), of which messages half as large go as deltas, if it + /// was larger than two chunks (smaller ones are lost for other reasons than their + /// size). Never below a chunk: a delta then is no smaller per message. + pub(crate) fn delta_threshold_for(&self, rate: Option<f64>, failed: Option<usize>) -> usize { + let chunk = self.chunk_size as usize; + let mut threshold = self.delta_threshold(); + if let Some(rate) = rate.filter(|rate| rate.is_finite() && *rate > 0.0) { + threshold = threshold.min((rate * WHOLE_WITHIN.as_secs_f64()) as usize); + } + if let Some(failed) = failed.filter(|failed| *failed > 2 * chunk) { + threshold = threshold.min(failed / 2); + } + threshold.max(chunk) + } + /// Starts downloading a delta offered by `peer`; false if one from it is running. pub(crate) fn start_delta(&self, peer: &str) -> bool { self.deltas.lock_or_recover().insert(peer.to_owned()) @@ -216,21 +239,18 @@ async fn prepare(state: &AppState, id: BlobId, delta: Option<DeltaSpec>) -> Opti // The spec must be the one the ID names. let spec = delta.filter(|spec| spec.id() == delta_id)?; let doc = Arc::clone(&state.doc); - tokio::task::spawn_blocking(move || delta_bytes(&mut doc.blocking_write().doc, &spec)) - .await - .ok() - .flatten() - .map(Arc::new)? + tokio::task::spawn_blocking(move || { + changes::delta_bytes(&doc.blocking_read().doc, &spec.from, &spec.to) + }) + .await + .ok() + .flatten() + .map(Arc::new)? } BlobId::Change(hash) => { let doc = Arc::clone(&state.doc); tokio::task::spawn_blocking(move || { - // get_change_by_hash needs the document mutably. - let mut replica = doc.blocking_write(); - replica - .doc - .get_change_by_hash(&hash) - .map(|change| change.raw_bytes().to_vec()) + changes::change_bytes(&doc.blocking_read().doc, &hash) }) .await .ok() @@ -586,14 +606,9 @@ pub(crate) async fn fetch_change( state: &AppState, hash: ChangeHash, peers: Vec<String>, -) -> Result<automerge::Change> { +) -> Result<crate::document::Change> { let bytes = fetch(state, BlobId::Change(hash), peers, None, None).await?; - let change = automerge::Change::from_bytes(bytes) - .map_err(|error| anyhow!("downloaded change {hash} doesn't load: {error}"))?; - if change.hash() != hash { - bail!("downloaded change has hash {}, not {hash}", change.hash()); - } - Ok(change) + changes::change_from_bytes(bytes, hash).context("downloaded") } /// Downloads the changes of `spec` from `peers` (any that hold its `to`). The caller @@ -606,120 +621,9 @@ pub(crate) async fn fetch_delta( fetch(state, BlobId::Delta(spec.id()), peers, None, Some(spec)).await } -/// The bytes of delta `spec` from `doc`: the changes in the history of `spec.to` but -/// not of `spec.from`, each as Automerge encodes it, sorted by hash and concatenated -/// (`AutoCommit::load_incremental` reads that). The same on every node holding `to`, -/// whatever else it has. None if `doc` lacks part of `to`. -pub(crate) fn delta_bytes(doc: &mut AutoCommit, spec: &DeltaSpec) -> Option<Vec<u8>> { - if spec - .to - .iter() - .any(|head| doc.get_change_by_hash(head).is_none()) - { - return None; - } - // Heads of `from` we don't have can't hide anything we have. - let from: Vec<ChangeHash> = spec - .from - .iter() - .copied() - .filter(|head| doc.get_change_by_hash(head).is_some()) - .collect(); - let since: HashMap<ChangeHash, (Vec<ChangeHash>, Vec<u8>)> = doc - .get_changes(&from) - .iter() - .map(|change| { - ( - change.hash(), - (change.deps().to_vec(), change.raw_bytes().to_vec()), - ) - }) - .collect(); - // Of those, the ones `to` depends on: what we have beyond `to` stays out. - let mut included: HashSet<ChangeHash> = HashSet::new(); - let mut pending: Vec<ChangeHash> = spec.to.clone(); - while let Some(hash) = pending.pop() { - if let Some((deps, _)) = since.get(&hash) - && included.insert(hash) - { - pending.extend(deps.iter().copied()); - } - } - let mut hashes: Vec<ChangeHash> = included.into_iter().collect(); - hashes.sort_unstable(); - let mut bytes = Vec::new(); - for hash in hashes { - bytes.extend_from_slice(&since[&hash].1); - } - Some(bytes) -} - #[cfg(test)] mod tests { use super::*; - use automerge::{ActorId, ROOT, transaction::Transactable}; - - fn random_write(rng: &mut crate::testrand::Rng, doc: &mut AutoCommit) { - let key = format!("k{}", rng.below(6)); - doc.put(ROOT, key, rng.below(100) as i64).unwrap(); - doc.commit(); - } - - fn sorted(mut heads: Vec<ChangeHash>) -> Vec<ChangeHash> { - heads.sort_unstable(); - heads - } - - /// Over random histories: two nodes that reached the same heads through different - /// merge orders, one with more on top, produce the same bytes for a delta to those - /// heads; a document at `from` that loads them is at `to`. - #[test] - fn delta_bytes_are_the_same_everywhere_and_complete() { - crate::testrand::for_seeds(200, |rng| { - let actor = |i: u8| ActorId::from(&[i; 16][..]); - let mut base = AutoCommit::new().with_actor(actor(0)); - for _ in 0..rng.between(0, 5) { - random_write(rng, &mut base); - } - let from = base.get_heads(); - let mut writers: Vec<AutoCommit> = (1..=rng.between(1, 3)) - .map(|i| base.fork().with_actor(actor(i as u8))) - .collect(); - for _ in 0..rng.between(1, 15) { - let writer = rng.below(writers.len()); - random_write(rng, &mut writers[writer]); - } - let mut one = base.fork().with_actor(actor(10)); - let mut two = base.fork().with_actor(actor(11)); - for writer in writers.iter_mut() { - one.merge(writer).unwrap(); - } - for writer in writers.iter_mut().rev() { - two.merge(writer).unwrap(); - } - let to = one.get_heads(); - assert_eq!(sorted(to.clone()), sorted(two.get_heads())); - // Beyond `to`: must stay out of the delta. - random_write(rng, &mut two); - - let spec = DeltaSpec::new(from.clone(), to.clone()); - let bytes = delta_bytes(&mut one, &spec).unwrap(); - assert_eq!(delta_bytes(&mut two, &spec).unwrap(), bytes); - - let mut behind = base.fork(); - behind.load_incremental(&bytes).unwrap(); - assert_eq!(sorted(behind.get_heads()), sorted(to)); - }); - } - - #[test] - fn no_delta_without_its_heads() { - let mut doc = AutoCommit::new(); - doc.put(ROOT, "k", 1_i64).unwrap(); - doc.commit(); - let spec = DeltaSpec::new(Vec::new(), vec![ChangeHash([7; 32])]); - assert!(delta_bytes(&mut doc, &spec).is_none()); - } #[test] fn chunking() { diff --git a/src/swarm/bootstrap.rs b/src/swarm/bootstrap.rs index a9ab1b2..4f6c072 100644 --- a/src/swarm/bootstrap.rs +++ b/src/swarm/bootstrap.rs @@ -18,7 +18,6 @@ use std::collections::HashSet; use std::path::Path; use anyhow::{Context, Result, bail}; -use automerge::{ActorId, AutoCommit}; use serde_json::Value as JsonValue; use tokio::time::Instant; use tracing::{debug, info, warn}; @@ -28,13 +27,13 @@ use crate::document; use crate::document::meta::{self, SWARM}; use crate::document::replica::Epoch; use crate::document::store; +use crate::document::{ActorId, Document}; use crate::files::{FileKind, Files}; use crate::force::Force; use crate::remake::snapshot; use crate::state::AppState; use crate::swarm::peer::{remember_peer, send_hello_once}; -const BOOTSTRAP_POLL: Duration = Duration::from_millis(500); /// How often a node that is still joining says why. const BOOTSTRAP_REPORT: Duration = Duration::from_secs(300); /// Retrying seeds that didn't resolve: first after this, doubling up to the maximum. @@ -112,6 +111,7 @@ pub(crate) async fn bootstrap(state: &AppState, seeds: &[String], start: Start) let mut resolve_retry = RESOLVE_RETRY; let mut next_resolve = Instant::now(); let mut next_report = Instant::now() + BOOTSTRAP_REPORT; + state.transport.on_seeds_found(state.join_progress.clone()); loop { if let Some(swarm_id) = state.swarm_id().await { // A node joining a compacted swarm rejoins the newer epoch first; it is @@ -147,7 +147,19 @@ pub(crate) async fn bootstrap(state: &AppState, seeds: &[String], start: Start) next_report = now + BOOTSTRAP_REPORT; report_joining(state, seeds, &unresolved).await; } - tokio::time::sleep(BOOTSTRAP_POLL).await; + // Until something changes (`join_progress`), or it's time to resolve or report. + let resolving = async { + if unresolved.is_empty() { + std::future::pending().await + } else { + tokio::time::sleep_until(next_resolve).await + } + }; + tokio::select! { + () = state.join_progress.notified() => {} + () = resolving => {} + () = tokio::time::sleep_until(next_report) => {} + } } } @@ -156,7 +168,7 @@ fn swarm_id_path() -> Vec<String> { } /// The swarm ID the document names, if any. -fn read_swarm_id(doc: &AutoCommit) -> Option<String> { +fn read_swarm_id(doc: &Document) -> Option<String> { match store::read_path(doc, &swarm_id_path()) { Ok(Some(JsonValue::String(id))) if !id.is_empty() => Some(id), _ => None, @@ -184,7 +196,7 @@ impl Origin { /// A document the node starts from, ready at once: on the swarm and epoch it names. pub(crate) struct StartingDocument { /// Under a fresh actor. - pub(crate) doc: AutoCommit, + pub(crate) doc: Document, pub(crate) epoch: Epoch, pub(crate) swarm_id: String, pub(crate) origin: Origin, @@ -281,7 +293,7 @@ pub(crate) fn starting_document( /// The swarm and epoch `doc` is on. fn identify( - mut doc: AutoCommit, + doc: Document, origin: Origin, earlier_actors: Vec<ActorId>, ) -> Result<StartingDocument> { @@ -289,8 +301,7 @@ fn identify( let Some(swarm_id) = read_swarm_id(&doc) else { bail!("{what} names no swarm (no _continuum/swarm); it predates swarm IDs in documents"); }; - let epoch = - snapshot::epoch_of(&mut doc).map_err(|reason| anyhow::anyhow!("{what}: {reason}"))?; + let epoch = snapshot::epoch_of(&doc).map_err(|reason| anyhow::anyhow!("{what}: {reason}"))?; Ok(StartingDocument { doc, epoch, @@ -300,15 +311,13 @@ fn identify( }) } -/// A backup to recover from, as `GET /admin/document` saves it, under a fresh actor. -fn load_backup(path: &Path, files: &Files) -> Result<AutoCommit> { +/// A backup to recover from, as `GET /admin/document` saves it (`document::load`). +fn load_backup(path: &Path, files: &Files) -> Result<Document> { let bytes = files .read(FileKind::Backup, path) .with_context(|| format!("Reading the backup {}", path.display()))? .with_context(|| format!("The backup {} doesn't exist", path.display()))?; - Ok(AutoCommit::load(&bytes) - .with_context(|| format!("Loading the backup {}", path.display()))? - .with_actor(ActorId::random())) + crate::document::load(&bytes).with_context(|| format!("Loading the backup {}", path.display())) } /// Resolves the seeds in `unresolved`, says HELLO to each that resolves, and keeps the diff --git a/src/swarm/changes.rs b/src/swarm/changes.rs new file mode 100644 index 0000000..0f22f52 --- /dev/null +++ b/src/swarm/changes.rs @@ -0,0 +1,79 @@ +//! The change publisher: after document changes, which store paths they touched +//! (`AppState::change_events`, for the API's streams, the dashboard and remakes), and +//! a nudge to topology when the member list changed, and to the peer worker when the +//! forget records did. + +use std::sync::Arc; + +use tracing::{info, warn}; + +use crate::document::changes; +use crate::document::store::{ChangeSet, InLists}; +use crate::state::AppState; + +pub(crate) async fn run_change_publisher(state: AppState) { + let mut changed = state.change_tx.subscribe(); + // The replica's heads (sorted, as of its last refresh: every change refreshes them). + let mut last_heads = state.doc.read().await.heads().to_vec(); + + info!("Starting change publisher"); + + while changed.changed().await.is_ok() { + let before = last_heads.clone(); + + let doc = Arc::clone(&state.doc); + let (heads, paths, conflicts) = tokio::task::spawn_blocking(move || { + let replica = doc.blocking_read(); + let heads = replica.heads().to_vec(); + if heads == before { + return (heads, Vec::new(), Vec::new()); + } + + // After an epoch switch the previous heads belong to the old document: + // report the root as changed, which every stream is related to. + if !changes::missing(&replica.doc, &before).is_empty() { + return (heads, vec![Vec::new()], Vec::new()); + } + + let mut paths = Vec::new(); + let mut conflicts = Vec::new(); + for changed in replica.changed_since(&before, InLists::Index) { + if changed.conflict { + conflicts.push(changed.path.clone()); + } + paths.push(changed.path); + } + (heads, paths, conflicts) + }) + .await + .expect("change publisher task panicked"); + + if heads == last_heads { + continue; + } + last_heads = heads; + + for path in &conflicts { + warn!(path = ?path, "Concurrent writes produced a conflict"); + } + + if paths + .iter() + .any(|path| crate::swarm::topology::affects_neighbours(path)) + { + state.topology_changed.notify_one(); + } + if paths + .iter() + .any(|path| crate::swarm::peer::affects_forget_records(path)) + { + state + .forget_records_changed + .store(true, std::sync::atomic::Ordering::Release); + state.peers_due.now(); + } + + // Fails only when nobody is subscribed. + let _ = state.change_events.send(Arc::new(ChangeSet { paths })); + } +} diff --git a/src/swarm/document_file.rs b/src/swarm/document_file.rs new file mode 100644 index 0000000..e9deca0 --- /dev/null +++ b/src/swarm/document_file.rs @@ -0,0 +1,110 @@ +//! The worker that keeps the document file (`document::file`) up to date: it looks at +//! the node (ready, the document's epoch and heads, this node's actors), and the file +//! says what is due. While a ready node rejoins, the file follows the document before +//! the rejoin, which takes the node's writes meanwhile (`state::StaleView`): a restart +//! then resumes on it, and rejoins again. + +use std::sync::Arc; + +use crate::document::Document; +use crate::document::file::Due; +use crate::document::replica::Epoch; +use crate::locks::RwLockExt; +use crate::state::AppState; + +/// Keeps the document file up to date. Runs for as long as the node does. +pub(crate) async fn run(state: AppState) { + let Some(file) = state.document_file.clone() else { + return; + }; + let mut tick = tokio::time::interval(file.append_every()); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + tick.tick().await; + flush(&state).await; + } +} + +/// Writes what is due now (`DocumentFile::due`). Also called at shutdown. +pub(crate) async fn flush(state: &AppState) { + let Some(file) = state.document_file.clone() else { + return; + }; + if state.remake.is_rejoining() { + if let Some((epoch, stale)) = state.stale_document() { + flush_before_rejoin(&file, epoch, stale).await; + } + return; + } + if !state.is_ready().await { + return; + } + let own = state.remake.own_actors(); + let (epoch, heads) = { + let replica = state.doc.read().await; + (replica.epoch, replica.heads().to_vec()) + }; + match file.due(epoch, own.len(), &heads) { + Due::Rewrite(reason) => { + let (epoch, copy) = { + let replica = state.doc.read().await; + (replica.epoch, replica.doc.clone()) + }; + file.rewrite(epoch, copy, own, reason).await; + } + Due::Append { epoch, since } => { + let (heads, increment) = { + let replica = state.doc.read().await; + // Switched since: the next flush rewrites the file instead. + if replica.epoch != epoch { + return; + } + let heads = replica.heads().to_vec(); + (heads, crate::document::save_since(&replica.doc, &since)) + }; + file.append(since, heads, increment).await; + } + Due::Nothing => {} + } +} + +/// [`flush`] of the document before a rejoin (`epoch`'s), while it takes this node's +/// writes. Only appends: a rewrite (another epoch, or the increments outgrew the full +/// save) waits for the end of the rejoin, so the file keeps the actors it had. +async fn flush_before_rejoin( + file: &Arc<crate::document::file::DocumentFile>, + epoch: Epoch, + stale: Arc<std::sync::RwLock<Document>>, +) { + let heads = { + let stale = Arc::clone(&stale); + // Sorted, as the live document's (`Replica::heads`) the file compares with. + tokio::task::spawn_blocking(move || { + let mut heads = stale.read_or_recover().get_heads(); + heads.sort_unstable(); + heads + }) + .await + .expect("document read task panicked") + }; + let Some(actors) = file.actors() else { + return; + }; + if let Due::Append { + epoch: written, + since, + } = file.due(epoch, actors, &heads) + && written == epoch + { + let since_then = since.clone(); + let (heads, increment) = tokio::task::spawn_blocking(move || { + let doc = stale.read_or_recover(); + let mut heads = doc.get_heads(); + heads.sort_unstable(); + (heads, crate::document::save_since(&doc, &since_then)) + }) + .await + .expect("document read task panicked"); + file.append(since, heads, increment).await; + } +} diff --git a/src/swarm/links.rs b/src/swarm/links.rs new file mode 100644 index 0000000..b84f57a --- /dev/null +++ b/src/swarm/links.rs @@ -0,0 +1,453 @@ +//! How members are linked, for a map of the swarm (the dashboard, `api::dashboard`): a +//! node asks a member with LINKS, and the member answers with a LINKS_REPLY listing its +//! neighbours and the peers it holds dormant (`own_links`), each with the transport its +//! messages go over, whether only through a relay, the round trip and whether it is +//! dormant: as the member measures them, which no other node can. +//! +//! Every node answers, whatever its build: one with a dashboard asks any member, which +//! may have none. Only members (in the member list) are answered, as the links name +//! members and where they're reached; and each at most every `ANSWER_EVERY`. Nothing +//! is written to the document: asked for while someone looks, forgotten after. +//! +//! A LINKS may subscribe (`subscribe_secs`): the member then sends a LINKS_REPLY again +//! whenever its links change (`changed`: a peer, a transport, relayed, dormant, or a +//! round trip that moved by a fifth), looked at every `WATCH_EVERY` while anyone +//! subscribes (round trips change without telling), until the subscription runs out +//! (the asker renews it with its next LINKS, so one that is gone stops it by not +//! renewing) or a LINKS with 0 ends it. + +use std::collections::HashMap; +use std::sync::Mutex; +use std::time::Duration; + +use tokio::time::Instant; +use tracing::debug; + +use crate::locks::MutexExt; +use crate::state::AppState; +use crate::swarm::{members, topology}; +use crate::transport::paths::{self, ClockOffset}; +use crate::transport::{ + Connection, Destination, Link, LinksReply, LinksRequest, OtherPath, ParsedMessage, +}; + +/// Links one reply lists at most (a member has about `--neighbours`, more if others +/// prefer it). +pub(crate) const MAX_LINKS: usize = 256; +/// Other transports in a link at most. +pub(crate) const MAX_OTHERS: usize = 8; +/// A member answers each other one at most this often. +const ANSWER_EVERY: Duration = Duration::from_secs(5); +/// Replies kept from members at most (one each), and requests out. +const MAX_KEPT: usize = 4096; +/// Subscriptions a member keeps at most, and how long one lasts at most. +const MAX_SUBSCRIBERS: usize = 64; +const MAX_SUBSCRIPTION: Duration = Duration::from_secs(300); +/// How often a member with subscribers looks whether its links changed. +const WATCH_EVERY: Duration = Duration::from_secs(2); +/// A reply to a LINKS that doesn't subscribe is taken for this long. +const REPLY_WITHIN: Duration = Duration::from_secs(30); + +/// The requests out and the replies in, and when each asker was last answered. +#[derive(Default)] +pub(crate) struct Links { + inner: Mutex<Inner>, + /// Someone subscribed (`answer`): `watch` looks again. + subscribed: tokio::sync::Notify, +} + +#[derive(Default)] +struct Inner { + /// Our LINKS out, by member: its nonce, and until when replies to it are taken (its + /// subscription's end, or `REPLY_WITHIN`). + asked: HashMap<String, (u64, Instant)>, + /// The latest LINKS_REPLY of each member: when, and its links. + replies: HashMap<String, (Instant, Vec<Link>)>, + /// When we last answered each member. + answered: HashMap<String, Instant>, + /// Who subscribed to our links: the nonce to answer with, and until when. + subscribers: HashMap<String, (u64, Instant)>, + /// The links subscribers were last sent (`watch`). + published: Option<Vec<Link>>, +} + +impl Links { + /// The latest links each member replied with, and how long ago. + #[cfg_attr(not(feature = "dashboard"), allow(dead_code))] + pub(crate) fn replies(&self) -> Vec<(String, Duration, Vec<Link>)> { + let now = Instant::now(); + self.inner + .lock_or_recover() + .replies + .iter() + .map(|(member, (at, links))| { + ( + member.clone(), + now.saturating_duration_since(*at), + links.clone(), + ) + }) + .collect() + } + + /// The members our subscriptions are with (not run out). + #[cfg_attr(not(feature = "dashboard"), allow(dead_code))] + pub(crate) fn subscribed(&self) -> Vec<String> { + let now = Instant::now(); + self.inner + .lock_or_recover() + .asked + .iter() + .filter(|(_, (_, until))| now < *until) + .map(|(member, _)| member.clone()) + .collect() + } +} + +/// This node's links: its neighbours, and the peers it holds dormant, as it measures +/// them. A neighbour that goes dormant is one no more (it is unreachable now), but its +/// link stays listed, dormant, until it is heard from again: left out, the link would +/// show only as that peer last reported it, from before. +pub(crate) async fn own_links(state: &AppState) -> Vec<Link> { + let mut links = Vec::new(); + for (peer, peer_state) in state.peers_snapshot().await { + let dormant = peer_state.lock().await.liveness.is_dormant(); + if !dormant && !topology::is_neighbour(state, &peer) { + continue; + } + let destination = state.transport.destination(&peer); + let transport = destination + .as_ref() + .and_then(|destination| scheme_of(&destination.address)); + links.push(Link { + others: others(destination.as_ref(), transport.as_deref()), + transport, + relayed: destination + .as_ref() + .is_some_and(|destination| destination.connection == Some(Connection::Relayed)), + // Ranked up before down: the first is down only if all are. + down: destination + .as_ref() + .and_then(|destination| destination.paths.first()) + .is_some_and(|path| path.down), + // The path in use's (`Transport::rtt`, the first of `destination.paths`). + rtt_us: state + .transport + .rtt(&peer) + .map(|rtt| u32::try_from(rtt.as_micros()).unwrap_or(u32::MAX)), + dormant, + clock: state.transport.clock_offset(&peer), + peer, + }); + } + links.sort_by(|a, b| a.peer.cmp(&b.peer)); + links.truncate(MAX_LINKS); + links +} + +/// The scheme of `address` (`mycelium:…`): its transport (`paths::scheme_of`), if it +/// has one. +fn scheme_of(address: &str) -> Option<String> { + Some(paths::scheme_of(address)) + .filter(|scheme| !scheme.is_empty()) + .map(str::to_owned) +} + +/// The transports besides `used` the peer proved itself at, with a path up: each with +/// the round trip of its fastest path, if any is measured, and whether all of them go +/// through a relay. +fn others(destination: Option<&Destination>, used: Option<&str>) -> Vec<OtherPath> { + let mut others: Vec<OtherPath> = Vec::new(); + for path in destination.map_or(&[][..], |destination| &destination.paths[..]) { + let Some(transport) = scheme_of(&path.address) else { + continue; + }; + if path.down || used == Some(transport.as_str()) { + continue; + } + let rtt_us = path + .rtt_ms + .filter(|ms| ms.is_finite() && *ms >= 0.0) + .map(|ms| (ms * 1000.0).min(f64::from(u32::MAX)) as u32); + let relayed = path.connection == Some(Connection::Relayed); + match others.iter_mut().find(|other| other.transport == transport) { + Some(other) => { + other.rtt_us = match (other.rtt_us, rtt_us) { + (Some(a), Some(b)) => Some(a.min(b)), + (a, b) => a.or(b), + }; + other.relayed &= relayed; + } + None => others.push(OtherPath { + transport, + rtt_us, + relayed, + }), + } + } + others.sort_by(|a, b| a.transport.cmp(&b.transport)); + others.truncate(MAX_OTHERS); + others +} + +/// Asks `member` for its links (a LINKS), and with `subscribe` for its changes for +/// that long; its replies land in `AppState::links`. +#[cfg_attr(not(feature = "dashboard"), allow(dead_code))] +pub(crate) async fn ask(state: &AppState, member: &str, subscribe: Duration) { + let nonce = crate::transport::paths::random_nonce(); + { + let mut inner = state.links.inner.lock_or_recover(); + if inner.asked.len() >= MAX_KEPT && !inner.asked.contains_key(member) { + inner.asked.clear(); + } + let until = Instant::now() + subscribe.max(REPLY_WITHIN); + inner.asked.insert(member.to_owned(), (nonce, until)); + } + let (epoch, epoch_seq) = state.epoch_stamp(); + let request = ParsedMessage::Links(LinksRequest { + epoch, + epoch_seq, + nonce, + subscribe_secs: u32::try_from(subscribe.as_secs()).unwrap_or(u32::MAX), + }); + if let Err(error) = state.transport.send_message(member, &request).await { + debug!(member = %member, error = %format!("{error:#}"), "LINKS failed"); + } +} + +/// Ends our subscription with `member` (a LINKS with 0); its replies aren't taken any +/// more. +#[cfg_attr(not(feature = "dashboard"), allow(dead_code))] +pub(crate) async fn unsubscribe(state: &AppState, member: &str) { + let nonce = { + let mut inner = state.links.inner.lock_or_recover(); + match inner.asked.remove(member) { + Some((nonce, _)) => nonce, + None => return, + } + }; + let (epoch, epoch_seq) = state.epoch_stamp(); + let request = ParsedMessage::Links(LinksRequest { + epoch, + epoch_seq, + nonce, + subscribe_secs: 0, + }); + if let Err(error) = state.transport.send_message(member, &request).await { + debug!(member = %member, error = %format!("{error:#}"), "LINKS failed"); + } +} + +/// A LINKS from `peer`: answered if it's a member, and not answered lately. +pub(crate) async fn answer(state: &AppState, peer: &str, request: LinksRequest) { + let member = { + let replica = state.doc.read().await; + members::members(&replica.doc) + .iter() + .any(|member| member == peer) + }; + if !member { + debug!(peer = %peer, "Not answering LINKS from a peer that isn't a member"); + return; + } + { + let now = Instant::now(); + let mut inner = state.links.inner.lock_or_recover(); + // A subscription is taken, renewed or ended even when the reply is due later. + if request.subscribe_secs == 0 { + inner.subscribers.remove(peer); + } else if inner.subscribers.len() < MAX_SUBSCRIBERS || inner.subscribers.contains_key(peer) + { + let lasts = + Duration::from_secs(u64::from(request.subscribe_secs)).min(MAX_SUBSCRIPTION); + inner + .subscribers + .insert(peer.to_owned(), (request.nonce, now + lasts)); + state.links.subscribed.notify_one(); + } + if inner + .answered + .get(peer) + .is_some_and(|at| now.saturating_duration_since(*at) < ANSWER_EVERY) + { + return; + } + if inner.answered.len() >= MAX_KEPT { + inner + .answered + .retain(|_, at| now.saturating_duration_since(*at) < ANSWER_EVERY); + } + inner.answered.insert(peer.to_owned(), now); + } + let (epoch, epoch_seq) = state.epoch_stamp(); + let reply = ParsedMessage::LinksReply(LinksReply { + epoch, + epoch_seq, + nonce: request.nonce, + links: own_links(state).await, + }); + if let Err(error) = state.transport.send_message(peer, &reply).await { + debug!(peer = %peer, error = %format!("{error:#}"), "LINKS_REPLY failed"); + } +} + +/// A LINKS_REPLY from `peer`: kept if it answers our latest LINKS to it, while its +/// replies are taken (a subscription's go on). +pub(crate) fn received(state: &AppState, peer: &str, mut reply: LinksReply) { + let mut inner = state.links.inner.lock_or_recover(); + let taken = inner + .asked + .get(peer) + .is_some_and(|(nonce, until)| *nonce == reply.nonce && Instant::now() < *until); + if !taken { + return; + } + reply.links.truncate(MAX_LINKS); + for link in &mut reply.links { + link.others.truncate(MAX_OTHERS); + } + if inner.replies.len() >= MAX_KEPT && !inner.replies.contains_key(peer) { + inner.replies.clear(); + } + inner + .replies + .insert(peer.to_owned(), (Instant::now(), reply.links)); +} + +/// For subscribers (`answer`): every `WATCH_EVERY`, while there are any, this node's +/// links; sent to each of them when they changed since they were last sent. Without +/// any, it waits for one. Runs for the node's life. +pub(crate) async fn watch(state: AppState) { + let mut every = tokio::time::interval(WATCH_EVERY); + every.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + every.tick().await; + let subscribers: Vec<(String, u64)> = { + let now = Instant::now(); + let mut inner = state.links.inner.lock_or_recover(); + inner.subscribers.retain(|_, (_, until)| now < *until); + if inner.subscribers.is_empty() { + inner.published = None; + } + inner + .subscribers + .iter() + .map(|(peer, (nonce, _))| (peer.clone(), *nonce)) + .collect() + }; + if subscribers.is_empty() { + // A new subscriber was just answered (`answer`, or within `ANSWER_EVERY`): + // the next look is due an interval later. + state.links.subscribed.notified().await; + every.reset(); + continue; + } + let links = own_links(&state).await; + { + let mut inner = state.links.inner.lock_or_recover(); + if inner + .published + .as_ref() + .is_some_and(|published| !changed(published, &links)) + { + continue; + } + inner.published = Some(links.clone()); + } + let (epoch, epoch_seq) = state.epoch_stamp(); + for (peer, nonce) in subscribers { + let reply = ParsedMessage::LinksReply(LinksReply { + epoch, + epoch_seq, + nonce, + links: links.clone(), + }); + if let Err(error) = state.transport.send_message(&peer, &reply).await { + debug!(peer = %peer, error = %format!("{error:#}"), "LINKS_REPLY failed"); + } + } + } +} + +/// Whether `new` says something `old` didn't: another peer, transport, relay or +/// dormancy, a round trip measured or moved by more than a fifth (and 1 ms), or a clock +/// measured or moved beyond both bounds. Round trips and clock samples wobble; small +/// moves aren't news. +fn changed(old: &[Link], new: &[Link]) -> bool { + let clock_moved = |a: Option<ClockOffset>, b: Option<ClockOffset>| match (a, b) { + (Some(a), Some(b)) => { + a.offset_ms.abs_diff(b.offset_ms) > a.within_ms.saturating_add(b.within_ms) + } + (a, b) => a.is_some() != b.is_some(), + }; + let moved = |a: Option<u32>, b: Option<u32>| match (a, b) { + (Some(a), Some(b)) => { + let by = a.abs_diff(b); + by > 1000 && by > a.max(b) / 5 + } + (a, b) => a.is_some() != b.is_some(), + }; + old.len() != new.len() + || old.iter().zip(new).any(|(a, b)| { + a.peer != b.peer + || a.transport != b.transport + || a.relayed != b.relayed + || a.down != b.down + || a.dormant != b.dormant + || moved(a.rtt_us, b.rtt_us) + || clock_moved(a.clock, b.clock) + || a.others.len() != b.others.len() + || a.others.iter().zip(&b.others).any(|(x, y)| { + x.transport != y.transport + || x.relayed != y.relayed + || moved(x.rtt_us, y.rtt_us) + }) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn link(peer: &str, rtt_us: Option<u32>) -> Link { + Link { + peer: peer.into(), + transport: Some("libp2p".into()), + relayed: false, + down: false, + rtt_us, + dormant: false, + others: Vec::new(), + clock: None, + } + } + + #[test] + fn only_news_is_a_change() { + let old = [link("a", Some(10_000))]; + assert!(!changed(&old, &[link("a", Some(11_000))])); + assert!(changed(&old, &[link("a", Some(13_000))])); + assert!(changed(&old, &[link("a", None)])); + assert!(changed(&old, &[link("b", Some(10_000))])); + assert!(changed(&old, &[])); + let mut dormant = link("a", Some(10_000)); + dormant.dormant = true; + assert!(changed(&old, &[dormant])); + let mut down = link("a", Some(10_000)); + down.down = true; + assert!(changed(&old, &[down])); + // Small round trips: a millisecond's wobble isn't news. + assert!(!changed(&[link("a", Some(400))], &[link("a", Some(1_300))])); + // A clock: news when measured, and when it moved beyond both bounds. + let clocked = |offset_ms| Link { + clock: Some(ClockOffset { + offset_ms, + within_ms: 50, + }), + ..link("a", Some(10_000)) + }; + assert!(changed(&old, &[clocked(0)])); + assert!(!changed(&[clocked(0)], &[clocked(80)])); + assert!(changed(&[clocked(0)], &[clocked(5_000)])); + } +} diff --git a/src/swarm/members.rs b/src/swarm/members.rs index 3bfb9cf..05c6501 100644 --- a/src/swarm/members.rs +++ b/src/swarm/members.rs @@ -18,12 +18,12 @@ use std::collections::HashMap; use std::time::Duration; -use automerge::AutoCommit; use serde_json::{Value as JsonValue, json}; use tracing::{debug, info, warn}; use crate::document::meta; use crate::document::store::{self, StoreError}; +use crate::document::{Document, ReadDoc, Transactable}; use crate::state::AppState; pub(crate) fn members_dir() -> Vec<String> { @@ -37,7 +37,7 @@ pub(crate) fn member_path(pk: &str) -> Vec<String> { } /// Public keys of all registered members, sorted. -pub(crate) fn members(doc: &AutoCommit) -> Vec<String> { +pub(crate) fn members(doc: &impl ReadDoc) -> Vec<String> { let mut members = store::keys_at(doc, &members_dir()).unwrap_or_default(); members.sort(); members @@ -67,14 +67,18 @@ fn tier_path(pk: &str) -> Vec<String> { } /// `pk`'s tier; transient unless its entry says stable. -pub(crate) fn tier(doc: &AutoCommit, pk: &str) -> Tier { +pub(crate) fn tier(doc: &Document, pk: &str) -> Tier { match store::read_path(doc, &tier_path(pk)) { Ok(Some(JsonValue::String(tier))) if tier == "stable" => Tier::Stable, _ => Tier::Transient, } } -pub(crate) fn set_tier(doc: &mut AutoCommit, pk: &str, tier: Tier) -> Result<(), StoreError> { +pub(crate) fn set_tier( + doc: &mut impl Transactable, + pk: &str, + tier: Tier, +) -> Result<(), StoreError> { store::write_path(doc, &tier_path(pk), &json!(tier.as_str())) } @@ -85,7 +89,7 @@ fn alias_path(pk: &str) -> Vec<String> { } /// `pk`'s alias, if it gave itself one. -pub(crate) fn alias(doc: &AutoCommit, pk: &str) -> Option<String> { +pub(crate) fn alias(doc: &Document, pk: &str) -> Option<String> { match store::read_path(doc, &alias_path(pk)) { Ok(Some(JsonValue::String(alias))) => Some(alias), _ => None, @@ -93,14 +97,14 @@ pub(crate) fn alias(doc: &AutoCommit, pk: &str) -> Option<String> { } /// The aliases of all members that have one. -pub(crate) fn aliases(doc: &AutoCommit) -> HashMap<String, String> { +pub(crate) fn aliases(doc: &Document) -> HashMap<String, String> { members(doc) .into_iter() .filter_map(|pk| alias(doc, &pk).map(|alias| (pk, alias))) .collect() } -fn set_alias(doc: &mut AutoCommit, pk: &str, alias: Option<&str>) -> Result<(), StoreError> { +fn set_alias(doc: &mut impl Transactable, pk: &str, alias: Option<&str>) -> Result<(), StoreError> { match alias { Some(alias) => store::write_path(doc, &alias_path(pk), &json!(alias)), None => match store::delete_path(doc, &alias_path(pk), false) { @@ -116,12 +120,14 @@ fn addresses_path(pk: &str) -> Vec<String> { path } -/// Addresses a member most likely has, at most: one per transport. -const MAX_ADDRESSES: usize = 8; +/// Addresses a member lists, at most: a few per transport (libp2p's up to +/// `PUBLISHED_DIRECT` and `PUBLISHED_RELAYED`, and where it relays, again as many to +/// reserve a slot at), taken in turns by transport (`Paths::local_addresses`). +const MAX_ADDRESSES: usize = 16; /// Where `pk` says it can be reached (at most `MAX_ADDRESSES`); none if its entry /// doesn't say. -pub(crate) fn addresses(doc: &AutoCommit, pk: &str) -> Vec<String> { +pub(crate) fn addresses(doc: &Document, pk: &str) -> Vec<String> { match store::read_path(doc, &addresses_path(pk)) { Ok(Some(JsonValue::Array(items))) => items .into_iter() @@ -135,9 +141,31 @@ pub(crate) fn addresses(doc: &AutoCommit, pk: &str) -> Vec<String> { } } +/// What the member list tells this node besides who its neighbours are: where members +/// are (`learn_addresses`), what they call themselves (`set_aliases`), and the relays +/// listed (`relays`). Done whenever the neighbours are recomputed (`topology::refresh`). +pub(crate) fn apply(state: &AppState, doc: &Document) { + learn_addresses(state, doc); + set_aliases(state, aliases(doc)); + state.transport.learn_relays(&relays(doc)); +} + +/// Hands the members' `aliases` to the logger, under every name that stands for each: +/// its node ID, and its addresses (`logging`, which renames them bare too). +fn set_aliases(state: &AppState, aliases: HashMap<String, String>) { + let mut names = HashMap::new(); + for (member, alias) in aliases { + for address in state.transport.identities_of(&member) { + names.insert(address, alias.clone()); + } + names.insert(member, alias); + } + crate::logging::set_aliases(names); +} + /// Hands every other member's addresses to the transport (`Transport::learn_addresses`): /// the member list is how a node finds where members it never met are. -pub(crate) fn learn_addresses(state: &AppState, doc: &AutoCommit) { +pub(crate) fn learn_addresses(state: &AppState, doc: &Document) { let me = state.local_id(); for pk in members(doc) { if pk != me { @@ -153,7 +181,7 @@ const MAX_RELAYS: usize = 32; /// addresses (`libp2p:<multiaddr>`, with or without the relay's peer ID) of relays that /// aren't members, e.g. nodes that only relay. Every member's transport may /// pick them (`Transport::learn_relays`), after the relays it is configured with. -pub(crate) fn relays(doc: &AutoCommit) -> Vec<String> { +pub(crate) fn relays(doc: &Document) -> Vec<String> { match store::read_path(doc, &meta::path(&[meta::RELAYS])) { Ok(Some(JsonValue::Array(items))) => items .into_iter() @@ -173,7 +201,7 @@ fn alive_path(pk: &str) -> Vec<String> { path } -fn millis_at(doc: &AutoCommit, path: &[String]) -> Option<u64> { +fn millis_at(doc: &impl ReadDoc, path: &[String]) -> Option<u64> { match store::read_path(doc, path) { Ok(Some(JsonValue::Number(number))) => number.as_u64(), _ => None, @@ -181,29 +209,57 @@ fn millis_at(doc: &AutoCommit, path: &[String]) -> Option<u64> { } /// When `pk` last said it is alive (Unix ms): its last renewal, or its registration. -pub(crate) fn alive_at(doc: &AutoCommit, pk: &str) -> Option<u64> { +pub(crate) fn alive_at(doc: &impl ReadDoc, pk: &str) -> Option<u64> { let mut since = member_path(pk); since.push("since".to_owned()); millis_at(doc, &alive_path(pk)).max(millis_at(doc, &since)) } /// The members that said they are alive within `window` before `now_ms` (sorted): the -/// electorate of a remake (`remake::coordinator`). -pub(crate) fn electorate(doc: &AutoCommit, now_ms: u64, window: Duration) -> Vec<String> { +/// electorate of a remake (`remake::coordinator`). A stamp further in the future than a +/// renewal interval (`alive_renewal`) doesn't count: its node's clock is that far ahead +/// (it warns about it, `swarm::skew`), and the stamp would never age out. +pub(crate) fn electorate(doc: &impl ReadDoc, now_ms: u64, window: Duration) -> Vec<String> { let oldest = now_ms.saturating_sub(window.as_millis() as u64); + let newest = now_ms.saturating_add(alive_renewal(window).as_millis() as u64); members(doc) .into_iter() - .filter(|pk| alive_at(doc, pk).is_some_and(|at| at >= oldest)) + .filter(|pk| alive_at(doc, pk).is_some_and(|at| (oldest..=newest).contains(&at))) .collect() } +fn window_path(pk: &str) -> Vec<String> { + let mut path = member_path(pk); + path.push("window".to_owned()); + path +} + +/// The shortest `--electorate-window` among this node's (`own`) and those the members +/// publish in their entries (ms, `window`): every node renews its alive stamp by it +/// (`alive_renewal`), so a coordinator with a shorter window than others' still counts +/// them all. Below `MIN_ELECTORATE_WINDOW` (which no node accepts) counts as that. +pub(crate) fn shortest_window(doc: &impl ReadDoc, own: Duration) -> Duration { + members(doc) + .iter() + .filter_map(|pk| millis_at(doc, &window_path(pk))) + .map(|ms| Duration::from_millis(ms).max(MIN_ELECTORATE_WINDOW)) + .fold(own, Duration::min) +} + /// How often a node renews its `alive` stamp: often enough that a live node never -/// drops out of the electorate (a seventh of the window), but at most hourly and at -/// least daily, so idle swarms stay quiet. +/// drops out of the electorate (a seventh of the shortest window, `shortest_window`), +/// but at most hourly and at least daily, so idle swarms stay quiet. fn alive_renewal(window: Duration) -> Duration { - (window / 7).clamp(Duration::from_secs(3600), Duration::from_secs(24 * 3600)) + (window / 7).clamp(RENEWAL_MOST_OFTEN, Duration::from_secs(24 * 3600)) } +const RENEWAL_MOST_OFTEN: Duration = Duration::from_secs(3600); + +/// The shortest `--electorate-window`: seven renewals at most often, so a live node +/// renews within a seventh of it, as with any longer one. +pub(crate) const MIN_ELECTORATE_WINDOW: Duration = + Duration::from_secs(7 * RENEWAL_MOST_OFTEN.as_secs()); + /// Registers this node if it is ready and its entry is missing, keeps its alias as /// configured (set, changed or removed after a restart with another `--alias`) and its /// addresses as the transport gives them, and renews its `alive` stamp now and then @@ -214,7 +270,8 @@ pub(crate) async fn ensure_registered(state: &AppState) { } let local = state.local_id().to_owned(); let own = member_path(&local); - let (registered, recorded_alias, recorded_addresses, alive) = { + let window = state.electorate_window.as_millis() as u64; + let (registered, recorded_alias, recorded_addresses, alive, recorded_window, shortest) = { let replica = state.doc.read().await; let registered = store::read_path(&replica.doc, &own) .ok() @@ -225,6 +282,8 @@ pub(crate) async fn ensure_registered(state: &AppState) { alias(&replica.doc, &local), addresses(&replica.doc, &local), millis_at(&replica.doc, &alive_path(&local)), + millis_at(&replica.doc, &window_path(&local)), + shortest_window(&replica.doc, state.electorate_window), ) }; let wanted = state.alias.clone(); @@ -233,7 +292,12 @@ pub(crate) async fn ensure_registered(state: &AppState) { let now = state.clock.now_millis().as_u64(); if !registered { - let mut entry = json!({ "since": now, "alive": now, "addresses": own_addresses }); + let mut entry = json!({ + "since": now, + "alive": now, + "addresses": own_addresses, + "window": window, + }); if let Some(alias) = &wanted { entry["alias"] = json!(alias); } @@ -250,7 +314,21 @@ pub(crate) async fn ensure_registered(state: &AppState) { return; } - let renewal = alive_renewal(state.electorate_window).as_millis() as u64; + if recorded_window != Some(window) { + let path = window_path(&local); + match state + .mutate_doc(move |doc| store::write_path(doc, &path, &json!(window))) + .await + { + Ok(()) => debug!( + window_ms = window, + "Electorate window published in the member list" + ), + Err(error) => warn!(error = %error, "Failed to record our electorate window"), + } + } + + let renewal = alive_renewal(shortest).as_millis() as u64; if alive.is_none_or(|at| now.saturating_sub(at) >= renewal) { let path = alive_path(&local); match state @@ -292,7 +370,7 @@ pub(crate) async fn ensure_registered(state: &AppState) { } /// Removes `pk`'s entry (an operator forgot it). Absent is fine. -pub(crate) fn unregister(doc: &mut AutoCommit, pk: &str) -> Result<(), StoreError> { +pub(crate) fn unregister(doc: &mut impl Transactable, pk: &str) -> Result<(), StoreError> { match store::delete_path(doc, &member_path(pk), false) { Err(StoreError::NotFound) => Ok(()), result => result, @@ -302,39 +380,72 @@ pub(crate) fn unregister(doc: &mut AutoCommit, pk: &str) -> Result<(), StoreErro #[cfg(test)] mod tests { use super::*; + use crate::document::testing::change; #[test] fn electorate_by_alive_stamp() { let day = 24 * 3600 * 1000; let now = 100 * day; - let mut doc = AutoCommit::new(); + let mut doc = Document::new(); let entries = [ ("fresh", json!({ "since": 1, "alive": now - day })), ("stale", json!({ "since": 1, "alive": now - 8 * day })), ("new", json!({ "since": now - day })), ("old", json!({ "since": 1 })), + // A clock a day ahead is tolerated (one renewal interval); a week isn't. + ("ahead", json!({ "since": 1, "alive": now + day })), + ("future", json!({ "since": 1, "alive": now + 7 * day })), ]; - for (pk, entry) in entries { - store::write_path(&mut doc, &member_path(pk), &entry).unwrap(); - } + change(&mut doc, |doc| { + for (pk, entry) in entries { + store::write_path(doc, &member_path(pk), &entry).unwrap(); + } + }); assert_eq!( electorate(&doc, now, Duration::from_secs(7 * 24 * 3600)), - ["fresh", "new"] + ["ahead", "fresh", "new"] ); } #[test] fn listed_relays() { - let mut doc = AutoCommit::new(); + let mut doc = Document::new(); assert!(relays(&doc).is_empty()); let relay = "libp2p:/ip6/fd00::1/tcp/9661"; - store::write_path(&mut doc, &meta::path(&[meta::RELAYS]), &json!([relay, 7])).unwrap(); + change(&mut doc, |doc| { + store::write_path(doc, &meta::path(&[meta::RELAYS]), &json!([relay, 7])).unwrap() + }); assert_eq!(relays(&doc), [relay]); // Anything but text is ignored. - store::write_path(&mut doc, &meta::path(&[meta::RELAYS]), &json!(relay)).unwrap(); + change(&mut doc, |doc| { + store::write_path(doc, &meta::path(&[meta::RELAYS]), &json!(relay)).unwrap() + }); assert!(relays(&doc).is_empty()); // Not a list. } #[test] + fn shortest_published_window() { + let day = Duration::from_secs(24 * 3600); + let mut doc = Document::new(); + assert_eq!(shortest_window(&doc, 7 * day), 7 * day); + change(&mut doc, |doc| { + store::write_path( + doc, + &member_path("a"), + &json!({ "window": 30 * 86_400_000_u64 }), + ) + .unwrap(); + store::write_path(doc, &member_path("b"), &json!({ "window": 86_400_000 })).unwrap(); + store::write_path(doc, &member_path("c"), &json!({ "since": 1 })).unwrap(); + }); + assert_eq!(shortest_window(&doc, 7 * day), day); + // Below what any node accepts: as the shortest it accepts. + change(&mut doc, |doc| { + store::write_path(doc, &window_path("c"), &json!(1)).unwrap(); + }); + assert_eq!(shortest_window(&doc, 7 * day), MIN_ELECTORATE_WINDOW); + } + + #[test] fn renewal_interval() { let day = Duration::from_secs(24 * 3600); assert_eq!(alive_renewal(7 * day), day); diff --git a/src/swarm/mod.rs b/src/swarm/mod.rs index d424529..8259ed5 100644 --- a/src/swarm/mod.rs +++ b/src/swarm/mod.rs @@ -4,11 +4,15 @@ pub(crate) mod auth; pub(crate) mod blob; pub(crate) mod bootstrap; +pub(crate) mod changes; +pub(crate) mod document_file; +pub(crate) mod links; pub(crate) mod members; pub(crate) mod pair; pub(crate) mod peer; pub(crate) mod relay; pub(crate) mod rtt; +pub(crate) mod skew; pub(crate) mod source; pub(crate) mod stability; pub(crate) mod state_file; diff --git a/src/swarm/peer/admit.rs b/src/swarm/peer/admit.rs index 1800775..6e02150 100644 --- a/src/swarm/peer/admit.rs +++ b/src/swarm/peer/admit.rs @@ -4,6 +4,7 @@ use tracing::{debug, info, trace, warn}; use super::handshake::remember_peer; use crate::state::{AppState, HelloState, Standing}; +use crate::swarm::topology; use crate::transport::ParsedMessage; /// With a swarm key, only peers that proved it get past this point. A HELLO or @@ -50,7 +51,7 @@ pub(super) async fn admit(state: &AppState, peer: &str, message: &ParsedMessage) if peer_state.standing.is_admitted() { warn!(peer = %peer, "Peer no longer proves the swarm key; dropping its membership"); } - peer_state.standing = Standing::Stranger; + topology::change_peer(state, &mut peer_state, |p| p.standing = Standing::Stranger); peer_state.hello_state = HelloState::NotSent; peer_state.reset_sync(); } diff --git a/src/swarm/peer/epoch.rs b/src/swarm/peer/epoch.rs index 56711b1..d00a903 100644 --- a/src/swarm/peer/epoch.rs +++ b/src/swarm/peer/epoch.rs @@ -6,7 +6,6 @@ use tracing::{debug, info, warn}; use crate::document::replica::{Epoch, EpochRelation}; use crate::state::AppState; -use crate::swarm::sync::SyncEvent; use crate::transport; /// Records the epoch a peer advertised and compares it with ours. Returns true when the @@ -14,7 +13,8 @@ use crate::transport; /// /// `seq` is the peer's `epoch_seq` with it. A message with a lower one than we have was /// sent before the peer's last epoch change and delivered late: its epoch isn't -/// recorded, so it can't flip our view of the peer back. The count starts over when the +/// recorded, so it can't flip our view of the peer back. The message itself still +/// counts: if its epoch is ours, its sync is applied (it is data of our epoch). The count starts over when the /// peer restarts, which its HELLO or HELLO_REPLY tells by a new run (`run`). pub(super) async fn observe_peer_epoch( state: &AppState, @@ -66,9 +66,7 @@ pub(super) async fn observe_peer_epoch( EpochRelation::Same => { if previous.is_some() { info!(peer = %peer, epoch = %remote, "Peer is on our epoch again; resuming sync"); - let _ = state - .sync_events - .send(SyncEvent::PeerNeedsSync(peer.to_owned())); + state.request_sync(peer); } } // Expected after a remake: either side switches or catches up. diff --git a/src/swarm/peer/forget.rs b/src/swarm/peer/forget.rs index 6386e3e..68266d5 100644 --- a/src/swarm/peer/forget.rs +++ b/src/swarm/peer/forget.rs @@ -5,11 +5,11 @@ use std::collections::HashMap; use std::time::Duration; -use automerge::AutoCommit; use serde_json::Value as JsonValue; use tokio::time::Instant; use tracing::{info, warn}; +use crate::document::Document; use crate::document::meta; use crate::document::store::{self, StoreError}; use crate::state::{AppState, Standing}; @@ -24,7 +24,14 @@ pub(crate) fn forgotten_path(pk: &str) -> Vec<String> { path } -fn forgotten_peers(doc: &AutoCommit) -> Vec<String> { +/// Whether a change at `path` may change the forget records: one among them, or above +/// them (the root, after an epoch switch). +pub(crate) fn affects_forget_records(path: &[String]) -> bool { + let dir = forgotten_dir(); + path.starts_with(&dir) || dir.starts_with(path) +} + +fn forgotten_peers(doc: &Document) -> Vec<String> { store::keys_at(doc, &forgotten_dir()).unwrap_or_default() } diff --git a/src/swarm/peer/handshake.rs b/src/swarm/peer/handshake.rs index 3e8fbf4..072545b 100644 --- a/src/swarm/peer/handshake.rs +++ b/src/swarm/peer/handshake.rs @@ -10,7 +10,6 @@ use tokio::time::Instant; use tracing::{debug, info, instrument, trace, warn}; use crate::state::{AppState, HelloState, PeerState, Standing}; -use crate::swarm::sync::SyncEvent; use crate::swarm::topology; use crate::transport::{self, ParsedMessage}; @@ -148,15 +147,11 @@ async fn transmit_hello(state: &AppState, peer: &str) -> Result<()> { let mut peer_state = peer_state_mutex.lock().await; if peer_state.hello_state == HelloState::Sending { - let was_unreachable = peer_state.unreachable_since.is_some(); - peer_state.hello_sent(Instant::now()); - if !was_unreachable && peer_state.unreachable_since.is_some() { - state.topology_changed.notify_one(); - } + topology::change_peer(state, &mut peer_state, |p| p.hello_sent(Instant::now())); let wait = peer_state.next_hello_wait(); - peer_state.hello_state = HelloState::AwaitingReply { - next_attempt: Instant::now() + wait, - }; + let next_attempt = Instant::now() + wait; + peer_state.hello_state = HelloState::AwaitingReply { next_attempt }; + state.peers_due.by(next_attempt); } } @@ -260,6 +255,11 @@ pub(super) async fn send_hello_reply( /// (it says whom we greet; a listed peer that isn't one would get nothing from us, so /// it isn't remembered either). Where the sender reaches them is a hint to the /// transport (at most `MAX_LISTED_ADDRESSES` each). +/// +/// A listed peer we can't reach (dormant, or unanswered HELLOs) is probed at once +/// rather than when its probe is due (`probe_now`): the sender reaches it, so it's likely +/// back for us too, as after a partition, whose sides would otherwise find each other +/// pair by pair, each at its own probe (up to `dormant-probe-max` apart). pub(super) async fn discover_peers( state: &AppState, from_peer: &str, @@ -272,9 +272,6 @@ pub(super) async fn discover_peers( if discovered_peer == from_peer || !state.transport.is_peer_id(discovered_peer) { continue; } - if view_known && !topology::is_neighbour(state, discovered_peer) { - continue; - } if taken == MAX_LISTED_PEERS { debug!( from = %from_peer, @@ -283,9 +280,18 @@ pub(super) async fn discover_peers( ); break; } + let addresses = &listed.addresses[..listed.addresses.len().min(MAX_LISTED_ADDRESSES)]; + if probe_now(state, discovered_peer).await { + taken += 1; + state.transport.learn_addresses(discovered_peer, addresses); + debug!(peer = %discovered_peer, by = %from_peer, "A peer we can't reach is reached by another; probing it now"); + continue; + } + if view_known && !topology::is_neighbour(state, discovered_peer) { + continue; + } taken += 1; - let addresses = &listed.addresses[..listed.addresses.len().min(MAX_LISTED_ADDRESSES)]; state.transport.learn_addresses(discovered_peer, addresses); remember_peer(state, discovered_peer).await; if let Err(error) = send_hello_once(state, discovered_peer).await { @@ -298,6 +304,53 @@ pub(super) async fn discover_peers( } } +/// Whenever others may reach this node where they couldn't before +/// (`Transport::on_reachable`: a relay granted it a slot), the members it can't reach +/// are probed at once (`probe_now`): they may have given up on it meanwhile, waiting +/// for their next probe, up to `dormant-probe-max` away. Two nodes behind NATs that +/// restart together each fail to reach the other until it holds a slot; the second to +/// get one reaches the first this way. Runs for the node's life. +pub(crate) async fn probe_when_reachable(state: AppState, reachable: Arc<tokio::sync::Notify>) { + loop { + reachable.notified().await; + let mut probed = 0; + for (peer, _) in state.peers_snapshot().await { + if probe_now(&state, &peer).await { + probed += 1; + } + } + if probed > 0 { + debug!( + probed, + "Reachable anew; probing the members we can't reach now" + ); + } + } +} + +/// Brings `peer`'s next probe forward to now, if it is a member we can't reach (dormant, +/// or unanswered HELLOs) whose probe is pending: the peer worker sends it (woken for +/// it). Whether it was. +async fn probe_now(state: &AppState, peer: &str) -> bool { + let Some(peer_state) = state.get_peer_state(peer).await else { + return false; + }; + let mut peer_state = peer_state.lock().await; + if !matches!(peer_state.standing, Standing::Member { .. }) + || !(peer_state.liveness.is_dormant() || peer_state.is_unreachable()) + { + return false; + } + match &mut peer_state.hello_state { + HelloState::AwaitingReply { next_attempt } => { + *next_attempt = Instant::now(); + state.peers_due.now(); + true + } + _ => false, + } +} + /// New neighbours get our HELLO (we may not have known them before: the member list is /// how a partial mesh discovers them) and a sync opening. The opening matters for a /// neighbour whose handshake is already complete, e.g. one that returned from dormancy @@ -314,9 +367,7 @@ pub(crate) async fn apply_neighbour_change(state: &AppState, change: topology::N } // Skipped there unless the handshake is complete; otherwise its reply restarts // sync anyway. - let _ = state - .sync_events - .send(SyncEvent::PeerNeedsSync(peer.clone())); + state.request_sync(peer); } for peer in &change.removed { if let Some(peer_state) = state.get_peer_state(peer).await { @@ -372,11 +423,12 @@ pub(super) async fn establish_peer( } if !matches!(peer_state.standing, Standing::Member { .. }) { - peer_state.standing = Standing::Member { - swarm_id: swarm_id.clone(), - }; // A member our list may lack yet: a neighbour then (`topology::refresh`). - state.topology_changed.notify_one(); + topology::change_peer(state, &mut peer_state, |p| { + p.standing = Standing::Member { + swarm_id: swarm_id.clone(), + } + }); } // Stop our retries and sync from a clean slate. A new handshake from the peer always @@ -386,6 +438,8 @@ pub(super) async fn establish_peer( if from_hello || peer_state.hello_state != HelloState::Established { peer_state.hello_state = HelloState::Established; peer_state.reset_sync(); + // Keepalives start. + state.peers_due.now(); info!(peer = %peer, from_hello, "Handshake complete; sync state reset"); } diff --git a/src/swarm/peer/inbound.rs b/src/swarm/peer/inbound.rs index 2e89739..18d29d5 100644 --- a/src/swarm/peer/inbound.rs +++ b/src/swarm/peer/inbound.rs @@ -4,14 +4,16 @@ use std::time::Duration; use anyhow::{Result, anyhow, bail}; use tokio::time::Instant; -use tracing::{debug, info, warn}; +use tracing::{debug, warn}; use super::admit::admit; use super::epoch::observe_peer_epoch; use super::handshake::{ discover_peers, ensure_handshake, establish_peer, peer_swarm, remember_peer, send_hello_reply, }; -use super::keepalive::{heal_if_diverged, local_status, note_alive, note_status, status_received}; +use super::keepalive::{ + heal_if_diverged, heard_from, local_status, note_alive, note_status, status_received, +}; use super::lanes::SyncLanes; use crate::state::AppState; use crate::swarm::sync::SyncEvent; @@ -31,7 +33,7 @@ pub(crate) async fn handle_message( return Ok(()); } // PING and PONG only say the peer is alive (`keepalive::note_alive`). - if matches!(message, ParsedMessage::Ping(_) | ParsedMessage::Pong(_)) { + if matches!(message, ParsedMessage::Ping(_) | ParsedMessage::Pong { .. }) { note_alive(state, &peer, &message).await; return Ok(()); } @@ -56,32 +58,24 @@ pub(crate) async fn handle_message( remember_peer(state, &peer).await; - // Any inbound message proves liveness. When we heard from it before tells a - // restarted peer how long it was away (in our HELLO_REPLY). - let (reactivated, previously_seen) = { - let peer_state_mutex = state.get_or_create_peer_state(&peer).await; - let mut peer_state = peer_state_mutex.lock().await; - let previously_seen = peer_state.last_seen; - if peer_state.unreachable_since.is_some() { - // Reachable again (`mark_seen` clears it). - state.topology_changed.notify_one(); - } - (peer_state.mark_seen(Instant::now()), previously_seen) - }; - crate::swarm::source::heard_from(state, &peer); - - if reactivated { - info!(peer = %peer, "Dormant peer is active again; re-handshaking"); - } - let run = match &message { transport::ParsedMessage::Hello(hello) => Some(hello.run_id), transport::ParsedMessage::HelloReply(reply) => Some(reply.sender_run_id), _ => None, }; - let same_epoch = observe_peer_epoch(state, &peer, epoch, epoch_seq, run).await; - //debug!(from = %peer, message = ?message, "Received message"); + // Any inbound message proves liveness (but from a run that said farewell: its next + // one does). When we heard from it before tells a restarted peer how long it was + // away (in our HELLO_REPLY). + let peer_state = state.get_or_create_peer_state(&peer).await; + let (reactivated, previously_seen) = + heard_from(state, &peer, &peer_state, Instant::now(), |peer_state| { + if let Some(run) = run { + peer_state.returned(run); + } + }) + .await; + let same_epoch = observe_peer_epoch(state, &peer, epoch, epoch_seq, run).await; match message { transport::ParsedMessage::Hello(hello) => { @@ -162,10 +156,16 @@ pub(crate) async fn handle_message( transport::ParsedMessage::StatusReply(remote) => { on_status(state, &peer, &remote, reactivated).await; } + transport::ParsedMessage::Links(request) => { + crate::swarm::links::answer(state, &peer, request).await; + } + transport::ParsedMessage::LinksReply(reply) => { + crate::swarm::links::received(state, &peer, reply); + } transport::ParsedMessage::Who(_) | transport::ParsedMessage::Me(_) | transport::ParsedMessage::Ping(_) - | transport::ParsedMessage::Pong(_) => {} + | transport::ParsedMessage::Pong { .. } => {} } Ok(()) diff --git a/src/swarm/peer/keepalive.rs b/src/swarm/peer/keepalive.rs index ea7ade2..8597dcd 100644 --- a/src/swarm/peer/keepalive.rs +++ b/src/swarm/peer/keepalive.rs @@ -16,8 +16,7 @@ use tokio::time::Instant; use tracing::{debug, info, warn}; use super::handshake::ensure_handshake; -use crate::state::{AppState, Standing}; -use crate::swarm::sync::SyncEvent; +use crate::state::{AppState, PeerState}; use crate::swarm::topology; use crate::transport::{self, ParsedMessage}; @@ -49,12 +48,15 @@ pub(super) async fn note_alive(state: &AppState, peer: &str, message: &ParsedMes let now = Instant::now(); match message { ParsedMessage::Ping(nonce) if !state.transport.answers_pings() => { - let pong = ParsedMessage::Pong(*nonce); + let pong = ParsedMessage::Pong { + nonce: *nonce, + at: state.clock.now_millis(), + }; if let Err(error) = state.transport.send_message(peer, &pong).await { debug!(peer = %peer, error = %format!("{error:#}"), "PONG failed"); } } - ParsedMessage::Pong(_) => { + ParsedMessage::Pong { .. } => { if let Some(rtt) = state.transport.rtt(peer) { state.lock_rtt().measured(peer, rtt, now); } @@ -64,23 +66,46 @@ pub(super) async fn note_alive(state: &AppState, peer: &str, message: &ParsedMes let Some(peer_state) = state.get_peer_state(peer).await else { return; }; - let reactivated = { - let mut peer_state = peer_state.lock().await; - if matches!(message, ParsedMessage::Ping(_)) { + let ping = matches!(message, ParsedMessage::Ping(_)); + let (reactivated, _) = heard_from(state, peer, &peer_state, now, |peer_state| { + if ping { // Its keepalive (or a path probe): the next is ours. peer_state.keepalive_received_at = Some(now); + if let Some(at) = peer_state.keepalive_due_at(&state.timing) { + state.peers_due.by(at); + } } - if peer_state.unreachable_since.is_some() { - // Reachable again (`mark_seen` clears it). - state.topology_changed.notify_one(); - } - peer_state.mark_seen(now) + }) + .await; + if reactivated { + ensure_handshake(state, peer).await; + } +} + +/// A message from `peer` came in: it is seen (`PeerState::mark_seen`: active, and +/// reachable, again), and the catch-up source hears of it. `also` runs on its state +/// under the same lock. Returns whether it was dormant (the caller re-handshakes), and +/// when it was last seen before. +pub(super) async fn heard_from( + state: &AppState, + peer: &str, + peer_state: &tokio::sync::Mutex<PeerState>, + now: Instant, + also: impl FnOnce(&mut PeerState), +) -> (bool, Option<Instant>) { + let (reactivated, previously_seen) = { + let mut peer_state = peer_state.lock().await; + also(&mut peer_state); + let previously_seen = peer_state.last_seen; + // Reachable (again). + let reactivated = topology::change_peer(state, &mut peer_state, |p| p.mark_seen(now)); + (reactivated, previously_seen) }; crate::swarm::source::heard_from(state, peer); if reactivated { info!(peer = %peer, "Dormant peer is active again; re-handshaking"); - ensure_handshake(state, peer).await; } + (reactivated, previously_seen) } /// How long shutting down waits for the last STATUSes (`farewell`). @@ -120,7 +145,8 @@ pub(crate) fn announce(state: &AppState) { /// catching up from us picks another source now, instead of once we go dormant. Bounded /// in time: our transport may be gone already. pub(crate) async fn farewell(state: &AppState) { - state.stopping.store(true, Ordering::Release); + state.closing.send_replace(true); + info!("Saying farewell to our neighbours"); status_to_neighbours(state, Some(FAREWELL_TIMEOUT)).await; } @@ -137,20 +163,16 @@ pub(crate) async fn local_status(state: &AppState, peer: &str) -> transport::Sta ) }; let syncs_with_you = match state.get_peer_state(peer).await { - Some(peer_state) => { - let peer_state = peer_state.lock().await; - !peer_state.liveness.is_dormant() - && matches!(peer_state.standing, Standing::Member { .. }) - && peer_state.epoch == Some(epoch) - } + Some(peer_state) => peer_state.lock().await.syncs_on(epoch), None => false, }; transport::Status { heads, epoch, epoch_seq, - ready: state.is_ready().await && !state.stopping.load(Ordering::Acquire), + ready: state.is_ready().await && !state.is_closing(), syncs_with_you, + leaving: state.is_closing(), } } @@ -162,6 +184,9 @@ pub(super) async fn status_received(state: &AppState, peer: &str) { let now = Instant::now(); peer_state.keepalive_received_at = Some(now); peer_state.last_status_at = Some(now); + if let Some(at) = peer_state.keepalive_due_at(&state.timing) { + state.peers_due.by(at); + } } } @@ -169,11 +194,24 @@ pub(super) async fn status_received(state: &AppState, peer: &str) { pub(super) async fn note_status(state: &AppState, peer: &str, remote: &transport::Status) { if let Some(peer_state) = state.get_peer_state(peer).await { let mut peer_state = peer_state.lock().await; + let left = topology::change_peer(state, &mut peer_state, |p| { + // A neighbour that only connects out stays one as long as it says so. + p.syncs_with_us = Some(remote.syncs_with_you); + // Its farewell: dormant now, as if silent for long (a message from it, as + // its HELLO when it's back, makes it active again). + remote.leaving && p.say_farewell(Instant::now()) + }); + if left { + // Probed from now on. + state.peers_due.now(); + } let became_ready = remote.ready && peer_state.reports_ready != Some(true); peer_state.reports_ready = Some(remote.ready); - peer_state.syncs_with_us = Some(remote.syncs_with_you); peer_state.reported_heads = remote.heads.heads().map(<[_]>::len); drop(peer_state); + if left { + info!(peer = %peer, "Peer said farewell; dormant until it is heard from again"); + } // Off the consumer: telling which heads we lack takes the document's lock. { let (state, peer, remote) = (state.clone(), peer.to_owned(), remote.clone()); @@ -184,9 +222,7 @@ pub(super) async fn note_status(state: &AppState, peer: &str, remote: &transport if became_ready { // We may have left the opening to it while it wasn't ready, and nothing else // would make us look again. - let _ = state - .sync_events - .send(SyncEvent::PeerNeedsSync(peer.to_owned())); + state.request_sync(peer); } } crate::swarm::source::reported(state, peer, remote.ready, Some(remote.syncs_with_you)); diff --git a/src/swarm/peer/mod.rs b/src/swarm/peer/mod.rs index 326f3a9..cf427a2 100644 --- a/src/swarm/peer/mod.rs +++ b/src/swarm/peer/mod.rs @@ -20,11 +20,12 @@ mod keepalive; mod lanes; mod worker; -pub(crate) use forget::{forget_unless_alive, forgotten_path}; +pub(crate) use forget::{affects_forget_records, forget_unless_alive, forgotten_path}; pub(crate) use handshake::{ - apply_neighbour_change, peer_swarm, rehandshake, remember_peer, send_hello_once, + apply_neighbour_change, peer_swarm, probe_when_reachable, rehandshake, remember_peer, + send_hello_once, }; pub(crate) use inbound::handle_message; pub(crate) use keepalive::{announce, farewell, send_status}; pub(crate) use lanes::SyncLanes; -pub(crate) use worker::run_peer_worker; +pub(crate) use worker::{Due, run_peer_worker}; diff --git a/src/swarm/peer/worker.rs b/src/swarm/peer/worker.rs index f3d0e84..6196571 100644 --- a/src/swarm/peer/worker.rs +++ b/src/swarm/peer/worker.rs @@ -1,11 +1,12 @@ -//! The peer worker: every `PEER_TICK`, keepalives due, dormancy, HELLO retries and -//! overdue sync replies; every `FORGET_CHECK_INTERVAL` (and at once when the node -//! becomes ready), forget records, the member registry, stability, measurements and -//! the state file. +//! The peer worker: keepalives due, dormancy, HELLO retries and overdue sync replies, +//! when the earliest of them is due (or something may have made one due sooner, +//! `Due`); every `FORGET_CHECK_INTERVAL` (and at once when the node becomes ready), +//! forget records, the member registry, stability, measurements and the state file; +//! forget records also at once when they changed. use std::time::Duration; -use tokio::time::{Instant, MissedTickBehavior}; +use tokio::time::Instant; use tracing::{debug, info, warn}; use super::forget::{AppliedForgets, apply_forget_records, forget_strangers}; @@ -13,17 +14,51 @@ use super::handshake::retry_hello_if_due; use super::keepalive::{send_ping, send_status}; use crate::locks::MutexExt; use crate::state::{AppState, Keepalive}; -use crate::swarm::sync::SyncEvent; use crate::swarm::topology; -const PEER_TICK: Duration = Duration::from_millis(500); +/// The least time between two passes, however often something is due. +const MIN_PASS_GAP: Duration = Duration::from_millis(20); const FORGET_CHECK_INTERVAL: Duration = Duration::from_secs(5); -pub(crate) async fn run_peer_worker(state: AppState) { - let mut tick = tokio::time::interval(PEER_TICK); - tick.set_missed_tick_behavior(MissedTickBehavior::Delay); +/// When the peer worker is next due. What makes something of its due sooner than it +/// knows (a HELLO sent, a sync message waiting for its answer, a keepalive received, a +/// new peer, new neighbours) says so (`by`); what only puts things off needn't. +#[derive(Default)] +pub(crate) struct Due { + /// Its next pass while it waits; None while it runs. + next: std::sync::Mutex<Option<Instant>>, + wake: tokio::sync::Notify, +} + +impl Due { + /// Something is due at `at`: a pass then at the latest. + pub(crate) fn by(&self, at: Instant) { + if self.next.lock_or_recover().is_none_or(|next| at < next) { + self.wake.notify_one(); + } + } + /// A pass now. + pub(crate) fn now(&self) { + self.wake.notify_one(); + } + + fn passing(&self) { + *self.next.lock_or_recover() = None; + } + + /// Until `at`, or until something is due sooner. + async fn until(&self, at: Instant) { + *self.next.lock_or_recover() = Some(at); + tokio::select! { + () = self.wake.notified() => {} + () = tokio::time::sleep_until(at) => {} + } + } +} + +pub(crate) async fn run_peer_worker(state: AppState) { info!("Starting peer maintenance worker"); let mut last_forget_check: Option<Instant> = None; @@ -31,7 +66,8 @@ pub(crate) async fn run_peer_worker(state: AppState) { let mut was_ready = false; loop { - tick.tick().await; + state.peers_due.passing(); + let mut next = Instant::now() + FORGET_CHECK_INTERVAL; // While joining, keep handshakes fast; dormancy would slow probes to minutes. let ready = state.is_ready().await; @@ -52,8 +88,11 @@ pub(crate) async fn run_peer_worker(state: AppState) { let (became_dormant, keepalive, unanswered) = { let mut peer_state = peer_state_mutex.lock().await; - let became_dormant = - ready && neighbour && peer_state.check_dormancy(&state.timing, now); + let became_dormant = ready + && neighbour + && topology::change_peer(&state, &mut peer_state, |p| { + p.check_dormancy(&state.timing, now) + }); let keepalive = if neighbour { peer_state.take_keepalive_due(&state.timing, now) } else { @@ -69,13 +108,10 @@ pub(crate) async fn run_peer_worker(state: AppState) { if let Some(changes) = unanswered { debug!(peer = %peer, changes, "No answer to our sync message; sending again"); crate::swarm::source::reply_timed_out(&state, peer); - let _ = state - .sync_events - .send(SyncEvent::PeerNeedsSync(peer.clone())); + state.request_sync(peer); } if became_dormant { - state.topology_changed.notify_one(); warn!( peer = %peer, silent_secs = state.timing.dormant_after.as_secs(), @@ -104,19 +140,45 @@ pub(crate) async fn run_peer_worker(state: AppState) { if let Err(error) = retry_hello_if_due(&state, peer).await { warn!(peer = %peer, error = %format!("{error:#}"), "HELLO retry failed"); } + + let due = peer_state_mutex + .lock() + .await + .next_due(&state.timing, neighbour, ready); + let probe = crate::swarm::source::probe_at(&state, peer); + next = [Some(next), due, probe] + .into_iter() + .flatten() + .min() + .unwrap_or(next); } - if last_forget_check.is_none_or(|t| t.elapsed() >= FORGET_CHECK_INTERVAL) { + let records_changed = state + .forget_records_changed + .swap(false, std::sync::atomic::Ordering::AcqRel); + let periodic = last_forget_check.is_none_or(|t| t.elapsed() >= FORGET_CHECK_INTERVAL); + if records_changed || periodic { apply_forget_records(&state, &mut forgets_applied).await; + } + if periodic { forget_strangers(&state).await; crate::swarm::members::ensure_registered(&state).await; crate::swarm::stability::tick(&state).await; + crate::swarm::skew::check(&state).await; crate::swarm::stats::measure_saved_size(&state); crate::swarm::source::warn_if_stalled(&state); crate::swarm::state_file::update(&state, false).await; { let known = state.peer_states.read().await; - state.traffic.retain(|peer| known.contains_key(peer)); + // And their addresses, where the layers below the node count + // (`Traffic::node`). + let addresses: std::collections::HashSet<String> = known + .keys() + .flat_map(|peer| state.transport.identities_of(peer)) + .collect(); + state + .traffic + .retain(|peer| known.contains_key(peer) || addresses.contains(peer)); state .throughput .lock_or_recover() @@ -126,5 +188,12 @@ pub(crate) async fn run_peer_worker(state: AppState) { state.topology_changed.notify_one(); last_forget_check = Some(Instant::now()); } + if let Some(checked) = last_forget_check { + next = next.min(checked + FORGET_CHECK_INTERVAL); + } + + let passed = Instant::now(); + state.peers_due.until(next).await; + tokio::time::sleep_until(passed + MIN_PASS_GAP).await; } } diff --git a/src/swarm/relay.rs b/src/swarm/relay.rs index 6172f67..1736cec 100644 --- a/src/swarm/relay.rs +++ b/src/swarm/relay.rs @@ -28,7 +28,6 @@ use tracing::debug; use crate::locks::MutexExt; use crate::state::{AppState, Standing}; use crate::swarm::peer::send_status; -use crate::swarm::sync::SyncEvent; use crate::swarm::topology; use crate::transport::{self, HeadsSummary}; @@ -131,16 +130,9 @@ pub(crate) async fn pull_if_behind( } // Those of the listed heads we lack. With a digest only a sample is listed; if we // have all of it, a later keepalive lists others. - let missing: Vec<_> = { - // AutoCommit's lookups take &mut self. - let mut replica = state.doc.write().await; - remote - .heads - .listed() - .iter() - .filter(|head| replica.doc.get_change_by_hash(head).is_none()) - .copied() - .collect() + let missing = { + let replica = state.doc.read().await; + crate::document::changes::missing(&replica.doc, remote.heads.listed()) }; if missing.is_empty() { return false; @@ -183,9 +175,7 @@ pub(crate) async fn pull_if_behind( peer_state.lock().await.sync.pull_from(theirs); state.stats.pull_started(); debug!(peer = %peer, "Neighbour has changes we lack; pulling"); - let _ = state - .sync_events - .send(SyncEvent::PeerNeedsSync(peer.to_owned())); + state.request_sync(peer); true } diff --git a/src/swarm/skew.rs b/src/swarm/skew.rs new file mode 100644 index 0000000..69c85f5 --- /dev/null +++ b/src/swarm/skew.rs @@ -0,0 +1,277 @@ +//! Whether this node's clock agrees with its peers' (`Transport::clock_offset`, from +//! their PONGs). Each peer's clock is known within half a round trip; this node's own +//! is off when it differs from most of its peers' by more than `--clock-skew-warn` for +//! sure, then the log says so (every `WARN_EVERY` while it lasts). A clock set anew +//! between two checks (by NTP, a resumed machine, by hand) by more than the limit shows +//! as a jump, this node's when most peers' clocks moved alike, else those peers'; the +//! log says so too. Clocks are the operating system's to set (NTP); the propagation +//! figures only take this node's own offset off (`AppState::clock_ahead_ms`). +//! +//! Two nodes alone can't tell whose clock is off, or jumped: each says it is its own. + +use std::collections::HashMap; + +use std::time::Duration; + +use serde::Serialize; +use tokio::time::Instant; +use tracing::{info, warn}; + +use crate::locks::MutexExt; +use crate::state::AppState; +use crate::transport::paths::ClockOffset; + +/// While this node's clock is off, the log says so this often. +const WARN_EVERY: Duration = Duration::from_secs(600); + +/// How this node's clock compares with its peers'. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct Skew { + /// Peers whose clocks were compared (active ones with a PONG lately). + pub(crate) peers: usize, + /// How far this node's clock is ahead of theirs (behind if negative): the median. + pub(crate) offset_ms: Option<i64>, + /// Peers whose clocks are behind ours by more than the limit for sure, and ahead. + pub(crate) behind: usize, + pub(crate) ahead: usize, + /// Whether ours is off: more than half of them differ from it beyond the limit, + /// the same way. Never with a limit of 0. + pub(crate) off: bool, + /// The limit (`--clock-skew-warn`). + pub(crate) limit_ms: u64, +} + +/// Ours against the peers' clocks `offsets` (each ahead of ours), with `limit`. +pub(crate) fn of(offsets: &[ClockOffset], limit: Duration) -> Skew { + let limit_ms = u64::try_from(limit.as_millis()).unwrap_or(u64::MAX); + let beyond = |offset_ms: i64, within_ms: u64| { + offset_ms > 0 && offset_ms.unsigned_abs() > within_ms.saturating_add(limit_ms) + }; + let ahead = offsets + .iter() + .filter(|o| beyond(o.offset_ms, o.within_ms)) + .count(); + let behind = offsets + .iter() + .filter(|o| beyond(-o.offset_ms, o.within_ms)) + .count(); + let mut ours: Vec<i64> = offsets.iter().map(|o| -o.offset_ms).collect(); + ours.sort_unstable(); + let peers = offsets.len(); + Skew { + peers, + offset_ms: ours.get(peers / 2).copied(), + behind, + ahead, + off: limit_ms > 0 && (ahead * 2 > peers || behind * 2 > peers), + limit_ms, + } +} + +/// The clocks of the peers that are active now, by peer. +async fn offsets(state: &AppState) -> Vec<(String, ClockOffset)> { + let mut offsets = Vec::new(); + for (peer, peer_state) in state.peers_snapshot().await { + if peer_state.lock().await.liveness.is_dormant() { + continue; + } + if let Some(offset) = state.transport.clock_offset(&peer) { + offsets.push((peer, offset)); + } + } + offsets +} + +/// Ours against the clocks of the peers that are active now. +pub(crate) async fn measure(state: &AppState) -> Skew { + let offsets: Vec<ClockOffset> = offsets(state).await.into_iter().map(|(_, o)| o).collect(); + of(&offsets, state.clock_skew_warn) +} + +/// What the checks remember between them (`AppState::clock_skew`). +#[derive(Debug, Default)] +pub(crate) struct Watch { + /// When the log last said this node's clock is off, while it is. + warned: Option<Instant>, + /// Each peer's clock at the last check. + last: HashMap<String, ClockOffset>, +} + +/// Clocks set anew between two checks, by more than the limit for sure. +#[derive(Debug, Default, PartialEq, Eq)] +struct Jumps { + /// Ours, by how much (forward if positive): most peers' clocks moved alike. + ours: Option<i64>, + /// Otherwise the peers' whose clocks moved, each by how much. + peers: Vec<(String, i64)>, +} + +/// Which clocks jumped from `last` to `now` (each peer's ahead of ours), by more than +/// `limit_ms` beyond both bounds. None with a limit of 0. +fn jumps( + last: &HashMap<String, ClockOffset>, + now: &[(String, ClockOffset)], + limit_ms: u64, +) -> Jumps { + if limit_ms == 0 { + return Jumps::default(); + } + let moves: Vec<(&String, i64, bool)> = now + .iter() + .filter_map(|(peer, offset)| { + let before = last.get(peer)?; + let moved = offset.offset_ms - before.offset_ms; + let bound = offset + .within_ms + .saturating_add(before.within_ms) + .saturating_add(limit_ms); + Some((peer, moved, moved.unsigned_abs() > bound)) + }) + .collect(); + let forward = moves + .iter() + .filter(|(_, moved, sure)| *sure && *moved > 0) + .count(); + let back = moves + .iter() + .filter(|(_, moved, sure)| *sure && *moved < 0) + .count(); + if forward * 2 > moves.len() || back * 2 > moves.len() { + // Ours moved the other way, by the median. + let mut ours: Vec<i64> = moves.iter().map(|(_, moved, _)| -moved).collect(); + ours.sort_unstable(); + return Jumps { + ours: ours.get(ours.len() / 2).copied(), + peers: Vec::new(), + }; + } + Jumps { + ours: None, + peers: moves + .into_iter() + .filter(|(_, _, sure)| *sure) + .map(|(peer, moved, _)| (peer.clone(), moved)) + .collect(), + } +} + +/// Notes how far this node's clock is ahead of its peers' (`AppState::clock_ahead_ms`), +/// and says in the log when it is off, and when it agrees again. From the peer worker, +/// every few seconds. +pub(crate) async fn check(state: &AppState) { + let offsets = offsets(state).await; + let skew = of( + &offsets.iter().map(|(_, o)| *o).collect::<Vec<_>>(), + state.clock_skew_warn, + ); + if let Some(offset_ms) = skew.offset_ms { + state + .clock_ahead_ms + .store(offset_ms, std::sync::atomic::Ordering::Relaxed); + } + let now = Instant::now(); + let mut watch = state.clock_skew.lock_or_recover(); + let jumped = jumps(&watch.last, &offsets, skew.limit_ms); + if let Some(ms) = jumped.ours { + warn!( + jump_ms = ms, + peers = offsets.len(), + "This node's clock jumped (set anew)" + ); + } + for (peer, ms) in &jumped.peers { + info!(peer = %peer, jump_ms = ms, "A peer's clock jumped (set anew)"); + } + watch.last = offsets.into_iter().collect(); + let warned = &mut watch.warned; + if skew.off { + if warned.is_none_or(|at| now.saturating_duration_since(at) >= WARN_EVERY) { + warn!( + offset_ms = skew.offset_ms, + peers = skew.peers, + behind = skew.behind, + ahead = skew.ahead, + limit_ms = state.clock_skew_warn.as_millis() as u64, + "This node's clock differs from most of its peers'; check its time synchronisation (NTP)" + ); + *warned = Some(now); + } + } else if warned.take().is_some() { + info!( + offset_ms = skew.offset_ms, + peers = skew.peers, + "This node's clock agrees with its peers' again" + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn offset(offset_ms: i64, within_ms: u64) -> ClockOffset { + ClockOffset { + offset_ms, + within_ms, + } + } + + /// Off only beyond the limit and the bound, and only against most peers. + #[test] + fn off_against_most_peers_for_sure() { + let limit = Duration::from_secs(2); + // Every peer 5 s ahead: ours is 5 s behind. + let behind = of( + &[offset(5_000, 100), offset(5_100, 50), offset(4_900, 300)], + limit, + ); + assert_eq!(behind.offset_ms, Some(-5_000)); + assert_eq!((behind.ahead, behind.behind, behind.off), (3, 0, true)); + // One peer off: that peer's clock, not ours. + let one = of( + &[offset(5_000, 100), offset(10, 50), offset(-20, 30)], + limit, + ); + assert_eq!((one.ahead, one.off), (1, false)); + // Within the bound: not for sure. + let unsure = of(&[offset(2_500, 600), offset(2_500, 600)], limit); + assert!(!unsure.off); + // A limit of 0: never. + assert!(!of(&[offset(60_000, 10)], Duration::ZERO).off); + // No peers: nothing known. + let none = of(&[], limit); + assert_eq!((none.peers, none.offset_ms, none.off), (0, None, false)); + } + + /// Most peers' clocks moved alike: ours jumped; a few moved: theirs. + #[test] + fn jumps_are_ours_when_most_moved_alike() { + let peers = |offsets: &[i64]| -> Vec<(String, ClockOffset)> { + offsets + .iter() + .enumerate() + .map(|(i, &ms)| (format!("p{i}"), offset(ms, 50))) + .collect() + }; + let last: HashMap<_, _> = peers(&[0, 10, -20]).into_iter().collect(); + // Ours went 30 s forward: theirs are 30 s behind it now. + let ours = jumps(&last, &peers(&[-30_000, -29_990, -30_020]), 2_000); + assert_eq!(ours.ours, Some(30_000)); + assert!(ours.peers.is_empty()); + // One peer's went 10 s back. + let one = jumps(&last, &peers(&[-10_000, 10, -20]), 2_000); + assert_eq!(one.ours, None); + assert_eq!(one.peers, vec![("p0".to_owned(), -10_000)]); + // Within the bounds and the limit: nothing. + assert_eq!( + jumps(&last, &peers(&[1_000, 10, -20]), 2_000), + Jumps::default() + ); + // A limit of 0: never. + assert_eq!( + jumps(&last, &peers(&[-30_000, -30_000, -30_000]), 0), + Jumps::default() + ); + } +} diff --git a/src/swarm/source.rs b/src/swarm/source.rs index d80e4d7..1288d86 100644 --- a/src/swarm/source.rs +++ b/src/swarm/source.rs @@ -37,7 +37,6 @@ use tracing::{debug, info, warn}; use crate::locks::MutexExt; use crate::state::{AppState, CatchUpSource, PeerState}; -use crate::swarm::sync::SyncEvent; /// How long a clearly closer peer (by round-trip time) gets to claim the source role /// before whoever asks next takes it. @@ -147,6 +146,7 @@ pub(crate) fn defer(state: &AppState, peer: &str, peer_state: &PeerState) -> Opt last_probe: None, refusals: 0, }); + state.peers_due.by(now + state.timing.catch_up_probe); None } @@ -286,18 +286,39 @@ pub(crate) fn probe_due(state: &AppState, peer: &str, now: Instant) -> bool { due } -/// The initial sync is done: no source any more, and the peers we held back only need -/// to learn our heads now. -pub(crate) fn finished(state: &AppState, from: &str) { +/// When a STATUS to `peer` is next due (`probe_due`), if it is the source. +pub(crate) fn probe_at(state: &AppState, peer: &str) -> Option<Instant> { + if !catching_up(state) { + return None; + } + let probe = state.timing.catch_up_probe; + let source = lock(state); + let current = source.as_ref().filter(|current| current.peer == peer)?; + let heard = current.last_heard + probe; + Some(match current.last_probe { + Some(last) => heard.max(last + probe), + None => heard, + }) +} + +/// No catch-up going on: no source, no head start, nobody avoided. When one ends, and +/// before one starts (a rejoin: `Host::reset_initial_sync`). +pub(crate) fn reset(state: &AppState) { *lock(state) = None; *lock_preference(state) = None; *lock_avoided(state) = None; +} + +/// The initial sync is done: no source any more, and the peers we held back only need +/// to learn our heads now. +pub(crate) fn finished(state: &AppState, from: &str) { + reset(state); let state = state.clone(); let from = from.to_owned(); tokio::spawn(async move { let peers: Vec<String> = state.peer_states.read().await.keys().cloned().collect(); for other in peers.into_iter().filter(|other| *other != from) { - let _ = state.sync_events.send(SyncEvent::PeerNeedsSync(other)); + state.request_sync(&other); } }); } @@ -308,7 +329,7 @@ pub(crate) fn resync_everyone(state: &AppState) { tokio::spawn(async move { let peers: Vec<String> = state.peer_states.read().await.keys().cloned().collect(); for other in peers { - let _ = state.sync_events.send(SyncEvent::PeerNeedsSync(other)); + state.request_sync(&other); } }); } diff --git a/src/swarm/state_file.rs b/src/swarm/state_file.rs index a65fa5b..415fb1a 100644 --- a/src/swarm/state_file.rs +++ b/src/swarm/state_file.rs @@ -185,7 +185,7 @@ pub(crate) async fn update(state: &AppState, now: bool) { /// Where to rejoin through: the addresses of members alive within the electorate /// window, stable ones first, of at most `MAX_SEEDS` members, sorted. Changes only when /// members join, leave, move or change tier, so the file rarely does. -fn seeds(doc: &automerge::AutoCommit, me: &str, window: Duration, now: u64) -> Vec<String> { +fn seeds(doc: &crate::document::Document, me: &str, window: Duration, now: u64) -> Vec<String> { let (mut stable, transient): (Vec<String>, Vec<String>) = members::electorate(doc, now, window) .into_iter() .filter(|pk| pk != me) diff --git a/src/swarm/stats.rs b/src/swarm/stats.rs index b5548d5..04f41b2 100644 --- a/src/swarm/stats.rs +++ b/src/swarm/stats.rs @@ -278,8 +278,8 @@ pub(crate) fn measure_saved_size(state: &AppState) { let doc = std::sync::Arc::clone(&state.doc); // Both steps take time proportional to the document: off the async threads. let measured = match tokio::task::spawn_blocking(move || { - let mut copy = doc.blocking_read().doc.clone(); - copy.save().len() as u64 + let copy = doc.blocking_read().doc.clone(); + crate::document::save(©).len() as u64 }) .await { diff --git a/src/swarm/sync.rs b/src/swarm/sync.rs index df4bb2d..3bfbf7d 100644 --- a/src/swarm/sync.rs +++ b/src/swarm/sync.rs @@ -11,7 +11,7 @@ use tokio::time::Instant; use tracing::{debug, info, trace, warn}; use crate::locks::MutexExt; -use crate::state::{AppState, Liveness, PeerState, Standing}; +use crate::state::{AppState, PeerState, Standing}; use crate::swarm::peer::{peer_swarm, send_hello_once}; use crate::swarm::source; use crate::swarm::throughput::SyncBytes; @@ -25,12 +25,6 @@ pub(crate) enum SyncEvent { PeerNeedsSync(String), } -/// Store paths touched by one batch of document changes. -#[derive(Debug)] -pub(crate) struct ChangeSet { - pub(crate) paths: Vec<Vec<String>>, -} - async fn deliver_sync_to_peer(state: &AppState, peer: &str) -> Result<()> { if state.remake.fetching_snapshot() { // Rejoining: the snapshot comes as a download, not in a sync message. Sync @@ -48,7 +42,7 @@ async fn deliver_sync_to_peer(state: &AppState, peer: &str) -> Result<()> { let validated_swarm = { let peer_state = peer_state_mutex.lock().await; - if matches!(peer_state.liveness, Liveness::Dormant { .. }) { + if peer_state.liveness.is_dormant() { // Reactivation re-handshakes, and PeerValidated then restarts sync. trace!("Skipping sync with dormant peer"); return Ok(()); @@ -74,12 +68,7 @@ async fn deliver_sync_to_peer(state: &AppState, peer: &str) -> Result<()> { ); // Done earlier when the initial sync completes (every peer is enqueued // then); this only matters if the source goes quiet. - let tx = state.sync_events.clone(); - let peer = peer.to_owned(); - tokio::spawn(async move { - tokio::time::sleep(wait).await; - let _ = tx.send(SyncEvent::PeerNeedsSync(peer)); - }); + state.request_sync_after(peer, wait); return Ok(()); } @@ -139,7 +128,16 @@ async fn deliver_sync_to_peer(state: &AppState, peer: &str) -> Result<()> { let ready = state.is_ready().await; // Before this message is counted. let bytes_before = SyncBytes::of(&state.traffic, peer); - let delta_threshold = state.blobs.delta_threshold(); + // Lower towards a peer a message that large wouldn't reach whole: a slow link + // takes a delta chunk by chunk, where a message whole would fail and be sent + // again, whole, every time. + let rate = state + .throughput + .lock_or_recover() + .get(peer) + .map(|estimate| estimate.bytes_per_sec); + let failed = peer_state_mutex.lock().await.failed_whole(Instant::now()); + let delta_threshold = state.blobs.delta_threshold_for(rate, failed); // The session the message comes from, for its send's outcome. let (session, generated) = tokio::task::spawn_blocking( @@ -147,7 +145,7 @@ async fn deliver_sync_to_peer(state: &AppState, peer: &str) -> Result<()> { // Label the message with the epoch of the document it was generated // from, read under the same lock. let (session, epoch, epoch_seq, body) = { - let mut replica = doc.blocking_write(); + let replica = doc.blocking_read(); let mut peer_state = psm.blocking_lock(); let (epoch, epoch_seq) = (replica.epoch, replica.epoch_seq); // Under the same locks as the epoch label: the message belongs to @@ -155,9 +153,8 @@ async fn deliver_sync_to_peer(state: &AppState, peer: &str) -> Result<()> { let message = peer_state .sync - .generate(&mut replica.doc, Instant::now(), bytes_before); - let body = message.map(|message| { - let payload = transport::SyncPayload::encode(message); + .generate(&replica.doc, Instant::now(), bytes_before); + let body = message.map(|payload| { if payload.0.len() <= delta_threshold { return transport::SyncBody::Message(payload); } @@ -210,6 +207,17 @@ async fn deliver_sync_to_peer(state: &AppState, peer: &str) -> Result<()> { let Some(encoded) = generated else { return Ok(()); }; + // Its answer is due. + let reply_due = peer_state_mutex + .lock() + .await + .sync + .reply + .as_ref() + .map(|wait| wait.deadline); + if let Some(at) = reply_due { + state.peers_due.by(at); + } (session, encoded) }; @@ -247,27 +255,22 @@ async fn send_generated( ); if regenerate { - state - .sync_events - .send(SyncEvent::PeerNeedsSync(peer.to_owned())) - .map_err(|error| anyhow!("Failed to notify sync worker: {error}"))?; + state.request_sync(peer); } Ok(()) } Err(error) => { - let Some(delay) = peer_state_mutex - .lock() - .await - .sync - .send_failed(session, Instant::now()) - else { + let delay = { + let mut peer_state = peer_state_mutex.lock().await; + let now = Instant::now(); + peer_state.whole_failed(encoded.payload_len, now); + peer_state.sync.send_failed(session, now) + }; + let Some(delay) = delay else { // The new session may not have opened yet. debug!("Sync message refused, but a reset already discarded it"); - state - .sync_events - .send(SyncEvent::PeerNeedsSync(peer.to_owned())) - .map_err(|error| anyhow!("Failed to notify sync worker: {error}"))?; + state.request_sync(peer); return Ok(()); }; @@ -277,12 +280,7 @@ async fn send_generated( "Sync delivery failed; generating a fresh message later" ); - let tx = state.sync_events.clone(); - let peer = peer.to_owned(); - tokio::spawn(async move { - tokio::time::sleep(delay).await; - let _ = tx.send(SyncEvent::PeerNeedsSync(peer)); - }); + state.request_sync_after(peer, delay); Ok(()) } @@ -372,12 +370,13 @@ pub(crate) async fn handle_sync( let message = payload; let peer = peer.to_owned(); let clock = Arc::clone(&state.clock); + let ahead_ms = state.clock_ahead_ms.load(Ordering::Relaxed); // Some((caught_up, heads changed)) when applied, None when the message belongs // to another epoch. tokio::task::spawn_blocking(move || -> Result<Option<(bool, bool)>> { // Before the lock: decoding a large message takes a while. - let message = message.decode()?; + let message = crate::document::sync::decode(&message)?; let mut replica = doc.blocking_write(); // Authoritative check, under the same lock as the apply: the document may @@ -390,9 +389,9 @@ pub(crate) async fn handle_sync( peer_state.reports_ready = Some(from_ready); let heads_before = replica.heads().to_vec(); - peer_state - .sync - .receive(&mut replica.doc, message) + let sync = &mut peer_state.sync; + replica + .logged(|doc, log| (sync.receive_logged(doc, message, log), true)) .context("Automerge sync failed")?; peer_state.last_sync_rx = Some(Instant::now()); if let Some(wait) = &peer_state.sync.reply { @@ -408,16 +407,14 @@ pub(crate) async fn handle_sync( stats.heads_seen(heads_after.len()); let changed = heads_after != heads_before; if changed && measure_propagation { - let now_ms = clock.now_millis().as_u64() as i64; - // Time 0: not stamped (snapshots, older nodes). - let delays: Vec<i64> = replica - .doc - .get_changes(&heads_before) - .iter() - .map(|change| change.timestamp()) - .filter(|&time| time > 0) - .map(|time| now_ms - time) - .collect(); + // Now by the peers' clocks (their median): this node's own clock skew + // doesn't count; the writer's still does. + let now_ms = clock.now_millis().as_u64() as i64 - ahead_ms; + let delays: Vec<i64> = + crate::document::changes::commit_times(&replica.doc, &heads_before) + .into_iter() + .map(|time| now_ms - time) + .collect(); stats.propagated(delays); } @@ -427,7 +424,7 @@ pub(crate) async fn handle_sync( // catching up has them, and so does a ready peer whose document is empty // (a misconfigured second swarm, or a genesis node in its first seconds, // before it registered itself). - check_caught_up && from_ready && peer_state.sync.have_all_theirs(&mut replica.doc), + check_caught_up && from_ready && peer_state.sync.have_all_theirs(&replica.doc), changed, ))) }) @@ -449,6 +446,7 @@ pub(crate) async fn handle_sync( .stats .caught_up(state.traffic.total().received.sync.bytes); source::finished(state, peer); + state.join_progress.notify_one(); } state.notify_store_changed(); @@ -492,7 +490,7 @@ fn fetch_delta(state: &AppState, peer: &str, epoch: transport::Epoch, spec: tran } } // Our next message says what we have now; the peer answers with the rest. - let _ = state.sync_events.send(SyncEvent::PeerNeedsSync(peer)); + state.request_sync(&peer); }); } @@ -523,16 +521,9 @@ async fn download_and_apply_delta( bail!("the epoch changed during the download"); } let heads_before = replica.heads().to_vec(); - replica - .doc - .load_incremental(&bytes) - .context("Loading the downloaded changes")?; + crate::document::changes::load_delta(&mut replica.doc, &bytes)?; replica.refresh_heads(); - let missing = spec - .to - .iter() - .filter(|head| replica.doc.get_change_by_hash(head).is_none()) - .count(); + let missing = crate::document::changes::missing(&replica.doc, &spec.to).len(); if missing > 0 { bail!("{missing} of the offered heads are still missing after loading"); } @@ -623,87 +614,3 @@ pub(crate) async fn run_sync_worker(state: AppState, mut rx: mpsc::UnboundedRece } } } - -// --- change publisher -------------------------------------------------------- - -fn patch_store_path(patch: &automerge::Patch) -> Vec<String> { - let mut path: Vec<String> = patch - .path - .iter() - .map(|(_, prop)| match prop { - automerge::Prop::Map(key) => key.clone(), - automerge::Prop::Seq(index) => index.to_string(), - }) - .collect(); - - match &patch.action { - automerge::PatchAction::PutMap { key, .. } - | automerge::PatchAction::DeleteMap { key, .. } => path.push(key.clone()), - // List, text and counter changes: the containing object changed. - _ => {} - } - path -} - -pub(crate) fn paths_related(a: &[String], b: &[String]) -> bool { - a.starts_with(b) || b.starts_with(a) -} - -pub(crate) async fn run_change_publisher(state: AppState) { - let mut changed = state.change_tx.subscribe(); - let mut last_heads = state.write_doc(|doc| doc.get_heads()).await; - - info!("Starting change publisher"); - - while changed.changed().await.is_ok() { - let before = last_heads.clone(); - - let (heads, paths, conflicts) = state - .write_doc(move |doc| { - let heads = doc.get_heads(); - if heads == before { - return (heads, Vec::new(), Vec::new()); - } - - // After an epoch switch the previous heads belong to the old document: - // report the root as changed, which every stream is related to. - if !before.iter().all(|h| doc.get_change_by_hash(h).is_some()) { - return (heads, vec![Vec::new()], Vec::new()); - } - - let mut paths = Vec::new(); - let mut conflicts = Vec::new(); - for patch in doc.diff(&before, &heads) { - let path = patch_store_path(&patch); - if matches!( - patch.action, - automerge::PatchAction::PutMap { conflict: true, .. } - ) { - conflicts.push(path.clone()); - } - paths.push(path); - } - (heads, paths, conflicts) - }) - .await; - - if heads == last_heads { - continue; - } - last_heads = heads; - - for path in &conflicts { - warn!(path = ?path, "Concurrent writes produced a conflict"); - } - - if paths - .iter() - .any(|path| crate::swarm::topology::affects_neighbours(path)) - { - state.topology_changed.notify_one(); - } - - // Fails only when nobody is subscribed. - let _ = state.change_events.send(Arc::new(ChangeSet { paths })); - } -} diff --git a/src/swarm/topology.rs b/src/swarm/topology.rs deleted file mode 100644 index 1d98637..0000000 --- a/src/swarm/topology.rs +++ /dev/null @@ -1,843 +0,0 @@ -//! Which members a node syncs with: its neighbours. -//! -//! In a full mesh every change reaches every node from every other node (Automerge -//! decides per pair, so nobody knows that a third peer already passed it on), so the -//! copies of one write grow with N². Above `k` members, each node keeps only a few -//! neighbours, and changes travel over several hops. -//! -//! Every node computes the graph from the member list and which members it can reach, -//! with no coordination: -//! * a ring ordered by key hash: each member's nearest reachable member in both -//! directions, which keeps the reachable members connected; -//! * plus, for each member, the `k - 2` reachable others with the lowest pair score (a -//! hash of both keys, so both ends agree on it), which keeps paths short. -//! -//! A link exists if either end chose it, so links are symmetric and a node has at least -//! `k` neighbours. With at most `k + 1` reachable members the graph is the full mesh: -//! small swarms behave exactly as before. A member joining, leaving or going offline -//! changes only links near it. -//! -//! Members on several transports (Mycelium, libp2p) get such a graph per transport, -//! among the members on it (as their addresses in the member list say), and a node's -//! neighbours are the union of its neighbours on each transport it is on -//! (`neighbours_across`). Each of those graphs is connected (its ring), and a member on -//! two transports is in both: as long as members on several transports link the -//! transports, the swarm stays connected, and such a member has about `k` neighbours per -//! transport. With one transport, it is the one graph as before. - -use std::collections::{BTreeMap, BTreeSet}; - -use sha2::{Digest, Sha256}; -use tracing::info; - -use crate::locks::RwLockExt; -use crate::state::AppState; -use crate::swarm::members; - -/// This node's current neighbours, and the member list they were computed from. -#[derive(Debug, Default)] -pub(crate) struct NeighbourView { - members: Vec<String>, - k: usize, - unreachable: BTreeSet<String>, - stable: BTreeSet<String>, - /// The transports (address schemes) of each member, and ours. - transports: BTreeMap<String, BTreeSet<String>>, - local_transports: BTreeSet<String>, - /// Peers that proved they are of our swarm but aren't in our member list (yet): their - /// registration hasn't reached us. They are neighbours too (`refresh`). - unlisted: BTreeSet<String>, - neighbours: BTreeSet<String>, - /// Every reachable member is a neighbour. - full_mesh: bool, -} - -/// Whether the neighbours are computed from the member list yet (before, everyone -/// counts as a neighbour). -pub(crate) fn view_known(state: &AppState) -> bool { - state.neighbours.read_or_recover().is_some() -} - -/// Whether every reachable member is our neighbour (also while the view is unknown). -pub(crate) fn is_full_mesh(state: &AppState) -> bool { - state - .neighbours - .read_or_recover() - .as_ref() - .is_none_or(|view| view.full_mesh) -} - -/// The current view in numbers, once known. -#[derive(Debug, Clone, Copy)] -pub(crate) struct Summary { - /// Registered members, us included. - pub(crate) members: usize, - /// Members other than us not currently judged unreachable. - pub(crate) reachable: usize, - pub(crate) stable: usize, - pub(crate) neighbours: usize, - pub(crate) stable_neighbours: usize, - pub(crate) full_mesh: bool, -} - -pub(crate) fn summary(state: &AppState) -> Option<Summary> { - let local = state.local_id(); - state - .neighbours - .read_or_recover() - .as_ref() - .map(|view| Summary { - members: view.members.len(), - reachable: view - .members - .iter() - .filter(|m| m.as_str() != local && !view.unreachable.contains(*m)) - .count(), - stable: view.stable.len(), - neighbours: view.neighbours.len(), - stable_neighbours: view - .neighbours - .iter() - .filter(|m| view.stable.contains(*m)) - .count(), - full_mesh: view.full_mesh, - }) -} - -/// Whether we initiate contact with `peer`: HELLOs, keepalives, dormancy checks, sync -/// openings. Everyone is a neighbour until the view is known (while joining, or before -/// any member registered), so joining works as in a full mesh. We always *answer* -/// anyone who contacts us. -pub(crate) fn is_neighbour(state: &AppState, peer: &str) -> bool { - match &*state.neighbours.read_or_recover() { - Some(view) => view.neighbours.contains(peer), - None => true, - } -} - -/// How the neighbours changed. -pub(crate) struct NeighbourChange { - pub(crate) added: Vec<String>, - pub(crate) removed: Vec<String>, -} - -/// Recomputes the neighbours whenever something they depend on may have changed -/// (`AppState::topology_changed`): the member list (`swarm::sync::run_change_publisher`), -/// a peer becoming unreachable or reachable again (`swarm::peer`), and every 5 s as a -/// fallback. A burst of notifications coalesces into one recomputation. -pub(crate) async fn run_worker(state: AppState) { - loop { - state.topology_changed.notified().await; - if let Some(change) = refresh(&state).await { - crate::swarm::peer::apply_neighbour_change(&state, change).await; - } - } -} - -/// Whether a change at `path` may change the neighbours: one in the member list, or the -/// root (the whole document replaced, as after an epoch switch). -pub(crate) fn affects_neighbours(path: &[String]) -> bool { - path.is_empty() || path.starts_with(&members::members_dir()) -} - -/// Hands the members' `aliases` to the logger, under every name that stands for each: -/// its node ID, and its addresses (`logging`, which renames them bare too). -fn set_aliases(state: &AppState, aliases: std::collections::HashMap<String, String>) { - let mut names = std::collections::HashMap::new(); - for (member, alias) in aliases { - for address in state.transport.identities_of(&member) { - names.insert(address, alias.clone()); - } - names.insert(member, alias); - } - crate::logging::set_aliases(names); -} - -/// Recomputes the neighbours once ready, if the member list, the reachable members or -/// `k` changed. -pub(crate) async fn refresh(state: &AppState) -> Option<NeighbourChange> { - if !state.is_ready().await { - return None; - } - let (current, stable, transports, alive) = { - let replica = state.doc.read().await; - members::learn_addresses(state, &replica.doc); - set_aliases(state, members::aliases(&replica.doc)); - state.transport.learn_relays(&members::relays(&replica.doc)); - let current = members::members(&replica.doc); - // Those that said they are alive within the electorate window. - let alive: BTreeSet<String> = members::electorate( - &replica.doc, - state.clock.now_millis().0, - state.electorate_window, - ) - .into_iter() - .collect(); - let stable: BTreeSet<String> = current - .iter() - .filter(|m| members::tier(&replica.doc, m) == members::Tier::Stable) - .cloned() - .collect(); - let transports: BTreeMap<String, BTreeSet<String>> = current - .iter() - .map(|m| (m.clone(), schemes(&members::addresses(&replica.doc, m)))) - .collect(); - (current, stable, transports, alive) - }; - let local_transports: BTreeSet<String> = state.transport.transports().into_iter().collect(); - let k = state.neighbours_k; - // Those that don't answer, those on none of our transports, and those that haven't - // said they are alive for longer than the electorate window, unless we heard from - // them since we started (one that comes back says so to us first: its HELLO). - let mut unreachable = BTreeSet::new(); - for member in ¤t { - if member == state.local_id() { - continue; - } - if !state.transport.can_reach(member) { - unreachable.insert(member.clone()); - continue; - } - let (gone, heard) = match state.get_peer_state(member).await { - Some(peer_state) => { - let peer_state = peer_state.lock().await; - (peer_state.is_unreachable(), peer_state.last_seen.is_some()) - } - None => (false, false), - }; - if gone || (!heard && !alive.contains(member)) { - unreachable.insert(member.clone()); - } - } - // A node that joined while the genesis node hadn't registered yet has the genesis - // document without the genesis node in it, and registers itself there: then neither - // lists the other, and with each the other's neighbour only by the member list, - // neither would ever sync with the other, and their registrations never pass. So a - // peer that proved itself a member of our swarm counts until the list says otherwise: - // syncing brings each the other's registration. - let mut unlisted = BTreeSet::new(); - for (peer, peer_state) in state.peers_snapshot().await { - if current.contains(&peer) || peer == state.local_id() { - continue; - } - let peer_state = peer_state.lock().await; - if matches!(peer_state.standing, crate::state::Standing::Member { .. }) - && !peer_state.is_unreachable() - { - unlisted.insert(peer); - } - } - let previous = { - let view = state.neighbours.read_or_recover(); - if view.as_ref().is_some_and(|view| { - view.members == current - && view.unlisted == unlisted - && view.k == k - && view.unreachable == unreachable - && view.stable == stable - && view.transports == transports - && view.local_transports == local_transports - }) { - return None; - } - view.as_ref().map(|view| view.neighbours.clone()) - }; - - // Before the first view everyone known counted as a neighbour. - let previous = match previous { - Some(previous) => previous, - None => state.peer_states.read().await.keys().cloned().collect(), - }; - let mut computed = neighbours_across( - &transports, - state.local_id(), - &local_transports, - k, - &|m| unreachable.contains(m), - &|m| stable.contains(m), - ); - let listed_neighbours = computed.len(); - computed.extend(unlisted.iter().cloned()); - let change = NeighbourChange { - added: computed - .iter() - .filter(|m| !previous.contains(*m)) - .cloned() - .collect(), - removed: previous - .iter() - .filter(|m| !computed.contains(*m)) - .cloned() - .collect(), - }; - info!( - members = current.len(), - unreachable = unreachable.len(), - stable = stable.len(), - k, - neighbours = computed.len(), - unlisted = unlisted.len(), - added = change.added.len(), - removed = change.removed.len(), - "Neighbours computed from the member list" - ); - let reachable_others = current - .iter() - .filter(|m| m.as_str() != state.local_id() && !unreachable.contains(*m)) - .count(); - *state.neighbours.write_or_recover() = Some(NeighbourView { - full_mesh: listed_neighbours >= reachable_others, - members: current, - k, - unreachable, - stable, - transports, - local_transports, - unlisted, - neighbours: computed, - }); - Some(change) -} - -const RING_LABEL: &[u8] = b"continuum/topology-ring/1"; -const PAIR_LABEL: &[u8] = b"continuum/topology-pair/2"; - -/// How many members may prefer the same stable member, as a multiple of `k`. Without a -/// bound, 20 stable members among 2000 get about 500 neighbours each: every -/// announcement in the swarm lands on them, and the backbone saturates first. Members -/// that find a stable member full prefer the next one, then go by pair score, so the -/// graph stays connected and symmetric. -const STABLE_FANIN_FACTOR: usize = 4; - -/// The transports `addresses` are on: their schemes (`mycelium` of `mycelium:<key>`); -/// the empty one for an address without (a transport whose identities are addresses). -pub(crate) fn schemes(addresses: &[String]) -> BTreeSet<String> { - addresses - .iter() - .map(|address| { - address - .split_once(':') - .map_or("", |(scheme, _)| scheme) - .to_owned() - }) - .collect() -} - -/// The neighbours of `local` among `members` (by the transports each is on), on each of -/// `local_transports`: on each, its neighbours among the members on it -/// (`neighbours`), all together. -pub(crate) fn neighbours_across( - members: &BTreeMap<String, BTreeSet<String>>, - local: &str, - local_transports: &BTreeSet<String>, - k: usize, - unreachable: &dyn Fn(&str) -> bool, - stable: &dyn Fn(&str) -> bool, -) -> BTreeSet<String> { - let mut all = BTreeSet::new(); - for transport in local_transports { - let on_it: Vec<String> = members - .iter() - .filter(|(member, transports)| *member != local && transports.contains(transport)) - .map(|(member, _)| member.clone()) - .collect(); - all.extend(neighbours(&on_it, local, k, unreachable, stable)); - } - all -} - -/// The neighbours of `local` among `members` (which may or may not contain `local`), -/// skipping members for which `unreachable` holds. -pub(crate) fn neighbours( - members: &[String], - local: &str, - k: usize, - unreachable: &dyn Fn(&str) -> bool, - stable: &dyn Fn(&str) -> bool, -) -> BTreeSet<String> { - let mut all: Vec<String> = members.to_vec(); - all.push(local.to_owned()); - graph(&all, k, &|m: &str| m != local && unreachable(m), stable) - .remove(local) - .unwrap_or_default() -} - -/// Every reachable member's neighbours. Hashes each member once and scores pairs by -/// mixing the two hashes (`pair_score`), so the N² scores are a few nanoseconds each: -/// tens of milliseconds for thousands of members. `stable` members (`swarm::stability`) -/// are preferred for half of each member's scored slots, each by at most -/// `STABLE_FANIN_FACTOR * k` members. -/// -/// Each reachable member walks the ring (by key hash) both ways to the first reachable -/// member, which gives a ring over the reachable members, and adds the `k - 2` -/// reachable members with the lowest pair score. A link exists if either end chose it. -/// With at most `k + 1` reachable members, the reachable members form a full mesh. -/// -/// Each node judges reachability by itself, so nodes may briefly disagree; that only -/// adds links (whoever chose a link initiates, the other answers), it never cuts one. -/// The preference budget is spent in key order, which every node shares, so nodes that -/// agree on reachability compute the same links. -pub(crate) fn graph( - members: &[String], - k: usize, - unreachable: &dyn Fn(&str) -> bool, - stable: &dyn Fn(&str) -> bool, -) -> BTreeMap<String, BTreeSet<String>> { - let k = k.max(2); - let all: BTreeSet<&str> = members.iter().map(String::as_str).collect(); - let live: BTreeSet<&str> = all.iter().copied().filter(|m| !unreachable(m)).collect(); - let mut links: BTreeMap<String, BTreeSet<String>> = live - .iter() - .map(|m| ((*m).to_owned(), BTreeSet::new())) - .collect(); - - let mut link = |a: &str, b: &str| { - if a != b { - links.get_mut(a).expect("live member").insert(b.to_owned()); - links.get_mut(b).expect("live member").insert(a.to_owned()); - } - }; - - if live.len() <= k + 1 { - for a in &live { - for b in &live { - link(a, b); - } - } - return links; - } - - // The ring over all members, so its order doesn't change when one goes offline. - let mut ring: Vec<([u8; 32], &str)> = - all.iter().map(|m| (hash(RING_LABEL, &[*m]), *m)).collect(); - ring.sort_unstable(); - let n = ring.len(); - let position: BTreeMap<&str, usize> = - ring.iter().enumerate().map(|(i, (_, m))| (*m, i)).collect(); - let first_live = |from: usize, step: usize| { - (1..n) - .map(|d| ring[(from + d * step) % n].1) - .find(|m| live.contains(m)) - .expect("more than k + 1 live members") - }; - - // Live members in key order, with their pair keys; and those of them that are stable. - let keyed: Vec<(&str, u64)> = live.iter().map(|m| (*m, pair_key(m))).collect(); - let stable_keyed: Vec<(&str, u64)> = keyed.iter().copied().filter(|(m, _)| stable(m)).collect(); - - // How many members have preferred each stable member so far (`keyed` is in key - // order, the same on every node). - let budget = STABLE_FANIN_FACTOR * k; - let mut preferred_by: BTreeMap<&str, usize> = BTreeMap::new(); - // Half the slots (rounded up) prefer stable members, so long-running nodes form a - // backbone; the rest go by pair score alone, keeping paths short. A stable member - // enough others prefer already is skipped, so no hub grows with the swarm. - let stable_slots = (k - 2).div_ceil(2); - - for &(member, key) in &keyed { - let at = position[member]; - let next = first_live(at, 1); - let prev = first_live(at, n - 1); - link(member, next); - link(member, prev); - let skip = [member, next, prev]; - // The stable candidates in score order (there are few). - let mut stable_candidates = scored(&stable_keyed, key, skip); - stable_candidates.sort_unstable(); - let preferred: Vec<&str> = stable_candidates - .iter() - .map(|(_, other)| *other) - .filter(|other| preferred_by.get(other).copied().unwrap_or(0) < budget) - .take(stable_slots) - .collect(); - for other in preferred.iter().copied() { - *preferred_by.entry(other).or_default() += 1; - } - // The rest: the lowest scores among the others that aren't preferred, which are - // among the `need + preferred` lowest overall. Selecting those is linear; only - // they are sorted. - let need = k - 2 - preferred.len(); - let mut candidates = scored(&keyed, key, skip); - let lowest = (need + preferred.len()).min(candidates.len()); - if lowest < candidates.len() { - candidates.select_nth_unstable(lowest); - candidates.truncate(lowest); - } - candidates.sort_unstable(); - let rest = candidates - .iter() - .map(|(_, other)| *other) - .filter(|other| !preferred.contains(other)) - .take(need); - for partner in preferred.iter().copied().chain(rest) { - link(member, partner); - } - } - links -} - -/// `others` but those in `skip`, scored as pairs with the member whose key is `key`. -fn scored<'a>(others: &[(&'a str, u64)], key: u64, skip: [&str; 3]) -> Vec<(u64, &'a str)> { - others - .iter() - .filter(|(other, _)| !skip.contains(other)) - .map(|&(other, other_key)| (mix_pair(key, other_key), other)) - .collect() -} - -/// Symmetric: the same for (a, b) and (b, a). -pub(crate) fn pair_score(a: &str, b: &str) -> u64 { - mix_pair(pair_key(a), pair_key(b)) -} - -/// What a member contributes to its pair scores: computed once per member. -fn pair_key(member: &str) -> u64 { - let digest = hash(PAIR_LABEL, &[member]); - u64::from_be_bytes(digest[..8].try_into().expect("8 bytes")) -} - -/// The score of a pair from its members' keys, symmetric: SplitMix64's finalizer over -/// the lower key and the mixed higher one. Keys are uniform (SHA-256), so for each member -/// the others' scores are in an order unrelated to anyone else's. -fn mix_pair(a: u64, b: u64) -> u64 { - let (low, high) = if a <= b { (a, b) } else { (b, a) }; - mix(low ^ mix(high)) -} - -fn mix(mut z: u64) -> u64 { - z = z.wrapping_add(0x9e37_79b9_7f4a_7c15); - z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); - z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); - z ^ (z >> 31) -} - -fn hash(label: &[u8], fields: &[&str]) -> [u8; 32] { - let mut hasher = Sha256::new(); - for field in [label] - .into_iter() - .chain(fields.iter().map(|f| f.as_bytes())) - { - hasher.update((field.len() as u32).to_be_bytes()); - hasher.update(field); - } - hasher.finalize().into() -} - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::VecDeque; - - fn members(n: usize) -> Vec<String> { - (0..n).map(|i| format!("{:064x}", i * 7919 + 13)).collect() - } - - fn all_reachable(_: &str) -> bool { - false - } - - fn none_stable(_: &str) -> bool { - false - } - - fn graph(members: &[String], k: usize) -> BTreeMap<String, BTreeSet<String>> { - super::graph(members, k, &all_reachable, &none_stable) - } - - fn connected(g: &BTreeMap<String, BTreeSet<String>>) -> bool { - let Some(start) = g.keys().next() else { - return true; - }; - let mut seen = BTreeSet::from([start.clone()]); - let mut queue = VecDeque::from([start.clone()]); - while let Some(m) = queue.pop_front() { - for next in &g[&m] { - if seen.insert(next.clone()) { - queue.push_back(next.clone()); - } - } - } - seen.len() == g.len() - } - - fn edges(graph: &BTreeMap<String, BTreeSet<String>>) -> BTreeSet<(String, String)> { - graph - .iter() - .flat_map(|(a, ns)| { - ns.iter().map(move |b| { - if a < b { - (a.clone(), b.clone()) - } else { - (b.clone(), a.clone()) - } - }) - }) - .collect() - } - - fn hex(bytes: [u8; 32]) -> String { - bytes.iter().map(|byte| format!("{byte:02x}")).collect() - } - - /// Over random swarms (size, `k`, who is reachable, who is stable): links only - /// between reachable members, never to oneself, symmetric; a full mesh while small, - /// otherwise at least `k` links each and connected; independent of the order members - /// are listed in; and each member's own view (`neighbours`) agrees with the graph. - #[test] - fn random_swarms() { - crate::testrand::for_seeds(300, |rng| { - let n = rng.between(1, 60); - let k = rng.between(2, 14); - let mut all: Vec<String> = (0..n).map(|_| hex(rng.bytes32())).collect(); - let unreachable: BTreeSet<String> = - all.iter().filter(|_| rng.chance(0.3)).cloned().collect(); - let stable: BTreeSet<String> = - all.iter().filter(|_| rng.chance(0.3)).cloned().collect(); - let is_unreachable = |m: &str| unreachable.contains(m); - let is_stable = |m: &str| stable.contains(m); - - let g = super::graph(&all, k, &is_unreachable, &is_stable); - // Owned: `all` is shuffled below. - let live: BTreeSet<String> = all - .iter() - .filter(|m| !unreachable.contains(*m)) - .cloned() - .collect(); - assert_eq!(g.keys().cloned().collect::<BTreeSet<_>>(), live); - for (a, ns) in &g { - assert!(!ns.contains(a), "{a} links to itself"); - for b in ns { - assert!(live.contains(b), "{a} links to unreachable {b}"); - assert!(g[b].contains(a), "{a} -> {b} but not back"); - } - } - if live.len() <= k + 1 { - assert!( - g.values().all(|ns| ns.len() + 1 == live.len()), - "not a full mesh" - ); - } else { - assert!( - g.values().all(|ns| ns.len() >= k), - "a member with fewer than k links" - ); - assert!(connected(&g), "not connected"); - } - - rng.shuffle(&mut all); - assert_eq!(super::graph(&all, k, &is_unreachable, &is_stable), g); - - if let Some(local) = live.iter().nth(rng.below(live.len())) { - let others: Vec<String> = all.iter().filter(|m| *m != local).cloned().collect(); - assert_eq!( - neighbours(&others, local, k, &is_unreachable, &is_stable), - g[local] - ); - } - }); - } - - #[test] - fn small_swarms_are_a_full_mesh() { - let ms = members(13); - for m in &ms { - assert_eq!( - neighbours(&ms, m, 12, &all_reachable, &none_stable).len(), - 12 - ); - } - } - - #[test] - fn links_are_symmetric_and_degree_at_least_k() { - let ms = members(250); - let g = graph(&ms, 8); - for (a, ns) in &g { - assert!(ns.len() >= 8, "{a} has {} neighbours", ns.len()); - assert!(!ns.contains(a)); - for b in ns { - assert!(g[b].contains(a), "{a} -> {b} not symmetric"); - } - } - } - - #[test] - fn graph_is_connected() { - for (n, k) in [(250, 8), (250, 2), (60, 3)] { - assert!(connected(&graph(&members(n), k)), "n={n} k={k}"); - } - } - - #[test] - fn one_member_joining_changes_few_links() { - let ms = members(200); - let before = edges(&graph(&ms, 8)); - let mut more = ms.clone(); - more.push(format!("{:064x}", 999_999)); - let after = edges(&graph(&more, 8)); - let changed = before.symmetric_difference(&after).count(); - assert!(changed <= 4 * 8, "{changed} links changed"); - } - - #[test] - fn reachable_members_stay_connected_when_most_are_offline() { - let ms = members(100); - // Every tenth member is online. - let online: BTreeSet<String> = ms.iter().step_by(10).cloned().collect(); - let offline = |m: &str| !online.contains(m); - let g = super::graph(&ms, 12, &offline, &none_stable); - assert_eq!(g.len(), 10); - // Ten reachable members and k = 12: a full mesh among them. - assert!(g.values().all(|ns| ns.len() == 9)); - - let online: BTreeSet<String> = ms.iter().step_by(4).cloned().collect(); - let offline = |m: &str| !online.contains(m); - let g = super::graph(&ms, 8, &offline, &none_stable); - assert_eq!(g.len(), 25); - assert!(connected(&g)); - assert!( - g.values() - .all(|ns| ns.len() >= 8 && ns.iter().all(|n| online.contains(n))) - ); - } - - #[test] - fn stable_members_become_hubs_with_bounded_degree() { - let ms = members(250); - let stable: BTreeSet<String> = ms.iter().step_by(12).take(20).cloned().collect(); - let is_stable = |m: &str| stable.contains(m); - let g = super::graph(&ms, 12, &all_reachable, &is_stable); - assert!(connected(&g)); - - let degree = |pred: &dyn Fn(&String) -> bool| -> (usize, f64) { - let degrees: Vec<usize> = g - .iter() - .filter(|(m, _)| pred(m)) - .map(|(_, ns)| ns.len()) - .collect(); - let max = *degrees.iter().max().unwrap(); - ( - max, - degrees.iter().sum::<usize>() as f64 / degrees.len() as f64, - ) - }; - let (stable_max, stable_mean) = degree(&|m| stable.contains(m)); - let (_, transient_mean) = degree(&|m| !stable.contains(m)); - // 230 transient members put about 5 of their 10 scored slots on 20 stable ones. - assert!( - stable_mean > 3.0 * transient_mean, - "{stable_mean} vs {transient_mean}" - ); - assert!( - stable_max <= 100, - "a stable member has {stable_max} neighbours" - ); - } - - /// A hub's degree in a swarm of `n` with `stable_count` stable members stays within - /// the bound. - fn hub_degree_is_bounded(n: usize, stable_count: usize) { - // Few stable members among many: without the fan-in budget each would be chosen - // by most of the swarm (about 130 neighbours with 10 of 250). With it, a hub has - // at most the budget plus the links others choose by pair score and the ring. - let k = 12; - let ms = members(n); - let stable: BTreeSet<String> = ms.iter().step_by(n / stable_count).cloned().collect(); - let is_stable = |m: &str| stable.contains(m); - let g = super::graph(&ms, k, &all_reachable, &is_stable); - assert!(connected(&g), "n={n} stable={stable_count}"); - let hub_max = g - .iter() - .filter(|(m, _)| stable.contains(*m)) - .map(|(_, ns)| ns.len()) - .max() - .unwrap(); - assert!( - hub_max <= 8 * k, - "n={n} stable={stable_count}: a stable member has {hub_max} neighbours" - ); - } - - /// Two transports with members on one or the other, and two members on both: every - /// link is over a shared transport, and the two members on both connect the rest. - #[test] - fn members_on_two_transports_bridge_the_others() { - let k = 8; - let ms = members(201); - let on = |names: &[&str]| -> BTreeSet<String> { - names.iter().map(|name| (*name).to_owned()).collect() - }; - let transports: BTreeMap<String, BTreeSet<String>> = ms - .iter() - .enumerate() - .map(|(i, m)| { - let transports = match i { - 0 | 1 => on(&["a", "b"]), - i if i % 2 == 0 => on(&["a"]), - _ => on(&["b"]), - }; - (m.clone(), transports) - }) - .collect(); - let no_stable = |_: &str| false; - let mut graph: BTreeMap<String, BTreeSet<String>> = BTreeMap::new(); - for (m, own) in &transports { - for n in neighbours_across(&transports, m, own, k, &all_reachable, &no_stable) { - assert!( - !transports[&n].is_disjoint(own), - "{m} and {n} share no transport" - ); - graph.entry(m.clone()).or_default().insert(n.clone()); - graph.entry(n).or_default().insert(m.clone()); - } - } - assert_eq!(graph.len(), ms.len()); - assert!(connected(&graph)); - let bridge = &ms[0]; - assert!(graph[bridge].iter().any(|n| !transports[n].contains("b"))); - assert!(graph[bridge].iter().any(|n| !transports[n].contains("a"))); - - // One transport: the one graph, as without transports. - let one: BTreeMap<String, BTreeSet<String>> = - ms.iter().map(|m| (m.clone(), on(&[""]))).collect(); - for m in &ms[..5] { - assert_eq!( - neighbours_across(&one, m, &on(&[""]), k, &all_reachable, &no_stable), - neighbours(&ms, m, k, &all_reachable, &no_stable) - ); - } - } - - #[test] - fn schemes_of_addresses() { - let addresses = - |all: &[&str]| -> Vec<String> { all.iter().map(|a| (*a).to_owned()).collect() }; - assert_eq!( - schemes(&addresses(&[ - "mycelium:ab", - "libp2p:/ip4/1.2.3.4/tcp/1", - "libp2p:/ip6/::1/tcp/1" - ])), - BTreeSet::from(["libp2p".to_owned(), "mycelium".to_owned()]) - ); - assert_eq!(schemes(&addresses(&["a"])), BTreeSet::from([String::new()])); - assert!(schemes(&[]).is_empty()); - } - - #[test] - fn hub_degree_is_bounded_however_few_stable_members() { - for (n, stable_count) in [(250, 10), (400, 5)] { - hub_degree_is_bounded(n, stable_count); - } - } - - /// The same at the sizes the bound was designed for; slow in debug builds, so run - /// on request (`cargo test --release -- --ignored`). - #[test] - #[ignore] - fn hub_degree_is_bounded_in_large_swarms() { - for (n, stable_count) in [(1000, 10), (1000, 20), (2000, 20)] { - hub_degree_is_bounded(n, stable_count); - } - } -} diff --git a/src/swarm/topology/graph.rs b/src/swarm/topology/graph.rs new file mode 100644 index 0000000..fb33170 --- /dev/null +++ b/src/swarm/topology/graph.rs @@ -0,0 +1,236 @@ +//! The graph itself, from the members alone: a ring by hash, plus each member's +//! lowest-scoring pairs, stable members preferred; per transport, and across them. Pure: +//! the same members give every node the same graph. + +use sha2::{Digest, Sha256}; + +use super::*; + +pub(super) const RING_LABEL: &[u8] = b"continuum/topology-ring/1"; + +pub(super) const PAIR_LABEL: &[u8] = b"continuum/topology-pair/2"; + +/// How many members may prefer the same stable member, as a multiple of `k`. Without a +/// bound, 20 stable members among 2000 get about 500 neighbours each: every +/// announcement in the swarm lands on them, and the backbone saturates first. Members +/// that find a stable member full prefer the next one, then go by pair score, so the +/// graph stays connected and symmetric. +pub(super) const STABLE_FANIN_FACTOR: usize = 4; + +/// The transports `addresses` are on: their schemes (`mycelium` of `mycelium:<key>`); +/// the empty one for an address without (a transport whose identities are addresses). +pub(crate) fn schemes(addresses: &[String]) -> BTreeSet<String> { + addresses + .iter() + .map(|address| { + address + .split_once(':') + .map_or("", |(scheme, _)| scheme) + .to_owned() + }) + .collect() +} + +/// The neighbours of `local` among `members` (by the transports each is on), on each of +/// `local_transports`: on each, its neighbours among the members on it +/// (`neighbours`), all together. +pub(crate) fn neighbours_across( + members: &BTreeMap<String, BTreeSet<String>>, + local: &str, + local_transports: &BTreeSet<String>, + k: usize, + unreachable: &dyn Fn(&str) -> bool, + stable: &dyn Fn(&str) -> bool, +) -> BTreeSet<String> { + let mut all = BTreeSet::new(); + for transport in local_transports { + let on_it: Vec<String> = members + .iter() + .filter(|(member, transports)| *member != local && transports.contains(transport)) + .map(|(member, _)| member.clone()) + .collect(); + all.extend(neighbours(&on_it, local, k, unreachable, stable)); + } + all +} + +/// The neighbours of `local` among `members` (which may or may not contain `local`), +/// skipping members for which `unreachable` holds. +pub(crate) fn neighbours( + members: &[String], + local: &str, + k: usize, + unreachable: &dyn Fn(&str) -> bool, + stable: &dyn Fn(&str) -> bool, +) -> BTreeSet<String> { + let mut all: Vec<String> = members.to_vec(); + all.push(local.to_owned()); + graph(&all, k, &|m: &str| m != local && unreachable(m), stable) + .remove(local) + .unwrap_or_default() +} + +/// Every reachable member's neighbours. Hashes each member once and scores pairs by +/// mixing the two hashes (`pair_score`), so the N² scores are a few nanoseconds each: +/// tens of milliseconds for thousands of members. `stable` members (`swarm::stability`) +/// are preferred for half of each member's scored slots, each by at most +/// `STABLE_FANIN_FACTOR * k` members. +/// +/// Each reachable member walks the ring (by key hash) both ways to the first reachable +/// member, which gives a ring over the reachable members, and adds the `k - 2` +/// reachable members with the lowest pair score. A link exists if either end chose it. +/// With at most `k + 1` reachable members, the reachable members form a full mesh. +/// +/// Each node judges reachability by itself, so nodes may briefly disagree; that only +/// adds links (whoever chose a link initiates, the other answers), it never cuts one. +/// The preference budget is spent in key order, which every node shares, so nodes that +/// agree on reachability compute the same links. +pub(crate) fn graph( + members: &[String], + k: usize, + unreachable: &dyn Fn(&str) -> bool, + stable: &dyn Fn(&str) -> bool, +) -> BTreeMap<String, BTreeSet<String>> { + let k = k.max(2); + let all: BTreeSet<&str> = members.iter().map(String::as_str).collect(); + let live: BTreeSet<&str> = all.iter().copied().filter(|m| !unreachable(m)).collect(); + let mut links: BTreeMap<String, BTreeSet<String>> = live + .iter() + .map(|m| ((*m).to_owned(), BTreeSet::new())) + .collect(); + + let mut link = |a: &str, b: &str| { + if a != b { + links.get_mut(a).expect("live member").insert(b.to_owned()); + links.get_mut(b).expect("live member").insert(a.to_owned()); + } + }; + + if live.len() <= k + 1 { + for a in &live { + for b in &live { + link(a, b); + } + } + return links; + } + + // The ring over all members, so its order doesn't change when one goes offline. + let mut ring: Vec<([u8; 32], &str)> = + all.iter().map(|m| (hash(RING_LABEL, &[*m]), *m)).collect(); + ring.sort_unstable(); + let n = ring.len(); + let position: BTreeMap<&str, usize> = + ring.iter().enumerate().map(|(i, (_, m))| (*m, i)).collect(); + let first_live = |from: usize, step: usize| { + (1..n) + .map(|d| ring[(from + d * step) % n].1) + .find(|m| live.contains(m)) + .expect("more than k + 1 live members") + }; + + // Live members in key order, with their pair keys; and those of them that are stable. + let keyed: Vec<(&str, u64)> = live.iter().map(|m| (*m, pair_key(m))).collect(); + let stable_keyed: Vec<(&str, u64)> = keyed.iter().copied().filter(|(m, _)| stable(m)).collect(); + + // How many members have preferred each stable member so far (`keyed` is in key + // order, the same on every node). + let budget = STABLE_FANIN_FACTOR * k; + let mut preferred_by: BTreeMap<&str, usize> = BTreeMap::new(); + // Half the slots (rounded up) prefer stable members, so long-running nodes form a + // backbone; the rest go by pair score alone, keeping paths short. A stable member + // enough others prefer already is skipped, so no hub grows with the swarm. + let stable_slots = (k - 2).div_ceil(2); + + for &(member, key) in &keyed { + let at = position[member]; + let next = first_live(at, 1); + let prev = first_live(at, n - 1); + link(member, next); + link(member, prev); + let skip = [member, next, prev]; + // The stable candidates in score order (there are few). + let mut stable_candidates = scored(&stable_keyed, key, skip); + stable_candidates.sort_unstable(); + let preferred: Vec<&str> = stable_candidates + .iter() + .map(|(_, other)| *other) + .filter(|other| preferred_by.get(other).copied().unwrap_or(0) < budget) + .take(stable_slots) + .collect(); + for other in preferred.iter().copied() { + *preferred_by.entry(other).or_default() += 1; + } + // The rest: the lowest scores among the others that aren't preferred, which are + // among the `need + preferred` lowest overall. Selecting those is linear; only + // they are sorted. + let need = k - 2 - preferred.len(); + let mut candidates = scored(&keyed, key, skip); + let lowest = (need + preferred.len()).min(candidates.len()); + if lowest < candidates.len() { + candidates.select_nth_unstable(lowest); + candidates.truncate(lowest); + } + candidates.sort_unstable(); + let rest = candidates + .iter() + .map(|(_, other)| *other) + .filter(|other| !preferred.contains(other)) + .take(need); + for partner in preferred.iter().copied().chain(rest) { + link(member, partner); + } + } + links +} + +/// `others` but those in `skip`, scored as pairs with the member whose key is `key`. +pub(super) fn scored<'a>( + others: &[(&'a str, u64)], + key: u64, + skip: [&str; 3], +) -> Vec<(u64, &'a str)> { + others + .iter() + .filter(|(other, _)| !skip.contains(other)) + .map(|&(other, other_key)| (mix_pair(key, other_key), other)) + .collect() +} + +/// Symmetric: the same for (a, b) and (b, a). +pub(crate) fn pair_score(a: &str, b: &str) -> u64 { + mix_pair(pair_key(a), pair_key(b)) +} + +/// What a member contributes to its pair scores: computed once per member. +pub(super) fn pair_key(member: &str) -> u64 { + let digest = hash(PAIR_LABEL, &[member]); + u64::from_be_bytes(digest[..8].try_into().expect("8 bytes")) +} + +/// The score of a pair from its members' keys, symmetric: SplitMix64's finalizer over +/// the lower key and the mixed higher one. Keys are uniform (SHA-256), so for each member +/// the others' scores are in an order unrelated to anyone else's. +pub(super) fn mix_pair(a: u64, b: u64) -> u64 { + let (low, high) = if a <= b { (a, b) } else { (b, a) }; + mix(low ^ mix(high)) +} + +pub(super) fn mix(mut z: u64) -> u64 { + z = z.wrapping_add(0x9e37_79b9_7f4a_7c15); + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^ (z >> 31) +} + +pub(super) fn hash(label: &[u8], fields: &[&str]) -> [u8; 32] { + let mut hasher = Sha256::new(); + for field in [label] + .into_iter() + .chain(fields.iter().map(|f| f.as_bytes())) + { + hasher.update((field.len() as u32).to_be_bytes()); + hasher.update(field); + } + hasher.finalize().into() +} diff --git a/src/swarm/topology/mod.rs b/src/swarm/topology/mod.rs new file mode 100644 index 0000000..30fa56e --- /dev/null +++ b/src/swarm/topology/mod.rs @@ -0,0 +1,377 @@ +//! Which members a node syncs with: its neighbours. +//! +//! In a full mesh every change reaches every node from every other node (Automerge +//! decides per pair, so nobody knows that a third peer already passed it on), so the +//! copies of one write grow with N². Above `k` members, each node keeps only a few +//! neighbours, and changes travel over several hops. +//! +//! Every node computes the graph from the member list and which members it can reach, +//! with no coordination: +//! * a ring ordered by key hash: each member's nearest reachable member in both +//! directions, which keeps the reachable members connected; +//! * plus, for each member, the `k - 2` reachable others with the lowest pair score (a +//! hash of both keys, so both ends agree on it), which keeps paths short. +//! +//! A link exists if either end chose it, so links are symmetric and a node has at least +//! `k` neighbours. With at most `k + 1` reachable members the graph is the full mesh: +//! small swarms behave exactly as before. A member joining, leaving or going offline +//! changes only links near it. +//! +//! Members on several transports (Mycelium, libp2p) get such a graph per transport, +//! among the members on it (as their addresses in the member list say), and a node's +//! neighbours are the union of its neighbours on each transport it is on +//! (`neighbours_across`). Each of those graphs is connected (its ring), and a member on +//! two transports is in both: as long as members on several transports link the +//! transports, the swarm stays connected, and such a member has about `k` neighbours per +//! transport. With one transport, it is the one graph as before. + +use std::collections::{BTreeMap, BTreeSet}; + +use tracing::info; + +use crate::locks::RwLockExt; +use crate::state::{AppState, PeerState}; +use crate::swarm::members; + +mod graph; + +pub(crate) use graph::pair_score; +use graph::*; + +/// This node's current neighbours, and the member list they were computed from. +#[derive(Debug, Default)] +pub(crate) struct NeighbourView { + members: Vec<String>, + k: usize, + unreachable: BTreeSet<String>, + stable: BTreeSet<String>, + /// The transports (address schemes) of each member, and ours. + transports: BTreeMap<String, BTreeSet<String>>, + local_transports: BTreeSet<String>, + /// Peers that proved they are of our swarm but aren't in our member list (yet): their + /// registration hasn't reached us. They are neighbours too (`refresh`). + unlisted: BTreeSet<String>, + /// Members the member list gives no address for on our transports, that reach us + /// anyway: they only connect out (no listen address). They are neighbours too while + /// they say they sync with us (`refresh`). + outbound: BTreeSet<String>, + neighbours: BTreeSet<String>, + /// Every reachable member is a neighbour. + full_mesh: bool, +} + +/// Whether the neighbours are computed from the member list yet (before, everyone +/// counts as a neighbour). +pub(crate) fn view_known(state: &AppState) -> bool { + state.neighbours.read_or_recover().is_some() +} + +/// Whether every reachable member is our neighbour (also while the view is unknown). +pub(crate) fn is_full_mesh(state: &AppState) -> bool { + state + .neighbours + .read_or_recover() + .as_ref() + .is_none_or(|view| view.full_mesh) +} + +/// The current view in numbers, once known. +#[derive(Debug, Clone, Copy)] +pub(crate) struct Summary { + /// Registered members, us included. + pub(crate) members: usize, + /// Members other than us not currently judged unreachable. + pub(crate) reachable: usize, + pub(crate) stable: usize, + pub(crate) neighbours: usize, + pub(crate) stable_neighbours: usize, + pub(crate) full_mesh: bool, +} + +pub(crate) fn summary(state: &AppState) -> Option<Summary> { + let local = state.local_id(); + state + .neighbours + .read_or_recover() + .as_ref() + .map(|view| Summary { + members: view.members.len(), + reachable: view + .members + .iter() + .filter(|m| m.as_str() != local && !view.unreachable.contains(*m)) + .count(), + stable: view.stable.len(), + neighbours: view.neighbours.len(), + stable_neighbours: view + .neighbours + .iter() + .filter(|m| view.stable.contains(*m)) + .count(), + full_mesh: view.full_mesh, + }) +} + +/// Whether we initiate contact with `peer`: HELLOs, keepalives, dormancy checks, sync +/// openings. Everyone is a neighbour until the view is known (while joining, or before +/// any member registered), so joining works as in a full mesh. We always *answer* +/// anyone who contacts us. +pub(crate) fn is_neighbour(state: &AppState, peer: &str) -> bool { + match &*state.neighbours.read_or_recover() { + Some(view) => view.neighbours.contains(peer), + None => true, + } +} + +/// Every member's neighbours as the member list makes them, as this node sees it (whom +/// it can reach, who is stable): the graph each node computes its own part of. Ours as +/// they are (with members that only connect out, or aren't listed yet). For a map of +/// the swarm (the dashboard); None until the view is known. A link exists if either end +/// chose it: draw one if either lists the other. About `members²` work: compute it when +/// the member list may have changed, not on every look. +#[cfg_attr(not(feature = "dashboard"), allow(dead_code))] +pub(crate) fn intended_graph(state: &AppState) -> Option<BTreeMap<String, BTreeSet<String>>> { + let view = state.neighbours.read_or_recover(); + let view = view.as_ref()?; + let unreachable = |member: &str| view.unreachable.contains(member); + let stable = |member: &str| view.stable.contains(member); + let graph = view + .members + .iter() + .map(|member| { + let neighbours = if member == state.local_id() { + view.neighbours.clone() + } else { + let own = view.transports.get(member).cloned().unwrap_or_default(); + neighbours_across( + &view.transports, + member, + &own, + view.k, + &unreachable, + &stable, + ) + }; + (member.clone(), neighbours) + }) + .collect(); + Some(graph) +} + +/// How the neighbours changed. +pub(crate) struct NeighbourChange { + pub(crate) added: Vec<String>, + pub(crate) removed: Vec<String>, +} + +/// Runs `change` on a peer's state, and has the neighbours recomputed (`run_worker`) if +/// it changed what they are chosen from (`PeerState::neighbour_inputs`). Changes to a +/// peer that may matter to the neighbours go through here. +pub(crate) fn change_peer<T>( + state: &AppState, + peer_state: &mut PeerState, + change: impl FnOnce(&mut PeerState) -> T, +) -> T { + let before = peer_state.neighbour_inputs(); + let result = change(peer_state); + if peer_state.neighbour_inputs() != before { + state.topology_changed.notify_one(); + } + result +} + +/// Recomputes the neighbours whenever something they depend on may have changed +/// (`AppState::topology_changed`): the member list (`swarm::changes::run_change_publisher`), +/// a peer's state (`change_peer`), and every 5 s as a fallback (`swarm::peer`). A burst +/// of notifications coalesces into one recomputation. +pub(crate) async fn run_worker(state: AppState) { + loop { + state.topology_changed.notified().await; + if let Some(change) = refresh(&state).await { + crate::swarm::peer::apply_neighbour_change(&state, change).await; + } + } +} + +/// Whether a change at `path` may change the neighbours: one in the member list, or the +/// root (the whole document replaced, as after an epoch switch). +pub(crate) fn affects_neighbours(path: &[String]) -> bool { + path.is_empty() || path.starts_with(&members::members_dir()) +} + +/// Recomputes the neighbours once ready, if the member list, the reachable members or +/// `k` changed. +pub(crate) async fn refresh(state: &AppState) -> Option<NeighbourChange> { + if !state.is_ready().await { + return None; + } + let (current, stable, transports, alive) = { + let replica = state.doc.read().await; + members::apply(state, &replica.doc); + let current = members::members(&replica.doc); + // Those that said they are alive within the electorate window. + let alive: BTreeSet<String> = members::electorate( + &replica.doc, + state.clock.now_millis().0, + state.electorate_window, + ) + .into_iter() + .collect(); + let stable: BTreeSet<String> = current + .iter() + .filter(|m| members::tier(&replica.doc, m) == members::Tier::Stable) + .cloned() + .collect(); + let transports: BTreeMap<String, BTreeSet<String>> = current + .iter() + .map(|m| (m.clone(), schemes(&members::addresses(&replica.doc, m)))) + .collect(); + (current, stable, transports, alive) + }; + let local_transports: BTreeSet<String> = state.transport.transports().into_iter().collect(); + let k = state.neighbours_k; + // Those that don't answer, those on none of our transports, and those that haven't + // said they are alive for longer than the electorate window, unless we heard from + // them since we started (one that comes back says so to us first: its HELLO). + let mut unreachable = BTreeSet::new(); + for member in ¤t { + if member == state.local_id() { + continue; + } + if !state.transport.can_reach(member) { + unreachable.insert(member.clone()); + continue; + } + let (gone, heard) = match state.get_peer_state(member).await { + Some(peer_state) => { + let peer_state = peer_state.lock().await; + (peer_state.is_unreachable(), peer_state.last_seen.is_some()) + } + None => (false, false), + }; + if gone || (!heard && !alive.contains(member)) { + unreachable.insert(member.clone()); + } + } + // A node that joined while the genesis node hadn't registered yet has the genesis + // document without the genesis node in it, and registers itself there: then neither + // lists the other, and with each the other's neighbour only by the member list, + // neither would ever sync with the other, and their registrations never pass. So a + // peer that proved itself a member of our swarm counts until the list says otherwise: + // syncing brings each the other's registration. + let mut unlisted = BTreeSet::new(); + for (peer, peer_state) in state.peers_snapshot().await { + if current.contains(&peer) || peer == state.local_id() { + continue; + } + let peer_state = peer_state.lock().await; + if matches!(peer_state.standing, crate::state::Standing::Member { .. }) + && !peer_state.is_unreachable() + { + unlisted.insert(peer); + } + } + // A member that lists no address on our transports, but reached us anyway, only + // connects out: it isn't in the graph on our side (it's on none of our transports, as + // far as the list goes), while on its side we are, so it would sync with us while we + // never sent it our changes (only answers to its). It picks its neighbours; those it + // picked keep it, as it says (STATUS). + let mut outbound = BTreeSet::new(); + for member in ¤t { + if member == state.local_id() + || unreachable.contains(member) + || transports + .get(member) + .is_some_and(|on| !on.is_disjoint(&local_transports)) + { + continue; + } + if let Some(peer_state) = state.get_peer_state(member).await { + let peer_state = peer_state.lock().await; + if matches!(peer_state.standing, crate::state::Standing::Member { .. }) + && peer_state.syncs_with_us != Some(false) + { + outbound.insert(member.clone()); + } + } + } + let previous = { + let view = state.neighbours.read_or_recover(); + if view.as_ref().is_some_and(|view| { + view.members == current + && view.unlisted == unlisted + && view.outbound == outbound + && view.k == k + && view.unreachable == unreachable + && view.stable == stable + && view.transports == transports + && view.local_transports == local_transports + }) { + return None; + } + view.as_ref().map(|view| view.neighbours.clone()) + }; + + // Before the first view everyone known counted as a neighbour. + let previous = match previous { + Some(previous) => previous, + None => state.peer_states.read().await.keys().cloned().collect(), + }; + let mut computed = neighbours_across( + &transports, + state.local_id(), + &local_transports, + k, + &|m| unreachable.contains(m), + &|m| stable.contains(m), + ); + computed.extend(outbound.iter().cloned()); + let listed_neighbours = computed.len(); + computed.extend(unlisted.iter().cloned()); + let change = NeighbourChange { + added: computed + .iter() + .filter(|m| !previous.contains(*m)) + .cloned() + .collect(), + removed: previous + .iter() + .filter(|m| !computed.contains(*m)) + .cloned() + .collect(), + }; + info!( + members = current.len(), + unreachable = unreachable.len(), + stable = stable.len(), + k, + neighbours = computed.len(), + unlisted = unlisted.len(), + outbound = outbound.len(), + added = change.added.len(), + removed = change.removed.len(), + "Neighbours computed from the member list" + ); + let reachable_others = current + .iter() + .filter(|m| m.as_str() != state.local_id() && !unreachable.contains(*m)) + .count(); + *state.neighbours.write_or_recover() = Some(NeighbourView { + full_mesh: listed_neighbours >= reachable_others, + members: current, + k, + unreachable, + stable, + transports, + local_transports, + unlisted, + outbound, + neighbours: computed, + }); + // New neighbours are kept alive and judged. + state.peers_due.now(); + Some(change) +} + +#[cfg(test)] +mod tests; diff --git a/src/swarm/topology/tests.rs b/src/swarm/topology/tests.rs new file mode 100644 index 0000000..d3a3e38 --- /dev/null +++ b/src/swarm/topology/tests.rs @@ -0,0 +1,313 @@ +use super::*; +use std::collections::VecDeque; + +fn members(n: usize) -> Vec<String> { + (0..n).map(|i| format!("{:064x}", i * 7919 + 13)).collect() +} + +fn all_reachable(_: &str) -> bool { + false +} + +fn none_stable(_: &str) -> bool { + false +} + +fn graph(members: &[String], k: usize) -> BTreeMap<String, BTreeSet<String>> { + super::graph(members, k, &all_reachable, &none_stable) +} + +fn connected(g: &BTreeMap<String, BTreeSet<String>>) -> bool { + let Some(start) = g.keys().next() else { + return true; + }; + let mut seen = BTreeSet::from([start.clone()]); + let mut queue = VecDeque::from([start.clone()]); + while let Some(m) = queue.pop_front() { + for next in &g[&m] { + if seen.insert(next.clone()) { + queue.push_back(next.clone()); + } + } + } + seen.len() == g.len() +} + +fn edges(graph: &BTreeMap<String, BTreeSet<String>>) -> BTreeSet<(String, String)> { + graph + .iter() + .flat_map(|(a, ns)| { + ns.iter().map(move |b| { + if a < b { + (a.clone(), b.clone()) + } else { + (b.clone(), a.clone()) + } + }) + }) + .collect() +} + +fn hex(bytes: [u8; 32]) -> String { + crate::transport::Hex(&bytes).to_string() +} + +/// Over random swarms (size, `k`, who is reachable, who is stable): links only +/// between reachable members, never to oneself, symmetric; a full mesh while small, +/// otherwise at least `k` links each and connected; independent of the order members +/// are listed in; and each member's own view (`neighbours`) agrees with the graph. +#[test] +fn random_swarms() { + crate::testrand::for_seeds(300, |rng| { + let n = rng.between(1, 60); + let k = rng.between(2, 14); + let mut all: Vec<String> = (0..n).map(|_| hex(rng.bytes32())).collect(); + let unreachable: BTreeSet<String> = + all.iter().filter(|_| rng.chance(0.3)).cloned().collect(); + let stable: BTreeSet<String> = all.iter().filter(|_| rng.chance(0.3)).cloned().collect(); + let is_unreachable = |m: &str| unreachable.contains(m); + let is_stable = |m: &str| stable.contains(m); + + let g = super::graph(&all, k, &is_unreachable, &is_stable); + // Owned: `all` is shuffled below. + let live: BTreeSet<String> = all + .iter() + .filter(|m| !unreachable.contains(*m)) + .cloned() + .collect(); + assert_eq!(g.keys().cloned().collect::<BTreeSet<_>>(), live); + for (a, ns) in &g { + assert!(!ns.contains(a), "{a} links to itself"); + for b in ns { + assert!(live.contains(b), "{a} links to unreachable {b}"); + assert!(g[b].contains(a), "{a} -> {b} but not back"); + } + } + if live.len() <= k + 1 { + assert!( + g.values().all(|ns| ns.len() + 1 == live.len()), + "not a full mesh" + ); + } else { + assert!( + g.values().all(|ns| ns.len() >= k), + "a member with fewer than k links" + ); + assert!(connected(&g), "not connected"); + } + + rng.shuffle(&mut all); + assert_eq!(super::graph(&all, k, &is_unreachable, &is_stable), g); + + if let Some(local) = live.iter().nth(rng.below(live.len())) { + let others: Vec<String> = all.iter().filter(|m| *m != local).cloned().collect(); + assert_eq!( + neighbours(&others, local, k, &is_unreachable, &is_stable), + g[local] + ); + } + }); +} + +#[test] +fn small_swarms_are_a_full_mesh() { + let ms = members(13); + for m in &ms { + assert_eq!( + neighbours(&ms, m, 12, &all_reachable, &none_stable).len(), + 12 + ); + } +} + +#[test] +fn links_are_symmetric_and_degree_at_least_k() { + let ms = members(250); + let g = graph(&ms, 8); + for (a, ns) in &g { + assert!(ns.len() >= 8, "{a} has {} neighbours", ns.len()); + assert!(!ns.contains(a)); + for b in ns { + assert!(g[b].contains(a), "{a} -> {b} not symmetric"); + } + } +} + +#[test] +fn graph_is_connected() { + for (n, k) in [(250, 8), (250, 2), (60, 3)] { + assert!(connected(&graph(&members(n), k)), "n={n} k={k}"); + } +} + +#[test] +fn one_member_joining_changes_few_links() { + let ms = members(200); + let before = edges(&graph(&ms, 8)); + let mut more = ms.clone(); + more.push(format!("{:064x}", 999_999)); + let after = edges(&graph(&more, 8)); + let changed = before.symmetric_difference(&after).count(); + assert!(changed <= 4 * 8, "{changed} links changed"); +} + +#[test] +fn reachable_members_stay_connected_when_most_are_offline() { + let ms = members(100); + // Every tenth member is online. + let online: BTreeSet<String> = ms.iter().step_by(10).cloned().collect(); + let offline = |m: &str| !online.contains(m); + let g = super::graph(&ms, 12, &offline, &none_stable); + assert_eq!(g.len(), 10); + // Ten reachable members and k = 12: a full mesh among them. + assert!(g.values().all(|ns| ns.len() == 9)); + + let online: BTreeSet<String> = ms.iter().step_by(4).cloned().collect(); + let offline = |m: &str| !online.contains(m); + let g = super::graph(&ms, 8, &offline, &none_stable); + assert_eq!(g.len(), 25); + assert!(connected(&g)); + assert!( + g.values() + .all(|ns| ns.len() >= 8 && ns.iter().all(|n| online.contains(n))) + ); +} + +#[test] +fn stable_members_become_hubs_with_bounded_degree() { + let ms = members(250); + let stable: BTreeSet<String> = ms.iter().step_by(12).take(20).cloned().collect(); + let is_stable = |m: &str| stable.contains(m); + let g = super::graph(&ms, 12, &all_reachable, &is_stable); + assert!(connected(&g)); + + let degree = |pred: &dyn Fn(&String) -> bool| -> (usize, f64) { + let degrees: Vec<usize> = g + .iter() + .filter(|(m, _)| pred(m)) + .map(|(_, ns)| ns.len()) + .collect(); + let max = *degrees.iter().max().unwrap(); + ( + max, + degrees.iter().sum::<usize>() as f64 / degrees.len() as f64, + ) + }; + let (stable_max, stable_mean) = degree(&|m| stable.contains(m)); + let (_, transient_mean) = degree(&|m| !stable.contains(m)); + // 230 transient members put about 5 of their 10 scored slots on 20 stable ones. + assert!( + stable_mean > 3.0 * transient_mean, + "{stable_mean} vs {transient_mean}" + ); + assert!( + stable_max <= 100, + "a stable member has {stable_max} neighbours" + ); +} + +/// A hub's degree in a swarm of `n` with `stable_count` stable members stays within +/// the bound. +fn hub_degree_is_bounded(n: usize, stable_count: usize) { + // Few stable members among many: without the fan-in budget each would be chosen + // by most of the swarm (about 130 neighbours with 10 of 250). With it, a hub has + // at most the budget plus the links others choose by pair score and the ring. + let k = 12; + let ms = members(n); + let stable: BTreeSet<String> = ms.iter().step_by(n / stable_count).cloned().collect(); + let is_stable = |m: &str| stable.contains(m); + let g = super::graph(&ms, k, &all_reachable, &is_stable); + assert!(connected(&g), "n={n} stable={stable_count}"); + let hub_max = g + .iter() + .filter(|(m, _)| stable.contains(*m)) + .map(|(_, ns)| ns.len()) + .max() + .unwrap(); + assert!( + hub_max <= 8 * k, + "n={n} stable={stable_count}: a stable member has {hub_max} neighbours" + ); +} + +/// Two transports with members on one or the other, and two members on both: every +/// link is over a shared transport, and the two members on both connect the rest. +#[test] +fn members_on_two_transports_bridge_the_others() { + let k = 8; + let ms = members(201); + let on = |names: &[&str]| -> BTreeSet<String> { + names.iter().map(|name| (*name).to_owned()).collect() + }; + let transports: BTreeMap<String, BTreeSet<String>> = ms + .iter() + .enumerate() + .map(|(i, m)| { + let transports = match i { + 0 | 1 => on(&["a", "b"]), + i if i % 2 == 0 => on(&["a"]), + _ => on(&["b"]), + }; + (m.clone(), transports) + }) + .collect(); + let no_stable = |_: &str| false; + let mut graph: BTreeMap<String, BTreeSet<String>> = BTreeMap::new(); + for (m, own) in &transports { + for n in neighbours_across(&transports, m, own, k, &all_reachable, &no_stable) { + assert!( + !transports[&n].is_disjoint(own), + "{m} and {n} share no transport" + ); + graph.entry(m.clone()).or_default().insert(n.clone()); + graph.entry(n).or_default().insert(m.clone()); + } + } + assert_eq!(graph.len(), ms.len()); + assert!(connected(&graph)); + let bridge = &ms[0]; + assert!(graph[bridge].iter().any(|n| !transports[n].contains("b"))); + assert!(graph[bridge].iter().any(|n| !transports[n].contains("a"))); + + // One transport: the one graph, as without transports. + let one: BTreeMap<String, BTreeSet<String>> = + ms.iter().map(|m| (m.clone(), on(&[""]))).collect(); + for m in &ms[..5] { + assert_eq!( + neighbours_across(&one, m, &on(&[""]), k, &all_reachable, &no_stable), + neighbours(&ms, m, k, &all_reachable, &no_stable) + ); + } +} + +#[test] +fn schemes_of_addresses() { + let addresses = |all: &[&str]| -> Vec<String> { all.iter().map(|a| (*a).to_owned()).collect() }; + assert_eq!( + schemes(&addresses(&[ + "mycelium:ab", + "libp2p:/ip4/1.2.3.4/tcp/1", + "libp2p:/ip6/::1/tcp/1" + ])), + BTreeSet::from(["libp2p".to_owned(), "mycelium".to_owned()]) + ); + assert_eq!(schemes(&addresses(&["a"])), BTreeSet::from([String::new()])); + assert!(schemes(&[]).is_empty()); +} + +#[test] +fn hub_degree_is_bounded_however_few_stable_members() { + for (n, stable_count) in [(250, 10), (400, 5)] { + hub_degree_is_bounded(n, stable_count); + } +} + +/// The same at the sizes the bound was designed for; slow in debug builds, so run +/// on request (`cargo test --release -- --ignored`). +#[test] +#[ignore] +fn hub_degree_is_bounded_in_large_swarms() { + for (n, stable_count) in [(1000, 10), (1000, 20), (2000, 20)] { + hub_degree_is_bounded(n, stable_count); + } +} diff --git a/src/transport/identity.rs b/src/transport/identity.rs index c597d77..1df84ec 100644 --- a/src/transport/identity.rs +++ b/src/transport/identity.rs @@ -44,7 +44,7 @@ use crate::transport::paths::{Gate, PathHandler, Paths, scheme_of}; use crate::transport::stats::{IdentityCounts, IdentityStats, TransportStats}; use crate::transport::traffic::Traffic; use crate::transport::{ - Destination, EncodedMessage, HandlerFn, IdentityProof, MessageHandler, MessageKind, + Connection, Destination, EncodedMessage, HandlerFn, IdentityProof, MessageHandler, MessageKind, ParsedMessage, Transport, }; @@ -58,8 +58,9 @@ const ASK_INTERVAL: Duration = Duration::from_secs(5); const OTHER_PATHS_ASK_INTERVAL: Duration = Duration::from_secs(60); /// How long resolving a seed waits for its ME. const RESOLVE_TIMEOUT: Duration = Duration::from_secs(10); -/// Addresses kept per node from hints: about one per transport. -const MAX_HINTS: usize = 8; +/// Addresses kept per node from hints: as many as a member lists +/// (`members::MAX_ADDRESSES`). +const MAX_HINTS: usize = 16; /// Addresses with a proven node, nodes with hints, and addresses asked, each at most: /// anyone can send us proofs (keys are free), so the books are bounded. const MAX_ENTRIES: usize = 4096; @@ -187,6 +188,8 @@ impl Book { } IdentityStats { proven, + // Asked of the transports outside this lock (`IdentityTransport::stats`). + relayed: std::collections::BTreeMap::new(), hinted: self .hints .keys() @@ -242,12 +245,15 @@ impl Book { impl IdentityTransport { /// Over `transport`, whose addresses have `config.scheme`, and `config.more`. - /// `firewall`: the debug firewall, if any, which paths applies by address. + /// `firewall`: the debug firewall, if any, which paths applies by address; `clock`: + /// the node's wall clock; `keepalive`: the node's keepalive interval (`Paths::new`). pub(crate) fn new( transport: Arc<dyn Transport>, config: IdentityConfig, traffic: Arc<Traffic>, firewall: Option<Arc<Firewall>>, + clock: Arc<crate::clock::Clock>, + keepalive: Duration, ) -> Self { let node_id = config.key.id(); let shared = Arc::new_cyclic(|shared: &std::sync::Weak<Shared>| { @@ -259,6 +265,8 @@ impl IdentityTransport { .collect(), Some(Arc::clone(&traffic)), firewall.is_some().then_some(gate), + clock, + keepalive, )), firewall, key: config.key, @@ -294,6 +302,22 @@ impl Shared { } impl Shared { + /// Of the nodes in `proven`, those each transport reaches only through a relay, by + /// transport (`IdentityStats::relayed`); asked of the transports, so not under the + /// book's lock. + fn relayed( + &self, + proven: &HashMap<String, Vec<String>>, + ) -> std::collections::BTreeMap<String, usize> { + let mut relayed = std::collections::BTreeMap::new(); + for address in proven.values().flatten() { + if self.paths.connection(address) == Some(Connection::Relayed) { + *relayed.entry(scheme_of(address).to_owned()).or_insert(0) += 1; + } + } + relayed + } + fn lock(&self) -> MutexGuard<'_, Book> { self.book.lock_or_recover() } @@ -435,24 +459,25 @@ impl Shared { } /// Where to send to `peer`: an address it proved itself at, else one it is said to - /// be at; the most preferred transport's first. - fn destination(&self, peer: &str) -> Option<String> { - self.proven_at(peer) + /// be at; the most preferred transport's first. `sending` as `Paths::rank` has it: + /// for a message about to go, or only a look. + fn destination(&self, peer: &str, sending: bool) -> Option<String> { + self.proven_at(peer, sending) .into_iter() .next() - .or_else(|| self.hinted_at(peer).into_iter().next()) + .or_else(|| self.hinted_at(peer, sending).into_iter().next()) } /// The addresses `peer` proved itself at, best first (`Paths::rank`). - fn proven_at(&self, peer: &str) -> Vec<String> { + fn proven_at(&self, peer: &str, sending: bool) -> Vec<String> { let proven = self.lock().proven.get(peer).cloned().unwrap_or_default(); - self.paths.rank(&proven) + self.paths.rank(&proven, sending) } /// Where `peer` is said to be, best first. - fn hinted_at(&self, peer: &str) -> Vec<String> { + fn hinted_at(&self, peer: &str, sending: bool) -> Vec<String> { let hints = self.lock().hints.get(peer).cloned().unwrap_or_default(); - self.paths.rank(&hints) + self.paths.rank(&hints, sending) } } @@ -470,6 +495,10 @@ impl Transport for IdentityTransport { self.shared.paths.delivery_limit() } + fn sync_reply_timeout(&self) -> Duration { + self.shared.paths.sync_reply_timeout() + } + fn local_addresses(&self) -> Vec<String> { self.shared.paths.local_addresses() } @@ -533,7 +562,7 @@ impl Transport for IdentityTransport { } fn can_reach(&self, peer: &str) -> bool { - self.shared.destination(peer).is_some() + self.shared.destination(peer, false).is_some() } fn transports(&self) -> Vec<String> { @@ -548,16 +577,24 @@ impl Transport for IdentityTransport { /// The address `encode` picks, the transport's connection there, and every path /// to `peer` as paths measures it, best first. fn destination(&self, peer: &str) -> Option<Destination> { - let proven = self.shared.proven_at(peer); + let proven = self.shared.proven_at(peer, false); let (address, is_proven) = match proven.first() { Some(address) => (address.clone(), true), - None => (self.shared.hinted_at(peer).into_iter().next()?, false), + None => ( + self.shared.hinted_at(peer, false).into_iter().next()?, + false, + ), }; let connection = self.shared.paths.connection(&address); + let punching = match connection { + Some(Connection::Relayed) => self.shared.paths.punching(&address), + _ => None, + }; Some(Destination { address: self.shared.paths.location(&address), proven: is_proven, connection, + punching, paths: self.shared.paths.info(&proven), }) } @@ -569,15 +606,29 @@ impl Transport for IdentityTransport { /// The best path's, where messages to `peer` go. fn rtt(&self, peer: &str) -> Option<Duration> { - let address = self.shared.proven_at(peer).into_iter().next()?; + let address = self.shared.proven_at(peer, false).into_iter().next()?; self.shared.paths.rtt(&address) } + /// The narrowest of its proven paths'. + fn clock_offset(&self, peer: &str) -> Option<crate::transport::paths::ClockOffset> { + self.shared + .proven_at(peer, false) + .iter() + .filter_map(|address| self.shared.paths.clock_offset(address)) + .min_by_key(|offset| offset.within_ms) + } + /// The identity layer's, and each transport's. fn stats(&self) -> TransportStats { let mut stats = self.shared.paths.stats(); + let (mut identity, proven) = { + let books = self.shared.lock(); + (books.stats(), books.proven.clone()) + }; + identity.relayed = self.shared.relayed(&proven); stats.merge(TransportStats { - identity: Some(self.shared.lock().stats()), + identity: Some(identity), ..TransportStats::default() }); stats @@ -587,12 +638,20 @@ impl Transport for IdentityTransport { self.shared.paths.discovered_seeds() } + fn on_seeds_found(&self, found: std::sync::Arc<tokio::sync::Notify>) { + self.shared.paths.on_seeds_found(found) + } + + fn on_reachable(&self, reachable: std::sync::Arc<tokio::sync::Notify>) { + self.shared.paths.on_reachable(reachable) + } + /// A node ID resolves to itself if this node knows where it is; anything else is /// an address (`<scheme>:...`) or what one of the transports resolves to one (the /// first that does), asked WHO. async fn resolve_seed(&self, seed: &str) -> Result<String> { if identity::is_node_id(seed) { - if self.shared.destination(seed).is_some() { + if self.shared.destination(seed, false).is_some() { return Ok(seed.to_owned()); } bail!("a node ID alone doesn't say where that node is; give its address"); @@ -628,9 +687,10 @@ impl Transport for IdentityTransport { } /// For the address `destination` picks (a send there is refused until `peer` - /// proved itself there, `send_encoded`). + /// proved itself there, `send_encoded`). Only a look at the choice: the send makes + /// it (encoding again for another path if it changes). fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> { - let Some(address) = self.shared.destination(peer) else { + let Some(address) = self.shared.destination(peer, false) else { bail!("no address known for {peer} on this node's transports"); }; let mut encoded = self.shared.paths.encode(&address, message)?; @@ -641,7 +701,7 @@ impl Transport for IdentityTransport { /// To the addresses `peer` proved itself at, best first, the next when a send /// fails (`Paths::send_first`); refused if it proved itself nowhere yet. async fn send_encoded(&self, message: &EncodedMessage) -> Result<()> { - let proven = self.shared.proven_at(&message.peer); + let proven = self.shared.proven_at(&message.peer, true); if !proven.is_empty() { self.shared.ask_other_paths(&message.peer); } diff --git a/src/transport/libp2p.rs b/src/transport/libp2p.rs deleted file mode 100644 index 7351b16..0000000 --- a/src/transport/libp2p.rs +++ /dev/null @@ -1,2311 +0,0 @@ -//! libp2p as a transport (`--transports libp2p`, in builds with the `libp2p` feature): -//! direct connections over TCP (Noise, Yamux) and QUIC, to addresses this node knows. -//! -//! A node's transport identity is its libp2p peer ID; where it is, its listen addresses -//! (multiaddrs ending in the peer ID, `locations`). Seeds are such multiaddrs, or ones -//! without the peer ID, which a connection finds out. Messages go one per stream of the -//! `/continuum/1` protocol: their length (4 bytes, big endian), then the message as -//! `wire` encodes it. A send is done once the stream is written and closed; a connection -//! that can't be made, or a stream that can't be written, refuses it. A stream's protocol -//! goes with its first data, without waiting for the other side to agree -//! (`swarm_config`): opening a stream costs no round trip. -//! -//! Connected peers tell each other where they listen (libp2p's identify protocol), so a -//! node learns the addresses of peers that dialled it. A node publishes where it can be -//! reached (`local_addresses`): the addresses it announces (`--libp2p-announce`, e.g. a -//! public address forwarded to it), or else those it listens on, loopback left out. -//! -//! A node nobody can dial (behind NAT, or a closed port) can still be reached through a -//! relay: a node of the swarm that relays (`--libp2p-relay`), or one configured -//! (`--libp2p-relays`), on which it reserves a slot; its address through the relay -//! (`<relay>/p2p-circuit/p2p/<it>`) is published with the others. Over a relayed connection, both sides then try to -//! connect directly (libp2p's DCUtR, hole punching); the relay carries the traffic until -//! that works, or for good if it doesn't, within its limits (`RELAY_CIRCUIT_DURATION`, -//! `RELAY_CIRCUIT_BYTES`; a circuit that ends is made again on the next send). -//! -//! Of several connections to one peer (both dialled at once, or AutoNAT dialled back), -//! all but one are closed a few seconds later (`DUPLICATE_GRACE`, `Book::duplicates`), -//! by the side with the smaller peer ID. -//! -//! A relay says so where it publishes its addresses: next to each, the same address -//! ending in `/p2p-circuit` (where to reserve a slot), which other nodes find in the -//! member list. A node that needs a relay picks up to `AUTO_RELAYS` relays -//! (`relay_check`): those configured first, then those listed in the document -//! (`learn_relays`), then members that relay. It needs one when -//! it has no address others could dial, or when AutoNAT says it can't be dialled: -//! AutoNAT asks connected peers (nodes of the swarm) to dial it back. Once AutoNAT says -//! it can be dialled, it lets the relays it picked go. -//! -//! With `--libp2p-mdns`, a node finds libp2p peers on its LAN (multicast DNS). Their -//! LAN addresses are noted, and those that turn out to speak continuum (identify says -//! so) are seeds while the node joins (`discovered_seeds`): nodes on one LAN find each -//! other without configured seeds. -//! -//! It uses public libp2p infrastructure only with `--libp2p-public`: then it joins the -//! IPFS DHT (Kademlia, through the IPFS bootstrap nodes), where it finds addresses of -//! peers the member list doesn't have, and public peers that relay; those are picked -//! after members that relay. Public relays keep circuits short (by default 2 minutes and -//! 128 KiB): enough to punch holes through NATs, not to carry a swarm's traffic. Without -//! it, relays are those configured, or members that say they relay. - -use std::collections::HashMap; -use std::path::Path; -use std::str::FromStr; -use std::sync::{Arc, Mutex, MutexGuard}; -use std::time::Duration; - -use ::libp2p::multiaddr::Protocol; -use ::libp2p::swarm::dial_opts::{DialOpts, PeerCondition}; -use ::libp2p::swarm::{ConnectionId, SwarmEvent}; -use ::libp2p::{Multiaddr, PeerId, StreamProtocol, Swarm, identity::Keypair}; -use anyhow::{Context, Result, anyhow, bail}; -use async_trait::async_trait; -use futures::{AsyncReadExt, AsyncWriteExt, StreamExt}; -use tokio::sync::{Semaphore, mpsc, oneshot, watch}; -use tracing::{debug, info, trace, warn}; - -use crate::locks::MutexExt; -use crate::transport::stats::{ - Connections, DialFailures, HolePunches, Libp2pStats, Reachability, RelaySlots, TransportStats, -}; -use crate::transport::{ - Addressed, Connection, EncodedMessage, HandlerFn, MessageHandler, ParsedMessage, Transport, - hand_on_probe, is_probe, wire, -}; - -/// The scheme of this transport's addresses (`transport::paths`). -pub(crate) const SCHEME: &str = "libp2p"; - -const PROTOCOL: StreamProtocol = StreamProtocol::new("/continuum/1"); -/// Sends taking this long are logged with where the time went (at debug). -const SLOW_SEND: Duration = Duration::from_millis(5); -/// The largest message read: anyone who can connect can send anything, so this bounds -/// what one stream costs. Sync messages above 16 chunks travel as blobs. -const MAX_MESSAGE: usize = 128 << 20; -/// How long a connection may take to come up. -const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); -/// A connection nobody uses is closed after this; neighbours keep theirs busy. -const IDLE_TIMEOUT: Duration = Duration::from_secs(300); -/// How long a stream may take to open and to carry a message, before what its size adds -/// (`stream_timeout`). -const STREAM_TIMEOUT: Duration = Duration::from_secs(10); -/// The slowest a message is expected to go, in bytes per second. -const MIN_RATE: u64 = 1 << 20; -/// Inbound streams read at once. -const READS: usize = 32; -/// Addresses kept per peer, and peers with addresses at most. -const ADDRESSES_PER_PEER: usize = 8; -const MAX_PEERS: usize = 4096; - -/// A relayed connection (circuit) through this node lasts at most this long and carries -/// at most this much: generous, since the swarm's own relays may carry sync traffic for -/// good where hole punching fails (libp2p's defaults, 2 minutes and 128 KiB, are for -/// the punching only). -const RELAY_CIRCUIT_DURATION: Duration = Duration::from_secs(3600); -const RELAY_CIRCUIT_BYTES: u64 = 1 << 30; -/// Circuits through this node at once, and per peer. -const RELAY_CIRCUITS: usize = 64; -const RELAY_CIRCUITS_PER_PEER: usize = 8; -/// A configured relay that couldn't be reached to find out its peer ID is tried again -/// after this. -const RELAY_RETRY: Duration = Duration::from_secs(30); -/// Relays picked among the members, at most, and how often a node checks whether it -/// needs them. -const AUTO_RELAYS: usize = 2; -const RELAY_CHECK: Duration = Duration::from_secs(30); -/// A relay that lost the slot before granting it (it doesn't answer, or another peer -/// answers at its address: a relay restarted without its libp2p key) isn't picked again -/// for this long. -const RELAY_PAUSE: Duration = Duration::from_secs(600); -/// A peer nobody mentioned for this long (the member list, identify, mDNS, a -/// connection) is forgotten, unless connected: mostly old peer IDs of nodes restarted -/// without their libp2p key, whose new ones the member list has since. Members are -/// mentioned every few seconds (`swarm::topology`). Checked every `PRUNE_EVERY`. -const STALE_AFTER: Duration = Duration::from_secs(600); -const PRUNE_EVERY: Duration = Duration::from_secs(60); -/// Several connections to one peer (both dialled at once, or AutoNAT dialled back to -/// see whether it can) are cut to one (`Book::duplicates`) this long after the last one -/// opened: long enough for what it was dialled for. Looked for every `DEDUPE_EVERY`. -const DUPLICATE_GRACE: Duration = Duration::from_secs(10); -const DEDUPE_EVERY: Duration = Duration::from_secs(5); - -/// How long a message of `bytes` may take on a stream. -fn stream_timeout(bytes: usize) -> Duration { - STREAM_TIMEOUT + Duration::from_millis(bytes as u64 * 1000 / MIN_RATE) -} - -/// What a libp2p transport is made with. -pub(crate) struct Libp2pConfig { - pub(crate) key: Keypair, - /// Multiaddrs to listen on. - pub(crate) listen: Vec<Multiaddr>, - /// Multiaddrs to publish instead of the listen addresses, if any. - pub(crate) announce: Vec<Multiaddr>, - /// Relays to reserve a slot on (multiaddrs; without the relay's peer ID, a connection - /// finds it out). - pub(crate) relays: Vec<Multiaddr>, - /// Relay for other nodes. - pub(crate) relay: bool, - /// Speak continuum (`PROTOCOL`): not on a node that only relays (`--relay-only`), - /// which nobody should take for a node. - pub(crate) continuum: bool, - /// Find peers on the LAN (mDNS). - pub(crate) mdns: bool, - /// Use public infrastructure: the IPFS DHT, public relays. - pub(crate) public: bool, - /// libp2p's in-memory transport instead of TCP and QUIC: for tests. - pub(crate) memory: bool, -} - -impl Libp2pConfig { - /// The key in `path` (32 raw bytes, an Ed25519 secret key), or a new one each start - /// without a file: the peer ID only says where a node is, so a new one costs nothing - /// but the WHO that finds it again (`transport::identity`). - pub(crate) fn load_key(path: Option<&Path>) -> Result<Keypair> { - let Some(path) = path else { - return Ok(Keypair::generate_ed25519()); - }; - let mut bytes = std::fs::read(path) - .with_context(|| format!("Reading the libp2p key from {}", path.display()))?; - crate::swarm::auth::warn_if_readable_by_others(path, "libp2p key"); - if bytes.len() != 32 { - bail!( - "The libp2p key in {} has {} bytes; it has 32", - path.display(), - bytes.len() - ); - } - Keypair::ed25519_from_bytes(&mut bytes) - .map_err(|error| anyhow!("The libp2p key in {}: {error}", path.display())) - } -} - -enum Command { - /// Connect to `peer` (at `addresses`) unless connected. - Connect { - peer: PeerId, - addresses: Vec<Multiaddr>, - done: oneshot::Sender<Result<(), String>>, - }, - /// Connect to `address` to find out which peer is there. - Identify { - address: Multiaddr, - done: oneshot::Sender<Result<PeerId, String>>, - }, - /// See whether relays are needed now (one was just learned of). - CheckRelays, - /// Find out which peer relays at `address`, named without the relay's peer ID - /// (`Book::finding`). - FindRelay { address: Multiaddr }, -} - -/// Who named a relay to pick, in the order they are picked. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Named { - /// `Libp2pConfig::relays`. - Configured, - /// The relays listed in the document (`Transport::learn_relays`). - Listed, -} - -impl Named { - fn as_str(self) -> &'static str { - match self { - Self::Configured => "configured", - Self::Listed => "listed", - } - } -} - -/// A relay named to pick. -#[derive(Clone, Debug, PartialEq, Eq)] -struct NamedRelay { - /// Where, with the relay's peer ID. - at: Multiaddr, - /// As named: without the peer ID, it is found out again when the relay doesn't - /// grant a slot (it may have a new one). - given: Multiaddr, - named: Named, -} - -/// Where peers are, and where we listen. -#[derive(Default)] -struct Book { - addresses: HashMap<PeerId, Vec<Multiaddr>>, - listening: Vec<Multiaddr>, - /// `Libp2pConfig::announce`. - announced: Vec<Multiaddr>, - /// `Libp2pConfig::relay`: this node relays, and says so. - serving: bool, - /// Relays named to pick (configured, listed), by peer: picked first. - named_relays: HashMap<PeerId, NamedRelay>, - /// Relays named without their peer ID, being found out (`Command::FindRelay`). - finding: HashMap<Multiaddr, Named>, - /// The relays listed in the document, as last learned (`learn_listed`). - listed: Vec<Multiaddr>, - /// Members that say they relay, and where (with the relay's peer ID). - relays: HashMap<PeerId, Multiaddr>, - /// With `Libp2pConfig::public`: other peers that relay (identify says so), and - /// where. - public_relays: HashMap<PeerId, Multiaddr>, - /// Relays picked that never granted a slot, and when (`RELAY_PAUSE`). - failed_relays: HashMap<PeerId, tokio::time::Instant>, - /// Relays that granted us a slot we hold. - slots: std::collections::HashSet<PeerId>, - /// Open connections: to whom, whether through a relay, and the remote address. - connections: HashMap<ConnectionId, (PeerId, bool, Multiaddr)>, - /// What AutoNAT says, and the address it confirmed while public. - reachability: Reachability, - public_address: Option<Multiaddr>, - /// Since start (`stats`). - counts: Counts, - /// When each peer was last mentioned (`STALE_AFTER`). - seen: HashMap<PeerId, tokio::time::Instant>, - /// Peers found on the LAN (mDNS) not known yet to speak continuum. - lan: std::collections::HashSet<PeerId>, - /// Peers that speak continuum (identify says so). - continuum: std::collections::HashSet<PeerId>, - /// Peers found on the LAN that speak continuum, with their address: seeds not - /// handed out yet (`discovered_seeds`). - discovered: Vec<Multiaddr>, -} - -/// What `Libp2pStats` counts since start. -#[derive(Default)] -struct Counts { - slots_granted: u64, - slots_lost: u64, - slots_not_granted: u64, - hole_punches: HolePunches, - dial_failures: DialFailures, -} - -/// Counts a dial that failed with `error` (not one that wasn't needed). -/// A dial error, shortly: for each address tried (without the peer ID the caller names -/// already), the innermost cause, once. libp2p's own text nests each transport's errors -/// in the next ("Multiple dial errors occurred: ..."), repeating the cause at each level. -fn describe_dial_error(error: &::libp2p::swarm::DialError) -> String { - use ::libp2p::swarm::DialError; - let DialError::Transport(errors) = error else { - return error.to_string(); - }; - let described: Vec<String> = errors - .iter() - .map(|(address, error)| { - let mut address = address.clone(); - if let Some(Protocol::P2p(_)) = address.iter().last() { - address.pop(); - } - format!("{address}: {}", innermost_causes(error)) - }) - .collect(); - if described.is_empty() { - return error.to_string(); - } - described.join("; ") -} - -/// The innermost of `error`'s causes, each distinct one once: those libp2p lists in a -/// "Multiple dial errors occurred:" text (one per line, ` - <cause>: <its cause>...`), -/// or the last of the source chain. -fn innermost_causes(error: &(dyn std::error::Error + 'static)) -> String { - let mut innermost = error; - while let Some(source) = innermost.source() { - innermost = source; - } - let text = innermost.to_string(); - let mut causes: Vec<&str> = Vec::new(); - for line in text.lines() { - let Some(listed) = line.trim().strip_prefix("- ") else { - continue; - }; - // A line ends in the next level's heading when that level lists more. - let cause = listed - .split(": ") - .filter(|part| !part.starts_with("Multiple dial errors")) - .last() - .unwrap_or(listed) - .trim(); - if !cause.is_empty() && !causes.contains(&cause) { - causes.push(cause); - } - } - if causes.is_empty() { - let cause = text.rsplit(": ").next().unwrap_or(&text).trim(); - return cause.to_owned(); - } - causes.join(", ") -} - -fn count_dial_failure(failures: &mut DialFailures, error: &::libp2p::swarm::DialError) { - use ::libp2p::swarm::DialError; - match error { - DialError::DialPeerConditionFalse(_) => {} - DialError::Transport(_) => failures.unreachable += 1, - DialError::WrongPeerId { .. } => failures.wrong_peer_id += 1, - DialError::NoAddresses => failures.no_addresses += 1, - _ => failures.other += 1, - } -} - -impl Book { - /// How we are connected to `peer`: directly if any connection is direct. - fn connection(&self, peer: &PeerId) -> Option<Connection> { - let mut found = None; - for (connected, relayed, _) in self.connections.values() { - if connected == peer { - if !relayed { - return Some(Connection::Direct); - } - found = Some(Connection::Relayed); - } - } - found - } - - /// The connections to `peer` to close, of several: all but one, the oldest direct - /// one (the oldest relayed one if none is direct). - fn duplicates(&self, peer: &PeerId) -> Vec<ConnectionId> { - let mut connections: Vec<(bool, ConnectionId)> = self - .connections - .iter() - .filter(|(_, (connected, _, _))| connected == peer) - .map(|(id, (_, relayed, _))| (*relayed, *id)) - .collect(); - // Direct (false) first, then the oldest. - connections.sort(); - connections.into_iter().skip(1).map(|(_, id)| id).collect() - } - - /// The remote address of a connection to `peer`: a direct one if there is one. - fn location(&self, peer: &PeerId) -> Option<&Multiaddr> { - let mut found = None; - for (connected, relayed, address) in self.connections.values() { - if connected == peer { - if !relayed { - return Some(address); - } - found = Some(address); - } - } - found - } - - fn stats(&self) -> Libp2pStats { - let mut connections = Connections::default(); - for (_, relayed, _) in self.connections.values() { - if *relayed { - connections.relayed += 1; - } else { - connections.direct += 1; - } - } - Libp2pStats { - reachability: self.reachability, - public_address: self.public_address.as_ref().map(Multiaddr::to_string), - connections, - relay_slots: RelaySlots { - held: self.slots.len(), - paused: self - .failed_relays - .values() - .filter(|at| at.elapsed() < RELAY_PAUSE) - .count(), - granted: self.counts.slots_granted, - lost: self.counts.slots_lost, - not_granted: self.counts.slots_not_granted, - }, - hole_punches: self.counts.hole_punches, - dial_failures: self.counts.dial_failures, - } - } - - /// `peer` was mentioned just now. - fn touch(&mut self, peer: PeerId) { - self.seen.insert(peer, tokio::time::Instant::now()); - } - - /// Forgets the peers not mentioned within `STALE_AFTER` before `now`, but those - /// `connected`. Returns how many. - fn prune(&mut self, now: tokio::time::Instant, connected: impl Fn(&PeerId) -> bool) -> usize { - let stale: Vec<PeerId> = self - .addresses - .keys() - .chain(self.relays.keys()) - .chain(self.public_relays.keys()) - .chain(self.continuum.iter()) - .chain(self.lan.iter()) - .filter(|peer| { - !connected(peer) - && self - .seen - .get(peer) - .is_none_or(|at| now.saturating_duration_since(*at) >= STALE_AFTER) - }) - .copied() - .collect::<std::collections::HashSet<_>>() - .into_iter() - .collect(); - for peer in &stale { - self.addresses.remove(peer); - self.relays.remove(peer); - self.public_relays.remove(peer); - self.continuum.remove(peer); - self.lan.remove(peer); - self.failed_relays.remove(peer); - self.seen.remove(peer); - } - stale.len() - } - - /// `peer`, found on the LAN, speaks continuum: a seed, at the address we know first. - fn discover(&mut self, peer: PeerId) { - let Some(address) = self.addresses.get(&peer).and_then(|known| known.first()) else { - return; - }; - if let Ok(address) = address.clone().with_p2p(peer) - && !self.discovered.contains(&address) - && self.discovered.len() < MAX_PEERS - { - self.discovered.push(address); - } - } - - /// Where others could dial us directly: the announced addresses, or else those we - /// listen on (`published`), without the peer ID. - fn direct(&self) -> Vec<Multiaddr> { - if !self.announced.is_empty() { - return self.announced.clone(); - } - let direct: Vec<Multiaddr> = self - .listening - .iter() - .filter(|address| !is_relayed(address)) - .cloned() - .collect(); - published(&direct) - } -} - -/// Whether `address` only reaches this host (or this link): no use to other hosts, and -/// dialled by one, it reaches that host itself. -fn is_local(address: &Multiaddr) -> bool { - match address.iter().next() { - Some(Protocol::Ip4(ip)) => ip.is_loopback() || ip.is_link_local() || ip.is_unspecified(), - Some(Protocol::Ip6(ip)) => { - ip.is_loopback() || ip.is_unicast_link_local() || ip.is_unspecified() - } - _ => false, - } -} - -/// The addresses of `all` worth publishing: all but local ones, unless they're all -/// local (a test on one host). -fn published(all: &[Multiaddr]) -> Vec<Multiaddr> { - let reachable: Vec<Multiaddr> = all - .iter() - .filter(|address| !is_local(address)) - .cloned() - .collect(); - if reachable.is_empty() { - all.to_vec() - } else { - reachable - } -} - -impl Book { - /// `named` names the relay at `given`; returns it if its peer is to be found out. - /// `local`: this node, never its own relay. - fn name_relay(&mut self, given: Multiaddr, named: Named, local: PeerId) -> Option<Multiaddr> { - match split_peer(&given) { - Some(peer) if peer == local => None, - Some(peer) => { - // Configured wins over listed. - if self - .named_relays - .get(&peer) - .is_none_or(|known| known.named == Named::Listed || named == Named::Configured) - { - self.named_relays.insert( - peer, - NamedRelay { - at: given.clone(), - given, - named, - }, - ); - } - None - } - None => { - let known = self.finding.contains_key(&given) - || self.named_relays.values().any(|relay| relay.given == given); - if known { - return None; - } - self.finding.insert(given.clone(), named); - Some(given) - } - } - } - - /// The relay named at `given` (without a peer ID) is `peer`: unless it is no longer - /// named (or is this node). Returns whether that's news. - fn relay_found(&mut self, given: Multiaddr, peer: PeerId, local: PeerId) -> bool { - let Some(named) = self.finding.remove(&given) else { - return false; - }; - let Ok(at) = given.clone().with_p2p(peer) else { - return false; - }; - if peer == local - || (named == Named::Listed - && self - .named_relays - .get(&peer) - .is_some_and(|known| known.named == Named::Configured)) - { - return false; - } - self.named_relays - .insert(peer, NamedRelay { at, given, named }); - true - } - - /// `relay` didn't grant a slot: if it was named without its peer ID, its address is - /// to be found out again, returned. - fn find_again(&mut self, relay: PeerId) -> Option<Multiaddr> { - if self - .named_relays - .get(&relay) - .is_none_or(|known| split_peer(&known.given).is_some()) - { - return None; - } - let known = self.named_relays.remove(&relay)?; - self.finding.insert(known.given.clone(), known.named); - Some(known.given) - } - - /// The relays listed in the document are `listed` now: returns those to find out, - /// None if nothing changed. - fn learn_listed(&mut self, listed: Vec<Multiaddr>, local: PeerId) -> Option<Vec<Multiaddr>> { - if listed == self.listed { - return None; - } - self.named_relays - .retain(|_, relay| relay.named != Named::Listed || listed.contains(&relay.given)); - self.finding - .retain(|given, named| *named != Named::Listed || listed.contains(given)); - self.listed = listed.clone(); - Some( - listed - .into_iter() - .filter_map(|given| self.name_relay(given, Named::Listed, local)) - .collect(), - ) - } - - /// The named relays `named` names, by peer, where. - fn named(&self, named: Named) -> HashMap<PeerId, Multiaddr> { - self.named_relays - .iter() - .filter(|(_, relay)| relay.named == named) - .map(|(peer, relay)| (*peer, relay.at.clone())) - .collect() - } - - /// Whether all of `addresses` are known of `peer` (`remember`). - fn knows(&self, peer: PeerId, addresses: &[Multiaddr]) -> bool { - let Some(known) = self.addresses.get(&peer) else { - return false; - }; - addresses.iter().all(|address| { - let mut address = address.clone(); - if let Some(Protocol::P2p(_)) = address.iter().last() { - address.pop(); - } - known.contains(&address) - }) - } - - /// `peer` is at `address` (without its `/p2p` part): kept first. - fn remember(&mut self, peer: PeerId, mut address: Multiaddr) { - if let Some(Protocol::P2p(_)) = address.iter().last() { - address.pop(); - } - if !self.addresses.contains_key(&peer) && self.addresses.len() >= MAX_PEERS { - return; - } - self.touch(peer); - let addresses = self.addresses.entry(peer).or_default(); - addresses.retain(|known| *known != address); - addresses.insert(0, address); - addresses.truncate(ADDRESSES_PER_PEER); - } -} - -/// Whether `address` goes through a relay. -fn is_relayed(address: &Multiaddr) -> bool { - address - .iter() - .any(|protocol| matches!(protocol, Protocol::P2pCircuit)) -} - -/// The relay `address` goes through, if it goes through one: the peer before -/// `/p2p-circuit`. -fn relay_of(address: &Multiaddr) -> Option<PeerId> { - let mut relay = None; - for protocol in address.iter() { - match protocol { - Protocol::P2p(peer) => relay = Some(peer), - Protocol::P2pCircuit => return relay, - _ => {} - } - } - None -} - -/// The addresses of `known` this node can dial: not those through itself as the relay -/// (a relay reaches the nodes it relays for only while they are connected to it). -fn dialable(known: &[Multiaddr], local: PeerId) -> Vec<Multiaddr> { - known - .iter() - .filter(|address| relay_of(address) != Some(local)) - .cloned() - .collect() -} - -/// `address` with the peer ID it ends in, if it ends in one. -fn split_peer(address: &Multiaddr) -> Option<PeerId> { - match address.iter().last() { - Some(Protocol::P2p(peer)) => Some(peer), - _ => None, - } -} - -pub(crate) struct Libp2pTransport { - local_id: String, - peer_id: PeerId, - control: libp2p_stream::Control, - commands: mpsc::UnboundedSender<Command>, - book: Arc<Mutex<Book>>, - incoming: tokio::sync::Mutex<Option<libp2p_stream::IncomingStreams>>, - stopped: watch::Receiver<bool>, -} - -fn lock(book: &Mutex<Book>) -> MutexGuard<'_, Book> { - book.lock_or_recover() -} - -impl Libp2pTransport { - /// Builds the swarm, listens, and runs it in a task of its own. - pub(crate) fn start(config: Libp2pConfig) -> Result<Self> { - let peer_id = config.key.public().to_peer_id(); - let features = Features::of(&config); - let mut swarm = if config.memory { - memory_swarm(config.key, features)? - } else { - network_swarm(config.key, features)? - }; - for address in config.listen { - swarm - .listen_on(address.clone()) - .with_context(|| format!("Listening on {address}"))?; - } - for address in &config.announce { - // Identify tells peers these. - swarm.add_external_address(address.clone()); - } - let mut control = swarm.behaviour().stream.new_control(); - let incoming = if config.continuum { - Some( - control - .accept(PROTOCOL) - .map_err(|error| anyhow!("Accepting {PROTOCOL}: {error}"))?, - ) - } else { - None - }; - let (commands, receiver) = mpsc::unbounded_channel(); - let mut book = Book { - announced: config.announce, - serving: config.relay, - ..Book::default() - }; - for relay in config.relays { - if let Some(address) = book.name_relay(relay, Named::Configured, peer_id) { - let _ = commands.send(Command::FindRelay { address }); - } - } - let book = Arc::new(Mutex::new(book)); - let (stopping, stopped) = watch::channel(false); - tokio::spawn(run( - swarm, - receiver, - commands.clone(), - Arc::clone(&book), - Relaying { - serving: config.relay, - public: config.public, - local: peer_id, - }, - stopping, - )); - info!(peer_id = %peer_id, "Transport started"); - Ok(Self { - local_id: peer_id.to_string(), - peer_id, - control, - commands, - book, - incoming: tokio::sync::Mutex::new(incoming), - stopped, - }) - } - - /// Connected to `peer`, if it can be had in time. - async fn connect(&self, peer: PeerId) -> Result<()> { - let addresses = lock(&self.book) - .addresses - .get(&peer) - .map(|known| dialable(known, self.peer_id)) - .unwrap_or_default(); - let (done, connected) = oneshot::channel(); - self.commands - .send(Command::Connect { - peer, - addresses, - done, - }) - .map_err(|_| anyhow!("the libp2p swarm stopped"))?; - match tokio::time::timeout(CONNECT_TIMEOUT, connected).await { - Ok(Ok(Ok(()))) => Ok(()), - Ok(Ok(Err(error))) => bail!("couldn't connect to {peer}: {error}"), - Ok(Err(_)) => bail!("the libp2p swarm stopped"), - Err(_) => bail!("connecting to {peer} took longer than {CONNECT_TIMEOUT:?}"), - } - } -} - -/// The protocol version identify says (informational: nothing checks it). -const IDENTIFY_VERSION: &str = "/continuum/1"; - -#[derive(::libp2p::swarm::NetworkBehaviour)] -struct Behaviour { - stream: libp2p_stream::Behaviour, - identify: ::libp2p::identify::Behaviour, - relay_client: ::libp2p::relay::client::Behaviour, - /// Only with `Libp2pConfig::relay`. - relay: ::libp2p::swarm::behaviour::toggle::Toggle<::libp2p::relay::Behaviour>, - dcutr: ::libp2p::dcutr::Behaviour, - autonat: ::libp2p::autonat::Behaviour, - /// Only with `Libp2pConfig::mdns`. - mdns: ::libp2p::swarm::behaviour::toggle::Toggle<::libp2p::mdns::tokio::Behaviour>, - /// Only with `Libp2pConfig::public`: the IPFS DHT. - kad: ::libp2p::swarm::behaviour::toggle::Toggle< - ::libp2p::kad::Behaviour<::libp2p::kad::store::MemoryStore>, - >, -} - -/// What the behaviour has besides what every node has. -#[derive(Clone, Copy)] -struct Features { - relay: bool, - mdns: bool, - public: bool, -} - -impl Features { - fn of(config: &Libp2pConfig) -> Self { - Self { - relay: config.relay, - mdns: config.mdns, - public: config.public, - } - } -} - -/// The IPFS DHT's protocol, and its bootstrap nodes (all at `IPFS_BOOTSTRAP_ADDRESS`). -const IPFS_DHT: StreamProtocol = StreamProtocol::new("/ipfs/kad/1.0.0"); -const IPFS_BOOTSTRAP_ADDRESS: &str = "/dnsaddr/bootstrap.libp2p.io"; -const IPFS_BOOTSTRAP_PEERS: [&str; 4] = [ - "QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", - "QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", - "QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb", - "QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt", -]; -/// How often a node on the DHT refreshes its routing table. -const DHT_BOOTSTRAP: Duration = Duration::from_secs(300); - -impl Behaviour { - fn new( - key: &Keypair, - relay_client: ::libp2p::relay::client::Behaviour, - features: Features, - ) -> std::io::Result<Self> { - let Features { - relay, - mdns, - public, - } = features; - let peer_id = key.public().to_peer_id(); - let kad = public.then(|| { - let mut dht = ::libp2p::kad::Behaviour::with_config( - peer_id, - ::libp2p::kad::store::MemoryStore::new(peer_id), - ::libp2p::kad::Config::new(IPFS_DHT), - ); - let bootstrap: Multiaddr = IPFS_BOOTSTRAP_ADDRESS.parse().expect("a multiaddr"); - for peer in IPFS_BOOTSTRAP_PEERS { - let peer: PeerId = peer.parse().expect("a peer ID"); - let _ = dht.add_address(&peer, bootstrap.clone()); - } - dht - }); - let identify = ::libp2p::identify::Config::new(IDENTIFY_VERSION.to_owned(), key.public()) - .with_agent_version(format!("continuum/{}", env!("CARGO_PKG_VERSION"))); - let relay = relay.then(|| { - ::libp2p::relay::Behaviour::new( - peer_id, - ::libp2p::relay::Config { - max_circuits: RELAY_CIRCUITS, - max_circuits_per_peer: RELAY_CIRCUITS_PER_PEER, - max_circuit_duration: RELAY_CIRCUIT_DURATION, - max_circuit_bytes: RELAY_CIRCUIT_BYTES, - ..Default::default() - }, - ) - }); - let mdns = match mdns { - true => Some(::libp2p::mdns::tokio::Behaviour::new( - ::libp2p::mdns::Config::default(), - peer_id, - )?), - false => None, - }; - Ok(Self { - stream: libp2p_stream::Behaviour::new(), - identify: ::libp2p::identify::Behaviour::new(identify), - relay_client, - relay: relay.into(), - dcutr: ::libp2p::dcutr::Behaviour::new(peer_id), - autonat: ::libp2p::autonat::Behaviour::new( - peer_id, - ::libp2p::autonat::Config { - // Swarms on private networks (a LAN, WireGuard) count too. - only_global_ips: false, - ..Default::default() - }, - ), - mdns: mdns.into(), - kad: kad.into(), - }) - } -} - -fn network_swarm(key: Keypair, features: Features) -> Result<Swarm<Behaviour>> { - Ok(::libp2p::SwarmBuilder::with_existing_identity(key) - .with_tokio() - .with_tcp( - ::libp2p::tcp::Config::default(), - ::libp2p::noise::Config::new, - ::libp2p::yamux::Config::default, - )? - .with_quic() - .with_dns()? - .with_relay_client( - ::libp2p::noise::Config::new, - ::libp2p::yamux::Config::default, - )? - .with_behaviour(|key, relay_client| { - Behaviour::new(key, relay_client, features) - .map_err(|error| Box::new(error) as Box<dyn std::error::Error + Send + Sync>) - })? - .with_swarm_config(|config| swarm_config(config, features)) - .build()) -} - -/// The swarm's settings. Streams are opened without waiting for the other side to agree -/// on the protocol (`V1Lazy`): the protocol and the first data go together, saving a -/// round trip per message (one stream each, module docs). That is safe only towards -/// peers sure to speak the protocol, so not with public libp2p infrastructure -/// (`Libp2pConfig::public`), whose nodes speak only some: there a protocol they don't -/// would fail late, and unclearly. -fn swarm_config(config: ::libp2p::swarm::Config, features: Features) -> ::libp2p::swarm::Config { - let config = config.with_idle_connection_timeout(IDLE_TIMEOUT); - if features.public { - config - } else { - config.with_substream_upgrade_protocol_override(::libp2p::core::upgrade::Version::V1Lazy) - } -} - -fn memory_swarm(key: Keypair, features: Features) -> Result<Swarm<Behaviour>> { - use ::libp2p::Transport as _; - use ::libp2p::core::transport::{MemoryTransport, OrTransport}; - use ::libp2p::core::upgrade::Version; - - let peer_id = key.public().to_peer_id(); - let (relay_transport, relay_client) = ::libp2p::relay::client::new(peer_id); - let transport = OrTransport::new(relay_transport, MemoryTransport::default()) - .boxed() - .upgrade(Version::V1) - .authenticate(::libp2p::noise::Config::new(&key)?) - .multiplex(::libp2p::yamux::Config::default()) - .timeout(CONNECT_TIMEOUT) - .boxed(); - Ok(Swarm::new( - transport, - Behaviour::new(&key, relay_client, features)?, - peer_id, - swarm_config(::libp2p::swarm::Config::with_tokio_executor(), features), - )) -} - -/// How a node relates to relays. -struct Relaying { - /// It relays for others. - serving: bool, - /// Public relays may be picked, after members. - public: bool, - local: PeerId, -} - -/// The relays to pick among `offers`, if this node needs relays, keeping `using`: a -/// node needs them when it has no address others could dial (`direct`), or AutoNAT says -/// it can't be dialled; up to `AUTO_RELAYS`, in an order of its own (the lowest hash of -/// both peer IDs), so that not every node picks the same. -fn relay_check( - local: PeerId, - direct: bool, - nat: &::libp2p::autonat::NatStatus, - offers: &HashMap<PeerId, Multiaddr>, - using: &[PeerId], -) -> Vec<(PeerId, Multiaddr)> { - use ::libp2p::autonat::NatStatus; - let needed = match nat { - NatStatus::Private => true, - NatStatus::Public(_) => false, - NatStatus::Unknown => !direct, - }; - if !needed || using.len() >= AUTO_RELAYS { - return Vec::new(); - } - let score = |relay: &PeerId| { - use std::hash::{Hash, Hasher}; - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - (local, relay).hash(&mut hasher); - hasher.finish() - }; - let mut candidates: Vec<(PeerId, Multiaddr)> = offers - .iter() - .filter(|(relay, _)| **relay != local && !using.contains(relay)) - .map(|(relay, address)| (*relay, address.clone())) - .collect(); - candidates.sort_by_key(|(relay, _)| score(relay)); - candidates.truncate(AUTO_RELAYS - using.len()); - candidates -} - -/// Reserves slots on the relays `relay_check` picks (configured ones first, then listed -/// ones, members, public ones), and lets those picked go once AutoNAT says this node can -/// be dialled. -fn pick_relays( - swarm: &mut Swarm<Behaviour>, - book: &Mutex<Book>, - relaying: &Relaying, - nat: &::libp2p::autonat::NatStatus, - picked: &mut HashMap<::libp2p::core::transport::ListenerId, PeerId>, -) { - if nat.is_public() && !picked.is_empty() { - info!("Reachable directly (AutoNAT); letting the relays picked go"); - let mut book = lock(book); - for (listener, relay) in std::mem::take(picked) { - swarm.remove_listener(listener); - book.slots.remove(&relay); - } - return; - } - let (direct, configured, listed, members, public) = { - let mut book = lock(book); - book.failed_relays - .retain(|_, at| at.elapsed() < RELAY_PAUSE); - let failed = book.failed_relays.clone(); - let usable = |offers: &HashMap<PeerId, Multiaddr>| -> HashMap<PeerId, Multiaddr> { - offers - .iter() - .filter(|(relay, _)| !failed.contains_key(relay)) - .map(|(relay, address)| (*relay, address.clone())) - .collect() - }; - let public = if relaying.public { - usable(&book.public_relays) - } else { - HashMap::new() - }; - ( - !book.direct().is_empty(), - usable(&book.named(Named::Configured)), - usable(&book.named(Named::Listed)), - usable(&book.relays), - public, - ) - }; - let mut using: Vec<PeerId> = picked.values().copied().collect(); - // Each kind for what the ones before left. - for (kind, offers) in [ - (Named::Configured.as_str(), configured), - (Named::Listed.as_str(), listed), - ("member", members), - ("public", public), - ] { - for (relay, address) in relay_check(relaying.local, direct, nat, &offers, &using) { - using.push(relay); - match swarm.listen_on(address.clone().with(Protocol::P2pCircuit)) { - Ok(listener) => { - info!(relay = %Addressed(SCHEME, &relay), address = %address, kind, "Picked a relay"); - picked.insert(listener, relay); - } - Err(error) => { - debug!(relay = %Addressed(SCHEME, &relay), kind, error = %error, "Couldn't use a relay"); - } - } - } - } -} - -/// Asks the DHT (with `--libp2p-public`) where `peer` is, for the next attempt. -fn find_on_dht(swarm: &mut Swarm<Behaviour>, peer: PeerId) { - if let Some(dht) = swarm.behaviour_mut().kad.as_mut() { - dht.get_closest_peers(peer); - } -} - -/// Drives the swarm: dials what `commands` ask for and notes where we listen, until the -/// transport is dropped. `again`: to schedule commands. -async fn run( - mut swarm: Swarm<Behaviour>, - mut commands: mpsc::UnboundedReceiver<Command>, - again: mpsc::UnboundedSender<Command>, - book: Arc<Mutex<Book>>, - relaying: Relaying, - stopping: watch::Sender<bool>, -) { - let serving = relaying.serving; - let mut nat = ::libp2p::autonat::NatStatus::Unknown; - // Relays picked, by listener. - let mut picked: HashMap<::libp2p::core::transport::ListenerId, PeerId> = HashMap::new(); - let mut check = tokio::time::interval(RELAY_CHECK); - check.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - let mut dht_bootstrap = tokio::time::interval(DHT_BOOTSTRAP); - dht_bootstrap.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - let mut connecting: HashMap<PeerId, Vec<oneshot::Sender<Result<(), String>>>> = HashMap::new(); - let mut identifying: HashMap< - ConnectionId, - (Multiaddr, oneshot::Sender<Result<PeerId, String>>), - > = HashMap::new(); - let mut prune = tokio::time::interval(PRUNE_EVERY); - prune.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - // Peers that may have several connections, and when to cut them to one. Only the - // side with the smaller peer ID does that, so the two never close different ones. - let mut dedupe_due: HashMap<PeerId, tokio::time::Instant> = HashMap::new(); - // Connections closed as duplicates, until they are: their "Connection closed" says so. - let mut closing_duplicates: std::collections::HashSet<ConnectionId> = - std::collections::HashSet::new(); - let mut dedupe = tokio::time::interval(DEDUPE_EVERY); - dedupe.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - loop { - tokio::select! { - _ = dedupe.tick(), if !dedupe_due.is_empty() => { - let now = tokio::time::Instant::now(); - let due: Vec<PeerId> = dedupe_due - .iter() - .filter(|(_, at)| **at <= now) - .map(|(peer, _)| *peer) - .collect(); - for peer in due { - dedupe_due.remove(&peer); - let duplicates = lock(&book).duplicates(&peer); - for connection in duplicates { - if swarm.close_connection(connection) { - closing_duplicates.insert(connection); - } - } - } - } - _ = prune.tick() => { - let forgotten = lock(&book).prune(tokio::time::Instant::now(), |peer| swarm.is_connected(peer)); - if forgotten > 0 { - debug!(forgotten, "Forgot peers nobody mentions any more"); - } - } - _ = dht_bootstrap.tick(), if relaying.public => { - if let Some(dht) = swarm.behaviour_mut().kad.as_mut() - && let Err(error) = dht.bootstrap() - { - warn!(error = %error, "Couldn't join the IPFS DHT"); - } - } - _ = check.tick() => { - pick_relays(&mut swarm, &book, &relaying, &nat, &mut picked); - } - command = commands.recv() => match command { - None => break, - Some(Command::CheckRelays) => { - pick_relays(&mut swarm, &book, &relaying, &nat, &mut picked); - } - Some(Command::Connect { peer, addresses, done }) => { - if swarm.is_connected(&peer) { - let _ = done.send(Ok(())); - continue; - } - let waiting = connecting.entry(peer).or_default(); - waiting.push(done); - if waiting.len() > 1 { - continue; // Already dialing. - } - let mut opts = DialOpts::peer_id(peer) - .addresses(addresses) - .condition(PeerCondition::DisconnectedAndNotDialing); - if relaying.public { - // Addresses the DHT knows too. - opts = opts.extend_addresses_through_behaviour(); - } - if let Err(error) = swarm.dial(opts.build()) { - count_dial_failure(&mut lock(&book).counts.dial_failures, &error); - for done in connecting.remove(&peer).into_iter().flatten() { - let _ = done.send(Err(error.to_string())); - } - find_on_dht(&mut swarm, peer); - } - } - Some(Command::FindRelay { address }) => { - let again = again.clone(); - let local = relaying.local; - let book = Arc::clone(&book); - tokio::spawn(async move { - let (done, identified) = oneshot::channel(); - if again - .send(Command::Identify { address: address.clone(), done }) - .is_err() - { - return; - } - match tokio::time::timeout(CONNECT_TIMEOUT, identified).await { - Ok(Ok(Ok(peer))) => { - if lock(&book).relay_found(address.clone(), peer, local) { - debug!(relay = %Addressed(SCHEME, &address), peer = %Addressed(SCHEME, &peer), "Found a named relay"); - let _ = again.send(Command::CheckRelays); - } - } - _ => { - if !lock(&book).finding.contains_key(&address) { - return; // No longer named. - } - warn!(relay = %Addressed(SCHEME, &address), "Couldn't reach a named relay; trying again later"); - tokio::time::sleep(RELAY_RETRY).await; - let _ = again.send(Command::FindRelay { address }); - } - } - }); - } - Some(Command::Identify { address, done }) => { - let opts = DialOpts::unknown_peer_id().address(address.clone()).build(); - let id = opts.connection_id(); - match swarm.dial(opts) { - Ok(()) => { - identifying.insert(id, (address, done)); - } - Err(error) => { - let _ = done.send(Err(error.to_string())); - } - } - } - }, - event = swarm.select_next_some() => match event { - SwarmEvent::ConnectionEstablished { peer_id, connection_id, endpoint, .. } => { - { - let mut book = lock(&book); - book.touch(peer_id); - let relayed = endpoint.is_relayed(); - book.connections.insert( - connection_id, - (peer_id, relayed, endpoint.get_remote_address().clone()), - ); - if relaying.local < peer_id - && book.connections.values().filter(|(connected, _, _)| *connected == peer_id).count() > 1 - { - dedupe_due.insert(peer_id, tokio::time::Instant::now() + DUPLICATE_GRACE); - } - debug!( - peer = %Addressed(SCHEME, &peer_id), - connection = %connection_id, - dialled = endpoint.is_dialer(), - address = %endpoint.get_remote_address(), - relayed, - "Connection opened" - ); - } - for done in connecting.remove(&peer_id).into_iter().flatten() { - let _ = done.send(Ok(())); - } - if let Some((address, done)) = identifying.remove(&connection_id) { - lock(&book).remember(peer_id, address); - let _ = done.send(Ok(peer_id)); - } - } - SwarmEvent::ConnectionClosed { peer_id, connection_id, cause, .. } => { - lock(&book).connections.remove(&connection_id); - let cause = match cause { - _ if closing_duplicates.remove(&connection_id) => "a duplicate".to_owned(), - // By us, or idle: libp2p doesn't tell which. - None => "closed here".to_owned(), - Some(error) => error.to_string(), - }; - debug!( - peer = %Addressed(SCHEME, &peer_id), - connection = %connection_id, - cause = ?cause, - "Connection closed" - ); - } - SwarmEvent::OutgoingConnectionError { connection_id, peer_id, error } => { - count_dial_failure(&mut lock(&book).counts.dial_failures, &error); - if let Some((_, done)) = identifying.remove(&connection_id) { - let _ = done.send(Err(describe_dial_error(&error))); - } - if let Some(peer) = peer_id - && !swarm.is_connected(&peer) - { - if let Some(waiting) = connecting.remove(&peer) { - for done in waiting { - let _ = done.send(Err(describe_dial_error(&error))); - } - find_on_dht(&mut swarm, peer); - } - } - } - SwarmEvent::NewListenAddr { address, .. } => { - info!(address = %address, "Listening"); - // A relay tells those reserving where it is: its external addresses. - if serving && !is_local(&address) && !is_relayed(&address) { - swarm.add_external_address(address.clone()); - } - let mut book = lock(&book); - if !book.listening.contains(&address) { - book.listening.push(address); - } - } - SwarmEvent::ExpiredListenAddr { address, .. } => { - info!(address = %address, "No longer listening"); - if serving { - swarm.remove_external_address(&address); - } - lock(&book).listening.retain(|known| *known != address); - } - SwarmEvent::ListenerClosed { listener_id, addresses, reason } => { - lock(&book) - .listening - .retain(|known| !addresses.contains(known)); - // A relay picked: the next check picks again, another one if this one - // never granted the slot. - if let Some(relay) = picked.remove(&listener_id) { - let lost = { - let mut book = lock(&book); - let lost = book.slots.remove(&relay); - if lost { - book.counts.slots_lost += 1; - } else { - book.counts.slots_not_granted += 1; - book.failed_relays.insert(relay, tokio::time::Instant::now()); - // Named without its peer ID: maybe another one now. - if let Some(address) = book.find_again(relay) { - let _ = again.send(Command::FindRelay { address }); - } - } - lost - }; - if lost { - debug!(relay = %Addressed(SCHEME, &relay), reason = ?reason, "Lost the slot on a relay"); - } else { - info!( - relay = %Addressed(SCHEME, &relay), - pause = ?RELAY_PAUSE, - "A relay didn't grant a slot (unreachable, or another peer at its address); picking others for a while" - ); - } - } - } - SwarmEvent::ListenerError { error, .. } => { - warn!(error = %error, "Listener failed"); - } - SwarmEvent::Behaviour(BehaviourEvent::RelayClient( - ::libp2p::relay::client::Event::ReservationReqAccepted { relay_peer_id, renewal, .. }, - )) => { - { - let mut book = lock(&book); - book.slots.insert(relay_peer_id); - book.failed_relays.remove(&relay_peer_id); - if !renewal { - book.counts.slots_granted += 1; - } - } - if !renewal { - info!(relay = %Addressed(SCHEME, &relay_peer_id), "Reachable through a relay"); - } - } - SwarmEvent::Behaviour(BehaviourEvent::Autonat( - ::libp2p::autonat::Event::StatusChanged { old, new }, - )) => { - info!(old = ?old, new = ?new, "Reachability (AutoNAT)"); - { - use ::libp2p::autonat::NatStatus; - let mut book = lock(&book); - (book.reachability, book.public_address) = match &new { - NatStatus::Unknown => (Reachability::Unknown, None), - NatStatus::Private => (Reachability::Private, None), - NatStatus::Public(address) => { - (Reachability::Public, Some(address.clone())) - } - }; - } - nat = new; - } - SwarmEvent::Behaviour(BehaviourEvent::Relay(event)) => { - debug!(event = ?event, "Relay event"); - } - SwarmEvent::Behaviour(BehaviourEvent::Dcutr(event)) => match event.result { - Ok(_) => { - lock(&book).counts.hole_punches.succeeded += 1; - info!(peer = %Addressed(SCHEME, &event.remote_peer_id), "Connected directly to a peer behind a relay"); - } - Err(error) => { - lock(&book).counts.hole_punches.failed += 1; - debug!(peer = %Addressed(SCHEME, &event.remote_peer_id), error = %error, "Couldn't connect directly to a peer behind a relay; the relay carries on"); - } - }, - SwarmEvent::Behaviour(BehaviourEvent::Identify( - ::libp2p::identify::Event::Received { connection_id, peer_id, info }, - )) => { - let mut book = lock(&book); - let listen = published(&info.listen_addrs); - // Every new connection identifies again: logged when it says something new. - if book.knows(peer_id, &listen) { - trace!(peer = %Addressed(SCHEME, &peer_id), connection = %connection_id, "Peer identified again"); - } else { - debug!( - peer = %Addressed(SCHEME, &peer_id), - connection = %connection_id, - listen = ?info.listen_addrs, - observed = %info.observed_addr, - agent = %info.agent_version, - "Peer identified" - ); - } - // Last first: `remember` puts each in front. - for address in listen.into_iter().rev() { - book.remember(peer_id, address); - } - if relaying.public - && info.protocols.contains(&::libp2p::relay::HOP_PROTOCOL_NAME) - && (book.public_relays.len() < MAX_PEERS - || book.public_relays.contains_key(&peer_id)) - && let Some(address) = book - .addresses - .get(&peer_id) - .and_then(|known| known.first()) - .and_then(|address| address.clone().with_p2p(peer_id).ok()) - { - book.public_relays.insert(peer_id, address); - } - if info.protocols.contains(&PROTOCOL) { - book.continuum.insert(peer_id); - if book.lan.remove(&peer_id) { - book.discover(peer_id); - } - } - } - SwarmEvent::Behaviour(BehaviourEvent::Mdns( - ::libp2p::mdns::Event::Discovered(found), - )) => { - for (peer, address) in found { - if peer == relaying.local { - continue; - } - debug!(peer = %Addressed(SCHEME, &peer), address = %address, "Peer on the LAN"); - let continuum = { - let mut book = lock(&book); - book.remember(peer, address.clone()); - book.continuum.contains(&peer) - }; - if continuum { - lock(&book).discover(peer); - } else if lock(&book).lan.len() < MAX_PEERS { - // Connected, identify says whether it speaks continuum. - lock(&book).lan.insert(peer); - let _ = swarm.dial( - DialOpts::peer_id(peer) - .addresses(vec![address]) - .condition(PeerCondition::DisconnectedAndNotDialing) - .build(), - ); - } - } - } - _ => {} - }, - } - } - let _ = stopping.send(true); -} - -/// One message from an inbound stream. -async fn read_message(mut stream: ::libp2p::Stream) -> Result<Vec<u8>> { - let mut length = [0u8; 4]; - tokio::time::timeout(STREAM_TIMEOUT, stream.read_exact(&mut length)) - .await - .map_err(|_| anyhow!("no length within {STREAM_TIMEOUT:?}"))??; - let length = u32::from_be_bytes(length) as usize; - if length > MAX_MESSAGE { - bail!("a message of {length} bytes; at most {MAX_MESSAGE} are read"); - } - let mut body = vec![0u8; length]; - tokio::time::timeout(stream_timeout(length), stream.read_exact(&mut body)) - .await - .map_err(|_| { - anyhow!( - "{length} bytes took longer than {:?}", - stream_timeout(length) - ) - })??; - Ok(body) -} - -#[async_trait] -impl Transport for Libp2pTransport { - fn local_id(&self) -> &str { - &self.local_id - } - - fn is_peer_id(&self, id: &str) -> bool { - PeerId::from_str(id).is_ok() - } - - /// The longest a message may take on a stream. - fn delivery_limit(&self) -> Duration { - stream_timeout(MAX_MESSAGE) - } - - /// Where we can be reached, each with our peer ID: the announced addresses, or else - /// those we listen on (`published`), and those through relays. A relay adds each - /// direct one again, ending in `/p2p-circuit`: where to reserve a slot. - fn local_addresses(&self) -> Vec<String> { - let book = lock(&self.book); - let direct: Vec<Multiaddr> = book - .direct() - .into_iter() - .filter_map(|address| address.with_p2p(self.peer_id).ok()) - .collect(); - let relayed = book - .listening - .iter() - .filter(|address| is_relayed(address)) - .filter_map(|address| address.clone().with_p2p(self.peer_id).ok()); - let offers: Vec<Multiaddr> = if book.serving { - direct - .iter() - .map(|address| address.clone().with(Protocol::P2pCircuit)) - .collect() - } else { - Vec::new() - }; - direct - .into_iter() - .chain(relayed) - .chain(offers) - .map(|address| address.to_string()) - .collect() - } - - /// A peer ID, or a multiaddr ending in one (noted as where that peer is). - fn locate(&self, target: &str) -> Option<String> { - if let Ok(peer) = PeerId::from_str(target) { - return Some(peer.to_string()); - } - let mut address = Multiaddr::from_str(target).ok()?; - // A relay saying where to reserve a slot: `<address>/p2p/<relay>/p2p-circuit`. - if let Some(Protocol::P2pCircuit) = address.iter().last() { - address.pop(); - let relay = split_peer(&address)?; - if relay != self.peer_id { - let new = { - let mut book = lock(&self.book); - let new = book.relays.get(&relay) != Some(&address); - if new && (book.relays.len() < MAX_PEERS || book.relays.contains_key(&relay)) { - book.relays.insert(relay, address.clone()); - } - book.remember(relay, address); - new - }; - if new { - let _ = self.commands.send(Command::CheckRelays); - } - } - return Some(relay.to_string()); - } - let peer = split_peer(&address)?; - if peer != self.peer_id { - lock(&self.book).remember(peer, address); - } - Some(peer.to_string()) - } - - /// Multiaddrs, with or without the relay's peer ID; others are ignored. - fn learn_relays(&self, targets: &[String]) { - let listed = targets - .iter() - .filter_map(|target| Multiaddr::from_str(target).ok()) - .collect(); - let Some(found) = lock(&self.book).learn_listed(listed, self.peer_id) else { - return; - }; - for address in found { - let _ = self.commands.send(Command::FindRelay { address }); - } - let _ = self.commands.send(Command::CheckRelays); - } - - fn connection(&self, target: &str) -> Option<Connection> { - let peer = PeerId::from_str(target).ok()?; - lock(&self.book).connection(&peer) - } - - fn location(&self, target: &str) -> Option<String> { - let peer = PeerId::from_str(target).ok()?; - let address = lock(&self.book).location(&peer)?.clone(); - Some( - address - .with_p2p(peer) - .unwrap_or_else(|address| address) - .to_string(), - ) - } - - fn stats(&self) -> TransportStats { - TransportStats { - libp2p: Some(lock(&self.book).stats()), - ..TransportStats::default() - } - } - - /// Nodes found on the LAN (mDNS) that speak continuum. - fn discovered_seeds(&self) -> Vec<String> { - std::mem::take(&mut lock(&self.book).discovered) - .iter() - .map(Multiaddr::to_string) - .collect() - } - - /// Where `target` is, each address with its peer ID. - fn locations(&self, target: &str) -> Vec<String> { - let Ok(peer) = PeerId::from_str(target) else { - return Vec::new(); - }; - lock(&self.book) - .addresses - .get(&peer) - .into_iter() - .flatten() - .filter_map(|address| address.clone().with_p2p(peer).ok()) - .map(|address| address.to_string()) - .collect() - } - - /// A multiaddr ending in a peer ID is that peer; one without is connected to, to - /// find out. A bare peer ID resolves if this transport knows where it is. - async fn resolve_seed(&self, seed: &str) -> Result<String> { - if let Ok(peer) = PeerId::from_str(seed) { - if lock(&self.book).addresses.contains_key(&peer) { - return Ok(peer.to_string()); - } - bail!("no address known for libp2p peer {peer}; give a multiaddr"); - } - let address = - Multiaddr::from_str(seed).with_context(|| format!("{seed} is no multiaddr"))?; - if let Some(peer) = self.locate(seed) { - return Ok(peer); - } - let (done, identified) = oneshot::channel(); - self.commands - .send(Command::Identify { address, done }) - .map_err(|_| anyhow!("the libp2p swarm stopped"))?; - match tokio::time::timeout(CONNECT_TIMEOUT, identified).await { - Ok(Ok(Ok(peer))) => Ok(peer.to_string()), - Ok(Ok(Err(error))) => bail!("couldn't connect to {seed}: {error}"), - Ok(Err(_)) => bail!("the libp2p swarm stopped"), - Err(_) => bail!("connecting to {seed} took longer than {CONNECT_TIMEOUT:?}"), - } - } - - fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> { - let body = wire::encode(message)?; - Ok(EncodedMessage { - peer: peer.to_owned(), - address: peer.to_owned(), - payload_len: body.len(), - body: body.into(), - kind: message.kind(), - message: None, - }) - } - - async fn send_encoded(&self, message: &EncodedMessage) -> Result<()> { - let peer = PeerId::from_str(&message.peer) - .with_context(|| format!("{} is no libp2p peer ID", message.peer))?; - let started = std::time::Instant::now(); - self.connect(peer).await?; - let connected = started.elapsed(); - let mut control = self.control.clone(); - let body = message.body.clone(); - let write = async move { - let mut stream = control.open_stream(peer, PROTOCOL).await?; - let opened = started.elapsed(); - stream.write_all(&(body.len() as u32).to_be_bytes()).await?; - stream.write_all(&body).await?; - stream.close().await?; - Ok::<_, anyhow::Error>(opened) - }; - let timeout = stream_timeout(message.body.len()); - let opened = tokio::time::timeout(timeout, write) - .await - .map_err(|_| anyhow!("sending to {peer} took longer than {timeout:?}"))? - .with_context(|| format!("Sending to {peer}"))?; - let took = started.elapsed(); - if took >= SLOW_SEND { - // Where a send's time goes: the swarm task (connecting), the stream's protocol - // negotiation (opening), or the connection (writing). - debug!( - peer = %Addressed(SCHEME, &peer), - kind = ?message.kind, - bytes = message.body.len(), - connect_ms = connected.as_secs_f64() * 1000.0, - open_ms = (opened - connected).as_secs_f64() * 1000.0, - write_ms = (took - opened).as_secs_f64() * 1000.0, - "Slow send" - ); - } - Ok(()) - } - - /// Reads inbound streams (`READS` at once) and handles their messages one at a - /// time, in the order they were read; PINGs and PONGs at once (`is_probe`). - async fn start_consumer(&self, handler: MessageHandler) -> Result<()> { - let Some(mut incoming) = self.incoming.lock().await.take() else { - bail!("libp2p consumer already started, or this node only relays"); - }; - let handler: Arc<HandlerFn> = Arc::from(handler); - let (sender, mut received) = mpsc::channel::<(PeerId, ParsedMessage, usize)>(READS); - let reads = Arc::new(Semaphore::new(READS)); - let probes = Arc::clone(&handler); - tokio::spawn(async move { - while let Some((peer, stream)) = incoming.next().await { - let Ok(permit) = Arc::clone(&reads).acquire_owned().await else { - break; - }; - let sender = sender.clone(); - let probes = Arc::clone(&probes); - tokio::spawn(async move { - match read_message(stream).await { - Ok(body) => match wire::decode(&body) { - Ok(message) if is_probe(&message) => { - hand_on_probe( - SCHEME, - &probes, - peer.to_string(), - message, - body.len(), - ); - } - Ok(message) => { - let _ = sender.send((peer, message, body.len())).await; - } - Err(error) => { - debug!(peer = %Addressed(SCHEME, &peer), error = %error, "Dropping a message that doesn't decode"); - } - }, - Err(error) => { - debug!(peer = %Addressed(SCHEME, &peer), error = %format!("{error:#}"), "Dropping an unreadable stream"); - } - } - drop(permit); - }); - } - }); - while let Some((peer, message, payload_len)) = received.recv().await { - if let Err(error) = handler(peer.to_string(), message, payload_len).await { - warn!(peer = %Addressed(SCHEME, &peer), error = %format!("{error:#}"), "Failed to process inbound message"); - } - } - Ok(()) - } - - async fn wait_for_exit(&self) -> std::io::Result<std::process::ExitStatus> { - let mut stopped = self.stopped.clone(); - let _ = stopped.wait_for(|stopped| *stopped).await; - Err(std::io::Error::other("the libp2p swarm stopped")) - } -} - -#[cfg(test)] -mod tests { - use std::sync::atomic::{AtomicU64, Ordering}; - - use super::*; - use crate::document::replica::Epoch; - use crate::transport::{HeadsSummary, Status}; - - /// Memory ports are process-wide. - static NEXT_PORT: AtomicU64 = AtomicU64::new(40_000); - - /// A node on the in-memory transport, listening on a port of its own (or nowhere), - /// and its address with its peer ID (or just its peer ID). - fn memory_node_with( - listen: bool, - relays: Vec<Multiaddr>, - relay: bool, - ) -> (Libp2pTransport, Multiaddr) { - let port = NEXT_PORT.fetch_add(1, Ordering::Relaxed); - let key = Keypair::generate_ed25519(); - let address: Multiaddr = format!("/memory/{port}").parse().unwrap(); - let peer = key.public().to_peer_id(); - let transport = Libp2pTransport::start(Libp2pConfig { - key, - listen: if listen { - vec![address.clone()] - } else { - Vec::new() - }, - announce: Vec::new(), - relays, - relay, - continuum: true, - mdns: false, - public: false, - memory: true, - }) - .unwrap(); - let address = if listen { - address.with_p2p(peer).unwrap() - } else { - Multiaddr::empty().with(Protocol::P2p(peer)) - }; - (transport, address) - } - - fn memory_node() -> (Libp2pTransport, Multiaddr) { - memory_node_with(true, Vec::new(), false) - } - - fn status() -> ParsedMessage { - ParsedMessage::Status(Status { - heads: HeadsSummary::of(Vec::new()), - epoch: Epoch::GENESIS, - epoch_seq: 3, - ready: true, - syncs_with_you: false, - }) - } - - /// Starts `transport`'s consumer, forwarding what it receives. - fn received( - transport: Arc<Libp2pTransport>, - ) -> mpsc::UnboundedReceiver<(String, ParsedMessage)> { - let (sender, receiver) = mpsc::unbounded_channel(); - tokio::spawn(async move { - transport - .start_consumer(Box::new(move |from, message, _| { - let sender = sender.clone(); - Box::pin(async move { - let _ = sender.send((from, message)); - Ok(()) - }) - })) - .await - }); - receiver - } - - #[tokio::test] - async fn messages_both_ways_over_one_connection() { - let (a, a_address) = memory_node(); - let (b, _) = memory_node(); - let (a, b) = (Arc::new(a), Arc::new(b)); - let mut at_a = received(Arc::clone(&a)); - let mut at_b = received(Arc::clone(&b)); - - assert_eq!( - b.resolve_seed(&a_address.to_string()).await.unwrap(), - a.local_id() - ); - b.send_message(a.local_id(), &status()).await.unwrap(); - let (from, message) = at_a.recv().await.unwrap(); - assert_eq!(from, b.local_id()); - assert!(matches!(message, ParsedMessage::Status(status) if status.epoch_seq == 3)); - - // a knows no address of b, but b's connection is there. - a.send_message(b.local_id(), &status()).await.unwrap(); - let (from, _) = at_b.recv().await.unwrap(); - assert_eq!(from, a.local_id()); - - assert_eq!(a.connection(b.local_id()), Some(Connection::Direct)); - assert_eq!(b.connection(a.local_id()), Some(Connection::Direct)); - let stats = a.stats().libp2p.unwrap(); - assert!(stats.connections.direct >= 1); - assert_eq!(stats.connections.relayed, 0); - } - - #[tokio::test] - async fn a_seed_without_a_peer_id_is_connected_to_to_find_out() { - let (a, a_address) = memory_node(); - let (b, _) = memory_node(); - let mut bare = a_address.clone(); - bare.pop(); - assert_eq!( - b.resolve_seed(&bare.to_string()).await.unwrap(), - a.local_id() - ); - // And b now knows where a is. - assert_eq!(b.locations(a.local_id()), [a_address.to_string()]); - } - - #[tokio::test] - async fn listen_addresses_carry_the_peer_id() { - let (a, a_address) = memory_node(); - let deadline = tokio::time::Instant::now() + Duration::from_secs(5); - while a.local_addresses().is_empty() && tokio::time::Instant::now() < deadline { - tokio::time::sleep(Duration::from_millis(10)).await; - } - assert_eq!(a.local_addresses(), [a_address.to_string()]); - assert_eq!( - a.locate(&a_address.to_string()).as_deref(), - Some(a.local_id()) - ); - } - - /// Waits up to 5 s for `condition`. - async fn eventually(mut condition: impl FnMut() -> bool) -> bool { - let deadline = tokio::time::Instant::now() + Duration::from_secs(5); - while !condition() { - if tokio::time::Instant::now() >= deadline { - return false; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - true - } - - /// A node learns where a peer that dialled it listens (identify), and can then - /// pass it on. - #[tokio::test] - async fn a_dialled_node_learns_where_the_dialler_listens() { - let (a, a_address) = memory_node(); - let (b, b_address) = memory_node(); - let (a, b) = (Arc::new(a), Arc::new(b)); - let _at_a = received(Arc::clone(&a)); - b.resolve_seed(&a_address.to_string()).await.unwrap(); - b.send_message(a.local_id(), &status()).await.unwrap(); - let b_id = b.local_id().to_owned(); - assert!( - eventually(|| a.locations(&b_id) == [b_address.to_string()]).await, - "a knows {:?} of b", - a.locations(&b_id) - ); - } - - #[tokio::test] - async fn announced_addresses_are_published_instead() { - let key = Keypair::generate_ed25519(); - let peer = key.public().to_peer_id(); - let port = NEXT_PORT.fetch_add(1, Ordering::Relaxed); - let announced: Multiaddr = "/ip4/203.0.113.7/tcp/9661".parse().unwrap(); - let transport = Libp2pTransport::start(Libp2pConfig { - key, - listen: vec![format!("/memory/{port}").parse().unwrap()], - announce: vec![announced.clone()], - relays: Vec::new(), - relay: false, - continuum: true, - mdns: false, - public: false, - memory: true, - }) - .unwrap(); - assert_eq!( - transport.local_addresses(), - [announced.with_p2p(peer).unwrap().to_string()] - ); - } - - #[test] - fn local_addresses_are_left_out_unless_there_are_only_local_ones() { - let parse = |text: &str| text.parse::<Multiaddr>().unwrap(); - let loopback = parse("/ip4/127.0.0.1/tcp/1"); - let link = parse("/ip6/fe80::1/tcp/1"); - let lan = parse("/ip4/192.168.1.2/tcp/1"); - let wireguard = parse("/ip6/fd00::2/tcp/1"); - assert_eq!( - published(&[ - loopback.clone(), - link.clone(), - lan.clone(), - wireguard.clone() - ]), - [lan, wireguard] - ); - assert_eq!( - published(&[loopback.clone(), link.clone()]), - [loopback, link] - ); - } - - /// b listens nowhere but through the relay r; a reaches it there. - #[tokio::test] - async fn a_node_nobody_can_dial_is_reached_through_a_relay() { - let (_r, r_address) = memory_node_with(true, Vec::new(), true); - let (b, _) = memory_node_with(false, vec![r_address.clone()], false); - let b = Arc::new(b); - let mut at_b = received(Arc::clone(&b)); - assert!( - eventually(|| !b.local_addresses().is_empty()).await, - "b got no slot on the relay" - ); - let relayed = b.local_addresses()[0].clone(); - assert!( - relayed.starts_with(&format!("{r_address}/p2p-circuit/")), - "{relayed}" - ); - - assert!(eventually(|| b.stats().libp2p.unwrap().relay_slots.held == 1).await); - let slots = b.stats().libp2p.unwrap().relay_slots; - assert_eq!((slots.granted, slots.not_granted), (1, 0)); - - let (a, _) = memory_node(); - assert_eq!(a.resolve_seed(&relayed).await.unwrap(), b.local_id()); - a.send_message(b.local_id(), &status()).await.unwrap(); - let (from, _) = at_b.recv().await.unwrap(); - assert_eq!(from, a.local_id()); - assert!(a.connection(b.local_id()).is_some()); - } - - /// A relay that is no node (`--relay-only`) relays, and takes no messages itself. - #[tokio::test] - async fn a_relay_that_is_no_node() { - let port = NEXT_PORT.fetch_add(1, Ordering::Relaxed); - let key = Keypair::generate_ed25519(); - let r_peer = key.public().to_peer_id(); - let r_listen: Multiaddr = format!("/memory/{port}").parse().unwrap(); - let r = Libp2pTransport::start(Libp2pConfig { - key, - listen: vec![r_listen.clone()], - announce: Vec::new(), - relays: Vec::new(), - relay: true, - continuum: false, - mdns: false, - public: false, - memory: true, - }) - .unwrap(); - let r_address = r_listen.with_p2p(r_peer).unwrap(); - let (b, _) = memory_node_with(false, vec![r_address.clone()], false); - let b = Arc::new(b); - let mut at_b = received(Arc::clone(&b)); - assert!( - eventually(|| !b.local_addresses().is_empty()).await, - "b got no slot on the relay" - ); - let (a, _) = memory_node(); - let relayed = b.local_addresses()[0].clone(); - assert_eq!(a.resolve_seed(&relayed).await.unwrap(), b.local_id()); - a.send_message(b.local_id(), &status()).await.unwrap(); - let (from, _) = at_b.recv().await.unwrap(); - assert_eq!(from, a.local_id()); - - // The relay itself doesn't speak continuum: it takes no messages (a send to it - // isn't refused, though: the protocol goes with the first data, `swarm_config`, - // so the relay's refusal comes too late; nodes never send to it, as it never - // proves itself a node). - assert_eq!( - a.resolve_seed(&r_address.to_string()).await.unwrap(), - r.local_id() - ); - assert!( - r.start_consumer(Box::new(|_, _, _| Box::pin(async { Ok(()) }))) - .await - .is_err() - ); - } - - /// A relay configured without its peer ID is found out, and used. - #[tokio::test] - async fn a_relay_configured_without_its_peer_id() { - let (_r, r_address) = memory_node_with(true, Vec::new(), true); - let mut bare = r_address.clone(); - bare.pop(); - let (b, _) = memory_node_with(false, vec![bare], false); - assert!( - eventually(|| !b.local_addresses().is_empty()).await, - "b got no slot on the relay" - ); - assert!(b.local_addresses()[0].starts_with(&format!("{r_address}/p2p-circuit/"))); - } - - fn peer() -> PeerId { - Keypair::generate_ed25519().public().to_peer_id() - } - - #[test] - fn relays_are_picked_when_needed() { - use ::libp2p::autonat::NatStatus; - let local = peer(); - let offers: HashMap<PeerId, Multiaddr> = (0..4) - .map(|port| (peer(), format!("/memory/{port}").parse().unwrap())) - .collect(); - let check = |direct, nat: &NatStatus, using: &[PeerId]| { - relay_check(local, direct, nat, &offers, using) - }; - // Dialable, or not known yet with an address to dial: none. - assert!(check(true, &NatStatus::Public(Multiaddr::empty()), &[]).is_empty()); - assert!(check(false, &NatStatus::Public(Multiaddr::empty()), &[]).is_empty()); - assert!(check(true, &NatStatus::Unknown, &[]).is_empty()); - // No address to dial, or AutoNAT says it can't be dialled: some. - assert_eq!(check(false, &NatStatus::Unknown, &[]).len(), AUTO_RELAYS); - let picked = check(true, &NatStatus::Private, &[]); - assert_eq!(picked.len(), AUTO_RELAYS); - // One in use: the next in the same order. - let more = check(true, &NatStatus::Private, &[picked[0].0]); - assert_eq!(more.len(), AUTO_RELAYS - 1); - assert_eq!(more[0].0, picked[1].0); - let all: Vec<PeerId> = picked.iter().map(|(relay, _)| *relay).collect(); - assert!(check(true, &NatStatus::Private, &all).is_empty()); - // Never itself. - let only_us = HashMap::from([(local, Multiaddr::empty())]); - assert!(relay_check(local, false, &NatStatus::Private, &only_us, &[]).is_empty()); - } - - /// A relay says so in its addresses; b, which nobody can dial, learns of it (as from - /// the member list) and becomes reachable through it. - #[tokio::test] - async fn a_member_that_relays_is_picked() { - let (r, _) = memory_node_with(true, Vec::new(), true); - assert!(eventually(|| !r.local_addresses().is_empty()).await); - let offer = r - .local_addresses() - .into_iter() - .find(|address| address.ends_with("/p2p-circuit")) - .expect("the relay offers itself"); - let (b, _) = memory_node_with(false, Vec::new(), false); - assert_eq!(b.locate(&offer).as_deref(), Some(r.local_id())); - assert!( - eventually(|| b - .local_addresses() - .iter() - .any(|address| address.contains("/p2p-circuit/"))) - .await, - "b isn't reachable through r: {:?}", - b.local_addresses() - ); - } - - /// A peer on the LAN that speaks continuum becomes a seed once, at its address. - #[test] - fn lan_peers_become_seeds_once() { - let mut book = Book::default(); - let lan = peer(); - let address: Multiaddr = "/ip4/192.168.1.5/tcp/9661".parse().unwrap(); - book.discover(lan); // No address known: nothing. - assert!(book.discovered.is_empty()); - book.remember(lan, address.clone()); - book.discover(lan); - book.discover(lan); - assert_eq!(book.discovered, [address.with_p2p(lan).unwrap()]); - } - - #[test] - fn named_relays() { - let (local, r1, r2) = (peer(), peer(), peer()); - let at = |port: u16| -> Multiaddr { format!("/ip6/fd00::1/tcp/{port}").parse().unwrap() }; - let mut book = Book::default(); - // With a peer ID: known at once; without: to be found out, once. - assert_eq!( - book.name_relay(at(1).with_p2p(r1).unwrap(), Named::Configured, local), - None - ); - assert_eq!( - book.name_relay(at(2), Named::Configured, local), - Some(at(2)) - ); - assert_eq!(book.name_relay(at(2), Named::Configured, local), None); - assert_eq!( - book.name_relay(at(3).with_p2p(local).unwrap(), Named::Configured, local), - None - ); - assert!(book.relay_found(at(2), r2, local)); - assert_eq!(book.named(Named::Configured).len(), 2); - // A relay named with its peer ID isn't found out again; one without is. - assert_eq!(book.find_again(r1), None); - assert_eq!(book.find_again(r2), Some(at(2))); - assert!(!book.named(Named::Configured).contains_key(&r2)); - - // Listed: configured wins; unlisted ones go, unless also configured. - let r3 = peer(); - let listed = vec![ - at(1).with_p2p(r1).unwrap(), - at(4).with_p2p(r3).unwrap(), - at(5), - ]; - assert_eq!(book.learn_listed(listed.clone(), local), Some(vec![at(5)])); - assert_eq!(book.learn_listed(listed, local), None); - assert_eq!(book.named_relays[&r1].named, Named::Configured); - assert_eq!(book.named(Named::Listed).keys().collect::<Vec<_>>(), [&r3]); - assert_eq!(book.learn_listed(Vec::new(), local), Some(Vec::new())); - assert!(book.named(Named::Listed).is_empty()); - assert!(!book.finding.contains_key(&at(5))); - assert!(!book.relay_found(at(5), peer(), local)); // No longer listed. - assert_eq!(book.named(Named::Configured).len(), 1); - } - - #[test] - fn known_addresses() { - let mut book = Book::default(); - let (relay, us) = (peer(), peer()); - let direct: Multiaddr = "/ip6/fd00::1/tcp/9661".parse().unwrap(); - let relayed: Multiaddr = format!("/ip6/fd00::2/tcp/9661/p2p/{relay}/p2p-circuit/p2p/{us}") - .parse() - .unwrap(); - assert!(!book.knows(us, &[direct.clone()])); - book.remember(us, relayed.clone()); - book.remember(us, direct.clone()); - // In any order, with or without the peer ID. - assert!(book.knows(us, &[relayed.clone(), direct.clone()])); - assert!(book.knows(us, &[direct.clone().with_p2p(us).unwrap()])); - assert!(!book.knows(us, &["/ip6/fd00::3/tcp/9661".parse().unwrap()])); - } - - /// Peers nobody mentions any more are forgotten, but connected ones. - #[test] - fn stale_peers_are_forgotten() { - let (old, current, connected) = (peer(), peer(), peer()); - let address: Multiaddr = "/ip6/fd00::1/tcp/9661".parse().unwrap(); - let mut book = Book::default(); - for peer in [old, current, connected] { - book.remember(peer, address.clone()); - } - book.relays.insert(old, address.clone()); - book.continuum.insert(old); - // From after the mentions above (taken at `Instant::now()`), however fast or - // slow the clock moved meanwhile. - let later = tokio::time::Instant::now() + STALE_AFTER; - // `current` was mentioned again since. - book.seen.insert(current, later); - assert_eq!(book.prune(later, |peer| *peer == connected), 1); - assert!(!book.addresses.contains_key(&old)); - assert!(!book.relays.contains_key(&old) && !book.continuum.contains(&old)); - assert!(book.addresses.contains_key(¤t)); - assert!(book.addresses.contains_key(&connected)); - assert_eq!(book.prune(later, |_| false), 1); // `connected`, once it isn't. - } - - /// libp2p's nested dial errors come out as each address's cause, once. - #[test] - fn dial_errors_are_short() { - use ::libp2p::core::transport::TransportError; - let peer = peer(); - let address: Multiaddr = format!("/ip6/fd00::1/tcp/9661/p2p/{peer}").parse().unwrap(); - let nested = std::io::Error::other( - "Multiple dial errors occurred:\n - Network is unreachable (os error 101): Multiple dial errors occurred:\n - Network is unreachable (os error 101): Network is unreachable (os error 101)", - ); - let error = - ::libp2p::swarm::DialError::Transport(vec![(address, TransportError::Other(nested))]); - assert_eq!( - describe_dial_error(&error), - "/ip6/fd00::1/tcp/9661: Network is unreachable (os error 101)" - ); - let refused = ::libp2p::swarm::DialError::Transport(vec![( - "/ip4/10.0.0.1/tcp/1".parse().unwrap(), - TransportError::Other(std::io::Error::from(std::io::ErrorKind::ConnectionRefused)), - )]); - assert_eq!( - describe_dial_error(&refused), - "/ip4/10.0.0.1/tcp/1: connection refused" - ); - } - - /// A relay doesn't dial the nodes it relays for through itself. - #[test] - fn no_circuits_through_ourselves() { - let (us, other_relay, target) = (peer(), peer(), peer()); - let through = |relay: PeerId| -> Multiaddr { - format!("/ip6/fd00::1/tcp/9661/p2p/{relay}/p2p-circuit") - .parse() - .unwrap() - }; - let direct: Multiaddr = "/ip6/fd00::2/tcp/9661".parse().unwrap(); - assert_eq!(relay_of(&through(us)), Some(us)); - assert_eq!(relay_of(&direct), None); - assert_eq!(relay_of(&direct.clone().with_p2p(target).unwrap()), None); - assert_eq!( - dialable(&[through(us), through(other_relay), direct.clone()], us), - [through(other_relay), direct] - ); - } - - #[tokio::test] - async fn unknown_peers_are_refused() { - let (b, _) = memory_node(); - let stranger = Keypair::generate_ed25519() - .public() - .to_peer_id() - .to_string(); - assert!(b.send_message(&stranger, &status()).await.is_err()); - assert!(b.resolve_seed(&stranger).await.is_err()); - assert_eq!(b.connection(&stranger), None); - } - - #[test] - fn a_direct_connection_wins_over_a_relayed_one() { - let mut book = Book::default(); - let (relayed_only, both) = (peer(), peer()); - let mut id = 0; - let mut connect = |book: &mut Book, peer: PeerId, relayed: bool| { - id += 1; - let address: Multiaddr = format!("/ip4/10.0.0.{id}/tcp/1").parse().unwrap(); - book.connections - .insert(ConnectionId::new_unchecked(id), (peer, relayed, address)); - }; - connect(&mut book, relayed_only, true); - connect(&mut book, both, true); - connect(&mut book, both, false); - assert_eq!(book.connection(&relayed_only), Some(Connection::Relayed)); - assert_eq!(book.connection(&both), Some(Connection::Direct)); - assert_eq!(book.connection(&peer()), None); - assert_eq!( - book.location(&both).map(|address| address.to_string()), - Some("/ip4/10.0.0.3/tcp/1".to_owned()) - ); - let connections = book.stats().connections; - assert_eq!((connections.direct, connections.relayed), (1, 2)); - // Of several, the oldest direct one stays; a relayed one only without a direct. - connect(&mut book, both, false); - assert_eq!( - book.duplicates(&both), - [ - ConnectionId::new_unchecked(4), - ConnectionId::new_unchecked(2) - ] - ); - assert!(book.duplicates(&relayed_only).is_empty()); - } -} diff --git a/src/transport/libp2p/behaviour.rs b/src/transport/libp2p/behaviour.rs new file mode 100644 index 0000000..0b9be6b --- /dev/null +++ b/src/transport/libp2p/behaviour.rs @@ -0,0 +1,375 @@ +//! The libp2p behaviours this transport combines (`Behaviour`), hole punching with LAN +//! candidates (`HolePunching`), and building the swarm (TCP and QUIC, or in memory). + +use super::*; + +/// The protocol version identify says (informational: nothing checks it). +pub(super) const IDENTIFY_VERSION: &str = "/continuum/1"; + +#[derive(::libp2p::swarm::NetworkBehaviour)] +pub(super) struct Behaviour { + /// First: connections it denies need nothing from the others. + pub(super) ignoring: Ignoring, + pub(super) stream: libp2p_stream::Behaviour, + pub(super) identify: Identifying, + pub(super) relay_client: ::libp2p::relay::client::Behaviour, + /// Only with `Libp2pConfig::relay`. + pub(super) relay: ::libp2p::swarm::behaviour::toggle::Toggle<::libp2p::relay::Behaviour>, + pub(super) dcutr: HolePunching, + /// Each connection's round trip (`Book::rtts`): of several to one peer, the fastest + /// is kept. + pub(super) ping: ::libp2p::ping::Behaviour, + pub(super) autonat: ::libp2p::autonat::Behaviour, + /// Only with `Libp2pConfig::mdns`. + pub(super) mdns: ::libp2p::swarm::behaviour::toggle::Toggle<::libp2p::mdns::tokio::Behaviour>, + /// Only with `Libp2pConfig::upnp`: maps our listen ports on the router, and confirms + /// the mapped addresses as ours (identify then tells them). + pub(super) upnp: ::libp2p::swarm::behaviour::toggle::Toggle<::libp2p::upnp::tokio::Behaviour>, + /// Only with `Libp2pConfig::public`: the IPFS DHT. + pub(super) kad: ::libp2p::swarm::behaviour::toggle::Toggle< + ::libp2p::kad::Behaviour<::libp2p::kad::store::MemoryStore>, + >, +} + +/// libp2p's hole punching (DCUtR), offering our listen addresses too. DCUtR offers only +/// where others saw us (external address candidates, from identify): seen by a relay +/// out on the internet, two nodes on one LAN are both at their router's public address, +/// and punching through it needs the router to hairpin, which many don't. Their LAN +/// addresses work, the firewalls on the way being stateful. Only DCUtR is told: +/// AutoNAT would test every candidate, and fail on all of these. +pub(super) struct HolePunching { + pub(super) inner: ::libp2p::dcutr::Behaviour, + /// Where we listen (`offered`). + pub(super) listening: Vec<Multiaddr>, + /// `Libp2pConfig::ignore`: not offered. + pub(super) ignored: Vec<IpRange>, +} + +/// Whether DCUtR offers our listen address `address` too: neither loopback nor +/// link-local (as for publishing), nor through a relay. +pub(super) fn offered(address: &Multiaddr) -> bool { + !is_local(address) && !is_relayed(address) +} + +impl HolePunching { + pub(super) fn new(local: PeerId, ignored: Vec<IpRange>) -> Self { + Self { + inner: ::libp2p::dcutr::Behaviour::new(local), + listening: Vec::new(), + ignored, + } + } + + /// Tells DCUtR where we listen, as if others had seen us there. Again after every + /// other candidate, as DCUtR keeps the latest 20. + pub(super) fn offer_listening(&mut self) { + use ::libp2p::swarm::{FromSwarm, NetworkBehaviour, NewExternalAddrCandidate}; + for addr in &self.listening { + self.inner + .on_swarm_event(FromSwarm::NewExternalAddrCandidate( + NewExternalAddrCandidate { addr }, + )); + } + } +} + +impl ::libp2p::swarm::NetworkBehaviour for HolePunching { + type ConnectionHandler = + <::libp2p::dcutr::Behaviour as ::libp2p::swarm::NetworkBehaviour>::ConnectionHandler; + type ToSwarm = ::libp2p::dcutr::Event; + + fn handle_pending_inbound_connection( + &mut self, + connection_id: ConnectionId, + local_addr: &Multiaddr, + remote_addr: &Multiaddr, + ) -> Result<(), ::libp2p::swarm::ConnectionDenied> { + self.inner + .handle_pending_inbound_connection(connection_id, local_addr, remote_addr) + } + + fn handle_established_inbound_connection( + &mut self, + connection_id: ConnectionId, + peer: PeerId, + local_addr: &Multiaddr, + remote_addr: &Multiaddr, + ) -> Result<::libp2p::swarm::THandler<Self>, ::libp2p::swarm::ConnectionDenied> { + if is_relayed(local_addr) { + // DCUtR offers its candidates on this connection: where others saw us, and + // these. + debug!(peer = %Addressed(SCHEME, &peer), listening = ?self.listening, "Reached through a relay; hole punching offers where we listen too"); + } + self.inner.handle_established_inbound_connection( + connection_id, + peer, + local_addr, + remote_addr, + ) + } + + fn handle_pending_outbound_connection( + &mut self, + connection_id: ConnectionId, + maybe_peer: Option<PeerId>, + addresses: &[Multiaddr], + effective_role: ::libp2p::core::Endpoint, + ) -> Result<Vec<Multiaddr>, ::libp2p::swarm::ConnectionDenied> { + self.inner.handle_pending_outbound_connection( + connection_id, + maybe_peer, + addresses, + effective_role, + ) + } + + fn handle_established_outbound_connection( + &mut self, + connection_id: ConnectionId, + peer: PeerId, + addr: &Multiaddr, + role_override: ::libp2p::core::Endpoint, + port_use: ::libp2p::core::transport::PortUse, + ) -> Result<::libp2p::swarm::THandler<Self>, ::libp2p::swarm::ConnectionDenied> { + self.inner.handle_established_outbound_connection( + connection_id, + peer, + addr, + role_override, + port_use, + ) + } + + fn on_swarm_event(&mut self, event: ::libp2p::swarm::FromSwarm) { + use ::libp2p::swarm::FromSwarm; + self.inner.on_swarm_event(event); + match event { + FromSwarm::NewListenAddr(new) + if offered(new.addr) && !ignored(&self.ignored, new.addr) => + { + if !self.listening.contains(new.addr) { + debug!(address = %new.addr, "Offering where we listen for hole punching"); + self.listening.push(new.addr.clone()); + } + self.offer_listening(); + } + FromSwarm::ExpiredListenAddr(expired) => { + self.listening.retain(|address| address != expired.addr); + } + FromSwarm::NewExternalAddrCandidate(_) => self.offer_listening(), + _ => {} + } + } + + fn on_connection_handler_event( + &mut self, + peer_id: PeerId, + connection_id: ConnectionId, + event: ::libp2p::swarm::THandlerOutEvent<Self>, + ) { + self.inner + .on_connection_handler_event(peer_id, connection_id, event) + } + + fn poll( + &mut self, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll< + ::libp2p::swarm::ToSwarm<Self::ToSwarm, ::libp2p::swarm::THandlerInEvent<Self>>, + > { + self.inner.poll(cx) + } +} + +/// What the behaviour has besides what every node has. +#[derive(Clone, Copy)] +pub(super) struct Features { + pub(super) relay: bool, + pub(super) mdns: bool, + pub(super) upnp: bool, + pub(super) public: bool, + /// How long a connection may idle (`idle_timeout`). + pub(super) idle_timeout: Duration, +} + +impl Features { + pub(super) fn of(config: &Libp2pConfig) -> Self { + Self { + relay: config.relay, + mdns: config.mdns, + upnp: config.upnp, + public: config.public, + idle_timeout: idle_timeout(config.keepalive), + } + } +} + +/// The IPFS DHT's protocol, and its bootstrap nodes (all at `IPFS_BOOTSTRAP_ADDRESS`). +pub(super) const IPFS_DHT: StreamProtocol = StreamProtocol::new("/ipfs/kad/1.0.0"); + +pub(super) const IPFS_BOOTSTRAP_ADDRESS: &str = "/dnsaddr/bootstrap.libp2p.io"; + +pub(super) const IPFS_BOOTSTRAP_PEERS: [&str; 4] = [ + "QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", + "QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", + "QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb", + "QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt", +]; + +/// How often a node on the DHT refreshes its routing table. +pub(super) const DHT_BOOTSTRAP: Duration = Duration::from_secs(300); + +impl Behaviour { + pub(super) fn new( + key: &Keypair, + relay_client: ::libp2p::relay::client::Behaviour, + features: Features, + ignored: Vec<IpRange>, + ) -> std::io::Result<Self> { + let Features { + relay, + mdns, + upnp, + public, + idle_timeout: _, + } = features; + let peer_id = key.public().to_peer_id(); + let kad = public.then(|| { + let mut dht = ::libp2p::kad::Behaviour::with_config( + peer_id, + ::libp2p::kad::store::MemoryStore::new(peer_id), + ::libp2p::kad::Config::new(IPFS_DHT), + ); + let bootstrap: Multiaddr = IPFS_BOOTSTRAP_ADDRESS.parse().expect("a multiaddr"); + for peer in IPFS_BOOTSTRAP_PEERS { + let peer: PeerId = peer.parse().expect("a peer ID"); + let _ = dht.add_address(&peer, bootstrap.clone()); + } + dht + }); + let identify = ::libp2p::identify::Config::new(IDENTIFY_VERSION.to_owned(), key.public()) + .with_agent_version(format!("continuum/{}", env!("CARGO_PKG_VERSION"))) + // Connected peers hear of a new interface at once (`explore`). + .with_push_listen_addr_updates(true); + let relay = relay.then(|| { + ::libp2p::relay::Behaviour::new( + peer_id, + ::libp2p::relay::Config { + max_circuits: RELAY_CIRCUITS, + max_circuits_per_peer: RELAY_CIRCUITS_PER_PEER, + max_circuit_duration: RELAY_CIRCUIT_DURATION, + max_circuit_bytes: RELAY_CIRCUIT_BYTES, + reservation_rate_limiters: Vec::new(), + circuit_src_rate_limiters: Vec::new(), + ..Default::default() + }, + ) + }); + let mdns = match mdns { + true => Some(::libp2p::mdns::tokio::Behaviour::new( + ::libp2p::mdns::Config::default(), + peer_id, + )?), + false => None, + }; + Ok(Self { + ignoring: Ignoring { + ranges: ignored.clone(), + }, + stream: libp2p_stream::Behaviour::new(), + identify: Identifying { + inner: ::libp2p::identify::Behaviour::new(identify), + ranges: ignored.clone(), + // Set by `Libp2pTransport::start`. + announced: Vec::new(), + }, + relay_client, + relay: relay.into(), + dcutr: HolePunching::new(peer_id, ignored), + ping: ::libp2p::ping::Behaviour::new(::libp2p::ping::Config::new()), + autonat: ::libp2p::autonat::Behaviour::new( + peer_id, + ::libp2p::autonat::Config { + // Swarms on private networks (a LAN, WireGuard) count too. + only_global_ips: false, + ..Default::default() + }, + ), + mdns: mdns.into(), + upnp: upnp.then(::libp2p::upnp::tokio::Behaviour::default).into(), + kad: kad.into(), + }) + } +} + +pub(super) fn network_swarm( + key: Keypair, + features: Features, + ignored: Vec<IpRange>, +) -> Result<Swarm<Behaviour>> { + Ok(::libp2p::SwarmBuilder::with_existing_identity(key) + .with_tokio() + .with_tcp( + ::libp2p::tcp::Config::default(), + ::libp2p::noise::Config::new, + ::libp2p::yamux::Config::default, + )? + .with_quic() + .with_dns()? + .with_relay_client( + ::libp2p::noise::Config::new, + ::libp2p::yamux::Config::default, + )? + .with_behaviour(|key, relay_client| { + Behaviour::new(key, relay_client, features, ignored.clone()) + .map_err(|error| Box::new(error) as Box<dyn std::error::Error + Send + Sync>) + })? + .with_swarm_config(|config| swarm_config(config, features)) + .build()) +} + +/// The swarm's settings. Streams are opened without waiting for the other side to agree +/// on the protocol (`V1Lazy`): the protocol and the first data go together, saving a +/// round trip per message (one stream each, module docs). That is safe only towards +/// peers sure to speak the protocol, so not with public libp2p infrastructure +/// (`Libp2pConfig::public`), whose nodes speak only some: there a protocol they don't +/// would fail late, and unclearly. +pub(super) fn swarm_config( + config: ::libp2p::swarm::Config, + features: Features, +) -> ::libp2p::swarm::Config { + let config = config + .with_idle_connection_timeout(features.idle_timeout) + .with_dial_concurrency_factor(DIAL_CONCURRENCY); + if features.public { + config + } else { + config.with_substream_upgrade_protocol_override(::libp2p::core::upgrade::Version::V1Lazy) + } +} + +pub(super) fn memory_swarm( + key: Keypair, + features: Features, + ignored: Vec<IpRange>, +) -> Result<Swarm<Behaviour>> { + use ::libp2p::Transport as _; + use ::libp2p::core::transport::{MemoryTransport, OrTransport}; + use ::libp2p::core::upgrade::Version; + + let peer_id = key.public().to_peer_id(); + let (relay_transport, relay_client) = ::libp2p::relay::client::new(peer_id); + let transport = OrTransport::new(relay_transport, MemoryTransport::default()) + .boxed() + .upgrade(Version::V1) + .authenticate(::libp2p::noise::Config::new(&key)?) + .multiplex(::libp2p::yamux::Config::default()) + .timeout(CONNECT_TIMEOUT) + .boxed(); + Ok(Swarm::new( + transport, + Behaviour::new(&key, relay_client, features, ignored)?, + peer_id, + swarm_config(::libp2p::swarm::Config::with_tokio_executor(), features), + )) +} diff --git a/src/transport/libp2p/book.rs b/src/transport/libp2p/book.rs new file mode 100644 index 0000000..0c61714 --- /dev/null +++ b/src/transport/libp2p/book.rs @@ -0,0 +1,779 @@ +//! The book: who is where (peers' addresses, connections and their round trips), the +//! relays and their slots, hole punching and upgrades, and the counts the figures show. + +use super::*; + +/// Who named a relay to pick, in the order they are picked. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Named { + /// `Libp2pConfig::relays`. + Configured, + /// The relays listed in the document (`Transport::learn_relays`). + Listed, +} + +impl Named { + pub(super) fn as_str(self) -> &'static str { + match self { + Self::Configured => "configured", + Self::Listed => "listed", + } + } +} + +/// A relay named to pick. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct NamedRelay { + /// Where, with the relay's peer ID. + pub(super) at: Multiaddr, + /// As named: without the peer ID, it is found out again when the relay doesn't + /// grant a slot (it may have a new one). + pub(super) given: Multiaddr, + pub(super) named: Named, +} + +/// Where peers are, and where we listen. +#[derive(Default)] +pub(super) struct Book { + pub(super) addresses: HashMap<PeerId, Vec<Multiaddr>>, + /// Where connected peers say they are, by their last identify (`identified`): while + /// connected, the only direct addresses of theirs kept. Forgotten when the last + /// connection to the peer closes, as a peer restarted elsewhere is found by what + /// the others say. + pub(super) identified: HashMap<PeerId, Vec<Multiaddr>>, + /// Addresses of directly connected peers dialled for a better path (`explore`): + /// not again while connected, unless our own interfaces change. + pub(super) explored: HashMap<PeerId, std::collections::HashSet<Multiaddr>>, + /// Addresses where a dial for a peer reached another (`wrongPeerId`), say a NATed + /// peer's LAN address in a range our LAN uses too: not that peer's, whatever + /// identify or the member list say, so not kept, dialled or explored again. As long + /// as the peer is known (`prune`), `WRONG_PER_PEER` each. + pub(super) wrong: HashMap<PeerId, std::collections::HashSet<Multiaddr>>, + pub(super) listening: Vec<Multiaddr>, + /// Where the router forwards to our listen addresses (UPnP, `Libp2pConfig::upnp`). + pub(super) mapped: Vec<Multiaddr>, + /// `Libp2pConfig::announce`. + pub(super) announced: Vec<Multiaddr>, + /// `Libp2pConfig::ignore`: not published (but announced), not dialled. + pub(super) ignored: Vec<IpRange>, + /// `Libp2pConfig::prefer`: addresses there first (`preference`). + pub(super) preferred: Vec<IpRange>, + /// Which address ranges have a route from here (`routes`): those without are left + /// out, as if ignored. + pub(super) routes: Routes, + /// `Libp2pConfig::relay`: this node relays, and says so. + pub(super) serving: bool, + /// Relays named to pick (configured, listed), by peer: picked first. + pub(super) named_relays: HashMap<PeerId, NamedRelay>, + /// Relays named without their peer ID, being found out (`Command::FindRelay`). + pub(super) finding: HashMap<Multiaddr, Named>, + /// The relays listed in the document, as last learned (`learn_listed`). + pub(super) listed: Vec<Multiaddr>, + /// Members that say they relay, and where (with the relay's peer ID). + pub(super) relays: HashMap<PeerId, Multiaddr>, + /// With `Libp2pConfig::public`: other peers that relay (identify says so), and + /// where. + pub(super) public_relays: HashMap<PeerId, Multiaddr>, + /// Relays not to be picked, until when (`RELAY_PAUSE`, `RELAY_RETRY`). + pub(super) failed_relays: HashMap<PeerId, tokio::time::Instant>, + /// Relays that granted us a slot we hold. + pub(super) slots: std::collections::HashSet<PeerId>, + /// Relays we hold a slot on whose duplicate connections we closed, and when + /// (`duplicates`): a slot lost then moves to the connection kept, it isn't lost. + pub(super) rehoming: HashMap<PeerId, tokio::time::Instant>, + /// Peers holding a slot on this node, as a relay. + pub(super) slots_here: std::collections::HashSet<PeerId>, + /// Peers connected only through a relay, dialled directly (`upgrade`): when to again + /// at the soonest, and how long to wait after that. + pub(super) upgrades: HashMap<PeerId, (tokio::time::Instant, Duration)>, + /// The shortest round trip measured on each open connection (libp2p's ping), since + /// the last connection to the same peer opened (they are compared then). + pub(super) rtts: HashMap<ConnectionId, Duration>, + /// Peers whose hole punching failed: how often in a row, and when to try again + /// (`PUNCH_RETRIES`). + pub(super) punching: HashMap<PeerId, (usize, tokio::time::Instant)>, + /// Open connections: to whom, whether through a relay, and the remote address. + pub(super) connections: HashMap<ConnectionId, (PeerId, bool, Multiaddr)>, + /// What AutoNAT says, and the address it confirmed while public. + pub(super) reachability: Reachability, + pub(super) public_address: Option<Multiaddr>, + /// Since start (`stats`). + pub(super) counts: Counts, + /// When each peer was last mentioned (`STALE_AFTER`). + pub(super) seen: HashMap<PeerId, tokio::time::Instant>, + /// Peers found on the LAN (mDNS) not known yet to speak continuum. + pub(super) lan: std::collections::HashSet<PeerId>, + /// Peers that speak continuum (identify says so). + pub(super) continuum: std::collections::HashSet<PeerId>, + /// Peers identify says don't speak continuum, and whether they relay: not sent to + /// (`send_encoded`), so that a seed that's no node says so. + pub(super) not_nodes: HashMap<PeerId, bool>, + /// Peers found on the LAN that speak continuum, with their address: seeds not + /// handed out yet (`discovered_seeds`). + pub(super) discovered: Vec<Multiaddr>, + /// Told when a seed is found (`Transport::on_seeds_found`). + pub(super) seeds_found: Option<Arc<tokio::sync::Notify>>, + /// Told when others may reach this node where they couldn't before + /// (`Transport::on_reachable`). + pub(super) reachable: Option<Arc<tokio::sync::Notify>>, +} + +/// What `Libp2pStats` counts since start. +#[derive(Default)] +pub(super) struct Counts { + pub(super) slots_granted: u64, + pub(super) slots_lost: u64, + pub(super) slots_not_granted: u64, + pub(super) hole_punches: HolePunches, + pub(super) dial_failures: DialFailures, +} + +impl Book { + /// How we are connected to `peer`: directly if any connection is direct. + pub(super) fn connection(&self, peer: &PeerId) -> Option<Connection> { + book_connection(&self.connections, peer) + } + + /// The connections to `peer` to close, of several: all but the one to keep + /// (`ranked`). + /// + /// Between a relay and a node with a slot on it too: messages go over any of the + /// connections (libp2p-stream picks one at random for each), so a slower one kept + /// would carry half of them. A slot lives on one connection, which libp2p doesn't + /// tell; if that one closes, the slot ends, and the node reserves it again at once, + /// over the connection kept (`Book::rehoming`, not counted as lost); circuits + /// through it end, and are made again with the next message. + pub(super) fn duplicates(&self, peer: &PeerId) -> Vec<ConnectionId> { + self.ranked(peer).into_iter().skip(1).collect() + } + + /// The connections to `peer`, the one to keep first: direct ones before relayed + /// ones, measured ones by their round trip (`rtts`, the shortest since the last of + /// them opened: over the LAN before a VPN's hub, whichever opened first), then the + /// oldest. + pub(super) fn ranked(&self, peer: &PeerId) -> Vec<ConnectionId> { + let mut connections: Vec<(bool, Duration, ConnectionId)> = self + .connections + .iter() + .filter(|(_, (connected, _, _))| connected == peer) + .map(|(id, (_, relayed, _))| { + let rtt = self.rtts.get(id).copied().unwrap_or(Duration::MAX); + (*relayed, rtt, *id) + }) + .collect(); + connections.sort(); + connections.into_iter().map(|(_, _, id)| id).collect() + } + + /// The remote address of the connection to `peer` that is kept (`ranked`). + pub(super) fn location(&self, peer: &PeerId) -> Option<&Multiaddr> { + let kept = *self.ranked(peer).first()?; + self.connections.get(&kept).map(|(_, _, address)| address) + } + + pub(super) fn stats(&self) -> Libp2pStats { + let mut connections = Connections::default(); + for (_, relayed, _) in self.connections.values() { + if *relayed { + connections.relayed += 1; + } else { + connections.direct += 1; + } + } + Libp2pStats { + reachability: self.reachability, + public_address: self.public_address.as_ref().map(Multiaddr::to_string), + connections, + relay_slots: RelaySlots { + held: self.slots.len(), + paused: self + .failed_relays + .values() + .filter(|until| tokio::time::Instant::now() < **until) + .count(), + granted: self.counts.slots_granted, + lost: self.counts.slots_lost, + not_granted: self.counts.slots_not_granted, + }, + hole_punches: self.counts.hole_punches, + dial_failures: self.counts.dial_failures, + } + } + + /// `peer` was mentioned just now. + pub(super) fn touch(&mut self, peer: PeerId) { + self.seen.insert(peer, tokio::time::Instant::now()); + } + + /// Forgets the peers not mentioned within `STALE_AFTER` before `now`, but those + /// `connected`. Returns how many. + pub(super) fn prune( + &mut self, + now: tokio::time::Instant, + connected: impl Fn(&PeerId) -> bool, + ) -> usize { + let stale: Vec<PeerId> = self + .addresses + .keys() + .chain(self.relays.keys()) + .chain(self.public_relays.keys()) + .chain(self.continuum.iter()) + .chain(self.lan.iter()) + .filter(|peer| { + !connected(peer) + && self + .seen + .get(peer) + .is_none_or(|at| now.saturating_duration_since(*at) >= STALE_AFTER) + }) + .copied() + .collect::<std::collections::HashSet<_>>() + .into_iter() + .collect(); + for peer in &stale { + self.addresses.remove(peer); + self.identified.remove(peer); + self.explored.remove(peer); + self.wrong.remove(peer); + self.relays.remove(peer); + self.public_relays.remove(peer); + self.continuum.remove(peer); + self.not_nodes.remove(peer); + self.lan.remove(peer); + self.failed_relays.remove(peer); + self.upgrades.remove(peer); + self.punching.remove(peer); + self.seen.remove(peer); + } + stale.len() + } + + /// `peer`, found on the LAN, speaks continuum: a seed, at the address we know first. + pub(super) fn discover(&mut self, peer: PeerId) { + let Some(address) = self.addresses.get(&peer).and_then(|known| known.first()) else { + return; + }; + if let Ok(address) = address.clone().with_p2p(peer) + && !self.discovered.contains(&address) + && self.discovered.len() < MAX_PEERS + { + self.discovered.push(address); + if let Some(found) = &self.seeds_found { + found.notify_one(); + } + } + } + + /// Where others could dial us directly: the announced addresses, or else those we + /// listen on (`published`), where the router forwards to them (`mapped`), and where + /// AutoNAT found others can dial us (`public_address`: a port forwarded by hand, say), + /// without the peer ID. + pub(super) fn direct(&self) -> Vec<Multiaddr> { + if !self.announced.is_empty() { + return self.announced.clone(); + } + let direct: Vec<Multiaddr> = self + .listening + .iter() + .filter(|address| !is_relayed(address)) + .cloned() + .collect(); + let mut direct = published(&direct); + let confirmed = self.public_address.iter().filter_map(|address| { + let address = without_peer(address.clone()); + (!address.is_empty() && !is_relayed(&address)).then_some(address) + }); + // First: only so many are published (`PUBLISHED_DIRECT`), and a node listening + // everywhere has many. + let mut first: Vec<Multiaddr> = Vec::new(); + for address in self.mapped.iter().cloned().chain(confirmed) { + if !first.contains(&address) { + first.push(address); + } + } + direct.retain(|address| !first.contains(address)); + direct.sort_by_key(|address| self.preferred_range(address)); + first.extend(direct); + first.retain(|address| { + !ignored(&self.ignored, address) + && ip_of(address).is_none_or(|ip| self.routes.publishable(ip)) + }); + first + } +} + +/// Whether `address` only reaches this host (or this link): no use to other hosts, and +/// dialled by one, it reaches that host itself. +pub(super) fn is_local(address: &Multiaddr) -> bool { + match address.iter().next() { + Some(Protocol::Ip4(ip)) => ip.is_loopback() || ip.is_link_local() || ip.is_unspecified(), + Some(Protocol::Ip6(ip)) => { + ip.is_loopback() || ip.is_unicast_link_local() || ip.is_unspecified() + } + _ => false, + } +} + +/// The addresses of `all` worth publishing: all but local ones, unless they're all +/// local (a test on one host). +pub(super) fn published(all: &[Multiaddr]) -> Vec<Multiaddr> { + let reachable: Vec<Multiaddr> = all + .iter() + .filter(|address| !is_local(address)) + .cloned() + .collect(); + if reachable.is_empty() { + all.to_vec() + } else { + reachable + } +} + +impl Book { + /// `named` names the relay at `given`; returns it if its peer is to be found out. + /// `local`: this node, never its own relay. + pub(super) fn name_relay( + &mut self, + given: Multiaddr, + named: Named, + local: PeerId, + ) -> Option<Multiaddr> { + match split_peer(&given) { + Some(peer) if peer == local => None, + Some(peer) => { + // Configured wins over listed. + if self + .named_relays + .get(&peer) + .is_none_or(|known| known.named == Named::Listed || named == Named::Configured) + { + self.named_relays.insert( + peer, + NamedRelay { + at: given.clone(), + given, + named, + }, + ); + } + None + } + None => { + let known = self.finding.contains_key(&given) + || self.named_relays.values().any(|relay| relay.given == given); + if known { + return None; + } + self.finding.insert(given.clone(), named); + Some(given) + } + } + } + + /// The relay named at `given` (without a peer ID) is `peer`: unless it is no longer + /// named (or is this node). Returns whether that's news. + pub(super) fn relay_found(&mut self, given: Multiaddr, peer: PeerId, local: PeerId) -> bool { + let Some(named) = self.finding.remove(&given) else { + return false; + }; + let Ok(at) = given.clone().with_p2p(peer) else { + return false; + }; + if peer == local + || (named == Named::Listed + && self + .named_relays + .get(&peer) + .is_some_and(|known| known.named == Named::Configured)) + { + return false; + } + self.named_relays + .insert(peer, NamedRelay { at, given, named }); + true + } + + /// `relay` didn't grant a slot: if it was named without its peer ID, its address is + /// to be found out again, returned. + pub(super) fn find_again(&mut self, relay: PeerId) -> Option<Multiaddr> { + if self + .named_relays + .get(&relay) + .is_none_or(|known| split_peer(&known.given).is_some()) + { + return None; + } + let known = self.named_relays.remove(&relay)?; + self.finding.insert(known.given.clone(), known.named); + Some(known.given) + } + + /// The relays listed in the document are `listed` now: returns those to find out, + /// None if nothing changed. + pub(super) fn learn_listed( + &mut self, + listed: Vec<Multiaddr>, + local: PeerId, + ) -> Option<Vec<Multiaddr>> { + if listed == self.listed { + return None; + } + self.named_relays + .retain(|_, relay| relay.named != Named::Listed || listed.contains(&relay.given)); + self.finding + .retain(|given, named| *named != Named::Listed || listed.contains(given)); + self.listed = listed.clone(); + Some( + listed + .into_iter() + .filter_map(|given| self.name_relay(given, Named::Listed, local)) + .collect(), + ) + } + + /// The named relays `named` names, by peer, where. + pub(super) fn named(&self, named: Named) -> HashMap<PeerId, Multiaddr> { + self.named_relays + .iter() + .filter(|(_, relay)| relay.named == named) + .map(|(peer, relay)| (*peer, relay.at.clone())) + .collect() + } + + /// Where `peer` may be reached through the relays we know (named, then members that + /// relay): for a peer known by its ID only (a seed `/p2p/<peer ID>`), which may hold + /// a slot on one of them. `local`: this node, never its own relay. + pub(super) fn through_relays(&mut self, peer: PeerId, local: PeerId) -> Vec<Multiaddr> { + let mut relays: Vec<(PeerId, Multiaddr)> = self + .named_relays + .iter() + .map(|(relay, named)| (*relay, named.at.clone())) + .collect(); + relays.sort_by_key(|(_, address)| address.to_string()); + let mut members: Vec<(PeerId, Multiaddr)> = self + .relays + .iter() + .filter(|(relay, _)| !self.named_relays.contains_key(relay)) + .map(|(relay, address)| (*relay, address.clone())) + .collect(); + members.sort_by_key(|(_, address)| address.to_string()); + relays + .into_iter() + .chain(members) + .filter(|(relay, address)| *relay != local && *relay != peer && !self.left_out(address)) + .map(|(_, address)| address.with(Protocol::P2pCircuit).with(Protocol::P2p(peer))) + .take(ADDRESSES_PER_PEER) + .collect() + } + + /// Whether all of `addresses` are known of `peer` (`remember`). + pub(super) fn knows(&self, peer: PeerId, addresses: &[Multiaddr]) -> bool { + let Some(known) = self.addresses.get(&peer) else { + return false; + }; + addresses + .iter() + .all(|address| known.contains(&without_peer(address.clone()))) + } + + /// `peer` is at `address` (without its `/p2p` part): kept first. Not a direct + /// address the peer's identify left out (`identified`): one it no longer has, or + /// one it doesn't publish (`Libp2pConfig::ignore`). Nor one in a range this node + /// leaves out, directly or through a relay there (`left_out`): never dialled, it + /// would only take a place, and be passed on. + /// A dial for `peer` at `address` reached another peer: `address` isn't `peer`'s. + /// Forgotten, and not taken again (`wrong`). + pub(super) fn not_theirs(&mut self, peer: PeerId, address: Multiaddr) { + let address = without_peer(address); + if let Some(known) = self.addresses.get_mut(&peer) { + known.retain(|known| *known != address); + } + let wrong = self.wrong.entry(peer).or_default(); + if wrong.len() < WRONG_PER_PEER { + wrong.insert(address); + } + } + + pub(super) fn remember(&mut self, peer: PeerId, address: Multiaddr) { + let address = without_peer(address); + if !self.addresses.contains_key(&peer) && self.addresses.len() >= MAX_PEERS { + return; + } + if self.left_out(&address) { + return; + } + if self + .wrong + .get(&peer) + .is_some_and(|wrong| wrong.contains(&address)) + { + return; + } + if self + .identified + .get(&peer) + .is_some_and(|said| is_ip(&address) && !said.contains(&address)) + { + return; + } + self.touch(peer); + let addresses = self.addresses.entry(peer).or_default(); + addresses.retain(|known| *known != address); + addresses.insert(0, address); + addresses.truncate(ADDRESSES_PER_PEER); + } + + /// Whether `address` can only be reached over a range left out: ignored + /// (`left_out`), or without a route from here (`routes`), directly or through a + /// relay there. + pub(super) fn left_out(&mut self, address: &Multiaddr) -> bool { + excluded(&self.ignored, &mut self.routes, address) + } + + /// Where `peer` can be dialled: its known addresses, but those through this node, + /// and those left out (`left_out`). What every dial of ours goes to (connecting, + /// upgrading a relayed connection, retrying a hole punch), so none dials into a + /// range left out. + pub(super) fn usable(&mut self, peer: &PeerId, local: PeerId) -> Vec<Multiaddr> { + let Some(known) = self.addresses.get(peer) else { + return Vec::new(); + }; + let mut usable = dialable(known, local); + usable.retain(|address| !excluded(&self.ignored, &mut self.routes, address)); + usable + } + + /// Asks again which ranges have a route (`Routes::recheck`); the addresses in those + /// that lost theirs go. + pub(super) fn recheck_routes(&mut self) { + let lost = self.routes.recheck(); + if lost.is_empty() { + return; + } + for known in self.addresses.values_mut() { + known.retain(|address| { + ip_of(address).is_none_or(|ip| !lost.iter().any(|range| range.contains(ip))) + }); + } + } + + /// The addresses `explore` dials `peer` at, noted as dialled: a node of the swarm + /// connected directly, at addresses of its not dialled so yet, and with `again` + /// not only those at an IP it isn't connected at (an inbound connection's remote + /// port isn't where the peer listens). + pub(super) fn to_explore(&mut self, peer: PeerId, again: bool) -> Vec<Multiaddr> { + if !self.continuum.contains(&peer) + || book_connection(&self.connections, &peer) != Some(Connection::Direct) + { + return Vec::new(); + } + let used: Vec<Option<std::net::IpAddr>> = self + .connections + .values() + .filter(|(connected, relayed, _)| *connected == peer && !relayed) + .map(|(_, _, address)| ip_of(address)) + .collect(); + let tried = self.explored.entry(peer).or_default(); + let candidates: Vec<Multiaddr> = self + .addresses + .get(&peer) + .into_iter() + .flatten() + .filter(|address| is_ip(address) && !left_out(&self.ignored, address)) + .filter(|address| again || !used.contains(&ip_of(address))) + .filter(|address| !tried.contains(*address)) + .cloned() + .collect(); + let candidates: Vec<Multiaddr> = candidates + .into_iter() + .filter(|address| !self.left_out(address)) + .take(EXPLORED_AT_ONCE) + .collect(); + self.explored + .entry(peer) + .or_default() + .extend(candidates.iter().cloned()); + candidates + } + + /// `peer`, connected, identified itself at `listen` (`published`): its direct + /// addresses are those from now on, while connected (`remember`); those through + /// relays and by name stay. Kept in order of `preference`: of a peer with many, + /// the ones that matter stay (`ADDRESSES_PER_PEER`). + pub(super) fn identified(&mut self, peer: PeerId, listen: Vec<Multiaddr>) { + let mut listen: Vec<Multiaddr> = listen.into_iter().map(without_peer).collect(); + listen.sort_by_key(|address| self.preference(address)); + if let Some(known) = self.addresses.get_mut(&peer) { + known.retain(|address| !is_ip(address) || listen.contains(address)); + } + self.identified.insert(peer, listen.clone()); + // Last first: `remember` puts each in front. + for address in listen.into_iter().rev() { + self.remember(peer, address); + } + } +} + +impl Book { + /// Where a peer's `address` comes, first first: in a range `libp2p-prefer` names + /// (by the range's place), then public, then through a relay or by name, then a + /// private one (another LAN, a container bridge). + fn preference(&self, address: &Multiaddr) -> (u8, usize) { + let range = self.preferred_range(address); + if range != usize::MAX { + return (0, range); + } + match ip_of(address).filter(|_| is_ip(address)) { + Some(ip) if is_public(ip) => (1, 0), + Some(_) => (3, 0), + None => (2, 0), + } + } + + /// The place of the first range of `libp2p-prefer` that has `address`; for an + /// address none has, `usize::MAX`: after them. + fn preferred_range(&self, address: &Multiaddr) -> usize { + ip_of(address) + .filter(|_| is_ip(address)) + .and_then(|ip| self.preferred.iter().position(|range| range.contains(ip))) + .unwrap_or(usize::MAX) + } +} + +/// The IP address `address` starts with, if any. +pub(super) fn ip_of(address: &Multiaddr) -> Option<std::net::IpAddr> { + match address.iter().next() { + Some(Protocol::Ip4(ip)) => Some(ip.into()), + Some(Protocol::Ip6(ip)) => Some(ip.into()), + _ => None, + } +} + +/// Whether `ip` can be reached from the internet, as far as its range tells. For IPv6, +/// only global unicast (`2000::/3`) is: other ranges in use are an overlay's +/// (Mycelium's `400::/7`), a translator's (`64:ff9b::/96`) or local, and, ranked as +/// public, would be preferred over the LAN (`preference`). +pub(super) fn is_public(ip: std::net::IpAddr) -> bool { + use std::net::IpAddr; + match ip { + IpAddr::V4(ip) => { + let [a, b, c, _] = ip.octets(); + // 100.64.0.0/10: carrier-grade NAT. + let shared = a == 100 && (b & 0xc0) == 64; + // 198.18.0.0/15: benchmarking; 240.0.0.0/4: reserved (and broadcast). + let benchmarking = a == 198 && (b & 0xfe) == 18; + let reserved = a >= 240; + // 192.0.0.0/24: protocol assignments. + let protocol = a == 192 && b == 0 && c == 0; + !(ip.is_private() + || ip.is_loopback() + || ip.is_link_local() + || ip.is_unspecified() + || ip.is_documentation() + || ip.is_multicast() + || shared + || benchmarking + || reserved + || protocol) + } + IpAddr::V6(ip) => { + let [first, second, ..] = ip.segments(); + // 2001:db8::/32: documentation. + let documentation = first == 0x2001 && second == 0x0db8; + (first & 0xe000) == 0x2000 && !documentation + } + } +} + +/// Whether `address` is a direct one at an IP address (not through a relay, nor a name). +pub(super) fn is_ip(address: &Multiaddr) -> bool { + matches!( + address.iter().next(), + Some(Protocol::Ip4(_) | Protocol::Ip6(_)) + ) && !is_relayed(address) +} + +/// Whether `address` goes through a relay. +pub(super) fn is_relayed(address: &Multiaddr) -> bool { + address + .iter() + .any(|protocol| matches!(protocol, Protocol::P2pCircuit)) +} + +/// The relay `address` goes through, if it goes through one: the peer before +/// `/p2p-circuit`. +pub(super) fn relay_of(address: &Multiaddr) -> Option<PeerId> { + let mut relay = None; + for protocol in address.iter() { + match protocol { + Protocol::P2p(peer) => relay = Some(peer), + Protocol::P2pCircuit => return relay, + _ => {} + } + } + None +} + +/// How `peer` is connected, of `connections` (`Book::connections`): directly if one is +/// direct. +pub(super) fn book_connection( + connections: &HashMap<ConnectionId, (PeerId, bool, Multiaddr)>, + peer: &PeerId, +) -> Option<Connection> { + let mut found = None; + for (connected, relayed, _) in connections.values() { + if connected == peer { + if !relayed { + return Some(Connection::Direct); + } + found = Some(Connection::Relayed); + } + } + found +} + +/// The addresses of `known` this node can dial: not those through itself as the relay +/// (a relay reaches the nodes it relays for only while they are connected to it). +/// `Book::left_out`, by the fields it needs, for where the book is borrowed in parts. +pub(super) fn excluded(ignored: &[IpRange], routes: &mut Routes, address: &Multiaddr) -> bool { + if left_out(ignored, address) { + return true; + } + let at = if is_relayed(address) { + address + .iter() + .take_while(|protocol| !matches!(protocol, Protocol::P2pCircuit)) + .collect::<Multiaddr>() + } else { + address.clone() + }; + ip_of(&at).is_some_and(|ip| !routes.routable(ip)) +} + +pub(super) fn dialable(known: &[Multiaddr], local: PeerId) -> Vec<Multiaddr> { + known + .iter() + .filter(|address| relay_of(address) != Some(local)) + .cloned() + .collect() +} + +/// `address` without the peer ID (`/p2p` part) it ends in, if it ends in one. +pub(super) fn without_peer(mut address: Multiaddr) -> Multiaddr { + if let Some(Protocol::P2p(_)) = address.iter().last() { + address.pop(); + } + address +} + +/// `address` with the peer ID it ends in, if it ends in one. +pub(super) fn split_peer(address: &Multiaddr) -> Option<PeerId> { + match address.iter().last() { + Some(Protocol::P2p(peer)) => Some(peer), + _ => None, + } +} + +pub(super) fn lock(book: &Mutex<Book>) -> MutexGuard<'_, Book> { + book.lock_or_recover() +} diff --git a/src/transport/libp2p/config.rs b/src/transport/libp2p/config.rs new file mode 100644 index 0000000..712f2e2 --- /dev/null +++ b/src/transport/libp2p/config.rs @@ -0,0 +1,177 @@ +//! What the options make of the libp2p transport (`Libp2pConfig`), and their checks. + +use super::*; + +/// What a libp2p transport is made with. +pub(crate) struct Libp2pConfig { + pub(crate) key: Keypair, + /// Multiaddrs to listen on. + pub(crate) listen: Vec<Multiaddr>, + /// Multiaddrs to publish instead of the listen addresses, if any. + pub(crate) announce: Vec<Multiaddr>, + /// Relays to reserve a slot on (multiaddrs; without the relay's peer ID, a connection + /// finds it out). + pub(crate) relays: Vec<Multiaddr>, + /// Relay for other nodes. + pub(crate) relay: bool, + /// Speak continuum (`PROTOCOL`): not on a node that only relays (`--relay-only`), + /// which nobody should take for a node. + pub(crate) continuum: bool, + /// Find peers on the LAN (mDNS). + pub(crate) mdns: bool, + /// Ask the router to forward our listen ports (UPnP), and publish where. + pub(crate) upnp: bool, + /// Address ranges neither published nor dialled, nor connected over (`Ignoring`). + pub(crate) ignore: Vec<IpRange>, + /// Address ranges that matter most, best first (`Book::preference`). + pub(crate) prefer: Vec<IpRange>, + /// Hosts on the internet (names or addresses) whose routes tell whether there is + /// one to the internet at all (`Routes::internet`). + pub(crate) internet_hosts: Vec<String>, + /// Use public infrastructure: the IPFS DHT, public relays. + pub(crate) public: bool, + /// libp2p's in-memory transport instead of TCP and QUIC: for tests. + pub(crate) memory: bool, + /// The node's keepalive interval (`--keepalive`): connections idle for a few of + /// them are closed (`idle_timeout`). + pub(crate) keepalive: Duration, +} + +/// The multiaddrs of option `option`. +pub(super) fn parse_addresses(option: &str, addresses: &[String]) -> Result<Vec<Multiaddr>> { + addresses + .iter() + .map(|address| { + address + .parse::<Multiaddr>() + .with_context(|| format!("{option} {address:?}")) + }) + .collect() +} + +/// The ranges of option `option`. +pub(super) fn parse_ranges(option: &str, ranges: &[String]) -> Result<Vec<IpRange>> { + ranges + .iter() + .map(|range| range.parse().with_context(|| option.to_owned())) + .collect() +} + +/// This transport's options, checked when the configuration is loaded +/// (`config::Settings::check`), so `--print-config` tells a wrong one too: the +/// multiaddrs and ranges parse, and a relay that is no node (`--relay-only`) can be +/// dialled and keeps its peer ID. +pub(crate) fn check(settings: &crate::config::Settings) -> Result<()> { + parse_addresses("libp2p-listen", &settings.libp2p_listen)?; + parse_addresses("libp2p-announce", &settings.libp2p_announce)?; + parse_addresses("libp2p-relays", &settings.libp2p_relays)?; + let ignore = parse_ranges("libp2p-ignore", &settings.libp2p_ignore)?; + let prefer = parse_ranges("libp2p-prefer", &settings.libp2p_prefer)?; + check_preferred( + &ignore, + &prefer, + &settings.libp2p_ignore, + &settings.libp2p_prefer, + )?; + if settings.libp2p_key_create && settings.libp2p_key_file.is_none() { + bail!("libp2p-key-create creates the key at libp2p-key-file; set that too"); + } + if settings.relay_only { + if settings.libp2p_listen.is_empty() { + bail!("relay-only needs libp2p-listen: where nodes reach the relay"); + } + if settings.libp2p_key_file.is_none() { + bail!( + "relay-only needs libp2p-key-file: without it, the relay's peer ID changes with every start and nodes lose it" + ); + } + } + Ok(()) +} + +/// A range both ignored and preferred: refused where all of the preferred one is +/// ignored (it would name nothing usable), a warning where only part of it is (the +/// ignored part stays left out). `ignore_given` and `prefer_given`: as written, for +/// the messages. Ranges nest or are apart, so there's nothing in between. +pub(super) fn check_preferred( + ignore: &[IpRange], + prefer: &[IpRange], + ignore_given: &[String], + prefer_given: &[String], +) -> Result<()> { + for (preferred, preferred_given) in prefer.iter().zip(prefer_given) { + for (ignored, ignored_given) in ignore.iter().zip(ignore_given) { + if ignored.covers(preferred) { + bail!( + "libp2p-prefer {preferred_given:?} is all in libp2p-ignore {ignored_given:?}: nothing there is used" + ); + } + if preferred.covers(ignored) { + warn!( + "libp2p-prefer {preferred_given:?} has libp2p-ignore {ignored_given:?} in it: addresses there stay left out" + ); + } + } + } + Ok(()) +} + +impl Libp2pConfig { + /// The configuration `settings` give; `relay_only`: a relay that is no node + /// (`--relay-only`). + pub(crate) fn from_settings( + settings: &crate::config::Settings, + relay_only: bool, + ) -> Result<Self> { + let parse = parse_addresses; + Ok(Self { + key: Self::load_key( + settings.libp2p_key_file.as_deref(), + settings.libp2p_key_create, + )?, + listen: parse("libp2p-listen", &settings.libp2p_listen)?, + announce: parse("libp2p-announce", &settings.libp2p_announce)?, + relays: parse("libp2p-relays", &settings.libp2p_relays)?, + relay: settings.libp2p_relay || relay_only, + continuum: !relay_only, + mdns: settings.libp2p_mdns, + upnp: settings.libp2p_upnp, + ignore: parse_ranges("libp2p-ignore", &settings.libp2p_ignore)?, + prefer: parse_ranges("libp2p-prefer", &settings.libp2p_prefer)?, + internet_hosts: settings.libp2p_internet_hosts.clone(), + public: settings.libp2p_public, + memory: false, + keepalive: settings.keepalive, + }) + } + + /// The key in `path` (32 raw bytes, an Ed25519 secret key), created there with + /// `create` if there is no file; or a new one each start without a path: the peer + /// ID only says where a node is, so a new one costs nothing but the WHO that finds + /// it again (`transport::identity`). + pub(crate) fn load_key(path: Option<&Path>, create: bool) -> Result<Keypair> { + let Some(path) = path else { + return Ok(Keypair::generate_ed25519()); + }; + let (mut bytes, created) = crate::files::key_file( + path, + "libp2p key", + create, + "libp2p-key-create", + "a new peer ID", + 32, + )?; + if created { + info!(path = %path.display(), "Created a new libp2p key"); + } + if bytes.len() != 32 { + bail!( + "The libp2p key in {} has {} bytes; it has 32", + path.display(), + bytes.len() + ); + } + Keypair::ed25519_from_bytes(&mut bytes) + .map_err(|error| anyhow!("The libp2p key in {}: {error}", path.display())) + } +} diff --git a/src/transport/libp2p/driver.rs b/src/transport/libp2p/driver.rs new file mode 100644 index 0000000..6ef04e7 --- /dev/null +++ b/src/transport/libp2p/driver.rs @@ -0,0 +1,624 @@ +//! The swarm's task (`run`): it owns the swarm, takes `Command`s from the transport, and +//! answers the swarm's events (connections, identify, relays, hole punching, AutoNAT, +//! mDNS, UPnP, the DHT), keeping the book. + +use super::*; + +pub(super) enum Command { + /// Connect to `peer` (at `addresses`) unless connected. + Connect { + peer: PeerId, + addresses: Vec<Multiaddr>, + done: oneshot::Sender<Result<(), String>>, + }, + /// Connect to `address` to find out which peer is there. + Identify { + address: Multiaddr, + done: oneshot::Sender<Result<PeerId, String>>, + }, + /// See whether relays are needed now (one was just learned of). + CheckRelays, + /// Find out which peer relays at `address`, named without the relay's peer ID + /// (`Book::finding`). + FindRelay { address: Multiaddr }, + /// Where `--libp2p-internet-hosts` are now (`Routes::internet`). + Internet(Vec<std::net::IpAddr>), +} + +/// Drives the swarm: dials what `commands` ask for and notes where we listen, until the +/// transport is dropped. `again`: to schedule commands. +pub(super) async fn run( + mut swarm: Swarm<Behaviour>, + mut commands: mpsc::UnboundedReceiver<Command>, + again: mpsc::UnboundedSender<Command>, + book: Arc<Mutex<Book>>, + relaying: Relaying, + stopping: watch::Sender<bool>, +) { + let serving = relaying.serving; + let mut nat = ::libp2p::autonat::NatStatus::Unknown; + // Relays picked, by listener. + let mut picked: HashMap<::libp2p::core::transport::ListenerId, PeerId> = HashMap::new(); + let mut check = tokio::time::interval(RELAY_CHECK); + check.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut dht_bootstrap = tokio::time::interval(DHT_BOOTSTRAP); + dht_bootstrap.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut connecting: HashMap<PeerId, Vec<oneshot::Sender<Result<(), String>>>> = HashMap::new(); + let mut identifying: HashMap< + ConnectionId, + (Multiaddr, oneshot::Sender<Result<PeerId, String>>), + > = HashMap::new(); + let mut prune = tokio::time::interval(PRUNE_EVERY); + prune.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + // Peers that may have several connections, and when to cut them to one. Only the + // side with the smaller peer ID does that, so the two never close different ones. + let mut dedupe_due: HashMap<PeerId, tokio::time::Instant> = HashMap::new(); + // Connections closed as duplicates, until they are: their "Connection closed" says so. + let mut closing_duplicates: std::collections::HashSet<ConnectionId> = + std::collections::HashSet::new(); + let mut dedupe = tokio::time::interval(DEDUPE_EVERY); + dedupe.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + tokio::select! { + _ = dedupe.tick(), if !dedupe_due.is_empty() => { + let now = tokio::time::Instant::now(); + let due: Vec<PeerId> = dedupe_due + .iter() + .filter(|(_, at)| **at <= now) + .map(|(peer, _)| *peer) + .collect(); + for peer in due { + dedupe_due.remove(&peer); + let reserving = picked.values().any(|relay| *relay == peer); + let duplicates = { + let mut book = lock(&book); + let duplicates = book.duplicates(&peer); + if !duplicates.is_empty() && (reserving || book.slots.contains(&peer)) { + book.rehoming.insert(peer, now); + } + duplicates + }; + for connection in duplicates { + if swarm.close_connection(connection) { + closing_duplicates.insert(connection); + } + } + } + } + _ = prune.tick() => { + let forgotten = lock(&book).prune(tokio::time::Instant::now(), |peer| swarm.is_connected(peer)); + if forgotten > 0 { + debug!(forgotten, "Forgot peers nobody mentions any more"); + } + } + _ = dht_bootstrap.tick(), if relaying.public => { + if let Some(dht) = swarm.behaviour_mut().kad.as_mut() + && let Err(error) = dht.bootstrap() + { + warn!(error = %error, "Couldn't join the IPFS DHT"); + } + } + _ = check.tick() => { + let hosts = { + let mut book = lock(&book); + book.recheck_routes(); + book.routes.internet_hosts.clone() + }; + // Names are resolved aside, not to hold up the swarm. + if !hosts.is_empty() { + let again = again.clone(); + tokio::spawn(async move { + let _ = again.send(Command::Internet(resolve_internet_hosts(&hosts).await)); + }); + } + pick_relays(&mut swarm, &book, &relaying, &nat, &mut picked); + retry_punches(&mut swarm, &book, relaying.local); + } + command = commands.recv() => match command { + None => break, + Some(Command::CheckRelays) => { + pick_relays(&mut swarm, &book, &relaying, &nat, &mut picked); + } + Some(Command::Internet(hosts)) => lock(&book).routes.internet(&hosts), + Some(Command::Connect { peer, addresses, done }) => { + if swarm.is_connected(&peer) { + let _ = done.send(Ok(())); + upgrade(&mut swarm, &book, peer, relaying.local); + continue; + } + let waiting = connecting.entry(peer).or_default(); + waiting.push(done); + if waiting.len() > 1 { + continue; // Already dialing. + } + let mut opts = DialOpts::peer_id(peer) + .addresses(addresses) + .condition(PeerCondition::DisconnectedAndNotDialing); + if relaying.public { + // Addresses the DHT knows too. + opts = opts.extend_addresses_through_behaviour(); + } + if let Err(error) = swarm.dial(opts.build()) { + count_dial_failure(&mut lock(&book).counts.dial_failures, &error); + for done in connecting.remove(&peer).into_iter().flatten() { + let _ = done.send(Err(error.to_string())); + } + find_on_dht(&mut swarm, peer); + } + } + Some(Command::FindRelay { address }) => { + let again = again.clone(); + let local = relaying.local; + let book = Arc::clone(&book); + tokio::spawn(async move { + let (done, identified) = oneshot::channel(); + if again + .send(Command::Identify { address: address.clone(), done }) + .is_err() + { + return; + } + match tokio::time::timeout(CONNECT_TIMEOUT, identified).await { + Ok(Ok(Ok(peer))) => { + if lock(&book).relay_found(address.clone(), peer, local) { + debug!(relay = %Addressed(SCHEME, &address), peer = %Addressed(SCHEME, &peer), "Found a named relay"); + let _ = again.send(Command::CheckRelays); + } + } + failed => { + if !lock(&book).finding.contains_key(&address) { + return; // No longer named. + } + let error = match failed { + Ok(Ok(Err(error))) => error, + Ok(_) => "the libp2p swarm stopped".to_owned(), + Err(_) => format!("no answer within {CONNECT_TIMEOUT:?}"), + }; + warn!(relay = %Addressed(SCHEME, &address), error = %error, retry = ?RELAY_RETRY, "Couldn't reach a named relay; trying again later"); + tokio::time::sleep(RELAY_RETRY).await; + let _ = again.send(Command::FindRelay { address }); + } + } + }); + } + Some(Command::Identify { address, done }) => { + let opts = DialOpts::unknown_peer_id().address(address.clone()).build(); + let id = opts.connection_id(); + match swarm.dial(opts) { + Ok(()) => { + identifying.insert(id, (address, done)); + } + Err(error) => { + let _ = done.send(Err(error.to_string())); + } + } + } + }, + event = swarm.select_next_some() => match event { + SwarmEvent::ConnectionEstablished { peer_id, connection_id, endpoint, .. } => { + { + let mut book = lock(&book); + book.touch(peer_id); + let relayed = endpoint.is_relayed(); + book.connections.insert( + connection_id, + (peer_id, relayed, endpoint.get_remote_address().clone()), + ); + if !relayed { + book.upgrades.remove(&peer_id); + book.punching.remove(&peer_id); + } + let to_peer: Vec<ConnectionId> = book + .connections + .iter() + .filter(|(_, (connected, _, _))| *connected == peer_id) + .map(|(id, _)| *id) + .collect(); + if to_peer.len() > 1 { + // A new path to compare: each is judged by its round trips + // from now on, not an old one by its best over hours + // (`Book::ranked`). + for connection in &to_peer { + book.rtts.remove(connection); + } + } + if relaying.local < peer_id && to_peer.len() > 1 { + // Only through relays: maybe a punch under way. + let grace = match book.connection(&peer_id) { + Some(Connection::Relayed) => PUNCH_GRACE, + _ => DUPLICATE_GRACE, + }; + dedupe_due.insert(peer_id, tokio::time::Instant::now() + grace); + } + debug!( + peer = %Addressed(SCHEME, &peer_id), + connection = %connection_id, + dialled = endpoint.is_dialer(), + address = %endpoint.get_remote_address(), + relayed, + "Connection opened" + ); + } + for done in connecting.remove(&peer_id).into_iter().flatten() { + let _ = done.send(Ok(())); + } + if let Some((address, done)) = identifying.remove(&connection_id) { + lock(&book).remember(peer_id, address); + let _ = done.send(Ok(peer_id)); + } + } + SwarmEvent::ConnectionClosed { peer_id, connection_id, cause, .. } => { + { + let mut book = lock(&book); + book.connections.remove(&connection_id); + book.rtts.remove(&connection_id); + if book.connection(&peer_id).is_none() { + book.identified.remove(&peer_id); + book.explored.remove(&peer_id); + } + } + let cause = match cause { + _ if closing_duplicates.remove(&connection_id) => "a duplicate".to_owned(), + // By us, or idle: libp2p doesn't tell which. + None => "closed here".to_owned(), + Some(error) => error.to_string(), + }; + debug!( + peer = %Addressed(SCHEME, &peer_id), + connection = %connection_id, + cause = %cause, + "Connection closed" + ); + } + SwarmEvent::OutgoingConnectionError { connection_id, peer_id, error } => { + count_dial_failure(&mut lock(&book).counts.dial_failures, &error); + // Another peer there: not this one's address, though it said so (a + // NATed peer's LAN address, the same range as ours). + if let (Some(peer), ::libp2p::swarm::DialError::WrongPeerId { obtained, address }) = + (peer_id, &error) + { + debug!( + peer = %Addressed(SCHEME, &peer), + address = %address, + obtained = %obtained, + "Another peer at an address of this one; not dialling it there again" + ); + lock(&book).not_theirs(peer, address.clone()); + } + if let Some((_, done)) = identifying.remove(&connection_id) { + let _ = done.send(Err(describe_dial_error(&error))); + } + if let Some(peer) = peer_id + && !swarm.is_connected(&peer) + && let Some(waiting) = connecting.remove(&peer) + { + for done in waiting { + let _ = done.send(Err(describe_dial_error(&error))); + } + find_on_dht(&mut swarm, peer); + } + // A dial that kept ours from going (`upgrade`) is over. + if let Some(peer) = peer_id { + upgrade(&mut swarm, &book, peer, relaying.local); + } + } + SwarmEvent::NewListenAddr { address, .. } => { + // libp2p listens on every interface it's told to; the ranges left + // out are only kept out of what's published, told and connected. + let left_out = ignored(&lock(&book).ignored, &address); + if left_out { + debug!(address = %address, "Listening, but left out (libp2p-ignore): neither published nor connected over"); + } else { + info!(address = %address, "Listening"); + } + // A relay tells those reserving where it is: its external addresses. + if serving && !left_out && !is_local(&address) && !is_relayed(&address) { + swarm.add_external_address(address.clone()); + } + let new_path = !left_out && !is_local(&address) && !is_relayed(&address); + { + let mut book = lock(&book); + if !book.listening.contains(&address) { + book.listening.push(address); + } + // A new interface (ethernet besides the WLAN, say): every + // address may be reached better now. + if new_path { + book.explored.clear(); + } + } + if new_path { + let peers: Vec<PeerId> = swarm.connected_peers().copied().collect(); + for peer in peers { + explore(&mut swarm, &book, peer, true); + } + } + } + SwarmEvent::ExpiredListenAddr { address, .. } => { + info!(address = %address, "No longer listening"); + if serving { + swarm.remove_external_address(&address); + } + lock(&book).listening.retain(|known| *known != address); + } + SwarmEvent::ListenerClosed { listener_id, addresses, reason } => { + lock(&book) + .listening + .retain(|known| !addresses.contains(known)); + // A relay picked: the next check picks again, another one if this one + // never granted the slot. + if let Some(relay) = picked.remove(&listener_id) { + let refused = match &reason { + Ok(()) => false, + Err(error) => relay_refused(&format!("{error:?}"), &innermost_causes(error)), + }; + let (lost, pause, rehomed) = { + let mut book = lock(&book); + let lost = book.slots.remove(&relay); + // Its connection closed as a duplicate (`duplicates`): again + // at once, over the one kept. + let rehomed = lost + && book.rehoming.remove(&relay).is_some_and(|at| { + tokio::time::Instant::now().saturating_duration_since(at) + < REHOMING_WITHIN + }); + let pause = match (refused, lost) { + (true, _) => Some(RELAY_PAUSE), + (false, true) => None, + (false, false) => Some(RELAY_RETRY), + }; + if lost { + if !rehomed { + book.counts.slots_lost += 1; + } + } else { + book.counts.slots_not_granted += 1; + // Named without its peer ID: maybe another one now. + if let Some(address) = book.find_again(relay) { + let _ = again.send(Command::FindRelay { address }); + } + } + if let Some(pause) = pause { + book.failed_relays.insert(relay, tokio::time::Instant::now() + pause); + } + (lost, pause, rehomed) + }; + // Again at once, or others meanwhile. + let _ = again.send(Command::CheckRelays); + let reason = relay_failure(&reason); + if rehomed { + debug!(relay = %Addressed(SCHEME, &relay), "A duplicate connection to a relay closed with our slot on it; reserving again over the one kept"); + continue; + } + match (lost, pause) { + (true, None) => { + info!(relay = %Addressed(SCHEME, &relay), reason = %reason, "Lost the slot on a relay; reserving again"); + } + (true, Some(pause)) => { + info!(relay = %Addressed(SCHEME, &relay), pause = ?pause, reason = %reason, "Lost the slot on a relay; picking others for a while"); + } + (false, pause) => { + info!( + relay = %Addressed(SCHEME, &relay), + pause = ?pause.unwrap_or_default(), + reason = %reason, + "A relay didn't grant a slot; picking others for a while" + ); + } + } + } + } + SwarmEvent::ListenerError { error, .. } => { + warn!(error = %error, "Listener failed"); + } + SwarmEvent::Behaviour(BehaviourEvent::RelayClient( + ::libp2p::relay::client::Event::ReservationReqAccepted { relay_peer_id, renewal, .. }, + )) => { + { + let mut book = lock(&book); + book.slots.insert(relay_peer_id); + book.failed_relays.remove(&relay_peer_id); + if !renewal { + book.counts.slots_granted += 1; + // Peers that couldn't reach us may now, through it. + if let Some(reachable) = &book.reachable { + reachable.notify_one(); + } + } + } + if !renewal { + info!(relay = %Addressed(SCHEME, &relay_peer_id), "Reachable through a relay"); + } + } + SwarmEvent::Behaviour(BehaviourEvent::Autonat( + ::libp2p::autonat::Event::StatusChanged { old, new }, + )) => { + info!(old = ?old, new = ?new, "Reachability (AutoNAT)"); + { + use ::libp2p::autonat::NatStatus; + let mut book = lock(&book); + (book.reachability, book.public_address) = match &new { + NatStatus::Unknown => (Reachability::Unknown, None), + NatStatus::Private => (Reachability::Private, None), + NatStatus::Public(address) => { + (Reachability::Public, Some(address.clone())) + } + }; + if matches!(new, NatStatus::Public(_)) + && !matches!(old, NatStatus::Public(_)) + && let Some(reachable) = &book.reachable + { + reachable.notify_one(); + } + } + nat = new; + } + SwarmEvent::Behaviour(BehaviourEvent::Relay(event)) => { + { + use ::libp2p::relay::Event; + let mut book = lock(&book); + match &event { + Event::ReservationReqAccepted { src_peer_id, .. } => { + book.slots_here.insert(*src_peer_id); + } + Event::ReservationClosed { src_peer_id } + | Event::ReservationTimedOut { src_peer_id } => { + book.slots_here.remove(src_peer_id); + } + _ => {} + } + } + debug!(event = ?event, "Relay event"); + } + SwarmEvent::Behaviour(BehaviourEvent::Ping(::libp2p::ping::Event { + peer, + connection, + result, + })) => match result { + Ok(rtt) => { + let mut book = lock(&book); + if book.connections.contains_key(&connection) { + let shortest = book.rtts.entry(connection).or_insert(rtt); + *shortest = (*shortest).min(rtt); + } + trace!(peer = %Addressed(SCHEME, &peer), connection = %connection, rtt = ?rtt, "Connection's round trip"); + } + Err(error) => { + trace!(peer = %Addressed(SCHEME, &peer), connection = %connection, error = %error, "Connection's ping failed"); + } + }, + SwarmEvent::Behaviour(BehaviourEvent::Upnp(event)) => match event { + ::libp2p::upnp::Event::NewExternalAddr { + local_addr, + external_addr: address, + } => { + info!(address = %address, to = %local_addr, "The router forwards to us (UPnP)"); + let mut book = lock(&book); + if !book.mapped.contains(&address) { + book.mapped.push(address); + } + } + ::libp2p::upnp::Event::ExpiredExternalAddr { + local_addr, + external_addr: address, + } => { + info!(address = %address, to = %local_addr, "The router no longer forwards to us (UPnP)"); + lock(&book).mapped.retain(|mapped| *mapped != address); + } + ::libp2p::upnp::Event::GatewayNotFound => { + info!("No router that maps ports (UPnP) answered"); + } + ::libp2p::upnp::Event::NonRoutableGateway => { + warn!("The router that maps ports (UPnP) is behind another NAT itself; its mappings don't reach us from outside"); + } + }, + SwarmEvent::Behaviour(BehaviourEvent::Dcutr(event)) => match event.result { + Ok(_) => { + let mut book = lock(&book); + book.counts.hole_punches.succeeded += 1; + book.punching.remove(&event.remote_peer_id); + drop(book); + info!(peer = %Addressed(SCHEME, &event.remote_peer_id), "Connected directly to a peer behind a relay"); + } + Err(error) => { + let retry = { + let mut book = lock(&book); + book.counts.hole_punches.failed += 1; + let failed = book + .punching + .get(&event.remote_peer_id) + .map_or(0, |(failed, _)| *failed); + let retry = PUNCH_RETRIES[failed.min(PUNCH_RETRIES.len() - 1)]; + book.punching.insert( + event.remote_peer_id, + (failed + 1, tokio::time::Instant::now() + retry), + ); + retry + }; + debug!(peer = %Addressed(SCHEME, &event.remote_peer_id), error = %error, retry = ?retry, "Couldn't connect directly to a peer behind a relay; the relay carries on"); + } + }, + SwarmEvent::Behaviour(BehaviourEvent::Identify( + ::libp2p::identify::Event::Received { connection_id, peer_id, info }, + )) => { + { + let mut book = lock(&book); + let listen = published(&info.listen_addrs); + // Every new connection identifies again: logged when it says something new. + if book.knows(peer_id, &listen) { + trace!(peer = %Addressed(SCHEME, &peer_id), connection = %connection_id, "Peer identified again"); + } else { + debug!( + peer = %Addressed(SCHEME, &peer_id), + connection = %connection_id, + listen = ?info.listen_addrs, + observed = %info.observed_addr, + agent = %info.agent_version, + "Peer identified" + ); + } + book.identified(peer_id, listen); + if relaying.public + && info.protocols.contains(&::libp2p::relay::HOP_PROTOCOL_NAME) + && (book.public_relays.len() < MAX_PEERS + || book.public_relays.contains_key(&peer_id)) + && let Some(address) = book + .addresses + .get(&peer_id) + .and_then(|known| known.first()) + .and_then(|address| address.clone().with_p2p(peer_id).ok()) + { + book.public_relays.insert(peer_id, address); + } + if !info.protocols.contains(&PROTOCOL) + && (book.not_nodes.len() < MAX_PEERS + || book.not_nodes.contains_key(&peer_id)) + { + let relays = info.protocols.contains(&::libp2p::relay::HOP_PROTOCOL_NAME); + book.not_nodes.insert(peer_id, relays); + } + if info.protocols.contains(&PROTOCOL) { + book.not_nodes.remove(&peer_id); + book.continuum.insert(peer_id); + if book.lan.remove(&peer_id) { + book.discover(peer_id); + } + } + } + upgrade(&mut swarm, &book, peer_id, relaying.local); + explore(&mut swarm, &book, peer_id, false); + } + SwarmEvent::Behaviour(BehaviourEvent::Mdns( + ::libp2p::mdns::Event::Discovered(found), + )) => { + for (peer, address) in found { + if peer == relaying.local { + continue; + } + debug!(peer = %Addressed(SCHEME, &peer), address = %address, "Peer on the LAN"); + let continuum = { + let mut book = lock(&book); + book.remember(peer, address.clone()); + book.continuum.contains(&peer) + }; + upgrade(&mut swarm, &book, peer, relaying.local); + if continuum { + lock(&book).discover(peer); + } else if lock(&book).lan.len() < MAX_PEERS { + // Connected, identify says whether it speaks continuum. + lock(&book).lan.insert(peer); + let _ = swarm.dial( + DialOpts::peer_id(peer) + .addresses(vec![address]) + .condition(PeerCondition::DisconnectedAndNotDialing) + .build(), + ); + } + } + } + _ => {} + }, + } + } + let _ = stopping.send(true); +} diff --git a/src/transport/libp2p/errors.rs b/src/transport/libp2p/errors.rs new file mode 100644 index 0000000..bb6b090 --- /dev/null +++ b/src/transport/libp2p/errors.rs @@ -0,0 +1,127 @@ +//! What libp2p's errors say, for the log and the figures: dial failures, relay +//! reservations refused or lost. + +use super::*; + +/// A dial error, shortly: for each address tried (without the peer ID the caller names +/// already), the innermost cause, once. libp2p's own text nests each transport's errors +/// in the next ("Multiple dial errors occurred: ..."), repeating the cause at each level. +pub(super) fn describe_dial_error(error: &::libp2p::swarm::DialError) -> String { + use ::libp2p::swarm::DialError; + let DialError::Transport(errors) = error else { + return error.to_string(); + }; + let described: Vec<String> = errors + .iter() + .map(|(address, error)| { + let address = without_peer(address.clone()); + format!("{address}: {}", innermost_causes(error)) + }) + .collect(); + if described.is_empty() { + return error.to_string(); + } + described.join("; ") +} + +/// Why a slot on a relay ended, or was never granted, shortly: libp2p nests the +/// relay's answer several types deep (`Left(Left(Left(Reservation(...))))`), and the +/// common cases deserve a sentence. +pub(super) fn relay_failure(reason: &std::io::Result<()>) -> String { + let Err(error) = reason else { + return "closed".to_owned(); + }; + relay_failure_text(&format!("{error:?}"), &innermost_causes(error)) +} + +/// `relay_failure` on the error's debug form, and its innermost cause as text. +pub(super) fn relay_failure_text(debug: &str, innermost: &str) -> String { + let known = [ + ( + "NoAddressesInReservation", + "the relay gave no address to be reached through it", + ), + ("ResourceLimitExceeded", "the relay has no slot left"), + ("PermissionDenied", "the relay doesn't let us reserve"), + ("NoReservation", "the relay holds no slot for us"), + ( + "connection gated", + "the relay refused the connection (connection gated)", + ), + ( + "UnsupportedProtocol", + "the relay doesn't relay (no circuit relay protocol)", + ), + ("Timeout", "the relay didn't answer in time"), + ]; + if let Some((_, said)) = known.iter().find(|(pattern, _)| debug.contains(pattern)) { + return (*said).to_owned(); + } + // The protocol refused, but only after its first data (`swarm_config`). + if innermost == "Failed" { + return "the relay refused the circuit relay protocol (is it a relay?)".to_owned(); + } + innermost.to_owned() +} + +/// Whether a slot ended, or wasn't granted, because the relay refused it (its error's +/// debug form and innermost cause, as for `relay_failure_text`), rather than because it +/// didn't answer or the connection closed: only then is it worth not asking again for a +/// while. +pub(super) fn relay_refused(debug: &str, innermost: &str) -> bool { + let refusals = [ + "NoAddressesInReservation", + "ResourceLimitExceeded", + "PermissionDenied", + "connection gated", + "UnsupportedProtocol", + "WrongPeerId", + // `Ignoring`: in a range left out, whoever dialled. + "IgnoredAddress", + ]; + refusals.iter().any(|pattern| debug.contains(pattern)) || innermost == "Failed" +} + +/// The innermost of `error`'s causes, each distinct one once: those libp2p lists in a +/// "Multiple dial errors occurred:" text (one per line, ` - <cause>: <its cause>...`), +/// or the last of the source chain. +pub(super) fn innermost_causes(error: &(dyn std::error::Error + 'static)) -> String { + let mut innermost = error; + while let Some(source) = innermost.source() { + innermost = source; + } + let text = innermost.to_string(); + let mut causes: Vec<&str> = Vec::new(); + for line in text.lines() { + let Some(listed) = line.trim().strip_prefix("- ") else { + continue; + }; + // A line ends in the next level's heading when that level lists more. + let cause = listed + .split(": ") + .filter(|part| !part.starts_with("Multiple dial errors")) + .last() + .unwrap_or(listed) + .trim(); + if !cause.is_empty() && !causes.contains(&cause) { + causes.push(cause); + } + } + if causes.is_empty() { + let cause = text.rsplit(": ").next().unwrap_or(&text).trim(); + return cause.to_owned(); + } + causes.join(", ") +} + +/// Counts a dial that failed with `error` (not one that wasn't needed). +pub(super) fn count_dial_failure(failures: &mut DialFailures, error: &::libp2p::swarm::DialError) { + use ::libp2p::swarm::DialError; + match error { + DialError::DialPeerConditionFalse(_) => {} + DialError::Transport(_) => failures.unreachable += 1, + DialError::WrongPeerId { .. } => failures.wrong_peer_id += 1, + DialError::NoAddresses => failures.no_addresses += 1, + _ => failures.other += 1, + } +} diff --git a/src/transport/libp2p/ignore.rs b/src/transport/libp2p/ignore.rs new file mode 100644 index 0000000..eca9342 --- /dev/null +++ b/src/transport/libp2p/ignore.rs @@ -0,0 +1,326 @@ +//! `--libp2p-ignore`: address ranges neither published nor dialled, nor connected over +//! (`Ignoring`, a behaviour that denies connections there), nor told peers by identify +//! (`Identifying`). + +use std::fmt; + +use super::*; + +/// An address range (`10.244.0.0/16`, `42::/16`; a bare address is a range of one), for +/// `Libp2pConfig::ignore`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) struct IpRange { + pub(super) network: std::net::IpAddr, + pub(super) prefix: u8, +} + +impl FromStr for IpRange { + type Err = anyhow::Error; + + fn from_str(text: &str) -> Result<Self> { + let (address, prefix) = match text.split_once('/') { + Some((address, prefix)) => (address, Some(prefix)), + None => (text, None), + }; + let network: std::net::IpAddr = address + .parse() + .with_context(|| format!("{text:?} is no address range (e.g. 10.244.0.0/16)"))?; + let bits = if network.is_ipv4() { 32 } else { 128 }; + let prefix = match prefix { + Some(prefix) => prefix + .parse::<u8>() + .ok() + .filter(|prefix| *prefix <= bits) + .with_context(|| format!("{text:?}: the prefix is 0 to {bits}"))?, + None => bits, + }; + Ok(Self { network, prefix }) + } +} + +impl fmt::Display for IpRange { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}/{}", self.network, self.prefix) + } +} + +impl IpRange { + /// The range of the `prefix` bits of `ip` (the rest zero). + pub(super) fn masked(ip: std::net::IpAddr, prefix: u8) -> Self { + use std::net::IpAddr; + let network = match ip { + IpAddr::V4(ip) => { + let mask = u32::MAX.checked_shl(32 - u32::from(prefix)).unwrap_or(0); + IpAddr::V4((u32::from(ip) & mask).into()) + } + IpAddr::V6(ip) => { + let mask = u128::MAX.checked_shl(128 - u32::from(prefix)).unwrap_or(0); + IpAddr::V6((u128::from(ip) & mask).into()) + } + }; + Self { network, prefix } + } + + /// Whether every address of `other` is in this range too. + pub(super) fn covers(&self, other: &IpRange) -> bool { + self.network.is_ipv4() == other.network.is_ipv4() + && self.prefix <= other.prefix + && self.contains(other.network) + } + + pub(super) fn contains(&self, ip: std::net::IpAddr) -> bool { + use std::net::IpAddr; + match (self.network, ip) { + (IpAddr::V4(network), IpAddr::V4(ip)) => { + let mask = u32::MAX + .checked_shl(32 - u32::from(self.prefix)) + .unwrap_or(0); + u32::from(network) & mask == u32::from(ip) & mask + } + (IpAddr::V6(network), IpAddr::V6(ip)) => { + let mask = u128::MAX + .checked_shl(128 - u32::from(self.prefix)) + .unwrap_or(0); + u128::from(network) & mask == u128::from(ip) & mask + } + _ => false, + } + } +} + +/// Whether `address` is in one of `ranges`: a direct address there (one through a relay +/// is judged by the connection to the relay). +pub(super) fn ignored(ranges: &[IpRange], address: &Multiaddr) -> bool { + if ranges.is_empty() || is_relayed(address) { + return false; + } + let ip = match address.iter().next() { + Some(Protocol::Ip4(ip)) => std::net::IpAddr::V4(ip), + Some(Protocol::Ip6(ip)) => std::net::IpAddr::V6(ip), + _ => return false, + }; + ranges.iter().any(|range| range.contains(ip)) +} + +/// Whether `address` can only be reached over a range of `ranges`: a direct address +/// there, or one through a relay whose own address is there (dialling the relay is +/// refused, `Ignoring`). For what's kept and dialled; connections are judged by +/// `ignored`. +pub(super) fn left_out(ranges: &[IpRange], address: &Multiaddr) -> bool { + if !is_relayed(address) { + return ignored(ranges, address); + } + let relay: Multiaddr = address + .iter() + .take_while(|protocol| !matches!(protocol, Protocol::P2pCircuit)) + .collect(); + ignored(ranges, &without_peer(relay)) +} + +/// Keeps connections off the address ranges `libp2p-ignore` names, whoever dialled: +/// hole punching and identify bring addresses there too, and a peer may dial us there. +pub(super) struct Ignoring { + pub(super) ranges: Vec<IpRange>, +} + +#[derive(Debug, thiserror::Error)] +#[error("{0} is in a range left out (libp2p-ignore)")] +pub(super) struct IgnoredAddress(Multiaddr); + +impl ::libp2p::swarm::NetworkBehaviour for Ignoring { + type ConnectionHandler = ::libp2p::swarm::dummy::ConnectionHandler; + type ToSwarm = std::convert::Infallible; + + fn handle_pending_inbound_connection( + &mut self, + _connection_id: ConnectionId, + _local_addr: &Multiaddr, + remote_addr: &Multiaddr, + ) -> Result<(), ::libp2p::swarm::ConnectionDenied> { + if ignored(&self.ranges, remote_addr) { + return Err(::libp2p::swarm::ConnectionDenied::new(IgnoredAddress( + remote_addr.clone(), + ))); + } + Ok(()) + } + + fn handle_established_inbound_connection( + &mut self, + _connection_id: ConnectionId, + _peer: PeerId, + _local_addr: &Multiaddr, + _remote_addr: &Multiaddr, + ) -> Result<::libp2p::swarm::THandler<Self>, ::libp2p::swarm::ConnectionDenied> { + Ok(::libp2p::swarm::dummy::ConnectionHandler) + } + + /// Dials libp2p makes itself (hole punching, AutoNAT, the DHT) to addresses all in + /// ranges left out are refused before they go out; ours never go there + /// (`Book::usable`). A dial with an address elsewhere too goes, and a connection it + /// makes in a range left out is refused once established (below). + fn handle_pending_outbound_connection( + &mut self, + _connection_id: ConnectionId, + _maybe_peer: Option<PeerId>, + addresses: &[Multiaddr], + _effective_role: ::libp2p::core::Endpoint, + ) -> Result<Vec<Multiaddr>, ::libp2p::swarm::ConnectionDenied> { + if let Some(first) = addresses.first() + && addresses + .iter() + .all(|address| ignored(&self.ranges, address)) + { + return Err(::libp2p::swarm::ConnectionDenied::new(IgnoredAddress( + first.clone(), + ))); + } + Ok(Vec::new()) + } + + fn handle_established_outbound_connection( + &mut self, + _connection_id: ConnectionId, + _peer: PeerId, + addr: &Multiaddr, + _role_override: ::libp2p::core::Endpoint, + _port_use: ::libp2p::core::transport::PortUse, + ) -> Result<::libp2p::swarm::THandler<Self>, ::libp2p::swarm::ConnectionDenied> { + if ignored(&self.ranges, addr) { + return Err(::libp2p::swarm::ConnectionDenied::new(IgnoredAddress( + addr.clone(), + ))); + } + Ok(::libp2p::swarm::dummy::ConnectionHandler) + } + + fn on_swarm_event(&mut self, _event: ::libp2p::swarm::FromSwarm) {} + + fn on_connection_handler_event( + &mut self, + _peer_id: PeerId, + _connection_id: ConnectionId, + event: ::libp2p::swarm::THandlerOutEvent<Self>, + ) { + match event {} + } + + fn poll( + &mut self, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll< + ::libp2p::swarm::ToSwarm<Self::ToSwarm, ::libp2p::swarm::THandlerInEvent<Self>>, + > { + std::task::Poll::Pending + } +} + +/// libp2p's identify, not told our listen addresses, nor external ones (found by UPnP, +/// AutoNAT, a relay), in the ranges `libp2p-ignore` names: peers would remember them, +/// show them, and dial them in vain. Announced addresses (`--libp2p-announce`) are told +/// whatever range they are in. +pub(super) struct Identifying { + pub(super) inner: ::libp2p::identify::Behaviour, + pub(super) ranges: Vec<IpRange>, + pub(super) announced: Vec<Multiaddr>, +} + +impl Identifying { + /// An external address identify isn't told: in an ignored range, and not announced. + fn left_out(&self, address: &Multiaddr) -> bool { + ignored(&self.ranges, address) && !self.announced.contains(address) + } +} + +impl ::libp2p::swarm::NetworkBehaviour for Identifying { + type ConnectionHandler = + <::libp2p::identify::Behaviour as ::libp2p::swarm::NetworkBehaviour>::ConnectionHandler; + type ToSwarm = ::libp2p::identify::Event; + + fn handle_pending_inbound_connection( + &mut self, + connection_id: ConnectionId, + local_addr: &Multiaddr, + remote_addr: &Multiaddr, + ) -> Result<(), ::libp2p::swarm::ConnectionDenied> { + self.inner + .handle_pending_inbound_connection(connection_id, local_addr, remote_addr) + } + + fn handle_established_inbound_connection( + &mut self, + connection_id: ConnectionId, + peer: PeerId, + local_addr: &Multiaddr, + remote_addr: &Multiaddr, + ) -> Result<::libp2p::swarm::THandler<Self>, ::libp2p::swarm::ConnectionDenied> { + self.inner.handle_established_inbound_connection( + connection_id, + peer, + local_addr, + remote_addr, + ) + } + + fn handle_pending_outbound_connection( + &mut self, + connection_id: ConnectionId, + maybe_peer: Option<PeerId>, + addresses: &[Multiaddr], + effective_role: ::libp2p::core::Endpoint, + ) -> Result<Vec<Multiaddr>, ::libp2p::swarm::ConnectionDenied> { + self.inner.handle_pending_outbound_connection( + connection_id, + maybe_peer, + addresses, + effective_role, + ) + } + + fn handle_established_outbound_connection( + &mut self, + connection_id: ConnectionId, + peer: PeerId, + addr: &Multiaddr, + role_override: ::libp2p::core::Endpoint, + port_use: ::libp2p::core::transport::PortUse, + ) -> Result<::libp2p::swarm::THandler<Self>, ::libp2p::swarm::ConnectionDenied> { + self.inner.handle_established_outbound_connection( + connection_id, + peer, + addr, + role_override, + port_use, + ) + } + + fn on_swarm_event(&mut self, event: ::libp2p::swarm::FromSwarm) { + use ::libp2p::swarm::FromSwarm; + match event { + FromSwarm::NewListenAddr(new) if ignored(&self.ranges, new.addr) => {} + FromSwarm::ExpiredListenAddr(expired) if ignored(&self.ranges, expired.addr) => {} + FromSwarm::ExternalAddrConfirmed(confirmed) if self.left_out(confirmed.addr) => {} + FromSwarm::ExternalAddrExpired(expired) if self.left_out(expired.addr) => {} + _ => self.inner.on_swarm_event(event), + } + } + + fn on_connection_handler_event( + &mut self, + peer_id: PeerId, + connection_id: ConnectionId, + event: ::libp2p::swarm::THandlerOutEvent<Self>, + ) { + self.inner + .on_connection_handler_event(peer_id, connection_id, event) + } + + fn poll( + &mut self, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll< + ::libp2p::swarm::ToSwarm<Self::ToSwarm, ::libp2p::swarm::THandlerInEvent<Self>>, + > { + self.inner.poll(cx) + } +} diff --git a/src/transport/libp2p/mod.rs b/src/transport/libp2p/mod.rs new file mode 100644 index 0000000..056b1b2 --- /dev/null +++ b/src/transport/libp2p/mod.rs @@ -0,0 +1,698 @@ +//! libp2p as a transport (`--libp2p-enable`, in builds with the `libp2p` feature): +//! direct connections over TCP (Noise, Yamux) and QUIC, to addresses this node knows. +//! +//! A node's transport identity is its libp2p peer ID; where it is, its listen addresses +//! (multiaddrs ending in the peer ID, `locations`). Seeds are such multiaddrs, or ones +//! without the peer ID, which a connection finds out. Messages go one per stream of the +//! `/continuum/1` protocol: their length (4 bytes, big endian), then the message as +//! `wire` encodes it. A send is done once the stream is written and closed; a connection +//! that can't be made, or a stream that can't be written, refuses it. A stream's protocol +//! goes with its first data, without waiting for the other side to agree +//! (`swarm_config`): opening a stream costs no round trip. +//! +//! Connected peers tell each other where they listen (libp2p's identify protocol), so a +//! node learns the addresses of peers that dialled it. While connected, a peer's +//! identify is all a node goes by for its direct addresses (`Book::identified`): those +//! it no longer has, or no longer publishes, are forgotten, whoever still names them. A node publishes where it can be +//! reached (`local_addresses`): the addresses it announces (`--libp2p-announce`, e.g. a +//! public address forwarded to it), or else those it listens on, loopback left out. +//! +//! A node nobody can dial (behind NAT, or a closed port) can still be reached through a +//! relay: a node of the swarm that relays (`--libp2p-relay`), or one configured +//! (`--libp2p-relays`), on which it reserves a slot; its address through the relay +//! (`<relay>/p2p-circuit/p2p/<it>`) is published with the others. Over a relayed connection, both sides then try to +//! connect directly (libp2p's DCUtR, hole punching); the relay carries the traffic until +//! that works, or for good if it doesn't (punched again now and then, `PUNCH_RETRIES`), +//! within its limits (`RELAY_CIRCUIT_DURATION`, +//! `RELAY_CIRCUIT_BYTES`; a circuit that ends is made again on the next send). Hole +//! punching needs both sides to listen (`HolePunching`: at their LAN addresses too); +//! where only one does, the other dials it directly once it knows where (`upgrade`). +//! +//! Of several connections to one peer (both dialled at once, or AutoNAT dialled back), +//! all but one are closed a few seconds later (`DUPLICATE_GRACE`, `Book::duplicates`), +//! by the side with the smaller peer ID: it keeps the direct one with the shortest round +//! trip (libp2p's ping, on each connection), so a path over the LAN wins over one +//! through a VPN, whichever opened first; between a relay and a node with a slot on it +//! too, where a slot on a connection closed is reserved again over the one kept. A +//! peer connected directly is dialled again where it might be reached better +//! (`explore`): at addresses it tells of later (a new interface: identify pushes them +//! at once), and at all of them when this node gains an interface. Otherwise a +//! connection made over the WLAN would stay there once ethernet comes. +//! +//! A relay says so where it publishes its addresses: next to each, the same address +//! ending in `/p2p-circuit` (where to reserve a slot), which other nodes find in the +//! member list. A node that needs a relay picks up to `AUTO_RELAYS` relays +//! (`relay_check`): those configured first, then those listed in the document +//! (`learn_relays`), then members that relay. It needs one when +//! it has no address others could dial, or when AutoNAT says it can't be dialled: +//! AutoNAT asks connected peers (nodes of the swarm) to dial it back. Once AutoNAT says +//! it can be dialled, it lets the relays it picked go. +//! +//! With `--libp2p-ignore`, address ranges are left out (VPNs): not published, not told +//! peers by identify (`Identifying`), not offered for hole punching, not dialled by us +//! (`Book::usable`) nor by libp2p unless with an address elsewhere too, and connections +//! there refused (`Ignoring`). Ranges this node has no route to are left +//! out alike, as long as it has none (`routes`): without IPv6 to the internet, peers' +//! public IPv6 addresses; and with `--libp2p-internet-hosts`, its own aren't published +//! then. With `--libp2p-prefer`, +//! addresses in the ranges named come first, where only so many are kept +//! (`Book::preference`). +//! +//! With `--libp2p-upnp`, a node asks its router to forward its listen ports (UPnP), and +//! publishes the router's public address for them with its own (`Book::direct`): then +//! it can be dialled from elsewhere, whatever the other side's NAT. +//! +//! With `--libp2p-mdns`, a node finds libp2p peers on its LAN (multicast DNS). Their +//! LAN addresses are noted, and those that turn out to speak continuum (identify says +//! so) are seeds while the node joins (`discovered_seeds`): nodes on one LAN find each +//! other without configured seeds. +//! +//! It uses public libp2p infrastructure only with `--libp2p-public`: then it joins the +//! IPFS DHT (Kademlia, through the IPFS bootstrap nodes), where it finds addresses of +//! peers the member list doesn't have, and public peers that relay; those are picked +//! after members that relay. Public relays keep circuits short (by default 2 minutes and +//! 128 KiB): enough to punch holes through NATs, not to carry a swarm's traffic. Without +//! it, relays are those configured, or members that say they relay. + +use std::collections::HashMap; +use std::path::Path; +use std::str::FromStr; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::time::Duration; + +use ::libp2p::multiaddr::Protocol; +use ::libp2p::swarm::dial_opts::{DialOpts, PeerCondition}; +use ::libp2p::swarm::{ConnectionId, SwarmEvent}; +use ::libp2p::{Multiaddr, PeerId, StreamProtocol, Swarm, identity::Keypair}; +use anyhow::{Context, Result, anyhow, bail}; +use async_trait::async_trait; +use futures::{AsyncReadExt, AsyncWriteExt, StreamExt}; +use tokio::sync::{Semaphore, mpsc, oneshot, watch}; +use tracing::{debug, info, trace, warn}; + +use crate::locks::MutexExt; +use crate::transport::stats::{ + Connections, DialFailures, HolePunches, Libp2pStats, Reachability, RelaySlots, TransportStats, +}; +use crate::transport::{ + Addressed, Connection, EncodedMessage, HandlerFn, MessageHandler, ParsedMessage, Transport, + hand_on_probe, is_probe, wire, +}; + +mod behaviour; +mod book; +mod config; +mod driver; +mod errors; +mod ignore; +mod relays; +mod routes; + +use behaviour::*; +use book::*; +pub(crate) use config::{Libp2pConfig, check}; +use driver::*; +use errors::*; +use ignore::*; +use relays::*; +use routes::*; + +/// The scheme of this transport's addresses (`transport::paths`). +pub(crate) const SCHEME: &str = "libp2p"; + +const PROTOCOL: StreamProtocol = StreamProtocol::new("/continuum/1"); +/// Sends taking this long are logged with where the time went (at debug). +const SLOW_SEND: Duration = Duration::from_millis(5); +/// The largest message read: anyone who can connect can send anything, so this bounds +/// what one stream costs. Sync messages above 16 chunks travel as blobs. +const MAX_MESSAGE: usize = 128 << 20; +/// How long a connection may take to come up. +const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); +/// A connection nobody uses is closed after this at least; neighbours keep theirs busy, +/// with a keepalive within one and a half intervals (`PeerState::take_keepalive_due`), +/// so a long interval lengthens it (`idle_timeout`). +const IDLE_TIMEOUT: Duration = Duration::from_secs(300); + +/// How long a connection may idle before it is closed: `IDLE_TIMEOUT`, or three +/// keepalive intervals if that is longer. +fn idle_timeout(keepalive: Duration) -> Duration { + IDLE_TIMEOUT.max(keepalive * 3) +} +/// Addresses of one dial tried at once (libp2p's default: 8). Hole punching (`HolePunching`) +/// dials every address both sides offer, where they listen (LAN, VPNs, public IPv6) and +/// where others saw them, often more than 8, while the other side punches towards all of +/// them at the same moment: an address dialled only after others failed (each after +/// QUIC's 5 s handshake timeout) misses its punch, and the LAN's is often among the +/// last. Dials of the member list's addresses (`ADDRESSES_PER_PEER`) are unchanged. +const DIAL_CONCURRENCY: std::num::NonZeroU8 = std::num::NonZeroU8::new(32).unwrap(); +/// How long a stream may take to open and to carry a message, before what its size adds +/// (`stream_timeout`). +const STREAM_TIMEOUT: Duration = Duration::from_secs(10); +/// The slowest a message is expected to go, in bytes per second. +const MIN_RATE: u64 = 1 << 20; +/// Inbound streams read at once. +const READS: usize = 32; +/// Addresses kept per peer, and peers with addresses at most. +const ADDRESSES_PER_PEER: usize = 8; +/// Addresses known not to be a peer's (`Book::wrong`), per peer at most. +const WRONG_PER_PEER: usize = 32; +/// Of this node's own, the direct ones (the best first) and those through relays it +/// publishes at most (`local_addresses`): a node listening everywhere has many direct +/// ones, and those through a relay are what reaches it from behind a NAT. +const PUBLISHED_DIRECT: usize = 4; +const PUBLISHED_RELAYED: usize = 2; +const MAX_PEERS: usize = 4096; + +/// A relayed connection (circuit) through this node lasts at most this long and carries +/// at most this much: generous, since the swarm's own relays may carry sync traffic for +/// good where hole punching fails (libp2p's defaults, 2 minutes and 128 KiB, are for +/// the punching only). +const RELAY_CIRCUIT_DURATION: Duration = Duration::from_secs(3600); +const RELAY_CIRCUIT_BYTES: u64 = 1 << 30; +/// Circuits through this node at once, and per peer (either end: a node behind the +/// relay is an end of every circuit to it). libp2p's rate limits on circuits and slots +/// (a burst, then one every minute or two, per peer and per IP address: every node +/// behind one NAT shares one) are left out: they're for public relays serving +/// strangers, and refused circuits used them up too. +const RELAY_CIRCUITS: usize = 1024; +const RELAY_CIRCUITS_PER_PEER: usize = 64; +/// A configured relay that couldn't be reached to find out its peer ID is tried again +/// after this. +const RELAY_RETRY: Duration = Duration::from_secs(30); +/// Relays picked among the members, at most, and how often a node checks whether it +/// needs them. +const AUTO_RELAYS: usize = 2; +const RELAY_CHECK: Duration = Duration::from_secs(30); +/// A relay that refused a slot (`relay_refused`: none left, not allowed, not a relay, +/// another peer at its address: a relay restarted without its libp2p key) isn't picked +/// again for this long. One that didn't grant it for another reason (it doesn't answer, +/// the connection closed) isn't for `RELAY_RETRY`; a slot lost is reserved again at +/// once. +const RELAY_PAUSE: Duration = Duration::from_secs(600); +/// A peer connected only through a relay is dialled directly where it can be reached +/// (`upgrade`): first at once, then after a failure no sooner than this, doubling up to +/// `RELAY_PAUSE`. +const UPGRADE_RETRY: Duration = Duration::from_secs(30); +/// A directly connected peer's addresses dialled at once for a better path (`explore`). +const EXPLORED_AT_ONCE: usize = 4; +/// A peer nobody mentioned for this long (the member list, identify, mDNS, a +/// connection) is forgotten, unless connected: mostly old peer IDs of nodes restarted +/// without their libp2p key, whose new ones the member list has since. Members are +/// mentioned every few seconds (`swarm::topology`). Checked every `PRUNE_EVERY`. +const STALE_AFTER: Duration = Duration::from_secs(600); +const PRUNE_EVERY: Duration = Duration::from_secs(60); +/// Several connections to one peer (both dialled at once, AutoNAT dialling back to see +/// whether it can, or a better path explored) are cut to one, the fastest +/// (`Book::duplicates`), this long after the last one opened: long enough for what it +/// was dialled for, and for each to be pinged since (a new one at once, as it opens; +/// the others within libp2p's ping interval, 15 s). Looked for every `DEDUPE_EVERY`. +const DUPLICATE_GRACE: Duration = Duration::from_secs(20); +/// The same while a peer is connected only through relays: a new relayed connection is +/// a hole punch under way (`PUNCH_RETRIES`), which takes its tries (up to a QUIC +/// handshake's 5 s each, three times) on that connection. +const PUNCH_GRACE: Duration = Duration::from_secs(60); +/// A slot lost this soon after its relay's duplicate connections were closed moved to +/// the one kept (`Book::rehoming`). +const REHOMING_WITHIN: Duration = Duration::from_secs(10); +/// After libp2p's hole punching gave up on a peer (three tries, as the relayed +/// connection opened), it tries again after these, the last one over and over: what +/// failed may have been passing (a NAT's mapping, a busy link). Each time through a new +/// connection through the relay, on which the peer starts the punch (`retry_punches`). +const PUNCH_RETRIES: [Duration; 3] = [ + Duration::from_secs(60), + Duration::from_secs(300), + Duration::from_secs(1800), +]; +const DEDUPE_EVERY: Duration = Duration::from_secs(5); + +/// How long a message of `bytes` may take on a stream. +fn stream_timeout(bytes: usize) -> Duration { + STREAM_TIMEOUT + Duration::from_millis(bytes as u64 * 1000 / MIN_RATE) +} + +pub(crate) struct Libp2pTransport { + local_id: String, + peer_id: PeerId, + control: libp2p_stream::Control, + commands: mpsc::UnboundedSender<Command>, + book: Arc<Mutex<Book>>, + incoming: tokio::sync::Mutex<Option<libp2p_stream::IncomingStreams>>, + stopped: watch::Receiver<bool>, +} + +impl Libp2pTransport { + /// Builds the swarm, listens, and runs it in a task of its own. + pub(crate) fn start(config: Libp2pConfig) -> Result<Self> { + let peer_id = config.key.public().to_peer_id(); + let features = Features::of(&config); + let mut swarm = if config.memory { + memory_swarm(config.key, features, config.ignore.clone())? + } else { + network_swarm(config.key, features, config.ignore.clone())? + }; + for address in config.listen { + swarm + .listen_on(address.clone()) + .with_context(|| format!("Listening on {address}"))?; + } + // Told peers whatever range they are in (`Identifying`). + swarm.behaviour_mut().identify.announced = config.announce.clone(); + for address in &config.announce { + // Identify tells peers these. + swarm.add_external_address(address.clone()); + } + let mut control = swarm.behaviour().stream.new_control(); + let incoming = if config.continuum { + Some( + control + .accept(PROTOCOL) + .map_err(|error| anyhow!("Accepting {PROTOCOL}: {error}"))?, + ) + } else { + None + }; + let (commands, receiver) = mpsc::unbounded_channel(); + let mut book = Book { + announced: config.announce, + ignored: config.ignore.clone(), + preferred: config.prefer.clone(), + routes: Routes::asking(config.internet_hosts.clone()), + serving: config.relay, + ..Book::default() + }; + for relay in config.relays { + if let Some(address) = book.name_relay(relay, Named::Configured, peer_id) { + let _ = commands.send(Command::FindRelay { address }); + } + } + let book = Arc::new(Mutex::new(book)); + let (stopping, stopped) = watch::channel(false); + tokio::spawn(run( + swarm, + receiver, + commands.clone(), + Arc::clone(&book), + Relaying { + serving: config.relay, + public: config.public, + local: peer_id, + }, + stopping, + )); + info!(peer_id = %peer_id, "Transport started"); + Ok(Self { + local_id: peer_id.to_string(), + peer_id, + control, + commands, + book, + incoming: tokio::sync::Mutex::new(incoming), + stopped, + }) + } + + /// Connected to `peer`, if it can be had in time. + async fn connect(&self, peer: PeerId) -> Result<()> { + let addresses = { + let mut book = lock(&self.book); + let known = book.usable(&peer, self.peer_id); + if known.is_empty() { + book.through_relays(peer, self.peer_id) + } else { + known + } + }; + let (done, connected) = oneshot::channel(); + self.commands + .send(Command::Connect { + peer, + addresses, + done, + }) + .map_err(|_| anyhow!("the libp2p swarm stopped"))?; + match tokio::time::timeout(CONNECT_TIMEOUT, connected).await { + Ok(Ok(Ok(()))) => Ok(()), + Ok(Ok(Err(error))) => bail!("couldn't connect to {peer}: {error}"), + Ok(Err(_)) => bail!("the libp2p swarm stopped"), + Err(_) => bail!("connecting to {peer} took longer than {CONNECT_TIMEOUT:?}"), + } + } +} + +/// One message from an inbound stream. +async fn read_message(mut stream: ::libp2p::Stream) -> Result<Vec<u8>> { + let mut length = [0u8; 4]; + tokio::time::timeout(STREAM_TIMEOUT, stream.read_exact(&mut length)) + .await + .map_err(|_| anyhow!("no length within {STREAM_TIMEOUT:?}"))??; + let length = u32::from_be_bytes(length) as usize; + if length > MAX_MESSAGE { + bail!("a message of {length} bytes; at most {MAX_MESSAGE} are read"); + } + let mut body = vec![0u8; length]; + tokio::time::timeout(stream_timeout(length), stream.read_exact(&mut body)) + .await + .map_err(|_| { + anyhow!( + "{length} bytes took longer than {:?}", + stream_timeout(length) + ) + })??; + Ok(body) +} + +#[async_trait] +impl Transport for Libp2pTransport { + fn local_id(&self) -> &str { + &self.local_id + } + + fn is_peer_id(&self, id: &str) -> bool { + PeerId::from_str(id).is_ok() + } + + /// The longest a message may take on a stream. + fn delivery_limit(&self) -> Duration { + stream_timeout(MAX_MESSAGE) + } + + /// Where we can be reached, each with our peer ID: the announced addresses, or else + /// those we listen on (`published`), and those through relays; at most + /// `PUBLISHED_DIRECT` and `PUBLISHED_RELAYED` of them, so the direct ones never + /// crowd out the relayed ones. A relay adds each direct one again, ending in + /// `/p2p-circuit`: where to reserve a slot. + fn local_addresses(&self) -> Vec<String> { + let book = lock(&self.book); + let direct: Vec<Multiaddr> = book + .direct() + .into_iter() + .filter_map(|address| address.with_p2p(self.peer_id).ok()) + .take(PUBLISHED_DIRECT) + .collect(); + let relayed = book + .listening + .iter() + .filter(|address| is_relayed(address)) + .filter_map(|address| address.clone().with_p2p(self.peer_id).ok()) + .take(PUBLISHED_RELAYED); + let offers: Vec<Multiaddr> = if book.serving { + direct + .iter() + .map(|address| address.clone().with(Protocol::P2pCircuit)) + .collect() + } else { + Vec::new() + }; + direct + .into_iter() + .chain(relayed) + .chain(offers) + .map(|address| address.to_string()) + .collect() + } + + /// A peer ID, or a multiaddr ending in one (noted as where that peer is). + fn locate(&self, target: &str) -> Option<String> { + if let Ok(peer) = PeerId::from_str(target) { + return Some(peer.to_string()); + } + let mut address = Multiaddr::from_str(target).ok()?; + // A relay saying where to reserve a slot: `<address>/p2p/<relay>/p2p-circuit`. + if let Some(Protocol::P2pCircuit) = address.iter().last() { + address.pop(); + let relay = split_peer(&address)?; + if relay != self.peer_id { + let new = { + let mut book = lock(&self.book); + // One in a range left out is no use (`Book::left_out`). + let new = book.relays.get(&relay) != Some(&address) && !book.left_out(&address); + if new && (book.relays.len() < MAX_PEERS || book.relays.contains_key(&relay)) { + book.relays.insert(relay, address.clone()); + } + book.remember(relay, address); + new + }; + if new { + let _ = self.commands.send(Command::CheckRelays); + } + } + return Some(relay.to_string()); + } + let peer = split_peer(&address)?; + address.pop(); + // A peer ID only says nothing about where. + if peer != self.peer_id && !address.is_empty() { + lock(&self.book).remember(peer, address); + } + Some(peer.to_string()) + } + + /// Multiaddrs, with or without the relay's peer ID; others are ignored. + fn learn_relays(&self, targets: &[String]) { + let listed = targets + .iter() + .filter_map(|target| Multiaddr::from_str(target).ok()) + .collect(); + let Some(found) = lock(&self.book).learn_listed(listed, self.peer_id) else { + return; + }; + for address in found { + let _ = self.commands.send(Command::FindRelay { address }); + } + let _ = self.commands.send(Command::CheckRelays); + } + + fn connection(&self, target: &str) -> Option<Connection> { + let peer = PeerId::from_str(target).ok()?; + lock(&self.book).connection(&peer) + } + + fn punching(&self, target: &str) -> Option<crate::transport::Punching> { + let peer = PeerId::from_str(target).ok()?; + let (failed, next) = *lock(&self.book).punching.get(&peer)?; + Some(crate::transport::Punching { + failed, + next_in_secs: next + .saturating_duration_since(tokio::time::Instant::now()) + .as_secs(), + }) + } + + fn location(&self, target: &str) -> Option<String> { + let peer = PeerId::from_str(target).ok()?; + let address = lock(&self.book).location(&peer)?.clone(); + Some( + address + .with_p2p(peer) + .unwrap_or_else(|address| address) + .to_string(), + ) + } + + fn stats(&self) -> TransportStats { + TransportStats { + libp2p: Some(lock(&self.book).stats()), + ..TransportStats::default() + } + } + + fn on_seeds_found(&self, found: Arc<tokio::sync::Notify>) { + lock(&self.book).seeds_found = Some(found); + } + + fn on_reachable(&self, reachable: Arc<tokio::sync::Notify>) { + lock(&self.book).reachable = Some(reachable); + } + + /// Nodes found on the LAN (mDNS) that speak continuum. + fn discovered_seeds(&self) -> Vec<String> { + std::mem::take(&mut lock(&self.book).discovered) + .iter() + .map(Multiaddr::to_string) + .collect() + } + + /// Where `target` is, each address with its peer ID. + fn locations(&self, target: &str) -> Vec<String> { + let Ok(peer) = PeerId::from_str(target) else { + return Vec::new(); + }; + lock(&self.book) + .addresses + .get(&peer) + .into_iter() + .flatten() + .filter_map(|address| address.clone().with_p2p(peer).ok()) + .map(|address| address.to_string()) + .collect() + } + + /// A multiaddr ending in a peer ID is that peer; one without is connected to, to + /// find out. A bare peer ID resolves if this transport knows where it is. + async fn resolve_seed(&self, seed: &str) -> Result<String> { + let address = match PeerId::from_str(seed) { + Ok(peer) => Multiaddr::empty().with(Protocol::P2p(peer)), + Err(_) => { + Multiaddr::from_str(seed).with_context(|| format!("{seed} is no multiaddr"))? + } + }; + // A peer ID only (`/p2p/<peer ID>`): reached where it's known, or through the + // relays we know. + if let [Protocol::P2p(peer)] = address.iter().collect::<Vec<_>>()[..] { + let mut book = lock(&self.book); + if book.addresses.contains_key(&peer) + || !book.through_relays(peer, self.peer_id).is_empty() + { + return Ok(peer.to_string()); + } + bail!( + "no address known for libp2p peer {peer}, and no relay to reach it through; \ + give a multiaddr, or a relay (libp2p-relays)" + ); + } + if let Some(peer) = self.locate(seed) { + return Ok(peer); + } + let (done, identified) = oneshot::channel(); + self.commands + .send(Command::Identify { address, done }) + .map_err(|_| anyhow!("the libp2p swarm stopped"))?; + match tokio::time::timeout(CONNECT_TIMEOUT, identified).await { + Ok(Ok(Ok(peer))) => Ok(peer.to_string()), + Ok(Ok(Err(error))) => bail!("couldn't connect to {seed}: {error}"), + Ok(Err(_)) => bail!("the libp2p swarm stopped"), + Err(_) => bail!("connecting to {seed} took longer than {CONNECT_TIMEOUT:?}"), + } + } + + fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> { + wire::encoded(peer, message) + } + + async fn send_encoded(&self, message: &EncodedMessage) -> Result<()> { + let peer = PeerId::from_str(&message.peer) + .with_context(|| format!("{} is no libp2p peer ID", message.peer))?; + let started = std::time::Instant::now(); + self.connect(peer).await?; + // The protocol goes with the first data (`swarm_config`), so a peer that doesn't + // speak it seems to take the message: once identify has told, say so. + if let Some(relays) = lock(&self.book).not_nodes.get(&peer).copied() { + if relays { + bail!( + "{peer} is a relay, not a node: give a node behind it as the seed \ + (<relay>/p2p-circuit/p2p/<peer ID>, or /p2p/<peer ID> with libp2p-relays)" + ); + } + bail!("{peer} doesn't speak continuum"); + } + let connected = started.elapsed(); + let mut control = self.control.clone(); + let body = message.body.clone(); + let write = async move { + let mut stream = control.open_stream(peer, PROTOCOL).await?; + let opened = started.elapsed(); + stream.write_all(&(body.len() as u32).to_be_bytes()).await?; + stream.write_all(&body).await?; + stream.close().await?; + // Kept until the peer has closed its side too. Over QUIC, dropping a stream + // stops its reading half (STOP_SENDING), and the peer's answer to the + // protocol (sent with the first data, `swarm_config`) then fails, and with it + // the stream, before the peer has read the message: identify, which reads + // its answers, works, while every message is lost. Yamux (TCP) doesn't mind. + // Off the send, which is done. + tokio::spawn(async move { + let mut rest = [0u8; 64]; + let _ = tokio::time::timeout(STREAM_TIMEOUT, async { + while let Ok(read) = stream.read(&mut rest).await { + if read == 0 { + break; + } + } + }) + .await; + }); + Ok::<_, anyhow::Error>(opened) + }; + let timeout = stream_timeout(message.body.len()); + let opened = tokio::time::timeout(timeout, write) + .await + .map_err(|_| anyhow!("sending to {peer} took longer than {timeout:?}"))? + .with_context(|| format!("Sending to {peer}"))?; + let took = started.elapsed(); + if took >= SLOW_SEND { + // Where a send's time goes: the swarm task (connecting), the stream's protocol + // negotiation (opening), or the connection (writing). + debug!( + peer = %Addressed(SCHEME, &peer), + kind = ?message.kind, + bytes = message.body.len(), + connect_ms = connected.as_secs_f64() * 1000.0, + open_ms = (opened - connected).as_secs_f64() * 1000.0, + write_ms = (took - opened).as_secs_f64() * 1000.0, + "Slow send" + ); + } + Ok(()) + } + + /// Reads inbound streams (`READS` at once) and handles their messages one at a + /// time, in the order they were read; PINGs and PONGs at once (`is_probe`). + async fn start_consumer(&self, handler: MessageHandler) -> Result<()> { + let Some(mut incoming) = self.incoming.lock().await.take() else { + bail!("libp2p consumer already started, or this node only relays"); + }; + let handler: Arc<HandlerFn> = Arc::from(handler); + let (sender, mut received) = mpsc::channel::<(PeerId, ParsedMessage, usize)>(READS); + let reads = Arc::new(Semaphore::new(READS)); + let probes = Arc::clone(&handler); + tokio::spawn(async move { + while let Some((peer, stream)) = incoming.next().await { + let Ok(permit) = Arc::clone(&reads).acquire_owned().await else { + break; + }; + let sender = sender.clone(); + let probes = Arc::clone(&probes); + tokio::spawn(async move { + match read_message(stream).await { + Ok(body) => match wire::decode(&body) { + Ok(message) if is_probe(&message) => { + hand_on_probe( + SCHEME, + &probes, + peer.to_string(), + message, + body.len(), + ); + } + Ok(message) => { + let _ = sender.send((peer, message, body.len())).await; + } + Err(error) => { + debug!(peer = %Addressed(SCHEME, &peer), error = %error, "Dropping a message that doesn't decode"); + } + }, + Err(error) => { + debug!(peer = %Addressed(SCHEME, &peer), error = %format!("{error:#}"), "Dropping an unreadable stream"); + } + } + drop(permit); + }); + } + }); + while let Some((peer, message, payload_len)) = received.recv().await { + if let Err(error) = handler(peer.to_string(), message, payload_len).await { + warn!(peer = %Addressed(SCHEME, &peer), error = %format!("{error:#}"), "Failed to process inbound message"); + } + } + Ok(()) + } + + async fn wait_for_exit(&self) -> std::io::Result<std::process::ExitStatus> { + let mut stopped = self.stopped.clone(); + let _ = stopped.wait_for(|stopped| *stopped).await; + Err(std::io::Error::other("the libp2p swarm stopped")) + } +} + +#[cfg(test)] +mod tests; diff --git a/src/transport/libp2p/relays.rs b/src/transport/libp2p/relays.rs new file mode 100644 index 0000000..1a57635 --- /dev/null +++ b/src/transport/libp2p/relays.rs @@ -0,0 +1,271 @@ +//! Reaching peers past NATs: picking relays and keeping slots on them, upgrading +//! relayed connections to direct ones, retrying hole punches, and finding peers on the +//! DHT. + +use super::*; + +/// How a node relates to relays. +pub(super) struct Relaying { + /// It relays for others. + pub(super) serving: bool, + /// Public relays may be picked, after members. + pub(super) public: bool, + pub(super) local: PeerId, +} + +/// The relays to pick among `offers`, if this node needs relays, keeping `using`: a +/// node needs them when it has no address others could dial (`direct`), or AutoNAT says +/// it can't be dialled; up to `AUTO_RELAYS`, in an order of its own (the lowest hash of +/// both peer IDs), so that not every node picks the same. +pub(super) fn relay_check( + local: PeerId, + direct: bool, + nat: &::libp2p::autonat::NatStatus, + offers: &HashMap<PeerId, Multiaddr>, + using: &[PeerId], +) -> Vec<(PeerId, Multiaddr)> { + use ::libp2p::autonat::NatStatus; + let needed = match nat { + NatStatus::Private => true, + NatStatus::Public(_) => false, + NatStatus::Unknown => !direct, + }; + if !needed || using.len() >= AUTO_RELAYS { + return Vec::new(); + } + let score = |relay: &PeerId| { + use std::hash::{Hash, Hasher}; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + (local, relay).hash(&mut hasher); + hasher.finish() + }; + let mut candidates: Vec<(PeerId, Multiaddr)> = offers + .iter() + .filter(|(relay, _)| **relay != local && !using.contains(relay)) + .map(|(relay, address)| (*relay, address.clone())) + .collect(); + candidates.sort_by_key(|(relay, _)| score(relay)); + candidates.truncate(AUTO_RELAYS - using.len()); + candidates +} + +/// Reserves slots on the relays `relay_check` picks (configured ones first, then listed +/// ones, members, public ones), and lets those picked go once AutoNAT says this node can +/// be dialled. +pub(super) fn pick_relays( + swarm: &mut Swarm<Behaviour>, + book: &Mutex<Book>, + relaying: &Relaying, + nat: &::libp2p::autonat::NatStatus, + picked: &mut HashMap<::libp2p::core::transport::ListenerId, PeerId>, +) { + if nat.is_public() && !picked.is_empty() { + info!("Reachable directly (AutoNAT); letting the relays picked go"); + let mut book = lock(book); + for (listener, relay) in std::mem::take(picked) { + swarm.remove_listener(listener); + book.slots.remove(&relay); + } + return; + } + let (direct, configured, listed, members, public) = { + let mut book = lock(book); + let now = tokio::time::Instant::now(); + book.failed_relays.retain(|_, until| now < *until); + let failed = book.failed_relays.clone(); + // Not paused, and not in a range left out (ignored, or without a route): those + // would only be denied or fail, again and again. + let usable = |book: &mut Book, offers: HashMap<PeerId, Multiaddr>| { + offers + .into_iter() + .filter(|(relay, address)| !failed.contains_key(relay) && !book.left_out(address)) + .collect::<HashMap<PeerId, Multiaddr>>() + }; + let public = if relaying.public { + let offers = book.public_relays.clone(); + usable(&mut book, offers) + } else { + HashMap::new() + }; + let (configured, listed, members) = ( + book.named(Named::Configured), + book.named(Named::Listed), + book.relays.clone(), + ); + ( + !book.direct().is_empty(), + usable(&mut book, configured), + usable(&mut book, listed), + usable(&mut book, members), + public, + ) + }; + let mut using: Vec<PeerId> = picked.values().copied().collect(); + // Each kind for what the ones before left. + for (kind, offers) in [ + (Named::Configured.as_str(), configured), + (Named::Listed.as_str(), listed), + ("member", members), + ("public", public), + ] { + for (relay, address) in relay_check(relaying.local, direct, nat, &offers, &using) { + using.push(relay); + match swarm.listen_on(address.clone().with(Protocol::P2pCircuit)) { + Ok(listener) => { + info!(relay = %Addressed(SCHEME, &relay), address = %address, kind, "Picked a relay"); + picked.insert(listener, relay); + } + Err(error) => { + debug!(relay = %Addressed(SCHEME, &relay), kind, error = %error, "Couldn't use a relay"); + } + } + } + } +} + +/// Dials `peer` directly if it's connected only through a relay and is known elsewhere +/// too (the member list, identify, mDNS), since the first connection, made through a +/// relay (a relayed seed), would otherwise be kept for good: a peer connected isn't +/// dialled again, and hole punching needs both sides to listen. The direct connection, +/// once open, wins over the relayed one (`Book::duplicates`). A dial refused because +/// another to the peer is under way (hole punching's, say) doesn't count as a try: the +/// next comes when that one fails (`driver`), not `UPGRADE_RETRY` later. +pub(super) fn upgrade( + swarm: &mut Swarm<Behaviour>, + book: &Mutex<Book>, + peer: PeerId, + local: PeerId, +) { + let (direct, now, wait) = { + let mut book = lock(book); + if book.connection(&peer) != Some(Connection::Relayed) { + return; + } + let mut direct = book.usable(&peer, local); + direct.retain(|address| !is_relayed(address)); + if direct.is_empty() { + return; + } + let now = tokio::time::Instant::now(); + let (due, wait) = book + .upgrades + .get(&peer) + .copied() + .unwrap_or((now, UPGRADE_RETRY)); + if now < due { + return; + } + (direct, now, wait) + }; + let dialled = swarm.dial( + DialOpts::peer_id(peer) + .addresses(direct.clone()) + .condition(PeerCondition::NotDialing) + .build(), + ); + match dialled { + Ok(()) => { + debug!(peer = %Addressed(SCHEME, &peer), addresses = ?direct, "Connected only through a relay; dialling directly"); + lock(book) + .upgrades + .insert(peer, (now + wait, (wait * 2).min(RELAY_PAUSE))); + } + Err(error) => { + debug!(peer = %Addressed(SCHEME, &peer), error = %error, "Connected only through a relay; can't dial directly yet"); + } + } +} + +/// Dials `peer`, connected directly, at its other addresses, in case one is a better +/// path: one that came since (a new interface of its, identify tells), or, `again` +/// (one of ours came), every one, as our side may reach them better now, the one in +/// use too. Each once while connected (`Book::explored`); a connection that opens is +/// pinged at once, and of several the fastest is kept (`Book::duplicates`). Without +/// it, a connection stays on the path it was made on (the WLAN) when a better one comes +/// (ethernet): a connected peer isn't dialled again. +pub(super) fn explore(swarm: &mut Swarm<Behaviour>, book: &Mutex<Book>, peer: PeerId, again: bool) { + let candidates = lock(book).to_explore(peer, again); + if candidates.is_empty() { + return; + } + debug!(peer = %Addressed(SCHEME, &peer), addresses = ?candidates, "Connected directly; dialling other addresses, for a better path"); + if let Err(error) = swarm.dial( + DialOpts::peer_id(peer) + .addresses(candidates) + .condition(PeerCondition::Always) + .build(), + ) { + debug!(peer = %Addressed(SCHEME, &peer), error = %error, "Couldn't dial for a better path"); + } +} + +/// Tries hole punching again with the peers it failed with that are due (`punching`). +/// libp2p punches as a relayed connection opens, started by the side that was dialled +/// through the relay; that side is where a punch's failure is told, so it is this one. +/// A new relayed connection starts a new punch: we dial the peer through a relay if it +/// has a slot on one; else we close our relayed connections to it, and its next message +/// (a probe, within a minute) dials us through ours again. Peers connected directly by +/// now are dropped; those not connected now are kept, failures and all. +pub(super) fn retry_punches(swarm: &mut Swarm<Behaviour>, book: &Mutex<Book>, local: PeerId) { + let now = tokio::time::Instant::now(); + let mut close: Vec<ConnectionId> = Vec::new(); + let due: Vec<(PeerId, usize, Vec<Multiaddr>)> = { + let mut book = lock(book); + let book = &mut *book; + let mut due = Vec::new(); + book.punching.retain(|peer, (failed, next)| { + match book_connection(&book.connections, peer) { + Some(Connection::Direct) => return false, + // Kept, failures and all: one we closed to have it dial anew comes back. + None => return true, + Some(Connection::Relayed) if now < *next => return true, + Some(Connection::Relayed) => {} + } + // As `Book::usable`, by parts: `punching` is borrowed. + let through: Vec<Multiaddr> = book + .addresses + .get(peer) + .map(|known| dialable(known, local)) + .unwrap_or_default() + .into_iter() + .filter(|address| { + is_relayed(address) && !excluded(&book.ignored, &mut book.routes, address) + }) + .collect(); + // Until its punch fails again (or the peer tries from its side). + *next = now + PUNCH_RETRIES[(*failed).min(PUNCH_RETRIES.len() - 1)]; + if through.is_empty() { + close.extend( + book.connections + .iter() + .filter(|(_, (connected, relayed, _))| connected == peer && *relayed) + .map(|(id, _)| *id), + ); + } else { + due.push((*peer, *failed, through)); + } + true + }); + due + }; + for connection in close { + debug!(connection = %connection, "Hole punching again: closing a relayed connection, for the peer to dial anew"); + swarm.close_connection(connection); + } + for (peer, failed, through) in due { + debug!(peer = %Addressed(SCHEME, &peer), failed, "Hole punching again, through a new relayed connection"); + let _ = swarm.dial( + DialOpts::peer_id(peer) + .addresses(through) + .condition(PeerCondition::Always) + .build(), + ); + } +} + +/// Asks the DHT (with `--libp2p-public`) where `peer` is, for the next attempt. +pub(super) fn find_on_dht(swarm: &mut Swarm<Behaviour>, peer: PeerId) { + if let Some(dht) = swarm.behaviour_mut().kad.as_mut() { + dht.get_closest_peers(peer); + } +} diff --git a/src/transport/libp2p/routes.rs b/src/transport/libp2p/routes.rs new file mode 100644 index 0000000..3108f0c --- /dev/null +++ b/src/transport/libp2p/routes.rs @@ -0,0 +1,203 @@ +//! Addresses without a route from here, left out as if `--libp2p-ignore` named them: a +//! node without IPv6 to the internet doesn't keep, show or dial peers' public IPv6 +//! addresses; one without a route to another LAN, that LAN's. And with +//! `--libp2p-internet-hosts`, whether there's a route to the internet at all, per +//! family: a node without one doesn't publish its own public addresses of that family. +//! +//! Whether there is a route is asked of the operating system: a UDP socket "connected" +//! to the address (no packet is sent) fails where the routing table has none. Asked +//! per range (`ROUTE_PREFIX_V4`, `ROUTE_PREFIX_V6`) of the addresses heard of, and +//! asked again every `ROUTES_RECHECK`: a range gets a route (IPv6 comes up), and its +//! addresses are taken again as they're heard of next (the member list, identify); +//! one loses its route, and the addresses there are dropped. + +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, UdpSocket}; + +use super::*; + +/// The ranges routes are asked for: an IPv4 /24 or an IPv6 /64, as far as a route +/// usually differs within them. +const ROUTE_PREFIX_V4: u8 = 24; +const ROUTE_PREFIX_V6: u8 = 64; +/// How long an answer is taken for one. +const ROUTES_RECHECK: Duration = Duration::from_secs(30); +/// Ranges remembered at most. +const MAX_RANGES: usize = 1024; +/// How long resolving `--libp2p-internet-hosts` may take, each time. +const RESOLVE_WITHIN: Duration = Duration::from_secs(5); + +/// Whether the routing table has a route to `ip`. +pub(super) fn has_route(ip: IpAddr) -> bool { + let any: SocketAddr = match ip { + IpAddr::V4(_) => (Ipv4Addr::UNSPECIFIED, 0).into(), + IpAddr::V6(_) => (Ipv6Addr::UNSPECIFIED, 0).into(), + }; + UdpSocket::bind(any) + .and_then(|socket| socket.connect((ip, 9))) + .is_ok() +} + +/// The answers, by range, and the question asked (`has_route`, or a stand-in in tests). +pub(super) struct Routes { + known: HashMap<IpRange, (bool, tokio::time::Instant)>, + /// Whether there is a route to the internet, per family (IPv4, IPv6), as last + /// asked (`internet`): own public addresses are published only then. True where + /// nobody asks (no `--libp2p-internet-hosts` of that family); false until the first + /// answer where someone does (resolving takes up to `RESOLVE_WITHIN`), so the first + /// member list entry doesn't list addresses nobody can reach. + internet: [bool; 2], + /// `--libp2p-internet-hosts`: names or addresses on the internet, whose routes + /// tell whether there is one to the internet at all. + pub(super) internet_hosts: Vec<String>, + probe: fn(IpAddr) -> bool, +} + +impl Default for Routes { + fn default() -> Self { + // Tests don't depend on the host's routes; those that care set `probe`. + #[cfg(test)] + let probe: fn(IpAddr) -> bool = |_| true; + #[cfg(not(test))] + let probe: fn(IpAddr) -> bool = has_route; + Self::with_probe(probe) + } +} + +/// The range of `ip` routes are asked for. +fn range_of(ip: IpAddr) -> IpRange { + let prefix = if ip.is_ipv4() { + ROUTE_PREFIX_V4 + } else { + ROUTE_PREFIX_V6 + }; + IpRange::masked(ip, prefix) +} + +impl Routes { + /// With `--libp2p-internet-hosts` `internet_hosts`. + pub(super) fn asking(internet_hosts: Vec<String>) -> Self { + let unknown = !internet_hosts.is_empty(); + Self { + internet: [!unknown, !unknown], + internet_hosts, + ..Self::default() + } + } + + pub(super) fn with_probe(probe: fn(IpAddr) -> bool) -> Self { + Self { + known: HashMap::new(), + internet: [true, true], + internet_hosts: Vec::new(), + probe, + } + } + + /// Whether `ip` has a route from here: asked if not known lately, told when the + /// answer is news (debug). + pub(super) fn routable(&mut self, ip: IpAddr) -> bool { + let range = range_of(ip); + let now = tokio::time::Instant::now(); + if let Some((routable, at)) = self.known.get(&range) + && now.saturating_duration_since(*at) < ROUTES_RECHECK + { + return *routable; + } + let routable = (self.probe)(ip); + let before = self.known.get(&range).map(|(routable, _)| *routable); + if before.is_none() && self.known.len() >= MAX_RANGES { + self.known.clear(); + } + self.known.insert(range, (routable, now)); + announce(range, before, routable); + routable + } + + /// Asks again for the ranges known; returns those that lost their route, whose + /// addresses are to go. + pub(super) fn recheck(&mut self) -> Vec<IpRange> { + let now = tokio::time::Instant::now(); + let mut lost = Vec::new(); + for (range, (routable, at)) in &mut self.known { + let now_routable = (self.probe)(range.network); + if *routable != now_routable { + announce(*range, Some(*routable), now_routable); + if !now_routable { + lost.push(*range); + } + } + *routable = now_routable; + *at = now; + } + lost + } + + /// Whether there's a route to the internet, per family, by the addresses + /// `--libp2p-internet-hosts` resolved to (`resolve_internet_hosts`): one of them + /// with a route is enough. A family none of them has isn't judged (taken as having + /// one). + pub(super) fn internet(&mut self, hosts: &[IpAddr]) { + for (family, name) in ["IPv4", "IPv6"].into_iter().enumerate() { + let of_family: Vec<IpAddr> = hosts + .iter() + .copied() + .filter(|ip| usize::from(ip.is_ipv6()) == family) + .collect(); + let routable = of_family.is_empty() || of_family.iter().any(|ip| (self.probe)(*ip)); + if routable != self.internet[family] { + if routable { + debug!( + family = name, + "A route to the internet again: own public addresses published again" + ); + } else { + debug!( + family = name, + "No route to the internet: own public addresses not published" + ); + } + } + self.internet[family] = routable; + } + } + + /// Whether an own address `ip` is worth publishing: not a public one of a family + /// with no route to the internet. + pub(super) fn publishable(&self, ip: IpAddr) -> bool { + !is_public(ip) || self.internet[usize::from(ip.is_ipv6())] + } +} + +fn announce(range: IpRange, before: Option<bool>, routable: bool) { + match (before, routable) { + (None | Some(true), false) => { + debug!(range = %range, "No route to an address range: its addresses left out until there is"); + } + (Some(false), true) => { + debug!(range = %range, "A route to an address range again: its addresses taken again"); + } + _ => {} + } +} + +/// The addresses `hosts` (names or addresses) resolve to, within `RESOLVE_WITHIN`; a +/// name that doesn't resolve is left out (told at debug level). +pub(super) async fn resolve_internet_hosts(hosts: &[String]) -> Vec<IpAddr> { + let mut addresses = Vec::new(); + for host in hosts { + if let Ok(ip) = host.parse::<IpAddr>() { + addresses.push(ip); + continue; + } + match tokio::time::timeout(RESOLVE_WITHIN, tokio::net::lookup_host((host.as_str(), 0))) + .await + { + Ok(Ok(found)) => addresses.extend(found.map(|address| address.ip())), + Ok(Err(error)) => { + debug!(host = %host, error = %error, "An internet host didn't resolve") + } + Err(_) => debug!(host = %host, "An internet host didn't resolve in time"), + } + } + addresses +} diff --git a/src/transport/libp2p/tests.rs b/src/transport/libp2p/tests.rs new file mode 100644 index 0000000..0f52e23 --- /dev/null +++ b/src/transport/libp2p/tests.rs @@ -0,0 +1,1026 @@ +use std::sync::atomic::{AtomicU64, Ordering}; + +use super::*; +use crate::document::replica::Epoch; +use crate::transport::{HeadsSummary, Status}; + +/// Memory ports are process-wide. +static NEXT_PORT: AtomicU64 = AtomicU64::new(40_000); + +/// A node on the in-memory transport, listening on a port of its own (or nowhere), +/// and its address with its peer ID (or just its peer ID). +fn memory_node_with( + listen: bool, + relays: Vec<Multiaddr>, + relay: bool, +) -> (Libp2pTransport, Multiaddr) { + let port = NEXT_PORT.fetch_add(1, Ordering::Relaxed); + let key = Keypair::generate_ed25519(); + let address: Multiaddr = format!("/memory/{port}").parse().unwrap(); + let peer = key.public().to_peer_id(); + let transport = Libp2pTransport::start(Libp2pConfig { + key, + listen: if listen { + vec![address.clone()] + } else { + Vec::new() + }, + announce: Vec::new(), + relays, + relay, + continuum: true, + mdns: false, + upnp: false, + ignore: Vec::new(), + prefer: Vec::new(), + internet_hosts: Vec::new(), + public: false, + memory: true, + keepalive: Duration::from_secs(60), + }) + .unwrap(); + let address = if listen { + address.with_p2p(peer).unwrap() + } else { + Multiaddr::empty().with(Protocol::P2p(peer)) + }; + (transport, address) +} + +fn memory_node() -> (Libp2pTransport, Multiaddr) { + memory_node_with(true, Vec::new(), false) +} + +fn status() -> ParsedMessage { + ParsedMessage::Status(Status { + heads: HeadsSummary::of(Vec::new()), + epoch: Epoch::GENESIS, + epoch_seq: 3, + ready: true, + syncs_with_you: false, + leaving: false, + }) +} + +/// Starts `transport`'s consumer, forwarding what it receives. +fn received(transport: Arc<Libp2pTransport>) -> mpsc::UnboundedReceiver<(String, ParsedMessage)> { + let (sender, receiver) = mpsc::unbounded_channel(); + tokio::spawn(async move { + transport + .start_consumer(Box::new(move |from, message, _| { + let sender = sender.clone(); + Box::pin(async move { + let _ = sender.send((from, message)); + Ok(()) + }) + })) + .await + }); + receiver +} + +#[tokio::test] +async fn messages_both_ways_over_one_connection() { + let (a, a_address) = memory_node(); + let (b, _) = memory_node(); + let (a, b) = (Arc::new(a), Arc::new(b)); + let mut at_a = received(Arc::clone(&a)); + let mut at_b = received(Arc::clone(&b)); + + assert_eq!( + b.resolve_seed(&a_address.to_string()).await.unwrap(), + a.local_id() + ); + b.send_message(a.local_id(), &status()).await.unwrap(); + let (from, message) = at_a.recv().await.unwrap(); + assert_eq!(from, b.local_id()); + assert!(matches!(message, ParsedMessage::Status(status) if status.epoch_seq == 3)); + + // a knows no address of b, but b's connection is there. + a.send_message(b.local_id(), &status()).await.unwrap(); + let (from, _) = at_b.recv().await.unwrap(); + assert_eq!(from, a.local_id()); + + assert_eq!(a.connection(b.local_id()), Some(Connection::Direct)); + assert_eq!(b.connection(a.local_id()), Some(Connection::Direct)); + let stats = a.stats().libp2p.unwrap(); + assert!(stats.connections.direct >= 1); + assert_eq!(stats.connections.relayed, 0); +} + +#[tokio::test] +async fn a_seed_without_a_peer_id_is_connected_to_to_find_out() { + let (a, a_address) = memory_node(); + let (b, _) = memory_node(); + let mut bare = a_address.clone(); + bare.pop(); + assert_eq!( + b.resolve_seed(&bare.to_string()).await.unwrap(), + a.local_id() + ); + // And b now knows where a is. + assert_eq!(b.locations(a.local_id()), [a_address.to_string()]); +} + +#[tokio::test] +async fn listen_addresses_carry_the_peer_id() { + let (a, a_address) = memory_node(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while a.local_addresses().is_empty() && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert_eq!(a.local_addresses(), [a_address.to_string()]); + assert_eq!( + a.locate(&a_address.to_string()).as_deref(), + Some(a.local_id()) + ); +} + +/// Waits up to 5 s for `condition`. +async fn eventually(mut condition: impl FnMut() -> bool) -> bool { + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while !condition() { + if tokio::time::Instant::now() >= deadline { + return false; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + true +} + +/// A node learns where a peer that dialled it listens (identify), and can then +/// pass it on. +#[tokio::test] +async fn a_dialled_node_learns_where_the_dialler_listens() { + let (a, a_address) = memory_node(); + let (b, b_address) = memory_node(); + let (a, b) = (Arc::new(a), Arc::new(b)); + let _at_a = received(Arc::clone(&a)); + b.resolve_seed(&a_address.to_string()).await.unwrap(); + b.send_message(a.local_id(), &status()).await.unwrap(); + let b_id = b.local_id().to_owned(); + assert!( + eventually(|| a.locations(&b_id) == [b_address.to_string()]).await, + "a knows {:?} of b", + a.locations(&b_id) + ); +} + +#[tokio::test] +async fn announced_addresses_are_published_instead() { + let key = Keypair::generate_ed25519(); + let peer = key.public().to_peer_id(); + let port = NEXT_PORT.fetch_add(1, Ordering::Relaxed); + let announced: Multiaddr = "/ip4/203.0.113.7/tcp/9661".parse().unwrap(); + let transport = Libp2pTransport::start(Libp2pConfig { + key, + listen: vec![format!("/memory/{port}").parse().unwrap()], + announce: vec![announced.clone()], + relays: Vec::new(), + relay: false, + continuum: true, + mdns: false, + upnp: false, + ignore: Vec::new(), + prefer: Vec::new(), + internet_hosts: Vec::new(), + public: false, + memory: true, + keepalive: Duration::from_secs(60), + }) + .unwrap(); + assert_eq!( + transport.local_addresses(), + [announced.with_p2p(peer).unwrap().to_string()] + ); +} + +#[test] +fn address_ranges() { + let range = |text: &str| text.parse::<IpRange>().unwrap(); + let address = |text: &str| text.parse::<Multiaddr>().unwrap(); + let ranges = [range("10.244.0.0/16"), range("42::/16"), range("192.0.2.7")]; + for left_out in [ + "/ip4/10.244.3.2/udp/9661/quic-v1", + "/ip6/42:1:a46e:be4a:cb58:e460:bf0:142c/tcp/9661", + "/ip4/192.0.2.7/tcp/1", + ] { + assert!(ignored(&ranges, &address(left_out)), "{left_out}"); + } + for kept in [ + "/ip4/10.23.1.144/udp/9661/quic-v1", + "/ip4/10.245.0.1/tcp/1", + "/ip6/2a0a:4cc0:c1:5eb0::1/udp/9661/quic-v1", + "/ip4/192.0.2.8/tcp/1", + "/dns4/example.org/tcp/1", + ] { + assert!(!ignored(&ranges, &address(kept)), "{kept}"); + } + // Through a relay: judged by the connection to the relay, not here. + let relayed = format!("/ip4/10.244.0.1/tcp/1/p2p/{}/p2p-circuit", peer()); + assert!(!ignored(&ranges, &address(&relayed))); + assert!(range("0.0.0.0/0").contains("203.0.113.1".parse().unwrap())); + assert!(!range("0.0.0.0/0").contains("::1".parse().unwrap())); + for wrong in ["10.244.0.0/33", "42::/129", "nowhere", "10.0.0.0/x"] { + assert!(wrong.parse::<IpRange>().is_err(), "{wrong}"); + } + // Mycelium's range: a prefix that isn't a multiple of 8. + let mycelium = [range("400::/7")]; + assert!(ignored( + &mycelium, + &address("/ip6/45f:fa21:4bdd:1::1/tcp/9661") + )); + assert!(ignored(&mycelium, &address("/ip6/5ff::1/tcp/9661"))); + assert!(!ignored(&mycelium, &address("/ip6/600::1/tcp/9661"))); +} + +/// A preferred range all ignored is refused; one with an ignored range in it is fine +/// (with a warning). +#[test] +fn preferred_and_ignored_ranges() { + let check = |ignore: &[&str], prefer: &[&str]| { + let given = |ranges: &[&str]| { + ranges + .iter() + .map(|range| range.to_string()) + .collect::<Vec<_>>() + }; + let (ignore, prefer) = (given(ignore), given(prefer)); + super::config::check_preferred( + &super::config::parse_ranges("libp2p-ignore", &ignore).unwrap(), + &super::config::parse_ranges("libp2p-prefer", &prefer).unwrap(), + &ignore, + &prefer, + ) + }; + assert!(check(&["10.244.0.0/16"], &["10.23.0.0/16"]).is_ok()); + assert!(check(&["10.0.0.0/8"], &["10.23.0.0/16"]).is_err()); + assert!(check(&["10.23.0.0/16"], &["10.23.0.0/16"]).is_err()); + assert!(check(&["10.23.5.0/24"], &["10.23.0.0/16"]).is_ok()); + assert!(check(&["42::/16"], &["10.0.0.0/8"]).is_ok()); +} + +/// Ranges left out aren't published; announced addresses are, whatever they are. +#[test] +fn ignored_addresses_are_not_published() { + let parse = |text: &str| text.parse::<Multiaddr>().unwrap(); + let lan = parse("/ip4/10.23.1.5/udp/9663/quic-v1"); + let vpn = parse("/ip4/10.244.3.2/udp/9663/quic-v1"); + let mut book = Book { + listening: vec![lan.clone(), vpn.clone()], + ignored: vec!["10.244.0.0/16".parse().unwrap()], + ..Book::default() + }; + assert_eq!(book.direct(), [lan]); + book.announced = vec![vpn.clone()]; + assert_eq!(book.direct(), [vpn]); +} + +/// A port forwarded by hand: once AutoNAT found the node dialable at the router's +/// address, that is published with the listen addresses; announced ones still win. +#[test] +fn the_address_autonat_confirmed_is_published() { + let parse = |text: &str| text.parse::<Multiaddr>().unwrap(); + let lan = parse("/ip4/10.23.1.5/udp/9663/quic-v1"); + let public = parse("/ip4/91.65.180.254/udp/9663/quic-v1"); + let mut book = Book { + listening: vec![lan.clone()], + ..Book::default() + }; + assert_eq!(book.direct(), [lan.clone()]); + book.public_address = Some(public.clone().with_p2p(peer()).unwrap()); + assert_eq!(book.direct(), [public.clone(), lan.clone()]); + let announced = parse("/dns4/example.org/udp/9663/quic-v1"); + book.announced = vec![announced.clone()]; + assert_eq!(book.direct(), [announced]); +} + +#[test] +fn local_addresses_are_left_out_unless_there_are_only_local_ones() { + let parse = |text: &str| text.parse::<Multiaddr>().unwrap(); + let loopback = parse("/ip4/127.0.0.1/tcp/1"); + let link = parse("/ip6/fe80::1/tcp/1"); + let lan = parse("/ip4/192.168.1.2/tcp/1"); + let wireguard = parse("/ip6/fd00::2/tcp/1"); + assert_eq!( + published(&[ + loopback.clone(), + link.clone(), + lan.clone(), + wireguard.clone() + ]), + [lan, wireguard] + ); + assert_eq!( + published(&[loopback.clone(), link.clone()]), + [loopback, link] + ); +} + +/// b listens nowhere but through the relay r; a reaches it there. +#[tokio::test] +async fn a_node_nobody_can_dial_is_reached_through_a_relay() { + let (_r, r_address) = memory_node_with(true, Vec::new(), true); + let (b, _) = memory_node_with(false, vec![r_address.clone()], false); + let b = Arc::new(b); + let mut at_b = received(Arc::clone(&b)); + assert!( + eventually(|| !b.local_addresses().is_empty()).await, + "b got no slot on the relay" + ); + let relayed = b.local_addresses()[0].clone(); + assert!( + relayed.starts_with(&format!("{r_address}/p2p-circuit/")), + "{relayed}" + ); + + assert!(eventually(|| b.stats().libp2p.unwrap().relay_slots.held == 1).await); + let slots = b.stats().libp2p.unwrap().relay_slots; + assert_eq!((slots.granted, slots.not_granted), (1, 0)); + + let (a, _) = memory_node(); + assert_eq!(a.resolve_seed(&relayed).await.unwrap(), b.local_id()); + a.send_message(b.local_id(), &status()).await.unwrap(); + let (from, _) = at_b.recv().await.unwrap(); + assert_eq!(from, a.local_id()); + assert!(a.connection(b.local_id()).is_some()); +} + +/// A seed that's a peer ID only is reached through the relays the node knows. +#[tokio::test] +async fn a_peer_id_seed_is_reached_through_a_relay() { + let (_r, r_address) = memory_node_with(true, Vec::new(), true); + let (b, _) = memory_node_with(false, vec![r_address.clone()], false); + let b = Arc::new(b); + let mut at_b = received(Arc::clone(&b)); + assert!( + eventually(|| !b.local_addresses().is_empty()).await, + "b got no slot on the relay" + ); + let (a, _) = memory_node_with(true, vec![r_address.clone()], false); + let seed = format!("/p2p/{}", b.local_id()); + assert_eq!(a.resolve_seed(&seed).await.unwrap(), b.local_id()); + assert_eq!(a.resolve_seed(b.local_id()).await.unwrap(), b.local_id()); + // Nothing to dial is remembered. + assert!(!lock(&a.book).addresses.contains_key(&b.peer_id)); + a.send_message(b.local_id(), &status()).await.unwrap(); + let (from, _) = at_b.recv().await.unwrap(); + assert_eq!(from, a.local_id()); + + // Without a relay, there's nowhere to look. + let (c, _) = memory_node(); + assert!(c.resolve_seed(&seed).await.is_err()); +} + +/// b listens nowhere; a, which can be dialled, reaches it through the relay r: b +/// then dials a directly (identify told it where), and both keep that. +#[tokio::test] +async fn a_relayed_connection_is_upgraded_where_one_side_can_be_dialled() { + let (_r, r_address) = memory_node_with(true, Vec::new(), true); + let (b, _) = memory_node_with(false, vec![r_address.clone()], false); + let b = Arc::new(b); + let mut at_b = received(Arc::clone(&b)); + assert!( + eventually(|| !b.local_addresses().is_empty()).await, + "b got no slot on the relay" + ); + let (a, _) = memory_node(); + let relayed = b.local_addresses()[0].clone(); + assert_eq!(a.resolve_seed(&relayed).await.unwrap(), b.local_id()); + a.send_message(b.local_id(), &status()).await.unwrap(); + at_b.recv().await.unwrap(); + assert!( + eventually(|| b.connection(a.local_id()) == Some(Connection::Direct)).await, + "b didn't dial a directly" + ); + assert_eq!(a.connection(b.local_id()), Some(Connection::Direct)); + // The relayed one goes (`DUPLICATE_GRACE`); the direct one stays. + let deadline = tokio::time::Instant::now() + DUPLICATE_GRACE * 2; + while a.stats().libp2p.unwrap().connections.relayed > 0 { + assert!( + tokio::time::Instant::now() < deadline, + "the relayed connection stayed" + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } + b.send_message(a.local_id(), &status()).await.unwrap(); + assert_eq!(a.connection(b.local_id()), Some(Connection::Direct)); +} + +/// A relay that is no node (`--relay-only`) relays, and takes no messages itself. +#[tokio::test] +async fn a_relay_that_is_no_node() { + let port = NEXT_PORT.fetch_add(1, Ordering::Relaxed); + let key = Keypair::generate_ed25519(); + let r_peer = key.public().to_peer_id(); + let r_listen: Multiaddr = format!("/memory/{port}").parse().unwrap(); + let r = Libp2pTransport::start(Libp2pConfig { + key, + listen: vec![r_listen.clone()], + announce: Vec::new(), + relays: Vec::new(), + relay: true, + continuum: false, + mdns: false, + upnp: false, + ignore: Vec::new(), + prefer: Vec::new(), + internet_hosts: Vec::new(), + public: false, + memory: true, + keepalive: Duration::from_secs(60), + }) + .unwrap(); + let r_address = r_listen.with_p2p(r_peer).unwrap(); + let (b, _) = memory_node_with(false, vec![r_address.clone()], false); + let b = Arc::new(b); + let mut at_b = received(Arc::clone(&b)); + assert!( + eventually(|| !b.local_addresses().is_empty()).await, + "b got no slot on the relay" + ); + let (a, _) = memory_node(); + let relayed = b.local_addresses()[0].clone(); + assert_eq!(a.resolve_seed(&relayed).await.unwrap(), b.local_id()); + a.send_message(b.local_id(), &status()).await.unwrap(); + let (from, _) = at_b.recv().await.unwrap(); + assert_eq!(from, a.local_id()); + + // The relay itself doesn't speak continuum: it takes no messages, and a send to + // it is refused once identify has told (the relay's own refusal comes too late: + // the protocol goes with the first data, `swarm_config`), saying what to give + // as a seed instead. + assert_eq!( + a.resolve_seed(&r_address.to_string()).await.unwrap(), + r.local_id() + ); + assert!( + eventually(|| lock(&a.book).not_nodes.get(&r.peer_id) == Some(&true)).await, + "identify didn't tell" + ); + let refused = a.send_message(r.local_id(), &status()).await.unwrap_err(); + assert!( + refused.to_string().contains("is a relay, not a node"), + "{refused:#}" + ); + assert!( + r.start_consumer(Box::new(|_, _, _| Box::pin(async { Ok(()) }))) + .await + .is_err() + ); +} + +/// A relay configured without its peer ID is found out, and used. +#[tokio::test] +async fn a_relay_configured_without_its_peer_id() { + let (_r, r_address) = memory_node_with(true, Vec::new(), true); + let mut bare = r_address.clone(); + bare.pop(); + let (b, _) = memory_node_with(false, vec![bare], false); + assert!( + eventually(|| !b.local_addresses().is_empty()).await, + "b got no slot on the relay" + ); + assert!(b.local_addresses()[0].starts_with(&format!("{r_address}/p2p-circuit/"))); +} + +fn peer() -> PeerId { + Keypair::generate_ed25519().public().to_peer_id() +} + +#[test] +fn relays_are_picked_when_needed() { + use ::libp2p::autonat::NatStatus; + let local = peer(); + let offers: HashMap<PeerId, Multiaddr> = (0..4) + .map(|port| (peer(), format!("/memory/{port}").parse().unwrap())) + .collect(); + let check = + |direct, nat: &NatStatus, using: &[PeerId]| relay_check(local, direct, nat, &offers, using); + // Dialable, or not known yet with an address to dial: none. + assert!(check(true, &NatStatus::Public(Multiaddr::empty()), &[]).is_empty()); + assert!(check(false, &NatStatus::Public(Multiaddr::empty()), &[]).is_empty()); + assert!(check(true, &NatStatus::Unknown, &[]).is_empty()); + // No address to dial, or AutoNAT says it can't be dialled: some. + assert_eq!(check(false, &NatStatus::Unknown, &[]).len(), AUTO_RELAYS); + let picked = check(true, &NatStatus::Private, &[]); + assert_eq!(picked.len(), AUTO_RELAYS); + // One in use: the next in the same order. + let more = check(true, &NatStatus::Private, &[picked[0].0]); + assert_eq!(more.len(), AUTO_RELAYS - 1); + assert_eq!(more[0].0, picked[1].0); + let all: Vec<PeerId> = picked.iter().map(|(relay, _)| *relay).collect(); + assert!(check(true, &NatStatus::Private, &all).is_empty()); + // Never itself. + let only_us = HashMap::from([(local, Multiaddr::empty())]); + assert!(relay_check(local, false, &NatStatus::Private, &only_us, &[]).is_empty()); +} + +/// A relay says so in its addresses; b, which nobody can dial, learns of it (as from +/// the member list) and becomes reachable through it. +#[tokio::test] +async fn a_member_that_relays_is_picked() { + let (r, _) = memory_node_with(true, Vec::new(), true); + assert!(eventually(|| !r.local_addresses().is_empty()).await); + let offer = r + .local_addresses() + .into_iter() + .find(|address| address.ends_with("/p2p-circuit")) + .expect("the relay offers itself"); + let (b, _) = memory_node_with(false, Vec::new(), false); + assert_eq!(b.locate(&offer).as_deref(), Some(r.local_id())); + assert!( + eventually(|| b + .local_addresses() + .iter() + .any(|address| address.contains("/p2p-circuit/"))) + .await, + "b isn't reachable through r: {:?}", + b.local_addresses() + ); +} + +/// A peer on the LAN that speaks continuum becomes a seed once, at its address. +#[test] +fn public_ranges() { + let public = |ip: &str| book::is_public(ip.parse().unwrap()); + assert!(public("203.0.114.5")); + assert!(public("2a01:4f8::1")); + for not in [ + "10.1.2.3", + "100.64.0.1", + "192.0.2.1", + "198.18.0.1", + "224.0.0.1", + "255.255.255.255", + "::1", + "fd00::1", + "fe80::1", + "2001:db8::1", + "64:ff9b::1", + // Mycelium's overlay. + "45f:fa21:4bdd::1", + ] { + assert!(!public(not), "{not}"); + } +} + +/// An address where a dial reached another peer isn't kept for the one dialled, even +/// when it tells it again (identify) or the member list does. +#[test] +fn addresses_of_another_peer_are_forgotten() { + let mut book = Book::default(); + let them = peer(); + let lan: Multiaddr = "/ip4/192.168.1.5/tcp/9661".parse().unwrap(); + let elsewhere: Multiaddr = "/ip4/203.0.114.9/tcp/9661".parse().unwrap(); + book.remember(them, lan.clone()); + book.remember(them, elsewhere.clone()); + book.not_theirs(them, lan.clone().with_p2p(them).unwrap()); + assert_eq!(book.addresses[&them], [elsewhere.clone()]); + book.remember(them, lan.clone()); + book.identified(them, vec![lan, elsewhere.clone()]); + assert_eq!(book.addresses[&them], [elsewhere]); +} + +#[test] +fn lan_peers_become_seeds_once() { + let mut book = Book::default(); + let lan = peer(); + let address: Multiaddr = "/ip4/192.168.1.5/tcp/9661".parse().unwrap(); + book.discover(lan); // No address known: nothing. + assert!(book.discovered.is_empty()); + book.remember(lan, address.clone()); + book.discover(lan); + book.discover(lan); + assert_eq!(book.discovered, [address.with_p2p(lan).unwrap()]); +} + +#[test] +fn named_relays() { + let (local, r1, r2) = (peer(), peer(), peer()); + let at = |port: u16| -> Multiaddr { format!("/ip6/fd00::1/tcp/{port}").parse().unwrap() }; + let mut book = Book::default(); + // With a peer ID: known at once; without: to be found out, once. + assert_eq!( + book.name_relay(at(1).with_p2p(r1).unwrap(), Named::Configured, local), + None + ); + assert_eq!( + book.name_relay(at(2), Named::Configured, local), + Some(at(2)) + ); + assert_eq!(book.name_relay(at(2), Named::Configured, local), None); + assert_eq!( + book.name_relay(at(3).with_p2p(local).unwrap(), Named::Configured, local), + None + ); + assert!(book.relay_found(at(2), r2, local)); + assert_eq!(book.named(Named::Configured).len(), 2); + // A relay named with its peer ID isn't found out again; one without is. + assert_eq!(book.find_again(r1), None); + assert_eq!(book.find_again(r2), Some(at(2))); + assert!(!book.named(Named::Configured).contains_key(&r2)); + + // Listed: configured wins; unlisted ones go, unless also configured. + let r3 = peer(); + let listed = vec![ + at(1).with_p2p(r1).unwrap(), + at(4).with_p2p(r3).unwrap(), + at(5), + ]; + assert_eq!(book.learn_listed(listed.clone(), local), Some(vec![at(5)])); + assert_eq!(book.learn_listed(listed, local), None); + assert_eq!(book.named_relays[&r1].named, Named::Configured); + assert_eq!(book.named(Named::Listed).keys().collect::<Vec<_>>(), [&r3]); + assert_eq!(book.learn_listed(Vec::new(), local), Some(Vec::new())); + assert!(book.named(Named::Listed).is_empty()); + assert!(!book.finding.contains_key(&at(5))); + assert!(!book.relay_found(at(5), peer(), local)); // No longer listed. + assert_eq!(book.named(Named::Configured).len(), 1); +} + +#[test] +fn known_addresses() { + let mut book = Book::default(); + let (relay, us) = (peer(), peer()); + let direct: Multiaddr = "/ip6/fd00::1/tcp/9661".parse().unwrap(); + let relayed: Multiaddr = format!("/ip6/fd00::2/tcp/9661/p2p/{relay}/p2p-circuit/p2p/{us}") + .parse() + .unwrap(); + assert!(!book.knows(us, &[direct.clone()])); + book.remember(us, relayed.clone()); + book.remember(us, direct.clone()); + // In any order, with or without the peer ID. + assert!(book.knows(us, &[relayed.clone(), direct.clone()])); + assert!(book.knows(us, &[direct.clone().with_p2p(us).unwrap()])); + assert!(!book.knows(us, &["/ip6/fd00::3/tcp/9661".parse().unwrap()])); +} + +/// A connected peer's identify says where it is: older direct addresses go, and others +/// aren't taken while it stays connected; those through relays and by name stay. +#[test] +fn identify_replaces_direct_addresses() { + let mut book = Book::default(); + let (relay, them) = (peer(), peer()); + let old: Multiaddr = "/ip4/10.244.0.7/tcp/9661".parse().unwrap(); + let lan: Multiaddr = "/ip4/192.168.1.7/tcp/9661".parse().unwrap(); + let named: Multiaddr = "/dns4/node.example/tcp/9661".parse().unwrap(); + let relayed: Multiaddr = format!("/ip4/192.0.2.1/tcp/9661/p2p/{relay}/p2p-circuit") + .parse() + .unwrap(); + book.remember(them, old.clone()); + book.remember(them, named.clone()); + book.remember(them, relayed.clone()); + + book.identified(them, vec![lan.clone().with_p2p(them).unwrap()]); + assert!(book.knows(them, &[lan.clone(), named.clone(), relayed.clone()])); + assert!(!book.knows(them, &[old.clone()])); + // The member list, say, still has the old one. + book.remember(them, old.clone()); + assert!(!book.knows(them, &[old.clone()])); + + // Disconnected (`driver`): taken again, it may have moved there. + book.identified.remove(&them); + book.remember(them, old.clone()); + assert!(book.knows(them, &[old])); +} + +/// Of a peer with more addresses than are kept, those that matter stay: in the ranges +/// `libp2p-prefer` names first, then public ones, then through relays, then other +/// private ones. +#[test] +fn identified_addresses_preferred_first() { + let mut book = Book { + preferred: vec!["10.23.0.0/16".parse().unwrap()], + ..Book::default() + }; + let (relay, them) = (peer(), peer()); + let lan: Multiaddr = "/ip4/10.23.7.7/tcp/9661".parse().unwrap(); + // Outside the documentation ranges, which aren't public (`is_public`). + let public: Multiaddr = "/ip4/203.0.114.7/tcp/9661".parse().unwrap(); + let relayed: Multiaddr = format!("/ip4/192.0.2.1/tcp/9661/p2p/{relay}/p2p-circuit") + .parse() + .unwrap(); + let bridges: Vec<Multiaddr> = (0..ADDRESSES_PER_PEER) + .map(|i| format!("/ip4/172.17.{i}.1/tcp/9661").parse().unwrap()) + .collect(); + let mut listen = bridges.clone(); + listen.extend([relayed.clone(), public.clone(), lan.clone()]); + + book.identified(them, listen); + let known = &book.addresses[&them]; + assert_eq!(known.len(), ADDRESSES_PER_PEER); + assert_eq!(known[..3], [lan.clone(), public, relayed]); + + // Our own: those listened on in the preferred ranges published first. + book.listening = vec![bridges[0].clone(), lan.clone()]; + assert_eq!(book.direct(), [lan, bridges[0].clone()]); +} + +/// A peer's addresses in a range this node leaves out aren't kept: never dialled, they +/// would take a place, and be passed on. +#[test] +fn ignored_addresses_not_kept() { + let mut book = Book { + ignored: vec!["10.244.0.0/16".parse().unwrap()], + ..Book::default() + }; + let them = peer(); + let vpn: Multiaddr = "/ip4/10.244.3.2/tcp/9661".parse().unwrap(); + let lan: Multiaddr = "/ip4/10.23.1.144/tcp/9661".parse().unwrap(); + let relay = peer(); + // Through a relay there: dialling the relay is refused. + let via_vpn: Multiaddr = format!("/ip4/10.244.0.1/tcp/9661/p2p/{relay}/p2p-circuit") + .parse() + .unwrap(); + let via_lan: Multiaddr = format!("/ip4/10.23.0.1/tcp/9661/p2p/{relay}/p2p-circuit") + .parse() + .unwrap(); + book.remember(them, vpn.clone()); + book.remember(them, via_vpn.clone()); + book.identified(them, vec![vpn.clone(), lan.clone(), via_lan.clone()]); + assert!(book.knows(them, &[lan, via_lan])); + assert!(!book.knows(them, &[vpn])); + assert!(!book.knows(them, &[via_vpn])); +} + +/// A node connected directly over the WLAN dials the ethernet address it later hears +/// of, once; and when an interface of ours comes, every address again, the one in use +/// too. +#[test] +fn better_paths_explored() { + let mut book = Book::default(); + let them = peer(); + let wlan: Multiaddr = "/ip4/192.168.1.7/tcp/9661".parse().unwrap(); + let ethernet: Multiaddr = "/ip4/192.168.2.7/tcp/9661".parse().unwrap(); + book.continuum.insert(them); + book.remember(them, wlan.clone()); + // Connected (inbound: from another port) over the WLAN. + book.connections.insert( + ConnectionId::new_unchecked(1), + (them, false, "/ip4/192.168.1.7/tcp/40000".parse().unwrap()), + ); + assert!(book.to_explore(them, false).is_empty()); + book.identified(them, vec![wlan.clone(), ethernet.clone()]); + assert_eq!(book.to_explore(them, false), [ethernet.clone()]); + assert!(book.to_explore(them, false).is_empty()); + // An interface of ours came (`driver`): all of them again. + book.explored.clear(); + let mut again = book.to_explore(them, true); + again.sort(); + let mut both = vec![wlan, ethernet]; + both.sort(); + assert_eq!(again, both); +} + +/// Addresses without a route from here are left out, as if ignored: not kept, nor +/// dialled; and own public ones of a family without a route to the internet (by +/// `--libp2p-internet-hosts`) aren't published. The routing table is asked for real only by `has_route`. +#[test] +fn unroutable_addresses_left_out() { + assert!(has_route("127.0.0.1".parse().unwrap())); + // No IPv6 here. + let mut book = Book { + routes: Routes::with_probe(|ip| ip.is_ipv4()), + listening: vec![ + "/ip4/192.168.1.2/tcp/9661".parse().unwrap(), + "/ip6/2a0a:4cc0::2/tcp/9661".parse().unwrap(), + ], + ..Book::default() + }; + let them = peer(); + let v4: Multiaddr = "/ip4/192.168.1.7/tcp/9661".parse().unwrap(); + let v6: Multiaddr = "/ip6/2a0a:4cc0::7/tcp/9661".parse().unwrap(); + book.identified(them, vec![v4.clone(), v6.clone()]); + assert!(book.knows(them, &[v4])); + assert!(!book.knows(them, &[v6])); + // Nobody asked whether there's a route to the internet: everything is published. + assert_eq!(book.direct().len(), 2); + book.routes.internet(&[ + "1.1.1.1".parse().unwrap(), + "2606:4700:4700::1111".parse().unwrap(), + ]); + assert_eq!( + book.direct(), + ["/ip4/192.168.1.2/tcp/9661".parse::<Multiaddr>().unwrap()] + ); +} + +/// `--libp2p-internet-hosts` takes addresses as they are (names are resolved). +#[tokio::test] +async fn internet_hosts_resolved() { + let resolved = resolve_internet_hosts(&["1.1.1.1".to_owned(), "::1".to_owned()]).await; + assert_eq!( + resolved, + [ + "1.1.1.1".parse::<std::net::IpAddr>().unwrap(), + "::1".parse().unwrap() + ] + ); +} + +/// Peers nobody mentions any more are forgotten, but connected ones. +#[test] +fn stale_peers_are_forgotten() { + let (old, current, connected) = (peer(), peer(), peer()); + let address: Multiaddr = "/ip6/fd00::1/tcp/9661".parse().unwrap(); + let mut book = Book::default(); + for peer in [old, current, connected] { + book.remember(peer, address.clone()); + } + book.relays.insert(old, address.clone()); + book.continuum.insert(old); + // From after the mentions above (taken at `Instant::now()`), however fast or + // slow the clock moved meanwhile. + let later = tokio::time::Instant::now() + STALE_AFTER; + // `current` was mentioned again since. + book.seen.insert(current, later); + assert_eq!(book.prune(later, |peer| *peer == connected), 1); + assert!(!book.addresses.contains_key(&old)); + assert!(!book.relays.contains_key(&old) && !book.continuum.contains(&old)); + assert!(book.addresses.contains_key(¤t)); + assert!(book.addresses.contains_key(&connected)); + assert_eq!(book.prune(later, |_| false), 1); // `connected`, once it isn't. +} + +#[test] +fn relay_failures_are_short() { + assert_eq!( + relay_failure_text( + "Custom { kind: Other, error: Other(Left(Left(Left(Reservation(Protocol(NoAddressesInReservation)))))) }", + "x" + ), + "the relay gave no address to be reached through it" + ); + assert_eq!( + relay_failure_text( + "Codec(Error(Custom { kind: Other, error: Failed }))", + "Failed" + ), + "the relay refused the circuit relay protocol (is it a relay?)" + ); + assert_eq!( + relay_failure_text("Io(…)", "connection reset"), + "connection reset" + ); + assert_eq!(relay_failure(&Ok(())), "closed"); +} + +/// Only refusals pause a relay; a slot that ended otherwise is asked for again. +#[test] +fn only_refusals_pause_relays() { + let refused = [ + ("Reservation(Protocol(NoAddressesInReservation))", "x"), + ("Reservation(Status(ResourceLimitExceeded))", "x"), + ("Reservation(Status(PermissionDenied))", "x"), + ("Denied { cause: \"connection gated\" }", "x"), + ("Upgrade(UnsupportedProtocol)", "x"), + ("Dial(WrongPeerId { obtained: … })", "x"), + ( + "Codec(Error(Custom { kind: Other, error: Failed }))", + "Failed", + ), + ]; + for (debug, innermost) in refused { + assert!(relay_refused(debug, innermost), "{debug}"); + } + let not = [ + ("Io(Custom { kind: Other, error: Timeout })", "Timeout"), + ( + "Io(Custom { kind: Other, error: Right(Closed) })", + "connection is closed", + ), + ("Io(…)", "connection reset"), + ]; + for (debug, innermost) in not { + assert!(!relay_refused(debug, innermost), "{debug}"); + } +} + +#[test] +fn listen_addresses_offered_for_hole_punching() { + let parse = |text: &str| text.parse::<Multiaddr>().unwrap(); + assert!(offered(&parse("/ip4/192.168.1.2/udp/9661/quic-v1"))); + assert!(offered(&parse("/ip6/fd00::2/tcp/9661"))); + assert!(!offered(&parse("/ip4/127.0.0.1/udp/9661/quic-v1"))); + assert!(!offered(&parse("/ip6/fe80::1/tcp/9661"))); + let relay = PeerId::random(); + assert!(!offered(&parse(&format!( + "/ip4/192.168.1.2/tcp/9661/p2p/{relay}/p2p-circuit" + )))); +} + +/// libp2p's nested dial errors come out as each address's cause, once. +#[test] +fn dial_errors_are_short() { + use ::libp2p::core::transport::TransportError; + let peer = peer(); + let address: Multiaddr = format!("/ip6/fd00::1/tcp/9661/p2p/{peer}").parse().unwrap(); + let nested = std::io::Error::other( + "Multiple dial errors occurred:\n - Network is unreachable (os error 101): Multiple dial errors occurred:\n - Network is unreachable (os error 101): Network is unreachable (os error 101)", + ); + let error = + ::libp2p::swarm::DialError::Transport(vec![(address, TransportError::Other(nested))]); + assert_eq!( + describe_dial_error(&error), + "/ip6/fd00::1/tcp/9661: Network is unreachable (os error 101)" + ); + let refused = ::libp2p::swarm::DialError::Transport(vec![( + "/ip4/10.0.0.1/tcp/1".parse().unwrap(), + TransportError::Other(std::io::Error::from(std::io::ErrorKind::ConnectionRefused)), + )]); + assert_eq!( + describe_dial_error(&refused), + "/ip4/10.0.0.1/tcp/1: connection refused" + ); +} + +/// A relay doesn't dial the nodes it relays for through itself. +#[test] +fn no_circuits_through_ourselves() { + let (us, other_relay, target) = (peer(), peer(), peer()); + let through = |relay: PeerId| -> Multiaddr { + format!("/ip6/fd00::1/tcp/9661/p2p/{relay}/p2p-circuit") + .parse() + .unwrap() + }; + let direct: Multiaddr = "/ip6/fd00::2/tcp/9661".parse().unwrap(); + assert_eq!(relay_of(&through(us)), Some(us)); + assert_eq!(relay_of(&direct), None); + assert_eq!(relay_of(&direct.clone().with_p2p(target).unwrap()), None); + assert_eq!( + dialable(&[through(us), through(other_relay), direct.clone()], us), + [through(other_relay), direct] + ); +} + +#[tokio::test] +async fn unknown_peers_are_refused() { + let (b, _) = memory_node(); + let stranger = Keypair::generate_ed25519() + .public() + .to_peer_id() + .to_string(); + assert!(b.send_message(&stranger, &status()).await.is_err()); + assert!(b.resolve_seed(&stranger).await.is_err()); + assert_eq!(b.connection(&stranger), None); +} + +#[test] +fn a_direct_connection_wins_over_a_relayed_one() { + let mut book = Book::default(); + let (relayed_only, both) = (peer(), peer()); + let mut id = 0; + let mut connect = |book: &mut Book, peer: PeerId, relayed: bool| { + id += 1; + let address: Multiaddr = format!("/ip4/10.0.0.{id}/tcp/1").parse().unwrap(); + book.connections + .insert(ConnectionId::new_unchecked(id), (peer, relayed, address)); + }; + connect(&mut book, relayed_only, true); + connect(&mut book, both, true); + connect(&mut book, both, false); + assert_eq!(book.connection(&relayed_only), Some(Connection::Relayed)); + assert_eq!(book.connection(&both), Some(Connection::Direct)); + assert_eq!(book.connection(&peer()), None); + assert_eq!( + book.location(&both).map(|address| address.to_string()), + Some("/ip4/10.0.0.3/tcp/1".to_owned()) + ); + let connections = book.stats().connections; + assert_eq!((connections.direct, connections.relayed), (1, 2)); + // Of several, the oldest direct one stays; a relayed one only without a direct. + connect(&mut book, both, false); + assert_eq!( + book.duplicates(&both), + [ + ConnectionId::new_unchecked(4), + ConnectionId::new_unchecked(2) + ] + ); + assert!(book.duplicates(&relayed_only).is_empty()); + // Also where one of them carries a slot, ours or the peer's: messages would go over + // either. + book.slots.insert(both); + assert_eq!(book.duplicates(&both).len(), 2); + book.slots.clear(); + book.slots_here.insert(both); + assert_eq!(book.duplicates(&both).len(), 2); + book.slots_here.clear(); + // Once measured, the fastest direct one stays, however young. + book.rtts + .insert(ConnectionId::new_unchecked(3), Duration::from_millis(40)); + book.rtts + .insert(ConnectionId::new_unchecked(4), Duration::from_millis(2)); + assert_eq!( + book.duplicates(&both), + [ + ConnectionId::new_unchecked(3), + ConnectionId::new_unchecked(2) + ] + ); + assert_eq!( + book.location(&both).map(|address| address.to_string()), + Some("/ip4/10.0.0.4/tcp/1".to_owned()) + ); +} diff --git a/src/transport/mod.rs b/src/transport/mod.rs index fb13ad9..96f7651 100644 --- a/src/transport/mod.rs +++ b/src/transport/mod.rs @@ -1,7 +1,5 @@ use anyhow::Result; use async_trait::async_trait; -use automerge::ChangeHash; -use automerge::sync::Message; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::boxed::Box; @@ -11,6 +9,8 @@ use std::sync::atomic::{AtomicUsize, Ordering}; pub(crate) use crate::document::replica::Epoch; +use crate::document::ChangeHash; + // Used by the transports that don't authenticate their peers themselves (`tor`). #[cfg_attr(not(feature = "tor"), allow(dead_code))] pub(crate) mod challenge; @@ -18,6 +18,7 @@ pub(crate) mod firewall; pub(crate) mod identity; #[cfg(feature = "libp2p")] pub(crate) mod libp2p; +#[cfg(feature = "mycelium")] pub(crate) mod mycelium; pub(crate) mod paths; #[cfg(test)] @@ -28,6 +29,67 @@ pub(crate) mod tor; pub(crate) mod traffic; pub(crate) mod wire; +#[cfg(not(any(feature = "mycelium", feature = "libp2p", feature = "tor")))] +compile_error!( + "continuum needs a transport: build with at least one of the mycelium, libp2p and tor features, e.g. `cargo build --features mycelium`" +); + +/// The options of the transports this build has, each checked by its transport when the +/// configuration is loaded (`config::Settings::check`, after the transports enabled): +/// Mycelium's always (its conflicts are errors wherever they come from, as before); +/// libp2p's and Tor's if the node uses them. Options of transports not built aren't +/// checked: they're ignored, and said so (`config::Loaded::unused_transport_options`). +pub(crate) fn check(settings: &crate::config::Settings) -> Result<()> { + let uses = |name: &str| settings.transports.iter().any(|used| used == name); + let _ = uses; + #[cfg(feature = "mycelium")] + mycelium::check(settings)?; + #[cfg(feature = "libp2p")] + if uses("libp2p") { + libp2p::check(settings)?; + } + #[cfg(feature = "tor")] + if uses("tor") { + tor::TorTransport::check(settings)?; + } + Ok(()) +} + +/// Starts the transport `name` (one of `config::TRANSPORTS`) as `settings` configure +/// it: its scheme and the transport, or None if `stop` came first (some take a while to +/// start: a supervised Mycelium daemon, arti). `key`: the node key, which some prove +/// themselves. +pub(crate) async fn start( + name: &str, + settings: &crate::config::Settings, + key: &std::sync::Arc<crate::identity::NodeKey>, + stop: tokio::sync::watch::Receiver<bool>, +) -> Result<Option<(&'static str, std::sync::Arc<dyn Transport>)>> { + use std::sync::Arc; + let _ = (key, &stop); + let started: Option<(&'static str, Arc<dyn Transport>)> = match name { + #[cfg(feature = "mycelium")] + "mycelium" => mycelium::start(settings, stop) + .await? + .map(|transport| (mycelium::SCHEME, Arc::new(transport) as Arc<dyn Transport>)), + #[cfg(feature = "libp2p")] + "libp2p" => { + let _ = stop; + let config = libp2p::Libp2pConfig::from_settings(settings, false)?; + Some(( + libp2p::SCHEME, + Arc::new(libp2p::Libp2pTransport::start(config)?) as Arc<dyn Transport>, + )) + } + #[cfg(feature = "tor")] + "tor" => tor::TorTransport::from_settings(settings, key, stop) + .await? + .map(|transport| (tor::SCHEME, Arc::new(transport) as Arc<dyn Transport>)), + other => anyhow::bail!("transports: no {other} in this build"), + }; + Ok(started) +} + /// Random identifier of one run of a node (from start to exit). pub(crate) type RunId = [u8; 16]; @@ -137,6 +199,9 @@ pub(crate) struct Status { /// The sender would sync with the recipient: counts it as an active member on its /// own epoch. pub(crate) syncs_with_you: bool, + /// The sender is shutting down: its farewell (`keepalive::farewell`). The + /// recipient holds it dormant at once, rather than after `dormant_after` of silence. + pub(crate) leaving: bool, } /// Cheap to clone: large parts are shared (`bytes::Bytes`). @@ -154,8 +219,17 @@ pub(crate) enum ParsedMessage { Me(IdentityProof), /// Asks for a PONG with this nonce, over the same path (`transport::paths`). Ping(u64), - /// Answers a PING. - Pong(u64), + /// Answers a PING: its nonce, and when it was answered by the answerer's clock + /// (`at`), for comparing clocks (`transport::paths`). + Pong { + nonce: u64, + at: crate::clock::UnixMillis, + }, + /// Asks a member how it is linked to its peers, for a map of the swarm + /// (`swarm::links`). + Links(LinksRequest), + /// Answers a LINKS. + LinksReply(LinksReply), } /// A node's proof that it holds the key of its node ID (`identity`), sending from its @@ -281,6 +355,8 @@ pub(crate) enum MessageKind { Me, Ping, Pong, + Links, + LinksReply, } impl ParsedMessage { @@ -295,7 +371,9 @@ impl ParsedMessage { ParsedMessage::Who(_) => MessageKind::Who, ParsedMessage::Me(_) => MessageKind::Me, ParsedMessage::Ping(_) => MessageKind::Ping, - ParsedMessage::Pong(_) => MessageKind::Pong, + ParsedMessage::Pong { .. } => MessageKind::Pong, + ParsedMessage::Links(_) => MessageKind::Links, + ParsedMessage::LinksReply(_) => MessageKind::LinksReply, } } @@ -310,14 +388,79 @@ impl ParsedMessage { Some((status.epoch, status.epoch_seq)) } ParsedMessage::Blob(blob) => Some((blob.epoch, blob.epoch_seq)), + ParsedMessage::Links(request) => Some((request.epoch, request.epoch_seq)), + ParsedMessage::LinksReply(reply) => Some((reply.epoch, reply.epoch_seq)), ParsedMessage::Who(_) | ParsedMessage::Me(_) | ParsedMessage::Ping(_) - | ParsedMessage::Pong(_) => None, + | ParsedMessage::Pong { .. } => None, } } } +/// LINKS: what a member is asked for a map of the swarm (`swarm::links`). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct LinksRequest { + #[serde(with = "crate::transport::wire::codec::epoch")] + pub(crate) epoch: Epoch, + /// See `SyncEnvelope::epoch_seq`. + pub(crate) epoch_seq: u32, + /// Repeated in the reply: which request it answers. + pub(crate) nonce: u64, + /// For this long (seconds), a LINKS_REPLY again whenever the links change (a + /// subscription; the next LINKS renews it). 0: this reply only, and a subscription + /// ends. + pub(crate) subscribe_secs: u32, +} + +/// LINKS_REPLY: how the sender is linked to its neighbours, as it measures them. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct LinksReply { + #[serde(with = "crate::transport::wire::codec::epoch")] + pub(crate) epoch: Epoch, + /// See `SyncEnvelope::epoch_seq`. + pub(crate) epoch_seq: u32, + /// The request's. + pub(crate) nonce: u64, + /// At most `swarm::links::MAX_LINKS`. + pub(crate) links: Vec<Link>, +} + +/// One of a member's links, in a LINKS_REPLY. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct Link { + /// The peer's node ID. + pub(crate) peer: String, + /// The transport its messages go over now (the scheme of the path in use), if any. + pub(crate) transport: Option<String>, + /// Only through a relay (libp2p). + pub(crate) relayed: bool, + /// The path in use is down: none to the peer answers (`paths`), whatever its + /// connection was. + pub(crate) down: bool, + /// The round trip on that path, in microseconds, if measured. + pub(crate) rtt_us: Option<u32>, + /// Silent for long (`peers.md`, dormancy). + pub(crate) dormant: bool, + /// The peer's other transports (`swarm::links::MAX_OTHERS` at most): where else + /// it proved itself, those paths up. + pub(crate) others: Vec<OtherPath>, + /// How far the peer's clock is ahead of the member's (`paths::ClockOffset`), if + /// measured: with every member's, a map of the swarm shows whose clock is off. + pub(crate) clock: Option<paths::ClockOffset>, +} + +/// A transport of a link besides the one in use, in a LINKS_REPLY. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct OtherPath { + /// Its scheme (`mycelium`, `libp2p`, `tor`). + pub(crate) transport: String, + /// The round trip of its fastest path, in microseconds, if any is measured. + pub(crate) rtt_us: Option<u32>, + /// Only through a relay (libp2p): no direct connection at any of its paths. + pub(crate) relayed: bool, +} + #[derive(Clone, Serialize, Deserialize)] pub(crate) struct HelloRequest { /// The sender's swarm, if it knows it (a new node doesn't yet). A receiver in @@ -361,17 +504,34 @@ pub(crate) struct Destination { pub(crate) proven: bool, /// The transport's connection to it, if the transport has connections. pub(crate) connection: Option<Connection>, + /// While it is connected only through a relay: how hole punching went. + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) punching: Option<Punching>, /// Every address it proved itself at, best first, as the paths layer measures them /// (`transport::paths`). pub(crate) paths: Vec<stats::PathInfo>, } +/// Hole punching to a peer connected only through a relay (libp2p), once a punch failed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct Punching { + /// Punches that failed in a row. + pub(crate) failed: usize, + /// When it is tried again, in seconds from now (0: due). + pub(crate) next_in_secs: u64, +} + /// How a transport with connections (libp2p) is connected to a peer. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] #[serde(rename_all = "camelCase")] +// Only libp2p and Tor have connections; without both (the cargo features) nothing makes +// one. +#[cfg_attr(not(any(feature = "libp2p", feature = "tor")), allow(dead_code))] pub(crate) enum Connection { + /// libp2p's to the peer itself; Tor's always (an onion circuit). Direct, - /// Only through a relay. + /// Only through a relay (libp2p). Relayed, } @@ -489,7 +649,7 @@ pub(crate) enum SyncBody { /// The changes in the history of `to` but not of `from`: what a peer at `from` lacks of /// a document at `to`. Every node holding `to` produces the same bytes for it -/// (`swarm::blob::delta_bytes`). +/// (`document::changes::delta_bytes`). #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub(crate) struct DeltaSpec { /// Sorted. @@ -523,23 +683,13 @@ impl DeltaSpec { } } -/// An Automerge sync message in Automerge's own encoding. Decoded only where it is -/// applied (`decode`), after the sender proved to be a member and off the consumer: a -/// large message from anyone else costs nothing but its bytes. +/// A sync message as the document's engine encodes it (`document::sync`): bytes to the +/// transports. Decoded only where it is applied (`document::sync::decode`), after the sender +/// proved to be a member and off the consumer: a large message from anyone else costs +/// nothing but its bytes. #[derive(Clone)] pub(crate) struct SyncPayload(pub(crate) bytes::Bytes); -impl SyncPayload { - pub(crate) fn encode(message: Message) -> Self { - Self(message.encode().into()) - } - - pub(crate) fn decode(&self) -> Result<Message> { - Message::decode(&self.0) - .map_err(|error| anyhow::anyhow!("Automerge decode failed: {error:?}")) - } -} - /// Sizes instead of contents: logged messages stay short. impl std::fmt::Debug for SyncPayload { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -585,7 +735,7 @@ impl<T: std::fmt::Display> std::fmt::Display for Addressed<T> { /// consumer hands them on as soon as they are read (`hand_on_probe`), not after the /// messages read before them are handled (which can take a while: sync). pub(crate) fn is_probe(message: &ParsedMessage) -> bool { - matches!(message, ParsedMessage::Ping(_) | ParsedMessage::Pong(_)) + matches!(message, ParsedMessage::Ping(_) | ParsedMessage::Pong { .. }) } /// Hands a PING or PONG (`is_probe`) to `handler` in a task of its own; `scheme` is the @@ -618,6 +768,13 @@ pub(crate) trait Transport: Send + Sync { /// a message sent is delivered or lost for good. fn delivery_limit(&self) -> std::time::Duration; + /// The first wait for an answer to a sync message (`Timing::sync_reply_timeout`): by + /// default `SYNC_REPLY_TIMEOUT`, longer on a transport whose round trips take + /// seconds. + fn sync_reply_timeout(&self) -> std::time::Duration { + crate::state::SYNC_REPLY_TIMEOUT + } + /// Where this node can be reached, as addresses other nodes can use: published in /// the member list (`swarm::members`). By default its identity, for a transport /// whose identities are addresses (Mycelium: a public key). @@ -668,6 +825,19 @@ pub(crate) trait Transport: Send + Sync { Vec::new() } + /// Has the transport tell `found` whenever it finds a seed by itself + /// (`discovered_seeds`). By default it never does. + fn on_seeds_found(&self, found: std::sync::Arc<tokio::sync::Notify>) { + let _ = found; + } + + /// Has the transport tell `reachable` whenever others may reach this node where they + /// couldn't before (libp2p: a relay granted it a slot, or AutoNAT found it public). + /// By default it never does. + fn on_reachable(&self, reachable: std::sync::Arc<tokio::sync::Notify>) { + let _ = reachable; + } + /// Whether this node has a way to reach `peer` at all: an address of it on one of /// its transports. Members it can't reach aren't its neighbours (`swarm::topology`). /// By default yes, as for `local_addresses`. @@ -712,6 +882,13 @@ pub(crate) trait Transport: Send + Sync { None } + /// How hole punching to `target` went, if a punch failed and it is still connected + /// only through a relay (libp2p). By default never. + fn punching(&self, target: &str) -> Option<Punching> { + let _ = target; + None + } + /// The smoothed round-trip time of the path messages to `peer` go over, if /// measured (`transport::paths`). By default none. fn rtt(&self, peer: &str) -> Option<std::time::Duration> { @@ -719,6 +896,13 @@ pub(crate) trait Transport: Send + Sync { None } + /// How far `peer`'s clock is ahead of ours, as its PONGs say (`paths::ClockOffset`): + /// the narrowest bound of its paths. By default unknown. + fn clock_offset(&self, peer: &str) -> Option<paths::ClockOffset> { + let _ = peer; + None + } + /// Whether the transport answers PINGs with PONGs itself (the paths layer, /// `transport::paths`); if not (a bare transport, in tests), the node does. By /// default not. diff --git a/src/transport/mycelium/mod.rs b/src/transport/mycelium/mod.rs index 5e8dc05..9078134 100644 --- a/src/transport/mycelium/mod.rs +++ b/src/transport/mycelium/mod.rs @@ -91,9 +91,145 @@ pub(crate) struct MyceliumTransport { local_id: String, } +/// This transport's options, checked when the configuration is loaded +/// (`config::Settings::check`): a daemon elsewhere (`mycelium-rest-url`) by a URL, and +/// none of the options only a supervised one takes with it; a private network with +/// both its key and its name. +pub(crate) fn check(settings: &crate::config::Settings) -> Result<()> { + if let Some(url) = &settings.mycelium_rest_url { + Url::parse(url).with_context(|| format!("mycelium-rest-url {url:?}"))?; + if let Some((name, _)) = supervised_only(settings).iter().find(|(_, set)| *set) { + anyhow::bail!( + "{name} is for a supervised Mycelium daemon and can't be combined with mycelium-rest-url" + ); + } + } + if settings.mycelium_key_create && settings.mycelium_key_file.is_none() { + anyhow::bail!("mycelium-key-create creates the key at mycelium-key-file; set that too"); + } + // A private network has both; one alone would be ignored or invented. + if settings.mycelium_network_key_file.is_some() != settings.mycelium_network_name.is_some() { + anyhow::bail!("mycelium-network-key-file and mycelium-network-name go together"); + } + Ok(()) +} + +/// The options only a supervised Mycelium daemon takes, and whether each is set: the +/// other side of `config::CONFLICTS`' `mycelium-rest-url` (a test keeps them the same). +pub(crate) fn supervised_only(settings: &crate::config::Settings) -> [(&'static str, bool); 5] { + [ + ("mycelium-key-file", settings.mycelium_key_file.is_some()), + ("mycelium-key-create", settings.mycelium_key_create), + ( + "mycelium-network-key-file", + settings.mycelium_network_key_file.is_some(), + ), + ( + "mycelium-network-name", + settings.mycelium_network_name.is_some(), + ), + ( + "mycelium-tcp-listen-port", + settings.mycelium_tcp_listen_port.is_some(), + ), + ] +} + +/// The transport as `settings` configure it (`transport::start`): its daemon supervised +/// (started, which can take a while) or one running elsewhere. None if `stop` came +/// first. +pub(crate) async fn start( + settings: &crate::config::Settings, + mut stop: tokio::sync::watch::Receiver<bool>, +) -> Result<Option<MyceliumTransport>> { + // The daemon's key, 32 raw bytes (as `daemon` hands it an ephemeral one): read, + // or made, here, so a missing file fails before the daemon starts. + if let Some(path) = &settings.mycelium_key_file { + let (bytes, created) = crate::files::key_file( + path, + "Mycelium key", + settings.mycelium_key_create, + "mycelium-key-create", + "a new Mycelium address", + 32, + )?; + if bytes.len() != 32 { + anyhow::bail!( + "The Mycelium key in {} has {} bytes; it has 32", + path.display(), + bytes.len() + ); + } + if created { + tracing::info!(path = %path.display(), "Created a new Mycelium key"); + } + } + let spawn_config = MyceliumSpawnConfig { + rest_url_opt: settings + .mycelium_rest_url + .as_deref() + .and_then(|url| Url::parse(url).ok()), + key_file_opt: settings.mycelium_key_file.clone(), + network_key_file: settings.mycelium_network_key_file.clone(), + network_name: settings.mycelium_network_name.clone(), + tcp_port: settings.mycelium_tcp_listen_port, + peers: settings.mycelium_peers.clone(), + }; + let setup = tokio::select! { + setup = MyceliumSetup::bootstrap(spawn_config) => setup?, + _ = stop.wait_for(|stop| *stop) => return Ok(None), + }; + // Without a password it may carry (the option is secret). + let mut shown_url = setup.rest_url.clone(); + let _ = shown_url.set_password(None); + tracing::debug!(url = shown_url.to_string(), "Using Mycelium REST API"); + tokio::select! { + transport = MyceliumTransport::connect(setup) => Ok(Some(transport?)), + _ = stop.wait_for(|stop| *stop) => Ok(None), + } +} + +/// How long the daemon's API may take to come up: a daemon just started listens a +/// moment later (`admin_info`). +const API_READY_WITHIN: Duration = Duration::from_secs(30); + +/// The daemon's identity, from its API at `url` (`/api/v1/admin`), once the API answers: +/// a daemon we just started doesn't listen yet, so refused connections are tried again +/// (50 ms, doubling up to a second) for `API_READY_WITHIN`; anything else is an error. +async fn admin_info(client: &reqwest::Client, url: Url) -> Result<MyceliumAdminInfo> { + let deadline = tokio::time::Instant::now() + API_READY_WITHIN; + let mut wait = Duration::from_millis(50); + loop { + match client.get(url.clone()).send().await { + Ok(response) => { + return response + .error_for_status()? + .json() + .await + .context("Querying local Mycelium identity"); + } + Err(error) if error.is_connect() && tokio::time::Instant::now() + wait < deadline => { + tracing::debug!(url = %url, error = %error, "Mycelium's API doesn't answer yet; trying again"); + tokio::time::sleep(wait).await; + wait = (wait * 2).min(Duration::from_secs(1)); + } + Err(error) => { + return Err(error).with_context(|| { + format!( + "Querying local Mycelium identity (no answer within {API_READY_WITHIN:?})" + ) + }); + } + } + } +} + impl MyceliumTransport { pub(crate) async fn connect(setup: MyceliumSetup) -> Result<Self> { let rest_url = setup.rest_url.clone(); + // reqwest's TLS has no crypto provider of its own (`Cargo.toml`): ring, for the + // whole process, unless one is installed already. + let _ = rustls::crypto::ring::default_provider().install_default(); let client = reqwest::Client::builder() .timeout(REQUEST_TIMEOUT) @@ -102,14 +238,7 @@ impl MyceliumTransport { .timeout(Duration::from_secs(MYCELIUM_POLL_TIMEOUT_SECS + 30)) .build()?; - let admin: MyceliumAdminInfo = client - .get(rest_url.join("/api/v1/admin")?) - .send() - .await? - .error_for_status()? - .json() - .await - .context("Querying local Mycelium identity")?; + let admin = admin_info(&client, rest_url.join("/api/v1/admin")?).await?; let transport = Self { setup: tokio::sync::Mutex::new(setup), @@ -246,31 +375,17 @@ impl Transport for MyceliumTransport { .into()); } - // Mycelium's message ID, for GET /api/v1/messages/status/{id} when chasing - // a message that never arrived. - if tracing::enabled!(tracing::Level::DEBUG) { - let Ok(body) = response + // Mycelium's message ID, for GET /api/v1/messages/status/{id} when chasing a + // message that never arrived. The message itself is logged above, in the paths + // layer (`transport::paths`). + if tracing::enabled!(tracing::Level::TRACE) { + let id = response .bytes() .await - .map_err(anyhow::Error::from) - .and_then(|bytes| Ok(serde_json::from_slice::<serde_json::Value>(&bytes)?)) - else { - tracing::debug!(to = %Addressed(SCHEME, &message.peer), "Sent message (couldn't decode response from Mycelium REST API)"); - return Ok(()); - }; - let Ok(outbound) = serde_json::from_slice::<OutboundMessage>(&message.body) else { - tracing::debug!(to = %Addressed(SCHEME, &message.peer), body_len = %message.body.len(), "Sent message (couldn't parse outbound message)"); - return Ok(()); - }; - let Ok(parsed) = decode_payload(&outbound.topic, &outbound.payload) else { - tracing::debug!(to = %Addressed(SCHEME, &message.peer), message = ?outbound, "Sent message (couldn't decode outbound message)"); - return Ok(()); - }; - let Some(id) = body.get("id").and_then(|id| id.as_str()) else { - tracing::debug!(to = %Addressed(SCHEME, &message.peer), message = ?parsed, "Sent message (couldn't read id from Mycelium REST API response)"); - return Ok(()); - }; - tracing::debug!(to = %Addressed(SCHEME, &message.peer), message = ?parsed, id = %id, "Sent message"); + .ok() + .and_then(|bytes| serde_json::from_slice::<serde_json::Value>(&bytes).ok()) + .and_then(|body| body.get("id")?.as_str().map(str::to_owned)); + tracing::trace!(to = %Addressed(SCHEME, &message.peer), id = ?id, "Mycelium took the message"); } Ok(()) } @@ -300,12 +415,11 @@ impl Transport for MyceliumTransport { while let Some(inbound_message) = rx.recv().await { match parse_message(&inbound_message) { Ok((sender, parsed)) => { - tracing::debug!( + // The message itself is logged above, in the paths layer. + tracing::trace!( from = %Addressed(SCHEME, &inbound_message.src_pk), - message = ?parsed, id = %inbound_message.id, - //payload_bytes = raw_bytes.len(), - "Received message" + "Mycelium delivered a message" ); let payload_len = base64_decoded_len(&inbound_message.payload); if is_probe(&parsed) { @@ -464,6 +578,13 @@ pub(crate) fn encode_message(peer: &str, message: &ParsedMessage) -> Result<Enco }) } +/// Mycelium's envelope, for simulations that run on it (`transport::sim::Codec`). +#[cfg(test)] +pub(crate) const CODEC: crate::transport::sim::Codec = crate::transport::sim::Codec { + encode: encode_message, + decode: decode_message, +}; + /// The message in a body made by [`encode_message`]. #[cfg(test)] pub(crate) fn decode_message(body: &[u8]) -> Result<ParsedMessage> { @@ -562,10 +683,10 @@ pub(crate) enum TransportError { #[cfg(test)] mod tests { use super::*; + use crate::document::ChangeHash; use crate::transport::{ BlobBody, BlobId, BlobMessage, Epoch, HeadsSummary, MAX_LISTED_HEADS, Status, }; - use automerge::ChangeHash; /// Mycelium splits messages into chunks of this size. const MYCELIUM_CHUNK: usize = 1300; @@ -595,6 +716,7 @@ mod tests { epoch_seq: u32::MAX, ready: true, syncs_with_you: true, + leaving: false, }; let payload = wire::encode(&ParsedMessage::StatusReply(status.clone())).unwrap(); // Mycelium's message data: a length byte, the topic, then our payload. @@ -657,20 +779,51 @@ mod tests { )); } + /// Every kind decodes from Mycelium's API body (its envelope: the topic, the payload + /// in base64) to what was encoded. #[test] - fn messages_round_trip_through_the_api_body() { - let status = ParsedMessage::Status(Status { - heads: HeadsSummary::of(vec![ChangeHash([1; 32])]), - epoch: Epoch::GENESIS, - epoch_seq: 0, - ready: true, - syncs_with_you: true, - }); - let encoded = encode_message("peer", &status).unwrap(); - assert!(matches!( - decode_message(&encoded.body).unwrap(), - ParsedMessage::Status(_) - )); + fn every_kind_round_trips_through_the_api_body() { + for message in wire::samples() { + let encoded = encode_message("peer", &message).unwrap(); + assert_eq!(encoded.kind, message.kind()); + let decoded = decode_message(&encoded.body).unwrap(); + assert_eq!( + wire::encode(&decoded).unwrap(), + wire::encode(&message).unwrap(), + "{message:?}" + ); + assert_eq!(encoded.payload_len, wire::encode(&message).unwrap().len()); + } + } + + /// A swarm whose every message goes through Mycelium's envelope: joining, writes + /// reaching everyone, and concurrent writes merging, as the simulations do with the + /// wire encoding alone. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn a_swarm_converges_over_the_envelope() { + use crate::simtest::{Sim, read, write}; + use crate::swarm::bootstrap::Start; + use serde_json::json; + const SOON: Duration = Duration::from_secs(10); + + let sim = Sim::with_codec(11, CODEC); + let a = sim.start("a", &[], Start::Genesis); + let b = sim.start("b", &["a"], Start::Join); + let c = sim.start("c", &["a"], Start::Join); + let (a, b, c) = (&a, &b, &c); + sim.wait_ready(&[a, b, c], SOON).await; + write(b, "users/bob", json!({"name": "Bob"})).await; + sim.wait_until("c has b's write", SOON, move || async move { + read(c, "users/bob/name").await == Some(json!("Bob")) + }) + .await; + write(a, "users/alice", json!(1)).await; + write(c, "users/carol", json!(3)).await; + sim.wait_converged(&[a, b, c], SOON).await; + for node in [a, b, c] { + assert_eq!(read(node, "users/alice").await, Some(json!(1))); + assert_eq!(read(node, "users/carol").await, Some(json!(3))); + } } #[test] diff --git a/src/transport/paths.rs b/src/transport/paths.rs index 3baa092..164574d 100644 --- a/src/transport/paths.rs +++ b/src/transport/paths.rs @@ -8,7 +8,7 @@ //! transport of its address, hands every transport's messages to one handler under the //! sender's address, and puts together what the transports say about themselves //! (addresses, seeds they found, figures). The transports are in order of preference, -//! as configured (`--transports`). +//! as configured (`--transport-order`). //! //! A message to a node with several addresses goes to the first of them that takes it //! (`send_first`), in the order `rank` gives. A send that fails is tried at once at the @@ -16,7 +16,8 @@ //! next carries on within the same send. //! //! Paths measures the addresses of such nodes itself (and every path the node's -//! keepalive PINGs go over, below): every `PROBE_EVERY`, each address it had a choice of +//! keepalive PINGs go over, below): every `PROBE_EVERY`, or every keepalive interval if +//! that is longer, each address it had a choice of //! within `ACTIVE_FOR` gets a PING with a random nonce, which the //! other side's paths answers at once with a PONG, over the same path. Answers give each //! path's round-trip time (smoothed); `UNANSWERED_DOWN` PINGs in a row unanswered, or a @@ -30,6 +31,11 @@ //! alive. The node's keepalive is a PING too (`swarm::peer::keepalive`), which paths //! sends like any message, and notes as the probe of the path it went over. //! +//! Every message on every transport passes here, so this is where messages are logged +//! (debug): `Sent message` once a transport took one, `Received message` as one +//! arrives, each with its address and what it says. Failed sends are logged by those who +//! sent them, with what they do about it. +//! //! The debug firewall (`transport::firewall`) works here, below everything else, through //! a `Gate`: messages from an address it blocks are dropped as they arrive, and sends to //! one are dropped silently, as by a lossy network. @@ -42,6 +48,7 @@ use anyhow::{Context, Result, anyhow}; use tokio::time::Instant; use tracing::{debug, info}; +use crate::clock::{Clock, UnixMillis}; use crate::locks::MutexExt; use crate::transport::stats::{PathCounts, PathInfo, PathsStats, TransportStats}; use crate::transport::traffic::Traffic; @@ -84,15 +91,16 @@ pub(crate) fn scheme_of(address: &str) -> &str { /// A path whose send failed is down for this long (unless it answers a PING, or a /// send there works again). const DOWN_FOR: Duration = Duration::from_secs(60); -/// How often a path in use is pinged, how long an answer may take, and how many PINGs -/// in a row unanswered make it down. +/// How often a path in use is pinged at most (every keepalive interval, if that is +/// longer: `Table::probe_every`), how long an answer may take, and how many PINGs in a +/// row unanswered make it down. const PROBE_EVERY: Duration = Duration::from_secs(30); const PROBE_TIMEOUT: Duration = Duration::from_secs(15); const UNANSWERED_DOWN: u32 = 3; /// How often the probe task looks for paths due. const PROBE_TICK: Duration = Duration::from_secs(5); -/// Paths this node had a choice of within this long are probed; ones it didn't use for -/// twice as long are forgotten. +/// Paths this node had a choice of, or sent over, within this long are probed; ones it +/// didn't use for twice as long are forgotten. const ACTIVE_FOR: Duration = Duration::from_secs(300); /// The path in use is changed for one this much faster (a fraction of its round-trip /// time), and at least `SWITCH_MIN`. @@ -100,14 +108,48 @@ const SWITCH_GAIN: f64 = 0.25; const SWITCH_MIN: Duration = Duration::from_millis(5); /// Paths kept at most: they are of peers this node chose among, but bounded anyway. const MAX_PATHS: usize = 4096; +/// A path's clock sample with the narrowest bound is kept this long at most; then the +/// next one takes its place. (A clock set anew shows at once: `Table::clock`.) +const CLOCK_KEPT: Duration = Duration::from_secs(300); + +/// How far a peer's clock is ahead of ours (behind if negative), from a PING and its +/// PONG: `offset_ms`, give or take `within_ms`. The PONG says its sender's time when it +/// answered, which was after our PING went and before the PONG came: within half the +/// round trip of its middle, however differently long the two ways took. +#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ClockOffset { + pub(crate) offset_ms: i64, + pub(crate) within_ms: u64, +} + +impl ClockOffset { + /// From a PONG stamped `at` by its sender, which came at `heard` by our clock, `rtt` + /// after its PING went. + pub(crate) fn of(at: UnixMillis, heard: UnixMillis, rtt: Duration) -> Self { + let half_us = rtt.as_micros() / 2; + let middle_us = i128::from(heard.as_u64()) * 1000 - half_us as i128; + let offset_us = i128::from(at.as_u64()) * 1000 - middle_us; + Self { + offset_ms: i64::try_from(offset_us / 1000).unwrap_or(i64::MAX), + // And a millisecond for the stamps' resolution. + within_ms: u64::try_from(half_us.div_ceil(1000)).unwrap_or(u64::MAX) + 1, + } + } +} /// What paths knows about one path (an address as paths keep it). #[derive(Debug, Default)] struct PathState { /// When it was last among several to choose from (`rank`). used: Option<Instant>, + /// When a message last went over it: a peer's only path is measured too, if its + /// keepalives don't (a busy pair has none: `PeerState::take_keepalive_due`). + carried: Option<Instant>, /// Smoothed round-trip time of its PINGs. rtt: Option<Duration>, + /// Where the transport went when `rtt` was measured (`Paths::location`). + measured_at: Option<String>, /// The PING out, by nonce, and when it went. probe: Option<(u64, Instant)>, last_probe: Option<Instant>, @@ -115,6 +157,9 @@ struct PathState { unanswered: u32, /// When a send there last failed. failed_at: Option<Instant>, + /// The peer's clock as its PONGs on this path say (the narrowest bound lately), and + /// when that sample came. + clock: Option<(ClockOffset, Instant)>, } impl PathState { @@ -127,29 +172,58 @@ impl PathState { } /// Every path's state, and which one is in use for each set to choose from. -#[derive(Debug, Default)] +#[derive(Debug)] struct Table { paths: HashMap<String, PathState>, /// By the set's addresses (sorted, joined): the one in use. chosen: HashMap<String, String>, counts: PathCounts, + /// How often a path in use is pinged: `PROBE_EVERY`, or the node's keepalive + /// interval if that is longer, so that the probes don't add to the keepalives a + /// long interval is chosen to spare. A busy pair, whose keepalives don't go, still + /// gets its paths measured this often. + probe_every: Duration, +} + +impl Default for Table { + fn default() -> Self { + Self::new(PROBE_EVERY) + } } impl Table { + fn new(keepalive: Duration) -> Self { + Self { + paths: HashMap::new(), + chosen: HashMap::new(), + counts: PathCounts::default(), + probe_every: PROBE_EVERY.max(keepalive), + } + } + /// `usable`, with each one's place in the configured order, best first (as - /// `Paths::rank` says), noting the choice. - fn rank(&mut self, mut usable: Vec<(String, usize)>, now: Instant) -> Vec<String> { + /// `Paths::rank` says). With `sending`, for a message about to go: they count as + /// used (`ACTIVE_FOR`), and the one in use may change for a faster one. Without, a + /// look at the choice as it stands, changing nothing (`/peers`, figures, LINKS). + fn rank( + &mut self, + mut usable: Vec<(String, usize)>, + now: Instant, + sending: bool, + ) -> Vec<String> { usable.sort_by_key(|(_, preference)| *preference); usable.dedup_by(|a, b| a.0 == b.0); if usable.len() < 2 { return usable.into_iter().map(|(address, _)| address).collect(); } - if self.paths.len() >= MAX_PATHS { - self.forget_unused(now); - } - for (address, _) in &usable { - if self.paths.len() < MAX_PATHS || self.paths.contains_key(address) { - self.paths.entry(address.clone()).or_default().used = Some(now); + if sending { + if self.paths.len() >= MAX_PATHS { + self.forget_unused(now); + } + for (address, _) in &usable { + if self.paths.len() < MAX_PATHS || self.paths.contains_key(address) { + self.paths.entry(address.clone()).or_default().used = Some(now); + } } } let rtt = |table: &Self, address: &str| table.paths.get(address).and_then(|path| path.rtt); @@ -180,6 +254,7 @@ impl Table { .min_by_key(|(_, rtt)| *rtt); if let (Some((fastest, fastest_rtt)), Some(current_rtt)) = (fastest, rtt(self, ¤t)) + && sending && *fastest != current && current_rtt.saturating_sub(fastest_rtt) >= SWITCH_MIN.max(current_rtt.mul_f64(SWITCH_GAIN)) @@ -194,7 +269,7 @@ impl Table { self.counts.switches += 1; current = fastest.clone(); } - if self.chosen.len() < MAX_PATHS || self.chosen.contains_key(&key) { + if sending && (self.chosen.len() < MAX_PATHS || self.chosen.contains_key(&key)) { self.chosen.insert(key, current.clone()); } // The rest: measured ones by round-trip time, then unmeasured ones in the @@ -228,15 +303,22 @@ impl Table { } /// A send to `address` worked. - fn sent(&mut self, address: &str) { - if let Some(path) = self.paths.get_mut(address) { - path.failed_at = None; + fn sent(&mut self, address: &str, now: Instant) { + if self.paths.len() >= MAX_PATHS && !self.paths.contains_key(address) { + self.forget_unused(now); + if self.paths.len() >= MAX_PATHS { + return; + } } + let path = self.paths.entry(address.to_owned()).or_default(); + path.failed_at = None; + path.carried = Some(now); } - /// The PINGs due now, with their nonces (from `nonce`): to every path used within - /// `ACTIVE_FOR` that wasn't pinged within `PROBE_EVERY` and has no PING out. A PING - /// out for `PROBE_TIMEOUT` counts as unanswered. + /// The PINGs due now, with their nonces (from `nonce`): to every path chosen among + /// others or sent over within `ACTIVE_FOR` that wasn't pinged (by us, or by a + /// keepalive) within `probe_every` and has no PING out. A PING out for + /// `PROBE_TIMEOUT` counts as unanswered. fn due(&mut self, now: Instant, mut nonce: impl FnMut() -> u64) -> Vec<(String, u64)> { self.forget_unused(now); let mut due = Vec::new(); @@ -251,12 +333,13 @@ impl Table { info!(address = %address, "A path stopped answering PINGs; using others"); } } - let active = path - .used - .is_some_and(|used| now.saturating_duration_since(used) < ACTIVE_FOR); + let lately = |at: Option<Instant>| { + at.is_some_and(|at| now.saturating_duration_since(at) < ACTIVE_FOR) + }; + let active = lately(path.used) || lately(path.carried); let rested = path .last_probe - .is_none_or(|last| now.saturating_duration_since(last) >= PROBE_EVERY); + .is_none_or(|last| now.saturating_duration_since(last) >= self.probe_every); if active && rested && path.probe.is_none() { let nonce = nonce(); path.probe = Some((nonce, now)); @@ -269,8 +352,8 @@ impl Table { } /// The node sent a PING with `nonce` to `address` (its keepalive): the probe of that - /// path, which its PONG measures, even if it is a peer's only path (then it isn't - /// pinged otherwise: `due` pings only paths there was a choice of). + /// path, which its PONG measures, even if it is a peer's only path; `due` then + /// doesn't ping it again within `probe_every`. fn pinged(&mut self, address: &str, nonce: u64, now: Instant) { if self.paths.len() >= MAX_PATHS && !self.paths.contains_key(address) { self.forget_unused(now); @@ -295,7 +378,23 @@ impl Table { } } - /// A PONG with `nonce` from `address`: the round trip, if it answers our PING out. + /// The transport of `address` now goes to `location` (a libp2p peer over another + /// connection: through a relay before, directly now): round trips measured before + /// were of another way, and would weigh on the next ones for minutes (`pong` + /// smooths), so they are dropped. + fn located(&mut self, address: &str, location: &str) { + let Some(path) = self.paths.get_mut(address) else { + return; + }; + if path.measured_at.as_deref() != Some(location) { + if path.measured_at.is_some() { + path.rtt = None; + } + path.measured_at = Some(location.to_owned()); + } + } + + /// A PONG with `nonce` from `address`: its round trip, if it answers our PING out. fn pong(&mut self, address: &str, nonce: u64, now: Instant) -> Option<Duration> { let path = self.paths.get_mut(address)?; let (expected, sent) = path.probe?; @@ -314,7 +413,26 @@ impl Table { None => sample, }); self.counts.probes_answered += 1; - path.rtt + Some(sample) + } + + /// A clock sample from a PONG over `address`: kept if its bound is as narrow as the + /// one kept, or that one is older than `CLOCK_KEPT`, or the two can't both be right + /// (their bounds don't overlap: a clock was set anew). The narrowest bound is the + /// round trip least delayed by queues and relays. + fn clock(&mut self, address: &str, offset: ClockOffset, now: Instant) { + let Some(path) = self.paths.get_mut(address) else { + return; + }; + let replaces = path.clock.is_none_or(|(kept, at)| { + offset.within_ms <= kept.within_ms + || now.saturating_duration_since(at) >= CLOCK_KEPT + || offset.offset_ms.abs_diff(kept.offset_ms) + > offset.within_ms.saturating_add(kept.within_ms) + }); + if replaces { + path.clock = Some((offset, now)); + } } /// Forgets the paths neither used nor pinged for twice `ACTIVE_FOR`, and choices @@ -324,7 +442,7 @@ impl Table { at.is_some_and(|at| now.saturating_duration_since(at) < ACTIVE_FOR * 2) }; self.paths - .retain(|_, path| recent(path.used) || recent(path.last_probe)); + .retain(|_, path| recent(path.used) || recent(path.carried) || recent(path.last_probe)); let paths = &self.paths; self.chosen .retain(|key, _| key.split(' ').all(|address| paths.contains_key(address))); @@ -338,6 +456,7 @@ impl Table { .and_then(|path| path.rtt) .map(|rtt| rtt.as_secs_f64() * 1000.0), down: path.is_some_and(|path| path.is_down(now)), + connection: None, } } } @@ -349,17 +468,22 @@ pub(crate) struct Paths { traffic: Option<Arc<Traffic>>, /// The debug firewall, if any. gate: Option<Weak<dyn Gate>>, + /// The node's wall clock, which its PONGs say the time by. + clock: Arc<Clock>, /// A std mutex: never held across an await. table: Mutex<Table>, } impl Paths { /// Over `transports`, by scheme, in order of preference; its own PINGs and PONGs - /// counted in `traffic`, the firewall through `gate`. + /// counted in `traffic`, the firewall through `gate`, PONGs stamped by `clock`; + /// paths pinged every `keepalive` at least (`Table::probe_every`). pub(crate) fn new( transports: Vec<(&'static str, Arc<dyn Transport>)>, traffic: Option<Arc<Traffic>>, gate: Option<Weak<dyn Gate>>, + clock: Arc<Clock>, + keepalive: Duration, ) -> Self { Self { links: transports @@ -368,7 +492,8 @@ impl Paths { .collect(), traffic, gate, - table: Mutex::default(), + clock, + table: Mutex::new(Table::new(keepalive)), } } @@ -421,14 +546,18 @@ impl Paths { /// The usable ones of `addresses` (as paths keep them), best first: paths that are /// up before those down, the one in use first, then by round-trip time, then in - /// the configured order (module docs). Several addresses are paths to measure. - pub(crate) fn rank(&self, addresses: &[String]) -> Vec<String> { + /// the configured order (module docs). For a send (`sending`): several addresses + /// are paths to measure, and the one in use may change; else only a look + /// (`Table::rank`). + pub(crate) fn rank(&self, addresses: &[String], sending: bool) -> Vec<String> { let usable: Vec<(String, usize)> = addresses .iter() .filter(|address| self.usable(address)) .map(|address| (address.clone(), self.preference(address))) .collect(); - self.table.lock_or_recover().rank(usable, Instant::now()) + self.table + .lock_or_recover() + .rank(usable, Instant::now(), sending) } /// The smoothed round-trip time of the path to `address`, if measured. @@ -436,6 +565,12 @@ impl Paths { self.table.lock_or_recover().paths.get(address)?.rtt } + /// The peer's clock at `address`, as its PONGs there say (`ClockOffset`). + pub(crate) fn clock_offset(&self, address: &str) -> Option<ClockOffset> { + let table = self.table.lock_or_recover(); + table.paths.get(address)?.clock.map(|(offset, _)| offset) + } + /// What paths knows about each of `addresses`. pub(crate) fn info(&self, addresses: &[String]) -> Vec<PathInfo> { let now = Instant::now(); @@ -449,9 +584,11 @@ impl Paths { self.located(infos) } - /// `infos` with where each path goes (`located`); asked outside the table's lock. + /// `infos` with where each path goes (`located`) and the transport's connection + /// there; asked outside the table's lock. fn located(&self, mut infos: Vec<PathInfo>) -> Vec<PathInfo> { for info in &mut infos { + info.connection = self.connection(&info.address); info.address = self.location(&info.address); } infos @@ -507,17 +644,40 @@ impl Paths { .unwrap_or_default() } - /// Each transport's (`Transport::local_addresses`), with its scheme. - pub(crate) fn local_addresses(&self) -> Vec<String> { + /// The longest of the transports' (`Transport::sync_reply_timeout`): an answer may + /// come over any of them. + pub(crate) fn sync_reply_timeout(&self) -> Duration { self.links .iter() - .flat_map(|link| { + .map(|link| link.inner.sync_reply_timeout()) + .max() + .unwrap_or(crate::state::SYNC_REPLY_TIMEOUT) + } + + /// Each transport's (`Transport::local_addresses`), with its scheme: taken in turns, + /// each transport's first in the configured order, then each one's second, and so on, + /// so a list cut short (`members::MAX_ADDRESSES`) still has every transport's best. + pub(crate) fn local_addresses(&self) -> Vec<String> { + let mut each: Vec<std::vec::IntoIter<String>> = self + .links + .iter() + .map(|link| { link.inner .local_addresses() .into_iter() .map(|target| link.address(&target)) + .collect::<Vec<_>>() + .into_iter() }) - .collect() + .collect(); + let mut addresses = Vec::new(); + loop { + let before = addresses.len(); + addresses.extend(each.iter_mut().filter_map(Iterator::next)); + if addresses.len() == before { + return addresses; + } + } } /// The schemes of the transports, in order of preference. @@ -541,6 +701,20 @@ impl Paths { .collect() } + /// Each transport's (`Transport::on_seeds_found`). + pub(crate) fn on_seeds_found(&self, found: std::sync::Arc<tokio::sync::Notify>) { + for link in &self.links { + link.inner.on_seeds_found(found.clone()); + } + } + + /// Each transport's (`Transport::on_reachable`). + pub(crate) fn on_reachable(&self, reachable: std::sync::Arc<tokio::sync::Notify>) { + for link in &self.links { + link.inner.on_reachable(reachable.clone()); + } + } + /// The address of the node a seed leads to, as paths keep it: an address of ours /// (`<scheme>:...`) whose transport reads it at once; with a scheme of ours, what /// that transport resolves the rest to (an address the transport can't read as an @@ -623,7 +797,7 @@ impl Paths { match self.send(&encoded).await { Ok(()) => { let mut table = self.table.lock_or_recover(); - table.sent(address); + table.sent(address, Instant::now()); if !failures.is_empty() { table.counts.failovers += 1; drop(table); @@ -665,7 +839,16 @@ impl Paths { let mut inner = message.clone(); inner.peer = target.clone(); inner.address = target; - link.inner.send_encoded(&inner).await + link.inner.send_encoded(&inner).await?; + match &message.message { + Some(parsed) => { + debug!(to = %message.address, bytes = message.payload_len, message = ?parsed, "Sent message"); + } + None => { + debug!(to = %message.address, bytes = message.payload_len, kind = ?message.kind, "Sent message"); + } + } + Ok(()) } /// How the transport of `address` is connected there (`Transport::connection`). @@ -674,6 +857,12 @@ impl Paths { link.inner.connection(&target) } + /// How hole punching to `address` went (`Transport::punching`). + pub(crate) fn punching(&self, address: &str) -> Option<crate::transport::Punching> { + let (link, target) = self.split(address)?; + link.inner.punching(&target) + } + /// Paths' own figures, and every transport's. pub(crate) fn stats(&self) -> TransportStats { let mut stats = TransportStats::default(); @@ -696,11 +885,14 @@ impl Paths { stats } - /// Sends a PING (or with `pong`, a PONG) with `nonce` to `address`, counted - /// (`traffic`). + /// Sends a PING (or with `pong`, a PONG) with `nonce` to `address`, counted and + /// logged (`traffic`). async fn send_probe(&self, address: &str, pong: bool, nonce: u64) -> Result<()> { let message = if pong { - ParsedMessage::Pong(nonce) + ParsedMessage::Pong { + nonce, + at: self.clock.now_millis(), + } } else { ParsedMessage::Ping(nonce) }; @@ -711,6 +903,12 @@ impl Paths { Ok(()) => traffic.sent(address, encoded.kind, encoded.payload_len), Err(_) => traffic.refused(address, encoded.kind, encoded.payload_len), } + // Logged too, under the address: the PINGs and PONGs that come back go up to + // the node and are logged there, and an exchange is seen whole. + let way = if sent.is_ok() { "sent" } else { "refused" }; + traffic + .log + .note(way, address, &message, encoded.payload_len); } sent } @@ -762,6 +960,7 @@ impl Paths { tracing::trace!(from = %address, "Firewall dropped inbound message"); return Ok(()); } + debug!(from = %address, bytes = payload_len, message = ?message, "Received message"); match message { ParsedMessage::Ping(nonce) => { let paths = Arc::clone(&self); @@ -772,10 +971,15 @@ impl Paths { } }); } - ParsedMessage::Pong(nonce) => { - self.table - .lock_or_recover() - .pong(&address, nonce, Instant::now()); + ParsedMessage::Pong { nonce, at } => { + let heard = self.clock.now_millis(); + let location = self.location(&address); + let now = Instant::now(); + let mut table = self.table.lock_or_recover(); + table.located(&address, &location); + if let Some(rtt) = table.pong(&address, nonce, now) { + table.clock(&address, ClockOffset::of(at, heard, rtt), now); + } } _ => {} } @@ -891,6 +1095,8 @@ mod tests { ], None, None, + Arc::default(), + Duration::ZERO, )) }; (on_both("a"), on_both("b")) @@ -952,27 +1158,39 @@ mod tests { let mut table = Table::default(); let mut now = Instant::now(); let set = paths(&["libp2p:x", "mycelium:x"]); - assert_eq!(table.rank(set.clone(), now), ["libp2p:x", "mycelium:x"]); + assert_eq!( + table.rank(set.clone(), now, true), + ["libp2p:x", "mycelium:x"] + ); // libp2p a bit slower: not enough to switch. for _ in 0..8 { now = answer(&mut table, now, &[("libp2p:x", 110), ("mycelium:x", 100)]); } - assert_eq!(table.rank(set.clone(), now), ["libp2p:x", "mycelium:x"]); + assert_eq!( + table.rank(set.clone(), now, true), + ["libp2p:x", "mycelium:x"] + ); assert_eq!(table.counts.switches, 0); // Much slower: switched, and it stays switched while that holds. for _ in 0..8 { now = answer(&mut table, now, &[("libp2p:x", 300), ("mycelium:x", 100)]); } - assert_eq!(table.rank(set.clone(), now), ["mycelium:x", "libp2p:x"]); - assert_eq!(table.rank(set.clone(), now), ["mycelium:x", "libp2p:x"]); + assert_eq!( + table.rank(set.clone(), now, true), + ["mycelium:x", "libp2p:x"] + ); + assert_eq!( + table.rank(set.clone(), now, true), + ["mycelium:x", "libp2p:x"] + ); assert_eq!(table.counts.switches, 1); // Small differences under SWITCH_MIN never switch. let lan = paths(&["libp2p:y", "mycelium:y"]); - table.rank(lan.clone(), now); + table.rank(lan.clone(), now, true); for _ in 0..8 { now = answer(&mut table, now, &[("libp2p:y", 4), ("mycelium:y", 1)]); } - assert_eq!(table.rank(lan, now), ["libp2p:y", "mycelium:y"]); + assert_eq!(table.rank(lan, now, true), ["libp2p:y", "mycelium:y"]); } #[test] @@ -980,14 +1198,17 @@ mod tests { let mut table = Table::default(); let mut now = Instant::now(); let set = paths(&["libp2p:x", "mycelium:x"]); - table.rank(set.clone(), now); + table.rank(set.clone(), now, true); // libp2p stops answering: down after UNANSWERED_DOWN PINGs, then last. for _ in 0..=UNANSWERED_DOWN { now = answer(&mut table, now, &[("mycelium:x", 100)]); - table.rank(set.clone(), now); + table.rank(set.clone(), now, true); } assert!(table.info("libp2p:x", now).down); - assert_eq!(table.rank(set.clone(), now), ["mycelium:x", "libp2p:x"]); + assert_eq!( + table.rank(set.clone(), now, true), + ["mycelium:x", "libp2p:x"] + ); // It answers again: up, and measured. now = answer(&mut table, now, &[("libp2p:x", 50), ("mycelium:x", 100)]); assert!(!table.info("libp2p:x", now).down); @@ -996,10 +1217,36 @@ mod tests { table.failed("mycelium:x", now); assert!(table.info("mycelium:x", now).down); assert!(!table.info("mycelium:x", now + DOWN_FOR).down); - table.sent("mycelium:x"); + table.sent("mycelium:x", now); assert!(!table.info("mycelium:x", now).down); } + /// A look at the choice (`/peers`, figures) changes nothing: no path counts as + /// used, and no faster one is switched to. + #[test] + fn a_look_changes_nothing() { + let mut table = Table::default(); + let mut now = Instant::now(); + let set = paths(&["libp2p:x", "mycelium:x"]); + assert_eq!( + table.rank(set.clone(), now, false), + ["libp2p:x", "mycelium:x"] + ); + assert!(table.paths.is_empty() && table.chosen.is_empty()); + table.rank(set.clone(), now, true); + for _ in 0..4 { + now = answer(&mut table, now, &[("libp2p:x", 500), ("mycelium:x", 10)]); + } + // Much faster, but only looked at: still the one in use. + assert_eq!( + table.rank(set.clone(), now, false), + ["libp2p:x", "mycelium:x"] + ); + assert_eq!(table.counts.switches, 0); + assert_eq!(table.rank(set, now, true), ["mycelium:x", "libp2p:x"]); + assert_eq!(table.counts.switches, 1); + } + #[test] fn only_paths_in_use_are_pinged() { let mut table = Table::default(); @@ -1010,9 +1257,9 @@ mod tests { nonce }; // One address: nothing to choose, nothing to measure. - table.rank(paths(&["libp2p:z"]), now); + table.rank(paths(&["libp2p:z"]), now, true); assert!(table.due(now, &mut next).is_empty()); - table.rank(paths(&["libp2p:x", "mycelium:x"]), now); + table.rank(paths(&["libp2p:x", "mycelium:x"]), now, true); assert_eq!(table.due(now, &mut next).len(), 2); // Not again within PROBE_EVERY; a stale answer is ignored. assert!(table.due(now + PROBE_TIMEOUT, &mut next).is_empty()); @@ -1023,12 +1270,65 @@ mod tests { assert!(table.paths.is_empty() && table.chosen.is_empty()); } + /// A PONG's time against the middle of its round trip: the peer's clock, give or + /// take half the round trip; the narrowest bound kept, until it is old. + #[test] + fn clocks_compare_by_the_middle_of_the_round_trip() { + let rtt = Duration::from_millis(200); + let heard = UnixMillis(10_000); + let ahead = ClockOffset::of(UnixMillis(14_900), heard, rtt); + assert_eq!((ahead.offset_ms, ahead.within_ms), (5_000, 101)); + let behind = ClockOffset::of(UnixMillis(6_900), heard, rtt); + assert_eq!(behind.offset_ms, -3_000); + + let mut table = Table::default(); + let now = Instant::now(); + table.sent("libp2p:z", now); + table.clock("libp2p:z", ahead, now); + let wider = ClockOffset::of(UnixMillis(15_000), heard, rtt * 4); + table.clock("libp2p:z", wider, now); + assert_eq!( + table.paths["libp2p:z"].clock.map(|(kept, _)| kept), + Some(ahead) + ); + table.clock("libp2p:z", wider, now + CLOCK_KEPT); + assert_eq!( + table.paths["libp2p:z"].clock.map(|(kept, _)| kept), + Some(wider) + ); + // A clock set anew: a sample the kept one contradicts takes its place at once, + // its bound wider or not. + let set_anew = ClockOffset::of(UnixMillis(25_000), heard, rtt * 8); + table.clock("libp2p:z", set_anew, now + CLOCK_KEPT); + assert_eq!( + table.paths["libp2p:z"].clock.map(|(kept, _)| kept), + Some(set_anew) + ); + } + + /// A peer's only path is measured too once messages go over it, unless its + /// keepalives just did. + #[test] + fn a_path_sent_over_is_pinged() { + let mut table = Table::default(); + let now = Instant::now(); + table.sent("libp2p:z", now); + assert_eq!(table.due(now, || 1), [("libp2p:z".to_owned(), 1)]); + table.pong("libp2p:z", 1, now + Duration::from_millis(20)); + assert!(table.info("libp2p:z", now).rtt_ms.is_some()); + // A keepalive's PING within PROBE_EVERY: none of ours. + let later = now + PROBE_EVERY; + table.pinged("libp2p:z", 2, later); + table.sent("libp2p:z", later); + assert!(table.due(later, || 3).is_empty()); + } + /// A PING's round trip counts from when it goes, not from when it was found due. #[test] fn round_trips_count_from_the_send() { let mut table = Table::default(); let now = Instant::now(); - table.rank(paths(&["libp2p:x", "mycelium:x"]), now); + table.rank(paths(&["libp2p:x", "mycelium:x"]), now, true); let due = table.due(now, || 7); assert_eq!(due.len(), 2); // Sent a second late (after another path's slow send). @@ -1040,6 +1340,27 @@ mod tests { assert!((rtt - 100.0).abs() < 1e-6, "{rtt}"); } + /// Round trips of another way to the peer (through a relay, then directly) don't + /// weigh on those of the new one. + #[test] + fn a_path_that_goes_elsewhere_is_measured_anew() { + let mut table = Table::default(); + let now = Instant::now(); + table.rank(paths(&["libp2p:x", "mycelium:x"]), now, true); + let ping = |table: &mut Table, nonce: u64, at: Instant, ms: u64| { + table.pinged("libp2p:x", nonce, at); + table.pong("libp2p:x", nonce, at + Duration::from_millis(ms)); + }; + table.located("libp2p:x", "libp2p:/relayed"); + ping(&mut table, 1, now, 120); + table.located("libp2p:x", "libp2p:/relayed"); + ping(&mut table, 2, now + Duration::from_secs(30), 120); + table.located("libp2p:x", "libp2p:/lan"); + ping(&mut table, 3, now + Duration::from_secs(60), 2); + let rtt = table.info("libp2p:x", now).rtt_ms.unwrap(); + assert!((rtt - 2.0).abs() < 1e-6, "{rtt}"); + } + /// The node's keepalive PING on a path is its probe: paths doesn't ping it too, and /// the PONG measures it. #[test] @@ -1051,7 +1372,7 @@ mod tests { nonce += 1; nonce }; - table.rank(paths(&["libp2p:x", "mycelium:x"]), now); + table.rank(paths(&["libp2p:x", "mycelium:x"]), now, true); table.pinged("libp2p:x", 7, now); let due = table.due(now, &mut next); assert_eq!(due.len(), 1); @@ -1105,11 +1426,15 @@ mod tests { vec![("sim", net.transport("b") as Arc<dyn Transport>)], None, Some(Arc::downgrade(&gate)), + Arc::default(), + Duration::ZERO, )); let a = Arc::new(Paths::new( vec![("sim", net.transport("a") as Arc<dyn Transport>)], None, None, + Arc::default(), + Duration::ZERO, )); let received = |paths: &Arc<Paths>| { let (sender, received) = tokio::sync::mpsc::unbounded_channel(); @@ -1167,7 +1492,7 @@ mod tests { Ok(seed.to_owned()) } fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> { - crate::transport::mycelium::encode_message(peer, message) + (crate::transport::sim::WIRE.encode)(peer, message) } async fn send_encoded(&self, _message: &EncodedMessage) -> Result<()> { Err(anyhow!("broken")) @@ -1187,6 +1512,7 @@ mod tests { epoch_seq: 0, ready: false, syncs_with_you: false, + leaving: false, }) } @@ -1202,11 +1528,15 @@ mod tests { ], None, None, + Arc::default(), + Duration::ZERO, ); let b = Arc::new(Paths::new( vec![("sim", net.transport("b") as Arc<dyn Transport>)], None, None, + Arc::default(), + Duration::ZERO, )); let (sender, mut received) = tokio::sync::mpsc::unbounded_channel(); let handler: Arc<PathHandler> = Arc::new(move |from, _, _| { @@ -1219,13 +1549,13 @@ mod tests { tokio::spawn(async move { b.start_consumer(handler).await }); let addresses = vec!["sim:b".to_owned(), "broken:b".to_owned()]; - let ranked = a.rank(&addresses); + let ranked = a.rank(&addresses, true); assert_eq!(ranked, ["broken:b", "sim:b"]); // The configured order. let mut encoded = a.encode(&ranked[0], &some_status()).unwrap(); encoded.peer = "node-b".to_owned(); assert_eq!(a.send_first(&encoded, &ranked).await.unwrap(), "sim:b"); assert_eq!(received.recv().await.unwrap(), "sim:a"); - assert_eq!(a.rank(&addresses), ["sim:b", "broken:b"]); + assert_eq!(a.rank(&addresses, true), ["sim:b", "broken:b"]); // Without the message kept, nothing else can be tried: the transport's error. encoded.message = None; diff --git a/src/transport/sim.rs b/src/transport/sim.rs index 500050b..2f167dc 100644 --- a/src/transport/sim.rs +++ b/src/transport/sim.rs @@ -1,8 +1,9 @@ //! A simulated network for tests: many nodes in one process, each with a //! [`SimTransport`] on a shared [`SimNet`]. //! -//! Messages go through the Mycelium wire encoding (`mycelium::encode_message`), so the -//! codec is exercised as in production. The net delays, loses and duplicates them per +//! Messages go through the wire encoding (`WIRE`, as libp2p and Tor send it), so the +//! codec is exercised as in production; a net can use a transport's envelope instead +//! (`SimNet::with_codec`; Mycelium's tests run a swarm on its envelope). The net delays, loses and duplicates them per //! link, from a seeded generator; nodes can be taken down (everything to and from them //! is lost) and brought up again; and a test can inject a message as if a node had sent //! it, e.g. one that arrives late. @@ -24,11 +25,14 @@ use bytes::Bytes; use tokio::sync::{mpsc, oneshot}; use tracing::{debug, warn}; -use super::mycelium::{decode_message, encode_message}; use super::{EncodedMessage, MessageHandler, MessageKind, ParsedMessage, Transport}; use crate::locks::MutexExt; use crate::testrand::Rng; +/// How long the simulated net may take to deliver a message: well above its delays. +/// A catch-up source that stops sending sync messages is given up after this. +pub(crate) const DELIVERY_LIMIT: Duration = Duration::from_secs(3); + /// How one direction of a link behaves. #[derive(Clone, Debug)] pub(crate) struct LinkParams { @@ -79,6 +83,9 @@ struct NetInner { links: HashMap<(String, String), LinkParams>, down: HashSet<String>, inboxes: HashMap<String, mpsc::UnboundedSender<Delivery>>, + /// Per node, its current run (`restart`): what a transport of an earlier run sends + /// is lost, as from a process that is gone. + runs: HashMap<String, u64>, counts: NetCounts, } @@ -86,17 +93,26 @@ struct NetInner { #[derive(Clone)] pub(crate) struct SimNet { inner: Arc<Mutex<NetInner>>, + codec: Codec, } impl SimNet { + /// Messages in the wire encoding alone (`WIRE`). pub(crate) fn new(seed: u64) -> Self { + Self::with_codec(seed, WIRE) + } + + /// Messages put into bytes by `codec`. + pub(crate) fn with_codec(seed: u64, codec: Codec) -> Self { Self { + codec, inner: Arc::new(Mutex::new(NetInner { rng: Rng::new(seed), default_link: LinkParams::default(), links: HashMap::new(), down: HashSet::new(), inboxes: HashMap::new(), + runs: HashMap::new(), counts: NetCounts::default(), })), } @@ -109,15 +125,31 @@ impl SimNet { /// A transport for a new node `id` on this net. pub(crate) fn transport(&self, id: &str) -> Arc<SimTransport> { let (tx, rx) = mpsc::unbounded_channel(); - let previous = self.lock().inboxes.insert(id.to_owned(), tx); - assert!(previous.is_none(), "node {id} is already on the net"); + let run = { + let mut inner = self.lock(); + let previous = inner.inboxes.insert(id.to_owned(), tx); + assert!(previous.is_none(), "node {id} is already on the net"); + *inner.runs.entry(id.to_owned()).or_default() + }; Arc::new(SimTransport { id: id.to_owned(), + run, net: self.clone(), inbox: tokio::sync::Mutex::new(Some(rx)), }) } + /// Node `id`'s process is gone, as if it crashed: its transport neither sends nor + /// receives any more (messages on their way to it are lost), and [`transport`] + /// may give `id` a new one, for its next run. + /// + /// [`transport`]: Self::transport + pub(crate) fn restart(&self, id: &str) { + let mut inner = self.lock(); + inner.inboxes.remove(id); + *inner.runs.entry(id.to_owned()).or_default() += 1; + } + /// How links without their own parameters behave. pub(crate) fn set_default_link(&self, params: LinkParams) { self.lock().default_link = params; @@ -145,12 +177,13 @@ impl SimNet { } /// Hands a message to the net: decides now whether and when it arrives. - fn send(&self, from: &str, message: &EncodedMessage) { + fn send(&self, from: &str, run: u64, message: &EncodedMessage) { let delays = { let mut guard = self.lock(); let inner = &mut *guard; inner.counts.sent += 1; - if inner.down.contains(from) || inner.down.contains(&message.peer) { + let gone = inner.runs.get(from).is_some_and(|current| *current != run); + if gone || inner.down.contains(from) || inner.down.contains(&message.peer) { inner.counts.lost += 1; return; } @@ -215,7 +248,7 @@ impl SimNet { /// net. For messages a real node wouldn't send at that moment, e.g. one sent long /// ago and delivered late. pub(crate) async fn inject(&self, from: &str, to: &str, message: &ParsedMessage) -> Result<()> { - let encoded = encode_message(to, message)?; + let encoded = (self.codec.encode)(to, message)?; let (handled_tx, handled_rx) = oneshot::channel(); let delivery = Delivery { from: from.to_owned(), @@ -245,10 +278,35 @@ impl SimNet { /// are node IDs. pub(crate) struct SimTransport { id: String, + /// The run of its node it belongs to (`SimNet::restart`). + run: u64, net: SimNet, inbox: tokio::sync::Mutex<Option<mpsc::UnboundedReceiver<Delivery>>>, } +/// How a simulated net puts messages into bytes and back (`SimNet::with_codec`): the +/// wire encoding alone (`WIRE`, as libp2p and Tor send it), or a transport's envelope +/// around it (Mycelium's: `mycelium::CODEC`), so a whole swarm runs on that envelope. +#[derive(Clone, Copy)] +pub(crate) struct Codec { + pub(crate) encode: fn(&str, &ParsedMessage) -> Result<EncodedMessage>, + pub(crate) decode: fn(&[u8]) -> Result<ParsedMessage>, +} + +/// The wire encoding alone (`transport::wire`). +pub(crate) const WIRE: Codec = Codec { + encode: wire_encode, + decode: wire_decode, +}; + +fn wire_encode(peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> { + super::wire::encoded(peer, message) +} + +fn wire_decode(body: &[u8]) -> Result<ParsedMessage> { + Ok(super::wire::decode(body)?) +} + #[async_trait] impl Transport for SimTransport { fn local_id(&self) -> &str { @@ -260,8 +318,8 @@ impl Transport for SimTransport { !id.is_empty() } - fn delivery_limit(&self) -> std::time::Duration { - crate::simtest::DELIVERY_LIMIT + fn delivery_limit(&self) -> Duration { + DELIVERY_LIMIT } /// A seed is a node ID; one no node has doesn't resolve (yet). @@ -274,13 +332,13 @@ impl Transport for SimTransport { } fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> { - encode_message(peer, message) + (self.net.codec.encode)(peer, message) } /// Always accepted: the net loses messages silently, like Mycelium once it has /// taken them. async fn send_encoded(&self, message: &EncodedMessage) -> Result<()> { - self.net.send(&self.id, message); + self.net.send(&self.id, self.run, message); Ok(()) } @@ -290,7 +348,7 @@ impl Transport for SimTransport { bail!("consumer of {} already started", self.id); }; while let Some(delivery) = inbox.recv().await { - match decode_message(&delivery.body) { + match (self.net.codec.decode)(&delivery.body) { Ok(message) => { debug!(node = %self.id, from = %delivery.from, message = ?message, "Simulated delivery"); if let Err(error) = handler(delivery.from, message, delivery.payload_len).await @@ -328,6 +386,7 @@ mod tests { epoch_seq: 0, ready: true, syncs_with_you: false, + leaving: false, }) } diff --git a/src/transport/stats.rs b/src/transport/stats.rs index 398ffdf..68fc3c6 100644 --- a/src/transport/stats.rs +++ b/src/transport/stats.rs @@ -6,6 +6,8 @@ use std::collections::BTreeMap; use serde::Serialize; +use super::Connection; + #[derive(Debug, Clone, Default, Serialize)] #[serde(rename_all = "camelCase")] pub(crate) struct TransportStats { @@ -32,12 +34,12 @@ impl TransportStats { } } -/// How the paths to peers with several addresses do (`transport::paths`). +/// How the paths to peers do (`transport::paths`). #[derive(Debug, Clone, Default, Serialize)] #[serde(rename_all = "camelCase")] pub(crate) struct PathsStats { - /// Every path measured: those of peers this node chose among lately, and those its - /// keepalives went over. + /// Every path measured: those of peers this node chose among or sent to lately, and + /// those its keepalives went over. pub(crate) paths: Vec<PathInfo>, #[serde(flatten)] pub(crate) counts: PathCounts, @@ -58,6 +60,10 @@ pub(crate) struct PathInfo { /// Its PINGs went unanswered three times in a row, or a send there failed within a /// minute: used only when no other path works. pub(crate) down: bool, + /// The transport's connection there, if it has connections (libp2p: direct or + /// relayed; `Transport::connection`). + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) connection: Option<Connection>, } /// Counts since start. @@ -82,6 +88,9 @@ pub(crate) struct IdentityStats { /// Nodes proven at an address, by transport (scheme); a node on several counts on /// each. pub(crate) proven: BTreeMap<String, usize>, + /// Of those, the nodes the transport reaches only through a relay (no direct + /// connection at its address there), by transport; absent where none is. + pub(crate) relayed: BTreeMap<String, usize>, /// Nodes known only from hints (the member list, HELLO_REPLYs), proven nowhere. pub(crate) hinted: usize, #[serde(flatten)] diff --git a/src/transport/tor.rs b/src/transport/tor.rs index 39e0f31..7ecd624 100644 --- a/src/transport/tor.rs +++ b/src/transport/tor.rs @@ -1,4 +1,4 @@ -//! Tor as a transport (`--transports tor`, in builds with the `tor` feature), for +//! Tor as a transport (`--tor-enable`, in builds with the `tor` feature), for //! reachability: every node runs an onion service (arti), reachable from anywhere //! without a forwarded port or a relay. Not for anonymity: a node's other addresses are //! in the member list next to its onion address. @@ -65,15 +65,19 @@ pub(crate) const SCHEME: &str = "tor"; /// The onion service's nickname in arti's state (one per node). const NICKNAME: &str = "continuum"; -/// The largest message read. Sync messages above 16 chunks travel as blobs, and a chunk -/// is at most 4 MiB. -const MAX_MESSAGE: usize = 8 << 20; +/// The largest message read and sent: the largest sync message a node of any +/// configuration sends whole (`MAX_SYNC_MESSAGE`: 16 chunks of 4 MiB; larger ones travel +/// as blobs), with room for its envelope. Over Tor at `MIN_RATE` that is a quarter of an +/// hour, which is the transport's delivery limit then. +const MAX_MESSAGE: usize = crate::swarm::blob::MAX_SYNC_MESSAGE + (1 << 20); /// How long reaching an onion service may take (building circuits, the rendezvous). const CONNECT_TIMEOUT: Duration = Duration::from_secs(60); /// How long the challenge handshake may take, a few round trips. const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(30); /// How long a message may take, before what its size adds (`stream_timeout`). const STREAM_TIMEOUT: Duration = Duration::from_secs(30); +/// The first wait for an answer to a sync message (`Transport::sync_reply_timeout`). +const SYNC_REPLY_TIMEOUT: Duration = Duration::from_secs(30); /// The slowest a message is expected to go over Tor, in bytes per second. const MIN_RATE: u64 = 64 << 10; /// Messages read and not handled yet, at most. @@ -194,6 +198,39 @@ pub(crate) struct TorTransport { } impl TorTransport { + /// This transport's options, checked when the configuration is loaded + /// (`config::Settings::check`), if the node uses it: where arti keeps its state. + pub(crate) fn check(settings: &crate::config::Settings) -> Result<()> { + if settings.tor_state_dir.is_none() { + bail!( + "tor needs tor-state-dir: where arti keeps its state, the onion service's key among it" + ); + } + Ok(()) + } + + /// The transport as `settings` configure it (`transport::start`), proving `key`. + /// None if `stop` came first (bootstrapping arti can take a while). + pub(crate) async fn from_settings( + settings: &crate::config::Settings, + key: &Arc<NodeKey>, + mut stop: watch::Receiver<bool>, + ) -> Result<Option<Self>> { + let state_dir = settings + .tor_state_dir + .clone() + .context("tor needs tor-state-dir")?; + let config = TorConfig { + key: Arc::clone(key), + state_dir, + port: settings.tor_port, + }; + tokio::select! { + transport = Self::start(config) => transport.map(Some), + _ = stop.wait_for(|stop| *stop) => Ok(None), + } + } + /// Launches our onion service, and bootstraps arti in the background (that takes a /// while: it fetches what it needs to know of the Tor network). Until it is done, /// dials wait for it (arti bootstraps on demand), and the onion service is published @@ -534,6 +571,12 @@ impl Transport for TorTransport { stream_timeout(MAX_MESSAGE) } + /// Round trips over Tor take a second or more, and the first message to a peer + /// waits for a connection (`CONNECT_TIMEOUT`, `HANDSHAKE_TIMEOUT`). + fn sync_reply_timeout(&self) -> Duration { + SYNC_REPLY_TIMEOUT + } + fn local_addresses(&self) -> Vec<String> { vec![self.shared.local_id.clone()] } @@ -544,6 +587,15 @@ impl Transport for TorTransport { parse_location(target) } + /// Direct while a connection to its onion service is open: an onion circuit, with no + /// relay of ours in between. + fn connection(&self, target: &str) -> Option<crate::transport::Connection> { + let location = parse_location(target)?; + lock(&self.shared.connections) + .contains_key(&location) + .then_some(crate::transport::Connection::Direct) + } + /// An onion service is its own transport identity: nothing to find out. async fn resolve_seed(&self, seed: &str) -> Result<String> { let location = parse_location(seed) @@ -555,15 +607,7 @@ impl Transport for TorTransport { } fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> { - let body = wire::encode(message)?; - Ok(EncodedMessage { - peer: peer.to_owned(), - address: peer.to_owned(), - payload_len: body.len(), - body: body.into(), - kind: message.kind(), - message: None, - }) + wire::encoded(peer, message) } async fn send_encoded(&self, message: &EncodedMessage) -> Result<()> { diff --git a/src/transport/traffic.rs b/src/transport/traffic.rs index b237db2..b7907fb 100644 --- a/src/transport/traffic.rs +++ b/src/transport/traffic.rs @@ -8,7 +8,8 @@ //! A message dropped by the debug firewall (in the paths layer, below) still counts as //! sent (the firewall plays a lossy network) but not as received. -use std::collections::HashMap; +use std::collections::{HashMap, VecDeque}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, MutexGuard}; use anyhow::Result; @@ -29,6 +30,11 @@ impl Counter { self.messages += 1; self.bytes += bytes as u64; } + + fn absorb(&mut self, other: Counter) { + self.messages += other.messages; + self.bytes += other.bytes; + } } #[derive(Debug, Default, Clone, Copy, Serialize)] @@ -45,9 +51,42 @@ pub(crate) struct ByKind { pub(crate) identity: Counter, /// PING and PONG (`transport::paths`), counted by the paths layer under the address. pub(crate) paths: Counter, + /// LINKS and LINKS_REPLY (`swarm::links`). + pub(crate) links: Counter, } impl ByKind { + fn absorb(&mut self, other: &ByKind) { + for kind in [ + MessageKind::Hello, + MessageKind::HelloReply, + MessageKind::Sync, + MessageKind::Status, + MessageKind::StatusReply, + MessageKind::Blob, + MessageKind::Who, + MessageKind::Ping, + MessageKind::Links, + ] { + let theirs = *other.get(kind); + self.get_mut(kind).absorb(theirs); + } + } + + fn get(&self, kind: MessageKind) -> &Counter { + match kind { + MessageKind::Hello => &self.hello, + MessageKind::HelloReply => &self.hello_reply, + MessageKind::Sync => &self.sync, + MessageKind::Status => &self.status, + MessageKind::StatusReply => &self.status_reply, + MessageKind::Blob => &self.blob, + MessageKind::Who | MessageKind::Me => &self.identity, + MessageKind::Ping | MessageKind::Pong => &self.paths, + MessageKind::Links | MessageKind::LinksReply => &self.links, + } + } + fn get_mut(&mut self, kind: MessageKind) -> &mut Counter { match kind { MessageKind::Hello => &mut self.hello, @@ -58,6 +97,7 @@ impl ByKind { MessageKind::Blob => &mut self.blob, MessageKind::Who | MessageKind::Me => &mut self.identity, MessageKind::Ping | MessageKind::Pong => &mut self.paths, + MessageKind::Links | MessageKind::LinksReply => &mut self.links, } } } @@ -82,6 +122,169 @@ struct Table { #[derive(Debug, Default)] pub(crate) struct Traffic { table: Mutex<Table>, + /// The messages one by one, while the dashboard looks. + pub(crate) log: MessageLog, +} + +/// Messages kept in the log (`MessageLog`) at most, and the text of one. +const LOGGED: usize = 1000; +const LOGGED_TEXT: usize = 8 * 1024; + +/// The node's messages one by one, for the dashboard's console (`api::dashboard`): +/// kept only while it looks (`watch`), the latest `LOGGED`. Those this layer sees: the +/// node's own, not the paths layer's PINGs and PONGs nor the identity layer's WHO and +/// ME (counted under their addresses, `ByKind`), but the PINGs and PONGs the node is +/// handed. +#[derive(Debug, Default)] +pub(crate) struct MessageLog { + watching: AtomicBool, + inner: Mutex<LogInner>, +} + +#[derive(Debug, Default)] +struct LogInner { + next: u64, + entries: VecDeque<Logged>, +} + +/// One message, as the dashboard shows it. +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct Logged { + /// Counts up, from 1. + pub(crate) seq: u64, + /// Unix milliseconds, by the system's clock. + pub(crate) at: u64, + /// `sent`, `received` or `refused` (the transport wouldn't send it). + pub(crate) way: &'static str, + pub(crate) peer: String, + /// The message's type, as the docs name it: `HELLO_REPLY`, `SYNC`, … + #[serde(rename = "type")] + pub(crate) kind: &'static str, + /// Its kind among the counters (`ByKind`'s names): `helloReply`, `sync`, … + pub(crate) counted: &'static str, + pub(crate) bytes: usize, + /// The message, written out (`Debug`; sizes for its bytes), `LOGGED_TEXT` at most. + pub(crate) text: String, + /// What ties it to other messages (`related`): those with the same are one + /// exchange. + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) related: Option<String>, + /// The address it went to or came from, where the paths layer logged it under that + /// (its own PINGs and PONGs) and the node there is known: `peer` is the node then + /// (`api::dashboard`). + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) address: Option<String>, +} + +/// What ties `message`, with `peer`, to other messages: a PING and its PONG (the +/// nonce alone: a PING may go to an address and its PONG come back up from the node +/// there, `paths`), a LINKS and its replies (the nonce), a HELLO and its replies (the run of the +/// node that sent the HELLO, which the reply echoes), a blob's requests and answers, from +/// any peer (its ID), and a delta offered in a SYNC with the blob it is fetched as. None +/// for the rest (a STATUS and its reply carry nothing of the kind). +fn related(peer: &str, message: &ParsedMessage) -> Option<String> { + use crate::transport::{Hex, SyncBody}; + Some(match message { + ParsedMessage::Ping(nonce) | ParsedMessage::Pong { nonce, .. } => { + format!("ping {nonce}") + } + ParsedMessage::Links(request) => format!("links {peer} {}", request.nonce), + ParsedMessage::LinksReply(reply) => format!("links {peer} {}", reply.nonce), + ParsedMessage::Hello(hello) => format!("hello {peer} {}", Hex(&hello.run_id)), + ParsedMessage::HelloReply(reply) => format!("hello {peer} {}", Hex(&reply.run_id)), + ParsedMessage::Blob(blob) => format!("blob {}", blob.id), + ParsedMessage::Sync(envelope) => match &envelope.body { + SyncBody::Delta(spec) => { + format!("blob {}", crate::transport::BlobId::Delta(spec.id())) + } + SyncBody::Message(_) => return None, + }, + _ => return None, + }) +} + +// Read by the dashboard only. +#[cfg_attr(not(feature = "dashboard"), allow(dead_code))] +impl MessageLog { + /// Keeps messages from now on (`true`), or no more, forgetting those kept. + pub(crate) fn watch(&self, on: bool) { + self.watching.store(on, Ordering::Relaxed); + if !on { + self.inner.lock_or_recover().entries.clear(); + } + } + + pub(crate) fn note( + &self, + way: &'static str, + peer: &str, + message: &ParsedMessage, + bytes: usize, + ) { + if !self.watching.load(Ordering::Relaxed) { + return; + } + let at = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |since| since.as_millis() as u64); + let mut text = format!("{message:#?}"); + if text.len() > LOGGED_TEXT { + let mut cut = LOGGED_TEXT; + while !text.is_char_boundary(cut) { + cut -= 1; + } + text.truncate(cut); + text.push_str("\n…"); + } + let (kind, counted) = names(message.kind()); + let mut inner = self.inner.lock_or_recover(); + inner.next += 1; + let seq = inner.next; + inner.entries.push_back(Logged { + seq, + at, + way, + peer: peer.to_owned(), + kind, + counted, + bytes, + text, + related: related(peer, message), + address: None, + }); + if inner.entries.len() > LOGGED { + inner.entries.pop_front(); + } + } + + /// The messages after `seq`, the latest `most` of them. + pub(crate) fn after(&self, seq: u64, most: usize) -> Vec<Logged> { + let inner = self.inner.lock_or_recover(); + let newer: Vec<&Logged> = inner.entries.iter().filter(|e| e.seq > seq).collect(); + newer[newer.len().saturating_sub(most)..] + .iter() + .map(|entry| (*entry).clone()) + .collect() + } +} + +/// A message kind's type name, and its counter's (`ByKind`). +fn names(kind: MessageKind) -> (&'static str, &'static str) { + match kind { + MessageKind::Hello => ("HELLO", "hello"), + MessageKind::HelloReply => ("HELLO_REPLY", "helloReply"), + MessageKind::Sync => ("SYNC", "sync"), + MessageKind::Status => ("STATUS", "status"), + MessageKind::StatusReply => ("STATUS_REPLY", "statusReply"), + MessageKind::Blob => ("BLOB", "blob"), + MessageKind::Who => ("WHO", "identity"), + MessageKind::Me => ("ME", "identity"), + MessageKind::Ping => ("PING", "paths"), + MessageKind::Pong => ("PONG", "paths"), + MessageKind::Links => ("LINKS", "links"), + MessageKind::LinksReply => ("LINKS_REPLY", "links"), + } } impl Traffic { @@ -115,6 +318,22 @@ impl Traffic { self.lock().peers.get(peer).copied() } + /// A node's traffic: counted under it, and under `addresses`, where it is (the + /// layers below the node count what they send and receive on their own there: WHO + /// and ME, the paths layer's PINGs and PONGs). + pub(crate) fn node(&self, node: &str, addresses: &[String]) -> PeerTraffic { + let table = self.lock(); + let mut traffic = table.peers.get(node).copied().unwrap_or_default(); + for address in addresses.iter().filter(|address| address.as_str() != node) { + if let Some(there) = table.peers.get(address) { + traffic.sent.absorb(&there.sent); + traffic.received.absorb(&there.received); + traffic.refused.absorb(&there.refused); + } + } + traffic + } + pub(crate) fn total(&self) -> PeerTraffic { self.lock().total } @@ -152,6 +371,10 @@ impl Transport for CountingTransport { self.inner.delivery_limit() } + fn sync_reply_timeout(&self) -> std::time::Duration { + self.inner.sync_reply_timeout() + } + fn local_addresses(&self) -> Vec<String> { self.inner.local_addresses() } @@ -176,6 +399,14 @@ impl Transport for CountingTransport { self.inner.discovered_seeds() } + fn on_seeds_found(&self, found: std::sync::Arc<tokio::sync::Notify>) { + self.inner.on_seeds_found(found) + } + + fn on_reachable(&self, reachable: std::sync::Arc<tokio::sync::Notify>) { + self.inner.on_reachable(reachable) + } + fn transports(&self) -> Vec<String> { self.inner.transports() } @@ -204,6 +435,10 @@ impl Transport for CountingTransport { self.inner.rtt(peer) } + fn clock_offset(&self, peer: &str) -> Option<crate::transport::paths::ClockOffset> { + self.inner.clock_offset(peer) + } + async fn resolve_seed(&self, seed: &str) -> Result<String> { self.inner.resolve_seed(seed).await } @@ -221,6 +456,12 @@ impl Transport for CountingTransport { self.traffic .refused(&message.peer, message.kind, message.payload_len); } + if let Some(parsed) = &message.message { + let way = if result.is_ok() { "sent" } else { "refused" }; + self.traffic + .log + .note(way, &message.peer, parsed, message.payload_len); + } result } @@ -229,6 +470,7 @@ impl Transport for CountingTransport { self.inner .start_consumer(Box::new(move |sender, message, payload_len| { traffic.received(&sender, message.kind(), payload_len); + traffic.log.note("received", &sender, &message, payload_len); handler(sender, message, payload_len) })) .await diff --git a/src/transport/wire.rs b/src/transport/wire.rs index fcbee85..7717200 100644 --- a/src/transport/wire.rs +++ b/src/transport/wire.rs @@ -10,11 +10,11 @@ use thiserror::Error; -use super::{MessageKind, ParsedMessage}; +use super::{EncodedMessage, MessageKind, ParsedMessage}; /// The wire format's version, the first byte of every payload. Bump it with every change /// to a message type or a codec (`golden_bytes` fails then). -pub(crate) const VERSION: u8 = 6; +pub(crate) const VERSION: u8 = 14; /// What the second byte of a payload says. fn tag(kind: MessageKind) -> u8 { @@ -29,6 +29,8 @@ fn tag(kind: MessageKind) -> u8 { MessageKind::Me => 7, MessageKind::Ping => 8, MessageKind::Pong => 9, + MessageKind::Links => 10, + MessageKind::LinksReply => 11, } } @@ -50,6 +52,21 @@ pub(crate) enum WireError { Decode(#[from] rmp_serde::decode::Error), } +/// `message` to `peer`, encoded as it goes on the wire alone (no envelope: libp2p, +/// Tor, the simulator); `peer` is its address too. +#[cfg_attr(not(any(feature = "libp2p", feature = "tor", test)), allow(dead_code))] +pub(crate) fn encoded(peer: &str, message: &ParsedMessage) -> anyhow::Result<EncodedMessage> { + let body = encode(message)?; + Ok(EncodedMessage { + peer: peer.to_owned(), + address: peer.to_owned(), + payload_len: body.len(), + body: body.into(), + kind: message.kind(), + message: None, + }) +} + /// The payload of `message`. pub(crate) fn encode(message: &ParsedMessage) -> Result<Vec<u8>, WireError> { let mut payload = vec![VERSION, tag(message.kind())]; @@ -64,9 +81,10 @@ pub(crate) fn encode(message: &ParsedMessage) -> Result<Vec<u8>, WireError> { ParsedMessage::Who(proof) | ParsedMessage::Me(proof) => { rmp_serde::encode::write(&mut payload, proof)? } - ParsedMessage::Ping(nonce) | ParsedMessage::Pong(nonce) => { - rmp_serde::encode::write(&mut payload, nonce)? - } + ParsedMessage::Ping(nonce) => rmp_serde::encode::write(&mut payload, nonce)?, + ParsedMessage::Pong { nonce, at } => rmp_serde::encode::write(&mut payload, &(nonce, at))?, + ParsedMessage::Links(request) => rmp_serde::encode::write(&mut payload, request)?, + ParsedMessage::LinksReply(reply) => rmp_serde::encode::write(&mut payload, reply)?, } Ok(payload) } @@ -98,7 +116,12 @@ pub(crate) fn decode(payload: &[u8]) -> Result<ParsedMessage, WireError> { } else if is(MessageKind::Ping) { ParsedMessage::Ping(rmp_serde::from_slice(body)?) } else if is(MessageKind::Pong) { - ParsedMessage::Pong(rmp_serde::from_slice(body)?) + let (nonce, at) = rmp_serde::from_slice(body)?; + ParsedMessage::Pong { nonce, at } + } else if is(MessageKind::Links) { + ParsedMessage::Links(rmp_serde::from_slice(body)?) + } else if is(MessageKind::LinksReply) { + ParsedMessage::LinksReply(rmp_serde::from_slice(body)?) } else { return Err(WireError::UnknownKind(kind)); }) @@ -180,7 +203,7 @@ pub(crate) mod codec { /// An Automerge change hash. pub(crate) mod change_hash { use super::*; - use automerge::ChangeHash; + use crate::document::ChangeHash; pub(crate) fn serialize<S: Serializer>( value: &ChangeHash, @@ -219,8 +242,8 @@ pub(crate) mod codec { /// empty), or as a digest (none listed). pub(crate) mod heads { use super::*; + use crate::document::ChangeHash; use crate::transport::{HeadsSummary, MAX_LISTED_HEADS}; - use automerge::ChangeHash; pub(crate) fn serialize<S: Serializer>( value: &HeadsSummary, @@ -267,7 +290,7 @@ pub(crate) mod codec { /// Change hashes, each a 32-byte binary. pub(crate) mod change_hashes { use super::*; - use automerge::ChangeHash; + use crate::document::ChangeHash; pub(crate) fn serialize<S: Serializer>( value: &[ChangeHash], @@ -325,15 +348,134 @@ pub(crate) mod codec { } } +/// One message of every kind, for the codecs' tests (this one's, and each transport's +/// envelope around it). +#[cfg(test)] +pub(crate) fn samples() -> Vec<ParsedMessage> { + use crate::document::ChangeHash; + use crate::document::replica::Epoch; + use crate::transport::{ + BlobBody, BlobId, BlobMessage, DeltaSpec, HeadsSummary, HelloReply, HelloRequest, + IdentityProof, ListedPeer, Status, SyncBody, SyncEnvelope, + }; + let epoch = Epoch { + generation: 2, + id: [3; 32], + }; + let status = Status { + heads: HeadsSummary::of(vec![ChangeHash([1; 32])]), + epoch, + epoch_seq: 1, + ready: true, + syncs_with_you: true, + leaving: false, + }; + let proof = IdentityProof { + key: [8; 32], + signature: [9; 64], + }; + vec![ + ParsedMessage::Hello(HelloRequest { + swarm_id: Some("swarm".into()), + epoch, + epoch_seq: 1, + auth: vec![1, 2, 3], + ready: true, + run_id: [4; 16], + handshake: 7, + }), + ParsedMessage::HelloReply(HelloReply { + swarm_id: "swarm".into(), + peers: vec![ListedPeer { + id: "p".into(), + addresses: vec!["a".into()], + }], + epoch, + epoch_seq: 1, + auth: Vec::new(), + ready: false, + run_id: [4; 16], + away_ms: Some(5), + sender_run_id: [6; 16], + }), + ParsedMessage::Sync(SyncEnvelope { + swarm_id: "swarm".into(), + epoch, + epoch_seq: 1, + body: SyncBody::Delta(DeltaSpec { + from: Vec::new(), + to: vec![ChangeHash([2; 32])], + }), + ready: true, + }), + ParsedMessage::Status(status.clone()), + ParsedMessage::StatusReply(status), + ParsedMessage::Blob(BlobMessage { + epoch, + epoch_seq: 1, + id: BlobId::Content([5; 32]), + body: BlobBody::Missing, + }), + ParsedMessage::Who(proof.clone()), + ParsedMessage::Me(proof), + ParsedMessage::Ping(7), + ParsedMessage::Pong { + nonce: u64::MAX, + at: crate::clock::UnixMillis(5), + }, + ParsedMessage::Links(crate::transport::LinksRequest { + epoch, + epoch_seq: 1, + nonce: 3, + subscribe_secs: 60, + }), + ParsedMessage::LinksReply(crate::transport::LinksReply { + epoch, + epoch_seq: 1, + nonce: 3, + links: vec![crate::transport::Link { + peer: "p".into(), + transport: Some("libp2p".into()), + relayed: true, + down: false, + rtt_us: Some(1500), + dormant: false, + others: vec![crate::transport::OtherPath { + transport: "mycelium".into(), + rtt_us: None, + relayed: false, + }], + clock: Some(crate::transport::paths::ClockOffset { + offset_ms: -2_500, + within_ms: 40, + }), + }], + }), + ] +} + #[cfg(test)] mod tests { use super::*; + use crate::document::ChangeHash; use crate::document::replica::Epoch; use crate::transport::{ BlobBody, BlobId, BlobMessage, HeadsSummary, HelloReply, HelloRequest, IdentityProof, ListedPeer, MAX_LISTED_HEADS, Status, }; - use automerge::ChangeHash; + + /// Every kind decodes to what was encoded: re-encoded, the same bytes. + #[test] + fn every_kind_round_trips() { + let samples = samples(); + assert_eq!(samples.len(), 12, "a kind is missing from `samples`"); + for message in samples { + let bytes = encode(&message).unwrap(); + let decoded = decode(&bytes).unwrap(); + assert_eq!(decoded.kind(), message.kind()); + assert_eq!(encode(&decoded).unwrap(), bytes, "{message:?}"); + } + } #[test] fn handshakes_round_trip() { @@ -398,6 +540,7 @@ mod tests { epoch_seq: 0, ready: true, syncs_with_you: false, + leaving: false, }; let decoded = decode(&encode(&ParsedMessage::StatusReply(status.clone())).unwrap()).unwrap(); @@ -420,6 +563,7 @@ mod tests { epoch_seq: 0, ready: true, syncs_with_you: true, + leaving: false, }; assert!(decode(&encode(&ParsedMessage::Status(status)).unwrap()).is_err()); } @@ -454,13 +598,15 @@ mod tests { } fn hex(bytes: &[u8]) -> String { - bytes.iter().map(|byte| format!("{byte:02x}")).collect() + crate::transport::Hex(bytes).to_string() } /// The exact bytes of a few messages. If this fails after a change to a message type /// or a codec, the change is a new wire format: bump `VERSION` and take the new bytes. #[test] fn golden_bytes() { + // The version byte follows `VERSION`: bumped for a change to other messages. + let v = format!("{VERSION:02x}"); let epoch = Epoch { generation: 2, id: [3; 32], @@ -479,7 +625,7 @@ mod tests { }); assert_eq!( hex(&encode(&hello).unwrap()), - format!("060097a5737761726d{epoch_hex}01c403010203c3{}07", run("04")) + format!("{v}0097a5737761726d{epoch_hex}01c403010203c3{}07", run("04")) ); let reply = ParsedMessage::HelloReply(HelloReply { @@ -499,7 +645,7 @@ mod tests { assert_eq!( hex(&encode(&reply).unwrap()), format!( - "060199a5737761726d9192a17091a161{epoch_hex}01c400c2{}05{}", + "{v}0199a5737761726d9192a17091a161{epoch_hex}01c400c2{}05{}", run("04"), run("06") ) @@ -511,10 +657,11 @@ mod tests { epoch_seq: 1, ready: true, syncs_with_you: false, + leaving: false, }); assert_eq!( hex(&encode(&status).unwrap()), - format!("0603959290c400{epoch_hex}01c3c2") + format!("{v}03969290c400{epoch_hex}01c3c2c2") ); let who = ParsedMessage::Who(IdentityProof { @@ -523,7 +670,7 @@ mod tests { }); assert_eq!( hex(&encode(&who).unwrap()), - format!("060692c420{}c440{}", "08".repeat(32), "09".repeat(64)) + format!("{v}0692c420{}c440{}", "08".repeat(32), "09".repeat(64)) ); let ParsedMessage::Me(proof) = decode( &encode(&ParsedMessage::Me(IdentityProof { @@ -537,14 +684,16 @@ mod tests { }; assert_eq!((proof.key, proof.signature), ([8; 32], [9; 64])); - assert_eq!(hex(&encode(&ParsedMessage::Ping(7)).unwrap()), "060807"); - assert_eq!( - hex(&encode(&ParsedMessage::Pong(u64::MAX)).unwrap()), - "0609cfffffffffffffffff" - ); - let ParsedMessage::Pong(nonce) = decode(&[VERSION, 9, 0x2a]).unwrap() else { + assert_eq!(hex(&encode(&ParsedMessage::Ping(7)).unwrap()), format!("{v}0807")); + let pong = ParsedMessage::Pong { + nonce: u64::MAX, + at: crate::clock::UnixMillis(5), + }; + assert_eq!(hex(&encode(&pong).unwrap()), format!("{v}0992cfffffffffffffffff05")); + let ParsedMessage::Pong { nonce, at } = decode(&[VERSION, 9, 0x92, 0x2a, 0x07]).unwrap() + else { panic!("not a PONG"); }; - assert_eq!(nonce, 42); + assert_eq!((nonce, at.as_u64()), (42, 7)); } } diff --git a/src/units.rs b/src/units.rs index ac08ebf..28adab9 100644 --- a/src/units.rs +++ b/src/units.rs @@ -171,6 +171,17 @@ pub(crate) fn optional_duration<'de, D: serde::Deserializer<'de>>( duration_option::deserialize(d).map(Some) } +/// Writes an optional `Duration` as [`duration_option`] does; `None` as null. +pub(crate) fn serialize_optional_duration<S: serde::Serializer>( + value: &Option<Duration>, + s: S, +) -> Result<S::Ok, S::Error> { + match value { + Some(value) => duration_option::serialize(value, s), + None => s.serialize_none(), + } +} + /// Serde for a size option: a string as [`parse_size`] reads it, or a number of bytes; /// written as [`format_size`]. pub(crate) mod size_option { |