1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
|
//! The store a node of a swarm holding one holds (`docs/store.md`, What the store takes
//! from the document): read from its document, each event taken as the store would take
//! it from anywhere, and by the node's own: what it refused by what it took before, and
//! what its operator had it drop, which it keeps in a file of its own (`--store-file`).
//! It keeps nothing else of the store across a restart: whenever it starts, it reads the
//! store anew from its document, from the root.
//!
//! A worker reads the events the document holds that it hasn't read yet, whenever the
//! document changes (`run_worker`). Which change put an event, and how often, changes
//! nothing; nor does another document (a drop, a rejoin), whose events it reads as it
//! comes, the store going on.
use std::collections::{BTreeMap, BTreeSet, HashSet};
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use anyhow::{Context, Result, bail};
use serde::{Deserialize, Serialize};
use tracing::{error, info, warn};
use super::fold::Federation;
use super::store::Id;
use super::store::memory::{Offered, Store};
use crate::document::{ReadDoc, events, meta, store};
use crate::files::{FileKind, Files};
use crate::locks::MutexExt;
/// What a node keeps of its own of its store across restarts, as its file holds it: the
/// events it refused by what it took before, and those its operator had it drop, by ID.
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct Kept {
#[serde(default)]
refused: BTreeSet<String>,
#[serde(default)]
dropped: BTreeSet<String>,
}
impl Kept {
fn of(own: &[Id], dropped: &[Id]) -> Self {
Kept {
refused: own.iter().map(Id::to_string).collect(),
dropped: dropped.iter().map(Id::to_string).collect(),
}
}
/// The IDs it holds: what was refused, and what dropped.
fn ids(&self) -> Option<(Vec<Id>, Vec<Id>)> {
let ids = |set: &BTreeSet<String>| {
set.iter()
.map(|id| Id::from_hex(id))
.collect::<Option<Vec<Id>>>()
};
Some((ids(&self.refused)?, ids(&self.dropped)?))
}
}
/// The store a node holds, if its swarm holds one, and the file it keeps its own in.
#[derive(Default)]
pub(crate) struct Holding {
file: Option<(PathBuf, Arc<Files>)>,
// A std mutex: never held while waiting for the document.
inner: Mutex<Inner>,
}
#[derive(Default)]
struct Inner {
/// None until a document holding a store holds its root.
store: Option<Store<Federation>>,
/// The document read from (`Replica::epoch_seq`), and its events read.
epoch_seq: Option<u32>,
read: HashSet<Id>,
/// What the file holds, or will once written.
kept: Kept,
/// The root that is none, logged once.
none: Option<Id>,
}
/// What a document holds that its store wasn't offered from it.
pub(crate) struct Unread {
epoch_seq: u32,
/// The root, where the store isn't started yet.
root: Option<Vec<u8>>,
events: BTreeMap<Id, Vec<u8>>,
}
impl Holding {
/// Keeping the node's own in the file at `path`, if any, which holds what it kept
/// before: a missing file nothing. One that doesn't read is an error: the node would
/// take what it refused.
pub(crate) fn open(path: Option<PathBuf>, files: Arc<Files>) -> Result<Self> {
let Some(path) = path else {
return Ok(Holding::default());
};
let kept = match files
.read(FileKind::Store, &path)
.with_context(|| format!("Reading the store file {}", path.display()))?
{
None => Kept::default(),
Some(bytes) => match serde_json::from_slice::<Kept>(&bytes) {
Ok(kept) if kept.ids().is_some() => kept,
_ => bail!(
"The store file {} doesn't read: move it away only knowing the node may then take what it refused",
path.display()
),
},
};
Ok(Holding {
file: Some((path, files)),
inner: Mutex::new(Inner {
kept,
..Inner::default()
}),
})
}
/// What `doc` (the document `epoch_seq`) holds that the store wasn't offered from it,
/// if it holds a store: everything, from another document.
pub(crate) fn unread(&self, doc: &impl ReadDoc, epoch_seq: u32) -> Option<Unread> {
if !matches!(
crate::access::policy::of(doc),
crate::access::policy::Policy::Store(_)
) {
return None;
}
let swarm = match store::read_plain(doc, &meta::path(&[meta::SWARM])) {
Ok(Some(serde_json::Value::String(id))) => Id::from_hex(&id)?,
_ => return None,
};
let inner = self.inner.lock_or_recover();
let same = inner.epoch_seq == Some(epoch_seq);
let events = events::ids(doc)
.into_iter()
.filter(|id| !same || !inner.read.contains(id))
.filter_map(|id| Some((id, events::bytes(doc, &id)?)))
.collect();
let root = match &inner.store {
None => Some(events::bytes(doc, &swarm)?),
Some(_) => None,
};
Some(Unread {
epoch_seq,
root,
events,
})
}
/// Offers the store what was `unread`: started from the root first, by what the node
/// kept of its own. What it refuses of its own from then on, it keeps.
pub(crate) fn read(&self, unread: Unread) {
let mut inner = self.inner.lock_or_recover();
let inner = &mut *inner;
if inner.epoch_seq != Some(unread.epoch_seq) {
inner.epoch_seq = Some(unread.epoch_seq);
inner.read.clear();
}
if inner.store.is_none() {
let Some(root) = unread.root else {
return;
};
let id = Id::of(&root);
let (own, dropped) = inner.kept.ids().unwrap_or_default();
match Store::new(root, Vec::new()) {
Ok(store) => {
info!(store = %id, refused = own.len(), dropped = dropped.len(), "Reading the store from the document");
inner.store = Some(store.refusing(&own, &dropped));
}
Err(why) => {
if inner.none.replace(id) != Some(id) {
error!(store = %id, why = %why.0, "The document's root is no store's: not reading it");
}
return;
}
}
}
let store = inner.store.as_mut().expect("started");
let (mut taken, mut refused) = (0, 0);
for (id, bytes) in unread.events {
inner.read.insert(id);
match store.offer(bytes) {
Offered::Taken(ids) => taken += ids.len(),
Offered::Refused(why) => {
refused += 1;
info!(event = %id, why = ?why, "The store refuses an event the document holds");
}
Offered::Held | Offered::Waiting => {}
}
}
if taken > 0 || refused > 0 {
info!(taken, refused, "Read events from the document");
}
let (own, dropped) = store.own();
let kept = Kept::of(&own, &dropped);
if kept != inner.kept {
for id in kept.refused.difference(&inner.kept.refused) {
warn!(event = %id, "Refused an event its history takes, by what this node took before: kept so");
}
inner.kept = kept;
self.keep(&inner.kept);
}
}
/// Writes what the node keeps of its own to its file, if it has one.
fn keep(&self, kept: &Kept) {
let Some((path, files)) = &self.file else {
return;
};
let written = serde_json::to_vec_pretty(kept)
.map_err(std::io::Error::from)
.and_then(|mut bytes| {
bytes.push(b'\n');
files.write_atomic(FileKind::Store, path, &bytes)
});
if let Err(error) = written {
error!(path = %path.display(), error = %error, "Couldn't write the store file: a restart may take what this node refused");
}
}
/// `f` of the store, where the node holds one.
pub(crate) fn with<T>(&self, f: impl FnOnce(&Store<Federation>) -> T) -> Option<T> {
self.inner.lock_or_recover().store.as_ref().map(f)
}
}
/// Reads the store from the document whenever it changes.
pub(crate) async fn run_worker(state: crate::state::AppState) {
let mut changed = state.change_tx.subscribe();
loop {
let holding = Arc::clone(&state.holding);
let doc = Arc::clone(&state.doc);
let read = tokio::task::spawn_blocking(move || {
// Read from the document under its lock; taken without it.
let unread = {
let replica = doc.blocking_read();
holding.unread(&replica.doc, replica.epoch_seq)
};
if let Some(unread) = unread {
holding.read(unread);
}
});
if let Err(error) = read.await {
error!(error = %error, "Reading the store from the document failed");
}
if changed.changed().await.is_err() {
return;
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::document::typed::{TimestampFormat, Val};
use crate::document::{Document, ScalarValue};
use crate::federation::event::Checkpoint;
use crate::federation::harness::{self, Party, Spec};
use crate::federation::store::Fold;
use crate::federation::version::Version;
fn root_of(spec: &Spec) -> Vec<u8> {
Checkpoint {
at: Vec::new(),
keeps: Vec::new(),
version: Version::read(&spec.json()).expect("a version"),
}
.bytes()
}
fn put(doc: &mut Document, event: &[u8]) {
crate::document::testing::change(doc, |doc| {
store::write_val_path(
doc,
&events::event(&Id::of(event)),
&Val::Scalar(ScalarValue::Bytes(event.to_vec())),
store::Writing::sticky(TimestampFormat::default()),
)
.unwrap();
});
}
fn read(holding: &Holding, doc: &Document, epoch_seq: u32) {
holding.read(holding.unread(doc, epoch_seq).unwrap());
}
#[test]
fn the_store_takes_the_events_the_document_holds() {
let tv = Party::new("tv", 1);
let first = Spec::first(&[&tv], &[]);
let root = root_of(&first);
let mut doc = events::started(Federation::NAME, &root).unwrap();
let holding = Holding::default();
assert!(holding.unread(&Document::new(), 0).is_none());
read(&holding, &doc, 0);
assert_eq!(holding.with(|store| store.id()), Some(Id::of(&root)));
let write = tv.write(&[Id::of(&root)], "home/tv/x", harness::str("1"));
let on = tv.write(&[Id::of(&write)], "home/tv/x", harness::str("2"));
// What it builds on comes later: it waits for it.
put(&mut doc, &on);
read(&holding, &doc, 0);
assert_eq!(
holding.with(|store| store.is_waiting(&Id::of(&on))),
Some(true)
);
put(&mut doc, &write);
read(&holding, &doc, 0);
assert_eq!(
holding.with(|store| store.taken().to_vec()),
Some(vec![Id::of(&write), Id::of(&on)])
);
// Another document holding them: nothing new.
let other = doc.fork();
read(&holding, &other, 1);
assert_eq!(holding.with(|store| store.taken().len()), Some(2));
}
/// Two version events changing version 0, each backed by tv and one other.
#[test]
fn a_node_read_anew_refuses_what_it_refused_of_its_own() {
let (tv, lass, makefu) = (
Party::new("tv", 1),
Party::new("lass", 2),
Party::new("makefu", 3),
);
let first = Spec::first(&[&tv, &lass, &makefu], &[]);
let root = root_of(&first);
let on = first.hash();
let one = first.after(on);
let mut two = first.after(on);
two.rules.push(("names/x".to_owned(), harness::str("tv")));
let p1 = tv.propose(&[Id::of(&root)], &one.json());
let p2 = lass.propose(&[Id::of(&root)], &two.json());
let votes = [
tv.vote(&[Id::of(&p1)], on, one.hash(), true),
lass.vote(&[Id::of(&p1)], on, one.hash(), true),
tv.vote(&[Id::of(&p2)], on, two.hash(), true),
makefu.vote(&[Id::of(&p2)], on, two.hash(), true),
];
let v1 = harness::version_event(&[Id::of(&votes[0]), Id::of(&votes[1])], one.hash());
let v2 = harness::version_event(&[Id::of(&votes[2]), Id::of(&votes[3])], two.hash());
let before: Vec<&Vec<u8>> = [&p1, &p2].into_iter().chain(&votes).collect();
let taken = |holding: &Holding| holding.with(|store| store.taken().to_vec()).unwrap();
// The one a store reading all of them at once takes, and the other, which a node
// took as it came first.
let mut all = events::started(Federation::NAME, &root).unwrap();
for event in before.iter().copied().chain([&v1, &v2]) {
put(&mut all, event);
}
let forgetting = Holding::default();
read(&forgetting, &all, 0);
let (later, first_taken) = if taken(&forgetting).contains(&Id::of(&v1)) {
(v1, v2)
} else {
(v2, v1)
};
let dir = crate::files::scratch_dir("held");
let path = dir.join("store.json");
let files = Arc::new(Files::default());
let holding = Holding::open(Some(path.clone()), Arc::clone(&files)).unwrap();
let mut doc = events::started(Federation::NAME, &root).unwrap();
for event in before.iter().copied().chain([&first_taken]) {
put(&mut doc, event);
}
read(&holding, &doc, 0);
put(&mut doc, &later);
read(&holding, &doc, 0);
assert!(taken(&holding).contains(&Id::of(&first_taken)));
assert!(!taken(&holding).contains(&Id::of(&later)));
// Restarted: the store read anew, all at once, refuses the later as before.
let anew = Holding::open(Some(path.clone()), Arc::clone(&files)).unwrap();
read(&anew, &all, 0);
assert!(taken(&anew).contains(&Id::of(&first_taken)));
assert!(!taken(&anew).contains(&Id::of(&later)));
std::fs::write(&path, b"{\"refused\": [\"not an ID\"]}").unwrap();
assert!(Holding::open(Some(path), files).is_err());
}
}
|