summaryrefslogtreecommitdiffstats
path: root/src/remake/kind.rs
diff options
context:
space:
mode:
Diffstat (limited to 'src/remake/kind.rs')
-rw-r--r--src/remake/kind.rs267
1 files changed, 204 insertions, 63 deletions
diff --git a/src/remake/kind.rs b/src/remake/kind.rs
index b709a05..8131f7f 100644
--- a/src/remake/kind.rs
+++ b/src/remake/kind.rs
@@ -10,27 +10,30 @@
//! * where a write made in the old epoch after the base goes in the new one
//! ([`RemakeKind::translate`]).
//!
-//! Kinds: [`Compact`] (same content, no history) and [`Replace`] (an operator's
-//! document, with rules for where later writes go).
+//! Kinds: [`Compact`] (same content, no history), [`Replace`] (an operator's document
+//! in store form, with rules for where later writes go) and [`Import`] (the same with
+//! an Automerge document, e.g. one exported with `GET /admin/document`).
//!
//! Protocol data under `_continuum` is not a kind's business: the generic part carries
//! it forward and replays it unchanged.
use automerge::AutoCommit;
+use base64::engine::{Engine, general_purpose::STANDARD as BASE64};
use serde::{Deserialize, Serialize};
use serde_json::Value as JsonValue;
-use super::snapshot::{canonical, META_ROOT};
+use super::snapshot::{META_ROOT, canonical};
use crate::document::store::{self, StoreError};
/// Name of the kind used when a request doesn't name one.
pub(crate) const DEFAULT_KIND: &str = "compact";
-/// What the coordinator stores in the proposal for every node.
+/// What the coordinator prepares for every node.
#[derive(Debug, Default)]
pub(crate) struct Prepared {
- /// Input to [`RemakeKind::build`]; replicated with the proposal (old epoch only).
- pub(crate) content: JsonValue,
+ /// Input to [`RemakeKind::build`], if the kind takes any. The proposal refers to it
+ /// by hash; voters fetch it from the coordinator and each other (`content`).
+ pub(crate) content: Option<Vec<u8>>,
/// Input to [`RemakeKind::translate`]; also kept in the new epoch's metadata, so
/// nodes that rejoin later can translate their writes.
pub(crate) translation: JsonValue,
@@ -62,7 +65,7 @@ pub(crate) trait RemakeKind: Send + Sync + 'static {
fn build(
&self,
old_at_base: &AutoCommit,
- content: &JsonValue,
+ content: Option<&[u8]>,
target: &mut AutoCommit,
) -> Result<(), StoreError>;
@@ -75,6 +78,7 @@ pub(crate) fn by_name(name: &str) -> Option<&'static dyn RemakeKind> {
match name {
"compact" => Some(&Compact),
"replace" => Some(&Replace),
+ "import" => Some(&Import),
_ => None,
}
}
@@ -104,7 +108,7 @@ impl RemakeKind for Compact {
fn build(
&self,
old_at_base: &AutoCommit,
- _content: &JsonValue,
+ _content: Option<&[u8]>,
target: &mut AutoCommit,
) -> Result<(), StoreError> {
store::materialize_into(old_at_base, target, &is_reserved)
@@ -196,45 +200,26 @@ impl RemakeKind for Replace {
));
}
- let mut rules = Vec::with_capacity(spec.rules.len());
- for rule in spec.rules {
- let from = rule.from.segments()?;
- let to = match (rule.to, rule.drop) {
- (Some(to), false) => Some(to.segments()?),
- (None, true) => None,
- _ => {
- return Err(format!(
- "rule for {from:?} needs exactly one of \"to\" and \"drop\""
- ))
- }
- };
- if is_reserved(&from) || to.as_deref().is_some_and(is_reserved) {
- return Err(format!(
- "rules must not touch the reserved key {META_ROOT:?}"
- ));
- }
- if rules.iter().any(|existing: &Rule| existing.from == from) {
- return Err(format!("two rules for {from:?}"));
- }
- rules.push(Rule { from, to });
- }
-
+ let content =
+ serde_json::to_vec(&canonical(spec.document)).map_err(|error| error.to_string())?;
Ok(Prepared {
- content: canonical(spec.document),
- translation: serde_json::to_value(Rules { rules })
- .map_err(|error| error.to_string())?,
+ content: Some(content),
+ translation: prepare_rules(spec.rules)?,
})
}
fn build(
&self,
_old_at_base: &AutoCommit,
- content: &JsonValue,
+ content: Option<&[u8]>,
target: &mut AutoCommit,
) -> Result<(), StoreError> {
- // Canonical again: the content went through the old epoch's document, and the
- // operations (hence the epoch ID) must not depend on any key order.
- let JsonValue::Object(document) = canonical(content.clone()) else {
+ let content = content.ok_or_else(|| anyhow::anyhow!("replace without content"))?;
+ let document: JsonValue = serde_json::from_slice(content)
+ .map_err(|error| anyhow::anyhow!("replacement document: {error}"))?;
+ // Canonical again: the operations (hence the epoch ID) must not depend on any
+ // key order.
+ let JsonValue::Object(document) = canonical(document) else {
return Ok(()); // propose only accepts objects
};
for (key, value) in document {
@@ -246,31 +231,130 @@ impl RemakeKind for Replace {
}
fn translate(&self, path: &[String], translation: &JsonValue) -> Translation {
- let Ok(Rules { rules }) = serde_json::from_value::<Rules>(translation.clone()) else {
- return Translation::Quarantine("unreadable replacement rules".to_owned());
- };
- let Some(rule) = rules
- .iter()
- .filter(|rule| path.starts_with(&rule.from))
- .max_by_key(|rule| rule.from.len())
- else {
- return Translation::Quarantine("no replacement rule covers this path".to_owned());
+ translate_by_rules(path, translation)
+ }
+}
+
+/// Validates rules from a request; the translation to store.
+fn prepare_rules(specs: Vec<RuleSpec>) -> Result<JsonValue, String> {
+ let mut rules = Vec::with_capacity(specs.len());
+ for rule in specs {
+ let from = rule.from.segments()?;
+ let to = match (rule.to, rule.drop) {
+ (Some(to), false) => Some(to.segments()?),
+ (None, true) => None,
+ _ => {
+ return Err(format!(
+ "rule for {from:?} needs exactly one of \"to\" and \"drop\""
+ ));
+ }
};
- match &rule.to {
- None => Translation::Drop,
- Some(to) => {
- let mut target = to.clone();
- target.extend_from_slice(&path[rule.from.len()..]);
- if target.as_slice() == path {
- Translation::Same
- } else {
- Translation::To(target)
- }
+ if is_reserved(&from) || to.as_deref().is_some_and(is_reserved) {
+ return Err(format!(
+ "rules must not touch the reserved key {META_ROOT:?}"
+ ));
+ }
+ if rules.iter().any(|existing: &Rule| existing.from == from) {
+ return Err(format!("two rules for {from:?}"));
+ }
+ rules.push(Rule { from, to });
+ }
+ serde_json::to_value(Rules { rules }).map_err(|error| error.to_string())
+}
+
+/// Where `path` goes under the stored rules: the longest matching `from` wins.
+fn translate_by_rules(path: &[String], translation: &JsonValue) -> Translation {
+ let Ok(Rules { rules }) = serde_json::from_value::<Rules>(translation.clone()) else {
+ return Translation::Quarantine("unreadable replacement rules".to_owned());
+ };
+ let Some(rule) = rules
+ .iter()
+ .filter(|rule| path.starts_with(&rule.from))
+ .max_by_key(|rule| rule.from.len())
+ else {
+ return Translation::Quarantine("no replacement rule covers this path".to_owned());
+ };
+ match &rule.to {
+ None => Translation::Drop,
+ Some(to) => {
+ let mut target = to.clone();
+ target.extend_from_slice(&path[rule.from.len()..]);
+ if target.as_slice() == path {
+ Translation::Same
+ } else {
+ Translation::To(target)
}
}
}
}
+/// Import: like [`Replace`], but the new document is an Automerge document (bytes, as
+/// saved), so Automerge types the store's JSON can't express survive: text, counters,
+/// bytes, timestamps, floats. Its `_continuum` (protocol data of whatever swarm it came
+/// from) is left out, as is its history: the content is the merged view.
+///
+/// Spec: `{ "document": "<base64 of the saved document>", "rules": […] }`, rules as for
+/// [`Replace`]. To restore a backup of this swarm with every later write kept where it
+/// was: `"rules": [{"from": "", "to": ""}]`.
+pub(crate) struct Import;
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct ImportSpec {
+ document: String,
+ #[serde(default)]
+ rules: Vec<RuleSpec>,
+}
+
+/// The document in a request (base64 of a saved Automerge document).
+fn load_encoded(encoded: &str) -> Result<AutoCommit, String> {
+ let bytes = BASE64
+ .decode(encoded.trim())
+ .map_err(|error| format!("the document is not valid base64: {error}"))?;
+ AutoCommit::load(&bytes)
+ .map_err(|error| format!("the document is not an Automerge document: {error}"))
+}
+
+impl RemakeKind for Import {
+ fn name(&self) -> &'static str {
+ "import"
+ }
+
+ fn propose(&self, spec: &JsonValue) -> Result<Prepared, String> {
+ let spec: ImportSpec =
+ serde_json::from_value(spec.clone()).map_err(|error| error.to_string())?;
+ let imported = load_encoded(&spec.document)?;
+
+ // Keep only what goes into the new epoch: the merged view without protocol data
+ // and history. Usually much smaller than what was uploaded, and it is what every
+ // voter downloads.
+ let mut content = AutoCommit::new();
+ store::materialize_into(&imported, &mut content, &is_reserved)
+ .map_err(|error| format!("the document can't be read: {error}"))?;
+
+ Ok(Prepared {
+ content: Some(content.save()),
+ translation: prepare_rules(spec.rules)?,
+ })
+ }
+
+ fn build(
+ &self,
+ _old_at_base: &AutoCommit,
+ content: Option<&[u8]>,
+ target: &mut AutoCommit,
+ ) -> Result<(), StoreError> {
+ let content = content.ok_or_else(|| anyhow::anyhow!("import without content"))?;
+ let imported = AutoCommit::load(content)?;
+ // Deterministic: the same bytes give the same view, copied in a fixed order.
+ store::materialize_into(&imported, target, &is_reserved)
+ }
+
+ fn translate(&self, path: &[String], translation: &JsonValue) -> Translation {
+ translate_by_rules(path, translation)
+ }
+}
+
/// The translations of several consecutive remakes, applied in order: a node that
/// rejoins several epochs later carries its writes through every remake in between.
#[derive(Default, Clone)]
@@ -332,7 +416,7 @@ mod tests {
fn build(
&self,
_old: &AutoCommit,
- _content: &JsonValue,
+ _content: Option<&[u8]>,
_target: &mut AutoCommit,
) -> Result<(), StoreError> {
Ok(())
@@ -399,10 +483,10 @@ mod tests {
assert!(replace(json!({ "document": [] })).is_err());
assert!(replace(json!({ "document": { META_ROOT: {} } })).is_err());
assert!(replace(json!({ "document": {}, "rules": [{ "from": "a" }] })).is_err());
- assert!(replace(
- json!({ "document": {}, "rules": [{ "from": "a", "to": "b", "drop": true }] })
- )
- .is_err());
+ assert!(
+ replace(json!({ "document": {}, "rules": [{ "from": "a", "to": "b", "drop": true }] }))
+ .is_err()
+ );
assert!(
replace(json!({ "document": {}, "rules": [{ "from": "a//b", "to": "c" }] })).is_err()
);
@@ -448,13 +532,70 @@ mod tests {
}
#[test]
+ fn import_keeps_types_and_drops_protocol_data() {
+ use automerge::{ObjType, ROOT, ReadDoc, transaction::Transactable};
+ use serde_json::json;
+
+ let mut source = AutoCommit::new();
+ let text = source.put_object(ROOT, "notes", ObjType::Text).unwrap();
+ source.splice_text(&text, 0, 0, "hello").unwrap();
+ source
+ .put(ROOT, "hits", automerge::ScalarValue::counter(3))
+ .unwrap();
+ let meta = source.put_object(ROOT, META_ROOT, ObjType::Map).unwrap();
+ source.put(&meta, "members", "elsewhere").unwrap();
+ let encoded = BASE64.encode(source.save());
+
+ assert!(
+ Import
+ .propose(&json!({ "document": "not base64!" }))
+ .is_err()
+ );
+ assert!(
+ Import
+ .propose(&json!({ "document": BASE64.encode(b"junk") }))
+ .is_err()
+ );
+
+ let prepared = Import
+ .propose(&json!({ "document": encoded, "rules": [{ "from": "", "to": "" }] }))
+ .unwrap();
+ let build = || {
+ let mut target = AutoCommit::new().with_actor(automerge::ActorId::from(&b"fixed"[..]));
+ Import
+ .build(&AutoCommit::new(), prepared.content.as_deref(), &mut target)
+ .unwrap();
+ target.commit_with(automerge::transaction::CommitOptions::default().with_time(0));
+ target
+ };
+ let mut a = build();
+ let mut b = build();
+ assert_eq!(a.get_heads(), b.get_heads());
+
+ assert!(a.get(ROOT, META_ROOT).unwrap().is_none());
+ let (notes, notes_id) = a.get(ROOT, "notes").unwrap().unwrap();
+ assert!(matches!(notes, automerge::Value::Object(ObjType::Text)));
+ assert_eq!(a.text(&notes_id).unwrap(), "hello");
+ assert!(matches!(
+ a.get(ROOT, "hits").unwrap().unwrap().0,
+ automerge::Value::Scalar(ref s) if matches!(s.as_ref(), automerge::ScalarValue::Counter(_))
+ ));
+
+ // Writes after the base keep their paths.
+ assert_eq!(
+ Translator::single(&Import, prepared.translation).translate(&path(&["x", "y"])),
+ Translation::Same
+ );
+ }
+
+ #[test]
fn replace_builds_the_document_deterministically() {
use serde_json::json;
let build = |document: serde_json::Value| {
let prepared = replace(json!({ "document": document })).unwrap();
let mut target = AutoCommit::new().with_actor(automerge::ActorId::from(&b"fixed"[..]));
Replace
- .build(&AutoCommit::new(), &prepared.content, &mut target)
+ .build(&AutoCommit::new(), prepared.content.as_deref(), &mut target)
.unwrap();
target.commit_with(automerge::transaction::CommitOptions::default().with_time(0));
(
generated by cgit v1.3.1 (git 2.54.0) at 2026-10-24 20:19:36 +0000