1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
|
//! Deflate for what goes over the wire: large sync messages (`swarm::sync`). Automerge
//! sends changes as they are stored, uncompressed; text and JSON in them shrink to about
//! half.
use std::io::{Read, Write};
use anyhow::{Result, bail};
use flate2::Compression;
use flate2::read::DeflateDecoder;
use flate2::write::DeflateEncoder;
/// Below this, deflating saves too little to be worth it.
pub(crate) const WORTH_FROM: usize = 1024;
/// Not 1 (`Compression::fast`): `flate2` runs on zlib-rs here (another dependency asks
/// for it), whose level 1 is zlib-ng's quick deflate, with fixed Huffman codes only.
/// Those can't use a skewed alphabet: 64 KiB of random hex shrinks to 87% at level 1, to
/// 54% at level 2 (as at level 6), still quick.
const LEVEL: u32 = 2;
/// `bytes` deflated, if that makes them at least a tenth smaller.
pub(crate) fn deflate(bytes: &[u8]) -> Option<Vec<u8>> {
if bytes.len() < WORTH_FROM {
return None;
}
let mut encoder =
DeflateEncoder::new(Vec::with_capacity(bytes.len() / 2), Compression::new(LEVEL));
encoder.write_all(bytes).ok()?;
let deflated = encoder.finish().ok()?;
(deflated.len() < bytes.len() - bytes.len() / 10).then_some(deflated)
}
/// `bytes` inflated, if they come to at most `limit` bytes: a small message can't make
/// us hold a large one.
pub(crate) fn inflate(bytes: &[u8], limit: usize) -> Result<Vec<u8>> {
let mut inflated = Vec::with_capacity((bytes.len() * 3).min(limit));
DeflateDecoder::new(bytes)
.take(limit as u64 + 1)
.read_to_end(&mut inflated)?;
if inflated.len() > limit {
bail!("inflates to more than {limit} bytes");
}
Ok(inflated)
}
#[cfg(test)]
mod tests {
use super::*;
/// What deflates inflates back; little, or what doesn't shrink, isn't deflated; more
/// than the limit is refused.
#[test]
fn deflates_and_inflates_within_limits() {
let text = "a key, a value; ".repeat(1000);
let deflated = deflate(text.as_bytes()).expect("text shrinks");
assert!(deflated.len() < text.len() / 4);
assert_eq!(inflate(&deflated, text.len()).unwrap(), text.as_bytes());
assert!(
inflate(&deflated, text.len() - 1).is_err(),
"over the limit"
);
assert!(deflate(b"short").is_none());
// Hex has 4 bits in 8: about half, as Huffman codes alone get it.
let mut rng = crate::testrand::Rng::new(2);
let hex: String = (0..4096)
.map(|_| format!("{:016x}", rng.next_u64()))
.collect();
let deflated = deflate(hex.as_bytes()).expect("hex shrinks");
assert!(
deflated.len() < hex.len() * 6 / 10,
"{} of {}",
deflated.len(),
hex.len()
);
let mut rng = crate::testrand::Rng::new(1);
let noise: Vec<u8> = (0..4096).map(|_| rng.next_u64() as u8).collect();
assert!(deflate(&noise).is_none(), "noise doesn't shrink");
}
}
|
