dialcache/protocol/
text.rs

1//! Text payload conversion.
2//!
3//! Text frame payloads and decompressed `0x01` envelopes may hold any bytes.
4//! `formal/guides/PROTOCOL.md` ("Text payload domain") fixes how they become text:
5//! the WHATWG UTF-8 decoder with replacement error handling and without BOM
6//! removal, so every maximal ill-formed subpart becomes exactly one U+FFFD.
7
8/// Decode bytes as UTF-8 with one U+FFFD per maximal ill-formed subpart and
9/// no BOM removal (the WHATWG decoder with replacement error handling).
10///
11/// The result is always valid UTF-8. Well-formed input is returned unchanged,
12/// so valid text costs one validation pass and one copy.
13pub fn replacement_utf8(bytes: &[u8]) -> Vec<u8> {
14    String::from_utf8_lossy(bytes).into_owned().into_bytes()
15}
16
17#[cfg(test)]
18mod tests {
19    use super::replacement_utf8;
20
21    fn decoded(bytes: &[u8]) -> String {
22        String::from_utf8(replacement_utf8(bytes)).expect("replacement output is UTF-8")
23    }
24
25    #[test]
26    fn protocol_replacement_table() {
27        // The representative outcomes fixed by formal/guides/PROTOCOL.md.
28        assert_eq!(decoded(&[0x22, 0xFF, 0x22]), "\"\u{FFFD}\"");
29        assert_eq!(decoded(&[0xE2, 0x82]), "\u{FFFD}");
30        assert_eq!(decoded(&[0xE2, 0x82, 0x41]), "\u{FFFD}A");
31        assert_eq!(decoded(&[0xED, 0xA0, 0x80]), "\u{FFFD}\u{FFFD}\u{FFFD}");
32        assert_eq!(
33            decoded(&[0xF4, 0x90, 0x80, 0x80]),
34            "\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}"
35        );
36        assert_eq!(decoded(&[0xEF, 0xBB, 0xBF, 0x61]), "\u{FEFF}a");
37    }
38
39    #[test]
40    fn well_formed_input_is_unchanged() {
41        let text = "\u{80}\u{7FF}\u{800}\u{D7FF}\u{E000}\u{FFFF}\u{10000}\u{10FFFF}\0";
42        assert_eq!(replacement_utf8(text.as_bytes()), text.as_bytes());
43        assert!(replacement_utf8(b"").is_empty());
44    }
45
46    #[test]
47    fn isolated_continuations_and_overlong_forms_replace_separately() {
48        assert_eq!(decoded(&[0x80, 0xBF]), "\u{FFFD}\u{FFFD}");
49        assert_eq!(
50            decoded(&[0xC0, 0xAF, 0xE0, 0x80, 0x80]),
51            "\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}"
52        );
53        // A lead that reaches end of input after accepted continuations.
54        assert_eq!(decoded(&[0xF0, 0x9F, 0x98]), "\u{FFFD}");
55        // The narrowed second byte after F0 rejects 8F; 8F is then a lone continuation.
56        assert_eq!(
57            decoded(&[0xF0, 0x8F, 0x80, 0x80]),
58            "\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}"
59        );
60        // A complete prefix interrupted by a new lead keeps the second sequence intact.
61        assert_eq!(decoded(&[0xE2, 0x82, 0xC3, 0xA9]), "\u{FFFD}\u{E9}");
62        // Leads beyond F4 and FE/FF are single-byte subparts.
63        assert_eq!(
64            decoded(&[0xF5, 0x80, 0xFE, 0xFF, 0x41]),
65            "\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}A"
66        );
67    }
68}
dialcache/protocol/text.rs

dialcache/protocol/
text.rs
