Test validation of encrypted message backup files

Add a utility binary that compresses and encrypts backup files per the spec. Use the encryptor binary to encrypt the unencrypted test case file and include it as an additional golden test. Check that, in addition to calling via the library, the binary also accepts valid test files.
2026-04-25 17:25:18 +02:00 · 2024-01-24 09:38:28 -05:00
parent 56e63eb765
commit dc7c4eab1f
10 changed files with 323 additions and 49 deletions
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,3 +1,7 @@
 # Prevent auto-merging of generated acknowledgment files
 acknowledgments/acknowledgments.* -merge -text
 acknowledgments/acknowledgments.*.hbs merge text=auto
+
+# Treat encrypted and unencrypted message backup files as binary
+*.binproto binary
+*.binproto.encrypted binary
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -203,6 +203,21 @@ dependencies = [
 "syn 2.0.48",
 ]

+[[package]]
+name = "assert_cmd"
+version = "2.0.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00ad3f3a942eee60335ab4342358c161ee296829e0d16ff42fc1d6cb07815467"
+dependencies = [
+ "anstyle",
+ "bstr",
+ "doc-comment",
+ "predicates",
+ "predicates-core",
+ "predicates-tree",
+ "wait-timeout",
+]
+
 [[package]]
 name = "assert_matches"
 version = "1.5.0"
@@ -408,6 +423,17 @@ dependencies = [
 "fslock",
 ]

+[[package]]
+name = "bstr"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c48f0051a4b4c5e0b6d365cd04af53aeaa209e3cc15ec2cdb69e73cc87fbd0dc"
+dependencies = [
+ "memchr",
+ "regex-automata",
+ "serde",
+]
+
 [[package]]
 name = "bumpalo"
 version = "3.14.0"
@@ -894,6 +920,12 @@ dependencies = [
 "libc",
 ]

+[[package]]
+name = "difflib"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8"
+
 [[package]]
 name = "digest"
 version = "0.10.7"
@@ -937,6 +969,12 @@ dependencies = [
 "syn 2.0.48",
 ]

+[[package]]
+name = "doc-comment"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
+
 [[package]]
 name = "downcast-rs"
 version = "1.2.0"
@@ -1793,6 +1831,7 @@ dependencies = [
 "aes",
 "array-concat",
 "arrayvec",
+ "assert_cmd",
 "assert_matches",
 "async-compression",
 "cbc",
@@ -2505,6 +2544,33 @@ version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "94e851c7654eed9e68d7d27164c454961a616cf8c203d500607ef22c737b51bb"

+[[package]]
+name = "predicates"
+version = "3.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68b87bfd4605926cdfefc1c3b5f8fe560e3feca9d5552cf68c466d3d8236c7e8"
+dependencies = [
+ "anstyle",
+ "difflib",
+ "predicates-core",
+]
+
+[[package]]
+name = "predicates-core"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174"
+
+[[package]]
+name = "predicates-tree"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf"
+dependencies = [
+ "predicates-core",
+ "termtree",
+]
+
 [[package]]
 name = "prettyplease"
 version = "0.2.16"
@@ -3247,6 +3313,12 @@ dependencies = [
 "winapi-util",
 ]

+[[package]]
+name = "termtree"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76"
+
 [[package]]
 name = "test-case"
 version = "3.3.1"
--- a/rust/message-backup/Cargo.toml
+++ b/rust/message-backup/Cargo.toml
@@ -39,6 +39,7 @@ uuid = "1.1.2"
 signal-crypto = { path = "../crypto" }

 array-concat = "0.5.2"
+assert_cmd = "2.0.13"
 assert_matches = "1.5.0"
 dir-test = "0.2.0"
 futures = { version = "0.3.29", features = ["executor"] }
@@ -48,4 +49,4 @@ test-log = "0.2.14"

 [build-dependencies]
 protobuf = "3.3.0"
-protobuf-codegen = "3.3.0"
+protobuf-codegen = "3.3.0"
--- a/rust/message-backup/examples/encrypt_backup.rs
+++ b/rust/message-backup/examples/encrypt_backup.rs
@@ -0,0 +1,136 @@
+//
+// Copyright 2024 Signal Messenger, LLC.
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+
+use std::fmt::Display;
+use std::io::{stdout, Read as _, Write};
+
+use aes::cipher::block_padding::Pkcs7;
+use aes::cipher::{BlockEncryptMut, KeyIvInit};
+use aes::Aes256;
+use async_compression::futures::bufread::GzipEncoder;
+use clap::builder::TypedValueParser;
+use clap::Parser;
+use clap_stdin::FileOrStdin;
+use futures::io::Cursor;
+use futures::AsyncReadExt;
+use hmac::Mac;
+use libsignal_message_backup::args::{parse_aci, parse_hex_bytes};
+use libsignal_message_backup::key::{BackupKey, MessageBackupKey};
+use libsignal_protocol::Aci;
+use sha2::Sha256;
+
+const DEFAULT_ACI: Aci = Aci::from_uuid_bytes([0x11; 16]);
+const DEFAULT_MASTER_KEY: [u8; 32] = [b'M'; 32];
+
+#[derive(Parser)]
+/// Compresses and encrypts an unencrypted backup file.
+struct CliArgs {
+    /// the file to read from, or '-' to read from stdin
+    filename: FileOrStdin,
+
+    /// the ACI to encrypt the backup file for
+    #[arg(
+        long,
+        value_parser=parse_aci.map(WrapCliArg),
+        default_value_t=WrapCliArg(DEFAULT_ACI)
+    )]
+    aci: WrapCliArg<Aci>,
+
+    /// master key used (with the ACI) to derive the backup keys
+    #[arg(
+        long,
+        value_parser=parse_hex_bytes::<32>.map(WrapCliArg),
+        default_value_t=WrapCliArg(DEFAULT_MASTER_KEY)
+    )]
+    master_key: WrapCliArg<[u8; BackupKey::MASTER_KEY_LEN]>,
+}
+
+fn main() {
+    let CliArgs {
+        filename,
+        master_key: WrapCliArg(master_key),
+        aci: WrapCliArg(aci),
+    } = CliArgs::parse();
+
+    let backup_key = BackupKey::derive_from_master_key(&master_key);
+    let backup_id = backup_key.derive_backup_id(&aci);
+    let key = MessageBackupKey::derive(&backup_key, &backup_id);
+
+    eprintln!("reading from {:?}", filename.source);
+
+    let contents = read_file(filename);
+    eprintln!("read {} bytes", contents.len());
+
+    let compressed_contents = gzip_compress(contents);
+    eprintln!("compressed to {} bytes", compressed_contents.len());
+
+    let MessageBackupKey {
+        hmac_key,
+        aes_key,
+        iv,
+    } = &key;
+
+    let encrypted_contents = aes_cbc_encrypt(aes_key, iv, compressed_contents);
+    eprintln!("encrypted to {} bytes", encrypted_contents.len());
+
+    let hmac = hmac_checksum(hmac_key, &encrypted_contents);
+    write_bytes("encrypted", encrypted_contents);
+
+    write_bytes("HMAC", hmac);
+}
+
+fn read_file(filename: FileOrStdin) -> Vec<u8> {
+    let source = filename.source.clone();
+    let mut contents = Vec::new();
+    filename
+        .into_reader()
+        .unwrap_or_else(|e| panic!("failed to read {source:?}: {e}"))
+        .read_to_end(&mut contents)
+        .expect("IO error");
+    contents
+}
+
+fn aes_cbc_encrypt(aes_key: &[u8; 32], iv: &[u8; 16], compressed_contents: Vec<u8>) -> Vec<u8> {
+    let encryptor = cbc::Encryptor::<Aes256>::new(aes_key.into(), iv.into());
+
+    encryptor.encrypt_padded_vec_mut::<Pkcs7>(&compressed_contents)
+}
+fn hmac_checksum(hmac_key: &[u8; 32], encrypted_contents: &[u8]) -> [u8; 32] {
+    let mut hmac = hmac::Hmac::<Sha256>::new_from_slice(hmac_key).expect("correct key size");
+    hmac.update(encrypted_contents);
+    hmac.finalize().into_bytes().into()
+}
+
+fn gzip_compress(contents: Vec<u8>) -> Vec<u8> {
+    let mut compressed_contents = Vec::new();
+    futures::executor::block_on(
+        GzipEncoder::new(Cursor::new(contents)).read_to_end(&mut compressed_contents),
+    )
+    .expect("failed to compress");
+
+    compressed_contents
+}
+
+fn write_bytes(label: &'static str, bytes: impl AsRef<[u8]>) {
+    let bytes = bytes.as_ref();
+    stdout().write_all(bytes).expect("failed to write");
+    eprintln!("wrote {} {label} bytes", bytes.len())
+}
+
+/// Wrapper struct to provide custom [`Display`] impls.
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
+struct WrapCliArg<T>(T);
+
+impl Display for WrapCliArg<Aci> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.0.service_id_string())
+    }
+}
+
+impl Display for WrapCliArg<[u8; 32]> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", hex::ToHex::encode_hex::<String>(&self.0))
+    }
+}
--- a/rust/message-backup/src/args.rs
+++ b/rust/message-backup/src/args.rs
@@ -0,0 +1,36 @@
+//
+// Copyright 2024 Signal Messenger, LLC.
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+
+use libsignal_protocol::Aci;
+
+#[derive(Debug, thiserror::Error)]
+pub enum ParseHexError<const N: usize> {
+    #[error("character {c} at position {index} is not a hex digit")]
+    InvalidHexCharacter { c: char, index: usize },
+    #[error("got {count} hex digits, expected {}",  2 * N)]
+    WrongNumberOfDigits { count: usize },
+}
+
+pub fn parse_hex_bytes<const N: usize>(input: &str) -> Result<[u8; N], ParseHexError<N>>
+where
+    [u8; N]: hex::FromHex<Error = hex::FromHexError>,
+{
+    hex::FromHex::from_hex(input).map_err(|e| match e {
+        hex::FromHexError::InvalidHexCharacter { c, index } => {
+            ParseHexError::InvalidHexCharacter { c, index }
+        }
+        hex::FromHexError::InvalidStringLength | hex::FromHexError::OddLength => {
+            ParseHexError::WrongNumberOfDigits { count: input.len() }
+        }
+    })
+}
+
+pub fn parse_aci(input: &str) -> Result<Aci, AciParseError> {
+    Aci::parse_from_service_id_string(input).ok_or(AciParseError)
+}
+
+/// invalid ACI, expected a UUID like "55555555-5555-5555-5555-555555555555"
+#[derive(Debug, thiserror::Error, displaydoc::Display)]
+pub struct AciParseError;
--- a/rust/message-backup/src/bin/validator/args.rs
+++ b/rust/message-backup/src/bin/validator/args.rs
@@ -3,38 +3,6 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 //

-use libsignal_protocol::Aci;
-
-#[derive(Debug, thiserror::Error)]
-pub(crate) enum ParseHexError<const N: usize> {
-    #[error("character {c} at position {index} is not a hex digit")]
-    InvalidHexCharacter { c: char, index: usize },
-    #[error("got {count} hex digits, expected {}",  2 * N)]
-    WrongNumberOfDigits { count: usize },
-}
-
-pub(crate) fn parse_hex_bytes<const N: usize>(input: &str) -> Result<[u8; N], ParseHexError<N>>
-where
-    [u8; N]: hex::FromHex<Error = hex::FromHexError>,
-{
-    hex::FromHex::from_hex(input).map_err(|e| match e {
-        hex::FromHexError::InvalidHexCharacter { c, index } => {
-            ParseHexError::InvalidHexCharacter { c, index }
-        }
-        hex::FromHexError::InvalidStringLength | hex::FromHexError::OddLength => {
-            ParseHexError::WrongNumberOfDigits { count: input.len() }
-        }
-    })
-}
-
-pub(crate) fn parse_aci(input: &str) -> Result<Aci, AciParseError> {
-    Aci::parse_from_service_id_string(input).ok_or(AciParseError)
-}
-
-/// invalid ACI, expected a UUID like "55555555-5555-5555-5555-555555555555"
-#[derive(Debug, thiserror::Error, displaydoc::Display)]
-pub(crate) struct AciParseError;
-
 pub(crate) enum ParseVerbosity {
    None,
    PrintOneLine,
--- a/rust/message-backup/src/bin/validator/main.rs
+++ b/rust/message-backup/src/bin/validator/main.rs
@@ -5,17 +5,19 @@

 use std::io::Read as _;

-use args::ParseVerbosity;
 use clap::{Args, Parser};
 use futures::io::{AllowStdIo, Cursor};
 use futures::AsyncRead;

+use libsignal_message_backup::args::{parse_aci, parse_hex_bytes};
 use libsignal_message_backup::frame::FramesReader;
 use libsignal_message_backup::key::{BackupKey, MessageBackupKey};
 use libsignal_message_backup::unknown::FormatPath;
 use libsignal_message_backup::{BackupReader, Error, FoundUnknownField, ReadResult};
 use libsignal_protocol::Aci;

+use crate::args::ParseVerbosity;
+
 mod args;

 /// Validates, and optionally prints the contents of, message backup files.
@@ -51,10 +53,10 @@ struct Cli {
 #[group(conflicts_with = "KeyParts")]
 struct DeriveKey {
    /// account master key, used with the ACI to derive the message backup key
-    #[arg(long, value_parser=args::parse_hex_bytes::<32>, requires="aci")]
+    #[arg(long, value_parser=parse_hex_bytes::<32>, requires="aci")]
    master_key: Option<[u8; BackupKey::MASTER_KEY_LEN]>,
    /// ACI for the backup creator
-    #[arg(long, value_parser=args::parse_aci, requires="master_key")]
+    #[arg(long, value_parser=parse_aci, requires="master_key")]
    aci: Option<Aci>,
 }

@@ -62,13 +64,13 @@ struct DeriveKey {
 #[group(conflicts_with = "DeriveKey")]
 struct KeyParts {
    /// HMAC key, used if the master key is not provided
-    #[arg(long, value_parser=args::parse_hex_bytes::<32>, requires_all=["aes_key", "iv"])]
+    #[arg(long, value_parser=parse_hex_bytes::<32>, requires_all=["aes_key", "iv"])]
    hmac_key: Option<[u8; MessageBackupKey::HMAC_KEY_LEN]>,
    /// AES encryption key, used if the master key is not provided
-    #[arg(long, value_parser=args::parse_hex_bytes::<32>, requires_all=["hmac_key", "iv"])]
+    #[arg(long, value_parser=parse_hex_bytes::<32>, requires_all=["hmac_key", "iv"])]
    aes_key: Option<[u8; MessageBackupKey::AES_KEY_LEN]>,
    /// AES IV bytes, used if the master key is not provided
-    #[arg(long, value_parser=args::parse_hex_bytes::<16>, requires_all=["hmac_key", "aes_key"])]
+    #[arg(long, value_parser=parse_hex_bytes::<16>, requires_all=["hmac_key", "aes_key"])]
    iv: Option<[u8; MessageBackupKey::IV_LEN]>,
 }

--- a/rust/message-backup/src/lib.rs
+++ b/rust/message-backup/src/lib.rs
@@ -12,6 +12,7 @@ use crate::key::MessageBackupKey;
 use crate::parse::VarintDelimitedReader;
 use crate::unknown::{PathPart, UnknownValue, VisitUnknownFieldsExt as _};

+pub mod args;
 pub mod backup;
 pub mod frame;
 pub mod key;
--- a/rust/message-backup/tests/res/test-cases/valid/new_account.binproto.encrypted
+++ b/rust/message-backup/tests/res/test-cases/valid/new_account.binproto.encrypted
@@ -0,0 +1,3 @@
+Ò<>ajÞ<6A>œ<EFBFBD>ãü´Û	bF`¼u¨ºÛUÉ¡"%e†
+´qŽ4LÊ7¬Ò<EFBFBD>ï3Ä¶!ôØf¡-#	#•ï<>Wƒ2CQw)íN|<¶¸ÏÁ²#‰Ûì”@êL£a4OŒü]GÓ›®ƒ
+ç÷ëv«„4ñL·gB7¯ÅF2xñ!òdÈVë5FDt•\‘–- ý8[èËrKËšJoš ê’NõõQ+n:š†W˜<57>sø;GÝ+øFX÷)À:íOë¥ò›2
ŽKleËn˜ÎºWùNÇ›<C387>ê‡è{V|§ÓÐ·ù
--- a/rust/message-backup/tests/test_cases.rs
+++ b/rust/message-backup/tests/test_cases.rs
@@ -3,20 +3,67 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 //

+use std::path::{Path, PathBuf};
+
+use assert_cmd::Command;
 use dir_test::{dir_test, Fixture};
 use futures::io::AllowStdIo;
-
+use futures::AsyncRead;
+use libsignal_message_backup::key::{BackupKey, MessageBackupKey};
 use libsignal_message_backup::{BackupReader, ReadResult};
+use libsignal_protocol::Aci;

 #[dir_test(
-    dir: "$CARGO_MANIFEST_DIR/tests/res/test-cases",
-    glob: "valid/*.binproto",
-    loader: read_file_async,
-    postfix: "binproto"
-)]
-fn is_valid_binproto(input: Fixture<AllowStdIo<std::fs::File>>) {
-    let input = input.into_content();
-    let mut reader = BackupReader::new_unencrypted(input);
+        dir: "$CARGO_MANIFEST_DIR/tests/res/test-cases",
+        glob: "valid/*.binproto",
+        loader: PathBuf::from,
+        postfix: "binproto"
+    )]
+fn is_valid_binproto(input: Fixture<PathBuf>) {
+    let path = input.into_content();
+    // Check via the library interface.
+    let reader = BackupReader::new_unencrypted(read_file_async(&path));
+    validate(reader);
+
+    // The CLI tool should agree.
+    validator_command().arg(path).ok().expect("command failed");
+}
+
+#[dir_test(
+        dir: "$CARGO_MANIFEST_DIR/tests/res/test-cases",
+        glob: "valid/*.binproto.encrypted",
+        loader: PathBuf::from,
+        postfix: "encrypted"
+    )]
+fn is_valid_encrypted_proto(input: Fixture<PathBuf>) {
+    const ACI: Aci = Aci::from_uuid_bytes([0x11; 16]);
+    const MASTER_KEY: [u8; 32] = [b'M'; 32];
+    let backup_key = BackupKey::derive_from_master_key(&MASTER_KEY);
+    let key = MessageBackupKey::derive(&backup_key, &backup_key.derive_backup_id(&ACI));
+
+    let path = input.into_content();
+    // Check via the library interface.
+    let reader = futures::executor::block_on(BackupReader::new_encrypted_compressed(
+        &key,
+        read_file_async(&path),
+    ))
+    .expect("invalid HMAC");
+    validate(reader);
+
+    // The CLI tool should agree.
+    validator_command()
+        .args([
+            "--aci".to_string(),
+            ACI.service_id_string(),
+            "--master-key".to_string(),
+            hex::encode(MASTER_KEY),
+            path.to_string_lossy().into_owned(),
+        ])
+        .ok()
+        .expect("command failed");
+}
+
+fn validate(mut reader: BackupReader<impl AsyncRead + Unpin>) {
    reader.visitor = |msg| println!("{msg:#?}");

    let ReadResult {
@@ -29,7 +76,11 @@ fn is_valid_binproto(input: Fixture<AllowStdIo<std::fs::File>>) {
    println!("got backup:\n{backup:#?}");
 }

-fn read_file_async(path: &str) -> AllowStdIo<std::fs::File> {
+fn validator_command() -> Command {
+    Command::cargo_bin("validator").expect("bin not found")
+}
+
+fn read_file_async(path: &Path) -> AllowStdIo<std::fs::File> {
    let file = std::fs::File::open(path).expect("can read");
    AllowStdIo::new(file)
 }