Added a new stream which signs and verifies each sector of data

it processes.

Cargo.lock

@@ -35,6 +35,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "anes"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
+
 [[package]]
 name = "ansi_term"
 version = "0.12.1"
@@ -91,7 +97,7 @@ dependencies = [
  "bitflags",
  "cexpr",
  "clang-sys",
- "clap",
+ "clap 2.34.0",
  "env_logger",
  "lazy_static",
  "lazycell",
@@ -160,6 +166,7 @@ dependencies = [
  "brotli",
  "btserde",
  "chrono",
+ "criterion",
  "ctor",
  "env_logger",
  "foreign-types",
@@ -225,6 +232,12 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "cast"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
+
 [[package]]
 name = "cc"
 version = "1.0.73"
@@ -261,6 +274,33 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "ciborium"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0c137568cc60b904a7724001b35ce2630fd00d5d84805fbb608ab89509d788f"
+dependencies = [
+ "ciborium-io",
+ "ciborium-ll",
+ "serde",
+]
+
+[[package]]
+name = "ciborium-io"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "346de753af073cc87b52b2083a506b38ac176a44cfb05497b622e27be899b369"
+
+[[package]]
+name = "ciborium-ll"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "213030a2b5a4e0c0892b6652260cf6ccac84827b83a85a534e178e3906c4cf1b"
+dependencies = [
+ "ciborium-io",
+ "half",
+]
+
 [[package]]
 name = "clang-sys"
 version = "1.4.0"
@@ -282,11 +322,32 @@ dependencies = [
  "atty",
  "bitflags",
  "strsim",
- "textwrap",
+ "textwrap 0.11.0",
  "unicode-width",
  "vec_map",
 ]
 
+[[package]]
+name = "clap"
+version = "3.2.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5"
+dependencies = [
+ "bitflags",
+ "clap_lex",
+ "indexmap",
+ "textwrap 0.16.0",
+]
+
+[[package]]
+name = "clap_lex"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5"
+dependencies = [
+ "os_str_bytes",
+]
+
 [[package]]
 name = "codespan-reporting"
 version = "0.11.1"
@@ -303,6 +364,85 @@ version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
 
+[[package]]
+name = "criterion"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb"
+dependencies = [
+ "anes",
+ "atty",
+ "cast",
+ "ciborium",
+ "clap 3.2.23",
+ "criterion-plot",
+ "itertools",
+ "lazy_static",
+ "num-traits",
+ "oorandom",
+ "plotters",
+ "rayon",
+ "regex",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "tinytemplate",
+ "walkdir",
+]
+
+[[package]]
+name = "criterion-plot"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
+dependencies = [
+ "cast",
+ "itertools",
+]
+
+[[package]]
+name = "crossbeam-channel"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521"
+dependencies = [
+ "cfg-if",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc"
+dependencies = [
+ "cfg-if",
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a"
+dependencies = [
+ "autocfg",
+ "cfg-if",
+ "crossbeam-utils",
+ "memoffset 0.7.1",
+ "scopeguard",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "ctor"
 version = "0.1.23"
@@ -458,6 +598,12 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
 
+[[package]]
+name = "half"
+version = "1.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"
+
 [[package]]
 name = "harness"
 version = "0.0.1"
@@ -469,6 +615,12 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "hashbrown"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+
 [[package]]
 name = "heck"
 version = "0.4.0"
@@ -520,6 +672,16 @@ dependencies = [
  "cxx-build",
 ]
 
+[[package]]
+name = "indexmap"
+version = "1.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399"
+dependencies = [
+ "autocfg",
+ "hashbrown",
+]
+
 [[package]]
 name = "io-uring"
 version = "0.5.7"
@@ -530,6 +692,21 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "itertools"
+version = "0.10.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
+dependencies = [
+ "either",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc"
+
 [[package]]
 name = "js-sys"
 version = "0.3.60"
@@ -620,6 +797,15 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "memoffset"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "minimal-lexical"
 version = "0.2.1"
@@ -647,7 +833,7 @@ dependencies = [
  "bitflags",
  "cfg-if",
  "libc",
- "memoffset",
+ "memoffset 0.6.5",
 ]
 
 [[package]]
@@ -660,7 +846,7 @@ dependencies = [
  "bitflags",
  "cfg-if",
  "libc",
- "memoffset",
+ "memoffset 0.6.5",
  "pin-utils",
 ]
 
@@ -704,6 +890,16 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "num_cpus"
+version = "1.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5"
+dependencies = [
+ "hermit-abi",
+ "libc",
+]
+
 [[package]]
 name = "oid"
 version = "0.2.1"
@@ -719,6 +915,12 @@ version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1"
 
+[[package]]
+name = "oorandom"
+version = "11.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
+
 [[package]]
 name = "openssl"
 version = "0.10.41"
@@ -778,6 +980,12 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "os_str_bytes"
+version = "6.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee"
+
 [[package]]
 name = "peeking_take_while"
 version = "0.1.2"
@@ -847,6 +1055,34 @@ version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae"
 
+[[package]]
+name = "plotters"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97"
+dependencies = [
+ "num-traits",
+ "plotters-backend",
+ "plotters-svg",
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "plotters-backend"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142"
+
+[[package]]
+name = "plotters-svg"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f"
+dependencies = [
+ "plotters-backend",
+]
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.43"
@@ -893,6 +1129,29 @@ version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
 
+[[package]]
+name = "rayon"
+version = "1.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e060280438193c554f654141c9ea9417886713b7acd75974c85b18a69a88e0b"
+dependencies = [
+ "crossbeam-deque",
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3"
+dependencies = [
+ "crossbeam-channel",
+ "crossbeam-deque",
+ "crossbeam-utils",
+ "num_cpus",
+]
+
 [[package]]
 name = "rdrand"
 version = "0.4.0"
@@ -949,12 +1208,33 @@ version = "1.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8"
 
+[[package]]
+name = "ryu"
+version = "1.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09"
+
+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
 [[package]]
 name = "scoped-tls"
 version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2"
 
+[[package]]
+name = "scopeguard"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
+
 [[package]]
 name = "scratch"
 version = "1.0.2"
@@ -1017,6 +1297,17 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "serde_json"
+version = "1.0.89"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
+
 [[package]]
 name = "shlex"
 version = "1.1.0"
@@ -1158,6 +1449,12 @@ dependencies = [
  "unicode-width",
 ]
 
+[[package]]
+name = "textwrap"
+version = "0.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d"
+
 [[package]]
 name = "thiserror"
 version = "1.0.37"
@@ -1189,6 +1486,16 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "tinytemplate"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "tokio"
 version = "1.21.2"
@@ -1306,6 +1613,17 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "walkdir"
+version = "2.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56"
+dependencies = [
+ "same-file",
+ "winapi",
+ "winapi-util",
+]
+
 [[package]]
 name = "wasi"
 version = "0.10.0+wasi-snapshot-preview1"
@@ -1372,6 +1690,16 @@ version = "0.2.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f"
 
+[[package]]
+name = "web-sys"
+version = "0.3.60"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
 [[package]]
 name = "which"
 version = "4.3.0"

Cargo.toml

@@ -8,3 +8,6 @@ members = [
     "crates/btfuse",
     "crates/swtpm-harness",
 ]
+
+[profile.bench]
+debug = true

README.md

@@ -164,3 +164,42 @@ and mouse and keyboard support.
   -A video streaming server.
   -An application for managing contacts.
   -A secure messaging and chat application.
+
+# Performance Measurement with flamegraph on Linux
+In order to use the `flamegraph` crate to on Linux you need to setup unprivileged access to the
+`perf` utility by your user account. After installing `perf` (on Arch with `pacman -S perf`),
+perform the following as root (if `perf` is not installed at `/usr/bin/perf` use `which perf` to
+locate it):
+```
+# groupadd perf_users
+# cd /usr/bin
+# ls -alhF perf
+-rwxr-xr-x  2 root root  11M Oct 19 15:12 perf
+# chgrp perf_users perf
+# ls -alhF
+-rwxr-xr-x  2 root perf_users  11M Oct 19 15:12 perf
+# chmod o-rwx perf
+# ls -alhF
+-rwxr-x---  2 root perf_users  11M Oct 19 15:12 perf
+# setcap "cap_perfmon,cap_sys_ptrace,cap_syslog=ep" perf
+# setcap -v "cap_perfmon,cap_sys_ptrace,cap_syslog=ep" perf
+perf: OK
+# getcap perf
+perf = cap_sys_ptrace,cap_syslog,cap_perfmon+ep
+```
+(source: https://www.kernel.org/doc/html/latest/admin-guide/perf-security.html)
+Finally add your user account to the perf_users group:
+```
+# usermod -aG perf_users <your username>
+```
+You'll need to logout and back in for the new group to take effect. You can confirm you're in the
+group with:
+```
+> groups
+tss audio <your username> perf_users
+```
+The you can run flamegraph as your user account with:
+```
+> cargo flamegraph --unit-test <crate name> -- test::<test name>
+```
+(source: https://crates.io/crates/flamegraph)

crates/btfuse/src/main.rs

@@ -257,8 +257,7 @@ mod test {
             });
             started_rx
                 .recv_timeout(Duration::from_secs(1))
-                .expect("failed to received started signal from `FuseDaemon`");
-            println!("sent started signal");
+                .expect("failed to receive started signal from `FuseDaemon`");
             TestCase {
                 temp_path_rx: rx,
                 mnt_path: None,

crates/btlib/Cargo.toml

@@ -34,4 +34,9 @@ chrono = "0.4.23"
 tempdir = { version = "0.3.7" }
 ctor = { version = "0.1.22" }
 lazy_static = { version = "1.4.0" }
-vm-memory = { version = "0.9.0" }
+vm-memory = { version = "0.9.0" }
+criterion = "0.4.0"
+
+[[bench]]
+name = "block_benches"
+harness = false

crates/btlib/benches/block_benches.rs

@@ -0,0 +1,58 @@
+use std::{fs::OpenOptions, time::Duration};
+
+use btlib::{
+    crypto::{ConcreteCreds, Creds},
+    BlockOpenOptions, BlockPath, Epoch, Principaled, SECTOR_SZ_DEFAULT,
+};
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use tempdir::TempDir;
+
+fn block_write(c: &mut Criterion) {
+    let temp_dir = TempDir::new("block_bench").expect("failed to create temp dir");
+    let root_creds = ConcreteCreds::generate().expect("failed to generate root_creds");
+    let mut node_creds = ConcreteCreds::generate().expect("failed to generate node_creds");
+    let components = vec!["nodes".to_string(), "phone".to_string()];
+    let writecap = root_creds
+        .issue_writecap(
+            node_creds.principal(),
+            components.clone(),
+            Epoch::now() + Duration::from_secs(3600),
+        )
+        .expect("failed to issue writecap");
+    node_creds.set_writecap(writecap);
+
+    let mut fs_path = temp_dir.path().to_owned();
+    fs_path.extend(components.iter());
+    std::fs::create_dir_all(&fs_path).expect("failed to create fs_path");
+
+    let node_path = BlockPath::new(root_creds.principal(), components);
+    let file_path = fs_path.join("file.txt");
+
+    c.bench_function("block_write", |b| {
+        b.iter(|| {
+            let file = OpenOptions::new()
+                .create(true)
+                .truncate(true)
+                .read(true)
+                .write(true)
+                .open(&file_path)
+                .expect("failed to open file");
+            let mut block = BlockOpenOptions::new()
+                .with_inner(file)
+                .with_creds(node_creds.clone())
+                .with_encrypt(true)
+                .with_block_path(node_path.clone())
+                .open()
+                .expect("failed to open block");
+
+            const ZEROS: [u8; SECTOR_SZ_DEFAULT] = [0u8; SECTOR_SZ_DEFAULT];
+            const ITER: usize = (512 * 1024) / ZEROS.len();
+            for zeros in std::iter::repeat(ZEROS.as_slice()).take(ITER) {
+                block.write(black_box(zeros)).expect("write failed");
+            }
+        })
+    });
+}
+
+criterion_group!(benches, block_write);
+criterion_main!(benches);

crates/btlib/src/blocktree.rs

@@ -468,7 +468,8 @@ mod private {
             };
             write_to(&sb, &mut sb_block)?;
             sb_block.mut_meta_body().access_secrets(|secrets| {
-                secrets.inode = SpecInodes::Sb.into();
+                secrets.block_id.generation = generation;
+                secrets.block_id.inode = SpecInodes::Sb.into();
                 secrets.mode = FileType::Reg.value() | 0o666;
                 secrets.uid = 0;
                 secrets.gid = 0;
@@ -486,7 +487,8 @@ mod private {
             )?;
             write_to(&Directory::new(), &mut root_block)?;
             root_block.mut_meta_body().access_secrets(|secrets| {
-                secrets.inode = SpecInodes::RootDir.into();
+                secrets.block_id.generation = generation;
+                secrets.block_id.inode = SpecInodes::RootDir.into();
                 secrets.mode = FileType::Dir.value() | 0o777;
                 secrets.uid = 0;
                 secrets.gid = 0;
@@ -977,8 +979,9 @@ mod private {
                 self.open_then_take_handle(inode, block_path, |handle, value| {
                     let block = value.block_mut(handle)?;
                     Ok(block.mut_meta_body().access_secrets(|secrets| {
-                        secrets.inode = inode;
-                        secrets.mode = args.mode;
+                        secrets.block_id.generation = self.generation;
+                        secrets.block_id.inode = inode;
+                        secrets.mode = args.mode & !args.umask;
                         secrets.uid = ctx.uid;
                         secrets.gid = ctx.gid;
                         let now = Epoch::now();
@@ -1355,6 +1358,8 @@ mod private {
             let stat = self.open_value(inode, block_path, |value| {
                 let stat = value.borrow_block(|block| {
                     let stat = block.mut_meta_body().access_secrets(|secrets| {
+                        secrets.block_id.generation = self.generation;
+                        secrets.block_id.inode = inode;
                         secrets.mode = libc::S_IFDIR | mode & !umask;
                         secrets.uid = ctx.uid;
                         secrets.gid = ctx.gid;

crates/btlib/src/crypto/mod.rs

@@ -1,8 +1,11 @@
-pub mod merkle_stream;
 pub mod tpm;
+
+mod merkle_stream;
 pub use merkle_stream::MerkleStream;
-pub mod secret_stream;
+mod secret_stream;
 pub use secret_stream::SecretStream;
+mod sign_stream;
+pub use sign_stream::SignStream;
 
 use crate::{
     fmt, io, BigArray, BlockMeta, BlockPath, Deserialize, Epoch, Formatter, Hashable, Principal,
@@ -1584,7 +1587,6 @@ impl ConcreteCreds {
         }
     }
 
-    #[cfg(test)]
     pub fn generate() -> Result<ConcreteCreds> {
         let encrypt = Encrypt::RSA_OAEP_3072_SHA_256.generate()?;
         let sign = Sign::RSA_PSS_3072_SHA_256.generate()?;
@@ -1781,6 +1783,9 @@ pub trait Signer {
 
     /// Starts a new signing operation and returns the struct representing it.
     fn init_sign(&self) -> Result<Self::Op<'_>>;
+    /// Returns a signature over the given parts. It's critical that subsequent invocations
+    /// of this method on the same instance return a [Signature] with `data` fields of the same
+    /// length.
     fn sign<'a, I: Iterator<Item = &'a [u8]>>(&self, parts: I) -> Result<Signature>;
 
     fn ser_sign<T: Serialize>(&self, value: &T) -> Result<Signed<T>> {

crates/btlib/src/crypto/sign_stream.rs

@@ -0,0 +1,405 @@
+pub use private::SignStream;
+
+mod private {
+    use crate::{
+        bterr,
+        crypto::{Error, Result, Signature, Signer, Verifier},
+        Block, BlockMeta, Decompose, MetaAccess, ReadExt, Sectored,
+    };
+    use btserde::{read_from, to_vec, write_to};
+
+    use std::io::{self, Read, Seek, SeekFrom, Write};
+
+    /// The length of the array in the `SignStream::index_bytes` field.
+    const INDEX_BYTES_LEN: usize = std::mem::size_of::<u64>();
+
+    /// A stream which signs each sector of data written and verifies each sector read.
+    ///
+    /// Note that the correct [BlockId] needs to be configured for the inner stream before the
+    /// first call to `read` or `write`. Upon the first such call the `id_bytes` field will be
+    /// initialized by serializing the current value of the [BlockId] obtained from the inner stream
+    /// using the [MetaAccess] trait.
+    pub struct SignStream<T, C> {
+        inner: T,
+        creds: C,
+        /// The `btserde` serialization of the `BlockId` of this stream. This data is signed
+        /// along with the index of each sector to ensure sectors cannot be reordered or moved
+        /// between blocks.
+        id_bytes: Option<Vec<u8>>,
+        /// The 0-based index of the next sector to be read or written.
+        index: u64,
+        /// The `btserde` serialization of index.
+        index_bytes: [u8; INDEX_BYTES_LEN],
+        /// The sector size of this stream. This is the size of the buffer expected by the
+        /// `read` and `write` methods.
+        out_sz: usize,
+        /// The length of the `data` field expected from [Signature] instances.
+        sig_len: usize,
+    }
+
+    impl<T: Sectored, C: Signer> SignStream<T, C> {
+        #[allow(dead_code)]
+        pub fn new(inner: T, creds: C) -> Result<SignStream<T, C>> {
+            // TODO: This is way too brittle. If creds that produce a different sized signature
+            // are ever used in the future, then this will break.
+            let (extra, sig_len) = {
+                let sig = creds.sign(std::iter::empty())?;
+                let vec = to_vec(&sig)?;
+                (vec.len(), sig.data.len())
+            };
+            let in_sz = inner.sector_sz();
+            if in_sz < extra {
+                return Err(bterr!(Error::custom("sector size is too small")));
+            }
+            let out_sz = in_sz - extra;
+            Ok(SignStream {
+                inner,
+                creds,
+                id_bytes: None,
+                index: 0,
+                index_bytes: [0u8; INDEX_BYTES_LEN],
+                out_sz,
+                sig_len,
+            })
+        }
+    }
+
+    impl<T, C> SignStream<T, C> {
+        fn out_to_index(&self, outer_offset: u64) -> u64 {
+            outer_offset / self.out_sz as u64
+        }
+
+        /// Asserts that the `data` field in the given [Signature] is the correct length.
+        fn assert_sig_len(&self, sig: &Signature) -> Result<()> {
+            let actual = sig.data.len();
+            if self.sig_len != actual {
+                Err(Error::IncorrectSize {
+                    expected: self.sig_len,
+                    actual,
+                })
+            } else {
+                Ok(())
+            }
+        }
+
+        fn set_index(&mut self, index: u64) -> Result<()> {
+            let mut slice = self.index_bytes.as_mut_slice();
+            write_to(&index, &mut slice)?;
+            self.index = index;
+            Ok(())
+        }
+
+        fn incr_index(&mut self) -> Result<()> {
+            self.set_index(self.index + 1)
+        }
+    }
+
+    impl<T: MetaAccess, C> SignStream<T, C> {
+        fn sig_input<'b, 's: 'b>(
+            &'s self,
+            buf: &'b [u8],
+        ) -> Result<impl Iterator<Item = &'b [u8]>> {
+            let id_bytes = self
+                .id_bytes
+                .as_ref()
+                .ok_or_else(|| Error::custom("id_bytes has not been initialized"))?;
+            Ok([id_bytes, self.index_bytes.as_slice(), buf].into_iter())
+        }
+
+        fn init_id_bytes(&mut self) -> Result<()> {
+            if self.id_bytes.is_none() {
+                let vec = to_vec(self.inner.meta_body().block_id()?)?;
+                self.id_bytes = Some(vec);
+            }
+            Ok(())
+        }
+    }
+
+    impl<T: Sectored, C> SignStream<T, C> {
+        fn index_to_in(&self, index: u64) -> u64 {
+            self.inner.offset_at(index as usize) as u64
+        }
+
+        fn index_to_out(&self, index: u64) -> u64 {
+            self.out_sz as u64 * index
+        }
+    }
+
+    impl<T: Sectored + Seek, C> SignStream<T, C> {
+        fn reset_inner_pos(&mut self) -> io::Result<u64> {
+            self.inner
+                .seek(SeekFrom::Start(self.index_to_in(self.index)))
+        }
+    }
+
+    impl<T, C> Sectored for SignStream<T, C> {
+        fn sector_sz(&self) -> usize {
+            self.out_sz
+        }
+    }
+
+    impl<T: Write + MetaAccess, C: Signer> Write for SignStream<T, C> {
+        fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+            self.assert_sector_sz(buf.len())?;
+            self.inner.write_all(buf)?;
+            self.init_id_bytes()?;
+            let sig = self.creds.sign(self.sig_input(buf)?)?;
+            self.assert_sig_len(&sig)?;
+            write_to(&sig, &mut self.inner)?;
+            self.incr_index()?;
+            Ok(self.out_sz)
+        }
+
+        fn flush(&mut self) -> std::io::Result<()> {
+            self.inner.flush()
+        }
+    }
+
+    impl<T: Read + Seek + Sectored + MetaAccess, C: Verifier> Read for SignStream<T, C> {
+        fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+            self.assert_at_least_sector_sz(buf.len())?;
+            let buf = &mut buf[..self.sector_sz()];
+            let read = self.inner.fill_buf(buf)?;
+            if 0 == read {
+                return Ok(0);
+            }
+            self.assert_sector_sz(read)?;
+            let sig: Signature = match read_from(&mut self.inner) {
+                Ok(sig) => sig,
+                Err(err) => {
+                    self.reset_inner_pos()?;
+                    return Err(err.into());
+                }
+            };
+            self.init_id_bytes()?;
+            let result = self.creds.verify(self.sig_input(buf)?, sig.as_slice());
+            if let Err(err) = result {
+                self.reset_inner_pos()?;
+                return Err(err.into());
+            }
+            if let Err(err) = self.incr_index() {
+                self.reset_inner_pos()?;
+                return Err(err.into());
+            }
+            Ok(read)
+        }
+    }
+
+    impl<T: Seek + Sectored, C> Seek for SignStream<T, C> {
+        fn seek(&mut self, out_pos: std::io::SeekFrom) -> std::io::Result<u64> {
+            let out_pos = match out_pos {
+                SeekFrom::Start(from_start) => from_start,
+                SeekFrom::Current(from_curr) => {
+                    self.index_to_out(self.index).wrapping_add_signed(from_curr)
+                }
+                SeekFrom::End(_) => {
+                    return Err(bterr!(io::Error::new(
+                        io::ErrorKind::Unsupported,
+                        "seeking from end is not supported"
+                    )))
+                }
+            };
+            let index = self.out_to_index(out_pos);
+            let in_pos = self.index_to_in(index);
+            self.inner.seek(SeekFrom::Start(in_pos))?;
+            self.set_index(index)?;
+            Ok(out_pos)
+        }
+    }
+
+    impl<T: AsRef<BlockMeta>, C> AsRef<BlockMeta> for SignStream<T, C> {
+        fn as_ref(&self) -> &BlockMeta {
+            self.inner.as_ref()
+        }
+    }
+
+    impl<T: AsMut<BlockMeta>, C> AsMut<BlockMeta> for SignStream<T, C> {
+        fn as_mut(&mut self) -> &mut BlockMeta {
+            self.inner.as_mut()
+        }
+    }
+
+    impl<T: MetaAccess, C> MetaAccess for SignStream<T, C> {}
+
+    impl<T: Block + Sectored, C: Signer + Verifier> Block for SignStream<T, C> {
+        fn flush_meta(&mut self) -> crate::Result<()> {
+            self.inner.flush_meta()
+        }
+    }
+
+    impl<T, C> Decompose<T> for SignStream<T, C> {
+        fn into_inner(self) -> T {
+            self.inner
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::io::{Read, Seek, SeekFrom, Write};
+
+    use super::*;
+    use crate::{
+        crypto::{ConcreteCreds, Error},
+        test_helpers::{node_creds, Randomizer, SectoredCursor},
+        Decompose, Sectored, SECTOR_SZ_DEFAULT,
+    };
+
+    fn sign_stream_with_block_id(
+        sect_sz: usize,
+    ) -> SignStream<SectoredCursor<Vec<u8>>, &'static ConcreteCreds> {
+        let cursor = SectoredCursor::new(Vec::new(), sect_sz).require_sect_sz(false);
+        SignStream::new(cursor, node_creds()).unwrap()
+    }
+
+    fn sign_stream_with_sz(
+        sect_sz: usize,
+    ) -> SignStream<SectoredCursor<Vec<u8>>, &'static ConcreteCreds> {
+        sign_stream_with_block_id(sect_sz)
+    }
+
+    fn sign_stream() -> SignStream<SectoredCursor<Vec<u8>>, &'static ConcreteCreds> {
+        sign_stream_with_sz(SECTOR_SZ_DEFAULT)
+    }
+
+    #[test]
+    fn new_empty() {
+        let _ = sign_stream();
+    }
+
+    #[test]
+    fn write() {
+        let mut stream = sign_stream();
+        let data = vec![1u8; stream.sector_sz()];
+
+        stream.write(&data).expect("write failed");
+    }
+
+    #[test]
+    fn seek() {
+        let in_sect_sz = SECTOR_SZ_DEFAULT;
+        let mut stream = sign_stream_with_sz(in_sect_sz);
+        let out_sect_sz = stream.sector_sz();
+        let expected: u64 = out_sect_sz.try_into().unwrap();
+        let data = vec![1u8; out_sect_sz];
+        stream.write(&data).expect("first write failed");
+        stream.write(&data).expect("second write failed");
+
+        let actual = stream.seek(SeekFrom::Start(expected)).expect("seek failed");
+
+        assert_eq!(expected, actual);
+        let expected: u64 = in_sect_sz.try_into().unwrap();
+        let actual = stream
+            .into_inner()
+            .stream_position()
+            .expect("stream_position failed");
+        assert_eq!(expected, actual);
+    }
+
+    #[test]
+    fn write_read_once() {
+        let mut stream = sign_stream();
+        let sect_sz = stream.sector_sz();
+        let expected = vec![1u8; sect_sz];
+        stream.write(&expected).expect("write failed");
+        stream.seek(SeekFrom::Start(0)).expect("seek failed");
+        let mut actual = vec![0u8; sect_sz];
+
+        let read = stream.read(&mut actual).expect("read failed");
+
+        assert_eq!(sect_sz, read);
+        assert_eq!(expected, actual);
+    }
+
+    fn fill_vec<T: Clone>(vec: &mut Vec<T>, value: T) -> &mut Vec<T> {
+        vec.clear();
+        vec.extend(std::iter::repeat(value).take(vec.capacity()));
+        vec
+    }
+
+    #[test]
+    fn write_read_many() {
+        const ITER: u8 = 16;
+        let mut stream = sign_stream();
+        let sect_sz = stream.sector_sz();
+        let mut expected = Vec::with_capacity(sect_sz);
+        let mut actual = Vec::with_capacity(sect_sz);
+
+        for k in 0..ITER {
+            fill_vec(&mut expected, k);
+            stream.write(&expected).expect("write failed");
+        }
+
+        stream.seek(SeekFrom::Start(0)).expect("seek failed");
+
+        for k in 0..ITER {
+            let read = stream.read(fill_vec(&mut actual, 0)).expect("read failed");
+            assert_eq!(sect_sz, read);
+            fill_vec(&mut expected, k);
+            assert_eq!(expected, actual);
+        }
+    }
+
+    #[test]
+    fn write_read_random() {
+        const ITER: usize = 16;
+        let mut stream = sign_stream();
+        let sect_sz = stream.sector_sz();
+        let rando = Randomizer::new([37; Randomizer::HASH.len()]);
+        let indices: Vec<usize> = rando.take(ITER).map(|e| e % ITER).collect();
+        let mut expected = Vec::with_capacity(sect_sz);
+        let mut actual = Vec::with_capacity(sect_sz);
+        // Fill the stream with zeros.
+        for _ in 0..ITER {
+            stream
+                .write(fill_vec(&mut expected, 0))
+                .expect("write failed");
+        }
+
+        for index in indices.iter().map(|e| *e) {
+            let offset = stream.offset_at(index);
+            stream
+                .seek(SeekFrom::Start(offset as u64))
+                .expect("seek failed");
+            fill_vec(&mut expected, (index + 1) as u8);
+            stream.write(&expected).expect("write failed");
+        }
+
+        for index in indices.iter().map(|e| *e) {
+            let offset = stream.offset_at(index);
+            stream
+                .seek(SeekFrom::Start(offset as u64))
+                .expect("seek failed");
+            fill_vec(&mut expected, (index + 1) as u8);
+            stream.read(fill_vec(&mut actual, 0)).expect("read failed");
+            assert_eq!(expected, actual);
+        }
+    }
+
+    #[test]
+    fn modify_inner_is_err() {
+        let in_sect_sz = SECTOR_SZ_DEFAULT;
+        let inner = SectoredCursor::new(Vec::new(), in_sect_sz).require_sect_sz(false);
+        let mut stream = SignStream::new(inner, node_creds()).expect("SignStream::new failed");
+        let out_sect_sz = stream.sector_sz();
+        let sect = vec![0u8; out_sect_sz];
+        stream.write(&sect).expect("write failed");
+        let mut inner = stream.into_inner();
+        inner.seek(SeekFrom::Start(0)).expect("seek failed");
+        inner.write(&[1u8]).expect("second write failed");
+        inner.seek(SeekFrom::Start(0)).expect("seek failed");
+        let mut stream =
+            SignStream::new(inner, node_creds()).expect("second SignStream::new failed");
+        let mut buf = vec![0u8; out_sect_sz];
+
+        let result = stream.read(&mut buf);
+
+        let actual_err = result.err().unwrap().into_inner().unwrap();
+        let actual: &Error = actual_err.downcast_ref().unwrap();
+        let is_match = match actual {
+            Error::InvalidSignature => true,
+            _ => false,
+        };
+        assert!(is_match);
+    }
+}

crates/btlib/src/lib.rs

@@ -44,6 +44,13 @@ use std::{
 };
 use strum_macros::{Display, EnumDiscriminants, FromRepr};
 
+#[macro_export]
+macro_rules! bterr {
+    ($err:expr) => {
+        $err
+    };
+}
+
 #[derive(Debug)]
 pub enum Error {
     MissingWritecap,
@@ -166,7 +173,7 @@ impl<T, E: Display> StrInIoErr<T> for std::result::Result<T, E> {
 }
 
 /// The default sector size to use for new blocks.
-const SECTOR_SZ_DEFAULT: usize = 4096;
+pub const SECTOR_SZ_DEFAULT: usize = 4096;
 
 /// ### THE BLOCK TRAIT
 ///
@@ -207,14 +214,31 @@ pub trait Sectored {
     // Returns the size of the sector for this stream.
     fn sector_sz(&self) -> usize;
 
+    /// Returns `Err(Error::IncorrectSize)` if the given size is not equal to the sector size.
     fn assert_sector_sz(&self, actual: usize) -> Result<()> {
         let expected = self.sector_sz();
-        if expected == actual {
-            Ok(())
+        if expected != actual {
+            Err(Error::IncorrectSize { expected, actual })
         } else {
+            Ok(())
+        }
+    }
+
+    /// Returns `Err(Error::IncorrectSize)` if the given size is less than the sector size.
+    fn assert_at_least_sector_sz(&self, actual: usize) -> Result<()> {
+        let expected = self.sector_sz();
+        if actual < expected {
             Err(Error::IncorrectSize { expected, actual })
+        } else {
+            Ok(())
         }
     }
+
+    /// Returns the offset (in bytes) from the beginning of this stream that the given 0-based
+    /// sector index corresponds to.
+    fn offset_at(&self, index: usize) -> usize {
+        index * self.sector_sz()
+    }
 }
 
 /// A version of the `Write` trait, which allows integrity information to be supplied when flushing.
@@ -325,11 +349,30 @@ trait HashExt: Hashable {
 
 impl<T: Hashable> HashExt for T {}
 
+/// A unique identifier for a block.
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Hash)]
+pub struct BlockId {
+    generation: u64,
+    inode: u64,
+}
+
+impl BlockId {
+    pub fn new(generation: u64, inode: u64) -> BlockId {
+        BlockId { generation, inode }
+    }
+}
+
+impl Default for BlockId {
+    fn default() -> Self {
+        BlockId::new(0, 0)
+    }
+}
+
 /// Metadata that is encrypted.
 #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone, Hash)]
 pub struct BlockMetaSecrets {
-    /// The inode number of the file.
-    inode: u64,
+    /// The identifier for the block these secrets are for.
+    block_id: BlockId,
     /// Mode of file.
     mode: u32,
     /// Owner UID of file.
@@ -348,16 +391,46 @@ pub struct BlockMetaSecrets {
     nlink: u32,
     /// The sector size used by the block.
     sect_sz: u32,
+    /// User controlled metadata.
     tags: BTreeMap<String, Vec<u8>>,
 }
 
 impl BlockMetaSecrets {
     pub fn new() -> BlockMetaSecrets {
-        Self::default()
+        Self {
+            block_id: BlockId::default(),
+            mode: 0,
+            uid: 0,
+            gid: 0,
+            atime: Epoch::default(),
+            mtime: Epoch::default(),
+            ctime: Epoch::default(),
+            size: 0,
+            nlink: 0,
+            sect_sz: SECTOR_SZ_DEFAULT.try_into().unwrap(),
+            tags: BTreeMap::new(),
+        }
     }
 
     pub fn attr(&self) -> Attr {
-        self.into()
+        Attr {
+            ino: self.block_id.inode,
+            size: self.size,
+            atime: self.atime.value(),
+            mtime: self.mtime.value(),
+            ctime: self.ctime.value(),
+            atimensec: 0,
+            mtimensec: 0,
+            ctimensec: 0,
+            mode: self.mode,
+            nlink: self.nlink,
+            uid: self.uid,
+            gid: self.gid,
+            rdev: 0,
+            blksize: self.sect_sz,
+            blocks: self.sectors(),
+            flags: 0,
+        }
     }
 
     pub fn stat(&self) -> stat64 {
@@ -373,46 +446,31 @@ impl BlockMetaSecrets {
             self.size / sect_sz + 1
         }
     }
+
+    pub fn block_id(&self) -> &BlockId {
+        &self.block_id
+    }
+
+    pub fn sector_sz(&self) -> u32 {
+        self.sect_sz
+    }
 }
 
 impl Default for BlockMetaSecrets {
     fn default() -> Self {
-        Self {
-            inode: 0,
-            mode: 0,
-            uid: 0,
-            gid: 0,
-            atime: Epoch::default(),
-            mtime: Epoch::default(),
-            ctime: Epoch::default(),
-            size: 0,
-            nlink: 0,
-            sect_sz: SECTOR_SZ_DEFAULT.try_into().unwrap(),
-            tags: BTreeMap::new(),
-        }
+        Self::new()
+    }
+}
+
+impl AsRef<BlockId> for BlockMetaSecrets {
+    fn as_ref(&self) -> &BlockId {
+        self.block_id()
     }
 }
 
 impl From<&BlockMetaSecrets> for Attr {
     fn from(value: &BlockMetaSecrets) -> Self {
-        Attr {
-            ino: value.inode,
-            size: value.size,
-            atime: value.atime.value(),
-            mtime: value.mtime.value(),
-            ctime: value.ctime.value(),
-            atimensec: 0,
-            mtimensec: 0,
-            ctimensec: 0,
-            mode: value.mode,
-            nlink: value.nlink,
-            uid: value.uid,
-            gid: value.gid,
-            rdev: 0,
-            blksize: value.sect_sz,
-            blocks: value.sectors(),
-            flags: 0,
-        }
+        value.attr()
     }
 }
 
@@ -464,7 +522,7 @@ impl BlockMetaBody {
         })
     }
 
-    pub fn unlock_block_key<C: Creds>(&mut self, creds: &C) -> Result<()> {
+    pub fn unlock_block_key<C: Decrypter + Principaled>(&mut self, creds: &C) -> Result<()> {
         if self.block_key.is_some() {
             return Ok(());
         }
@@ -480,26 +538,25 @@ impl BlockMetaBody {
         &mut self,
         accessor: F,
     ) -> Result<T> {
-        let secrets = match self.secrets_struct.as_mut() {
-            Some(secrets) => secrets,
-            None => {
-                let block_key = self.block_key()?;
-                self.secrets_struct = Some(block_key.ser_decrypt(&self.secrets)?);
-                self.secrets_struct.as_mut().unwrap()
-            }
-        };
-        let prev = secrets.default_hash();
+        self.decrypt_secrets()?;
+        let secrets = self.secrets_struct.as_mut().unwrap();
         let output = accessor(secrets)?;
-        if prev != secrets.default_hash() {
-            self.secrets = self
-                .block_key
-                .as_ref()
-                .ok_or(Error::NoBlockKey)?
-                .ser_encrypt(secrets)?;
-        }
+        self.secrets = self
+            .block_key
+            .as_ref()
+            .ok_or(Error::NoBlockKey)?
+            .ser_encrypt(secrets)?;
         Ok(output)
     }
 
+    pub fn decrypt_secrets(&mut self) -> Result<()> {
+        if self.secrets_struct.is_none() {
+            let block_key = self.block_key()?;
+            self.secrets_struct = Some(block_key.ser_decrypt(&self.secrets)?);
+        }
+        Ok(())
+    }
+
     pub fn secrets(&self) -> Result<&BlockMetaSecrets> {
         self.secrets_struct
             .as_ref()
@@ -510,7 +567,16 @@ impl BlockMetaBody {
         self.block_key.as_ref().ok_or(Error::NoBlockKey)
     }
 
-    pub fn use_block_key_for<C: Creds>(&mut self, creds: &C) -> Result<&SymKey> {
+    pub fn block_id(&self) -> Result<&BlockId> {
+        let secrets = self.secrets()?;
+        Ok(secrets.block_id())
+    }
+
+    pub fn integrity(&self) -> Option<&[u8]> {
+        self.integrity.as_ref().map(|hash| hash.as_slice())
+    }
+
+    pub fn use_readcap_for<C: Creds>(&mut self, creds: &C) -> Result<&SymKey> {
         let readcap = self
             .readcaps
             .get(&creds.principal())
@@ -524,10 +590,6 @@ impl BlockMetaBody {
         &mut self.block_key
     }
 
-    pub fn integrity(&self) -> Option<&[u8]> {
-        self.integrity.as_ref().map(|hash| hash.as_slice())
-    }
-
     pub fn add_readcap_for<E: Encrypter>(&mut self, owner: Principal, key: &E) -> Result<()> {
         let block_key = self.block_key()?;
         self.readcaps.insert(owner, key.ser_encrypt(block_key)?);
@@ -579,6 +641,7 @@ struct BlockStream<T, C> {
     meta: BlockMeta,
     meta_body_buf: Vec<u8>,
     creds: C,
+    sect_sz: usize,
 }
 
 impl<T: Read + Seek, C: Creds> BlockStream<T, C> {
@@ -588,11 +651,12 @@ impl<T: Read + Seek, C: Creds> BlockStream<T, C> {
             Some(mut meta) => {
                 meta.assert_valid(&block_path)?;
                 meta.body.path = block_path;
+                meta.body.use_readcap_for(&creds)?;
                 // We need to use the writecap and signing_key provided by the current credentials.
                 meta.body.writecap =
                     Some(creds.writecap().ok_or(Error::MissingWritecap)?.to_owned());
                 meta.body.signing_key = creds.public_sign().to_owned();
-                meta.body.use_block_key_for(&creds)?;
+                meta.body.decrypt_secrets()?;
                 meta
             }
             None => {
@@ -604,11 +668,13 @@ impl<T: Read + Seek, C: Creds> BlockStream<T, C> {
                 meta
             }
         };
+        let sect_sz = meta.body.secrets()?.sector_sz().try_into().box_err()?;
         Ok(BlockStream {
             trailered,
-            meta,
             meta_body_buf: Vec::new(),
             creds,
+            sect_sz,
+            meta,
         })
     }
 }
@@ -629,10 +695,7 @@ impl<T: Write + Seek, C: Signer + Principaled + Decrypter> Write for BlockStream
     }
 
     fn flush(&mut self) -> io::Result<()> {
-        Err(io::Error::new(
-            io::ErrorKind::Unsupported,
-            "flush is not supported, use flush_integ instead",
-        ))
+        self.sign_flush_meta()
     }
 }
 
@@ -673,12 +736,24 @@ impl<T, C> AsMut<BlockMeta> for BlockStream<T, C> {
 
 impl<T, C> MetaAccess for BlockStream<T, C> {}
 
+impl<T, C> Sectored for BlockStream<T, C> {
+    fn sector_sz(&self) -> usize {
+        self.sect_sz
+    }
+}
+
 impl<T: Read + Write + Seek, C: Creds> Block for BlockStream<T, C> {
     fn flush_meta(&mut self) -> Result<()> {
         self.sign_flush_meta().map_err(|err| err.into())
     }
 }
 
+impl<T, C> Decompose<(T, C)> for BlockStream<T, C> {
+    fn into_inner(self) -> (T, C) {
+        (self.trailered.into_inner(), self.creds)
+    }
+}
+
 pub struct BlockOpenOptions<T, C> {
     inner: T,
     creds: C,
@@ -743,11 +818,6 @@ impl<T: Read + Write + Seek + 'static, C: Creds + 'static> BlockOpenOptions<T, C
         let block_key = stream.meta_body().block_key().map(|e| e.to_owned())?;
         let mut stream = MerkleStream::new(stream)?;
         stream.assert_root_integrity()?;
-        let sect_sz = stream.sector_sz();
-        stream.mut_meta_body().access_secrets(|secrets| {
-            secrets.sect_sz = sect_sz.try_into()?;
-            Ok(())
-        })?;
         if self.encrypt {
             let stream = SecretStream::new(block_key).try_compose(stream)?;
             let stream = SectoredBuf::new().try_compose(stream)?;
@@ -1143,14 +1213,10 @@ impl<F: FnOnce()> Drop for DropTrigger<F> {
 mod tests {
     use std::{fs::OpenOptions, io::Cursor, path::PathBuf};
 
-    use crate::crypto::{
-        tpm::{TpmCredStore, TpmCreds},
-        ConcreteCreds, CredStore, CredsPriv,
-    };
+    use crate::crypto::{ConcreteCreds, CredsPriv, SignStream};
 
     use super::*;
     use btserde::{from_vec, to_vec};
-    use swtpm_harness::SwtpmHarness;
     use tempdir::TempDir;
     use test_helpers::*;
 
@@ -1202,28 +1268,111 @@ mod tests {
         assert_eq!(UID, actual_uid);
     }
 
-    struct BlockTestCase {
-        root_creds: TpmCreds,
-        node_creds: TpmCreds,
-        _swtpm: SwtpmHarness,
+    struct InMemTestCase {
+        node_creds: ConcreteCreds,
+        block_path: BlockPath,
+        block_id: BlockId,
+    }
+
+    type EncBlock = SectoredBuf<
+        SecretStream<SignStream<BlockStream<Cursor<Vec<u8>>, ConcreteCreds>, ConcreteCreds>>,
+    >;
+
+    impl InMemTestCase {
+        fn new() -> InMemTestCase {
+            let components = vec!["nodes".to_string(), "phone".to_string()];
+            let node_creds = {
+                let mut node_creds = node_creds().clone();
+                let writecap = root_creds()
+                    .issue_writecap(
+                        node_creds.principal(),
+                        components.clone(),
+                        Epoch::now() + Duration::from_secs(3600),
+                    )
+                    .expect("failed to issue writecap");
+                node_creds.set_writecap(writecap);
+                node_creds
+            };
+            let block_path = BlockPath::new(root_creds().principal(), components);
+            let block_id = BlockId::default();
+            Self {
+                node_creds,
+                block_path,
+                block_id,
+            }
+        }
+
+        fn stream(&self, vec: Vec<u8>) -> EncBlock {
+            let inner = Cursor::new(vec);
+            let mut stream =
+                BlockStream::new(inner, self.node_creds.clone(), self.block_path.clone()).unwrap();
+            stream
+                .mut_meta_body()
+                .access_secrets(|secrets| {
+                    secrets.block_id = self.block_id.clone();
+                    Ok(())
+                })
+                .unwrap();
+            let block_key = stream.meta_body().block_key().unwrap().to_owned();
+            let stream = SignStream::new(stream, self.node_creds.clone()).unwrap();
+            let stream = SecretStream::new(block_key).try_compose(stream).unwrap();
+            SectoredBuf::new().try_compose(stream).unwrap()
+        }
+
+        fn into_vec(stream: EncBlock) -> Vec<u8> {
+            stream
+                .into_inner()
+                .into_inner()
+                .into_inner()
+                .into_inner()
+                .0
+                .into_inner()
+        }
+    }
+
+    #[test]
+    fn block_write_read_with_cursor() {
+        const EXPECTED: &[u8] = b"Silly sordid sulking sultans.";
+        let case = InMemTestCase::new();
+        let mut stream = case.stream(Vec::new());
+
+        stream.write_all(EXPECTED).unwrap();
+        stream.flush().unwrap();
+
+        let vec = InMemTestCase::into_vec(stream);
+        let mut stream = case.stream(vec);
+        let mut actual = [0u8; EXPECTED.len()];
+        stream.read(&mut actual).unwrap();
+
+        assert_eq!(EXPECTED, actual);
+    }
+
+    #[test]
+    fn block_write_multiple() {
+        const ITER: usize = 16;
+        let case = InMemTestCase::new();
+        let mut stream = case.stream(Vec::new());
+        let expected = vec![1u8; stream.sector_sz()];
+
+        for _ in 0..ITER {
+            stream.write(&expected).unwrap();
+        }
+        stream.flush().unwrap();
+    }
+
+    pub struct BlockTestCase {
         temp_dir: TempDir,
         root_path: BlockPath,
         node_path: BlockPath,
+        root_creds: ConcreteCreds,
+        node_creds: ConcreteCreds,
     }
 
     impl BlockTestCase {
-        const ROOT_PASSWORD: &'static str = "(1337Prestidigitation7331)";
-
         fn new() -> BlockTestCase {
             let temp_dir = TempDir::new("block_test").expect("failed to create temp dir");
-            let swtpm = SwtpmHarness::new().expect("failed to start swtpm");
-            let context = swtpm.context().expect("failed to retrieve context");
-            let cred_store = TpmCredStore::new(context, swtpm.state_path().to_owned())
-                .expect("failed to create TpmCredStore");
-            let root_creds = cred_store
-                .gen_root_creds(Self::ROOT_PASSWORD)
-                .expect("failed to get root creds");
-            let mut node_creds = cred_store.node_creds().expect("failed to get node creds");
+            let root_creds = test_helpers::ROOT_CREDS.clone();
+            let mut node_creds = test_helpers::NODE_CREDS.clone();
             let components = vec!["nodes".to_string(), "phone".to_string()];
             let writecap = root_creds
                 .issue_writecap(
@@ -1232,15 +1381,12 @@ mod tests {
                     Epoch::now() + Duration::from_secs(3600),
                 )
                 .expect("failed to issue writecap");
-            cred_store
-                .assign_node_writecap(&mut node_creds, writecap)
-                .expect("failed to assign node writecap");
+            node_creds.set_writecap(writecap);
             let case = BlockTestCase {
                 temp_dir,
-                _swtpm: swtpm,
-                node_creds,
                 node_path: BlockPath::new(root_creds.principal(), components),
                 root_path: BlockPath::new(root_creds.principal(), vec![]),
+                node_creds,
                 root_creds,
             };
             std::fs::create_dir_all(case.fs_path(&case.node_path))

crates/btlib/src/test_helpers.rs

@@ -100,6 +100,14 @@ lazy_static! {
     };
 }
 
+pub fn node_creds() -> &'static ConcreteCreds {
+    &NODE_CREDS
+}
+
+pub fn root_creds() -> &'static ConcreteCreds {
+    &ROOT_CREDS
+}
+
 /// Converts the given error to a serde_block_tree error by turning it into a message.
 fn convert_err<E: Display>(err: E) -> Error {
     Error::Message(err.to_string())
@@ -574,6 +582,7 @@ pub struct SectoredCursor<T: FromVec> {
     cursor: BtCursor<T>,
     sect_sz: usize,
     meta: BlockMeta,
+    require_sect_sz: bool,
 }
 
 impl<T: FromVec> SectoredCursor<T> {
@@ -582,8 +591,14 @@ impl<T: FromVec> SectoredCursor<T> {
             cursor: BtCursor::new(inner),
             sect_sz,
             meta: BlockMeta::new(&*NODE_CREDS).unwrap(),
+            require_sect_sz: true,
         }
     }
+
+    pub fn require_sect_sz(mut self, require: bool) -> Self {
+        self.require_sect_sz = require;
+        self
+    }
 }
 
 impl<T: FromVec> Sectored for SectoredCursor<T> {
@@ -594,7 +609,9 @@ impl<T: FromVec> Sectored for SectoredCursor<T> {
 
 impl Write for SectoredCursor<Vec<u8>> {
     fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
-        self.assert_sector_sz(buf.len())?;
+        if self.require_sect_sz {
+            self.assert_sector_sz(buf.len())?;
+        }
         self.cursor.write(buf)
     }
 
@@ -605,7 +622,9 @@ impl Write for SectoredCursor<Vec<u8>> {
 
 impl<const N: usize> Write for SectoredCursor<[u8; N]> {
     fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
-        self.assert_sector_sz(buf.len())?;
+        if self.require_sect_sz {
+            self.assert_sector_sz(buf.len())?;
+        }
         self.cursor.write(buf)
     }
 
@@ -616,7 +635,9 @@ impl<const N: usize> Write for SectoredCursor<[u8; N]> {
 
 impl<T: FromVec> Read for SectoredCursor<T> {
     fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
-        self.assert_sector_sz(buf.len())?;
+        if self.require_sect_sz {
+            self.assert_sector_sz(buf.len())?;
+        }
         self.cursor.read(buf)
     }
 }

crates/btlib/src/trailered.rs

@@ -36,7 +36,8 @@ mod private {
             if 0 == end {
                 return Ok((Self::empty(inner), None));
             }
-            inner.seek(SeekFrom::End(-8))?;
+            let offset: i64 = std::mem::size_of::<i64>().try_into().unwrap();
+            inner.seek(SeekFrom::End(-offset))?;
             let offset: i64 = read_from(&mut inner)?;
             let body_len = inner.seek(SeekFrom::End(offset))?;
             let trailer: D = read_from(&mut inner)?;