* Started querying the file metadata for its preferred blksize.

* Improved interoperability between usize and u64 by assuming
  `size_of::<usize>() <= size_of::<u64>()`.

TODO.txt

@@ -4,16 +4,12 @@
 Fix bug where writing to a block that already has a Writecap in its header using the creds of
 a different node produces an invalid signature (a signature using the creds of the other node).
 
-- 1
+!- 1
 Fix BufSectored so it doesn't have to write to the first sector every flush.
 
 - 2
 Track position and dirty-ness in Trailered.
 
-- 3
-Implement a stream which is both Read and Write and which can transparently compress and decompress
-data written to and read from it.
-
 - 4
 Remove TryCompose?
 
@@ -24,8 +20,9 @@ Move crypto::{encrypt, decrypt} into corresponding {EncrypterExt, DecrypterExt}.
 Add a ser_sign_into method to SignerExt which serializes a value into a provided Vec<u8> and returns
 a signature over this data. Update BlockStream::flush_integ to use this method.
 
-- 8
+!- 8
 Convert all sector sizes to u32 for portability.
+(I ended up using u64 but keeping usize as the return type for Sectored::sector_sz)
 
 - 9
 Create an extension trait for u64 with a method for adding an i64 to it. Use this in
@@ -66,7 +63,7 @@ to find their readcap and to rotate the block and create new readcaps for each o
 the dictionary, but prevent an attacker from being able to identify when two blocks contain
 readcaps for the same principal.
 
-- 18, 3, mdcarr941@gmail.com, 8665339,
+!- 18, 3, mdcarr941@gmail.com, 8665339, ???
 SECURITY: Remove the path field from BlockMeta. It isn't needed as the block path should be
 independently know by any verified. This will ensure that path names are not stored in cleartext.
 
@@ -101,4 +98,7 @@ modifications, a new field with the serde(skip) attribute needs to be added to B
 store the hash of BlockMetaSecrets that was computed just after decryption. 
 
 - 24, 3, mdcarr941@gmail.com, 7dbb358,
-Move `BlockRecord.frags` into `BlockMetaSecrets`.
+Move `BlockRecord.frags` into `BlockMetaSecrets`.
+
+- 25, 2, mdcarr941@gmail.com, 02d8cb,
+Implement `Blocktree::batch_forget`.

crates/btlib/src/blocktree.rs

@@ -877,7 +877,8 @@ mod private {
                 io::Error::new(io::ErrorKind::Unsupported, "can't lookup server entry")
             })?;
             let stat = self.open_value(inode, block_path, |value| {
-                let stat = value.borrow_block(|block| Ok(block.meta_body().secrets()?.stat()))?;
+                let stat =
+                    value.borrow_block(|block| Ok(block.meta_body().secrets()?.stat()?))?;
                 value.incr_lookup_count();
                 Ok(stat)
             })?;
@@ -1018,7 +1019,7 @@ mod private {
                         secrets.ctime = now;
                         secrets.mtime = now;
                         secrets.nlink = 1;
-                        Ok((handle, secrets.stat()))
+                        Ok((handle, secrets.stat()?))
                     })?)
                 })?;
 
@@ -1200,7 +1201,7 @@ mod private {
             _handle: Option<Self::Handle>,
         ) -> io::Result<(stat64, Duration)> {
             debug!("Blocktree::getattr called for inode {inode}");
-            let stat = self.access_meta(inode, |meta| Ok(meta.body.secrets()?.stat()))?;
+            let stat = self.access_meta(inode, |meta| Ok(meta.body.secrets()?.stat()?))?;
             Ok((stat, self.attr_timeout()))
         }
 
@@ -1283,7 +1284,7 @@ mod private {
                         let meta = block.mut_meta_body();
                         let (mode, stat) = meta.access_secrets(|secrets| {
                             secrets.nlink += 1;
-                            Ok((secrets.mode, secrets.stat()))
+                            Ok((secrets.mode, secrets.stat()?))
                         })?;
                         let file_type = FileType::from_value(mode)?;
                         block.flush_meta()?;
@@ -1351,7 +1352,7 @@ mod private {
                     if valid.intersects(SetattrValid::KILL_SUIDGID) {
                         secrets.mode &= !(libc::S_ISUID | libc::S_ISGID)
                     }
-                    Ok(secrets.stat())
+                    secrets.stat()
                 })?;
                 block.flush_meta()?;
                 Ok(stat)
@@ -1392,7 +1393,7 @@ mod private {
                         secrets.mode = libc::S_IFDIR | mode & !umask;
                         secrets.uid = ctx.uid;
                         secrets.gid = ctx.gid;
-                        Ok(secrets.stat())
+                        secrets.stat()
                     })?;
                     block.write_dir(&Directory::new())?;
                     block.flush_meta()?;

crates/btlib/src/crypto/sign_stream.rs

@@ -118,7 +118,7 @@ mod private {
 
     impl<T: Sectored, C> SignStream<T, C> {
         fn index_to_in(&self, index: u64) -> u64 {
-            self.inner.offset_at(index as usize) as u64
+            self.inner.offset_at(index)
         }
 
         fn index_to_out(&self, index: u64) -> u64 {
@@ -348,7 +348,7 @@ mod tests {
         let mut stream = sign_stream();
         let sect_sz = stream.sector_sz();
         let rando = Randomizer::new([37; Randomizer::HASH.len()]);
-        let indices: Vec<usize> = rando.take(ITER).map(|e| e % ITER).collect();
+        let indices: Vec<u64> = rando.take(ITER).map(|e| (e % ITER) as u64).collect();
         let mut expected = Vec::with_capacity(sect_sz);
         let mut actual = Vec::with_capacity(sect_sz);
         // Fill the stream with zeros.

crates/btlib/src/error.rs

@@ -49,10 +49,18 @@ impl Error {
         Self(err)
     }
 
+    pub fn map<T>(result: ::anyhow::Result<T>) -> Result<T> {
+        result.map_err(Error::new)
+    }
+
     pub fn downcast<E: std::error::Error + Send + Sync + 'static>(
         self,
     ) -> std::result::Result<E, Self> {
-        self.0.downcast::<E>().bterr()
+        self.0.downcast::<E>().map_err(Self::new)
+    }
+
+    pub fn context<C: Display + Send + Sync + 'static>(self, context: C) -> Self {
+        Self::new(self.0.context(context))
     }
 
     pub fn downcast_ref<E: Display + ::std::fmt::Debug + Send + Sync + 'static>(
@@ -168,9 +176,9 @@ pub trait BtErr<T> {
     fn bterr(self) -> Result<T>;
 }
 
-impl<T> BtErr<T> for std::result::Result<T, anyhow::Error> {
+impl<T, E: ::std::error::Error + Send + Sync + 'static> BtErr<T> for ::std::result::Result<T, E> {
     fn bterr(self) -> Result<T> {
-        self.map_err(Error::new)
+        self.map_err(|err| bterr!(err))
     }
 }
 

crates/btlib/src/lib.rs

@@ -25,7 +25,7 @@ use crypto::{
     AsymKeyPub, Ciphertext, Creds, Decrypter, DecrypterExt, Encrypter, EncrypterExt, HashKind,
     MerkleStream, PublicCreds, SecretStream, Sign, Signature, Signer, SymKey, SymKeyKind, VarHash,
 };
-use error::BoxInIoErr;
+use error::{BoxInIoErr, BtErr};
 use trailered::Trailered;
 
 use ::log::error;
@@ -35,13 +35,14 @@ use sectored_buf::SectoredBuf;
 use serde::{Deserialize, Serialize};
 use serde_big_array::BigArray;
 use std::{
-    collections::{btree_map, hash_map::DefaultHasher, BTreeMap, HashMap},
+    collections::{btree_map, BTreeMap, HashMap},
     convert::{Infallible, TryFrom},
     fmt::{self, Display, Formatter},
-    hash::{Hash as Hashable, Hasher},
+    hash::Hash as Hashable,
     io::{self, Read, Seek, SeekFrom, Write},
     net::SocketAddr,
     ops::{Add, Sub},
+    os::unix::prelude::MetadataExt,
     time::{Duration, SystemTime},
 };
 use strum_macros::{Display, EnumDiscriminants, FromRepr};
@@ -69,8 +70,14 @@ impl Display for BlockError {
 
 impl std::error::Error for BlockError {}
 
+// This assertion ensures that conversions from `usize` to `u64` will not cause truncation. This
+// prevents this code from compiling for 128 bit platforms, but that's not really a concern for the
+// foreseeable future.
+const_assert!(::std::mem::size_of::<usize>() <= ::std::mem::size_of::<u64>());
 /// The default sector size to use for new blocks.
 pub const SECTOR_SZ_DEFAULT: usize = 4096;
+/// `SECTOR_SZ_DEFAULT` converted to a `u64`.
+pub const SECTOR_U64_DEFAULT: u64 = SECTOR_SZ_DEFAULT as u64;
 
 /// ### THE BLOCK TRAIT
 ///
@@ -108,9 +115,16 @@ impl<T: Block> Block for &mut T {
 
 // A trait for streams which only allow reads and writes in fixed sized units called sectors.
 pub trait Sectored {
-    // Returns the size of the sector for this stream.
+    /// Returns the size of the sector for this stream.
     fn sector_sz(&self) -> usize;
 
+    /// Returns the sector size as a `u64`.
+    fn sector_sz64(&self) -> u64 {
+        // This is guaranteed not to truncate thanks to the `const_assert!` above
+        // `SECTOR_SZ_DEFAULT`.
+        self.sector_sz() as u64
+    }
+
     /// Returns `Err(Error::IncorrectSize)` if the given size is not equal to the sector size.
     fn assert_sector_sz(&self, actual: usize) -> Result<()> {
         let expected = self.sector_sz();
@@ -133,8 +147,16 @@ pub trait Sectored {
 
     /// Returns the offset (in bytes) from the beginning of this stream that the given 0-based
     /// sector index corresponds to.
-    fn offset_at(&self, index: usize) -> usize {
-        index * self.sector_sz()
+    fn offset_at(&self, index: u64) -> u64 {
+        index * self.sector_sz64()
+    }
+}
+
+impl Sectored for ::std::fs::File {
+    fn sector_sz(&self) -> usize {
+        self.metadata()
+            .map(|e| e.blksize().try_into().bterr().unwrap())
+            .unwrap_or(SECTOR_SZ_DEFAULT)
     }
 }
 
@@ -235,17 +257,6 @@ trait ReadExt: Read {
 
 impl<T: Read> ReadExt for T {}
 
-trait HashExt: Hashable {
-    /// Returns the hash produced by [DefaultHasher].
-    fn default_hash(&self) -> u64 {
-        let mut hasher = DefaultHasher::new();
-        self.hash(&mut hasher);
-        hasher.finish()
-    }
-}
-
-impl<T: Hashable> HashExt for T {}
-
 /// A unique identifier for a block.
 #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Hash)]
 pub struct BlockId {
@@ -287,7 +298,7 @@ pub struct BlockMetaSecrets {
     /// Number of hard links to the file.
     nlink: u32,
     /// The sector size used by the block.
-    sect_sz: u32,
+    sect_sz: u64,
     /// User controlled metadata.
     tags: BTreeMap<String, Vec<u8>>,
 }
@@ -304,13 +315,13 @@ impl BlockMetaSecrets {
             ctime: Epoch::default(),
             size: 0,
             nlink: 0,
-            sect_sz: SECTOR_SZ_DEFAULT.try_into().unwrap(),
+            sect_sz: SECTOR_U64_DEFAULT,
             tags: BTreeMap::new(),
         }
     }
 
-    pub fn attr(&self) -> Attr {
-        Attr {
+    pub fn attr(&self) -> Result<Attr> {
+        Ok(Attr {
             ino: self.block_id.inode,
             size: self.size,
             atime: self.atime.value(),
@@ -324,23 +335,25 @@ impl BlockMetaSecrets {
             uid: self.uid,
             gid: self.gid,
             rdev: 0,
-            blksize: self.sect_sz,
+            blksize: self
+                .sect_sz
+                .try_into()
+                .map_err(|_| bterr!("BlockMetaSecrets::sect_sz could not be converted to a u32"))?,
             blocks: self.sectors(),
             flags: 0,
-        }
+        })
     }
 
-    pub fn stat(&self) -> stat64 {
-        self.attr().into()
+    pub fn stat(&self) -> Result<stat64> {
+        self.attr().map(|e| e.into())
     }
 
     /// Returns the number of sectors occupied by the block's data.
     pub fn sectors(&self) -> u64 {
-        let sect_sz = self.sect_sz as u64;
-        if self.size % sect_sz == 0 {
-            self.size / sect_sz
+        if self.size % self.sect_sz == 0 {
+            self.size / self.sect_sz
         } else {
-            self.size / sect_sz + 1
+            self.size / self.sect_sz + 1
         }
     }
 
@@ -348,7 +361,7 @@ impl BlockMetaSecrets {
         &self.block_id
     }
 
-    pub fn sector_sz(&self) -> u32 {
+    pub fn sector_sz(&self) -> u64 {
         self.sect_sz
     }
 }
@@ -365,8 +378,10 @@ impl AsRef<BlockId> for BlockMetaSecrets {
     }
 }
 
-impl From<&BlockMetaSecrets> for Attr {
-    fn from(value: &BlockMetaSecrets) -> Self {
+impl TryFrom<&BlockMetaSecrets> for Attr {
+    type Error = crate::Error;
+
+    fn try_from(value: &BlockMetaSecrets) -> Result<Self> {
         value.attr()
     }
 }
@@ -543,7 +558,7 @@ struct BlockStream<T, C> {
     sect_sz: usize,
 }
 
-impl<T: Read + Seek, C: Creds> BlockStream<T, C> {
+impl<T: Read + Seek + Sectored, C: Creds> BlockStream<T, C> {
     fn new(inner: T, creds: C, block_path: BlockPath) -> Result<BlockStream<T, C>> {
         let (trailered, meta) = Trailered::<_, BlockMeta>::new(inner)?;
         let meta = match meta {
@@ -572,10 +587,14 @@ impl<T: Read + Seek, C: Creds> BlockStream<T, C> {
                         .ok_or(BlockError::MissingWritecap)?
                         .to_owned(),
                 );
+                meta.body.access_secrets(|secrets| {
+                    secrets.sect_sz = trailered.sector_sz64();
+                    Ok(())
+                })?;
                 meta
             }
         };
-        let sect_sz = meta.body.secrets()?.sector_sz().try_into().box_err()?;
+        let sect_sz = meta.body.secrets()?.sector_sz().try_into().bterr()?;
         Ok(BlockStream {
             trailered,
             meta_body_buf: Vec::new(),
@@ -718,7 +737,7 @@ impl<T, C> BlockOpenOptions<T, C> {
     }
 }
 
-impl<T: Read + Write + Seek + 'static, C: Creds + 'static> BlockOpenOptions<T, C> {
+impl<T: Read + Write + Seek + Sectored + 'static, C: Creds + 'static> BlockOpenOptions<T, C> {
     pub fn open(self) -> Result<Box<dyn Block>> {
         let block_path = self.block_path.ok_or(BlockError::NoBlockPath)?;
         let stream = BlockStream::new(self.inner, self.creds, block_path)?;
@@ -1126,7 +1145,7 @@ mod tests {
 
     #[test]
     fn brotli_compress_decompress() {
-        const SECT_SZ: usize = SECTOR_SZ_DEFAULT;
+        const SECT_SZ: usize = SECTOR_U64_DEFAULT as usize;
         const SECT_CT: usize = 16;
         let params = BrotliParams::new(SECT_SZ, 8, 20);
         let mut memory = Cursor::new([0u8; SECT_SZ * SECT_CT]);
@@ -1179,7 +1198,9 @@ mod tests {
     }
 
     type EncBlock = SectoredBuf<
-        SecretStream<SignStream<BlockStream<Cursor<Vec<u8>>, ConcreteCreds>, ConcreteCreds>>,
+        SecretStream<
+            SignStream<BlockStream<SectoredCursor<Vec<u8>>, ConcreteCreds>, ConcreteCreds>,
+        >,
     >;
 
     impl InMemTestCase {
@@ -1207,7 +1228,7 @@ mod tests {
         }
 
         fn stream(&self, vec: Vec<u8>) -> EncBlock {
-            let inner = Cursor::new(vec);
+            let inner = SectoredCursor::new(vec, SECTOR_SZ_DEFAULT).require_sect_sz(false);
             let mut stream =
                 BlockStream::new(inner, self.node_creds.clone(), self.block_path.clone()).unwrap();
             stream
@@ -1351,7 +1372,7 @@ mod tests {
     fn block_can_create_empty() {
         let case = BlockTestCase::new();
         BlockOpenOptions::new()
-            .with_inner(BtCursor::new(Vec::<u8>::new()))
+            .with_inner(SectoredCursor::new(Vec::<u8>::new(), SECTOR_SZ_DEFAULT))
             .with_creds(case.node_creds)
             .with_encrypt(true)
             .with_block_path(case.root_path)

crates/btlib/src/test_helpers.rs

@@ -218,8 +218,12 @@ pub fn make_block_with<C: CredsPub>(creds: &C) -> Box<dyn Block> {
         secrets_struct: Some(secrets),
     };
     let sig = Signature::copy_from(Sign::RSA_PSS_3072_SHA_256, &SIGNATURE);
-    let mut stream = BlockStream::new(BtCursor::new(Vec::new()), creds, path)
-        .expect("create block stream failed");
+    let mut stream = BlockStream::new(
+        SectoredCursor::new(Vec::new(), SECTOR_SZ_DEFAULT).require_sect_sz(false),
+        creds,
+        path,
+    )
+    .expect("create block stream failed");
     stream.meta.body = header;
     stream.meta.sig = sig;
     let stream = MerkleStream::new(stream).expect("create merkle stream failed");
@@ -608,6 +612,12 @@ impl<T: FromVec> Sectored for SectoredCursor<T> {
     }
 }
 
+impl<T: FromVec> Decompose<T> for SectoredCursor<T> {
+    fn into_inner(self) -> T {
+        self.cursor.into_inner()
+    }
+}
+
 impl Write for SectoredCursor<Vec<u8>> {
     fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
         if self.require_sect_sz {

crates/btlib/src/trailered.rs

@@ -9,7 +9,9 @@ mod private {
     use btserde::{read_from, write_to};
     use serde::{de::DeserializeOwned, Serialize};
 
-    use crate::{Block, BlockMeta, BoxInIoErr, Decompose, MetaAccess, Result, WriteInteg};
+    use crate::{
+        Block, BlockMeta, BoxInIoErr, Decompose, MetaAccess, Result, Sectored, WriteInteg,
+    };
 
     /// A struct which wraps a stream and which writes a trailing data structure to it when flushed.
     pub struct Trailered<T, D> {
@@ -168,6 +170,12 @@ mod private {
             self.inner.flush_meta()
         }
     }
+
+    impl<T: Sectored, D> Sectored for Trailered<T, D> {
+        fn sector_sz(&self) -> usize {
+            self.inner.sector_sz()
+        }
+    }
 }
 
 #[cfg(test)]