diff --git a/include/common/dump.rs b/include/common/dump.rs
index 6d4f9a487..d0a154eb0 100644
--- a/include/common/dump.rs
+++ b/include/common/dump.rs
@@ -47,7 +47,7 @@ where
     println!("{}", what);
     for y in 0..h {
         for x in 0..w {
-            print!(" {:0len$}", get(x * y), len = len);
+            print!(" {:0len$}", get(x + y * w), len = len);
         }
         println!();
     }
diff --git a/src/cdef_apply.rs b/src/cdef_apply.rs
index 15f3884a3..e8151b6ec 100644
--- a/src/cdef_apply.rs
+++ b/src/cdef_apply.rs
@@ -165,7 +165,7 @@ fn adjust_strength(strength: c_int, var: c_uint) -> c_int {
 pub(crate) unsafe fn rav1d_cdef_brow<BD: BitDepth>(
     c: &Rav1dContext,
     tc: &mut Rav1dTaskContext,
-    f: &mut Rav1dFrameData,
+    f: &Rav1dFrameData,
     p: &[*mut BD::Pixel; 3],
     lflvl_offset: i32,
     by_start: c_int,
diff --git a/src/decode.rs b/src/decode.rs
index a250ab58b..a6ce08f48 100644
--- a/src/decode.rs
+++ b/src/decode.rs
@@ -755,7 +755,7 @@ unsafe fn read_pal_indices(
 
 unsafe fn read_vartx_tree(
     t: &mut Rav1dTaskContext,
-    f: &mut Rav1dFrameData,
+    f: &Rav1dFrameData,
     b: &mut Av1Block,
     bs: BlockSize,
     bx4: c_int,
@@ -1108,40 +1108,11 @@ unsafe fn obmc_lowest_px(
 unsafe fn decode_b(
     c: &Rav1dContext,
     t: &mut Rav1dTaskContext,
-    f: &mut Rav1dFrameData,
-    bl: BlockLevel,
-    bs: BlockSize,
-    bp: BlockPartition,
-    intra_edge_flags: EdgeFlags,
-) -> Result<(), ()> {
-    // Pull out the current block from Rav1dFrameData so that we can operate on
-    // it without borrow check errors.
-    let (mut b_mem, b_idx) = if t.frame_thread.pass != 0 {
-        let b_idx = (t.b.y as isize * f.b4_stride + t.b.x as isize) as usize;
-        (mem::take(&mut f.frame_thread.b[b_idx]), Some(b_idx))
-    } else {
-        (Default::default(), None)
-    };
-    let b = &mut b_mem;
-    let res = decode_b_inner(c, t, f, bl, bs, bp, intra_edge_flags, b);
-    if let Some(i) = b_idx {
-        let _old_b = mem::replace(&mut f.frame_thread.b[i], b_mem);
-        // TODO(SJC): We should be able to compare Av1Blocks, but there are C
-        // unions in them.
-        // assert_eq!(old_b, Default::default());
-    }
-    res
-}
-
-unsafe fn decode_b_inner(
-    c: &Rav1dContext,
-    t: &mut Rav1dTaskContext,
-    f: &mut Rav1dFrameData,
+    f: &Rav1dFrameData,
     bl: BlockLevel,
     bs: BlockSize,
     bp: BlockPartition,
     intra_edge_flags: EdgeFlags,
-    b: &mut Av1Block,
 ) -> Result<(), ()> {
     use std::fmt;
 
@@ -1155,6 +1126,18 @@ unsafe fn decode_b_inner(
         }
     }
 
+    let mut b_guard;
+    let mut b_mem = Av1Block::default();
+    let b = if t.frame_thread.pass != 0 {
+        b_guard = f
+            .frame_thread
+            .b
+            .index_mut((t.b.y as isize * f.b4_stride + t.b.x as isize) as usize);
+        &mut *b_guard
+    } else {
+        &mut b_mem
+    };
+
     let ts = &mut *f.ts.offset(t.ts as isize);
     let bd_fn = f.bd_fn();
     let b_dim = &dav1d_block_dimensions[bs as usize];
@@ -1887,7 +1870,7 @@ unsafe fn decode_b_inner(
                 TileStateRef::Local => &ts.lflvlmem,
             };
             rav1d_create_lf_mask_intra(
-                &mut f.lf.mask[t.lf_mask.unwrap()],
+                &f.lf.mask[t.lf_mask.unwrap()],
                 &f.lf.level,
                 f.b4_stride,
                 &lflvl[b.seg_id as usize],
@@ -2905,7 +2888,7 @@ unsafe fn decode_b_inner(
                 TileStateRef::Local => &ts.lflvlmem,
             };
             rav1d_create_lf_mask_inter(
-                &mut f.lf.mask[t.lf_mask.unwrap()],
+                &f.lf.mask[t.lf_mask.unwrap()],
                 &f.lf.level,
                 f.b4_stride,
                 // In C, the inner dimensions (`ref`, `is_gmv`) are offset,
@@ -2997,7 +2980,7 @@ unsafe fn decode_b_inner(
     if b.skip == 0 {
         let mask = !0u32 >> 32 - bw4 << (bx4 & 15);
         let bx_idx = (bx4 & 16) >> 4;
-        for noskip_mask in &mut f.lf.mask[t.lf_mask.unwrap()].noskip_mask[by4 as usize >> 1..]
+        for noskip_mask in &f.lf.mask[t.lf_mask.unwrap()].noskip_mask[by4 as usize >> 1..]
             [..(bh4 as usize + 1) / 2]
         {
             noskip_mask[bx_idx as usize].fetch_or(mask as u16, Ordering::Relaxed);
@@ -3245,7 +3228,7 @@ unsafe fn decode_b_inner(
 unsafe fn decode_sb(
     c: &Rav1dContext,
     t: &mut Rav1dTaskContext,
-    f: &mut Rav1dFrameData,
+    f: &Rav1dFrameData,
     bl: BlockLevel,
     edge_index: EdgeIndex,
 ) -> Result<(), ()> {
@@ -3318,7 +3301,10 @@ unsafe fn decode_sb(
                 );
             }
         } else {
-            let b = &f.frame_thread.b[(t.b.y as isize * f.b4_stride + t.b.x as isize) as usize];
+            let b = f
+                .frame_thread
+                .b
+                .index((t.b.y as isize * f.b4_stride + t.b.x as isize) as usize);
             bp = if b.bl == bl {
                 b.bp
             } else {
@@ -3479,7 +3465,10 @@ unsafe fn decode_sb(
                 );
             }
         } else {
-            let b = &f.frame_thread.b[(t.b.y as isize * f.b4_stride + t.b.x as isize) as usize];
+            let b = &f
+                .frame_thread
+                .b
+                .index((t.b.y as isize * f.b4_stride + t.b.x as isize) as usize);
             is_split = b.bl != bl;
         }
 
@@ -3532,7 +3521,10 @@ unsafe fn decode_sb(
                 );
             }
         } else {
-            let b = &f.frame_thread.b[(t.b.y as isize * f.b4_stride + t.b.x as isize) as usize];
+            let b = &f
+                .frame_thread
+                .b
+                .index((t.b.y as isize * f.b4_stride + t.b.x as isize) as usize);
             is_split = b.bl != bl;
         }
 
diff --git a/src/internal.rs b/src/internal.rs
index 6d0843118..0f8a6dce2 100644
--- a/src/internal.rs
+++ b/src/internal.rs
@@ -402,7 +402,7 @@ pub(crate) struct Rav1dFrameContext_bd_fn {
     pub filter_sbrow_deblock_cols: filter_sbrow_fn,
     pub filter_sbrow_deblock_rows: filter_sbrow_fn,
     pub filter_sbrow_cdef:
-        unsafe fn(&Rav1dContext, &mut Rav1dFrameData, &mut Rav1dTaskContext, c_int) -> (),
+        unsafe fn(&Rav1dContext, &Rav1dFrameData, &mut Rav1dTaskContext, c_int) -> (),
     pub filter_sbrow_resize: filter_sbrow_fn,
     pub filter_sbrow_lr: filter_sbrow_fn,
     pub backup_ipred_edge: backup_ipred_edge_fn,
@@ -444,7 +444,7 @@ impl Rav1dFrameContext_bd_fn {
 
     pub unsafe fn recon_b_intra(
         &self,
-        f: &mut Rav1dFrameData,
+        f: &Rav1dFrameData,
         context: &mut Rav1dTaskContext,
         block_size: BlockSize,
         flags: EdgeFlags,
@@ -455,7 +455,7 @@ impl Rav1dFrameContext_bd_fn {
 
     pub unsafe fn recon_b_inter(
         &self,
-        f: &mut Rav1dFrameData,
+        f: &Rav1dFrameData,
         context: &mut Rav1dTaskContext,
         block_size: BlockSize,
         block: &Av1Block,
@@ -465,7 +465,7 @@ impl Rav1dFrameContext_bd_fn {
 
     pub unsafe fn read_coef_blocks(
         &self,
-        f: &mut Rav1dFrameData,
+        f: &Rav1dFrameData,
         context: &mut Rav1dTaskContext,
         block_size: BlockSize,
         block: &Av1Block,
@@ -548,7 +548,7 @@ pub struct Rav1dFrameContext_frame_thread {
     pub next_tile_row: [AtomicI32; 2],
 
     /// Indexed using `t.b.y * f.b4_stride + t.b.x`.
-    pub b: Vec<Av1Block>,
+    pub b: DisjointMut<Vec<Av1Block>>,
 
     pub cbi: Vec<[Atomic<CodedBlockInfo>; 3]>,
 
diff --git a/src/lf_apply.rs b/src/lf_apply.rs
index c962ddc7f..5fc94103d 100644
--- a/src/lf_apply.rs
+++ b/src/lf_apply.rs
@@ -585,14 +585,14 @@ unsafe fn filter_plane_rows_uv<BD: BitDepth>(
 }
 
 pub(crate) unsafe fn rav1d_loopfilter_sbrow_cols<BD: BitDepth>(
-    f: &mut Rav1dFrameData,
+    f: &Rav1dFrameData,
     p: &mut [&mut [BD::Pixel]; 3],
     p_offset: &[usize; 2],
     lflvl_offset: usize,
     sby: c_int,
     start_of_tile_row: c_int,
 ) {
-    let lflvl = &mut f.lf.mask[lflvl_offset..];
+    let lflvl = &f.lf.mask[lflvl_offset..];
     let mut have_left; // Don't filter outside the frame
     let seq_hdr = &***f.seq_hdr.as_ref().unwrap();
     let is_sb64 = (seq_hdr.sb128 == 0) as c_int;
@@ -751,7 +751,7 @@ pub(crate) unsafe fn rav1d_loopfilter_sbrow_cols<BD: BitDepth>(
 }
 
 pub(crate) unsafe fn rav1d_loopfilter_sbrow_rows<BD: BitDepth>(
-    f: &mut Rav1dFrameData,
+    f: &Rav1dFrameData,
     p: &mut [&mut [BD::Pixel]; 3],
     p_offset: &[usize; 2],
     lflvl_offset: usize,
diff --git a/src/lr_apply.rs b/src/lr_apply.rs
index db29ec9ad..ebd8bdd45 100644
--- a/src/lr_apply.rs
+++ b/src/lr_apply.rs
@@ -259,7 +259,7 @@ unsafe fn lr_sbrow<BD: BitDepth>(
 
 pub(crate) unsafe fn rav1d_lr_sbrow<BD: BitDepth>(
     c: &Rav1dContext,
-    f: &mut Rav1dFrameData,
+    f: &Rav1dFrameData,
     dst: &mut [&mut [BD::Pixel]; 3],
     dst_offset: &[usize; 2],
     sby: c_int,
diff --git a/src/recon.rs b/src/recon.rs
index c76a00f09..19de52c58 100644
--- a/src/recon.rs
+++ b/src/recon.rs
@@ -133,22 +133,22 @@ macro_rules! debug_block_info {
 pub(crate) use debug_block_info;
 
 pub(crate) type recon_b_intra_fn =
-    unsafe fn(&mut Rav1dFrameData, &mut Rav1dTaskContext, BlockSize, EdgeFlags, &Av1Block) -> ();
+    unsafe fn(&Rav1dFrameData, &mut Rav1dTaskContext, BlockSize, EdgeFlags, &Av1Block) -> ();
 
 pub(crate) type recon_b_inter_fn =
-    unsafe fn(&mut Rav1dFrameData, &mut Rav1dTaskContext, BlockSize, &Av1Block) -> Result<(), ()>;
+    unsafe fn(&Rav1dFrameData, &mut Rav1dTaskContext, BlockSize, &Av1Block) -> Result<(), ()>;
 
 pub(crate) type filter_sbrow_fn =
-    unsafe fn(&Rav1dContext, &mut Rav1dFrameData, &mut Rav1dTaskContext, c_int) -> ();
+    unsafe fn(&Rav1dContext, &Rav1dFrameData, &mut Rav1dTaskContext, c_int) -> ();
 
 pub(crate) type backup_ipred_edge_fn = unsafe fn(&Rav1dFrameData, &mut Rav1dTaskContext) -> ();
 
 pub(crate) type read_coef_blocks_fn =
-    unsafe fn(&mut Rav1dFrameData, &mut Rav1dTaskContext, BlockSize, &Av1Block) -> ();
+    unsafe fn(&Rav1dFrameData, &mut Rav1dTaskContext, BlockSize, &Av1Block) -> ();
 
 pub(crate) type copy_pal_block_fn = unsafe fn(
     t: &mut Rav1dTaskContext,
-    f: &mut Rav1dFrameData,
+    f: &Rav1dFrameData,
     bx4: usize,
     by4: usize,
     bw4: usize,
@@ -157,7 +157,7 @@ pub(crate) type copy_pal_block_fn = unsafe fn(
 
 pub(crate) type read_pal_plane_fn = unsafe fn(
     t: &mut Rav1dTaskContext,
-    f: &mut Rav1dFrameData,
+    f: &Rav1dFrameData,
     b: &mut Av1Block,
     pl: bool,
     sz_ctx: u8,
@@ -167,7 +167,7 @@ pub(crate) type read_pal_plane_fn = unsafe fn(
 
 pub(crate) type read_pal_uv_fn = unsafe fn(
     t: &mut Rav1dTaskContext,
-    f: &mut Rav1dFrameData,
+    f: &Rav1dFrameData,
     b: &mut Av1Block,
     sz_ctx: u8,
     bx4: usize,
@@ -477,7 +477,7 @@ fn get_lo_ctx(
 }
 
 unsafe fn decode_coefs<BD: BitDepth>(
-    f: &mut Rav1dFrameData,
+    f: &Rav1dFrameData,
     t: *mut Rav1dTaskContext,
     a: &mut [u8],
     l: &mut [u8],
@@ -1624,7 +1624,7 @@ enum CfSelect {
 }
 
 impl CfSelect {
-    fn set<BD: BitDepth>(
+    unsafe fn set<BD: BitDepth>(
         self,
         f: &Rav1dFrameData,
         t: &mut Rav1dTaskContext,
@@ -1655,7 +1655,7 @@ impl CfSelect {
 }
 
 unsafe fn read_coef_tree<BD: BitDepth>(
-    f: &mut Rav1dFrameData,
+    f: &Rav1dFrameData,
     t: *mut Rav1dTaskContext,
     bs: BlockSize,
     b: &Av1Block,
@@ -1865,7 +1865,7 @@ unsafe fn read_coef_tree<BD: BitDepth>(
 }
 
 pub(crate) unsafe fn rav1d_read_coef_blocks<BD: BitDepth>(
-    f: &mut Rav1dFrameData,
+    f: &Rav1dFrameData,
     t: &mut Rav1dTaskContext,
     bs: BlockSize,
     b: &Av1Block,
@@ -2463,7 +2463,7 @@ unsafe fn warp_affine<BD: BitDepth>(
 }
 
 pub(crate) unsafe fn rav1d_recon_b_intra<BD: BitDepth>(
-    f: &mut Rav1dFrameData,
+    f: &Rav1dFrameData,
     t: &mut Rav1dTaskContext,
     bs: BlockSize,
     intra_edge_flags: EdgeFlags,
@@ -3246,7 +3246,7 @@ pub(crate) unsafe fn rav1d_recon_b_intra<BD: BitDepth>(
 }
 
 pub(crate) unsafe fn rav1d_recon_b_inter<BD: BitDepth>(
-    f: &mut Rav1dFrameData,
+    f: &Rav1dFrameData,
     t: &mut Rav1dTaskContext,
     bs: BlockSize,
     b: &Av1Block,
@@ -3684,11 +3684,14 @@ pub(crate) unsafe fn rav1d_recon_b_inter<BD: BitDepth>(
                             if t.frame_thread.pass != 2 {
                                 t.tl_4x4_filter
                             } else {
-                                f.frame_thread.b[(t.b.y as usize - 1) as usize
-                                    * f.b4_stride as usize
-                                    + t.b.x as usize
-                                    - 1]
-                                .filter2d()
+                                f.frame_thread
+                                    .b
+                                    .index(
+                                        (t.b.y as usize - 1) as usize * f.b4_stride as usize
+                                            + t.b.x as usize
+                                            - 1,
+                                    )
+                                    .filter2d()
                             },
                         )?;
                     }
@@ -3720,8 +3723,12 @@ pub(crate) unsafe fn rav1d_recon_b_inter<BD: BitDepth>(
                             if t.frame_thread.pass != 2 {
                                 left_filter_2d
                             } else {
-                                f.frame_thread.b
-                                    [t.b.y as usize * f.b4_stride as usize + t.b.x as usize - 1]
+                                f.frame_thread
+                                    .b
+                                    .index(
+                                        (t.b.y as isize * f.b4_stride + t.b.x as isize - 1)
+                                            as usize,
+                                    )
                                     .filter2d()
                             },
                         )?;
@@ -3753,8 +3760,12 @@ pub(crate) unsafe fn rav1d_recon_b_inter<BD: BitDepth>(
                             if t.frame_thread.pass != 2 {
                                 top_filter_2d
                             } else {
-                                f.frame_thread.b
-                                    [(t.b.y as usize - 1) * f.b4_stride as usize + t.b.x as usize]
+                                f.frame_thread
+                                    .b
+                                    .index(
+                                        (t.b.y as usize - 1) * f.b4_stride as usize
+                                            + t.b.x as usize,
+                                    )
                                     .filter2d()
                             },
                         )?;
@@ -4142,7 +4153,7 @@ pub(crate) unsafe fn rav1d_recon_b_inter<BD: BitDepth>(
 
 pub(crate) unsafe fn rav1d_filter_sbrow_deblock_cols<BD: BitDepth>(
     c: &Rav1dContext,
-    f: &mut Rav1dFrameData,
+    f: &Rav1dFrameData,
     _t: &mut Rav1dTaskContext,
     sby: c_int,
 ) {
@@ -4206,7 +4217,7 @@ pub(crate) unsafe fn rav1d_filter_sbrow_deblock_cols<BD: BitDepth>(
 
 pub(crate) unsafe fn rav1d_filter_sbrow_deblock_rows<BD: BitDepth>(
     c: &Rav1dContext,
-    f: &mut Rav1dFrameData,
+    f: &Rav1dFrameData,
     _t: &mut Rav1dTaskContext,
     sby: c_int,
 ) {
@@ -4267,7 +4278,7 @@ pub(crate) unsafe fn rav1d_filter_sbrow_deblock_rows<BD: BitDepth>(
 
 pub(crate) unsafe fn rav1d_filter_sbrow_cdef<BD: BitDepth>(
     c: &Rav1dContext,
-    f: &mut Rav1dFrameData,
+    f: &Rav1dFrameData,
     tc: &mut Rav1dTaskContext,
     sby: c_int,
 ) {
@@ -4310,7 +4321,7 @@ pub(crate) unsafe fn rav1d_filter_sbrow_cdef<BD: BitDepth>(
 
 pub(crate) unsafe fn rav1d_filter_sbrow_resize<BD: BitDepth>(
     _c: &Rav1dContext,
-    f: &mut Rav1dFrameData,
+    f: &Rav1dFrameData,
     _t: &mut Rav1dTaskContext,
     sby: c_int,
 ) {
@@ -4378,7 +4389,7 @@ pub(crate) unsafe fn rav1d_filter_sbrow_resize<BD: BitDepth>(
 
 pub(crate) unsafe fn rav1d_filter_sbrow_lr<BD: BitDepth>(
     c: &Rav1dContext,
-    f: &mut Rav1dFrameData,
+    f: &Rav1dFrameData,
     _t: &mut Rav1dTaskContext,
     sby: c_int,
 ) {
@@ -4414,7 +4425,7 @@ pub(crate) unsafe fn rav1d_filter_sbrow_lr<BD: BitDepth>(
 
 pub(crate) unsafe fn rav1d_filter_sbrow<BD: BitDepth>(
     c: &Rav1dContext,
-    f: &mut Rav1dFrameData,
+    f: &Rav1dFrameData,
     t: &mut Rav1dTaskContext,
     sby: c_int,
 ) {
@@ -4487,7 +4498,7 @@ pub(crate) unsafe fn rav1d_backup_ipred_edge<BD: BitDepth>(
 
 pub(crate) unsafe fn rav1d_copy_pal_block_y<BD: BitDepth>(
     t: &mut Rav1dTaskContext,
-    f: &mut Rav1dFrameData,
+    f: &Rav1dFrameData,
     bx4: usize,
     by4: usize,
     bw4: usize,
@@ -4513,7 +4524,7 @@ pub(crate) unsafe fn rav1d_copy_pal_block_y<BD: BitDepth>(
 
 pub(crate) unsafe fn rav1d_copy_pal_block_uv<BD: BitDepth>(
     t: &mut Rav1dTaskContext,
-    f: &mut Rav1dFrameData,
+    f: &Rav1dFrameData,
     bx4: usize,
     by4: usize,
     bw4: usize,
@@ -4542,7 +4553,7 @@ pub(crate) unsafe fn rav1d_copy_pal_block_uv<BD: BitDepth>(
 
 pub(crate) unsafe fn rav1d_read_pal_plane<BD: BitDepth>(
     t: &mut Rav1dTaskContext,
-    f: &mut Rav1dFrameData,
+    f: &Rav1dFrameData,
     b: &mut Av1Block,
     pl: bool,
     sz_ctx: u8,
@@ -4733,7 +4744,7 @@ pub(crate) unsafe fn rav1d_read_pal_plane<BD: BitDepth>(
 
 pub(crate) unsafe fn rav1d_read_pal_uv<BD: BitDepth>(
     t: &mut Rav1dTaskContext,
-    f: &mut Rav1dFrameData,
+    f: &Rav1dFrameData,
     b: &mut Av1Block,
     sz_ctx: u8,
     bx4: usize,