diff --git a/src/decode.rs b/src/decode.rs
index f3c310775..fcd0d75e4 100644
--- a/src/decode.rs
+++ b/src/decode.rs
@@ -4609,7 +4609,7 @@ pub(crate) unsafe fn rav1d_decode_frame_init(
     // at, as long as the offsets are valid.
     let has_chroma = (f.cur.p.layout != Rav1dPixelLayout::I400) as usize;
     f.lf.p = array::from_fn(|i| has_chroma * i);
-    f.lf.sr_p = array::from_fn(|i| f.sr_cur.p.data.data[has_chroma * i].cast());
+    f.lf.sr_p = array::from_fn(|i| has_chroma * i);
 
     Ok(())
 }
diff --git a/src/internal.rs b/src/internal.rs
index 6881e45ca..c4a7a3020 100644
--- a/src/internal.rs
+++ b/src/internal.rs
@@ -468,8 +468,8 @@ pub struct Rav1dFrameContext_lf {
 
     // in-loop filter per-frame state keeping
     pub start_of_tile_row: Vec<u8>,
-    pub p: [usize; 3], // Offsets into `f.cur.data.data`.
-    pub sr_p: [*mut DynPixel; 3],
+    pub p: [usize; 3],         // Offsets into `f.cur.data.data`.
+    pub sr_p: [usize; 3],      // Offsets into `f.sr_cur.p.data.data`.
     pub restore_planes: c_int, // enum LrRestorePlanes
 }
 
diff --git a/src/recon.rs b/src/recon.rs
index 3c3b8e548..8c31f865d 100644
--- a/src/recon.rs
+++ b/src/recon.rs
@@ -4671,11 +4671,14 @@ pub(crate) unsafe fn rav1d_filter_sbrow_resize<BD: BitDepth>(
             .offset(y as isize * BD::pxstride(f.cur.stride[1]) >> ss_ver),
     ];
     let sr_p: [*mut BD::Pixel; 3] = [
-        (f.lf.sr_p[0] as *mut BD::Pixel)
+        f.sr_cur.p.data.data[f.lf.sr_p[0]]
+            .cast::<BD::Pixel>()
             .offset((y as isize * BD::pxstride(f.sr_cur.p.stride[0])) as isize),
-        (f.lf.sr_p[1] as *mut BD::Pixel)
+        f.sr_cur.p.data.data[f.lf.sr_p[1]]
+            .cast::<BD::Pixel>()
             .offset((y as isize * BD::pxstride(f.sr_cur.p.stride[1]) >> ss_ver) as isize),
-        (f.lf.sr_p[2] as *mut BD::Pixel)
+        f.sr_cur.p.data.data[f.lf.sr_p[2]]
+            .cast::<BD::Pixel>()
             .offset((y as isize * BD::pxstride(f.sr_cur.p.stride[1]) >> ss_ver) as isize),
     ];
     let has_chroma =
@@ -4732,15 +4735,15 @@ pub(crate) unsafe fn rav1d_filter_sbrow_lr<BD: BitDepth>(
     let h = (*f).sr_cur.p.p.h + 127 & !127;
     let mut sr_p: [&mut [BD::Pixel]; 3] = [
         slice::from_raw_parts_mut(
-            f.lf.sr_p[0] as *mut BD::Pixel,
+            f.sr_cur.p.data.data[f.lf.sr_p[0]].cast::<BD::Pixel>(),
             (h as isize * BD::pxstride(f.sr_cur.p.stride[0])) as usize,
         ),
         slice::from_raw_parts_mut(
-            f.lf.sr_p[1] as *mut BD::Pixel,
+            f.sr_cur.p.data.data[f.lf.sr_p[1]].cast::<BD::Pixel>(),
             (h as isize * BD::pxstride(f.sr_cur.p.stride[1])) as usize >> ss_ver,
         ),
         slice::from_raw_parts_mut(
-            f.lf.sr_p[2] as *mut BD::Pixel,
+            f.sr_cur.p.data.data[f.lf.sr_p[2]].cast::<BD::Pixel>(),
             (h as isize * BD::pxstride(f.sr_cur.p.stride[1])) as usize >> ss_ver,
         ),
     ];