diff --git a/src/decode.rs b/src/decode.rs
index 8e267f8bc..622c33770 100644
--- a/src/decode.rs
+++ b/src/decode.rs
@@ -126,7 +126,6 @@ use crate::src::msac::rav1d_msac_decode_uniform;
 use crate::src::msac::rav1d_msac_init;
 use crate::src::picture::rav1d_picture_alloc_copy;
 use crate::src::picture::rav1d_thread_picture_alloc;
-use crate::src::picture::rav1d_thread_picture_unref;
 use crate::src::picture::Rav1dThreadPicture;
 use crate::src::qm::dav1d_qm_tbl;
 use crate::src::recon::debug_block_info;
@@ -4571,12 +4570,12 @@ pub(crate) unsafe fn rav1d_decode_frame_exit(
     // TODO(kkysen) use array::zip when stable
     for i in 0..7 {
         if f.refp[i].p.frame_hdr.is_some() {
-            rav1d_thread_picture_unref(&mut f.refp[i]);
+            let _ = mem::take(&mut f.refp[i]);
         }
         let _ = mem::take(&mut f.ref_mvs[i]);
     }
     let _ = mem::take(&mut f.cur);
-    rav1d_thread_picture_unref(&mut f.sr_cur);
+    let _ = mem::take(&mut f.sr_cur);
     let _ = mem::take(&mut f.in_cdf);
     if let Some(frame_hdr) = &f.frame_hdr {
         if frame_hdr.refresh_context != 0 {
@@ -4690,14 +4689,14 @@ pub unsafe fn rav1d_submit_frame(c: &mut Rav1dContext) -> Rav1dResult {
             if error.is_err() {
                 c.cached_error = mem::replace(&mut error, Ok(()));
                 *c.cached_error_props.get_mut().unwrap() = out_delayed.p.m.clone();
-                rav1d_thread_picture_unref(out_delayed);
+                let _ = mem::take(out_delayed);
             } else if out_delayed.p.data.is_some() {
                 let progress = out_delayed.progress.as_ref().unwrap()[1].load(Ordering::Relaxed);
                 if (out_delayed.visible || c.output_invisible_frames) && progress != FRAME_ERROR {
                     c.out = out_delayed.clone();
                     c.event_flags |= out_delayed.flags.into();
                 }
-                rav1d_thread_picture_unref(out_delayed);
+                let _ = mem::take(out_delayed);
             }
         }
         (f, out_delayed as *mut _, Some(task_thread_lock))
@@ -4717,13 +4716,13 @@ pub unsafe fn rav1d_submit_frame(c: &mut Rav1dContext) -> Rav1dResult {
         }
         for i in 0..7 {
             if f.refp[i].p.frame_hdr.is_some() {
-                rav1d_thread_picture_unref(&mut f.refp[i]);
+                let _ = mem::take(&mut f.refp[i]);
             }
             let _ = mem::take(&mut f.ref_mvs[i]);
         }
-        rav1d_thread_picture_unref(out);
+        let _ = mem::take(&mut *out);
         let _ = mem::take(&mut f.cur);
-        rav1d_thread_picture_unref(&mut f.sr_cur);
+        let _ = mem::take(&mut f.sr_cur);
         let _ = mem::take(&mut f.mvs);
         let _ = mem::take(&mut f.seq_hdr);
         let _ = mem::take(&mut f.frame_hdr);
@@ -4768,7 +4767,7 @@ pub unsafe fn rav1d_submit_frame(c: &mut Rav1dContext) -> Rav1dResult {
                 || bpc != c.refs[refidx].p.p.p.bpc
             {
                 for j in 0..i {
-                    rav1d_thread_picture_unref(&mut f.refp[j]);
+                    let _ = mem::take(&mut f.refp[j]);
                 }
                 on_error(f, c, out);
                 return Err(EINVAL);
@@ -4963,7 +4962,7 @@ pub unsafe fn rav1d_submit_frame(c: &mut Rav1dContext) -> Rav1dResult {
     for i in 0..8 {
         if refresh_frame_flags & (1 << i) != 0 {
             if c.refs[i].p.p.frame_hdr.is_some() {
-                rav1d_thread_picture_unref(&mut c.refs[i].p);
+                let _ = mem::take(&mut c.refs[i].p);
             }
             c.refs[i].p = f.sr_cur.clone();
 
@@ -4985,11 +4984,11 @@ pub unsafe fn rav1d_submit_frame(c: &mut Rav1dContext) -> Rav1dResult {
     if c.n_fc == 1 {
         let res = rav1d_decode_frame(c, f);
         if res.is_err() {
-            rav1d_thread_picture_unref(&mut c.out);
+            let _ = mem::take(&mut c.out);
             for i in 0..8 {
                 if refresh_frame_flags & (1 << i) != 0 {
                     if c.refs[i].p.p.frame_hdr.is_some() {
-                        rav1d_thread_picture_unref(&mut c.refs[i].p);
+                        let _ = mem::take(&mut c.refs[i].p);
                     }
                     let _ = mem::take(&mut c.cdf[i]);
                     let _ = mem::take(&mut c.refs[i].segmap);
diff --git a/src/lib.rs b/src/lib.rs
index ab2d2dc76..796fcf8b1 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -44,7 +44,6 @@ use crate::src::obu::rav1d_parse_sequence_header;
 use crate::src::picture::dav1d_default_picture_alloc;
 use crate::src::picture::dav1d_default_picture_release;
 use crate::src::picture::rav1d_picture_alloc_copy;
-use crate::src::picture::rav1d_thread_picture_unref;
 use crate::src::picture::PictureFlags;
 use crate::src::picture::Rav1dThreadPicture;
 use crate::src::refmvs::rav1d_refmvs_clear;
@@ -392,7 +391,7 @@ unsafe fn output_image(c: &mut Rav1dContext, out: &mut Rav1dPicture) -> Rav1dRes
     } else {
         res = rav1d_apply_grain(c, out, &(*r#in).p);
     }
-    rav1d_thread_picture_unref(&mut *r#in);
+    let _ = mem::take(&mut *r#in);
 
     if !c.all_layers && c.max_spatial_id && c.out.p.data.is_some() {
         *r#in = mem::take(&mut c.out);
@@ -411,7 +410,7 @@ unsafe fn output_picture_ready(c: &mut Rav1dContext, drain: bool) -> bool {
             {
                 return true;
             }
-            rav1d_thread_picture_unref(&mut c.cache);
+            let _ = mem::take(&mut c.cache);
             c.cache = mem::take(&mut c.out);
             return false;
         } else {
@@ -469,7 +468,7 @@ unsafe fn drain_picture(c: &mut Rav1dContext, out: &mut Rav1dPicture) -> Rav1dRe
         let error = mem::replace(&mut *f.task_thread.retval.try_lock().unwrap(), Ok(()));
         if error.is_err() {
             *c.cached_error_props.get_mut().unwrap() = out_delayed.p.m.clone();
-            rav1d_thread_picture_unref(out_delayed);
+            let _ = mem::take(out_delayed);
             return error;
         }
         if out_delayed.p.data.is_some() {
@@ -478,7 +477,7 @@ unsafe fn drain_picture(c: &mut Rav1dContext, out: &mut Rav1dPicture) -> Rav1dRe
                 c.out = out_delayed.clone();
                 c.event_flags |= out_delayed.flags.into();
             }
-            rav1d_thread_picture_unref(out_delayed);
+            let _ = mem::take(out_delayed);
             if output_picture_ready(c, false) {
                 return output_image(c, out);
             }
@@ -661,17 +660,17 @@ pub unsafe extern "C" fn dav1d_apply_grain(
 pub(crate) unsafe fn rav1d_flush(c: *mut Rav1dContext) {
     let _ = mem::take(&mut (*c).in_0);
     if (*c).out.p.frame_hdr.is_some() {
-        rav1d_thread_picture_unref(&mut (*c).out);
+        let _ = mem::take(&mut (*c).out);
     }
     if (*c).cache.p.frame_hdr.is_some() {
-        rav1d_thread_picture_unref(&mut (*c).cache);
+        let _ = mem::take(&mut (*c).cache);
     }
     (*c).drain = 0 as c_int;
     (*c).cached_error = Ok(());
     let mut i = 0;
     while i < 8 {
         if (*c).refs[i as usize].p.p.frame_hdr.is_some() {
-            rav1d_thread_picture_unref(&mut (*((*c).refs).as_mut_ptr().offset(i as isize)).p);
+            let _ = mem::take(&mut (*((*c).refs).as_mut_ptr().offset(i as isize)).p);
         }
         let _ = mem::take(&mut (*c).refs[i as usize].segmap);
         let _ = mem::take(&mut (*c).refs[i as usize].refmvs);
@@ -730,7 +729,7 @@ pub(crate) unsafe fn rav1d_flush(c: *mut Rav1dContext) {
             *f.task_thread.retval.try_lock().unwrap() = Ok(());
             let out_delayed = &mut (*c).frame_thread.out_delayed[next as usize];
             if out_delayed.p.frame_hdr.is_some() {
-                rav1d_thread_picture_unref(out_delayed);
+                let _ = mem::take(out_delayed);
             }
             n = n.wrapping_add(1);
             next = next.wrapping_add(1);
@@ -829,9 +828,7 @@ impl Drop for Rav1dContext {
                         .frame_hdr
                         .is_some()
                     {
-                        rav1d_thread_picture_unref(
-                            &mut self.frame_thread.out_delayed[n_2 as usize],
-                        );
+                        let _ = mem::take(&mut self.frame_thread.out_delayed[n_2 as usize]);
                     }
                     n_2 = n_2.wrapping_add(1);
                 }
@@ -841,9 +838,7 @@ impl Drop for Rav1dContext {
             let mut n_4 = 0;
             while n_4 < 8 {
                 if self.refs[n_4 as usize].p.p.frame_hdr.is_some() {
-                    rav1d_thread_picture_unref(
-                        &mut (*(self.refs).as_mut_ptr().offset(n_4 as isize)).p,
-                    );
+                    let _ = mem::take(&mut (*(self.refs).as_mut_ptr().offset(n_4 as isize)).p);
                 }
                 let _ = mem::take(&mut self.refs[n_4 as usize].refmvs);
                 let _ = mem::take(&mut self.refs[n_4 as usize].segmap);
diff --git a/src/obu.rs b/src/obu.rs
index 0c7c2553e..3516ee7bc 100644
--- a/src/obu.rs
+++ b/src/obu.rs
@@ -64,7 +64,6 @@ use crate::src::internal::Rav1dTileGroupHeader;
 use crate::src::levels::ObuMetaType;
 use crate::src::log::Rav1dLog as _;
 use crate::src::picture::rav1d_picture_copy_props;
-use crate::src::picture::rav1d_thread_picture_unref;
 use crate::src::picture::PictureFlags;
 use crate::src::thread_task::FRAME_ERROR;
 use std::array;
@@ -2136,7 +2135,7 @@ unsafe fn parse_obus(
         // update refs with only the headers in case we skip the frame
         for i in 0..8 {
             if c.frame_hdr.as_ref().unwrap().refresh_frame_flags & (1 << i) != 0 {
-                rav1d_thread_picture_unref(&mut c.refs[i as usize].p);
+                let _ = mem::take(&mut c.refs[i as usize].p);
                 c.refs[i as usize].p.p.frame_hdr = c.frame_hdr.clone();
                 c.refs[i as usize].p.p.seq_hdr = c.seq_hdr.clone();
             }
@@ -2291,7 +2290,7 @@ unsafe fn parse_obus(
                     let _ = mem::take(&mut c.mastering_display);
                     for i in 0..8 {
                         if c.refs[i as usize].p.p.frame_hdr.is_some() {
-                            rav1d_thread_picture_unref(&mut c.refs[i as usize].p);
+                            let _ = mem::take(&mut c.refs[i as usize].p);
                         }
                         let _ = mem::take(&mut c.refs[i as usize].segmap);
                         let _ = mem::take(&mut c.refs[i as usize].refmvs);
@@ -2571,7 +2570,7 @@ unsafe fn parse_obus(
                 if error.is_err() {
                     c.cached_error = mem::replace(&mut *error, Ok(()));
                     *c.cached_error_props.get_mut().unwrap() = out_delayed.p.m.clone();
-                    rav1d_thread_picture_unref(out_delayed);
+                    let _ = mem::take(out_delayed);
                 } else if out_delayed.p.data.is_some() {
                     let progress =
                         out_delayed.progress.as_ref().unwrap()[1].load(Ordering::Relaxed);
@@ -2580,7 +2579,7 @@ unsafe fn parse_obus(
                         c.out = out_delayed.clone();
                         c.event_flags |= out_delayed.flags.into();
                     }
-                    rav1d_thread_picture_unref(out_delayed);
+                    let _ = mem::take(out_delayed);
                 }
                 *out_delayed = c.refs[frame_hdr.existing_frame_idx as usize].p.clone();
                 out_delayed.visible = true;
@@ -2610,7 +2609,7 @@ unsafe fn parse_obus(
                     }
 
                     if c.refs[i as usize].p.p.frame_hdr.is_some() {
-                        rav1d_thread_picture_unref(&mut c.refs[i as usize].p);
+                        let _ = mem::take(&mut c.refs[i as usize].p);
                     }
                     c.refs[i as usize].p = c.refs[r as usize].p.clone();
 
diff --git a/src/picture.rs b/src/picture.rs
index dc6f30b68..172121239 100644
--- a/src/picture.rs
+++ b/src/picture.rs
@@ -310,8 +310,3 @@ pub(crate) unsafe fn rav1d_picture_alloc_copy(
     );
     Ok(())
 }
-
-pub(crate) unsafe fn rav1d_thread_picture_unref(p: *mut Rav1dThreadPicture) {
-    let _ = mem::take(&mut (*p).p);
-    let _ = mem::take(&mut (*p).progress);
-}