diff --git a/src/internal.rs b/src/internal.rs index b8fe283c7..343d81f37 100644 --- a/src/internal.rs +++ b/src/internal.rs @@ -73,9 +73,15 @@ use crate::src::refmvs::refmvs_tile; use crate::src::refmvs::Rav1dRefmvsDSPContext; use atomig::Atomic; use libc::ptrdiff_t; +use std::cell::UnsafeCell; use std::ffi::c_int; use std::ffi::c_uint; use std::mem; +use std::ops::Add; +use std::ops::AddAssign; +use std::ops::Index; +use std::ops::IndexMut; +use std::ops::Sub; use std::ptr; use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicI32; @@ -304,7 +310,7 @@ unsafe impl Send for Rav1dContext {} // TODO(SJC): Remove when Rav1dContext is thread-safe unsafe impl Sync for Rav1dContext {} -#[derive(Clone)] +#[derive(Clone, Default)] #[repr(C)] pub struct Rav1dTask { // frame thread id @@ -319,7 +325,7 @@ pub struct Rav1dTask { pub deblock_progress: c_int, pub deps_skip: c_int, // only used in task queue - pub next: *mut Rav1dTask, + pub next: Option, } #[repr(C)] @@ -453,17 +459,128 @@ pub struct Rav1dFrameContext_lf { pub restore_planes: c_int, // enum LrRestorePlanes } +#[derive(Default)] #[repr(C)] pub struct Rav1dFrameContext_task_thread_pending_tasks { - pub head: *mut Rav1dTask, - pub tail: *mut Rav1dTask, + pub head: Option, + pub tail: Option, } -impl Default for Rav1dFrameContext_task_thread_pending_tasks { - fn default() -> Self { - Self { - head: ptr::null_mut(), - tail: ptr::null_mut(), +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub enum Rav1dTaskIndex { + Task(usize), + TileTask(usize), + Init, +} + +impl Rav1dTaskIndex { + pub fn raw_index(self) -> Option { + match self { + Self::Task(i) => Some(i), + Self::TileTask(i) => Some(i), + Self::Init => None, + } + } +} + +impl Sub for Rav1dTaskIndex { + type Output = Self; + + fn sub(self, rhs: Self) -> Self::Output { + match (self, rhs) { + (Self::Task(x), Self::Task(y)) => Self::Task(x - y), + (Self::TileTask(x), Self::TileTask(y)) => Self::TileTask(x - y), + _ => panic!("Cannot subtract {rhs:?} from {self:?}"), + } + } +} + +impl PartialOrd for Rav1dTaskIndex { + fn partial_cmp(&self, other: &Self) -> Option { + match (self, other) { + (Self::Task(x), Self::Task(y)) => x.partial_cmp(y), + (Self::TileTask(x), Self::TileTask(y)) => x.partial_cmp(y), + _ => None, + } + } +} + +impl Add for Rav1dTaskIndex { + type Output = Self; + + fn add(self, rhs: usize) -> Self::Output { + match self { + Self::Task(i) => Self::Task(i + rhs), + Self::TileTask(i) => Self::TileTask(i + rhs), + Self::Init => panic!("Cannot index from the init task"), + } + } +} + +impl AddAssign for Rav1dTaskIndex { + fn add_assign(&mut self, rhs: usize) { + match self { + Self::Task(i) => *i += rhs, + Self::TileTask(i) => *i += rhs, + Self::Init => panic!("Cannot index from the init task"), + } + } +} + +#[derive(Default)] +pub struct Rav1dTasks { + tasks: Vec, + tile_tasks_vec: Vec, + init_task: Rav1dTask, + pub tile_tasks: [Option; 2], + pub head: Option, + pub tail: Option, + // Points to the task directly before the cur pointer in the queue. + // This cur pointer is theoretical here, we actually keep track of the + // "prev_t" variable. This is needed to not loose the tasks in + // [head;cur-1] when picking one for execution. + pub cur_prev: Option, +} + +impl Rav1dTasks { + pub fn grow_tasks(&mut self, new_len: usize) { + if new_len > self.tasks.len() { + self.tasks.clear(); + self.tasks.resize_with(new_len, Default::default); + } + } + + pub fn grow_tile_tasks(&mut self, new_len: usize) { + if new_len > self.tile_tasks_vec.len() { + self.tile_tasks_vec.clear(); + self.tile_tasks_vec.resize_with(new_len, Default::default); + self.tile_tasks[0] = Some(Rav1dTaskIndex::TileTask(0)); + } + } + + pub fn init_task(&mut self) -> &mut Rav1dTask { + &mut self.init_task + } +} + +impl Index for Rav1dTasks { + type Output = Rav1dTask; + + fn index(&self, index: Rav1dTaskIndex) -> &Self::Output { + match index { + Rav1dTaskIndex::Task(index) => &self.tasks[index], + Rav1dTaskIndex::TileTask(index) => &self.tile_tasks_vec[index], + Rav1dTaskIndex::Init => &self.init_task, + } + } +} + +impl IndexMut for Rav1dTasks { + fn index_mut(&mut self, index: Rav1dTaskIndex) -> &mut Self::Output { + match index { + Rav1dTaskIndex::Task(index) => &mut self.tasks[index], + Rav1dTaskIndex::TileTask(index) => &mut self.tile_tasks_vec[index], + Rav1dTaskIndex::Init => &mut self.init_task, } } } @@ -473,29 +590,24 @@ pub(crate) struct Rav1dFrameContext_task_thread { pub lock: Mutex<()>, pub cond: Condvar, pub ttd: Arc, - pub tasks: *mut Rav1dTask, - pub tile_tasks: [*mut Rav1dTask; 2], - pub init_task: Rav1dTask, - pub num_tasks: c_int, - pub num_tile_tasks: c_int, + pub tasks: UnsafeCell, pub init_done: AtomicI32, pub done: [AtomicI32; 2], pub retval: Rav1dResult, pub update_set: bool, // whether we need to update CDF reference pub error: AtomicI32, pub task_counter: AtomicI32, - pub task_head: *mut Rav1dTask, - pub task_tail: *mut Rav1dTask, - // Points to the task directly before the cur pointer in the queue. - // This cur pointer is theoretical here, we actually keep track of the - // "prev_t" variable. This is needed to not loose the tasks in - // [head;cur-1] when picking one for execution. - pub task_cur_prev: *mut Rav1dTask, // async task insertion pub pending_tasks_merge: AtomicI32, pub pending_tasks: Mutex, } +impl Rav1dFrameContext_task_thread { + pub unsafe fn tasks(&self) -> *mut Rav1dTasks { + self.tasks.get() + } +} + // threading (refer to tc[] for per-thread things) #[repr(C)] pub struct FrameTileThreadData { diff --git a/src/lib.rs b/src/lib.rs index c0b36cd05..fcdd813a9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -35,7 +35,6 @@ use crate::src::internal::Rav1dContext; use crate::src::internal::Rav1dContextTaskThread; use crate::src::internal::Rav1dContextTaskType; use crate::src::internal::Rav1dFrameData; -use crate::src::internal::Rav1dTask; use crate::src::internal::Rav1dTaskContext; use crate::src::internal::Rav1dTaskContext_task_thread; use crate::src::internal::TaskThreadData; @@ -75,6 +74,7 @@ use libc::pthread_attr_destroy; use libc::pthread_attr_init; use libc::pthread_attr_setstacksize; use libc::pthread_attr_t; +use std::cell::UnsafeCell; use std::cmp; use std::ffi::c_char; use std::ffi::c_int; @@ -327,6 +327,7 @@ pub(crate) unsafe fn rav1d_open(c_out: &mut *mut Rav1dContext, s: &Rav1dSettings let mut n: c_uint = 0 as c_int as c_uint; while n < (*c).n_fc { let f: &mut Rav1dFrameData = &mut *((*c).fc).offset(n as isize); + f.task_thread.tasks = UnsafeCell::new(Default::default()); if n_tc > 1 { f.task_thread.lock = Mutex::new(()); f.task_thread.cond = Condvar::new(); @@ -797,12 +798,10 @@ pub(crate) unsafe fn rav1d_flush(c: *mut Rav1dContext) { } let mut i_1: c_uint = 0 as c_int as c_uint; while i_1 < (*c).n_fc { - let ref mut fresh1 = (*((*c).fc).offset(i_1 as isize)).task_thread.task_head; - *fresh1 = 0 as *mut Rav1dTask; - let ref mut fresh2 = (*((*c).fc).offset(i_1 as isize)).task_thread.task_tail; - *fresh2 = 0 as *mut Rav1dTask; - let ref mut fresh3 = (*((*c).fc).offset(i_1 as isize)).task_thread.task_cur_prev; - *fresh3 = 0 as *mut Rav1dTask; + let tasks = &mut *(*((*c).fc).offset(i_1 as isize)).task_thread.tasks(); + tasks.head = None; + tasks.tail = None; + tasks.cur_prev = None; *(*((*c).fc).offset(i_1 as isize)) .task_thread .pending_tasks @@ -917,11 +916,7 @@ impl Drop for Rav1dContext { } mem::take(&mut f.frame_thread_progress.frame); // TODO: remove when context is owned mem::take(&mut f.frame_thread_progress.copy_lpf); // TODO: remove when context is owned - freep(&mut f.task_thread.tasks as *mut *mut Rav1dTask as *mut c_void); - freep( - &mut *(f.task_thread.tile_tasks).as_mut_ptr().offset(0) as *mut *mut Rav1dTask - as *mut c_void, - ); + mem::take(&mut f.task_thread.tasks); // TODO: remove when context is owned rav1d_free_aligned(f.ts as *mut c_void); rav1d_free_aligned(f.ipred_edge[0] as *mut c_void); free(f.a as *mut c_void); diff --git a/src/thread_task.rs b/src/thread_task.rs index 500776d9c..ea373e582 100644 --- a/src/thread_task.rs +++ b/src/thread_task.rs @@ -10,7 +10,6 @@ use crate::src::decode::rav1d_decode_frame_exit; use crate::src::decode::rav1d_decode_frame_init; use crate::src::decode::rav1d_decode_frame_init_cdf; use crate::src::decode::rav1d_decode_tile_sbrow; -use crate::src::error::Rav1dError::EGeneric; use crate::src::error::Rav1dError::EINVAL; use crate::src::error::Rav1dError::ENOMEM; use crate::src::error::Rav1dResult; @@ -18,9 +17,9 @@ use crate::src::fg_apply::rav1d_apply_grain_row; use crate::src::fg_apply::rav1d_prep_grain; use crate::src::internal::Rav1dContext; use crate::src::internal::Rav1dFrameData; -use crate::src::internal::Rav1dTask; use crate::src::internal::Rav1dTaskContext; use crate::src::internal::Rav1dTaskContext_task_thread; +use crate::src::internal::Rav1dTaskIndex; use crate::src::internal::Rav1dTileState; use crate::src::internal::TaskThreadData; use crate::src::internal::TaskThreadData_delayed_fg; @@ -39,17 +38,13 @@ use crate::src::internal::RAV1D_TASK_TYPE_SUPER_RESOLUTION; use crate::src::internal::RAV1D_TASK_TYPE_TILE_ENTROPY; use crate::src::internal::RAV1D_TASK_TYPE_TILE_RECONSTRUCTION; use crate::src::picture::Rav1dThreadPicture; -use libc::memset; -use libc::realloc; use std::cmp; use std::ffi::c_char; use std::ffi::c_int; use std::ffi::c_long; use std::ffi::c_uint; -use std::ffi::c_void; use std::mem; use std::process::abort; -use std::ptr; use std::sync::atomic::AtomicI32; use std::sync::atomic::AtomicU32; use std::sync::atomic::Ordering; @@ -104,7 +99,7 @@ unsafe fn reset_task_cur(c: &Rav1dContext, ttd: &TaskThreadData, mut frame_idx: reset_frame_idx = u32::MAX; } if ttd.cur.load(Ordering::Relaxed) == 0 - && ((*(c.fc).offset(first as isize)).task_thread.task_cur_prev).is_null() + && ((*(*(c.fc).offset(first as isize)).task_thread.tasks()).cur_prev).is_none() { return 0 as c_int; } @@ -138,14 +133,15 @@ unsafe fn reset_task_cur(c: &Rav1dContext, ttd: &TaskThreadData, mut frame_idx: ttd.cur .store(min_frame_idx.wrapping_sub(first), Ordering::Relaxed); while ttd.cur.load(Ordering::Relaxed) < c.n_fc { - if !((*(c.fc).offset( + if (*(*(c.fc).offset( first .wrapping_add(ttd.cur.load(Ordering::Relaxed)) .wrapping_rem(c.n_fc) as isize, )) .task_thread - .task_head) - .is_null() + .tasks()) + .head + .is_some() { break; } @@ -156,10 +152,10 @@ unsafe fn reset_task_cur(c: &Rav1dContext, ttd: &TaskThreadData, mut frame_idx: } let mut i: c_uint = ttd.cur.load(Ordering::Relaxed); while i < c.n_fc { - let ref mut fresh0 = (*(c.fc).offset(first.wrapping_add(i).wrapping_rem(c.n_fc) as isize)) + (*(*(c.fc).offset(first.wrapping_add(i).wrapping_rem(c.n_fc) as isize)) .task_thread - .task_cur_prev; - *fresh0 = 0 as *mut Rav1dTask; + .tasks()) + .cur_prev = None; i = i.wrapping_add(1); } return 1 as c_int; @@ -191,30 +187,29 @@ unsafe fn reset_task_cur_async(ttd: &TaskThreadData, mut frame_idx: c_uint, n_fr unsafe fn insert_tasks_between( c: &Rav1dContext, - f: &mut Rav1dFrameData, - first: *mut Rav1dTask, - last: *mut Rav1dTask, - a: *mut Rav1dTask, - b: *mut Rav1dTask, + f: &Rav1dFrameData, + first: Rav1dTaskIndex, + last: Rav1dTaskIndex, + a: Option, + b: Option, cond_signal: c_int, ) { let ttd: &TaskThreadData = &*f.task_thread.ttd; if c.flush.load(Ordering::SeqCst) != 0 { return; } - if !(a.is_null() || (*a).next == b) { - unreachable!(); - } - if a.is_null() { - f.task_thread.task_head = first; + let tasks = &mut *f.task_thread.tasks(); + if let Some(a) = a { + assert_eq!(tasks[a].next, b); + tasks[a].next = Some(first); } else { - (*a).next = first; + tasks.head = Some(first); } - if b.is_null() { - f.task_thread.task_tail = last; + if b.is_none() { + tasks.tail = Some(last); } - (*last).next = b; - reset_task_cur(c, ttd, (*first).frame_idx); + tasks[last].next = b; + reset_task_cur(c, ttd, tasks[first].frame_idx); if cond_signal != 0 && ttd.cond_signaled.fetch_or(1, Ordering::SeqCst) == 0 { ttd.cond.notify_one(); } @@ -222,45 +217,45 @@ unsafe fn insert_tasks_between( unsafe fn insert_tasks( c: &Rav1dContext, - f: &mut Rav1dFrameData, - first: *mut Rav1dTask, - last: *mut Rav1dTask, + f: &Rav1dFrameData, + first: Rav1dTaskIndex, + last: Rav1dTaskIndex, cond_signal: c_int, ) { - let mut t_ptr: *mut Rav1dTask; - let mut prev_t: *mut Rav1dTask = 0 as *mut Rav1dTask; + let tasks = &*f.task_thread.tasks(); + let mut prev_t = None; let mut current_block_34: u64; - t_ptr = f.task_thread.task_head; - while !t_ptr.is_null() { - if (*t_ptr).type_0 as c_uint == RAV1D_TASK_TYPE_TILE_ENTROPY as c_int as c_uint { - if (*first).type_0 as c_uint > RAV1D_TASK_TYPE_TILE_ENTROPY as c_int as c_uint { + let mut maybe_t = tasks.head; + while let Some(t) = maybe_t { + if tasks[t].type_0 as c_uint == RAV1D_TASK_TYPE_TILE_ENTROPY as c_int as c_uint { + if tasks[first].type_0 as c_uint > RAV1D_TASK_TYPE_TILE_ENTROPY as c_int as c_uint { current_block_34 = 11174649648027449784; - } else if (*first).sby > (*t_ptr).sby { + } else if tasks[first].sby > tasks[t].sby { current_block_34 = 11174649648027449784; } else { - if (*first).sby < (*t_ptr).sby { - insert_tasks_between(c, f, first, last, prev_t, t_ptr, cond_signal); + if tasks[first].sby < tasks[t].sby { + insert_tasks_between(c, f, first, last, prev_t, Some(t), cond_signal); return; } current_block_34 = 15904375183555213903; } } else { - if (*first).type_0 as c_uint == RAV1D_TASK_TYPE_TILE_ENTROPY as c_int as c_uint { - insert_tasks_between(c, f, first, last, prev_t, t_ptr, cond_signal); + if tasks[first].type_0 as c_uint == RAV1D_TASK_TYPE_TILE_ENTROPY as c_int as c_uint { + insert_tasks_between(c, f, first, last, prev_t, Some(t), cond_signal); return; } - if (*first).sby > (*t_ptr).sby { + if tasks[first].sby > tasks[t].sby { current_block_34 = 11174649648027449784; } else { - if (*first).sby < (*t_ptr).sby { - insert_tasks_between(c, f, first, last, prev_t, t_ptr, cond_signal); + if tasks[first].sby < tasks[t].sby { + insert_tasks_between(c, f, first, last, prev_t, Some(t), cond_signal); return; } - if (*first).type_0 as c_uint > (*t_ptr).type_0 as c_uint { + if tasks[first].type_0 as c_uint > tasks[t].type_0 as c_uint { current_block_34 = 11174649648027449784; } else { - if ((*first).type_0 as c_uint) < (*t_ptr).type_0 as c_uint { - insert_tasks_between(c, f, first, last, prev_t, t_ptr, cond_signal); + if (tasks[first].type_0 as c_uint) < tasks[t].type_0 as c_uint { + insert_tasks_between(c, f, first, last, prev_t, Some(t), cond_signal); return; } current_block_34 = 15904375183555213903; @@ -269,78 +264,78 @@ unsafe fn insert_tasks( } match current_block_34 { 15904375183555213903 => { - if !((*first).type_0 as c_uint + if !(tasks[first].type_0 as c_uint == RAV1D_TASK_TYPE_TILE_RECONSTRUCTION as c_int as c_uint - || (*first).type_0 as c_uint == RAV1D_TASK_TYPE_TILE_ENTROPY as c_int as c_uint) + || tasks[first].type_0 as c_uint == RAV1D_TASK_TYPE_TILE_ENTROPY as c_int as c_uint) { unreachable!(); } - if !((*first).type_0 as c_uint == (*t_ptr).type_0 as c_uint) { + if !(tasks[first].type_0 as c_uint == tasks[t].type_0 as c_uint) { unreachable!(); } - if !((*t_ptr).sby == (*first).sby) { + if !(tasks[t].sby == tasks[first].sby) { unreachable!(); } - let p = ((*first).type_0 as c_uint + let p = (tasks[first].type_0 as c_uint == RAV1D_TASK_TYPE_TILE_ENTROPY as c_int as c_uint) as c_int; - let t_tile_idx = - first.offset_from(f.task_thread.tile_tasks[p as usize]) as c_long as c_int; - let p_tile_idx = - t_ptr.offset_from(f.task_thread.tile_tasks[p as usize]) as c_long as c_int; + let t_tile_idx = first - tasks.tile_tasks[p as usize].unwrap(); + let p_tile_idx = t - tasks.tile_tasks[p as usize].unwrap(); if !(t_tile_idx != p_tile_idx) { unreachable!(); } if !(t_tile_idx > p_tile_idx) { - insert_tasks_between(c, f, first, last, prev_t, t_ptr, cond_signal); + insert_tasks_between(c, f, first, last, prev_t, Some(t), cond_signal); return; } } _ => {} } - prev_t = t_ptr; - t_ptr = (*t_ptr).next; + prev_t = Some(t); + maybe_t = tasks[t].next; } - insert_tasks_between(c, f, first, last, prev_t, 0 as *mut Rav1dTask, cond_signal); + insert_tasks_between(c, f, first, last, prev_t, None, cond_signal); } #[inline] unsafe fn insert_task( c: &Rav1dContext, - f: &mut Rav1dFrameData, - t: *mut Rav1dTask, + f: &Rav1dFrameData, + t: Rav1dTaskIndex, cond_signal: c_int, ) { insert_tasks(c, f, t, t, cond_signal); } #[inline] -unsafe fn add_pending(f: &Rav1dFrameData, t: *mut Rav1dTask) { +unsafe fn add_pending(f: &Rav1dFrameData, t: Rav1dTaskIndex) { + let tasks = &mut *f.task_thread.tasks(); let mut pending_tasks = f.task_thread.pending_tasks.lock().unwrap(); - (*t).next = 0 as *mut Rav1dTask; - if pending_tasks.head.is_null() { - pending_tasks.head = t; + tasks[t].next = None; + if pending_tasks.head.is_none() { + pending_tasks.head = Some(t); } else { - (*pending_tasks.tail).next = t; + tasks[pending_tasks.tail.unwrap()].next = Some(t); } - pending_tasks.tail = t; + pending_tasks.tail = Some(t); f.task_thread.pending_tasks_merge.store(1, Ordering::SeqCst); } #[inline] -unsafe fn merge_pending_frame(c: &Rav1dContext, f: &mut Rav1dFrameData) -> c_int { +unsafe fn merge_pending_frame(c: &Rav1dContext, f: &Rav1dFrameData) -> c_int { + let tasks = &*f.task_thread.tasks(); let merge = f.task_thread.pending_tasks_merge.load(Ordering::SeqCst); if merge != 0 { - let mut t = { + let mut next_t = { let mut pending_tasks = f.task_thread.pending_tasks.lock().unwrap(); let old_head = mem::take(&mut *pending_tasks).head; f.task_thread.pending_tasks_merge.store(0, Ordering::SeqCst); old_head }; - while !t.is_null() { - let tmp: *mut Rav1dTask = (*t).next; + while let Some(t) = next_t { + let tmp = tasks[t].next; insert_task(c, f, t, 0 as c_int); - t = tmp; + next_t = tmp; } } return merge; @@ -361,8 +356,7 @@ unsafe fn create_filter_sbrow( c: &Rav1dContext, f: &mut Rav1dFrameData, pass: c_int, - res_t: *mut *mut Rav1dTask, -) -> c_int { +) -> Rav1dResult { let frame_hdr = &***f.frame_hdr.as_ref().unwrap(); let has_deblock = (frame_hdr.loopfilter.level_y[0] != 0 || frame_hdr.loopfilter.level_y[1] != 0) as c_int; @@ -370,20 +364,11 @@ unsafe fn create_filter_sbrow( let has_cdef = seq_hdr.cdef; let has_resize = (frame_hdr.size.width[0] != frame_hdr.size.width[1]) as c_int; let has_lr = f.lf.restore_planes; - let mut tasks: *mut Rav1dTask = f.task_thread.tasks; + let tasks = &mut *f.task_thread.tasks(); let uses_2pass = (c.n_fc > 1 as c_uint) as c_int; - let num_tasks = f.sbh * (1 + uses_2pass); - if num_tasks > f.task_thread.num_tasks { - let size: usize = (::core::mem::size_of::()).wrapping_mul(num_tasks as usize); - tasks = realloc(f.task_thread.tasks as *mut c_void, size) as *mut Rav1dTask; - if tasks.is_null() { - return -(1 as c_int); - } - memset(tasks as *mut c_void, 0 as c_int, size); - f.task_thread.tasks = tasks; - f.task_thread.num_tasks = num_tasks; - } - tasks = tasks.offset((f.sbh * (pass & 1)) as isize); + let num_tasks = (f.sbh * (1 + uses_2pass)) as usize; + tasks.grow_tasks(num_tasks); + let task_idx = Rav1dTaskIndex::Task((f.sbh * (pass & 1)) as usize); if pass & 1 != 0 { f.frame_thread_progress.entropy = AtomicI32::new(0); } else { @@ -399,11 +384,11 @@ unsafe fn create_filter_sbrow( f.frame_thread_progress.deblock.store(0, Ordering::SeqCst); } f.frame_thread.next_tile_row[(pass & 1) as usize] = 0 as c_int; - let t: *mut Rav1dTask = &mut *tasks.offset(0) as *mut Rav1dTask; - (*t).sby = 0 as c_int; - (*t).recon_progress = 1 as c_int; - (*t).deblock_progress = 0 as c_int; - (*t).type_0 = (if pass == 1 { + let t = &mut tasks[task_idx]; + t.sby = 0 as c_int; + t.recon_progress = 1 as c_int; + t.deblock_progress = 0 as c_int; + t.type_0 = (if pass == 1 { RAV1D_TASK_TYPE_ENTROPY_PROGRESS as c_int } else if has_deblock != 0 { RAV1D_TASK_TYPE_DEBLOCK_COLS as c_int @@ -414,9 +399,8 @@ unsafe fn create_filter_sbrow( } else { RAV1D_TASK_TYPE_RECONSTRUCTION_PROGRESS as c_int }) as TaskType; - (*t).frame_idx = (f as *mut Rav1dFrameData).offset_from(c.fc) as c_long as c_int as c_uint; - *res_t = t; - return 0 as c_int; + t.frame_idx = (f as *mut Rav1dFrameData).offset_from(c.fc) as c_long as c_int as c_uint; + Ok(task_idx) } pub(crate) unsafe fn rav1d_task_create_tile_sbrow( @@ -425,70 +409,60 @@ pub(crate) unsafe fn rav1d_task_create_tile_sbrow( pass: c_int, _cond_signal: c_int, ) -> Rav1dResult { - let mut tasks: *mut Rav1dTask = f.task_thread.tile_tasks[0]; + let tasks = &mut *f.task_thread.tasks(); let uses_2pass = (c.n_fc > 1 as c_uint) as c_int; let frame_hdr = &***f.frame_hdr.as_ref().unwrap(); let num_tasks = frame_hdr.tiling.cols * frame_hdr.tiling.rows; if pass < 2 { let alloc_num_tasks = num_tasks * (1 + uses_2pass); - if alloc_num_tasks > f.task_thread.num_tile_tasks { - let size: usize = - (::core::mem::size_of::()).wrapping_mul(alloc_num_tasks as usize); - tasks = realloc(f.task_thread.tile_tasks[0] as *mut c_void, size) as *mut Rav1dTask; - if tasks.is_null() { - return Err(EGeneric); - } - memset(tasks as *mut c_void, 0 as c_int, size); - f.task_thread.tile_tasks[0] = tasks; - f.task_thread.num_tile_tasks = alloc_num_tasks; - } - f.task_thread.tile_tasks[1] = tasks.offset(num_tasks as isize); - } - tasks = tasks.offset((num_tasks * (pass & 1)) as isize); - let mut pf_t: *mut Rav1dTask = 0 as *mut Rav1dTask; - if create_filter_sbrow(c, f, pass, &mut pf_t) != 0 { - return Err(EGeneric); + tasks.grow_tile_tasks(alloc_num_tasks as usize); + tasks.tile_tasks[1] = Some(Rav1dTaskIndex::TileTask(num_tasks as usize)); } - let mut prev_t: *mut Rav1dTask = 0 as *mut Rav1dTask; + let tile_tasks = tasks.tile_tasks[0].map(|t| t + (num_tasks * (pass & 1)) as usize); + let mut pf_t = Some(create_filter_sbrow(c, f, pass)?); + let mut prev_t = None; let mut tile_idx = 0; while tile_idx < num_tasks { let ts: *mut Rav1dTileState = &mut *(f.ts).offset(tile_idx as isize) as *mut Rav1dTileState; - let t: *mut Rav1dTask = &mut *tasks.offset(tile_idx as isize) as *mut Rav1dTask; - (*t).sby = (*ts).tiling.row_start >> f.sb_shift; - if !pf_t.is_null() && (*t).sby != 0 { - (*prev_t).next = pf_t; + let t_idx = tile_tasks.unwrap() + (tile_idx as usize); + let t = &mut tasks[t_idx]; + t.sby = (*ts).tiling.row_start >> f.sb_shift; + if pf_t.is_some() && t.sby != 0 { + tasks[prev_t.unwrap()].next = pf_t; prev_t = pf_t; - pf_t = 0 as *mut Rav1dTask; + pf_t = None; } - (*t).recon_progress = 0 as c_int; - (*t).deblock_progress = 0 as c_int; - (*t).deps_skip = 0 as c_int; - (*t).type_0 = (if pass != 1 as c_int { + // re-borrow to avoid conflict with tasks[prev_t] above + let t = &mut tasks[t_idx]; + t.recon_progress = 0 as c_int; + t.deblock_progress = 0 as c_int; + t.deps_skip = 0 as c_int; + t.type_0 = (if pass != 1 as c_int { RAV1D_TASK_TYPE_TILE_RECONSTRUCTION as c_int } else { RAV1D_TASK_TYPE_TILE_ENTROPY as c_int }) as TaskType; - (*t).frame_idx = (f as *mut Rav1dFrameData).offset_from(c.fc) as c_long as c_int as c_uint; - if !prev_t.is_null() { - (*prev_t).next = t; + t.frame_idx = (f as *mut Rav1dFrameData).offset_from(c.fc) as c_long as c_int as c_uint; + if let Some(prev_t) = prev_t { + tasks[prev_t].next = Some(t_idx); } - prev_t = t; + prev_t = Some(t_idx); tile_idx += 1; } - if !pf_t.is_null() { - (*prev_t).next = pf_t; + if pf_t.is_some() { + tasks[prev_t.unwrap()].next = pf_t; prev_t = pf_t; } - (*prev_t).next = 0 as *mut Rav1dTask; + tasks[prev_t.unwrap()].next = None; f.task_thread.done[(pass & 1) as usize].store(0, Ordering::SeqCst); let mut pending_tasks = f.task_thread.pending_tasks.lock().unwrap(); - if !(pending_tasks.head.is_null() || pass == 2) { + if !(pending_tasks.head.is_none() || pass == 2) { unreachable!(); } - if pending_tasks.head.is_null() { - pending_tasks.head = tasks.offset(0); + if pending_tasks.head.is_none() { + pending_tasks.head = tile_tasks; } else { - (*pending_tasks.tail).next = tasks.offset(0); + tasks[pending_tasks.tail.unwrap()].next = tile_tasks; } pending_tasks.tail = prev_t; f.task_thread.pending_tasks_merge.store(1, Ordering::SeqCst); @@ -498,14 +472,15 @@ pub(crate) unsafe fn rav1d_task_create_tile_sbrow( pub(crate) unsafe fn rav1d_task_frame_init(c: &Rav1dContext, f: &mut Rav1dFrameData) { f.task_thread.init_done.store(0, Ordering::SeqCst); - let t: *mut Rav1dTask = &mut f.task_thread.init_task; - (*t).type_0 = RAV1D_TASK_TYPE_INIT; + let tasks = f.task_thread.tasks(); + let t = (*tasks).init_task(); + t.type_0 = RAV1D_TASK_TYPE_INIT; // TODO(SJC): add a frame context index so we don't have to rely on linear offsets - (*t).frame_idx = (f as *mut Rav1dFrameData).offset_from(c.fc) as c_long as c_int as c_uint; - (*t).sby = 0 as c_int; - (*t).deblock_progress = 0 as c_int; - (*t).recon_progress = (*t).deblock_progress; - insert_task(c, f, t, 1 as c_int); + t.frame_idx = (f as *mut Rav1dFrameData).offset_from(c.fc) as c_long as c_int as c_uint; + t.sby = 0 as c_int; + t.deblock_progress = 0 as c_int; + t.recon_progress = t.deblock_progress; + insert_task(c, f, Rav1dTaskIndex::Init, 1 as c_int); } pub(crate) unsafe fn rav1d_task_delayed_fg( @@ -529,19 +504,19 @@ pub(crate) unsafe fn rav1d_task_delayed_fg( unsafe fn ensure_progress<'l, 'ttd: 'l>( ttd: &'ttd TaskThreadData, f: &Rav1dFrameData, - t: *mut Rav1dTask, + t_idx: Rav1dTaskIndex, type_0: TaskType, state: &AtomicI32, - target: *mut c_int, task_thread_lock: &'l mut Option>, ) -> c_int { let p1 = state.load(Ordering::SeqCst); - if p1 < (*t).sby { - (*t).type_0 = type_0; - (*t).deblock_progress = 0 as c_int; - (*t).recon_progress = (*t).deblock_progress; - *target = (*t).sby; - add_pending(f, t); + let tasks = &mut *f.task_thread.tasks(); + let t = &mut tasks[t_idx]; + if p1 < t.sby { + t.type_0 = type_0; + t.recon_progress = 0 as c_int; + t.deblock_progress = t.sby; + add_pending(f, t_idx); *task_thread_lock = Some(ttd.delayed_fg.lock().unwrap()); return 1 as c_int; } @@ -549,12 +524,14 @@ unsafe fn ensure_progress<'l, 'ttd: 'l>( } #[inline] -unsafe fn check_tile(t: *mut Rav1dTask, f: &mut Rav1dFrameData, frame_mt: c_int) -> c_int { - let tp = ((*t).type_0 as c_uint == RAV1D_TASK_TYPE_TILE_ENTROPY as c_int as c_uint) as c_int; - let tile_idx = t.offset_from(f.task_thread.tile_tasks[tp as usize]) as c_long as c_int; +unsafe fn check_tile(t_idx: Rav1dTaskIndex, f: &Rav1dFrameData, frame_mt: c_int) -> c_int { + let tasks = &mut *f.task_thread.tasks(); + let t = &tasks[t_idx]; + let tp = (t.type_0 as c_uint == RAV1D_TASK_TYPE_TILE_ENTROPY as c_int as c_uint) as c_int; + let tile_idx = (t_idx - tasks.tile_tasks[tp as usize].unwrap()).raw_index().expect("t_idx was not a valid tile task"); let ts: *mut Rav1dTileState = &mut *(f.ts).offset(tile_idx as isize) as *mut Rav1dTileState; let p1 = (*ts).progress[tp as usize].load(Ordering::SeqCst); - if p1 < (*t).sby { + if p1 < t.sby { return 1 as c_int; } let mut error = (p1 == TILE_ERROR) as c_int; @@ -569,7 +546,7 @@ unsafe fn check_tile(t: *mut Rav1dTask, f: &mut Rav1dFrameData, frame_mt: c_int) } let frame_hdr = &***f.frame_hdr.as_ref().unwrap(); if error == 0 && frame_mt != 0 && !frame_hdr.frame_type.is_key_or_intra() { - let p: *const Rav1dThreadPicture = &mut f.sr_cur; + let p: *const Rav1dThreadPicture = &f.sr_cur; let ss_ver = ((*p).p.p.layout as c_uint == Rav1dPixelLayout::I420 as c_int as c_uint) as c_int; let p_b: c_uint = (((*t).sby + 1) << f.sb_shift + 2) as c_uint; @@ -616,7 +593,7 @@ unsafe fn check_tile(t: *mut Rav1dTask, f: &mut Rav1dFrameData, frame_mt: c_int) _ => {} } n += 1; - (*t).deps_skip += 1; + tasks[t_idx].deps_skip += 1; } } return 0 as c_int; @@ -829,21 +806,23 @@ pub unsafe fn rav1d_worker_task(c: &Rav1dContext, task_thread: Arc 1 as c_uint { // run init tasks second 'init_tasks: for i in 0..c.n_fc { let first = ttd.first.load(Ordering::SeqCst); let f = &mut *(c.fc).offset(first.wrapping_add(i).wrapping_rem(c.n_fc) as isize); + let tasks = &*f.task_thread.tasks(); if f.task_thread.init_done.load(Ordering::SeqCst) != 0 { continue 'init_tasks; } - let Some(t) = f.task_thread.task_head.as_mut() else { + let Some(t_idx) = tasks.head else { continue 'init_tasks; }; + let t = &tasks[t_idx]; if t.type_0 as c_uint == RAV1D_TASK_TYPE_INIT as c_int as c_uint { - break 'found (f, t, None); + break 'found (f, t_idx, None); } if t.type_0 as c_uint == RAV1D_TASK_TYPE_INIT_CDF as c_int as c_uint { // XXX This can be a simple else, if adding tasks of both @@ -862,7 +841,7 @@ pub unsafe fn rav1d_worker_task(c: &Rav1dContext, task_thread: Arc 1 as c_uint) as c_int) == 0 { - break 'found (f, t, prev_t); + if check_tile(t_idx, f, (c.n_fc > 1 as c_uint) as c_int) == 0 { + break 'found (f, t_idx, prev_t); } } else if t.recon_progress != 0 { let p = (t.type_0 as c_uint @@ -935,24 +916,25 @@ pub unsafe fn rav1d_worker_task(c: &Rav1dContext, task_thread: Arc> 5) as usize] .load(Ordering::SeqCst); if p1_1 as c_uint & (1 as c_uint) << (t.sby - 1 & 31) != 0 { - break 'found (f, t, prev_t); + break 'found (f, t_idx, prev_t); } } else { if t.deblock_progress == 0 { @@ -963,14 +945,14 @@ pub unsafe fn rav1d_worker_task(c: &Rav1dContext, task_thread: Arc RAV1D_TASK_TYPE_INIT_CDF as c_int as c_uint - && (f.task_thread.task_head).is_null() + if tasks[t_idx].type_0 as c_uint > RAV1D_TASK_TYPE_INIT_CDF as c_int as c_uint + && tasks.head.is_none() { ttd.cur.fetch_add(1, Ordering::Relaxed); } - t.next = 0 as *mut Rav1dTask; + tasks[t_idx].next = None; + let tile_tasks = tasks.tile_tasks.clone(); + let t = &mut tasks[t_idx]; // we don't need to check cond_signaled here, since we found a task // after the last signal so we want to re-signal the next waiting thread // and again won't need to signal after that @@ -1037,7 +1021,7 @@ pub unsafe fn rav1d_worker_task(c: &Rav1dContext, task_thread: Arc { let p_1 = (t.type_0 as c_uint - == RAV1D_TASK_TYPE_TILE_ENTROPY as c_int as c_uint) - as c_int; - // TODO(sjc): t is a reference to an array element, we - // need to replace this pointer arithmetic with proper - // indexing - let tile_idx = (t as *const Rav1dTask) - .offset_from(f.task_thread.tile_tasks[p_1 as usize]) - as c_long as c_int; + == RAV1D_TASK_TYPE_TILE_ENTROPY as c_int as c_uint) + as c_int; + let tile_idx = (t_idx - tile_tasks[p_1 as usize].unwrap()).raw_index().unwrap(); let ts_0: *mut Rav1dTileState = &mut *(f.ts).offset(tile_idx as isize) as *mut Rav1dTileState; tc.ts = ts_0; @@ -1153,7 +1132,7 @@ pub unsafe fn rav1d_worker_task(c: &Rav1dContext, task_thread: Arc