From c5cab395356ebff0045340b94b80f245ca6b435f Mon Sep 17 00:00:00 2001 From: Stephen Crane Date: Tue, 12 Mar 2024 18:50:30 -0700 Subject: [PATCH] `struct Rav1dFrameContext_task_thread`: Make Rav1dTask lists indexed Rav1dTasks are stored in two arrays and linked into a linked list by pointers. This change refactors the lists into a `Rav1dTasks` structure that links tasks by index instead of pointer. We still access the Rav1dTasks structure with unsafe interior mutability, because we would have borrowing errors between the Rav1dFrameData reference lifetime and the lifetime of borrows out of the Rav1dTasks structure. This will be fixed soon when we split `Rav1dFrameData` into an immutable, interiorly mutable portion and a mutex protected mutable portion. --- src/internal.rs | 153 +++++++++++++++--- src/lib.rs | 19 +-- src/thread_task.rs | 394 +++++++++++++++++++++------------------------ 3 files changed, 323 insertions(+), 243 deletions(-) diff --git a/src/internal.rs b/src/internal.rs index fd6a3c3fd..87212cb50 100644 --- a/src/internal.rs +++ b/src/internal.rs @@ -73,9 +73,16 @@ use crate::src::refmvs::refmvs_tile; use crate::src::refmvs::Rav1dRefmvsDSPContext; use atomig::Atomic; use libc::ptrdiff_t; +use std::cell::UnsafeCell; +use std::cmp; use std::ffi::c_int; use std::ffi::c_uint; use std::mem; +use std::ops::Add; +use std::ops::AddAssign; +use std::ops::Index; +use std::ops::IndexMut; +use std::ops::Sub; use std::ptr; use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicI32; @@ -127,6 +134,12 @@ pub enum TaskType { FgApply = 12, } +impl Default for TaskType { + fn default() -> Self { + Self::Init + } +} + #[repr(C)] pub(crate) struct Rav1dContext_frame_thread { pub out_delayed: Box<[Rav1dThreadPicture]>, @@ -294,7 +307,7 @@ unsafe impl Send for Rav1dContext {} // TODO(SJC): Remove when Rav1dContext is thread-safe unsafe impl Sync for Rav1dContext {} -#[derive(Clone)] +#[derive(Clone, Default)] #[repr(C)] pub struct Rav1dTask { // frame thread id @@ -309,7 +322,7 @@ pub struct Rav1dTask { pub deblock_progress: c_int, pub deps_skip: c_int, // only used in task queue - pub next: *mut Rav1dTask, + pub next: Option, } #[repr(C)] @@ -474,17 +487,120 @@ pub struct Rav1dFrameContext_lf { pub restore_planes: c_int, // enum LrRestorePlanes } +#[derive(Default)] #[repr(C)] pub struct Rav1dFrameContext_task_thread_pending_tasks { - pub head: *mut Rav1dTask, - pub tail: *mut Rav1dTask, + pub head: Option, + pub tail: Option, } -impl Default for Rav1dFrameContext_task_thread_pending_tasks { - fn default() -> Self { - Self { - head: ptr::null_mut(), - tail: ptr::null_mut(), +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub enum Rav1dTaskIndex { + Task(usize), + TileTask(usize), + Init, +} + +impl Rav1dTaskIndex { + pub fn raw_index(self) -> Option { + match self { + Self::Task(i) => Some(i), + Self::TileTask(i) => Some(i), + Self::Init => None, + } + } +} + +impl Sub for Rav1dTaskIndex { + type Output = Self; + + fn sub(self, rhs: Self) -> Self::Output { + match (self, rhs) { + (Self::Task(x), Self::Task(y)) => Self::Task(x - y), + (Self::TileTask(x), Self::TileTask(y)) => Self::TileTask(x - y), + _ => panic!("Cannot subtract {rhs:?} from {self:?}"), + } + } +} + +impl PartialOrd for Rav1dTaskIndex { + fn partial_cmp(&self, other: &Self) -> Option { + match (self, other) { + (Self::Task(x), Self::Task(y)) => x.partial_cmp(y), + (Self::TileTask(x), Self::TileTask(y)) => x.partial_cmp(y), + _ => None, + } + } +} + +impl Add for Rav1dTaskIndex { + type Output = Self; + + fn add(self, rhs: usize) -> Self::Output { + match self { + Self::Task(i) => Self::Task(i + rhs), + Self::TileTask(i) => Self::TileTask(i + rhs), + Self::Init => panic!("Cannot add to the init task"), + } + } +} + +impl AddAssign for Rav1dTaskIndex { + fn add_assign(&mut self, rhs: usize) { + *self = *self + rhs; + } +} + +#[derive(Default)] +pub struct Rav1dTasks { + tasks: Vec, + tile_tasks_vec: Vec, + init_task: Rav1dTask, + pub tile_tasks: [Option; 2], + pub head: Option, + pub tail: Option, + // Points to the task directly before the cur pointer in the queue. + // This cur pointer is theoretical here, we actually keep track of the + // "prev_t" variable. This is needed to not loose the tasks in + // [head;cur-1] when picking one for execution. + pub cur_prev: Option, +} + +impl Rav1dTasks { + pub fn grow_tasks(&mut self, new_len: usize) { + if new_len > self.tasks.len() { + self.tasks.clear(); + self.tasks.resize_with(new_len, Default::default); + } + } + + pub fn grow_tile_tasks(&mut self, new_len: usize) { + if new_len > self.tile_tasks_vec.len() { + self.tile_tasks_vec.clear(); + self.tile_tasks_vec.resize_with(new_len, Default::default); + self.tile_tasks[0] = Some(Rav1dTaskIndex::TileTask(0)); + } + } +} + +impl Index for Rav1dTasks { + type Output = Rav1dTask; + + fn index(&self, index: Rav1dTaskIndex) -> &Self::Output { + match index { + Rav1dTaskIndex::Task(index) => &self.tasks[index], + Rav1dTaskIndex::TileTask(index) => &self.tile_tasks_vec[index], + Rav1dTaskIndex::Init => &self.init_task, + } + } +} + +impl IndexMut for Rav1dTasks { + fn index_mut(&mut self, index: Rav1dTaskIndex) -> &mut Self::Output { + match index { + Rav1dTaskIndex::Task(index) => &mut self.tasks[index], + Rav1dTaskIndex::TileTask(index) => &mut self.tile_tasks_vec[index], + Rav1dTaskIndex::Init => &mut self.init_task, } } } @@ -494,29 +610,24 @@ pub(crate) struct Rav1dFrameContext_task_thread { pub lock: Mutex<()>, pub cond: Condvar, pub ttd: Arc, - pub tasks: *mut Rav1dTask, - pub tile_tasks: [*mut Rav1dTask; 2], - pub init_task: Rav1dTask, - pub num_tasks: c_int, - pub num_tile_tasks: c_int, + pub tasks: UnsafeCell, pub init_done: AtomicI32, pub done: [AtomicI32; 2], pub retval: Rav1dResult, pub update_set: bool, // whether we need to update CDF reference pub error: AtomicI32, pub task_counter: AtomicI32, - pub task_head: *mut Rav1dTask, - pub task_tail: *mut Rav1dTask, - // Points to the task directly before the cur pointer in the queue. - // This cur pointer is theoretical here, we actually keep track of the - // "prev_t" variable. This is needed to not loose the tasks in - // [head;cur-1] when picking one for execution. - pub task_cur_prev: *mut Rav1dTask, // async task insertion pub pending_tasks_merge: AtomicI32, pub pending_tasks: Mutex, } +impl Rav1dFrameContext_task_thread { + pub unsafe fn tasks(&self) -> *mut Rav1dTasks { + self.tasks.get() + } +} + // threading (refer to tc[] for per-thread things) #[repr(C)] pub struct FrameTileThreadData { diff --git a/src/lib.rs b/src/lib.rs index 30d7408de..ab0c4f2e3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -33,7 +33,6 @@ use crate::src::internal::Rav1dContext; use crate::src::internal::Rav1dContextTaskThread; use crate::src::internal::Rav1dContextTaskType; use crate::src::internal::Rav1dFrameData; -use crate::src::internal::Rav1dTask; use crate::src::internal::Rav1dTaskContext; use crate::src::internal::Rav1dTaskContext_task_thread; use crate::src::internal::TaskThreadData; @@ -70,6 +69,7 @@ use libc::pthread_attr_destroy; use libc::pthread_attr_init; use libc::pthread_attr_setstacksize; use libc::pthread_attr_t; +use std::cell::UnsafeCell; use std::cmp; use std::ffi::c_char; use std::ffi::c_int; @@ -337,6 +337,7 @@ pub(crate) unsafe fn rav1d_open(c_out: &mut *mut Rav1dContext, s: &Rav1dSettings for n in 0..n_fc { let f: &mut Rav1dFrameData = &mut *((*c).fc).offset(n as isize); f.index = n; + addr_of_mut!(f.task_thread.tasks).write(UnsafeCell::new(Default::default())); addr_of_mut!(f.frame_thread).write(Default::default()); if n_tc > 1 { f.task_thread.lock = Mutex::new(()); @@ -800,12 +801,10 @@ pub(crate) unsafe fn rav1d_flush(c: *mut Rav1dContext) { } let mut i_1: c_uint = 0 as c_int as c_uint; while i_1 < (*c).n_fc { - let ref mut fresh1 = (*((*c).fc).offset(i_1 as isize)).task_thread.task_head; - *fresh1 = 0 as *mut Rav1dTask; - let ref mut fresh2 = (*((*c).fc).offset(i_1 as isize)).task_thread.task_tail; - *fresh2 = 0 as *mut Rav1dTask; - let ref mut fresh3 = (*((*c).fc).offset(i_1 as isize)).task_thread.task_cur_prev; - *fresh3 = 0 as *mut Rav1dTask; + let tasks = &mut *(*((*c).fc).offset(i_1 as isize)).task_thread.tasks(); + tasks.head = None; + tasks.tail = None; + tasks.cur_prev = None; *(*((*c).fc).offset(i_1 as isize)) .task_thread .pending_tasks @@ -913,11 +912,7 @@ impl Drop for Rav1dContext { let _ = mem::take(&mut f.frame_thread); // TODO: remove when context is owned mem::take(&mut f.frame_thread_progress.frame); // TODO: remove when context is owned mem::take(&mut f.frame_thread_progress.copy_lpf); // TODO: remove when context is owned - freep(&mut f.task_thread.tasks as *mut *mut Rav1dTask as *mut c_void); - freep( - &mut *(f.task_thread.tile_tasks).as_mut_ptr().offset(0) as *mut *mut Rav1dTask - as *mut c_void, - ); + mem::take(&mut f.task_thread.tasks); // TODO: remove when context is owned rav1d_free_aligned(f.ts as *mut c_void); rav1d_free_aligned(f.ipred_edge[0] as *mut c_void); free(f.a as *mut c_void); diff --git a/src/thread_task.rs b/src/thread_task.rs index ede81038d..1afeb6fb4 100644 --- a/src/thread_task.rs +++ b/src/thread_task.rs @@ -10,7 +10,6 @@ use crate::src::decode::rav1d_decode_frame_exit; use crate::src::decode::rav1d_decode_frame_init; use crate::src::decode::rav1d_decode_frame_init_cdf; use crate::src::decode::rav1d_decode_tile_sbrow; -use crate::src::error::Rav1dError::EGeneric; use crate::src::error::Rav1dError::EINVAL; use crate::src::error::Rav1dError::ENOMEM; use crate::src::error::Rav1dResult; @@ -18,25 +17,20 @@ use crate::src::fg_apply::rav1d_apply_grain_row; use crate::src::fg_apply::rav1d_prep_grain; use crate::src::internal::Rav1dContext; use crate::src::internal::Rav1dFrameData; -use crate::src::internal::Rav1dTask; use crate::src::internal::Rav1dTaskContext; use crate::src::internal::Rav1dTaskContext_task_thread; +use crate::src::internal::Rav1dTaskIndex; use crate::src::internal::Rav1dTileState; use crate::src::internal::TaskThreadData; use crate::src::internal::TaskThreadData_delayed_fg; use crate::src::internal::TaskType; use crate::src::picture::Rav1dThreadPicture; -use libc::memset; -use libc::realloc; use std::cmp; use std::ffi::c_char; use std::ffi::c_int; -use std::ffi::c_long; use std::ffi::c_uint; -use std::ffi::c_void; use std::mem; use std::process::abort; -use std::ptr; use std::sync::atomic::AtomicI32; use std::sync::atomic::AtomicU32; use std::sync::atomic::Ordering; @@ -91,7 +85,7 @@ unsafe fn reset_task_cur(c: &Rav1dContext, ttd: &TaskThreadData, mut frame_idx: reset_frame_idx = u32::MAX; } if ttd.cur.load(Ordering::Relaxed) == 0 - && ((*(c.fc).offset(first as isize)).task_thread.task_cur_prev).is_null() + && ((*(*(c.fc).offset(first as isize)).task_thread.tasks()).cur_prev).is_none() { return 0 as c_int; } @@ -125,14 +119,15 @@ unsafe fn reset_task_cur(c: &Rav1dContext, ttd: &TaskThreadData, mut frame_idx: ttd.cur .store(min_frame_idx.wrapping_sub(first), Ordering::Relaxed); while ttd.cur.load(Ordering::Relaxed) < c.n_fc { - if !((*(c.fc).offset( + if (*(*(c.fc).offset( first .wrapping_add(ttd.cur.load(Ordering::Relaxed)) .wrapping_rem(c.n_fc) as isize, )) .task_thread - .task_head) - .is_null() + .tasks()) + .head + .is_some() { break; } @@ -143,10 +138,10 @@ unsafe fn reset_task_cur(c: &Rav1dContext, ttd: &TaskThreadData, mut frame_idx: } let mut i: c_uint = ttd.cur.load(Ordering::Relaxed); while i < c.n_fc { - let ref mut fresh0 = (*(c.fc).offset(first.wrapping_add(i).wrapping_rem(c.n_fc) as isize)) + (*(*(c.fc).offset(first.wrapping_add(i).wrapping_rem(c.n_fc) as isize)) .task_thread - .task_cur_prev; - *fresh0 = 0 as *mut Rav1dTask; + .tasks()) + .cur_prev = None; i = i.wrapping_add(1); } return 1 as c_int; @@ -178,30 +173,29 @@ unsafe fn reset_task_cur_async(ttd: &TaskThreadData, mut frame_idx: c_uint, n_fr unsafe fn insert_tasks_between( c: &Rav1dContext, - f: &mut Rav1dFrameData, - first: *mut Rav1dTask, - last: *mut Rav1dTask, - a: *mut Rav1dTask, - b: *mut Rav1dTask, + f: &Rav1dFrameData, + first: Rav1dTaskIndex, + last: Rav1dTaskIndex, + a: Option, + b: Option, cond_signal: c_int, ) { let ttd: &TaskThreadData = &*f.task_thread.ttd; if c.flush.load(Ordering::SeqCst) != 0 { return; } - if !(a.is_null() || (*a).next == b) { - unreachable!(); - } - if a.is_null() { - f.task_thread.task_head = first; + let tasks = &mut *f.task_thread.tasks(); + if let Some(a) = a { + assert_eq!(tasks[a].next, b); + tasks[a].next = Some(first); } else { - (*a).next = first; + tasks.head = Some(first); } - if b.is_null() { - f.task_thread.task_tail = last; + if b.is_none() { + tasks.tail = Some(last); } - (*last).next = b; - reset_task_cur(c, ttd, (*first).frame_idx); + tasks[last].next = b; + reset_task_cur(c, ttd, tasks[first].frame_idx); if cond_signal != 0 && ttd.cond_signaled.fetch_or(1, Ordering::SeqCst) == 0 { ttd.cond.notify_one(); } @@ -209,45 +203,45 @@ unsafe fn insert_tasks_between( unsafe fn insert_tasks( c: &Rav1dContext, - f: &mut Rav1dFrameData, - first: *mut Rav1dTask, - last: *mut Rav1dTask, + f: &Rav1dFrameData, + first: Rav1dTaskIndex, + last: Rav1dTaskIndex, cond_signal: c_int, ) { - let mut t_ptr: *mut Rav1dTask; - let mut prev_t: *mut Rav1dTask = 0 as *mut Rav1dTask; + let tasks = &*f.task_thread.tasks(); + let mut prev_t = None; let mut current_block_34: u64; - t_ptr = f.task_thread.task_head; - while !t_ptr.is_null() { - if (*t_ptr).type_0 == TaskType::TileEntropy { - if (*first).type_0 > TaskType::TileEntropy { + let mut maybe_t = tasks.head; + while let Some(t) = maybe_t { + if tasks[t].type_0 == TaskType::TileEntropy { + if tasks[first].type_0 > TaskType::TileEntropy { current_block_34 = 11174649648027449784; - } else if (*first).sby > (*t_ptr).sby { + } else if tasks[first].sby > tasks[t].sby { current_block_34 = 11174649648027449784; } else { - if (*first).sby < (*t_ptr).sby { - insert_tasks_between(c, f, first, last, prev_t, t_ptr, cond_signal); + if tasks[first].sby < tasks[t].sby { + insert_tasks_between(c, f, first, last, prev_t, Some(t), cond_signal); return; } current_block_34 = 15904375183555213903; } } else { - if (*first).type_0 == TaskType::TileEntropy { - insert_tasks_between(c, f, first, last, prev_t, t_ptr, cond_signal); + if tasks[first].type_0 == TaskType::TileEntropy { + insert_tasks_between(c, f, first, last, prev_t, Some(t), cond_signal); return; } - if (*first).sby > (*t_ptr).sby { + if tasks[first].sby > tasks[t].sby { current_block_34 = 11174649648027449784; } else { - if (*first).sby < (*t_ptr).sby { - insert_tasks_between(c, f, first, last, prev_t, t_ptr, cond_signal); + if tasks[first].sby < tasks[t].sby { + insert_tasks_between(c, f, first, last, prev_t, Some(t), cond_signal); return; } - if (*first).type_0 as c_uint > (*t_ptr).type_0 as c_uint { + if tasks[first].type_0 as c_uint > tasks[t].type_0 as c_uint { current_block_34 = 11174649648027449784; } else { - if ((*first).type_0 as c_uint) < (*t_ptr).type_0 as c_uint { - insert_tasks_between(c, f, first, last, prev_t, t_ptr, cond_signal); + if (tasks[first].type_0 as c_uint) < tasks[t].type_0 as c_uint { + insert_tasks_between(c, f, first, last, prev_t, Some(t), cond_signal); return; } current_block_34 = 15904375183555213903; @@ -257,75 +251,70 @@ unsafe fn insert_tasks( match current_block_34 { 15904375183555213903 => { if !matches!( - (*first).type_0, + tasks[first].type_0, TaskType::TileReconstruction | TaskType::TileEntropy ) { unreachable!(); } - if !((*first).type_0 == (*t_ptr).type_0) { + if !(tasks[first].type_0 == tasks[t].type_0) { unreachable!(); } - if !((*t_ptr).sby == (*first).sby) { + if !(tasks[t].sby == tasks[first].sby) { unreachable!(); } - let p = (*first).type_0 == TaskType::TileEntropy; - let t_tile_idx = - first.offset_from(f.task_thread.tile_tasks[p as usize]) as c_long as c_int; - let p_tile_idx = - t_ptr.offset_from(f.task_thread.tile_tasks[p as usize]) as c_long as c_int; + let p = tasks[first].type_0 == TaskType::TileEntropy; + let t_tile_idx = first - tasks.tile_tasks[p as usize].unwrap(); + let p_tile_idx = t - tasks.tile_tasks[p as usize].unwrap(); if !(t_tile_idx != p_tile_idx) { unreachable!(); } if !(t_tile_idx > p_tile_idx) { - insert_tasks_between(c, f, first, last, prev_t, t_ptr, cond_signal); + insert_tasks_between(c, f, first, last, prev_t, Some(t), cond_signal); return; } } _ => {} } - prev_t = t_ptr; - t_ptr = (*t_ptr).next; + prev_t = Some(t); + maybe_t = tasks[t].next; } - insert_tasks_between(c, f, first, last, prev_t, 0 as *mut Rav1dTask, cond_signal); + insert_tasks_between(c, f, first, last, prev_t, None, cond_signal); } #[inline] -unsafe fn insert_task( - c: &Rav1dContext, - f: &mut Rav1dFrameData, - t: *mut Rav1dTask, - cond_signal: c_int, -) { +unsafe fn insert_task(c: &Rav1dContext, f: &Rav1dFrameData, t: Rav1dTaskIndex, cond_signal: c_int) { insert_tasks(c, f, t, t, cond_signal); } #[inline] -unsafe fn add_pending(f: &Rav1dFrameData, t: *mut Rav1dTask) { +unsafe fn add_pending(f: &Rav1dFrameData, t: Rav1dTaskIndex) { + let tasks = &mut *f.task_thread.tasks(); let mut pending_tasks = f.task_thread.pending_tasks.lock().unwrap(); - (*t).next = 0 as *mut Rav1dTask; - if pending_tasks.head.is_null() { - pending_tasks.head = t; + tasks[t].next = None; + if pending_tasks.head.is_none() { + pending_tasks.head = Some(t); } else { - (*pending_tasks.tail).next = t; + tasks[pending_tasks.tail.unwrap()].next = Some(t); } - pending_tasks.tail = t; + pending_tasks.tail = Some(t); f.task_thread.pending_tasks_merge.store(1, Ordering::SeqCst); } #[inline] -unsafe fn merge_pending_frame(c: &Rav1dContext, f: &mut Rav1dFrameData) -> c_int { +unsafe fn merge_pending_frame(c: &Rav1dContext, f: &Rav1dFrameData) -> c_int { + let tasks = &*f.task_thread.tasks(); let merge = f.task_thread.pending_tasks_merge.load(Ordering::SeqCst); if merge != 0 { - let mut t = { + let mut next_t = { let mut pending_tasks = f.task_thread.pending_tasks.lock().unwrap(); let old_head = mem::take(&mut *pending_tasks).head; f.task_thread.pending_tasks_merge.store(0, Ordering::SeqCst); old_head }; - while !t.is_null() { - let tmp: *mut Rav1dTask = (*t).next; + while let Some(t) = next_t { + let tmp = tasks[t].next; insert_task(c, f, t, 0 as c_int); - t = tmp; + next_t = tmp; } } return merge; @@ -346,8 +335,7 @@ unsafe fn create_filter_sbrow( c: &Rav1dContext, f: &mut Rav1dFrameData, pass: c_int, - res_t: *mut *mut Rav1dTask, -) -> c_int { +) -> Rav1dResult { let frame_hdr = &***f.frame_hdr.as_ref().unwrap(); let has_deblock = (frame_hdr.loopfilter.level_y[0] != 0 || frame_hdr.loopfilter.level_y[1] != 0) as c_int; @@ -355,20 +343,11 @@ unsafe fn create_filter_sbrow( let has_cdef = seq_hdr.cdef; let has_resize = (frame_hdr.size.width[0] != frame_hdr.size.width[1]) as c_int; let has_lr = f.lf.restore_planes; - let mut tasks: *mut Rav1dTask = f.task_thread.tasks; + let tasks = &mut *f.task_thread.tasks(); let uses_2pass = (c.n_fc > 1 as c_uint) as c_int; - let num_tasks = f.sbh * (1 + uses_2pass); - if num_tasks > f.task_thread.num_tasks { - let size: usize = (::core::mem::size_of::()).wrapping_mul(num_tasks as usize); - tasks = realloc(f.task_thread.tasks as *mut c_void, size) as *mut Rav1dTask; - if tasks.is_null() { - return -(1 as c_int); - } - memset(tasks as *mut c_void, 0 as c_int, size); - f.task_thread.tasks = tasks; - f.task_thread.num_tasks = num_tasks; - } - tasks = tasks.offset((f.sbh * (pass & 1)) as isize); + let num_tasks = (f.sbh * (1 + uses_2pass)) as usize; + tasks.grow_tasks(num_tasks); + let task_idx = Rav1dTaskIndex::Task((f.sbh * (pass & 1)) as usize); if pass & 1 != 0 { f.frame_thread_progress.entropy = AtomicI32::new(0); } else { @@ -384,11 +363,11 @@ unsafe fn create_filter_sbrow( f.frame_thread_progress.deblock.store(0, Ordering::SeqCst); } f.frame_thread.next_tile_row[(pass & 1) as usize] = 0 as c_int; - let t: *mut Rav1dTask = &mut *tasks.offset(0) as *mut Rav1dTask; - (*t).sby = 0 as c_int; - (*t).recon_progress = 1 as c_int; - (*t).deblock_progress = 0 as c_int; - (*t).type_0 = if pass == 1 { + let t = &mut tasks[task_idx]; + t.sby = 0 as c_int; + t.recon_progress = 1 as c_int; + t.deblock_progress = 0 as c_int; + t.type_0 = if pass == 1 { TaskType::EntropyProgress } else if has_deblock != 0 { TaskType::DeblockCols @@ -399,9 +378,8 @@ unsafe fn create_filter_sbrow( } else { TaskType::ReconstructionProgress }; - (*t).frame_idx = f.index as c_uint; - *res_t = t; - return 0 as c_int; + t.frame_idx = f.index as c_uint; + Ok(task_idx) } pub(crate) unsafe fn rav1d_task_create_tile_sbrow( @@ -410,70 +388,60 @@ pub(crate) unsafe fn rav1d_task_create_tile_sbrow( pass: c_int, _cond_signal: c_int, ) -> Rav1dResult { - let mut tasks: *mut Rav1dTask = f.task_thread.tile_tasks[0]; + let tasks = &mut *f.task_thread.tasks(); let uses_2pass = (c.n_fc > 1 as c_uint) as c_int; let frame_hdr = &***f.frame_hdr.as_ref().unwrap(); let num_tasks = frame_hdr.tiling.cols * frame_hdr.tiling.rows; if pass < 2 { let alloc_num_tasks = num_tasks * (1 + uses_2pass); - if alloc_num_tasks > f.task_thread.num_tile_tasks { - let size: usize = - (::core::mem::size_of::()).wrapping_mul(alloc_num_tasks as usize); - tasks = realloc(f.task_thread.tile_tasks[0] as *mut c_void, size) as *mut Rav1dTask; - if tasks.is_null() { - return Err(EGeneric); - } - memset(tasks as *mut c_void, 0 as c_int, size); - f.task_thread.tile_tasks[0] = tasks; - f.task_thread.num_tile_tasks = alloc_num_tasks; - } - f.task_thread.tile_tasks[1] = tasks.offset(num_tasks as isize); - } - tasks = tasks.offset((num_tasks * (pass & 1)) as isize); - let mut pf_t: *mut Rav1dTask = 0 as *mut Rav1dTask; - if create_filter_sbrow(c, f, pass, &mut pf_t) != 0 { - return Err(EGeneric); + tasks.grow_tile_tasks(alloc_num_tasks as usize); + tasks.tile_tasks[1] = Some(Rav1dTaskIndex::TileTask(num_tasks as usize)); } - let mut prev_t: *mut Rav1dTask = 0 as *mut Rav1dTask; + let tile_tasks = tasks.tile_tasks[0].map(|t| t + (num_tasks * (pass & 1)) as usize); + let mut pf_t = Some(create_filter_sbrow(c, f, pass)?); + let mut prev_t = None; let mut tile_idx = 0; while tile_idx < num_tasks { let ts: *mut Rav1dTileState = &mut *(f.ts).offset(tile_idx as isize) as *mut Rav1dTileState; - let t: *mut Rav1dTask = &mut *tasks.offset(tile_idx as isize) as *mut Rav1dTask; - (*t).sby = (*ts).tiling.row_start >> f.sb_shift; - if !pf_t.is_null() && (*t).sby != 0 { - (*prev_t).next = pf_t; + let t_idx = tile_tasks.unwrap() + (tile_idx as usize); + let t = &mut tasks[t_idx]; + t.sby = (*ts).tiling.row_start >> f.sb_shift; + if pf_t.is_some() && t.sby != 0 { + tasks[prev_t.unwrap()].next = pf_t; prev_t = pf_t; - pf_t = 0 as *mut Rav1dTask; + pf_t = None; } - (*t).recon_progress = 0 as c_int; - (*t).deblock_progress = 0 as c_int; - (*t).deps_skip = 0 as c_int; - (*t).type_0 = if pass != 1 { + // re-borrow to avoid conflict with tasks[prev_t] above + let t = &mut tasks[t_idx]; + t.recon_progress = 0 as c_int; + t.deblock_progress = 0 as c_int; + t.deps_skip = 0 as c_int; + t.type_0 = if pass != 1 { TaskType::TileReconstruction } else { TaskType::TileEntropy }; - (*t).frame_idx = f.index as c_uint; - if !prev_t.is_null() { - (*prev_t).next = t; + t.frame_idx = f.index as c_uint; + if let Some(prev_t) = prev_t { + tasks[prev_t].next = Some(t_idx); } - prev_t = t; + prev_t = Some(t_idx); tile_idx += 1; } - if !pf_t.is_null() { - (*prev_t).next = pf_t; + if pf_t.is_some() { + tasks[prev_t.unwrap()].next = pf_t; prev_t = pf_t; } - (*prev_t).next = 0 as *mut Rav1dTask; + tasks[prev_t.unwrap()].next = None; f.task_thread.done[(pass & 1) as usize].store(0, Ordering::SeqCst); let mut pending_tasks = f.task_thread.pending_tasks.lock().unwrap(); - if !(pending_tasks.head.is_null() || pass == 2) { + if !(pending_tasks.head.is_none() || pass == 2) { unreachable!(); } - if pending_tasks.head.is_null() { - pending_tasks.head = tasks.offset(0); + if pending_tasks.head.is_none() { + pending_tasks.head = tile_tasks; } else { - (*pending_tasks.tail).next = tasks.offset(0); + tasks[pending_tasks.tail.unwrap()].next = tile_tasks; } pending_tasks.tail = prev_t; f.task_thread.pending_tasks_merge.store(1, Ordering::SeqCst); @@ -483,13 +451,15 @@ pub(crate) unsafe fn rav1d_task_create_tile_sbrow( pub(crate) unsafe fn rav1d_task_frame_init(c: &Rav1dContext, f: &mut Rav1dFrameData) { f.task_thread.init_done.store(0, Ordering::SeqCst); - let t: *mut Rav1dTask = &mut f.task_thread.init_task; - (*t).type_0 = TaskType::Init; - (*t).frame_idx = f.index as c_uint; - (*t).sby = 0 as c_int; - (*t).deblock_progress = 0 as c_int; - (*t).recon_progress = (*t).deblock_progress; - insert_task(c, f, t, 1 as c_int); + let tasks = f.task_thread.tasks(); + let t_idx = Rav1dTaskIndex::Init; + let t = &mut (*tasks)[t_idx]; + t.type_0 = TaskType::Init; + t.frame_idx = f.index as c_uint; + t.sby = 0 as c_int; + t.deblock_progress = 0 as c_int; + t.recon_progress = t.deblock_progress; + insert_task(c, f, t_idx, 1 as c_int); } pub(crate) unsafe fn rav1d_task_delayed_fg( @@ -513,19 +483,19 @@ pub(crate) unsafe fn rav1d_task_delayed_fg( unsafe fn ensure_progress<'l, 'ttd: 'l>( ttd: &'ttd TaskThreadData, f: &Rav1dFrameData, - t: *mut Rav1dTask, + t_idx: Rav1dTaskIndex, type_0: TaskType, state: &AtomicI32, - target: *mut c_int, task_thread_lock: &'l mut Option>, ) -> c_int { let p1 = state.load(Ordering::SeqCst); - if p1 < (*t).sby { - (*t).type_0 = type_0; - (*t).deblock_progress = 0 as c_int; - (*t).recon_progress = (*t).deblock_progress; - *target = (*t).sby; - add_pending(f, t); + let tasks = &mut *f.task_thread.tasks(); + let t = &mut tasks[t_idx]; + if p1 < t.sby { + t.type_0 = type_0; + t.recon_progress = 0 as c_int; + t.deblock_progress = t.sby; + add_pending(f, t_idx); *task_thread_lock = Some(ttd.delayed_fg.lock().unwrap()); return 1 as c_int; } @@ -533,12 +503,16 @@ unsafe fn ensure_progress<'l, 'ttd: 'l>( } #[inline] -unsafe fn check_tile(t: *mut Rav1dTask, f: &mut Rav1dFrameData, frame_mt: c_int) -> c_int { - let tp = (*t).type_0 == TaskType::TileEntropy; - let tile_idx = t.offset_from(f.task_thread.tile_tasks[tp as usize]) as c_long as c_int; +unsafe fn check_tile(t_idx: Rav1dTaskIndex, f: &Rav1dFrameData, frame_mt: c_int) -> c_int { + let tasks = &mut *f.task_thread.tasks(); + let t = &tasks[t_idx]; + let tp = t.type_0 == TaskType::TileEntropy; + let tile_idx = (t_idx - tasks.tile_tasks[tp as usize].unwrap()) + .raw_index() + .expect("t_idx was not a valid tile task"); let ts: *mut Rav1dTileState = &mut *(f.ts).offset(tile_idx as isize) as *mut Rav1dTileState; let p1 = (*ts).progress[tp as usize].load(Ordering::SeqCst); - if p1 < (*t).sby { + if p1 < t.sby { return 1 as c_int; } let mut error = (p1 == TILE_ERROR) as c_int; @@ -553,7 +527,7 @@ unsafe fn check_tile(t: *mut Rav1dTask, f: &mut Rav1dFrameData, frame_mt: c_int) } let frame_hdr = &***f.frame_hdr.as_ref().unwrap(); if error == 0 && frame_mt != 0 && !frame_hdr.frame_type.is_key_or_intra() { - let p: *const Rav1dThreadPicture = &mut f.sr_cur; + let p: *const Rav1dThreadPicture = &f.sr_cur; let ss_ver = ((*p).p.p.layout as c_uint == Rav1dPixelLayout::I420 as c_int as c_uint) as c_int; let p_b: c_uint = (((*t).sby + 1) << f.sb_shift + 2) as c_uint; @@ -600,7 +574,7 @@ unsafe fn check_tile(t: *mut Rav1dTask, f: &mut Rav1dFrameData, frame_mt: c_int) _ => {} } n += 1; - (*t).deps_skip += 1; + tasks[t_idx].deps_skip += 1; } } return 0 as c_int; @@ -813,21 +787,23 @@ pub unsafe fn rav1d_worker_task(c: &Rav1dContext, task_thread: Arc 1 as c_uint { // run init tasks second 'init_tasks: for i in 0..c.n_fc { let first = ttd.first.load(Ordering::SeqCst); let f = &mut *(c.fc).offset(first.wrapping_add(i).wrapping_rem(c.n_fc) as isize); + let tasks = &*f.task_thread.tasks(); if f.task_thread.init_done.load(Ordering::SeqCst) != 0 { continue 'init_tasks; } - let Some(t) = f.task_thread.task_head.as_mut() else { + let Some(t_idx) = tasks.head else { continue 'init_tasks; }; + let t = &tasks[t_idx]; if t.type_0 == TaskType::Init { - break 'found (f, t, None); + break 'found (f, t_idx, None); } if t.type_0 == TaskType::InitCdf { // XXX This can be a simple else, if adding tasks of both @@ -846,7 +822,7 @@ pub unsafe fn rav1d_worker_task(c: &Rav1dContext, task_thread: Arc 1 as c_uint) as c_int) == 0 { - break 'found (f, t, prev_t); + if check_tile(t_idx, f, (c.n_fc > 1 as c_uint) as c_int) == 0 { + break 'found (f, t_idx, prev_t); } } else if t.recon_progress != 0 { let p = t.type_0 == TaskType::EntropyProgress; @@ -914,27 +892,25 @@ pub unsafe fn rav1d_worker_task(c: &Rav1dContext, task_thread: Arc> 5) as usize] .load(Ordering::SeqCst); if p1_1 as c_uint & (1 as c_uint) << (t.sby - 1 & 31) != 0 { - break 'found (f, t, prev_t); + break 'found (f, t_idx, prev_t); } } else { if t.deblock_progress == 0 { @@ -945,14 +921,14 @@ pub unsafe fn rav1d_worker_task(c: &Rav1dContext, task_thread: Arc TaskType::InitCdf && (f.task_thread.task_head).is_null() { + if tasks[t_idx].type_0 > TaskType::InitCdf && tasks.head.is_none() { ttd.cur.fetch_add(1, Ordering::Relaxed); } - t.next = 0 as *mut Rav1dTask; + tasks[t_idx].next = None; + let tile_tasks = tasks.tile_tasks; + let t = &mut tasks[t_idx]; // we don't need to check cond_signaled here, since we found a task // after the last signal so we want to re-signal the next waiting thread // and again won't need to signal after that @@ -1017,7 +995,7 @@ pub unsafe fn rav1d_worker_task(c: &Rav1dContext, task_thread: Arc { let p_1 = t.type_0 == TaskType::TileEntropy; - // TODO(sjc): t is a reference to an array element, we - // need to replace this pointer arithmetic with proper - // indexing - let tile_idx = (t as *const Rav1dTask) - .offset_from(f.task_thread.tile_tasks[p_1 as usize]) - as c_long as c_int; + let tile_idx = (t_idx - tile_tasks[p_1 as usize].unwrap()) + .raw_index() + .unwrap(); let ts_0: *mut Rav1dTileState = &mut *(f.ts).offset(tile_idx as isize) as *mut Rav1dTileState; tc.ts = ts_0; @@ -1128,7 +1103,7 @@ pub unsafe fn rav1d_worker_task(c: &Rav1dContext, task_thread: Arc