diff --git a/src/looprestoration.rs b/src/looprestoration.rs index 6fc493947..391baa154 100644 --- a/src/looprestoration.rs +++ b/src/looprestoration.rs @@ -1,3 +1,4 @@ +use crate::include::common::bitdepth::BitDepth; use crate::include::stddef::ptrdiff_t; use crate::include::stdint::int16_t; use crate::include::stdint::uint32_t; @@ -136,3 +137,160 @@ decl_looprestorationfilter_fns! { fn dav1d_wiener_filter7_16bpc_neon, fn dav1d_wiener_filter5_16bpc_neon, } + +// 256 * 1.5 + 3 + 3 = 390 +const REST_UNIT_STRIDE: usize = 390; + +// TODO Reuse p when no padding is needed (add and remove lpf pixels in p) +// TODO Chroma only requires 2 rows of padding. +// TODO(randomPoison): Temporarily pub until remaining looprestoration fns have +// been deduplicated. +#[inline(never)] +pub(crate) unsafe fn padding( + mut dst: &mut [BD::Pixel; 70 /*(64 + 3 + 3)*/ * REST_UNIT_STRIDE], + mut p: *const BD::Pixel, + stride: ptrdiff_t, + mut left: *const [BD::Pixel; 4], + mut lpf: *const BD::Pixel, + mut unit_w: libc::c_int, + stripe_h: libc::c_int, + edges: LrEdgeFlags, +) { + let stride = BD::pxstride(stride as usize); + + let have_left = (edges & LR_HAVE_LEFT != 0) as libc::c_int; + let have_right = (edges & LR_HAVE_RIGHT != 0) as libc::c_int; + + // Copy more pixels if we don't have to pad them + unit_w += 3 * have_left + 3 * have_right; + let mut dst_l = &mut dst[(3 * (have_left == 0) as libc::c_int) as usize..]; + p = p.offset(-((3 * have_left) as isize)); + lpf = lpf.offset(-((3 * have_left) as isize)); + + if edges & LR_HAVE_TOP != 0 { + // Copy previous loop filtered rows + let above_1 = std::slice::from_raw_parts(lpf, stride + unit_w as usize); + let above_2 = &above_1[stride..]; + + BD::pixel_copy(dst_l, above_1, unit_w as usize); + BD::pixel_copy(&mut dst_l[REST_UNIT_STRIDE..], above_1, unit_w as usize); + BD::pixel_copy(&mut dst_l[2 * REST_UNIT_STRIDE..], above_2, unit_w as usize); + } else { + // Pad with first row + let p = std::slice::from_raw_parts(p, unit_w as usize); + + BD::pixel_copy(dst_l, p, unit_w as usize); + BD::pixel_copy(&mut dst_l[REST_UNIT_STRIDE..], p, unit_w as usize); + BD::pixel_copy(&mut dst_l[2 * REST_UNIT_STRIDE..], p, unit_w as usize); + + if have_left != 0 { + let left = &(*left.offset(0))[1..]; + BD::pixel_copy(dst_l, left, 3); + BD::pixel_copy(&mut dst_l[REST_UNIT_STRIDE..], left, 3); + BD::pixel_copy(&mut dst_l[2 * REST_UNIT_STRIDE..], left, 3); + } + } + + let mut dst_tl = &mut dst_l[3 * REST_UNIT_STRIDE..]; + if edges & LR_HAVE_BOTTOM != 0 { + // Copy next loop filtered rows + let below_1 = std::slice::from_raw_parts(lpf.offset(6 * stride as isize), unit_w as usize); + let below_2 = std::slice::from_raw_parts(lpf.offset(7 * stride as isize), unit_w as usize); + + BD::pixel_copy( + &mut dst_tl[stripe_h as usize * REST_UNIT_STRIDE..], + below_1, + unit_w as usize, + ); + BD::pixel_copy( + &mut dst_tl[(stripe_h + 1) as usize * REST_UNIT_STRIDE..], + below_2, + unit_w as usize, + ); + BD::pixel_copy( + &mut dst_tl[(stripe_h + 2) as usize * REST_UNIT_STRIDE..], + below_2, + unit_w as usize, + ); + } else { + // Pad with last row + let src = std::slice::from_raw_parts( + p.offset(((stripe_h - 1) as isize * stride as isize) as isize), + unit_w as usize, + ); + + BD::pixel_copy( + &mut dst_tl[stripe_h as usize * REST_UNIT_STRIDE..], + src, + unit_w as usize, + ); + BD::pixel_copy( + &mut dst_tl[(stripe_h + 1) as usize * REST_UNIT_STRIDE..], + src, + unit_w as usize, + ); + BD::pixel_copy( + &mut dst_tl[(stripe_h + 2) as usize * REST_UNIT_STRIDE..], + src, + unit_w as usize, + ); + + if have_left != 0 { + let left = &(*left.offset((stripe_h - 1) as isize))[1..]; + + BD::pixel_copy(&mut dst_tl[stripe_h as usize * REST_UNIT_STRIDE..], left, 3); + BD::pixel_copy( + &mut dst_tl[(stripe_h + 1) as usize * REST_UNIT_STRIDE..], + left, + 3, + ); + BD::pixel_copy( + &mut dst_tl[(stripe_h + 2) as usize * REST_UNIT_STRIDE..], + left, + 3, + ); + } + } + + // Inner UNIT_WxSTRIPE_H + let len = (unit_w - 3 * have_left) as usize; + for j in 0..stripe_h as usize { + let p = std::slice::from_raw_parts( + p.offset((j * stride + 3 * have_left as usize) as isize), + len, + ); + BD::pixel_copy( + &mut dst_tl[j * REST_UNIT_STRIDE + (3 * have_left) as usize..], + p, + len, + ); + } + + if have_right == 0 { + // Pad 3x(STRIPE_H+6) with last column + for j in 0..stripe_h as usize + 6 { + let mut row_last = dst_l[(unit_w - 1) as usize + j * REST_UNIT_STRIDE]; + let mut pad = &mut dst_l[unit_w as usize + j * REST_UNIT_STRIDE..]; + BD::pixel_set(pad, row_last, 3); + } + } + + if have_left == 0 { + // Pad 3x(STRIPE_H+6) with first column + for j in 0..stripe_h as usize + 6 { + let offset = j * REST_UNIT_STRIDE; + let val = dst[3 + offset]; + BD::pixel_set(&mut dst[offset..], val, 3); + } + } else { + let dst = &mut dst[3 * REST_UNIT_STRIDE..]; + + for j in 0..stripe_h as usize { + BD::pixel_copy( + &mut dst[j * REST_UNIT_STRIDE..], + &(*left.offset(j as isize))[1..], + 3, + ); + } + }; +} diff --git a/src/looprestoration_tmpl_16.rs b/src/looprestoration_tmpl_16.rs index fd5d5f5e3..b7334b3b4 100644 --- a/src/looprestoration_tmpl_16.rs +++ b/src/looprestoration_tmpl_16.rs @@ -1,3 +1,4 @@ +use crate::include::common::bitdepth::BitDepth16; use crate::include::stddef::*; use crate::include::stdint::*; #[cfg(all(feature = "asm", any(target_arch = "arm", target_arch = "aarch64"),))] @@ -5,9 +6,6 @@ use crate::src::align::Align16; use ::libc; #[cfg(feature = "asm")] use cfg_if::cfg_if; -extern "C" { - fn memcpy(_: *mut libc::c_void, _: *const libc::c_void, _: libc::c_ulong) -> *mut libc::c_void; -} #[cfg(all(feature = "asm", any(target_arch = "arm", target_arch = "aarch64")))] extern "C" { @@ -134,10 +132,6 @@ use crate::src::tables::dav1d_sgr_x_by_x; pub type pixel = uint16_t; pub type coef = int32_t; use crate::src::looprestoration::LrEdgeFlags; -use crate::src::looprestoration::LR_HAVE_BOTTOM; -use crate::src::looprestoration::LR_HAVE_LEFT; -use crate::src::looprestoration::LR_HAVE_RIGHT; -use crate::src::looprestoration::LR_HAVE_TOP; pub type const_left_pixel_row = *const [pixel; 4]; use crate::src::looprestoration::LooprestorationParams; @@ -145,15 +139,9 @@ use crate::include::common::attributes::clz; use crate::include::common::intops::iclip; use crate::include::common::intops::imax; use crate::include::common::intops::umin; +use crate::src::looprestoration::padding; use crate::src::looprestoration::Dav1dLoopRestorationDSPContext; -#[inline] -unsafe extern "C" fn pixel_set(dst: *mut pixel, val: libc::c_int, num: libc::c_int) { - let mut n = 0; - while n < num { - *dst.offset(n as isize) = val as pixel; - n += 1; - } -} + #[inline] unsafe extern "C" fn PXSTRIDE(x: ptrdiff_t) -> ptrdiff_t { if x & 1 != 0 { @@ -161,179 +149,6 @@ unsafe extern "C" fn PXSTRIDE(x: ptrdiff_t) -> ptrdiff_t { } return x >> 1; } -#[inline(never)] -unsafe extern "C" fn padding( - mut dst: *mut pixel, - mut p: *const pixel, - stride: ptrdiff_t, - mut left: *const [pixel; 4], - mut lpf: *const pixel, - mut unit_w: libc::c_int, - stripe_h: libc::c_int, - edges: LrEdgeFlags, -) { - let have_left = - (edges as libc::c_uint & LR_HAVE_LEFT as libc::c_int as libc::c_uint != 0) as libc::c_int; - let have_right = - (edges as libc::c_uint & LR_HAVE_RIGHT as libc::c_int as libc::c_uint != 0) as libc::c_int; - unit_w += 3 * have_left + 3 * have_right; - let mut dst_l: *mut pixel = dst.offset((3 * (have_left == 0) as libc::c_int) as isize); - p = p.offset(-((3 * have_left) as isize)); - lpf = lpf.offset(-((3 * have_left) as isize)); - if edges as libc::c_uint & LR_HAVE_TOP as libc::c_int as libc::c_uint != 0 { - let above_1: *const pixel = lpf; - let above_2: *const pixel = above_1.offset(PXSTRIDE(stride) as isize); - memcpy( - dst_l as *mut libc::c_void, - above_1 as *const libc::c_void, - (unit_w << 1) as libc::c_ulong, - ); - memcpy( - dst_l.offset(390) as *mut libc::c_void, - above_1 as *const libc::c_void, - (unit_w << 1) as libc::c_ulong, - ); - memcpy( - dst_l.offset((2 * 390) as isize) as *mut libc::c_void, - above_2 as *const libc::c_void, - (unit_w << 1) as libc::c_ulong, - ); - } else { - memcpy( - dst_l as *mut libc::c_void, - p as *const libc::c_void, - (unit_w << 1) as libc::c_ulong, - ); - memcpy( - dst_l.offset(390) as *mut libc::c_void, - p as *const libc::c_void, - (unit_w << 1) as libc::c_ulong, - ); - memcpy( - dst_l.offset((2 * 390) as isize) as *mut libc::c_void, - p as *const libc::c_void, - (unit_w << 1) as libc::c_ulong, - ); - if have_left != 0 { - memcpy( - dst_l as *mut libc::c_void, - &*(*left.offset(0)).as_ptr().offset(1) as *const pixel as *const libc::c_void, - ((3 as libc::c_int) << 1) as libc::c_ulong, - ); - memcpy( - dst_l.offset(390) as *mut libc::c_void, - &*(*left.offset(0)).as_ptr().offset(1) as *const pixel as *const libc::c_void, - ((3 as libc::c_int) << 1) as libc::c_ulong, - ); - memcpy( - dst_l.offset((2 * 390) as isize) as *mut libc::c_void, - &*(*left.offset(0)).as_ptr().offset(1) as *const pixel as *const libc::c_void, - ((3 as libc::c_int) << 1) as libc::c_ulong, - ); - } - } - let mut dst_tl: *mut pixel = dst_l.offset((3 * 390) as isize); - if edges as libc::c_uint & LR_HAVE_BOTTOM as libc::c_int as libc::c_uint != 0 { - let below_1: *const pixel = lpf.offset(6 * PXSTRIDE(stride)); - let below_2: *const pixel = below_1.offset(PXSTRIDE(stride) as isize); - memcpy( - dst_tl.offset((stripe_h * 390) as isize) as *mut libc::c_void, - below_1 as *const libc::c_void, - (unit_w << 1) as libc::c_ulong, - ); - memcpy( - dst_tl.offset(((stripe_h + 1) * 390) as isize) as *mut libc::c_void, - below_2 as *const libc::c_void, - (unit_w << 1) as libc::c_ulong, - ); - memcpy( - dst_tl.offset(((stripe_h + 2) * 390) as isize) as *mut libc::c_void, - below_2 as *const libc::c_void, - (unit_w << 1) as libc::c_ulong, - ); - } else { - let src: *const pixel = p.offset(((stripe_h - 1) as isize * PXSTRIDE(stride)) as isize); - memcpy( - dst_tl.offset((stripe_h * 390) as isize) as *mut libc::c_void, - src as *const libc::c_void, - (unit_w << 1) as libc::c_ulong, - ); - memcpy( - dst_tl.offset(((stripe_h + 1) * 390) as isize) as *mut libc::c_void, - src as *const libc::c_void, - (unit_w << 1) as libc::c_ulong, - ); - memcpy( - dst_tl.offset(((stripe_h + 2) * 390) as isize) as *mut libc::c_void, - src as *const libc::c_void, - (unit_w << 1) as libc::c_ulong, - ); - if have_left != 0 { - memcpy( - dst_tl.offset((stripe_h * 390) as isize) as *mut libc::c_void, - &*(*left.offset((stripe_h - 1) as isize)).as_ptr().offset(1) as *const pixel - as *const libc::c_void, - ((3 as libc::c_int) << 1) as libc::c_ulong, - ); - memcpy( - dst_tl.offset(((stripe_h + 1) * 390) as isize) as *mut libc::c_void, - &*(*left.offset((stripe_h - 1) as isize)).as_ptr().offset(1) as *const pixel - as *const libc::c_void, - ((3 as libc::c_int) << 1) as libc::c_ulong, - ); - memcpy( - dst_tl.offset(((stripe_h + 2) * 390) as isize) as *mut libc::c_void, - &*(*left.offset((stripe_h - 1) as isize)).as_ptr().offset(1) as *const pixel - as *const libc::c_void, - ((3 as libc::c_int) << 1) as libc::c_ulong, - ); - } - } - let mut j = 0; - while j < stripe_h { - memcpy( - dst_tl.offset((3 * have_left) as isize) as *mut libc::c_void, - p.offset((3 * have_left) as isize) as *const libc::c_void, - (unit_w - 3 * have_left << 1) as libc::c_ulong, - ); - dst_tl = dst_tl.offset(390); - p = p.offset(PXSTRIDE(stride) as isize); - j += 1; - } - if have_right == 0 { - let mut pad: *mut pixel = dst_l.offset(unit_w as isize); - let mut row_last: *mut pixel = &mut *dst_l.offset((unit_w - 1) as isize) as *mut pixel; - let mut j_0 = 0; - while j_0 < stripe_h + 6 { - pixel_set(pad, *row_last as libc::c_int, 3 as libc::c_int); - pad = pad.offset(390); - row_last = row_last.offset(390); - j_0 += 1; - } - } - if have_left == 0 { - let mut j_1 = 0; - while j_1 < stripe_h + 6 { - pixel_set(dst, *dst_l as libc::c_int, 3 as libc::c_int); - dst = dst.offset(390); - dst_l = dst_l.offset(390); - j_1 += 1; - } - } else { - dst = dst.offset((3 * 390) as isize); - let mut j_2 = 0; - while j_2 < stripe_h { - memcpy( - dst as *mut libc::c_void, - &*(*left.offset(j_2 as isize)).as_ptr().offset(1) as *const pixel - as *const libc::c_void, - ((3 as libc::c_int) << 1) as libc::c_ulong, - ); - dst = dst.offset(390); - j_2 += 1; - } - }; -} unsafe extern "C" fn wiener_c( mut p: *mut libc::c_void, @@ -359,7 +174,8 @@ unsafe extern "C" fn wiener_c( ) } -unsafe extern "C" fn wiener_rust( +// TODO(randompoison): Temporarily public until we can move this to `looprestoration.rs`. +pub(crate) unsafe extern "C" fn wiener_rust( mut p: *mut pixel, stride: ptrdiff_t, left: *const [pixel; 4], @@ -372,7 +188,7 @@ unsafe extern "C" fn wiener_rust( ) { let mut tmp: [pixel; 27300] = [0; 27300]; let mut tmp_ptr: *mut pixel = tmp.as_mut_ptr(); - padding(tmp.as_mut_ptr(), p, stride, left, lpf, w, h, edges); + padding::(&mut tmp, p, stride, left, lpf, w, h, edges); let mut hor: [uint16_t; 27300] = [0; 27300]; let mut hor_ptr: *mut uint16_t = hor.as_mut_ptr(); let filter: *const [int16_t; 8] = ((*params).filter.0).as_ptr(); @@ -759,7 +575,7 @@ unsafe extern "C" fn sgr_5x5_rust( ) { let mut tmp: [pixel; 27300] = [0; 27300]; let mut dst: [coef; 24576] = [0; 24576]; - padding(tmp.as_mut_ptr(), p, stride, left, lpf, w, h, edges); + padding::(&mut tmp, p, stride, left, lpf, w, h, edges); selfguided_filter( dst.as_mut_ptr(), tmp.as_mut_ptr(), @@ -825,7 +641,7 @@ unsafe extern "C" fn sgr_3x3_rust( ) { let mut tmp: [pixel; 27300] = [0; 27300]; let mut dst: [coef; 24576] = [0; 24576]; - padding(tmp.as_mut_ptr(), p, stride, left, lpf, w, h, edges); + padding::(&mut tmp, p, stride, left, lpf, w, h, edges); selfguided_filter( dst.as_mut_ptr(), tmp.as_mut_ptr(), @@ -892,7 +708,7 @@ unsafe extern "C" fn sgr_mix_rust( let mut tmp: [pixel; 27300] = [0; 27300]; let mut dst0: [coef; 24576] = [0; 24576]; let mut dst1: [coef; 24576] = [0; 24576]; - padding(tmp.as_mut_ptr(), p, stride, left, lpf, w, h, edges); + padding::(&mut tmp, p, stride, left, lpf, w, h, edges); selfguided_filter( dst0.as_mut_ptr(), tmp.as_mut_ptr(), @@ -1028,6 +844,9 @@ unsafe extern "C" fn wiener_filter_neon( edges: LrEdgeFlags, bitdepth_max: libc::c_int, ) { + use crate::src::looprestoration::LR_HAVE_BOTTOM; + use crate::src::looprestoration::LR_HAVE_TOP; + let filter: *const [int16_t; 8] = (*params).filter.0.as_ptr(); let mut mid: Align16<[int16_t; 68 * 384]> = Align16([0; 68 * 384]); let mut mid_stride: libc::c_int = w + 7 & !7; @@ -1199,6 +1018,9 @@ unsafe extern "C" fn dav1d_sgr_filter1_neon( edges: LrEdgeFlags, bitdepth_max: libc::c_int, ) { + use crate::src::looprestoration::LR_HAVE_BOTTOM; + use crate::src::looprestoration::LR_HAVE_TOP; + let mut sumsq_mem: Align16<[int32_t; 27208]> = Align16([0; 27208]); let sumsq: *mut int32_t = &mut *sumsq_mem .0 @@ -1254,6 +1076,9 @@ unsafe extern "C" fn dav1d_sgr_filter2_neon( edges: LrEdgeFlags, bitdepth_max: libc::c_int, ) { + use crate::src::looprestoration::LR_HAVE_BOTTOM; + use crate::src::looprestoration::LR_HAVE_TOP; + let mut sumsq_mem: Align16<[int32_t; 27208]> = Align16([0; 27208]); let sumsq: *mut int32_t = &mut *sumsq_mem .0 diff --git a/src/looprestoration_tmpl_8.rs b/src/looprestoration_tmpl_8.rs index 4337b3a2b..314627004 100644 --- a/src/looprestoration_tmpl_8.rs +++ b/src/looprestoration_tmpl_8.rs @@ -1,15 +1,11 @@ +use crate::include::common::bitdepth::BitDepth8; use crate::include::stddef::*; use crate::include::stdint::*; #[cfg(all(feature = "asm", any(target_arch = "arm", target_arch = "aarch64"),))] use crate::src::align::Align16; use ::libc; -use ::libc::size_t; #[cfg(feature = "asm")] use cfg_if::cfg_if; -extern "C" { - fn memcpy(_: *mut libc::c_void, _: *const libc::c_void, _: size_t) -> *mut libc::c_void; - fn memset(_: *mut libc::c_void, _: libc::c_int, _: size_t) -> *mut libc::c_void; -} #[cfg(all(feature = "asm", any(target_arch = "arm", target_arch = "aarch64")))] extern "C" { @@ -133,11 +129,8 @@ use crate::src::tables::dav1d_sgr_x_by_x; pub type pixel = uint8_t; pub type coef = int16_t; use crate::src::looprestoration::LrEdgeFlags; -use crate::src::looprestoration::LR_HAVE_BOTTOM; -use crate::src::looprestoration::LR_HAVE_LEFT; -use crate::src::looprestoration::LR_HAVE_RIGHT; -use crate::src::looprestoration::LR_HAVE_TOP; pub type const_left_pixel_row = *const [pixel; 4]; +use crate::src::looprestoration::padding; use crate::src::looprestoration::Dav1dLoopRestorationDSPContext; use crate::src::looprestoration::LooprestorationParams; @@ -145,179 +138,6 @@ use crate::include::common::intops::iclip; use crate::include::common::intops::iclip_u8; use crate::include::common::intops::imax; use crate::include::common::intops::umin; -#[inline(never)] -unsafe extern "C" fn padding( - mut dst: *mut pixel, - mut p: *const pixel, - stride: ptrdiff_t, - mut left: *const [pixel; 4], - mut lpf: *const pixel, - mut unit_w: libc::c_int, - stripe_h: libc::c_int, - edges: LrEdgeFlags, -) { - let have_left = - (edges as libc::c_uint & LR_HAVE_LEFT as libc::c_int as libc::c_uint != 0) as libc::c_int; - let have_right = - (edges as libc::c_uint & LR_HAVE_RIGHT as libc::c_int as libc::c_uint != 0) as libc::c_int; - unit_w += 3 * have_left + 3 * have_right; - let mut dst_l: *mut pixel = dst.offset((3 * (have_left == 0) as libc::c_int) as isize); - p = p.offset(-((3 * have_left) as isize)); - lpf = lpf.offset(-((3 * have_left) as isize)); - if edges as libc::c_uint & LR_HAVE_TOP as libc::c_int as libc::c_uint != 0 { - let above_1: *const pixel = lpf; - let above_2: *const pixel = above_1.offset(stride as isize); - memcpy( - dst_l as *mut libc::c_void, - above_1 as *const libc::c_void, - unit_w as size_t, - ); - memcpy( - dst_l.offset(390) as *mut libc::c_void, - above_1 as *const libc::c_void, - unit_w as size_t, - ); - memcpy( - dst_l.offset((2 * 390) as isize) as *mut libc::c_void, - above_2 as *const libc::c_void, - unit_w as size_t, - ); - } else { - memcpy( - dst_l as *mut libc::c_void, - p as *const libc::c_void, - unit_w as size_t, - ); - memcpy( - dst_l.offset(390) as *mut libc::c_void, - p as *const libc::c_void, - unit_w as size_t, - ); - memcpy( - dst_l.offset((2 * 390) as isize) as *mut libc::c_void, - p as *const libc::c_void, - unit_w as size_t, - ); - if have_left != 0 { - memcpy( - dst_l as *mut libc::c_void, - &*(*left.offset(0)).as_ptr().offset(1) as *const pixel as *const libc::c_void, - 3, - ); - memcpy( - dst_l.offset(390) as *mut libc::c_void, - &*(*left.offset(0)).as_ptr().offset(1) as *const pixel as *const libc::c_void, - 3, - ); - memcpy( - dst_l.offset((2 * 390) as isize) as *mut libc::c_void, - &*(*left.offset(0)).as_ptr().offset(1) as *const pixel as *const libc::c_void, - 3, - ); - } - } - let mut dst_tl: *mut pixel = dst_l.offset((3 * 390) as isize); - if edges as libc::c_uint & LR_HAVE_BOTTOM as libc::c_int as libc::c_uint != 0 { - let below_1: *const pixel = lpf.offset((6 * stride) as isize); - let below_2: *const pixel = below_1.offset(stride as isize); - memcpy( - dst_tl.offset((stripe_h * 390) as isize) as *mut libc::c_void, - below_1 as *const libc::c_void, - unit_w as size_t, - ); - memcpy( - dst_tl.offset(((stripe_h + 1) * 390) as isize) as *mut libc::c_void, - below_2 as *const libc::c_void, - unit_w as size_t, - ); - memcpy( - dst_tl.offset(((stripe_h + 2) * 390) as isize) as *mut libc::c_void, - below_2 as *const libc::c_void, - unit_w as size_t, - ); - } else { - let src: *const pixel = p.offset(((stripe_h - 1) as isize * stride) as isize); - memcpy( - dst_tl.offset((stripe_h * 390) as isize) as *mut libc::c_void, - src as *const libc::c_void, - unit_w as size_t, - ); - memcpy( - dst_tl.offset(((stripe_h + 1) * 390) as isize) as *mut libc::c_void, - src as *const libc::c_void, - unit_w as size_t, - ); - memcpy( - dst_tl.offset(((stripe_h + 2) * 390) as isize) as *mut libc::c_void, - src as *const libc::c_void, - unit_w as size_t, - ); - if have_left != 0 { - memcpy( - dst_tl.offset((stripe_h * 390) as isize) as *mut libc::c_void, - &*(*left.offset((stripe_h - 1) as isize)).as_ptr().offset(1) as *const pixel - as *const libc::c_void, - 3, - ); - memcpy( - dst_tl.offset(((stripe_h + 1) * 390) as isize) as *mut libc::c_void, - &*(*left.offset((stripe_h - 1) as isize)).as_ptr().offset(1) as *const pixel - as *const libc::c_void, - 3, - ); - memcpy( - dst_tl.offset(((stripe_h + 2) * 390) as isize) as *mut libc::c_void, - &*(*left.offset((stripe_h - 1) as isize)).as_ptr().offset(1) as *const pixel - as *const libc::c_void, - 3, - ); - } - } - let mut j = 0; - while j < stripe_h { - memcpy( - dst_tl.offset((3 * have_left) as isize) as *mut libc::c_void, - p.offset((3 * have_left) as isize) as *const libc::c_void, - (unit_w - 3 * have_left) as size_t, - ); - dst_tl = dst_tl.offset(390); - p = p.offset(stride as isize); - j += 1; - } - if have_right == 0 { - let mut pad: *mut pixel = dst_l.offset(unit_w as isize); - let mut row_last: *mut pixel = &mut *dst_l.offset((unit_w - 1) as isize) as *mut pixel; - let mut j_0 = 0; - while j_0 < stripe_h + 6 { - memset(pad as *mut libc::c_void, *row_last as libc::c_int, 3); - pad = pad.offset(390); - row_last = row_last.offset(390); - j_0 += 1; - } - } - if have_left == 0 { - let mut j_1 = 0; - while j_1 < stripe_h + 6 { - memset(dst as *mut libc::c_void, *dst_l as libc::c_int, 3); - dst = dst.offset(390); - dst_l = dst_l.offset(390); - j_1 += 1; - } - } else { - dst = dst.offset((3 * 390) as isize); - let mut j_2 = 0; - while j_2 < stripe_h { - memcpy( - dst as *mut libc::c_void, - &*(*left.offset(j_2 as isize)).as_ptr().offset(1) as *const pixel - as *const libc::c_void, - 3, - ); - dst = dst.offset(390); - j_2 += 1; - } - }; -} unsafe extern "C" fn wiener_c( mut p: *mut libc::c_void, @@ -342,6 +162,7 @@ unsafe extern "C" fn wiener_c( ) } +// TODO(randompoison): Temporarily public until we can move this to `looprestoration.rs`. unsafe extern "C" fn wiener_rust( mut p: *mut pixel, stride: ptrdiff_t, @@ -354,7 +175,7 @@ unsafe extern "C" fn wiener_rust( ) { let mut tmp: [pixel; 27300] = [0; 27300]; let mut tmp_ptr: *mut pixel = tmp.as_mut_ptr(); - padding(tmp.as_mut_ptr(), p, stride, left, lpf, w, h, edges); + padding::(&mut tmp, p, stride, left, lpf, w, h, edges); let mut hor: [uint16_t; 27300] = [0; 27300]; let mut hor_ptr: *mut uint16_t = hor.as_mut_ptr(); let filter: *const [int16_t; 8] = ((*params).filter.0).as_ptr(); @@ -743,7 +564,7 @@ unsafe extern "C" fn sgr_5x5_rust( ) { let mut tmp: [pixel; 27300] = [0; 27300]; let mut dst: [coef; 24576] = [0; 24576]; - padding(tmp.as_mut_ptr(), p, stride, left, lpf, w, h, edges); + padding::(&mut tmp, p, stride, left, lpf, w, h, edges); selfguided_filter( dst.as_mut_ptr(), tmp.as_mut_ptr(), @@ -804,7 +625,7 @@ unsafe extern "C" fn sgr_3x3_rust( ) { let mut tmp: [pixel; 27300] = [0; 27300]; let mut dst: [coef; 24576] = [0; 24576]; - padding(tmp.as_mut_ptr(), p, stride, left, lpf, w, h, edges); + padding::(&mut tmp, p, stride, left, lpf, w, h, edges); selfguided_filter( dst.as_mut_ptr(), tmp.as_mut_ptr(), @@ -866,7 +687,7 @@ unsafe extern "C" fn sgr_mix_rust( let mut tmp: [pixel; 27300] = [0; 27300]; let mut dst0: [coef; 24576] = [0; 24576]; let mut dst1: [coef; 24576] = [0; 24576]; - padding(tmp.as_mut_ptr(), p, stride, left, lpf, w, h, edges); + padding::(&mut tmp, p, stride, left, lpf, w, h, edges); selfguided_filter( dst0.as_mut_ptr(), tmp.as_mut_ptr(), @@ -994,6 +815,9 @@ unsafe extern "C" fn wiener_filter_neon( params: *const LooprestorationParams, edges: LrEdgeFlags, ) { + use crate::src::looprestoration::LR_HAVE_BOTTOM; + use crate::src::looprestoration::LR_HAVE_TOP; + let filter: *const [int16_t; 8] = (*params).filter.0.as_ptr(); let mut mid: Align16<[int16_t; 68 * 384]> = Align16([0; 68 * 384]); let mut mid_stride: libc::c_int = w + 7 & !7; @@ -1094,6 +918,9 @@ unsafe extern "C" fn dav1d_sgr_filter1_neon( strength: libc::c_int, edges: LrEdgeFlags, ) { + use crate::src::looprestoration::LR_HAVE_BOTTOM; + use crate::src::looprestoration::LR_HAVE_TOP; + let mut sumsq_mem: Align16<[int32_t; 27208]> = Align16([0; 27208]); let sumsq: *mut int32_t = &mut *sumsq_mem .0 @@ -1148,6 +975,9 @@ unsafe extern "C" fn dav1d_sgr_filter2_neon( strength: libc::c_int, edges: LrEdgeFlags, ) { + use crate::src::looprestoration::LR_HAVE_BOTTOM; + use crate::src::looprestoration::LR_HAVE_TOP; + let mut sumsq_mem: Align16<[int32_t; 27208]> = Align16([0; 27208]); let sumsq: *mut int32_t = &mut *sumsq_mem .0