aom: Add SVE2 implementation of HBD convolve_y_sr for 12-tap filters

From b28f21b032044b6ad51104ec4fad26f6223fe77a Mon Sep 17 00:00:00 2001
From: Salome Thirot <[EMAIL REDACTED]>
Date: Mon, 26 Feb 2024 14:12:17 +0000
Subject: [PATCH] Add SVE2 implementation of HBD convolve_y_sr for 12-tap
 filters

Add SVE2 implementation of av1_highbd_convolve_y_sr for 12-tap filters.

Change-Id: Ia5bd02068722796e31f4c99e9e6bdf71eb646539
---
 av1/common/arm/highbd_convolve_sve2.c | 127 +++++++++++++++++++++++++-
 1 file changed, 126 insertions(+), 1 deletion(-)

diff --git a/av1/common/arm/highbd_convolve_sve2.c b/av1/common/arm/highbd_convolve_sve2.c
index 95e697314..caa8661df 100644
--- a/av1/common/arm/highbd_convolve_sve2.c
+++ b/av1/common/arm/highbd_convolve_sve2.c
@@ -477,6 +477,125 @@ static INLINE void aom_tbl2x2_s16(int16x8_t t0[2], int16x8_t t1[2],
   res[1] = aom_tbl2_s16(t0[1], t1[1], tbl);
 }
 
+static INLINE uint16x4_t highbd_convolve12_4_y(int16x8_t s0[2], int16x8_t s1[2],
+                                               int16x8_t s2[2],
+                                               int16x8_t filter_0_7,
+                                               int16x8_t filter_4_11,
+                                               uint16x4_t max) {
+  int64x2_t sum[2];
+
+  sum[0] = aom_svdot_lane_s16(vdupq_n_s64(0), s0[0], filter_0_7, 0);
+  sum[0] = aom_svdot_lane_s16(sum[0], s1[0], filter_0_7, 1);
+  sum[0] = aom_svdot_lane_s16(sum[0], s2[0], filter_4_11, 1);
+
+  sum[1] = aom_svdot_lane_s16(vdupq_n_s64(0), s0[1], filter_0_7, 0);
+  sum[1] = aom_svdot_lane_s16(sum[1], s1[1], filter_0_7, 1);
+  sum[1] = aom_svdot_lane_s16(sum[1], s2[1], filter_4_11, 1);
+
+  int32x4_t res_s32 = vcombine_s32(vmovn_s64(sum[0]), vmovn_s64(sum[1]));
+
+  uint16x4_t res = vqrshrun_n_s32(res_s32, FILTER_BITS);
+
+  return vmin_u16(res, max);
+}
+
+static INLINE void highbd_convolve_y_sr_12tap_sve2(
+    const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride,
+    int width, int height, const int16_t *y_filter_ptr, int bd) {
+  const int16x8_t y_filter_0_7 = vld1q_s16(y_filter_ptr);
+  const int16x8_t y_filter_4_11 = vld1q_s16(y_filter_ptr + 4);
+
+  uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kDotProdMergeBlockTbl);
+  // Scale indices by size of the true vector length to avoid reading from an
+  // 'undefined' portion of a vector on a system with SVE vectors > 128-bit.
+  uint16x8_t correction0 =
+      vreinterpretq_u16_u64(vdupq_n_u64(svcnth() * 0x0001000000000000ULL));
+  merge_block_tbl.val[0] = vaddq_u16(merge_block_tbl.val[0], correction0);
+
+  uint16x8_t correction1 =
+      vreinterpretq_u16_u64(vdupq_n_u64(svcnth() * 0x0001000100000000ULL));
+  merge_block_tbl.val[1] = vaddq_u16(merge_block_tbl.val[1], correction1);
+
+  uint16x8_t correction2 =
+      vreinterpretq_u16_u64(vdupq_n_u64(svcnth() * 0x0001000100010000ULL));
+  merge_block_tbl.val[2] = vaddq_u16(merge_block_tbl.val[2], correction2);
+
+  const uint16x4_t max = vdup_n_u16((1 << bd) - 1);
+
+  do {
+    int16_t *s = (int16_t *)src;
+    uint16_t *d = dst;
+    int h = height;
+
+    int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, sA;
+    load_s16_4x11(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7, &s8,
+                  &s9, &sA);
+    s += 11 * src_stride;
+
+    int16x8_t s0123[2], s1234[2], s2345[2], s3456[2], s4567[2], s5678[2],
+        s6789[2], s789A[2];
+    transpose_concat_4x4(s0, s1, s2, s3, s0123);
+    transpose_concat_4x4(s1, s2, s3, s4, s1234);
+    transpose_concat_4x4(s2, s3, s4, s5, s2345);
+    transpose_concat_4x4(s3, s4, s5, s6, s3456);
+    transpose_concat_4x4(s4, s5, s6, s7, s4567);
+    transpose_concat_4x4(s5, s6, s7, s8, s5678);
+    transpose_concat_4x4(s6, s7, s8, s9, s6789);
+    transpose_concat_4x4(s7, s8, s9, sA, s789A);
+
+    do {
+      int16x4_t sB, sC, sD, sE;
+      load_s16_4x4(s, src_stride, &sB, &sC, &sD, &sE);
+
+      int16x8_t s89AB[2], s9ABC[2], sABCD[2], sBCDE[2];
+      transpose_concat_4x4(sB, sC, sD, sE, sBCDE);
+
+      // Use the above transpose and reuse data from the previous loop to get
+      // the rest.
+      aom_tbl2x2_s16(s789A, sBCDE, merge_block_tbl.val[0], s89AB);
+      aom_tbl2x2_s16(s789A, sBCDE, merge_block_tbl.val[1], s9ABC);
+      aom_tbl2x2_s16(s789A, sBCDE, merge_block_tbl.val[2], sABCD);
+
+      uint16x4_t d0 = highbd_convolve12_4_y(s0123, s4567, s89AB, y_filter_0_7,
+                                            y_filter_4_11, max);
+      uint16x4_t d1 = highbd_convolve12_4_y(s1234, s5678, s9ABC, y_filter_0_7,
+                                            y_filter_4_11, max);
+      uint16x4_t d2 = highbd_convolve12_4_y(s2345, s6789, sABCD, y_filter_0_7,
+                                            y_filter_4_11, max);
+      uint16x4_t d3 = highbd_convolve12_4_y(s3456, s789A, sBCDE, y_filter_0_7,
+                                            y_filter_4_11, max);
+
+      store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
+
+      // Prepare block for next iteration - re-using as much as possible.
+      // Shuffle everything up four rows.
+      s0123[0] = s4567[0];
+      s0123[1] = s4567[1];
+      s1234[0] = s5678[0];
+      s1234[1] = s5678[1];
+      s2345[0] = s6789[0];
+      s2345[1] = s6789[1];
+      s3456[0] = s789A[0];
+      s3456[1] = s789A[1];
+      s4567[0] = s89AB[0];
+      s4567[1] = s89AB[1];
+      s5678[0] = s9ABC[0];
+      s5678[1] = s9ABC[1];
+      s6789[0] = sABCD[0];
+      s6789[1] = sABCD[1];
+      s789A[0] = sBCDE[0];
+      s789A[1] = sBCDE[1];
+
+      s += 4 * src_stride;
+      d += 4 * dst_stride;
+      h -= 4;
+    } while (h != 0);
+    src += 4;
+    dst += 4;
+    width -= 4;
+  } while (width != 0);
+}
+
 static INLINE uint16x4_t highbd_convolve8_4_y(int16x8_t samples_lo[2],
                                               int16x8_t samples_hi[2],
                                               int16x8_t filter,
@@ -793,7 +912,7 @@ void av1_highbd_convolve_y_sr_sve2(const uint16_t *src, int src_stride,
   }
   const int y_filter_taps = get_filter_tap(filter_params_y, subpel_y_qn);
 
-  if (y_filter_taps != 8 && y_filter_taps != 4) {
+  if (y_filter_taps == 6) {
     av1_highbd_convolve_y_sr_neon(src, src_stride, dst, dst_stride, w, h,
                                   filter_params_y, subpel_y_qn, bd);
     return;
@@ -805,6 +924,12 @@ void av1_highbd_convolve_y_sr_sve2(const uint16_t *src, int src_stride,
 
   src -= vert_offset * src_stride;
 
+  if (y_filter_taps > 8) {
+    highbd_convolve_y_sr_12tap_sve2(src, src_stride, dst, dst_stride, w, h,
+                                    y_filter_ptr, bd);
+    return;
+  }
+
   if (y_filter_taps == 4) {
     highbd_convolve_y_sr_4tap_sve2(src + 2 * src_stride, src_stride, dst,
                                    dst_stride, w, h, y_filter_ptr, bd);