aom: Add 4-tap specialization of aom_highbd_convolve8_vert_sve

From cd0a8b7cc087bd02227d4af7001972ef244e3988 Mon Sep 17 00:00:00 2001
From: Salome Thirot <[EMAIL REDACTED]>
Date: Fri, 23 Feb 2024 17:57:08 +0000
Subject: [PATCH] Add 4-tap specialization of aom_highbd_convolve8_vert_sve

This function is run mostly with 2-tap and 4-tap filters, so add a
specialised path that allows to use half as many instructions as the
generic 8-tap path for these filters.

Change-Id: Ib273a1b8906541946edc9c11420319a1a66fe390
---
 aom_dsp/arm/highbd_convolve8_sve.c | 166 ++++++++++++++++++++++++++---
 1 file changed, 150 insertions(+), 16 deletions(-)

diff --git a/aom_dsp/arm/highbd_convolve8_sve.c b/aom_dsp/arm/highbd_convolve8_sve.c
index 9830b7e5d..e57c41a0b 100644
--- a/aom_dsp/arm/highbd_convolve8_sve.c
+++ b/aom_dsp/arm/highbd_convolve8_sve.c
@@ -398,22 +398,10 @@ static INLINE uint16x8_t highbd_convolve8_8_v(int16x8_t samples_lo[4],
   return vminq_u16(res, max);
 }
 
-void aom_highbd_convolve8_vert_sve(const uint8_t *src8, ptrdiff_t src_stride,
-                                   uint8_t *dst8, ptrdiff_t dst_stride,
-                                   const int16_t *filter_x, int x_step_q4,
-                                   const int16_t *filter_y, int y_step_q4,
-                                   int width, int height, int bd) {
-  assert(y_step_q4 == 16);
-  assert(w >= 4 && h >= 4);
-  (void)filter_x;
-  (void)y_step_q4;
-  (void)x_step_q4;
-
-  uint16_t *src = CONVERT_TO_SHORTPTR(src8);
-  uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
-
-  src -= (SUBPEL_TAPS / 2 - 1) * src_stride;
-
+static INLINE void highbd_convolve8_vert_8tap_sve(
+    const uint16_t *src, ptrdiff_t src_stride, uint16_t *dst,
+    ptrdiff_t dst_stride, const int16_t *filter_y, int width, int height,
+    int bd) {
   const int16x8_t y_filter = vld1q_s16(filter_y);
 
   uint8x16_t merge_block_tbl[3];
@@ -545,3 +533,149 @@ void aom_highbd_convolve8_vert_sve(const uint8_t *src8, ptrdiff_t src_stride,
     } while (width != 0);
   }
 }
+
+static INLINE uint16x4_t highbd_convolve4_4_v(int16x8_t s[2], int16x8_t filter,
+                                              uint16x4_t max) {
+  int64x2_t sum01 = aom_svdot_lane_s16(vdupq_n_s64(0), s[0], filter, 0);
+  int64x2_t sum23 = aom_svdot_lane_s16(vdupq_n_s64(0), s[1], filter, 0);
+
+  int32x4_t sum0123 = vcombine_s32(vmovn_s64(sum01), vmovn_s64(sum23));
+  uint16x4_t res = vqrshrun_n_s32(sum0123, FILTER_BITS);
+
+  return vmin_u16(res, max);
+}
+
+static INLINE uint16x8_t highbd_convolve4_8_v(int16x8_t s[4], int16x8_t filter,
+                                              uint16x8_t max) {
+  int64x2_t sum01 = aom_svdot_lane_s16(vdupq_n_s64(0), s[0], filter, 0);
+  int64x2_t sum23 = aom_svdot_lane_s16(vdupq_n_s64(0), s[1], filter, 0);
+  int64x2_t sum45 = aom_svdot_lane_s16(vdupq_n_s64(0), s[2], filter, 0);
+  int64x2_t sum67 = aom_svdot_lane_s16(vdupq_n_s64(0), s[3], filter, 0);
+
+  int32x4_t s0123 = vcombine_s32(vmovn_s64(sum01), vmovn_s64(sum23));
+  int32x4_t s4567 = vcombine_s32(vmovn_s64(sum45), vmovn_s64(sum67));
+
+  uint16x8_t res = vcombine_u16(vqrshrun_n_s32(s0123, FILTER_BITS),
+                                vqrshrun_n_s32(s4567, FILTER_BITS));
+
+  return vminq_u16(res, max);
+}
+
+static INLINE void highbd_convolve8_vert_4tap_sve(
+    const uint16_t *src, ptrdiff_t src_stride, uint16_t *dst,
+    ptrdiff_t dst_stride, const int16_t *filter_y, int width, int height,
+    int bd) {
+  const int16x8_t y_filter =
+      vcombine_s16(vld1_s16(filter_y + 2), vdup_n_s16(0));
+
+  uint8x16_t merge_block_tbl[3];
+  merge_block_tbl[0] = vld1q_u8(kDotProdMergeBlockTbl);
+  merge_block_tbl[1] = vld1q_u8(kDotProdMergeBlockTbl + 16);
+  merge_block_tbl[2] = vld1q_u8(kDotProdMergeBlockTbl + 32);
+
+  if (width == 4) {
+    const uint16x4_t max = vdup_n_u16((1 << bd) - 1);
+    int16_t *s = (int16_t *)src;
+
+    int16x4_t s0, s1, s2;
+    load_s16_4x3(s, src_stride, &s0, &s1, &s2);
+    s += 3 * src_stride;
+
+    do {
+      int16x4_t s3, s4, s5, s6;
+      load_s16_4x4(s, src_stride, &s3, &s4, &s5, &s6);
+
+      // This operation combines a conventional transpose and the sample permute
+      // required before computing the dot product.
+      int16x8_t s0123[2], s1234[2], s2345[2], s3456[2];
+      transpose_concat_4x4(s0, s1, s2, s3, s0123);
+      transpose_concat_4x4(s1, s2, s3, s4, s1234);
+      transpose_concat_4x4(s2, s3, s4, s5, s2345);
+      transpose_concat_4x4(s3, s4, s5, s6, s3456);
+
+      uint16x4_t d0 = highbd_convolve4_4_v(s0123, y_filter, max);
+      uint16x4_t d1 = highbd_convolve4_4_v(s1234, y_filter, max);
+      uint16x4_t d2 = highbd_convolve4_4_v(s2345, y_filter, max);
+      uint16x4_t d3 = highbd_convolve4_4_v(s3456, y_filter, max);
+
+      store_u16_4x4(dst, dst_stride, d0, d1, d2, d3);
+
+      // Shuffle everything up four rows.
+      s0 = s4;
+      s1 = s5;
+      s2 = s6;
+
+      s += 4 * src_stride;
+      dst += 4 * dst_stride;
+      height -= 4;
+    } while (height != 0);
+  } else {
+    const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
+    do {
+      int h = height;
+      int16_t *s = (int16_t *)src;
+      uint16_t *d = dst;
+
+      int16x8_t s0, s1, s2;
+      load_s16_8x3(s, src_stride, &s0, &s1, &s2);
+      s += 3 * src_stride;
+
+      do {
+        int16x8_t s3, s4, s5, s6;
+        load_s16_8x4(s, src_stride, &s3, &s4, &s5, &s6);
+
+        // This operation combines a conventional transpose and the sample
+        // permute required before computing the dot product.
+        int16x8_t s0123[4], s1234[4], s2345[4], s3456[4];
+        transpose_concat_8x4(s0, s1, s2, s3, s0123);
+        transpose_concat_8x4(s1, s2, s3, s4, s1234);
+        transpose_concat_8x4(s2, s3, s4, s5, s2345);
+        transpose_concat_8x4(s3, s4, s5, s6, s3456);
+
+        uint16x8_t d0 = highbd_convolve4_8_v(s0123, y_filter, max);
+        uint16x8_t d1 = highbd_convolve4_8_v(s1234, y_filter, max);
+        uint16x8_t d2 = highbd_convolve4_8_v(s2345, y_filter, max);
+        uint16x8_t d3 = highbd_convolve4_8_v(s3456, y_filter, max);
+
+        store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+
+        // Shuffle everything up four rows.
+        s0 = s4;
+        s1 = s5;
+        s2 = s6;
+
+        s += 4 * src_stride;
+        d += 4 * dst_stride;
+        h -= 4;
+      } while (h != 0);
+      src += 8;
+      dst += 8;
+      width -= 8;
+    } while (width != 0);
+  }
+}
+
+void aom_highbd_convolve8_vert_sve(const uint8_t *src8, ptrdiff_t src_stride,
+                                   uint8_t *dst8, ptrdiff_t dst_stride,
+                                   const int16_t *filter_x, int x_step_q4,
+                                   const int16_t *filter_y, int y_step_q4,
+                                   int width, int height, int bd) {
+  assert(y_step_q4 == 16);
+  assert(w >= 4 && h >= 4);
+  (void)filter_x;
+  (void)y_step_q4;
+  (void)x_step_q4;
+
+  const uint16_t *src = CONVERT_TO_SHORTPTR(src8);
+  uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
+
+  src -= (SUBPEL_TAPS / 2 - 1) * src_stride;
+
+  if (get_filter_taps_convolve8(filter_y) <= 4) {
+    highbd_convolve8_vert_4tap_sve(src + 2 * src_stride, src_stride, dst,
+                                   dst_stride, filter_y, width, height, bd);
+  } else {
+    highbd_convolve8_vert_8tap_sve(src, src_stride, dst, dst_stride, filter_y,
+                                   width, height, bd);
+  }
+}