aom: rtc: Use best_sad for pruning palette in nonrd key

From c612b961347bce8c3a9890de14ed5726dc219d83 Mon Sep 17 00:00:00 2001
From: Marco Paniconi <[EMAIL REDACTED]>
Date: Wed, 8 May 2024 23:46:26 -0700
Subject: [PATCH] rtc: Use best_sad for pruning palette in nonrd key

Affects screen mode for speed >= 10: use the best_sad
to skip palette testing, instead of best_dist. This is
more consistent and improves in particular lower Q behavior
for all intra coding.

Change-Id: I832d61f6a6a4fbe39e6978556463bfab08784011
---
 av1/encoder/nonrd_pickmode.c | 12 +++++++-----
 av1/encoder/speed_features.c |  2 +-
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/av1/encoder/nonrd_pickmode.c b/av1/encoder/nonrd_pickmode.c
index 4032d942f..ed64056e4 100644
--- a/av1/encoder/nonrd_pickmode.c
+++ b/av1/encoder/nonrd_pickmode.c
@@ -1648,14 +1648,16 @@ void av1_nonrd_pick_intra_mode(AV1_COMP *cpi, MACROBLOCK *x, RD_STATS *rd_cost,
     }
   }
 
-  const int64_t thresh_dist = cpi->sf.rt_sf.prune_palette_nonrd ? 60000 : 20000;
-  const int64_t best_dist_norm = best_rdc.dist >> (b_width_log2_lookup[bsize] +
-                                                   b_height_log2_lookup[bsize]);
+  const unsigned int thresh_sad = cpi->sf.rt_sf.prune_palette_nonrd ? 100 : 20;
+  const unsigned int best_sad_norm =
+      args.best_sad >>
+      (b_width_log2_lookup[bsize] + b_height_log2_lookup[bsize]);
 
   // Try palette if it's enabled.
   bool try_palette =
-      best_dist_norm > thresh_dist && cpi->oxcf.tool_cfg.enable_palette &&
-      bsize <= BLOCK_16X16 && x->source_variance > 200 &&
+      (!args.prune_mode_based_on_sad || best_sad_norm > thresh_sad) &&
+      cpi->oxcf.tool_cfg.enable_palette && bsize <= BLOCK_16X16 &&
+      x->source_variance > 200 &&
       av1_allow_palette(cpi->common.features.allow_screen_content_tools,
                         mi->bsize);
   if (try_palette) {
diff --git a/av1/encoder/speed_features.c b/av1/encoder/speed_features.c
index 4ddd4903a..671986600 100644
--- a/av1/encoder/speed_features.c
+++ b/av1/encoder/speed_features.c
@@ -1579,6 +1579,7 @@ static void set_rt_speed_feature_framesize_dependent(const AV1_COMP *const cpi,
       sf->rt_sf.nonrd_aggressive_skip = 1;
       sf->rt_sf.thresh_active_maps_skip_lf_cdef = 90;
       sf->rt_sf.hybrid_intra_pickmode = 0;
+      sf->rt_sf.prune_intra_mode_using_best_sad_so_far = true;
     }
     if (speed >= 11) {
       sf->rt_sf.skip_lf_screen = 2;
@@ -1588,7 +1589,6 @@ static void set_rt_speed_feature_framesize_dependent(const AV1_COMP *const cpi,
       sf->rt_sf.increase_color_thresh_palette = 0;
       sf->rt_sf.prune_h_pred_using_best_mode_so_far = true;
       sf->rt_sf.enable_intra_mode_pruning_using_neighbors = true;
-      sf->rt_sf.prune_intra_mode_using_best_sad_so_far = true;
     }
     sf->rt_sf.skip_newmv_flat_blocks_screen = 1;
     sf->rt_sf.use_idtx_nonrd = 1;