aom: rtc: Adjust color_thresh_palette for nonrd keyframe

From 06f3b3bae25956313f5dd51cedf39b2e04713264 Mon Sep 17 00:00:00 2001
From: Marco Paniconi <[EMAIL REDACTED]>
Date: Thu, 9 May 2024 22:26:27 -0700
Subject: [PATCH] rtc: Adjust color_thresh_palette for nonrd keyframe

For keyframes with hybrid_intra_pickmode = 0:
reduce color_thresh_palette down, to exit palette search
early when it's likely not the best mode.

bdrate on rtc_screen for speed 11, all-intra with CQ mode:
avg_psnr/ovr_psnr/ssim:-6.3, -5.3, -7.1
Average Instruction count reduction ~17%.

For the default IPPP CBR mode:
avg_psnr/ovr_psnr/ssim: 0.153, 0.619, 0.427
Some loss on screen_recording_crd clip, that will
be looked into.

Change-Id: I2fd8acb674d09c34629f08f6e593ec97a807d9d4
---
 av1/encoder/block.h          | 3 +++
 av1/encoder/encodeframe.c    | 1 +
 av1/encoder/nonrd_pickmode.c | 2 ++
 av1/encoder/palette.c        | 2 +-
 4 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/av1/encoder/block.h b/av1/encoder/block.h
index 1baf3f942..9bee0b8d0 100644
--- a/av1/encoder/block.h
+++ b/av1/encoder/block.h
@@ -1325,6 +1325,9 @@ typedef struct macroblock {
   //! Coding block distortion value for uv/color, minimum over the inter modes.
   int64_t min_dist_inter_uv;
 
+  //! Threshold on the number of colors for testing palette mode.
+  int color_palette_thresh;
+
   //! The buffer used by search_tx_type() to swap dqcoeff in macroblockd_plane
   // so we can keep dqcoeff of the best tx_type.
   tran_low_t *dqcoeff_buf;
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 07382eb6c..4c178b18c 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -1211,6 +1211,7 @@ static AOM_INLINE void encode_sb_row(AV1_COMP *cpi, ThreadData *td,
     x->sb_me_partition = 0;
     x->sb_me_mv.as_int = 0;
     x->sb_force_fixed_part = 1;
+    x->color_palette_thresh = 64;
 
     if (cpi->oxcf.mode == ALLINTRA) {
       x->intra_sb_rdmult_modifier = 128;
diff --git a/av1/encoder/nonrd_pickmode.c b/av1/encoder/nonrd_pickmode.c
index d08e9c0ac..7b204c78a 100644
--- a/av1/encoder/nonrd_pickmode.c
+++ b/av1/encoder/nonrd_pickmode.c
@@ -1668,6 +1668,8 @@ void av1_nonrd_pick_intra_mode(AV1_COMP *cpi, MACROBLOCK *x, RD_STATS *rd_cost,
   if (try_palette) {
     const TxfmSearchInfo *txfm_info = &x->txfm_search_info;
     const unsigned int intra_ref_frame_cost = 0;
+    x->color_palette_thresh = (best_sad_norm < 500) ? 32 : 64;
+
     // Search palette mode for Luma plane in intra frame.
     av1_search_palette_mode_luma(cpi, x, bsize, intra_ref_frame_cost, ctx,
                                  &this_rdc, best_rdc.rdcost);
diff --git a/av1/encoder/palette.c b/av1/encoder/palette.c
index 45b56199c..6ae1c6cf6 100644
--- a/av1/encoder/palette.c
+++ b/av1/encoder/palette.c
@@ -564,7 +564,7 @@ void av1_rd_pick_palette_intra_sby(
   }
 
   uint8_t *const color_map = xd->plane[0].color_index_map;
-  int color_thresh_palette = 64;
+  int color_thresh_palette = x->color_palette_thresh;
   // Allow for larger color_threshold for palette search, based on color,
   // scene_change, and block source variance.
   // Since palette is Y based, only allow larger threshold if block