aom: Global motion: Move some checks earlier

From 4564cb71c567597d726bbef109cf2f017b235925 Mon Sep 17 00:00:00 2001
From: Rachel Barker <[EMAIL REDACTED]>
Date: Tue, 19 Dec 2023 13:40:29 +0000
Subject: [PATCH] Global motion: Move some checks earlier

* Add an early-exit criterion to avoid doing some pointless work
  when we get a non-warpable model

* Check the signaling cost of a model before comparing it to the
  current best error ratio

* Remove some dead code

No change to encoder output, noise-level speed impact

Change-Id: Ic95d2edfe1f4baaa06a661a1e42072a293485778
---
 av1/encoder/global_motion.c        |  5 ++++
 av1/encoder/global_motion_facade.c | 39 +++++++++---------------------
 2 files changed, 16 insertions(+), 28 deletions(-)

diff --git a/av1/encoder/global_motion.c b/av1/encoder/global_motion.c
index 73910de121..0847a39e68 100644
--- a/av1/encoder/global_motion.c
+++ b/av1/encoder/global_motion.c
@@ -541,6 +541,11 @@ int64_t av1_refine_integerized_param(
   }
 
   wm->wmtype = get_wmtype(wm);
+  // Recompute shear params for the refined model
+  // This should never fail, because we only ever consider warp-able models
+  if (!av1_get_shear_params(wm)) {
+    assert(0);
+  }
   return best_error;
 }
 
diff --git a/av1/encoder/global_motion_facade.c b/av1/encoder/global_motion_facade.c
index 02a4e70ed3..4679a2cb83 100644
--- a/av1/encoder/global_motion_facade.c
+++ b/av1/encoder/global_motion_facade.c
@@ -115,6 +115,9 @@ static AOM_INLINE void compute_global_motion_for_ref_frame(
       WarpedMotionParams tmp_wm_params;
       av1_convert_model_to_params(motion_models[i].params, &tmp_wm_params);
 
+      // Check that the generated model is warp-able
+      if (!av1_get_shear_params(&tmp_wm_params)) continue;
+
       // Skip models that we won't use (IDENTITY or TRANSLATION)
       //
       // For IDENTITY type models, we don't need to evaluate anything because
@@ -151,6 +154,14 @@ static AOM_INLINE void compute_global_motion_for_ref_frame(
 
       double erroradvantage = (double)warp_error / ref_frame_error;
 
+      // Check that the model signaling cost is not too high
+      if (!av1_is_enough_erroradvantage(
+              erroradvantage,
+              gm_get_params_cost(&tmp_wm_params, ref_params,
+                                 cm->features.allow_high_precision_mv))) {
+        continue;
+      }
+
       if (erroradvantage < best_erroradv) {
         best_erroradv = erroradvantage;
         // Save the wm_params modified by
@@ -161,34 +172,6 @@ static AOM_INLINE void compute_global_motion_for_ref_frame(
       }
     }
   }
-
-  if (!av1_get_shear_params(&cm->global_motion[frame]))
-    cm->global_motion[frame] = default_warp_params;
-
-#if 0
-  // We never choose translational models, so this code is disabled
-  if (cm->global_motion[frame].wmtype == TRANSLATION) {
-    cm->global_motion[frame].wmmat[0] =
-        convert_to_trans_prec(cm->features.allow_high_precision_mv,
-                              cm->global_motion[frame].wmmat[0]) *
-        GM_TRANS_ONLY_DECODE_FACTOR;
-    cm->global_motion[frame].wmmat[1] =
-        convert_to_trans_prec(cm->features.allow_high_precision_mv,
-                              cm->global_motion[frame].wmmat[1]) *
-        GM_TRANS_ONLY_DECODE_FACTOR;
-  }
-#endif
-
-  if (cm->global_motion[frame].wmtype == IDENTITY) return;
-
-  // If the best error advantage found doesn't meet the threshold for
-  // this motion type, revert to IDENTITY.
-  if (!av1_is_enough_erroradvantage(
-          best_erroradv,
-          gm_get_params_cost(&cm->global_motion[frame], ref_params,
-                             cm->features.allow_high_precision_mv))) {
-    cm->global_motion[frame] = default_warp_params;
-  }
 }
 
 // Computes global motion for the given reference frame.