From: Sarah Parker <sarahparker@google.com>
Date: Tue, 18 Oct 2016 23:27:04 +0000 (-0700)
Subject: Add clamping to parameter search
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=081783dc671240054acdee61d8ff8c9ff064e8de;p=libvpx

Add clamping to parameter search

This fixes mismatches due to overflowing low precision parameters.

Change-Id: If34e39ca7ab0adc9688d46b0e8ed62cbb6fdaff0
---

diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 773e368c0..3f93d5369 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -4447,7 +4447,9 @@ static void refine_integerized_param(WarpedMotionParams *wm,
     best_param = curr_param;
     for (i = 0; i < n_refinements; i++) {
       // look to the left
-      *param = curr_param - step;
+      *param =
+          (int16_t)clamp(curr_param - step, p < 2 ? GM_TRANS_MIN : GM_ALPHA_MIN,
+                         p < 2 ? GM_TRANS_MAX : GM_ALPHA_MAX);
       step_error =
           av1_warp_erroradv(wm,
 #if CONFIG_AOM_HIGHBITDEPTH
@@ -4464,7 +4466,9 @@ static void refine_integerized_param(WarpedMotionParams *wm,
       }
 
       // look to the right
-      *param = curr_param + step;
+      *param =
+          (int16_t)clamp(curr_param + step, p < 2 ? GM_TRANS_MIN : GM_ALPHA_MIN,
+                         p < 2 ? GM_TRANS_MAX : GM_ALPHA_MAX);
       step_error =
           av1_warp_erroradv(wm,
 #if CONFIG_AOM_HIGHBITDEPTH