Fix rounding in ARNR calculation

The rounding of the ARNR filter output prior to
normalization by the filter strength was incorrect
when strength = 0.

In this case 1 << (strength - 1) would not create the
required rounding of 0, rather it would outrange. This
patch fixes this issue.

Change-Id: I771809ba34d6052b17d34c870ea11ff67b418dab

diff --git a/vp8/encoder/temporal_filter.c b/vp8/encoder/temporal_filter.c
index 513b2bfead0c8b100e318b2cb1111af0b26fc617..4dc0d95922112fdb97c06accb8663f11f372aa20 100644 (file)
--- a/vp8/encoder/temporal_filter.c
+++ b/vp8/encoder/temporal_filter.c
@@ -98,6 +98,7 @@ void vp8_temporal_filter_apply_c
     unsigned int i, j, k;
     int modifier;
     int byte = 0;
+    const int rounding = strength > 0 ? 1 << (strength - 1) : 0;
 
     for (i = 0,k = 0; i < block_size; i++)
     {
@@ -114,7 +115,7 @@ void vp8_temporal_filter_apply_c
              */
             modifier  *= modifier;
             modifier  *= 3;
-            modifier  += 1 << (strength - 1);
+            modifier  += rounding;
             modifier >>= strength;
 
             if (modifier > 16)

diff --git a/vp9/encoder/vp9_temporal_filter.c b/vp9/encoder/vp9_temporal_filter.c
index a5234cd9e40b8e314362ba1d15ddeab9a7b9319f..6eff200801425a957a7239969c4cff4dec867371 100644 (file)
--- a/vp9/encoder/vp9_temporal_filter.c
+++ b/vp9/encoder/vp9_temporal_filter.c
@@ -99,6 +99,7 @@ void vp9_temporal_filter_apply_c(uint8_t *frame1,
   unsigned int i, j, k;
   int modifier;
   int byte = 0;
+  const int rounding = strength > 0 ? 1 << (strength - 1) : 0;
 
   for (i = 0, k = 0; i < block_size; i++) {
     for (j = 0; j < block_size; j++, k++) {
@@ -111,7 +112,7 @@ void vp9_temporal_filter_apply_c(uint8_t *frame1,
       // modifier =  (int)roundf(coeff > 16 ? 0 : 16-coeff);
       modifier  *= modifier;
       modifier  *= 3;
-      modifier  += 1 << (strength - 1);
+      modifier  += rounding;
       modifier >>= strength;
 
       if (modifier > 16)