From 24aa59cc51a19fc7618c910c7825bd163fbb291b Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Thu, 6 Oct 2016 11:36:08 -0700 Subject: [PATCH] Fix left shift of negative integer in hbd directional predictors Change-Id: Id78139ae2dfa2d521bd50618b4a81cf24e09e391 --- av1/common/reconintra.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/av1/common/reconintra.c b/av1/common/reconintra.c index 3ebfc973c..3b2221f3f 100644 --- a/av1/common/reconintra.c +++ b/av1/common/reconintra.c @@ -868,7 +868,7 @@ static void highbd_dr_prediction_z1(uint16_t *dst, ptrdiff_t stride, int bs, for (c = 0; c < bs; ++c) { x = (c << 8) - y * dx; base = x >> 8; - shift = x - (base << 8); + shift = x & 0xFF; if (base < 2 * bs - 1) { val = highbd_intra_subpel_interp(base, shift, above, 0, 2 * bs - 1, filter_type); @@ -897,7 +897,7 @@ static void highbd_dr_prediction_z2(uint16_t *dst, ptrdiff_t stride, int bs, x = (c << 8) - y * dx; base = x >> 8; if (base >= -1) { - shift = x - (base << 8); + shift = x & 0xFF; val = highbd_intra_subpel_interp(base, shift, above, -1, bs - 1, filter_type); } else { @@ -905,7 +905,7 @@ static void highbd_dr_prediction_z2(uint16_t *dst, ptrdiff_t stride, int bs, y = (r << 8) - x * dy; base = y >> 8; if (base >= 0) { - shift = y - (base << 8); + shift = y & 0xFF; val = highbd_intra_subpel_interp(base, shift, left, 0, bs - 1, filter_type); } else { @@ -935,7 +935,7 @@ static void highbd_dr_prediction_z3(uint16_t *dst, ptrdiff_t stride, int bs, x = c + 1; y = (r << 8) - x * dy; base = y >> 8; - shift = y - (base << 8); + shift = y & 0xFF; if (base < 2 * bs - 1) { val = highbd_intra_subpel_interp(base, shift, left, 0, 2 * bs - 1, filter_type); -- 2.50.1