[libav-devel] [PATCH] swscale: fix off-by-one in second coefficient in bilinear filters.

[Luca Barbato]

On 13/04/12 14:37, Ronald S. Bultje wrote:
> From: "Ronald S. Bultje" <rsbultje at gmail.com>
> 
> If coefficient A is 12-bits xixed-point number "X", then the other

fixed?

> coefficient is (1 << 12) - X, not (1 << 12) - X - 1.



> ---
>  libswscale/output.c |   14 +++++++-------
>  1 file changed, 7 insertions(+), 7 deletions(-)
> 
> diff --git a/libswscale/output.c b/libswscale/output.c
> index 5bc25fc..c0f9b36 100644
> --- a/libswscale/output.c
> +++ b/libswscale/output.c
> @@ -353,7 +353,7 @@ yuv2mono_2_c_template(SwsContext *c, const int16_t *buf[2],
>  {
>      const int16_t *buf0  = buf[0],  *buf1  = buf[1];
>      const uint8_t * const d128 = dither_8x8_220[y & 7];
> -    int  yalpha1 = 4095 - yalpha;
> +    int  yalpha1 = 4096 - yalpha;
>      int i;
>  
>      for (i = 0; i < dstW; i += 8) {
> @@ -505,8 +505,8 @@ yuv2422_2_c_template(SwsContext *c, const int16_t *buf[2],
>      const int16_t *buf0  = buf[0],  *buf1  = buf[1],
>                    *ubuf0 = ubuf[0], *ubuf1 = ubuf[1],
>                    *vbuf0 = vbuf[0], *vbuf1 = vbuf[1];
> -    int  yalpha1 = 4095 - yalpha;
> -    int uvalpha1 = 4095 - uvalpha;
> +    int  yalpha1 = 4096 - yalpha;
> +    int uvalpha1 = 4096 - uvalpha;
>      int i;
>  
>      for (i = 0; i < ((dstW + 1) >> 1); i++) {
> @@ -648,8 +648,8 @@ yuv2rgb48_2_c_template(SwsContext *c, const int32_t *buf[2],
>      const int32_t *buf0  = buf[0],  *buf1  = buf[1],
>                    *ubuf0 = ubuf[0], *ubuf1 = ubuf[1],
>                    *vbuf0 = vbuf[0], *vbuf1 = vbuf[1];
> -    int  yalpha1 = 4095 - yalpha;
> -    int uvalpha1 = 4095 - uvalpha;
> +    int  yalpha1 = 4096 - yalpha;
> +    int uvalpha1 = 4096 - uvalpha;
>      int i;
>  
>      for (i = 0; i < ((dstW + 1) >> 1); i++) {
> @@ -995,8 +995,8 @@ yuv2rgb_2_c_template(SwsContext *c, const int16_t *buf[2],
>                    *vbuf0 = vbuf[0], *vbuf1 = vbuf[1],
>                    *abuf0 = hasAlpha ? abuf[0] : NULL,
>                    *abuf1 = hasAlpha ? abuf[1] : NULL;
> -    int  yalpha1 = 4095 - yalpha;
> -    int uvalpha1 = 4095 - uvalpha;
> +    int  yalpha1 = 4096 - yalpha;
> +    int uvalpha1 = 4096 - uvalpha;
>      int i;
>  
>      for (i = 0; i < ((dstW + 1) >> 1); i++) {


seems ok, what about making it a #define?

lu

-- 

Luca Barbato
Gentoo/linux
http://dev.gentoo.org/~lu_zero