[libav-commits] imgconvert: avoid undefined left shift in avcodec_find_best_pix_fmt

Janne Grunau git at libav.org
Sat Oct 6 09:43:30 CEST 2012


Module: libav
Branch: release/0.7
Commit: 8812b5f164109553f009ce385e17a1af16b6ea53

Author:    Janne Grunau <janne-libav at jannau.net>
Committer: Anton Khirnov <anton at khirnov.net>
Date:      Mon Jul  2 10:46:39 2012 +0200

imgconvert: avoid undefined left shift in avcodec_find_best_pix_fmt

CC: libav-stable at libav.org
(cherry picked from commit 39bb27bf79bc4c2d8beaed637a14176264cb1916)

Signed-off-by: Anton Khirnov <anton at khirnov.net>
(cherry picked from commit 7a7229b52d1900279041991fadbd29b27e8dfe95)

Signed-off-by: Anton Khirnov <anton at khirnov.net>

---

 libavcodec/imgconvert.c |    3 ++-
 1 files changed, 2 insertions(+), 1 deletions(-)

diff --git a/libavcodec/imgconvert.c b/libavcodec/imgconvert.c
index 351ed7a..8da11a5 100644
--- a/libavcodec/imgconvert.c
+++ b/libavcodec/imgconvert.c
@@ -615,7 +615,8 @@ static enum PixelFormat avcodec_find_best_pix_fmt1(int64_t pix_fmt_mask,
     /* find exact color match with smallest size */
     dst_pix_fmt = PIX_FMT_NONE;
     min_dist = 0x7fffffff;
-    for(i = 0;i < PIX_FMT_NB; i++) {
+    /* test only the first 64 pixel formats to avoid undefined behaviour */
+    for (i = 0; i < 64; i++) {
         if (pix_fmt_mask & (1ULL << i)) {
             loss = avcodec_get_pix_fmt_loss(i, src_pix_fmt, has_alpha) & loss_mask;
             if (loss == 0) {



More information about the libav-commits mailing list