[libav-bugs] [Bug 342] New: Crash decoding Lagarith RGB24 on Windows 64-bit

bugzilla-daemon at aruru.libav.org bugzilla-daemon at aruru.libav.org
Thu Aug 2 21:58:42 CEST 2012


https://bugzilla.libav.org/show_bug.cgi?id=342

           Summary: Crash decoding Lagarith RGB24 on Windows 64-bit
           Product: Libav
           Version: git HEAD
          Platform: X86
        OS/Version: Windows
            Status: NEW
          Severity: major
          Priority: Normal
         Component: libavcodec
        AssignedTo: bugzilla at libav.org
        ReportedBy: h.leppkes at gmail.com


Created attachment 335
  --> https://bugzilla.libav.org/attachment.cgi?id=335
Crash sample

I encountered a crash decoding a RGB24 Lagarith file on Windows 64-bit. Linux
does not seem to exhibit the crash in my tests, neiter does 32-bit Windows.

I tracked the crash into "add_hfyu_left_prediction_sse4" in
libavcodec/x86/dsputil_yasm.asm.
The crash happens with -cpuflags sse4.1, but not with -cpuflags ssse3 or lower.

The gdb backtrace and output of avconv are rather useless, as is disass.

(gdb) bt
#0  0x0000000000000004 in ?? ()
#1  0x4028000000000000 in ?? ()
#2  0x0000000000000000 in ?? ()

(gdb) disass $pc-32,$pc+32
Dump of assembler code from 0xffffffffffffffe4 to 0x24:
End of assembler dump.

(gdb) info all-registers
rax            0x0      0
rbx            0x1      1
rcx            0x2b17100        45183232
rdx            0x2b17100        45183232
rsi            0x280    640
rdi            0xfffffffffffffd80       -640
rbp            0x27f    0x27f
rsp            0x22ef60 0x22ef60
r8             0x1      1
r9             0x0      0
r10            0x1e0    480
r11            0x35f6   13814
r12            0x280    640
r13            0x780    1920
r14            0x24d00a0        38600864
r15            0x2b16c00        45181952
rip            0x4      0x4
eflags         0x10206  [ PF IF RF ]
cs             0x33     51
ss             0x206002b        33947691
ds             0x0      0
es             0x0      0
fs             0x0      0
gs             0x2b0000 2818048
st0            -inf     (raw 0xffff0000000000000000)
st1            -nan(0x000000006)        (raw 0xffff0000000000000006)
st2            -nan(0xd00000000000000)  (raw 0xffff0d00000000000000)
st3            -nan(0x505ffff0101ffff)  (raw 0xffff0505ffff0101ffff)
st4            -nan(0x3030303ffffffff)  (raw 0xffff03030303ffffffff)
st5            -nan(0x707070707070707)  (raw 0xffff0707070707070707)
st6            9.9999999999999995e-021  (raw 0x3fbcbce5086492111800)
st7            1        (raw 0x3fff8000000000000000)
fctrl          0x420037f        69206911
fstat          0xff0420 16712736
ftag           0xff     255
fiseg          0x33     51
fioff          0x54c279 5554809
foseg          0x2b     43
fooff          0x22f678 2291320
fop            0x0      0
xmm0           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>},
  v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0,
0x0, 0x0}, v2_int64 = {0x0, 0x0},
  uint128 = 0x00000000000000000000000000000000}
xmm1           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>},
  v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0,
0x0, 0x0}, v2_int64 = {0x0, 0x0},
  uint128 = 0x00000000000000000000000000000000}
xmm2           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>},
  v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0,
0x0, 0x0}, v2_int64 = {0x0, 0x0},
  uint128 = 0x00000000000000000000000000000000}
xmm3           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0xff, 0xff, 0x1, 0x1, 0xff,
    0xff, 0x5, 0x5, 0xff, 0xff, 0x9, 0x9, 0xff, 0xff, 0xd, 0xd}, v8_int16 =
{0xffff, 0x101, 0xffff, 0x505, 0xffff,
    0x909, 0xffff, 0xd0d}, v4_int32 = {0x101ffff, 0x505ffff, 0x909ffff,
0xd0dffff}, v2_int64 = {0x505ffff0101ffff,
    0xd0dffff0909ffff}, uint128 = 0x0d0dffff0909ffff0505ffff0101ffff}
xmm4           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0xff, 0xff, 0xff, 0xff, 0x3,
    0x3, 0x3, 0x3, 0xff, 0xff, 0xff, 0xff, 0xb, 0xb, 0xb, 0xb}, v8_int16 =
{0xffff, 0xffff, 0x303, 0x303, 0xffff,
    0xffff, 0xb0b, 0xb0b}, v4_int32 = {0xffffffff, 0x3030303, 0xffffffff,
0xb0b0b0b}, v2_int64 = {0x3030303ffffffff,
    0xb0b0b0bffffffff}, uint128 = 0x0b0b0b0bffffffff03030303ffffffff}
xmm5           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0xf <repeats 16 times>},
  v8_int16 = {0xf0f, 0xf0f, 0xf0f, 0xf0f, 0xf0f, 0xf0f, 0xf0f, 0xf0f}, v4_int32
= {0xf0f0f0f, 0xf0f0f0f, 0xf0f0f0f,
    0xf0f0f0f}, v2_int64 = {0xf0f0f0f0f0f0f0f, 0xf0f0f0f0f0f0f0f}, uint128 =
0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f}
xmm6           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double =
{0x8000000000000000, 0x0}, v16_int8 = {0xff, 0xff, 0xff,
    0xff, 0xff, 0xff, 0xff, 0xff, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7},
v8_int16 = {0xffff, 0xffff, 0xffff,
    0xffff, 0x707, 0x707, 0x707, 0x707}, v4_int32 = {0xffffffff, 0xffffffff,
0x7070707, 0x7070707}, v2_int64 = {
    0xffffffffffffffff, 0x707070707070707}, uint128 =
0x0707070707070707ffffffffffffffff}
xmm7           {v4_float = {0x0, 0x4, 0x0, 0x0}, v2_double = {0x3e9, 0x0},
v16_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x48,
    0x8f, 0x40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x0, 0x0,
0x4800, 0x408f, 0x0, 0x0, 0x0, 0x0},
  v4_int32 = {0x0, 0x408f4800, 0x0, 0x0}, v2_int64 = {0x408f480000000000, 0x0},
  uint128 = 0x0000000000000000408f480000000000}
xmm8           {v4_float = {0x0, 0x6, 0x0, 0x0}, v2_double = {0x2eec, 0x0},
v16_int8 = {0x0, 0x0, 0x0, 0x0, 0x0,
    0x76, 0xc7, 0x40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x0,
0x0, 0x7600, 0x40c7, 0x0, 0x0, 0x0,
    0x0}, v4_int32 = {0x0, 0x40c77600, 0x0, 0x0}, v2_int64 =
{0x40c7760000000000, 0x0},
  uint128 = 0x000000000000000040c7760000000000}
xmm9           {v4_float = {0x2b020000, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0xfc, 0xa9, 0xf1, 0xd2,
    0x4d, 0x62, 0x40, 0x3f, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 =
{0xa9fc, 0xd2f1, 0x624d, 0x3f40, 0x0,
    0x0, 0x0, 0x0}, v4_int32 = {0xd2f1a9fc, 0x3f40624d, 0x0, 0x0}, v2_int64 =
{0x3f40624dd2f1a9fc, 0x0},
  uint128 = 0x00000000000000003f40624dd2f1a9fc}
xmm10          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>},
  v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0,
0x0, 0x0}, v2_int64 = {0x0, 0x0},
  uint128 = 0x00000000000000000000000000000000}
xmm11          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>},
  v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0,
0x0, 0x0}, v2_int64 = {0x0, 0x0},
  uint128 = 0x00000000000000000000000000000000}
xmm12          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>},
  v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0,
0x0, 0x0}, v2_int64 = {0x0, 0x0},
  uint128 = 0x00000000000000000000000000000000}
xmm13          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>},
  v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0,
0x0, 0x0}, v2_int64 = {0x0, 0x0},
  uint128 = 0x00000000000000000000000000000000}
xmm14          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>},
  v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0,
0x0, 0x0}, v2_int64 = {0x0, 0x0},
  uint128 = 0x00000000000000000000000000000000}
xmm15          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>},
  v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0,
0x0, 0x0}, v2_int64 = {0x0, 0x0},
  uint128 = 0x00000000000000000000000000000000}
mxcsr          0x1fa0   [ PE IM DM ZM OM UM PM ]

-- 
Configure bugmail: https://bugzilla.libav.org/userprefs.cgi?tab=email
------- You are receiving this mail because: -------
You are watching all bug changes.


More information about the libav-bugs mailing list