[libav-bugs] [Bug 1016] New: Null deref at libavcodec/x86/dcadsp.asm ff_synth_filter_inner_avx()

bugzilla at libav.org bugzilla at libav.org
Mon Jan 2 21:31:31 CET 2017


https://bugzilla.libav.org/show_bug.cgi?id=1016

            Bug ID: 1016
           Summary: Null deref at libavcodec/x86/dcadsp.asm
                    ff_synth_filter_inner_avx()
           Product: Libav
           Version: git HEAD
          Hardware: Other
                OS: Linux
            Status: NEW
          Severity: enhancement
          Priority: ---
         Component: libavcodec
          Assignee: bugzilla at libav.org
          Reporter: jan.s.ruge at gmail.com

Created attachment 643
  --> https://bugzilla.libav.org/attachment.cgi?id=643&action=edit
Causess avplay to crash with a null deref

By fuzzing i found a NULL dereference in ff_synth_filter_inner_avx.
A samplefile causing avplay to crash was appended to the Bugreport
git HEAD ee164727dd64c199b87118917e674b17c25e0da3

$ ./gdb/avplay crash-null-0x27532a6.dts 
avplay version 13_dev0, Copyright (c) 2003-2016 the Libav developers
  built on Jan  2 2017 19:06:58 with gcc 5.4.0 (Ubuntu 5.4.0-6ubuntu1~16.04.4)
20160609
Segmentation fault (core dumped)

$ gdb gdb/avplay /tmp/core_avplay.30571 
Reading symbols from gdb/avplay...done.
[New LWP 30571]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `./gdb/avplay crash-null-0x27532a6.dts'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0  ff_synth_filter_inner_avx () at libavcodec/x86/dcadsp.asm:334
334    SYNTH_FILTER
gdb-peda$ bt
#0  ff_synth_filter_inner_avx () at libavcodec/x86/dcadsp.asm:334
#1  0x00007ffff7f16de0 in ?? ()
#2  0x00000000008f858a in synth_filter_avx (imdct=<optimized out>,
synth_buf_ptr=<optimized out>, synth_buf_offset=0x0, synth_buf2=0x7ffff7f17e00,
window=0xd50f50 <ff_dca_fir_32bands_nonperfect>, out=0x0, in=0x7ffff7f12b60,
scale=7.62939453e-06)
    at libavcodec/x86/dcadsp_init.c:64
#3  0x00000000005a7fc7 in dca_qmf_32_subbands (samples_in=<optimized out>,
sb_act=0x2, synth=0x7ffff7f45608, imdct=0x7ffff7f455a0,
synth_buf_ptr=0x7ffff7f16e00, synth_buf_offset=0x7ffff7f16de0,
synth_buf2=0x7ffff7f17e00, 
    window=0xd50f50 <ff_dca_fir_32bands_nonperfect>, samples_out=0x0,
raXin=0x7ffff7f12b60, scale=7.62939453e-06) at libavcodec/dcadsp.c:87
#4  0x00000000005a1b2e in qmf_32_subbands (scale=7.62939453e-06,
samples_out=<optimized out>, samples_in=0x7fffffffd740, chans=0x0,
s=0x7ffff7f12040) at libavcodec/dcadec.c:560
#5  dca_filter_channels (s=s at entry=0x7ffff7f12040,
block_index=block_index at entry=0x0, upsample=upsample at entry=0x0) at
libavcodec/dcadec.c:974
#6  0x00000000005a756b in dca_decode_frame (avctx=<optimized out>,
data=0x18880a0, got_frame_ptr=0x7fffffffe024, avpkt=<optimized out>) at
libavcodec/dcadec.c:1498
#7  0x00000000005ab61d in decode_simple_internal (frame=0x18880a0,
avctx=0x1886780) at libavcodec/decode.c:334
#8  decode_simple_receive_frame (frame=<optimized out>, avctx=<optimized out>)
at libavcodec/decode.c:390
#9  decode_receive_frame_internal (avctx=0x1886780, frame=0x18880a0) at
libavcodec/decode.c:408
#10 0x00000000005ab9b8 in avcodec_send_packet (avctx=avctx at entry=0x1886780,
avpkt=avpkt at entry=0x7fffffffe0d0) at libavcodec/decode.c:445
#11 0x0000000000543a0d in try_decode_frame (st=st at entry=0x18735a0,
avpkt=avpkt at entry=0x7fffffffe1b0, options=0x1876b80, s=0x1872f00) at
libavformat/utils.c:1950
#12 0x0000000000547d76 in avformat_find_stream_info (ic=0x1872f00,
options=0x1876b80) at libavformat/utils.c:2459
#13 0x000000000044d1f3 in stream_setup (is=0x106ec80 <player_state>) at
avplay.c:2316
#14 stream_open (iformat=<optimized out>, filename=<optimized out>,
is=0x106ec80 <player_state>) at avplay.c:2550
#15 main (argc=argc at entry=0x2, argv=argv at entry=0x7fffffffe568) at avplay.c:3050
#16 0x00007ffff63e7830 in __libc_start_main (main=0x44cfa0 <main>, argc=0x2,
argv=0x7fffffffe568, init=<optimized out>, fini=<optimized out>,
rtld_fini=<optimized out>, stack_end=0x7fffffffe558) at ../csu/libc-start.c:291
#17 0x000000000044e589 in _start ()

gdb-peda$ disas $pc
...
   0x0000000000a5cece <+542>:    jge    0xa5cdf3
<ff_synth_filter_inner_avx+323>
   0x0000000000a5ced4 <+548>:    vmulps ymm1,ymm1,ymm0
   0x0000000000a5ced8 <+552>:    vmulps ymm2,ymm2,ymm0
   0x0000000000a5cedc <+556>:    vmulps ymm7,ymm7,ymm0
   0x0000000000a5cee0 <+560>:    vmulps ymm8,ymm8,ymm0
   0x0000000000a5cee4 <+564>:    vmovaps YMMWORD PTR [rsi],ymm3
   0x0000000000a5cee8 <+568>:    vmovaps YMMWORD PTR [rsi+0x40],ymm4
   0x0000000000a5ceed <+573>:    vmovaps YMMWORD PTR [rsi+0x20],ymm9
   0x0000000000a5cef2 <+578>:    vmovaps YMMWORD PTR [rsi+0x60],ymm10
=> 0x0000000000a5cef7 <+583>:    vmovaps YMMWORD PTR [rcx],ymm1
   0x0000000000a5cefb <+587>:    vmovaps YMMWORD PTR [rcx+0x40],ymm2
   0x0000000000a5cf00 <+592>:    vmovaps YMMWORD PTR [rcx+0x20],ymm7
   0x0000000000a5cf05 <+597>:    vmovaps YMMWORD PTR [rcx+0x60],ymm8
   0x0000000000a5cf0a <+602>:    pop    rbx
   0x0000000000a5cf0b <+603>:    vzeroupper 
   0x0000000000a5cf0e <+606>:    ret  
...
gdb-peda$ info all
rax            0x7ffff7f16e00    0x7ffff7f16e00
rbx            0x0    0x0
rcx            0x0    0x0
rdx            0xd50f50    0xd50f50
rsi            0x7ffff7f17e00    0x7ffff7f17e00
rdi            0x7ffff7f16e00    0x7ffff7f16e00
rbp            0x7ffff7f16e00    0x7ffff7f16e00
rsp            0x7fffffffd3d0    0x7fffffffd3d0
r8             0x0    0x0
r9             0x700    0x700
r10            0x7ffff7f16e00    0x7ffff7f16e00
r11            0xd50f50    0xd50f50
r12            0x7ffff7f17e00    0x7ffff7f17e00
r13            0xd50f50    0xd50f50
r14            0x0    0x0
r15            0x1    0x1
rip            0xa5cef7    0xa5cef7 <ff_synth_filter_inner_avx+583>
eflags         0x10246    [ PF ZF IF RF ]
cs             0x33    0x33
ss             0x2b    0x2b
ds             0x0    0x0
es             0x0    0x0
fs             0x0    0x0
gs             0x0    0x0
st0            0    (raw 0x00000000000000000000)
st1            0    (raw 0x00000000000000000000)
st2            0    (raw 0x00000000000000000000)
st3            0    (raw 0x00000000000000000000)
st4            0    (raw 0x00000000000000000000)
st5            0    (raw 0x00000000000000000000)
st6            0    (raw 0x00000000000000000000)
st7            0    (raw 0x00000000000000000000)
fctrl          0x37f    0x37f
fstat          0x0    0x0
ftag           0xffff    0xffff
fiseg          0x0    0x0
fioff          0x0    0x0
foseg          0x0    0x0
fooff          0x0    0x0
fop            0x0    0x0
mxcsr          0x1fa0    [ PE IM DM ZM OM UM PM ]
ymm0           {
  v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 
  v4_double = {0x0, 0x0, 0x0, 0x0}, 
  v32_int8 = {0x0, 0x0, 0x0, 0x37, 0x0, 0x0, 0x0, 0x37, 0x0, 0x0, 0x0, 0x37,
0x0, 0x0, 0x0, 0x37, 0x0, 0x0, 0x0, 0x37, 0x0, 0x0, 0x0, 0x37, 0x0, 0x0, 0x0,
0x37, 0x0, 0x0, 0x0, 0x37}, 
  v16_int16 = {0x0, 0x3700, 0x0, 0x3700, 0x0, 0x3700, 0x0, 0x3700, 0x0, 0x3700,
0x0, 0x3700, 0x0, 0x3700, 0x0, 0x3700}, 
  v8_int32 = {0x37000000, 0x37000000, 0x37000000, 0x37000000, 0x37000000,
0x37000000, 0x37000000, 0x37000000}, 
  v4_int64 = {0x3700000037000000, 0x3700000037000000, 0x3700000037000000,
0x3700000037000000}, 
  v2_int128 = {0x37000000370000003700000037000000,
0x37000000370000003700000037000000}
}
ymm1           {
  v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 
  v4_double = {0x0, 0x0, 0x0, 0x0}, 
  v32_int8 = {0x0 <repeats 32 times>}, 
  v16_int16 = {0x0 <repeats 16 times>}, 
  v8_int32 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 
  v4_int64 = {0x0, 0x0, 0x0, 0x0}, 
  v2_int128 = {0x00000000000000000000000000000000,
0x00000000000000000000000000000000}
}
ymm2           {
  v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 
  v4_double = {0x0, 0x0, 0x0, 0x0}, 
  v32_int8 = {0x0 <repeats 32 times>}, 
  v16_int16 = {0x0 <repeats 16 times>}, 
  v8_int32 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 
  v4_int64 = {0x0, 0x0, 0x0, 0x0}, 
  v2_int128 = {0x00000000000000000000000000000000,
0x00000000000000000000000000000000}
}
ymm3           {
  v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 
  v4_double = {0x0, 0x0, 0x0, 0x0}, 
  v32_int8 = {0x0 <repeats 32 times>}, 
  v16_int16 = {0x0 <repeats 16 times>}, 
  v8_int32 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 
  v4_int64 = {0x0, 0x0, 0x0, 0x0}, 
  v2_int128 = {0x00000000000000000000000000000000,
0x00000000000000000000000000000000}
}
ymm4           {
  v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 
  v4_double = {0x0, 0x0, 0x0, 0x0}, 
  v32_int8 = {0x0 <repeats 32 times>}, 
  v16_int16 = {0x0 <repeats 16 times>}, 
  v8_int32 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 
  v4_int64 = {0x0, 0x0, 0x0, 0x0}, 
  v2_int128 = {0x00000000000000000000000000000000,
0x00000000000000000000000000000000}
}
ymm5           {
  v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 
  v4_double = {0x0, 0x0, 0x0, 0x0}, 
  v32_int8 = {0x0, 0x0, 0x0, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x80,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x80, 0x0, 0x0, 0x0, 0x0}, 
  v16_int16 = {0x0, 0x8000, 0x0, 0x0, 0x0, 0x8000, 0x0, 0x0, 0x0, 0x8000, 0x0,
0x0, 0x0, 0x8000, 0x0, 0x0}, 
  v8_int32 = {0x80000000, 0x0, 0x80000000, 0x0, 0x80000000, 0x0, 0x80000000,
0x0}, 
  v4_int64 = {0x80000000, 0x80000000, 0x80000000, 0x80000000}, 
  v2_int128 = {0x00000000800000000000000080000000,
0x00000000800000000000000080000000}
}
ymm6           {
  v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 
  v4_double = {0x0, 0x0, 0x0, 0x0}, 
  v32_int8 = {0x0 <repeats 15 times>, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x80}, 
  v16_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8000, 0x0, 0x0, 0x0,
0x8000, 0x0, 0x0, 0x0, 0x8000}, 
  v8_int32 = {0x0, 0x0, 0x0, 0x80000000, 0x0, 0x80000000, 0x0, 0x80000000}, 
  v4_int64 = {0x0, 0x8000000000000000, 0x8000000000000000, 0x8000000000000000}, 
  v2_int128 = {0x80000000000000000000000000000000,
0x80000000000000008000000000000000}
}
ymm7           {
  v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 
  v4_double = {0x0, 0x0, 0x0, 0x0}, 
  v32_int8 = {0x0 <repeats 32 times>}, 
  v16_int16 = {0x0 <repeats 16 times>}, 
  v8_int32 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 
  v4_int64 = {0x0, 0x0, 0x0, 0x0}, 
  v2_int128 = {0x00000000000000000000000000000000,
0x00000000000000000000000000000000}
}
ymm8           {
  v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 
  v4_double = {0x0, 0x0, 0x0, 0x0}, 
  v32_int8 = {0x0 <repeats 32 times>}, 
  v16_int16 = {0x0 <repeats 16 times>}, 
  v8_int32 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 
  v4_int64 = {0x0, 0x0, 0x0, 0x0}, 
  v2_int128 = {0x00000000000000000000000000000000,
0x00000000000000000000000000000000}
}
ymm9           {
  v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 
  v4_double = {0x0, 0x0, 0x0, 0x0}, 
  v32_int8 = {0x0 <repeats 32 times>}, 
  v16_int16 = {0x0 <repeats 16 times>}, 
  v8_int32 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 
  v4_int64 = {0x0, 0x0, 0x0, 0x0}, 
  v2_int128 = {0x00000000000000000000000000000000,
0x00000000000000000000000000000000}
}
ymm10          {
  v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 
  v4_double = {0x0, 0x0, 0x0, 0x0}, 
  v32_int8 = {0x0 <repeats 32 times>}, 
  v16_int16 = {0x0 <repeats 16 times>}, 
  v8_int32 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 
  v4_int64 = {0x0, 0x0, 0x0, 0x0}, 
  v2_int128 = {0x00000000000000000000000000000000,
0x00000000000000000000000000000000}
}
ymm11          {
  v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 
  v4_double = {0x0, 0x0, 0x0, 0x0}, 
  v32_int8 = {0x0, 0x0, 0x0, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x80,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x80, 0x0 <repeats 12 times>}, 
  v16_int16 = {0x0, 0x8000, 0x0, 0x0, 0x0, 0x8000, 0x0, 0x0, 0x0, 0x8000, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0}, 
  v8_int32 = {0x80000000, 0x0, 0x80000000, 0x0, 0x80000000, 0x0, 0x0, 0x0}, 
  v4_int64 = {0x80000000, 0x80000000, 0x80000000, 0x0}, 
  v2_int128 = {0x00000000800000000000000080000000,
0x00000000000000000000000080000000}
}
ymm12          {
  v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 
  v4_double = {0x0, 0x0, 0x0, 0x0}, 
  v32_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x80, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x80}, 
  v16_int16 = {0x0, 0x0, 0x0, 0x8000, 0x0, 0x0, 0x0, 0x8000, 0x0, 0x0, 0x0,
0x8000, 0x0, 0x0, 0x0, 0x8000}, 
  v8_int32 = {0x0, 0x80000000, 0x0, 0x80000000, 0x0, 0x80000000, 0x0,
0x80000000}, 
  v4_int64 = {0x8000000000000000, 0x8000000000000000, 0x8000000000000000,
0x8000000000000000}, 
  v2_int128 = {0x80000000000000008000000000000000,
0x80000000000000008000000000000000}
}
ymm13          {
  v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 
  v4_double = {0x0, 0x0, 0x0, 0x0}, 
  v32_int8 = {0x62, 0xb1, 0x68, 0xe6, 0x6b, 0x2b, 0xe6, 0xbd, 0x0 <repeats 24
times>}, 
  v16_int16 = {0xb162, 0xe668, 0x2b6b, 0xbde6, 0x0 <repeats 12 times>}, 
  v8_int32 = {0xe668b162, 0xbde62b6b, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 
  v4_int64 = {0xbde62b6be668b162, 0x0, 0x0, 0x0}, 
  v2_int128 = {0x0000000000000000bde62b6be668b162,
0x00000000000000000000000000000000}
}
ymm14          {
  v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 
  v4_double = {0x0, 0x0, 0x0, 0x0}, 
  v32_int8 = {0x0, 0x0, 0x0, 0x0, 0xb0, 0x20, 0xd2, 0xbd, 0x0 <repeats 24
times>}, 
  v16_int16 = {0x0, 0x0, 0x20b0, 0xbdd2, 0x0 <repeats 12 times>}, 
  v8_int32 = {0x0, 0xbdd220b0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 
  v4_int64 = {0xbdd220b000000000, 0x0, 0x0, 0x0}, 
  v2_int128 = {0x0000000000000000bdd220b000000000,
0x00000000000000000000000000000000}
}
ymm15          {
  v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 
  v4_double = {0x0, 0x0, 0x0, 0x0}, 
  v32_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x80, 0x0 <repeats 24 times>}, 
  v16_int16 = {0x0, 0x0, 0x0, 0x8000, 0x0 <repeats 12 times>}, 
  v8_int32 = {0x0, 0x80000000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 
  v4_int64 = {0x8000000000000000, 0x0, 0x0, 0x0}, 
  v2_int128 = {0x00000000000000008000000000000000,
0x00000000000000000000000000000000}
}

-- 
You are receiving this mail because:
You are watching all bug changes.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.libav.org/pipermail/libav-bugs/attachments/20170102/00b9f88e/attachment-0001.html>


More information about the libav-bugs mailing list