How do I use overlay filter to overlay a video/gif on the main video like ffmpeg cli?

[DavieHR]

Overview

when i overlay a video on the main video stream, pyav raise out a error:
av.error.BlockingIOError: [Errno 11] Resource temporarily unavailable
the duration overlay video is 00:00:10s, but the duration of main video is 00:10:00.

Expected behavior

ffmpeg cli command is：

ffmpeg -i $origin_video_path ${overlay_inst} \
-filter_complex \"[1:v]scale=${width}:${height}:flags=bicubic[bg1]; \
                         [2:v]scale=${width}:${height}:flags=bicubic[bg2]; \
                         [0:v][bg1]overlay=($cordinate_x-w/2):($cordinate_y-h/2):enable='between(t,${start_time},${end_time}):shortest=1'[out1]; \
                         [out1][bg2]overlay=($cordinate_+100-w/2):($cordinate_y+100-h/2):enable='between(t,${start_time}+10,${end_time}+10):shortest=1'\"[out2] \
        -map [out2] -map 0:a? \
        ${save_video_path}"

Actual behavior

what i wrote pyav codes are:

def pyav_overlay_demo(
                        main_video,
                        video_to_overlay_path,
                        output_path
                     ):
    in_video = av.open(main_video, mode = 'r')
    overlay_video = av.open(video_to_overlay_path, mode = 'r', options = {"-ignore_loop": '-1'})

    output_video = av.open(output_path, 'w')

    main_video_stream, main_audio_stream = in_video.streams.video[0], in_video.streams.audio[0]
    overlay_video_stream, _ = overlay_video.streams.video[0], None


    output_stream = dict(
                          video = output_video.add_stream(codec_name = 'h264', rate = main_video_stream.average_rate),
                          audio = output_video.add_stream(codec_name = 'aac', rate = main_audio_stream.average_rate)
                        )

    graph = av.filter.Graph()
    
    source = graph.add_buffer(main_video_stream)
    overlayed = graph.add_buffer(overlay_video_stream)
    resize_filter = graph.add("scale", "256:256:flags=bicubic")
    overlay_filter = graph.add("overlay", "W/2-w/2:H/2-h/2:enable=\'between(t,0,2)\':shortest=1")
    sink = graph.add("buffersink")

    overlayed.link_to(resize_filter, 0, 0)

    source.link_to(overlay_filter, 0, 0)
    resize_filter.link_to(overlay_filter, 0, 1)
    overlay_filter.link_to(sink, 0, 0)
    graph.configure()
    frames = []
    count = 0 
    for packet in overlay_video.demux(overlay_video.streams.video[0]):
        for ifr in packet.decode():
            overlayed.push(ifr)
            count += 1
            #frames.append(ifr)

    for packet in in_video.demux(in_video.streams.video[0]):
        for ifr in packet.decode():
            index = ifr.index
            typ = packet.stream.type
            if typ == 'audio':
                ifr.pts = None
                for p in output_stream[typ].encode(ifr):
                    output_video.mux(p)
            else:
                source.push(ifr)
                #overlayed.push(frames[index % len(frames)])
                ofr = sink.pull()
                #print(index % len(frames))
                for p in output_stream[typ].encode(ofr):
                    output_video.mux(p)
    for packet in output_stream['video'].encode():
        output_video.mux(packet)
    output_video.close()

Traceback:

Traceback (most recent call last):
  File "overlay.py", line 86, in <module>
    pyav_overlay_demo(main_video, overlay_video, save_video)
  File "overlay.py", line 74, in pyav_overlay_demo
    ofr = sink.pull()
  File "av/filter/context.pyx", line 119, in av.filter.context.FilterContext.pull
  File "av/error.pyx", line 336, in av.error.err_check
av.error.BlockingIOError: [Errno 11] Resource temporarily unavailable

Investigation

for the gif type stream, i put an option: -ignore_loop -1 in overlay_video;
but nothing happened. The error still is "Resource temporarily unavailable"

Research

I have done the following:

• Checked the PyAV documentation
• Searched on Google
• Searched on Stack Overflow
• Looked through old GitHub issues
• Asked on PyAV Gitter
• ... and waited 72 hours for a response.

Additional context

no

[DavieHR]

@rawler @heni @Knio @pklaus