From fdbbffb3aa2e17f65556f4c5574552f7130a1285 Mon Sep 17 00:00:00 2001
From: germanocaumo <germanocaumo@gmail.com>
Date: Fri, 4 Sep 2020 20:58:29 -0300
Subject: [PATCH] Improve screenshare audio recording sync: use libopus decoder
and encoder, its better than built-in ffmpeg/flac don't mix screenshare audio
with mics, was generating desync with bad audio segments, encode it together
with video file (TODO: needs adjustments in playback)
---
.../core/lib/recordandplayback/edl/audio.rb | 37 +++++++------------
.../generators/audio_processor.rb | 27 +-------------
.../recordandplayback/generators/events.rb | 3 +-
.../lib/recordandplayback/generators/video.rb | 24 +++++++++++-
4 files changed, 38 insertions(+), 53 deletions(-)
diff --git a/record-and-playback/core/lib/recordandplayback/edl/audio.rb b/record-and-playback/core/lib/recordandplayback/edl/audio.rb
index 84fb6be4ee..0348de3214 100644
--- a/record-and-playback/core/lib/recordandplayback/edl/audio.rb
+++ b/record-and-playback/core/lib/recordandplayback/edl/audio.rb
@@ -25,6 +25,8 @@ module BigBlueButton
FFMPEG_WF_CODEC = 'libvorbis'
FFMPEG_WF_ARGS = ['-c:a', FFMPEG_WF_CODEC, '-q:a', '2', '-f', 'ogg']
WF_EXT = 'ogg'
+ FFMPEG_WF_SCREENSHARE_CODEC = 'libopus'
+ FFMPEG_WF_SCREENSHARE_ARGS = ['-c:a', FFMPEG_WF_SCREENSHARE_CODEC, '-b:a', '48K', '-f', 'opus']
def self.dump(edl)
BigBlueButton.logger.debug "EDL Dump:"
@@ -41,26 +43,7 @@ module BigBlueButton
end
end
- def self.mixer(inputs, output_basename)
- BigBlueButton.logger.debug "Mixing audio files"
-
- ffmpeg_cmd = [*FFMPEG]
- inputs.each do |input|
- ffmpeg_cmd += ['-i', input]
- end
- ffmpeg_cmd += ['-filter_complex', "amix"]
-
- output = "#{output_basename}.#{WF_EXT}"
- ffmpeg_cmd += [*FFMPEG_WF_ARGS, output]
-
- BigBlueButton.logger.info "Running audio mixer..."
- exitstatus = BigBlueButton.exec_ret(*ffmpeg_cmd)
- raise "ffmpeg failed, exit code #{exitstatus}" if exitstatus != 0
-
- output
- end
-
- def self.render(edl, output_basename)
+ def self.render(edl, output_basename, screenshare)
sections = []
audioinfo = {}
@@ -128,7 +111,7 @@ module BigBlueButton
filter = "[#{input_index}] "
filter << "atempo=#{speed},atrim=start=#{ms_to_s(audio[:timestamp])},"
filter << "asetpts=PTS-STARTPTS,"
- filter << "#{FFMPEG_AFORMAT},apad,atrim=end=#{ms_to_s(duration)} ,afifo[out#{output_index}]"
+ filter << "#{FFMPEG_AFORMAT},apad,atrim=end=#{ms_to_s(duration)} [out#{output_index}]"
ffmpeg_filters << filter
ffmpeg_inputs << {
@@ -147,7 +130,7 @@ module BigBlueButton
BigBlueButton.logger.info " Using input #{audio[:filename]}"
filter = "[#{input_index}] "
- filter << "#{FFMPEG_AFORMAT},apad,atrim=end=#{ms_to_s(duration)} ,afifo[out#{output_index}]"
+ filter << "#{FFMPEG_AFORMAT},apad,atrim=end=#{ms_to_s(duration)} [out#{output_index}]"
ffmpeg_filters << filter
ffmpeg_inputs << {
@@ -174,7 +157,9 @@ module BigBlueButton
if audioinfo[input[:filename]][:format][:format_name] == 'wav'
ffmpeg_cmd += ['-ignore_length', '1']
end
- ffmpeg_cmd += ['-vsync', 'vfr']
+ if (screenshare)
+ ffmpeg_cmd += ['-c:a', 'libopus']
+ end
ffmpeg_cmd += ['-i', input[:filename]]
end
ffmpeg_filter = ffmpeg_filters.join(' ; ')
@@ -192,7 +177,11 @@ module BigBlueButton
ffmpeg_cmd += ['-filter_complex', ffmpeg_filter]
output = "#{output_basename}.#{WF_EXT}"
- ffmpeg_cmd += [*FFMPEG_WF_ARGS, output]
+ if (screenshare)
+ ffmpeg_cmd += [*FFMPEG_WF_SCREENSHARE_ARGS, output]
+ else
+ ffmpeg_cmd += [*FFMPEG_WF_ARGS, output]
+ end
BigBlueButton.logger.info "Running audio processing..."
exitstatus = BigBlueButton.exec_ret(*ffmpeg_cmd)
diff --git a/record-and-playback/core/lib/recordandplayback/generators/audio_processor.rb b/record-and-playback/core/lib/recordandplayback/generators/audio_processor.rb
index df589797e8..905326c76f 100755
--- a/record-and-playback/core/lib/recordandplayback/generators/audio_processor.rb
+++ b/record-and-playback/core/lib/recordandplayback/generators/audio_processor.rb
@@ -55,32 +55,7 @@ module BigBlueButton
# getting users audio...
@audio_file = BigBlueButton::EDL::Audio.render(
- audio_edl, File.join(target_dir, 'recording'))
-
- # and mixing it with deskshare audio
- if BigBlueButton::Events.screenshare_has_audio?(events_xml)
- BigBlueButton.logger.info("AudioProcessor.process: processing Deskshare audio...")
-
- deskshare_dir = "#{archive_dir}/deskshare"
- mixed_dir = "#{archive_dir}/mixed"
-
- deskshare_audio_edl = BigBlueButton::AudioEvents.create_deskshare_audio_edl(events, deskshare_dir)
- BigBlueButton::EDL::Audio.dump(deskshare_audio_edl)
-
- BigBlueButton.logger.info "Applying recording start/stop events to Deskshare audio"
- deskshare_audio_edl = BigBlueButton::Events.edl_match_recording_marks_audio(
- deskshare_audio_edl, events, start_time, end_time)
- BigBlueButton.logger.debug "Trimmed Deskshare Audio EDL:"
- BigBlueButton::EDL::Audio.dump(deskshare_audio_edl)
-
- audio_inputs = []
- audio_inputs << @audio_file
- audio_inputs << BigBlueButton::EDL::Audio.render(deskshare_audio_edl, deskshare_dir)
-
- @audio_file = BigBlueButton::EDL::Audio.mixer(audio_inputs, mixed_dir)
- else
- BigBlueButton.logger.info("AudioProcessor.process: no Deskshare audio to process.")
- end
+ audio_edl, File.join(target_dir, 'recording'), false)
ogg_format = {
:extension => 'ogg',
diff --git a/record-and-playback/core/lib/recordandplayback/generators/events.rb b/record-and-playback/core/lib/recordandplayback/generators/events.rb
index 6dd353a388..94be3e5e8b 100755
--- a/record-and-playback/core/lib/recordandplayback/generators/events.rb
+++ b/record-and-playback/core/lib/recordandplayback/generators/events.rb
@@ -693,8 +693,7 @@ module BigBlueButton
end
# Check if any screenshare files has audio
- def self.screenshare_has_audio?(events_xml)
- events = Nokogiri::XML(File.open(events_xml))
+ def self.screenshare_has_audio?(events)
events.xpath('/recording/event[@eventname="StartWebRTCDesktopShareEvent"]').each do |event|
filename = event.at_xpath('filename').text
fileHasAudio = !BigBlueButton::EDL::Audio.audio_info(filename)[:audio].nil?
diff --git a/record-and-playback/core/lib/recordandplayback/generators/video.rb b/record-and-playback/core/lib/recordandplayback/generators/video.rb
index fdcf0ed049..1af7f61a37 100755
--- a/record-and-playback/core/lib/recordandplayback/generators/video.rb
+++ b/record-and-playback/core/lib/recordandplayback/generators/video.rb
@@ -115,6 +115,28 @@ module BigBlueButton
deskshare_video_file = BigBlueButton::EDL::Video.render(
deskshare_video_edl, deskshare_layout, "#{target_dir}/deskshare")
+ # process deskshare audio
+ deskshare_audio_file = nil
+ if BigBlueButton::Events.screenshare_has_audio?(events)
+ BigBlueButton.logger.info("Processing Deskshare audio...")
+
+ deskshare_dir = "#{temp_dir}/#{meeting_id}/deskshare"
+
+ deskshare_audio_edl = BigBlueButton::AudioEvents.create_deskshare_audio_edl(events, deskshare_dir)
+ BigBlueButton::EDL::Audio.dump(deskshare_audio_edl)
+
+ BigBlueButton.logger.info "Applying recording start/stop events to Deskshare audio"
+ deskshare_audio_edl = BigBlueButton::Events.edl_match_recording_marks_audio(
+ deskshare_audio_edl, events, start_time, end_time)
+ BigBlueButton.logger.debug "Trimmed Deskshare Audio EDL:"
+ BigBlueButton::EDL::Audio.dump(deskshare_audio_edl)
+
+ deskshare_audio_file = BigBlueButton::EDL::Audio.render(deskshare_audio_edl, deskshare_dir, true)
+
+ else
+ BigBlueButton.logger.info("No Deskshare audio to process.")
+ end
+
formats = [
{
extension: 'webm',
@@ -153,7 +175,7 @@ module BigBlueButton
formats.reject!{ |format| ! video_formats.include? format[:extension] }
formats.each do |format|
filename = BigBlueButton::EDL::encode(
- nil, deskshare_video_file, format, "#{target_dir}/deskshare", 0)
+ deskshare_audio_file, deskshare_video_file, format, "#{target_dir}/deskshare", 0)
end
end
--
GitLab