From 4928ca91d16cbbcaaa172e475901a26bc214fae6 Mon Sep 17 00:00:00 2001
From: germanocaumo <germanocaumo@gmail.com>
Date: Fri, 11 Sep 2020 19:19:01 -0300
Subject: [PATCH] Mix screenshare audio with mics again to avoid playback
autoplay problems in iOS, fixed desync with aresample
---
.../core/lib/recordandplayback/edl/audio.rb | 38 ++++++++++++++++---
.../generators/audio_processor.rb | 25 ++++++++++++
.../lib/recordandplayback/generators/video.rb | 24 +-----------
3 files changed, 58 insertions(+), 29 deletions(-)
diff --git a/record-and-playback/core/lib/recordandplayback/edl/audio.rb b/record-and-playback/core/lib/recordandplayback/edl/audio.rb
index 0348de3214..db24d0b24f 100644
--- a/record-and-playback/core/lib/recordandplayback/edl/audio.rb
+++ b/record-and-playback/core/lib/recordandplayback/edl/audio.rb
@@ -22,6 +22,7 @@ module BigBlueButton
module Audio
FFMPEG_AEVALSRC = "aevalsrc=s=48000:c=stereo:exprs=0|0"
FFMPEG_AFORMAT = "aformat=sample_fmts=s16:sample_rates=48000:channel_layouts=stereo"
+ FFMPEG_AFORMAT_SCREENSHARE = "aresample=async=1000,aformat=sample_fmts=s16:sample_rates=48000:channel_layouts=stereo"
FFMPEG_WF_CODEC = 'libvorbis'
FFMPEG_WF_ARGS = ['-c:a', FFMPEG_WF_CODEC, '-q:a', '2', '-f', 'ogg']
WF_EXT = 'ogg'
@@ -43,6 +44,25 @@ module BigBlueButton
end
end
+ def self.mixer(inputs, output_basename)
+ BigBlueButton.logger.debug "Mixing audio files"
+
+ ffmpeg_cmd = [*FFMPEG]
+ inputs.each do |input|
+ ffmpeg_cmd += ['-i', input]
+ end
+ ffmpeg_cmd += ['-filter_complex', "amix"]
+
+ output = "#{output_basename}.#{WF_EXT}"
+ ffmpeg_cmd += [*FFMPEG_WF_ARGS, output]
+
+ BigBlueButton.logger.info "Running audio mixer..."
+ exitstatus = BigBlueButton.exec_ret(*ffmpeg_cmd)
+ raise "ffmpeg failed, exit code #{exitstatus}" if exitstatus != 0
+
+ output
+ end
+
def self.render(edl, output_basename, screenshare)
sections = []
audioinfo = {}
@@ -111,7 +131,11 @@ module BigBlueButton
filter = "[#{input_index}] "
filter << "atempo=#{speed},atrim=start=#{ms_to_s(audio[:timestamp])},"
filter << "asetpts=PTS-STARTPTS,"
- filter << "#{FFMPEG_AFORMAT},apad,atrim=end=#{ms_to_s(duration)} [out#{output_index}]"
+ if screenshare
+ filter << "#{FFMPEG_AFORMAT_SCREENSHARE},apad,atrim=end=#{ms_to_s(duration)} [out#{output_index}]"
+ else
+ filter << "#{FFMPEG_AFORMAT},apad,atrim=end=#{ms_to_s(duration)} [out#{output_index}]"
+ end
ffmpeg_filters << filter
ffmpeg_inputs << {
@@ -130,7 +154,11 @@ module BigBlueButton
BigBlueButton.logger.info " Using input #{audio[:filename]}"
filter = "[#{input_index}] "
- filter << "#{FFMPEG_AFORMAT},apad,atrim=end=#{ms_to_s(duration)} [out#{output_index}]"
+ if screenshare
+ filter << "#{FFMPEG_AFORMAT_SCREENSHARE},apad,atrim=end=#{ms_to_s(duration)} [out#{output_index}]"
+ else
+ filter << "#{FFMPEG_AFORMAT},apad,atrim=end=#{ms_to_s(duration)} [out#{output_index}]"
+ end
ffmpeg_filters << filter
ffmpeg_inputs << {
@@ -157,9 +185,7 @@ module BigBlueButton
if audioinfo[input[:filename]][:format][:format_name] == 'wav'
ffmpeg_cmd += ['-ignore_length', '1']
end
- if (screenshare)
- ffmpeg_cmd += ['-c:a', 'libopus']
- end
+ screenshare ? ffmpeg_cmd += ['-c:a', 'libopus'] : nil
ffmpeg_cmd += ['-i', input[:filename]]
end
ffmpeg_filter = ffmpeg_filters.join(' ; ')
@@ -177,7 +203,7 @@ module BigBlueButton
ffmpeg_cmd += ['-filter_complex', ffmpeg_filter]
output = "#{output_basename}.#{WF_EXT}"
- if (screenshare)
+ if screenshare
ffmpeg_cmd += [*FFMPEG_WF_SCREENSHARE_ARGS, output]
else
ffmpeg_cmd += [*FFMPEG_WF_ARGS, output]
diff --git a/record-and-playback/core/lib/recordandplayback/generators/audio_processor.rb b/record-and-playback/core/lib/recordandplayback/generators/audio_processor.rb
index 905326c76f..4add823dfa 100755
--- a/record-and-playback/core/lib/recordandplayback/generators/audio_processor.rb
+++ b/record-and-playback/core/lib/recordandplayback/generators/audio_processor.rb
@@ -57,6 +57,31 @@ module BigBlueButton
@audio_file = BigBlueButton::EDL::Audio.render(
audio_edl, File.join(target_dir, 'recording'), false)
+ # and mixing it with deskshare audio
+ if BigBlueButton::Events.screenshare_has_audio?(events_xml)
+ BigBlueButton.logger.info("AudioProcessor.process: processing Deskshare audio...")
+
+ deskshare_dir = "#{archive_dir}/deskshare"
+ mixed_dir = "#{archive_dir}/mixed"
+
+ deskshare_audio_edl = BigBlueButton::AudioEvents.create_deskshare_audio_edl(events, deskshare_dir)
+ BigBlueButton::EDL::Audio.dump(deskshare_audio_edl)
+
+ BigBlueButton.logger.info "Applying recording start/stop events to Deskshare audio"
+ deskshare_audio_edl = BigBlueButton::Events.edl_match_recording_marks_audio(
+ deskshare_audio_edl, events, start_time, end_time)
+ BigBlueButton.logger.debug "Trimmed Deskshare Audio EDL:"
+ BigBlueButton::EDL::Audio.dump(deskshare_audio_edl)
+
+ audio_inputs = []
+ audio_inputs << @audio_file
+ audio_inputs << BigBlueButton::EDL::Audio.render(deskshare_audio_edl, deskshare_dir, true)
+
+ @audio_file = BigBlueButton::EDL::Audio.mixer(audio_inputs, mixed_dir)
+ else
+ BigBlueButton.logger.info("AudioProcessor.process: no Deskshare audio to process.")
+ end
+
ogg_format = {
:extension => 'ogg',
:parameters => [ [ '-c:a', 'copy', '-f', 'ogg' ] ]
diff --git a/record-and-playback/core/lib/recordandplayback/generators/video.rb b/record-and-playback/core/lib/recordandplayback/generators/video.rb
index 1af7f61a37..fdcf0ed049 100755
--- a/record-and-playback/core/lib/recordandplayback/generators/video.rb
+++ b/record-and-playback/core/lib/recordandplayback/generators/video.rb
@@ -115,28 +115,6 @@ module BigBlueButton
deskshare_video_file = BigBlueButton::EDL::Video.render(
deskshare_video_edl, deskshare_layout, "#{target_dir}/deskshare")
- # process deskshare audio
- deskshare_audio_file = nil
- if BigBlueButton::Events.screenshare_has_audio?(events)
- BigBlueButton.logger.info("Processing Deskshare audio...")
-
- deskshare_dir = "#{temp_dir}/#{meeting_id}/deskshare"
-
- deskshare_audio_edl = BigBlueButton::AudioEvents.create_deskshare_audio_edl(events, deskshare_dir)
- BigBlueButton::EDL::Audio.dump(deskshare_audio_edl)
-
- BigBlueButton.logger.info "Applying recording start/stop events to Deskshare audio"
- deskshare_audio_edl = BigBlueButton::Events.edl_match_recording_marks_audio(
- deskshare_audio_edl, events, start_time, end_time)
- BigBlueButton.logger.debug "Trimmed Deskshare Audio EDL:"
- BigBlueButton::EDL::Audio.dump(deskshare_audio_edl)
-
- deskshare_audio_file = BigBlueButton::EDL::Audio.render(deskshare_audio_edl, deskshare_dir, true)
-
- else
- BigBlueButton.logger.info("No Deskshare audio to process.")
- end
-
formats = [
{
extension: 'webm',
@@ -175,7 +153,7 @@ module BigBlueButton
formats.reject!{ |format| ! video_formats.include? format[:extension] }
formats.each do |format|
filename = BigBlueButton::EDL::encode(
- deskshare_audio_file, deskshare_video_file, format, "#{target_dir}/deskshare", 0)
+ nil, deskshare_video_file, format, "#{target_dir}/deskshare", 0)
end
end
--
GitLab