From 0d48dd01cb5362bbd2255caf70c0fda46e133126 Mon Sep 17 00:00:00 2001
From: germanocaumo <germanocaumo@gmail.com>
Date: Fri, 28 Aug 2020 16:35:51 -0300
Subject: [PATCH] Re-add screenshare audio recording processing
---
.../core/lib/recordandplayback/edl/audio.rb | 24 ++++++-
.../lib/recordandplayback/generators/audio.rb | 71 +++++++++++++++++++
.../generators/audio_processor.rb | 28 +++++++-
.../recordandplayback/generators/events.rb | 13 ++++
4 files changed, 132 insertions(+), 4 deletions(-)
diff --git a/record-and-playback/core/lib/recordandplayback/edl/audio.rb b/record-and-playback/core/lib/recordandplayback/edl/audio.rb
index e6c500c0d8..84fb6be4ee 100644
--- a/record-and-playback/core/lib/recordandplayback/edl/audio.rb
+++ b/record-and-playback/core/lib/recordandplayback/edl/audio.rb
@@ -41,6 +41,25 @@ module BigBlueButton
end
end
+ def self.mixer(inputs, output_basename)
+ BigBlueButton.logger.debug "Mixing audio files"
+
+ ffmpeg_cmd = [*FFMPEG]
+ inputs.each do |input|
+ ffmpeg_cmd += ['-i', input]
+ end
+ ffmpeg_cmd += ['-filter_complex', "amix"]
+
+ output = "#{output_basename}.#{WF_EXT}"
+ ffmpeg_cmd += [*FFMPEG_WF_ARGS, output]
+
+ BigBlueButton.logger.info "Running audio mixer..."
+ exitstatus = BigBlueButton.exec_ret(*ffmpeg_cmd)
+ raise "ffmpeg failed, exit code #{exitstatus}" if exitstatus != 0
+
+ output
+ end
+
def self.render(edl, output_basename)
sections = []
audioinfo = {}
@@ -109,7 +128,7 @@ module BigBlueButton
filter = "[#{input_index}] "
filter << "atempo=#{speed},atrim=start=#{ms_to_s(audio[:timestamp])},"
filter << "asetpts=PTS-STARTPTS,"
- filter << "#{FFMPEG_AFORMAT},apad,atrim=end=#{ms_to_s(duration)} [out#{output_index}]"
+ filter << "#{FFMPEG_AFORMAT},apad,atrim=end=#{ms_to_s(duration)} ,afifo[out#{output_index}]"
ffmpeg_filters << filter
ffmpeg_inputs << {
@@ -128,7 +147,7 @@ module BigBlueButton
BigBlueButton.logger.info " Using input #{audio[:filename]}"
filter = "[#{input_index}] "
- filter << "#{FFMPEG_AFORMAT},apad,atrim=end=#{ms_to_s(duration)} [out#{output_index}]"
+ filter << "#{FFMPEG_AFORMAT},apad,atrim=end=#{ms_to_s(duration)} ,afifo[out#{output_index}]"
ffmpeg_filters << filter
ffmpeg_inputs << {
@@ -155,6 +174,7 @@ module BigBlueButton
if audioinfo[input[:filename]][:format][:format_name] == 'wav'
ffmpeg_cmd += ['-ignore_length', '1']
end
+ ffmpeg_cmd += ['-vsync', 'vfr']
ffmpeg_cmd += ['-i', input[:filename]]
end
ffmpeg_filter = ffmpeg_filters.join(' ; ')
diff --git a/record-and-playback/core/lib/recordandplayback/generators/audio.rb b/record-and-playback/core/lib/recordandplayback/generators/audio.rb
index 98c94dbd73..80c11ea88b 100755
--- a/record-and-playback/core/lib/recordandplayback/generators/audio.rb
+++ b/record-and-playback/core/lib/recordandplayback/generators/audio.rb
@@ -73,5 +73,76 @@ module BigBlueButton
return audio_edl
end
+ def self.create_deskshare_audio_edl(events, deskshare_dir)
+ audio_edl = []
+
+ initial_timestamp = BigBlueButton::Events.first_event_timestamp(events)
+ final_timestamp = BigBlueButton::Events.last_event_timestamp(events)
+ filename = ""
+
+ # Initially start with silence
+ audio_edl << {
+ :timestamp => 0,
+ :audio => nil
+ }
+
+ deskshare_start = {
+ :filename => "",
+ :timestamp => 0
+ }
+
+ events.xpath('/recording/event[@module="Deskshare" or (@module="bbb-webrtc-sfu" and (@eventname="StartWebRTCDesktopShareEvent" or @eventname="StopWebRTCDesktopShareEvent"))]').each do |event|
+ # check if deskshare has audio
+ filename = event.at_xpath('filename').text
+ fileHasAudio = !BigBlueButton::EDL::Audio.audio_info(filename)[:audio].nil?
+ if (fileHasAudio)
+ timestamp = event['timestamp'].to_i - initial_timestamp
+ # Determine the audio filename
+ case event['eventname']
+ when 'DeskshareStartedEvent', 'DeskshareStoppedEvent'
+ filename = event.at_xpath('file').text
+ filename = "#{deskshare_dir}/#{File.basename(filename)}"
+ when 'StartWebRTCDesktopShareEvent', 'StopWebRTCDesktopShareEvent'
+ uri = event.at_xpath('filename').text
+ filename = "#{deskshare_dir}/#{File.basename(uri)}"
+ end
+ raise "Couldn't determine audio filename" if filename.nil?
+
+ # Add the audio to the EDL
+ case event['eventname']
+ when 'DeskshareStartedEvent', 'StartWebRTCDesktopShareEvent'
+ audio_edl << {
+ :timestamp => timestamp,
+ :audio => { :filename => filename, :timestamp => 0 }
+ }
+ when 'DeskshareStoppedEvent', 'StopWebRTCDesktopShareEvent'
+ if audio_edl.last[:audio] && audio_edl.last[:audio][:filename] == filename
+ # Fill in the original/expected audo duration when available
+ duration = event.at_xpath('duration')
+ if !duration.nil?
+ duration = duration.text.to_i
+ audio_edl.last[:original_duration] = duration * 1000
+ else
+ audio_edl.last[:original_duration] = timestamp - audio_edl.last[:timestamp]
+ end
+ audio_edl << {
+ :timestamp => timestamp,
+ :audio => nil
+ }
+ end
+ end
+ else
+ BigBlueButton.logger.debug " Screenshare without audio, ignoring..."
+ end
+ end
+
+ audio_edl << {
+ :timestamp => final_timestamp - initial_timestamp,
+ :audio => nil
+ }
+
+ return audio_edl
+ end
+
end
end
diff --git a/record-and-playback/core/lib/recordandplayback/generators/audio_processor.rb b/record-and-playback/core/lib/recordandplayback/generators/audio_processor.rb
index 9f5129c144..df589797e8 100755
--- a/record-and-playback/core/lib/recordandplayback/generators/audio_processor.rb
+++ b/record-and-playback/core/lib/recordandplayback/generators/audio_processor.rb
@@ -36,7 +36,6 @@ module BigBlueButton
def self.process(archive_dir, file_basename)
BigBlueButton.logger.info("AudioProcessor.process: Processing audio...")
- audio_dir = "#{archive_dir}/audio"
events_xml = "#{archive_dir}/events.xml"
events = Nokogiri::XML(File.open(events_xml))
@@ -52,12 +51,37 @@ module BigBlueButton
BigBlueButton::EDL::Audio.dump(audio_edl)
target_dir = File.dirname(file_basename)
- audio_dir = "#{archive_dir}/audio"
events_xml = "#{archive_dir}/events.xml"
+ # getting users audio...
@audio_file = BigBlueButton::EDL::Audio.render(
audio_edl, File.join(target_dir, 'recording'))
+ # and mixing it with deskshare audio
+ if BigBlueButton::Events.screenshare_has_audio?(events_xml)
+ BigBlueButton.logger.info("AudioProcessor.process: processing Deskshare audio...")
+
+ deskshare_dir = "#{archive_dir}/deskshare"
+ mixed_dir = "#{archive_dir}/mixed"
+
+ deskshare_audio_edl = BigBlueButton::AudioEvents.create_deskshare_audio_edl(events, deskshare_dir)
+ BigBlueButton::EDL::Audio.dump(deskshare_audio_edl)
+
+ BigBlueButton.logger.info "Applying recording start/stop events to Deskshare audio"
+ deskshare_audio_edl = BigBlueButton::Events.edl_match_recording_marks_audio(
+ deskshare_audio_edl, events, start_time, end_time)
+ BigBlueButton.logger.debug "Trimmed Deskshare Audio EDL:"
+ BigBlueButton::EDL::Audio.dump(deskshare_audio_edl)
+
+ audio_inputs = []
+ audio_inputs << @audio_file
+ audio_inputs << BigBlueButton::EDL::Audio.render(deskshare_audio_edl, deskshare_dir)
+
+ @audio_file = BigBlueButton::EDL::Audio.mixer(audio_inputs, mixed_dir)
+ else
+ BigBlueButton.logger.info("AudioProcessor.process: no Deskshare audio to process.")
+ end
+
ogg_format = {
:extension => 'ogg',
:parameters => [ [ '-c:a', 'copy', '-f', 'ogg' ] ]
diff --git a/record-and-playback/core/lib/recordandplayback/generators/events.rb b/record-and-playback/core/lib/recordandplayback/generators/events.rb
index 236c95d229..6dd353a388 100755
--- a/record-and-playback/core/lib/recordandplayback/generators/events.rb
+++ b/record-and-playback/core/lib/recordandplayback/generators/events.rb
@@ -692,5 +692,18 @@ module BigBlueButton
end
end
+ # Check if any screenshare files has audio
+ def self.screenshare_has_audio?(events_xml)
+ events = Nokogiri::XML(File.open(events_xml))
+ events.xpath('/recording/event[@eventname="StartWebRTCDesktopShareEvent"]').each do |event|
+ filename = event.at_xpath('filename').text
+ fileHasAudio = !BigBlueButton::EDL::Audio.audio_info(filename)[:audio].nil?
+ if fileHasAudio
+ return true
+ end
+ end
+ return false
+ end
+
end
end
--
GitLab