From c6e09f52a1c45f91b348dc84cd7f4837c9e7ce56 Mon Sep 17 00:00:00 2001
From: Calvin Walton <calvin.walton@kepstin.ca>
Date: Fri, 10 May 2019 14:06:23 -0400
Subject: [PATCH] RaP Caption inbox: read uploaded caption json file
Some additional validation/normalization is added for the language tag,
and some cleanup has been done for logging.
---
record-and-playback/core/Gemfile | 20 +++--
record-and-playback/core/Gemfile.lock | 6 ++
.../core/scripts/rap-caption-inbox.rb | 84 +++++++++++++++----
.../systemd/bbb-rap-caption-inbox.service | 9 ++
4 files changed, 96 insertions(+), 23 deletions(-)
create mode 100644 record-and-playback/core/systemd/bbb-rap-caption-inbox.service
diff --git a/record-and-playback/core/Gemfile b/record-and-playback/core/Gemfile
index 1ad4deb09d..adbfc21e84 100644
--- a/record-and-playback/core/Gemfile
+++ b/record-and-playback/core/Gemfile
@@ -19,16 +19,18 @@
source "https://rubygems.org"
-gem "redis"
-gem "nokogiri"
-gem "loofah"
-gem "rubyzip"
+gem "absolute_time"
gem "builder"
-gem "trollop", "2.1.3"
-gem "open4"
gem "fastimage"
-gem "absolute_time"
-gem "jwt"
-gem "java_properties"
gem "fnv"
+gem "java_properties"
+gem "journald-logger"
+gem "jwt"
+gem "locale"
+gem "loofah"
+gem "nokogiri"
+gem "open4"
gem "rb-inotify"
+gem "redis"
+gem "rubyzip"
+gem "trollop", "2.1.3"
diff --git a/record-and-playback/core/Gemfile.lock b/record-and-playback/core/Gemfile.lock
index b3fdcdf735..24dc5b1cbf 100644
--- a/record-and-playback/core/Gemfile.lock
+++ b/record-and-playback/core/Gemfile.lock
@@ -8,7 +8,11 @@ GEM
ffi (1.10.0)
fnv (0.2.0)
java_properties (0.0.4)
+ journald-logger (2.0.4)
+ journald-native (~> 1.0)
+ journald-native (1.0.11)
jwt (2.1.0)
+ locale (2.1.2)
loofah (2.2.3)
crass (~> 1.0.2)
nokogiri (>= 1.5.9)
@@ -31,7 +35,9 @@ DEPENDENCIES
fastimage
fnv
java_properties
+ journald-logger
jwt
+ locale
loofah
nokogiri
open4
diff --git a/record-and-playback/core/scripts/rap-caption-inbox.rb b/record-and-playback/core/scripts/rap-caption-inbox.rb
index 848acd2538..b9569efe3f 100755
--- a/record-and-playback/core/scripts/rap-caption-inbox.rb
+++ b/record-and-playback/core/scripts/rap-caption-inbox.rb
@@ -18,24 +18,22 @@
# along with BigBlueButton. If not, see <http://www.gnu.org/licenses/>.
require "rubygems"
+require "bundler/setup"
+
+require File.expand_path("../../lib/recordandplayback", __FILE__)
+
+require "journald-logger"
+require "locale"
require "rb-inotify"
require "yaml"
-require "logger"
-props = YAML::load(File.open("bigbluebutton.yml"))
+# Read configuration and set up logger
-log_dir = props["log_dir"]
-logger = Logger.new(STDERR)
-logger.level = Logger::INFO
-# TODO: I need to set the Bigbluebutton logger here if I load the other rap code
+props = YAML::load(File.open(File.expand_path("../bigbluebutton.yml", __FILE__)))
-def handle_caption_file(filename)
- # There's a possible race condition where we can be notified twice for a new
- # file. That's fine, just do nothing the second time.
- return unless File.exist?(filename)
-
- logger.info("Found new caption index file #{filename}")
-end
+log_dir = props["log_dir"]
+logger = Journald::Logger.new("bbb-rap-caption-inbox")
+BigBlueButton.logger = logger
captions_dir = props["captions_dir"]
unless captions_dir
@@ -44,6 +42,64 @@ unless captions_dir
end
captions_inbox_dir = File.join(captions_dir, "inbox")
+# Internal error classes
+
+# Base class for internal errors
+class CaptionError < StandardError
+end
+
+# Indicates that uploaded caption files are invalid (unrecoverable)
+class InvalidCaptionError < CaptionError
+end
+
+# Implementation
+
+def caption_file_notify(json_filename)
+ # There's a possible race condition where we can be notified twice for a new
+ # file. That's fine, just do nothing the second time.
+ return unless File.exist?(json_filename)
+
+ logger.info("Found new caption index file #{json_filename}")
+
+ # TODO: Rather than do anything directly in this script, it should create a
+ # queue job (resque?) that does the actual work.
+
+ new_caption_info = File.open(json_filename) { |file| JSON.parse(file) }
+ logger.tag(record_id: new_caption_info["record_id"]) do
+
+ # TODO: This is racy if multiple tools are editing the captions.json file
+ captions_info = begin
+ File.open(
+ File.join(captions_dir, new_caption_info["record_id"], "captions.json")
+ ) do |file|
+ JSON.parse(file)
+ end
+ rescue
+ # No captions file or cannot be read, assume none present
+ []
+ end
+
+ langtag = Locale::Tag::Rfc.parse(new_caption_info["lang"])
+ raise InvalidCaptionError, "Language tag is not well-formed" unless langtag
+
+ # Remove the info for an existing matching track
+ captions_info.delete_if do |caption_info|
+ caption_info["lang"] == new_caption_info["lang"] &&
+ caption_info["kind"] == new_caption_info["kind"]
+ end
+
+ captions_info << {
+ "kind" => new_caption_info["kind"],
+ "label" => new_caption_info["label"],
+ "lang" => langtag.to_s,
+ "source" => "upload",
+ }
+
+ dest_filename = "#{captions_info["kind"]}_#{captions_info["lang"]}.vtt"
+
+ end
+end
+
logger.info("Setting up inotify watch on #{captions_inbox_dir}")
notifier = INotify::Notifier.new
notifier.watch(captions_inbox_dir, :moved_to, :create) do |event|
@@ -54,7 +110,7 @@ end
logger.info("Checking for missed/skipped caption files")
Dir.glob(File.join(captions_inbox_dir, "*-track.json")).each do |filename|
- handle_caption_file(filename)
+ caption_file_notify(filename)
end
logger.info("Waiting for new caption files...")
diff --git a/record-and-playback/core/systemd/bbb-rap-caption-inbox.service b/record-and-playback/core/systemd/bbb-rap-caption-inbox.service
new file mode 100644
index 0000000000..2796163c81
--- /dev/null
+++ b/record-and-playback/core/systemd/bbb-rap-caption-inbox.service
@@ -0,0 +1,9 @@
+[Unit]
+Description=BigBlueButton recording caption upload handler
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bigbluebutton/core/scripts/rap-caption-inbox.rb
+User=bigbluebutton
+Slice=bbb_record_core.slice
+Restart=on-failure
--
GitLab