diff --git a/bigbluebutton-html5/imports/api/captions/server/methods.js b/bigbluebutton-html5/imports/api/captions/server/methods.js
index 2444e7baddd1e26f59937fc79206eb4494abb601..771580d706254482a35143bee9b3eff76b6e6d23 100644
--- a/bigbluebutton-html5/imports/api/captions/server/methods.js
+++ b/bigbluebutton-html5/imports/api/captions/server/methods.js
@@ -1,6 +1,8 @@
import { Meteor } from 'meteor/meteor';
import takeOwnership from '/imports/api/captions/server/methods/takeOwnership';
+import appendText from '/imports/api/captions/server/methods/appendText';
Meteor.methods({
takeOwnership,
+ appendText,
});
diff --git a/bigbluebutton-html5/imports/api/captions/server/methods/appendText.js b/bigbluebutton-html5/imports/api/captions/server/methods/appendText.js
new file mode 100644
index 0000000000000000000000000000000000000000..1afe735c0e6a5546eb30150efc36dcb44ed585b9
--- /dev/null
+++ b/bigbluebutton-html5/imports/api/captions/server/methods/appendText.js
@@ -0,0 +1,30 @@
+import axios from 'axios';
+import { check } from 'meteor/check';
+import Logger from '/imports/startup/server/logger';
+import {
+ generatePadId,
+} from '/imports/api/captions/server/helpers';
+import {
+ appendTextURL,
+} from '/imports/api/note/server/helpers';
+
+export default function appendText(body, text, locale) {
+ const { meetingId } = body;
+
+ check(meetingId, String);
+ check(text, String);
+ check(locale, String);
+
+ const padId = generatePadId(meetingId, locale);
+
+ axios({
+ method: 'get',
+ url: appendTextURL(padId, text),
+ responseType: 'json',
+ }).then((response) => {
+ const { status } = response;
+ if (status === 200) {
+ Logger.verbose(`Appended text for padId:${padId}`);
+ }
+ }).catch(error => Logger.error(`Could not append captions for padId=${padId}: ${error}`));
+}
diff --git a/bigbluebutton-html5/imports/api/note/server/helpers.js b/bigbluebutton-html5/imports/api/note/server/helpers.js
index 04a25ae1277034d622378ff5c333ca87b6c81d28..e9c3e640e59b910ed2afd5e8d4081cb251aeb4f2 100644
--- a/bigbluebutton-html5/imports/api/note/server/helpers.js
+++ b/bigbluebutton-html5/imports/api/note/server/helpers.js
@@ -10,6 +10,8 @@ const createPadURL = padId => `${BASE_URL}/createPad?apikey=${ETHERPAD.apikey}&p
const getReadOnlyIdURL = padId => `${BASE_URL}/getReadOnlyID?apikey=${ETHERPAD.apikey}&padID=${padId}`;
+const appendTextURL = (padId, text) => `${BASE_URL}/appendText?apikey=${ETHERPAD.apikey}&padID=${padId}&text=${encodeURIComponent(text)}`;
+
const generateNoteId = (meetingId) => {
const noteId = hashFNV32a(meetingId, true);
return noteId;
@@ -48,5 +50,6 @@ export {
getReadOnlyIdURL,
isEnabled,
getDataFromResponse,
+ appendTextURL,
processForNotePadOnly,
};
diff --git a/bigbluebutton-html5/imports/ui/components/captions/pad/component.jsx b/bigbluebutton-html5/imports/ui/components/captions/pad/component.jsx
index a8fb89c2302fec2c1a50d92a9ba658889b1a33db..23fcd6b55ad98879c014a6497091ea9f0cb2bc2c 100644
--- a/bigbluebutton-html5/imports/ui/components/captions/pad/component.jsx
+++ b/bigbluebutton-html5/imports/ui/components/captions/pad/component.jsx
@@ -1,9 +1,10 @@
-import React from 'react';
+import React, { PureComponent } from 'react';
import PropTypes from 'prop-types';
import { Session } from 'meteor/session';
import { defineMessages, injectIntl } from 'react-intl';
import injectWbResizeEvent from '/imports/ui/components/presentation/resize-wrapper/component';
import Button from '/imports/ui/components/button/component';
+import logger from '/imports/startup/client/logger';
import PadService from './service';
import CaptionsService from '/imports/ui/components/captions/service';
import { styles } from './styles';
@@ -21,11 +22,32 @@ const intlMessages = defineMessages({
id: 'app.captions.pad.ownership',
description: 'Label for taking ownership of closed captions pad',
},
+ interimResult: {
+ id: 'app.captions.pad.interimResult',
+ description: 'Title for speech recognition interim results',
+ },
+ dictationStart: {
+ id: 'app.captions.pad.dictationStart',
+ description: 'Label for starting speech recognition',
+ },
+ dictationStop: {
+ id: 'app.captions.pad.dictationStop',
+ description: 'Label for stoping speech recognition',
+ },
+ dictationOnDesc: {
+ id: 'app.captions.pad.dictationOnDesc',
+ description: 'Aria description for button that turns on speech recognition',
+ },
+ dictationOffDesc: {
+ id: 'app.captions.pad.dictationOffDesc',
+ description: 'Aria description for button that turns off speech recognition',
+ },
});
const propTypes = {
locale: PropTypes.string.isRequired,
ownerId: PropTypes.string.isRequired,
+ currentUserId: PropTypes.string.isRequired,
padId: PropTypes.string.isRequired,
readOnlyPadId: PropTypes.string.isRequired,
name: PropTypes.string.isRequired,
@@ -35,63 +57,204 @@ const propTypes = {
}).isRequired,
};
-const Pad = (props) => {
- const {
- locale,
- intl,
- padId,
- readOnlyPadId,
- ownerId,
- name,
- amIModerator,
- } = props;
-
- if (!amIModerator) {
- Session.set('openPanel', 'userlist');
+class Pad extends PureComponent {
+ static getDerivedStateFromProps(nextProps) {
+ if (nextProps.ownerId !== nextProps.currentUserId) {
+ return ({ listening: false });
+ }
return null;
}
- const url = PadService.getPadURL(padId, readOnlyPadId, ownerId);
-
- return (
- <div className={styles.pad}>
- <header className={styles.header}>
- <div className={styles.title}>
- <Button
- onClick={() => { Session.set('openPanel', 'userlist'); }}
- aria-label={intl.formatMessage(intlMessages.hide)}
- label={name}
- icon="left_arrow"
- className={styles.hideBtn}
- />
- </div>
- {CaptionsService.canIOwnThisPad(ownerId)
- ? (
+ constructor(props) {
+ super(props);
+
+ this.state = {
+ listening: false,
+ };
+
+ const { locale } = props;
+ this.recognition = CaptionsService.initSpeechRecognition(locale);
+
+ this.toggleListen = this.toggleListen.bind(this);
+ this.handleListen = this.handleListen.bind(this);
+ }
+
+ componentDidUpdate() {
+ const {
+ locale,
+ ownerId,
+ currentUserId,
+ } = this.props;
+
+ if (this.recognition) {
+ this.recognition.lang = locale;
+ if (ownerId !== currentUserId) this.recognition.stop();
+ }
+ }
+
+ toggleListen() {
+ const {
+ listening,
+ } = this.state;
+
+ this.setState({
+ listening: !listening,
+ }, this.handleListen);
+ }
+
+ handleListen() {
+ const {
+ locale,
+ } = this.props;
+
+ const {
+ listening,
+ } = this.state;
+
+ if (this.recognition) {
+ // Starts and stops the recognition when listening.
+ // Throws an error if start() is called on a recognition that has already been started.
+ if (listening) {
+ this.recognition.start();
+ } else {
+ this.recognition.stop();
+ }
+
+ // Stores the voice recognition results that have been verified.
+ let finalTranscript = '';
+
+ this.recognition.onresult = (event) => {
+ const {
+ resultIndex,
+ results,
+ } = event;
+
+ // Stores the first guess at what was recognised (Not always accurate).
+ let interimTranscript = '';
+
+ // Loops through the results to check if any of the entries have been validated,
+ // signaled by the isFinal flag.
+ for (let i = resultIndex; i < results.length; i += 1) {
+ const { transcript } = event.results[i][0];
+ if (results[i].isFinal) finalTranscript += `${transcript} `;
+ else interimTranscript += transcript;
+ }
+
+ // Adds the interimTranscript text to the iterimResultContainer to show
+ // what's being said while speaking.
+ if (this.iterimResultContainer) {
+ this.iterimResultContainer.innerHTML = interimTranscript;
+ }
+
+ const newEntry = finalTranscript !== '';
+
+ // Changes to the finalTranscript are shown to in the captions
+ if (newEntry) {
+ const text = finalTranscript.trimRight();
+ CaptionsService.appendText(text, locale);
+ finalTranscript = '';
+ }
+ };
+
+ this.recognition.onerror = (event) => {
+ logger.error({ logCode: 'captions_recognition' }, event.error);
+ };
+ }
+ }
+
+ render() {
+ const {
+ locale,
+ intl,
+ padId,
+ readOnlyPadId,
+ ownerId,
+ name,
+ amIModerator,
+ } = this.props;
+
+ if (!amIModerator) {
+ Session.set('openPanel', 'userlist');
+ return null;
+ }
+
+ const { listening } = this.state;
+ const url = PadService.getPadURL(padId, readOnlyPadId, ownerId);
+
+ return (
+ <div className={styles.pad}>
+ <header className={styles.header}>
+ <div className={styles.title}>
<Button
- icon="pen_tool"
- size="sm"
- ghost
- color="dark"
- hideLabel
- onClick={() => { CaptionsService.takeOwnership(locale); }}
- aria-label={intl.formatMessage(intlMessages.takeOwnership)}
- label={intl.formatMessage(intlMessages.takeOwnership)}
+ onClick={() => { Session.set('openPanel', 'userlist'); }}
+ aria-label={intl.formatMessage(intlMessages.hide)}
+ label={name}
+ icon="left_arrow"
+ className={styles.hideBtn}
/>
- ) : null
+ </div>
+ {CaptionsService.canIDictateThisPad(ownerId)
+ ? (
+ <span>
+ <Button
+ onClick={() => { this.toggleListen(); }}
+ label={listening
+ ? intl.formatMessage(intlMessages.dictationStop)
+ : intl.formatMessage(intlMessages.dictationStart)
+ }
+ aria-describedby="dictationBtnDesc"
+ color="primary"
+ disabled={!this.recognition}
+ />
+ <div id="dictationBtnDesc" hidden>
+ {listening
+ ? intl.formatMessage(intlMessages.dictationOffDesc)
+ : intl.formatMessage(intlMessages.dictationOnDesc)
+ }
+ </div>
+ </span>
+ ) : null
+ }
+ {CaptionsService.canIOwnThisPad(ownerId)
+ ? (
+ <Button
+ icon="pen_tool"
+ size="sm"
+ ghost
+ color="dark"
+ hideLabel
+ onClick={() => { CaptionsService.takeOwnership(locale); }}
+ aria-label={intl.formatMessage(intlMessages.takeOwnership)}
+ label={intl.formatMessage(intlMessages.takeOwnership)}
+ />
+ ) : null
}
- </header>
- <iframe
- title="etherpad"
- src={url}
- aria-describedby="padEscapeHint"
- />
- <span id="padEscapeHint" className={styles.hint} aria-hidden>
- {intl.formatMessage(intlMessages.tip)}
- </span>
- </div>
- );
-};
-
-Pad.propTypes = propTypes;
+ </header>
+ {listening ? (
+ <div>
+ <span className={styles.interimTitle}>
+ {intl.formatMessage(intlMessages.interimResult)}
+ </span>
+ <div
+ className={styles.processing}
+ ref={(node) => { this.iterimResultContainer = node; }}
+ />
+ </div>
+ ) : null
+ }
+ <iframe
+ title="etherpad"
+ src={url}
+ aria-describedby="padEscapeHint"
+ />
+ <span id="padEscapeHint" className={styles.hint} aria-hidden>
+ {intl.formatMessage(intlMessages.tip)}
+ </span>
+ </div>
+ );
+ }
+}
export default injectWbResizeEvent(injectIntl(Pad));
+
+Pad.propTypes = propTypes;
diff --git a/bigbluebutton-html5/imports/ui/components/captions/pad/container.jsx b/bigbluebutton-html5/imports/ui/components/captions/pad/container.jsx
index 98c22f4239d04b48db361b571e684aee306fba58..a8fcb24fc1de29f11001235eefa8741c1f27c5af 100644
--- a/bigbluebutton-html5/imports/ui/components/captions/pad/container.jsx
+++ b/bigbluebutton-html5/imports/ui/components/captions/pad/container.jsx
@@ -1,8 +1,9 @@
import React, { PureComponent } from 'react';
import { withTracker } from 'meteor/react-meteor-data';
import { Session } from 'meteor/session';
-import Pad from './component';
import CaptionsService from '/imports/ui/components/captions/service';
+import PadService from './service';
+import Pad from './component';
class PadContainer extends PureComponent {
render() {
@@ -25,12 +26,14 @@ export default withTracker(() => {
} = caption;
const { name } = caption ? caption.locale : '';
+
return {
locale,
name,
ownerId,
padId,
readOnlyPadId,
+ currentUserId: PadService.getCurrentUser().userId,
amIModerator: CaptionsService.amIModerator(),
};
})(PadContainer);
diff --git a/bigbluebutton-html5/imports/ui/components/captions/pad/service.js b/bigbluebutton-html5/imports/ui/components/captions/pad/service.js
index 237adc6f1765a040d07fa234f74d1775da896f13..a993748b313173f6dabebd8a47d54c0178258344 100644
--- a/bigbluebutton-html5/imports/ui/components/captions/pad/service.js
+++ b/bigbluebutton-html5/imports/ui/components/captions/pad/service.js
@@ -43,4 +43,5 @@ const getPadURL = (padId, readOnlyPadId, ownerId) => {
export default {
getPadURL,
+ getCurrentUser,
};
diff --git a/bigbluebutton-html5/imports/ui/components/captions/pad/styles.scss b/bigbluebutton-html5/imports/ui/components/captions/pad/styles.scss
index b9ff521619e196881be546b338f65e0b224b50d7..9bc5b5b84ffed9cf6fa7597f84bbc2d19efd3065 100644
--- a/bigbluebutton-html5/imports/ui/components/captions/pad/styles.scss
+++ b/bigbluebutton-html5/imports/ui/components/captions/pad/styles.scss
@@ -1,6 +1,22 @@
@import "/imports/ui/stylesheets/mixins/focus";
@import "/imports/ui/stylesheets/variables/_all";
+:root {
+ --speech-results-width: 22.3rem;
+}
+
+.processing {
+ margin-top: var(--lg-padding-y);
+ margin-bottom: var(--lg-padding-y);
+ min-height: var(--jumbo-padding-x);
+ width: var(--speech-results-width);
+ border: solid var(--border-size-small) var(--color-gray);
+}
+
+.interimTitle {
+ margin-top: var(--lg-padding-y);
+}
+
.pad {
background-color: var(--color-white);
padding: var(--md-padding-x);
diff --git a/bigbluebutton-html5/imports/ui/components/captions/service.js b/bigbluebutton-html5/imports/ui/components/captions/service.js
index af830a816e457cadb791e989aebe88d05e0add59..17b31c7f51f560e2655d16ccdd875ff9be00a5b0 100644
--- a/bigbluebutton-html5/imports/ui/components/captions/service.js
+++ b/bigbluebutton-html5/imports/ui/components/captions/service.js
@@ -68,6 +68,16 @@ const takeOwnership = (locale) => {
makeCall('takeOwnership', locale);
};
+const formatEntry = (entry) => {
+ const letterIndex = entry.charAt(0) === ' ' ? 1 : 0;
+ const formattedEntry = `${entry.charAt(letterIndex).toUpperCase() + entry.slice(letterIndex + 1)}.\n\n`;
+ return formattedEntry;
+};
+
+const appendText = (text, locale) => {
+ makeCall('appendText', formatEntry(text), locale);
+};
+
const canIOwnThisPad = (ownerId) => {
const { userID } = Auth;
if (!CAPTIONS_CONFIG.takeOwnership) return false;
@@ -75,6 +85,15 @@ const canIOwnThisPad = (ownerId) => {
return ownerId !== userID;
};
+const canIDictateThisPad = (ownerId) => {
+ const { userID } = Auth;
+ if (!CAPTIONS_CONFIG.enableDictation) return false;
+ if (ownerId === '') return false;
+ const SpeechRecognitionAPI = getSpeechRecognitionAPI();
+ if (!SpeechRecognitionAPI) return false;
+ return ownerId === userID;
+};
+
const setActiveCaptions = (locale) => {
Session.set('activeCaptions', locale);
};
@@ -137,13 +156,32 @@ const amIModerator = () => {
return mapUser(currentUser).isModerator;
};
+const getSpeechRecognitionAPI = () => {
+ return window.SpeechRecognition || window.webkitSpeechRecognition;
+};
+
+const initSpeechRecognition = (locale) => {
+ const SpeechRecognitionAPI = getSpeechRecognitionAPI();
+ let recognition = null;
+ if (SpeechRecognitionAPI) {
+ recognition = new SpeechRecognitionAPI();
+ recognition.continuous = true;
+ recognition.interimResults = true;
+ recognition.lang = locale;
+ }
+
+ return recognition;
+};
+
export default {
getCaptionsData,
getAvailableLocales,
getOwnedLocales,
takeOwnership,
+ appendText,
getCaptions,
canIOwnThisPad,
+ canIDictateThisPad,
getCaptionsSettings,
isCaptionsEnabled,
isCaptionsAvailable,
@@ -152,4 +190,5 @@ export default {
activateCaptions,
formatCaptionsText,
amIModerator,
+ initSpeechRecognition,
};
diff --git a/bigbluebutton-html5/private/config/settings.yml b/bigbluebutton-html5/private/config/settings.yml
index 6e8e747126a111f19caad55b572ee38c2a9e5f43..9644413dc35a8ef26965ac9d7f811eaad2d0d92b 100755
--- a/bigbluebutton-html5/private/config/settings.yml
+++ b/bigbluebutton-html5/private/config/settings.yml
@@ -144,6 +144,7 @@ public:
max_custom: 5
captions:
enabled: false
+ enableDictation: false
backgroundColor: "#000000"
fontColor: "#FFFFFF"
fontFamily: Calibri
diff --git a/bigbluebutton-html5/private/locales/en.json b/bigbluebutton-html5/private/locales/en.json
index d917c52cef3a6618c7a9967f39540870470fd6b6..0fb10450820472af6cbf778240c817895d76c5b0 100755
--- a/bigbluebutton-html5/private/locales/en.json
+++ b/bigbluebutton-html5/private/locales/en.json
@@ -37,6 +37,11 @@
"app.captions.pad.hide": "Hide closed captions",
"app.captions.pad.tip": "Press Esc to focus editor toolbar",
"app.captions.pad.ownership": "Take ownership",
+ "app.captions.pad.interimResult": "Interim results",
+ "app.captions.pad.dictationStart": "Start Dictation",
+ "app.captions.pad.dictationStop": "Stop Dictation",
+ "app.captions.pad.dictationOnDesc": "Turns speech recognition on",
+ "app.captions.pad.dictationOffDesc": "Turns speech recognition off",
"app.note.title": "Shared Notes",
"app.note.label": "Note",
"app.note.hideNoteLabel": "Hide note",