Unverified Commit df6c2c54 authored by Dan Caseley's avatar Dan Caseley Committed by GitHub
Browse files

[TESTS] Idempotence for e2e tests (#2308)



* Idempotence for Assorted

* Idempotence for Onboarding

* Idempotence for Rooms

* Remove redundant expects

* Fixes and improvements

* Remove unneeded sleeps

* Make stable following merge

* Try solving early taps without long sleep

Try solving early taps without long sleep (cont)

Temporary CircleCI hack for quicker testing

Add screenshots to CircleCI for failed tests

Try solving early taps without long sleep (cont. 2)

Revert "Temporary CircleCI hack for quicker testing"

This reverts commit 4abef3a5827910c05b12ac8b8380275b60e8af4f.

* Fix flaky test with a fluent wait on the tap

* Add some new sleeps to workaround #2324

* Add test artifacts to gitignore

* More longpress for dodgy taps, wait for pin response

Co-authored-by: default avatarDiego Mello <diegolmello@gmail.com>
parent a5c81b26
......@@ -135,6 +135,10 @@ commands:
name: Test
command: |
npx detox test << parameters.folder >> --configuration ios.sim.release --cleanup
when: always
- store_artifacts:
path: ./artifacts
# JOBS
jobs:
......
......@@ -58,6 +58,7 @@ buck-out/
coverage
artifacts
.vscode/
e2e/docker/rc_test_env/docker-compose.yml
e2e/docker/data/db
\ No newline at end of file
......@@ -29,10 +29,15 @@ const data = {
}
},
channels: {
public: {
detoxpublic: {
name: 'detox-public'
}
},
groups: {
private: {
name: `detox-private-${ value }`
}
},
registeringUser: {
username: `newuser${ value }`,
password: `password${ value }`,
......
......@@ -29,10 +29,15 @@ const data = {
}
},
channels: {
public: {
detoxpublic: {
name: 'detox-public'
}
},
groups: {
private: {
name: `detox-private-${ value }`
}
},
registeringUser: {
username: `newuser${ value }`,
password: `password${ value }`,
......
......@@ -29,10 +29,15 @@ const data = {
}
},
channels: {
public: {
detoxpublic: {
name: 'detox-public'
}
},
groups: {
private: {
name: `detox-private-${ value }`
}
},
registeringUser: {
username: `newuser${ value }`,
password: `password${ value }`,
......
......@@ -31,7 +31,6 @@ async function login(username, password) {
await waitFor(element(by.id('login-view'))).toBeVisible().withTimeout(2000);
await element(by.id('login-view-email')).replaceText(username);
await element(by.id('login-view-password')).replaceText(password);
await sleep(300);
await element(by.id('login-view-submit')).tap();
await waitFor(element(by.id('rooms-list-view'))).toBeVisible().withTimeout(10000);
}
......@@ -61,6 +60,33 @@ async function mockMessage(message) {
await element(by.label(`${ data.random }${ message }`)).atIndex(0).tap();
};
async function starMessage(message){
const messageLabel = `${ data.random }${ message }`
await waitFor(element(by.label(messageLabel))).toBeVisible().withTimeout(5000);
await element(by.label(messageLabel)).atIndex(0).longPress();
await expect(element(by.id('action-sheet'))).toExist();
await expect(element(by.id('action-sheet-handle'))).toBeVisible();
await element(by.id('action-sheet-handle')).swipe('up', 'fast', 0.5);
await element(by.label('Star')).tap();
await waitFor(element(by.id('action-sheet'))).toNotExist().withTimeout(5000);
};
async function pinMessage(message){
const messageLabel = `${ data.random }${ message }`
await waitFor(element(by.label(messageLabel)).atIndex(0)).toExist();
await element(by.label(messageLabel)).atIndex(0).longPress();
await expect(element(by.id('action-sheet'))).toExist();
await expect(element(by.id('action-sheet-handle'))).toBeVisible();
await element(by.id('action-sheet-handle')).swipe('up', 'fast', 0.5);
await element(by.label('Pin')).tap();
await waitFor(element(by.id('action-sheet'))).toNotExist().withTimeout(5000);
}
async function dismissReviewNag(){
await waitFor(element(by.text('Are you enjoying this app?'))).toExist().withTimeout(60000);
await element(by.label('No').and(by.type('_UIAlertControllerActionView'))).tap(); // Tap `no` on ask for review alert
}
async function tapBack() {
await element(by.id('header-back')).atIndex(0).tap();
}
......@@ -74,7 +100,22 @@ async function searchRoom(room) {
await expect(element(by.id('rooms-list-view-search-input'))).toExist();
await waitFor(element(by.id('rooms-list-view-search-input'))).toExist().withTimeout(5000);
await element(by.id('rooms-list-view-search-input')).typeText(room);
await sleep(2000);
}
async function tryTapping(theElement, timeout, longtap = false){
try {
if(longtap){
await theElement.longPress()
} else {
await theElement.tap()
}
} catch(e) {
if(timeout <= 0){ //TODO: Maths. How closely has the timeout been honoured here?
throw e
}
await sleep(100)
await tryTapping(theElement, timeout - 100)
}
}
module.exports = {
......@@ -84,7 +125,11 @@ module.exports = {
login,
logout,
mockMessage,
starMessage,
pinMessage,
dismissReviewNag,
tapBack,
sleep,
searchRoom
searchRoom,
tryTapping
};
\ No newline at end of file
......@@ -38,7 +38,7 @@ const createUser = async (username, password, name, email) => {
}
const createChannelIfNotExists = async (channelname) => {
console.log(`Creating channel ${channelname}`)
console.log(`Creating public channel ${channelname}`)
try {
await rocketchat.post('channels.create', {
"name": channelname
......@@ -49,7 +49,24 @@ const createChannelIfNotExists = async (channelname) => {
} catch (infoError) {
console.log(JSON.stringify(createError))
console.log(JSON.stringify(infoError))
throw "Failed to find or create channel"
throw "Failed to find or create public channel"
}
}
}
const createGroupIfNotExists = async (groupname) => {
console.log(`Creating private group ${groupname}`)
try {
await rocketchat.post('groups.create', {
"name": groupname
})
} catch (createError) {
try { //Maybe it exists already?
await rocketchat.get(`group.info?roomName=${groupname}`)
} catch (infoError) {
console.log(JSON.stringify(createError))
console.log(JSON.stringify(infoError))
throw "Failed to find or create private group"
}
}
}
......@@ -71,6 +88,15 @@ const setup = async () => {
}
}
await login(data.users.regular.username, data.users.regular.password)
for (var groupKey in data.groups) {
if (data.groups.hasOwnProperty(groupKey)) {
const group = data.groups[groupKey]
await createGroupIfNotExists(group.name)
}
}
return
}
......
......@@ -9,12 +9,12 @@ const checkServer = async(server) => {
await element(by.id('rooms-list-view-sidebar')).tap();
await waitFor(element(by.id('sidebar-view'))).toBeVisible().withTimeout(2000);
await waitFor(element(by.label(label))).toBeVisible().withTimeout(60000);
await expect(element(by.label(label))).toBeVisible();
await element(by.id('sidebar-close-drawer')).tap();
}
describe('Change server', () => {
before(async() => {
await device.launchApp({ permissions: { notifications: 'YES' }, delete: true });
await navigateToLogin();
await login(data.users.regular.username, data.users.regular.password);
await waitFor(element(by.id('rooms-list-view'))).toBeVisible().withTimeout(10000);
......@@ -28,8 +28,6 @@ describe('Change server', () => {
await sleep(5000);
await element(by.id('rooms-list-header-server-dropdown-button')).tap();
await waitFor(element(by.id('rooms-list-header-server-dropdown'))).toBeVisible().withTimeout(5000);
await expect(element(by.id('rooms-list-header-server-dropdown'))).toExist();
await sleep(1000);
await element(by.id('rooms-list-header-server-add')).tap();
// TODO: refactor
......@@ -37,19 +35,16 @@ describe('Change server', () => {
await element(by.id('new-server-view-input')).replaceText(data.alternateServer);
await element(by.id('new-server-view-button')).tap();
await waitFor(element(by.id('workspace-view'))).toBeVisible().withTimeout(60000);
await expect(element(by.id('workspace-view'))).toBeVisible();
await element(by.id('workspace-view-register')).tap();
await waitFor(element(by.id('register-view'))).toBeVisible().withTimeout(2000);
await expect(element(by.id('register-view'))).toBeVisible();
// Register new user
await element(by.id('register-view-name')).replaceText(data.registeringUser.username);
await element(by.id('register-view-username')).replaceText(data.registeringUser.username);
await element(by.id('register-view-email')).replaceText(data.registeringUser.email);
await element(by.id('register-view-password')).replaceText(data.registeringUser.password);
await sleep(1000);
await element(by.id('register-view-submit')).tap();
await waitFor(element(by.id('rooms-list-view'))).toBeVisible().withTimeout(60000);
await expect(element(by.id('rooms-list-view'))).toBeVisible();
// For a sanity test, to make sure roomslist is showing correct rooms
// app CANNOT show public room created on previous tests
......@@ -59,11 +54,8 @@ describe('Change server', () => {
});
it('should change back', async() => {
await sleep(5000);
await element(by.id('rooms-list-header-server-dropdown-button')).tap();
await waitFor(element(by.id('rooms-list-header-server-dropdown'))).toBeVisible().withTimeout(5000);
await expect(element(by.id('rooms-list-header-server-dropdown'))).toExist();
await sleep(1000);
await element(by.id(`rooms-list-header-server-${ data.server }`)).tap();
await waitFor(element(by.id('rooms-list-view'))).toBeVisible().withTimeout(10000);
await checkServer(data.server);
......
......@@ -23,28 +23,16 @@ describe('Broadcast room', () => {
await waitFor(element(by.id('select-users-view'))).toBeVisible().withTimeout(2000);
await element(by.id('select-users-view-search')).replaceText(otheruser.username);
await waitFor(element(by.id(`select-users-view-item-${ otheruser.username }`))).toBeVisible().withTimeout(60000);
await expect(element(by.id(`select-users-view-item-${ otheruser.username }`))).toBeVisible();
await element(by.id(`select-users-view-item-${ otheruser.username }`)).tap();
await waitFor(element(by.id(`selected-user-${ otheruser.username }`))).toBeVisible().withTimeout(5000);
await sleep(1000);
await element(by.id('selected-users-view-submit')).tap();
await sleep(1000);
await waitFor(element(by.id('create-channel-view'))).toExist().withTimeout(5000);
await element(by.id('create-channel-name')).replaceText(`broadcast${ data.random }`);
await sleep(2000);
await element(by.id('create-channel-broadcast')).tap();
if (device.getPlatform() === 'ios') { //Because this tap is FLAKY on iOS
await expect(element(by.id('create-channel-broadcast'))).toHaveValue('1')
}
await sleep(500);
await element(by.id('create-channel-broadcast')).longPress(); //https://github.com/facebook/react-native/issues/28032
await element(by.id('create-channel-submit')).tap();
await waitFor(element(by.id('room-view'))).toBeVisible().withTimeout(60000);
await expect(element(by.id('room-view'))).toBeVisible();
await waitFor(element(by.id(`room-view-title-broadcast${ data.random }`))).toBeVisible().withTimeout(60000);
await expect(element(by.id(`room-view-title-broadcast${ data.random }`))).toBeVisible();
await sleep(1000);
await element(by.id('room-view-header-actions')).tap();
await sleep(1000);
await waitFor(element(by.id('room-actions-view'))).toBeVisible().withTimeout(5000);
await element(by.id('room-actions-info')).tap();
await waitFor(element(by.id('room-info-view'))).toBeVisible().withTimeout(2000);
......@@ -64,25 +52,19 @@ describe('Broadcast room', () => {
it('should login as user without write message authorization and enter room', async() => {
await device.launchApp({ permissions: { notifications: 'YES' }, delete: true });
await navigateToLogin();
await element(by.id('login-view-email')).replaceText(otheruser.username);
await element(by.id('login-view-password')).replaceText(otheruser.password);
await sleep(1000);
await element(by.id('login-view-submit')).tap();
await login(otheruser.username, otheruser.password);
//await waitFor(element(by.id('two-factor'))).toBeVisible().withTimeout(5000);
//await expect(element(by.id('two-factor'))).toBeVisible();
//const code = GA.gen(data.alternateUserTOTPSecret);
//await element(by.id('two-factor-input')).replaceText(code);
//await sleep(1000);
//await element(by.id('two-factor-send')).tap();
await waitFor(element(by.id('rooms-list-view'))).toBeVisible().withTimeout(10000);
await searchRoom(`broadcast${ data.random }`);
await waitFor(element(by.id(`rooms-list-view-item-broadcast${ data.random }`))).toExist().withTimeout(60000);
await expect(element(by.id(`rooms-list-view-item-broadcast${ data.random }`))).toExist();
await element(by.id(`rooms-list-view-item-broadcast${ data.random }`)).tap();
await waitFor(element(by.id('room-view'))).toBeVisible().withTimeout(5000);
await waitFor(element(by.id(`room-view-title-broadcast${ data.random }`))).toBeVisible().withTimeout(60000);
await expect(element(by.id(`room-view-title-broadcast${ data.random }`))).toBeVisible();
await sleep(1000);
});
it('should not have messagebox', async() => {
......@@ -95,7 +77,6 @@ describe('Broadcast room', () => {
it('should have the message created earlier', async() => {
await waitFor(element(by.label(`${ data.random }message`)).atIndex(0)).toBeVisible().withTimeout(60000);
await expect(element(by.label(`${ data.random }message`)).atIndex(0)).toBeVisible();
});
it('should have reply button', async() => {
......@@ -104,9 +85,7 @@ describe('Broadcast room', () => {
it('should tap on reply button and navigate to direct room', async() => {
await element(by.id('message-broadcast-reply')).tap();
await sleep(1000);
await waitFor(element(by.id(`room-view-title-${ testuser.username }`))).toBeVisible().withTimeout(5000);
await expect(element(by.id(`room-view-title-${ testuser.username }`))).toBeVisible();
});
it('should reply broadcasted message', async() => {
......
......@@ -13,7 +13,7 @@ async function waitForToast() {
// await expect(element(by.id('toast'))).toBeVisible();
// await waitFor(element(by.id('toast'))).toBeNotVisible().withTimeout(10000);
// await expect(element(by.id('toast'))).toBeNotVisible();
await sleep(5000);
await sleep(1);
}
describe('Profile screen', () => {
......@@ -24,7 +24,6 @@ describe('Profile screen', () => {
await element(by.id('rooms-list-view-sidebar')).tap();
await waitFor(element(by.id('sidebar-view'))).toBeVisible().withTimeout(2000);
await waitFor(element(by.id('sidebar-profile'))).toBeVisible().withTimeout(2000);
await expect(element(by.id('sidebar-profile'))).toBeVisible();
await element(by.id('sidebar-profile')).tap();
await waitFor(element(by.id('profile-view'))).toBeVisible().withTimeout(2000);
});
......@@ -60,22 +59,18 @@ describe('Profile screen', () => {
it('should have reset avatar button', async() => {
await waitFor(element(by.id('profile-view-reset-avatar'))).toExist().whileElement(by.id('profile-view-list')).scroll(scrollDown, 'down');
await expect(element(by.id('profile-view-reset-avatar'))).toExist();
});
it('should have upload avatar button', async() => {
await waitFor(element(by.id('profile-view-upload-avatar'))).toExist().whileElement(by.id('profile-view-list')).scroll(scrollDown, 'down');
await expect(element(by.id('profile-view-upload-avatar'))).toExist();
});
it('should have avatar url button', async() => {
await waitFor(element(by.id('profile-view-avatar-url-button'))).toExist().whileElement(by.id('profile-view-list')).scroll(scrollDown, 'down');
await expect(element(by.id('profile-view-avatar-url-button'))).toExist();
});
it('should have submit button', async() => {
await waitFor(element(by.id('profile-view-submit'))).toExist().whileElement(by.id('profile-view-list')).scroll(scrollDown, 'down');
await expect(element(by.id('profile-view-submit'))).toExist();
});
});
......@@ -84,9 +79,7 @@ describe('Profile screen', () => {
await element(by.type('UIScrollView')).atIndex(1).swipe('down');
await element(by.id('profile-view-name')).replaceText(`${ profileChangeUser.username }new`);
await element(by.id('profile-view-username')).replaceText(`${ profileChangeUser.username }new`);
await sleep(1000);
await element(by.type('UIScrollView')).atIndex(1).swipe('up');
await sleep(1000);
await element(by.id('profile-view-submit')).tap();
await waitForToast();
});
......@@ -103,7 +96,6 @@ describe('Profile screen', () => {
it('should reset avatar', async() => {
await element(by.type('UIScrollView')).atIndex(1).swipe('up');
await sleep(1000);
await element(by.id('profile-view-reset-avatar')).tap();
await waitForToast();
});
......
const {
device, expect, element, by, waitFor
} = require('detox');
const { navigateToLogin, login } = require('../../helpers/app');
const data = require('../../data');
const testuser = data.users.regular
describe('Settings screen', () => {
before(async() => {
await device.launchApp({ newInstance: true });
await device.launchApp({ permissions: { notifications: 'YES' }, delete: true });
await navigateToLogin();
await login(testuser.username, testuser.password);
await waitFor(element(by.id('rooms-list-view'))).toBeVisible().withTimeout(10000);
await expect(element(by.id('rooms-list-view'))).toBeVisible();
await element(by.id('rooms-list-view-sidebar')).tap();
await waitFor(element(by.id('sidebar-view'))).toBeVisible().withTimeout(2000);
await waitFor(element(by.id('sidebar-settings'))).toBeVisible().withTimeout(2000);
......
......@@ -2,12 +2,12 @@ const {
device, expect, element, by, waitFor
} = require('detox');
const data = require('../../data');
const { mockMessage, tapBack, sleep, searchRoom } = require('../../helpers/app');
const { navigateToLogin, login, mockMessage, tapBack, sleep, searchRoom } = require('../../helpers/app');
const room = 'detox-public';
const testuser = data.users.regular
const room = data.channels.detoxpublic.name;
async function navigateToRoom() {
await sleep(2000);
await searchRoom(room);
await waitFor(element(by.id(`rooms-list-view-item-${ room }`)).atIndex(0)).toBeVisible().withTimeout(60000);
await element(by.id(`rooms-list-view-item-${ room }`)).atIndex(0).tap();
......@@ -15,15 +15,15 @@ async function navigateToRoom() {
}
async function navigateToRoomActions() {
await sleep(2000);
await element(by.id('room-view-header-actions')).tap();
await sleep(2000);
await waitFor(element(by.id('room-actions-view'))).toBeVisible().withTimeout(5000);
}
describe('Join public room', () => {
before(async() => {
await device.launchApp({ newInstance: true });
await device.launchApp({ permissions: { notifications: 'YES' }, delete: true });
await navigateToLogin();
await login(testuser.username, testuser.password);
await navigateToRoom();
});
......@@ -167,9 +167,7 @@ describe('Join public room', () => {
await element(by.text('Yes, leave it!')).tap();
await waitFor(element(by.id('rooms-list-view'))).toBeVisible().withTimeout(10000);
// await element(by.id('rooms-list-view-search')).typeText('');
await sleep(2000);
await waitFor(element(by.id(`rooms-list-view-item-${ room }`))).toBeNotVisible().withTimeout(60000);
await expect(element(by.id(`rooms-list-view-item-${ room }`))).toBeNotVisible();
});
});
});
const {
expect, element, by, waitFor
} = require('detox');
const { sleep } = require('../../helpers/app');
const { navigateToLogin, login, sleep } = require('../../helpers/app');
const data = require('../../data');
const testuser = data.users.regular
async function waitForToast() {
await sleep(5000);
await sleep(1);
}
describe('Status screen', () => {
before(async() => {
before(async () => {
await device.launchApp({ permissions: { notifications: 'YES' }, delete: true });
await navigateToLogin();
await login(testuser.username, testuser.password);
await element(by.id('rooms-list-view-sidebar')).tap();
await waitFor(element(by.id('sidebar-view'))).toBeVisible().withTimeout(2000);
await waitFor(element(by.id('sidebar-custom-status'))).toBeVisible().withTimeout(2000);
......@@ -17,30 +24,27 @@ describe('Status screen', () => {
await waitFor(element(by.id('status-view'))).toBeVisible().withTimeout(2000);
});
describe('Render', async() => {
it('should have status input', async() => {
describe('Render', async () => {
it('should have status input', async () => {
await expect(element(by.id('status-view-input'))).toBeVisible();
await expect(element(by.id('status-view-online'))).toExist();
await expect(element(by.id('status-view-busy'))).toExist();
await expect(element(by.id('status-view-away'))).toExist();
await expect(element(by.id('status-view-offline'))).toExist();
});
});
describe('Usage', async() => {
it('should change status', async() => {
await sleep(1000);
});
});
describe('Usage', async () => {
it('should change status', async () => {
await element(by.id('status-view-busy')).tap();
await sleep(1000);
await expect(element(by.id('status-view-current-busy'))).toExist();
});
});
it('should change status text', async() => {
it('should change status text', async () => {
await element(by.id('status-view-input')).replaceText('status-text-new');
await sleep(1000);
await element(by.id('status-view-submit')).tap();
await waitForToast();
await waitFor(element(by.label('status-text-new').withAncestor(by.id('sidebar-custom-status')))).toBeVisible().withTimeout(2000);
});
});
});
});
});
});
\ No newline at end of file
const detox = require('detox');
const config = require('../../package.json').detox;
const dataSetup = require('../helpers/data_setup')
const adapter = require('detox/runners/mocha/adapter');
before(async() => {
await dataSetup()
await detox.init(config, { launchApp: false });
await device.launchApp({ permissions: { notifications: 'YES' } });
await Promise.all([dataSetup(), detox.init(config, { launchApp: false })])
//await dataSetup()
//await detox.init(config, { launchApp: false });
//await device.launchApp({ permissions: { notifications: 'YES' } });
});
beforeEach(async function() {
await adapter.beforeEach(this);
});
afterEach(async function() {
await adapter.afterEach(this);
});
after(async() => {
......
......@@ -31,7 +31,6 @@ describe('Onboarding', () => {
it('should navigate to join a workspace', async() => {
await element(by.id('join-workspace')).tap();
await waitFor(element(by.id('new-server-view'))).toBeVisible().withTimeout(60000);
await expect(element(by.id('new-server-view'))).toBeVisible();
});
it('should enter an invalid server and get error', async() => {
......@@ -39,14 +38,12 @@ describe('Onboarding', () => {
await element(by.id('new-server-view-button')).tap();
const errorText = 'Oops!';
await waitFor(element(by.text(errorText))).toBeVisible().withTimeout(60000);
await expect(element(by.text(errorText))).toBeVisible();
await element(by.text('OK')).tap();
});
it('should tap on "Join our open workspace" and navigate', async() => {
await element(by.id('new-server-view-open')).tap();
await waitFor(element(by.id('workspace-view'))).toBeVisible().withTimeout(60000);
await expect(element(by.id('workspace-view'))).toBeVisible();
});
it('should enter a valid server without login services and navigate to login', async() => {
......@@ -57,7 +54,6 @@ describe('Onboarding', () => {
await element(by.id('new-server-view-input')).replaceText(data.server);
await element(by.id('new-server-view-button')).tap();
await waitFor(element(by.id('workspace-view'))).toBeVisible().withTimeout(60000);
await expect(element(b