Skip to content

Reduce FM Integration Test Impact & Run Time #4676

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Mar 30, 2021
Merged

Reduce FM Integration Test Impact & Run Time #4676

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b491044
Initial Commit
zwu52 Mar 24, 2021
c1877d5
Let get_delete_token run first
zwu52 Mar 24, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
157 changes: 157 additions & 0 deletions integration/messaging/test/test-receive-background.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
/**
* @license
* Copyright 2017 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

const testServer = require('./utils/test-server');
const sendMessage = require('./utils/sendMessage');
const retrieveToken = require('./utils/retrieveToken');
const seleniumAssistant = require('selenium-assistant');
const getReceivedBackgroundMessages = require('./utils/getReceivedBackgroundMessages');
const createPermittedWebDriver = require('./utils/createPermittedWebDriver');
const checkSendResponse = require('./utils/checkSendResponse');
const checkMessageReceived = require('./utils/checkMessageReceived');
const openNewTab = require('./utils/openNewTab');

const TEST_DOMAINS = ['valid-vapid-key-modern-sw'];

// 4 minutes. The fact that the flow includes making a request to the Send Service,
// storing/retrieving form indexedDb asynchronously makes these test units to have a execution time
// variance. Therefore, allowing these units to have a longer time to work is crucial.
const TIMEOUT_BACKGROUND_MESSAGE_TEST_UNIT_MILLISECONDS = 240000;

// 1 minute. Wait for object store to be created and received message to be stored in idb. This
// waiting time MUST be longer than the wait time for adding to db in the sw.
const WAIT_TIME_BEFORE_RETRIEVING_BACKGROUND_MESSAGES_MILLISECONDS = 60000;

const wait = ms => new Promise(res => setTimeout(res, ms));

// 50% of integration test run time is spent in testing receiving in background. Running these
// slower tests last so other tests can fail quickly if they do fail.
require('./test-token-delete');
require('./test-token-update');
require('./test-useValidManifest');
require('./test-useDefaultServiceWorker');
require('./test-receive-foreground');

describe('Firebase Messaging Integration Tests > Test Background Receive', function () {
this.retries(2);
let globalWebDriver;

before(async function () {
await testServer.start();
});

after(async function () {
await testServer.stop();
});

// TODO: enable testing for firefox
seleniumAssistant.getLocalBrowsers().forEach(assistantBrowser => {
if (assistantBrowser.getId() !== 'chrome') {
return;
}

TEST_DOMAINS.forEach(domain => {
describe(`Testing browser: ${assistantBrowser.getPrettyName()} : ${domain}`, function () {
before(async function () {
globalWebDriver = createPermittedWebDriver(
/* browser= */ assistantBrowser.getId()
);
});

it('Background app can receive a {} empty message from sw', async function () {
this.timeout(TIMEOUT_BACKGROUND_MESSAGE_TEST_UNIT_MILLISECONDS);

// Clearing the cache and db data by killing the previously instantiated driver. Note that
// ideally this call is placed inside the after/before hooks. However, Mocha forbids
// operations longer than 2s in hooks. Hence, this clearing call needs to be inside the
// test unit.
await seleniumAssistant.killWebDriver(globalWebDriver);

globalWebDriver = createPermittedWebDriver(
/* browser= */ assistantBrowser.getId()
);

prepareBackgroundApp(globalWebDriver, domain);

checkSendResponse(
await sendMessage({
to: await retrieveToken(globalWebDriver)
})
);

await wait(
WAIT_TIME_BEFORE_RETRIEVING_BACKGROUND_MESSAGES_MILLISECONDS
);

checkMessageReceived(
await getReceivedBackgroundMessages(globalWebDriver),
/* expectedNotificationPayload= */ null,
/* expectedDataPayload= */ null,
/* isLegacyPayload= */ false
);
});

it('Background app can receive a {"data"} message frow sw', async function () {
this.timeout(TIMEOUT_BACKGROUND_MESSAGE_TEST_UNIT_MILLISECONDS);

await seleniumAssistant.killWebDriver(globalWebDriver);

globalWebDriver = createPermittedWebDriver(
/* browser= */ assistantBrowser.getId()
);

prepareBackgroundApp(globalWebDriver, domain);

checkSendResponse(
await sendMessage({
to: await retrieveToken(globalWebDriver),
data: getTestDataPayload()
})
);

await wait(
WAIT_TIME_BEFORE_RETRIEVING_BACKGROUND_MESSAGES_MILLISECONDS
);

checkMessageReceived(
await getReceivedBackgroundMessages(globalWebDriver),
/* expectedNotificationPayload= */ null,
/* expectedDataPayload= */ getTestDataPayload()
);
});
});
});
});
});

async function prepareBackgroundApp(globalWebDriver, domain) {
await globalWebDriver.get(`${testServer.serverAddress}/${domain}/`);
// TODO: remove the try/catch block once the underlying bug has been resolved. Shift window focus
// away from app window so that background messages can be received/processed
try {
await openNewTab(globalWebDriver);
} catch (err) {
// ChromeDriver seems to have an open bug which throws "JavascriptError: javascript error:
// circular reference". Nevertheless, a new tab can still be opened. Hence, just catch and
// continue here.
console.log('FCM (ignored on purpose): ' + err);
}
}

function getTestDataPayload() {
return { hello: 'world' };
}
145 changes: 10 additions & 135 deletions integration/messaging/test/test-send.js → ...messaging/test/test-receive-foreground.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,41 +15,24 @@
* limitations under the License.
*/

const expect = require('chai').expect;
const testServer = require('./utils/test-server');
const sendMessage = require('./utils/sendMessage');
const retrieveToken = require('./utils/retrieveToken');
const seleniumAssistant = require('selenium-assistant');
const getReceivedBackgroundMessages = require('./utils/getReceivedBackgroundMessages');
const checkSendResponse = require('./utils/checkSendResponse');
const getReceivedForegroundMessages = require('./utils/getReceivedForegroundMessages');
const openNewTab = require('./utils/openNewTab');
const checkMessageReceived = require('./utils/checkMessageReceived');
const createPermittedWebDriver = require('./utils/createPermittedWebDriver');

const TEST_DOMAINS = ['valid-vapid-key', 'valid-vapid-key-modern-sw'];
const TEST_PROJECT_SENDER_ID = '750970317741';
const DEFAULT_COLLAPSE_KEY_VALUE = 'do_not_collapse';
const FIELD_FROM = 'from';
const FIELD_COLLAPSE_KEY_LEGACY = 'collapse_key';
const FIELD_COLLAPSE_KEY = 'collapseKey';

const FIELD_DATA = 'data';
const FIELD_NOTIFICATION = 'notification';

// 4 minutes. The fact that the flow includes making a request to the Send Service,
// storing/retrieving form indexedDb asynchronously makes these test units to have a execution time
// variance. Therefore, allowing these units to have a longer time to work is crucial.
const TIMEOUT_BACKGROUND_MESSAGE_TEST_UNIT_MILLISECONDS = 240000;

// Only testing 'valid-vapid-key' because 'valid-vapid-key-modern-sw' has the same behavior
const TEST_DOMAINS = ['valid-vapid-key'];
const TIMEOUT_FOREGROUND_MESSAGE_TEST_UNIT_MILLISECONDS = 120000;

// 1 minute. Wait for object store to be created and received message to be stored in idb. This
// waiting time MUST be longer than the wait time for adding to db in the sw.
const WAIT_TIME_BEFORE_RETRIEVING_BACKGROUND_MESSAGES_MILLISECONDS = 60000;
// Getting and deleting token is the entry step of using FM SDK. Let it run first and fail quickly.
require('./test-token-delete');

const wait = ms => new Promise(res => setTimeout(res, ms));

describe('Starting Integration Test > Sending and Receiving ', function () {
this.retries(3);
describe('Firebase Messaging Integration Tests > Test Foreground Receive', function () {
this.retries(2);
let globalWebDriver;

before(async function () {
Expand All @@ -73,68 +56,6 @@ describe('Starting Integration Test > Sending and Receiving ', function () {
/* browser= */ assistantBrowser.getId()
);
});

it('Background app can receive a {} empty message from sw', async function () {
this.timeout(TIMEOUT_BACKGROUND_MESSAGE_TEST_UNIT_MILLISECONDS);

// Clearing the cache and db data by killing the previously instantiated driver. Note that
// ideally this call is placed inside the after/before hooks. However, Mocha forbids
// operations longer than 2s in hooks. Hence, this clearing call needs to be inside the
// test unit.
await seleniumAssistant.killWebDriver(globalWebDriver);

globalWebDriver = createPermittedWebDriver(
/* browser= */ assistantBrowser.getId()
);

prepareBackgroundApp(globalWebDriver, domain);

checkSendResponse(
await sendMessage({
to: await retrieveToken(globalWebDriver)
})
);

await wait(
WAIT_TIME_BEFORE_RETRIEVING_BACKGROUND_MESSAGES_MILLISECONDS
);

checkMessageReceived(
await getReceivedBackgroundMessages(globalWebDriver),
/* expectedNotificationPayload= */ null,
/* expectedDataPayload= */ null,
/* isLegacyPayload= */ false
);
});

it('Background app can receive a {"data"} message frow sw', async function () {
this.timeout(TIMEOUT_BACKGROUND_MESSAGE_TEST_UNIT_MILLISECONDS);

await seleniumAssistant.killWebDriver(globalWebDriver);

globalWebDriver = createPermittedWebDriver(
/* browser= */ assistantBrowser.getId()
);

prepareBackgroundApp(globalWebDriver, domain);

checkSendResponse(
await sendMessage({
to: await retrieveToken(globalWebDriver),
data: getTestDataPayload()
})
);

await wait(
WAIT_TIME_BEFORE_RETRIEVING_BACKGROUND_MESSAGES_MILLISECONDS
);

checkMessageReceived(
await getReceivedBackgroundMessages(globalWebDriver),
/* expectedNotificationPayload= */ null,
/* expectedDataPayload= */ getTestDataPayload()
);
});
});

it('Foreground app can receive a {} empty message in onMessage', async function () {
Expand Down Expand Up @@ -241,35 +162,8 @@ describe('Starting Integration Test > Sending and Receiving ', function () {
});
});

function checkMessageReceived(
receivedMessages,
expectedNotificationPayload,
expectedDataPayload
) {
expect(receivedMessages).to.exist;

const message = receivedMessages[0];

expect(message[FIELD_FROM]).to.equal(TEST_PROJECT_SENDER_ID);
const collapseKey = !!message[FIELD_COLLAPSE_KEY_LEGACY]
? message[FIELD_COLLAPSE_KEY_LEGACY]
: message[FIELD_COLLAPSE_KEY];
expect(collapseKey).to.equal(DEFAULT_COLLAPSE_KEY_VALUE);

if (expectedNotificationPayload) {
expect(message[FIELD_NOTIFICATION]).to.deep.equal(
getTestNotificationPayload()
);
}

if (expectedDataPayload) {
expect(message[FIELD_DATA]).to.deep.equal(getTestDataPayload());
}
}

function checkSendResponse(response) {
expect(response).to.exist;
expect(response.success).to.equal(1);
function getTestDataPayload() {
return { hello: 'world' };
}

function getTestNotificationPayload() {
Expand All @@ -281,22 +175,3 @@ function getTestNotificationPayload() {
tag: 'test-tag'
};
}

function getTestDataPayload() {
return { hello: 'world' };
}

async function prepareBackgroundApp(globalWebDriver, domain) {
await globalWebDriver.get(`${testServer.serverAddress}/${domain}/`);

// TODO: remove the try/catch block once the underlying bug has been resolved. Shift window focus
// away from app window so that background messages can be received/processed
try {
await openNewTab(globalWebDriver);
} catch (err) {
// ChromeDriver seems to have an open bug which throws "JavascriptError: javascript error:
// circular reference". Nevertheless, a new tab can still be opened. Hence, just catch and
// continue here.
console.log('FCM (ignored on purpose): ' + err);
}
}
4 changes: 2 additions & 2 deletions ...ration/messaging/test/test-deleteToken.js → ...ation/messaging/test/test-token-delete.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,12 @@ const retrieveToken = require('./utils/retrieveToken');
const seleniumAssistant = require('selenium-assistant');
const createPermittedWebDriver = require('./utils/createPermittedWebDriver');

const TEST_SUITE_TIMEOUT_MS = 100000;
const TEST_SUITE_TIMEOUT_MS = 20000;
const TEST_DOMAIN = 'valid-vapid-key';

describe('Firebase Messaging Integration Tests > get and delete token', function () {
this.timeout(TEST_SUITE_TIMEOUT_MS);
this.retries(3);
this.retries(2);
let globalWebDriver;

before(async function () {
Expand Down
5 changes: 4 additions & 1 deletion ...ration/messaging/test/test-updateToken.js → ...ation/messaging/test/test-token-update.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,12 @@ const getErrors = require('./utils/getErrors');
const TEST_SUITE_TIMEOUT_MS = 70000;
const TEST_DOMAIN = 'valid-vapid-key';

// Getting and deleting token is the entry step of using FM SDK. Let it run first and fail quickly.
require('./test-token-delete');

describe('Firebase Messaging Integration Tests > update a token', function () {
this.timeout(TEST_SUITE_TIMEOUT_MS);
this.retries(3);
this.retries(2);

let globalWebDriver;

Expand Down
5 changes: 3 additions & 2 deletions integration/messaging/test/test-useDefaultServiceWorker.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,12 @@ const createPermittedWebDriver = require('./utils/createPermittedWebDriver');
const TEST_DOMAIN = 'default-sw';
const TEST_SUITE_TIMEOUT_MS = 70000;

// Getting and deleting token is the entry step of using FM SDK. Let it run first and fail quickly.
require('./test-token-delete');

describe(`Firebase Messaging Integration Tests > Use 'firebase-messaging-sw.js' by default`, function () {
this.timeout(TEST_SUITE_TIMEOUT_MS);

this.retries(3);

let globalWebDriver;

before(async function () {
Expand Down
5 changes: 3 additions & 2 deletions integration/messaging/test/test-useValidManifest.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,12 @@ const createPermittedWebDriver = require('./utils/createPermittedWebDriver');
const TEST_DOMAIN = 'valid-manifest';
const TEST_SUITE_TIMEOUT_MS = 70000;

// Getting and deleting token is the entry step of using FM SDK. Let it run first and fail quickly.
require('./test-token-delete');

describe(`Firebase Messaging Integration Tests > Use 'use valid manifest`, function () {
this.timeout(TEST_SUITE_TIMEOUT_MS);

this.retries(3);

let globalWebDriver;

before(async function () {
Expand Down
Loading