24
1
Fork 0

reimplemented l10n using dynamic import() (#1012)

this should greatly reduce the complexity of the l10n code
and build pipeline and eliminate the most common error
seen in sentry logs (no translate function)
This commit is contained in:
Danny Coates 2018-11-20 06:50:59 -08:00 committed by Donovan Preston
parent 5afa4e5c9b
commit 1e62aa976d
28 changed files with 145 additions and 280 deletions

View File

@ -5,7 +5,6 @@ node_modules
firefox
assets
docs
public
test
coverage
.nyc_output

View File

@ -2,3 +2,5 @@ dist
assets
firefox
coverage
app/locale.js
app/capabilities.js

View File

@ -26,7 +26,6 @@ import Raven from 'raven-js';
import { setApiUrlPrefix } from '../app/api';
import assets from '../common/assets';
import Header from '../app/ui/header';
import locale from '../common/locales';
import storage from '../app/storage';
import controller from '../app/controller';
import User from './user';
@ -36,9 +35,9 @@ import upload from './pages/upload';
import share from './pages/share';
import preferences from './pages/preferences';
import error from './pages/error';
import { getTranslator } from '../app/locale';
if (navigator.userAgent === 'Send Android') {
assets.setPrefix('/android_asset');
setApiUrlPrefix('https://send2.dev.lcip.org');
}
@ -71,30 +70,32 @@ function body(main) {
}
};
}
(async function start() {
const translate = await getTranslator('en-US');
app.use(async (state, emitter) => {
state.translate = translate;
state.capabilities = {
account: true
}; //TODO
state.storage = storage;
state.user = new User(storage);
state.raven = Raven;
app.use((state, emitter) => {
state.translate = locale.getTranslator();
state.capabilities = {
account: true
}; //TODO
state.storage = storage;
state.user = new User(storage);
state.raven = Raven;
window.finishLogin = async function(accountInfo) {
await state.user.finishLogin(accountInfo);
emitter.emit('render');
};
window.finishLogin = async function(accountInfo) {
await state.user.finishLogin(accountInfo);
emitter.emit('render');
};
// for debugging
window.appState = state;
window.appEmit = emitter.emit.bind(emitter);
});
app.route('/', body(home));
app.route('/upload', upload);
app.route('/share/:id', share);
app.route('/preferences', preferences);
app.route('/error', error);
//app.route('/debugging', require('./pages/debugging').default);
// add /api/filelist
app.mount('body');
// for debugging
window.appState = state;
window.appEmit = emitter.emit.bind(emitter);
});
app.route('/', body(home));
app.route('/upload', upload);
app.route('/share/:id', share);
app.route('/preferences', preferences);
app.route('/error', error);
//app.route('/debugging', require('./pages/debugging').default);
// add /api/filelist
app.mount('body');
})();

View File

@ -3,5 +3,4 @@
npm run build
rm -rf src/main/assets
mkdir -p src/main/assets
cp -R ../../dist/* src/main/assets
sed -i '' 's/url(/url(\/android_asset/g' src/main/assets/app.*.css
cp -R ../../dist/* src/main/assets

View File

@ -52,9 +52,9 @@ function checkStreams() {
}
}
function polyfillStreams() {
async function polyfillStreams() {
try {
require('@mattiasbuelens/web-streams-polyfill');
await import('@mattiasbuelens/web-streams-polyfill');
return true;
} catch (e) {
return false;
@ -64,7 +64,10 @@ function polyfillStreams() {
export default async function capabilities() {
const crypto = await checkCrypto();
const nativeStreams = checkStreams();
const polyStreams = nativeStreams ? false : polyfillStreams();
let polyStreams = false;
if (!nativeStreams) {
polyStreams = await polyfillStreams();
}
let account = typeof AUTH_CONFIG !== 'undefined';
try {
account = account && !!localStorage;

26
app/locale.js Normal file
View File

@ -0,0 +1,26 @@
import { FluentBundle } from 'fluent';
function makeBundle(locale, ftl) {
const bundle = new FluentBundle(locale, { useIsolating: false });
bundle.addMessages(ftl);
return bundle;
}
export async function getTranslator(locale) {
const bundles = [];
const { default: en } = await import('../public/locales/en-US/send.ftl');
if (locale !== 'en-US') {
const {
default: ftl
} = await import(`../public/locales/${locale}/send.ftl`);
bundles.push(makeBundle(locale, ftl));
}
bundles.push(makeBundle('en-US', en));
return function(id, data) {
for (let bundle of bundles) {
if (bundle.hasMessage(id)) {
return bundle.format(bundle.getMessage(id), data);
}
}
};
}

View File

@ -1,10 +1,11 @@
/* global LOCALE */
import 'core-js';
import 'fast-text-encoding'; // MS Edge support
import 'fluent-intl-polyfill';
import choo from 'choo';
import nanotiming from 'nanotiming';
import routes from './routes';
import capabilities from './capabilities';
import locale from '../common/locales';
import controller from './controller';
import dragManager from './dragManager';
import pasteManager from './pasteManager';
@ -14,6 +15,7 @@ import experiments from './experiments';
import Raven from 'raven-js';
import './main.css';
import User from './user';
import { getTranslator } from './locale';
(async function start() {
const app = routes(choo());
@ -28,11 +30,13 @@ import User from './user';
navigator.serviceWorker.register('/serviceWorker.js');
}
const translate = await getTranslator(LOCALE);
app.use((state, emitter) => {
state.capabilities = capa;
state.transfer = null;
state.fileInfo = null;
state.translate = locale.getTranslator();
state.translate = translate;
state.storage = storage;
state.raven = Raven;
state.user = new User(storage);

View File

@ -15,16 +15,6 @@ function chunkFileNames(compilation) {
}
class AndroidIndexPlugin {
apply(compiler) {
const assets = {};
compiler.hooks.compilation.tap(NAME, compilation => {
compilation.hooks.moduleAsset.tap(NAME, (mod, file) => {
if (mod.userRequest) {
assets[
path.join(path.dirname(file), path.basename(mod.userRequest))
] = file;
}
});
});
compiler.hooks.emit.tap(NAME, compilation => {
const files = chunkFileNames(compilation);
const page = html`
@ -36,9 +26,8 @@ class AndroidIndexPlugin {
name="viewport"
content="width=device-width, initial-scale=1"
/>
<base href="file:///android_asset/" />
<link href="${files['app.css']}" rel="stylesheet" />
<script src="${files['vendor.js']}"></script>
<script src="${assets['public/locales/en-US/send.ftl']}"></script>
<script src="${files['android.js']}"></script>
</head>
<body></body>

View File

@ -1,62 +0,0 @@
const { FluentResource } = require('fluent/compat');
const fs = require('fs');
function toJSON(resource) {
return JSON.stringify(Array.from(resource));
}
module.exports = function(source) {
const localeExp = /([^/]+)\/[^/]+\.ftl$/;
const result = localeExp.exec(this.resourcePath);
const locale = result && result[1];
if (!locale) {
throw new Error(`couldn't find locale in: ${this.resourcePath}`);
}
// Parse the current language's translation file.
const locResource = FluentResource.fromString(source);
let enResource;
// If the current language is not en-US, also parse en-US to provide a
// fallback for missing translations.
if (locale !== 'en-US') {
const en_ftl = fs.readFileSync(
require.resolve('../public/locales/en-US/send.ftl'),
'utf8'
);
enResource = FluentResource.fromString(en_ftl);
}
return `
module.exports = \`
if (typeof window === 'undefined') {
var fluent = require('fluent');
}
(function () {
let bundles = [
['${locale}', ${toJSON(locResource)}],
${enResource ? `['en-US', ${toJSON(enResource)}]` : ''}
].map(([locale, entries]) => {
let bundle = new fluent.FluentBundle(locale, {useIsolating: false});
bundle.addResource(new fluent.FluentResource(entries));
return bundle;
});
function translate(id, data) {
for (let bundle of bundles) {
if (bundle.hasMessage(id)) {
let message = bundle.getMessage(id);
return bundle.format(message, data);
}
}
}
if (typeof window === 'undefined') {
module.exports = translate;
}
else {
window.translate = translate;
}
})();
\``;
};

View File

@ -1,33 +0,0 @@
/*
This code is included by both the server and frontend via
common/locales.js
When included from the server the export will be the function.
When included from the frontend (via webpack) the export will
be an object mapping ftl files to js files. Example:
"public/locales/en-US/send.ftl":"public/locales/en-US/send.6b4f8354.js"
*/
const fs = require('fs');
const path = require('path');
function kv(d) {
return `"${d}": require('../public/locales/${d}/send.ftl')`;
}
module.exports = function() {
const dirs = fs.readdirSync(path.join(__dirname, '..', 'public', 'locales'));
const code = `
module.exports = {
translate: function (id, data) { return window.translate(id, data) },
${dirs.map(kv).join(',\n')}
};`;
return {
code,
dependencies: dirs.map(d =>
require.resolve(`../public/locales/${d}/send.ftl`)
),
cacheable: true
};
};

View File

@ -1,17 +1,9 @@
# Custom Loaders
## Fluent Loader
The fluent loader "compiles" `.ftl` files into `.js` files directly usable by both the frontend and server for localization.
## Generate Asset Map
This loader enumerates all the files in `assets/` so that `common/assets.js` can provide mappings from the source filename to the hashed filename used on the site.
## Generate L10N Map
This loader enumerates all the ftl files in `public/locales` so that the fluent loader can create it's js files.
## Version Plugin
Creates a `version.json` file that gets exposed by the `/__version__` route from the `package.json` file and current git commit hash.

View File

@ -1,52 +0,0 @@
const gen = require('../build/generate_l10n_map');
const isServer = typeof gen === 'function';
const prefix = '';
let manifest = {};
try {
// eslint-disable-next-line node/no-missing-require
manifest = require('../dist/manifest.json');
} catch (e) {
// use middleware
}
const locales = isServer ? manifest : gen;
function getLocale(name) {
return prefix + locales[`public/locales/${name}/send.ftl`];
}
function serverTranslator(name) {
// eslint-disable-next-line security/detect-non-literal-require
return require(`../dist/${locales[`public/locales/${name}/send.ftl`]}`);
}
function browserTranslator() {
return locales.translate;
}
const translator = isServer ? serverTranslator : browserTranslator;
const instance = {
get: getLocale,
getTranslator: translator,
setMiddleware: function(middleware) {
if (middleware) {
const _eval = require('require-from-string');
instance.get = function getLocaleWithMiddleware(name) {
const f = middleware.fileSystem.readFileSync(
middleware.getFilenameFromUrl('/manifest.json')
);
return prefix + JSON.parse(f)[`public/locales/${name}/send.ftl`];
};
instance.getTranslator = function(name) {
const f = middleware.fileSystem.readFileSync(
middleware.getFilenameFromUrl(instance.get(name))
);
return _eval(f.toString());
};
}
}
};
module.exports = instance;

View File

@ -1,3 +1,3 @@
# Common Code
This directory contains code loaded by both the frontend `app` and backend `server`. The code here can be challenging to understand at first because the contexts for the two (three counting the dev server) environments that include them are quite different, but the purpose of these modules are quite simple, to provide mappings from the source assets (`copy-16.png`) to the concrete production assets (`copy-16.db66e0bf.svg`), similarly for localizations.
This directory contains code loaded by both the frontend `app` and backend `server`. The code here can be challenging to understand at first because the contexts for the two (three counting the dev server) environments that include them are quite different, but the purpose of these modules are quite simple, to provide mappings from the source assets (`copy-16.png`) to the concrete production assets (`copy-16.db66e0bf.svg`).

33
package-lock.json generated
View File

@ -520,6 +520,15 @@
"@babel/helper-plugin-utils": "^7.0.0"
}
},
"@babel/plugin-syntax-dynamic-import": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/@babel/plugin-syntax-dynamic-import/-/plugin-syntax-dynamic-import-7.0.0.tgz",
"integrity": "sha512-Gt9xNyRrCHCiyX/ZxDGOcBnlJl0I3IWicpZRC4CdC0P5a/I07Ya2OAMEBU+J7GmRFVmIetqEYRko6QYRuKOESw==",
"dev": true,
"requires": {
"@babel/helper-plugin-utils": "^7.0.0"
}
},
"@babel/plugin-syntax-json-strings": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.0.0.tgz",
@ -816,24 +825,6 @@
"regexpu-core": "^4.1.3"
}
},
"@babel/polyfill": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/@babel/polyfill/-/polyfill-7.0.0.tgz",
"integrity": "sha512-dnrMRkyyr74CRelJwvgnnSUDh2ge2NCTyHVwpOdvRMHtJUyxLtMAfhBN3s64pY41zdw0kgiLPh6S20eb1NcX6Q==",
"dev": true,
"requires": {
"core-js": "^2.5.7",
"regenerator-runtime": "^0.11.1"
},
"dependencies": {
"regenerator-runtime": {
"version": "0.11.1",
"resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz",
"integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg==",
"dev": true
}
}
},
"@babel/preset-env": {
"version": "7.1.6",
"resolved": "https://registry.npmjs.org/@babel/preset-env/-/preset-env-7.1.6.tgz",
@ -16470,6 +16461,12 @@
"unpipe": "1.0.0"
}
},
"raw-loader": {
"version": "0.5.1",
"resolved": "https://registry.npmjs.org/raw-loader/-/raw-loader-0.5.1.tgz",
"integrity": "sha1-DD0L6u2KAclm2Xh793goElKpeao=",
"dev": true
},
"read-file-stdin": {
"version": "0.2.1",
"resolved": "https://registry.npmjs.org/read-file-stdin/-/read-file-stdin-0.2.1.tgz",

View File

@ -61,7 +61,7 @@
"devDependencies": {
"@babel/core": "^7.1.6",
"@babel/plugin-proposal-class-properties": "^7.1.0",
"@babel/polyfill": "^7.0.0",
"@babel/plugin-syntax-dynamic-import": "^7.0.0",
"@babel/preset-env": "^7.1.6",
"@dannycoates/webpack-dev-server": "^3.1.4",
"@fullhuman/postcss-purgecss": "^1.1.0",
@ -73,6 +73,7 @@
"base64-js": "^1.3.0",
"content-disposition": "^0.5.2",
"copy-webpack-plugin": "^4.5.2",
"core-js": "^2.5.7",
"crc": "^3.8.0",
"cross-env": "^5.2.0",
"css-loader": "^1.0.0",
@ -105,8 +106,8 @@
"proxyquire": "^2.1.0",
"puppeteer": "1.9.0",
"raven-js": "^3.27.0",
"raw-loader": "^0.5.1",
"redis-mock": "^0.39.0",
"require-from-string": "^2.0.2",
"rimraf": "^2.6.2",
"sinon": "^7.1.1",
"string-hash": "^1.1.3",

View File

@ -1,5 +1,4 @@
const assets = require('../../common/assets');
const locales = require('../../common/locales');
const routes = require('../routes');
const pages = require('../routes/pages');
const tests = require('../../test/frontend/routes');
@ -17,7 +16,6 @@ module.exports = function(app, devServer) {
wsapp.listen(8081, config.listen_address);
assets.setMiddleware(devServer.middleware);
locales.setMiddleware(devServer.middleware);
app.use(morgan('dev', { stream: process.stderr }));
function android(req, res) {
const index = devServer.middleware.fileSystem.readFileSync(

View File

@ -1,5 +1,4 @@
const assets = require('../../common/assets');
const locales = require('../../common/locales');
const routes = require('../routes');
const pages = require('../routes/pages');
const tests = require('../../test/frontend/routes');
@ -7,7 +6,6 @@ const expressWs = require('express-ws');
module.exports = function(app, devServer) {
assets.setMiddleware(devServer.middleware);
locales.setMiddleware(devServer.middleware);
expressWs(app, null, { perMessageDeflate: false });
app.ws('/api/ws', require('../routes/ws'));
routes(app);

View File

@ -6,6 +6,7 @@ module.exports = function(state) {
return state.cspNonce
? html`
<script nonce="${state.cspNonce}">
const LOCALE = '${state.locale}';
const downloadMetadata = ${
state.downloadMetadata
? raw(JSON.stringify(state.downloadMetadata))

View File

@ -1,6 +1,5 @@
const html = require('choo/html');
const assets = require('../common/assets');
const locales = require('../common/locales');
const initScript = require('./initScript');
module.exports = function(state, body = '') {
@ -17,6 +16,7 @@ module.exports = function(state, body = '') {
<!DOCTYPE html>
<html lang="${state.locale}">
<head>
<base href="/" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
@ -95,9 +95,6 @@ module.exports = function(state, body = '') {
${firaTag}
<script defer src="/jsconfig.js"></script>
<!-- <script defer src="${assets.get('runtime.js')}"></script> -->
<script defer src="${assets.get('vendor.js')}"></script>
<script defer src="${locales.get(state.locale)}"></script>
<script defer src="${assets.get('cryptofill.js')}"></script>
<script defer src="${assets.get('app.js')}"></script>
</head>

26
server/locale.js Normal file
View File

@ -0,0 +1,26 @@
const fs = require('fs');
const path = require('path');
const { FluentBundle } = require('fluent');
const localesPath = path.resolve(__dirname, '../public/locales');
const locales = fs.readdirSync(localesPath);
function makeBundle(locale) {
const bundle = new FluentBundle(locale, { useIsolating: false });
bundle.addMessages(
fs.readFileSync(path.resolve(localesPath, locale, 'send.ftl'))
);
return [locale, bundle];
}
const bundles = new Map(locales.map(makeBundle));
module.exports = function getTranslator(locale) {
const defaultBundle = bundles.get('en-US');
const bundle = bundles.get(locale) || defaultBundle;
return function(id, data) {
if (bundle.hasMessage(id)) {
return bundle.format(bundle.getMessage(id), data);
}
return defaultBundle.format(defaultBundle.getMessage(id), data);
};
};

View File

@ -1,14 +1,14 @@
const config = require('./config');
const layout = require('./layout');
const locales = require('../common/locales');
const assets = require('../common/assets');
const getTranslator = require('./locale');
module.exports = function(req) {
const locale = req.language || 'en-US';
return {
locale,
capabilities: { account: false },
translate: locales.getTranslator(locale),
translate: getTranslator(locale),
title: 'Firefox Send',
description:
'Encrypt and send files with a link that automatically expires to ensure your important documents dont stay online forever.',

View File

@ -14,6 +14,7 @@ class DownloadPage extends Page {
* @throws ElementNotFound
*/
waitForPageToLoad() {
super.waitForPageToLoad();
browser.waitForExist(this.downloadButton);
return this;
}

View File

@ -12,6 +12,7 @@ class HomePage extends Page {
}
waitForPageToLoad() {
super.waitForPageToLoad();
browser.waitForExist(this.uploadInput);
this.showUploadInput();
return this;

View File

@ -1,4 +1,4 @@
/* global browser */
/* global browser window */
class Page {
constructor(path) {
this.path = path;
@ -15,6 +15,12 @@ class Page {
* @throws ElementNotFound
*/
waitForPageToLoad() {
browser.waitUntil(function() {
return browser.execute(function() {
return typeof window.appState !== 'undefined';
});
}, 3000);
browser.pause(100);
return this;
}
}

View File

@ -14,4 +14,10 @@ You can also run them in headless Chrome by using `npm run test:frontend`. The r
Unit tests reside in `test/backend`
Backend test can be run with `npm run test:backend`. [Sinon](http://sinonjs.org/) and [proxyquire](https://github.com/thlorenz/proxyquire) are used for mocking.
Backend test can be run with `npm run test:backend`. [Sinon](http://sinonjs.org/) and [proxyquire](https://github.com/thlorenz/proxyquire) are used for mocking.
## Integration
Integration tests include UI tests that run with Selenium.
The preferred way to run these locally is with `npm run test-integration` which requires docker. To watch the tests connect with VNC. On mac enter `vnc://localhost:5900` in Safari and use the password `secret` to connect. For info on debugging a test see the [wdio debug docs](http://webdriver.io/api/utility/debug.html).

View File

@ -8,7 +8,6 @@ module.exports = {
const express = require('express');
const expressWs = require('express-ws');
const assets = require('../common/assets');
const locales = require('../common/locales');
const routes = require('../server/routes');
const tests = require('./frontend/routes');
const app = express();
@ -18,7 +17,6 @@ module.exports = {
});
app.use(wpm);
assets.setMiddleware(wpm);
locales.setMiddleware(wpm);
expressWs(app, null, { perMessageDeflate: false });
app.ws('/api/ws', require('../server/routes/ws'));
routes(app);

View File

@ -17,10 +17,10 @@ exports.config = Object.assign({}, common.config, {
maxInstances: 1,
services: ['docker', require('./testServer')],
dockerOptions: {
image: 'selenium/standalone-firefox',
image: 'selenium/standalone-firefox-debug',
healthCheck: 'http://localhost:4444',
options: {
p: ['4444:4444'],
p: ['4444:4444', '5900:5900'],
mount: `type=bind,source=${dir},destination=${dir},consistency=delegated`,
shmSize: '2g'
}

View File

@ -12,13 +12,13 @@ const webJsOptions = {
[
'@babel/preset-env',
{
modules: false,
useBuiltIns: 'entry'
}
]
],
// yo-yoify converts html template strings to direct dom api calls
plugins: [
'@babel/plugin-syntax-dynamic-import',
'yo-yoify',
['@babel/plugin-proposal-class-properties', { loose: false }]
]
@ -89,17 +89,13 @@ const serviceWorker = {
const web = {
target: 'web',
entry: {
// babel-polyfill and fluent are directly included in vendor
// because they are not explicitly referenced by app
vendor: ['@babel/polyfill', 'fluent'], //TODO: remove @babel/polyfill
app: ['./app/main.js'],
android: ['./android/android.js'],
ios: ['./ios/ios.js']
},
output: {
filename: '[name].[hash:8].js',
path: path.resolve(__dirname, 'dist'),
publicPath: '/'
path: path.resolve(__dirname, 'dist')
},
module: {
rules: [
@ -117,21 +113,6 @@ const web = {
}
]
},
{
// fluent gets exposed as a global so that each language script
// can load independently and share it.
include: [path.dirname(require.resolve('fluent'))],
use: [
{
loader: 'expose-loader',
options: 'fluent'
},
{
loader: 'babel-loader',
options: webJsOptions
}
]
},
{
loader: 'babel-loader',
include: [
@ -148,7 +129,10 @@ const web = {
{
// Strip asserts from our deps, mainly choojs family
include: [path.resolve(__dirname, 'node_modules')],
exclude: [path.resolve(__dirname, 'node_modules/crc')],
exclude: [
path.resolve(__dirname, 'node_modules/crc'),
path.resolve(__dirname, 'node_modules/fluent')
],
loader: 'webpack-unassert-loader'
}
]
@ -197,18 +181,8 @@ const web = {
})
},
{
// creates a js script for each ftl
test: /\.ftl$/,
use: [
{
loader: 'file-loader',
options: {
name: '[path][name].[hash:8].js'
}
},
'extract-loader',
'./build/fluent_loader'
]
use: 'raw-loader'
},
{
// creates test.js for /test
@ -219,11 +193,6 @@ const web = {
// loads all assets from assets/ for use by common/assets.js
test: require.resolve('./build/generate_asset_map.js'),
use: ['babel-loader', 'val-loader']
},
{
// loads all the ftl from public/locales for use by common/locales.js
test: require.resolve('./build/generate_l10n_map.js'),
use: ['babel-loader', 'val-loader']
}
]
},
@ -236,12 +205,10 @@ const web = {
]),
new webpack.EnvironmentPlugin(['NODE_ENV']),
new webpack.IgnorePlugin(/\.\.\/dist/), // used in common/*.js
new webpack.IgnorePlugin(/require-from-string/), // used in common/locales.js
new webpack.HashedModuleIdsPlugin(),
new ExtractTextPlugin({
filename: '[name].[hash:8].css'
}),
new VersionPlugin(),
new VersionPlugin(), // used for the /__version__ route
new AndroidIndexPlugin(),
new ManifestPlugin() // used by server side to resolve hashed assets
],