Chapril/drop.chapril.org-firefoxsend
Chapril/drop.chapril.org-firefoxsend
25
1
Fork 0
Code Issues Releases Activity

Merge pull request #855 from mozilla/streaming

Browse Source 
add streaming upload
This commit is contained in:
Emily Hou 2018-06-25 11:37:06 -07:00 committed by GitHub
commit 9bfdf86bec
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 774 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

138
app/api.js
View File

@ -91,47 +91,107 @@ export async function setPassword(id, owner_token, keychain) {
return response.ok;
}
export function uploadFile(
encrypted,
function asyncInitWebSocket(server) {
return new Promise(resolve => {
const ws = new WebSocket(server);
ws.onopen = () => {
resolve(ws);
};
});
}
function listenForResponse(ws, canceller) {
return new Promise((resolve, reject) => {
ws.addEventListener('message', function(msg) {
try {
const response = JSON.parse(msg.data);
if (response.error) {
throw new Error(response.error);
} else {
resolve({
url: response.url,
id: response.id,
ownerToken: response.owner
});
}
} catch (e) {
ws.close();
canceller.cancelled = true;
canceller.error = e;
reject(e);
}
});
});
}
async function upload(
stream,
streamInfo,
metadata,
verifierB64,
keychain,
onprogress
onprogress,
canceller
) {
const xhr = new XMLHttpRequest();
const upload = {
cancel: function() {
xhr.abort();
},
result: new Promise(function(resolve, reject) {
xhr.addEventListener('loadend', function() {
const authHeader = xhr.getResponseHeader('WWW-Authenticate');
if (authHeader) {
keychain.nonce = parseNonce(authHeader);
}
if (xhr.status === 200) {
const responseObj = JSON.parse(xhr.responseText);
return resolve({
url: responseObj.url,
id: responseObj.id,
ownerToken: responseObj.owner
});
}
reject(new Error(xhr.status));
});
})
const host = window.location.hostname;
const port = window.location.port;
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
const ws = await asyncInitWebSocket(`${protocol}//${host}:${port}/api/ws`);
try {
const metadataHeader = arrayToB64(new Uint8Array(metadata));
const fileMeta = {
fileMetadata: metadataHeader,
authorization: `send-v1 ${verifierB64}`
};
const blob = new Blob([encrypted], { type: 'application/octet-stream' });
xhr.upload.addEventListener('progress', function(event) {
if (event.lengthComputable) {
onprogress([event.loaded, event.total]);
const responsePromise = listenForResponse(ws, canceller);
ws.send(JSON.stringify(fileMeta));
const reader = stream.getReader();
let state = await reader.read();
let size = 0;
while (!state.done) {
const buf = state.value;
if (canceller.cancelled) {
throw canceller.error;
}
});
xhr.open('post', '/api/upload', true);
xhr.setRequestHeader('X-File-Metadata', arrayToB64(new Uint8Array(metadata)));
xhr.setRequestHeader('Authorization', `send-v1 ${verifierB64}`);
xhr.send(blob);
return upload;
ws.send(buf);
onprogress([Math.min(streamInfo.fileSize, size), streamInfo.fileSize]);
size += streamInfo.recordSize;
state = await reader.read();
}
const footer = new Uint8Array([0]);
ws.send(footer);
const response = await responsePromise; //promise only fufills if response is good
ws.close();
return response;
} catch (e) {
ws.close(4000);
throw e;
}
}
export function uploadWs(encrypted, info, metadata, verifierB64, onprogress) {
const canceller = { cancelled: false };
return {
cancel: function() {
canceller.error = new Error(0);
canceller.cancelled = true;
},
result: upload(
encrypted,
info,
metadata,
verifierB64,
onprogress,
canceller
)
};
}
function download(id, keychain, onprogress, canceller) {
@ -151,11 +211,7 @@ function download(id, keychain, onprogress, canceller) {
}
const blob = new Blob([xhr.response]);
const fileReader = new FileReader();
fileReader.readAsArrayBuffer(blob);
fileReader.onload = function() {
resolve(this.result);
};
resolve(blob);
});
xhr.addEventListener('progress', function(event) {
if (event.lengthComputable && event.target.status === 200) {

292
app/ece.js Normal file
View File

@ -0,0 +1,292 @@
require('buffer');
import { ReadableStream, TransformStream } from 'web-streams-polyfill';
const NONCE_LENGTH = 12;
const TAG_LENGTH = 16;
const KEY_LENGTH = 16;
const MODE_ENCRYPT = 'encrypt';
const MODE_DECRYPT = 'decrypt';
const RS = 1048576;
const encoder = new TextEncoder();
function generateSalt(len) {
const randSalt = new Uint8Array(len);
window.crypto.getRandomValues(randSalt);
return randSalt.buffer;
}
class ECETransformer {
constructor(mode, ikm, rs, salt) {
this.mode = mode;
this.prevChunk;
this.seq = 0;
this.firstchunk = true;
this.rs = rs;
this.ikm = ikm.buffer;
this.salt = salt;
}
async generateKey() {
const inputKey = await window.crypto.subtle.importKey(
'raw',
this.ikm,
'HKDF',
false,
['deriveKey']
);
return window.crypto.subtle.deriveKey(
{
name: 'HKDF',
salt: this.salt,
info: encoder.encode('Content-Encoding: aes128gcm\0'),
hash: 'SHA-256'
},
inputKey,
{
name: 'AES-GCM',
length: 128
},
false,
['encrypt', 'decrypt']
);
}
async generateNonceBase() {
const inputKey = await window.crypto.subtle.importKey(
'raw',
this.ikm,
'HKDF',
false,
['deriveKey']
);
const base = await window.crypto.subtle.exportKey(
'raw',
await window.crypto.subtle.deriveKey(
{
name: 'HKDF',
salt: this.salt,
info: encoder.encode('Content-Encoding: nonce\0'),
hash: 'SHA-256'
},
inputKey,
{
name: 'AES-GCM',
length: 128
},
true,
['encrypt', 'decrypt']
)
);
return Buffer.from(base.slice(0, NONCE_LENGTH));
}
generateNonce(seq) {
if (seq > 0xffffffff) {
throw new Error('record sequence number exceeds limit');
}
const nonce = Buffer.from(this.nonceBase);
const m = nonce.readUIntBE(nonce.length - 4, 4);
const xor = (m ^ seq) >>> 0; //forces unsigned int xor
nonce.writeUIntBE(xor, nonce.length - 4, 4);
return nonce;
}
pad(data, isLast) {
const len = data.length;
if (len + TAG_LENGTH >= this.rs) {
throw new Error('data too large for record size');
}
if (isLast) {
const padding = Buffer.alloc(1);
padding.writeUInt8(2, 0);
return Buffer.concat([data, padding]);
} else {
const padding = Buffer.alloc(this.rs - len - TAG_LENGTH);
padding.fill(0);
padding.writeUInt8(1, 0);
return Buffer.concat([data, padding]);
}
}
unpad(data, isLast) {
for (let i = data.length - 1; i >= 0; i--) {
if (data[i]) {
if (isLast) {
if (data[i] !== 2) {
throw new Error('delimiter of final record is not 2');
}
} else {
if (data[i] !== 1) {
throw new Error('delimiter of not final record is not 1');
}
}
return data.slice(0, i);
}
}
throw new Error('no delimiter found');
}
createHeader() {
const nums = Buffer.alloc(5);
nums.writeUIntBE(this.rs, 0, 4);
nums.writeUIntBE(0, 4, 1);
return Buffer.concat([Buffer.from(this.salt), nums]);
}
readHeader(buffer) {
if (buffer.length < 21) {
throw new Error('chunk too small for reading header');
}
const header = {};
header.salt = buffer.buffer.slice(0, KEY_LENGTH);
header.rs = buffer.readUIntBE(KEY_LENGTH, 4);
const idlen = buffer.readUInt8(KEY_LENGTH + 4);
header.length = idlen + KEY_LENGTH + 5;
return header;
}
async encryptRecord(buffer, seq, isLast) {
const nonce = this.generateNonce(seq);
const encrypted = await window.crypto.subtle.encrypt(
{ name: 'AES-GCM', iv: nonce },
this.key,
this.pad(buffer, isLast)
);
return Buffer.from(encrypted);
}
async decryptRecord(buffer, seq, isLast) {
const nonce = this.generateNonce(seq);
const data = await window.crypto.subtle.decrypt(
{
name: 'AES-GCM',
iv: nonce,
tagLength: 128
},
this.key,
buffer
);
return this.unpad(Buffer.from(data), isLast);
}
async start(controller) {
if (this.mode === MODE_ENCRYPT) {
this.key = await this.generateKey();
this.nonceBase = await this.generateNonceBase();
controller.enqueue(this.createHeader());
} else if (this.mode !== MODE_DECRYPT) {
throw new Error('mode must be either encrypt or decrypt');
}
}
async transformPrevChunk(isLast, controller) {
if (this.mode === MODE_ENCRYPT) {
controller.enqueue(
await this.encryptRecord(this.prevChunk, this.seq, isLast)
);
this.seq++;
} else {
if (this.seq === 0) {
//the first chunk during decryption contains only the header
const header = this.readHeader(this.prevChunk);
this.salt = header.salt;
this.rs = header.rs;
this.key = await this.generateKey();
this.nonceBase = await this.generateNonceBase();
} else {
controller.enqueue(
await this.decryptRecord(this.prevChunk, this.seq - 1, isLast)
);
}
this.seq++;
}
}
async transform(chunk, controller) {
if (!this.firstchunk) {
await this.transformPrevChunk(false, controller);
}
this.firstchunk = false;
this.prevChunk = Buffer.from(chunk.buffer);
}
async flush(controller) {
if (this.prevChunk) {
await this.transformPrevChunk(true, controller);
}
}
}
class BlobSlicer {
constructor(blob, rs, mode) {
this.blob = blob;
this.index = 0;
this.mode = mode;
this.chunkSize = mode === MODE_ENCRYPT ? rs - 17 : rs;
}
pull(controller) {
return new Promise((resolve, reject) => {
const bytesLeft = this.blob.size - this.index;
if (bytesLeft <= 0) {
controller.close();
return resolve();
}
let size = 1;
if (this.mode === MODE_DECRYPT && this.index === 0) {
size = Math.min(21, bytesLeft);
} else {
size = Math.min(this.chunkSize, bytesLeft);
}
const blob = this.blob.slice(this.index, this.index + size);
const reader = new FileReader();
reader.onload = () => {
controller.enqueue(new Uint8Array(reader.result));
resolve();
};
reader.onerror = reject;
reader.readAsArrayBuffer(blob);
this.index += size;
});
}
}
class BlobSliceStream extends ReadableStream {
constructor(blob, size, mode) {
super(new BlobSlicer(blob, size, mode));
}
}
/*
input: a blob containing data to be transformed
key: Uint8Array containing key of size KEY_LENGTH
mode: string, either 'encrypt' or 'decrypt'
rs: int containing record size, optional
salt: ArrayBuffer containing salt of KEY_LENGTH length, optional
*/
export default class ECE {
constructor(input, key, mode, rs, salt) {
if (rs === undefined) {
rs = RS;
}
if (salt === undefined) {
salt = generateSalt(KEY_LENGTH);
}
this.streamInfo = {
recordSize: rs,
fileSize: 21 + input.size + 16 * Math.floor(input.size / (rs - 17))
};
const inputStream = new BlobSliceStream(input, rs, mode);
const ts = new TransformStream(new ECETransformer(mode, key, rs, salt));
this.stream = inputStream.pipeThrough(ts);
}
}

2
app/fileManager.js
View File

@ -52,7 +52,7 @@ export default function(state, emitter) {
checkFiles();
});
emitter.on('navigate', checkFiles);
//emitter.on('navigate', checkFiles);
emitter.on('render', () => {
lastRender = Date.now();

25
app/fileReceiver.js
View File

@ -51,6 +51,21 @@ export default class FileReceiver extends Nanobus {
this.state = 'ready';
}
async streamToArrayBuffer(stream, streamSize) {
const reader = stream.getReader();
const result = new Uint8Array(streamSize);
let offset = 0;
let state = await reader.read();
while (!state.done) {
result.set(state.value, offset);
offset += state.value.length;
state = await reader.read();
}
return result.buffer;
}
async download(noSave = false) {
this.state = 'downloading';
this.downloadRequest = await downloadFile(
@ -61,13 +76,21 @@ export default class FileReceiver extends Nanobus {
this.emit('progress');
}
);
try {
const ciphertext = await this.downloadRequest.result;
this.downloadRequest = null;
this.msg = 'decryptingFile';
this.state = 'decrypting';
this.emit('decrypting');
const plaintext = await this.keychain.decryptFile(ciphertext);
const dec = await this.keychain.decryptStream(ciphertext);
const plainstream = dec.stream;
const plaintext = await this.streamToArrayBuffer(
plainstream,
dec.streamInfo.fileSize
);
if (!noSave) {
await saveFile({
plaintext,

21
app/fileSender.js
View File

@ -3,7 +3,7 @@ import Nanobus from 'nanobus';
import OwnedFile from './ownedFile';
import Keychain from './keychain';
import { arrayToB64, bytes } from './utils';
import { uploadFile } from './api';
import { uploadWs } from './api';
export default class FileSender extends Nanobus {
constructor(file) {
@ -59,28 +59,31 @@ export default class FileSender extends Nanobus {
async upload() {
const start = Date.now();
const plaintext = await this.readFile();
if (this.cancelled) {
throw new Error(0);
}
this.msg = 'encryptingFile';
this.emit('encrypting');
const encrypted = await this.keychain.encryptFile(plaintext);
const enc = this.keychain.encryptStream(this.file);
const metadata = await this.keychain.encryptMetadata(this.file);
const authKeyB64 = await this.keychain.authKeyB64();
if (this.cancelled) {
throw new Error(0);
}
this.uploadRequest = uploadFile(
encrypted,
this.uploadRequest = uploadWs(
enc.stream,
enc.streamInfo,
metadata,
authKeyB64,
this.keychain,
p => {
this.progress = p;
this.emit('progress');
}
);
if (this.cancelled) {
throw new Error(0);
}
this.msg = 'fileSizeProgress';
this.emit('progress'); // HACK to kick MS Edge
try {

12
app/keychain.js
View File

@ -1,5 +1,5 @@
import { arrayToB64, b64ToArray } from './utils';
import ECE from './ece.js';
const encoder = new TextEncoder();
const decoder = new TextDecoder();
@ -179,6 +179,16 @@ export default class Keychain {
return ciphertext;
}
encryptStream(plaintext) {
const enc = new ECE(plaintext, this.rawSecret, 'encrypt');
return enc;
}
decryptStream(encstream) {
const dec = new ECE(encstream, this.rawSecret, 'decrypt');
return dec;
}
async decryptFile(ciphertext) {
const encryptKey = await this.encryptKeyPromise;
const plaintext = await window.crypto.subtle.decrypt(

4
docs/build.md
View File

@ -2,11 +2,11 @@ Send has two build configurations, development and production. Both can be run v
# Development
`npm start` launches a `webpack-dev-server` on port 8080 that compiles the assets and watches files for changes. It also serves the backend API and frontend unit tests via the `server/dev.js` entrypoint. The frontend tests can be run in the browser by navigating to http://localhost:8080/test and will rerun automatically as the watched files are saved with changes.
`npm start` launches a `webpack-dev-server` on port 8080 that compiles the assets and watches files for changes. It also serves the backend API and frontend unit tests via the `server/bin/dev.js` entrypoint. The frontend tests can be run in the browser by navigating to http://localhost:8080/test and will rerun automatically as the watched files are saved with changes.
# Production
`npm run build` compiles the assets and writes the files to the `dist/` directory. `npm run prod` launches an Express server on port 1443 that serves the backend API and frontend static assets from `dist/` via the `server/prod.js` entrypoint.
`npm run build` compiles the assets and writes the files to the `dist/` directory. `npm run prod` launches an Express server on port 1443 that serves the backend API and frontend static assets from `dist/` via the `server/bin/prod.js` entrypoint.
# Notable differences

126
package-lock.json generated
View File

@ -485,8 +485,7 @@
"async-limiter": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz",
"integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg==",
"dev": true
"integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg=="
},
"asynckit": {
"version": "0.4.0",
@ -2613,8 +2612,7 @@
"core-util-is": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=",
"dev": true
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
},
"cosmiconfig": {
"version": "4.0.0",
@ -3628,7 +3626,6 @@
"version": "3.5.4",
"resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.5.4.tgz",
"integrity": "sha512-JzYSLYMhoVVBe8+mbHQ4KgpvHpm0DZpJuL8PY93Vyv1fW7jYJ90LoXa1di/CVbJM+TgMs91rbDapE/RNIfnJsA==",
"dev": true,
"requires": {
"end-of-stream": "1.4.1",
"inherits": "2.0.3",
@ -3640,7 +3637,6 @@
"version": "2.3.5",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.5.tgz",
"integrity": "sha512-tK0yDhrkygt/knjowCUiWP9YdV7c5R+8cR0r/kt9ZhBU906Fs6RpQJCEilamRJj1Nx2rWI6LkW9gKqjTkshhEw==",
"dev": true,
"requires": {
"core-util-is": "1.0.2",
"inherits": "2.0.3",
@ -3655,7 +3651,6 @@
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz",
"integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==",
"dev": true,
"requires": {
"safe-buffer": "5.1.1"
}
@ -3719,7 +3714,6 @@
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz",
"integrity": "sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q==",
"dev": true,
"requires": {
"once": "1.4.0"
}
@ -4373,6 +4367,14 @@
}
}
},
"express-ws": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/express-ws/-/express-ws-4.0.0.tgz",
"integrity": "sha512-KEyUw8AwRET2iFjFsI1EJQrJ/fHeGiJtgpYgEWG3yDv4l/To/m3a2GaYfeGyB3lsWdvbesjF5XCMx+SVBgAAYw==",
"requires": {
"ws": "5.2.0"
}
},
"extend": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.1.tgz",
@ -6496,6 +6498,15 @@
"ctype": "0.5.3"
}
},
"http_ece": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/http_ece/-/http_ece-1.0.5.tgz",
"integrity": "sha1-tgZg+q8UIVEC0Uk+pyDc2StTNy8=",
"dev": true,
"requires": {
"urlsafe-base64": "1.0.0"
}
},
"https-browserify": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz",
@ -12113,7 +12124,6 @@
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
"integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
"dev": true,
"requires": {
"wrappy": "1.0.2"
}
@ -15771,8 +15781,7 @@
"process-nextick-args": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
"integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==",
"dev": true
"integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw=="
},
"progress": {
"version": "2.0.0",
@ -15894,6 +15903,19 @@
"proxy-from-env": "1.0.0",
"rimraf": "2.6.2",
"ws": "3.3.3"
},
"dependencies": {
"ws": {
"version": "3.3.3",
"resolved": "https://registry.npmjs.org/ws/-/ws-3.3.3.tgz",
"integrity": "sha512-nnWLa/NwZSt4KQJu51MYlCcSQ5g7INpOrOMt4XV8j4dqTXdmlUmSHQ8/oLC069ckre0fRsgfvsKwbTdtKLCDkA==",
"dev": true,
"requires": {
"async-limiter": "1.0.0",
"safe-buffer": "5.1.1",
"ultron": "1.1.1"
}
}
}
},
"q": {
@ -17655,8 +17677,7 @@
"stream-shift": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.0.tgz",
"integrity": "sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI=",
"dev": true
"integrity": "sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI="
},
"stream-to-observable": {
"version": "0.2.0",
@ -18713,8 +18734,7 @@
"ultron": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.1.tgz",
"integrity": "sha512-UIEXBNeYmKptWH6z8ZnqTeS8fV74zG0/eRU9VGkpzz+LIJNs8W/zM/L+7ctCkRrgbNnnR0xxw4bKOr0cW0N0Og==",
"dev": true
"integrity": "sha512-UIEXBNeYmKptWH6z8ZnqTeS8fV74zG0/eRU9VGkpzz+LIJNs8W/zM/L+7ctCkRrgbNnnR0xxw4bKOr0cW0N0Og=="
},
"unassert": {
"version": "1.5.1",
@ -19004,6 +19024,12 @@
}
}
},
"urlsafe-base64": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/urlsafe-base64/-/urlsafe-base64-1.0.0.tgz",
"integrity": "sha1-I/iQaabGL0bPOh07ABac77kL4MY=",
"dev": true
},
"use": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/use/-/use-2.0.2.tgz",
@ -19123,8 +19149,7 @@
"util-deprecate": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=",
"dev": true
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
},
"util.promisify": {
"version": "1.0.0",
@ -19295,6 +19320,12 @@
"minimalistic-assert": "1.0.0"
}
},
"web-streams-polyfill": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-1.3.2.tgz",
"integrity": "sha1-NxkkXpCSgtk5Z4JfRLzVUOnAOZU=",
"dev": true
},
"webidl-conversions": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz",
@ -19794,6 +19825,53 @@
"integrity": "sha512-nqHUnMXmBzT0w570r2JpJxfiSD1IzoI+HGVdd3aZ0yNi3ngvQ4jv1dtHt5VGxfI2yj5yqImPhOK4vmIh2xMbGg==",
"dev": true
},
"websocket-stream": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/websocket-stream/-/websocket-stream-5.1.2.tgz",
"integrity": "sha512-lchLOk435iDWs0jNuL+hiU14i3ERSrMA0IKSiJh7z6X/i4XNsutBZrtqu2CPOZuA4G/zabiqVAos0vW+S7GEVw==",
"requires": {
"duplexify": "3.5.4",
"inherits": "2.0.3",
"readable-stream": "2.3.6",
"safe-buffer": "5.1.1",
"ws": "3.3.3",
"xtend": "4.0.1"
},
"dependencies": {
"readable-stream": {
"version": "2.3.6",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
"integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==",
"requires": {
"core-util-is": "1.0.2",
"inherits": "2.0.3",
"isarray": "1.0.0",
"process-nextick-args": "2.0.0",
"safe-buffer": "5.1.1",
"string_decoder": "1.1.1",
"util-deprecate": "1.0.2"
}
},
"string_decoder": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"requires": {
"safe-buffer": "5.1.1"
}
},
"ws": {
"version": "3.3.3",
"resolved": "https://registry.npmjs.org/ws/-/ws-3.3.3.tgz",
"integrity": "sha512-nnWLa/NwZSt4KQJu51MYlCcSQ5g7INpOrOMt4XV8j4dqTXdmlUmSHQ8/oLC069ckre0fRsgfvsKwbTdtKLCDkA==",
"requires": {
"async-limiter": "1.0.0",
"safe-buffer": "5.1.1",
"ultron": "1.1.1"
}
}
}
},
"whatwg-encoding": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.3.tgz",
@ -19882,8 +19960,7 @@
"wrappy": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
"dev": true
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
},
"wreck": {
"version": "12.5.1",
@ -19922,14 +19999,11 @@
}
},
"ws": {
"version": "3.3.3",
"resolved": "https://registry.npmjs.org/ws/-/ws-3.3.3.tgz",
"integrity": "sha512-nnWLa/NwZSt4KQJu51MYlCcSQ5g7INpOrOMt4XV8j4dqTXdmlUmSHQ8/oLC069ckre0fRsgfvsKwbTdtKLCDkA==",
"dev": true,
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/ws/-/ws-5.2.0.tgz",
"integrity": "sha512-c18dMeW+PEQdDFzkhDsnBAlS4Z8KGStBQQUcQ5mf7Nf689jyGk0594L+i9RaQuf4gog6SvWLJorz2NfSaqxZ7w==",
"requires": {
"async-limiter": "1.0.0",
"safe-buffer": "5.1.1",
"ultron": "1.1.1"
"async-limiter": "1.0.0"
}
},
"x-is-function": {

8
package.json
View File

@ -31,7 +31,7 @@
"test-integration": "docker-compose up --abort-on-container-exit --exit-code-from integration-tests --build --remove-orphans --quiet-pull && docker-compose down",
"test-integration-stage": "cross-env BASE_URL=https://send.stage.mozaws.net npm run test-integration",
"start": "npm run clean && cross-env NODE_ENV=development webpack-dev-server",
"prod": "node server/prod.js"
"prod": "node server/bin/prod.js"
},
"lint-staged": {
"*.js": [
@ -82,6 +82,7 @@
"git-rev-sync": "^1.10.0",
"github-changes": "^1.1.2",
"html-loader": "^0.5.5",
"http_ece": "^1.0.5",
"husky": "^0.14.3",
"lint-staged": "^7.0.0",
"mocha": "^5.0.4",
@ -109,6 +110,7 @@
"svgo-loader": "^2.1.0",
"testpilot-ga": "^0.3.0",
"val-loader": "^1.1.0",
"web-streams-polyfill": "^1.3.2",
"webpack": "^3.11.0",
"webpack-dev-middleware": "^2.0.6",
"webpack-dev-server": "2.9.1",
@ -122,13 +124,15 @@
"cldr-core": "^32.0.0",
"convict": "^4.0.1",
"express": "^4.16.2",
"express-ws": "^4.0.0",
"fluent": "^0.6.3",
"fluent-langneg": "^0.1.0",
"helmet": "^3.12.0",
"mkdirp": "^0.5.1",
"mozlog": "^2.2.0",
"raven": "^2.4.2",
"redis": "^2.8.0"
"redis": "^2.8.0",
"websocket-stream": "^5.1.2"
},
"availableLanguages": [
"en-US",

23
server/bin/dev.js Normal file
View File

@ -0,0 +1,23 @@
const assets = require('../../common/assets');
const locales = require('../../common/locales');
const routes = require('../routes');
const pages = require('../routes/pages');
const tests = require('../../test/frontend/routes');
const express = require('express');
const expressWs = require('express-ws');
const config = require('../config');
const wsapp = express();
expressWs(wsapp, null, { perMessageDeflate: false });
wsapp.ws('/api/ws', require('../routes/ws'));
wsapp.listen(8081, config.listen_address);
module.exports = function(app, devServer) {
assets.setMiddleware(devServer.middleware);
locales.setMiddleware(devServer.middleware);
routes(app);
tests(app);
// webpack-dev-server routes haven't been added yet
// so wait for next tick to add 404 handler
process.nextTick(() => app.use(pages.notfound));
};

12
server/prod.js → server/bin/prod.js
View File

@ -1,20 +1,22 @@
const express = require('express');
const path = require('path');
const Raven = require('raven');
const config = require('./config');
const routes = require('./routes');
const pages = require('./routes/pages');
const config = require('../config');
const routes = require('../routes');
const pages = require('../routes/pages');
const expressWs = require('express-ws');
if (config.sentry_dsn) {
Raven.config(config.sentry_dsn).install();
}
const app = express();
expressWs(app, null, { perMessageDeflate: false });
app.ws('/api/ws', require('../routes/ws'));
routes(app);
app.use(
express.static(path.resolve(__dirname, '../dist/'), {
express.static(path.resolve(__dirname, '../../dist/'), {
setHeaders: function(res) {
res.set('Cache-Control', 'public, max-age=31536000, immutable');
res.removeHeader('Pragma');

18
server/bin/test.js Normal file
View File

@ -0,0 +1,18 @@
const assets = require('../../common/assets');
const locales = require('../../common/locales');
const routes = require('../routes');
const pages = require('../routes/pages');
const tests = require('../../test/frontend/routes');
const expressWs = require('express-ws');
module.exports = function(app, devServer) {
assets.setMiddleware(devServer.middleware);
locales.setMiddleware(devServer.middleware);
expressWs(app, null, { perMessageDeflate: false });
app.ws('/api/ws', require('../routes/ws'));
routes(app);
tests(app);
// webpack-dev-server routes haven't been added yet
// so wait for next tick to add 404 handler
process.nextTick(() => app.use(pages.notfound));
};

15
server/dev.js
View File

@ -1,15 +0,0 @@
const assets = require('../common/assets');
const locales = require('../common/locales');
const routes = require('./routes');
const pages = require('./routes/pages');
const tests = require('../test/frontend/routes');
module.exports = function(app, devServer) {
assets.setMiddleware(devServer.middleware);
locales.setMiddleware(devServer.middleware);
routes(app);
tests(app);
// webpack-dev-server routes haven't been added yet
// so wait for next tick to add 404 handler
process.nextTick(() => app.use(pages.notfound));
};

73
server/routes/ws.js Normal file
View File

@ -0,0 +1,73 @@
const crypto = require('crypto');
const storage = require('../storage');
const config = require('../config');
const mozlog = require('../log');
const Limiter = require('../limiter');
const Parser = require('../streamparser');
const wsStream = require('websocket-stream/stream');
const log = mozlog('send.upload');
module.exports = async function(ws, req) {
let fileStream;
ws.on('close', e => {
if (e !== 1000 && fileStream !== undefined) {
fileStream.destroy();
}
});
ws.once('message', async function(message) {
try {
const newId = crypto.randomBytes(5).toString('hex');
const owner = crypto.randomBytes(10).toString('hex');
const fileInfo = JSON.parse(message);
const metadata = fileInfo.fileMetadata;
const auth = fileInfo.authorization;
if (!metadata || !auth) {
ws.send(
JSON.stringify({
error: 400
})
);
return ws.close();
}
const meta = {
owner,
metadata,
auth: auth.split(' ')[1],
nonce: crypto.randomBytes(16).toString('base64')
};
const protocol = config.env === 'production' ? 'https' : req.protocol;
const url = `${protocol}://${req.get('host')}/download/${newId}/`;
const limiter = new Limiter(config.max_file_size);
const parser = new Parser();
fileStream = wsStream(ws, { binary: true })
.pipe(limiter)
.pipe(parser);
storage.set(newId, fileStream, meta);
ws.send(
JSON.stringify({
url,
owner: meta.owner,
id: newId,
authentication: `send-v1 ${meta.nonce}`
})
);
} catch (e) {
log.error('upload', e);
ws.send(
JSON.stringify({
error: e === 'limit' ? 413 : 500
})
);
ws.close();
}
});
};

23
server/streamparser.js Normal file
View File

@ -0,0 +1,23 @@
const { Transform } = require('stream');
class StreamParser extends Transform {
constructor() {
super();
let res;
this.promise = new Promise(resolve => {
res = resolve;
});
this.res = res;
}
_transform(chunk, encoding, callback) {
if (chunk.byteLength === 1 && chunk[0] === 0) {
this.res();
} else {
this.push(chunk);
}
callback();
}
}
module.exports = StreamParser;

2
test/frontend/runner.js
View File

@ -7,7 +7,7 @@ const webpack = require('webpack');
const config = require('../../webpack.config');
const middleware = require('webpack-dev-middleware');
const express = require('express');
const devRoutes = require('../../server/dev');
const devRoutes = require('../../server/bin/test');
const app = express();
const wpm = middleware(webpack(config), { logLevel: 'silent' });

13
test/frontend/tests/api-tests.js
View File

@ -3,21 +3,22 @@ import * as api from '../../../app/api';
import Keychain from '../../../app/keychain';
const encoder = new TextEncoder();
const plaintext = encoder.encode('hello world!');
const plaintext = new Blob([encoder.encode('hello world!')]);
const metadata = {
name: 'test.txt',
type: 'text/plain'
};
describe('API', function() {
describe('uploadFile', function() {
describe('websocket upload', function() {
it('returns file info on success', async function() {
const keychain = new Keychain();
const encrypted = await keychain.encryptFile(plaintext);
const enc = keychain.encryptStream(plaintext);
const meta = await keychain.encryptMetadata(metadata);
const verifierB64 = await keychain.authKeyB64();
const p = function() {};
const up = api.uploadFile(encrypted, meta, verifierB64, keychain, p);
const up = api.uploadWs(enc.stream, enc.streamInfo, meta, verifierB64, p);
const result = await up.result;
assert.ok(result.url);
assert.ok(result.id);
@ -26,11 +27,11 @@ describe('API', function() {
it('can be cancelled', async function() {
const keychain = new Keychain();
const encrypted = await keychain.encryptFile(plaintext);
const enc = keychain.encryptStream(plaintext);
const meta = await keychain.encryptMetadata(metadata);
const verifierB64 = await keychain.authKeyB64();
const p = function() {};
const up = api.uploadFile(encrypted, meta, verifierB64, keychain, p);
const up = api.uploadWs(enc.stream, enc.streamInfo, meta, verifierB64, p);
up.cancel();
try {
await up.result;

64
test/frontend/tests/streaming-tests.js Normal file
View File

@ -0,0 +1,64 @@
const ece = require('http_ece');
require('buffer');
import assert from 'assert';
import { b64ToArray } from '../../../app/utils';
import ECE from '../../../app/ece.js';
const rs = 36;
const str = 'You are the dancing queen, young and sweet, only seventeen.';
const testSalt = 'I1BsxtFttlv3u_Oo94xnmw';
const keystr = 'yqdlZ-tYemfogSmv7Ws5PQ';
const buffer = Buffer.from(str);
const params = {
version: 'aes128gcm',
rs: rs,
salt: testSalt,
keyid: '',
key: keystr
};
const encrypted = ece.encrypt(buffer, params);
const decrypted = ece.decrypt(encrypted, params);
describe('Streaming', function() {
//testing against http_ece's implementation
describe('ECE', function() {
const key = b64ToArray(keystr);
const salt = b64ToArray(testSalt).buffer;
const blob = new Blob([str], { type: 'text/plain' });
it('can encrypt', async function() {
const encStream = new ECE(blob, key, 'encrypt', rs, salt).stream;
const reader = encStream.getReader();
let result = Buffer.from([]);
let state = await reader.read();
while (!state.done) {
result = Buffer.concat([result, state.value]);
state = await reader.read();
}
assert.deepEqual(result, encrypted);
});
it('can decrypt', async function() {
const encBlob = new Blob([encrypted]);
const decStream = await new ECE(encBlob, key, 'decrypt', rs).stream;
const reader = decStream.getReader();
let result = Buffer.from([]);
let state = await reader.read();
while (!state.done) {
result = Buffer.concat([result, state.value]);
state = await reader.read();
}
assert.deepEqual(result, decrypted);
});
});
});

6
test/frontend/tests/workflow-tests.js
View File

@ -93,7 +93,7 @@ describe('Upload / Download flow', function() {
fs.cancel(); // before encrypting
try {
await up;
assert.fail('not cancelled');
assert.fail('not cancelled 1');
} catch (e) {
assert.equal(e.message, '0');
}
@ -101,7 +101,7 @@ describe('Upload / Download flow', function() {
fs.once('encrypting', () => fs.cancel());
try {
await fs.upload();
assert.fail('not cancelled');
assert.fail('not cancelled 2');
} catch (e) {
assert.equal(e.message, '0');
}
@ -109,7 +109,7 @@ describe('Upload / Download flow', function() {
fs.once('progress', () => fs.cancel());
try {
await fs.upload();
assert.fail('not cancelled');
assert.fail('not cancelled 3');
} catch (e) {
assert.equal(e.message, '0');
}

9
webpack.config.js
View File

@ -209,6 +209,13 @@ module.exports = {
devServer: {
compress: true,
host: '0.0.0.0',
before: IS_DEV ? require('./server/dev') : undefined
before: IS_DEV ? require('./server/bin/dev') : undefined,
proxy: {
'/api/ws': {
target: 'ws://localhost:8081',
ws: true,
secure: false
}
}
}
};