diff --git a/app/archive.js b/app/archive.js index 20766055..572bb071 100644 --- a/app/archive.js +++ b/app/archive.js @@ -1,4 +1,4 @@ -/* global LIMITS */ +/* global LIMITS DEFAULTS */ import { blobStream, concatStream } from './streams'; function isDupe(newFile, array) { @@ -17,6 +17,9 @@ function isDupe(newFile, array) { export default class Archive { constructor(files = []) { this.files = Array.from(files); + this.timeLimit = DEFAULTS.EXPIRE_SECONDS; + this.dlimit = 1; + this.password = null; } get name() { @@ -73,5 +76,8 @@ export default class Archive { clear() { this.files = []; + this.dlimit = 1; + this.timeLimit = DEFAULTS.EXPIRE_SECONDS; + this.password = null; } } diff --git a/app/controller.js b/app/controller.js index 47675d6b..95fa9ad3 100644 --- a/app/controller.js +++ b/app/controller.js @@ -1,4 +1,4 @@ -/* global DEFAULTS LIMITS */ +/* global LIMITS */ import FileSender from './fileSender'; import FileReceiver from './fileReceiver'; import { copyToClipboard, delay, openLinksInNewTab, percent } from './utils'; @@ -50,37 +50,27 @@ export default function(state, emitter) { emitter.on('logout', () => { state.user.logout(); - state.timeLimit = DEFAULTS.EXPIRE_SECONDS; - state.downloadCount = 1; + metrics.loggedOut({ trigger: 'button' }); emitter.emit('pushState', '/'); }); - emitter.on('changeLimit', async ({ file, value }) => { - const ok = await file.changeLimit(value, state.user); - if (!ok) { - return; - } - state.storage.writeFile(file); - metrics.changedDownloadLimit(file); - }); - emitter.on('removeUpload', file => { state.archive.remove(file); render(); }); - emitter.on('delete', async ({ file, location }) => { + emitter.on('delete', async ownedFile => { try { metrics.deletedUpload({ - size: file.size, - time: file.time, - speed: file.speed, - type: file.type, - ttl: file.expiresAt - Date.now(), + size: ownedFile.size, + time: ownedFile.time, + speed: ownedFile.speed, + type: ownedFile.type, + ttl: ownedFile.expiresAt - Date.now(), location }); - state.storage.remove(file.id); - await file.del(); + state.storage.remove(ownedFile.id); + await ownedFile.del(); } catch (e) { state.raven.captureException(e); } @@ -100,20 +90,35 @@ export default function(state, emitter) { state.archive.addFiles(files, maxSize); } catch (e) { if (e.message === 'fileTooBig' && maxSize < LIMITS.MAX_FILE_SIZE) { - state.modal = signupDialog(); - } else { - state.modal = okDialog( - state.translate(e.message, { - size: bytes(maxSize), - count: LIMITS.MAX_FILES_PER_ARCHIVE - }) - ); + return emitter.emit('signup-cta', 'size'); } + state.modal = okDialog( + state.translate(e.message, { + size: bytes(maxSize), + count: LIMITS.MAX_FILES_PER_ARCHIVE + }) + ); } render(); }); - emitter.on('upload', async ({ type, dlimit, password }) => { + emitter.on('signup-cta', source => { + state.modal = signupDialog(source); + render(); + }); + + emitter.on('authenticate', async (code, oauthState) => { + try { + await state.user.finishLogin(code, oauthState); + await state.user.syncFileList(); + emitter.emit('replaceState', '/'); + } catch (e) { + emitter.emit('replaceState', '/error'); + setTimeout(render); + } + }); + + emitter.on('upload', async () => { if (state.storage.files.length >= LIMITS.MAX_ARCHIVES_PER_USER) { state.modal = okDialog( state.translate('tooManyArchives', { @@ -122,8 +127,7 @@ export default function(state, emitter) { ); return render(); } - const size = state.archive.size; - if (!state.timeLimit) state.timeLimit = DEFAULTS.EXPIRE_SECONDS; + const archive = state.archive; const sender = new FileSender(); sender.on('progress', updateProgress); @@ -135,41 +139,38 @@ export default function(state, emitter) { const links = openLinksInNewTab(); await delay(200); + const start = Date.now(); try { - metrics.startedUpload({ size, type }); - - const ownedFile = await sender.upload( - state.archive, - state.timeLimit, - dlimit, - state.user.bearerToken - ); - ownedFile.type = type; + const ownedFile = await sender.upload(archive, state.user.bearerToken); state.storage.totalUploads += 1; - metrics.completedUpload(ownedFile); + const duration = Date.now() - start; + metrics.completedUpload(archive, duration); state.storage.addFile(ownedFile); // TODO integrate password into /upload request - if (password) { - emitter.emit('password', { password, file: ownedFile }); + if (archive.password) { + emitter.emit('password', { + password: archive.password, + file: ownedFile + }); } state.modal = copyDialog(ownedFile.name, ownedFile.url); } catch (err) { if (err.message === '0') { //cancelled. do nothing - metrics.cancelledUpload({ size, type }); + const duration = Date.now() - start; + metrics.cancelledUpload(archive, duration); render(); } else { // eslint-disable-next-line no-console console.error(err); state.raven.captureException(err); - metrics.stoppedUpload({ size, type, err }); + metrics.stoppedUpload(archive); emitter.emit('pushState', '/error'); } } finally { openLinksInNewTab(links, false); - state.archive.clear(); - state.password = ''; + archive.clear(); state.uploading = false; state.transfer = null; await state.user.syncFileList(); @@ -183,7 +184,6 @@ export default function(state, emitter) { render(); await file.setPassword(password); state.storage.writeFile(file); - metrics.addedPassword({ size: file.size }); await delay(1000); } catch (err) { // eslint-disable-next-line no-console @@ -220,18 +220,20 @@ export default function(state, emitter) { state.transfer.on('complete', render); const links = openLinksInNewTab(); const size = file.size; + const start = Date.now(); try { - const start = Date.now(); - metrics.startedDownload({ size: file.size, ttl: file.ttl }); const dl = state.transfer.download({ stream: state.capabilities.streamDownload }); render(); await dl; - const time = Date.now() - start; - const speed = size / (time / 1000); state.storage.totalDownloads += 1; - metrics.completedDownload({ size, time, speed }); + const duration = Date.now() - start; + metrics.completedDownload({ + size, + duration, + password_protected: file.requiresPassword + }); } catch (err) { if (err.message === '0') { // download cancelled @@ -239,12 +241,16 @@ export default function(state, emitter) { render(); } else { // eslint-disable-next-line no-console - console.error(err); state.transfer = null; const location = err.message === '404' ? '/404' : '/error'; if (location === '/error') { state.raven.captureException(err); - metrics.stoppedDownload({ size, err }); + const duration = Date.now() - start; + metrics.stoppedDownload({ + size, + duration, + password_protected: file.requiresPassword + }); } emitter.emit('pushState', location); } @@ -253,9 +259,9 @@ export default function(state, emitter) { } }); - emitter.on('copy', ({ url, location }) => { + emitter.on('copy', ({ url }) => { copyToClipboard(url); - metrics.copiedLink({ location }); + // metrics.copiedLink({ location }); }); setInterval(() => { diff --git a/app/fileSender.js b/app/fileSender.js index cc937833..47955068 100644 --- a/app/fileSender.js +++ b/app/fileSender.js @@ -1,4 +1,3 @@ -/* global DEFAULTS */ import Nanobus from 'nanobus'; import OwnedFile from './ownedFile'; import Keychain from './keychain'; @@ -42,29 +41,24 @@ export default class FileSender extends Nanobus { } } - async upload( - file, - timeLimit = DEFAULTS.EXPIRE_SECONDS, - dlimit = 1, - bearerToken - ) { + async upload(archive, bearerToken) { const start = Date.now(); if (this.cancelled) { throw new Error(0); } this.msg = 'encryptingFile'; this.emit('encrypting'); - const totalSize = encryptedSize(file.size); - const encStream = await this.keychain.encryptStream(file.stream); - const metadata = await this.keychain.encryptMetadata(file); + const totalSize = encryptedSize(archive.size); + const encStream = await this.keychain.encryptStream(archive.stream); + const metadata = await this.keychain.encryptMetadata(archive); const authKeyB64 = await this.keychain.authKeyB64(); this.uploadRequest = uploadWs( encStream, metadata, authKeyB64, - timeLimit, - dlimit, + archive.timeLimit, + archive.dlimit, bearerToken, p => { this.progress = [p, totalSize]; @@ -88,18 +82,18 @@ export default class FileSender extends Nanobus { const ownedFile = new OwnedFile({ id: result.id, url: `${result.url}#${secretKey}`, - name: file.name, - size: file.size, - manifest: file.manifest, + name: archive.name, + size: archive.size, + manifest: archive.manifest, time: time, - speed: file.size / (time / 1000), + speed: archive.size / (time / 1000), createdAt: Date.now(), - expiresAt: Date.now() + timeLimit * 1000, + expiresAt: Date.now() + archive.timeLimit * 1000, secretKey: secretKey, nonce: this.keychain.nonce, ownerToken: result.ownerToken, - dlimit, - timeLimit: timeLimit + dlimit: archive.dlimit, + timeLimit: archive.timeLimit }); return ownedFile; diff --git a/app/metrics.js b/app/metrics.js index 1f3c25db..3c02e2a7 100644 --- a/app/metrics.js +++ b/app/metrics.js @@ -1,296 +1,172 @@ -import testPilotGA from 'testpilot-ga/src/TestPilotGA'; import storage from './storage'; - -let hasLocalStorage = false; -try { - hasLocalStorage = typeof localStorage !== 'undefined'; -} catch (e) { - // when disabled, any mention of localStorage throws an error -} - -const analytics = new testPilotGA({ - an: 'Firefox Send', - ds: 'web', - tid: window.GOOGLE_ANALYTICS_ID -}); +import { platform } from './utils'; let appState = null; -let experiment = null; +// let experiment = null; +const HOUR = 1000 * 60 * 60; +const events = []; +let session_id = Date.now(); +const lang = document.querySelector('html').lang; export default function initialize(state, emitter) { appState = state; + if (!appState.user.firstAction) { + appState.user.firstAction = appState.route === '/' ? 'upload' : 'download'; + } emitter.on('DOMContentLoaded', () => { - addExitHandlers(); - experiment = storage.enrolled[0]; - sendEvent(category(), 'visit', { - cm5: storage.totalUploads, - cm6: storage.files.length, - cm7: storage.totalDownloads + // experiment = storage.enrolled[0]; + addEvent('client_visit', { + entrypoint: appState.route === '/' ? 'upload' : 'download' }); }); - emitter.on('exit', exitEvent); emitter.on('experiment', experimentEvent); + window.addEventListener('unload', submitEvents); } -function category() { - switch (appState.route) { - case '/': - case '/share/:id': - return 'sender'; - case '/download/:id/:key': - case '/download/:id': - case '/completed': - return 'recipient'; - default: - return 'other'; - } +function sizeOrder(n) { + return Math.floor(Math.log10(n)); } -function sendEvent() { - const args = Array.from(arguments); - if (experiment && args[2]) { - args[2].xid = experiment[0]; - args[2].xvar = experiment[1]; +function submitEvents() { + if (navigator.doNotTrack === '1') { + return; } - return ( - hasLocalStorage && analytics.sendEvent.apply(analytics, args).catch(() => 0) + const data = new Blob( + [ + JSON.stringify({ + now: Date.now(), + session_id, + lang, + platform: platform(), + events + }) + ], + { type: 'application/json' } ); -} - -function urlToMetric(url) { - switch (url) { - case 'https://www.mozilla.org/': - return 'mozilla'; - case 'https://www.mozilla.org/about/legal': - return 'legal'; - case 'https://testpilot.firefox.com/about': - return 'about'; - case 'https://testpilot.firefox.com/privacy': - return 'privacy'; - case 'https://testpilot.firefox.com/terms': - return 'terms'; - case 'https://www.mozilla.org/privacy/websites/#cookies': - return 'cookies'; - case 'https://github.com/mozilla/send': - return 'github'; - case 'https://twitter.com/FxTestPilot': - return 'twitter'; - case 'https://www.mozilla.org/firefox/new/?scene=2': - return 'download-firefox'; - case 'https://qsurvey.mozilla.com/s3/txp-firefox-send': - return 'survey'; - case 'https://testpilot.firefox.com/': - case 'https://testpilot.firefox.com/experiments/send': - return 'testpilot'; - case 'https://www.mozilla.org/firefox/new/?utm_campaign=send-acquisition&utm_medium=referral&utm_source=send.firefox.com': - return 'promo'; - default: - return 'other'; + events.splice(0); + if (!navigator.sendBeacon) { + return; } + navigator.sendBeacon('/api/metrics', data); } -function setReferrer(state) { - if (category() === 'sender') { - if (state) { - storage.referrer = `${state}-upload`; - } - } else if (category() === 'recipient') { - if (state) { - storage.referrer = `${state}-download`; +async function addEvent(event_type, event_properties) { + const user_id = await appState.user.metricId(); + const device_id = await appState.user.deviceId(); + events.push({ + device_id, + event_properties, + event_type, + time: Date.now(), + user_id, + user_properties: { + anonymous: !appState.user.loggedIn, + first_action: appState.user.firstAction, + active_count: storage.files.length } + }); + if (events.length === 25) { + submitEvents(); } } -function externalReferrer() { - if (/^https:\/\/testpilot\.firefox\.com/.test(document.referrer)) { - return 'testpilot'; - } - return 'external'; -} - -function takeReferrer() { - const referrer = storage.referrer || externalReferrer(); - storage.referrer = null; - return referrer; -} - -function startedUpload(params) { - return sendEvent('sender', 'upload-started', { - cm1: params.size, - cm5: storage.totalUploads, - cm6: storage.files.length + 1, - cm7: storage.totalDownloads, - cd1: params.type, - cd5: takeReferrer() +function cancelledUpload(archive, duration) { + return addEvent('client_upload', { + download_limit: archive.dlimit, + duration: sizeOrder(duration), + file_count: archive.numFiles, + password_protected: !!archive.password, + size: sizeOrder(archive.size), + status: 'cancel', + time_limit: archive.timeLimit }); } -function cancelledUpload(params) { - setReferrer('cancelled'); - return sendEvent('sender', 'upload-stopped', { - cm1: params.size, - cm5: storage.totalUploads, - cm6: storage.files.length, - cm7: storage.totalDownloads, - cd1: params.type, - cd2: 'cancelled' +function completedUpload(archive, duration) { + return addEvent('client_upload', { + download_limit: archive.dlimit, + duration: sizeOrder(duration), + file_count: archive.numFiles, + password_protected: !!archive.password, + size: sizeOrder(archive.size), + status: 'ok', + time_limit: archive.timeLimit }); } -function completedUpload(params) { - return sendEvent('sender', 'upload-stopped', { - cm1: params.size, - cm2: params.time, - cm3: params.speed, - cm5: storage.totalUploads, - cm6: storage.files.length, - cm7: storage.totalDownloads, - cd1: params.type, - cd2: 'completed' - }); -} - -function addedPassword(params) { - return sendEvent('sender', 'password-added', { - cm1: params.size, - cm5: storage.totalUploads, - cm6: storage.files.length, - cm7: storage.totalDownloads - }); -} - -function startedDownload(params) { - return sendEvent('recipient', 'download-started', { - cm1: params.size, - cm4: params.ttl, - cm5: storage.totalUploads, - cm6: storage.files.length, - cm7: storage.totalDownloads +function stoppedUpload(archive) { + return addEvent('client_upload', { + download_limit: archive.dlimit, + file_count: archive.numFiles, + password_protected: !!archive.password, + size: sizeOrder(archive.size), + status: 'error', + time_limit: archive.timeLimit }); } function stoppedDownload(params) { - return sendEvent('recipient', 'download-stopped', { - cm1: params.size, - cm5: storage.totalUploads, - cm6: storage.files.length, - cm7: storage.totalDownloads, - cd2: 'errored', - cd6: params.err - }); -} - -function cancelledDownload(params) { - setReferrer('cancelled'); - return sendEvent('recipient', 'download-stopped', { - cm1: params.size, - cm5: storage.totalUploads, - cm6: storage.files.length, - cm7: storage.totalDownloads, - cd2: 'cancelled' - }); -} - -function stoppedUpload(params) { - return sendEvent('sender', 'upload-stopped', { - cm1: params.size, - cm5: storage.totalUploads, - cm6: storage.files.length, - cm7: storage.totalDownloads, - cd1: params.type, - cd2: 'errored', - cd6: params.err - }); -} - -function changedDownloadLimit(params) { - return sendEvent('sender', 'download-limit-changed', { - cm1: params.size, - cm5: storage.totalUploads, - cm6: storage.files.length, - cm7: storage.totalDownloads, - cm8: params.dlimit + return addEvent('client_download', { + duration: sizeOrder(params.duration), + password_protected: params.password_protected, + size: sizeOrder(params.size), + status: 'error' }); } function completedDownload(params) { - return sendEvent('recipient', 'download-stopped', { - cm1: params.size, - cm2: params.time, - cm3: params.speed, - cm5: storage.totalUploads, - cm6: storage.files.length, - cm7: storage.totalDownloads, - cd2: 'completed' + return addEvent('client_download', { + duration: sizeOrder(params.duration), + password_protected: params.password_protected, + size: sizeOrder(params.size), + status: 'ok' }); } -function deletedUpload(params) { - return sendEvent(category(), 'upload-deleted', { - cm1: params.size, - cm2: params.time, - cm3: params.speed, - cm4: params.ttl, - cm5: storage.totalUploads, - cm6: storage.files.length, - cm7: storage.totalDownloads, - cd1: params.type, - cd4: params.location - }); -} - -function unsupported(params) { - return sendEvent(category(), 'unsupported', { - cd6: params.err - }); -} - -function copiedLink(params) { - return sendEvent('sender', 'copied', { - cd4: params.location - }); -} - -function exitEvent(target) { - return sendEvent(category(), 'exited', { - cd3: urlToMetric(target.currentTarget.href) +function deletedUpload(ownedFile) { + return addEvent('client_delete', { + age: Math.floor((Date.now() - ownedFile.createdAt) / HOUR), + downloaded: ownedFile.dtotal > 0, + status: 'ok' }); } function experimentEvent(params) { - return sendEvent(category(), 'experiment', params); + return addEvent('client_experiment', params); } -// eslint-disable-next-line no-unused-vars -function addExitHandlers() { - const links = Array.from(document.querySelectorAll('a')); - links.forEach(l => { - if (/^http/.test(l.getAttribute('href'))) { - l.addEventListener('click', exitEvent); - } +function submittedSignup(params) { + return addEvent('client_login', { + status: 'ok', + trigger: params.trigger }); } -function restart(state) { - setReferrer(state); - return sendEvent(category(), 'restarted', { - cd2: state +function canceledSignup(params) { + return addEvent('client_login', { + status: 'cancel', + trigger: params.trigger }); } +function loggedOut(params) { + addEvent('client_logout', { + status: 'ok', + trigger: params.trigger + }); + // flush events and start new anon session + submitEvents(); + session_id = Date.now(); +} + export { - copiedLink, - startedUpload, cancelledUpload, stoppedUpload, completedUpload, - changedDownloadLimit, deletedUpload, - startedDownload, - cancelledDownload, stoppedDownload, completedDownload, - addedPassword, - restart, - unsupported + submittedSignup, + canceledSignup, + loggedOut }; diff --git a/app/ownedFile.js b/app/ownedFile.js index 3fef67cb..d6d91a53 100644 --- a/app/ownedFile.js +++ b/app/ownedFile.js @@ -8,7 +8,6 @@ export default class OwnedFile { this.url = obj.url; this.name = obj.name; this.size = obj.size; - this.type = obj.type; this.manifest = obj.manifest; this.time = obj.time; this.speed = obj.speed; @@ -78,7 +77,6 @@ export default class OwnedFile { url: this.url, name: this.name, size: this.size, - type: this.type, manifest: this.manifest, time: this.time, speed: this.speed, diff --git a/app/routes.js b/app/routes.js index 6abd558e..8405db9b 100644 --- a/app/routes.js +++ b/app/routes.js @@ -11,14 +11,7 @@ module.exports = function(app = choo()) { app.route('/error', body(require('./ui/error'))); app.route('/blank', body(require('./ui/blank'))); app.route('/oauth', async function(state, emit) { - try { - await state.user.finishLogin(state.query.code, state.query.state); - await state.user.syncFileList(); - emit('replaceState', '/'); - } catch (e) { - emit('replaceState', '/error'); - setTimeout(() => emit('render')); - } + emit('authenticate', state.query.code, state.query.state); }); app.route('*', body(require('./ui/notFound'))); return app; diff --git a/app/storage.js b/app/storage.js index 34837318..d66391e0 100644 --- a/app/storage.js +++ b/app/storage.js @@ -1,4 +1,4 @@ -import { isFile } from './utils'; +import { arrayToB64, isFile } from './utils'; import OwnedFile from './ownedFile'; class Mem { @@ -58,6 +58,15 @@ class Storage { return fs; } + get id() { + let id = this.engine.getItem('device_id'); + if (!id) { + id = arrayToB64(crypto.getRandomValues(new Uint8Array(16))); + this.engine.setItem('device_id', id); + } + return id; + } + get totalDownloads() { return Number(this.engine.getItem('totalDownloads')); } diff --git a/app/ui/account.js b/app/ui/account.js index 3ddbd83a..cb67b540 100644 --- a/app/ui/account.js +++ b/app/ui/account.js @@ -1,6 +1,5 @@ const html = require('choo/html'); const Component = require('choo/component'); -const signupDialog = require('./signupDialog'); class Account extends Component { constructor(name, state, emit) { @@ -27,8 +26,7 @@ class Account extends Component { login(event) { event.preventDefault(); - this.state.modal = signupDialog(); - this.emit('render'); + this.emit('signup-cta', 'button'); } logout(event) { diff --git a/app/ui/archiveTile.js b/app/ui/archiveTile.js index 7a70e6f2..e6f65f33 100644 --- a/app/ui/archiveTile.js +++ b/app/ui/archiveTile.js @@ -34,7 +34,7 @@ function password(state) { @@ -44,7 +44,7 @@ function password(state) { ${expiryInfo(state.translate, { - dlimit: state.downloadCount || 1, + dlimit: state.archive.dlimit, dtotal: 0, - expiresAt: Date.now() + 500 + state.timeLimit * 1000 + expiresAt: Date.now() + 500 + state.archive.timeLimit * 1000 })} ${progressPercent} diff --git a/app/ui/expiryOptions.js b/app/ui/expiryOptions.js index f24da37d..3f15d619 100644 --- a/app/ui/expiryOptions.js +++ b/app/ui/expiryOptions.js @@ -3,7 +3,6 @@ const html = require('choo/html'); const raw = require('choo/html/raw'); const { secondsToL10nId } = require('../utils'); const selectbox = require('./selectbox'); -const signupDialog = require('./signupDialog'); module.exports = function(state, emit) { const el = html` @@ -29,17 +28,17 @@ module.exports = function(state, emit) { const dlCountSelect = el.querySelector('#dlCount'); el.replaceChild( selectbox( - state.downloadCount || 1, + state.archive.dlimit, counts, num => state.translate('downloadCount', { num }), value => { const max = state.user.maxDownloads; + state.archive.dlimit = Math.min(value, max); if (value > max) { - state.modal = signupDialog(); - value = max; + emit('signup-cta', 'count'); + } else { + emit('render'); } - state.downloadCount = value; - emit('render'); }, 'expire-after-dl-count-select' ), @@ -53,7 +52,7 @@ module.exports = function(state, emit) { const timeSelect = el.querySelector('#timespan'); el.replaceChild( selectbox( - state.timeLimit || 86400, + state.archive.timeLimit, expires, num => { const l10n = secondsToL10nId(num); @@ -61,12 +60,12 @@ module.exports = function(state, emit) { }, value => { const max = state.user.maxExpireSeconds; + state.archive.timeLimit = Math.min(value, max); if (value > max) { - state.modal = signupDialog(); - value = max; + emit('signup-cta', 'time'); + } else { + emit('render'); } - state.timeLimit = value; - emit('render'); }, 'expire-after-time-select' ), diff --git a/app/ui/signupDialog.js b/app/ui/signupDialog.js index d4583adf..e6e9d57d 100644 --- a/app/ui/signupDialog.js +++ b/app/ui/signupDialog.js @@ -1,8 +1,9 @@ /* global LIMITS */ const html = require('choo/html'); const { bytes, platform } = require('../utils'); +const { canceledSignup, submittedSignup } = require('../metrics'); -module.exports = function() { +module.exports = function(trigger) { return function(state, emit, close) { const hidden = platform() === 'android' ? 'hidden' : ''; let submitting = false; @@ -37,7 +38,7 @@ module.exports = function() { ${state.translate('deletePopupCancel')} + onclick=${cancel}>${state.translate('deletePopupCancel')} `; @@ -50,6 +51,11 @@ module.exports = function() { return a.length === 2 && a.every(s => s.length > 0); } + function cancel(event) { + canceledSignup({ trigger }); + close(event); + } + function submitEmail(event) { event.preventDefault(); if (submitting) { @@ -59,6 +65,7 @@ module.exports = function() { const el = document.getElementById('email-input'); const email = el.value; + submittedSignup({ trigger }); emit('login', emailish(email) ? email : null); } }; diff --git a/app/user.js b/app/user.js index def34d9c..3cfb68d3 100644 --- a/app/user.js +++ b/app/user.js @@ -9,6 +9,16 @@ import storage from './storage'; const textEncoder = new TextEncoder(); const textDecoder = new TextDecoder(); +const anonId = arrayToB64(crypto.getRandomValues(new Uint8Array(16))); + +async function hashId(id) { + const d = new Date(); + const month = d.getUTCMonth(); + const year = d.getUTCFullYear(); + const encoded = textEncoder.encode(`${id}:${year}:${month}`); + const hash = await crypto.subtle.digest('SHA-256', encoded); + return arrayToB64(new Uint8Array(hash.slice(16))); +} export default class User { constructor(storage) { @@ -25,6 +35,14 @@ export default class User { this.storage.user = data; } + get firstAction() { + return this.storage.get('firstAction'); + } + + set firstAction(action) { + this.storage.set('firstAction', action); + } + get avatar() { const defaultAvatar = assets.get('user.svg'); if (this.info.avatarDefault) { @@ -63,6 +81,14 @@ export default class User { return this.loggedIn ? LIMITS.MAX_DOWNLOADS : LIMITS.ANON.MAX_DOWNLOADS; } + async metricId() { + return this.loggedIn ? hashId(this.info.uid) : undefined; + } + + async deviceId() { + return this.loggedIn ? hashId(this.storage.id) : hashId(anonId); + } + async login(email) { const state = arrayToB64(crypto.getRandomValues(new Uint8Array(16))); storage.set('oauthState', state); diff --git a/docs/docker.md b/docs/docker.md index 9003412b..d230558a 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -16,7 +16,6 @@ Then you can run either `docker build` or `docker-compose up`. | `PORT` | Port the server will listen on (defaults to 1443). | `S3_BUCKET` | The S3 bucket name. | `REDIS_HOST` | Host name of the Redis server. -| `GOOGLE_ANALYTICS_ID` | Google Analytics ID | `SENTRY_CLIENT` | Sentry Client ID | `SENTRY_DSN` | Sentry DSN | `MAX_FILE_SIZE` | in bytes (defaults to 2147483648) @@ -28,7 +27,6 @@ Then you can run either `docker build` or `docker-compose up`. $ docker run --net=host -e 'NODE_ENV=production' \ -e 'S3_BUCKET=testpilot-p2p-dev' \ -e 'REDIS_HOST=dyf9s2r4vo3.bolxr4.0001.usw2.cache.amazonaws.com' \ - -e 'GOOGLE_ANALYTICS_ID=UA-35433268-78' \ -e 'SENTRY_CLIENT=https://51e23d7263e348a7a3b90a5357c61cb2@sentry.prod.mozaws.net/168' \ -e 'SENTRY_DSN=https://51e23d7263e348a7a3b90a5357c61cb2:65e23d7263e348a7a3b90a5357c61c44@sentry.prod.mozaws.net/168' \ mozilla/send:latest diff --git a/docs/metrics.md b/docs/metrics.md index b1c65dbe..79e83583 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -39,87 +39,89 @@ We will know this to be true when we can conduct six research tasks (surveys, A/ * FxA UI Engagement `--- DESIRED OUTCOME --->` Authenticate * **STRETCH** App Open or Visit `--- DESIRED OUTCOME --->` Successful Download -## Complete Schema - +## Amplitude Schema Please see, **See Amplitude HTTP API**(https://amplitude.zendesk.com/hc/en-us/articles/204771828) for HTTP API reference. -### Event Structure +## Metric Events -* `app_version` **string** ∙ app version `Android 1.5` or `Web 1.2.5` -* `country` **string** ∙ Can be captured using [FxA Geo Library](https://github.com/mozilla/fxa-geodb) -* `device_id` **string** ∙ required, should be a unique hash -* `event_properties` **dictionary** ∙ [see list below](#event-properties) -* `event_type` **string** ∙ [see list below](#events) -* `insert_id` **string** ∙ unique event id used by amplitude to dedupe events -* `language` **string** ∙ App language -* `os_name` **string** ∙ `Mac OS X`, `iOS`, `Windows`, etc. -* `os_version` **string** ∙ `10.01`, `400`, etc -* `region` **string** ∙ Can be captured using [FxA Geo Library](https://github.com/mozilla/fxa-geodb) -* `session_id` **long** ∙ start time in ms since epoch (this should only be changed at the start of a session, but sent with each ping), set to -1 if event is out of session, such as expiration -* `time` **long** ∙ The timestamp of the event in milliseconds since epoch -* `user_id` **string** ∙ required unless device ID is present, should be a double hash of FxA email -* `user_properties` **dictionary** ∙ [see list below](#user-properties). All user properties can be passed with all events. Amplitude will automatically drop user properties that do not change +In support of our KPIs we collect events from two separate contexts, server and client. The events are designed to have minimal correlation between contexts. -### User Properties +Server events collect lifecycle information about individual uploads but no user information; also time precision is truncated to hour increments. Client events collect information about how users interact with the UI but no upload identifiers. -* `Has account` **boolean** ∙ whether the user is account active -* `First action` **string** ∙ did this user `upload` or `download` first -* `Total uploads` **num** ∙ running sum of bundles uploaded -* `Total upload size` **float** ∙ running sum of total MB uploaded -* `Total downloads` **num** ∙ running count of bundles downloaded -* `Total download size` **float** ∙ running sum of total MB downloaded -* `Total clients` **num** ∙ running tally of total clients sharing a UID -* `Current uploads` **int** ∙ count of current unexpired files -* `User agent Browser` **string** ∙ browser or if app `App` derived from UA string -* `User Agent version` **string** ∙ browser version or if app `App Version` derived from UA string -* `UTM campaign` **string** ∙ referrer -* `UTM content` **string** ∙ referrer -* `UTM medium` **string** ∙ referrer -* `UTM source` **string** ∙ referrer -* `UTM term` **string** ∙ referrer -* `Experiments` **array of strings** ∙ set of experiments the user is in +### Server Events -### Event Properties +Server events allow us to aggregate data about file lifecycle without collecting data about individual users. In this context `user_id` and `user_properties` describe the uploaded archive. -1. `Bundle id` **string** ∙ Guid for bundle -2. `Bundle creation timestamp` **long** ∙ The timestamp of bundle creation in milliseconds since epoch -3. `Number of files` **int** ∙ Number of files in bundle -4. `Size of files` **float** ∙ Size of files in MB -5. `Transfer rate` **float** ∙ rate of transfter in bytes per second -6. `Total downloads` **int** ∙ number of downloads set -7. `Total duration` **string** ∙ Time for bundle expiry, one of `5 minutes` `one hour` etc -8. `Password added` **boolean** ∙ Did the user add a password to the bundle -9. `Remaining downloads` **int** ∙ number of remaining downloads for a file -10. `Remaining time` **long** ∙ time until a bundle expires -11. `Reason transfer stopped` **string** ∙ One of `completed`, `errored` or `canceled` -12. `FxA prompt trigger` **string** ∙ One of `time options`, `count options`, `bundle size`, `shoulder button` -13. `Location of URL copy` **string** ∙ Where did the user copy the share url `success-screen` or `upload-list` -14. `Site exit path` **string** ∙ Name of external link followed ... `download-firefox`, `twitter`, `github`, `cookies`, `terms`, `privacy`, `about`, `legal`, `mozilla` -15. `Expiry reason` **string** ∙ one of `time limit hit`, `download limit hit`, `user deleted` -16. `Error code` **String** ∙ Error code if added +* `session_id` -1 (not part of a session) +* `user_id` hash of (archive_id + owner_id) +* `app_version` package.json version +* `time` timestamp truncated to hour precision +* `country` +* `region` +* `event_type` [server_upload | server_download | server_delete] +* `user_properties` + * `download_limit` set number of downloads + * `time_limit` set expiry duration + * `size` approximate size (log10) + * `anonymous` true if anonymous, false if fxa +* `event_properties` + * `download_count` downloads completed + * `ttl` time remaining before expiry truncated to hour -### Event Types +### Client Events - The following list is of required `event_type` definitions. If adding new event types please use the syntax `Group - verb subject modifier` +Client events allow us to aggregate data about how the user interface is being used without tracking the lifecycle of individual files. In this context `user_id` and `user_properties` describe the user. The `user_id` and `device_id` change for all users at the beginning of each month. -| Event | Event Properties | Description | -|-------|------------------|-------------| -| `{ Uploader, Downloader, Unsupported } - visit` | `none` | When a user visits the site, or opens the app, grouped by interface at open. Note, a number of API properties and User Properties should be set with this event | -|`{ Uploader, Downloader, Unsupported } - exit` | `none` | When a user exits the site via click event on a link that directs to another domain | -| `Uploader - start bundle upload` | `1, 2, 3, 4, 6, 7, 8, 16 (if applicable)` | When a user begins to upload a bundle for the site | -| `Uploader - stop bundle upload` | `1, 2, 3, 4, 5, 6, 7, 8, 11, 16 (if applicable)` | When a user stops an upload or an upload stops for any reason | -| `Uploader - delete bundle` | `1, 2, 3, 4, 6, 7, 8, 9, 10` | When a user deletes their bundle | -| `Uploader - copy bundle url` | `1, 13` | When a user copies the url of a bundle they create | -| `Uploader - dismiss copy bundle dialog` | `1` | When a user dismisses the bundle copy dialog | -| `{ Uploader, Downloader } - start bundle download` | `1, 2, 3, 4, 6, 7, 8, 9, 10, 16 (if applicable)` | When a user begins to download a bundle. Remaining downloads should be decremented after event. | -| `{ Uploader, Downloader } - stop bundle download` | `1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16 (if applicable)` | When a a download ends for any reason | -| `Downloader - click try send cta` | `1` | When a downloader clicks the prompt to try Firefox Send | -| `Downloader - unlock bundle success` | `1` | When a downloader successfully unlocks a file | -| `Downloader - unlock bundle failure` | `1` | When a downloader fails to unlock a file (only send once per session) | -| `Uploader - trigger signup cta` | `12` | When an uploader triggers the CTA via change to expiry options | -| `Signup - interact with email` | `12` | when a user inputs anything into the email submission form | -| `Signup - cancel signup` | `12` | When a user opts out of signing up | -| `Signup - submit signup` | `12` | When a user submits a sign up to fxa and we begin OAuth dance | -| `Server - expire bundle` | `1, 2, 3, 4, 6, 7, 8, 9, 10, 15` | when the server expires a bundle for any reason | -| `Error` | `16` | Fallback event for any errors that occur. Use the error code event property to specify an error type | +* `session_id` timestamp +* `user_id` hash of (fxa_id + Date.year + Date.month) +* `device_id` hash of (localStorage random id + Date.year + Date.month) +* `platform` [web | android] +* `country` +* `region` +* `language` +* `time` timestamp +* `os_name` +* `event_type` [client_visit | client_upload | client_download | client_delete | client_login | client_logout] +* `event_properties` + * `browser` + * `browser_version` + * `status` [ ok | error | cancel ] + * Event specific properties (see below) +* `user_properties` + * `active_count` number of active uploads + * `anonymous` true if anonymous, false if fxa + * `experiments` list of experiment ids the user is participating in + * `first_action` how this use came to Send the first time [ upload | download ] + +#### Visit Event + + * `entrypoint` [ upload | download ] + +#### Upload Event + + * `download_limit` download limit + * `file_count` number of files + * `password_protected` boolean + * `size` approximate size (log10) + * `time_limit` time limit + * `duration` approximate transfer duration (log10) + +#### Download Event + + * `password_protected` boolean + * `size` approximate size (log10) + * `duration` approximate transfer duration (log10) + +#### Delete Event + + * `age` hours since uploaded + * `downloaded` downloaded at least once + +#### Login Event + + * `trigger` [button | time | count | size] + +#### Logout Event + + * `trigger` [button | timeout] diff --git a/package-lock.json b/package-lock.json index db46d793..c41f1591 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1825,9 +1825,9 @@ } }, "aws-sdk": { - "version": "2.400.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.400.0.tgz", - "integrity": "sha512-FJjRXajLnI52F0C1E4nFARk/907x4ZuffwFd6jiBQfaepT03wsY1PcJNEQ5CNHYA2qxAx4HezL2pKySEX+g87g==", + "version": "2.401.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.401.0.tgz", + "integrity": "sha512-mOI4gzKoP/g8Q0ToAaqTh7TijGG9PvGVVUkKmurXqBKy7GTPmy4JizfVkTrM+iBg7RAsx5H2lBxBFpdEFBa5fg==", "requires": { "buffer": "4.9.1", "events": "1.1.1", @@ -2054,6 +2054,11 @@ "tweetnacl": "^0.14.3" } }, + "big-integer": { + "version": "1.6.41", + "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.41.tgz", + "integrity": "sha512-d5AT9lMTYJ/ZE/4gzxb+5ttPcRWljVsvv7lF1w9KzkPhVUhBtHrjDo1J8swfZKepfLsliDhYa31zRYwcD0Yg9w==" + }, "big.js": { "version": "5.2.2", "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", @@ -3275,6 +3280,14 @@ "sha.js": "^2.4.8" } }, + "cron": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/cron/-/cron-1.5.0.tgz", + "integrity": "sha512-j7zMFLrcSta53xqOvETUt8ge+PM14GtF47gEGJJeVlM6qP24/eWHSgtiWiEiKBR2sHS8xZaBQZq4D7vFXg8dcQ==", + "requires": { + "moment-timezone": "^0.5.x" + } + }, "cross-env": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-5.2.0.tgz", @@ -4851,9 +4864,9 @@ "dev": true }, "esm": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/esm/-/esm-3.2.4.tgz", - "integrity": "sha512-wOuWtQCkkwD1WKQN/k3RsyGSSN+AmiUzdKftn8vaC+uV9JesYmQlODJxgXaaRz0LaaFIlUxZaUu5NPiUAjKAAA==" + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/esm/-/esm-3.1.1.tgz", + "integrity": "sha512-Md2pR4IbR37UqubbgbA4+wiBorOEFB05Oo+g4WJW7W2ajiOhUfjZt77NzzCoQdrCb40GdKcflitm+XHDF053OQ==" }, "espree": { "version": "5.0.0", @@ -5772,24 +5785,28 @@ "dependencies": { "abbrev": { "version": "1.1.1", - "bundled": true, + "resolved": false, + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", "dev": true, "optional": true }, "ansi-regex": { "version": "2.1.1", - "bundled": true, + "resolved": false, + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", "dev": true }, "aproba": { "version": "1.2.0", - "bundled": true, + "resolved": false, + "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", "dev": true, "optional": true }, "are-we-there-yet": { "version": "1.1.5", - "bundled": true, + "resolved": false, + "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==", "dev": true, "optional": true, "requires": { @@ -5799,12 +5816,14 @@ }, "balanced-match": { "version": "1.0.0", - "bundled": true, + "resolved": false, + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", "dev": true }, "brace-expansion": { "version": "1.1.11", - "bundled": true, + "resolved": false, + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", "dev": true, "requires": { "balanced-match": "^1.0.0", @@ -5813,34 +5832,40 @@ }, "chownr": { "version": "1.1.1", - "bundled": true, + "resolved": false, + "integrity": "sha512-j38EvO5+LHX84jlo6h4UzmOwi0UgW61WRyPtJz4qaadK5eY3BTS5TY/S1Stc3Uk2lIM6TPevAlULiEJwie860g==", "dev": true, "optional": true }, "code-point-at": { "version": "1.1.0", - "bundled": true, + "resolved": false, + "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", "dev": true }, "concat-map": { "version": "0.0.1", - "bundled": true, + "resolved": false, + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", "dev": true }, "console-control-strings": { "version": "1.1.0", - "bundled": true, + "resolved": false, + "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=", "dev": true }, "core-util-is": { "version": "1.0.2", - "bundled": true, + "resolved": false, + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", "dev": true, "optional": true }, "debug": { "version": "2.6.9", - "bundled": true, + "resolved": false, + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", "dev": true, "optional": true, "requires": { @@ -5849,25 +5874,29 @@ }, "deep-extend": { "version": "0.6.0", - "bundled": true, + "resolved": false, + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", "dev": true, "optional": true }, "delegates": { "version": "1.0.0", - "bundled": true, + "resolved": false, + "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=", "dev": true, "optional": true }, "detect-libc": { "version": "1.0.3", - "bundled": true, + "resolved": false, + "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=", "dev": true, "optional": true }, "fs-minipass": { "version": "1.2.5", - "bundled": true, + "resolved": false, + "integrity": "sha512-JhBl0skXjUPCFH7x6x61gQxrKyXsxB5gcgePLZCwfyCGGsTISMoIeObbrvVeP6Xmyaudw4TT43qV2Gz+iyd2oQ==", "dev": true, "optional": true, "requires": { @@ -5876,13 +5905,15 @@ }, "fs.realpath": { "version": "1.0.0", - "bundled": true, + "resolved": false, + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", "dev": true, "optional": true }, "gauge": { "version": "2.7.4", - "bundled": true, + "resolved": false, + "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", "dev": true, "optional": true, "requires": { @@ -5898,7 +5929,8 @@ }, "glob": { "version": "7.1.3", - "bundled": true, + "resolved": false, + "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", "dev": true, "optional": true, "requires": { @@ -5912,13 +5944,15 @@ }, "has-unicode": { "version": "2.0.1", - "bundled": true, + "resolved": false, + "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=", "dev": true, "optional": true }, "iconv-lite": { "version": "0.4.24", - "bundled": true, + "resolved": false, + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", "dev": true, "optional": true, "requires": { @@ -5927,7 +5961,8 @@ }, "ignore-walk": { "version": "3.0.1", - "bundled": true, + "resolved": false, + "integrity": "sha512-DTVlMx3IYPe0/JJcYP7Gxg7ttZZu3IInhuEhbchuqneY9wWe5Ojy2mXLBaQFUQmo0AW2r3qG7m1mg86js+gnlQ==", "dev": true, "optional": true, "requires": { @@ -5936,7 +5971,8 @@ }, "inflight": { "version": "1.0.6", - "bundled": true, + "resolved": false, + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", "dev": true, "optional": true, "requires": { @@ -5946,18 +5982,21 @@ }, "inherits": { "version": "2.0.3", - "bundled": true, + "resolved": false, + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", "dev": true }, "ini": { "version": "1.3.5", - "bundled": true, + "resolved": false, + "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", "dev": true, "optional": true }, "is-fullwidth-code-point": { "version": "1.0.0", - "bundled": true, + "resolved": false, + "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", "dev": true, "requires": { "number-is-nan": "^1.0.0" @@ -5965,13 +6004,15 @@ }, "isarray": { "version": "1.0.0", - "bundled": true, + "resolved": false, + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", "dev": true, "optional": true }, "minimatch": { "version": "3.0.4", - "bundled": true, + "resolved": false, + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", "dev": true, "requires": { "brace-expansion": "^1.1.7" @@ -5979,12 +6020,14 @@ }, "minimist": { "version": "0.0.8", - "bundled": true, + "resolved": false, + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", "dev": true }, "minipass": { "version": "2.3.5", - "bundled": true, + "resolved": false, + "integrity": "sha512-Gi1W4k059gyRbyVUZQ4mEqLm0YIUiGYfvxhF6SIlk3ui1WVxMTGfGdQ2SInh3PDrRTVvPKgULkpJtT4RH10+VA==", "dev": true, "requires": { "safe-buffer": "^5.1.2", @@ -5993,7 +6036,8 @@ }, "minizlib": { "version": "1.2.1", - "bundled": true, + "resolved": false, + "integrity": "sha512-7+4oTUOWKg7AuL3vloEWekXY2/D20cevzsrNT2kGWm+39J9hGTCBv8VI5Pm5lXZ/o3/mdR4f8rflAPhnQb8mPA==", "dev": true, "optional": true, "requires": { @@ -6002,7 +6046,8 @@ }, "mkdirp": { "version": "0.5.1", - "bundled": true, + "resolved": false, + "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", "dev": true, "requires": { "minimist": "0.0.8" @@ -6010,13 +6055,15 @@ }, "ms": { "version": "2.0.0", - "bundled": true, + "resolved": false, + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", "dev": true, "optional": true }, "needle": { "version": "2.2.4", - "bundled": true, + "resolved": false, + "integrity": "sha512-HyoqEb4wr/rsoaIDfTH2aVL9nWtQqba2/HvMv+++m8u0dz808MaagKILxtfeSN7QU7nvbQ79zk3vYOJp9zsNEA==", "dev": true, "optional": true, "requires": { @@ -6027,7 +6074,8 @@ }, "node-pre-gyp": { "version": "0.10.3", - "bundled": true, + "resolved": false, + "integrity": "sha512-d1xFs+C/IPS8Id0qPTZ4bUT8wWryfR/OzzAFxweG+uLN85oPzyo2Iw6bVlLQ/JOdgNonXLCoRyqDzDWq4iw72A==", "dev": true, "optional": true, "requires": { @@ -6045,7 +6093,8 @@ }, "nopt": { "version": "4.0.1", - "bundled": true, + "resolved": false, + "integrity": "sha1-0NRoWv1UFRk8jHUFYC0NF81kR00=", "dev": true, "optional": true, "requires": { @@ -6055,13 +6104,15 @@ }, "npm-bundled": { "version": "1.0.5", - "bundled": true, + "resolved": false, + "integrity": "sha512-m/e6jgWu8/v5niCUKQi9qQl8QdeEduFA96xHDDzFGqly0OOjI7c+60KM/2sppfnUU9JJagf+zs+yGhqSOFj71g==", "dev": true, "optional": true }, "npm-packlist": { "version": "1.2.0", - "bundled": true, + "resolved": false, + "integrity": "sha512-7Mni4Z8Xkx0/oegoqlcao/JpPCPEMtUvsmB0q7mgvlMinykJLSRTYuFqoQLYgGY8biuxIeiHO+QNJKbCfljewQ==", "dev": true, "optional": true, "requires": { @@ -6071,7 +6122,8 @@ }, "npmlog": { "version": "4.1.2", - "bundled": true, + "resolved": false, + "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", "dev": true, "optional": true, "requires": { @@ -6083,18 +6135,21 @@ }, "number-is-nan": { "version": "1.0.1", - "bundled": true, + "resolved": false, + "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", "dev": true }, "object-assign": { "version": "4.1.1", - "bundled": true, + "resolved": false, + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", "dev": true, "optional": true }, "once": { "version": "1.4.0", - "bundled": true, + "resolved": false, + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", "dev": true, "requires": { "wrappy": "1" @@ -6102,19 +6157,22 @@ }, "os-homedir": { "version": "1.0.2", - "bundled": true, + "resolved": false, + "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=", "dev": true, "optional": true }, "os-tmpdir": { "version": "1.0.2", - "bundled": true, + "resolved": false, + "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", "dev": true, "optional": true }, "osenv": { "version": "0.1.5", - "bundled": true, + "resolved": false, + "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==", "dev": true, "optional": true, "requires": { @@ -6124,19 +6182,22 @@ }, "path-is-absolute": { "version": "1.0.1", - "bundled": true, + "resolved": false, + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", "dev": true, "optional": true }, "process-nextick-args": { "version": "2.0.0", - "bundled": true, + "resolved": false, + "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==", "dev": true, "optional": true }, "rc": { "version": "1.2.8", - "bundled": true, + "resolved": false, + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", "dev": true, "optional": true, "requires": { @@ -6148,7 +6209,8 @@ "dependencies": { "minimist": { "version": "1.2.0", - "bundled": true, + "resolved": false, + "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=", "dev": true, "optional": true } @@ -6156,7 +6218,8 @@ }, "readable-stream": { "version": "2.3.6", - "bundled": true, + "resolved": false, + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "dev": true, "optional": true, "requires": { @@ -6171,7 +6234,8 @@ }, "rimraf": { "version": "2.6.3", - "bundled": true, + "resolved": false, + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", "dev": true, "optional": true, "requires": { @@ -6180,42 +6244,49 @@ }, "safe-buffer": { "version": "5.1.2", - "bundled": true, + "resolved": false, + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", "dev": true }, "safer-buffer": { "version": "2.1.2", - "bundled": true, + "resolved": false, + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "dev": true, "optional": true }, "sax": { "version": "1.2.4", - "bundled": true, + "resolved": false, + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==", "dev": true, "optional": true }, "semver": { "version": "5.6.0", - "bundled": true, + "resolved": false, + "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg==", "dev": true, "optional": true }, "set-blocking": { "version": "2.0.0", - "bundled": true, + "resolved": false, + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", "dev": true, "optional": true }, "signal-exit": { "version": "3.0.2", - "bundled": true, + "resolved": false, + "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=", "dev": true, "optional": true }, "string-width": { "version": "1.0.2", - "bundled": true, + "resolved": false, + "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", "dev": true, "requires": { "code-point-at": "^1.0.0", @@ -6225,7 +6296,8 @@ }, "string_decoder": { "version": "1.1.1", - "bundled": true, + "resolved": false, + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "dev": true, "optional": true, "requires": { @@ -6234,7 +6306,8 @@ }, "strip-ansi": { "version": "3.0.1", - "bundled": true, + "resolved": false, + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", "dev": true, "requires": { "ansi-regex": "^2.0.0" @@ -6242,13 +6315,15 @@ }, "strip-json-comments": { "version": "2.0.1", - "bundled": true, + "resolved": false, + "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", "dev": true, "optional": true }, "tar": { "version": "4.4.8", - "bundled": true, + "resolved": false, + "integrity": "sha512-LzHF64s5chPQQS0IYBn9IN5h3i98c12bo4NCO7e0sGM2llXQ3p2FGC5sdENN4cTW48O915Sh+x+EXx7XW96xYQ==", "dev": true, "optional": true, "requires": { @@ -6263,13 +6338,15 @@ }, "util-deprecate": { "version": "1.0.2", - "bundled": true, + "resolved": false, + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", "dev": true, "optional": true }, "wide-align": { "version": "1.1.3", - "bundled": true, + "resolved": false, + "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", "dev": true, "optional": true, "requires": { @@ -6278,12 +6355,14 @@ }, "wrappy": { "version": "1.0.2", - "bundled": true, + "resolved": false, + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", "dev": true }, "yallist": { "version": "3.0.3", - "bundled": true, + "resolved": false, + "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==", "dev": true } } @@ -6300,6 +6379,26 @@ "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", "dev": true }, + "fxa-geodb": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/fxa-geodb/-/fxa-geodb-1.0.4.tgz", + "integrity": "sha512-f+uNgA+6OxmLAHhZvMztwPrByhkaVmSrKcb5Q1TI7Zz/onSQPYCJs388are7nWQdXI94pncqmSPxmT9kOUllEA==", + "requires": { + "bluebird": "3.5.2", + "cron": "1.5.0", + "maxmind": "2.8.0", + "mkdirp": "0.5.1", + "mozlog": "2.2.0", + "request": "2.88.0" + }, + "dependencies": { + "bluebird": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.2.tgz", + "integrity": "sha512-dhHTWMI7kMx5whMQntl7Vr9C6BvV10lFXDAasnqnrMYhXVCzzk6IO9Fo2L75jXHT07WrOngL1WDXOp+yYS91Yg==" + } + } + }, "g-status": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/g-status/-/g-status-2.0.2.tgz", @@ -8732,6 +8831,15 @@ "integrity": "sha512-3Zs9P/0zzwTob2pdgT0CHZuMbnSUSp8MB1bddfm+HDmnFWHGT4jvEZRf+2RuPoa+cjdn/z25SEt5gFTqdhvJAg==", "dev": true }, + "maxmind": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/maxmind/-/maxmind-2.8.0.tgz", + "integrity": "sha512-U3/jQRUoMf4pQ/Tm7JNtGRaM9z82fATB2TiGgs0kEKMPZn/UbOnlyGMRItJ2+KWrwjz9a7PqRzy3/haq9XfUOQ==", + "requires": { + "big-integer": "^1.6.31", + "tiny-lru": "^1.6.1" + } + }, "md5": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/md5/-/md5-2.2.1.tgz", @@ -9077,6 +9185,14 @@ "resolved": "https://registry.npmjs.org/moment/-/moment-2.22.2.tgz", "integrity": "sha1-PCV/mDn8DpP/UxSWMiOeuQeD/2Y=" }, + "moment-timezone": { + "version": "0.5.23", + "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.23.tgz", + "integrity": "sha512-WHFH85DkCfiNMDX5D3X7hpNH3/PUhjTGcD0U1SgfBGZxJ3qUmJh5FdvaFjcClxOvB3rzdfj4oRffbI38jEnC1w==", + "requires": { + "moment": ">= 2.9.0" + } + }, "morgan": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.9.1.tgz", @@ -15047,14 +15163,14 @@ } }, "terser": { - "version": "3.16.1", - "resolved": "https://registry.npmjs.org/terser/-/terser-3.16.1.tgz", - "integrity": "sha512-JDJjgleBROeek2iBcSNzOHLKsB/MdDf+E/BOAJ0Tk9r7p9/fVobfv7LMJ/g/k3v9SXdmjZnIlFd5nfn/Rt0Xow==", + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/terser/-/terser-3.14.1.tgz", + "integrity": "sha512-NSo3E99QDbYSMeJaEk9YW2lTg3qS9V0aKGlb+PlOrei1X02r1wSBHCNX/O+yeTRFSWPKPIGj6MqvvdqV4rnVGw==", "dev": true, "requires": { "commander": "~2.17.1", "source-map": "~0.6.1", - "source-map-support": "~0.5.9" + "source-map-support": "~0.5.6" }, "dependencies": { "source-map": { @@ -15066,9 +15182,9 @@ } }, "terser-webpack-plugin": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-1.2.2.tgz", - "integrity": "sha512-1DMkTk286BzmfylAvLXwpJrI7dWa5BnFmscV/2dCr8+c56egFcbaeFAl7+sujAjdmpLam21XRdhA4oifLyiWWg==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-1.2.1.tgz", + "integrity": "sha512-GGSt+gbT0oKcMDmPx4SRSfJPE1XaN3kQRWG4ghxKQw9cn5G9x6aCKSsgYdvyM0na9NJ4Drv0RG6jbBByZ5CMjw==", "dev": true, "requires": { "cacache": "^11.0.2", @@ -15076,7 +15192,7 @@ "schema-utils": "^1.0.0", "serialize-javascript": "^1.4.0", "source-map": "^0.6.1", - "terser": "^3.16.1", + "terser": "^3.8.1", "webpack-sources": "^1.1.0", "worker-farm": "^1.5.2" }, @@ -15285,12 +15401,6 @@ } } }, - "testpilot-ga": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/testpilot-ga/-/testpilot-ga-0.3.0.tgz", - "integrity": "sha512-z4PJbw3KK0R0iflA+u/3BhWZrtsLHLu+0rMviMd8H1wp8qPV0rggNBjsKckBJCcXq4uEjXETGZzApHH7Tovpzw==", - "dev": true - }, "text-encoding": { "version": "0.6.4", "resolved": "https://registry.npmjs.org/text-encoding/-/text-encoding-0.6.4.tgz", @@ -15344,6 +15454,11 @@ "integrity": "sha1-QFQRqOfmM5/mTbmiNN4R3DHgK9Q=", "dev": true }, + "tiny-lru": { + "version": "1.6.4", + "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-1.6.4.tgz", + "integrity": "sha512-Et+J3Css66XPSLWjLF9wmgbECsGiExlEL+jxsFerTQF6N6dpxswDTPAfIrAbQKO5c1uhgq2xvo5zMk1W+kBDNA==" + }, "tmp": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", @@ -15540,6 +15655,11 @@ "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" }, + "ua-parser-js": { + "version": "0.7.19", + "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.19.tgz", + "integrity": "sha512-T3PVJ6uz8i0HzPxOF9SWzWAlfN/DavlpQqepn22xgve/5QecC+XMCAtmUNnY7C9StehaV6exjUCI801lOI7QlQ==" + }, "uglify-js": { "version": "3.4.9", "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.4.9.tgz", diff --git a/package.json b/package.json index e34a01ec..7a17b6cc 100644 --- a/package.json +++ b/package.json @@ -119,7 +119,6 @@ "svgo": "^1.1.1", "svgo-loader": "^2.2.0", "tailwindcss": "^0.7.4", - "testpilot-ga": "^0.3.0", "val-loader": "^1.1.1", "wdio-docker-service": "^1.4.2", "wdio-dot-reporter": "0.0.10", @@ -144,12 +143,14 @@ "express-ws": "github:dannycoates/express-ws", "fluent": "^0.10.0", "fluent-langneg": "^0.1.1", + "fxa-geodb": "^1.0.4", "helmet": "^3.15.0", "mkdirp": "^0.5.1", "mozlog": "^2.2.0", "node-fetch": "^2.3.0", "raven": "^2.6.4", "redis": "^2.8.0", + "ua-parser-js": "^0.7.19", "websocket-stream": "^5.1.2" }, "availableLanguages": [ diff --git a/server/amplitude.js b/server/amplitude.js new file mode 100644 index 00000000..ee9cc785 --- /dev/null +++ b/server/amplitude.js @@ -0,0 +1,161 @@ +const crypto = require('crypto'); +const geoip = require('fxa-geodb')(); +const fetch = require('node-fetch'); +const config = require('./config'); +const pkg = require('../package.json'); + +const HOUR = 1000 * 60 * 60; + +function truncateToHour(timestamp) { + return Math.floor(timestamp / HOUR) * HOUR; +} + +function orderOfMagnitude(n) { + return Math.floor(Math.log10(n)); +} + +function userId(fileId, ownerId) { + const hash = crypto.createHash('sha256'); + hash.update(fileId); + hash.update(ownerId); + return hash.digest('hex').substring(32); +} + +function location(ip) { + try { + return geoip(ip); + } catch (e) { + return {}; + } +} + +function statUploadEvent(data) { + const loc = location(data.ip); + const event = { + session_id: -1, + country: loc.country, + region: loc.state, + user_id: userId(data.id, data.owner), + app_version: pkg.version, + time: truncateToHour(Date.now()), + event_type: 'server_upload', + user_properties: { + download_limit: data.dlimit, + time_limit: data.timeLimit, + size: orderOfMagnitude(data.size), + anonymous: data.anonymous + }, + event_id: 0 + }; + return sendBatch([event]); +} + +function statDownloadEvent(data) { + const loc = location(data.ip); + const event = { + session_id: -1, + country: loc.country, + region: loc.state, + user_id: userId(data.id, data.owner), + app_version: pkg.version, + time: truncateToHour(Date.now()), + event_type: 'server_download', + event_properties: { + download_count: data.download_count, + ttl: data.ttl + }, + event_id: data.download_count + }; + return sendBatch([event]); +} + +function statDeleteEvent(data) { + const loc = location(data.ip); + const event = { + session_id: -1, + country: loc.country, + region: loc.state, + user_id: userId(data.id, data.owner), + app_version: pkg.version, + time: truncateToHour(Date.now()), + event_type: 'server_delete', + event_properties: { + download_count: data.download_count, + ttl: data.ttl + }, + event_id: data.download_count + 1 + }; + return sendBatch([event]); +} + +function clientEvent(event, ua, language, session_id, deltaT, platform, ip) { + const loc = location(ip); + const ep = event.event_properties || {}; + const up = event.user_properties || {}; + const event_properties = { + browser: ua.browser.name, + browser_version: ua.browser.version, + status: ep.status, + + age: ep.age, + downloaded: ep.downloaded, + download_limit: ep.download_limit, + duration: ep.duration, + file_count: ep.file_count, + password_protected: ep.password_protected, + size: ep.size, + time_limit: ep.time_limit, + trigger: ep.trigger, + ttl: ep.ttl + }; + const user_properties = { + active_count: up.active_count, + anonymous: up.anonymous, + experiments: up.experiments, + first_action: up.first_action + }; + return { + app_version: pkg.version, + country: loc.country, + device_id: event.device_id, + event_properties, + event_type: event.event_type, + language, + os_name: ua.os.name, + os_version: ua.os.version, + platform, + region: loc.state, + session_id, + time: event.time + deltaT, + user_id: event.user_id, + user_properties + }; +} + +async function sendBatch(events, timeout = 1000) { + if (!config.amplitude_id) { + return 200; + } + try { + const result = await fetch('https://api.amplitude.com/batch', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + api_key: config.amplitude_id, + events + }), + timeout + }); + return result.status; + } catch (e) { + return 500; + } +} + +module.exports = { + statUploadEvent, + statDownloadEvent, + statDeleteEvent, + clientEvent, + sendBatch +}; diff --git a/server/config.js b/server/config.js index 5a037544..e0f0bbca 100644 --- a/server/config.js +++ b/server/config.js @@ -80,6 +80,11 @@ const conf = convict({ arg: 'port', env: 'PORT' }, + amplitude_id: { + format: String, + default: '', + env: 'AMPLITUDE_ID' + }, analytics_id: { format: String, default: '', diff --git a/server/initScript.js b/server/initScript.js index 0bd35902..73072def 100644 --- a/server/initScript.js +++ b/server/initScript.js @@ -24,11 +24,6 @@ var SENTRY_ID = '${config.sentry_id}'; `; } -let ga = ''; -if (config.analytics_id) { - ga = `var GOOGLE_ANALYTICS_ID = '${config.analytics_id}';`; -} - module.exports = function(state) { const authConfig = state.authConfig ? `var AUTH_CONFIG = ${JSON.stringify(state.authConfig)};` @@ -71,7 +66,6 @@ module.exports = function(state) { state.downloadMetadata ? raw(JSON.stringify(state.downloadMetadata)) : '{}' }; ${authConfig}; - ${ga} ${sentry} `; return state.cspNonce diff --git a/server/routes/delete.js b/server/routes/delete.js index 71ada5f9..757b015b 100644 --- a/server/routes/delete.js +++ b/server/routes/delete.js @@ -1,9 +1,20 @@ const storage = require('../storage'); +const { statDeleteEvent } = require('../amplitude'); module.exports = async function(req, res) { try { - await storage.del(req.params.id); + const id = req.params.id; + const meta = req.meta; + const ttl = await storage.ttl(id); + await storage.del(id); res.sendStatus(200); + statDeleteEvent({ + id, + ip: req.ip, + owner: meta.owner, + download_count: meta.dl, + ttl + }); } catch (e) { res.sendStatus(404); } diff --git a/server/routes/download.js b/server/routes/download.js index 2f37de22..7828c880 100644 --- a/server/routes/download.js +++ b/server/routes/download.js @@ -1,6 +1,7 @@ const storage = require('../storage'); const mozlog = require('../log'); const log = mozlog('send.download'); +const { statDownloadEvent } = require('../amplitude'); module.exports = async function(req, res) { const id = req.params.id; @@ -21,6 +22,14 @@ module.exports = async function(req, res) { const dl = meta.dl + 1; const dlimit = meta.dlimit; + const ttl = await storage.ttl(id); + statDownloadEvent({ + id, + ip: req.ip, + owner: meta.owner, + download_count: dl, + ttl + }); try { if (dl >= dlimit) { await storage.del(id); diff --git a/server/routes/index.js b/server/routes/index.js index 5bc43d8c..3bc99b87 100644 --- a/server/routes/index.js +++ b/server/routes/index.js @@ -1,6 +1,7 @@ const crypto = require('crypto'); const express = require('express'); const helmet = require('helmet'); +const uaparser = require('ua-parser-js'); const storage = require('../storage'); const config = require('../config'); const auth = require('../middleware/auth'); @@ -12,6 +13,7 @@ const IS_DEV = config.env === 'development'; const ID_REGEX = '([0-9a-fA-F]{10})'; module.exports = function(app) { + app.set('trust proxy', true); app.use(helmet()); app.use( helmet.hsts({ @@ -19,6 +21,10 @@ module.exports = function(app) { force: !IS_DEV }) ); + app.use(function(req, res, next) { + req.ua = uaparser(req.header('user-agent')); + next(); + }); app.use(function(req, res, next) { req.cspNonce = crypto.randomBytes(16).toString('hex'); next(); @@ -35,12 +41,10 @@ module.exports = function(app) { 'wss://send.firefox.com', 'https://*.dev.lcip.org', 'https://*.accounts.firefox.com', - 'https://sentry.prod.mozaws.net', - 'https://www.google-analytics.com' + 'https://sentry.prod.mozaws.net' ], imgSrc: [ "'self'", - 'https://www.google-analytics.com', 'https://*.dev.lcip.org', 'https://firefoxusercontent.com' ], @@ -92,7 +96,7 @@ module.exports = function(app) { require('./params') ); app.post(`/api/info/:id${ID_REGEX}`, auth.owner, require('./info')); - + app.post('/api/metrics', require('./metrics')); app.get('/__version__', function(req, res) { res.sendFile(require.resolve('../../dist/version.json')); }); diff --git a/server/routes/metrics.js b/server/routes/metrics.js new file mode 100644 index 00000000..0d8015c1 --- /dev/null +++ b/server/routes/metrics.js @@ -0,0 +1,23 @@ +const { sendBatch, clientEvent } = require('../amplitude'); + +module.exports = async function(req, res) { + try { + const data = req.body; + const deltaT = Date.now() - data.now; + const events = data.events.map(e => + clientEvent( + e, + req.ua, + data.lang, + data.session_id + deltaT, + deltaT, + data.platform, + req.ip + ) + ); + const status = await sendBatch(events); + res.sendStatus(status); + } catch (e) { + res.sendStatus(500); + } +}; diff --git a/server/routes/ws.js b/server/routes/ws.js index e70c4cf1..49089d47 100644 --- a/server/routes/ws.js +++ b/server/routes/ws.js @@ -5,6 +5,7 @@ const mozlog = require('../log'); const Limiter = require('../limiter'); const wsStream = require('websocket-stream/stream'); const fxa = require('../fxa'); +const { statUploadEvent } = require('../amplitude'); const { Duplex } = require('stream'); @@ -105,6 +106,15 @@ module.exports = function(ws, req) { // in order to avoid having to check socket state and clean // up storage, possibly with an exception that we can catch. ws.send(JSON.stringify({ ok: true })); + statUploadEvent({ + id: newId, + ip: req.ip, + owner, + dlimit, + timeLimit, + anonymous: !user, + size: limiter.length + }); } } catch (e) { log.error('upload', e); diff --git a/test/backend/delete-tests.js b/test/backend/delete-tests.js index 1de32349..2984b34b 100644 --- a/test/backend/delete-tests.js +++ b/test/backend/delete-tests.js @@ -2,7 +2,8 @@ const sinon = require('sinon'); const proxyquire = require('proxyquire').noCallThru(); const storage = { - del: sinon.stub() + del: sinon.stub(), + ttl: sinon.stub() }; function request(id) { diff --git a/webpack.config.js b/webpack.config.js index e7943cdd..b33f4c86 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -101,7 +101,6 @@ const web = { path.resolve(__dirname, 'common'), // some dependencies need to get re-babeled because we // have different targets than their default configs - path.resolve(__dirname, 'node_modules/testpilot-ga/src'), path.resolve(__dirname, 'node_modules/fluent'), path.resolve(__dirname, 'node_modules/fluent-intl-polyfill'), path.resolve(__dirname, 'node_modules/intl-pluralrules')