From 8ad45971145dd27089031296769982e1ea5ce426 Mon Sep 17 00:00:00 2001 From: Danny Coates Date: Tue, 28 Jul 2020 09:31:09 -0700 Subject: [PATCH] replaced fxa-geodb with load balancer header --- package-lock.json | 68 --------------------------------------- package.json | 1 - server/amplitude.js | 48 ++++++++++++--------------- server/routes/delete.js | 2 ++ server/routes/download.js | 2 ++ server/routes/index.js | 13 ++++++++ server/routes/metrics.js | 3 +- server/routes/report.js | 2 ++ server/routes/ws.js | 2 ++ 9 files changed, 43 insertions(+), 98 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6a6525cb..44efb1fd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2380,11 +2380,6 @@ "tweetnacl": "^0.14.3" } }, - "big-integer": { - "version": "1.6.48", - "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.48.tgz", - "integrity": "sha512-j51egjPa7/i+RdiRuJbPdJ2FIUYYPhvYLjzoYbcMMm62ooO6F94fETG4MTs46zPAF9Brs04OajboA/qTGuz78w==" - }, "big.js": { "version": "5.2.2", "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", @@ -2437,11 +2432,6 @@ } } }, - "bluebird": { - "version": "3.5.2", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.2.tgz", - "integrity": "sha512-dhHTWMI7kMx5whMQntl7Vr9C6BvV10lFXDAasnqnrMYhXVCzzk6IO9Fo2L75jXHT07WrOngL1WDXOp+yYS91Yg==" - }, "bn.js": { "version": "4.11.8", "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz", @@ -3621,14 +3611,6 @@ "sha.js": "^2.4.8" } }, - "cron": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/cron/-/cron-1.5.0.tgz", - "integrity": "sha512-j7zMFLrcSta53xqOvETUt8ge+PM14GtF47gEGJJeVlM6qP24/eWHSgtiWiEiKBR2sHS8xZaBQZq4D7vFXg8dcQ==", - "requires": { - "moment-timezone": "^0.5.x" - } - }, "cross-env": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-6.0.3.tgz", @@ -6854,34 +6836,6 @@ "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", "dev": true }, - "fxa-geodb": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/fxa-geodb/-/fxa-geodb-1.0.4.tgz", - "integrity": "sha512-f+uNgA+6OxmLAHhZvMztwPrByhkaVmSrKcb5Q1TI7Zz/onSQPYCJs388are7nWQdXI94pncqmSPxmT9kOUllEA==", - "requires": { - "bluebird": "3.5.2", - "cron": "1.5.0", - "maxmind": "2.8.0", - "mkdirp": "0.5.1", - "mozlog": "2.2.0", - "request": "2.88.0" - }, - "dependencies": { - "minimist": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" - }, - "mkdirp": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", - "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", - "requires": { - "minimist": "0.0.8" - } - } - } - }, "gaxios": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-3.0.3.tgz", @@ -9734,15 +9688,6 @@ "integrity": "sha512-APMBEanjybaPzUrfqU0IMU5I0AswKMH7k8OTLs0vvV4KZpExkTkY87nR/zpbuTPj+gARop7aGUbl11pnDfW6xg==", "dev": true }, - "maxmind": { - "version": "2.8.0", - "resolved": "https://registry.npmjs.org/maxmind/-/maxmind-2.8.0.tgz", - "integrity": "sha512-U3/jQRUoMf4pQ/Tm7JNtGRaM9z82fATB2TiGgs0kEKMPZn/UbOnlyGMRItJ2+KWrwjz9a7PqRzy3/haq9XfUOQ==", - "requires": { - "big-integer": "^1.6.31", - "tiny-lru": "^1.6.1" - } - }, "md5.js": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz", @@ -10300,14 +10245,6 @@ "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==" }, - "moment-timezone": { - "version": "0.5.28", - "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.28.tgz", - "integrity": "sha512-TDJkZvAyKIVWg5EtVqRzU97w0Rb0YVbfpqyjgu6GwXCAohVRqwZjf4fOzDE6p1Ch98Sro/8hQQi65WDXW5STPw==", - "requires": { - "moment": ">= 2.9.0" - } - }, "morgan": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.10.0.tgz", @@ -15805,11 +15742,6 @@ "integrity": "sha1-QFQRqOfmM5/mTbmiNN4R3DHgK9Q=", "dev": true }, - "tiny-lru": { - "version": "1.6.4", - "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-1.6.4.tgz", - "integrity": "sha512-Et+J3Css66XPSLWjLF9wmgbECsGiExlEL+jxsFerTQF6N6dpxswDTPAfIrAbQKO5c1uhgq2xvo5zMk1W+kBDNA==" - }, "tmp": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", diff --git a/package.json b/package.json index c3425f2b..9eca19f4 100644 --- a/package.json +++ b/package.json @@ -142,7 +142,6 @@ "configstore": "github:dannycoates/configstore#master", "convict": "^5.2.0", "express": "^4.17.1", - "fxa-geodb": "^1.0.4", "helmet": "^3.21.2", "mkdirp": "^0.5.1", "mozlog": "^2.2.0", diff --git a/server/amplitude.js b/server/amplitude.js index 5a20ed27..116dc750 100644 --- a/server/amplitude.js +++ b/server/amplitude.js @@ -3,10 +3,6 @@ const fetch = require('node-fetch'); const config = require('./config'); const pkg = require('../package.json'); -const geoip = config.ip_db - ? require('fxa-geodb')({ dbPath: config.ip_db }) - : () => ({}); - const HOUR = 1000 * 60 * 60; function truncateToHour(timestamp) { @@ -24,20 +20,11 @@ function userId(fileId, ownerId) { return hash.digest('hex').substring(32); } -function location(ip) { - try { - return geoip(ip); - } catch (e) { - return {}; - } -} - function statUploadEvent(data) { - const loc = location(data.ip); const event = { session_id: -1, - country: loc.country, - region: loc.state, + country: data.country, + region: data.state, user_id: userId(data.id, data.owner), app_version: pkg.version, time: truncateToHour(Date.now()), @@ -57,11 +44,10 @@ function statUploadEvent(data) { } function statDownloadEvent(data) { - const loc = location(data.ip); const event = { session_id: -1, - country: loc.country, - region: loc.state, + country: data.country, + region: data.state, user_id: userId(data.id, data.owner), app_version: pkg.version, time: truncateToHour(Date.now()), @@ -77,11 +63,10 @@ function statDownloadEvent(data) { } function statDeleteEvent(data) { - const loc = location(data.ip); const event = { session_id: -1, - country: loc.country, - region: loc.state, + country: data.country, + region: data.state, user_id: userId(data.id, data.owner), app_version: pkg.version, time: truncateToHour(Date.now()), @@ -97,11 +82,10 @@ function statDeleteEvent(data) { } function statReportEvent(data) { - const loc = location(data.ip); const event = { session_id: -1, - country: loc.country, - region: loc.state, + country: data.country, + region: data.state, user_id: userId(data.id, data.owner), app_version: pkg.version, time: truncateToHour(Date.now()), @@ -118,8 +102,16 @@ function statReportEvent(data) { return sendBatch([event]); } -function clientEvent(event, ua, language, session_id, deltaT, platform, ip) { - const loc = location(ip); +function clientEvent( + event, + ua, + language, + session_id, + deltaT, + platform, + country, + state +) { const ep = event.event_properties || {}; const up = event.user_properties || {}; const event_properties = { @@ -155,7 +147,7 @@ function clientEvent(event, ua, language, session_id, deltaT, platform, ip) { }; return { app_version: pkg.version, - country: loc.country, + country: country, device_id: event.device_id, event_properties, event_type: event.event_type, @@ -163,7 +155,7 @@ function clientEvent(event, ua, language, session_id, deltaT, platform, ip) { os_name: ua.os.name, os_version: ua.os.version, platform, - region: loc.state, + region: state, session_id, time: event.time + deltaT, user_id: event.user_id, diff --git a/server/routes/delete.js b/server/routes/delete.js index 20556486..586cd9dc 100644 --- a/server/routes/delete.js +++ b/server/routes/delete.js @@ -11,6 +11,8 @@ module.exports = async function(req, res) { statDeleteEvent({ id, ip: req.ip, + country: req.geo.country, + state: req.geo.state, owner: meta.owner, download_count: meta.dl, ttl, diff --git a/server/routes/download.js b/server/routes/download.js index d714b3d0..d9a41e9c 100644 --- a/server/routes/download.js +++ b/server/routes/download.js @@ -31,6 +31,8 @@ module.exports = async function(req, res) { statDownloadEvent({ id, ip: req.ip, + country: req.geo.country, + state: req.geo.state, owner: meta.owner, download_count: dl, ttl, diff --git a/server/routes/index.js b/server/routes/index.js index 637e6a85..ff31f233 100644 --- a/server/routes/index.js +++ b/server/routes/index.js @@ -93,6 +93,19 @@ module.exports = function(app) { ); next(); }); + app.use(function(req, res, next) { + try { + // set by the load balancer + const [country, state] = req.header('X-Client-Geo-Location').split(','); + req.geo = { + country, + state + }; + } catch (e) { + req.geo = {}; + } + next(); + }); app.use(bodyParser.json()); app.use(bodyParser.text()); app.get('/', language, pages.index); diff --git a/server/routes/metrics.js b/server/routes/metrics.js index 059e330e..0f6f64aa 100644 --- a/server/routes/metrics.js +++ b/server/routes/metrics.js @@ -12,7 +12,8 @@ module.exports = async function(req, res) { data.session_id + deltaT, deltaT, data.platform, - req.ip + req.geo.country, + req.geo.state ) ); const status = await sendBatch(events); diff --git a/server/routes/report.js b/server/routes/report.js index 466e1f85..5598e926 100644 --- a/server/routes/report.js +++ b/server/routes/report.js @@ -9,6 +9,8 @@ module.exports = async function(req, res) { statReportEvent({ id, ip: req.ip, + country: req.geo.country, + state: req.geo.state, owner: meta.owner, reason: req.body.reason, download_limit: meta.dlimit, diff --git a/server/routes/ws.js b/server/routes/ws.js index f56fad1d..c32b63a9 100644 --- a/server/routes/ws.js +++ b/server/routes/ws.js @@ -111,6 +111,8 @@ module.exports = function(ws, req) { statUploadEvent({ id: newId, ip: req.ip, + country: req.geo.country, + state: req.geo.state, owner, dlimit, timeLimit,