2017-06-07 23:07:31 +02:00
|
|
|
const AWS = require('aws-sdk');
|
|
|
|
const s3 = new AWS.S3();
|
|
|
|
|
|
|
|
const conf = require('./config.js');
|
|
|
|
const fs = require('fs');
|
|
|
|
const path = require('path');
|
|
|
|
const crypto = require('crypto');
|
|
|
|
|
2017-06-09 19:44:12 +02:00
|
|
|
const notLocalHost = conf.notLocalHost;
|
2017-06-08 22:45:28 +02:00
|
|
|
|
|
|
|
const mozlog = require('./log.js');
|
|
|
|
|
2017-06-09 19:44:12 +02:00
|
|
|
const log = mozlog('portal.storage');
|
2017-06-08 22:45:28 +02:00
|
|
|
|
2017-06-08 01:16:38 +02:00
|
|
|
const redis = require('redis');
|
2017-06-08 01:46:48 +02:00
|
|
|
const redis_client = redis.createClient({
|
|
|
|
host: conf.redis_host
|
|
|
|
});
|
2017-06-08 01:16:38 +02:00
|
|
|
|
|
|
|
redis_client.on('error', err => {
|
2017-06-10 01:31:34 +02:00
|
|
|
log.info('Redis:', err);
|
2017-06-08 01:16:38 +02:00
|
|
|
});
|
|
|
|
|
2017-06-08 22:45:28 +02:00
|
|
|
if (notLocalHost) {
|
2017-06-07 23:07:31 +02:00
|
|
|
module.exports = {
|
2017-06-08 01:16:38 +02:00
|
|
|
filename: filename,
|
2017-06-09 00:01:04 +02:00
|
|
|
exists: exists,
|
2017-06-08 01:16:38 +02:00
|
|
|
length: awsLength,
|
|
|
|
get: awsGet,
|
|
|
|
set: awsSet,
|
|
|
|
delete: awsDelete,
|
2017-06-09 00:01:04 +02:00
|
|
|
forceDelete: awsForceDelete
|
2017-06-07 23:07:31 +02:00
|
|
|
};
|
|
|
|
} else {
|
|
|
|
module.exports = {
|
2017-06-08 01:16:38 +02:00
|
|
|
filename: filename,
|
2017-06-09 00:01:04 +02:00
|
|
|
exists: exists,
|
2017-06-08 01:16:38 +02:00
|
|
|
length: localLength,
|
|
|
|
get: localGet,
|
|
|
|
set: localSet,
|
|
|
|
delete: localDelete,
|
2017-06-09 00:01:04 +02:00
|
|
|
forceDelete: localForceDelete
|
2017-06-07 23:07:31 +02:00
|
|
|
};
|
|
|
|
}
|
|
|
|
|
2017-06-08 01:16:38 +02:00
|
|
|
function filename(id) {
|
|
|
|
return new Promise((resolve, reject) => {
|
|
|
|
redis_client.hget(id, 'filename', (err, reply) => {
|
|
|
|
if (!err) {
|
|
|
|
resolve(reply);
|
|
|
|
} else {
|
2017-06-23 19:53:11 +02:00
|
|
|
reject(err);
|
2017-06-08 01:16:38 +02:00
|
|
|
}
|
|
|
|
});
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2017-06-09 00:01:04 +02:00
|
|
|
function exists(id) {
|
2017-06-08 22:45:28 +02:00
|
|
|
return new Promise((resolve, reject) => {
|
2017-06-09 00:01:04 +02:00
|
|
|
redis_client.exists(id, (rediserr, reply) => {
|
2017-06-23 20:42:51 +02:00
|
|
|
if (reply === 1 && !rediserr) {
|
|
|
|
resolve();
|
|
|
|
} else {
|
|
|
|
reject(rediserr);
|
|
|
|
}
|
2017-06-08 22:45:28 +02:00
|
|
|
});
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2017-06-08 01:16:38 +02:00
|
|
|
function localLength(id) {
|
2017-06-07 23:07:31 +02:00
|
|
|
return new Promise((resolve, reject) => {
|
|
|
|
try {
|
2017-06-09 19:44:12 +02:00
|
|
|
resolve(fs.statSync(path.join(__dirname, '../static', id)).size);
|
2017-06-07 23:07:31 +02:00
|
|
|
} catch (err) {
|
|
|
|
reject();
|
|
|
|
}
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2017-06-08 01:16:38 +02:00
|
|
|
function localGet(id) {
|
2017-06-09 19:44:12 +02:00
|
|
|
return fs.createReadStream(path.join(__dirname, '../static', id));
|
2017-06-07 23:07:31 +02:00
|
|
|
}
|
|
|
|
|
2017-06-08 01:16:38 +02:00
|
|
|
function localSet(id, file, filename, url) {
|
2017-06-07 23:07:31 +02:00
|
|
|
return new Promise((resolve, reject) => {
|
2017-06-09 19:44:12 +02:00
|
|
|
const fstream = fs.createWriteStream(path.join(__dirname, '../static', id));
|
2017-06-07 23:07:31 +02:00
|
|
|
file.pipe(fstream);
|
|
|
|
fstream.on('close', () => {
|
2017-06-09 19:44:12 +02:00
|
|
|
const uuid = crypto.randomBytes(10).toString('hex');
|
2017-06-07 23:07:31 +02:00
|
|
|
|
2017-06-08 01:16:38 +02:00
|
|
|
redis_client.hmset([id, 'filename', filename, 'delete', uuid]);
|
|
|
|
redis_client.expire(id, 86400000);
|
2017-06-08 22:45:28 +02:00
|
|
|
log.info('localSet:', 'Upload Finished of ' + id);
|
2017-06-07 23:07:31 +02:00
|
|
|
resolve({
|
|
|
|
uuid: uuid,
|
|
|
|
url: url
|
|
|
|
});
|
|
|
|
});
|
2017-06-08 01:16:38 +02:00
|
|
|
|
2017-06-10 01:31:34 +02:00
|
|
|
fstream.on('error', () => {
|
|
|
|
log.error('localSet:', 'Failed upload of ' + id);
|
|
|
|
reject();
|
|
|
|
});
|
2017-06-07 23:07:31 +02:00
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2017-06-08 01:16:38 +02:00
|
|
|
function localDelete(id, delete_token) {
|
2017-06-07 23:07:31 +02:00
|
|
|
return new Promise((resolve, reject) => {
|
2017-06-08 01:16:38 +02:00
|
|
|
redis_client.hget(id, 'delete', (err, reply) => {
|
2017-06-07 23:07:31 +02:00
|
|
|
if (!reply || delete_token !== reply) {
|
2017-06-08 01:16:38 +02:00
|
|
|
reject();
|
2017-06-07 23:07:31 +02:00
|
|
|
} else {
|
2017-06-08 01:16:38 +02:00
|
|
|
redis_client.del(id);
|
2017-06-09 19:44:12 +02:00
|
|
|
resolve(fs.unlinkSync(path.join(__dirname, '../static', id)));
|
2017-06-07 23:07:31 +02:00
|
|
|
}
|
|
|
|
});
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2017-06-08 01:16:38 +02:00
|
|
|
function localForceDelete(id) {
|
2017-06-07 23:07:31 +02:00
|
|
|
return new Promise((resolve, reject) => {
|
2017-06-08 01:16:38 +02:00
|
|
|
redis_client.del(id);
|
2017-06-09 19:44:12 +02:00
|
|
|
resolve(fs.unlinkSync(path.join(__dirname, '../static', id)));
|
2017-06-07 23:07:31 +02:00
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2017-06-08 01:16:38 +02:00
|
|
|
function awsLength(id) {
|
2017-06-09 19:44:12 +02:00
|
|
|
const params = {
|
2017-06-07 23:07:31 +02:00
|
|
|
Bucket: conf.s3_bucket,
|
|
|
|
Key: id
|
|
|
|
};
|
|
|
|
return new Promise((resolve, reject) => {
|
|
|
|
s3.headObject(params, function(err, data) {
|
2017-06-08 01:16:38 +02:00
|
|
|
if (!err) {
|
|
|
|
resolve(data.ContentLength);
|
|
|
|
} else {
|
|
|
|
reject();
|
|
|
|
}
|
2017-06-07 23:07:31 +02:00
|
|
|
});
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2017-06-08 01:16:38 +02:00
|
|
|
function awsGet(id) {
|
2017-06-09 19:44:12 +02:00
|
|
|
const params = {
|
2017-06-07 23:07:31 +02:00
|
|
|
Bucket: conf.s3_bucket,
|
|
|
|
Key: id
|
|
|
|
};
|
|
|
|
|
2017-06-20 19:21:11 +02:00
|
|
|
try {
|
|
|
|
return s3.getObject(params).createReadStream();
|
2017-06-20 21:52:01 +02:00
|
|
|
} catch (err) {
|
2017-06-20 19:21:11 +02:00
|
|
|
log.info('GetFailed', 'Get Object from s3 failed.');
|
|
|
|
return null;
|
|
|
|
}
|
2017-06-07 23:07:31 +02:00
|
|
|
}
|
|
|
|
|
2017-06-08 01:16:38 +02:00
|
|
|
function awsSet(id, file, filename, url) {
|
2017-06-09 19:44:12 +02:00
|
|
|
const params = {
|
2017-06-07 23:07:31 +02:00
|
|
|
Bucket: conf.s3_bucket,
|
|
|
|
Key: id,
|
|
|
|
Body: file
|
|
|
|
};
|
|
|
|
|
|
|
|
return new Promise((resolve, reject) => {
|
2017-06-09 19:44:12 +02:00
|
|
|
s3.upload(params, function(err, _data) {
|
2017-06-07 23:07:31 +02:00
|
|
|
if (err) {
|
2017-06-08 22:45:28 +02:00
|
|
|
log.info('awsUploadError:', err.stack); // an error occurred
|
2017-06-08 01:16:38 +02:00
|
|
|
reject();
|
2017-06-07 23:07:31 +02:00
|
|
|
} else {
|
2017-06-09 19:44:12 +02:00
|
|
|
const uuid = crypto.randomBytes(10).toString('hex');
|
2017-06-07 23:07:31 +02:00
|
|
|
|
2017-06-08 01:16:38 +02:00
|
|
|
redis_client.hmset([id, 'filename', filename, 'delete', uuid]);
|
2017-06-07 23:07:31 +02:00
|
|
|
|
2017-06-08 01:16:38 +02:00
|
|
|
redis_client.expire(id, 86400000);
|
2017-06-09 00:01:04 +02:00
|
|
|
log.info('awsUploadFinish', 'Upload Finished of ' + filename);
|
2017-06-23 00:32:52 +02:00
|
|
|
resolve({
|
|
|
|
uuid: uuid,
|
|
|
|
url: url
|
|
|
|
});
|
2017-06-07 23:07:31 +02:00
|
|
|
}
|
|
|
|
});
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2017-06-08 01:16:38 +02:00
|
|
|
function awsDelete(id, delete_token) {
|
2017-06-07 23:07:31 +02:00
|
|
|
return new Promise((resolve, reject) => {
|
2017-06-08 01:16:38 +02:00
|
|
|
redis_client.hget(id, 'delete', (err, reply) => {
|
2017-06-07 23:07:31 +02:00
|
|
|
if (!reply || delete_token !== reply) {
|
2017-06-08 01:16:38 +02:00
|
|
|
reject();
|
2017-06-07 23:07:31 +02:00
|
|
|
} else {
|
2017-06-08 01:16:38 +02:00
|
|
|
redis_client.del(id);
|
2017-06-09 19:44:12 +02:00
|
|
|
const params = {
|
2017-06-07 23:07:31 +02:00
|
|
|
Bucket: conf.s3_bucket,
|
|
|
|
Key: id
|
|
|
|
};
|
|
|
|
|
2017-06-09 19:44:12 +02:00
|
|
|
s3.deleteObject(params, function(err, _data) {
|
2017-06-19 22:37:56 +02:00
|
|
|
err ? reject(err) : resolve(err);
|
2017-06-08 01:16:38 +02:00
|
|
|
});
|
2017-06-07 23:07:31 +02:00
|
|
|
}
|
|
|
|
});
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2017-06-08 01:16:38 +02:00
|
|
|
function awsForceDelete(id) {
|
2017-06-07 23:07:31 +02:00
|
|
|
return new Promise((resolve, reject) => {
|
2017-06-08 01:16:38 +02:00
|
|
|
redis_client.del(id);
|
2017-06-09 19:44:12 +02:00
|
|
|
const params = {
|
2017-06-07 23:07:31 +02:00
|
|
|
Bucket: conf.s3_bucket,
|
|
|
|
Key: id
|
|
|
|
};
|
|
|
|
|
2017-06-09 19:44:12 +02:00
|
|
|
s3.deleteObject(params, function(err, _data) {
|
2017-06-19 22:37:56 +02:00
|
|
|
err ? reject(err) : resolve(err);
|
2017-06-08 01:16:38 +02:00
|
|
|
});
|
2017-06-07 23:07:31 +02:00
|
|
|
});
|
|
|
|
}
|