From 00603ed0fa1477311cf5b6d1dfb5c69fc3cc7033 Mon Sep 17 00:00:00 2001 From: emad-salah Date: Sat, 7 Mar 2020 15:51:20 +0100 Subject: [PATCH 1/3] Added API Caching functionality --- .gitignore | 1 + package.json | 2 +- services/auth/secrets.json | 17 ++++++++++- src/routes.js | 1 + src/server.js | 59 +++++++++++++++++++++++++++++++------- utils/encryptionStore.js | 4 +-- 6 files changed, 69 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index dcfafeb5..19270c7e 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,6 @@ services/auth/secrets.json .directory radata/ +radata-*.tmp *.cert *.key diff --git a/package.json b/package.json index fb4a12d4..7ecb1d0f 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "src/server.js", "scripts": { "start": "node --experimental-modules src/server.js", - "dev": "node main.js -h 0.0.0.0", + "dev": "node --max-old-space-size=4096 main.js -h 0.0.0.0", "dev:watch": "nodemon main.js -- -h 0.0.0.0", "test": "jest --no-cache", "test:watch": "jest --no-cache --watch", diff --git a/services/auth/secrets.json b/services/auth/secrets.json index ab09bce8..529ade48 100644 --- a/services/auth/secrets.json +++ b/services/auth/secrets.json @@ -1,5 +1,20 @@ { "1582813836485": "b8328f50-596d-11ea-b251-cf923e62abf5", "1582815195785": "e2674790-5970-11ea-ba5d-4b7fda6488de", - "1582820933939": "3e9c1830-597e-11ea-acaa-2b96f5467d08" + "1582820933939": "3e9c1830-597e-11ea-acaa-2b96f5467d08", + "1582670203354": "4c2e13a0-581f-11ea-ad3d-a73b4b4c0361", + "1583339145556": "cccda140-5e34-11ea-8c4e-17a1f5fd1235", + "1583584195969": "5a3b3f10-606f-11ea-8523-9d59c298ff41", + "1583584284436": "8ef63d40-606f-11ea-ba0f-e74d20a4ef9a", + "1583584328860": "a970cdc0-606f-11ea-ba0f-e74d20a4ef9a", + "1583584658911": "6e2a96f0-6070-11ea-ba0f-e74d20a4ef9a", + "1583585004260": "3c02aa40-6071-11ea-987e-bd3595c9104b", + "1583585179728": "a498f500-6071-11ea-a5f5-1f051c0be7de", + "1583585675201": "cbec2310-6072-11ea-a5f5-1f051c0be7de", + "1583585744767": "f56310f0-6072-11ea-a5f5-1f051c0be7de", + "1583585838554": "2d49d3a0-6073-11ea-b4d0-f9a792ba6160", + "1583586016626": "976d7520-6073-11ea-a6b4-4380ee6b86a1", + "1583588647390": "b77c2fe0-6079-11ea-a0aa-8368c2d33cfe", + "1583590307894": "9538fd60-607d-11ea-92cb-ed88e8db1271", + "1583590742693": "98623460-607e-11ea-bf6c-611b3b0363dc" } diff --git a/src/routes.js b/src/routes.js index 494938b2..ed0bccb0 100644 --- a/src/routes.js +++ b/src/routes.js @@ -212,6 +212,7 @@ module.exports = async ( app.use((req, res, next) => { const deviceId = req.headers["x-shockwallet-device-id"]; + console.log("Decrypting route...") try { if (nonEncryptedRoutes.includes(req.path)) { return next(); diff --git a/src/server.js b/src/server.js index 8d5af925..04af6b85 100644 --- a/src/server.js +++ b/src/server.js @@ -9,6 +9,7 @@ const server = program => { const Https = require('https') const Http = require('http') const Express = require('express') + const Crypto = require('crypto') const LightningServices = require('../utils/lightningServices') const Encryption = require('../utils/encryptionStore') const app = Express() @@ -41,6 +42,25 @@ const server = program => { logger.info('Mainnet Mode:', !!program.mainnet) + const stringifyData = data => { + if (typeof data === 'object') { + const stringifiedData = JSON.stringify(data) + return stringifiedData + } + + if (data.toString) { + return data.toString() + } + + return data + } + + const hashData = data => { + return Crypto.createHash('SHA256') + .update(Buffer.from(stringifyData(data))) + .digest('hex') + } + const modifyResponseBody = (req, res, next) => { const deviceId = req.headers['x-shockwallet-device-id'] const oldSend = res.send @@ -50,18 +70,35 @@ const server = program => { if (args[0] && args[0].encryptedData && args[0].encryptionKey) { logger.warn('Response loop detected!') oldSend.apply(res, args) - } else { - // arguments[0] (or `data`) contains the response body - const authorized = Encryption.isAuthorizedDevice({ deviceId }) - const encryptedMessage = authorized - ? Encryption.encryptMessage({ - message: args[0], - deviceId - }) - : args[0] - args[0] = JSON.stringify(encryptedMessage) - oldSend.apply(res, args) + return } + + const dataHash = hashData(args[0]) + res.set('ETag', dataHash) + + console.log('ETag:', req.headers.etag) + console.log('Data Hash:', dataHash) + if (req.headers.etag === dataHash) { + console.log('Same Hash Detected!') + args[0] = null + res.status(304) + oldSend.apply(res, args) + return + } + + // arguments[0] (or `data`) contains the response body + const authorized = Encryption.isAuthorizedDevice({ deviceId }) + const encryptedMessage = authorized + ? Encryption.encryptMessage({ + message: args[0], + deviceId, + metadata: { + hash: dataHash + } + }) + : args[0] + args[0] = JSON.stringify(encryptedMessage) + oldSend.apply(res, args) } } next() diff --git a/utils/encryptionStore.js b/utils/encryptionStore.js index c67c6195..ba4f8191 100644 --- a/utils/encryptionStore.js +++ b/utils/encryptionStore.js @@ -49,7 +49,7 @@ const Encryption = { return encryptedData.toString() }, - encryptMessage: ({ deviceId, message }) => { + encryptMessage: ({ deviceId, message, metadata }) => { const parsedMessage = typeof message === 'object' ? JSON.stringify(message) : message const data = Buffer.from(parsedMessage) @@ -66,7 +66,7 @@ const Encryption = { Buffer.from(cipher.final()) ]) const encryptedData = encryptedBuffer.toString('base64') - return { encryptedData, encryptedKey, iv: iv.toString('hex') } + return { encryptedData, encryptedKey, iv: iv.toString('hex'), metadata } }, decryptMessage: ({ message, key, iv }) => { const data = Buffer.from(message, 'base64') From e2abc424896c78a1c7687181cbe526013281403e Mon Sep 17 00:00:00 2001 From: emad-salah Date: Tue, 10 Mar 2020 14:20:03 +0100 Subject: [PATCH 2/3] Data compression --- package.json | 1 + src/server.js | 3 +++ yarn.lock | 32 +++++++++++++++++++++++++++++++- 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 7ecb1d0f..3a9bfe8f 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "colors": "^1.3.0", "command-exists": "^1.2.6", "commander": "^2.9.0", + "compression": "^1.7.4", "cors": "^2.8.4", "debug": "^3.1.0", "dotenv": "^8.1.0", diff --git a/src/server.js b/src/server.js index 04af6b85..438b92af 100644 --- a/src/server.js +++ b/src/server.js @@ -15,6 +15,7 @@ const server = program => { const app = Express() const FS = require('../utils/fs') + const compression = require('compression') const bodyParser = require('body-parser') const session = require('express-session') const methodOverride = require('method-override') @@ -121,6 +122,8 @@ const server = program => { ) const auth = require('../services/auth/auth') + app.use(compression()) + app.use(async (req, res, next) => { logger.info('Route:', req.path) if (unprotectedRoutes[req.method][req.path]) { diff --git a/yarn.lock b/yarn.lock index f77ff876..51c6e263 100644 --- a/yarn.lock +++ b/yarn.lock @@ -703,7 +703,7 @@ abbrev@1: resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q== -accepts@~1.3.4, accepts@~1.3.7: +accepts@~1.3.4, accepts@~1.3.5, accepts@~1.3.7: version "1.3.7" resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.7.tgz#531bc726517a3b2b41f850021c6cc15eaab507cd" integrity sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA== @@ -1332,6 +1332,11 @@ bytebuffer@~5: dependencies: long "~3" +bytes@3.0.0: + version "3.0.0" + resolved "https://registry.npm.taobao.org/bytes/download/bytes-3.0.0.tgz#d32815404d689699f85a4ea4fa8755dd13a96048" + integrity sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg= + bytes@3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.0.tgz#f6cf7933a360e0588fa9fde85651cdc7f805d1f6" @@ -1614,6 +1619,26 @@ component-inherit@0.0.3: resolved "https://registry.yarnpkg.com/component-inherit/-/component-inherit-0.0.3.tgz#645fc4adf58b72b649d5cae65135619db26ff143" integrity sha1-ZF/ErfWLcrZJ1crmUTVhnbJv8UM= +compressible@~2.0.16: + version "2.0.18" + resolved "https://registry.npm.taobao.org/compressible/download/compressible-2.0.18.tgz#af53cca6b070d4c3c0750fbd77286a6d7cc46fba" + integrity sha1-r1PMprBw1MPAdQ+9dyhqbXzEb7o= + dependencies: + mime-db ">= 1.43.0 < 2" + +compression@^1.7.4: + version "1.7.4" + resolved "https://registry.npm.taobao.org/compression/download/compression-1.7.4.tgz#95523eff170ca57c29a0ca41e6fe131f41e5bb8f" + integrity sha1-lVI+/xcMpXwpoMpB5v4TH0Hlu48= + dependencies: + accepts "~1.3.5" + bytes "3.0.0" + compressible "~2.0.16" + debug "2.6.9" + on-headers "~1.0.2" + safe-buffer "5.1.2" + vary "~1.1.2" + concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" @@ -4195,6 +4220,11 @@ mime-db@1.40.0: resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.40.0.tgz#a65057e998db090f732a68f6c276d387d4126c32" integrity sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA== +"mime-db@>= 1.43.0 < 2": + version "1.43.0" + resolved "https://registry.npm.taobao.org/mime-db/download/mime-db-1.43.0.tgz#0a12e0502650e473d735535050e7c8f4eb4fae58" + integrity sha1-ChLgUCZQ5HPXNVNQUOfI9OtPrlg= + mime-types@^2.1.12, mime-types@~2.1.19, mime-types@~2.1.24: version "2.1.24" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.24.tgz#b6f8d0b3e951efb77dedeca194cff6d16f676f81" From 0fbc2366ba0272b69c4a2f899949ff02bc627ff6 Mon Sep 17 00:00:00 2001 From: emad-salah Date: Tue, 10 Mar 2020 14:45:39 +0100 Subject: [PATCH 3/3] Reduced hash length to preserve data usage --- src/server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.js b/src/server.js index 438b92af..1afa8104 100644 --- a/src/server.js +++ b/src/server.js @@ -74,7 +74,7 @@ const server = program => { return } - const dataHash = hashData(args[0]) + const dataHash = hashData(args[0]).slice(-8) res.set('ETag', dataHash) console.log('ETag:', req.headers.etag)