From bf9426ede7438add491cdf9ae8f81e439a83d142 Mon Sep 17 00:00:00 2001 From: Daniel Lugo Date: Mon, 3 Aug 2020 18:12:46 -0400 Subject: [PATCH 1/9] new mechanisms for fetching the feed --- src/routes.js | 65 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 52 insertions(+), 13 deletions(-) diff --git a/src/routes.js b/src/routes.js index a6f5cd75..56aeda18 100644 --- a/src/routes.js +++ b/src/routes.js @@ -13,6 +13,8 @@ const Common = require('shock-common') const isARealUsableNumber = require('lodash/isFinite') const Big = require('big.js') const size = require('lodash/size') +const { range } = require('ramda') +const flattenDeep = require('lodash/flattenDeep') const getListPage = require('../utils/paginate') const auth = require('../services/auth/auth') @@ -2047,22 +2049,59 @@ module.exports = async ( const { page: pageStr } = req.query const page = Number(pageStr) - if (!isARealUsableNumber(page)) { - return res.status(400).json({ - field: 'page', - errorMessage: 'page must be a number' + const MAX_PAGES_TO_FETCH_FOR_TRY_UNTIL = 4 + /** + * Similar to a "before" query param in cursor based pagination. We call + * it "try" because it is likely that this item lies beyond + * MAX_PAGES_TO_FETCH_FOR_TRY_UNTIL in which case we gracefully just send + * 2 pages and 205 response. + */ + const try_until = req.query + + if (pageStr) { + const page = Number(pageStr) + + if (!isARealUsableNumber(page)) { + return res.status(400).json({ + field: 'page', + errorMessage: 'page must be a number' + }) + } + + if (page < 1) { + return res.status(400).json({ + field: page, + errorMessage: 'page must be a positive number' + }) + } + + return res.status(200).json({ + posts: await GunGetters.getFeedPage(page) + }) + } else if (try_until) { + const promises = range(1, MAX_PAGES_TO_FETCH_FOR_TRY_UNTIL).map(p => + GunGetters.getFeedPage(p) + ) + + let results = await Promise.all(promises) + + const idxIfFound = results.findIndex(pp => + pp.some(p => p.id === try_until) + ) + + if (idxIfFound > -1) { + results = results.slice(0, idxIfFound) + } + + const posts = flattenDeep(results) + + return res.status(200).json({ + posts }) } - if (page < 1) { - return res.status(400).json({ - field: page, - errorMessage: 'page must be a positive number' - }) - } - - return res.status(200).json({ - posts: await GunGetters.getFeedPage(page) + return res.status(400).json({ + errorMessage: `Must provide at least a page or a try_until query param.` }) } catch (err) { return res.status(500).json({ From 16f7c0250f82d37d349984150b91188405cd29d6 Mon Sep 17 00:00:00 2001 From: Daniel Lugo Date: Tue, 4 Aug 2020 09:40:03 -0400 Subject: [PATCH 2/9] fix typo --- src/routes.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes.js b/src/routes.js index 56aeda18..57a107fc 100644 --- a/src/routes.js +++ b/src/routes.js @@ -2056,7 +2056,7 @@ module.exports = async ( * MAX_PAGES_TO_FETCH_FOR_TRY_UNTIL in which case we gracefully just send * 2 pages and 205 response. */ - const try_until = req.query + const try_until = req.query.try_until if (pageStr) { const page = Number(pageStr) From 5b9ef555aea1c59c2625bc734d7395d10345a071 Mon Sep 17 00:00:00 2001 From: Daniel Lugo Date: Tue, 4 Aug 2020 10:01:32 -0400 Subject: [PATCH 3/9] better name --- services/gunDB/contact-api/getters/feed.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/gunDB/contact-api/getters/feed.js b/services/gunDB/contact-api/getters/feed.js index e0cce941..5c8ef112 100644 --- a/services/gunDB/contact-api/getters/feed.js +++ b/services/gunDB/contact-api/getters/feed.js @@ -14,7 +14,7 @@ const Wall = require('./wall') * @param {number} pageRequested * @returns {[ number , number ]} */ -const calculateFeedPage = (numberOfPublicKeyGroups, pageRequested) => { +const calculateWallRequest = (numberOfPublicKeyGroups, pageRequested) => { // thanks to sebassdc return [ @@ -56,7 +56,7 @@ const getFeedPage = async page => { const pagedPublicKeys = R.splitEvery(10, shuffle(subbedPublicKeys)) - const [publicKeyGroupIdx, pageToRequest] = calculateFeedPage( + const [publicKeyGroupIdx, pageToRequest] = calculateWallRequest( pagedPublicKeys.length, page ) From cf600c8876ca1913f4dc2ecf5c46e61854476fd8 Mon Sep 17 00:00:00 2001 From: Daniel Lugo Date: Tue, 4 Aug 2020 10:44:11 -0400 Subject: [PATCH 4/9] asyncFilter() --- utils/helpers.js | 24 +++++++++++++++++++++ utils/helpers.spec.js | 49 +++++++++++++++++++++++++++++++++++++++++++ utils/index.js | 9 ++++++++ 3 files changed, 82 insertions(+) create mode 100644 utils/helpers.js create mode 100644 utils/helpers.spec.js create mode 100644 utils/index.js diff --git a/utils/helpers.js b/utils/helpers.js new file mode 100644 index 00000000..4fecdc2f --- /dev/null +++ b/utils/helpers.js @@ -0,0 +1,24 @@ +/** + * @format + */ + +/** + * @template T + * @typedef {(value: T) => Promise} AsyncFilterCallback + */ + +/** + * @template T + * @param {T[]} arr + * @param {AsyncFilterCallback} cb + * @returns {Promise} + */ +const asyncFilter = async (arr, cb) => { + const results = await Promise.all(arr.map(item => cb(item))) + + return arr.filter((_, i) => results[i]) +} + +module.exports = { + asyncFilter +} diff --git a/utils/helpers.spec.js b/utils/helpers.spec.js new file mode 100644 index 00000000..409f2264 --- /dev/null +++ b/utils/helpers.spec.js @@ -0,0 +1,49 @@ +/** + * @format + */ + +const { asyncFilter } = require('./helpers') + +const numbers = [1, 2, 3, 4] +const odds = [1, 3] +const evens = [2, 4] + +describe('asyncFilter()', () => { + it('returns an empty array when given one', async () => { + expect.hasAssertions() + const result = await asyncFilter([], () => true) + + expect(result).toStrictEqual([]) + }) + + it('rejects', async () => { + expect.hasAssertions() + + const result = await asyncFilter(numbers, () => false) + + expect(result).toStrictEqual([]) + }) + + it('rejects via calling with the correct value', async () => { + expect.hasAssertions() + + const result = await asyncFilter(numbers, v => v % 2 !== 0) + + expect(result).toStrictEqual(odds) + }) + + it('filters via calling with the correct value', async () => { + expect.hasAssertions() + + const result = await asyncFilter(numbers, v => v % 2 === 0) + + expect(result).toStrictEqual(evens) + }) + + it('handles promises', async () => { + expect.hasAssertions() + + const result = await asyncFilter(numbers, v => Promise.resolve(v % 2 === 0)) + expect(result).toStrictEqual(evens) + }) +}) diff --git a/utils/index.js b/utils/index.js new file mode 100644 index 00000000..90e80eee --- /dev/null +++ b/utils/index.js @@ -0,0 +1,9 @@ +/** + * @format + */ + +const { asyncFilter } = require('./helpers') + +module.exports = { + asyncFilter +} From 5289a9e398f2b2cfd75e2cb1deef2d64d4c056fe Mon Sep 17 00:00:00 2001 From: Daniel Lugo Date: Tue, 4 Aug 2020 12:22:13 -0400 Subject: [PATCH 5/9] avoid out of bound errors --- services/gunDB/contact-api/getters/feed.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/services/gunDB/contact-api/getters/feed.js b/services/gunDB/contact-api/getters/feed.js index 5c8ef112..22e160a7 100644 --- a/services/gunDB/contact-api/getters/feed.js +++ b/services/gunDB/contact-api/getters/feed.js @@ -6,6 +6,8 @@ const isFinite = require('lodash/isFinite') const shuffle = require('lodash/shuffle') const R = require('ramda') +const { asyncFilter } = require('../../../../utils') + const Follows = require('./follows') const Wall = require('./wall') @@ -61,7 +63,13 @@ const getFeedPage = async page => { page ) - const publicKeys = pagedPublicKeys[publicKeyGroupIdx] + const publicKeysRaw = pagedPublicKeys[publicKeyGroupIdx] + const publicKeys = await asyncFilter( + publicKeysRaw, + // reject public keys for which the page to request would result in an out + // of bounds error + async pk => pageToRequest <= (await Wall.getWallTotalPages(pk)) + ) const fetchedPages = await Promise.all( publicKeys.map(pk => Wall.getWallPage(pageToRequest, pk)) From 48894b27c4b286cb26de4cab95b068b3a1a4ed0c Mon Sep 17 00:00:00 2001 From: Daniel Lugo Date: Tue, 4 Aug 2020 12:25:49 -0400 Subject: [PATCH 6/9] fix try_until --- src/routes.js | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/routes.js b/src/routes.js index 57a107fc..df707b9f 100644 --- a/src/routes.js +++ b/src/routes.js @@ -13,8 +13,7 @@ const Common = require('shock-common') const isARealUsableNumber = require('lodash/isFinite') const Big = require('big.js') const size = require('lodash/size') -const { range } = require('ramda') -const flattenDeep = require('lodash/flattenDeep') +const { range, flatten } = require('ramda') const getListPage = require('../utils/paginate') const auth = require('../services/auth/auth') @@ -2046,16 +2045,17 @@ module.exports = async ( */ const apiGunFeedGet = async (req, res) => { try { - const { page: pageStr } = req.query - const page = Number(pageStr) - const MAX_PAGES_TO_FETCH_FOR_TRY_UNTIL = 4 + + const { page: pageStr } = req.query + /** * Similar to a "before" query param in cursor based pagination. We call * it "try" because it is likely that this item lies beyond * MAX_PAGES_TO_FETCH_FOR_TRY_UNTIL in which case we gracefully just send * 2 pages and 205 response. */ + // eslint-disable-next-line prefer-destructuring const try_until = req.query.try_until if (pageStr) { @@ -2079,9 +2079,8 @@ module.exports = async ( posts: await GunGetters.getFeedPage(page) }) } else if (try_until) { - const promises = range(1, MAX_PAGES_TO_FETCH_FOR_TRY_UNTIL).map(p => - GunGetters.getFeedPage(p) - ) + const pages = range(1, MAX_PAGES_TO_FETCH_FOR_TRY_UNTIL) + const promises = pages.map(p => GunGetters.getFeedPage(p)) let results = await Promise.all(promises) @@ -2090,10 +2089,10 @@ module.exports = async ( ) if (idxIfFound > -1) { - results = results.slice(0, idxIfFound) + results = results.slice(0, idxIfFound + 1) } - const posts = flattenDeep(results) + const posts = flatten(results) return res.status(200).json({ posts From 1d32d636c20028f0876e2277e3a40aaf6d017eb5 Mon Sep 17 00:00:00 2001 From: Daniel Lugo Date: Tue, 4 Aug 2020 12:47:19 -0400 Subject: [PATCH 7/9] handle try_until failures --- src/routes.js | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/routes.js b/src/routes.js index df707b9f..9cced4f8 100644 --- a/src/routes.js +++ b/src/routes.js @@ -2078,7 +2078,9 @@ module.exports = async ( return res.status(200).json({ posts: await GunGetters.getFeedPage(page) }) - } else if (try_until) { + } + + if (try_until) { const pages = range(1, MAX_PAGES_TO_FETCH_FOR_TRY_UNTIL) const promises = pages.map(p => GunGetters.getFeedPage(p)) @@ -2090,12 +2092,20 @@ module.exports = async ( if (idxIfFound > -1) { results = results.slice(0, idxIfFound + 1) + + const posts = flatten(results) + + return res.status(200).json({ + posts + }) } - const posts = flatten(results) + // we couldn't find the posts leading up to the requested post + // (try_until) Let's just return the ones we found with together with a + // 205 code (client should refresh UI) - return res.status(200).json({ - posts + return res.status(205).json({ + posts: results[0] || [] }) } From aa748323e265a5120c893f6ca772bd57800b5318 Mon Sep 17 00:00:00 2001 From: Daniel Lugo Date: Tue, 4 Aug 2020 12:52:47 -0400 Subject: [PATCH 8/9] remove redudant code --- services/gunDB/contact-api/getters/follows.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/services/gunDB/contact-api/getters/follows.js b/services/gunDB/contact-api/getters/follows.js index 55ed47f8..9a802cf7 100644 --- a/services/gunDB/contact-api/getters/follows.js +++ b/services/gunDB/contact-api/getters/follows.js @@ -29,11 +29,7 @@ exports.currentFollows = async () => { } // load sometimes returns an empty set on the first try - if (size(v) === 0) { - return true - } - - return false + return size(v) === 0 } ) From c0f36d71dde173c23c894470ffb292098b814c59 Mon Sep 17 00:00:00 2001 From: Daniel Lugo Date: Tue, 4 Aug 2020 13:30:54 -0400 Subject: [PATCH 9/9] ensure follows gets loaded on first try --- services/gunDB/contact-api/getters/follows.js | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/services/gunDB/contact-api/getters/follows.js b/services/gunDB/contact-api/getters/follows.js index 9a802cf7..da7de6e1 100644 --- a/services/gunDB/contact-api/getters/follows.js +++ b/services/gunDB/contact-api/getters/follows.js @@ -21,15 +21,25 @@ exports.currentFollows = async () => { * @type {Record} */ const raw = await Utils.tryAndWait( - // @ts-ignore - (_, user) => new Promise(res => user.get(Key.FOLLOWS).load(res)), + (_, user) => + new Promise(res => + // @ts-expect-error + user.get(Key.FOLLOWS).load(res) + ), v => { if (typeof v !== 'object' || v === null) { return true } // load sometimes returns an empty set on the first try - return size(v) === 0 + if (size(v) === 0) { + return true + } + + // sometimes it returns empty sub objects + return Object.values(v) + .filter(Common.Schema.isObj) + .some(obj => size(obj) === 0) } )