Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add http rsr #285

Merged
merged 15 commits into from
Jan 10, 2025
+132 −52
Merged

feat: add http rsr #285

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
2b5249e
add http success stats
Jan 9, 2025
6089859
change dependency and null test§
Jan 9, 2025
eb6c216
Update stats/lib/stats-fetchers.js
NikolasHaimerl Jan 9, 2025
8a0e62c
test edge cases
Jan 9, 2025
6c3aac6
merged with remote
Jan 9, 2025
2244c11
Update stats/test/handler.test.js
NikolasHaimerl Jan 9, 2025
fc6d4f6
change value check
Jan 9, 2025
274437f
formatting
Jan 9, 2025
8231dc4
Merge branch 'nhaimerl-add-http-rsr' of https://github.com/filecoin-s…
Jan 9, 2025
51a0fef
add miner test
Jan 10, 2025
975ad1d
miner stats unique test
Jan 10, 2025
f4918ee
formatting
Jan 10, 2025
677e36d
separated miner from summary test
Jan 10, 2025
c9a4d1b
move test
Jan 10, 2025
db23fcc
Merge branch 'main' into nhaimerl-add-http-rsr
NikolasHaimerl Jan 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

21 changes: 15 additions & 6 deletions stats/lib/stats-fetchers.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ export const fetchRetrievalSuccessRate = async (pgPools, filter) => {
// Fetch the "day" (DATE) as a string (TEXT) to prevent node-postgres for converting it into
// a JavaScript Date with a timezone, as that could change the date one day forward or back.
const { rows } = await pgPools.evaluate.query(`
SELECT day::text, SUM(total) as total, SUM(successful) as successful
SELECT day::text, SUM(total) as total, SUM(successful) as successful, SUM(successful_http) as successful_http
juliangruber marked this conversation as resolved.
Show resolved Hide resolved
bajtos marked this conversation as resolved.
Show resolved Hide resolved
FROM retrieval_stats
WHERE day >= $1 AND day <= $2 ${filter.nonZero === 'true' ? 'AND successful > 0' : ''}
GROUP BY day
Expand All @@ -20,7 +20,10 @@ export const fetchRetrievalSuccessRate = async (pgPools, filter) => {
day: r.day,
total: r.total,
successful: r.successful,
success_rate: r.total > 0 ? r.successful / r.total : null
success_rate: r.total > 0 ? r.successful / r.total : null,
successful_http: r.successful_http ?? null,
// successful_http might be null because the column was added later
success_rate_http: r.total > 0 && !(r.successful_http === undefined || r.successful_http === null) ? r.successful_http / r.total : null
juliangruber marked this conversation as resolved.
Show resolved Hide resolved
}))
return stats
}
Expand Down Expand Up @@ -207,7 +210,7 @@ export const fetchParticipantRewardTransfers = async (pgPools, { from, to }, add
*/
export const fetchMinersRSRSummary = async (pgPools, filter) => {
const { rows } = await pgPools.evaluate.query(`
SELECT miner_id, SUM(total) as total, SUM(successful) as successful
SELECT miner_id, SUM(total) as total, SUM(successful) as successful, SUM(successful_http) as successful_http
bajtos marked this conversation as resolved.
Show resolved Hide resolved
FROM retrieval_stats
WHERE day >= $1 AND day <= $2
GROUP BY miner_id
Expand All @@ -219,7 +222,10 @@ export const fetchMinersRSRSummary = async (pgPools, filter) => {
miner_id: r.miner_id,
total: r.total,
successful: r.successful,
success_rate: r.total > 0 ? r.successful / r.total : null
success_rate: r.total > 0 ? r.successful / r.total : null,
successful_http: r.successful_http ?? null,
// successful_http might be null because the column was added later
success_rate_http: r.total > 0 && !(r.successful_http === undefined || r.successful_http === null) ? r.successful_http / r.total : null
}))
return stats
}
Expand All @@ -232,7 +238,7 @@ export const fetchMinersRSRSummary = async (pgPools, filter) => {
*/
export const fetchDailyMinerRSRSummary = async (pgPools, { from, to }, minerId) => {
const { rows } = await pgPools.evaluate.query(`
SELECT day::TEXT, SUM(total) as total, SUM(successful) as successful
SELECT day::TEXT, SUM(total) as total, SUM(successful) as successful, SUM(successful_http) as successful_http
bajtos marked this conversation as resolved.
Show resolved Hide resolved
FROM retrieval_stats
WHERE miner_id = $1 AND day >= $2 AND day <= $3
GROUP BY day
Expand All @@ -246,7 +252,10 @@ export const fetchDailyMinerRSRSummary = async (pgPools, { from, to }, minerId)
day: r.day,
total: r.total,
successful: r.successful,
success_rate: r.total > 0 ? r.successful / r.total : null
success_rate: r.total > 0 ? r.successful / r.total : null,
successful_http: r.successful_http ?? null,
// successful_http might be null because the column was added later
success_rate_http: r.total > 0 && !(r.successful_http === undefined || r.successful_http === null) ? r.successful_http / r.total : null
}))
return stats
}
Expand Down
112 changes: 67 additions & 45 deletions stats/test/handler.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,20 +75,20 @@ describe('HTTP request handler', () => {

it('returns today stats for no query string', async () => {
const day = today()
await givenRetrievalStats(pgPools.evaluate, { day, total: 10, successful: 1 })
await givenRetrievalStats(pgPools.evaluate, { day, total: 10, successful: 1, successfulHttp: 0 })
const res = await fetch(new URL('/retrieval-success-rate', baseUrl), { redirect: 'follow' })
await assertResponseStatus(res, 200)
const stats = await res.json()
assert.deepStrictEqual(stats, [
{ day, success_rate: 0.1, successful: '1', total: '10' }
{ day, success_rate: 0.1, successful: '1', total: '10', successful_http: '0', success_rate_http: 0 }
NikolasHaimerl marked this conversation as resolved.
Show resolved Hide resolved
])
})

it('applies from & to in YYYY-MM-DD format', async () => {
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', total: 10, successful: 1 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-11', total: 20, successful: 1 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-12', total: 30, successful: 3 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-13', total: 40, successful: 1 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', total: 10, successful: 1, successfulHttp: 1 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-11', total: 20, successful: 1, successfulHttp: 0 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-12', total: 30, successful: 3, successfulHttp: 3 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-13', total: 40, successful: 1, successfulHttp: 1 })

const res = await fetch(
new URL(
Expand All @@ -101,8 +101,8 @@ describe('HTTP request handler', () => {
await assertResponseStatus(res, 200)
const stats = await res.json()
assert.deepStrictEqual(stats, [
{ day: '2024-01-11', success_rate: 0.05, successful: '1', total: '20' },
{ day: '2024-01-12', success_rate: 0.1, successful: '3', total: '30' }
{ day: '2024-01-11', success_rate: 0.05, successful: '1', total: '20', successful_http: '0', success_rate_http: 0 },
{ day: '2024-01-12', success_rate: 0.1, successful: '3', total: '30', successful_http: '3', success_rate_http: 0.1 }
])
})

Expand Down Expand Up @@ -145,10 +145,10 @@ describe('HTTP request handler', () => {
})

it('sums daily retrievals from all miners', async () => {
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1one', total: 10, successful: 1 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1two', total: 100, successful: 50 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-11', minerId: 'f1one', total: 20, successful: 1 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-11', minerId: 'f1two', total: 200, successful: 60 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1one', total: 10, successful: 1, successfulHttp: 1 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1two', total: 100, successful: 50, successfulHttp: 35 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-11', minerId: 'f1one', total: 20, successful: 1, successfulHttp: 0 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-11', minerId: 'f1two', total: 200, successful: 60, successfulHttp: 50 })

const res = await fetch(
new URL(
Expand All @@ -164,14 +164,14 @@ describe('HTTP request handler', () => {
await res.json()
)
assert.deepStrictEqual(stats, [
{ day: '2024-01-10', success_rate: 51 / 110, total: '110', successful: '51' },
{ day: '2024-01-11', success_rate: 61 / 220, total: '220', successful: '61' }
{ day: '2024-01-10', success_rate: 51 / 110, total: '110', successful: '51', successful_http: '36', success_rate_http: 36 / 110 },
{ day: '2024-01-11', success_rate: 61 / 220, total: '220', successful: '61', successful_http: '50', success_rate_http: 50 / 220 }
])
})

it('sorts items by date ascending', async () => {
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', total: 10, successful: 1 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', total: 10, successful: 5 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', total: 10, successful: 1, successfulHttp: 1 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', total: 10, successful: 5, successfulHttp: 3 })

const res = await fetch(
new URL(
Expand All @@ -186,14 +186,14 @@ describe('HTTP request handler', () => {
await res.json()
)
assert.deepStrictEqual(stats, [
{ day: '2024-01-10', success_rate: 5 / 10, total: '10', successful: '5' },
{ day: '2024-01-20', success_rate: 1 / 10, total: '10', successful: '1' }
{ day: '2024-01-10', success_rate: 5 / 10, total: '10', successful: '5', successful_http: '3', success_rate_http: 3 / 10 },
{ day: '2024-01-20', success_rate: 1 / 10, total: '10', successful: '1', successful_http: '1', success_rate_http: 1 / 10 }
])
})

it('filters out miners with zero RSR when asked', async () => {
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', total: 10, successful: 1, minerId: 'f1one' })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', total: 10, successful: 0, minerId: 'f1two' })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', total: 10, successful: 1, minerId: 'f1one', successfulHttp: 1 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', total: 10, successful: 0, minerId: 'f1two', successfulHttp: 0 })

const res = await fetch(
new URL(
Expand All @@ -208,19 +208,40 @@ describe('HTTP request handler', () => {
await res.json()
)
assert.deepStrictEqual(stats, [
{ day: '2024-01-20', success_rate: 1 / 10, successful: '1', total: '10' }
{ day: '2024-01-20', success_rate: 1 / 10, successful: '1', total: '10', successful_http: '1', success_rate_http: 1 / 10 }
])
})

it('preserves additional query string arguments when redirecting', async () => {
const day = today()
await givenRetrievalStats(pgPools.evaluate, { day, total: 10, successful: 1, minerId: 'f1one' })
await givenRetrievalStats(pgPools.evaluate, { day, total: 10, successful: 0, minerId: 'f1two' })
await givenRetrievalStats(pgPools.evaluate, { day, total: 10, successful: 1, minerId: 'f1one', successfulHttp: 1 })
await givenRetrievalStats(pgPools.evaluate, { day, total: 10, successful: 0, minerId: 'f1two', successfulHttp: 0 })
const res = await fetch(new URL('/retrieval-success-rate?nonZero=true', baseUrl), { redirect: 'follow' })
await assertResponseStatus(res, 200)
const stats = await res.json()
assert.deepStrictEqual(stats, [
{ day, success_rate: 0.1, successful: '1', total: '10' }
{ day, success_rate: 0.1, successful: '1', total: '10', successful_http: '1', success_rate_http: 0.1 }
])
})
it('handles successfulhttp edge cases', async () => {
NikolasHaimerl marked this conversation as resolved.
Show resolved Hide resolved
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', total: 10, successful: 1, successfulHttp: 0 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-21', total: 10, successful: 1, successfulHttp: undefined })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-22', total: 10, successful: 1, successfulHttp: null })

const res = await fetch(
new URL(
'/retrieval-success-rate?from=2024-01-20&to=2024-01-22',
baseUrl
), {
redirect: 'manual'
}
)
await assertResponseStatus(res, 200)
const stats = await res.json()
assert.deepStrictEqual(stats, [
{ day: '2024-01-20', success_rate: 0.1, successful: '1', total: '10', successful_http: '0', success_rate_http: 0 },
{ day: '2024-01-21', success_rate: 0.1, successful: '1', total: '10', successful_http: null, success_rate_http: null },
{ day: '2024-01-22', success_rate: 0.1, successful: '1', total: '10', successful_http: null, success_rate_http: null }
])
})
})
Expand Down Expand Up @@ -409,14 +430,14 @@ describe('HTTP request handler', () => {
describe('GET /miners/retrieval-success-rate/summary', () => {
it('returns a summary of miners RSR for the given date range', async () => {
// before the range
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1one', total: 10, successful: 1 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1two', total: 100, successful: 20 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1one', total: 10, successful: 1, successfulHttp: 1 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1two', total: 100, successful: 20, successfulHttp: 10 })
// in the range
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-11', minerId: 'f1one', total: 20, successful: 1 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-11', minerId: 'f1two', total: 200, successful: 150 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-11', minerId: 'f1one', total: 20, successful: 1, successfulHttp: 0 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-11', minerId: 'f1two', total: 200, successful: 150, successfulHttp: 100 })
// after the range
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-12', minerId: 'f1one', total: 30, successful: 1 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-12', minerId: 'f1two', total: 300, successful: 60 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-12', minerId: 'f1one', total: 30, successful: 1, successfulHttp: 1 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-12', minerId: 'f1two', total: 300, successful: 60, successfulHttp: 60 })

const res = await fetch(
new URL(
Expand All @@ -429,8 +450,8 @@ describe('HTTP request handler', () => {
await assertResponseStatus(res, 200)
const stats = await res.json()
assert.deepStrictEqual(stats, [
{ miner_id: 'f1one', success_rate: 0.05, total: '20', successful: '1' },
{ miner_id: 'f1two', success_rate: 0.75, total: '200', successful: '150' }
{ miner_id: 'f1one', success_rate: 0.05, total: '20', successful: '1', successful_http: '0', success_rate_http: 0 },
{ miner_id: 'f1two', success_rate: 0.75, total: '200', successful: '150', successful_http: '100', success_rate_http: 100 / 200 }
])
})
})
Expand Down Expand Up @@ -678,16 +699,16 @@ describe('HTTP request handler', () => {
describe('GET /miner/{id}/retrieval-success-rate/summary', () => {
it('lists daily retrieval stats summary for specified miner in given date range', async () => {
// before the range
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-09', minerId: 'f1one', total: 10, successful: 1 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-09', minerId: 'f1two', total: 100, successful: 20 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-09', minerId: 'f1one', total: 10, successful: 1, successfulHttp: 1 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-09', minerId: 'f1two', total: 100, successful: 20, successfulHttp: 10 })
// in the range
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', minerId: 'f1one', total: 20, successful: 1 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', minerId: 'f1two', total: 200, successful: 60 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1one', total: 10, successful: 1 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1two', total: 100, successful: 50 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', minerId: 'f1one', total: 20, successful: 1, successfulHttp: 0 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', minerId: 'f1two', total: 200, successful: 60, successfulHttp: 50 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1one', total: 10, successful: 1, successfulHttp: 1 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1two', total: 100, successful: 50, successfulHttp: 35 })
// after the range
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-21', minerId: 'f1one', total: 30, successful: 1 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-21', minerId: 'f1two', total: 300, successful: 60 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-21', minerId: 'f1one', total: 30, successful: 1, successfulHttp: 1 })
await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-21', minerId: 'f1two', total: 300, successful: 60, successfulHttp: 60 })

const res = await fetch(
new URL(
Expand All @@ -703,8 +724,8 @@ describe('HTTP request handler', () => {
await res.json()
)
assert.deepStrictEqual(stats, [
{ day: '2024-01-10', success_rate: 1 / 10, total: '10', successful: '1' },
{ day: '2024-01-20', success_rate: 1 / 20, total: '20', successful: '1' }
{ day: '2024-01-10', success_rate: 1 / 10, total: '10', successful: '1', successful_http: '1', success_rate_http: 1 / 10 },
{ day: '2024-01-20', success_rate: 1 / 20, total: '20', successful: '1', successful_http: '0', success_rate_http: 0 }
])
})
})
Expand All @@ -718,11 +739,12 @@ describe('HTTP request handler', () => {
* @param {string} [data.minerId]
* @param {number | bigint} data.total
* @param {number | bigint } data.successful
* @param {number | bigint} [data.successfulHttp]
*/
const givenRetrievalStats = async (pgPool, { day, minerId, total, successful }) => {
const givenRetrievalStats = async (pgPool, { day, minerId, total, successful, successfulHttp }) => {
await pgPool.query(
'INSERT INTO retrieval_stats (day, miner_id, total, successful) VALUES ($1, $2, $3, $4)',
[day, minerId ?? 'f1test', total, successful]
'INSERT INTO retrieval_stats (day, miner_id, total, successful, successful_http) VALUES ($1, $2, $3, $4, $5)',
[day, minerId ?? 'f1test', total, successful, successfulHttp]
)
}

Expand Down
Loading