From 2b5249eea2376f281e71c17e469ce1d4ac1afb49 Mon Sep 17 00:00:00 2001
From: Nikolas Haimerl <nikolas.haimerl@dfinity.org>
Date: Thu, 9 Jan 2025 10:15:53 +0100
Subject: [PATCH 01/12] add http success stats

---
 stats/lib/stats-fetchers.js | 21 ++++++---
 stats/test/handler.test.js  | 91 +++++++++++++++++++------------------
 2 files changed, 61 insertions(+), 51 deletions(-)

diff --git a/stats/lib/stats-fetchers.js b/stats/lib/stats-fetchers.js
index 3e2f2cc..f3540ac 100644
--- a/stats/lib/stats-fetchers.js
+++ b/stats/lib/stats-fetchers.js
@@ -7,7 +7,7 @@ export const fetchRetrievalSuccessRate = async (pgPools, filter) => {
   // Fetch the "day" (DATE) as a string (TEXT) to prevent node-postgres for converting it into
   // a JavaScript Date with a timezone, as that could change the date one day forward or back.
   const { rows } = await pgPools.evaluate.query(`
-    SELECT day::text, SUM(total) as total, SUM(successful) as successful
+    SELECT day::text, SUM(total) as total, SUM(successful) as successful, SUM(successful_http) as successful_http
     FROM retrieval_stats
     WHERE day >= $1 AND day <= $2 ${filter.nonZero === 'true' ? 'AND successful > 0' : ''}
     GROUP BY day
@@ -20,7 +20,10 @@ export const fetchRetrievalSuccessRate = async (pgPools, filter) => {
     day: r.day,
     total: r.total,
     successful: r.successful,
-    success_rate: r.total > 0 ? r.successful / r.total : null
+    success_rate: r.total > 0 ? r.successful / r.total : null,
+    successful_http: r.successful_http ?? null,
+    // Successful http might be null because the column was added later
+    success_rate_http: r.total > 0 && r.successful_http ? r.successful_http / r.total : null
   }))
   return stats
 }
@@ -207,7 +210,7 @@ export const fetchParticipantRewardTransfers = async (pgPools, { from, to }, add
  */
 export const fetchMinersRSRSummary = async (pgPools, filter) => {
   const { rows } = await pgPools.evaluate.query(`
-    SELECT miner_id, SUM(total) as total, SUM(successful) as successful
+    SELECT miner_id, SUM(total) as total, SUM(successful) as successful, SUM(successful_http) as successful_http
     FROM retrieval_stats
     WHERE day >= $1 AND day <= $2
     GROUP BY miner_id
@@ -219,7 +222,10 @@ export const fetchMinersRSRSummary = async (pgPools, filter) => {
     miner_id: r.miner_id,
     total: r.total,
     successful: r.successful,
-    success_rate: r.total > 0 ? r.successful / r.total : null
+    success_rate: r.total > 0 ? r.successful / r.total : null,
+    successful_http: r.successful_http ?? null,
+    // Successful http might be null because the column was added later
+    success_rate_http: r.total > 0 && r.successful_http ? r.successful_http / r.total : null
   }))
   return stats
 }
@@ -232,7 +238,7 @@ export const fetchMinersRSRSummary = async (pgPools, filter) => {
  */
 export const fetchDailyMinerRSRSummary = async (pgPools, { from, to }, minerId) => {
   const { rows } = await pgPools.evaluate.query(`
-    SELECT day::TEXT, SUM(total) as total, SUM(successful) as successful
+    SELECT day::TEXT, SUM(total) as total, SUM(successful) as successful, SUM(successful_http) as successful_http
     FROM retrieval_stats
     WHERE miner_id = $1 AND day >= $2 AND day <= $3
     GROUP BY day
@@ -246,7 +252,10 @@ export const fetchDailyMinerRSRSummary = async (pgPools, { from, to }, minerId)
     day: r.day,
     total: r.total,
     successful: r.successful,
-    success_rate: r.total > 0 ? r.successful / r.total : null
+    success_rate: r.total > 0 ? r.successful / r.total : null,
+    successful_http: r.successful_http ?? null,
+    // Successful http might be null because the column was added later
+    success_rate_http: r.total > 0 && r.successful_http ? r.successful_http / r.total : null
   }))
   return stats
 }
diff --git a/stats/test/handler.test.js b/stats/test/handler.test.js
index 74610f9..46d3d33 100644
--- a/stats/test/handler.test.js
+++ b/stats/test/handler.test.js
@@ -75,20 +75,20 @@ describe('HTTP request handler', () => {
 
     it('returns today stats for no query string', async () => {
       const day = today()
-      await givenRetrievalStats(pgPools.evaluate, { day, total: 10, successful: 1 })
+      await givenRetrievalStats(pgPools.evaluate, { day, total: 10, successful: 1, successfulHttp: 1 })
       const res = await fetch(new URL('/retrieval-success-rate', baseUrl), { redirect: 'follow' })
       await assertResponseStatus(res, 200)
       const stats = await res.json()
       assert.deepStrictEqual(stats, [
-        { day, success_rate: 0.1, successful: '1', total: '10' }
+        { day, success_rate: 0.1, successful: '1', total: '10', successful_http: '1' }
       ])
     })
 
     it('applies from & to in YYYY-MM-DD format', async () => {
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', total: 10, successful: 1 })
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-11', total: 20, successful: 1 })
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-12', total: 30, successful: 3 })
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-13', total: 40, successful: 1 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', total: 10, successful: 1, successfulHttp: 1 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-11', total: 20, successful: 1, successfulHttp: 0 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-12', total: 30, successful: 3, successfulHttp: 3 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-13', total: 40, successful: 1, successfulHttp: 1 })
 
       const res = await fetch(
         new URL(
@@ -101,8 +101,8 @@ describe('HTTP request handler', () => {
       await assertResponseStatus(res, 200)
       const stats = await res.json()
       assert.deepStrictEqual(stats, [
-        { day: '2024-01-11', success_rate: 0.05, successful: '1', total: '20' },
-        { day: '2024-01-12', success_rate: 0.1, successful: '3', total: '30' }
+        { day: '2024-01-11', success_rate: 0.05, successful: '1', total: '20', successful_http: '0' },
+        { day: '2024-01-12', success_rate: 0.1, successful: '3', total: '30', successful_http: '3' }
       ])
     })
 
@@ -145,10 +145,10 @@ describe('HTTP request handler', () => {
     })
 
     it('sums daily retrievals from all miners', async () => {
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1one', total: 10, successful: 1 })
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1two', total: 100, successful: 50 })
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-11', minerId: 'f1one', total: 20, successful: 1 })
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-11', minerId: 'f1two', total: 200, successful: 60 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1one', total: 10, successful: 1, successfulHttp: 1 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1two', total: 100, successful: 50, successfulHttp: 35 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-11', minerId: 'f1one', total: 20, successful: 1, successfulHttp: 0 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-11', minerId: 'f1two', total: 200, successful: 60, successfulHttp: 50 })
 
       const res = await fetch(
         new URL(
@@ -164,14 +164,14 @@ describe('HTTP request handler', () => {
         await res.json()
       )
       assert.deepStrictEqual(stats, [
-        { day: '2024-01-10', success_rate: 51 / 110, total: '110', successful: '51' },
-        { day: '2024-01-11', success_rate: 61 / 220, total: '220', successful: '61' }
+        { day: '2024-01-10', success_rate: 51 / 110, total: '110', successful: '51', successful_http: '36' },
+        { day: '2024-01-11', success_rate: 61 / 220, total: '220', successful: '61', successful_http: '50' }
       ])
     })
 
     it('sorts items by date ascending', async () => {
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', total: 10, successful: 1 })
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', total: 10, successful: 5 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', total: 10, successful: 1, successfulHttp: 1 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', total: 10, successful: 5, successfulHttp: 3 })
 
       const res = await fetch(
         new URL(
@@ -186,14 +186,14 @@ describe('HTTP request handler', () => {
         await res.json()
       )
       assert.deepStrictEqual(stats, [
-        { day: '2024-01-10', success_rate: 5 / 10, total: '10', successful: '5' },
-        { day: '2024-01-20', success_rate: 1 / 10, total: '10', successful: '1' }
+        { day: '2024-01-10', success_rate: 5 / 10, total: '10', successful: '5', successful_http: '3' },
+        { day: '2024-01-20', success_rate: 1 / 10, total: '10', successful: '1', successful_http: '1' }
       ])
     })
 
     it('filters out miners with zero RSR when asked', async () => {
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', total: 10, successful: 1, minerId: 'f1one' })
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', total: 10, successful: 0, minerId: 'f1two' })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', total: 10, successful: 1, minerId: 'f1one', successfulHttp: 1 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', total: 10, successful: 0, minerId: 'f1two', successfulHttp: 0 })
 
       const res = await fetch(
         new URL(
@@ -208,19 +208,19 @@ describe('HTTP request handler', () => {
         await res.json()
       )
       assert.deepStrictEqual(stats, [
-        { day: '2024-01-20', success_rate: 1 / 10, successful: '1', total: '10' }
+        { day: '2024-01-20', success_rate: 1 / 10, successful: '1', total: '10', successful_http: '1' }
       ])
     })
 
     it('preserves additional query string arguments when redirecting', async () => {
       const day = today()
-      await givenRetrievalStats(pgPools.evaluate, { day, total: 10, successful: 1, minerId: 'f1one' })
-      await givenRetrievalStats(pgPools.evaluate, { day, total: 10, successful: 0, minerId: 'f1two' })
+      await givenRetrievalStats(pgPools.evaluate, { day, total: 10, successful: 1, minerId: 'f1one', successfulHttp: 1 })
+      await givenRetrievalStats(pgPools.evaluate, { day, total: 10, successful: 0, minerId: 'f1two', successfulHttp: 0 })
       const res = await fetch(new URL('/retrieval-success-rate?nonZero=true', baseUrl), { redirect: 'follow' })
       await assertResponseStatus(res, 200)
       const stats = await res.json()
       assert.deepStrictEqual(stats, [
-        { day, success_rate: 0.1, successful: '1', total: '10' }
+        { day, success_rate: 0.1, successful: '1', total: '10', successful_http: '1' }
       ])
     })
   })
@@ -409,14 +409,14 @@ describe('HTTP request handler', () => {
   describe('GET /miners/retrieval-success-rate/summary', () => {
     it('returns a summary of miners RSR for the given date range', async () => {
       // before the range
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1one', total: 10, successful: 1 })
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1two', total: 100, successful: 20 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1one', total: 10, successful: 1, successfulHttp: 1 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1two', total: 100, successful: 20, successfulHttp: 10 })
       // in the range
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-11', minerId: 'f1one', total: 20, successful: 1 })
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-11', minerId: 'f1two', total: 200, successful: 150 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-11', minerId: 'f1one', total: 20, successful: 1, successfulHttp: 0 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-11', minerId: 'f1two', total: 200, successful: 150, successfulHttp: 100 })
       // after the range
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-12', minerId: 'f1one', total: 30, successful: 1 })
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-12', minerId: 'f1two', total: 300, successful: 60 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-12', minerId: 'f1one', total: 30, successful: 1, successfulHttp: 1 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-12', minerId: 'f1two', total: 300, successful: 60, successfulHttp: 60 })
 
       const res = await fetch(
         new URL(
@@ -429,8 +429,8 @@ describe('HTTP request handler', () => {
       await assertResponseStatus(res, 200)
       const stats = await res.json()
       assert.deepStrictEqual(stats, [
-        { miner_id: 'f1one', success_rate: 0.05, total: '20', successful: '1' },
-        { miner_id: 'f1two', success_rate: 0.75, total: '200', successful: '150' }
+        { miner_id: 'f1one', success_rate: 0.05, total: '20', successful: '1', successful_http: '0' },
+        { miner_id: 'f1two', success_rate: 0.75, total: '200', successful: '150', successful_http: '100' }
       ])
     })
   })
@@ -678,16 +678,16 @@ describe('HTTP request handler', () => {
   describe('GET /miner/{id}/retrieval-success-rate/summary', () => {
     it('lists daily retrieval stats summary for specified miner in given date range', async () => {
       // before the range
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-09', minerId: 'f1one', total: 10, successful: 1 })
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-09', minerId: 'f1two', total: 100, successful: 20 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-09', minerId: 'f1one', total: 10, successful: 1, successfulHttp: 1 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-09', minerId: 'f1two', total: 100, successful: 20, successfulHttp: 10 })
       // in the range
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', minerId: 'f1one', total: 20, successful: 1 })
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', minerId: 'f1two', total: 200, successful: 60 })
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1one', total: 10, successful: 1 })
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1two', total: 100, successful: 50 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', minerId: 'f1one', total: 20, successful: 1, successfulHttp: 0 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', minerId: 'f1two', total: 200, successful: 60, successfulHttp: 50 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1one', total: 10, successful: 1, successfulHttp: 1 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-10', minerId: 'f1two', total: 100, successful: 50, successfulHttp: 35 })
       // after the range
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-21', minerId: 'f1one', total: 30, successful: 1 })
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-21', minerId: 'f1two', total: 300, successful: 60 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-21', minerId: 'f1one', total: 30, successful: 1, successfulHttp: 1 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-21', minerId: 'f1two', total: 300, successful: 60, successfulHttp: 60 })
 
       const res = await fetch(
         new URL(
@@ -703,8 +703,8 @@ describe('HTTP request handler', () => {
         await res.json()
       )
       assert.deepStrictEqual(stats, [
-        { day: '2024-01-10', success_rate: 1 / 10, total: '10', successful: '1' },
-        { day: '2024-01-20', success_rate: 1 / 20, total: '20', successful: '1' }
+        { day: '2024-01-10', success_rate: 1 / 10, total: '10', successful: '1', successful_http: '1' },
+        { day: '2024-01-20', success_rate: 1 / 20, total: '20', successful: '1', successful_http: '0' }
       ])
     })
   })
@@ -718,11 +718,12 @@ describe('HTTP request handler', () => {
  * @param {string} [data.minerId]
  * @param {number | bigint} data.total
  * @param {number | bigint } data.successful
+ * @param {number | bigint} [data.successfulHttp]
  */
-const givenRetrievalStats = async (pgPool, { day, minerId, total, successful }) => {
+const givenRetrievalStats = async (pgPool, { day, minerId, total, successful, successfulHttp }) => {
   await pgPool.query(
-    'INSERT INTO retrieval_stats (day, miner_id, total, successful) VALUES ($1, $2, $3, $4)',
-    [day, minerId ?? 'f1test', total, successful]
+    'INSERT INTO retrieval_stats (day, miner_id, total, successful, successful_http) VALUES ($1, $2, $3, $4, $5)',
+    [day, minerId ?? 'f1test', total, successful, successfulHttp]
   )
 }
 

From 60898592b95a56541a38d6bfa13381822e3625f1 Mon Sep 17 00:00:00 2001
From: Nikolas Haimerl <nikolas.haimerl@dfinity.org>
Date: Thu, 9 Jan 2025 10:42:17 +0100
Subject: [PATCH 02/12] =?UTF-8?q?change=20dependency=20and=20null=20test?=
 =?UTF-8?q?=C2=A7?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 package-lock.json          |  2 +-
 stats/test/handler.test.js | 28 ++++++++++++++--------------
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index c1662da..96cbb02 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -30,7 +30,7 @@
       }
     },
     "db/node_modules/spark-evaluate": {
-      "resolved": "git+ssh://git@github.com/filecoin-station/spark-evaluate.git#607d836f6a60b9b22c57254d6df041dcd4c1cf5f",
+      "resolved": "git+ssh://git@github.com/filecoin-station/spark-evaluate.git#3ac022aac2d5e5df7d88d33c460b45e6567d01d6",
       "dependencies": {
         "@filecoin-station/spark-impact-evaluator": "^1.2.4",
         "@glif/filecoin-address": "^3.0.12",
diff --git a/stats/test/handler.test.js b/stats/test/handler.test.js
index 46d3d33..80ed3b9 100644
--- a/stats/test/handler.test.js
+++ b/stats/test/handler.test.js
@@ -75,12 +75,12 @@ describe('HTTP request handler', () => {
 
     it('returns today stats for no query string', async () => {
       const day = today()
-      await givenRetrievalStats(pgPools.evaluate, { day, total: 10, successful: 1, successfulHttp: 1 })
+      await givenRetrievalStats(pgPools.evaluate, { day, total: 10, successful: 1, successfulHttp: null })
       const res = await fetch(new URL('/retrieval-success-rate', baseUrl), { redirect: 'follow' })
       await assertResponseStatus(res, 200)
       const stats = await res.json()
       assert.deepStrictEqual(stats, [
-        { day, success_rate: 0.1, successful: '1', total: '10', successful_http: '1' }
+        { day, success_rate: 0.1, successful: '1', total: '10', successful_http: null, success_rate_http: null }
       ])
     })
 
@@ -101,8 +101,8 @@ describe('HTTP request handler', () => {
       await assertResponseStatus(res, 200)
       const stats = await res.json()
       assert.deepStrictEqual(stats, [
-        { day: '2024-01-11', success_rate: 0.05, successful: '1', total: '20', successful_http: '0' },
-        { day: '2024-01-12', success_rate: 0.1, successful: '3', total: '30', successful_http: '3' }
+        { day: '2024-01-11', success_rate: 0.05, successful: '1', total: '20', successful_http: '0', success_rate_http: 0 },
+        { day: '2024-01-12', success_rate: 0.1, successful: '3', total: '30', successful_http: '3', success_rate_http: 0.1 }
       ])
     })
 
@@ -164,8 +164,8 @@ describe('HTTP request handler', () => {
         await res.json()
       )
       assert.deepStrictEqual(stats, [
-        { day: '2024-01-10', success_rate: 51 / 110, total: '110', successful: '51', successful_http: '36' },
-        { day: '2024-01-11', success_rate: 61 / 220, total: '220', successful: '61', successful_http: '50' }
+        { day: '2024-01-10', success_rate: 51 / 110, total: '110', successful: '51', successful_http: '36', success_rate_http: 36 / 110 },
+        { day: '2024-01-11', success_rate: 61 / 220, total: '220', successful: '61', successful_http: '50', success_rate_http: 50 / 220 }
       ])
     })
 
@@ -186,8 +186,8 @@ describe('HTTP request handler', () => {
         await res.json()
       )
       assert.deepStrictEqual(stats, [
-        { day: '2024-01-10', success_rate: 5 / 10, total: '10', successful: '5', successful_http: '3' },
-        { day: '2024-01-20', success_rate: 1 / 10, total: '10', successful: '1', successful_http: '1' }
+        { day: '2024-01-10', success_rate: 5 / 10, total: '10', successful: '5', successful_http: '3', success_rate_http: 3 / 10 },
+        { day: '2024-01-20', success_rate: 1 / 10, total: '10', successful: '1', successful_http: '1', success_rate_http: 1 / 10 }
       ])
     })
 
@@ -208,7 +208,7 @@ describe('HTTP request handler', () => {
         await res.json()
       )
       assert.deepStrictEqual(stats, [
-        { day: '2024-01-20', success_rate: 1 / 10, successful: '1', total: '10', successful_http: '1' }
+        { day: '2024-01-20', success_rate: 1 / 10, successful: '1', total: '10', successful_http: '1', success_rate_http: 1 / 10 }
       ])
     })
 
@@ -220,7 +220,7 @@ describe('HTTP request handler', () => {
       await assertResponseStatus(res, 200)
       const stats = await res.json()
       assert.deepStrictEqual(stats, [
-        { day, success_rate: 0.1, successful: '1', total: '10', successful_http: '1' }
+        { day, success_rate: 0.1, successful: '1', total: '10', successful_http: '1', success_rate_http: 0.1 }
       ])
     })
   })
@@ -429,8 +429,8 @@ describe('HTTP request handler', () => {
       await assertResponseStatus(res, 200)
       const stats = await res.json()
       assert.deepStrictEqual(stats, [
-        { miner_id: 'f1one', success_rate: 0.05, total: '20', successful: '1', successful_http: '0' },
-        { miner_id: 'f1two', success_rate: 0.75, total: '200', successful: '150', successful_http: '100' }
+        { miner_id: 'f1one', success_rate: 0.05, total: '20', successful: '1', successful_http: '0', success_rate_http: 0 },
+        { miner_id: 'f1two', success_rate: 0.75, total: '200', successful: '150', successful_http: '100', success_rate_http: 100 / 200 }
       ])
     })
   })
@@ -703,8 +703,8 @@ describe('HTTP request handler', () => {
         await res.json()
       )
       assert.deepStrictEqual(stats, [
-        { day: '2024-01-10', success_rate: 1 / 10, total: '10', successful: '1', successful_http: '1' },
-        { day: '2024-01-20', success_rate: 1 / 20, total: '20', successful: '1', successful_http: '0' }
+        { day: '2024-01-10', success_rate: 1 / 10, total: '10', successful: '1', successful_http: '1', success_rate_http: 1 / 10 },
+        { day: '2024-01-20', success_rate: 1 / 20, total: '20', successful: '1', successful_http: '0', success_rate_http: 0 }
       ])
     })
   })

From eb6c2169076ac4bedfa64acb749cbcb54e727f30 Mon Sep 17 00:00:00 2001
From: Nikolas Haimerl <113891786+NikolasHaimerl@users.noreply.github.com>
Date: Thu, 9 Jan 2025 11:23:22 +0100
Subject: [PATCH 03/12] Update stats/lib/stats-fetchers.js

Co-authored-by: Julian Gruber <julian@juliangruber.com>
---
 stats/lib/stats-fetchers.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stats/lib/stats-fetchers.js b/stats/lib/stats-fetchers.js
index f3540ac..4db332a 100644
--- a/stats/lib/stats-fetchers.js
+++ b/stats/lib/stats-fetchers.js
@@ -22,7 +22,7 @@ export const fetchRetrievalSuccessRate = async (pgPools, filter) => {
     successful: r.successful,
     success_rate: r.total > 0 ? r.successful / r.total : null,
     successful_http: r.successful_http ?? null,
-    // Successful http might be null because the column was added later
+    // `successful_http` might be null because the column was added later
     success_rate_http: r.total > 0 && r.successful_http ? r.successful_http / r.total : null
   }))
   return stats

From 8a0e62cbe71f47a4b8a7732674fb2655ae944c1d Mon Sep 17 00:00:00 2001
From: Nikolas Haimerl <nikolas.haimerl@dfinity.org>
Date: Thu, 9 Jan 2025 12:00:44 +0100
Subject: [PATCH 04/12] test edge cases

---
 stats/lib/stats-fetchers.js |  6 +++---
 stats/test/handler.test.js  | 25 +++++++++++++++++++++++--
 2 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/stats/lib/stats-fetchers.js b/stats/lib/stats-fetchers.js
index f3540ac..3213428 100644
--- a/stats/lib/stats-fetchers.js
+++ b/stats/lib/stats-fetchers.js
@@ -23,7 +23,7 @@ export const fetchRetrievalSuccessRate = async (pgPools, filter) => {
     success_rate: r.total > 0 ? r.successful / r.total : null,
     successful_http: r.successful_http ?? null,
     // Successful http might be null because the column was added later
-    success_rate_http: r.total > 0 && r.successful_http ? r.successful_http / r.total : null
+    success_rate_http: r.total > 0 && !(r.successful_http === undefined || r.successful_http === null) ? r.successful_http / r.total : null
   }))
   return stats
 }
@@ -225,7 +225,7 @@ export const fetchMinersRSRSummary = async (pgPools, filter) => {
     success_rate: r.total > 0 ? r.successful / r.total : null,
     successful_http: r.successful_http ?? null,
     // Successful http might be null because the column was added later
-    success_rate_http: r.total > 0 && r.successful_http ? r.successful_http / r.total : null
+    success_rate_http: r.total > 0 && !(r.successful_http === undefined || r.successful_http === null) ? r.successful_http / r.total : null
   }))
   return stats
 }
@@ -255,7 +255,7 @@ export const fetchDailyMinerRSRSummary = async (pgPools, { from, to }, minerId)
     success_rate: r.total > 0 ? r.successful / r.total : null,
     successful_http: r.successful_http ?? null,
     // Successful http might be null because the column was added later
-    success_rate_http: r.total > 0 && r.successful_http ? r.successful_http / r.total : null
+    success_rate_http: r.total > 0 && !(r.successful_http === undefined || r.successful_http === null) ? r.successful_http / r.total : null
   }))
   return stats
 }
diff --git a/stats/test/handler.test.js b/stats/test/handler.test.js
index 80ed3b9..ee6a95f 100644
--- a/stats/test/handler.test.js
+++ b/stats/test/handler.test.js
@@ -75,12 +75,12 @@ describe('HTTP request handler', () => {
 
     it('returns today stats for no query string', async () => {
       const day = today()
-      await givenRetrievalStats(pgPools.evaluate, { day, total: 10, successful: 1, successfulHttp: null })
+      await givenRetrievalStats(pgPools.evaluate, { day, total: 10, successful: 1, successfulHttp: 0 })
       const res = await fetch(new URL('/retrieval-success-rate', baseUrl), { redirect: 'follow' })
       await assertResponseStatus(res, 200)
       const stats = await res.json()
       assert.deepStrictEqual(stats, [
-        { day, success_rate: 0.1, successful: '1', total: '10', successful_http: null, success_rate_http: null }
+        { day, success_rate: 0.1, successful: '1', total: '10', successful_http: '0', success_rate_http: 0 }
       ])
     })
 
@@ -223,6 +223,27 @@ describe('HTTP request handler', () => {
         { day, success_rate: 0.1, successful: '1', total: '10', successful_http: '1', success_rate_http: 0.1 }
       ])
     })
+    it('handles successfulhttp edge cases', async () => {
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', total: 10, successful: 1, successfulHttp: 0 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-21', total: 10, successful: 1, successfulHttp: undefined })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-22', total: 10, successful: 1, successfulHttp: null })
+
+      const res = await fetch(
+        new URL(
+          '/retrieval-success-rate?from=2024-01-20&to=2024-01-22',
+          baseUrl
+        ), {
+          redirect: 'manual'
+        }
+      )
+      await assertResponseStatus(res, 200)
+      const stats = await res.json()
+      assert.deepStrictEqual(stats, [
+        { day: '2024-01-20', success_rate: 0.1, successful: '1', total: '10', successful_http: '0', success_rate_http: 0 },
+        { day: '2024-01-21', success_rate: 0.1, successful: '1', total: '10', successful_http: null, success_rate_http: null },
+        { day: '2024-01-22', success_rate: 0.1, successful: '1', total: '10', successful_http: null, success_rate_http: null }
+      ])
+    })
   })
 
   describe('GET /participants/daily', () => {

From 2244c11b1f5ea534fccd61957707ff7ad686740b Mon Sep 17 00:00:00 2001
From: Nikolas Haimerl <113891786+NikolasHaimerl@users.noreply.github.com>
Date: Thu, 9 Jan 2025 14:58:43 +0100
Subject: [PATCH 05/12] Update stats/test/handler.test.js
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Miroslav Bajtoš <oss@bajtos.net>
---
 stats/test/handler.test.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stats/test/handler.test.js b/stats/test/handler.test.js
index ee6a95f..fbaa009 100644
--- a/stats/test/handler.test.js
+++ b/stats/test/handler.test.js
@@ -223,7 +223,7 @@ describe('HTTP request handler', () => {
         { day, success_rate: 0.1, successful: '1', total: '10', successful_http: '1', success_rate_http: 0.1 }
       ])
     })
-    it('handles successfulhttp edge cases', async () => {
+    it('handles successful_http values 0, null, undefined', async () => {
       await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', total: 10, successful: 1, successfulHttp: 0 })
       await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-21', total: 10, successful: 1, successfulHttp: undefined })
       await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-22', total: 10, successful: 1, successfulHttp: null })

From fc6d4f62f486db1c18da8406ea9db98303d933b1 Mon Sep 17 00:00:00 2001
From: Nikolas Haimerl <nikolas.haimerl@dfinity.org>
Date: Thu, 9 Jan 2025 15:01:30 +0100
Subject: [PATCH 06/12] change value check

---
 stats/lib/stats-fetchers.js | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/stats/lib/stats-fetchers.js b/stats/lib/stats-fetchers.js
index c18c564..c80c270 100644
--- a/stats/lib/stats-fetchers.js
+++ b/stats/lib/stats-fetchers.js
@@ -1,3 +1,8 @@
+/**
+ *
+ * @param { string } value
+ */
+const isValidNumber = value => value != null && !isNaN(+value)
 /** @typedef {import('@filecoin-station/spark-stats-db').PgPools} PgPools */
 /**
  * @param {PgPools} pgPools
@@ -23,7 +28,7 @@ export const fetchRetrievalSuccessRate = async (pgPools, filter) => {
     success_rate: r.total > 0 ? r.successful / r.total : null,
     successful_http: r.successful_http ?? null,
     // successful_http might be null because the column was added later
-    success_rate_http: r.total > 0 && !(r.successful_http === undefined || r.successful_http === null) ? r.successful_http / r.total : null
+    success_rate_http: r.total > 0 && isValidNumber(r.successful_http) ? r.successful_http / r.total : null
   }))
   return stats
 }
@@ -225,7 +230,7 @@ export const fetchMinersRSRSummary = async (pgPools, filter) => {
     success_rate: r.total > 0 ? r.successful / r.total : null,
     successful_http: r.successful_http ?? null,
     // successful_http might be null because the column was added later
-    success_rate_http: r.total > 0 && !(r.successful_http === undefined || r.successful_http === null) ? r.successful_http / r.total : null
+    success_rate_http: r.total > 0 && isValidNumber(r.successful_http) ? r.successful_http / r.total : null
   }))
   return stats
 }
@@ -255,7 +260,7 @@ export const fetchDailyMinerRSRSummary = async (pgPools, { from, to }, minerId)
     success_rate: r.total > 0 ? r.successful / r.total : null,
     successful_http: r.successful_http ?? null,
     // successful_http might be null because the column was added later
-    success_rate_http: r.total > 0 && !(r.successful_http === undefined || r.successful_http === null) ? r.successful_http / r.total : null
+    success_rate_http: r.total > 0 && isValidNumber(r.successful_http) ? r.successful_http / r.total : null
   }))
   return stats
 }

From 274437f2d477839224e93180a94b5c05940ce3d2 Mon Sep 17 00:00:00 2001
From: Nikolas Haimerl <nikolas.haimerl@dfinity.org>
Date: Thu, 9 Jan 2025 17:39:04 +0100
Subject: [PATCH 07/12] formatting

---
 stats/lib/stats-fetchers.js | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/stats/lib/stats-fetchers.js b/stats/lib/stats-fetchers.js
index c80c270..4729506 100644
--- a/stats/lib/stats-fetchers.js
+++ b/stats/lib/stats-fetchers.js
@@ -1,8 +1,3 @@
-/**
- *
- * @param { string } value
- */
-const isValidNumber = value => value != null && !isNaN(+value)
 /** @typedef {import('@filecoin-station/spark-stats-db').PgPools} PgPools */
 /**
  * @param {PgPools} pgPools
@@ -12,7 +7,11 @@ export const fetchRetrievalSuccessRate = async (pgPools, filter) => {
   // Fetch the "day" (DATE) as a string (TEXT) to prevent node-postgres for converting it into
   // a JavaScript Date with a timezone, as that could change the date one day forward or back.
   const { rows } = await pgPools.evaluate.query(`
-    SELECT day::text, SUM(total) as total, SUM(successful) as successful, SUM(successful_http) as successful_http
+    SELECT 
+    day::text, 
+    SUM(total) as total, 
+    SUM(successful) as successful, 
+    SUM(successful_http) as successful_http
     FROM retrieval_stats
     WHERE day >= $1 AND day <= $2 ${filter.nonZero === 'true' ? 'AND successful > 0' : ''}
     GROUP BY day
@@ -28,7 +27,7 @@ export const fetchRetrievalSuccessRate = async (pgPools, filter) => {
     success_rate: r.total > 0 ? r.successful / r.total : null,
     successful_http: r.successful_http ?? null,
     // successful_http might be null because the column was added later
-    success_rate_http: r.total > 0 && isValidNumber(r.successful_http) ? r.successful_http / r.total : null
+    success_rate_http: r.total > 0 && r.successful_http !== null ? r.successful_http / r.total : null
   }))
   return stats
 }
@@ -215,7 +214,11 @@ export const fetchParticipantRewardTransfers = async (pgPools, { from, to }, add
  */
 export const fetchMinersRSRSummary = async (pgPools, filter) => {
   const { rows } = await pgPools.evaluate.query(`
-    SELECT miner_id, SUM(total) as total, SUM(successful) as successful, SUM(successful_http) as successful_http
+    SELECT 
+    miner_id, 
+    SUM(total) as total, 
+    SUM(successful) as successful, 
+    SUM(successful_http) as successful_http
     FROM retrieval_stats
     WHERE day >= $1 AND day <= $2
     GROUP BY miner_id
@@ -230,7 +233,7 @@ export const fetchMinersRSRSummary = async (pgPools, filter) => {
     success_rate: r.total > 0 ? r.successful / r.total : null,
     successful_http: r.successful_http ?? null,
     // successful_http might be null because the column was added later
-    success_rate_http: r.total > 0 && isValidNumber(r.successful_http) ? r.successful_http / r.total : null
+    success_rate_http: r.total > 0 && r.successful_http !== null ? r.successful_http / r.total : null
   }))
   return stats
 }
@@ -243,7 +246,10 @@ export const fetchMinersRSRSummary = async (pgPools, filter) => {
  */
 export const fetchDailyMinerRSRSummary = async (pgPools, { from, to }, minerId) => {
   const { rows } = await pgPools.evaluate.query(`
-    SELECT day::TEXT, SUM(total) as total, SUM(successful) as successful, SUM(successful_http) as successful_http
+    SELECT 
+    day::TEXT, 
+    SUM(total) as total, SUM(successful) as successful, 
+    SUM(successful_http) as successful_http
     FROM retrieval_stats
     WHERE miner_id = $1 AND day >= $2 AND day <= $3
     GROUP BY day
@@ -260,7 +266,7 @@ export const fetchDailyMinerRSRSummary = async (pgPools, { from, to }, minerId)
     success_rate: r.total > 0 ? r.successful / r.total : null,
     successful_http: r.successful_http ?? null,
     // successful_http might be null because the column was added later
-    success_rate_http: r.total > 0 && isValidNumber(r.successful_http) ? r.successful_http / r.total : null
+    success_rate_http: r.total > 0 && r.successful_http !== null ? r.successful_http / r.total : null
   }))
   return stats
 }

From 51a0fefee382c858c034d5ba4357f68e064368c9 Mon Sep 17 00:00:00 2001
From: Nikolas Haimerl <nikolas.haimerl@dfinity.org>
Date: Fri, 10 Jan 2025 10:06:22 +0100
Subject: [PATCH 08/12] add miner test

---
 stats/test/handler.test.js | 33 +++++++++++++++++++++++++++------
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/stats/test/handler.test.js b/stats/test/handler.test.js
index fbaa009..5b852d4 100644
--- a/stats/test/handler.test.js
+++ b/stats/test/handler.test.js
@@ -224,11 +224,14 @@ describe('HTTP request handler', () => {
       ])
     })
     it('handles successful_http values 0, null, undefined', async () => {
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', total: 10, successful: 1, successfulHttp: 0 })
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-21', total: 10, successful: 1, successfulHttp: undefined })
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-22', total: 10, successful: 1, successfulHttp: null })
-
-      const res = await fetch(
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', minerId: 'f1one', total: 10, successful: 1, successfulHttp: 0 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-21', minerId: 'f1one', total: 10, successful: 1, successfulHttp: undefined })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-22', minerId: 'f1one', total: 10, successful: 1, successfulHttp: null })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-23', minerId: 'f2two', total: 10, successful: 1, successfulHttp: undefined })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-24', minerId: 'f3three', total: 10, successful: 1, successfulHttp: null })
+
+      // First we test each case separately 0, undefined and null
+      let res = await fetch(
         new URL(
           '/retrieval-success-rate?from=2024-01-20&to=2024-01-22',
           baseUrl
@@ -237,12 +240,30 @@ describe('HTTP request handler', () => {
         }
       )
       await assertResponseStatus(res, 200)
-      const stats = await res.json()
+      let stats = await res.json()
       assert.deepStrictEqual(stats, [
         { day: '2024-01-20', success_rate: 0.1, successful: '1', total: '10', successful_http: '0', success_rate_http: 0 },
         { day: '2024-01-21', success_rate: 0.1, successful: '1', total: '10', successful_http: null, success_rate_http: null },
         { day: '2024-01-22', success_rate: 0.1, successful: '1', total: '10', successful_http: null, success_rate_http: null }
       ])
+
+      res = await fetch(
+        new URL(
+          '/miners/retrieval-success-rate/summary?from=2024-01-20&to=2024-01-25',
+          baseUrl
+        ), {
+          redirect: 'manual'
+        }
+      )
+      await assertResponseStatus(res, 200)
+      stats = await res.json()
+      console.log(JSON.stringify(stats))
+      assert.deepStrictEqual(stats, [
+        { miner_id: 'f2two', total: '10', successful: '1', success_rate: 0.1, successful_http: null, success_rate_http: null },
+        // If there is a single number we expect any undefined or null values to be converted to 0 by Postgres
+        { miner_id: 'f1one', total: '30', successful: '3', success_rate: 0.1, successful_http: '0', success_rate_http: 0 },
+        { miner_id: 'f3three', total: '10', successful: '1', success_rate: 0.1, successful_http: null, success_rate_http: null }]
+      )
     })
   })
 

From 975ad1d76c1ff0dcf6a242f304dcd6362b6d125d Mon Sep 17 00:00:00 2001
From: Nikolas Haimerl <nikolas.haimerl@dfinity.org>
Date: Fri, 10 Jan 2025 10:20:57 +0100
Subject: [PATCH 09/12] miner stats unique test

---
 stats/test/handler.test.js | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/stats/test/handler.test.js b/stats/test/handler.test.js
index 5b852d4..7bb5a50 100644
--- a/stats/test/handler.test.js
+++ b/stats/test/handler.test.js
@@ -228,7 +228,7 @@ describe('HTTP request handler', () => {
       await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-21', minerId: 'f1one', total: 10, successful: 1, successfulHttp: undefined })
       await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-22', minerId: 'f1one', total: 10, successful: 1, successfulHttp: null })
       await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-23', minerId: 'f2two', total: 10, successful: 1, successfulHttp: undefined })
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-24', minerId: 'f3three', total: 10, successful: 1, successfulHttp: null })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-24', minerId: 'f3three', total: 20, successful: 2, successfulHttp: null })
 
       // First we test each case separately 0, undefined and null
       let res = await fetch(
@@ -249,7 +249,7 @@ describe('HTTP request handler', () => {
 
       res = await fetch(
         new URL(
-          '/miners/retrieval-success-rate/summary?from=2024-01-20&to=2024-01-25',
+          '/miners/retrieval-success-rate/summary?from=2024-01-20&to=2024-01-22',
           baseUrl
         ), {
           redirect: 'manual'
@@ -257,12 +257,25 @@ describe('HTTP request handler', () => {
       )
       await assertResponseStatus(res, 200)
       stats = await res.json()
-      console.log(JSON.stringify(stats))
       assert.deepStrictEqual(stats, [
-        { miner_id: 'f2two', total: '10', successful: '1', success_rate: 0.1, successful_http: null, success_rate_http: null },
         // If there is a single number we expect any undefined or null values to be converted to 0 by Postgres
-        { miner_id: 'f1one', total: '30', successful: '3', success_rate: 0.1, successful_http: '0', success_rate_http: 0 },
-        { miner_id: 'f3three', total: '10', successful: '1', success_rate: 0.1, successful_http: null, success_rate_http: null }]
+        { miner_id: 'f1one', total: '30', successful: '3', success_rate: 0.1, successful_http: '0', success_rate_http: 0 }
+      ])
+
+      res = await fetch(
+        new URL(
+          '/miners/retrieval-success-rate/summary?from=2024-01-23&to=2024-01-24',
+          baseUrl
+        ), {
+          redirect: 'manual'
+        }
+      )
+      await assertResponseStatus(res, 200)
+      stats = await res.json()
+      assert.deepStrictEqual(stats, [
+        { miner_id: 'f2two', total: '10', successful: '1', success_rate: 0.1, successful_http: null, success_rate_http: null },
+        { miner_id: 'f3three', total: '20', successful: '2', success_rate: 0.1, successful_http: null, success_rate_http: null }
+      ]
       )
     })
   })

From f4918ee55e46138df3824a250d3a04ae100e6b2b Mon Sep 17 00:00:00 2001
From: Nikolas Haimerl <nikolas.haimerl@dfinity.org>
Date: Fri, 10 Jan 2025 10:30:15 +0100
Subject: [PATCH 10/12] formatting

---
 stats/test/handler.test.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/stats/test/handler.test.js b/stats/test/handler.test.js
index 7bb5a50..c17985f 100644
--- a/stats/test/handler.test.js
+++ b/stats/test/handler.test.js
@@ -230,7 +230,6 @@ describe('HTTP request handler', () => {
       await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-23', minerId: 'f2two', total: 10, successful: 1, successfulHttp: undefined })
       await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-24', minerId: 'f3three', total: 20, successful: 2, successfulHttp: null })
 
-      // First we test each case separately 0, undefined and null
       let res = await fetch(
         new URL(
           '/retrieval-success-rate?from=2024-01-20&to=2024-01-22',

From 677e36dd2d2e72b0d01a6ec2f5da8c69766916fa Mon Sep 17 00:00:00 2001
From: Nikolas Haimerl <nikolas.haimerl@dfinity.org>
Date: Fri, 10 Jan 2025 11:19:19 +0100
Subject: [PATCH 11/12] separated miner from summary test

---
 stats/test/handler.test.js | 81 ++++++++++++++++++++------------------
 1 file changed, 43 insertions(+), 38 deletions(-)

diff --git a/stats/test/handler.test.js b/stats/test/handler.test.js
index c17985f..8d77abd 100644
--- a/stats/test/handler.test.js
+++ b/stats/test/handler.test.js
@@ -224,13 +224,11 @@ describe('HTTP request handler', () => {
       ])
     })
     it('handles successful_http values 0, null, undefined', async () => {
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', minerId: 'f1one', total: 10, successful: 1, successfulHttp: 0 })
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-21', minerId: 'f1one', total: 10, successful: 1, successfulHttp: undefined })
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-22', minerId: 'f1one', total: 10, successful: 1, successfulHttp: null })
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-23', minerId: 'f2two', total: 10, successful: 1, successfulHttp: undefined })
-      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-24', minerId: 'f3three', total: 20, successful: 2, successfulHttp: null })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', total: 10, successful: 1, successfulHttp: 0 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-21', total: 10, successful: 1, successfulHttp: undefined })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-22', total: 10, successful: 1, successfulHttp: null })
 
-      let res = await fetch(
+      const res = await fetch(
         new URL(
           '/retrieval-success-rate?from=2024-01-20&to=2024-01-22',
           baseUrl
@@ -239,43 +237,12 @@ describe('HTTP request handler', () => {
         }
       )
       await assertResponseStatus(res, 200)
-      let stats = await res.json()
+      const stats = await res.json()
       assert.deepStrictEqual(stats, [
         { day: '2024-01-20', success_rate: 0.1, successful: '1', total: '10', successful_http: '0', success_rate_http: 0 },
         { day: '2024-01-21', success_rate: 0.1, successful: '1', total: '10', successful_http: null, success_rate_http: null },
         { day: '2024-01-22', success_rate: 0.1, successful: '1', total: '10', successful_http: null, success_rate_http: null }
       ])
-
-      res = await fetch(
-        new URL(
-          '/miners/retrieval-success-rate/summary?from=2024-01-20&to=2024-01-22',
-          baseUrl
-        ), {
-          redirect: 'manual'
-        }
-      )
-      await assertResponseStatus(res, 200)
-      stats = await res.json()
-      assert.deepStrictEqual(stats, [
-        // If there is a single number we expect any undefined or null values to be converted to 0 by Postgres
-        { miner_id: 'f1one', total: '30', successful: '3', success_rate: 0.1, successful_http: '0', success_rate_http: 0 }
-      ])
-
-      res = await fetch(
-        new URL(
-          '/miners/retrieval-success-rate/summary?from=2024-01-23&to=2024-01-24',
-          baseUrl
-        ), {
-          redirect: 'manual'
-        }
-      )
-      await assertResponseStatus(res, 200)
-      stats = await res.json()
-      assert.deepStrictEqual(stats, [
-        { miner_id: 'f2two', total: '10', successful: '1', success_rate: 0.1, successful_http: null, success_rate_http: null },
-        { miner_id: 'f3three', total: '20', successful: '2', success_rate: 0.1, successful_http: null, success_rate_http: null }
-      ]
-      )
     })
   })
 
@@ -518,6 +485,44 @@ describe('HTTP request handler', () => {
         { day: '2024-01-13', rates: { OK: '0.5', IPNI_500: '0.5' } }
       ])
     })
+    it('handles successful_http values 0, null, undefined', async () => {
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', minerId: 'f1one', total: 10, successful: 1, successfulHttp: 0 })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-21', minerId: 'f1one', total: 10, successful: 1, successfulHttp: undefined })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-22', minerId: 'f1one', total: 10, successful: 1, successfulHttp: null })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-23', minerId: 'f2two', total: 10, successful: 1, successfulHttp: undefined })
+      await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-24', minerId: 'f3three', total: 20, successful: 2, successfulHttp: null })
+
+      let res = await fetch(
+        new URL(
+          '/miners/retrieval-success-rate/summary?from=2024-01-20&to=2024-01-22',
+          baseUrl
+        ), {
+          redirect: 'manual'
+        }
+      )
+      await assertResponseStatus(res, 200)
+      let stats = await res.json()
+      assert.deepStrictEqual(stats, [
+        // If there is a single number we expect any undefined or null values to be converted to 0 by Postgres
+        { miner_id: 'f1one', total: '30', successful: '3', success_rate: 0.1, successful_http: '0', success_rate_http: 0 }
+      ])
+
+      res = await fetch(
+        new URL(
+          '/miners/retrieval-success-rate/summary?from=2024-01-23&to=2024-01-24',
+          baseUrl
+        ), {
+          redirect: 'manual'
+        }
+      )
+      await assertResponseStatus(res, 200)
+      stats = await res.json()
+      assert.deepStrictEqual(stats, [
+        { miner_id: 'f2two', total: '10', successful: '1', success_rate: 0.1, successful_http: null, success_rate_http: null },
+        { miner_id: 'f3three', total: '20', successful: '2', success_rate: 0.1, successful_http: null, success_rate_http: null }
+      ]
+      )
+    })
   })
 
   describe('summary of eligible deals', () => {

From c9a4d1b292f3678204e202934115036470e46888 Mon Sep 17 00:00:00 2001
From: Nikolas Haimerl <nikolas.haimerl@dfinity.org>
Date: Fri, 10 Jan 2025 11:21:13 +0100
Subject: [PATCH 12/12] move test

---
 stats/test/handler.test.js | 62 +++++++++++++++++++-------------------
 1 file changed, 31 insertions(+), 31 deletions(-)

diff --git a/stats/test/handler.test.js b/stats/test/handler.test.js
index 8d77abd..04f57ec 100644
--- a/stats/test/handler.test.js
+++ b/stats/test/handler.test.js
@@ -454,37 +454,6 @@ describe('HTTP request handler', () => {
         { miner_id: 'f1two', success_rate: 0.75, total: '200', successful: '150', successful_http: '100', success_rate_http: 100 / 200 }
       ])
     })
-  })
-
-  describe('GET /retrieval-result-codes/daily', () => {
-    it('returns daily retrieval result codes for the given date range', async () => {
-      await pgPools.stats.query(`
-        INSERT INTO daily_retrieval_result_codes
-        (day, code, rate)
-        VALUES
-        ('2024-01-11', 'OK', 0.1),
-        ('2024-01-11', 'CAR_TOO_LARGE', 0.9),
-        ('2024-01-12', 'OK', 1),
-        ('2024-01-13', 'OK', 0.5),
-        ('2024-01-13', 'IPNI_500', 0.5)
-      `)
-
-      const res = await fetch(
-        new URL(
-          '/retrieval-result-codes/daily?from=2024-01-11&to=2024-01-13',
-          baseUrl
-        ), {
-          redirect: 'manual'
-        }
-      )
-      await assertResponseStatus(res, 200)
-      const stats = await res.json()
-      assert.deepStrictEqual(stats, [
-        { day: '2024-01-11', rates: { OK: '0.1', CAR_TOO_LARGE: '0.9' } },
-        { day: '2024-01-12', rates: { OK: '1' } },
-        { day: '2024-01-13', rates: { OK: '0.5', IPNI_500: '0.5' } }
-      ])
-    })
     it('handles successful_http values 0, null, undefined', async () => {
       await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-20', minerId: 'f1one', total: 10, successful: 1, successfulHttp: 0 })
       await givenRetrievalStats(pgPools.evaluate, { day: '2024-01-21', minerId: 'f1one', total: 10, successful: 1, successfulHttp: undefined })
@@ -525,6 +494,37 @@ describe('HTTP request handler', () => {
     })
   })
 
+  describe('GET /retrieval-result-codes/daily', () => {
+    it('returns daily retrieval result codes for the given date range', async () => {
+      await pgPools.stats.query(`
+        INSERT INTO daily_retrieval_result_codes
+        (day, code, rate)
+        VALUES
+        ('2024-01-11', 'OK', 0.1),
+        ('2024-01-11', 'CAR_TOO_LARGE', 0.9),
+        ('2024-01-12', 'OK', 1),
+        ('2024-01-13', 'OK', 0.5),
+        ('2024-01-13', 'IPNI_500', 0.5)
+      `)
+
+      const res = await fetch(
+        new URL(
+          '/retrieval-result-codes/daily?from=2024-01-11&to=2024-01-13',
+          baseUrl
+        ), {
+          redirect: 'manual'
+        }
+      )
+      await assertResponseStatus(res, 200)
+      const stats = await res.json()
+      assert.deepStrictEqual(stats, [
+        { day: '2024-01-11', rates: { OK: '0.1', CAR_TOO_LARGE: '0.9' } },
+        { day: '2024-01-12', rates: { OK: '1' } },
+        { day: '2024-01-13', rates: { OK: '0.5', IPNI_500: '0.5' } }
+      ])
+    })
+  })
+
   describe('summary of eligible deals', () => {
     describe('GET /miner/{id}/deals/eligible/summary', () => {
       it('redirects to spark-api', async () => {