diff --git a/package-lock.json b/package-lock.json index 6ac8c94..b20f7b6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,7 +22,8 @@ "dotenv": "^16.4.5", "fastify": "^4.26.1", "pg": "^8.11.3", - "query-string": "^7.1.3" + "query-string": "^7.1.3", + "redis": "^4.6.13" } }, "node_modules/@fastify/accept-negotiator": { @@ -215,6 +216,59 @@ "node": ">=14" } }, + "node_modules/@redis/bloom": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz", + "integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/client": { + "version": "1.5.14", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.5.14.tgz", + "integrity": "sha512-YGn0GqsRBFUQxklhY7v562VMOP0DcmlrHHs3IV1mFE3cbxe31IITUkqhBcIhVSI/2JqtWAJXg5mjV4aU+zD0HA==", + "dependencies": { + "cluster-key-slot": "1.1.2", + "generic-pool": "3.9.0", + "yallist": "4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@redis/graph": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz", + "integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/json": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.6.tgz", + "integrity": "sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/search": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.6.tgz", + "integrity": "sha512-mZXCxbTYKBQ3M2lZnEddwEAks0Kc7nauire8q20oA0oA/LoA+E/b5Y5KZn232ztPb1FkIGqo12vh3Lf+Vw5iTw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/time-series": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.5.tgz", + "integrity": "sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, "node_modules/abort-controller": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", @@ -488,6 +542,14 @@ "node": ">=0.8" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -1012,6 +1074,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/generic-pool": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz", + "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==", + "engines": { + "node": ">= 4" + } + }, "node_modules/get-intrinsic": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", @@ -1802,6 +1872,19 @@ "node": ">= 12.13.0" } }, + "node_modules/redis": { + "version": "4.6.13", + "resolved": "https://registry.npmjs.org/redis/-/redis-4.6.13.tgz", + "integrity": "sha512-MHgkS4B+sPjCXpf+HfdetBwbRz6vCtsceTmw1pHNYJAsYxrfpOP6dz+piJWGos8wqG7qb3vj/Rrc5qOlmInUuA==", + "dependencies": { + "@redis/bloom": "1.2.0", + "@redis/client": "1.5.14", + "@redis/graph": "1.1.1", + "@redis/json": "1.0.6", + "@redis/search": "1.1.6", + "@redis/time-series": "1.0.5" + } + }, "node_modules/require-from-string": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", diff --git a/package.json b/package.json index 647f21f..a0db13f 100755 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "dotenv": "^16.4.5", "fastify": "^4.26.1", "pg": "^8.11.3", - "query-string": "^7.1.3" + "query-string": "^7.1.3", + "redis": "^4.6.13" } } diff --git a/routes/mvt.js b/routes/mvt.js index 2871d65..9417490 100755 --- a/routes/mvt.js +++ b/routes/mvt.js @@ -1,6 +1,9 @@ // route query require("dotenv").config() +const { createClient } = require('redis'); +const crypto = require('crypto'); + const sql = (params, query) => { return ` WITH mvtgeom2 as ( @@ -30,7 +33,7 @@ const sql = (params, query) => { ) transformed_geom -- Add where clause only when filtering by bounds - ${params.z == 0 && params.x == 0 && params.y == 0 ? ` + ${String(params.z) == '0' && String(params.x) == '0' && String(params.y) == '0' ? ` WHERE ST_Intersects( ${process.env.TABLE_COLUMN}, @@ -92,6 +95,40 @@ const schema = { } } +const initializeRedis = async () => { + const client = createClient(); + + client.on('error', err => console.log('Redis Client Error', err)); + + return client.connect(); +} + +// cache results for at least 2 minutes +const cacheResults = async (params, results) => { + client = await initializeRedis() + client.set(params, results) +} + +const getCachedResults = async (cacheKey) => { + client = await initializeRedis() + return await client.get(cacheKey) +} + + +// Function to merge two objects and generate a hash +function mergeAndHash(params, query) { + const mergedObject = { ...params, ...query }; + const jsonString = JSON.stringify(mergedObject); + const hash = crypto.createHash('sha256').update(jsonString).digest('hex'); + + return hash; +} + +function arrayBufferToBase64(buffer) { + return Buffer.from(buffer).toString('base64'); +} + + // create route module.exports = function (fastify, opts, next) { fastify.route({ @@ -99,7 +136,21 @@ module.exports = function (fastify, opts, next) { url: '/mvt/:z/:x/:y', schema: schema, handler: function (request, reply) { - fastify.pg.connect(onConnect) + // check redis to see if we have cached something already + cacheKey = mergeAndHash(request.params, request.query) + getCachedResults(cacheKey).then(( + cacheResult + )=> { + if (cacheResult == null) { + fastify.pg.connect(onConnect) + } else { + reply.header('Content-Type', 'application/x-protobuf').send(Buffer.from(cacheResults, 'base64').buffer) + } + } + ).catch((error) => { + console.log(error) + } ) + function onConnect(err, client, release) { if (err) { @@ -116,6 +167,7 @@ module.exports = function (fastify, opts, next) { reply.send(err) } else { const mvt = result.rows[0].mvt + cacheResults(cacheKey, arrayBufferToBase64(mvt)) if (mvt.length === 0) { reply.code(204).send() }