diff --git a/README.md b/README.md index 027cf10bd..88e64468d 100644 --- a/README.md +++ b/README.md @@ -152,14 +152,6 @@ sequence by running this SQL command after the insert: Otherwise, you will get a unique constraint error when inserting via the API because Sequelize will try use an id that is now taken. -## Adding new image types - -Steps to add a new image type: -1. Open file app/routes/report/images.js -2. Add new object to IMAGES_CONFIG. Pattern will be the regex for the image key being uploaded -3. Open file app/constants.js -4. Add new entry to VALID_IMAGE_KEY_PATTERN based on the IMAGES_CONFIG pattern - ## Process Manager The production installation of IPR is run & managed by [pm2](http://pm2.keymetrics.io/) diff --git a/app/constants.js b/app/constants.js index d34e37498..80a120362 100644 --- a/app/constants.js +++ b/app/constants.js @@ -23,26 +23,6 @@ module.exports = { 'probeResults', 'proteinVariants', ], - VALID_IMAGE_KEY_PATTERN: `^${[ - 'mutSignature\\.(corPcors|barplot)\\.(dbs|indels|sbs)', - 'subtypePlot\\.\\S+', - '(cnv|loh)\\.[12345]', - 'cnvLoh.circos', - 'mutationBurden\\.(barplot|density|legend)_(sv|snv|indel|snv_indel)\\.(primary|secondary|tertiary|quaternary)', - 'circosSv\\.(genome|transcriptome)', - 'expDensity\\.(histogram|violin\\.(tcga|gtex|cser|hartwig|pediatric))\\.\\S+', - 'expression\\.(chart|legend|spearman\\.(tcga|gtex|cser|hartwig|target|pediatric|brca\\.(molecular|receptor)))', - 'microbial\\.circos\\.(genome|transcriptome)', - 'cibersort\\.(cd8_positive|combined)_t-cell_scatter', - 'mixcr\\.circos_trb_vj_gene_usage', - 'mixcr\\.dominance_vs_alpha_beta_t-cells_scatter', - 'scpPlot', - 'msi.scatter', - 'pathwayAnalysis.legend', - 'copyNumber(.*)', - ].map((patt) => { - return `(${patt})`; - }).join('|')}$`, REPORT_CREATE_BASE_URI: '/reports/create', REPORT_UPDATE_BASE_URI: '/reports/update', GERMLINE_CREATE_BASE_URI: '/germline/create', @@ -57,4 +37,6 @@ module.exports = { MASTER_REPORT_ACCESS: ['admin', 'manager'], ALL_PROJECTS_ACCESS: ['admin', 'all projects access'], UPDATE_METHODS: ['POST', 'PUT', 'DELETE'], + IMAGE_UPLOAD_LIMIT: 3000000, + IMAGE_SIZE_LIMIT: 20000, // in bytes }; diff --git a/app/libs/createReport.js b/app/libs/createReport.js index 9682089e1..072ef2f33 100644 --- a/app/libs/createReport.js +++ b/app/libs/createReport.js @@ -4,7 +4,7 @@ const path = require('path'); const db = require('../models'); const {uploadReportImage} = require('../routes/report/images'); const logger = require('../log'); -const {GENE_LINKED_VARIANT_MODELS, KB_PIVOT_MAPPING} = require('../constants'); +const {GENE_LINKED_VARIANT_MODELS, KB_PIVOT_MAPPING, IMAGE_UPLOAD_LIMIT} = require('../constants'); const {sanitizeHtml} = require('./helperFunctions'); const EXCLUDE_SECTIONS = new Set([ @@ -418,6 +418,26 @@ const createReport = async (data) => { // Create report sections try { await createReportSections(report, data, transaction); + + const result = await db.query( + `SELECT + SUM(pg_column_size("reports_image_data")) AS avg_size_bytes + FROM + "reports_image_data" + WHERE + "report_id" = :reportId`, + { + replacements: {reportId: report.id}, + type: 'select', + transaction, + }, + ); + + const totalImageSize = result[0].avg_size_bytes; + if (totalImageSize > IMAGE_UPLOAD_LIMIT) { + throw new Error(`Total image size exceeds ${IMAGE_UPLOAD_LIMIT / 1000000} megabytes`); + } + await transaction.commit(); return report; } catch (error) { diff --git a/app/libs/image.js b/app/libs/image.js index ec46cf7aa..23d2df60b 100644 --- a/app/libs/image.js +++ b/app/libs/image.js @@ -1,5 +1,34 @@ const sharp = require('sharp'); const db = require('../models'); +const {IMAGE_SIZE_LIMIT} = require('../constants'); + +/** + * Delete image from images table + * + * @param {string} ident - Ident of image to delete + * @param {boolean} force - Whether it is a hard delete or not + * @param {object} transaction - Transaction to run delete under + * @returns {Promise} - Returns the newly updated image data + */ +const autoDownsize = async (image, size, format = 'png') => { + // Base case: If the image size is less than or equal to size, return the image + if (image.info.size <= size) { + return image; + } + + // Process the image (resize and format) and update the size + const resizedImage = await sharp(image.data) + .resize( + Math.floor(image.info.width / 1.5), + Math.floor(image.info.height / 1.5), + {fit: 'inside', withoutEnlargement: true}, + ) + .toFormat(format.toLowerCase()) + .toBuffer({resolveWithObject: true}); + + // Recursive call with the resized image + return autoDownsize(resizedImage, size, format); +}; /** * Resize, reformat and return base64 representation of image. @@ -12,13 +41,14 @@ const db = require('../models'); * @returns {Promise} - Returns base64 representation of image * @throws {Promise} - File doesn't exist, incorrect permissions, etc. */ -const processImage = async (image, width, height, format = 'png') => { - const imageData = await sharp(image) - .resize(width, height, {fit: 'inside', withoutEnlargement: true}) +const processImage = async (image, size = IMAGE_SIZE_LIMIT, format = 'png') => { + let imageData = await sharp(image) .toFormat(format.toLowerCase()) - .toBuffer(); + .toBuffer({resolveWithObject: true}); + + imageData = await autoDownsize(imageData, size, format); - return imageData.toString('base64'); + return imageData.data.toString('base64'); }; /** @@ -37,11 +67,11 @@ const processImage = async (image, width, height, format = 'png') => { */ const uploadImage = async (image, options = {}) => { const { - data, width, height, filename, format, type, + data, filename, format, type, } = image; // Resize image - const imageData = await processImage(data, width, height, format.replace('image/', '')); + const imageData = await processImage(data, IMAGE_SIZE_LIMIT, format.replace('image/', '')); // Upload image data return db.models.image.create({ diff --git a/app/routes/report/image/index.js b/app/routes/report/image/index.js index 108b620ad..72c9525fa 100644 --- a/app/routes/report/image/index.js +++ b/app/routes/report/image/index.js @@ -5,7 +5,6 @@ const {Op} = require('sequelize'); const db = require('../../../models'); const logger = require('../../../log'); const {uploadReportImage} = require('../images'); -const {VALID_IMAGE_KEY_PATTERN} = require('../../../constants'); const router = express.Router({mergeParams: true}); @@ -62,19 +61,11 @@ router.route('/') return res.status(HTTP_STATUS.BAD_REQUEST).json({error: {message: 'No attached images to upload'}}); } - // Check for valid and duplicate keys const keys = []; - const pattern = new RegExp(VALID_IMAGE_KEY_PATTERN); for (let [key, value] of Object.entries(req.files)) { key = key.trim(); - // Check if key is valid - if (!pattern.test(key)) { - logger.error(`Invalid key: ${key}`); - return res.status(HTTP_STATUS.BAD_REQUEST).json({error: {message: `Invalid key: ${key}`}}); - } - // Check if key is a duplicate if (keys.includes(key) || Array.isArray(value)) { logger.error(`Duplicate keys are not allowed. Duplicate key: ${key}`); diff --git a/app/routes/report/images.js b/app/routes/report/images.js index c0e68b5fb..370475133 100644 --- a/app/routes/report/images.js +++ b/app/routes/report/images.js @@ -1,243 +1,10 @@ const logger = require('../../log'); const db = require('../../models'); const {processImage} = require('../../libs/image'); +const {IMAGE_SIZE_LIMIT} = require('../../constants'); -const DEFAULT_WIDTH = 500; -const DEFAULT_HEIGHT = 500; const DEFAULT_FORMAT = 'PNG'; -// takes first pattern match (order matters) -const IMAGES_CONFIG = [ - { - pattern: 'subtypePlot\\.ped_\\S+', - width: 420, - height: 900, - format: 'PNG', - }, - { - pattern: 'subtypePlot\\.\\S+', - width: 600, - height: 375, - format: 'PNG', - }, - { - pattern: '(cnv|loh)\\.[1]', - height: 166, - width: 1000, - format: 'PNG', - }, - { - pattern: '(cnv|loh)\\.[2345]', - height: 161, - width: 1000, - format: 'PNG', - }, - { - pattern: 'mutationBurden\\.(barplot|density|legend)_(sv|snv|indel|snv_indel)\\.(primary|secondary|tertiary|quaternary)', - width: 560, - height: 151, - format: 'PNG', - }, - { - pattern: 'cnvLoh.circos', - width: 1000, - height: 1000, - format: 'JPG', - }, - { - pattern: 'mutSignature.corPcors\\.(dbs|indels|sbs)', - width: 1000, - height: 2000, - format: 'JPG', - }, - { - pattern: 'mutSignature.barplot\\.sbs', - width: 480, - height: 480, - format: 'PNG', - }, - { - pattern: 'mutSignature.barplot\\.(dbs|indels)', - width: 2000, - height: 1000, - format: 'PNG', - }, - { - pattern: 'circosSv\\.(genome|transcriptome)', - width: 1001, - height: 900, - format: 'PNG', - }, - { - pattern: 'expDensity.histogram.\\S+', - width: 450, - height: 450, - format: 'PNG', - }, - { - pattern: 'expDensity.violin.\\S+', - width: 825, - height: 1965, - format: 'PNG', - }, - { - pattern: 'expression\\.chart', - width: 800, - height: 1500, - format: 'PNG', - }, - { - pattern: 'expression\\.legend', - width: 800, - height: 400, - format: 'PNG', - }, - { - pattern: 'microbial\\.circos\\.(genome|transcriptome)', - width: 900, - height: 900, - format: 'PNG', - }, - { - pattern: 'cibersort\\.(cd8_positive|combined)_t-cell_scatter', - width: 1020, - height: 1020, - format: 'PNG', - }, - { - pattern: 'mixcr\\.circos_trb_vj_gene_usage', - width: 1000, - height: 1000, - format: 'PNG', - }, - { - pattern: 'mixcr\\.dominance_vs_alpha_beta_t-cells_scatter', - width: 640, - height: 480, - format: 'PNG', - }, - { - pattern: 'scpPlot', - width: 1400, - height: 900, - format: 'PNG', - }, - { - pattern: 'msi.scatter', - width: 1000, - height: 1000, - format: 'PNG', - }, - { - pattern: 'pathwayAnalysis.legend', - width: 990, - height: 765, - format: 'PNG', - }, - { - pattern: 'expression.spearman.tcga', - width: 1100, - height: 3000, - format: 'PNG', - }, - { - pattern: 'expression.spearman.gtex', - width: 1100, - height: 4050, - format: 'PNG', - }, - { - pattern: 'expression.spearman.cser', - width: 1100, - height: 1125, - format: 'PNG', - }, - { - pattern: 'expression.spearman.hartwig', - width: 1100, - height: 1500, - format: 'PNG', - }, - { - pattern: 'expression.spearman.pediatric', - width: 1100, - height: 2775, - format: 'PNG', - }, - { - pattern: 'expression.spearman.target', - width: 640, - height: 480, - format: 'PNG', - }, - { - pattern: 'expression.spearman.brca\\.(molecular|receptor)', - width: 1100, - height: 450, - format: 'PNG', - }, - { - pattern: 'copyNumberChr(1|2)', - width: 1625, - height: 875, - format: 'PNG', - }, - { - pattern: 'copyNumberChr(3|4|5)', - width: 1300, - height: 875, - format: 'PNG', - }, - { - pattern: 'copyNumberChr(6|7|X)', - width: 1150, - height: 875, - format: 'PNG', - }, - { - pattern: 'copyNumberChr(8|9)', - width: 1100, - height: 875, - format: 'PNG', - }, - { - pattern: 'copyNumberChr(10|11|12|13|14)', - width: 1000, - height: 1000, - format: 'PNG', - }, - { - pattern: 'copyNumberChr15', - width: 1000, - height: 1150, - format: 'PNG', - }, - { - pattern: 'copyNumberChr(16|17|18)', - width: 900, - height: 1150, - format: 'PNG', - }, - { - pattern: 'copyNumberChr(19|20|Y)', - width: 700, - height: 1150, - format: 'PNG', - }, - { - pattern: 'copyNumberChr(21|22)', - width: 550, - height: 1150, - format: 'PNG', - }, - { - pattern: 'copyNumberLegend', - width: 300, - height: 1150, - format: 'PNG', - }, -]; - /** * Resize, reformat and upload a report image to the reports_image_data table * @@ -257,22 +24,10 @@ const IMAGES_CONFIG = [ const uploadReportImage = async (reportId, key, image, options = {}) => { logger.verbose(`Loading (${key}) image`); - let config; - for (const {pattern, ...conf} of IMAGES_CONFIG) { - const regexp = new RegExp(`^${pattern}$`); - if (regexp.exec(key)) { - config = conf; - break; - } - } - - if (!config) { - logger.warn(`No format/size configuration for ${key}. Using default values`); - config = {format: DEFAULT_FORMAT, height: DEFAULT_HEIGHT, width: DEFAULT_WIDTH}; - } + const config = {format: DEFAULT_FORMAT, size: IMAGE_SIZE_LIMIT}; try { - const imageData = await processImage(image, config.width, config.height, config.format); + const imageData = await processImage(image, config.size, config.format); return db.models.imageData.create({ reportId, diff --git a/app/routes/template/template.js b/app/routes/template/template.js index 0b46ac30d..8971f33a0 100644 --- a/app/routes/template/template.js +++ b/app/routes/template/template.js @@ -72,8 +72,6 @@ router.route('/:template([A-z0-9-]{36})') data: logo.data, filename: logo.name, format: logo.mimetype, - height: DEFAULT_LOGO_HEIGHT, - width: DEFAULT_LOGO_WIDTH, type: 'Logo', }, {transaction}), ); @@ -87,8 +85,6 @@ router.route('/:template([A-z0-9-]{36})') data: header.data, filename: header.name, format: header.mimetype, - height: DEFAULT_HEADER_HEIGHT, - width: DEFAULT_HEADER_WIDTH, type: 'Header', }, {transaction}), ); diff --git a/app/schemas/report/reportUpload/index.js b/app/schemas/report/reportUpload/index.js index 7c0d455db..fae16b02a 100644 --- a/app/schemas/report/reportUpload/index.js +++ b/app/schemas/report/reportUpload/index.js @@ -3,7 +3,7 @@ const {BASE_EXCLUDE} = require('../../exclude'); const variantSchemas = require('./variant'); const kbMatchesSchema = require('./kbMatches'); const schemaGenerator = require('../../schemaGenerator'); -const {VALID_IMAGE_KEY_PATTERN, UPLOAD_BASE_URI} = require('../../../constants'); +const {UPLOAD_BASE_URI} = require('../../../constants'); /** * Generate schema for uploading a report @@ -40,7 +40,6 @@ const generateReportUploadSchema = (isJsonSchema) => { }, key: { type: 'string', - pattern: VALID_IMAGE_KEY_PATTERN, }, }, }, diff --git a/test/routes/report/image/index.test.js b/test/routes/report/image/index.test.js index be0e99595..824c43878 100644 --- a/test/routes/report/image/index.test.js +++ b/test/routes/report/image/index.test.js @@ -260,19 +260,6 @@ describe('/reports/{REPORTID}/image', () => { })); }); - test('POST / - 400 Bad Request invalid key', async () => { - const res = await request - .post(`/api/reports/${report.ident}/image`) - .attach('INVALID_KEY', 'test/testData/images/golden.jpg') - .auth(username, password) - .expect(HTTP_STATUS.BAD_REQUEST); - - // Check invalid key error - expect(res.body.error).toEqual(expect.objectContaining({ - message: 'Invalid key: INVALID_KEY', - })); - }); - test('POST / - 400 Bad Request duplicate key', async () => { const res = await request .post(`/api/reports/${report.ident}/image`)