From 3b3a5a512733c84c4b1a35770224c37ff2e07394 Mon Sep 17 00:00:00 2001 From: Pierre Ozoux Date: Tue, 9 Apr 2024 18:42:41 +0200 Subject: [PATCH 1/7] feat: add file upload --- src/handlers/messages.test.ts | 1 + src/handlers/messages.ts | 105 +++++++++++++++++++++++++++++++++- 2 files changed, 104 insertions(+), 2 deletions(-) diff --git a/src/handlers/messages.test.ts b/src/handlers/messages.test.ts index 398dd82..ca61ecb 100644 --- a/src/handlers/messages.test.ts +++ b/src/handlers/messages.test.ts @@ -32,6 +32,7 @@ const rcMessage: RcMessage = { ts: { $date: '1970-01-02T06:51:51.0Z', // UNIX-TS: 111111000 }, + type: 'm.text', } const matrixMessage: MatrixMessage = { diff --git a/src/handlers/messages.ts b/src/handlers/messages.ts index 57d24ab..5e7a61b 100644 --- a/src/handlers/messages.ts +++ b/src/handlers/messages.ts @@ -8,6 +8,7 @@ import { getMessageId, getRoomId, getUserId, + getAccessToken, getUserMappingByName, save, } from '../helpers/storage' @@ -18,6 +19,7 @@ import { } from '../helpers/synapse' import emojiMap from '../emojis.json' import { executeAndHandleMissingMember } from './rooms' +import * as fs from 'fs' const applicationServiceToken = process.env.AS_TOKEN || '' if (!applicationServiceToken) { @@ -26,6 +28,16 @@ if (!applicationServiceToken) { throw new Error(message) } +type attachment = { + type?: string + description?: string + message_link?: string + image_url?: string + image_type?: string + title: string + title_link?: string +} + /** * Type of Rocket.Chat messages */ @@ -34,6 +46,14 @@ export type RcMessage = { t?: string // Event type rid: string // The unique id for the room msg: string // The content of the message. + attachments?: attachment[] + file?: { + _id: string + name: string + type: string + url: string + } + type: string tmid?: string ts: { $date: string @@ -81,6 +101,7 @@ export type MatrixMessage = { event_id: string } } + url?: string } /** @@ -132,7 +153,7 @@ export async function mapTextMessage( const htmled = converter.makeHtml(emojified) const matrixMessage: MatrixMessage = { type: 'm.room.message', - msgtype: 'm.text', + msgtype: rcMessage.type, body: emojified, } if (mentions && (mentions.room || mentions.user_ids)) { @@ -204,6 +225,41 @@ export async function createMessage( ).data.event_id } +/** + * Send a File to Synapse + * @param user_id The user the media will be posted by + * @param ts The timestamp to which the file will be dated + * @param filePath the path on the local filesystem + * @param fileName the filename + * @param content_type: Content type of the file + * @returns The Matrix Message/event ID + */ +export async function uploadFile( + user_id: string, + ts: number, + filePath: string, + fileName: string, + content_type: string +): Promise { + const fileStream = fs.createReadStream(filePath) + const accessToken = await getAccessToken(user_id) + log.http(`Uploading ${fileName}...`) + + return ( + await axios.post( + `/_matrix/media/v3/upload?user_id=${user_id}&ts=${ts}&filename=${fileName}`, + fileStream, + { + headers: { + 'Content-Type': content_type, + 'Content-Length': fs.statSync(filePath).size, + Authorization: `Bearer ${accessToken}`, + }, + } + ) + ).data.content_uri +} + /** * Add reactions to the event * @param reactions A Rocket.Chat reactions object @@ -324,6 +380,49 @@ export async function handle(rcMessage: RcMessage): Promise { return } + const ts = new Date(rcMessage.ts.$date).valueOf() + if (rcMessage.file) { + if (rcMessage.attachments?.length == 1) { + const path = './inputs/files/' + rcMessage.file._id + if (!fs.existsSync(path)) { + log.warn(`File doesn't exist locally, skipping Upload.`) + return + } + const mxcurl = await uploadFile( + rcMessage.u._id, + ts, + path, + rcMessage.file.name, + rcMessage.file.type + ) + rcMessage.msg = rcMessage.file.name + rcMessage.file.url = mxcurl + if (rcMessage.attachments[0].image_type) { + rcMessage.type = 'm.image' + } else { + rcMessage.type = 'm.file' + } + } else { + log.warn( + `Many attachments in ${rcMessage.u._id} not handled, skipping Upload.` + ) + return + } + } else if (rcMessage.attachments && rcMessage.attachments.length > 0) { + log.warn(`Attachment in ${rcMessage.u._id} not handled, skipping.`) + return + } else { + rcMessage.type = 'm.text' + } + + await handleMessage(rcMessage, room_id, ts) +} + +async function handleMessage( + rcMessage: RcMessage, + room_id: string, + ts: number +) { const user_id = await getUserId(rcMessage.u._id) if (!user_id) { log.warn( @@ -332,7 +431,9 @@ export async function handle(rcMessage: RcMessage): Promise { return } const matrixMessage = await mapMessage(rcMessage) - const ts = new Date(rcMessage.ts.$date).valueOf() + if (rcMessage.file) { + matrixMessage.url = rcMessage.file.url + } if (rcMessage.tmid) { const event_id = await getMessageId(rcMessage.tmid) From 7267b4b402305a9f733d23f09a9aa4f27f27837b Mon Sep 17 00:00:00 2001 From: Tunui Franken Date: Tue, 23 Jul 2024 14:53:16 +0200 Subject: [PATCH 2/7] Also remove media storage when resetting --- reset.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/reset.sh b/reset.sh index d893cc4..ec60345 100755 --- a/reset.sh +++ b/reset.sh @@ -24,6 +24,7 @@ set -u echo 'Resetting containers and databases' docker compose down sudo rm -f files/homeserver.db +sudo rm -rf files/media_store/local_{content,thumbnails} rm -f db.sqlite docker compose up -d From 6697c08da4d737cb4122004d6d677e509f484fb8 Mon Sep 17 00:00:00 2001 From: Tunui Franken Date: Wed, 24 Jul 2024 10:13:22 +0200 Subject: [PATCH 3/7] Send attachment descriptions as separate text message --- src/handlers/messages.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/handlers/messages.ts b/src/handlers/messages.ts index 5e7a61b..26866fb 100644 --- a/src/handlers/messages.ts +++ b/src/handlers/messages.ts @@ -395,6 +395,15 @@ export async function handle(rcMessage: RcMessage): Promise { rcMessage.file.name, rcMessage.file.type ) + if (rcMessage.attachments[0].description) { + // send the description as a separate text message + const saved_id = rcMessage._id + rcMessage._id = rcMessage.file._id + rcMessage.msg = rcMessage.attachments[0].description + rcMessage.type = 'm.text' + await handleMessage(rcMessage, room_id, ts) + rcMessage._id = saved_id + } rcMessage.msg = rcMessage.file.name rcMessage.file.url = mxcurl if (rcMessage.attachments[0].image_type) { From 5951071ad44daafb75f4a14b3f81476ce1f578a0 Mon Sep 17 00:00:00 2001 From: Tunui Franken Date: Mon, 29 Jul 2024 15:32:52 +0200 Subject: [PATCH 4/7] Add inputs/files/.gitkeep --- inputs/files/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 inputs/files/.gitkeep diff --git a/inputs/files/.gitkeep b/inputs/files/.gitkeep new file mode 100644 index 0000000..e69de29 From 7ce31fb48d7cad63db2e25e672322bb6efe2cdb5 Mon Sep 17 00:00:00 2001 From: Tunui Franken Date: Mon, 29 Jul 2024 17:24:59 +0200 Subject: [PATCH 5/7] Use async fs/promises API and handle file access errors in uploadFile --- src/handlers/messages.ts | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/src/handlers/messages.ts b/src/handlers/messages.ts index 26866fb..e608a47 100644 --- a/src/handlers/messages.ts +++ b/src/handlers/messages.ts @@ -19,7 +19,7 @@ import { } from '../helpers/synapse' import emojiMap from '../emojis.json' import { executeAndHandleMissingMember } from './rooms' -import * as fs from 'fs' +import fs from 'fs/promises' const applicationServiceToken = process.env.AS_TOKEN || '' if (!applicationServiceToken) { @@ -241,10 +241,18 @@ export async function uploadFile( fileName: string, content_type: string ): Promise { - const fileStream = fs.createReadStream(filePath) const accessToken = await getAccessToken(user_id) log.http(`Uploading ${fileName}...`) + let fd: fs.FileHandle | undefined + try { + fd = await fs.open(filePath) + } catch (err) { + log.warn(`Unable to open ${filePath}:`, err) + throw err + } + const fileStream = fd.createReadStream() + return ( await axios.post( `/_matrix/media/v3/upload?user_id=${user_id}&ts=${ts}&filename=${fileName}`, @@ -252,7 +260,7 @@ export async function uploadFile( { headers: { 'Content-Type': content_type, - 'Content-Length': fs.statSync(filePath).size, + 'Content-Length': (await fd.stat()).size, Authorization: `Bearer ${accessToken}`, }, } @@ -384,17 +392,19 @@ export async function handle(rcMessage: RcMessage): Promise { if (rcMessage.file) { if (rcMessage.attachments?.length == 1) { const path = './inputs/files/' + rcMessage.file._id - if (!fs.existsSync(path)) { - log.warn(`File doesn't exist locally, skipping Upload.`) + let mxcurl: string + try { + mxcurl = await uploadFile( + rcMessage.u._id, + ts, + path, + rcMessage.file.name, + rcMessage.file.type + ) + } catch (err) { + log.warn(`Error uploading file ${path}, skipping Upload.`) return } - const mxcurl = await uploadFile( - rcMessage.u._id, - ts, - path, - rcMessage.file.name, - rcMessage.file.type - ) if (rcMessage.attachments[0].description) { // send the description as a separate text message const saved_id = rcMessage._id From 1669df6df8184dde9077901072c572aa1d60366d Mon Sep 17 00:00:00 2001 From: Tunui Franken Date: Tue, 30 Jul 2024 10:18:41 +0200 Subject: [PATCH 6/7] uploadFile: move all code in try/catch and check for error types --- src/handlers/messages.ts | 42 ++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/src/handlers/messages.ts b/src/handlers/messages.ts index e608a47..d2e10d3 100644 --- a/src/handlers/messages.ts +++ b/src/handlers/messages.ts @@ -244,28 +244,32 @@ export async function uploadFile( const accessToken = await getAccessToken(user_id) log.http(`Uploading ${fileName}...`) - let fd: fs.FileHandle | undefined try { - fd = await fs.open(filePath) - } catch (err) { - log.warn(`Unable to open ${filePath}:`, err) + const fd = await fs.open(filePath) + const fileStream = fd.createReadStream() + return ( + await axios.post( + `/_matrix/media/v3/upload?user_id=${user_id}&ts=${ts}&filename=${fileName}`, + fileStream, + { + headers: { + 'Content-Type': content_type, + 'Content-Length': (await fd.stat()).size, + Authorization: `Bearer ${accessToken}`, + }, + } + ) + ).data.content_uri + } catch (err: any) { + if (err.code === 'EACCES' || err.code === 'ENOENT') { + log.warn(`Unable to open ${filePath}:`, err) + } else if (err instanceof AxiosError) { + log.warn(`Error during POST request of ${fileName}:`, err) + } else { + log.warn(`Other error while uploading ${filePath}:`, err) + } throw err } - const fileStream = fd.createReadStream() - - return ( - await axios.post( - `/_matrix/media/v3/upload?user_id=${user_id}&ts=${ts}&filename=${fileName}`, - fileStream, - { - headers: { - 'Content-Type': content_type, - 'Content-Length': (await fd.stat()).size, - Authorization: `Bearer ${accessToken}`, - }, - } - ) - ).data.content_uri } /** From d3c5b39b651b656500bdc1b0da94fe5b2850ac83 Mon Sep 17 00:00:00 2001 From: Tunui Franken Date: Tue, 30 Jul 2024 14:48:42 +0200 Subject: [PATCH 7/7] Document GridFS export for files with `mongofiles` --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 78c8602..21537f9 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,16 @@ mongoexport --collection=users --db=rocketchat --out=users.json Export them to `inputs/` +If you are using the `GridFS` storage mode, you will also need to export files: + +```shell +for file in $(mongofiles --db=rocketchat --prefix=rocketchat_uploads list | awk '{print $1}'); do + mongofiles --db=rocketchat --prefix=rocketchat_uploads get "$file" +done +``` + +Export them to `inputs/files/` + ### Configuring the Matrix Dev Server Generate a Synapse homeserver config with the following command (you might change `my.matrix.host` for the actual server name, as it can't be changed afterwards):