From b02db837cb7fb7e1f11eae90cc5e8f9c35bb474e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20H=C3=BCttemann?= Date: Mon, 5 Jun 2023 14:53:48 +0200 Subject: [PATCH 01/17] Update run instructions to be clearer --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9a4609e..c4a662d 100644 --- a/README.md +++ b/README.md @@ -19,13 +19,13 @@ Export them to `inputs/` ```shell docker-compose run --rm -e SYNAPSE_SERVER_NAME=my.matrix.host -e SYNAPSE_REPORT_STATS=no synapse generate docker-compose up -d -# Register a admin user +# Register an admin user docker-compose exec -it synapse register_new_matrix_user http://localhost:8008 -c /data/homeserver.yaml --admin --user verdiadmin --password verdiadmin ``` Then you can access the homeserver in [Element Web](https://app.element.io/#/login) or the [local admin interface](http://localhost:8080) as `http://localhost:8008` with the `verdiadmin` as username AND password. -You can store an access token for that user: +Store an access token for that user: ```shell curl --request POST \ @@ -35,6 +35,8 @@ curl --request POST \ > src/config/synapse_access_token.json ``` +To finally run the script, execute it via `npm start`. + ## Configuration Copy over `.env.example` to `.env` and insert your values. @@ -45,7 +47,7 @@ Copy over `.env.example` to `.env` and insert your values. ## Cleaning Up -To clean up the Synapse server and loal storage database, run (while the containers are stopped) +To clean up the Synapse server and local storage database, run (while the containers are stopped) ```shell sudo rm files/homeserver.db From 870128dd6e7f45c4fbee0a546f43741170c0ae4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20H=C3=BCttemann?= Date: Mon, 5 Jun 2023 15:33:32 +0200 Subject: [PATCH 02/17] Update entities to have more options --- src/app.ts | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/src/app.ts b/src/app.ts index aed672d..1def391 100644 --- a/src/app.ts +++ b/src/app.ts @@ -20,13 +20,33 @@ const AppDataSource = new DataSource({ }) const enum Entities { - Users = 'users.json', - Rooms = 'rocketchat_room.json', - Messages = 'rocketchat_message.json', + Users = 'users', + Rooms = 'rooms', + Messages = 'messages', +} + +type EntityConfig = { + filename: string + mappingType: number +} + +const entities: { [key in Entities]: EntityConfig } = { + users: { + filename: 'users.json', + mappingType: 0, + }, + rooms: { + filename: 'rocketchat_room.json', + mappingType: 1, + }, + messages: { + filename: 'rocketchat_message.json', + mappingType: 2, + }, } async function loadRcExport(entity: Entities) { - const rl = new lineByLine(`./inputs/${entity}`) + const rl = new lineByLine(`./inputs/${entities[entity].filename}`) let line: false | Buffer while ((line = rl.next())) { From ce5a668f1c141295f4fd7f1d28bd1b53ec911689 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20H=C3=BCttemann?= Date: Wed, 7 Jun 2023 18:07:36 +0200 Subject: [PATCH 03/17] Add Storage helper --- src/helpers/storage.ts | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 src/helpers/storage.ts diff --git a/src/helpers/storage.ts b/src/helpers/storage.ts new file mode 100644 index 0000000..6d31580 --- /dev/null +++ b/src/helpers/storage.ts @@ -0,0 +1,26 @@ +import { DataSource } from 'typeorm' +import { IdMapping } from '../entity/IdMapping' +import { Membership } from '../entity/Membership' + +const AppDataSource = new DataSource({ + type: 'sqlite', + database: 'db.sqlite', + entities: [IdMapping, Membership], + synchronize: true, + logging: false, +}) + +export function getMapping(id: string, type: number) { + return AppDataSource.manager.findOneBy(IdMapping, { + rcId: id, + type: type, + }) +} + +export async function save(entity: IdMapping | Membership) { + await AppDataSource.manager.save(entity) +} + +export async function getAccessToken(id: string) { + return (await getMapping(id, 0))?.accessToken +} From 98fe90264f9cd9e277ae66e3a5ba991576a958e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20H=C3=BCttemann?= Date: Wed, 7 Jun 2023 18:08:30 +0200 Subject: [PATCH 04/17] Move helper modules --- README.md | 2 +- package.json | 2 +- src/app.ts | 23 ++++++----------------- src/{ => helpers}/logger.ts | 0 src/{ => helpers}/synapse.ts | 13 +++++++++++-- src/users.ts | 4 ++-- 6 files changed, 21 insertions(+), 23 deletions(-) rename src/{ => helpers}/logger.ts (100%) rename src/{ => helpers}/synapse.ts (55%) diff --git a/README.md b/README.md index c4a662d..e54abe7 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Export them to `inputs/` ```shell docker-compose run --rm -e SYNAPSE_SERVER_NAME=my.matrix.host -e SYNAPSE_REPORT_STATS=no synapse generate docker-compose up -d -# Register an admin user +# Wait for the Server to boot, then register an admin user docker-compose exec -it synapse register_new_matrix_user http://localhost:8008 -c /data/homeserver.yaml --admin --user verdiadmin --password verdiadmin ``` diff --git a/package.json b/package.json index 2a7da95..4fbc321 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "lint-fix": "eslint src/ --fix --ext .ts", "prefix": "npm run format-fix", "fix": "npm run lint-fix", - "test": "jest", + "test": "rm -rf dist/ && jest", "compile": "rm -rf dist/ && tsc", "start": "npm run compile && node dist/app.js", "prepare": "husky install" diff --git a/src/app.ts b/src/app.ts index 1def391..548d761 100644 --- a/src/app.ts +++ b/src/app.ts @@ -2,23 +2,15 @@ import dotenv from 'dotenv' dotenv.config() import lineByLine from 'n-readlines' import 'reflect-metadata' -import { DataSource } from 'typeorm' import { IdMapping } from './entity/IdMapping' import { Membership } from './entity/Membership' -import log from './logger' -import { whoami } from './synapse' +import log from './helpers/logger' +import { whoami } from './helpers/synapse' import { RcUser, createUser } from './users' +import { getMapping, save, setMapping } from './helpers/storage' log.info('rocketchat2matrix starts.') -const AppDataSource = new DataSource({ - type: 'sqlite', - database: 'db.sqlite', - entities: [IdMapping, Membership], - synchronize: true, - logging: false, -}) - const enum Entities { Users = 'users', Rooms = 'rooms', @@ -65,10 +57,7 @@ async function loadRcExport(entity: Entities) { break } - let mapping = await AppDataSource.manager.findOneBy(IdMapping, { - rcId: rcUser._id, - type: 0, - }) + let mapping = await getMapping(rcUser._id, entities[entity].mappingType) if (mapping && mapping.matrixId) { log.debug('Mapping exists:', mapping) } else { @@ -79,7 +68,7 @@ async function loadRcExport(entity: Entities) { mapping.type = 0 mapping.accessToken = matrixUser.access_token - AppDataSource.manager.save(mapping) + await save(mapping) log.debug('Mapping added:', mapping) // Add user to room mapping (specific to users) @@ -88,7 +77,7 @@ async function loadRcExport(entity: Entities) { membership.rcRoomId = rcRoomId membership.rcUserId = rcUser._id - await AppDataSource.manager.save(membership) + await save(membership) log.debug(`${rcUser.username} membership for ${rcRoomId} created`) }) } diff --git a/src/logger.ts b/src/helpers/logger.ts similarity index 100% rename from src/logger.ts rename to src/helpers/logger.ts diff --git a/src/synapse.ts b/src/helpers/synapse.ts similarity index 55% rename from src/synapse.ts rename to src/helpers/synapse.ts index 0673cf6..70b8f30 100644 --- a/src/synapse.ts +++ b/src/helpers/synapse.ts @@ -1,9 +1,10 @@ -import { access_token } from './config/synapse_access_token.json' +import { access_token } from '../config/synapse_access_token.json' import axios from 'axios' import log from './logger' +import { getAccessToken } from './storage' axios.defaults.baseURL = 'http://localhost:8008' -axios.defaults.headers.common['Authorization'] = ` Bearer ${access_token}` +axios.defaults.headers.common['Authorization'] = `Bearer ${access_token}` axios.defaults.headers.post['Content-Type'] = 'application/json' export { default as axios } from 'axios' @@ -20,3 +21,11 @@ export const whoami = () => reject() }) }) + +export async function getUserSessionOptions(id: string) { + const accessToken = await getAccessToken(id) + if (!accessToken) { + throw new Error(`Could not retrieve access token for ID ${id}`) + } + return { headers: { Authorization: `Bearer ${accessToken}` } } +} diff --git a/src/users.ts b/src/users.ts index 0c4e02d..d139b26 100644 --- a/src/users.ts +++ b/src/users.ts @@ -1,5 +1,5 @@ -import log from './logger' -import { axios } from './synapse' +import log from './helpers/logger' +import { axios } from './helpers/synapse' import { createHmac } from 'node:crypto' export type RcUser = { From ae3353c6cbcbab47daa78c3a0809d786a40d542c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20H=C3=BCttemann?= Date: Wed, 7 Jun 2023 18:10:55 +0200 Subject: [PATCH 05/17] Add room handler --- src/rooms.test.ts | 77 +++++++++++++++++++++++++++++++++++++++++ src/rooms.ts | 88 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 src/rooms.test.ts create mode 100644 src/rooms.ts diff --git a/src/rooms.test.ts b/src/rooms.test.ts new file mode 100644 index 0000000..2abbd93 --- /dev/null +++ b/src/rooms.test.ts @@ -0,0 +1,77 @@ +import { MatrixRoomPresets, RcRoomTypes, mapRoom } from './rooms' + +const roomCreator = { + _id: 'roomcreatorid', + name: 'RoomCreator', + username: 'RoomCreator', + roles: [], + __rooms: [], +} + +test('mapping direct chats', () => { + expect( + mapRoom({ + _id: 'aliceidbobid', + t: RcRoomTypes.direct, + usernames: ['Alice', 'Bob'], + uids: ['aliceid', 'bobid'], + }) + ).toEqual({ + is_direct: true, + preset: MatrixRoomPresets.trusted, + creation_content: { + 'm.federate': false, + }, + _creatorId: 'aliceid', + }) +}) + +test('mapping public rooms', () => { + expect( + mapRoom({ + _id: 'randomRoomId', + fname: 'public', + description: 'Public chat room', + name: 'public', + t: RcRoomTypes.chat, + u: roomCreator, + }) + ).toEqual({ + preset: MatrixRoomPresets.public, + room_alias_name: 'public', + name: 'public', + topic: 'Public chat room', + creation_content: { + 'm.federate': false, + }, + _creatorId: roomCreator._id, + }) +}) + +test('mapping private rooms', () => { + expect( + mapRoom({ + _id: 'privateRoomId', + name: 'private', + fname: 'private', + description: 'Private chat room', + t: RcRoomTypes.private, + u: roomCreator, + }) + ).toEqual({ + preset: MatrixRoomPresets.private, + room_alias_name: 'private', + name: 'private', + topic: 'Private chat room', + creation_content: { + 'm.federate': false, + }, + _creatorId: roomCreator._id, + }) +}) + +test('mapping live chats', () => { + expect(() => + mapRoom({ _id: 'liveChatId', t: RcRoomTypes.live }) + ).toThrowError('Room type l is unknown') +}) diff --git a/src/rooms.ts b/src/rooms.ts new file mode 100644 index 0000000..df7afe6 --- /dev/null +++ b/src/rooms.ts @@ -0,0 +1,88 @@ +import log from './helpers/logger' +import { getAccessToken } from './helpers/storage' +import { axios, getUserSessionOptions } from './helpers/synapse' +import { RcUser } from './users' + +export const enum RcRoomTypes { + direct = 'd', + chat = 'c', + private = 'p', + live = 'l', +} + +export type RcRoom = { + _id: string + t: RcRoomTypes + uids?: string[] + usernames?: string[] + name?: string + u?: RcUser + topic?: string + fname?: string + description?: string +} + +export const enum MatrixRoomPresets { + private = 'private_chat', + public = 'public_chat', + trusted = 'trusted_private_chat', +} + +export type MatrixRoom = { + room_id?: string + name?: string + creation_content?: object + room_alias_name?: string + topic?: string + is_direct?: boolean + preset?: MatrixRoomPresets + _creatorId?: string +} + +export function mapRoom(rcRoom: RcRoom): MatrixRoom { + const room: MatrixRoom = { + creation_content: { + 'm.federate': false, + }, + } + rcRoom.name && (room.name = rcRoom.name) + rcRoom.name && (room.room_alias_name = rcRoom.name) + rcRoom.description && (room.topic = rcRoom.description) + + switch (rcRoom.t) { + case 'd': + room.is_direct = true + room.preset = MatrixRoomPresets.trusted + room._creatorId = rcRoom.uids?.[0] + break + + case 'c': + room.preset = MatrixRoomPresets.public + room._creatorId = rcRoom.u?._id + break + + case 'p': + room.preset = MatrixRoomPresets.private + room._creatorId = rcRoom.u?._id + break + + default: + const message = `Room type ${rcRoom.t} is unknown` + log.error(message) + throw new Error(message) + } + return room +} + +export async function createRoom(rcRoom: RcRoom): Promise { + const room: MatrixRoom = mapRoom(rcRoom) + room.room_id = ( + await axios.post( + '/_matrix/client/v3/createRoom', + room, + await getUserSessionOptions(room._creatorId!) + ) + ).data.room_id + + return room +} From 4692c7edea34345d57d0bb0bde4fde4237e23986 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20H=C3=BCttemann?= Date: Wed, 7 Jun 2023 18:23:20 +0200 Subject: [PATCH 06/17] Move handlers and clean up --- src/app.ts | 6 +++--- src/{ => handlers}/rooms.test.ts | 0 src/{ => handlers}/rooms.ts | 21 +++++++++++++-------- src/{ => handlers}/users.test.ts | 8 +++++++- src/{ => handlers}/users.ts | 4 ++-- src/helpers/storage.ts | 4 ++++ src/helpers/synapse.ts | 2 +- 7 files changed, 30 insertions(+), 15 deletions(-) rename src/{ => handlers}/rooms.test.ts (100%) rename src/{ => handlers}/rooms.ts (76%) rename src/{ => handlers}/users.test.ts (94%) rename src/{ => handlers}/users.ts (95%) diff --git a/src/app.ts b/src/app.ts index 548d761..43c4ad0 100644 --- a/src/app.ts +++ b/src/app.ts @@ -4,10 +4,10 @@ import lineByLine from 'n-readlines' import 'reflect-metadata' import { IdMapping } from './entity/IdMapping' import { Membership } from './entity/Membership' +import { RcUser, createUser } from './handlers/users' import log from './helpers/logger' +import { getMapping, initStorage, save } from './helpers/storage' import { whoami } from './helpers/synapse' -import { RcUser, createUser } from './users' -import { getMapping, save, setMapping } from './helpers/storage' log.info('rocketchat2matrix starts.') @@ -101,7 +101,7 @@ async function loadRcExport(entity: Entities) { async function main() { try { await whoami() - await AppDataSource.initialize() + await initStorage() await loadRcExport(Entities.Users) log.info('Done.') } catch (error) { diff --git a/src/rooms.test.ts b/src/handlers/rooms.test.ts similarity index 100% rename from src/rooms.test.ts rename to src/handlers/rooms.test.ts diff --git a/src/rooms.ts b/src/handlers/rooms.ts similarity index 76% rename from src/rooms.ts rename to src/handlers/rooms.ts index df7afe6..e5f022c 100644 --- a/src/rooms.ts +++ b/src/handlers/rooms.ts @@ -1,6 +1,5 @@ -import log from './helpers/logger' -import { getAccessToken } from './helpers/storage' -import { axios, getUserSessionOptions } from './helpers/synapse' +import log from '../helpers/logger' +import { axios, getUserSessionOptions } from '../helpers/synapse' import { RcUser } from './users' export const enum RcRoomTypes { @@ -36,7 +35,7 @@ export type MatrixRoom = { topic?: string is_direct?: boolean preset?: MatrixRoomPresets - _creatorId?: string + _creatorId: string } export function mapRoom(rcRoom: RcRoom): MatrixRoom { @@ -44,6 +43,7 @@ export function mapRoom(rcRoom: RcRoom): MatrixRoom { creation_content: { 'm.federate': false, }, + _creatorId: '', } rcRoom.name && (room.name = rcRoom.name) rcRoom.name && (room.room_alias_name = rcRoom.name) @@ -53,17 +53,17 @@ export function mapRoom(rcRoom: RcRoom): MatrixRoom { case 'd': room.is_direct = true room.preset = MatrixRoomPresets.trusted - room._creatorId = rcRoom.uids?.[0] + room._creatorId = rcRoom.uids?.[0] || '' break case 'c': room.preset = MatrixRoomPresets.public - room._creatorId = rcRoom.u?._id + room._creatorId = rcRoom.u?._id || '' break case 'p': room.preset = MatrixRoomPresets.private - room._creatorId = rcRoom.u?._id + room._creatorId = rcRoom.u?._id || '' break default: @@ -71,6 +71,11 @@ export function mapRoom(rcRoom: RcRoom): MatrixRoom { log.error(message) throw new Error(message) } + if (!room._creatorId) { + const message = `Creator ID could not be determined for room of type ${rcRoom.t}` + log.error(message) + throw new Error(message) + } return room } @@ -80,7 +85,7 @@ export async function createRoom(rcRoom: RcRoom): Promise { await axios.post( '/_matrix/client/v3/createRoom', room, - await getUserSessionOptions(room._creatorId!) + await getUserSessionOptions(room._creatorId) ) ).data.room_id diff --git a/src/users.test.ts b/src/handlers/users.test.ts similarity index 94% rename from src/users.test.ts rename to src/handlers/users.test.ts index 2849020..774e404 100644 --- a/src/users.test.ts +++ b/src/handlers/users.test.ts @@ -1,6 +1,12 @@ process.env.REGISTRATION_SHARED_SECRET = 'ThisIsSoSecretWow' import axios from 'axios' -import { MatrixUser, RcUser, createUser, generateHmac, mapUser } from './users' +import { + MatrixUser, + RcUser, + createUser, + generateHmac, + mapUser, +} from '../handlers/users' jest.mock('axios') const mockedAxios = axios as jest.Mocked diff --git a/src/users.ts b/src/handlers/users.ts similarity index 95% rename from src/users.ts rename to src/handlers/users.ts index d139b26..c6867ed 100644 --- a/src/users.ts +++ b/src/handlers/users.ts @@ -1,6 +1,6 @@ -import log from './helpers/logger' -import { axios } from './helpers/synapse' import { createHmac } from 'node:crypto' +import log from '../helpers/logger' +import { axios } from '../helpers/synapse' export type RcUser = { _id: string diff --git a/src/helpers/storage.ts b/src/helpers/storage.ts index 6d31580..7192955 100644 --- a/src/helpers/storage.ts +++ b/src/helpers/storage.ts @@ -10,6 +10,10 @@ const AppDataSource = new DataSource({ logging: false, }) +export async function initStorage() { + await AppDataSource.initialize() +} + export function getMapping(id: string, type: number) { return AppDataSource.manager.findOneBy(IdMapping, { rcId: id, diff --git a/src/helpers/synapse.ts b/src/helpers/synapse.ts index 70b8f30..aaad424 100644 --- a/src/helpers/synapse.ts +++ b/src/helpers/synapse.ts @@ -1,5 +1,5 @@ -import { access_token } from '../config/synapse_access_token.json' import axios from 'axios' +import { access_token } from '../config/synapse_access_token.json' import log from './logger' import { getAccessToken } from './storage' From 91fa37f82d727dc8730d06d072f55e6209bc63bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20H=C3=BCttemann?= Date: Mon, 12 Jun 2023 14:54:24 +0200 Subject: [PATCH 07/17] Make accessTokens nullable in storage --- src/entity/IdMapping.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/entity/IdMapping.ts b/src/entity/IdMapping.ts index f2c00d4..7a3bdc4 100644 --- a/src/entity/IdMapping.ts +++ b/src/entity/IdMapping.ts @@ -11,6 +11,6 @@ export class IdMapping { @Column('integer') type!: number // Type of the entity; 0 = user, 1 = room, 2 = message - @Column() + @Column({ nullable: true }) accessToken?: string // Access token for matrix users } From ef86d6e3694abce5a6f5ed3ee8bf56cdfd4913fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20H=C3=BCttemann?= Date: Mon, 12 Jun 2023 14:54:39 +0200 Subject: [PATCH 08/17] --wip-- [skip ci] --- src/app.ts | 28 ++++++++++++++++++++++++---- src/handlers/rooms.ts | 29 ++++++++++++++++++++--------- 2 files changed, 44 insertions(+), 13 deletions(-) diff --git a/src/app.ts b/src/app.ts index 43c4ad0..ae9906f 100644 --- a/src/app.ts +++ b/src/app.ts @@ -8,6 +8,7 @@ import { RcUser, createUser } from './handlers/users' import log from './helpers/logger' import { getMapping, initStorage, save } from './helpers/storage' import { whoami } from './helpers/synapse' +import { RcRoom, createRoom } from './handlers/rooms' log.info('rocketchat2matrix starts.') @@ -65,7 +66,7 @@ async function loadRcExport(entity: Entities) { mapping = new IdMapping() mapping.rcId = rcUser._id mapping.matrixId = matrixUser.user_id - mapping.type = 0 + mapping.type = entities[entity].mappingType mapping.accessToken = matrixUser.access_token await save(mapping) @@ -85,7 +86,25 @@ async function loadRcExport(entity: Entities) { break case Entities.Rooms: - log.debug(`Room: ${item.name}`) + const rcRoom: RcRoom = item + log.debug(`Room: ${rcRoom.name}`, rcRoom) + + let roomMapping = await getMapping( + rcRoom._id, + entities[entity].mappingType + ) + if (roomMapping && roomMapping.matrixId) { + log.debug('Mapping exists:', roomMapping) + } else { + const matrixRoom = await createRoom(rcRoom) + roomMapping = new IdMapping() + roomMapping.rcId = rcRoom._id + roomMapping.matrixId = matrixRoom.room_id + roomMapping.type = entities[entity].mappingType + + await save(roomMapping) + log.debug('Mapping added:', roomMapping) + } break case Entities.Messages: @@ -102,10 +121,11 @@ async function main() { try { await whoami() await initStorage() - await loadRcExport(Entities.Users) + // await loadRcExport(Entities.Users) + await loadRcExport(Entities.Rooms) log.info('Done.') } catch (error) { - log.error(`Encountered an error while booting up`) + log.error(`Encountered an error while booting up: ${error}`) } } diff --git a/src/handlers/rooms.ts b/src/handlers/rooms.ts index e5f022c..673e0ec 100644 --- a/src/handlers/rooms.ts +++ b/src/handlers/rooms.ts @@ -35,7 +35,7 @@ export type MatrixRoom = { topic?: string is_direct?: boolean preset?: MatrixRoomPresets - _creatorId: string + _creatorId?: string } export function mapRoom(rcRoom: RcRoom): MatrixRoom { @@ -72,22 +72,33 @@ export function mapRoom(rcRoom: RcRoom): MatrixRoom { throw new Error(message) } if (!room._creatorId) { - const message = `Creator ID could not be determined for room of type ${rcRoom.t}` - log.error(message) - throw new Error(message) + log.warn( + `Creator ID could not be determined for room ${rcRoom.name} of type ${rcRoom.t}.` + ) } return room } export async function createRoom(rcRoom: RcRoom): Promise { const room: MatrixRoom = mapRoom(rcRoom) + let sessionOptions = {} + if (room._creatorId) { + try { + sessionOptions = await getUserSessionOptions(room._creatorId) + log.debug('Room user session generated:', sessionOptions) + } catch (error) { + log.warn(error) + // TODO: Skip room, if it has 0-1 member or is a direct chat? + } + } + log.debug('Creating room:', room) + delete room._creatorId + room.room_id = ( - await axios.post( - '/_matrix/client/v3/createRoom', - room, - await getUserSessionOptions(room._creatorId) - ) + await axios.post('/_matrix/client/v3/createRoom', room, sessionOptions) ).data.room_id + // TODO: Add members + return room } From 997ebf3ebb7cc68163b61eabc4a89c5f66aafa47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20H=C3=BCttemann?= Date: Mon, 12 Jun 2023 15:23:28 +0200 Subject: [PATCH 09/17] Add public room visibility to public chats --- src/handlers/rooms.test.ts | 8 +++++++- src/handlers/rooms.ts | 18 +++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/handlers/rooms.test.ts b/src/handlers/rooms.test.ts index 2abbd93..7560380 100644 --- a/src/handlers/rooms.test.ts +++ b/src/handlers/rooms.test.ts @@ -1,4 +1,9 @@ -import { MatrixRoomPresets, RcRoomTypes, mapRoom } from './rooms' +import { + MatrixRoomPresets, + MatrixRoomVisibility, + RcRoomTypes, + mapRoom, +} from './rooms' const roomCreator = { _id: 'roomcreatorid', @@ -44,6 +49,7 @@ test('mapping public rooms', () => { creation_content: { 'm.federate': false, }, + visibility: MatrixRoomVisibility.public, _creatorId: roomCreator._id, }) }) diff --git a/src/handlers/rooms.ts b/src/handlers/rooms.ts index 673e0ec..e247c27 100644 --- a/src/handlers/rooms.ts +++ b/src/handlers/rooms.ts @@ -27,6 +27,11 @@ export const enum MatrixRoomPresets { trusted = 'trusted_private_chat', } +export const enum MatrixRoomVisibility { + private = 'private', + public = 'public', +} + export type MatrixRoom = { room_id?: string name?: string @@ -35,6 +40,7 @@ export type MatrixRoom = { topic?: string is_direct?: boolean preset?: MatrixRoomPresets + visibility?: MatrixRoomVisibility _creatorId?: string } @@ -50,24 +56,26 @@ export function mapRoom(rcRoom: RcRoom): MatrixRoom { rcRoom.description && (room.topic = rcRoom.description) switch (rcRoom.t) { - case 'd': + case RcRoomTypes.direct: room.is_direct = true room.preset = MatrixRoomPresets.trusted room._creatorId = rcRoom.uids?.[0] || '' break - case 'c': + case RcRoomTypes.chat: room.preset = MatrixRoomPresets.public + room.visibility = MatrixRoomVisibility.public room._creatorId = rcRoom.u?._id || '' break - case 'p': + case RcRoomTypes.private: room.preset = MatrixRoomPresets.private room._creatorId = rcRoom.u?._id || '' break + case RcRoomTypes.live: default: - const message = `Room type ${rcRoom.t} is unknown` + const message = `Room type ${rcRoom.t} is unknown or unimplemented` log.error(message) throw new Error(message) } @@ -98,7 +106,7 @@ export async function createRoom(rcRoom: RcRoom): Promise { await axios.post('/_matrix/client/v3/createRoom', room, sessionOptions) ).data.room_id - // TODO: Add members + // TODO: Invite members and let them join return room } From f4891fdc8374d49b2670f267b6c46789d1dc68f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20H=C3=BCttemann?= Date: Mon, 12 Jun 2023 15:28:59 +0200 Subject: [PATCH 10/17] Update reset instructions --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e54abe7..4024ba8 100644 --- a/README.md +++ b/README.md @@ -47,9 +47,10 @@ Copy over `.env.example` to `.env` and insert your values. ## Cleaning Up -To clean up the Synapse server and local storage database, run (while the containers are stopped) +To clean up the Synapse server and local storage database, run ```shell +docker-compose down sudo rm files/homeserver.db rm db.sqlite ``` From e706451d90390732cbdf917871b8de6e45d71b98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20H=C3=BCttemann?= Date: Mon, 12 Jun 2023 16:51:56 +0200 Subject: [PATCH 11/17] Add createMembership function --- src/app.ts | 24 +++++++++++++----------- src/handlers/rooms.ts | 12 ++++++++++++ src/helpers/storage.ts | 8 ++++++++ 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/src/app.ts b/src/app.ts index ae9906f..dcb22bb 100644 --- a/src/app.ts +++ b/src/app.ts @@ -3,10 +3,14 @@ dotenv.config() import lineByLine from 'n-readlines' import 'reflect-metadata' import { IdMapping } from './entity/IdMapping' -import { Membership } from './entity/Membership' import { RcUser, createUser } from './handlers/users' import log from './helpers/logger' -import { getMapping, initStorage, save } from './helpers/storage' +import { + createMembership, + getMapping, + initStorage, + save, +} from './helpers/storage' import { whoami } from './helpers/synapse' import { RcRoom, createRoom } from './handlers/rooms' @@ -73,14 +77,12 @@ async function loadRcExport(entity: Entities) { log.debug('Mapping added:', mapping) // Add user to room mapping (specific to users) - rcUser.__rooms.forEach(async (rcRoomId: string) => { - const membership = new Membership() - membership.rcRoomId = rcRoomId - membership.rcUserId = rcUser._id - - await save(membership) - log.debug(`${rcUser.username} membership for ${rcRoomId} created`) - }) + Promise.all( + rcUser.__rooms.map(async (rcRoomId: string) => { + await createMembership(rcRoomId, rcUser._id) + log.debug(`${rcUser.username} membership for ${rcRoomId} created`) + }) + ) } break @@ -121,7 +123,7 @@ async function main() { try { await whoami() await initStorage() - // await loadRcExport(Entities.Users) + await loadRcExport(Entities.Users) await loadRcExport(Entities.Rooms) log.info('Done.') } catch (error) { diff --git a/src/handlers/rooms.ts b/src/handlers/rooms.ts index e247c27..e247fbb 100644 --- a/src/handlers/rooms.ts +++ b/src/handlers/rooms.ts @@ -1,4 +1,5 @@ import log from '../helpers/logger' +import { createMembership } from '../helpers/storage' import { axios, getUserSessionOptions } from '../helpers/synapse' import { RcUser } from './users' @@ -60,6 +61,17 @@ export function mapRoom(rcRoom: RcRoom): MatrixRoom { room.is_direct = true room.preset = MatrixRoomPresets.trusted room._creatorId = rcRoom.uids?.[0] || '' + + if (rcRoom.uids) { + Promise.all( + rcRoom.uids.map(async (uid) => { + await createMembership(rcRoom._id, uid) + log.debug(`${uid} membership in direct chat ${rcRoom._id} created`) + }) + ) + } else { + throw new Error('Found a direct chat without uids. This is unexpected.') + } break case RcRoomTypes.chat: diff --git a/src/helpers/storage.ts b/src/helpers/storage.ts index 7192955..3ac80d4 100644 --- a/src/helpers/storage.ts +++ b/src/helpers/storage.ts @@ -28,3 +28,11 @@ export async function save(entity: IdMapping | Membership) { export async function getAccessToken(id: string) { return (await getMapping(id, 0))?.accessToken } + +export async function createMembership(rcRoomId: string, rcUserId: string) { + const membership = new Membership() + membership.rcRoomId = rcRoomId + membership.rcUserId = rcUserId + + await save(membership) +} From 5d5c751de867fd76815ed04008d9f6bd8c824e37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20H=C3=BCttemann?= Date: Mon, 12 Jun 2023 16:52:41 +0200 Subject: [PATCH 12/17] Fix promises to be awaited --- src/app.ts | 6 ++++-- src/handlers/rooms.ts | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/app.ts b/src/app.ts index dcb22bb..2146061 100644 --- a/src/app.ts +++ b/src/app.ts @@ -77,7 +77,7 @@ async function loadRcExport(entity: Entities) { log.debug('Mapping added:', mapping) // Add user to room mapping (specific to users) - Promise.all( + await Promise.all( rcUser.__rooms.map(async (rcRoomId: string) => { await createMembership(rcRoomId, rcUser._id) log.debug(`${rcUser.username} membership for ${rcRoomId} created`) @@ -123,11 +123,13 @@ async function main() { try { await whoami() await initStorage() + log.info('Parsing users') await loadRcExport(Entities.Users) + log.info('Parsing rooms') await loadRcExport(Entities.Rooms) log.info('Done.') } catch (error) { - log.error(`Encountered an error while booting up: ${error}`) + log.error(`Encountered an error while booting up: ${error}`, error) } } diff --git a/src/handlers/rooms.ts b/src/handlers/rooms.ts index e247fbb..33165d2 100644 --- a/src/handlers/rooms.ts +++ b/src/handlers/rooms.ts @@ -45,7 +45,7 @@ export type MatrixRoom = { _creatorId?: string } -export function mapRoom(rcRoom: RcRoom): MatrixRoom { +export async function mapRoom(rcRoom: RcRoom): Promise { const room: MatrixRoom = { creation_content: { 'm.federate': false, @@ -63,7 +63,7 @@ export function mapRoom(rcRoom: RcRoom): MatrixRoom { room._creatorId = rcRoom.uids?.[0] || '' if (rcRoom.uids) { - Promise.all( + await Promise.all( rcRoom.uids.map(async (uid) => { await createMembership(rcRoom._id, uid) log.debug(`${uid} membership in direct chat ${rcRoom._id} created`) @@ -100,7 +100,7 @@ export function mapRoom(rcRoom: RcRoom): MatrixRoom { } export async function createRoom(rcRoom: RcRoom): Promise { - const room: MatrixRoom = mapRoom(rcRoom) + const room: MatrixRoom = await mapRoom(rcRoom) let sessionOptions = {} if (room._creatorId) { try { From 4a54b63cc05fc1859b4c46be0bf069867e1d8b94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20H=C3=BCttemann?= Date: Tue, 13 Jun 2023 12:32:11 +0200 Subject: [PATCH 13/17] Fix error for duplicate room memberships --- src/handlers/rooms.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/handlers/rooms.ts b/src/handlers/rooms.ts index 33165d2..8889418 100644 --- a/src/handlers/rooms.ts +++ b/src/handlers/rooms.ts @@ -64,10 +64,13 @@ export async function mapRoom(rcRoom: RcRoom): Promise { if (rcRoom.uids) { await Promise.all( - rcRoom.uids.map(async (uid) => { - await createMembership(rcRoom._id, uid) - log.debug(`${uid} membership in direct chat ${rcRoom._id} created`) - }) + [...new Set(rcRoom.uids)] // Deduplicate users + .map(async (uid) => { + await createMembership(rcRoom._id, uid) + log.debug( + `${uid} membership in direct chat ${rcRoom._id} created` + ) + }) ) } else { throw new Error('Found a direct chat without uids. This is unexpected.') From f767753bb35f396132de42e2ff06ec79010d310f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20H=C3=BCttemann?= Date: Tue, 13 Jun 2023 14:48:48 +0200 Subject: [PATCH 14/17] Add instructions for rate limiting Fixes #19 --- README.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/README.md b/README.md index 4024ba8..0208421 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,36 @@ Export them to `inputs/` ```shell docker-compose run --rm -e SYNAPSE_SERVER_NAME=my.matrix.host -e SYNAPSE_REPORT_STATS=no synapse generate +``` + +To run the script without hitting rate limiting, you SHOULD add the following options to the freshly generated `files/homeserver.yaml`. **Do not leave these in the production setup!** + +```yaml +rc_joins: + local: + per_second: 1024 + burst_count: 2048 +rc_joins_per_room: + per_second: 1024 + burst_count: 2048 +rc_message: + per_second: 1024 + burst_count: 2048 +rc_invites: + per_room: + per_second: 1024 + burst_count: 2048 + per_user: + per_second: 1024 + burst_count: 2048 + per_issuer: + per_second: 1024 + burst_count: 2048 +``` + +Continue setting up the server: + +```shell docker-compose up -d # Wait for the Server to boot, then register an admin user docker-compose exec -it synapse register_new_matrix_user http://localhost:8008 -c /data/homeserver.yaml --admin --user verdiadmin --password verdiadmin From a06e379531617bc3b753e4c3920a7759047948b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20H=C3=BCttemann?= Date: Wed, 14 Jun 2023 13:07:52 +0200 Subject: [PATCH 15/17] Move Membership side-effects to own function --- src/handlers/rooms.ts | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/src/handlers/rooms.ts b/src/handlers/rooms.ts index 8889418..d9b6648 100644 --- a/src/handlers/rooms.ts +++ b/src/handlers/rooms.ts @@ -45,7 +45,7 @@ export type MatrixRoom = { _creatorId?: string } -export async function mapRoom(rcRoom: RcRoom): Promise { +export function mapRoom(rcRoom: RcRoom): MatrixRoom { const room: MatrixRoom = { creation_content: { 'm.federate': false, @@ -61,20 +61,6 @@ export async function mapRoom(rcRoom: RcRoom): Promise { room.is_direct = true room.preset = MatrixRoomPresets.trusted room._creatorId = rcRoom.uids?.[0] || '' - - if (rcRoom.uids) { - await Promise.all( - [...new Set(rcRoom.uids)] // Deduplicate users - .map(async (uid) => { - await createMembership(rcRoom._id, uid) - log.debug( - `${uid} membership in direct chat ${rcRoom._id} created` - ) - }) - ) - } else { - throw new Error('Found a direct chat without uids. This is unexpected.') - } break case RcRoomTypes.chat: @@ -102,8 +88,21 @@ export async function mapRoom(rcRoom: RcRoom): Promise { return room } +export async function parseMemberships(rcRoom: RcRoom) { + if (rcRoom.t == RcRoomTypes.direct && rcRoom.uids) { + await Promise.all( + [...new Set(rcRoom.uids)] // Deduplicate users + .map(async (uid) => { + await createMembership(rcRoom._id, uid) + log.debug(`${uid} membership in direct chat ${rcRoom._id} created`) + }) + ) + } +} + export async function createRoom(rcRoom: RcRoom): Promise { - const room: MatrixRoom = await mapRoom(rcRoom) + const room: MatrixRoom = mapRoom(rcRoom) + await parseMemberships(rcRoom) let sessionOptions = {} if (room._creatorId) { try { From 8b8b7b290d71ec4f8485a7f145c04aa9bc8ec3c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20H=C3=BCttemann?= Date: Wed, 14 Jun 2023 13:37:57 +0200 Subject: [PATCH 16/17] Add cleanup script to reset dev environment --- README.md | 2 +- reset.sh | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100755 reset.sh diff --git a/README.md b/README.md index 0208421..ac6c38e 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ Copy over `.env.example` to `.env` and insert your values. ## Cleaning Up -To clean up the Synapse server and local storage database, run +To clean up the Synapse server and local storage database, run either the convenience script `./reset.sh` or start with: ```shell docker-compose down diff --git a/reset.sh b/reset.sh new file mode 100755 index 0000000..73e39f1 --- /dev/null +++ b/reset.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -euo pipefail +IFS=$'\n\t' + +HOMESERVER="http://localhost:8008" + +docker-compose down +sudo rm -f files/homeserver.db +rm -f db.sqlite +docker-compose up -d + +sleep 1.5 +set +e +until docker-compose exec -it synapse register_new_matrix_user $HOMESERVER -c /data/homeserver.yaml --admin --user verdiadmin --password verdiadmin &> /dev/null +do + echo 'Retrying creating admin...' +done +set -e + +curl --request POST \ + --url $HOMESERVER/_matrix/client/v3/login \ + --header 'Content-Type: application/json' \ + --data '{"type": "m.login.password","user": "verdiadmin","password": "verdiadmin","device_id": "DEV"}' \ +> src/config/synapse_access_token.json 2> /dev/null From ab597195da0bfd0a1a57b3195005081984b9f8c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20H=C3=BCttemann?= Date: Wed, 14 Jun 2023 15:02:10 +0200 Subject: [PATCH 17/17] Add membership handling --- src/app.ts | 4 ++-- src/handlers/rooms.ts | 46 +++++++++++++++++++++++++++++++++++++++--- src/helpers/storage.ts | 13 ++++++++++++ 3 files changed, 58 insertions(+), 5 deletions(-) diff --git a/src/app.ts b/src/app.ts index 2146061..f50a1c3 100644 --- a/src/app.ts +++ b/src/app.ts @@ -51,7 +51,7 @@ async function loadRcExport(entity: Entities) { switch (entity) { case Entities.Users: const rcUser: RcUser = item - log.debug(`Parsing user: ${rcUser.name}: ${rcUser._id}`) + log.info(`Parsing user: ${rcUser.name}: ${rcUser._id}`) // Check for exclusion if ( @@ -89,7 +89,7 @@ async function loadRcExport(entity: Entities) { case Entities.Rooms: const rcRoom: RcRoom = item - log.debug(`Room: ${rcRoom.name}`, rcRoom) + log.info(`Parsing room ${rcRoom.name || 'with ID: ' + rcRoom._id}`) let roomMapping = await getMapping( rcRoom._id, diff --git a/src/handlers/rooms.ts b/src/handlers/rooms.ts index d9b6648..1f1d214 100644 --- a/src/handlers/rooms.ts +++ b/src/handlers/rooms.ts @@ -1,5 +1,10 @@ +import { IdMapping } from '../entity/IdMapping' import log from '../helpers/logger' -import { createMembership } from '../helpers/storage' +import { + createMembership, + getMapping, + getMemberships, +} from '../helpers/storage' import { axios, getUserSessionOptions } from '../helpers/synapse' import { RcUser } from './users' @@ -102,11 +107,13 @@ export async function parseMemberships(rcRoom: RcRoom) { export async function createRoom(rcRoom: RcRoom): Promise { const room: MatrixRoom = mapRoom(rcRoom) + const creatorId = room._creatorId || '' + delete room._creatorId await parseMemberships(rcRoom) let sessionOptions = {} if (room._creatorId) { try { - sessionOptions = await getUserSessionOptions(room._creatorId) + sessionOptions = await getUserSessionOptions(creatorId) log.debug('Room user session generated:', sessionOptions) } catch (error) { log.warn(error) @@ -114,13 +121,46 @@ export async function createRoom(rcRoom: RcRoom): Promise { } } log.debug('Creating room:', room) - delete room._creatorId room.room_id = ( await axios.post('/_matrix/client/v3/createRoom', room, sessionOptions) ).data.room_id // TODO: Invite members and let them join + const members = await getMemberships(rcRoom._id) + log.info(`Inviting members to room ${rcRoom._id}:`, members) + + const memberMappings = ( + await Promise.all( + members + .filter((rcMemberId) => rcMemberId != creatorId) + .map(async (rcMemberId) => await getMapping(rcMemberId, 0)) + ) + ) + .filter((mapping): mapping is IdMapping => mapping != null) + .map(async (mapping) => { + log.http(`Invite member ${mapping.rcId} aka. ${mapping.matrixId}`) + await axios.post( + `/_matrix/client/v3/rooms/${room.room_id}/invite`, + { user_id: mapping.matrixId }, + sessionOptions + ) + + log.http( + `Accepting invitation for member ${mapping.rcId} aka. ${mapping.matrixId}` + ) + await axios.post( + `/_matrix/client/v3/join/${room.room_id}`, + {}, + { + headers: { + Authorization: `Bearer ${mapping.accessToken}`, + }, + } + ) + }) + + await Promise.all(memberMappings) return room } diff --git a/src/helpers/storage.ts b/src/helpers/storage.ts index 3ac80d4..3b8c8d3 100644 --- a/src/helpers/storage.ts +++ b/src/helpers/storage.ts @@ -36,3 +36,16 @@ export async function createMembership(rcRoomId: string, rcUserId: string) { await save(membership) } + +export async function getMemberships(rcRoomId: string) { + return ( + await AppDataSource.manager.find(Membership, { + select: { + rcUserId: true, + }, + where: { + rcRoomId: rcRoomId, + }, + }) + ).map((entity) => entity.rcUserId) +}