diff --git a/README.md b/README.md index 9a4609e..ac6c38e 100644 --- a/README.md +++ b/README.md @@ -18,14 +18,44 @@ Export them to `inputs/` ```shell docker-compose run --rm -e SYNAPSE_SERVER_NAME=my.matrix.host -e SYNAPSE_REPORT_STATS=no synapse generate +``` + +To run the script without hitting rate limiting, you SHOULD add the following options to the freshly generated `files/homeserver.yaml`. **Do not leave these in the production setup!** + +```yaml +rc_joins: + local: + per_second: 1024 + burst_count: 2048 +rc_joins_per_room: + per_second: 1024 + burst_count: 2048 +rc_message: + per_second: 1024 + burst_count: 2048 +rc_invites: + per_room: + per_second: 1024 + burst_count: 2048 + per_user: + per_second: 1024 + burst_count: 2048 + per_issuer: + per_second: 1024 + burst_count: 2048 +``` + +Continue setting up the server: + +```shell docker-compose up -d -# Register a admin user +# Wait for the Server to boot, then register an admin user docker-compose exec -it synapse register_new_matrix_user http://localhost:8008 -c /data/homeserver.yaml --admin --user verdiadmin --password verdiadmin ``` Then you can access the homeserver in [Element Web](https://app.element.io/#/login) or the [local admin interface](http://localhost:8080) as `http://localhost:8008` with the `verdiadmin` as username AND password. -You can store an access token for that user: +Store an access token for that user: ```shell curl --request POST \ @@ -35,6 +65,8 @@ curl --request POST \ > src/config/synapse_access_token.json ``` +To finally run the script, execute it via `npm start`. + ## Configuration Copy over `.env.example` to `.env` and insert your values. @@ -45,9 +77,10 @@ Copy over `.env.example` to `.env` and insert your values. ## Cleaning Up -To clean up the Synapse server and loal storage database, run (while the containers are stopped) +To clean up the Synapse server and local storage database, run either the convenience script `./reset.sh` or start with: ```shell +docker-compose down sudo rm files/homeserver.db rm db.sqlite ``` diff --git a/package.json b/package.json index 2a7da95..4fbc321 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "lint-fix": "eslint src/ --fix --ext .ts", "prefix": "npm run format-fix", "fix": "npm run lint-fix", - "test": "jest", + "test": "rm -rf dist/ && jest", "compile": "rm -rf dist/ && tsc", "start": "npm run compile && node dist/app.js", "prepare": "husky install" diff --git a/reset.sh b/reset.sh new file mode 100755 index 0000000..73e39f1 --- /dev/null +++ b/reset.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -euo pipefail +IFS=$'\n\t' + +HOMESERVER="http://localhost:8008" + +docker-compose down +sudo rm -f files/homeserver.db +rm -f db.sqlite +docker-compose up -d + +sleep 1.5 +set +e +until docker-compose exec -it synapse register_new_matrix_user $HOMESERVER -c /data/homeserver.yaml --admin --user verdiadmin --password verdiadmin &> /dev/null +do + echo 'Retrying creating admin...' +done +set -e + +curl --request POST \ + --url $HOMESERVER/_matrix/client/v3/login \ + --header 'Content-Type: application/json' \ + --data '{"type": "m.login.password","user": "verdiadmin","password": "verdiadmin","device_id": "DEV"}' \ +> src/config/synapse_access_token.json 2> /dev/null diff --git a/src/app.ts b/src/app.ts index aed672d..f50a1c3 100644 --- a/src/app.ts +++ b/src/app.ts @@ -2,31 +2,48 @@ import dotenv from 'dotenv' dotenv.config() import lineByLine from 'n-readlines' import 'reflect-metadata' -import { DataSource } from 'typeorm' import { IdMapping } from './entity/IdMapping' -import { Membership } from './entity/Membership' -import log from './logger' -import { whoami } from './synapse' -import { RcUser, createUser } from './users' +import { RcUser, createUser } from './handlers/users' +import log from './helpers/logger' +import { + createMembership, + getMapping, + initStorage, + save, +} from './helpers/storage' +import { whoami } from './helpers/synapse' +import { RcRoom, createRoom } from './handlers/rooms' log.info('rocketchat2matrix starts.') -const AppDataSource = new DataSource({ - type: 'sqlite', - database: 'db.sqlite', - entities: [IdMapping, Membership], - synchronize: true, - logging: false, -}) - const enum Entities { - Users = 'users.json', - Rooms = 'rocketchat_room.json', - Messages = 'rocketchat_message.json', + Users = 'users', + Rooms = 'rooms', + Messages = 'messages', +} + +type EntityConfig = { + filename: string + mappingType: number +} + +const entities: { [key in Entities]: EntityConfig } = { + users: { + filename: 'users.json', + mappingType: 0, + }, + rooms: { + filename: 'rocketchat_room.json', + mappingType: 1, + }, + messages: { + filename: 'rocketchat_message.json', + mappingType: 2, + }, } async function loadRcExport(entity: Entities) { - const rl = new lineByLine(`./inputs/${entity}`) + const rl = new lineByLine(`./inputs/${entities[entity].filename}`) let line: false | Buffer while ((line = rl.next())) { @@ -34,7 +51,7 @@ async function loadRcExport(entity: Entities) { switch (entity) { case Entities.Users: const rcUser: RcUser = item - log.debug(`Parsing user: ${rcUser.name}: ${rcUser._id}`) + log.info(`Parsing user: ${rcUser.name}: ${rcUser._id}`) // Check for exclusion if ( @@ -45,10 +62,7 @@ async function loadRcExport(entity: Entities) { break } - let mapping = await AppDataSource.manager.findOneBy(IdMapping, { - rcId: rcUser._id, - type: 0, - }) + let mapping = await getMapping(rcUser._id, entities[entity].mappingType) if (mapping && mapping.matrixId) { log.debug('Mapping exists:', mapping) } else { @@ -56,27 +70,43 @@ async function loadRcExport(entity: Entities) { mapping = new IdMapping() mapping.rcId = rcUser._id mapping.matrixId = matrixUser.user_id - mapping.type = 0 + mapping.type = entities[entity].mappingType mapping.accessToken = matrixUser.access_token - AppDataSource.manager.save(mapping) + await save(mapping) log.debug('Mapping added:', mapping) // Add user to room mapping (specific to users) - rcUser.__rooms.forEach(async (rcRoomId: string) => { - const membership = new Membership() - membership.rcRoomId = rcRoomId - membership.rcUserId = rcUser._id - - await AppDataSource.manager.save(membership) - log.debug(`${rcUser.username} membership for ${rcRoomId} created`) - }) + await Promise.all( + rcUser.__rooms.map(async (rcRoomId: string) => { + await createMembership(rcRoomId, rcUser._id) + log.debug(`${rcUser.username} membership for ${rcRoomId} created`) + }) + ) } break case Entities.Rooms: - log.debug(`Room: ${item.name}`) + const rcRoom: RcRoom = item + log.info(`Parsing room ${rcRoom.name || 'with ID: ' + rcRoom._id}`) + + let roomMapping = await getMapping( + rcRoom._id, + entities[entity].mappingType + ) + if (roomMapping && roomMapping.matrixId) { + log.debug('Mapping exists:', roomMapping) + } else { + const matrixRoom = await createRoom(rcRoom) + roomMapping = new IdMapping() + roomMapping.rcId = rcRoom._id + roomMapping.matrixId = matrixRoom.room_id + roomMapping.type = entities[entity].mappingType + + await save(roomMapping) + log.debug('Mapping added:', roomMapping) + } break case Entities.Messages: @@ -92,11 +122,14 @@ async function loadRcExport(entity: Entities) { async function main() { try { await whoami() - await AppDataSource.initialize() + await initStorage() + log.info('Parsing users') await loadRcExport(Entities.Users) + log.info('Parsing rooms') + await loadRcExport(Entities.Rooms) log.info('Done.') } catch (error) { - log.error(`Encountered an error while booting up`) + log.error(`Encountered an error while booting up: ${error}`, error) } } diff --git a/src/entity/IdMapping.ts b/src/entity/IdMapping.ts index f2c00d4..7a3bdc4 100644 --- a/src/entity/IdMapping.ts +++ b/src/entity/IdMapping.ts @@ -11,6 +11,6 @@ export class IdMapping { @Column('integer') type!: number // Type of the entity; 0 = user, 1 = room, 2 = message - @Column() + @Column({ nullable: true }) accessToken?: string // Access token for matrix users } diff --git a/src/handlers/rooms.test.ts b/src/handlers/rooms.test.ts new file mode 100644 index 0000000..7560380 --- /dev/null +++ b/src/handlers/rooms.test.ts @@ -0,0 +1,83 @@ +import { + MatrixRoomPresets, + MatrixRoomVisibility, + RcRoomTypes, + mapRoom, +} from './rooms' + +const roomCreator = { + _id: 'roomcreatorid', + name: 'RoomCreator', + username: 'RoomCreator', + roles: [], + __rooms: [], +} + +test('mapping direct chats', () => { + expect( + mapRoom({ + _id: 'aliceidbobid', + t: RcRoomTypes.direct, + usernames: ['Alice', 'Bob'], + uids: ['aliceid', 'bobid'], + }) + ).toEqual({ + is_direct: true, + preset: MatrixRoomPresets.trusted, + creation_content: { + 'm.federate': false, + }, + _creatorId: 'aliceid', + }) +}) + +test('mapping public rooms', () => { + expect( + mapRoom({ + _id: 'randomRoomId', + fname: 'public', + description: 'Public chat room', + name: 'public', + t: RcRoomTypes.chat, + u: roomCreator, + }) + ).toEqual({ + preset: MatrixRoomPresets.public, + room_alias_name: 'public', + name: 'public', + topic: 'Public chat room', + creation_content: { + 'm.federate': false, + }, + visibility: MatrixRoomVisibility.public, + _creatorId: roomCreator._id, + }) +}) + +test('mapping private rooms', () => { + expect( + mapRoom({ + _id: 'privateRoomId', + name: 'private', + fname: 'private', + description: 'Private chat room', + t: RcRoomTypes.private, + u: roomCreator, + }) + ).toEqual({ + preset: MatrixRoomPresets.private, + room_alias_name: 'private', + name: 'private', + topic: 'Private chat room', + creation_content: { + 'm.federate': false, + }, + _creatorId: roomCreator._id, + }) +}) + +test('mapping live chats', () => { + expect(() => + mapRoom({ _id: 'liveChatId', t: RcRoomTypes.live }) + ).toThrowError('Room type l is unknown') +}) diff --git a/src/handlers/rooms.ts b/src/handlers/rooms.ts new file mode 100644 index 0000000..1f1d214 --- /dev/null +++ b/src/handlers/rooms.ts @@ -0,0 +1,166 @@ +import { IdMapping } from '../entity/IdMapping' +import log from '../helpers/logger' +import { + createMembership, + getMapping, + getMemberships, +} from '../helpers/storage' +import { axios, getUserSessionOptions } from '../helpers/synapse' +import { RcUser } from './users' + +export const enum RcRoomTypes { + direct = 'd', + chat = 'c', + private = 'p', + live = 'l', +} + +export type RcRoom = { + _id: string + t: RcRoomTypes + uids?: string[] + usernames?: string[] + name?: string + u?: RcUser + topic?: string + fname?: string + description?: string +} + +export const enum MatrixRoomPresets { + private = 'private_chat', + public = 'public_chat', + trusted = 'trusted_private_chat', +} + +export const enum MatrixRoomVisibility { + private = 'private', + public = 'public', +} + +export type MatrixRoom = { + room_id?: string + name?: string + creation_content?: object + room_alias_name?: string + topic?: string + is_direct?: boolean + preset?: MatrixRoomPresets + visibility?: MatrixRoomVisibility + _creatorId?: string +} + +export function mapRoom(rcRoom: RcRoom): MatrixRoom { + const room: MatrixRoom = { + creation_content: { + 'm.federate': false, + }, + _creatorId: '', + } + rcRoom.name && (room.name = rcRoom.name) + rcRoom.name && (room.room_alias_name = rcRoom.name) + rcRoom.description && (room.topic = rcRoom.description) + + switch (rcRoom.t) { + case RcRoomTypes.direct: + room.is_direct = true + room.preset = MatrixRoomPresets.trusted + room._creatorId = rcRoom.uids?.[0] || '' + break + + case RcRoomTypes.chat: + room.preset = MatrixRoomPresets.public + room.visibility = MatrixRoomVisibility.public + room._creatorId = rcRoom.u?._id || '' + break + + case RcRoomTypes.private: + room.preset = MatrixRoomPresets.private + room._creatorId = rcRoom.u?._id || '' + break + + case RcRoomTypes.live: + default: + const message = `Room type ${rcRoom.t} is unknown or unimplemented` + log.error(message) + throw new Error(message) + } + if (!room._creatorId) { + log.warn( + `Creator ID could not be determined for room ${rcRoom.name} of type ${rcRoom.t}.` + ) + } + return room +} + +export async function parseMemberships(rcRoom: RcRoom) { + if (rcRoom.t == RcRoomTypes.direct && rcRoom.uids) { + await Promise.all( + [...new Set(rcRoom.uids)] // Deduplicate users + .map(async (uid) => { + await createMembership(rcRoom._id, uid) + log.debug(`${uid} membership in direct chat ${rcRoom._id} created`) + }) + ) + } +} + +export async function createRoom(rcRoom: RcRoom): Promise { + const room: MatrixRoom = mapRoom(rcRoom) + const creatorId = room._creatorId || '' + delete room._creatorId + await parseMemberships(rcRoom) + let sessionOptions = {} + if (room._creatorId) { + try { + sessionOptions = await getUserSessionOptions(creatorId) + log.debug('Room user session generated:', sessionOptions) + } catch (error) { + log.warn(error) + // TODO: Skip room, if it has 0-1 member or is a direct chat? + } + } + log.debug('Creating room:', room) + + room.room_id = ( + await axios.post('/_matrix/client/v3/createRoom', room, sessionOptions) + ).data.room_id + + // TODO: Invite members and let them join + const members = await getMemberships(rcRoom._id) + log.info(`Inviting members to room ${rcRoom._id}:`, members) + + const memberMappings = ( + await Promise.all( + members + .filter((rcMemberId) => rcMemberId != creatorId) + .map(async (rcMemberId) => await getMapping(rcMemberId, 0)) + ) + ) + .filter((mapping): mapping is IdMapping => mapping != null) + .map(async (mapping) => { + log.http(`Invite member ${mapping.rcId} aka. ${mapping.matrixId}`) + await axios.post( + `/_matrix/client/v3/rooms/${room.room_id}/invite`, + { user_id: mapping.matrixId }, + sessionOptions + ) + + log.http( + `Accepting invitation for member ${mapping.rcId} aka. ${mapping.matrixId}` + ) + await axios.post( + `/_matrix/client/v3/join/${room.room_id}`, + {}, + { + headers: { + Authorization: `Bearer ${mapping.accessToken}`, + }, + } + ) + }) + + await Promise.all(memberMappings) + + return room +} diff --git a/src/users.test.ts b/src/handlers/users.test.ts similarity index 94% rename from src/users.test.ts rename to src/handlers/users.test.ts index 2849020..774e404 100644 --- a/src/users.test.ts +++ b/src/handlers/users.test.ts @@ -1,6 +1,12 @@ process.env.REGISTRATION_SHARED_SECRET = 'ThisIsSoSecretWow' import axios from 'axios' -import { MatrixUser, RcUser, createUser, generateHmac, mapUser } from './users' +import { + MatrixUser, + RcUser, + createUser, + generateHmac, + mapUser, +} from '../handlers/users' jest.mock('axios') const mockedAxios = axios as jest.Mocked diff --git a/src/users.ts b/src/handlers/users.ts similarity index 95% rename from src/users.ts rename to src/handlers/users.ts index 0c4e02d..c6867ed 100644 --- a/src/users.ts +++ b/src/handlers/users.ts @@ -1,6 +1,6 @@ -import log from './logger' -import { axios } from './synapse' import { createHmac } from 'node:crypto' +import log from '../helpers/logger' +import { axios } from '../helpers/synapse' export type RcUser = { _id: string diff --git a/src/logger.ts b/src/helpers/logger.ts similarity index 100% rename from src/logger.ts rename to src/helpers/logger.ts diff --git a/src/helpers/storage.ts b/src/helpers/storage.ts new file mode 100644 index 0000000..3b8c8d3 --- /dev/null +++ b/src/helpers/storage.ts @@ -0,0 +1,51 @@ +import { DataSource } from 'typeorm' +import { IdMapping } from '../entity/IdMapping' +import { Membership } from '../entity/Membership' + +const AppDataSource = new DataSource({ + type: 'sqlite', + database: 'db.sqlite', + entities: [IdMapping, Membership], + synchronize: true, + logging: false, +}) + +export async function initStorage() { + await AppDataSource.initialize() +} + +export function getMapping(id: string, type: number) { + return AppDataSource.manager.findOneBy(IdMapping, { + rcId: id, + type: type, + }) +} + +export async function save(entity: IdMapping | Membership) { + await AppDataSource.manager.save(entity) +} + +export async function getAccessToken(id: string) { + return (await getMapping(id, 0))?.accessToken +} + +export async function createMembership(rcRoomId: string, rcUserId: string) { + const membership = new Membership() + membership.rcRoomId = rcRoomId + membership.rcUserId = rcUserId + + await save(membership) +} + +export async function getMemberships(rcRoomId: string) { + return ( + await AppDataSource.manager.find(Membership, { + select: { + rcUserId: true, + }, + where: { + rcRoomId: rcRoomId, + }, + }) + ).map((entity) => entity.rcUserId) +} diff --git a/src/synapse.ts b/src/helpers/synapse.ts similarity index 55% rename from src/synapse.ts rename to src/helpers/synapse.ts index 0673cf6..aaad424 100644 --- a/src/synapse.ts +++ b/src/helpers/synapse.ts @@ -1,9 +1,10 @@ -import { access_token } from './config/synapse_access_token.json' import axios from 'axios' +import { access_token } from '../config/synapse_access_token.json' import log from './logger' +import { getAccessToken } from './storage' axios.defaults.baseURL = 'http://localhost:8008' -axios.defaults.headers.common['Authorization'] = ` Bearer ${access_token}` +axios.defaults.headers.common['Authorization'] = `Bearer ${access_token}` axios.defaults.headers.post['Content-Type'] = 'application/json' export { default as axios } from 'axios' @@ -20,3 +21,11 @@ export const whoami = () => reject() }) }) + +export async function getUserSessionOptions(id: string) { + const accessToken = await getAccessToken(id) + if (!accessToken) { + throw new Error(`Could not retrieve access token for ID ${id}`) + } + return { headers: { Authorization: `Bearer ${accessToken}` } } +}