Merge pull request 'Implement Room Workflow' (#16) from implement-rooms into main

Reviewed-on: https://git.verdigado.com/NB-Public/rocketchat2matrix/pulls/16

README.md

@@ -18,14 +18,44 @@ Export them to `inputs/`
 
 ```shell
 docker-compose run --rm -e SYNAPSE_SERVER_NAME=my.matrix.host -e SYNAPSE_REPORT_STATS=no synapse generate
+```
+
+To run the script without hitting rate limiting, you SHOULD add the following options to the freshly generated `files/homeserver.yaml`. **Do not leave these in the production setup!**
+
+```yaml
+rc_joins:
+  local:
+    per_second: 1024
+    burst_count: 2048
+rc_joins_per_room:
+  per_second: 1024
+  burst_count: 2048
+rc_message:
+  per_second: 1024
+  burst_count: 2048
+rc_invites:
+  per_room:
+    per_second: 1024
+    burst_count: 2048
+  per_user:
+    per_second: 1024
+    burst_count: 2048
+  per_issuer:
+    per_second: 1024
+    burst_count: 2048
+```
+
+Continue setting up the server:
+
+```shell
 docker-compose up -d
-# Register a admin user
+# Wait for the Server to boot, then register an admin user
 docker-compose exec -it synapse register_new_matrix_user http://localhost:8008 -c /data/homeserver.yaml --admin --user verdiadmin --password verdiadmin
 ```
 
 Then you can access the homeserver in [Element Web](https://app.element.io/#/login) or the [local admin interface](http://localhost:8080) as `http://localhost:8008` with the `verdiadmin` as username AND password.
 
-You can store an access token for that user:
+Store an access token for that user:
 
 ```shell
 curl --request POST \
@@ -35,6 +65,8 @@ curl --request POST \
 > src/config/synapse_access_token.json
 ```
 
+To finally run the script, execute it via `npm start`.
+
 ## Configuration
 
 Copy over `.env.example` to `.env` and insert your values.
@@ -45,9 +77,10 @@ Copy over `.env.example` to `.env` and insert your values.
 
 ## Cleaning Up
 
-To clean up the Synapse server and loal storage database, run (while the containers are stopped)
+To clean up the Synapse server and local storage database, run either the convenience script `./reset.sh` or start with:
 
 ```shell
+docker-compose down
 sudo rm files/homeserver.db
 rm db.sqlite
 ```

package.json

@@ -21,7 +21,7 @@
     "lint-fix": "eslint src/ --fix --ext .ts",
     "prefix": "npm run format-fix",
     "fix": "npm run lint-fix",
-    "test": "jest",
+    "test": "rm -rf dist/ && jest",
     "compile": "rm -rf dist/ && tsc",
     "start": "npm run compile && node dist/app.js",
     "prepare": "husky install"

reset.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+HOMESERVER="http://localhost:8008"
+
+docker-compose down
+sudo rm -f files/homeserver.db
+rm -f db.sqlite
+docker-compose up -d
+
+sleep 1.5
+set +e
+until docker-compose exec -it synapse register_new_matrix_user $HOMESERVER -c /data/homeserver.yaml --admin --user verdiadmin --password verdiadmin &> /dev/null
+do
+  echo 'Retrying creating admin...'
+done
+set -e
+
+curl --request POST \
+  --url $HOMESERVER/_matrix/client/v3/login \
+  --header 'Content-Type: application/json' \
+  --data '{"type": "m.login.password","user": "verdiadmin","password": "verdiadmin","device_id": "DEV"}' \
+> src/config/synapse_access_token.json 2> /dev/null

src/app.ts

@@ -2,31 +2,48 @@ import dotenv from 'dotenv'
 dotenv.config()
 import lineByLine from 'n-readlines'
 import 'reflect-metadata'
-import { DataSource } from 'typeorm'
 import { IdMapping } from './entity/IdMapping'
-import { Membership } from './entity/Membership'
+import { RcUser, createUser } from './handlers/users'
-import log from './logger'
+import log from './helpers/logger'
-import { whoami } from './synapse'
+import {
-import { RcUser, createUser } from './users'
+  createMembership,
+  getMapping,
+  initStorage,
+  save,
+} from './helpers/storage'
+import { whoami } from './helpers/synapse'
+import { RcRoom, createRoom } from './handlers/rooms'
 
 log.info('rocketchat2matrix starts.')
 
-const AppDataSource = new DataSource({
-  type: 'sqlite',
-  database: 'db.sqlite',
-  entities: [IdMapping, Membership],
-  synchronize: true,
-  logging: false,
-})
-
 const enum Entities {
-  Users = 'users.json',
+  Users = 'users',
-  Rooms = 'rocketchat_room.json',
+  Rooms = 'rooms',
-  Messages = 'rocketchat_message.json',
+  Messages = 'messages',
+}
+
+type EntityConfig = {
+  filename: string
+  mappingType: number
+}
+
+const entities: { [key in Entities]: EntityConfig } = {
+  users: {
+    filename: 'users.json',
+    mappingType: 0,
+  },
+  rooms: {
+    filename: 'rocketchat_room.json',
+    mappingType: 1,
+  },
+  messages: {
+    filename: 'rocketchat_message.json',
+    mappingType: 2,
+  },
 }
 
 async function loadRcExport(entity: Entities) {
-  const rl = new lineByLine(`./inputs/${entity}`)
+  const rl = new lineByLine(`./inputs/${entities[entity].filename}`)
 
   let line: false | Buffer
   while ((line = rl.next())) {
@@ -34,7 +51,7 @@ async function loadRcExport(entity: Entities) {
     switch (entity) {
       case Entities.Users:
         const rcUser: RcUser = item
-        log.debug(`Parsing user: ${rcUser.name}: ${rcUser._id}`)
+        log.info(`Parsing user: ${rcUser.name}: ${rcUser._id}`)
 
         // Check for exclusion
         if (
@@ -45,10 +62,7 @@ async function loadRcExport(entity: Entities) {
           break
         }
 
-        let mapping = await AppDataSource.manager.findOneBy(IdMapping, {
+        let mapping = await getMapping(rcUser._id, entities[entity].mappingType)
-          rcId: rcUser._id,
-          type: 0,
-        })
         if (mapping && mapping.matrixId) {
           log.debug('Mapping exists:', mapping)
         } else {
@@ -56,27 +70,43 @@ async function loadRcExport(entity: Entities) {
           mapping = new IdMapping()
           mapping.rcId = rcUser._id
           mapping.matrixId = matrixUser.user_id
-          mapping.type = 0
+          mapping.type = entities[entity].mappingType
           mapping.accessToken = matrixUser.access_token
 
-          AppDataSource.manager.save(mapping)
+          await save(mapping)
           log.debug('Mapping added:', mapping)
 
           // Add user to room mapping (specific to users)
-          rcUser.__rooms.forEach(async (rcRoomId: string) => {
+          await Promise.all(
-            const membership = new Membership()
+            rcUser.__rooms.map(async (rcRoomId: string) => {
-            membership.rcRoomId = rcRoomId
+              await createMembership(rcRoomId, rcUser._id)
-            membership.rcUserId = rcUser._id
+              log.debug(`${rcUser.username} membership for ${rcRoomId} created`)
-
+            })
-            await AppDataSource.manager.save(membership)
+          )
-            log.debug(`${rcUser.username} membership for ${rcRoomId} created`)
-          })
         }
 
         break
 
       case Entities.Rooms:
-        log.debug(`Room: ${item.name}`)
+        const rcRoom: RcRoom = item
+        log.info(`Parsing room ${rcRoom.name || 'with ID: ' + rcRoom._id}`)
+
+        let roomMapping = await getMapping(
+          rcRoom._id,
+          entities[entity].mappingType
+        )
+        if (roomMapping && roomMapping.matrixId) {
+          log.debug('Mapping exists:', roomMapping)
+        } else {
+          const matrixRoom = await createRoom(rcRoom)
+          roomMapping = new IdMapping()
+          roomMapping.rcId = rcRoom._id
+          roomMapping.matrixId = matrixRoom.room_id
+          roomMapping.type = entities[entity].mappingType
+
+          await save(roomMapping)
+          log.debug('Mapping added:', roomMapping)
+        }
         break
 
       case Entities.Messages:
@@ -92,11 +122,14 @@ async function loadRcExport(entity: Entities) {
 async function main() {
   try {
     await whoami()
-    await AppDataSource.initialize()
+    await initStorage()
+    log.info('Parsing users')
     await loadRcExport(Entities.Users)
+    log.info('Parsing rooms')
+    await loadRcExport(Entities.Rooms)
     log.info('Done.')
   } catch (error) {
-    log.error(`Encountered an error while booting up`)
+    log.error(`Encountered an error while booting up: ${error}`, error)
   }
 }
 

src/entity/IdMapping.ts

@@ -11,6 +11,6 @@ export class IdMapping {
   @Column('integer')
   type!: number // Type of the entity; 0 = user, 1 = room, 2 = message
 
-  @Column()
+  @Column({ nullable: true })
   accessToken?: string // Access token for matrix users
 }

src/handlers/rooms.test.ts

@@ -0,0 +1,83 @@
+import {
+  MatrixRoomPresets,
+  MatrixRoomVisibility,
+  RcRoomTypes,
+  mapRoom,
+} from './rooms'
+
+const roomCreator = {
+  _id: 'roomcreatorid',
+  name: 'RoomCreator',
+  username: 'RoomCreator',
+  roles: [],
+  __rooms: [],
+}
+
+test('mapping direct chats', () => {
+  expect(
+    mapRoom({
+      _id: 'aliceidbobid',
+      t: RcRoomTypes.direct,
+      usernames: ['Alice', 'Bob'],
+      uids: ['aliceid', 'bobid'],
+    })
+  ).toEqual({
+    is_direct: true,
+    preset: MatrixRoomPresets.trusted,
+    creation_content: {
+      'm.federate': false,
+    },
+    _creatorId: 'aliceid',
+  })
+})
+
+test('mapping public rooms', () => {
+  expect(
+    mapRoom({
+      _id: 'randomRoomId',
+      fname: 'public',
+      description: 'Public chat room',
+      name: 'public',
+      t: RcRoomTypes.chat,
+      u: roomCreator,
+    })
+  ).toEqual({
+    preset: MatrixRoomPresets.public,
+    room_alias_name: 'public',
+    name: 'public',
+    topic: 'Public chat room',
+    creation_content: {
+      'm.federate': false,
+    },
+    visibility: MatrixRoomVisibility.public,
+    _creatorId: roomCreator._id,
+  })
+})
+
+test('mapping private rooms', () => {
+  expect(
+    mapRoom({
+      _id: 'privateRoomId',
+      name: 'private',
+      fname: 'private',
+      description: 'Private chat room',
+      t: RcRoomTypes.private,
+      u: roomCreator,
+    })
+  ).toEqual({
+    preset: MatrixRoomPresets.private,
+    room_alias_name: 'private',
+    name: 'private',
+    topic: 'Private chat room',
+    creation_content: {
+      'm.federate': false,
+    },
+    _creatorId: roomCreator._id,
+  })
+})
+
+test('mapping live chats', () => {
+  expect(() =>
+    mapRoom({ _id: 'liveChatId', t: RcRoomTypes.live })
+  ).toThrowError('Room type l is unknown')
+})

src/handlers/rooms.ts

@@ -0,0 +1,166 @@
+import { IdMapping } from '../entity/IdMapping'
+import log from '../helpers/logger'
+import {
+  createMembership,
+  getMapping,
+  getMemberships,
+} from '../helpers/storage'
+import { axios, getUserSessionOptions } from '../helpers/synapse'
+import { RcUser } from './users'
+
+export const enum RcRoomTypes {
+  direct = 'd',
+  chat = 'c',
+  private = 'p',
+  live = 'l',
+}
+
+export type RcRoom = {
+  _id: string
+  t: RcRoomTypes
+  uids?: string[]
+  usernames?: string[]
+  name?: string
+  u?: RcUser
+  topic?: string
+  fname?: string
+  description?: string
+}
+
+export const enum MatrixRoomPresets {
+  private = 'private_chat',
+  public = 'public_chat',
+  trusted = 'trusted_private_chat',
+}
+
+export const enum MatrixRoomVisibility {
+  private = 'private',
+  public = 'public',
+}
+
+export type MatrixRoom = {
+  room_id?: string
+  name?: string
+  creation_content?: object
+  room_alias_name?: string
+  topic?: string
+  is_direct?: boolean
+  preset?: MatrixRoomPresets
+  visibility?: MatrixRoomVisibility
+  _creatorId?: string
+}
+
+export function mapRoom(rcRoom: RcRoom): MatrixRoom {
+  const room: MatrixRoom = {
+    creation_content: {
+      'm.federate': false,
+    },
+    _creatorId: '',
+  }
+  rcRoom.name && (room.name = rcRoom.name)
+  rcRoom.name && (room.room_alias_name = rcRoom.name)
+  rcRoom.description && (room.topic = rcRoom.description)
+
+  switch (rcRoom.t) {
+    case RcRoomTypes.direct:
+      room.is_direct = true
+      room.preset = MatrixRoomPresets.trusted
+      room._creatorId = rcRoom.uids?.[0] || ''
+      break
+
+    case RcRoomTypes.chat:
+      room.preset = MatrixRoomPresets.public
+      room.visibility = MatrixRoomVisibility.public
+      room._creatorId = rcRoom.u?._id || ''
+      break
+
+    case RcRoomTypes.private:
+      room.preset = MatrixRoomPresets.private
+      room._creatorId = rcRoom.u?._id || ''
+      break
+
+    case RcRoomTypes.live:
+    default:
+      const message = `Room type ${rcRoom.t} is unknown or unimplemented`
+      log.error(message)
+      throw new Error(message)
+  }
+  if (!room._creatorId) {
+    log.warn(
+      `Creator ID could not be determined for room ${rcRoom.name} of type ${rcRoom.t}.`
+    )
+  }
+  return room
+}
+
+export async function parseMemberships(rcRoom: RcRoom) {
+  if (rcRoom.t == RcRoomTypes.direct && rcRoom.uids) {
+    await Promise.all(
+      [...new Set(rcRoom.uids)] // Deduplicate users
+        .map(async (uid) => {
+          await createMembership(rcRoom._id, uid)
+          log.debug(`${uid} membership in direct chat ${rcRoom._id} created`)
+        })
+    )
+  }
+}
+
+export async function createRoom(rcRoom: RcRoom): Promise<MatrixRoom> {
+  const room: MatrixRoom = mapRoom(rcRoom)
+  const creatorId = room._creatorId || ''
+  delete room._creatorId
+  await parseMemberships(rcRoom)
+  let sessionOptions = {}
+  if (room._creatorId) {
+    try {
+      sessionOptions = await getUserSessionOptions(creatorId)
+      log.debug('Room user session generated:', sessionOptions)
+    } catch (error) {
+      log.warn(error)
+      // TODO: Skip room, if it has 0-1 member or is a direct chat?
+    }
+  }
+  log.debug('Creating room:', room)
+
+  room.room_id = (
+    await axios.post('/_matrix/client/v3/createRoom', room, sessionOptions)
+  ).data.room_id
+
+  // TODO: Invite members and let them join
+  const members = await getMemberships(rcRoom._id)
+  log.info(`Inviting members to room ${rcRoom._id}:`, members)
+
+  const memberMappings = (
+    await Promise.all(
+      members
+        .filter((rcMemberId) => rcMemberId != creatorId)
+        .map(async (rcMemberId) => await getMapping(rcMemberId, 0))
+    )
+  )
+    .filter((mapping): mapping is IdMapping => mapping != null)
+    .map(async (mapping) => {
+      log.http(`Invite member ${mapping.rcId} aka. ${mapping.matrixId}`)
+      await axios.post(
+        `/_matrix/client/v3/rooms/${room.room_id}/invite`,
+        { user_id: mapping.matrixId },
+        sessionOptions
+      )
+
+      log.http(
+        `Accepting invitation for member ${mapping.rcId} aka. ${mapping.matrixId}`
+      )
+      await axios.post(
+        `/_matrix/client/v3/join/${room.room_id}`,
+        {},
+        {
+          headers: {
+            Authorization: `Bearer ${mapping.accessToken}`,
+          },
+        }
+      )
+    })
+
+  await Promise.all(memberMappings)
+
+  return room
+}

src/handlers/users.test.ts

@@ -1,6 +1,12 @@
 process.env.REGISTRATION_SHARED_SECRET = 'ThisIsSoSecretWow'
 import axios from 'axios'
-import { MatrixUser, RcUser, createUser, generateHmac, mapUser } from './users'
+import {
+  MatrixUser,
+  RcUser,
+  createUser,
+  generateHmac,
+  mapUser,
+} from '../handlers/users'
 
 jest.mock('axios')
 const mockedAxios = axios as jest.Mocked<typeof axios>

src/handlers/users.ts

@@ -1,6 +1,6 @@
-import log from './logger'
-import { axios } from './synapse'
 import { createHmac } from 'node:crypto'
+import log from '../helpers/logger'
+import { axios } from '../helpers/synapse'
 
 export type RcUser = {
   _id: string

src/helpers/storage.ts

@@ -0,0 +1,51 @@
+import { DataSource } from 'typeorm'
+import { IdMapping } from '../entity/IdMapping'
+import { Membership } from '../entity/Membership'
+
+const AppDataSource = new DataSource({
+  type: 'sqlite',
+  database: 'db.sqlite',
+  entities: [IdMapping, Membership],
+  synchronize: true,
+  logging: false,
+})
+
+export async function initStorage() {
+  await AppDataSource.initialize()
+}
+
+export function getMapping(id: string, type: number) {
+  return AppDataSource.manager.findOneBy(IdMapping, {
+    rcId: id,
+    type: type,
+  })
+}
+
+export async function save(entity: IdMapping | Membership) {
+  await AppDataSource.manager.save(entity)
+}
+
+export async function getAccessToken(id: string) {
+  return (await getMapping(id, 0))?.accessToken
+}
+
+export async function createMembership(rcRoomId: string, rcUserId: string) {
+  const membership = new Membership()
+  membership.rcRoomId = rcRoomId
+  membership.rcUserId = rcUserId
+
+  await save(membership)
+}
+
+export async function getMemberships(rcRoomId: string) {
+  return (
+    await AppDataSource.manager.find(Membership, {
+      select: {
+        rcUserId: true,
+      },
+      where: {
+        rcRoomId: rcRoomId,
+      },
+    })
+  ).map((entity) => entity.rcUserId)
+}

src/helpers/synapse.ts

@@ -1,9 +1,10 @@
-import { access_token } from './config/synapse_access_token.json'
 import axios from 'axios'
+import { access_token } from '../config/synapse_access_token.json'
 import log from './logger'
+import { getAccessToken } from './storage'
 
 axios.defaults.baseURL = 'http://localhost:8008'
-axios.defaults.headers.common['Authorization'] = ` Bearer ${access_token}`
+axios.defaults.headers.common['Authorization'] = `Bearer ${access_token}`
 axios.defaults.headers.post['Content-Type'] = 'application/json'
 
 export { default as axios } from 'axios'
@@ -20,3 +21,11 @@ export const whoami = () =>
         reject()
       })
   })
+
+export async function getUserSessionOptions(id: string) {
+  const accessToken = await getAccessToken(id)
+  if (!accessToken) {
+    throw new Error(`Could not retrieve access token for ID ${id}`)
+  }
+  return { headers: { Authorization: `Bearer ${accessToken}` } }
+}