Merge pull request 'Handle event types' (#49) from event-based into main

Reviewed-on: https://git.verdigado.com/NB-Public/rocketchat2matrix/pulls/49

.env.example

@@ -1,3 +1,4 @@
 REGISTRATION_SHARED_SECRET='look in your synapses homeserver.yaml'
 AS_TOKEN='look in the app-service.yaml'
 EXCLUDED_USERS='rocket.cat' # Comma-separated list of usernames or IDs
+ADMIN_USERNAME='admin' # The login name of the Rocket.Chat admin

reset.sh

@@ -4,12 +4,14 @@ IFS=$'\n\t'
 
 HOMESERVER="http://localhost:8008"
 
+echo 'Resetting containers and databases'
 docker-compose down
 sudo rm -f files/homeserver.db
 rm -f db.sqlite
 docker-compose up -d
 
 sleep 1.5
+echo 'Creating admin user'
 set +e
 until docker-compose exec -it synapse register_new_matrix_user $HOMESERVER -c /data/homeserver.yaml --admin --user verdiadmin --password verdiadmin &> /dev/null
 do
@@ -17,8 +19,14 @@ do
 done
 set -e
 
+echo 'Saving admin access token'
 curl --request POST \
   --url $HOMESERVER/_matrix/client/v3/login \
   --header 'Content-Type: application/json' \
   --data '{"type": "m.login.password","user": "verdiadmin","password": "verdiadmin","device_id": "DEV"}' \
 > src/config/synapse_access_token.json 2> /dev/null
+
+echo 'Removing log files'
+rm ./*.log
+
+echo 'Done.'

src/app.ts

@@ -1,18 +1,26 @@
 import dotenv from 'dotenv'
 dotenv.config()
+import { AxiosError } from 'axios'
 import lineByLine from 'n-readlines'
 import 'reflect-metadata'
-import { handle as handleRoom } from './handlers/rooms'
-import { handle as handleUser } from './handlers/users'
-import { handle as handleMessage } from './handlers/messages'
-import log from './helpers/logger'
-import { initStorage } from './helpers/storage'
-import { whoami } from './helpers/synapse'
 import { Entity, entities } from './Entities'
-import { AxiosError } from 'axios'
+import { handle as handleMessage } from './handlers/messages'
+import { getFilteredMembers, handle as handleRoom } from './handlers/rooms'
+import { handle as handleUser } from './handlers/users'
+import log from './helpers/logger'
+import {
+  getAllMappingsByType,
+  getMappingByMatrixId,
+  getMemberships,
+  initStorage,
+} from './helpers/storage'
+import { axios, formatUserSessionOptions, whoami } from './helpers/synapse'
+
+const applicationServiceToken = process.env.AS_TOKEN || ''
 
 log.info('rocketchat2matrix starts.')
 
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
 async function loadRcExport(entity: Entity) {
   const rl = new lineByLine(`./inputs/${entities[entity].filename}`)
 
@@ -38,6 +46,59 @@ async function loadRcExport(entity: Entity) {
   }
 }
 
+async function removeExcessRoomMembers() {
+  const roomMappings = await getAllMappingsByType(
+    entities[Entity.Rooms].mappingType
+  )
+  if (!roomMappings) {
+    throw new Error(`No room mappings found`)
+  }
+
+  roomMappings.forEach(async (roomMapping) => {
+    log.info(
+      `Checking memberships for room ${roomMapping.rcId} / ${roomMapping.matrixId}:`
+    )
+    // get all memberships from db
+    const rcMemberIds = await getMemberships(roomMapping.rcId)
+    const memberMappings = await getFilteredMembers(rcMemberIds, '')
+    const memberNames: string[] = memberMappings.map(
+      (memberMapping) => memberMapping.matrixId || ''
+    )
+    // get each mx rooms' mx users
+    const actualMembers: string[] = Object.keys(
+      (
+        await axios.get(
+          `/_matrix/client/v3/rooms/${roomMapping.matrixId}/joined_members`,
+          formatUserSessionOptions(applicationServiceToken)
+        )
+      ).data.joined
+    )
+
+    // do action for any user in mx, but not in rc
+    await Promise.all(
+      actualMembers.map(async (actualMember) => {
+        if (!memberNames.includes(actualMember)) {
+          log.warn(
+            `Member ${actualMember} should not be in room ${roomMapping.matrixId}, removing`
+          )
+          const memberMapping = await getMappingByMatrixId(actualMember)
+          if (!memberMapping || !memberMapping.accessToken) {
+            throw new Error(
+              `Could not find access token for member ${actualMember}, this is a bug`
+            )
+          }
+
+          await axios.post(
+            `/_matrix/client/v3/rooms/${roomMapping.matrixId}/leave`,
+            {},
+            formatUserSessionOptions(memberMapping.accessToken)
+          )
+        }
+      })
+    )
+  })
+}
+
 async function main() {
   try {
     await whoami()
@@ -48,6 +109,9 @@ async function main() {
     await loadRcExport(Entity.Rooms)
     log.info('Parsing messages')
     await loadRcExport(Entity.Messages)
+    log.info('Checking room memberships')
+    await removeExcessRoomMembers()
+
     log.info('Done.')
   } catch (error) {
     if (error instanceof AxiosError) {

src/functions.js

@@ -1,93 +0,0 @@
-postToMatrix (endpoint, payload) {}
-mapUserId (id) {}
-mapChannelId (id) {}
-mapMessageId (id) {}
-generateHmac(user) {}
-
-mapRoom (rcRoom) {
-  const room = {
-    creation_content: {
-      'm.federate': false
-    },
-    name: rcRoom.name,
-    room_alias_name: rcRoom.name,
-    topic: rcRoom.description,
-    // TODO: Invite users (Rate Limit?)
-      // POST /_matrix/client/v3/rooms/{roomId}/invite
-      // {
-      //   "reason": "Welcome to the team!",
-      //   "user_id": "@cheeky_monkey:matrix.org"
-      // }
-  }
-
-  switch (rcRoom.t) {
-    case 'd':
-      room.is_direct = true
-      break;
-
-    case 'c':
-      room.preset = 'public_chat'
-      break;
-
-      case 'p':
-      room.preset = 'private_chat'
-      break;
-
-    default:
-      // log; 'l' for livechat, anything else is undefined
-      break;
-  }
-  // POST /_matrix/client/v3/createRoom
-}
-
-mapUser (rcUser) {
-  return {
-    'nonce': '',
-    'username': rcUser.username,
-    'displayname': rcUser.name,
-    'password': '',
-    'admin': rcUser.roles.includes('admin'),
-    'mac': '',
-   }
-}
-
-getUserRegisterNonce () {} // GET /_synapse/admin/v1/register
-
-createUser (rcUser) {
-  const user = mapUser(rcUser)
-  user.nonce = getUserRegisterNonce()
-  user.mac = generateHmac(user)
-  const mUser = postToMatrix('/_synapse/admin/v1/register', user) // POST /_synapse/admin/v1/register
-
-  // rcUser.__rooms.map(mapChannelId)
-  return mUser
-}
-
-mapMessage (rcMessage) {
-  const message = {
-    'content': {
-      'body': rc.msg,
-      // 'format': 'org.matrix.custom.html',
-      // 'formatted_body': '<b>This is an example text message</b>',
-      'msgtype': 'm.text',
-    },
-    'event_id': '$143273582443PhrSn:example.org', // TODO: ??
-    'origin_server_ts': new Date(rc.t.$date).valueOf(),
-    'room_id': mapChannelId(rcMessage.rid),
-    'sender': mapUserId(rc.u._id),
-    'type': 'm.room.message',
-    'unsigned': {
-      'age': 1234, // TODO: ??
-    },
-  }
-  // TODO: Other media types
-
-  if (rc.tmid) { // If it is a thread reply
-    message.content['m.relates_to'] = {
-      rel_type: 'm.thread',
-      event_id: mapMessageId(rc.tmid),
-    }
-  }
-
-  return message
-}

src/handlers/messages.ts

@@ -1,8 +1,17 @@
+import { AxiosError } from 'axios'
 import { Entity, entities } from '../Entities'
 import { IdMapping } from '../entity/IdMapping'
 import log from '../helpers/logger'
-import { getMessageId, getRoomId, getUserId, save } from '../helpers/storage'
+import {
+  getMapping,
+  getMappingByMatrixId,
+  getMessageId,
+  getRoomId,
+  getUserId,
+  save,
+} from '../helpers/storage'
 import { axios, formatUserSessionOptions } from '../helpers/synapse'
+import { acceptInvitation, inviteMember } from './rooms'
 
 const applicationServiceToken = process.env.AS_TOKEN || ''
 if (!applicationServiceToken) {
@@ -93,11 +102,6 @@ export async function handle(rcMessage: RcMessage): Promise<void> {
     return
   }
 
-  if (rcMessage.t) {
-    log.warn(`Message ${rcMessage._id} is of type ${rcMessage.t}, skipping.`)
-    return
-  }
-
   const room_id = await getRoomId(rcMessage.rid)
   if (!room_id) {
     log.warn(
@@ -106,6 +110,71 @@ export async function handle(rcMessage: RcMessage): Promise<void> {
     return
   }
 
+  if (rcMessage.t) {
+    switch (rcMessage.t) {
+      case 'ru': // User removed by
+      case 'ul': // User left
+      case 'ult': // User left team
+      case 'removed-user-from-team': // Removed user from team
+        log.info(
+          `Message ${rcMessage._id} is of type ${rcMessage.t}, removing member ${rcMessage.msg} from room ${room_id}`
+        )
+
+        const members = (
+          await axios.get(
+            `/_matrix/client/v3/rooms/${room_id}/joined_members`,
+            formatUserSessionOptions(applicationServiceToken)
+          )
+        ).data.joined
+        if (!members) {
+          const errorMessage = `Could not determine members of room ${room_id}, aborting`
+          log.error(errorMessage)
+          throw new Error(errorMessage)
+        }
+
+        const matrixUser =
+          Object.keys(members).find((key) =>
+            key.includes(rcMessage.msg.toLowerCase())
+          ) || ''
+
+        const userMapping = await getMappingByMatrixId(matrixUser)
+        if (!userMapping?.accessToken) {
+          log.warn(
+            `Could not get access token for ${rcMessage.msg}, maybe user is not a member, skipping.`
+          )
+          return
+        }
+
+        log.http(`User ${matrixUser} leaves room ${room_id}`)
+        await axios.post(
+          `/_matrix/client/v3/rooms/${room_id}/leave`,
+          { reason: `Event type ${rcMessage.t}` },
+          formatUserSessionOptions(userMapping.accessToken)
+        )
+        return
+
+      case 'uj': // User joined channel
+      case 'ujt': // User joined team
+      case 'ut': // User joined conversation
+
+      case 'au': // User added by
+      case 'added-user-to-team': // Added user to team
+      case 'r': // Room name changed
+      case 'rm': // Message removed
+        log.warn(
+          `Message ${rcMessage._id} is of type ${rcMessage.t}, for which Rocket.Chat does not provide the initial state information, skipping.`
+        )
+        return
+
+      case 'user-muted': // User muted by
+      default:
+        log.warn(
+          `Message ${rcMessage._id} is of unhandled type ${rcMessage.t}, skipping.`
+        )
+        return
+    }
+  }
+
   const user_id = await getUserId(rcMessage.u._id)
   if (!user_id) {
     log.warn(
@@ -134,13 +203,66 @@ export async function handle(rcMessage: RcMessage): Promise<void> {
     }
   }
 
-  const event_id = await createMessage(
-    matrixMessage,
-    room_id,
-    user_id,
-    ts,
-    rcMessage._id
-  )
+  try {
+    const event_id = await createMessage(
+      matrixMessage,
+      room_id,
+      user_id,
+      ts,
+      rcMessage._id
+    )
+    await createMapping(rcMessage._id, event_id)
+  } catch (error) {
+    if (
+      error instanceof AxiosError &&
+      error.response &&
+      error.response.data.errcode === 'M_FORBIDDEN' &&
+      error.response.data.error === `User ${user_id} not in room ${room_id}`
+    ) {
+      log.info(error.response.data.error + ', adding.')
 
-  createMapping(rcMessage._id, event_id)
+      const userMapping = await getMapping(
+        rcMessage.u._id,
+        entities[Entity.Users].mappingType
+      )
+      if (!userMapping || !userMapping.matrixId || !userMapping.accessToken) {
+        log.warn(`Could not determine joining user, skipping.`, rcMessage)
+        return
+      }
+
+      // Get room creator session or use empty axios options
+      let userSessionOptions = {}
+      const roomCreatorId = (
+        await axios.get(`/_synapse/admin/v1/rooms/${room_id}`)
+      ).data.creator
+      if (!roomCreatorId) {
+        log.warn(
+          `Could not determine room creator for room ${room_id}, using admin credentials.`
+        )
+      } else {
+        const creatorMapping = await getMappingByMatrixId(roomCreatorId)
+        if (!creatorMapping?.accessToken) {
+          log.warn(`Could not access token for ${roomCreatorId}, skipping.`)
+          return
+        }
+        userSessionOptions = formatUserSessionOptions(
+          creatorMapping.accessToken
+        )
+      }
+
+      await inviteMember(userMapping.matrixId, room_id, userSessionOptions)
+      await acceptInvitation(userMapping, room_id)
+
+      const event_id = await createMessage(
+        matrixMessage,
+        room_id,
+        user_id,
+        ts,
+        rcMessage._id
+      )
+      await createMapping(rcMessage._id, event_id)
+    } else {
+      throw error
+    }
+  }
 }

src/handlers/rooms.test.ts

@@ -1,5 +1,6 @@
 import { expect, jest, test } from '@jest/globals'
 import axios from 'axios'
+import { Entity, entities } from '../Entities'
 import { IdMapping } from '../entity/IdMapping'
 import * as storage from '../helpers/storage'
 import { SessionOptions } from '../helpers/synapse'
@@ -9,15 +10,14 @@ import {
   RcRoom,
   RcRoomTypes,
   acceptInvitation,
+  createDirectChatMemberships,
   createMapping,
   getCreator,
   getFilteredMembers,
   inviteMember,
   mapRoom,
-  createDirectChatMemberships,
   registerRoom,
 } from './rooms'
-import { Entity, entities } from '../Entities'
 
 jest.mock('axios')
 const mockedAxios = axios as jest.Mocked<typeof axios>
@@ -106,7 +106,9 @@ test('mapping private rooms', () => {
 test('mapping live chats', () => {
   expect(() =>
     mapRoom({ _id: 'liveChatId', t: RcRoomTypes.live })
-  ).toThrowError('Room type l is unknown')
+  ).toThrowError(
+    'Room with ID: liveChatId is a live chat. Migration not implemented'
+  )
 })
 
 test('getting creator', () => {

src/handlers/rooms.ts

@@ -1,3 +1,4 @@
+import { AxiosError } from 'axios'
 import { Entity, entities } from '../Entities'
 import { IdMapping } from '../entity/IdMapping'
 import log from '../helpers/logger'
@@ -83,10 +84,18 @@ export function mapRoom(rcRoom: RcRoom): MatrixRoom {
       break
 
     case RcRoomTypes.live:
+      const messageLivechat = `Room ${
+        rcRoom.name || 'with ID: ' + rcRoom._id
+      } is a live chat. Migration not implemented`
+      log.warn(messageLivechat)
+      throw new Error(messageLivechat)
+
     default:
-      const message = `Room type ${rcRoom.t} is unknown or unimplemented`
-      log.error(message)
-      throw new Error(message)
+      const messageUnknownRoom = `Room ${
+        rcRoom.name || 'with ID: ' + rcRoom._id
+      } is of type ${rcRoom.t}, which is unknown or unimplemented`
+      log.error(messageUnknownRoom)
+      throw new Error(messageUnknownRoom)
   }
   return room
 }
@@ -98,7 +107,7 @@ export function getCreator(rcRoom: RcRoom): string {
     return rcRoom.uids[0]
   } else {
     log.warn(
-      `Creator ID could not be determined for room ${rcRoom.name} of type ${rcRoom.t}. This is normal for the default room.`
+      `Creator ID could not be determined for room ${rcRoom.name} of type ${rcRoom.t}. This is normal for the default room. Using admin user.`
     )
     return ''
   }
@@ -152,11 +161,26 @@ export async function inviteMember(
   creatorSessionOptions: SessionOptions | object
 ): Promise<void> {
   log.http(`Invite member ${inviteeId}`)
-  await axios.post(
-    `/_matrix/client/v3/rooms/${roomId}/invite`,
-    { user_id: inviteeId },
-    creatorSessionOptions
-  )
+  try {
+    await axios.post(
+      `/_matrix/client/v3/rooms/${roomId}/invite`,
+      { user_id: inviteeId },
+      creatorSessionOptions
+    )
+  } catch (error) {
+    if (
+      error instanceof AxiosError &&
+      error.response &&
+      error.response.data.errcode === 'M_FORBIDDEN' &&
+      error.response.data.error === `${inviteeId} is already in the room.`
+    ) {
+      log.debug(
+        `User ${inviteeId} is already in room ${roomId}, probably because this user created the room as a fallback.`
+      )
+    } else {
+      throw error
+    }
+  }
 }
 
 export async function acceptInvitation(

src/handlers/users.test.ts

@@ -1,8 +1,10 @@
 process.env.REGISTRATION_SHARED_SECRET = 'ThisIsSoSecretWow'
 process.env.EXCLUDED_USERS = 'excludedUser1,excludedUser2'
+process.env.ADMIN_USERNAME = 'testAdmin'
 import { expect, jest, test } from '@jest/globals'
 import axios from 'axios'
-import * as storage from '../helpers/storage'
+import { Entity, entities } from '../Entities'
+import { IdMapping } from '../entity/IdMapping'
 import {
   MatrixUser,
   RcUser,
@@ -12,8 +14,7 @@ import {
   mapUser,
   userIsExcluded,
 } from '../handlers/users'
-import { IdMapping } from '../entity/IdMapping'
-import { Entity, entities } from '../Entities'
+import * as storage from '../helpers/storage'
 
 jest.mock('axios')
 const mockedAxios = axios as jest.Mocked<typeof axios>

src/handlers/users.ts

@@ -1,9 +1,10 @@
 import { createHmac } from 'node:crypto'
-import log from '../helpers/logger'
-import { axios } from '../helpers/synapse'
-import { createMembership, getUserId, save } from '../helpers/storage'
-import { IdMapping } from '../entity/IdMapping'
 import { Entity, entities } from '../Entities'
+import adminAccessToken from '../config/synapse_access_token.json'
+import { IdMapping } from '../entity/IdMapping'
+import log from '../helpers/logger'
+import { createMembership, getUserId, save } from '../helpers/storage'
+import { axios } from '../helpers/synapse'
 
 export type RcUser = {
   _id: string
@@ -41,15 +42,22 @@ export function mapUser(rcUser: RcUser): MatrixUser {
   }
 }
 
-const registration_shared_secret = process.env.REGISTRATION_SHARED_SECRET || ''
-if (!registration_shared_secret) {
+const registrationSharedSecret = process.env.REGISTRATION_SHARED_SECRET || ''
+if (!registrationSharedSecret) {
   const message = 'No REGISTRATION_SHARED_SECRET found in .env.'
   log.error(message)
   throw new Error(message)
 }
 
+const adminUsername = process.env.ADMIN_USERNAME || ''
+if (!adminUsername) {
+  const message = 'No ADMIN_USERNAME found in .env.'
+  log.error(message)
+  throw new Error(message)
+}
+
 export function generateHmac(user: MatrixUser): string {
-  const hmac = createHmac('sha1', registration_shared_secret)
+  const hmac = createHmac('sha1', registrationSharedSecret)
   hmac.write(
     `${user.nonce}\0${user.username}\0${user.password}\0${
       user.admin ? 'admin' : 'notadmin'
@@ -87,7 +95,7 @@ export function userIsExcluded(rcUser: RcUser): boolean {
     reasons.push(`username "${rcUser.username}" is on exclusion list`)
 
   if (reasons.length > 0) {
-    log.debug(`User ${rcUser.name} is excluded: ${reasons.join(', ')}`)
+    log.warn(`User ${rcUser.name} is excluded: ${reasons.join(', ')}`)
     return true
   }
   return false
@@ -124,15 +132,18 @@ export async function createUser(rcUser: RcUser): Promise<MatrixUser> {
 export async function handle(rcUser: RcUser): Promise<void> {
   log.info(`Parsing user: ${rcUser.name}: ${rcUser._id}`)
 
-  if (userIsExcluded(rcUser)) {
-    return undefined
-  }
-
   const matrixId = await getUserId(rcUser._id)
   if (matrixId) {
     log.debug(`Mapping exists: ${rcUser._id} -> ${matrixId}`)
   } else {
-    const matrixUser = await createUser(rcUser)
-    await createMapping(rcUser._id, matrixUser)
+    if (rcUser.username === adminUsername) {
+      log.info(
+        `User ${rcUser.username} is defined as admin in ENV, mapping as such`
+      )
+      await createMapping(rcUser._id, adminAccessToken as unknown as MatrixUser)
+    } else if (!userIsExcluded(rcUser)) {
+      const matrixUser = await createUser(rcUser)
+      await createMapping(rcUser._id, matrixUser)
+    }
   }
 }

src/helpers/logger.ts

@@ -2,7 +2,20 @@ import winston from 'winston'
 
 export default winston.createLogger({
   level: 'debug',
-  transports: [new winston.transports.Console()],
+  transports: [
+    new winston.transports.Console(),
+    new winston.transports.File({
+      filename: 'warn.log',
+      level: 'warn',
+      silent: process.env.NODE_ENV === 'test',
+      options: { flags: 'w' },
+    }),
+    new winston.transports.File({
+      filename: 'combined.log',
+      silent: process.env.NODE_ENV === 'test',
+      options: { flags: 'w' },
+    }),
+  ],
   format: winston.format.combine(
     winston.format.colorize({ all: true }),
     winston.format.simple()

src/helpers/storage.test.ts

@@ -1,5 +1,8 @@
 process.env.DATABASE = ':memory:'
 import { beforeAll, expect, test } from '@jest/globals'
+import { Entity, entities } from '../Entities'
+import { IdMapping } from '../entity/IdMapping'
+import { Membership } from '../entity/Membership'
 import {
   createMembership,
   getAccessToken,
@@ -11,9 +14,6 @@ import {
   initStorage,
   save,
 } from './storage'
-import { IdMapping } from '../entity/IdMapping'
-import { Membership } from '../entity/Membership'
-import { Entity, entities } from '../Entities'
 
 const mapping = new IdMapping()
 mapping.rcId = 'rcId'

src/helpers/storage.ts

@@ -1,7 +1,7 @@
 import { DataSource } from 'typeorm'
+import { Entity, entities } from '../Entities'
 import { IdMapping } from '../entity/IdMapping'
 import { Membership } from '../entity/Membership'
-import { Entity, entities } from '../Entities'
 
 const AppDataSource = new DataSource({
   type: 'sqlite',
@@ -25,6 +25,16 @@ export function getMapping(
   })
 }
 
+export function getAllMappingsByType(type: number): Promise<IdMapping[]> {
+  return AppDataSource.manager.findBy(IdMapping, { type })
+}
+
+export function getMappingByMatrixId(id: string): Promise<IdMapping | null> {
+  return AppDataSource.manager.findOneBy(IdMapping, {
+    matrixId: id,
+  })
+}
+
 export async function save(entity: IdMapping | Membership): Promise<void> {
   await AppDataSource.manager.save(entity)
 }