Fix reactions

- look up missing reaction keys/emojis via library
- skip, if no reaction found
- skip for missing users
- fix async function not completely awaited

.woodpecker.yaml

@@ -1,17 +1,17 @@
 variables:
-  - &node_image 'node:20.8.1-alpine'
+  - &node_image 'node:20.6.1-alpine'
   - &create_synapse_access_token >-
     echo '{"user_id":"ci-dummy","access_token":"ci-dummy","home_server":"ci-dummy","device_id":"ci-dummy"}' > src/config/synapse_access_token.json
 
 steps:
   lint-markdown:
-    image: markdownlint/markdownlint:0.13.0
+    image: markdownlint/markdownlint:0.12.0
     group: test
     commands:
       - mdl .
 
   check-pre-commit:
-    image: python:3.12.0
+    image: python:3.11.5
     group: test
     environment:
       - SKIP=no-commit-to-branch # Ignore "don't commit to protected branch" check

README.md

@@ -49,7 +49,7 @@ app_service_config_files:
   - /data/app-service.yaml
 ```
 
-Now edit `app-service.example.yaml` and save it at `files/app-service.yaml`, changing the tokens.
+Now edit `app-service.example.yaml` and save it at `files/app-service.yaml`, changing the tokens manually.
 
 Copy over `.env.example` to `.env` and insert your values.
 
@@ -101,4 +101,8 @@ Then you can restart with an empty but quite equal server, following the instruc
 
 - Getting data from Rocket.Chat via (currently) manual mongodb export
 - Room to Channel conversion:
-  - Read-only attributes of 2 verdigado channels not converted to power levels due to complexity
+  - Read-only attributes of channels not converted to power levels due to complexity
+- Reactions:
+  - So far only reactions used in our chats have been translated
+  - Individual logos of *netzbegruenung* and *verdigado* have been replaced by a generic sunflower
+  - Skin colour tones and genders have been ignored in the manual translation, using the neutral versions

docker-compose.yaml

@@ -3,7 +3,7 @@ version: '3'
 services:
 
   synapse:
-    image: docker.io/matrixdotorg/synapse:v1.94.0
+    image: docker.io/matrixdotorg/synapse:v1.91.2
     # Since synapse does not retry to connect to the database, restart upon
     # failure
     restart: "no"

package-lock.json

@@ -12,6 +12,7 @@
         "axios": "^1.5.0",
         "dotenv": "^16.3.1",
         "n-readlines": "^1.0.1",
+        "node-emoji": "^2.1.0",
         "reflect-metadata": "^0.1.13",
         "sqlite3": "^5.1.6",
         "typeorm": "^0.3.17",
@@ -1405,6 +1406,17 @@
       "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==",
       "dev": true
     },
+    "node_modules/@sindresorhus/is": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-3.1.2.tgz",
+      "integrity": "sha512-JiX9vxoKMmu8Y3Zr2RVathBL1Cdu4Nt4MuNWemt1Nc06A0RAin9c5FArkhGsyMBWfCu4zj+9b+GxtjAnE4qqLQ==",
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sindresorhus/is?sponsor=1"
+      }
+    },
     "node_modules/@sinonjs/commons": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.0.tgz",
@@ -2498,8 +2510,6 @@
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz",
       "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==",
-      "dev": true,
-      "peer": true,
       "engines": {
         "node": ">=10"
       }
@@ -3204,6 +3214,11 @@
       "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==",
       "dev": true
     },
+    "node_modules/emojilib": {
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/emojilib/-/emojilib-2.4.0.tgz",
+      "integrity": "sha512-5U0rVMU5Y2n2+ykNLQqMoqklN9ICBT/KsvC1Gz6vqHbz2AXXGkG+Pm5rMWk/8Vjrr/mY9985Hi8DYzn1F09Nyw=="
+    },
     "node_modules/enabled": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz",
@@ -6317,6 +6332,17 @@
       "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-4.3.0.tgz",
       "integrity": "sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ=="
     },
+    "node_modules/node-emoji": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-2.1.0.tgz",
+      "integrity": "sha512-tcsBm9C6FmPN5Wo7OjFi9lgMyJjvkAeirmjR/ax8Ttfqy4N8PoFic26uqFTIgayHPNI5FH4ltUvfh9kHzwcK9A==",
+      "dependencies": {
+        "@sindresorhus/is": "^3.1.2",
+        "char-regex": "^1.0.2",
+        "emojilib": "^2.4.0",
+        "skin-tone": "^2.0.0"
+      }
+    },
     "node_modules/node-fetch": {
       "version": "2.7.0",
       "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
@@ -7426,6 +7452,17 @@
       "dev": true,
       "peer": true
     },
+    "node_modules/skin-tone": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/skin-tone/-/skin-tone-2.0.0.tgz",
+      "integrity": "sha512-kUMbT1oBJCpgrnKoSr0o6wPtvRWT9W9UKvGLwfJYO2WuahZRHOpEyL1ckyMGgMWh0UdpmaoFqKKD29WTomNEGA==",
+      "dependencies": {
+        "unicode-emoji-modifier-base": "^1.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/slash": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz",
@@ -8292,6 +8329,14 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/unicode-emoji-modifier-base": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/unicode-emoji-modifier-base/-/unicode-emoji-modifier-base-1.0.0.tgz",
+      "integrity": "sha512-yLSH4py7oFH3oG/9K+XWrz1pSi3dfUrWEnInbxMfArOfc1+33BlGPQtLsOYwvdMy11AwUBetYuaRxSPqgkq+8g==",
+      "engines": {
+        "node": ">=4"
+      }
+    },
     "node_modules/unique-filename": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-1.1.1.tgz",

package.json

@@ -50,6 +50,7 @@
     "axios": "^1.5.0",
     "dotenv": "^16.3.1",
     "n-readlines": "^1.0.1",
+    "node-emoji": "^2.1.0",
     "reflect-metadata": "^0.1.13",
     "sqlite3": "^5.1.6",
     "typeorm": "^0.3.17",

src/app.ts

@@ -75,9 +75,13 @@ async function removeExcessRoomMembers() {
     )
 
     // do action for any user in mx, but not in rc
+    const adminUsername = process.env.ADMIN_USERNAME || ''
     await Promise.all(
       actualMembers.map(async (actualMember) => {
-        if (!memberNames.includes(actualMember)) {
+        if (
+          !memberNames.includes(actualMember) &&
+          !actualMember.includes(adminUsername) // exclude admin from removal
+        ) {
           log.warn(
             `Member ${actualMember} should not be in room ${roomMapping.matrixId}, removing`
           )

src/handlers/messages.ts

@@ -1,4 +1,5 @@
 import { AxiosError } from 'axios'
+import * as emoji from 'node-emoji'
 import { Entity, entities } from '../Entities'
 import { IdMapping } from '../entity/IdMapping'
 import log from '../helpers/logger'
@@ -8,9 +9,11 @@ import {
   getMessageId,
   getRoomId,
   getUserId,
+  getUserMappingByName,
   save,
 } from '../helpers/storage'
 import { axios, formatUserSessionOptions } from '../helpers/synapse'
+import reactionKeys from '../reactions.json'
 import { acceptInvitation, inviteMember } from './rooms'
 
 const applicationServiceToken = process.env.AS_TOKEN || ''
@@ -39,7 +42,11 @@ export type RcMessage = {
   pinned?: boolean
   drid?: string // The direct room id (if belongs to a direct room).
   // attachments?: any[] // An array of attachment objects, available only when the message has at least one attachment.
-  reactions?: object // Object containing reaction information associated with the message.
+  reactions?: {
+    [key: string]: {
+      usernames: string[]
+    }
+  }
 }
 
 export type MatrixMessage = {
@@ -93,6 +100,65 @@ export async function createMessage(
   ).data.event_id
 }
 
+export async function handleReactions(
+  reactions: object,
+  matrixMessageId: string,
+  matrixRoomId: string
+): Promise<void> {
+  for (const [reaction, value] of Object.entries(reactions)) {
+    // Lookup key/emoji
+    const reactionKey: string =
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      (reactionKeys as any)[reaction] || emoji.get(reaction.replaceAll(':', ''))
+    if (!reactionKey) {
+      log.warn(
+        `Could not find an emoji for ${reaction} for message ${matrixMessageId}, skipping`
+      )
+      return
+    }
+    await Promise.all(
+      value.usernames.map(async (rcUsername: string) => {
+        // generate transaction id
+        const transactionId = Buffer.from(
+          [matrixMessageId, reaction, rcUsername].join('\0')
+        ).toString('base64')
+        // lookup user access token
+        const userMapping = await getUserMappingByName(rcUsername)
+        if (!userMapping) {
+          log.warn(
+            `Could not find user mapping for name: ${rcUsername}, skipping reaction ${reaction} for message ${matrixMessageId}`
+          )
+          return
+        }
+        if (!userMapping.accessToken) {
+          throw new Error(
+            `User mapping for name ${rcUsername} has no access token`
+          )
+        }
+
+        const userSessionOptions = formatUserSessionOptions(
+          userMapping.accessToken
+        )
+        log.http(
+          `Adding reaction to message ${matrixMessageId} with symbol ${reactionKey} for user ${rcUsername}`
+        )
+        // put reaction
+        await axios.put(
+          `/_matrix/client/v3/rooms/${matrixRoomId}/send/m.reaction/${transactionId}`,
+          {
+            'm.relates_to': {
+              rel_type: 'm.annotation',
+              event_id: matrixMessageId,
+              key: reactionKey,
+            },
+          },
+          userSessionOptions
+        )
+      })
+    )
+  }
+}
+
 export async function handle(rcMessage: RcMessage): Promise<void> {
   log.info(`Parsing message with ID: ${rcMessage._id}`)
 
@@ -144,20 +210,7 @@ export async function handle(rcMessage: RcMessage): Promise<void> {
           )
           return
         }
-        const roomCreatorId = (
-          await axios.get(`/_synapse/admin/v1/rooms/${room_id}`)
-        ).data.creator
-        if (!roomCreatorId) {
-          log.warn(
-            `Could not determine room creator for room ${room_id}, using admin credentials.`
-          )
-        }
-        if (roomCreatorId == matrixUser) {
-          log.warn(
-            `Room creator ${roomCreatorId} left rocketchat room ${room_id}, skipping to prevent being unable to rejoin.`
-          )
-          return
-        }
+
         log.http(`User ${matrixUser} leaves room ${room_id}`)
         await axios.post(
           `/_matrix/client/v3/rooms/${room_id}/leave`,
@@ -224,6 +277,13 @@ export async function handle(rcMessage: RcMessage): Promise<void> {
       ts,
       rcMessage._id
     )
+    if (rcMessage.reactions) {
+      log.info(
+        `Parsing reactions for message ${rcMessage._id}`,
+        rcMessage.reactions
+      )
+      await handleReactions(rcMessage.reactions, event_id, room_id)
+    }
     await createMapping(rcMessage._id, event_id)
   } catch (error) {
     if (
@@ -273,6 +333,13 @@ export async function handle(rcMessage: RcMessage): Promise<void> {
         ts,
         rcMessage._id
       )
+      if (rcMessage.reactions) {
+        log.info(
+          `Parsing reactions for message ${rcMessage._id}`,
+          rcMessage.reactions
+        )
+        await handleReactions(rcMessage.reactions, event_id, room_id)
+      }
       await createMapping(rcMessage._id, event_id)
     } else {
       throw error

src/handlers/users.ts

@@ -4,7 +4,7 @@ import adminAccessToken from '../config/synapse_access_token.json'
 import { IdMapping } from '../entity/IdMapping'
 import log from '../helpers/logger'
 import { createMembership, getUserId, save } from '../helpers/storage'
-import { axios, getUserDomain } from '../helpers/synapse'
+import { axios } from '../helpers/synapse'
 
 export type RcUser = {
   _id: string
@@ -32,12 +32,6 @@ export type AccessToken = {
   user_id: string
 }
 
-export type UserInfo = {
-  admin: boolean
-  displayname: string
-  name: string
-}
-
 export function mapUser(rcUser: RcUser): MatrixUser {
   return {
     user_id: '',
@@ -81,13 +75,6 @@ async function registerUser(user: MatrixUser): Promise<AccessToken> {
   return (await axios.post('/_synapse/admin/v1/register', user)).data
 }
 
-async function getUserData(user: MatrixUser): Promise<UserInfo> {
-  return (await axios.get('/_synapse/admin/v2/users/@' + user.username + ":" + getUserDomain())).data
-}
-async function loginUser(user: MatrixUser): Promise<AccessToken> {
-  return (await axios.post('/_synapse/admin/v1/users/@' + user.username + ":" + getUserDomain() + "/login")).data
-}
-
 async function parseUserMemberships(rcUser: RcUser): Promise<void> {
   await Promise.all(
     rcUser.__rooms.map(async (rcRoomId: string) => {
@@ -130,30 +117,12 @@ export async function createMapping(
 
 export async function createUser(rcUser: RcUser): Promise<MatrixUser> {
   const user = mapUser(rcUser)
-  var user_exists = false;
-  try {
-    await getUserData(user)
-    user_exists = true
-  } catch (error) {
-    user_exists = false
-  }
-
-  if (user_exists) {
-    const userData = await getUserData(user)
-    user.user_id = userData.name
-    user.displayname = userData.displayname
-    user.admin = user.admin || userData.admin
-    const accessToken = await loginUser(user)
-    user.access_token = accessToken.access_token
-    log.info(`User ${rcUser.username} exists:`, user)
-  } else {
-    const nonce = await getUserRegistrationNonce()
-    const mac = generateHmac({ ...user, nonce })
-    const accessToken = await registerUser({ ...user, nonce, mac })
-    user.user_id = accessToken.user_id
-    user.access_token = accessToken.access_token
-    log.info(`User ${rcUser.username} created:`, user)
-  }
+  const nonce = await getUserRegistrationNonce()
+  const mac = generateHmac({ ...user, nonce })
+  const accessToken = await registerUser({ ...user, nonce, mac })
+  user.user_id = accessToken.user_id
+  user.access_token = accessToken.access_token
+  log.info(`User ${rcUser.username} created:`, user)
 
   await parseUserMemberships(rcUser)
 

src/helpers/storage.ts

@@ -1,4 +1,4 @@
-import { DataSource } from 'typeorm'
+import { DataSource, ILike } from 'typeorm'
 import { Entity, entities } from '../Entities'
 import { IdMapping } from '../entity/IdMapping'
 import { Membership } from '../entity/Membership'
@@ -35,6 +35,15 @@ export function getMappingByMatrixId(id: string): Promise<IdMapping | null> {
   })
 }
 
+export function getUserMappingByName(
+  username: string
+): Promise<IdMapping | null> {
+  return AppDataSource.manager.findOneBy(IdMapping, {
+    matrixId: ILike(`@${username.toLowerCase()}:%`),
+    type: entities[Entity.Users].mappingType,
+  })
+}
+
 export async function save(entity: IdMapping | Membership): Promise<void> {
   await AppDataSource.manager.save(entity)
 }

src/helpers/synapse.ts

@@ -3,7 +3,7 @@ import { access_token } from '../config/synapse_access_token.json'
 import log from './logger'
 import { getAccessToken } from './storage'
 
-axios.defaults.baseURL = 'https://m-rc.jennett-wheeler.co.uk'
+axios.defaults.baseURL = 'http://localhost:8008'
 axios.defaults.headers.common['Authorization'] = `Bearer ${access_token}`
 axios.defaults.headers.post['Content-Type'] = 'application/json'
 
@@ -14,10 +14,6 @@ export interface SessionOptions {
   [others: string]: unknown
 }
 
-export function getUserDomain(): string {
-  return "m-rc.jennett-wheeler.co.uk"
-}
-
 export { default as axios } from 'axios'
 export const whoami = () =>
   new Promise<void>((resolve, reject) => {