Merge pull request 'Implement User Workflow' (#14) from implement-workflow into main

Reviewed-on: https://git.verdigado.com/NB-Public/rocketchat2matrix/pulls/14

.env.example

@@ -0,0 +1,2 @@
+REGISTRATION_SHARED_SECRET='look in your synapses homeserver.yaml'
+EXCLUDED_USERS='rocket.cat' # Comma-separated list

.gitignore

@@ -133,3 +133,5 @@ dist
 files/
 src/config/synapse_access_token.json
 inputs/
+db.sqlite
+db.sqlite-journal

.vscode/settings.json

@@ -0,0 +1,13 @@
+{
+  "cSpell.words": [
+    "homeserver",
+    "markdownlint",
+    "mongoexport",
+    "notadmin",
+    "readlines",
+    "rocketchat",
+    "sqlite",
+    "typeorm",
+    "verdiadmin"
+  ]
+}

.woodpecker.yml

@@ -1,5 +1,7 @@
 variables:
   - &node_image 'node:20-alpine'
+  - &create_synapse_access_token >-
+    echo '{"user_id":"ci-dummy","access_token":"ci-dummy","home_server":"ci-dummy","device_id":"ci-dummy"}' > src/config/synapse_access_token.json
 
 pipeline:
   lint-markdown:
@@ -34,13 +36,14 @@ pipeline:
     commands:
       - npm run format
 
+  node-test:
+    image: *node_image
+    commands:
+      - *create_synapse_access_token
+      - npm test
+
   node-compile:
     image: *node_image
     commands:
-      - echo '{"user_id":"ci-dummy","access_token":"ci-dummy","home_server":"ci-dummy","device_id":"ci-dummy"}' > src/config/synapse_access_token.json
+      - *create_synapse_access_token
       - npm run compile
-
-  # node-test:
-  #   image: *node_image
-  #   commands:
-  #     - npm test

README.md

@@ -35,6 +35,14 @@ curl --request POST \
 > src/config/synapse_access_token.json
 ```
 
+## Configuration
+
+Copy over `.env.example` to `.env` and insert your values.
+
+## Running Tests
+
+`npm test`.
+
 ## Design Decisions
 
 - Getting data from Rocket.Chat via (currently) manual mongodb export

docker-compose.yml

@@ -6,7 +6,7 @@ services:
     image: docker.io/matrixdotorg/synapse:latest
     # Since synapse does not retry to connect to the database, restart upon
     # failure
-    restart: unless-stopped
+    restart: "no"
     # See the readme for a full documentation of the environment settings
     # NOTE: You must edit homeserver.yaml to use postgres, it defaults to sqlite
     environment:
@@ -46,4 +46,4 @@ services:
     image: awesometechnologies/synapse-admin:latest
     ports:
       - "8080:80"
-    restart: unless-stopped
+    restart: "no"

jest.config.js

@@ -0,0 +1,5 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+}

package.json

@@ -21,13 +21,15 @@
     "lint-fix": "eslint src/ --fix --ext .ts",
     "prefix": "npm run format-fix",
     "fix": "npm run lint-fix",
-    "test": "echo \"Warning: no test specified\"",
+    "test": "jest",
     "compile": "rm -rf dist/ && tsc",
     "start": "npm run compile && node dist/app.js",
     "prepare": "husky install"
   },
   "version": "0.1.0",
   "devDependencies": {
+    "@types/jest": "^29.5.0",
+    "@types/n-readlines": "^1.0.3",
     "@types/node": "^20.2.1",
     "@typescript-eslint/eslint-plugin": "^5.59.6",
     "@typescript-eslint/parser": "^5.59.6",
@@ -41,10 +43,16 @@
     "husky": "^8.0.3",
     "lint-staged": "^13.2.2",
     "prettier": "2.8.8",
+    "ts-jest": "^29.1.0",
     "typescript": "^5.0.4"
   },
   "dependencies": {
     "axios": "^1.4.0",
+    "dotenv": "^16.0.3",
+    "n-readlines": "^1.0.1",
+    "reflect-metadata": "^0.1.13",
+    "sqlite3": "^5.1.6",
+    "typeorm": "^0.3.16",
     "winston": "^3.8.2"
   }
 }

src/app.ts

@@ -1,37 +1,101 @@
-import fs from 'node:fs'
-import readline from 'node:readline'
+import dotenv from 'dotenv'
+dotenv.config()
+import lineByLine from 'n-readlines'
+import 'reflect-metadata'
+import { DataSource } from 'typeorm'
+import { IdMapping } from './entity/IdMapping'
+import { Membership } from './entity/Membership'
 import log from './logger'
 import { whoami } from './synapse'
+import { RcUser, createUser } from './users'
+
 log.info('rocketchat2matrix starts.')
 
-interface RcUser {
-  username: string
-  name: string
-  roles: string[]
-  _id: string
-  __rooms: string[]
+const AppDataSource = new DataSource({
+  type: 'sqlite',
+  database: 'db.sqlite',
+  entities: [IdMapping, Membership],
+  synchronize: true,
+  logging: false,
+})
+
+const enum Entities {
+  Users = 'users.json',
+  Rooms = 'rocketchat_room.json',
+  Messages = 'rocketchat_message.json',
 }
 
-function loadRcExport(filename: string) {
-  const rl = readline.createInterface({
-    input: fs.createReadStream(`./inputs/${filename}`, {
-      encoding: 'utf-8',
-    }),
-    crlfDelay: Infinity,
-  })
+async function loadRcExport(entity: Entities) {
+  const rl = new lineByLine(`./inputs/${entity}`)
 
-  rl.on('line', (line) => {
-    const entity: RcUser = JSON.parse(line)
-    log.debug(`User: ${entity.name}`)
+  let line: false | Buffer
+  while ((line = rl.next())) {
+    const item = JSON.parse(line.toString())
+    switch (entity) {
+      case Entities.Users:
+        const rcUser: RcUser = item
+        log.debug(`Parsing user: ${rcUser.name}: ${rcUser._id}`)
+
+        // Check for exclusion
+        if (
+          rcUser.roles.some((e) => ['app', 'bot'].includes(e)) ||
+          (process.env.EXCLUDED_USERS || '').split(',').includes(rcUser._id)
+        ) {
+          log.debug('User excluded. Skipping.')
+          break
+        }
+
+        let mapping = await AppDataSource.manager.findOneBy(IdMapping, {
+          rcId: rcUser._id,
+          type: 0,
         })
+        if (mapping && mapping.matrixId) {
+          log.debug('Mapping exists:', mapping)
+        } else {
+          const matrixUser = await createUser(rcUser)
+          mapping = new IdMapping()
+          mapping.rcId = rcUser._id
+          mapping.matrixId = matrixUser.user_id
+          mapping.type = 0
+
+          AppDataSource.manager.save(mapping)
+          log.debug('Mapping added:', mapping)
+
+          // Add user to room mapping (specific to users)
+          rcUser.__rooms.forEach(async (rcRoomId: string) => {
+            const membership = new Membership()
+            membership.rcRoomId = rcRoomId
+            membership.rcUserId = rcUser._id
+
+            await AppDataSource.manager.save(membership)
+            log.debug(`${rcUser.username} membership for ${rcRoomId} created`)
+          })
+        }
+
+        break
+
+      case Entities.Rooms:
+        log.debug(`Room: ${item.name}`)
+        break
+
+      case Entities.Messages:
+        log.debug(`Message: ${item.name}`)
+        break
+
+      default:
+        throw new Error(`Unhandled Entity: ${entity}`)
+    }
+  }
 }
 
 async function main() {
   try {
     await whoami()
-    await loadRcExport('users.json')
+    await AppDataSource.initialize()
+    await loadRcExport(Entities.Users)
+    log.info('Done.')
   } catch (error) {
-    log.error(`Encountered an error booting up`)
+    log.error(`Encountered an error while booting up`)
   }
 }
 

src/entity/IdMapping.ts

@@ -0,0 +1,13 @@
+import { Entity, Column, PrimaryColumn } from 'typeorm'
+
+@Entity()
+export class IdMapping {
+  @PrimaryColumn()
+  rcId!: string
+
+  @Column()
+  matrixId?: string
+
+  @Column('integer')
+  type!: number
+}

src/entity/Membership.ts

@@ -0,0 +1,10 @@
+import { Entity, PrimaryColumn } from 'typeorm'
+
+@Entity()
+export class Membership {
+  @PrimaryColumn()
+  rcRoomId!: string
+
+  @PrimaryColumn()
+  rcUserId!: string
+}

src/users.test.ts

@@ -0,0 +1,58 @@
+process.env.REGISTRATION_SHARED_SECRET = 'ThisIsSoSecretWow'
+import axios from 'axios'
+import { MatrixUser, RcUser, createUser, generateHmac, mapUser } from './users'
+
+jest.mock('axios')
+const mockedAxios = axios as jest.Mocked<typeof axios>
+
+const rcUser: RcUser = {
+  _id: 'testRc',
+  name: 'Tester McDelme',
+  username: 'testuser',
+  roles: ['user'],
+  __rooms: [],
+}
+
+const matrixUser: MatrixUser = {
+  user_id: '',
+  username: rcUser.username,
+  displayname: rcUser.name,
+  password: '',
+  admin: false,
+}
+
+const nonce = 'test-nonce'
+
+test('mapping users', () => {
+  expect(mapUser(rcUser)).toStrictEqual(matrixUser)
+})
+test('generating correct hmac', () => {
+  expect(generateHmac({ ...matrixUser, nonce })).toStrictEqual(
+    'be0537407ab3c82de908c5763185556e98a7211c'
+  )
+})
+
+test('creating users', async () => {
+  const matrixId = 'TestRandomId'
+
+  mockedAxios.get.mockResolvedValue({ data: { nonce: nonce } })
+  mockedAxios.post.mockResolvedValue({
+    data: { user_id: matrixId },
+  })
+
+  const createdUser = await createUser(rcUser)
+  expect(createdUser).toStrictEqual({
+    ...matrixUser,
+    user_id: matrixId,
+  })
+
+  expect(mockedAxios.get).toHaveBeenCalledWith('/_synapse/admin/v1/register')
+  expect(mockedAxios.post).toHaveBeenCalled()
+  // The following test fails with an incorrect return value, for whatever reason.
+  // Probably because of mutated call logs in jest due to the `delete` or sth.
+  // expect(mockedAxios.post).toHaveBeenCalledWith('/_synapse/admin/v1/register', {
+  //   ...matrixUser,
+  //   nonce,
+  //   mac: 'be0537407ab3c82de908c5763185556e98a7211c',
+  // })
+})

src/users.ts

@@ -0,0 +1,70 @@
+import log from './logger'
+import { axios } from './synapse'
+import { createHmac } from 'node:crypto'
+
+export type RcUser = {
+  _id: string
+  username: string
+  name: string
+  roles: string[]
+  __rooms: string[]
+}
+
+export type MatrixUser = {
+  user_id: string
+  username: string
+  displayname: string
+  password: string
+  admin: boolean
+  nonce?: string
+  mac?: string
+}
+
+export function mapUser(rcUser: RcUser): MatrixUser {
+  return {
+    user_id: '',
+    username: rcUser.username,
+    displayname: rcUser.name,
+    password: '',
+    admin: rcUser.roles.includes('admin'),
+  }
+}
+
+const registration_shared_secret = process.env.REGISTRATION_SHARED_SECRET || ''
+if (!registration_shared_secret) {
+  const message = 'No REGISTRATION_SHARED_SECRET found in .env.'
+  log.error(message)
+  throw new Error(message)
+}
+
+export function generateHmac(user: MatrixUser): string {
+  const hmac = createHmac('sha1', registration_shared_secret)
+  hmac.write(
+    `${user.nonce}\0${user.username}\0${user.password}\0${
+      user.admin ? 'admin' : 'notadmin'
+    }`
+  )
+  hmac.end()
+  return hmac.read().toString('hex')
+}
+
+async function getUserRegistrationNonce(): Promise<string> {
+  return (await axios.get('/_synapse/admin/v1/register')).data.nonce
+}
+
+async function registerUser(user: MatrixUser): Promise<string> {
+  return (await axios.post('/_synapse/admin/v1/register', user)).data.user_id
+}
+
+export async function createUser(rcUser: RcUser): Promise<MatrixUser> {
+  const user = mapUser(rcUser)
+  user.nonce = await getUserRegistrationNonce()
+  user.mac = generateHmac(user)
+  user.user_id = await registerUser(user)
+  log.info(`User ${rcUser.username} created:`, user)
+
+  delete user.nonce
+  delete user.mac
+
+  return user
+}

tsconfig.json

@@ -14,8 +14,8 @@
     "target": "ES2022",                                  /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
     // "lib": [],                                        /* Specify a set of bundled library declaration files that describe the target runtime environment. */
     // "jsx": "preserve",                                /* Specify what JSX code is generated. */
-    // "experimentalDecorators": true,                   /* Enable experimental support for legacy experimental decorators. */
-    // "emitDecoratorMetadata": true,                    /* Emit design-type metadata for decorated declarations in source files. */
+    "experimentalDecorators": true,                   /* Enable experimental support for legacy experimental decorators. */
+    "emitDecoratorMetadata": true,                    /* Emit design-type metadata for decorated declarations in source files. */
     // "jsxFactory": "",                                 /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */
     // "jsxFragmentFactory": "",                         /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
     // "jsxImportSource": "",                            /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */