Add nodejs wrapper for wasm sphynx client

This commit is contained in:
Sebastian Martinez
2023-10-09 14:06:46 +02:00
parent 7e16920358
commit 37958ccb4e
18 changed files with 3512 additions and 82 deletions
@@ -0,0 +1 @@
*/worker.js
@@ -0,0 +1,34 @@
# Nym NodeJS wrapper for Sphinx webassembly client
This package is a NodeJS client that uses the wasm from the [Sphinx webassembly client](https://github.com/nymtech/nym/blob/4890c528bcb519290de81ef968bb2ba1399914a4/wasm/client/README.md), runs it inside a NodeJS Worker thread and allows users to send and receive Sphinx packets to/from a Nym mixnet.
## Usage
```js
const { createNymMixnetClient } = require('../dist/cjs/index.js');
async () => {
const nym = await createNymMixnetClient();
nym.events.subscribeToTextMessageReceivedEvent(async (e) => {
if (e.args.payload === 'Hello') {
await nym.client.stop();
}
});
const nymApiUrl = 'https://validator.nymtech.net/api/';
// start the client and connect to a gateway
await nym.client.start({
nymApiUrl,
clientId: 'my-client',
});
nym.events.subscribeToConnected(async (e) => {
// send a message to yourself
const message = 'Hello';
const recipient = await nym.client.selfAddress();
await nym.client.send({ payload: { message, mimeType: 'text/plain' }, recipient });
});
};
```
@@ -6,11 +6,10 @@
"files": [
"dist/**/*"
],
"main": "dist/index.mjs",
"main": "dist/cjs/index.js",
"scripts": {
"build": "scripts/build-prod.sh",
"build:dev": "scripts/build.sh",
"build:dev:esm": "scripts/build-dev-esm.sh",
"build:worker": "rollup -c rollup-worker.config.mjs",
"clean": "rimraf dist",
"docs:dev": "run-p docs:watch docs:serve ",
@@ -23,12 +22,14 @@
"lint:fix": "eslint src --fix",
"start": "tsc -w",
"start:dev": "nodemon --watch src -e ts,json --exec 'yarn build:dev:esm'",
"test": "node --experimental-vm-modules --no-warnings node_modules/jest/bin/jest.js -c=jest.config.mjs --no-cache",
"tsc": "tsc --noEmit true"
},
"dependencies": {
"@nymproject/nym-client-wasm-node": ">=1.2.0-rc.10 || ^1",
"comlink": "^4.3.1"
"@nymproject/nym-client-wasm-node": "^1.2.0",
"comlink": "^4.3.1",
"fake-indexeddb": "^4.0.2",
"rollup-plugin-polyfill": "^4.2.0",
"ws": "^8.14.2"
},
"devDependencies": {
"@babel/core": "^7.15.0",
@@ -68,6 +69,7 @@
"rimraf": "^3.0.2",
"rollup": "^3.9.1",
"rollup-plugin-base64": "^1.0.1",
"rollup-plugin-modify": "^3.0.0",
"rollup-plugin-web-worker-loader": "^1.6.1",
"ts-jest": "^29.1.0",
"ts-loader": "^9.4.2",
@@ -75,6 +77,5 @@
"typescript": "^4.8.4"
},
"private": false,
"type": "module",
"types": "./dist/index.d.ts"
}
@@ -0,0 +1,31 @@
/* eslint-disable import/no-extraneous-dependencies */
import typescript from '@rollup/plugin-typescript';
import resolve from '@rollup/plugin-node-resolve';
import { wasm } from '@rollup/plugin-wasm';
import webWorkerLoader from 'rollup-plugin-web-worker-loader';
import replace from '@rollup/plugin-replace';
export default {
input: 'src/index.ts',
output: {
dir: 'dist/cjs',
format: 'cjs',
},
plugins: [
webWorkerLoader({ targetPlatform: 'node', inline: false }),
replace({
values: {
"createURLWorkerFactory('web-worker-0.js')":
"createURLWorkerFactory(require('path').resolve(__dirname, 'web-worker-0.js'))",
},
delimiters: ['', ''],
preventAssignment: true,
}),
resolve({ browser: false, extensions: ['.js', '.ts'] }),
wasm({ targetEnv: 'node', maxFileSize: 0 }),
typescript({
compilerOptions: { outDir: 'dist/cjs', target: 'es5' },
exclude: ['src/worker.ts'],
}),
],
};
@@ -0,0 +1,46 @@
/* eslint-disable import/no-extraneous-dependencies */
import typescript from '@rollup/plugin-typescript';
import resolve from '@rollup/plugin-node-resolve';
import { wasm } from '@rollup/plugin-wasm';
import commonjs from '@rollup/plugin-commonjs';
import modify from 'rollup-plugin-modify';
export default {
input: 'src/worker.ts',
output: {
dir: 'dist/cjs',
format: 'cjs',
},
external: ['util', 'fake-indexeddb'],
plugins: [
resolve({
browser: false,
preferBuiltins: true,
extensions: ['.js', '.ts'],
}),
commonjs(),
// TODO: One of the wasm functions calls `new WebSocket` at one point, which we aren't able to polyfill correctly yet.
modify({
find: 'const ret = new WebSocket(getStringFromWasm0(arg0, arg1));',
replace: 'const ws = require("ws"); const ret = new ws.WebSocket(getStringFromWasm0(arg0, arg1));',
}),
// TODO: `getObject(...).require` seems to generate a warning on Webpack but with Rollup we get a panic since it can't require.
// By hard coding the require here, we can workaround that.
// Reference: https://github.com/rust-random/getrandom/issues/224
modify({ find: 'getObject(arg0).require(getStringFromWasm0(arg1, arg2));', replace: 'require("crypto");' }),
// TODO: The NodeJS setTimeout returns a Timeout object instead of a timeout id as the browser API one does.
// check how we could polyfill this, instead of commenting it out.
modify({
find: /const ret = getObject\(arg0\).setTimeout\(getObject\(arg1\), arg2\);\n\s*?_assertNum\((.*?)\)/,
replace: (match) => match.replace('_assertNum(ret)', '// _assertNum(ret)'),
}),
wasm({ targetEnv: 'node', maxFileSize: 0, fileName: '[name].wasm' }),
typescript({
compilerOptions: {
outDir: 'dist/cjs',
declaration: false,
target: 'es5',
},
}),
],
};
@@ -0,0 +1,20 @@
#!/usr/bin/env bash
set -o errexit
set -o nounset
set -o pipefail
rm -rf dist || true
rm -rf ../../../../dist/ts/sdk/nodejs-client || true
# run the build
scripts/build.sh
node scripts/buildPackageJson.mjs
# move the output outside of the yarn/npm workspaces
mkdir -p ../../../../dist/ts/sdk
mv dist ../../../../dist/ts/sdk
mv ../../../../dist/ts/sdk/dist ../../../../dist/ts/sdk/nodejs-client
echo "Output can be found in:"
realpath ../../../../dist/ts/sdk/nodejs-client
+35
View File
@@ -0,0 +1,35 @@
#!/usr/bin/env bash
set -o errexit
set -o nounset
set -o pipefail
rm -rf dist || true
#-------------------------------------------------------
# WEB WORKER (mix-fetch WASM)
#-------------------------------------------------------
# The web worker needs to be bundled because the WASM bundle needs to be loaded synchronously and all dependencies
# must be included in the worker script (because it is not loaded as an ES Module)
# build the worker
rollup -c rollup-worker.config.mjs
# move it next to the Typescript `src/index.ts` so it can be inlined by rollup
rm -f src/*.js
mv dist/cjs/worker.js src/worker.js
#-------------------------------------------------------
# COMMON JS
#-------------------------------------------------------
# Some old build systems cannot fully handle ESM or ES2021, so build
# a CommonJS bundle targeting ES5
# build the SDK as a CommonJS bundle
rollup -c rollup-cjs.config.mjs
#-------------------------------------------------------
# CLEAN UP
#-------------------------------------------------------
cp README.md dist/cjs/README.md
@@ -0,0 +1,22 @@
import * as fs from 'fs';
// parse the package.json from the SDK, so we can keep fields like the name and version
const json = JSON.parse(fs.readFileSync('package.json').toString());
// defaults (NB: these are in the output file locations)
const browser = 'index.js';
const main = 'index.js';
const types = 'index.d.ts';
const getPackageJson = (type, suffix) => ({
name: `${json.name}${suffix ? `-${suffix}` : ''}`,
version: json.version,
license: json.license,
author: json.author,
type,
browser,
main,
types,
});
fs.writeFileSync('dist/cjs/package.json', JSON.stringify(getPackageJson('commonjs', 'commonjs'), null, 2));
@@ -1,43 +1,23 @@
import * as Comlink from 'comlink';
import { Worker } from 'worker_threads';
import InlineWasmWebWorker from 'web-worker:./worker';
import {
BinaryMessageReceivedEvent,
Client,
ConnectedEvent,
EventKinds,
IWebWorker,
Client,
Events,
IWebWorker,
LoadedEvent,
MimeTypes,
NymMixnetClient,
NymMixnetClientOptions,
RawMessageReceivedEvent,
StringMessageReceivedEvent,
} from './types';
import { createSubscriptions } from './subscriptions';
/**
* Options for the Nym mixnet client.
* @property autoConvertStringMimeTypes - An array of mime types.
* @example
* ```typescript
* const client = await createNymMixnetClient({
* autoConvertStringMimeTypes: [MimeTypes.ApplicationJson, MimeTypes.TextPlain],
* });
* ```
*/
export interface NymMixnetClientOptions {
autoConvertStringMimeTypes?: string[] | MimeTypes[];
}
/**
* The client for the Nym mixnet which gives access to client methods and event subscriptions.
* Returned by the {@link createNymMixnetClient} function.
*
*/
export interface NymMixnetClient {
client: Client;
events: Events;
}
import nodeEndpoint from './node-adapter';
/**
* Create a client to send and receive traffic from the Nym mixnet.
@@ -57,16 +37,16 @@ export const createNymMixnetClient = async (options?: NymMixnetClientOptions): P
const { getSubscriptions, addSubscription } = subscriptions;
// listen to messages from the worker, parse them and let the subscribers handle them, catching any unhandled exceptions
worker.addEventListener('message', (msg) => {
if (msg.data && msg.data.kind) {
const subscribers = getSubscriptions(msg.data.kind);
worker.addListener('message', (msg: { kind: EventKinds; args: any }) => {
if (msg.kind) {
const subscribers = getSubscriptions(msg.kind);
(subscribers || []).forEach((s) => {
try {
// let the subscriber handle the message
s(msg.data);
s(msg);
} catch (e) {
// eslint-disable-next-line no-console
console.error('Unhandled error in event handler', msg.data, e);
console.error('Unhandled error in event handler', msg, e);
}
});
}
@@ -85,14 +65,14 @@ export const createNymMixnetClient = async (options?: NymMixnetClientOptions): P
};
// let comlink handle interop with the web worker
const client: Client = Comlink.wrap<IWebWorker>(worker);
const client: Client = Comlink.wrap<IWebWorker>(nodeEndpoint(worker));
// set any options
if (options?.autoConvertStringMimeTypes) {
await client.setTextMimeTypes(options.autoConvertStringMimeTypes);
client.setTextMimeTypes(options.autoConvertStringMimeTypes);
} else {
// set some sensible defaults for text mime types
await client.setTextMimeTypes([MimeTypes.ApplicationJson, MimeTypes.TextPlain]);
client.setTextMimeTypes([MimeTypes.ApplicationJson, MimeTypes.TextPlain]);
}
// pass the client interop and subscription manage back to the caller
@@ -115,17 +95,13 @@ const createWorker = async () =>
// however, it will make this SDK bundle bigger because of Base64 inline data
const worker = new InlineWasmWebWorker();
worker.addEventListener('error', reject);
worker.addEventListener(
'message',
(msg) => {
worker.removeEventListener('error', reject);
if (msg.data?.kind === EventKinds.Loaded) {
resolve(worker);
} else {
reject(msg);
}
},
{ once: true },
);
worker.addListener('error', reject);
worker.addListener('message', (msg: { kind: EventKinds; args: any }) => {
worker.removeListener('error', reject);
if (msg.kind === EventKinds.Loaded) {
resolve(worker);
} else {
reject(msg);
}
});
});
@@ -0,0 +1,43 @@
/**
* @license
* Copyright 2019 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
// Borrowed from https://github.com/GoogleChromeLabs/comlink/blob/main/src/node-adapter.ts
import { Endpoint } from 'comlink';
export interface NodeEndpoint {
postMessage(message: any, transfer?: any[]): void;
on(type: string, listener: EventListenerOrEventListenerObject, options?: {}): void;
off(type: string, listener: EventListenerOrEventListenerObject, options?: {}): void;
start?: () => void;
}
export default function nodeEndpoint(nep: NodeEndpoint): Endpoint {
const listeners = new WeakMap();
return {
postMessage: nep.postMessage.bind(nep),
addEventListener: (_, eh) => {
const l = (data: any) => {
if ('handleEvent' in eh) {
eh.handleEvent({ data } as MessageEvent);
} else {
eh({ data } as MessageEvent);
}
};
nep.on('message', l);
listeners.set(eh, l);
},
removeEventListener: (_, eh) => {
const l = listeners.get(eh);
if (!l) {
return;
}
nep.off('message', l);
listeners.delete(eh);
},
start: nep.start && nep.start.bind(nep),
};
}
@@ -54,6 +54,7 @@ export const createSubscriptions = () => {
try {
handler(event);
} catch (e: any) {
// eslint-disable-next-line no-console
console.error(`Unhandled exception in handler for ${key}: `, e);
}
});
@@ -1,4 +1,29 @@
import type { DebugWasm } from '@nymproject/nym-client-wasm';
import type { DebugWasm } from '@nymproject/nym-client-wasm-node';
/**
* Options for the Nym mixnet client.
* @property autoConvertStringMimeTypes - An array of mime types.
* @example
* ```typescript
* const client = await createNymMixnetClient({
* autoConvertStringMimeTypes: [MimeTypes.ApplicationJson, MimeTypes.TextPlain],
* });
* ```
*/
export interface NymMixnetClientOptions {
autoConvertStringMimeTypes?: string[] | MimeTypes[];
}
/**
* The client for the Nym mixnet which gives access to client methods and event subscriptions.
* Returned by the {@link createNymMixnetClient} function.
*
*/
export interface NymMixnetClient {
client: Client;
events: Events;
}
/**
*
@@ -1,4 +1,6 @@
declare module 'web-worker:*' {
import { Worker } from 'worker_threads';
const WorkerFactory: new () => Worker;
export default WorkerFactory;
}
@@ -0,0 +1,261 @@
// eslint-disable-next-line import/first
import * as Comlink from 'comlink';
import * as crypto from 'node:crypto';
import * as fs from 'node:fs';
import * as process from 'node:process';
import { indexedDB } from 'fake-indexeddb';
import { parentPort } from 'worker_threads';
import '@nymproject/nym-client-wasm-node/nym_client_wasm_bg.wasm';
import {
ClientConfig,
NymClient,
NymClientBuilder,
decode_payload,
encode_payload_with_headers,
parse_utf8_string,
utf8_string_to_byte_array,
} from '@nymproject/nym-client-wasm-node';
import type {
BinaryMessageReceivedEvent,
ConnectedEvent,
IWebWorker,
LoadedEvent,
NymClientConfig,
OnRawPayloadFn,
RawMessageReceivedEvent,
StringMessageReceivedEvent,
} from './types';
import nodeEndpoint from './node-adapter';
import { EventKinds, MimeTypes } from './types';
// polyfill setup
const globalVar =
// eslint-disable-next-line @typescript-eslint/no-implied-eval, no-restricted-globals, no-nested-ternary
typeof WorkerGlobalScope !== 'undefined' ? self : typeof global !== 'undefined' ? global : Function('return this;')();
globalVar.indexedDB = indexedDB;
globalVar.fs = fs;
globalVar.process = process;
globalVar.performance = performance;
globalVar.TextEncoder = TextEncoder;
globalVar.TextDecoder = TextDecoder;
globalVar.crypto = crypto;
// eslint-disable-next-line no-console
console.log('[Nym WASM client] Starting Nym WASM web worker...');
/**
* Helper method to send typed messages.
* @param event The strongly typed message to send back to the calling thread.
* see https://nodejs.org/api/worker_threads.html#workerparentport
*/
// eslint-disable-next-line no-restricted-globals
const postMessageWithType = <E>(event: E) => parentPort?.postMessage(event);
/**
* This class holds the state of the Nym WASM client and provides any interop needed.
*/
class ClientWrapper {
client: NymClient | null = null;
builder: NymClientBuilder | null = null;
mimeTypes: string[] = [MimeTypes.TextPlain, MimeTypes.ApplicationJson];
/**
* Creates the WASM client and initialises it.
*/
init = (config: any, onRawPayloadHandler?: OnRawPayloadFn) => {
const onMessageHandler = (message: Uint8Array) => {
try {
if (onRawPayloadHandler) {
onRawPayloadHandler(message);
}
} catch (e) {
// eslint-disable-next-line no-console
console.error('Unhandled exception in `ClientWrapper.onRawPayloadHandler`: ', e);
}
};
this.builder = new NymClientBuilder(config, onMessageHandler);
};
/**
* Sets the mime-types that will be parsed for UTF-8 string content.
*
* @param mimeTypes An array of mime-types to treat as having string content.
*/
setTextMimeTypes = (mimeTypes: string[]) => {
this.mimeTypes = mimeTypes;
};
/**
* Gest the mime-types that are considered as string and will be automatically converted to byte arrays.
*/
getTextMimeTypes = () => this.mimeTypes;
/**
* Returns the address of this client.
*/
selfAddress = () => {
if (!this.client) {
// eslint-disable-next-line no-console
console.error('Client has not been initialised. Please call `init` first.');
return undefined;
}
return this.client.self_address();
};
/**
* Connects to the gateway and starts the client sending traffic.
*/
start = async () => {
if (!this.builder) {
// eslint-disable-next-line no-console
console.error('Client config has not been initialised. Please call `init` first.');
return;
}
// this is current limitation of wasm in rust - for async methods you can't take self by reference...
// I'm trying to figure out if I can somehow hack my way around it, but for time being you have to re-assign
// the object (it's the same one)
this.client = await this.builder.start_client();
};
/**
* Stops the client and cleans up.
*/
stop = () => {
if (!this.client) {
// eslint-disable-next-line no-console
console.error('Client has not been initialised. Please call `init` first.');
return;
}
this.client.free();
this.client = null;
};
send = async ({
payload,
recipient,
replySurbs = 0,
}: {
payload: Uint8Array;
recipient: string;
replySurbs?: number;
}) => {
if (!this.client) {
// eslint-disable-next-line no-console
console.error('Client has not been initialised. Please call `init` first.');
return;
}
// TODO: currently we don't do anything with the result, it needs some typing and exposed back on the main thread
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const res = await this.client.send_anonymous_message(payload, recipient, replySurbs);
};
}
// this wrapper handles any state that the wasm-pack interop needs, e.g. holding an instance of the instantiated WASM code
const wrapper = new ClientWrapper();
const startHandler = async (config: NymClientConfig) => {
// create the client, passing handlers for events
wrapper.init(new ClientConfig(config), async (message) => {
// fire an event with the raw message
postMessageWithType<RawMessageReceivedEvent>({
kind: EventKinds.RawMessageReceived,
args: { payload: message },
});
try {
// try to decode the payload to extract the mime-type, headers and payload body
const decodedPayload = decode_payload(message);
const { payload, headers } = decodedPayload;
const mimeType = decodedPayload.mimeType as MimeTypes;
if (wrapper.getTextMimeTypes().includes(mimeType)) {
const stringMessage = parse_utf8_string(payload);
// the payload is a string type (in the options at creation time, string mime-types are set, or fall back
// to defaults, such as `text/plain`, `application/json`, etc)
postMessageWithType<StringMessageReceivedEvent>({
kind: EventKinds.StringMessageReceived,
args: { mimeType, payload: stringMessage, payloadRaw: payload, headers },
});
return;
}
// the payload is a binary type
postMessageWithType<BinaryMessageReceivedEvent>({
kind: EventKinds.BinaryMessageReceived,
args: { mimeType, payload, headers },
});
} catch (e) {
// eslint-disable-next-line no-console
console.error('Failed to parse binary message', e);
}
});
// start the client sending traffic
await wrapper.start();
// get the address
const address = wrapper.selfAddress();
postMessageWithType<ConnectedEvent>({ kind: EventKinds.Connected, args: { address } });
};
// implement the public logic of this web worker (message exchange between the worker and caller is done by https://www.npmjs.com/package/comlink)
const webWorker: IWebWorker = {
start(config) {
// eslint-disable-next-line no-console
startHandler(config).catch((e) => console.error('[Nym WASM client] Failed to start', e));
},
stop() {
wrapper.stop();
},
selfAddress() {
return wrapper.selfAddress();
},
setTextMimeTypes(mimeTypes) {
wrapper.setTextMimeTypes(mimeTypes);
},
getTextMimeTypes() {
return wrapper.getTextMimeTypes();
},
send(args) {
const {
recipient,
replySurbs,
payload: { mimeType, headers },
} = args;
let payloadBytes = new Uint8Array();
if (mimeType && wrapper.getTextMimeTypes().includes(mimeType) && typeof args.payload.message === 'string') {
payloadBytes = utf8_string_to_byte_array(args.payload.message);
} else if (typeof args.payload.message !== 'string') {
payloadBytes = args.payload.message;
} else {
// eslint-disable-next-line no-console
console.error(
"[Nym WASM client] Payload is a string. It should be a UintArray, or the mime-type should be set with `setTextMimeTypes` or in the options for `init({ autoConvertStringMimeTypes: ['text/plain', 'application/json'] })` for auto-conversion",
);
return;
}
const payload = encode_payload_with_headers(mimeType || MimeTypes.ApplicationOctetStream, payloadBytes, headers);
wrapper
.send({ payload, recipient, replySurbs })
// eslint-disable-next-line no-console
.catch((e) => console.error('[Nym WASM client] Failed to send message', e));
},
rawSend(args) {
const { recipient, payload, replySurbs } = args;
wrapper
.send({ payload, replySurbs, recipient })
// eslint-disable-next-line no-console
.catch((e) => console.error('[Nym WASM client] Failed to send message', e));
},
};
// start comlink listening for messages and handle them above, if we are on a worker thread.
if (parentPort) {
Comlink.expose(webWorker, nodeEndpoint(parentPort));
}
// notify any listeners that the web worker has loaded - HOWEVER, the client has not been created and connected,
// listen for EventKinds.Connected before sending messages
postMessageWithType<LoadedEvent>({ kind: EventKinds.Loaded, args: { loaded: true } });
@@ -1,13 +1,7 @@
{
"compileOnSave": false,
"compilerOptions": {
"lib": [
"es2021",
"dom",
"dom.iterable",
"esnext",
"webworker"
],
"lib": ["es2021", "webworker"],
"outDir": "./dist/",
"module": "esnext",
"target": "esnext",
@@ -17,26 +11,9 @@
"skipLibCheck": true,
"skipDefaultLibCheck": true,
"declaration": true,
"allowSyntheticDefaultImports": true,
"baseUrl": "."
},
"include": [
"src/**/*.ts"
],
"exclude": [
"jest.config.js",
"webpack.config.js",
"webpack.prod.js",
"webpack.common.js",
"node_modules",
"**/node_modules",
"dist",
"**/dist",
"scripts",
"jest",
"__tests__",
"**/__tests__",
"__jest__",
"**/__jest__",
"config/*",
]
"include": ["src/**/*.ts"],
"exclude": ["node_modules", "**/node_modules", "dist", "**/dist", "scripts"]
}
File diff suppressed because it is too large Load Diff
+2
View File
@@ -36,6 +36,8 @@ packages=(
"ts/sdk/mix-fetch/esm"
"ts/sdk/mix-fetch/esm-full-fat"
"ts/sdk/nodejs-client/cjs"
"ts/sdk/node-tester/cjs"
"ts/sdk/node-tester/cjs-full-fat"
"ts/sdk/node-tester/esm"
+2
View File
@@ -23,6 +23,8 @@ packages2=(
"ts/sdk/mix-fetch/esm"
"ts/sdk/mix-fetch/esm-full-fat"
"ts/sdk/nodejs-client/cjs"
"ts/sdk/node-tester/cjs"
"ts/sdk/node-tester/cjs-full-fat"
"ts/sdk/node-tester/esm"