Polishing nodejs-client and mix-fetch-node

Signed-off-by: Sebastian Martinez <me@sebastinez.dev>
This commit is contained in:
Sebastian Martinez
2023-10-23 12:15:25 +02:00
parent 7129de4373
commit 06a96fa74a
28 changed files with 1249 additions and 183 deletions
+11 -1
View File
@@ -12,7 +12,7 @@ jobs:
uses: actions/setup-node@v3
with:
node-version: 18
registry-url: 'https://registry.npmjs.org'
registry-url: "https://registry.npmjs.org"
- name: Setup yarn
run: npm install -g yarn
@@ -28,6 +28,16 @@ jobs:
- name: Install wasm-opt
run: cargo install wasm-opt
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: "1.20"
- name: Install TinyGo
uses: acifani/setup-tinygo@v1
with:
tinygo-version: "0.27.0"
- name: Install dependencies
run: yarn
+4 -2
View File
@@ -10,6 +10,8 @@
"sdk/typescript/packages/mui-theme",
"sdk/typescript/packages/react-components",
"sdk/typescript/packages/validator-client",
"sdk/typescript/packages/mix-fetch-node",
"sdk/typescript/packages/nodejs-client",
"ts-packages/*",
"nym-wallet",
"nym-connect/**",
@@ -34,7 +36,7 @@
"prebuild:ci": "yarn dev:on && yarn",
"build:ci": "run-s build:types build:packages build:wasm build:ci:sdk",
"postbuild:ci": "yarn dev:off",
"build:ci:sdk": "lerna run --scope '{@nymproject/sdk,@nymproject/node-tester,@nymproject/sdk-react,@nymproject/mix-fetch}' build:dev --stream",
"build:ci:sdk": "lerna run --scope '{@nymproject/sdk,@nymproject/node-tester,@nymproject/sdk-react,@nymproject/mix-fetch,@nymproject/nodejs-client,@nymproject/mix-fetch-node}' build --stream",
"docs:prod:build": "run-s docs:prod:build:ws",
"docs:prod:build:ws": "lerna run docs:prod:build --stream",
"sdk:build": "./sdk/typescript/scripts/build-prod-sdk.sh",
@@ -53,4 +55,4 @@
"@npmcli/node-gyp": "^3.0.0",
"node-gyp": "^9.3.1"
}
}
}
@@ -1,4 +1,4 @@
import { createMixFetch, disconnectMixFetch } from '../dist/cjs/index.js';
const { createMixFetch, disconnectMixFetch } = require('@nymproject/mix-fetch-node-commonjs');
/**
* The main entry point
@@ -0,0 +1,14 @@
{
"name": "@nymproject/mix-fetch-node-js-example",
"version": "1.0.0",
"main": "index.js",
"license": "MIT",
"scripts": {
"start": "node index.js",
"start:server": "node server.js",
"test": "echo \"Error: no test specified\" && exit 1"
},
"dependencies": {
"@nymproject/mix-fetch-node-commonjs": "^1.2.1-rc.2"
}
}
@@ -0,0 +1,55 @@
const express = require('express');
const { mixFetch } = require('@nymproject/mix-fetch-node-commonjs');
const app = express();
app.use(express.static('public'));
app.get('/nym-fetch', async (req, res) => {
try {
const args = {
mode: 'unsafe-ignore-cors',
headers: {
'Content-Type': 'application/json',
},
};
const url = req.query.url;
if (!url) {
return res.status(400).send('input a valid url');
}
const extra = {
hiddenGateways: [
{
owner: 'n1ns3v70ul9gnl9l9fkyz8cyxfq75vjcmx8el0t3',
host: 'sandbox-gateway1.nymtech.net',
explicitIp: '35.158.238.80',
identityKey: 'HjNEDJuotWV8VD4ufeA1jeheTnfNJ7Jorevp57hgaZua',
sphinxKey: 'BoXeUD7ERGmzRauMjJD3itVNnQiH42ncUb6kcVLrb3dy',
},
],
};
const mixFetchOptions = {
nymApiUrl: 'https://sandbox-nym-api1.nymtech.net/api',
preferredGateway: 'HjNEDJuotWV8VD4ufeA1jeheTnfNJ7Jorevp57hgaZua',
preferredNetworkRequester:
'AzGdJ4MU78Ex22NEWfeycbN7bt3PFZr1MtKstAdhfELG.GSxnKnvKPjjQm3FdtsgG5KyhP6adGbPHRmFWDH4XfUpP@HjNEDJuotWV8VD4ufeA1jeheTnfNJ7Jorevp57hgaZua',
mixFetchOverride: {
requestTimeoutMs: 60_000,
},
forceTls: false,
extra,
};
const response = await mixFetch(url, args, mixFetchOptions);
const json = await response.json();
res.send(json);
} catch (error) {
console.log(error);
res.status(500).send(error.message);
}
});
app.listen(3000, () => console.log('Server running on port 3000'));
@@ -1,3 +0,0 @@
{
"presets": ["@babel/env"]
}
@@ -5,7 +5,7 @@ This package is a drop-in replacement for `fetch` in NodeJS to send HTTP request
## Usage
```js
const { mixFetch } = require('@nymproject/mix-fetch-node');
const { mixFetch } = require('@nymproject/mix-fetch-node-commonjs');
...
@@ -1,6 +1,6 @@
{
"name": "@nymproject/mix-fetch-node",
"version": "1.2.0",
"version": "1.2.1-rc.2",
"description": "This package is a drop-in replacement for `fetch` in NodeJS to send HTTP requests over the Nym Mixnet.",
"license": "Apache-2.0",
"author": "Nym Technologies SA",
@@ -28,7 +28,7 @@
"tsc": "tsc --noEmit true"
},
"dependencies": {
"@nymproject/mix-fetch-wasm-node": ">=1.2.0-rc.10 || ^1",
"@nymproject/mix-fetch-wasm-node": ">=1.2.0 || ^1",
"comlink": "^4.3.1",
"fake-indexeddb": "^5.0.0",
"node-fetch": "^3.3.2",
@@ -11,7 +11,7 @@ export default {
dir: 'dist/cjs',
format: 'cjs',
},
external: ['util', 'fake-indexeddb'],
onwarn,
plugins: [
resolve({
browser: false,
@@ -19,11 +19,6 @@ export default {
extensions: ['.js', '.ts'],
}),
commonjs(),
// TODO: One of the wasm functions calls `new WebSocket` at one point, which we aren't able to polyfill correctly yet.
modify({
find: 'const ret = new WebSocket(getStringFromWasm0(arg0, arg1));',
replace: 'const ws = require("ws"); const ret = new ws.WebSocket(getStringFromWasm0(arg0, arg1));',
}),
// TODO: `getObject(...).require` seems to generate a warning on Webpack but with Rollup we get a panic since it can't require.
// By hard coding the require here, we can workaround that.
// Reference: https://github.com/rust-random/getrandom/issues/224
@@ -38,3 +33,11 @@ export default {
}),
],
};
function onwarn(warning) {
// fake-indexeddb has a circular dependency that triggers a warning when rolled up
if (warning.code !== 'CIRCULAR_DEPENDENCY') {
// eslint-disable-next-line no-console
console.error(`(!) ${warning.message}`);
}
}
@@ -4,14 +4,14 @@ set -o errexit
set -o nounset
set -o pipefail
rm -rf ../../../../dist/ts/docs/tsdoc/nymproject/mix-fetch || true
rm -rf ../../../../dist/ts/docs/tsdoc/nymproject/mix-fetch-node || true
# run the build
yarn docs:generate:prod
# move the output outside of the yarn/npm workspaces
mkdir -p ../../../../dist/ts/docs/tsdoc/nymproject
mv docs ../../../../dist/ts/docs/tsdoc/nymproject/mix-fetch
mv docs ../../../../dist/ts/docs/tsdoc/nymproject/mix-fetch-node
echo "Output can be found in:"
realpath ../../../../dist/ts/docs/tsdoc/nymproject/mix-fetch
realpath ../../../../dist/ts/docs/tsdoc/nymproject/mix-fetch-node
@@ -5,7 +5,7 @@ set -o nounset
set -o pipefail
rm -rf dist || true
rm -rf ../../../../dist/ts/sdk/mix-fetch || true
rm -rf ../../../../dist/ts/sdk/mix-fetch-node || true
# run the build
scripts/build.sh
@@ -14,7 +14,7 @@ node scripts/buildPackageJson.mjs
# move the output outside of the yarn/npm workspaces
mkdir -p ../../../../dist/ts/sdk
mv dist ../../../../dist/ts/sdk
mv ../../../../dist/ts/sdk/dist ../../../../dist/ts/sdk/mix-fetch
mv ../../../../dist/ts/sdk/dist ../../../../dist/ts/sdk/mix-fetch-node
echo "Output can be found in:"
realpath ../../../../dist/ts/sdk/mix-fetch
realpath ../../../../dist/ts/sdk/mix-fetch-node
@@ -1,5 +1,4 @@
/* eslint-disable no-underscore-dangle */
import type { SetupMixFetchOps, IMixFetchFn } from './types';
import { createMixFetch as createMixFetchInternal } from './create-mix-fetch';
@@ -1,4 +1,5 @@
import './polyfill';
import { loadWasm } from './wasm-loading';
import { run } from './main';
@@ -1,10 +1,10 @@
/* eslint-disable no-console */
import type { IMixFetchWebWorker, LoadedEvent } from '../types';
import * as Comlink from 'comlink';
import { parentPort } from 'node:worker_threads';
import { setupMixFetch, disconnectMixFetch } from '@nymproject/mix-fetch-wasm-node';
import type { IMixFetchWebWorker, LoadedEvent } from '../types';
import nodeEndpoint from '../node-adapter';
import { EventKinds, ResponseBodyConfigMap, ResponseBodyConfigMapDefaults } from '../types';
import { handleResponseMimeTypes } from './handle-response-mime-types';
@@ -1,8 +1,8 @@
import { TextDecoder, TextEncoder } from 'node:util';
import * as crypto from 'node:crypto';
import * as fs from 'node:fs';
import WebSocket from 'ws';
import fetch, { Headers, Request, Response } from 'node-fetch';
import { TextDecoder, TextEncoder } from 'node:util';
import { Worker } from 'node:worker_threads';
import { indexedDB } from 'fake-indexeddb';
@@ -18,7 +18,6 @@ import { indexedDB } from 'fake-indexeddb';
(globalThis as any).Headers = Headers;
(globalThis as any).Request = Request;
(globalThis as any).Response = Response;
(globalThis as any).Headers = Headers;
(globalThis as any).fs = fs;
(globalThis as any).crypto = crypto;
(globalThis as any).WebSocket = WebSocket;
@@ -5,15 +5,13 @@ This package is a NodeJS client that uses the wasm from the [Sphinx webassembly
## Usage
```js
const { createNymMixnetClient } = require('../dist/cjs/index.js');
const { createNymMixnetClient } = require('@nymproject/nodejs-client-commonjs');
async () => {
const nym = await createNymMixnetClient();
nym.events.subscribeToTextMessageReceivedEvent(async (e) => {
if (e.args.payload === 'Hello') {
await nym.client.stop();
}
console.log("message received", e.args.payload);
});
const nymApiUrl = 'https://validator.nymtech.net/api/';
@@ -0,0 +1,25 @@
const { createNymMixnetClient } = require('../dist/cjs/index.js');
(async () => {
const nym = await createNymMixnetClient();
nym.events.subscribeToTextMessageReceivedEvent(async ({ args: { payload, mimeType } }) => {
console.log(`received message: ${payload}`);
console.log(`with mimeType: ${mimeType}`);
});
// start the client and connect to a gateway
await nym.client.start({
nymApiUrl: 'https://validator.nymtech.net/api/',
clientId: 'my-client',
});
nym.events.subscribeToConnected(async (e) => {
// send a message to yourself
const message = 'Hello';
const recipient = await nym.client.selfAddress();
console.log('main thread address: ', recipient);
console.log(`sending "${message}" to ourselves...`);
await nym.client.send({ payload: { message, mimeType: 'text/plain' }, recipient });
});
})();
@@ -1,6 +1,6 @@
{
"name": "@nymproject/nodejs-client",
"version": "1.2.0-rc.10",
"version": "1.2.1-rc.3",
"license": "Apache-2.0",
"author": "Nym Technologies SA",
"files": [
@@ -25,29 +25,20 @@
"tsc": "tsc --noEmit true"
},
"dependencies": {
"@nymproject/nym-client-wasm-node": "^1.2.0",
"@nymproject/nym-client-wasm-node": ">=1.2.0 || ^1",
"comlink": "^4.3.1",
"fake-indexeddb": "^4.0.2",
"rollup-plugin-polyfill": "^4.2.0",
"ws": "^8.14.2"
},
"devDependencies": {
"@babel/core": "^7.15.0",
"@babel/plugin-transform-async-to-generator": "^7.14.5",
"@babel/preset-env": "^7.15.0",
"@babel/preset-react": "^7.14.5",
"@babel/preset-typescript": "^7.15.0",
"@nymproject/eslint-config-react-typescript": "^1.0.0",
"@rollup/plugin-commonjs": "^24.0.1",
"@rollup/plugin-inject": "^5.0.3",
"@rollup/plugin-json": "^6.0.0",
"@rollup/plugin-node-resolve": "^15.0.1",
"@rollup/plugin-replace": "^5.0.2",
"@rollup/plugin-terser": "^0.2.1",
"@rollup/plugin-typescript": "^10.0.1",
"@rollup/plugin-url": "^8.0.1",
"@rollup/plugin-wasm": "^6.1.1",
"@types/jest": "^27.0.1",
"@types/node": "^16.7.13",
"@typescript-eslint/eslint-plugin": "^5.13.0",
"@typescript-eslint/parser": "^5.13.0",
@@ -62,8 +53,6 @@
"eslint-plugin-prettier": "^4.0.0",
"eslint-plugin-react": "^7.29.2",
"eslint-plugin-react-hooks": "^4.3.0",
"handlebars": "^4.7.8",
"jest": "^29.5.0",
"nodemon": "3.0.1",
"reload": "^3.2.1",
"rimraf": "^3.0.2",
@@ -71,7 +60,6 @@
"rollup-plugin-base64": "^1.0.1",
"rollup-plugin-modify": "^3.0.0",
"rollup-plugin-web-worker-loader": "^1.6.1",
"ts-jest": "^29.1.0",
"ts-loader": "^9.4.2",
"typedoc": "^0.24.8",
"typescript": "^4.8.4"
@@ -1,9 +1,9 @@
/* eslint-disable import/no-extraneous-dependencies */
import typescript from '@rollup/plugin-typescript';
import resolve from '@rollup/plugin-node-resolve';
import { wasm } from '@rollup/plugin-wasm';
import webWorkerLoader from 'rollup-plugin-web-worker-loader';
import replace from '@rollup/plugin-replace';
import resolve from '@rollup/plugin-node-resolve';
import typescript from '@rollup/plugin-typescript';
import webWorkerLoader from 'rollup-plugin-web-worker-loader';
import { wasm } from '@rollup/plugin-wasm';
export default {
input: 'src/index.ts',
@@ -1,9 +1,9 @@
/* eslint-disable import/no-extraneous-dependencies */
import typescript from '@rollup/plugin-typescript';
import resolve from '@rollup/plugin-node-resolve';
import { wasm } from '@rollup/plugin-wasm';
import commonjs from '@rollup/plugin-commonjs';
import modify from 'rollup-plugin-modify';
import resolve from '@rollup/plugin-node-resolve';
import typescript from '@rollup/plugin-typescript';
import { wasm } from '@rollup/plugin-wasm';
export default {
input: 'src/worker.ts',
@@ -11,7 +11,6 @@ export default {
dir: 'dist/cjs',
format: 'cjs',
},
external: ['util', 'fake-indexeddb'],
plugins: [
resolve({
browser: false,
@@ -19,11 +18,6 @@ export default {
extensions: ['.js', '.ts'],
}),
commonjs(),
// TODO: One of the wasm functions calls `new WebSocket` at one point, which we aren't able to polyfill correctly yet.
modify({
find: 'const ret = new WebSocket(getStringFromWasm0(arg0, arg1));',
replace: 'const ws = require("ws"); const ret = new ws.WebSocket(getStringFromWasm0(arg0, arg1));',
}),
// TODO: `getObject(...).require` seems to generate a warning on Webpack but with Rollup we get a panic since it can't require.
// By hard coding the require here, we can workaround that.
// Reference: https://github.com/rust-random/getrandom/issues/224
@@ -21,8 +21,8 @@ import nodeEndpoint from './node-adapter';
/**
* Create a client to send and receive traffic from the Nym mixnet.
* @required
* @returns
* @param options - An optional of options
* @returns { Promise<NymMixnetClient> } A new instance of the NymMixnetClient.
* @example
* ```typescript
* const client = await createNymMixnetClient();
@@ -0,0 +1,23 @@
import * as crypto from 'node:crypto';
import * as fs from 'node:fs';
import ws from 'ws';
import { TextDecoder, TextEncoder } from 'node:util';
import { Worker } from 'node:worker_threads';
import { indexedDB } from 'fake-indexeddb';
import { performance } from 'node:perf_hooks';
(globalThis as any).performance = performance;
(globalThis as any).TextDecoder = TextDecoder;
(globalThis as any).fs = fs;
(globalThis as any).crypto = crypto;
(globalThis as any).ws = ws;
(globalThis as any).Worker = Worker;
globalThis.process = process;
globalThis.TextEncoder = TextEncoder;
globalThis.Reflect = Reflect;
globalThis.Proxy = Proxy;
globalThis.Error = Error;
globalThis.Promise = Promise;
globalThis.Object = Object;
globalThis.indexedDB = indexedDB;
@@ -10,7 +10,6 @@ import type { DebugWasm } from '@nymproject/nym-client-wasm-node';
* });
* ```
*/
export interface NymMixnetClientOptions {
autoConvertStringMimeTypes?: string[] | MimeTypes[];
}
@@ -18,7 +17,8 @@ export interface NymMixnetClientOptions {
/**
* The client for the Nym mixnet which gives access to client methods and event subscriptions.
* Returned by the {@link createNymMixnetClient} function.
*
* @property client - The sphinx nym wasm client.
* @property events - Different streams of events provided by the client.
*/
export interface NymMixnetClient {
client: Client;
@@ -1,9 +1,6 @@
// eslint-disable-next-line import/first
import './polyfill';
import * as Comlink from 'comlink';
import * as crypto from 'node:crypto';
import * as fs from 'node:fs';
import * as process from 'node:process';
import { indexedDB } from 'fake-indexeddb';
import { parentPort } from 'worker_threads';
import '@nymproject/nym-client-wasm-node/nym_client_wasm_bg.wasm';
@@ -31,19 +28,6 @@ import type {
import nodeEndpoint from './node-adapter';
import { EventKinds, MimeTypes } from './types';
// polyfill setup
const globalVar =
// eslint-disable-next-line @typescript-eslint/no-implied-eval, no-restricted-globals, no-nested-ternary
typeof WorkerGlobalScope !== 'undefined' ? self : typeof global !== 'undefined' ? global : Function('return this;')();
globalVar.indexedDB = indexedDB;
globalVar.fs = fs;
globalVar.process = process;
globalVar.performance = performance;
globalVar.TextEncoder = TextEncoder;
globalVar.TextDecoder = TextDecoder;
globalVar.crypto = crypto;
// eslint-disable-next-line no-console
console.log('[Nym WASM client] Starting Nym WASM web worker...');
@@ -52,7 +36,6 @@ console.log('[Nym WASM client] Starting Nym WASM web worker...');
* @param event The strongly typed message to send back to the calling thread.
* see https://nodejs.org/api/worker_threads.html#workerparentport
*/
// eslint-disable-next-line no-restricted-globals
const postMessageWithType = <E>(event: E) => parentPort?.postMessage(event);
/**
@@ -152,8 +135,7 @@ class ClientWrapper {
return;
}
// TODO: currently we don't do anything with the result, it needs some typing and exposed back on the main thread
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const res = await this.client.send_anonymous_message(payload, recipient, replySurbs);
await this.client.send_anonymous_message(payload, recipient, replySurbs);
};
}
+1 -1
View File
@@ -14,7 +14,7 @@ rm -rf dist || true
yarn build:wasm
# build the Typescript SDK packages
yarn build:sdk
yarn build:ci:sdk
# build documentation
yarn docs:prod:build
+5 -42
View File
@@ -10,46 +10,9 @@ set -o pipefail
cd dist
#packages=(
#chat-app/parcel
#chat-app/plain-html
#chat-app/react-webpack-with-theme-example
#chrome-extension
#firefox-extension
#node-tester/parcel
#node-tester/plain-html
#node-tester/react
#react/mui-theme
#react/sdk-react
#)
packages=(
"wasm/client"
"wasm/mix-fetch"
"wasm/node-tester"
"wasm/extension-storage"
"node/wasm/client"
"node/wasm/mix-fetch"
"ts/sdk/mix-fetch/cjs"
"ts/sdk/mix-fetch/cjs-full-fat"
"ts/sdk/mix-fetch/esm"
"ts/sdk/mix-fetch/esm-full-fat"
"ts/sdk/nodejs-client/cjs"
"ts/sdk/mix-fetch-node/cjs"
"ts/sdk/node-tester/cjs"
"ts/sdk/node-tester/cjs-full-fat"
"ts/sdk/node-tester/esm"
"ts/sdk/node-tester/esm-full-fat"
"ts/sdk/sdk/cjs"
"ts/sdk/sdk/cjs-full-fat"
"ts/sdk/sdk/esm"
"ts/sdk/sdk/esm-full-fat"
"ts/sdk/contract-clients"
)
pushd () {
@@ -57,7 +20,7 @@ pushd () {
}
popd () {
command popd "$@" > /dev/null
command popd > /dev/null
}
echo "Summary of versions of packages to publish:"
@@ -65,8 +28,8 @@ echo ""
for item in "${packages[@]}"
do
pushd "$item"
cat package.json | jq -r '. | "📦 " + .version + " " +.name'
popd
jq -r '. | "📦 " + .version + " " +.name' < package.json
popd
done
echo ""
@@ -79,8 +42,8 @@ do
(( COUNTER++ ))
pushd "$item"
echo "🚀 Publishing $item... (${COUNTER} of ${#packages[@]})"
cat package.json | jq -r '. | .name + " " +.version'
npm publish --access=public
jq -r '. | .name + " " +.version' < package.json
npm publish --access=public --verbose
popd
echo ""
done
+2 -29
View File
@@ -7,34 +7,8 @@ set -o pipefail
cd dist
packages=(
"ts/sdk/node-tester/cjs"
"ts/sdk/node-tester/cjs-full-fat"
"ts/sdk/node-tester/esm"
"ts/sdk/node-tester/esm-full-fat"
)
packages2=(
"wasm/client"
"wasm/mix-fetch"
"wasm/node-tester"
"wasm/extension-storage"
"ts/sdk/mix-fetch/cjs"
"ts/sdk/mix-fetch/cjs-full-fat"
"ts/sdk/mix-fetch/esm"
"ts/sdk/mix-fetch/esm-full-fat"
"ts/sdk/nodejs-client/cjs"
"ts/sdk/mix-fetch-node/cjs"
"ts/sdk/node-tester/cjs"
"ts/sdk/node-tester/cjs-full-fat"
"ts/sdk/node-tester/esm"
"ts/sdk/node-tester/esm-full-fat"
"ts/sdk/sdk/cjs"
"ts/sdk/sdk/cjs-full-fat"
"ts/sdk/sdk/esm"
"ts/sdk/sdk/esm-full-fat"
)
pushd () {
@@ -42,7 +16,7 @@ pushd () {
}
popd () {
command popd "$@" > /dev/null
command popd > /dev/null
}
echo "Summary of versions of packages to publish:"
@@ -50,7 +24,7 @@ echo ""
for item in "${packages[@]}"
do
pushd "$item"
cat package.json | jq -r '. | "📦 " + .version + " " +.name'
jq -r '. | "📦 " + .version + " " +.name' < package.json
popd
done
@@ -68,4 +42,3 @@ do
done
echo ""
echo "✅ Done"
+1069 -29
View File
File diff suppressed because it is too large Load Diff