fix: come back quickly after a hub restart, and pin the dependencies

Two things helloeco ran into while wiring the gateway.

grpc-js waits a second before its second reconnect attempt and backs
off towards two minutes. That is right for a far-away service and
wrong for a hub on the same network: after a restart or a deploy a
gateway sits out the backoff with requests in hand. The client now
sets initial 200 ms and a 5 s ceiling, and a caller's own
channelOptions still win. The client exposes the options it ended up
with, which is the only way to tell a default that applies from one
that was silently replaced.

The dependencies are pinned exactly instead of by range. This package
is consumed by a handful of services, not by a library ecosystem, and
a surprise minor of grpc-js is what breaks a portal at three in the
morning. Raising one is a commit, with the skew job in front of it.

Guards: the defaults and the precedence are asserted (probed by
mutation), plus a case that a run works again once the hub is back.

Signed-off-by: flemming-it <sf@flemming.it>
This commit is contained in:
flemming-it 2026-10-27 03:00:55 +01:00
parent 986377f1df
commit 2131193f1f
4 changed files with 86 additions and 6 deletions

View file

@ -115,6 +115,24 @@ memory and is lost, while approvals live in SQLite and survive. A
gateway has to treat "approval still pending after a restart" as a run gateway has to treat "approval still pending after a restart" as a run
it must start again. it must start again.
## Versions and the channel
The dependencies are pinned exactly, not by range. This package is
consumed by a handful of services rather than by a library
ecosystem, and a surprise minor of grpc-js is the kind of thing that
breaks a portal at three in the morning. Raising one is a commit, and
the skew job runs against a released hub before it lands.
The client sets two channel options by default:
`grpc.initial_reconnect_backoff_ms: 200` and
`grpc.max_reconnect_backoff_ms: 5000`. grpc-js otherwise waits a
second before its second reconnect attempt and backs off towards two
minutes, which is right for a far-away service and wrong for a hub on
the same network: after a restart or a deploy, a gateway would sit
out that backoff with requests in hand. Anything in
`channelOptions` wins over these, and `client.channelOptions` says
what the client ended up with.
## Development ## Development
```bash ```bash

View file

@ -32,11 +32,11 @@
"check": "tsc -p tsconfig.json --noEmit && tsc -p tsconfig.test.json --noEmit" "check": "tsc -p tsconfig.json --noEmit && tsc -p tsconfig.test.json --noEmit"
}, },
"dependencies": { "dependencies": {
"@grpc/grpc-js": "^1.14.0", "@grpc/grpc-js": "1.14.6",
"@grpc/proto-loader": "^0.8.0" "@grpc/proto-loader": "0.8.1"
}, },
"devDependencies": { "devDependencies": {
"@types/node": "^22.10.0", "@types/node": "22.20.5",
"typescript": "^5.9.0" "typescript": "5.9.3"
} }
} }

View file

@ -39,7 +39,10 @@ export interface ChainClientOptions {
project?: string; project?: string;
/** Deadline for a single call, in milliseconds. Default 30000. */ /** Deadline for a single call, in milliseconds. Default 30000. */
deadlineMs?: number; deadlineMs?: number;
/** Passed through to grpc-js, e.g. message size limits. */ /**
* Passed through to grpc-js, e.g. message size limits. What is set
* here wins over the defaults below.
*/
channelOptions?: grpc.ChannelOptions; channelOptions?: grpc.ChannelOptions;
} }
@ -156,6 +159,14 @@ const DEFAULT_DEADLINE_MS = 30_000;
/** A connection to one hub. */ /** A connection to one hub. */
export class ChainClient { export class ChainClient {
/**
* The channel options this client was built with: its own defaults
* with the caller's on top. Readable so a service can assert what
* it is running with, which is the only way to tell a default that
* applies from one that was silently replaced.
*/
readonly channelOptions: grpc.ChannelOptions;
private readonly hub: grpc.Client; private readonly hub: grpc.Client;
private readonly admin: grpc.Client; private readonly admin: grpc.Client;
private readonly token: string | undefined; private readonly token: string | undefined;
@ -170,9 +181,20 @@ export class ChainClient {
options.tls === true ? undefined : options.tls.certChain, options.tls === true ? undefined : options.tls.certChain,
) )
: grpc.credentials.createInsecure(); : grpc.credentials.createInsecure();
const channelOptions = options.channelOptions ?? {}; // A hub that comes back should be usable again at once. grpc-js
// waits a second before the first reconnect and then backs off
// towards two minutes, which is right for a far-away service and
// wrong for a hub on the same network: after a restart or a
// deploy, a gateway would sit out the backoff with requests in
// hand. The caller's own options still win.
const channelOptions: grpc.ChannelOptions = {
'grpc.initial_reconnect_backoff_ms': 200,
'grpc.max_reconnect_backoff_ms': 5_000,
...(options.channelOptions ?? {}),
};
const Hub = hubServiceConstructor(); const Hub = hubServiceConstructor();
const HubAdmin = hubAdminServiceConstructor(); const HubAdmin = hubAdminServiceConstructor();
this.channelOptions = channelOptions;
this.hub = new Hub(options.address, credentials, channelOptions); this.hub = new Hub(options.address, credentials, channelOptions);
this.admin = new HubAdmin(options.address, credentials, channelOptions); this.admin = new HubAdmin(options.address, credentials, channelOptions);
this.token = options.token; this.token = options.token;

View file

@ -275,3 +275,43 @@ async function waitFor<T>(produce: () => Promise<T | undefined>, timeoutMs = 3_0
await new Promise((resolve) => setTimeout(resolve, 20)); await new Promise((resolve) => setTimeout(resolve, 20));
} }
} }
describe('the channel after a hub comes back', () => {
it('is built to retry soon, not after a second', () => {
// grpc-js waits a second before its second reconnect attempt and
// backs off towards two minutes. That is right for a far-away
// service and wrong for a hub on the same network: after a
// restart or a deploy, a gateway would sit out the backoff with
// requests in hand.
const client = new ChainClient({ address: '127.0.0.1:1' });
assert.equal(client.channelOptions['grpc.initial_reconnect_backoff_ms'], 200);
assert.equal(client.channelOptions['grpc.max_reconnect_backoff_ms'], 5_000);
client.close();
});
it('lets a caller override what the client set', () => {
const client = new ChainClient({
address: '127.0.0.1:1',
channelOptions: { 'grpc.initial_reconnect_backoff_ms': 1_234 },
});
assert.equal(client.channelOptions['grpc.initial_reconnect_backoff_ms'], 1_234);
// And the other default is still there.
assert.equal(client.channelOptions['grpc.max_reconnect_backoff_ms'], 5_000);
client.close();
});
it('works again once the hub is back', async () => {
const hub = new FakeHub();
const address = await hub.start();
const client = new ChainClient({ address, deadlineMs: 5_000 });
await client.health();
await hub.restart();
hub.succeeds({ report: 'wieder da' });
const result = await client.submit({ flowYaml: FLOW });
assert.equal(asText(result.outputs.report), 'wieder da');
client.close();
await hub.close();
});
});