fix: come back quickly after a hub restart, and pin the dependencies
Two things helloeco ran into while wiring the gateway. grpc-js waits a second before its second reconnect attempt and backs off towards two minutes. That is right for a far-away service and wrong for a hub on the same network: after a restart or a deploy a gateway sits out the backoff with requests in hand. The client now sets initial 200 ms and a 5 s ceiling, and a caller's own channelOptions still win. The client exposes the options it ended up with, which is the only way to tell a default that applies from one that was silently replaced. The dependencies are pinned exactly instead of by range. This package is consumed by a handful of services, not by a library ecosystem, and a surprise minor of grpc-js is what breaks a portal at three in the morning. Raising one is a commit, with the skew job in front of it. Guards: the defaults and the precedence are asserted (probed by mutation), plus a case that a run works again once the hub is back. Signed-off-by: flemming-it <sf@flemming.it>
This commit is contained in:
parent
986377f1df
commit
2131193f1f
4 changed files with 86 additions and 6 deletions
18
README.md
18
README.md
|
|
@ -115,6 +115,24 @@ memory and is lost, while approvals live in SQLite and survive. A
|
|||
gateway has to treat "approval still pending after a restart" as a run
|
||||
it must start again.
|
||||
|
||||
## Versions and the channel
|
||||
|
||||
The dependencies are pinned exactly, not by range. This package is
|
||||
consumed by a handful of services rather than by a library
|
||||
ecosystem, and a surprise minor of grpc-js is the kind of thing that
|
||||
breaks a portal at three in the morning. Raising one is a commit, and
|
||||
the skew job runs against a released hub before it lands.
|
||||
|
||||
The client sets two channel options by default:
|
||||
`grpc.initial_reconnect_backoff_ms: 200` and
|
||||
`grpc.max_reconnect_backoff_ms: 5000`. grpc-js otherwise waits a
|
||||
second before its second reconnect attempt and backs off towards two
|
||||
minutes, which is right for a far-away service and wrong for a hub on
|
||||
the same network: after a restart or a deploy, a gateway would sit
|
||||
out that backoff with requests in hand. Anything in
|
||||
`channelOptions` wins over these, and `client.channelOptions` says
|
||||
what the client ended up with.
|
||||
|
||||
## Development
|
||||
|
||||
```bash
|
||||
|
|
|
|||
|
|
@ -32,11 +32,11 @@
|
|||
"check": "tsc -p tsconfig.json --noEmit && tsc -p tsconfig.test.json --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@grpc/grpc-js": "^1.14.0",
|
||||
"@grpc/proto-loader": "^0.8.0"
|
||||
"@grpc/grpc-js": "1.14.6",
|
||||
"@grpc/proto-loader": "0.8.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.10.0",
|
||||
"typescript": "^5.9.0"
|
||||
"@types/node": "22.20.5",
|
||||
"typescript": "5.9.3"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -39,7 +39,10 @@ export interface ChainClientOptions {
|
|||
project?: string;
|
||||
/** Deadline for a single call, in milliseconds. Default 30000. */
|
||||
deadlineMs?: number;
|
||||
/** Passed through to grpc-js, e.g. message size limits. */
|
||||
/**
|
||||
* Passed through to grpc-js, e.g. message size limits. What is set
|
||||
* here wins over the defaults below.
|
||||
*/
|
||||
channelOptions?: grpc.ChannelOptions;
|
||||
}
|
||||
|
||||
|
|
@ -156,6 +159,14 @@ const DEFAULT_DEADLINE_MS = 30_000;
|
|||
|
||||
/** A connection to one hub. */
|
||||
export class ChainClient {
|
||||
/**
|
||||
* The channel options this client was built with: its own defaults
|
||||
* with the caller's on top. Readable so a service can assert what
|
||||
* it is running with, which is the only way to tell a default that
|
||||
* applies from one that was silently replaced.
|
||||
*/
|
||||
readonly channelOptions: grpc.ChannelOptions;
|
||||
|
||||
private readonly hub: grpc.Client;
|
||||
private readonly admin: grpc.Client;
|
||||
private readonly token: string | undefined;
|
||||
|
|
@ -170,9 +181,20 @@ export class ChainClient {
|
|||
options.tls === true ? undefined : options.tls.certChain,
|
||||
)
|
||||
: grpc.credentials.createInsecure();
|
||||
const channelOptions = options.channelOptions ?? {};
|
||||
// A hub that comes back should be usable again at once. grpc-js
|
||||
// waits a second before the first reconnect and then backs off
|
||||
// towards two minutes, which is right for a far-away service and
|
||||
// wrong for a hub on the same network: after a restart or a
|
||||
// deploy, a gateway would sit out the backoff with requests in
|
||||
// hand. The caller's own options still win.
|
||||
const channelOptions: grpc.ChannelOptions = {
|
||||
'grpc.initial_reconnect_backoff_ms': 200,
|
||||
'grpc.max_reconnect_backoff_ms': 5_000,
|
||||
...(options.channelOptions ?? {}),
|
||||
};
|
||||
const Hub = hubServiceConstructor();
|
||||
const HubAdmin = hubAdminServiceConstructor();
|
||||
this.channelOptions = channelOptions;
|
||||
this.hub = new Hub(options.address, credentials, channelOptions);
|
||||
this.admin = new HubAdmin(options.address, credentials, channelOptions);
|
||||
this.token = options.token;
|
||||
|
|
|
|||
|
|
@ -275,3 +275,43 @@ async function waitFor<T>(produce: () => Promise<T | undefined>, timeoutMs = 3_0
|
|||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
}
|
||||
}
|
||||
|
||||
describe('the channel after a hub comes back', () => {
|
||||
it('is built to retry soon, not after a second', () => {
|
||||
// grpc-js waits a second before its second reconnect attempt and
|
||||
// backs off towards two minutes. That is right for a far-away
|
||||
// service and wrong for a hub on the same network: after a
|
||||
// restart or a deploy, a gateway would sit out the backoff with
|
||||
// requests in hand.
|
||||
const client = new ChainClient({ address: '127.0.0.1:1' });
|
||||
assert.equal(client.channelOptions['grpc.initial_reconnect_backoff_ms'], 200);
|
||||
assert.equal(client.channelOptions['grpc.max_reconnect_backoff_ms'], 5_000);
|
||||
client.close();
|
||||
});
|
||||
|
||||
it('lets a caller override what the client set', () => {
|
||||
const client = new ChainClient({
|
||||
address: '127.0.0.1:1',
|
||||
channelOptions: { 'grpc.initial_reconnect_backoff_ms': 1_234 },
|
||||
});
|
||||
assert.equal(client.channelOptions['grpc.initial_reconnect_backoff_ms'], 1_234);
|
||||
// And the other default is still there.
|
||||
assert.equal(client.channelOptions['grpc.max_reconnect_backoff_ms'], 5_000);
|
||||
client.close();
|
||||
});
|
||||
|
||||
it('works again once the hub is back', async () => {
|
||||
const hub = new FakeHub();
|
||||
const address = await hub.start();
|
||||
const client = new ChainClient({ address, deadlineMs: 5_000 });
|
||||
await client.health();
|
||||
|
||||
await hub.restart();
|
||||
hub.succeeds({ report: 'wieder da' });
|
||||
const result = await client.submit({ flowYaml: FLOW });
|
||||
assert.equal(asText(result.outputs.report), 'wieder da');
|
||||
|
||||
client.close();
|
||||
await hub.close();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue