fix: come back quickly after a hub restart, and pin the dependencies
Two things helloeco ran into while wiring the gateway. grpc-js waits a second before its second reconnect attempt and backs off towards two minutes. That is right for a far-away service and wrong for a hub on the same network: after a restart or a deploy a gateway sits out the backoff with requests in hand. The client now sets initial 200 ms and a 5 s ceiling, and a caller's own channelOptions still win. The client exposes the options it ended up with, which is the only way to tell a default that applies from one that was silently replaced. The dependencies are pinned exactly instead of by range. This package is consumed by a handful of services, not by a library ecosystem, and a surprise minor of grpc-js is what breaks a portal at three in the morning. Raising one is a commit, with the skew job in front of it. Guards: the defaults and the precedence are asserted (probed by mutation), plus a case that a run works again once the hub is back. Signed-off-by: flemming-it <sf@flemming.it>
This commit is contained in:
parent
986377f1df
commit
2131193f1f
4 changed files with 86 additions and 6 deletions
18
README.md
18
README.md
|
|
@ -115,6 +115,24 @@ memory and is lost, while approvals live in SQLite and survive. A
|
||||||
gateway has to treat "approval still pending after a restart" as a run
|
gateway has to treat "approval still pending after a restart" as a run
|
||||||
it must start again.
|
it must start again.
|
||||||
|
|
||||||
|
## Versions and the channel
|
||||||
|
|
||||||
|
The dependencies are pinned exactly, not by range. This package is
|
||||||
|
consumed by a handful of services rather than by a library
|
||||||
|
ecosystem, and a surprise minor of grpc-js is the kind of thing that
|
||||||
|
breaks a portal at three in the morning. Raising one is a commit, and
|
||||||
|
the skew job runs against a released hub before it lands.
|
||||||
|
|
||||||
|
The client sets two channel options by default:
|
||||||
|
`grpc.initial_reconnect_backoff_ms: 200` and
|
||||||
|
`grpc.max_reconnect_backoff_ms: 5000`. grpc-js otherwise waits a
|
||||||
|
second before its second reconnect attempt and backs off towards two
|
||||||
|
minutes, which is right for a far-away service and wrong for a hub on
|
||||||
|
the same network: after a restart or a deploy, a gateway would sit
|
||||||
|
out that backoff with requests in hand. Anything in
|
||||||
|
`channelOptions` wins over these, and `client.channelOptions` says
|
||||||
|
what the client ended up with.
|
||||||
|
|
||||||
## Development
|
## Development
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|
|
||||||
|
|
@ -32,11 +32,11 @@
|
||||||
"check": "tsc -p tsconfig.json --noEmit && tsc -p tsconfig.test.json --noEmit"
|
"check": "tsc -p tsconfig.json --noEmit && tsc -p tsconfig.test.json --noEmit"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@grpc/grpc-js": "^1.14.0",
|
"@grpc/grpc-js": "1.14.6",
|
||||||
"@grpc/proto-loader": "^0.8.0"
|
"@grpc/proto-loader": "0.8.1"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/node": "^22.10.0",
|
"@types/node": "22.20.5",
|
||||||
"typescript": "^5.9.0"
|
"typescript": "5.9.3"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -39,7 +39,10 @@ export interface ChainClientOptions {
|
||||||
project?: string;
|
project?: string;
|
||||||
/** Deadline for a single call, in milliseconds. Default 30000. */
|
/** Deadline for a single call, in milliseconds. Default 30000. */
|
||||||
deadlineMs?: number;
|
deadlineMs?: number;
|
||||||
/** Passed through to grpc-js, e.g. message size limits. */
|
/**
|
||||||
|
* Passed through to grpc-js, e.g. message size limits. What is set
|
||||||
|
* here wins over the defaults below.
|
||||||
|
*/
|
||||||
channelOptions?: grpc.ChannelOptions;
|
channelOptions?: grpc.ChannelOptions;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -156,6 +159,14 @@ const DEFAULT_DEADLINE_MS = 30_000;
|
||||||
|
|
||||||
/** A connection to one hub. */
|
/** A connection to one hub. */
|
||||||
export class ChainClient {
|
export class ChainClient {
|
||||||
|
/**
|
||||||
|
* The channel options this client was built with: its own defaults
|
||||||
|
* with the caller's on top. Readable so a service can assert what
|
||||||
|
* it is running with, which is the only way to tell a default that
|
||||||
|
* applies from one that was silently replaced.
|
||||||
|
*/
|
||||||
|
readonly channelOptions: grpc.ChannelOptions;
|
||||||
|
|
||||||
private readonly hub: grpc.Client;
|
private readonly hub: grpc.Client;
|
||||||
private readonly admin: grpc.Client;
|
private readonly admin: grpc.Client;
|
||||||
private readonly token: string | undefined;
|
private readonly token: string | undefined;
|
||||||
|
|
@ -170,9 +181,20 @@ export class ChainClient {
|
||||||
options.tls === true ? undefined : options.tls.certChain,
|
options.tls === true ? undefined : options.tls.certChain,
|
||||||
)
|
)
|
||||||
: grpc.credentials.createInsecure();
|
: grpc.credentials.createInsecure();
|
||||||
const channelOptions = options.channelOptions ?? {};
|
// A hub that comes back should be usable again at once. grpc-js
|
||||||
|
// waits a second before the first reconnect and then backs off
|
||||||
|
// towards two minutes, which is right for a far-away service and
|
||||||
|
// wrong for a hub on the same network: after a restart or a
|
||||||
|
// deploy, a gateway would sit out the backoff with requests in
|
||||||
|
// hand. The caller's own options still win.
|
||||||
|
const channelOptions: grpc.ChannelOptions = {
|
||||||
|
'grpc.initial_reconnect_backoff_ms': 200,
|
||||||
|
'grpc.max_reconnect_backoff_ms': 5_000,
|
||||||
|
...(options.channelOptions ?? {}),
|
||||||
|
};
|
||||||
const Hub = hubServiceConstructor();
|
const Hub = hubServiceConstructor();
|
||||||
const HubAdmin = hubAdminServiceConstructor();
|
const HubAdmin = hubAdminServiceConstructor();
|
||||||
|
this.channelOptions = channelOptions;
|
||||||
this.hub = new Hub(options.address, credentials, channelOptions);
|
this.hub = new Hub(options.address, credentials, channelOptions);
|
||||||
this.admin = new HubAdmin(options.address, credentials, channelOptions);
|
this.admin = new HubAdmin(options.address, credentials, channelOptions);
|
||||||
this.token = options.token;
|
this.token = options.token;
|
||||||
|
|
|
||||||
|
|
@ -275,3 +275,43 @@ async function waitFor<T>(produce: () => Promise<T | undefined>, timeoutMs = 3_0
|
||||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
describe('the channel after a hub comes back', () => {
|
||||||
|
it('is built to retry soon, not after a second', () => {
|
||||||
|
// grpc-js waits a second before its second reconnect attempt and
|
||||||
|
// backs off towards two minutes. That is right for a far-away
|
||||||
|
// service and wrong for a hub on the same network: after a
|
||||||
|
// restart or a deploy, a gateway would sit out the backoff with
|
||||||
|
// requests in hand.
|
||||||
|
const client = new ChainClient({ address: '127.0.0.1:1' });
|
||||||
|
assert.equal(client.channelOptions['grpc.initial_reconnect_backoff_ms'], 200);
|
||||||
|
assert.equal(client.channelOptions['grpc.max_reconnect_backoff_ms'], 5_000);
|
||||||
|
client.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('lets a caller override what the client set', () => {
|
||||||
|
const client = new ChainClient({
|
||||||
|
address: '127.0.0.1:1',
|
||||||
|
channelOptions: { 'grpc.initial_reconnect_backoff_ms': 1_234 },
|
||||||
|
});
|
||||||
|
assert.equal(client.channelOptions['grpc.initial_reconnect_backoff_ms'], 1_234);
|
||||||
|
// And the other default is still there.
|
||||||
|
assert.equal(client.channelOptions['grpc.max_reconnect_backoff_ms'], 5_000);
|
||||||
|
client.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('works again once the hub is back', async () => {
|
||||||
|
const hub = new FakeHub();
|
||||||
|
const address = await hub.start();
|
||||||
|
const client = new ChainClient({ address, deadlineMs: 5_000 });
|
||||||
|
await client.health();
|
||||||
|
|
||||||
|
await hub.restart();
|
||||||
|
hub.succeeds({ report: 'wieder da' });
|
||||||
|
const result = await client.submit({ flowYaml: FLOW });
|
||||||
|
assert.equal(asText(result.outputs.report), 'wieder da');
|
||||||
|
|
||||||
|
client.close();
|
||||||
|
await hub.close();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue