Two things helloeco ran into while wiring the gateway. grpc-js waits a second before its second reconnect attempt and backs off towards two minutes. That is right for a far-away service and wrong for a hub on the same network: after a restart or a deploy a gateway sits out the backoff with requests in hand. The client now sets initial 200 ms and a 5 s ceiling, and a caller's own channelOptions still win. The client exposes the options it ended up with, which is the only way to tell a default that applies from one that was silently replaced. The dependencies are pinned exactly instead of by range. This package is consumed by a handful of services, not by a library ecosystem, and a surprise minor of grpc-js is what breaks a portal at three in the morning. Raising one is a commit, with the skew job in front of it. Guards: the defaults and the precedence are asserted (probed by mutation), plus a case that a run works again once the hub is back. Signed-off-by: flemming-it <sf@flemming.it>
317 lines
12 KiB
TypeScript
317 lines
12 KiB
TypeScript
// The client against the fake hub: hermetic, no hub binary, no
|
|
// module, no model. Every case here is one a portal gateway has to
|
|
// survive.
|
|
|
|
import assert from 'node:assert/strict';
|
|
import { after, before, beforeEach, describe, it } from 'node:test';
|
|
|
|
import { ChainClient, ChainError, asJson, asText } from '../src/index.js';
|
|
import { FakeHub } from '../src/fake_hub.js';
|
|
|
|
const FLOW = `name: verbrauchsausweis
|
|
inputs:
|
|
consumption: { type: json }
|
|
steps:
|
|
- id: score
|
|
uses: geg.verbrauch@^1
|
|
outputs:
|
|
report: $steps.score.text
|
|
`;
|
|
|
|
describe('client against the fake hub', () => {
|
|
let hub: FakeHub;
|
|
let client: ChainClient;
|
|
|
|
before(async () => {
|
|
hub = new FakeHub();
|
|
const address = await hub.start();
|
|
client = new ChainClient({ address, project: 'helloeco', deadlineMs: 5_000 });
|
|
});
|
|
|
|
after(async () => {
|
|
client.close();
|
|
await hub.close();
|
|
});
|
|
|
|
beforeEach(() => {
|
|
hub.setFallback({ type: 'succeed', outputs: {} });
|
|
hub.setHealth('SERVING');
|
|
});
|
|
|
|
it('reports health', async () => {
|
|
const health = await client.health();
|
|
assert.equal(health.state, 'SERVING');
|
|
});
|
|
|
|
it('runs a flow and reads its outputs', async () => {
|
|
hub.succeeds({ report: 'Endenergiebedarf 128 kWh/(m²·a)', data: { json: { class: 'D' } } });
|
|
const result = await client.submit({
|
|
flowYaml: FLOW,
|
|
inputs: { consumption: { json: { kwh: 15400 } } },
|
|
});
|
|
assert.equal(asText(result.outputs.report), 'Endenergiebedarf 128 kWh/(m²·a)');
|
|
assert.deepEqual(asJson(result.outputs.data), { class: 'D' });
|
|
assert.match(result.invocationId, /^run-/);
|
|
|
|
const seen = hub.submits.at(-1);
|
|
assert.equal(seen?.project, 'helloeco');
|
|
assert.deepEqual(seen?.inputs.consumption, { kind: 'json', json: { kwh: 15400 } });
|
|
});
|
|
|
|
it('follows a run step by step', async () => {
|
|
hub.succeeds({ report: 'done' }, ['check', 'score', 'render']);
|
|
const events = [];
|
|
for await (const event of client.submitStream({ flowYaml: FLOW })) {
|
|
events.push(event);
|
|
}
|
|
const started = events.filter((e) => e.type === 'stepStarted').map((e) => e.step);
|
|
assert.deepEqual(started, ['check', 'score', 'render']);
|
|
const last = events.at(-1);
|
|
assert.equal(last?.type, 'result');
|
|
if (last?.type === 'result') {
|
|
assert.equal(asText(last.result.outputs.report), 'done');
|
|
}
|
|
});
|
|
|
|
it('classifies an overloaded hub as retryable without blaming the request', async () => {
|
|
hub.fails('overloaded', 'too many runs in flight');
|
|
const err = await client.submit({ flowYaml: FLOW }).then(
|
|
() => undefined,
|
|
(e: unknown) => e as ChainError,
|
|
);
|
|
assert.ok(err instanceof ChainError);
|
|
assert.equal(err.kind, 'overloaded');
|
|
assert.equal(err.family, 'overloaded');
|
|
assert.equal(err.retryable, true);
|
|
assert.equal(err.reachedHub, true);
|
|
});
|
|
|
|
it('classifies a refused request as not retryable', async () => {
|
|
hub.fails('flow_rejected', "missing value '$inputs.consumption'");
|
|
const err = (await client.submit({ flowYaml: FLOW }).catch((e) => e)) as ChainError;
|
|
assert.equal(err.kind, 'flow_rejected');
|
|
assert.equal(err.retryable, false);
|
|
});
|
|
|
|
it('carries the class out of a stream too', async () => {
|
|
hub.fails('step_failed', "step 'render' failed: template has no page size");
|
|
const err = await (async () => {
|
|
try {
|
|
for await (const _event of client.submitStream({ flowYaml: FLOW })) {
|
|
// drain
|
|
}
|
|
return undefined;
|
|
} catch (e) {
|
|
return e as ChainError;
|
|
}
|
|
})();
|
|
assert.ok(err instanceof ChainError);
|
|
assert.equal(err.kind, 'step_failed');
|
|
assert.equal(err.family, 'failed');
|
|
assert.equal(err.step, 'render');
|
|
});
|
|
|
|
it('returns the same run for the same idempotency key', async () => {
|
|
hub.succeeds({ report: 'first' });
|
|
hub.succeeds({ report: 'second' });
|
|
const first = await client.submit({ flowYaml: FLOW, idempotencyKey: 'stripe-evt-1' });
|
|
const again = await client.submit({ flowYaml: FLOW, idempotencyKey: 'stripe-evt-1' });
|
|
assert.equal(again.invocationId, first.invocationId);
|
|
assert.equal(asText(again.outputs.report), 'first');
|
|
// The second submit must not have consumed the next scripted run.
|
|
const third = await client.submit({ flowYaml: FLOW, idempotencyKey: 'stripe-evt-2' });
|
|
assert.equal(asText(third.outputs.report), 'second');
|
|
assert.notEqual(third.invocationId, first.invocationId);
|
|
});
|
|
|
|
it('waits for an approval, then finishes with the reviewer on record', async () => {
|
|
hub.waitsForApproval({
|
|
prompt: 'Registriernummer 4711 vergeben und Ausweis zustellen?',
|
|
stepId: 'freigabe',
|
|
outputs: { report: 'zugestellt' },
|
|
});
|
|
const run = client.submit({ flowYaml: FLOW });
|
|
|
|
const pending = await waitFor(async () => {
|
|
const list = await client.listApprovals({ project: 'helloeco' });
|
|
return list.length > 0 ? list : undefined;
|
|
});
|
|
assert.equal(pending[0]?.prompt, 'Registriernummer 4711 vergeben und Ausweis zustellen?');
|
|
assert.equal(pending[0]?.stepId, 'freigabe');
|
|
|
|
await client.decideApproval({
|
|
approvalId: pending[0]!.approvalId,
|
|
approve: true,
|
|
reviewer: 'aussteller@example.org',
|
|
});
|
|
|
|
const result = await run;
|
|
assert.equal(asText(result.outputs.report), 'zugestellt');
|
|
assert.equal(hub.decisions.at(-1)?.reviewer, 'aussteller@example.org');
|
|
});
|
|
|
|
it('fails the run when the reviewer declines, with the reason', async () => {
|
|
hub.waitsForApproval({ prompt: 'Ausweis zustellen?', outputs: {} });
|
|
// The catch is attached right away: the run rejects while the
|
|
// test is still waiting for the approval row, and an unhandled
|
|
// rejection in between would fail the test for the wrong reason.
|
|
const run = client.submit({ flowYaml: FLOW }).catch((e: unknown) => e as ChainError);
|
|
const pending = await waitFor(async () => {
|
|
const list = await client.listApprovals();
|
|
return list.length > 0 ? list : undefined;
|
|
});
|
|
await client.decideApproval({
|
|
approvalId: pending[0]!.approvalId,
|
|
approve: false,
|
|
reviewer: 'aussteller@example.org',
|
|
reason: 'Verbrauchswerte unplausibel',
|
|
});
|
|
const err = (await run) as ChainError;
|
|
assert.equal(err.kind, 'approval_rejected');
|
|
assert.match(err.message, /unplausibel/);
|
|
});
|
|
|
|
it('refuses a rejection without a reason', async () => {
|
|
hub.waitsForApproval({ prompt: 'Ausweis zustellen?', outputs: {} });
|
|
const run = client.submit({ flowYaml: FLOW }).catch((e: unknown) => e as ChainError);
|
|
const pending = await waitFor(async () => {
|
|
const list = await client.listApprovals();
|
|
return list.length > 0 ? list : undefined;
|
|
});
|
|
const err = (await client
|
|
.decideApproval({ approvalId: pending[0]!.approvalId, approve: false })
|
|
.catch((e) => e)) as ChainError;
|
|
assert.equal(err.kind, 'flow_rejected');
|
|
await client.decideApproval({
|
|
approvalId: pending[0]!.approvalId,
|
|
approve: false,
|
|
reason: 'aufgeräumt',
|
|
});
|
|
await run;
|
|
});
|
|
|
|
it('follows a detached run by id', async () => {
|
|
hub.succeeds({ report: 'spät' });
|
|
const started = await client.submit({ flowYaml: FLOW, detach: true });
|
|
assert.deepEqual(started.outputs, {});
|
|
const status = await client.getStatus(started.invocationId);
|
|
assert.equal(status.phase, 'SUCCEEDED');
|
|
const result = await client.getResult(started.invocationId);
|
|
assert.equal(asText(result.outputs.report), 'spät');
|
|
});
|
|
|
|
it('says not_found for an invocation the hub never had', async () => {
|
|
const err = (await client.getStatus('run-does-not-exist').catch((e) => e)) as ChainError;
|
|
assert.equal(err.kind, 'not_found');
|
|
assert.equal(err.family, 'rejected');
|
|
});
|
|
});
|
|
|
|
describe('client against a hub that is not there', () => {
|
|
it('says the hub was never reached, not that a dependency failed', async () => {
|
|
const hub = new FakeHub();
|
|
const address = await hub.start();
|
|
const client = new ChainClient({ address, deadlineMs: 2_000 });
|
|
await client.health();
|
|
await hub.stop();
|
|
|
|
const err = (await client.health().catch((e) => e)) as ChainError;
|
|
assert.equal(err.family, 'unreachable');
|
|
assert.equal(err.reachedHub, false);
|
|
assert.equal(err.retryable, true);
|
|
client.close();
|
|
await hub.close();
|
|
});
|
|
});
|
|
|
|
describe('a hub restart while a run waits for an approval', () => {
|
|
it('loses the run, keeps the approval, and says so', async () => {
|
|
// This is today's platform behaviour, not a wish: run state lives
|
|
// in memory, approvals live in SQLite. A gateway must therefore
|
|
// treat "approval still pending after a restart" as a run it has
|
|
// to start again.
|
|
const hub = new FakeHub();
|
|
const address = await hub.start();
|
|
const client = new ChainClient({ address, project: 'helloeco', deadlineMs: 5_000 });
|
|
|
|
hub.waitsForApproval({ prompt: 'Ausweis zustellen?', outputs: { report: 'zugestellt' } });
|
|
const run = client.submit({ flowYaml: FLOW }).catch((e: unknown) => e as ChainError);
|
|
const pending = await waitFor(async () => {
|
|
const list = await client.listApprovals({ project: 'helloeco' });
|
|
return list.length > 0 ? list : undefined;
|
|
});
|
|
const approvalId = pending[0]!.approvalId;
|
|
|
|
await hub.restart();
|
|
|
|
const err = (await run) as ChainError;
|
|
assert.equal(err.family, 'unreachable');
|
|
|
|
// The decision still lands, and still has nothing to resume.
|
|
const stillPending = await client.listApprovals({ project: 'helloeco' });
|
|
assert.equal(stillPending.length, 1);
|
|
assert.equal(stillPending[0]?.approvalId, approvalId);
|
|
const decided = await client.decideApproval({
|
|
approvalId,
|
|
approve: true,
|
|
reviewer: 'aussteller@example.org',
|
|
});
|
|
assert.equal(decided.status, 'APPROVED');
|
|
const gone = (await client.getStatus(pending[0]!.flowExecution).catch((e) => e)) as ChainError;
|
|
assert.equal(gone.kind, 'not_found');
|
|
|
|
client.close();
|
|
await hub.close();
|
|
});
|
|
});
|
|
|
|
/** Poll until the producer returns something, or give up. */
|
|
async function waitFor<T>(produce: () => Promise<T | undefined>, timeoutMs = 3_000): Promise<T> {
|
|
const until = Date.now() + timeoutMs;
|
|
for (;;) {
|
|
const value = await produce();
|
|
if (value !== undefined) return value;
|
|
if (Date.now() > until) throw new Error('timed out waiting for the fake hub');
|
|
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
}
|
|
}
|
|
|
|
describe('the channel after a hub comes back', () => {
|
|
it('is built to retry soon, not after a second', () => {
|
|
// grpc-js waits a second before its second reconnect attempt and
|
|
// backs off towards two minutes. That is right for a far-away
|
|
// service and wrong for a hub on the same network: after a
|
|
// restart or a deploy, a gateway would sit out the backoff with
|
|
// requests in hand.
|
|
const client = new ChainClient({ address: '127.0.0.1:1' });
|
|
assert.equal(client.channelOptions['grpc.initial_reconnect_backoff_ms'], 200);
|
|
assert.equal(client.channelOptions['grpc.max_reconnect_backoff_ms'], 5_000);
|
|
client.close();
|
|
});
|
|
|
|
it('lets a caller override what the client set', () => {
|
|
const client = new ChainClient({
|
|
address: '127.0.0.1:1',
|
|
channelOptions: { 'grpc.initial_reconnect_backoff_ms': 1_234 },
|
|
});
|
|
assert.equal(client.channelOptions['grpc.initial_reconnect_backoff_ms'], 1_234);
|
|
// And the other default is still there.
|
|
assert.equal(client.channelOptions['grpc.max_reconnect_backoff_ms'], 5_000);
|
|
client.close();
|
|
});
|
|
|
|
it('works again once the hub is back', async () => {
|
|
const hub = new FakeHub();
|
|
const address = await hub.start();
|
|
const client = new ChainClient({ address, deadlineMs: 5_000 });
|
|
await client.health();
|
|
|
|
await hub.restart();
|
|
hub.succeeds({ report: 'wieder da' });
|
|
const result = await client.submit({ flowYaml: FLOW });
|
|
assert.equal(asText(result.outputs.report), 'wieder da');
|
|
|
|
client.close();
|
|
await hub.close();
|
|
});
|
|
});
|