chain-studio/test/guided_setup_test.dart
flemming-it c6da5025ce
Some checks failed
Security / Security check (push) Failing after 2s
fix(setup): wizard errors copyable above the dialog, CLI skew explained, exec transparency
Field test of the setup wizard surfaced three trust breaks in one run:
an unexplained macOS Documents permission prompt, a perceived crash,
and an error message whose copy button could not be reached.

Root causes and fixes:
- chain init failures were shown as a SnackBar, which lands BEHIND the
  wizard's modal barrier: dimmed, clipped, copy unreachable — and the
  click aimed at it hit the barrier, dismissing the whole wizard with
  all answers (the perceived crash). Errors now open a modal dialog
  ABOVE the wizard via showChainErrorDialog with a copyable detail
  block, and the wizard is no longer barrier-dismissible.
- When the resolved chain binary is older than Studio and rejects
  --plan-json, the wizard now explains the version skew in plain
  language (binary path + update path) instead of leaking a raw clap
  usage error. A missing binary gets its own localized story.
- Step 3 announces which chain binary the preview will execute; when
  that binary physically lives (symlinks resolved) in a TCC-protected
  folder, the wizard pre-explains the macOS folder prompt.

Supporting changes: FriendlyError passes through friendlyError()
unchanged so call sites can ship precise localized stories through the
shared presentation; SystemActions gains resolvedChainBinary() plus
run/resolve test seams; ChainErrorBox hugs its content instead of
filling an unbounded dialog; the wizard's answers file is written
synchronously (the async dart:io variants never complete under the
widget-test fake-async zone).

Verified: flutter analyze clean, 53 tests green (6 new wizard error-
path tests incl. clipboard round-trip), plus a live GUI walk on macOS
in dark + light with a stale binary (skew dialog, copy verified via
clipboard) and with the real binary (TCC pre-explanation with the
resolved path, full plan preview).

Signed-off-by: flemming-it <stefan.a.flemming@googlemail.com>
2026-07-15 00:10:01 +02:00

460 lines
16 KiB
Dart

// Guided-setup wizard — the three explained answer steps. Verifies
// that options render as localized cards with their one-line
// explanation (no English enum humanization) and that Weiter/Zurück
// walk the steps. CLI calls run through the SystemActions test seam
// so the error paths (stale binary, generic failure) are covered
// without a subprocess.
import 'dart:io' show Directory, Platform;
import 'package:flutter/material.dart';
import 'package:flutter/services.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:chain_studio/data/chain_log.dart';
import 'package:chain_studio/data/system_actions.dart';
import 'package:chain_studio/l10n/app_localizations.dart';
import 'package:chain_studio/widgets/guided_setup_dialog.dart';
Widget _host() => MaterialApp(
locale: const Locale('de'),
localizationsDelegates: AppLocalizations.localizationsDelegates,
supportedLocales: AppLocalizations.supportedLocales,
home: Builder(
builder: (context) => Scaffold(
body: Center(
child: ElevatedButton(
onPressed: () => GuidedSetupDialog.show(context),
child: const Text('open'),
),
),
),
),
);
/// Host that opens the dialog pre-seeded on the review / applied
/// step with [plan] (the parsed `--plan-json` shape), bypassing the
/// CLI subprocess.
Widget _seededHost(Map<String, dynamic> plan, {bool applied = false}) =>
MaterialApp(
locale: const Locale('de'),
localizationsDelegates: AppLocalizations.localizationsDelegates,
supportedLocales: AppLocalizations.supportedLocales,
home: Builder(
builder: (context) => Scaffold(
body: Center(
child: ElevatedButton(
onPressed: () => showDialog<void>(
context: context,
builder: (_) =>
GuidedSetupDialog(debugPlan: plan, debugApplied: applied),
),
child: const Text('open'),
),
),
),
),
);
Map<String, dynamic> _regulatedPlan({bool signed = true}) => {
'profile': 'enterprise',
'modules': ['text.extract', 'text.summarize'],
'starter_flow': 'extract-summarize',
'runbook': 'service',
'curated_docs': <String>[],
'require_signatures': signed,
'worm_audit': false,
'approval_step': true,
};
void main() {
testWidgets('step 1 shows scenario options in German with explanations', (
tester,
) async {
await tester.pumpWidget(_host());
await tester.tap(find.text('open'));
await tester.pumpAndSettle();
// Step counter + localized (German) scenario labels, not the
// English enum humanization.
expect(find.text('Schritt 1 von 3'), findsOneWidget);
expect(find.text('Regulierter Produktivbetrieb'), findsOneWidget);
expect(find.text('Erst mal ausprobieren'), findsOneWidget);
// Each option carries its one-line explanation.
expect(
find.textContaining('menschliche Freigabe vor jedem KI-Schritt'),
findsOneWidget,
);
// No leaked English enum labels.
expect(find.text('Regulated Production'), findsNothing);
expect(find.text('Trying Out'), findsNothing);
});
testWidgets('Weiter/Zurück walk the three answer steps', (tester) async {
await tester.pumpWidget(_host());
await tester.tap(find.text('open'));
await tester.pumpAndSettle();
// Step 1 → 2 (task).
await tester.tap(find.text('Weiter'));
await tester.pumpAndSettle();
expect(find.text('Schritt 2 von 3'), findsOneWidget);
expect(find.text('Text aus Dokumenten extrahieren'), findsOneWidget);
// Step 2 → 3 (environment) shows the two plain-language toggles.
await tester.tap(find.text('Weiter'));
await tester.pumpAndSettle();
expect(find.text('Schritt 3 von 3'), findsOneWidget);
expect(find.text('Dieser Rechner'), findsOneWidget);
expect(
find.text('Vor jedem KI-Schritt einen Menschen um Freigabe bitten'),
findsOneWidget,
);
// Zurück returns to step 2.
await tester.tap(find.text('Zurück'));
await tester.pumpAndSettle();
expect(find.text('Schritt 2 von 3'), findsOneWidget);
});
testWidgets(
'review of a regulated plan explains the signature dead end and offers the relaxation switch',
(tester) async {
await tester.pumpWidget(_seededHost(_regulatedPlan()));
await tester.tap(find.text('open'));
await tester.pumpAndSettle();
// The plain-language notice + the deliberate one-click switch.
expect(
find.textContaining('Der öffentliche Store liefert zurzeit unsignierte Pakete'),
findsOneWidget,
);
expect(
find.text('Installation aus dem öffentlichen Store erlauben'),
findsOneWidget,
);
// No raw CLI prose anywhere.
expect(find.textContaining('chain install'), findsNothing);
},
);
testWidgets(
'applied state renders clickable next steps — start hub, per-module install, open flow',
(tester) async {
await tester.pumpWidget(
_seededHost(_regulatedPlan(signed: false), applied: true),
);
await tester.tap(find.text('open'));
await tester.pumpAndSettle();
expect(find.text('Fertig — Ch∆In ist eingerichtet.'), findsOneWidget);
// Hub not probed up in tests → the start action + the
// install-needs-hub hint show, and install buttons exist
// (disabled) for each plan module.
expect(find.text('Hub starten'), findsOneWidget);
expect(
find.textContaining('Starten Sie zuerst den Hub'),
findsOneWidget,
);
expect(find.text('text.extract installieren'), findsOneWidget);
expect(find.text('text.summarize installieren'), findsOneWidget);
expect(
find.text('Beispiel-Flow „extract-summarize“ öffnen'),
findsOneWidget,
);
// No CLI text in the GUI path.
expect(find.textContaining('chain serve'), findsNothing);
expect(find.textContaining('chain install'), findsNothing);
},
);
testWidgets(
'applied regulated (signed) state stays clickable — signed-source dialog, trust hint, install buttons',
(tester) async {
await tester.pumpWidget(_seededHost(_regulatedPlan(), applied: true));
await tester.tap(find.text('open'));
await tester.pumpAndSettle();
// The plain-language story + the pin-the-publisher trust hint.
expect(find.textContaining('signierten Quelle'), findsOneWidget);
expect(
find.textContaining('Schlüssel des Herausgebers'),
findsOneWidget,
);
// No dead end: the stores dialog (with its pin-a-key field) is
// one click away, and the installs stay available for after
// the source is added.
expect(find.text('Signierte Quelle hinzufügen…'), findsOneWidget);
expect(find.text('text.extract installieren'), findsOneWidget);
expect(find.textContaining('chain store add'), findsNothing);
},
);
testWidgets(
'review preview names the machine being set up and the audit chain',
(tester) async {
await tester.pumpWidget(_seededHost(_regulatedPlan()));
await tester.tap(find.text('open'));
await tester.pumpAndSettle();
// Runbook honesty: a "home server" target still configures
// THIS machine — the preview must say so.
expect(
find.textContaining('Eingerichtet wird dieser Rechner'),
findsOneWidget,
);
// The regulated promise is stated even when WORM is off:
// the hash-chained audit log line is always there.
expect(
find.textContaining('hash-verketteten Prüfprotokoll'),
findsOneWidget,
);
},
);
testWidgets(
'step 1 offers the free-text path with an honest no-AI fallback hint',
(tester) async {
await tester.pumpWidget(
MaterialApp(
locale: const Locale('de'),
localizationsDelegates: AppLocalizations.localizationsDelegates,
supportedLocales: AppLocalizations.supportedLocales,
home: Builder(
builder: (context) => Scaffold(
body: Center(
child: ElevatedButton(
onPressed: () => showDialog<void>(
context: context,
builder: (_) =>
const GuidedSetupDialog(debugSkipAiProbe: true),
),
child: const Text('open'),
),
),
),
),
),
);
await tester.tap(find.text('open'));
await tester.pumpAndSettle();
expect(
find.text('Oder beschreiben Sie einfach, was Sie vorhaben'),
findsOneWidget,
);
// No system AI configured → honest fallback: the menu always
// works; no dead end, no disabled mystery button.
expect(
find.textContaining('Die Auswahl oben funktioniert immer'),
findsOneWidget,
);
expect(find.text('Vorschlagen lassen'), findsNothing);
},
);
group('CLI paths (SystemActions test seam)', () {
late Directory tmp;
setUp(() {
tmp = Directory.systemTemp.createTempSync('wizard-test-');
ChainLog.testPathOverride = '${tmp.path}/studio-errors.log';
});
tearDown(() {
ChainLog.testPathOverride = null;
SystemActions.debugRunFaiOverride = null;
SystemActions.debugResolveOverride = null;
tmp.deleteSync(recursive: true);
});
Future<void> walkToStep3(WidgetTester tester) async {
await tester.pumpWidget(_host());
await tester.tap(find.text('open'));
await tester.pumpAndSettle();
await tester.tap(find.text('Weiter'));
await tester.pumpAndSettle();
await tester.tap(find.text('Weiter'));
await tester.pumpAndSettle();
expect(find.text('Schritt 3 von 3'), findsOneWidget);
}
testWidgets('a stray click outside the wizard does not dismiss it', (
tester,
) async {
await tester.pumpWidget(_host());
await tester.tap(find.text('open'));
await tester.pumpAndSettle();
expect(find.text('Schritt 1 von 3'), findsOneWidget);
// The exact click that used to end the wizard: aiming at an
// error SnackBar dimmed below the modal barrier.
await tester.tapAt(const Offset(4, 4));
await tester.pumpAndSettle();
expect(find.text('Schritt 1 von 3'), findsOneWidget);
});
testWidgets(
'stale chain binary → explained, copyable error dialog above the intact wizard',
(tester) async {
const clapError =
"error: unexpected argument '--plan-json' found\n\n"
" tip: a similar argument exists: '--plan-out'\n\n"
'Usage: chain init --answers <ANSWERS> --plan-out <PLAN_OUT>';
SystemActions.debugResolveOverride = () => '/Users/op/.chain/bin/chain';
SystemActions.debugRunFaiOverride = (args) async =>
(ok: false, stdout: '', stderr: clapError);
final copied = <String>[];
tester.binding.defaultBinaryMessenger.setMockMethodCallHandler(
SystemChannels.platform,
(call) async {
if (call.method == 'Clipboard.setData') {
copied.add((call.arguments as Map)['text'] as String);
}
return null;
},
);
await walkToStep3(tester);
await tester.tap(find.text('Weiter')); // triggers `chain init`
await tester.pumpAndSettle();
// Plain-language headline naming the skew — not a raw clap
// usage wall — plus the binary that was executed.
expect(find.textContaining('älter als Studio'), findsOneWidget);
expect(
find.textContaining('/Users/op/.chain/bin/chain'),
findsWidgets,
);
// The verbatim CLI output is one copy-click away (the old
// SnackBar rendition sat behind the modal barrier where the
// copy button could not be reached at all).
await tester.tap(find.byIcon(Icons.content_copy));
await tester.pump();
expect(copied, hasLength(1));
expect(copied.single, contains('unexpected argument'));
// Let the button's transient "copied" checkmark reset so no
// timer outlives the test.
await tester.pump(const Duration(seconds: 2));
// Dismissing the error returns to the intact wizard — the
// failure must not cost the operator their answers.
await tester.tap(find.text('OK'));
await tester.pumpAndSettle();
expect(find.text('Schritt 3 von 3'), findsOneWidget);
},
);
testWidgets('generic init failure names the executed binary', (
tester,
) async {
SystemActions.debugResolveOverride = () => '/Users/op/.chain/bin/chain';
SystemActions.debugRunFaiOverride = (args) async =>
(ok: false, stdout: '', stderr: 'boom: config unreadable');
await walkToStep3(tester);
await tester.tap(find.text('Weiter'));
await tester.pumpAndSettle();
expect(
find.textContaining('Die Plan-Vorschau konnte nicht erstellt werden'),
findsOneWidget,
);
expect(find.textContaining('Ausgeführt wurde:'), findsOneWidget);
});
testWidgets('step 3 announces which binary the wizard will run', (
tester,
) async {
SystemActions.debugResolveOverride = () => '/Users/op/.chain/bin/chain';
SystemActions.debugRunFaiOverride = (args) async =>
(ok: false, stdout: '', stderr: 'unused');
await walkToStep3(tester);
expect(
find.textContaining('führt Studio das Programm „chain“ aus'),
findsOneWidget,
);
expect(
find.textContaining('/Users/op/.chain/bin/chain'),
findsOneWidget,
);
});
testWidgets(
'step 3 pre-explains the macOS folder prompt for a Documents-resident binary',
(tester) async {
final home = Platform.environment['HOME'];
if (!Platform.isMacOS || home == null || home.isEmpty) {
return; // TCC folder prompts are a macOS-only concern.
}
SystemActions.debugResolveOverride = () => '$home/Documents/dev/chain';
SystemActions.debugRunFaiOverride = (args) async =>
(ok: false, stdout: '', stderr: 'unused');
await walkToStep3(tester);
expect(
find.textContaining('nach Zugriff auf diesen Ordner'),
findsOneWidget,
);
},
);
testWidgets('step 3 says so when no chain binary can be found', (
tester,
) async {
SystemActions.debugResolveOverride = () => null;
await walkToStep3(tester);
expect(
find.textContaining('wurde auf diesem Rechner nicht gefunden'),
findsOneWidget,
);
});
});
group('parseSetupSuggestion', () {
test('accepts a valid JSON object embedded in prose', () {
final r = parseSetupSuggestion(
'Sure! Here is the mapping:\n'
'{"scenario":"regulated-production","intent":"classify-documents",'
'"target":"home-server","require_approval":true,'
'"data_must_stay_local":false}\n'
'Let me know if you need anything else.',
);
expect(r, isNotNull);
expect(r!['scenario'], 'regulated-production');
expect(r['intent'], 'classify-documents');
expect(r['target'], 'home-server');
expect(r['require_approval'], true);
expect(r['data_must_stay_local'], false);
});
test('rejects hallucinated enum values', () {
expect(
parseSetupSuggestion(
'{"scenario":"world-domination","intent":"hello-world",'
'"target":"this-laptop"}',
),
isNull,
);
});
test('rejects non-JSON replies', () {
expect(parseSetupSuggestion('I cannot help with that.'), isNull);
expect(parseSetupSuggestion('{broken json'), isNull);
});
test('missing booleans default to false', () {
final r = parseSetupSuggestion(
'{"scenario":"trying-out","intent":"hello-world",'
'"target":"this-laptop"}',
);
expect(r, isNotNull);
expect(r!['require_approval'], false);
expect(r['data_must_stay_local'], false);
});
});
}