Add voice-first UI mode and MCP command executor improvements (#1940)

## Summary

- **Voice-first UI mode**: New `.voice-mode` CSS layer for
large-screen/AR/car use cases — animated `VoiceOrb` component with
idle/wake-word-waiting/listening/thinking/speaking states, last-result
glanceable card, mode banner, and settings toggle. Auto-starts/stops
continuous wake word detection. Wake word changed from `"hey type
agent"` → `"type agent"` with normalization for `"typeagent"`.

- **MCP command executor improvements**: Live agent schema resolution
via new `getAgentSchemas()` dispatcher RPC (with static registry
fallback). Added `naturalLanguage` param to `execute_action` so
TypeAgent's NL cache gets populated. Fixed apostrophe-in-JSON bug using
`\u0027` Unicode escape. Updated tool descriptions to steer Claude
toward `discover_agents` + `execute_action` for multi-step
orchestration.

- **Dispatcher schema RPC**: New `ActionInfo`, `AgentSubSchemaInfo`,
`AgentSchemaInfo` types; `getAgentSchemas(agentName?)` wired through
dispatcher types, server, and client.

- **Shell agent manifests**: Added code, email, image, photo, video
agents to shell manifest registrations.

## Test plan

- [ ] Build: `pnpm run build` from `ts/` — no type errors
- [ ] Voice mode: `@shell voice on` → body gets `voice-mode` class, orb
appears, "Voice Mode" banner visible
- [ ] Voice orb states: submit a message → orb shows `thinking`,
response arrives → back to `idle`
- [ ] Wake word: with mic enabled, voice mode auto-starts continuous;
say "type agent, what time is it?" → `wake-word-waiting` → `listening` →
`thinking` → `idle`
- [ ] Settings: Settings panel shows "Voice mode" checkbox that persists
across reload
- [ ] `@shell voice off` → desktop UI unchanged, orb hidden
- [ ] MCP: `execute_action` with apostrophes in params (e.g., song
titles) no longer breaks JSON parsing
- [ ] MCP: `discover_agents` returns live schema data from dispatcher
- [ ] MCP: `execute_action` with `naturalLanguage` populates TypeAgent
NL cache

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

Author: steveluc

ts/packages/actionGrammar/src/generation/grammarGenerator.ts

@@ -17,7 +17,7 @@ export interface RuleRHS {
     // Parameters that map to the action
     actionParameters: Array<{
         parameterName: string;
-        parameterValue: string; // e.g., "$(track)" or fixed value like "kitchen"
+        parameterValue: string; // e.g., "track" (bare var name) or fixed value like "kitchen"; arrays: "[artist]"
     }>;
 }
 
@@ -97,7 +97,7 @@ WILDCARD RULES:
 2. Structure words (the, a, in, by, to, for, etc.) = fixed text
 3. Question words (what, where, when, who, why, how) = fixed text
 4. For array parameters: if only one item is given, use singular variable name
-   - Example Pattern: "by $(artist:wildcard)" → maps to artists: [$(artist)]
+   - Example Pattern: "by $(artist:wildcard)" → maps to artists: [artist]
 
 WILDCARD TYPES:
 - Built-in entity types: $(varName:CalendarDate), $(varName:CalendarTime), $(varName:CalendarTimeRange)
@@ -125,7 +125,7 @@ interface RuleRHS {
     matchPattern: string;  // The grammar pattern like "play $(track:wildcard) by $(artist:wildcard)"
     actionParameters: Array<{
         parameterName: string;
-        parameterValue: string; // e.g., "$(track)" or fixed value like "kitchen"
+        parameterValue: string; // e.g., "track" (bare var name) or fixed value like "kitchen"; arrays: "[artist]"
     }>;
 }
     
@@ -172,7 +172,7 @@ Output:
     { "parameterName": "deviceName", "sourceText": "kitchen", "targetValue": "kitchen", "isWildcard": true }
   ],
   "fixedPhrases": ["select", "device"],
-  "grammarPattern": { "matchPattern": "select $(deviceName:wildcard) device", "actionParameters": [ { "parameterName": "deviceName", "parameterValue": "$(deviceName)" } ] },
+  "grammarPattern": { "matchPattern": "select $(deviceName:wildcard) device", "actionParameters": [ { "parameterName": "deviceName", "parameterValue": "deviceName" } ] },
   "reasoning": "Fixed structure 'select...device'. Parameter deviceName is validated (checked_wildcard), use wildcard type."
 }
 
@@ -191,7 +191,7 @@ Output:
     { "parameterName": "artists", "sourceText": "queen", "targetValue": ["queen"], "isWildcard": true }
   ],
   "fixedPhrases": ["play", "by"],
-  "grammarPattern": { "matchPattern": "play $(trackName:wildcard) by $(artist:wildcard)", "actionParameters": [ { "parameterName": "trackName", "parameterValue": "$(trackName)" }, { "parameterName": "artists", "parameterValue": "[$(artist)]" } ] },
+  "grammarPattern": { "matchPattern": "play $(trackName:wildcard) by $(artist:wildcard)", "actionParameters": [ { "parameterName": "trackName", "parameterValue": "trackName" }, { "parameterName": "artists", "parameterValue": "[artist]" } ] },
   "reasoning": "Fixed structure 'play...by'. Both parameters are validated so adjacent wildcards are OK. Use singular 'artist' for array parameter 'artists'."
 }`;
 

ts/packages/actionGrammar/src/generation/schemaToGrammarGenerator.ts

@@ -70,7 +70,7 @@ CRITICAL SYNTAX RULES:
 4. Wildcard type annotations CANNOT contain pipes or unions
    For parameters with union types (e.g., CalendarTime | CalendarTimeRange):
    OPTION A: Create a sub-rule with alternation
-     @ <TimeSpec> = $(t:CalendarTime) -> $(t) | $(t:CalendarTimeRange) -> $(t)
+     @ <TimeSpec> = $(t:CalendarTime) -> t | $(t:CalendarTimeRange) -> t
    OPTION B: Just use one of the types
      @ <TimeExpr> = $(time:CalendarTime)
    WRONG: $(time:CalendarTime | CalendarTimeRange)

ts/packages/agentServer/server/src/server.ts

@@ -26,8 +26,9 @@ async function main() {
     const instanceDir = getInstanceDir();
 
     // did the launch request a specific config? (e.g. "test" to load "config.test.json")
+    const configIdx = process.argv.indexOf("--config");
     const configName =
-        process.argv[process.argv.indexOf("--config") + 1] || undefined;
+        configIdx !== -1 ? process.argv[configIdx + 1] : undefined;
 
     // Create single shared dispatcher with routing ClientIO
     const sharedDispatcher = await createSharedDispatcher("agent server", {

ts/packages/agents/calendar/src/calendarActionHandlerV3.ts

@@ -17,7 +17,15 @@
     "./agent/handlers": "./dist/codeActionHandler.js"
   },
   "scripts": {
-    "build": "npm run tsc",
+    "asc": "asc -i ./src/codeActionsSchema.ts -o ./dist/codeSchema.pas.json -t CodeActions -a CodeActivity",
+    "asc:all": "concurrently npm:asc npm:asc:debug npm:asc:display npm:asc:general npm:asc:editor npm:asc:workbench npm:asc:extension",
+    "asc:debug": "asc -i ./src/vscode/debugActionsSchema.ts -o ./dist/debugSchema.pas.json -t CodeDebugActions",
+    "asc:display": "asc -i ./src/vscode/displayActionsSchema.ts -o ./dist/displaySchema.pas.json -t CodeDisplayActions",
+    "asc:editor": "asc -i ./src/vscode/editorCodeActionsSchema.ts -o ./dist/editorSchema.pas.json -t EditorCodeActions",
+    "asc:extension": "asc -i ./src/vscode/extensionsActionsSchema.ts -o ./dist/extensionSchema.pas.json -t CodeWorkbenchExtensionActions",
+    "asc:general": "asc -i ./src/vscode/generalActionsSchema.ts -o ./dist/generalSchema.pas.json -t CodeGeneralActions",
+    "asc:workbench": "asc -i ./src/vscode/workbenchCommandActionsSchema.ts -o ./dist/workbenchSchema.pas.json -t CodeWorkbenchActions",
+    "build": "concurrently npm:tsc npm:asc:all",
     "clean": "rimraf --glob dist *.tsbuildinfo *.done.build.log",
     "prettier": "prettier --check . --ignore-path ../../../.prettierignore",
     "prettier:fix": "prettier --write . --ignore-path ../../../.prettierignore",
@@ -33,9 +41,11 @@
     "ws": "^8.17.1"
   },
   "devDependencies": {
+    "@typeagent/action-schema-compiler": "workspace:*",
     "@types/better-sqlite3": "7.6.13",
     "@types/debug": "^4.1.12",
     "@types/ws": "^8.5.10",
+    "concurrently": "^9.1.2",
     "prettier": "^3.5.3",
     "rimraf": "^6.0.1",
     "typescript": "~5.4.5"

ts/packages/agents/code/package.json

@@ -5,6 +5,7 @@
   "schema": {
     "description": "Code agent helps you with productivity in using an editor and performing actions like creating files, editor customization, writing code, intelligent code suggestions, real-time error detection, debugging, and source control tasks etc.",
     "schemaFile": "./codeActionsSchema.ts",
+    "compiledSchemaFile": "agents/code/dist/codeSchema.pas.json",
     "schemaType": {
       "action": "CodeActions",
       "activity": "CodeActivity"
@@ -15,41 +16,47 @@
       "schema": {
         "description": "Code agent that helps you debug code including handling actions like showing the debug panel, adding/toggling breakpoints, stepping in/out of code etc.",
         "schemaFile": "./vscode/debugActionsSchema.ts",
+        "compiledSchemaFile": "agents/code/dist/debugSchema.pas.json",
         "schemaType": "CodeDebugActions"
       }
     },
     "code-display": {
       "schema": {
         "description": "Code agent that helps you work with display keyboard bindings, Zoom in, Zoom out, do text search and replace, show extensions etc.",
         "schemaFile": "./vscode/displayActionsSchema.ts",
+        "compiledSchemaFile": "agents/code/dist/displaySchema.pas.json",
         "schemaType": "CodeDisplayActions"
       }
     },
     "code-general": {
       "schema": {
         "description": "Code agent that helps you perform actions like finding a file, go to a symbol or a line in a file, Open new vscode window, show the command palette, user settings json etc.",
         "schemaFile": "./vscode/generalActionsSchema.ts",
+        "compiledSchemaFile": "agents/code/dist/generalSchema.pas.json",
         "schemaType": "CodeGeneralActions"
       }
     },
     "code-editor": {
       "schema": {
         "description": "Code agent that helps you perform vscode editor actions like create a new file, go to a reference, reveal a declaration, clipboard copy or paste, add a comment line etc.",
         "schemaFile": "./vscode/editorCodeActionsSchema.ts",
+        "compiledSchemaFile": "agents/code/dist/editorSchema.pas.json",
         "schemaType": "EditorCodeActions"
       }
     },
     "code-workbench": {
       "schema": {
         "description": "Code agent that helps you perform vscode workbench actions like open a file, close a file etc.",
         "schemaFile": "./vscode/workbenchCommandActionsSchema.ts",
+        "compiledSchemaFile": "agents/code/dist/workbenchSchema.pas.json",
         "schemaType": "CodeWorkbenchActions"
       }
     },
     "code-extension": {
       "schema": {
         "description": "Code agent that helps you perform vscode extension actions like show, install, update, enable, disable, uninstall extensions etc.",
         "schemaFile": "./vscode/extensionsActionsSchema.ts",
+        "compiledSchemaFile": "agents/code/dist/extensionSchema.pas.json",
         "schemaType": "CodeWorkbenchExtensionActions"
       }
     }

ts/packages/agents/code/src/codeManifest.json

@@ -17,7 +17,8 @@
     "./agent/handlers": "./dist/emailActionHandler.js"
   },
   "scripts": {
-    "build": "npm run tsc",
+    "asc": "asc -i ./src/emailActionsSchema.ts -o ./dist/emailSchema.pas.json -t EmailAction",
+    "build": "concurrently npm:tsc npm:asc",
     "clean": "rimraf --glob dist *.tsbuildinfo *.done.build.log",
     "prettier": "prettier --check . --ignore-path ../../../.prettierignore",
     "prettier:fix": "prettier --write . --ignore-path ../../../.prettierignore",
@@ -34,7 +35,9 @@
     "typeagent": "workspace:*"
   },
   "devDependencies": {
+    "@typeagent/action-schema-compiler": "workspace:*",
     "@types/debug": "^4.1.12",
+    "concurrently": "^9.1.2",
     "prettier": "^3.5.3",
     "rimraf": "^6.0.1",
     "typescript": "~5.4.5"

ts/packages/agents/email/package.json

@@ -4,6 +4,7 @@
   "schema": {
     "description": "Email agent helps manage email messages. Use it to send, reply, forward, find messages etc.",
     "schemaFile": "./emailActionsSchema.ts",
+    "compiledSchemaFile": "agents/email/dist/emailSchema.pas.json",
     "schemaType": "EmailAction"
   }
 }

ts/packages/agents/email/src/emailManifest.json

@@ -17,7 +17,8 @@
     "./agent/handlers": "./dist/imageActionHandler.js"
   },
   "scripts": {
-    "build": "npm run tsc",
+    "asc": "asc -i ./src/imageActionSchema.ts -o ./dist/imageSchema.pas.json -t ImageAction",
+    "build": "concurrently npm:tsc npm:asc",
     "clean": "rimraf --glob dist *.tsbuildinfo *.done.build.log",
     "prettier": "prettier --check . --ignore-path ../../../.prettierignore",
     "prettier:fix": "prettier --write . --ignore-path ../../../.prettierignore",
@@ -30,6 +31,8 @@
     "typechat-utils": "workspace:*"
   },
   "devDependencies": {
+    "@typeagent/action-schema-compiler": "workspace:*",
+    "concurrently": "^9.1.2",
     "prettier": "^3.5.3",
     "rimraf": "^6.0.1",
     "typescript": "~5.4.5"

ts/packages/agents/image/package.json

@@ -4,6 +4,7 @@
   "schema": {
     "description": "Image agent that creates images using AI image modes.",
     "schemaFile": "./imageActionSchema.ts",
+    "compiledSchemaFile": "agents/image/dist/imageSchema.pas.json",
     "schemaType": "ImageAction"
   }
 }