auto retry when receive 5xx error from server with isRetryable = true

AGENTS.md

@@ -165,6 +165,14 @@ These are load-bearing. Breaking one causes silent failures in opencode.
     plan; never hardcode a plan→cap table. The TUI resolves the key from `api.state` (provider
     entry, then `config.provider.commandcode.options`, then `COMMANDCODE_API_KEY`), expanding
     `{env:VAR}` itself, and must never log the key — quota errors go through `redact()`.
+17. **Retryable upstream failures can arrive inside an HTTP 200 stream.** CommandCode's gateway
+    answers `200` and then emits an SSE `error` event carrying `statusCode`/`isRetryable` (e.g.
+    `{"type":"server_error","message":"Invalid error response format: Gateway request failed",
+    "statusCode":520,"isRetryable":true}`). `fetchWithRetry` buffers only the `start`/`start-step`
+    preamble and, if the first real event is such a retryable error and attempts remain, cancels the
+    body and retries with backoff. Once any content event is seen the error is surfaced, not
+    retried (retrying mid-stream would duplicate output). Non-retryable in-stream errors (400s)
+    surface immediately. Keep the preamble set in sync if upstream adds new pre-content events.
 
 ## Change workflow
 

README.md

@@ -257,7 +257,7 @@ The API key is never logged; quota errors pass through `redact()` like every oth
 | Finish reasons | Yes — unified (`stop`, `length`, `tool-calls`, `content-filter`, `error`, `other`) plus raw |
 | Sampling parameters | Yes — `temperature`, `topP`, `topK`, `stopSequences`, `seed`, presence/frequency penalties |
 | `reasoning_effort` | Yes — via `providerOptions.commandcode` |
-| Retry with backoff | Yes — 429/5xx and network errors, honouring `Retry-After` |
+| Retry with backoff | Yes — 429/5xx and network errors, honouring `Retry-After`; also retryable `error` events that arrive inside an HTTP 200 stream before any content (gateway 520s) |
 | Credential redaction | Yes — error bodies are scrubbed before surfacing |
 | Runtime toggles | Yes — `/cc-zdr` and `/cc-debug` flip the shared toggle file without a restart |
 | Sidebar status panel | Yes — a `sidebar_content` panel shows `zdr`/`debug` and live quota |

src/model.ts

@@ -109,6 +109,50 @@ function isRetryable(status: number): boolean {
   return status === 429 || (status >= 500 && status < 600);
 }
 
+// Upstream can answer HTTP 200 and then fail inside the stream (SSE `error` event).
+// CommandCode's gateway marks these with `statusCode`/`isRetryable`; retry before content.
+type StreamErrorInfo = { message: string; statusCode?: number; isRetryable?: boolean };
+
+function streamErrorInfo(evt: Record<string, any>): StreamErrorInfo | null {
+  if (evt?.["type"] !== "error") return null;
+  const err = evt["error"];
+  if (err === null || typeof err !== "object") return { message: String(err) };
+  const record = err as Record<string, unknown>;
+  const message = typeof record["message"] === "string" ? record["message"] : JSON.stringify(err);
+  return {
+    message,
+    ...(typeof record["statusCode"] === "number" ? { statusCode: record["statusCode"] } : {}),
+    ...(typeof record["isRetryable"] === "boolean" ? { isRetryable: record["isRetryable"] } : {}),
+  };
+}
+
+function isRetryableStreamError(info: StreamErrorInfo): boolean {
+  return info.isRetryable === true || (info.statusCode !== undefined && isRetryable(info.statusCode));
+}
+
+// Only `start`/`start-step` are safe to buffer while looking for an immediate stream error;
+// once real output appears a retry would duplicate the response.
+const STREAM_PREAMBLE = new Set(["start", "start-step"]);
+const MAX_PEEK_EVENTS = 32;
+
+type OpenedStream = {
+  response: Response;
+  prefix: Record<string, any>[];
+  events: AsyncGenerator<Record<string, any>>;
+};
+
+async function* emptyEvents(): AsyncGenerator<Record<string, any>> {
+  /* no-op */
+}
+
+async function* replayEvents(
+  prefix: Record<string, any>[],
+  rest: AsyncGenerator<Record<string, any>>,
+): AsyncGenerator<Record<string, any>> {
+  for (const evt of prefix) yield evt;
+  for await (const evt of rest) yield evt;
+}
+
 function retryDelay(attempt: number, retryAfter: string | null, maxDelay: number): number {
   if (retryAfter) {
     const seconds = Number(retryAfter);
@@ -186,7 +230,7 @@ class CommandCodeLanguageModel implements LanguageModelV3 {
     body: string,
     options: LanguageModelV3CallOptions,
     dbg: boolean,
-  ): Promise<Response> {
+  ): Promise<OpenedStream> {
     const headers = this.requestHeaders(options.headers, resolveZdr(options, this.opts.headers), dbg);
     let lastError: unknown;
     const url = `${this.opts.baseURL}${GENERATE_PATH}`;
@@ -247,8 +291,60 @@ class CommandCodeLanguageModel implements LanguageModelV3 {
           continue;
         }
       }
+      if (!response.ok || !response.body) {
         debugWhen(dbg, "fetch", "response", `url=${url}`, `attempt=${attempt}`, `status=${status}`, `elapsedMs=${Date.now() - started}`);
-      return response;
+        return { response, prefix: [], events: emptyEvents() };
+      }
+
+      // HTTP 200 can still hide an upstream failure. Buffer only the preamble; if the first
+      // real event is a retryable stream error, retry before anything reaches the client.
+      const events = iterateEvents(response.body, dbg);
+      const prefix: Record<string, any>[] = [];
+      let streamRetry: StreamErrorInfo | undefined;
+      while (prefix.length < MAX_PEEK_EVENTS) {
+        const next = await events.next();
+        if (next.done) break;
+        const evt = next.value;
+        prefix.push(evt);
+        const info = streamErrorInfo(evt);
+        if (info) {
+          if (isRetryableStreamError(info) && attempt < this.opts.maxRetries) streamRetry = info;
+          break;
+        }
+        if (!STREAM_PREAMBLE.has(evt["type"])) break;
+      }
+
+      if (streamRetry) {
+        const wait = retryDelay(attempt, null, this.opts.retryMaxDelay);
+        debugWhen(
+          dbg,
+          "fetch",
+          "stream-retry",
+          `url=${url}`,
+          `attempt=${attempt}`,
+          `status=${streamRetry.statusCode ?? "unknown"}`,
+          `message=${redact(streamRetry.message)}`,
+          `waitMs=${wait}`,
+        );
+        try {
+          await events.return(undefined as never);
+        } catch {
+          /* ignore */
+        }
+        try {
+          await response.body.cancel();
+        } catch {
+          /* ignore */
+        }
+        if (options.abortSignal?.aborted) {
+          throw options.abortSignal.reason ?? new Error("Aborted");
+        }
+        await sleep(wait);
+        continue;
+      }
+
+      debugWhen(dbg, "fetch", "response", `url=${url}`, `attempt=${attempt}`, `status=${status}`, `elapsedMs=${Date.now() - started}`);
+      return { response, prefix, events };
     }
 
     throw lastError instanceof Error ? lastError : new Error("Upstream unreachable");
@@ -279,11 +375,11 @@ class CommandCodeLanguageModel implements LanguageModelV3 {
     const dbg = resolveDebug(options);
     const body = transform(options, this.modelId);
     debugWhen(dbg, "doStream", `model=${this.modelId}`, `bodyBytes=${Buffer.byteLength(body, "utf8")}`);
-    const response = await this.fetchWithRetry(body, options, dbg);
+    const { response, prefix, events } = await this.fetchWithRetry(body, options, dbg);
     if (!response.ok || !response.body) throw await this.errorFrom(response, dbg);
     debugWhen(dbg, "doStream", "ok", `status=${response.status}`);
 
-    const stream = toReadableStream(this.streamParts(response.body, dbg));
+    const stream = toReadableStream(this.streamParts(prefix, events, dbg));
     return {
       stream,
       request: { body: JSON.parse(body) as unknown },
@@ -357,7 +453,11 @@ class CommandCodeLanguageModel implements LanguageModelV3 {
     return { content, finishReason, usage, warnings, request, response, ...(providerMetadata ? { providerMetadata } : {}) };
   }
 
-  private async *streamParts(body: ReadableStream<Uint8Array>, dbg: boolean): AsyncGenerator<LanguageModelV3StreamPart> {
+  private async *streamParts(
+    prefix: Record<string, any>[],
+    events: AsyncGenerator<Record<string, any>>,
+    dbg: boolean,
+  ): AsyncGenerator<LanguageModelV3StreamPart> {
     yield { type: "stream-start", warnings: [] };
 
     const textId = "text-0";
@@ -372,7 +472,7 @@ class CommandCodeLanguageModel implements LanguageModelV3 {
     let marketCost: number | undefined;
     let errored = false;
 
-    for await (const evt of iterateEvents(body, dbg)) {
+    for await (const evt of replayEvents(prefix, events)) {
       const payload = JSON.stringify(evt);
       debugWhen(dbg, "stream", `event=${evt.type}`, `payload=${payload && payload.length > 4096 ? payload.slice(0, 4096) + "…" : payload}`);
       switch (evt.type) {