diff --git a/src/__tests__/utils/auth-poll.test.ts b/src/__tests__/utils/auth-poll.test.ts new file mode 100644 index 0000000000..15f79b66ce --- /dev/null +++ b/src/__tests__/utils/auth-poll.test.ts @@ -0,0 +1,297 @@ +/** + * Tests for browser login polling + * + * The bug these cover: pollAuthStatus returned null for a pending user, for an + * HTTP error and for a dead transport alike, so waitForAuth polled a blocked + * host to timeout without printing anything. + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { pollAuthStatus, waitForAuth, WEB_URL } from '../../utils/auth'; + +const SESSION_ID = 'a'.repeat(64); +const CODE_VERIFIER = 'b'.repeat(43); +// Deliberately not the production default, so a code path that ignores the +// --web-url override fails these tests instead of passing by coincidence. +const WEB_HOST = 'https://test-host.example'; +const STATUS_URL = `${WEB_HOST}/api/auth/cli/status`; + +/** + * A Response body can only be read once, so every call must get a fresh one. + * Returns a factory for `mockImplementation`, not a shared object. + */ +function jsonResponse( + body: unknown, + init: { status?: number; statusText?: string } = {} +): () => Promise { + return async () => + new Response(JSON.stringify(body), { + status: init.status ?? 200, + statusText: init.statusText ?? '', + headers: { 'Content-Type': 'application/json' }, + }); +} + +/** Mimics Node's fetch: a bare TypeError carrying the real reason on `cause`. */ +function transportFailure(code: string, reason: string): TypeError { + const cause = new Error(reason) as Error & { code?: string }; + cause.code = code; + return new TypeError('fetch failed', { cause }); +} + +describe('pollAuthStatus', () => { + let fetchMock: ReturnType; + + beforeEach(() => { + fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('reports pending when the user has not authorised yet', async () => { + fetchMock.mockImplementation(jsonResponse({ status: 'pending' })); + + const result = await pollAuthStatus(SESSION_ID, CODE_VERIFIER, WEB_HOST); + + expect(result).toEqual({ status: 'pending' }); + expect(fetchMock).toHaveBeenCalledWith( + STATUS_URL, + expect.objectContaining({ method: 'POST' }) + ); + }); + + it('reports complete with the session when the browser has authorised', async () => { + fetchMock.mockImplementation( + jsonResponse({ + status: 'complete', + apiKey: 'fc-test-key', + teamName: 'Acme', + }) + ); + + const result = await pollAuthStatus(SESSION_ID, CODE_VERIFIER, WEB_HOST); + + expect(result).toEqual({ + status: 'complete', + session: { + apiKey: 'fc-test-key', + apiUrl: 'https://api.firecrawl.dev', + teamName: 'Acme', + }, + }); + }); + + it('reports unreachable, with the underlying cause, when the transport fails', async () => { + fetchMock.mockRejectedValue( + transportFailure('ECONNREFUSED', 'connect ECONNREFUSED 10.0.0.1:443') + ); + + const result = await pollAuthStatus(SESSION_ID, CODE_VERIFIER, WEB_HOST); + + expect(result.status).toBe('unreachable'); + if (result.status !== 'unreachable') + throw new Error('expected unreachable'); + expect(result.detail).toContain('connect ECONNREFUSED'); + expect(result.detail).toContain('ECONNREFUSED'); + }); + + it('separates a retryable rate limit from a refusal', async () => { + fetchMock.mockImplementation( + jsonResponse( + { error: 'Too many requests. Please try again later.' }, + { + status: 429, + } + ) + ); + await expect( + pollAuthStatus(SESSION_ID, CODE_VERIFIER, WEB_HOST) + ).resolves.toMatchObject({ status: 'server-busy' }); + + fetchMock.mockImplementation( + jsonResponse({ error: 'Session expired' }, { status: 410 }) + ); + await expect( + pollAuthStatus(SESSION_ID, CODE_VERIFIER, WEB_HOST) + ).resolves.toMatchObject({ + status: 'server-error', + detail: 'HTTP 410: Session expired', + }); + }); +}); + +describe('waitForAuth', () => { + let fetchMock: ReturnType; + + beforeEach(() => { + fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + vi.spyOn(process.stdout, 'write').mockReturnValue(true); + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + }); + + /** Drives the poll loop without waiting on the real 2 second interval. */ + async function drain(ms = 60_000): Promise { + await vi.advanceTimersByTimeAsync(ms); + } + + it('surfaces an error naming the host instead of polling a dead transport to timeout', async () => { + fetchMock.mockRejectedValue( + transportFailure('ENOTFOUND', 'getaddrinfo ENOTFOUND test-host.example') + ); + + const pending = waitForAuth(SESSION_ID, CODE_VERIFIER, WEB_HOST); + const assertion = expect(pending).rejects.toThrow( + /Cannot reach test-host\.example/ + ); + await drain(); + await assertion; + + // It gave up on the transport rather than running the full 5 minute timeout. + expect(fetchMock).toHaveBeenCalledTimes(3); + }); + + it('names the underlying cause and how to override the host', async () => { + fetchMock.mockRejectedValue( + transportFailure('ECONNRESET', 'CONNECT tunnel failed, response 403') + ); + + const pending = waitForAuth(SESSION_ID, CODE_VERIFIER, WEB_HOST); + const assertion = expect(pending).rejects.toThrow( + /CONNECT tunnel failed, response 403[\s\S]*--web-url/ + ); + await drain(); + await assertion; + }); + + it('keeps polling while the user has not authorised yet, then resolves', async () => { + fetchMock + .mockImplementationOnce(jsonResponse({ status: 'pending' })) + .mockImplementationOnce(jsonResponse({ status: 'pending' })) + .mockImplementationOnce(jsonResponse({ status: 'pending' })) + .mockImplementationOnce(jsonResponse({ status: 'pending' })) + .mockImplementation( + jsonResponse({ status: 'complete', apiKey: 'fc-late-key' }) + ); + + const pending = waitForAuth(SESSION_ID, CODE_VERIFIER, WEB_HOST); + await drain(); + + await expect(pending).resolves.toMatchObject({ apiKey: 'fc-late-key' }); + expect(fetchMock.mock.calls.length).toBeGreaterThan(4); + }); + + it('still honours the existing timeout for a user who never authorises', async () => { + fetchMock.mockImplementation(jsonResponse({ status: 'pending' })); + + const pending = waitForAuth(SESSION_ID, CODE_VERIFIER, WEB_HOST, 10_000); + const assertion = expect(pending).rejects.toThrow( + 'Authentication timed out. Please try again.' + ); + await drain(); + await assertion; + }); + + it('rides out a transient rate limit and completes', async () => { + fetchMock + .mockImplementationOnce( + jsonResponse({ error: 'Too many requests.' }, { status: 429 }) + ) + .mockImplementationOnce( + jsonResponse({ error: 'Too many requests.' }, { status: 429 }) + ) + .mockImplementation( + jsonResponse({ status: 'complete', apiKey: 'fc-after-429' }) + ); + + const pending = waitForAuth(SESSION_ID, CODE_VERIFIER, WEB_HOST); + await drain(); + + await expect(pending).resolves.toMatchObject({ apiKey: 'fc-after-429' }); + }); + + it('counts each failure budget only while that failure repeats', async () => { + // The transport budget is 3 and the server budget is 5. Five rounds of one + // dead transport then one rate limit exhaust both if a counter survives the + // other outcome, yet no failure of either kind ever repeats. + const rounds: Array<() => Promise> = []; + for (let round = 0; round < 5; round++) { + rounds.push(() => + Promise.reject(transportFailure('ECONNRESET', 'socket hang up')) + ); + rounds.push( + jsonResponse({ error: 'Too many requests.' }, { status: 429 }) + ); + } + const complete = jsonResponse({ + status: 'complete', + apiKey: 'fc-after-mixed', + }); + let call = 0; + fetchMock.mockImplementation(() => (rounds[call++] ?? complete)()); + + const pending = waitForAuth(SESSION_ID, CODE_VERIFIER, WEB_HOST); + await drain(); + + await expect(pending).resolves.toMatchObject({ + apiKey: 'fc-after-mixed', + }); + expect(fetchMock).toHaveBeenCalledTimes(rounds.length + 1); + }); + + it('stops at once when the server refuses the session', async () => { + fetchMock.mockImplementation( + jsonResponse({ error: 'Session expired' }, { status: 410 }) + ); + + const pending = waitForAuth(SESSION_ID, CODE_VERIFIER, WEB_HOST); + const assertion = expect(pending).rejects.toThrow( + /rejected the login poll: HTTP 410: Session expired/ + ); + await drain(); + await assertion; + + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it('polls the status endpoint on the configured host', async () => { + fetchMock.mockImplementation( + jsonResponse({ status: 'complete', apiKey: 'fc-key' }) + ); + + await waitForAuth(SESSION_ID, CODE_VERIFIER, WEB_HOST); + + expect(fetchMock).toHaveBeenCalledWith( + STATUS_URL, + expect.objectContaining({ method: 'POST' }) + ); + }); + + it('builds the same endpoint from the production default host', async () => { + fetchMock.mockImplementation( + jsonResponse({ status: 'complete', apiKey: 'fc-key' }) + ); + + await waitForAuth(SESSION_ID, CODE_VERIFIER, WEB_URL); + + expect(fetchMock).toHaveBeenCalledWith( + 'https://www.firecrawl.dev/api/auth/cli/status', + expect.objectContaining({ method: 'POST' }) + ); + }); +}); + +describe('default web URL', () => { + it('defaults to www, which the apex redirects to and allowlists usually permit', () => { + expect(WEB_URL).toBe('https://www.firecrawl.dev'); + }); +}); diff --git a/src/commands/login.ts b/src/commands/login.ts index c1c5b0c973..280d8677bc 100644 --- a/src/commands/login.ts +++ b/src/commands/login.ts @@ -10,10 +10,10 @@ import { manualLogin, interactiveLogin, isAuthenticated, + WEB_URL, } from '../utils/auth'; const DEFAULT_API_URL = 'https://api.firecrawl.dev'; -const WEB_URL = 'https://firecrawl.dev'; export interface LoginOptions { apiKey?: string; diff --git a/src/index.ts b/src/index.ts index bc13112ee2..27a0b9df8a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2164,7 +2164,7 @@ program .option('--api-url ', 'API URL (default: https://api.firecrawl.dev)') .option( '--web-url ', - 'Web URL for browser login (default: https://firecrawl.dev)' + 'Web URL for browser login (default: https://www.firecrawl.dev)' ) .option( '-m, --method ', @@ -2197,7 +2197,7 @@ program .option('--api-url ', 'API URL (default: https://api.firecrawl.dev)') .option( '--web-url ', - 'Web URL for browser login (default: https://firecrawl.dev)' + 'Web URL for browser login (default: https://www.firecrawl.dev)' ) .option( '-m, --method ', diff --git a/src/utils/auth.ts b/src/utils/auth.ts index 94f02c37f8..2ff3d32f8e 100644 --- a/src/utils/auth.ts +++ b/src/utils/auth.ts @@ -13,9 +13,16 @@ import { import { updateConfig, getApiKey } from './config'; const DEFAULT_API_URL = 'https://api.firecrawl.dev'; -const WEB_URL = 'https://firecrawl.dev'; +// The apex redirects to www, and egress allowlists often permit only www. +// Polling www directly avoids both the redirect and the blocked apex. +const WEB_URL = 'https://www.firecrawl.dev'; const AUTH_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes const POLL_INTERVAL_MS = 2000; // 2 seconds +const POLL_REQUEST_TIMEOUT_MS = 10000; // 10 seconds per request +// A dead transport fails on every attempt, so a small budget is enough. +const MAX_TRANSPORT_FAILURES = 3; +// The server rate limits at 30 polls per minute, so allow it to recover. +const MAX_SERVER_FAILURES = 5; /** * Prompt for input @@ -85,6 +92,77 @@ function generateCodeChallenge(verifier: string): string { return crypto.createHash('sha256').update(verifier).digest('base64url'); } +export interface AuthSession { + apiKey: string; + apiUrl?: string; + teamName?: string; +} + +/** + * Outcome of one poll. + * + * The caller must be able to tell a user who has not authorised yet from a + * server that refused and from a host the CLI cannot reach. Collapsing all + * three into one value is what made a dead transport look like a slow user. + */ +export type PollAuthResult = + | { status: 'pending' } + | { status: 'complete'; session: AuthSession } + | { status: 'server-busy'; detail: string } + | { status: 'server-error'; detail: string } + | { status: 'unreachable'; detail: string }; + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +/** + * Node reports DNS, TLS and proxy failures as a bare "fetch failed" TypeError + * and puts the real reason on `cause`, so unwrap it or the message says nothing. + */ +function describeFetchError(error: unknown): string { + const base = errorMessage(error); + const cause = + error instanceof Error + ? (error as Error & { cause?: unknown }).cause + : undefined; + if (!cause) return base; + + const causeText = errorMessage(cause).replace(/\s*\.\s*$/, ''); + const causeCode = (cause as { code?: unknown }).code; + const code = typeof causeCode === 'string' ? ` (${causeCode})` : ''; + return causeText && causeText !== base + ? `${base}: ${causeText}${code}` + : `${base}${code}`; +} + +function hostLabel(webUrl: string): string { + try { + return new URL(webUrl).host; + } catch { + return webUrl; + } +} + +function debugLog(message: string): void { + if (process.env.FIRECRAWL_DEBUG) { + process.stderr.write(`[firecrawl] ${message}\n`); + } +} + +async function describeHttpError(response: Response): Promise { + let serverMessage = ''; + try { + const body = await response.json(); + if (body && typeof body.error === 'string') serverMessage = body.error; + } catch { + // No body, or not JSON. The status line is the whole story. + } + return serverMessage + ? `HTTP ${response.status}: ${serverMessage}` + : `HTTP ${response.status} ${response.statusText}`.trim(); +} + /** * Poll the server for authentication status using PKCE verification * Uses POST to send the code_verifier securely (not in URL) @@ -93,11 +171,12 @@ async function pollAuthStatus( sessionId: string, codeVerifier: string, webUrl: string -): Promise<{ apiKey: string; apiUrl?: string; teamName?: string } | null> { +): Promise { const statusUrl = `${webUrl}/api/auth/cli/status`; + let response: Response; try { - const response = await fetch(statusUrl, { + response = await fetch(statusUrl, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -106,43 +185,83 @@ async function pollAuthStatus( session_id: sessionId, code_verifier: codeVerifier, }), + // Without this a black-holed connection never settles and the poll stalls. + signal: AbortSignal.timeout(POLL_REQUEST_TIMEOUT_MS), }); + } catch (error) { + return { status: 'unreachable', detail: describeFetchError(error) }; + } - if (!response.ok) { - return null; + if (!response.ok) { + const detail = await describeHttpError(response); + // Rate limits and server faults pass. Every other code is a refusal that + // more polling cannot change. + if (response.status === 429 || response.status >= 500) { + return { status: 'server-busy', detail }; } + return { status: 'server-error', detail }; + } - const data = await response.json(); - if (data.status === 'complete' && data.apiKey) { - return { + let data: { + status?: string; + apiKey?: string; + apiUrl?: string; + teamName?: string; + }; + try { + data = await response.json(); + } catch (error) { + return { + status: 'server-busy', + detail: `Unreadable response body: ${errorMessage(error)}`, + }; + } + + if (data.status === 'complete' && data.apiKey) { + return { + status: 'complete', + session: { apiKey: data.apiKey, apiUrl: data.apiUrl || DEFAULT_API_URL, teamName: data.teamName || undefined, - }; - } - - return null; - } catch { - return null; + }, + }; } + + return { status: 'pending' }; } /** * Wait for authentication with polling + * + * A pending user keeps the poll running until `timeoutMs`. A dead transport or + * a refusal ends it with a message that names the host and the reason. */ async function waitForAuth( sessionId: string, codeVerifier: string, webUrl: string, timeoutMs: number = AUTH_TIMEOUT_MS -): Promise<{ apiKey: string; apiUrl?: string; teamName?: string }> { +): Promise { const startTime = Date.now(); + const host = hostLabel(webUrl); let dots = 0; + let transportFailures = 0; + let serverFailures = 0; return new Promise((resolve, reject) => { + const clearLine = (): void => { + process.stdout.write('\r' + ' '.repeat(50) + '\r'); + }; + + const fail = (message: string): void => { + clearLine(); + reject(new Error(message)); + }; + const poll = async () => { if (Date.now() - startTime > timeoutMs) { - reject(new Error('Authentication timed out. Please try again.')); + fail('Authentication timed out. Please try again.'); return; } @@ -152,10 +271,54 @@ async function waitForAuth( dots++; const result = await pollAuthStatus(sessionId, codeVerifier, webUrl); - if (result) { - process.stdout.write('\r' + ' '.repeat(50) + '\r'); - resolve(result); - return; + + switch (result.status) { + case 'complete': + clearLine(); + resolve(result.session); + return; + + case 'pending': + transportFailures = 0; + serverFailures = 0; + break; + + case 'unreachable': + // Both budgets count consecutive failures, so each outcome clears the + // other counter. + serverFailures = 0; + transportFailures += 1; + debugLog(`cannot reach ${host}: ${result.detail}`); + if (transportFailures >= MAX_TRANSPORT_FAILURES) { + fail( + `Cannot reach ${host}: ${result.detail}. ` + + `${transportFailures} attempts to POST ${host}/api/auth/cli/status all failed to connect, ` + + `so the browser login cannot complete. Check DNS, proxy and firewall rules for ${host}, ` + + `or point the CLI at a host this network allows: firecrawl login --method browser --web-url ` + ); + return; + } + break; + + case 'server-busy': + transportFailures = 0; + serverFailures += 1; + debugLog(`${host} returned a retryable error: ${result.detail}`); + if (serverFailures >= MAX_SERVER_FAILURES) { + fail( + `${host} is not answering the login poll: ${result.detail}. ` + + `Please try again in a moment.` + ); + return; + } + break; + + case 'server-error': + fail( + `${host} rejected the login poll: ${result.detail}. ` + + `Run "firecrawl login" again to start a new session.` + ); + return; } setTimeout(poll, POLL_INTERVAL_MS); @@ -699,3 +862,8 @@ export async function ensureAuthenticated(): Promise { * Export for direct login command usage */ export { browserLogin, manualLogin, interactiveLogin }; + +/** + * Exported for tests that cover the pending, unreachable and refused cases + */ +export { pollAuthStatus, waitForAuth, WEB_URL };