Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,14 @@ serpapi login
```
Quote the expression with single quotes in bash/zsh to avoid shell interpretation of `$`, `|`, and `"`.
- `--api-key <key>` — Override API key (takes priority over environment and config file)
- `--timeout <seconds>` — HTTP request timeout (default `60`, env: `SERPAPI_TIMEOUT`). Use `0` to wait indefinitely. Slow engines such as `google_ai_mode`, or searches with `no_cache=true`, can take longer than the default; raise this if you see a `network_error` mentioning `Client.Timeout exceeded`. Note that a timed-out search may still complete on SerpApi's side and be billed and retrievable with `serpapi archive <search-id>`.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we consider google_ai_mode as a slow engine?

or searches with no_cache=true, can take longer than the default

This is also a bit confusing. All of the searches have no_cache=true by default.

```bash
# Give an AI Mode query up to two minutes
serpapi search --timeout 120 engine=google_ai_mode q="..."

# Or set it once for the shell session
export SERPAPI_TIMEOUT=120
```

## Configuration

Expand Down
55 changes: 51 additions & 4 deletions pkg/api/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"time"
Expand All @@ -15,24 +17,39 @@ import (
)

const (
defaultTimeout = 30 * time.Second
maxResponseBytes = 100 << 20 // 100 MB
// DefaultTimeout bounds a single HTTP request. Slow engines such as
// google_ai_mode can take well over 30s with no_cache=true, so this
// matches the 60s default used by the official serpapi-golang library.
DefaultTimeout = 60 * time.Second
maxResponseBytes = 100 << 20 // 100 MB
)

// Client is an HTTP client for the SerpApi service.
type Client struct {
apiKey string
baseURL string
timeout time.Duration
http *http.Client
}

// New creates a new SerpApi client. apiKey may be empty for unauthenticated requests.
// New creates a new SerpApi client with DefaultTimeout. apiKey may be empty
// for unauthenticated requests.
func New(apiKey string) *Client {
return NewWithTimeout(apiKey, DefaultTimeout)
}

// NewWithTimeout creates a new SerpApi client whose requests are bounded by
// timeout. A zero or negative timeout disables the limit entirely.
func NewWithTimeout(apiKey string, timeout time.Duration) *Client {
if timeout < 0 {
timeout = 0
}
return &Client{
apiKey: apiKey,
baseURL: "https://serpapi.com",
timeout: timeout,
http: &http.Client{
Timeout: defaultTimeout,
Timeout: timeout,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
},
Expand Down Expand Up @@ -64,6 +81,9 @@ func (c *Client) doGet(ctx context.Context, endpoint string, params map[string]s

resp, err := c.http.Do(req)
if err != nil {
if c.isTimeout(req, err) {
return nil, &clierrors.NetworkError{Message: c.timeoutMessage(), Cause: err}
}
return nil, &clierrors.NetworkError{Message: err.Error(), Cause: err}
}
defer resp.Body.Close()
Expand Down Expand Up @@ -104,6 +124,33 @@ func (c *Client) doGet(ctx context.Context, endpoint string, params map[string]s
return body, nil
}

// isTimeout reports whether err was caused by the client's own timeout rather
// than by the caller cancelling the request context.
func (c *Client) isTimeout(req *http.Request, err error) bool {
if c.timeout <= 0 {
return false
}
if ctxErr := req.Context().Err(); ctxErr != nil {
// The caller's context ended first (Ctrl-C, parent deadline, ...).
return false
}
var netErr net.Error
if errors.As(err, &netErr) && netErr.Timeout() {
return true
}
return errors.Is(err, context.DeadlineExceeded)
}

func (c *Client) timeoutMessage() string {
return fmt.Sprintf(
"Request timed out after %gs waiting for SerpApi to respond. "+
"Slow engines (e.g. google_ai_mode) or no_cache=true can take longer; "+
"retry with --timeout <seconds> (0 to wait indefinitely). "+
"The search may still have completed server-side and be available in your SerpApi search archive.",
c.timeout.Seconds(),
)
}

// checkAPIError returns an APIError if the JSON body contains a top-level "error" key.
func checkAPIError(body []byte) error {
var envelope struct {
Expand Down
97 changes: 97 additions & 0 deletions pkg/api/client_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
package api

import (
"context"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"

clierrors "github.com/serpapi/serpapi-cli/pkg/errors"
)

// newSlowServer returns a client pointed at a server that waits for `delay`
// (or until the request is cancelled) before answering with a tiny JSON body.
func newSlowServer(t *testing.T, delay time.Duration, timeout time.Duration) *Client {
t.Helper()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
select {
case <-time.After(delay):
case <-r.Context().Done():
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"ok":true}`))
}))
t.Cleanup(srv.Close)

client := NewWithTimeout("secret_key", timeout)
client.baseURL = srv.URL
return client
}

func TestNewUsesDefaultTimeout(t *testing.T) {
if got := New("k").http.Timeout; got != DefaultTimeout {
t.Fatalf("expected default timeout %s, got %s", DefaultTimeout, got)
}
if DefaultTimeout < 60*time.Second {
t.Fatalf("default timeout %s is too short for slow engines like google_ai_mode", DefaultTimeout)
}
}

func TestTimeoutProducesActionableNetworkError(t *testing.T) {
client := newSlowServer(t, 5*time.Second, 50*time.Millisecond)

_, err := client.Search(context.Background(), map[string]string{"q": "x"})
var netErr *clierrors.NetworkError
if !errors.As(err, &netErr) {
t.Fatalf("expected NetworkError, got %T: %v", err, err)
}
for _, want := range []string{"timed out", "--timeout", "archive"} {
if !strings.Contains(netErr.Message, want) {
t.Errorf("message %q should mention %q", netErr.Message, want)
}
}
if strings.Contains(netErr.Message, "api_key=") {
t.Errorf("timeout message should not echo the request URL: %q", netErr.Message)
}
}

func TestZeroTimeoutDisablesLimit(t *testing.T) {
client := newSlowServer(t, 200*time.Millisecond, 0)
if client.http.Timeout != 0 {
t.Fatalf("expected no http timeout, got %s", client.http.Timeout)
}

if _, err := client.Search(context.Background(), map[string]string{"q": "x"}); err != nil {
t.Fatalf("expected slow request to succeed with timeout disabled, got %v", err)
}
}

func TestLongerTimeoutAllowsSlowResponse(t *testing.T) {
client := newSlowServer(t, 200*time.Millisecond, 5*time.Second)
if _, err := client.Search(context.Background(), map[string]string{"q": "x"}); err != nil {
t.Fatalf("expected slow request to succeed within timeout, got %v", err)
}
}

func TestCallerCancellationIsNotReportedAsTimeout(t *testing.T) {
client := newSlowServer(t, 5*time.Second, 10*time.Second)

ctx, cancel := context.WithCancel(context.Background())
go func() {
time.Sleep(50 * time.Millisecond)
cancel()
}()

_, err := client.Search(ctx, map[string]string{"q": "x"})
var netErr *clierrors.NetworkError
if !errors.As(err, &netErr) {
t.Fatalf("expected NetworkError, got %T: %v", err, err)
}
if strings.Contains(netErr.Message, "timed out") {
t.Errorf("caller cancellation should not be reported as a timeout: %q", netErr.Message)
}
}
8 changes: 5 additions & 3 deletions pkg/cmd/account.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@ import (
"regexp"

"github.com/spf13/cobra"

"github.com/serpapi/serpapi-cli/pkg/api"
)

var accountCmd = &cobra.Command{
Expand All @@ -26,10 +24,14 @@ func runAccount(cmd *cobra.Command, args []string) error {
return err
}

client, err := newClient(apiKey)
if err != nil {
return err
}

sp := newSpinner("Fetching account...")
sp.Start()
defer sp.Stop()
client := api.New(apiKey)
result, err := client.Account(cmd.Context())
if err != nil {
return err
Expand Down
7 changes: 5 additions & 2 deletions pkg/cmd/archive.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (

"github.com/spf13/cobra"

"github.com/serpapi/serpapi-cli/pkg/api"
clierrors "github.com/serpapi/serpapi-cli/pkg/errors"
)

Expand Down Expand Up @@ -33,10 +32,14 @@ func runArchive(cmd *cobra.Command, args []string) error {
return err
}

client, err := newClient(apiKey)
if err != nil {
return err
}

sp := newSpinner("Fetching archive...")
sp.Start()
defer sp.Stop()
client := api.New(apiKey)
result, err := client.Archive(cmd.Context(), id)
if err != nil {
return err
Expand Down
7 changes: 5 additions & 2 deletions pkg/cmd/locations.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package cmd
import (
"github.com/spf13/cobra"

"github.com/serpapi/serpapi-cli/pkg/api"
"github.com/serpapi/serpapi-cli/pkg/params"
)

Expand All @@ -25,10 +24,14 @@ func runLocations(cmd *cobra.Command, args []string) error {
}

paramsMap := params.ParamsToMap(parsed)
client, err := newClient("")
if err != nil {
return err
}

sp := newSpinner("Fetching locations...")
sp.Start()
defer sp.Stop()
client := api.New("")
result, err := client.Locations(cmd.Context(), paramsMap)
if err != nil {
return err
Expand Down
11 changes: 8 additions & 3 deletions pkg/cmd/login.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (
"github.com/spf13/cobra"
"golang.org/x/term"

"github.com/serpapi/serpapi-cli/pkg/api"
"github.com/serpapi/serpapi-cli/pkg/config"
clierrors "github.com/serpapi/serpapi-cli/pkg/errors"
)
Expand All @@ -30,7 +29,10 @@ func init() {
func runLogin(cmd *cobra.Command, args []string) error {
// Check if already authenticated
if existingKey, ok := config.LoadAPIKey(); ok {
client := api.New(existingKey)
client, err := newClient(existingKey)
if err != nil {
return err
}
raw, err := client.Account(cmd.Context())
if err == nil {
var account struct {
Expand Down Expand Up @@ -80,7 +82,10 @@ func runLogin(cmd *cobra.Command, args []string) error {
return &clierrors.UsageError{Message: "API key cannot be empty."}
}

client := api.New(apiKey)
client, err := newClient(apiKey)
if err != nil {
return err
}
raw, err := client.Account(cmd.Context())
if err != nil {
return err
Expand Down
46 changes: 43 additions & 3 deletions pkg/cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,12 @@ import (
"encoding/json"
"fmt"
"os"
"strconv"
"time"

"github.com/spf13/cobra"

"github.com/serpapi/serpapi-cli/pkg/api"
"github.com/serpapi/serpapi-cli/pkg/config"
clierrors "github.com/serpapi/serpapi-cli/pkg/errors"
"github.com/serpapi/serpapi-cli/pkg/jq"
Expand All @@ -17,9 +20,10 @@ import (
)

var (
apiKeyFlag string
fieldsFlag string
jqFlag string
apiKeyFlag string
fieldsFlag string
jqFlag string
timeoutFlag string
)

var rootCmd = &cobra.Command{
Expand Down Expand Up @@ -49,6 +53,8 @@ func init() {
rootCmd.PersistentFlags().StringVar(&apiKeyFlag, "api-key", "", "SerpApi API key (env: SERPAPI_KEY)")
rootCmd.PersistentFlags().StringVar(&fieldsFlag, "fields", "", "Restrict JSON fields (server-side json_restrictor)")
rootCmd.PersistentFlags().StringVar(&jqFlag, "jq", "", "Apply jq filter to output")
rootCmd.PersistentFlags().StringVar(&timeoutFlag, "timeout", "",
fmt.Sprintf("HTTP request timeout in seconds, 0 to disable (default %d; env: SERPAPI_TIMEOUT)", int(api.DefaultTimeout/time.Second)))

// Wrap cobra flag-parsing errors as UsageError so they get exit code 2.
rootCmd.SetFlagErrorFunc(func(_ *cobra.Command, err error) error {
Expand Down Expand Up @@ -80,6 +86,40 @@ func resolveAPIKeyOptional() string {
return ""
}

// resolveTimeout merges --timeout with SERPAPI_TIMEOUT, falling back to
// api.DefaultTimeout. Values are whole or fractional seconds; 0 disables.
func resolveTimeout() (time.Duration, error) {
raw := timeoutFlag
source := "--timeout"
if raw == "" {
raw = os.Getenv("SERPAPI_TIMEOUT")
source = "SERPAPI_TIMEOUT"
}
if raw == "" {
return api.DefaultTimeout, nil
}
return parseTimeout(raw, source)
}

func parseTimeout(raw, source string) (time.Duration, error) {
secs, err := strconv.ParseFloat(raw, 64)
if err != nil || secs < 0 || secs != secs { // secs != secs rejects NaN
return 0, &clierrors.UsageError{
Message: fmt.Sprintf("Invalid %s value %q: expected a non-negative number of seconds", source, raw),
}
}
return time.Duration(secs * float64(time.Second)), nil
}

// newClient builds an API client honoring the configured request timeout.
func newClient(apiKey string) (*api.Client, error) {
timeout, err := resolveTimeout()
if err != nil {
return nil, err
}
return api.NewWithTimeout(apiKey, timeout), nil
}

// handleOutput applies --jq filtering and prints result.
func handleOutput(raw json.RawMessage) error {
if jqFlag != "" {
Expand Down
Loading
Loading