From e6ac7eccc3802c1ef8265e6e136604c21d054b63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 8 Oct 2026 18:22:58 +0300 Subject: [PATCH 1/2] alibaba: page through all instances --- pkg/providers/alibaba/instances.go | 47 +++++++++++++++---------- pkg/providers/alibaba/instances_test.go | 41 +++++++++++++++++++++ 2 files changed, 69 insertions(+), 19 deletions(-) create mode 100644 pkg/providers/alibaba/instances_test.go diff --git a/pkg/providers/alibaba/instances.go b/pkg/providers/alibaba/instances.go index 67e0c587..21f2615d 100644 --- a/pkg/providers/alibaba/instances.go +++ b/pkg/providers/alibaba/instances.go @@ -3,6 +3,7 @@ package alibaba import ( "context" + "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests" "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" "github.com/projectdiscovery/cloudlist/pkg/schema" ) @@ -22,29 +23,37 @@ func (d *instanceProvider) GetResource(ctx context.Context) (*schema.Resources, list := schema.NewResources() request := ecs.CreateDescribeInstancesRequest() + // The default page size is 10; 100 is the API maximum. + request.MaxResults = requests.NewInteger(100) - response, err := d.client.DescribeInstances(request) - if err != nil { - return nil, err - } - - for _, instance := range response.Instances.Instance { + for { + response, err := d.client.DescribeInstances(request) + if err != nil { + return nil, err + } - var ipv4, privateIPv4 string - if len(instance.PublicIpAddress.IpAddress) > 0 { - ipv4 = instance.PublicIpAddress.IpAddress[0] + for _, instance := range response.Instances.Instance { + + var ipv4, privateIPv4 string + if len(instance.PublicIpAddress.IpAddress) > 0 { + ipv4 = instance.PublicIpAddress.IpAddress[0] + } + if len(instance.NetworkInterfaces.NetworkInterface) > 0 && len(instance.NetworkInterfaces.NetworkInterface[0].PrivateIpSets.PrivateIpSet) > 0 { + privateIPv4 = instance.NetworkInterfaces.NetworkInterface[0].PrivateIpSets.PrivateIpSet[0].PrivateIpAddress + } + list.Append(&schema.Resource{ + ID: d.id, + Provider: providerName, + PublicIPv4: ipv4, + PrivateIpv4: privateIPv4, + Public: ipv4 != "", + Service: d.name(), + }) } - if len(instance.NetworkInterfaces.NetworkInterface) > 0 && len(instance.NetworkInterfaces.NetworkInterface[0].PrivateIpSets.PrivateIpSet) > 0 { - privateIPv4 = instance.NetworkInterfaces.NetworkInterface[0].PrivateIpSets.PrivateIpSet[0].PrivateIpAddress + if response.NextToken == "" { + break } - list.Append(&schema.Resource{ - ID: d.id, - Provider: providerName, - PublicIPv4: ipv4, - PrivateIpv4: privateIPv4, - Public: ipv4 != "", - Service: d.name(), - }) + request.NextToken = response.NextToken } return list, nil diff --git a/pkg/providers/alibaba/instances_test.go b/pkg/providers/alibaba/instances_test.go new file mode 100644 index 00000000..69d3cd08 --- /dev/null +++ b/pkg/providers/alibaba/instances_test.go @@ -0,0 +1,41 @@ +package alibaba + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/aliyun/alibaba-cloud-sdk-go/sdk" + "github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth/credentials" + "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestInstancesAllPages(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.NoError(t, r.ParseForm()) + w.Header().Set("Content-Type", "application/json") + if r.Form.Get("NextToken") == "page-2" { + _, _ = w.Write([]byte(`{"RequestId":"2","Instances":{"Instance":[{"PublicIpAddress":{"IpAddress":["47.88.1.2"]}}]},"NextToken":""}`)) + return + } + _, _ = w.Write([]byte(`{"RequestId":"1","Instances":{"Instance":[{"PublicIpAddress":{"IpAddress":["47.88.1.1"]}}]},"NextToken":"page-2"}`)) + })) + t.Cleanup(server.Close) + + client, err := ecs.NewClientWithOptions("cn-hangzhou", sdk.NewConfig().WithScheme("HTTP"), credentials.NewAccessKeyCredential("test", "test")) + require.NoError(t, err) + client.Domain = strings.TrimPrefix(server.URL, "http://") + + resources, err := (&instanceProvider{id: "test", client: client}).GetResource(context.Background()) + require.NoError(t, err) + + var ips []string + for _, r := range resources.Items { + ips = append(ips, r.PublicIPv4) + } + assert.ElementsMatch(t, []string{"47.88.1.1", "47.88.1.2"}, ips) +} From bb91c7c9488a4a288d8474f4bb2d43771a53fb4f Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 8 Oct 2026 20:49:52 +0200 Subject: [PATCH 2/2] keep instances --- pkg/providers/alibaba/alibaba.go | 7 ++++- pkg/providers/alibaba/instances.go | 4 ++- pkg/providers/alibaba/instances_test.go | 35 +++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/pkg/providers/alibaba/alibaba.go b/pkg/providers/alibaba/alibaba.go index 9177e537..16620305 100644 --- a/pkg/providers/alibaba/alibaba.go +++ b/pkg/providers/alibaba/alibaba.go @@ -5,6 +5,7 @@ import ( "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" "github.com/projectdiscovery/cloudlist/pkg/schema" + "github.com/projectdiscovery/gologger" ) var Services = []string{"instance"} @@ -79,9 +80,13 @@ func (p *Provider) Resources(ctx context.Context) (*schema.Resources, error) { finalResources := schema.NewResources() if p.ecsClient != nil { ecsprovider := &instanceProvider{client: p.ecsClient, id: p.id} - if resources, err := ecsprovider.GetResource(ctx); err == nil { + resources, err := ecsprovider.GetResource(ctx) + if resources != nil { finalResources.Merge(resources) } + if err != nil { + gologger.Warning().Msgf("alibaba: instance listing failed: %v", err) + } } return finalResources, nil } diff --git a/pkg/providers/alibaba/instances.go b/pkg/providers/alibaba/instances.go index 21f2615d..30b0366f 100644 --- a/pkg/providers/alibaba/instances.go +++ b/pkg/providers/alibaba/instances.go @@ -29,7 +29,9 @@ func (d *instanceProvider) GetResource(ctx context.Context) (*schema.Resources, for { response, err := d.client.DescribeInstances(request) if err != nil { - return nil, err + // A later page can fail after earlier instances were collected. + // Return them with the error so the caller can keep that page. + return list, err } for _, instance := range response.Instances.Instance { diff --git a/pkg/providers/alibaba/instances_test.go b/pkg/providers/alibaba/instances_test.go index 69d3cd08..da350cad 100644 --- a/pkg/providers/alibaba/instances_test.go +++ b/pkg/providers/alibaba/instances_test.go @@ -10,6 +10,7 @@ import ( "github.com/aliyun/alibaba-cloud-sdk-go/sdk" "github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth/credentials" "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" + "github.com/projectdiscovery/cloudlist/pkg/schema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -39,3 +40,37 @@ func TestInstancesAllPages(t *testing.T) { } assert.ElementsMatch(t, []string{"47.88.1.1", "47.88.1.2"}, ips) } + +func TestInstancesKeepsEarlierPageWhenLaterPageFails(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.NoError(t, r.ParseForm()) + if r.Form.Get("NextToken") == "page-2" { + w.WriteHeader(http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"RequestId":"1","Instances":{"Instance":[{"PublicIpAddress":{"IpAddress":["47.88.1.1"]}}]},"NextToken":"page-2"}`)) + })) + t.Cleanup(server.Close) + + client, err := ecs.NewClientWithOptions("cn-hangzhou", sdk.NewConfig().WithScheme("HTTP").WithAutoRetry(false), credentials.NewAccessKeyCredential("test", "test")) + require.NoError(t, err) + client.Domain = strings.TrimPrefix(server.URL, "http://") + + resources, err := (&instanceProvider{id: "test", client: client}).GetResource(context.Background()) + require.Error(t, err) + require.NotNil(t, resources) + require.Equal(t, []string{"47.88.1.1"}, publicIPs(resources)) + + kept, err := (&Provider{id: "test", ecsClient: client}).Resources(context.Background()) + require.NoError(t, err) + assert.Equal(t, []string{"47.88.1.1"}, publicIPs(kept)) +} + +func publicIPs(resources *schema.Resources) []string { + var ips []string + for _, item := range resources.Items { + ips = append(ips, item.PublicIPv4) + } + return ips +}