diff --git a/pkg/providers/alibaba/alibaba.go b/pkg/providers/alibaba/alibaba.go index 9177e537..16620305 100644 --- a/pkg/providers/alibaba/alibaba.go +++ b/pkg/providers/alibaba/alibaba.go @@ -5,6 +5,7 @@ import ( "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" "github.com/projectdiscovery/cloudlist/pkg/schema" + "github.com/projectdiscovery/gologger" ) var Services = []string{"instance"} @@ -79,9 +80,13 @@ func (p *Provider) Resources(ctx context.Context) (*schema.Resources, error) { finalResources := schema.NewResources() if p.ecsClient != nil { ecsprovider := &instanceProvider{client: p.ecsClient, id: p.id} - if resources, err := ecsprovider.GetResource(ctx); err == nil { + resources, err := ecsprovider.GetResource(ctx) + if resources != nil { finalResources.Merge(resources) } + if err != nil { + gologger.Warning().Msgf("alibaba: instance listing failed: %v", err) + } } return finalResources, nil } diff --git a/pkg/providers/alibaba/instances.go b/pkg/providers/alibaba/instances.go index 67e0c587..30b0366f 100644 --- a/pkg/providers/alibaba/instances.go +++ b/pkg/providers/alibaba/instances.go @@ -3,6 +3,7 @@ package alibaba import ( "context" + "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests" "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" "github.com/projectdiscovery/cloudlist/pkg/schema" ) @@ -22,29 +23,39 @@ func (d *instanceProvider) GetResource(ctx context.Context) (*schema.Resources, list := schema.NewResources() request := ecs.CreateDescribeInstancesRequest() + // The default page size is 10; 100 is the API maximum. + request.MaxResults = requests.NewInteger(100) + + for { + response, err := d.client.DescribeInstances(request) + if err != nil { + // A later page can fail after earlier instances were collected. + // Return them with the error so the caller can keep that page. + return list, err + } - response, err := d.client.DescribeInstances(request) - if err != nil { - return nil, err - } - - for _, instance := range response.Instances.Instance { - - var ipv4, privateIPv4 string - if len(instance.PublicIpAddress.IpAddress) > 0 { - ipv4 = instance.PublicIpAddress.IpAddress[0] + for _, instance := range response.Instances.Instance { + + var ipv4, privateIPv4 string + if len(instance.PublicIpAddress.IpAddress) > 0 { + ipv4 = instance.PublicIpAddress.IpAddress[0] + } + if len(instance.NetworkInterfaces.NetworkInterface) > 0 && len(instance.NetworkInterfaces.NetworkInterface[0].PrivateIpSets.PrivateIpSet) > 0 { + privateIPv4 = instance.NetworkInterfaces.NetworkInterface[0].PrivateIpSets.PrivateIpSet[0].PrivateIpAddress + } + list.Append(&schema.Resource{ + ID: d.id, + Provider: providerName, + PublicIPv4: ipv4, + PrivateIpv4: privateIPv4, + Public: ipv4 != "", + Service: d.name(), + }) } - if len(instance.NetworkInterfaces.NetworkInterface) > 0 && len(instance.NetworkInterfaces.NetworkInterface[0].PrivateIpSets.PrivateIpSet) > 0 { - privateIPv4 = instance.NetworkInterfaces.NetworkInterface[0].PrivateIpSets.PrivateIpSet[0].PrivateIpAddress + if response.NextToken == "" { + break } - list.Append(&schema.Resource{ - ID: d.id, - Provider: providerName, - PublicIPv4: ipv4, - PrivateIpv4: privateIPv4, - Public: ipv4 != "", - Service: d.name(), - }) + request.NextToken = response.NextToken } return list, nil diff --git a/pkg/providers/alibaba/instances_test.go b/pkg/providers/alibaba/instances_test.go new file mode 100644 index 00000000..da350cad --- /dev/null +++ b/pkg/providers/alibaba/instances_test.go @@ -0,0 +1,76 @@ +package alibaba + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/aliyun/alibaba-cloud-sdk-go/sdk" + "github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth/credentials" + "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" + "github.com/projectdiscovery/cloudlist/pkg/schema" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestInstancesAllPages(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.NoError(t, r.ParseForm()) + w.Header().Set("Content-Type", "application/json") + if r.Form.Get("NextToken") == "page-2" { + _, _ = w.Write([]byte(`{"RequestId":"2","Instances":{"Instance":[{"PublicIpAddress":{"IpAddress":["47.88.1.2"]}}]},"NextToken":""}`)) + return + } + _, _ = w.Write([]byte(`{"RequestId":"1","Instances":{"Instance":[{"PublicIpAddress":{"IpAddress":["47.88.1.1"]}}]},"NextToken":"page-2"}`)) + })) + t.Cleanup(server.Close) + + client, err := ecs.NewClientWithOptions("cn-hangzhou", sdk.NewConfig().WithScheme("HTTP"), credentials.NewAccessKeyCredential("test", "test")) + require.NoError(t, err) + client.Domain = strings.TrimPrefix(server.URL, "http://") + + resources, err := (&instanceProvider{id: "test", client: client}).GetResource(context.Background()) + require.NoError(t, err) + + var ips []string + for _, r := range resources.Items { + ips = append(ips, r.PublicIPv4) + } + assert.ElementsMatch(t, []string{"47.88.1.1", "47.88.1.2"}, ips) +} + +func TestInstancesKeepsEarlierPageWhenLaterPageFails(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.NoError(t, r.ParseForm()) + if r.Form.Get("NextToken") == "page-2" { + w.WriteHeader(http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"RequestId":"1","Instances":{"Instance":[{"PublicIpAddress":{"IpAddress":["47.88.1.1"]}}]},"NextToken":"page-2"}`)) + })) + t.Cleanup(server.Close) + + client, err := ecs.NewClientWithOptions("cn-hangzhou", sdk.NewConfig().WithScheme("HTTP").WithAutoRetry(false), credentials.NewAccessKeyCredential("test", "test")) + require.NoError(t, err) + client.Domain = strings.TrimPrefix(server.URL, "http://") + + resources, err := (&instanceProvider{id: "test", client: client}).GetResource(context.Background()) + require.Error(t, err) + require.NotNil(t, resources) + require.Equal(t, []string{"47.88.1.1"}, publicIPs(resources)) + + kept, err := (&Provider{id: "test", ecsClient: client}).Resources(context.Background()) + require.NoError(t, err) + assert.Equal(t, []string{"47.88.1.1"}, publicIPs(kept)) +} + +func publicIPs(resources *schema.Resources) []string { + var ips []string + for _, item := range resources.Items { + ips = append(ips, item.PublicIPv4) + } + return ips +}