diff --git a/pkg/providers/scaleway/containers.go b/pkg/providers/scaleway/containers.go new file mode 100644 index 00000000..bdc31c18 --- /dev/null +++ b/pkg/providers/scaleway/containers.go @@ -0,0 +1,118 @@ +package scaleway + +import ( + "context" + "net/url" + + "github.com/projectdiscovery/cloudlist/pkg/schema" + container "github.com/scaleway/scaleway-sdk-go/api/container/v1beta1" + function "github.com/scaleway/scaleway-sdk-go/api/function/v1beta1" + k8s "github.com/scaleway/scaleway-sdk-go/api/k8s/v1" + "github.com/scaleway/scaleway-sdk-go/scw" +) + +// containerProvider lists endpoints of Serverless Containers, Serverless +// Functions and Kapsule clusters. +type containerProvider struct { + id string + containerAPI *container.API + functionAPI *function.API + k8sAPI *k8s.API +} + +// GetContainers returns Serverless Containers domain names. +// Listing goes through namespaces because this SDK version always sends +// namespace_id on ListContainers. +func (d *containerProvider) GetContainers(ctx context.Context) (*schema.Resources, error) { + list := schema.NewResources() + var listingErr error + + err := forEachLocality(d.containerAPI.Regions(), func(region scw.Region) error { + namespaces, err := d.containerAPI.ListNamespaces(&container.ListNamespacesRequest{Region: region}, scw.WithAllPages(), scw.WithContext(ctx)) + if err != nil { + return err + } + for _, namespace := range namespaces.Namespaces { + resp, err := d.containerAPI.ListContainers(&container.ListContainersRequest{Region: region, NamespaceID: namespace.ID}, scw.WithAllPages(), scw.WithContext(ctx)) + if err != nil { + listingErr = err + continue + } + for _, c := range resp.Containers { + list.Append(d.dnsResource(c.DomainName, "container")) + } + } + return nil + }) + if listingErr != nil { + err = listingErr + } + if err != nil && len(list.Items) == 0 { + return nil, err + } + return list, err +} + +// GetFunctions returns Serverless Functions domain names. +func (d *containerProvider) GetFunctions(ctx context.Context) (*schema.Resources, error) { + list := schema.NewResources() + var listingErr error + + err := forEachLocality(d.functionAPI.Regions(), func(region scw.Region) error { + namespaces, err := d.functionAPI.ListNamespaces(&function.ListNamespacesRequest{Region: region}, scw.WithAllPages(), scw.WithContext(ctx)) + if err != nil { + return err + } + for _, namespace := range namespaces.Namespaces { + resp, err := d.functionAPI.ListFunctions(&function.ListFunctionsRequest{Region: region, NamespaceID: namespace.ID}, scw.WithAllPages(), scw.WithContext(ctx)) + if err != nil { + listingErr = err + continue + } + for _, f := range resp.Functions { + list.Append(d.dnsResource(f.DomainName, "function")) + } + } + return nil + }) + if listingErr != nil { + err = listingErr + } + if err != nil && len(list.Items) == 0 { + return nil, err + } + return list, err +} + +// GetKapsuleClusters returns Kapsule cluster API server hostnames. +func (d *containerProvider) GetKapsuleClusters(ctx context.Context) (*schema.Resources, error) { + list := schema.NewResources() + + err := forEachLocality(d.k8sAPI.Regions(), func(region scw.Region) error { + resp, err := d.k8sAPI.ListClusters(&k8s.ListClustersRequest{Region: region}, scw.WithAllPages(), scw.WithContext(ctx)) + if err != nil { + return err + } + for _, cluster := range resp.Clusters { + // ClusterURL is a full URL such as https://.api.k8s.fr-par.scw.cloud:6443 + if u, err := url.Parse(cluster.ClusterURL); err == nil { + list.Append(d.dnsResource(u.Hostname(), "kapsule")) + } + } + return nil + }) + if err != nil && len(list.Items) == 0 { + return nil, err + } + return list, err +} + +func (d *containerProvider) dnsResource(hostname, service string) *schema.Resource { + return &schema.Resource{ + Provider: providerName, + ID: d.id, + DNSName: hostname, + Public: true, + Service: service, + } +} diff --git a/pkg/providers/scaleway/ips.go b/pkg/providers/scaleway/ips.go new file mode 100644 index 00000000..e9673b97 --- /dev/null +++ b/pkg/providers/scaleway/ips.go @@ -0,0 +1,109 @@ +package scaleway + +import ( + "context" + "fmt" + "net" + + "github.com/projectdiscovery/cloudlist/pkg/schema" + "github.com/projectdiscovery/gologger" + "github.com/scaleway/scaleway-sdk-go/api/flexibleip/v1alpha1" + "github.com/scaleway/scaleway-sdk-go/api/instance/v1" + "github.com/scaleway/scaleway-sdk-go/api/lb/v1" + "github.com/scaleway/scaleway-sdk-go/scw" +) + +// ipProvider lists reserved IPs, which are returned whether or not they are +// attached, so detached IPs that ListServers never sees are covered too. +type ipProvider struct { + id string + instanceAPI *instance.API + flexibleIPAPI *flexibleip.API + lbAPI *lb.ZonedAPI +} + +// GetFlexibleIPs returns Instance flexible IPs and Elastic Metal flexible IPs. +func (d *ipProvider) GetFlexibleIPs(ctx context.Context) (*schema.Resources, error) { + list := schema.NewResources() + + instanceErr := forEachLocality(d.instanceAPI.Zones(), func(zone scw.Zone) error { + resp, err := d.instanceAPI.ListIPs(&instance.ListIPsRequest{Zone: zone}, scw.WithAllPages(), scw.WithContext(ctx)) + if err != nil { + return err + } + for _, ip := range resp.IPs { + list.Append(d.ipResource(ip.Address, "flexibleip")) + } + return nil + }) + + metalErr := forEachLocality(d.flexibleIPAPI.Zones(), func(zone scw.Zone) error { + resp, err := d.flexibleIPAPI.ListFlexibleIPs(&flexibleip.ListFlexibleIPsRequest{Zone: zone}, scw.WithAllPages(), scw.WithContext(ctx)) + if err != nil { + return err + } + for _, ip := range resp.FlexibleIPs { + list.Append(d.ipResource(ip.IPAddress.IP, "flexibleip")) + } + return nil + }) + if len(list.Items) == 0 && instanceErr != nil && metalErr != nil { + return nil, fmt.Errorf("scaleway: flexible ip listing failed: %v; %v", instanceErr, metalErr) + } + for _, err := range []error{instanceErr, metalErr} { + if err != nil { + gologger.Warning().Msgf("scaleway: flexible ip listing failed: %v", err) + } + } + return list, nil +} + +// GetLoadBalancerIPs returns load balancer IPs. +func (d *ipProvider) GetLoadBalancerIPs(ctx context.Context) (*schema.Resources, error) { + list := schema.NewResources() + + err := forEachLocality(d.lbAPI.Zones(), func(zone scw.Zone) error { + resp, err := d.lbAPI.ListIPs(&lb.ZonedAPIListIPsRequest{Zone: zone}, scw.WithAllPages(), scw.WithContext(ctx)) + if err != nil { + return err + } + for _, ip := range resp.IPs { + list.Append(d.ipResource(net.ParseIP(ip.IPAddress), "lb")) + } + return nil + }) + if err != nil { + return nil, err + } + return list, nil +} + +func (d *ipProvider) ipResource(ip net.IP, service string) *schema.Resource { + resource := &schema.Resource{ + Provider: providerName, + ID: d.id, + Public: true, + Service: service, + } + if ip.To4() != nil { + resource.PublicIPv4 = ip.String() + } else if ip != nil { + resource.PublicIPv6 = ip.String() + } + return resource +} + +// forEachLocality calls fn for every zone or region, skipping ones that fail +// (e.g. a product not offered there), and only errors when all of them failed. +func forEachLocality[T scw.Zone | scw.Region](localities []T, fn func(T) error) error { + var errs []error + for _, locality := range localities { + if err := fn(locality); err != nil { + errs = append(errs, err) + } + } + if len(errs) > 0 && len(errs) == len(localities) { + return fmt.Errorf("all localities failed: %v", errs) + } + return nil +} diff --git a/pkg/providers/scaleway/scaleway.go b/pkg/providers/scaleway/scaleway.go index 4c9bc046..40207604 100644 --- a/pkg/providers/scaleway/scaleway.go +++ b/pkg/providers/scaleway/scaleway.go @@ -4,11 +4,17 @@ import ( "context" "github.com/projectdiscovery/cloudlist/pkg/schema" + "github.com/projectdiscovery/gologger" + container "github.com/scaleway/scaleway-sdk-go/api/container/v1beta1" + "github.com/scaleway/scaleway-sdk-go/api/flexibleip/v1alpha1" + function "github.com/scaleway/scaleway-sdk-go/api/function/v1beta1" "github.com/scaleway/scaleway-sdk-go/api/instance/v1" + k8s "github.com/scaleway/scaleway-sdk-go/api/k8s/v1" + "github.com/scaleway/scaleway-sdk-go/api/lb/v1" "github.com/scaleway/scaleway-sdk-go/scw" ) -var Services = []string{"instance"} +var Services = []string{"instance", "flexibleip", "lb", "container", "function", "kapsule"} // Provider is a data provider for scaleway API type Provider struct { @@ -66,5 +72,29 @@ func (p *Provider) Resources(ctx context.Context) (*schema.Resources, error) { finalResources.Merge(resources) } } + + ips := &ipProvider{id: p.id, instanceAPI: instance.NewAPI(p.client), flexibleIPAPI: flexibleip.NewAPI(p.client), lbAPI: lb.NewZonedAPI(p.client)} + containers := &containerProvider{id: p.id, containerAPI: container.NewAPI(p.client), functionAPI: function.NewAPI(p.client), k8sAPI: k8s.NewAPI(p.client)} + for _, s := range []struct { + name string + getResources func(context.Context) (*schema.Resources, error) + }{ + {"flexibleip", ips.GetFlexibleIPs}, + {"lb", ips.GetLoadBalancerIPs}, + {"container", containers.GetContainers}, + {"function", containers.GetFunctions}, + {"kapsule", containers.GetKapsuleClusters}, + } { + if !p.services.Has(s.name) { + continue + } + resources, err := s.getResources(ctx) + if resources != nil { + finalResources.Merge(resources) + } + if err != nil { + gologger.Warning().Msgf("scaleway: %s listing failed: %v", s.name, err) + } + } return finalResources, nil } diff --git a/pkg/providers/scaleway/scaleway_test.go b/pkg/providers/scaleway/scaleway_test.go new file mode 100644 index 00000000..3f8dae71 --- /dev/null +++ b/pkg/providers/scaleway/scaleway_test.go @@ -0,0 +1,153 @@ +package scaleway + +import ( + "context" + "net/http" + "net/http/httptest" + "sort" + "strings" + "testing" + + "github.com/projectdiscovery/cloudlist/pkg/schema" + "github.com/scaleway/scaleway-sdk-go/scw" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newTestScalewayAPI(t *testing.T) *httptest.Server { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + query := r.URL.Query() + switch r.URL.Path { + case "/instance/v1/zones/fr-par-1/ips": + // Two pages, to check every page is fetched. + if query.Get("page") == "2" { + _, _ = w.Write([]byte(`{"ips":[{"address":"51.15.0.11"}],"total_count":2}`)) + return + } + _, _ = w.Write([]byte(`{"ips":[{"address":"51.15.0.10"}],"total_count":2}`)) + case "/flexible-ip/v1alpha1/zones/fr-par-2/fips": + _, _ = w.Write([]byte(`{"flexible_ips":[{"ip_address":"51.159.0.20/32"}],"total_count":1}`)) + case "/lb/v1/zones/nl-ams-1/ips": + // A failing zone must not hide results from the others. + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"message":"not found"}`)) + case "/lb/v1/zones/fr-par-1/ips": + _, _ = w.Write([]byte(`{"ips":[{"ip_address":"51.159.10.30"}],"total_count":1}`)) + case "/containers/v1beta1/regions/fr-par/namespaces", "/functions/v1beta1/regions/fr-par/namespaces": + _, _ = w.Write([]byte(`{"namespaces":[{"id":"ns-1"}],"total_count":1}`)) + case "/containers/v1beta1/regions/fr-par/containers": + assert.Equal(t, "ns-1", query.Get("namespace_id")) + _, _ = w.Write([]byte(`{"containers":[{"domain_name":"ns1-app.functions.fnc.fr-par.scw.cloud"}],"total_count":1}`)) + case "/functions/v1beta1/regions/fr-par/functions": + assert.Equal(t, "ns-1", query.Get("namespace_id")) + _, _ = w.Write([]byte(`{"functions":[{"domain_name":"ns1-fn.functions.fnc.fr-par.scw.cloud"}],"total_count":1}`)) + case "/k8s/v1/regions/nl-ams/clusters": + _, _ = w.Write([]byte(`{"clusters":[{"cluster_url":"https://abc.api.k8s.nl-ams.scw.cloud:6443"}],"total_count":1}`)) + default: + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(server.Close) + return server +} + +func TestResources(t *testing.T) { + server := newTestScalewayAPI(t) + client, err := scw.NewClient( + scw.WithAuth("SCWXXXXXXXXXXXXXXXXX", "11111111-1111-1111-1111-111111111111"), + scw.WithAPIURL(server.URL), + ) + require.NoError(t, err) + + services := schema.ServiceMap{} + for _, s := range Services { + services[s] = struct{}{} + } + provider := &Provider{id: "test", client: client, services: services} + + resources, err := provider.Resources(context.Background()) + require.NoError(t, err) + + var got []string + for _, r := range resources.Items { + got = append(got, r.Service+" "+r.PublicIPv4+r.DNSName) + assert.True(t, r.Public) + } + sort.Strings(got) + assert.Equal(t, []string{ + "container ns1-app.functions.fnc.fr-par.scw.cloud", + "flexibleip 51.15.0.10", + "flexibleip 51.15.0.11", + "flexibleip 51.159.0.20", + "function ns1-fn.functions.fnc.fr-par.scw.cloud", + "kapsule abc.api.k8s.nl-ams.scw.cloud", + "lb 51.159.10.30", + }, got) +} + +func TestFlexibleIPsKeptWhenMetalListingFails(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if strings.Contains(r.URL.Path, "/flexible-ip/") { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"message":"failed"}`)) + return + } + if strings.Contains(r.URL.Path, "/instance/") && strings.HasSuffix(r.URL.Path, "/ips") { + _, _ = w.Write([]byte(`{"ips":[{"address":"51.15.0.10"}],"total_count":1}`)) + return + } + _, _ = w.Write([]byte(`{}`)) + })) + t.Cleanup(server.Close) + + client, err := scw.NewClient( + scw.WithAuth("SCWXXXXXXXXXXXXXXXXX", "11111111-1111-1111-1111-111111111111"), + scw.WithAPIURL(server.URL), + ) + require.NoError(t, err) + + provider := &Provider{id: "test", client: client, services: schema.ServiceMap{"flexibleip": {}}} + resources, err := provider.Resources(context.Background()) + require.NoError(t, err) + + var got []string + for _, item := range resources.Items { + got = append(got, item.PublicIPv4) + } + assert.Contains(t, got, "51.15.0.10") +} + +func TestContainerKeptWhenLaterNamespaceFails(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/containers/v1beta1/regions/fr-par/namespaces": + _, _ = w.Write([]byte(`{"namespaces":[{"id":"ns-1"},{"id":"ns-2"}],"total_count":2}`)) + case "/containers/v1beta1/regions/fr-par/containers": + if r.URL.Query().Get("namespace_id") == "ns-2" { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"message":"failed"}`)) + return + } + _, _ = w.Write([]byte(`{"containers":[{"domain_name":"ns1-app.functions.fnc.fr-par.scw.cloud"}],"total_count":1}`)) + default: + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(server.Close) + + client, err := scw.NewClient( + scw.WithAuth("SCWXXXXXXXXXXXXXXXXX", "11111111-1111-1111-1111-111111111111"), + scw.WithAPIURL(server.URL), + ) + require.NoError(t, err) + + provider := &Provider{id: "test", client: client, services: schema.ServiceMap{"container": {}}} + resources, err := provider.Resources(context.Background()) + require.NoError(t, err) + require.Len(t, resources.Items, 1) + assert.Equal(t, "ns1-app.functions.fnc.fr-par.scw.cloud", resources.Items[0].DNSName) +}