diff --git a/PROVIDERS.md b/PROVIDERS.md index 2e038959..1e18e363 100644 --- a/PROVIDERS.md +++ b/PROVIDERS.md @@ -316,7 +316,9 @@ Digitalocean can be integrated by using the following configuration block. digitalocean_token: $DIGITALOCEAN_TOKEN ``` -`digitalocean_token` can be generated from the Digitalocean Control Panel. We recommend only giving Read Access to the token. +`digitalocean_token` can be generated from the Digitalocean Control Panel. We recommend only giving Read Access to the token. With custom scopes, the token needs `droplet:read`, `app:read`, `reserved_ip:read`, `load_balancer:read` and `kubernetes:read`. Services the token cannot read are skipped with a warning. + +Supported services: `droplet` (alias `instance`), `app`, `reservedip` (reserved IPv4 and IPv6, including unassigned ones), `loadbalancer`, `kubernetes` (DOKS control plane endpoints). References - 1. https://www.digitalocean.com/docs/apis-clis/api/create-personal-access-token/ diff --git a/go.mod b/go.mod index 729097b9..a77ec505 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/aliyun/alibaba-cloud-sdk-go v1.62.560 github.com/aws/aws-sdk-go v1.55.5 github.com/cloudflare/cloudflare-go v0.77.0 - github.com/digitalocean/godo v1.102.1 + github.com/digitalocean/godo v1.218.0 github.com/fastly/go-fastly/v3 v3.12.0 github.com/gophercloud/gophercloud v1.7.0 github.com/hashicorp/consul/api v1.20.0 @@ -54,7 +54,7 @@ require ( github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect - github.com/fatih/color v1.15.0 // indirect + github.com/fatih/color v1.16.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/jsonpointer v0.20.2 // indirect @@ -78,8 +78,8 @@ require ( github.com/gorilla/websocket v1.5.0 // indirect github.com/hashicorp/cronexpr v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-hclog v1.2.0 // indirect - github.com/hashicorp/go-retryablehttp v0.7.4 // indirect + github.com/hashicorp/go-hclog v1.6.3 // indirect + github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/serf v0.10.1 // indirect diff --git a/go.sum b/go.sum index 685d1e8e..d1b149cc 100644 --- a/go.sum +++ b/go.sum @@ -172,8 +172,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/digitalocean/godo v1.102.1 h1:BrNePwIXjQWjOJXVTBqkURMjm70BRR0qXbRKfHNBF24= -github.com/digitalocean/godo v1.102.1/go.mod h1:SaUYccN7r+CO1QtsbXGypAsgobDrmSfVMJESEfXgoEg= +github.com/digitalocean/godo v1.218.0 h1:Seawyy0Zt2SMfT1GsJMB2q5ttkn3VeOty3almGV26Qk= +github.com/digitalocean/godo v1.218.0/go.mod h1:xQsWpVCCbkDrWisHA72hPzPlnC+4W5w/McZY5ij9uvU= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= @@ -196,8 +196,9 @@ github.com/fastly/go-fastly/v3 v3.12.0 h1:rF+SF34yYcX2eig7aOXiDKdT/ipdfnxKHiNfsb github.com/fastly/go-fastly/v3 v3.12.0/go.mod h1:KOaCWsmkIKSASPzADl8PT/bTQIghOw/eEaxlHOu3jMA= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -307,9 +308,8 @@ github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtng github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-hclog v1.2.0 h1:La19f8d7WIlm4ogzNHB0JGqs5AUDAZ2UfCY4sJXcJdM= -github.com/hashicorp/go-hclog v1.2.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.0 h1:8exGP7ego3OmkfksihtSouGMZ+hQrhxx+FVELeXpVPE= github.com/hashicorp/go-immutable-radix v1.3.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= @@ -320,8 +320,8 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA= -github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= @@ -403,13 +403,15 @@ github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= @@ -567,6 +569,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= @@ -746,7 +749,6 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -760,7 +762,10 @@ golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/pkg/providers/digitalocean/digitalocean.go b/pkg/providers/digitalocean/digitalocean.go index 6cbfd83c..18d8d1d7 100644 --- a/pkg/providers/digitalocean/digitalocean.go +++ b/pkg/providers/digitalocean/digitalocean.go @@ -5,9 +5,10 @@ import ( "github.com/digitalocean/godo" "github.com/projectdiscovery/cloudlist/pkg/schema" + "github.com/projectdiscovery/gologger" ) -var Services = []string{"droplet", "app", "instance"} +var Services = []string{"droplet", "app", "instance", "reservedip", "loadbalancer", "kubernetes"} // Provider is a data provider for digitalocean API type Provider struct { @@ -90,5 +91,52 @@ func (p *Provider) Resources(ctx context.Context) (*schema.Resources, error) { finalResources.Merge(apps) } + // Newer services only warn on failure so a token scoped to the original + // services (droplet, app) keeps working after upgrade. + if p.services.Has("reservedip") { + reservedipprovider := &reservedIPProvider{ + client: p.client, + id: p.id, + extendedMetadata: p.extendedMetadata, + } + reservedIPs, err := reservedipprovider.GetResource(ctx) + if reservedIPs != nil { + finalResources.Merge(reservedIPs) + } + if err != nil { + gologger.Warning().Msgf("digitalocean: could not list reserved ips: %s", err) + } + } + + if p.services.Has("loadbalancer") { + loadbalancerprovider := &loadBalancerProvider{ + client: p.client, + id: p.id, + extendedMetadata: p.extendedMetadata, + } + loadBalancers, err := loadbalancerprovider.GetResource(ctx) + if loadBalancers != nil { + finalResources.Merge(loadBalancers) + } + if err != nil { + gologger.Warning().Msgf("digitalocean: could not list load balancers: %s", err) + } + } + + if p.services.Has("kubernetes") { + kubernetesprovider := &kubernetesProvider{ + client: p.client, + id: p.id, + extendedMetadata: p.extendedMetadata, + } + clusters, err := kubernetesprovider.GetResource(ctx) + if clusters != nil { + finalResources.Merge(clusters) + } + if err != nil { + gologger.Warning().Msgf("digitalocean: could not list kubernetes clusters: %s", err) + } + } + return finalResources, nil } diff --git a/pkg/providers/digitalocean/digitalocean_test.go b/pkg/providers/digitalocean/digitalocean_test.go new file mode 100644 index 00000000..97fed3b0 --- /dev/null +++ b/pkg/providers/digitalocean/digitalocean_test.go @@ -0,0 +1,191 @@ +package digitalocean + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/digitalocean/godo" + "github.com/projectdiscovery/cloudlist/pkg/schema" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newTestProvider(t *testing.T, routes map[string]string, services ...string) *Provider { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + key := r.URL.Path + if page := r.URL.Query().Get("page"); page != "" && page != "1" { + key += "?page=" + page + } + body, ok := routes[key] + if !ok { + t.Errorf("unexpected request: %s", r.URL) + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + + client, err := godo.New(http.DefaultClient, godo.SetBaseURL(server.URL+"/")) + require.NoError(t, err) + + serviceMap := make(schema.ServiceMap) + for _, s := range services { + serviceMap[s] = struct{}{} + } + return &Provider{id: "test", client: client, services: serviceMap, extendedMetadata: true} +} + +func resourcesByValue(resources *schema.Resources) map[string]*schema.Resource { + byValue := make(map[string]*schema.Resource) + for _, r := range resources.Items { + for _, v := range []string{r.DNSName, r.PublicIPv4, r.PublicIPv6} { + if v != "" { + byValue[v] = r + } + } + } + return byValue +} + +func TestReservedIPs(t *testing.T) { + provider := newTestProvider(t, map[string]string{ + "/v2/reserved_ips": `{ + "reserved_ips": [ + {"ip": "45.55.96.47", "region": {"slug": "nyc3"}, "droplet": {"id": 42, "name": "web-1"}, "project_id": "p1"} + ], + "links": {"pages": {"next": "http://example.com/v2/reserved_ips?page=2", "last": "http://example.com/v2/reserved_ips?page=2"}}, + "meta": {"total": 2} + }`, + "/v2/reserved_ips?page=2": `{ + "reserved_ips": [{"ip": "45.55.96.48", "region": {"slug": "nyc3"}, "droplet": null}], + "links": {"pages": {"first": "http://example.com/v2/reserved_ips?page=1", "prev": "http://example.com/v2/reserved_ips?page=1"}}, + "meta": {"total": 2} + }`, + "/v2/reserved_ipv6": `{ + "reserved_ipv6s": [{"ip": "2604:a880:800:14::42c3:d000", "region_slug": "nyc3", "reserved_at": "2024-01-01T00:00:00Z"}], + "links": {}, + "meta": {"total": 1} + }`, + }, "reservedip") + + resources, err := provider.Resources(context.Background()) + require.NoError(t, err) + byValue := resourcesByValue(resources) + require.Len(t, byValue, 3) + + attached := byValue["45.55.96.47"] + require.NotNil(t, attached) + assert.Equal(t, "reservedip", attached.Service) + assert.Equal(t, "42", attached.Metadata["droplet_id"]) + + unassigned := byValue["45.55.96.48"] + require.NotNil(t, unassigned, "second page must be fetched") + assert.NotContains(t, unassigned.Metadata, "droplet_id") + + ipv6 := byValue["2604:a880:800:14::42c3:d000"] + require.NotNil(t, ipv6) + assert.Equal(t, "nyc3", ipv6.Metadata["region_slug"]) + assert.Equal(t, "2024-01-01T00:00:00Z", ipv6.Metadata["reserved_at"]) +} + +func TestLoadBalancers(t *testing.T) { + provider := newTestProvider(t, map[string]string{ + "/v2/load_balancers": `{ + "load_balancers": [ + {"id": "lb-1", "name": "regional", "ip": "104.131.186.241", "ipv6": "2604:a880:400:d1::90c:6001", "type": "REGIONAL", "region": {"slug": "nyc1"}, "droplet_ids": [1, 2]}, + {"id": "lb-2", "name": "global", "type": "GLOBAL", "domains": [{"name": "app.example.com", "is_managed": true}]} + ], + "links": {}, + "meta": {"total": 2} + }`, + }, "loadbalancer") + + resources, err := provider.Resources(context.Background()) + require.NoError(t, err) + byValue := resourcesByValue(resources) + require.Len(t, byValue, 3) + + regional := byValue["104.131.186.241"] + require.NotNil(t, regional) + assert.Equal(t, "loadbalancer", regional.Service) + assert.Equal(t, "2", regional.Metadata["droplet_count"]) + require.NotNil(t, byValue["2604:a880:400:d1::90c:6001"]) + + global := byValue["app.example.com"] + require.NotNil(t, global) + assert.Equal(t, "GLOBAL", global.Metadata["type"]) +} + +func TestKubernetesClusters(t *testing.T) { + provider := newTestProvider(t, map[string]string{ + "/v2/kubernetes/clusters": `{ + "kubernetes_clusters": [ + {"id": "bd5f5959", "name": "prod", "region": "nyc1", "version": "1.31.1-do.0", "ipv4": "68.183.121.157", "endpoint": "https://bd5f5959.k8s.ondigitalocean.com", "status": {"state": "running"}} + ], + "links": {}, + "meta": {"total": 1} + }`, + }, "kubernetes") + + resources, err := provider.Resources(context.Background()) + require.NoError(t, err) + byValue := resourcesByValue(resources) + require.Len(t, byValue, 2) + + host := byValue["bd5f5959.k8s.ondigitalocean.com"] + require.NotNil(t, host) + assert.Equal(t, "kubernetes", host.Service) + assert.Equal(t, "running", host.Metadata["status"]) + require.NotNil(t, byValue["68.183.121.157"]) +} + +func TestResourcesSkipsUnreadableNewServices(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v2/load_balancers" { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"load_balancers": [{"id": "lb-1", "ip": "159.203.150.1"}], "links": {}, "meta": {"total": 1}}`)) + return + } + // Simulates a custom-scoped token without reserved_ip:read or kubernetes:read. + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"id": "forbidden", "message": "You are not authorized to perform this operation"}`)) + })) + t.Cleanup(server.Close) + + client, err := godo.New(http.DefaultClient, godo.SetBaseURL(server.URL+"/")) + require.NoError(t, err) + provider := &Provider{id: "test", client: client, services: schema.ServiceMap{ + "reservedip": struct{}{}, "loadbalancer": struct{}{}, "kubernetes": struct{}{}, + }} + + resources, err := provider.Resources(context.Background()) + require.NoError(t, err) + assert.Contains(t, resourcesByValue(resources), "159.203.150.1") +} + +func TestReservedIPv4KeptWhenIPv6Fails(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.URL.Path == "/v2/reserved_ips" { + _, _ = w.Write([]byte(`{"reserved_ips":[{"ip":"45.55.96.47"}],"links":{},"meta":{"total":1}}`)) + return + } + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"id":"forbidden","message":"denied"}`)) + })) + t.Cleanup(server.Close) + + client, err := godo.New(http.DefaultClient, godo.SetBaseURL(server.URL+"/")) + require.NoError(t, err) + provider := &Provider{id: "test", client: client, services: schema.ServiceMap{"reservedip": {}}} + + resources, err := provider.Resources(context.Background()) + require.NoError(t, err) + assert.Contains(t, resourcesByValue(resources), "45.55.96.47") +} diff --git a/pkg/providers/digitalocean/kubernetes.go b/pkg/providers/digitalocean/kubernetes.go new file mode 100644 index 00000000..b419f447 --- /dev/null +++ b/pkg/providers/digitalocean/kubernetes.go @@ -0,0 +1,99 @@ +package digitalocean + +import ( + "context" + "net/url" + "strings" + "time" + + "github.com/digitalocean/godo" + "github.com/projectdiscovery/cloudlist/pkg/schema" +) + +// kubernetesProvider is a DOKS cluster provider for digitalocean API +type kubernetesProvider struct { + id string + client *godo.Client + extendedMetadata bool +} + +func (d *kubernetesProvider) name() string { + return "kubernetes" +} + +// GetResource returns the control plane endpoints of all DOKS clusters. +func (d *kubernetesProvider) GetResource(ctx context.Context) (*schema.Resources, error) { + opt := &godo.ListOptions{PerPage: 200} + list := schema.NewResources() + + for { + clusters, resp, err := d.client.Kubernetes.List(ctx, opt) + if err != nil { + return list, err + } + + for _, cluster := range clusters { + if cluster == nil { + continue + } + + var metadata map[string]string + if d.extendedMetadata { + metadata = d.getClusterMetadata(cluster) + } + + // Endpoint is a URL (https://.k8s.ondigitalocean.com), not a hostname. + var hostname string + if u, err := url.Parse(cluster.Endpoint); err == nil { + hostname = u.Hostname() + } + + list.Append(&schema.Resource{ + Provider: providerName, + ID: d.id, + DNSName: hostname, + PublicIPv4: cluster.IPv4, + Public: true, + Service: d.name(), + Metadata: metadata, + }) + } + if resp.Links == nil || resp.Links.IsLastPage() { + break + } + + page, err := resp.Links.CurrentPage() + if err != nil { + return list, err + } + opt.Page = page + 1 + } + return list, nil +} + +func (d *kubernetesProvider) getClusterMetadata(cluster *godo.KubernetesCluster) map[string]string { + metadata := make(map[string]string) + + schema.AddMetadata(metadata, "cluster_id", &cluster.ID) + schema.AddMetadata(metadata, "name", &cluster.Name) + schema.AddMetadata(metadata, "region_slug", &cluster.RegionSlug) + schema.AddMetadata(metadata, "version", &cluster.VersionSlug) + schema.AddMetadata(metadata, "endpoint", &cluster.Endpoint) + schema.AddMetadata(metadata, "vpc_uuid", &cluster.VPCUUID) + + if cluster.Status != nil { + state := string(cluster.Status.State) + schema.AddMetadata(metadata, "status", &state) + } + if len(cluster.Tags) > 0 { + tags := strings.Join(cluster.Tags, ",") + schema.AddMetadata(metadata, "tags", &tags) + } + schema.AddMetadataInt(metadata, "node_pool_count", len(cluster.NodePools)) + if !cluster.CreatedAt.IsZero() { + createdAt := cluster.CreatedAt.Format(time.RFC3339) + schema.AddMetadata(metadata, "created_at", &createdAt) + } + + return metadata +} diff --git a/pkg/providers/digitalocean/loadbalancers.go b/pkg/providers/digitalocean/loadbalancers.go new file mode 100644 index 00000000..19e331c3 --- /dev/null +++ b/pkg/providers/digitalocean/loadbalancers.go @@ -0,0 +1,98 @@ +package digitalocean + +import ( + "context" + "strings" + + "github.com/digitalocean/godo" + "github.com/projectdiscovery/cloudlist/pkg/schema" +) + +// loadBalancerProvider is a load balancer provider for digitalocean API +type loadBalancerProvider struct { + id string + client *godo.Client + extendedMetadata bool +} + +func (d *loadBalancerProvider) name() string { + return "loadbalancer" +} + +// GetResource returns all the load balancer IPs and domains for a provider. +func (d *loadBalancerProvider) GetResource(ctx context.Context) (*schema.Resources, error) { + opt := &godo.ListOptions{PerPage: 200} + list := schema.NewResources() + + for { + lbs, resp, err := d.client.LoadBalancers.List(ctx, opt) + if err != nil { + return list, err + } + + for _, lb := range lbs { + var metadata map[string]string + if d.extendedMetadata { + metadata = d.getLoadBalancerMetadata(&lb) + } + + list.Append(&schema.Resource{ + Provider: providerName, + ID: d.id, + PublicIPv4: lb.IP, + PublicIPv6: lb.IPv6, + Public: true, + Service: d.name(), + Metadata: metadata, + }) + // Global load balancers are reached through their domains, not an IP. + for _, domain := range lb.Domains { + if domain == nil { + continue + } + list.Append(&schema.Resource{ + Provider: providerName, + ID: d.id, + DNSName: domain.Name, + Public: true, + Service: d.name(), + Metadata: metadata, + }) + } + } + if resp.Links == nil || resp.Links.IsLastPage() { + break + } + + page, err := resp.Links.CurrentPage() + if err != nil { + return list, err + } + opt.Page = page + 1 + } + return list, nil +} + +func (d *loadBalancerProvider) getLoadBalancerMetadata(lb *godo.LoadBalancer) map[string]string { + metadata := make(map[string]string) + + schema.AddMetadata(metadata, "load_balancer_id", &lb.ID) + schema.AddMetadata(metadata, "name", &lb.Name) + schema.AddMetadata(metadata, "type", &lb.Type) + schema.AddMetadata(metadata, "network", &lb.Network) + schema.AddMetadata(metadata, "status", &lb.Status) + schema.AddMetadata(metadata, "created_at", &lb.Created) + schema.AddMetadata(metadata, "vpc_uuid", &lb.VPCUUID) + schema.AddMetadata(metadata, "project_id", &lb.ProjectID) + + if lb.Region != nil { + schema.AddMetadata(metadata, "region_slug", &lb.Region.Slug) + } + if len(lb.Tags) > 0 { + tags := strings.Join(lb.Tags, ",") + schema.AddMetadata(metadata, "tags", &tags) + } + schema.AddMetadataInt(metadata, "droplet_count", len(lb.DropletIDs)) + + return metadata +} diff --git a/pkg/providers/digitalocean/reservedips.go b/pkg/providers/digitalocean/reservedips.go new file mode 100644 index 00000000..bd7f5049 --- /dev/null +++ b/pkg/providers/digitalocean/reservedips.go @@ -0,0 +1,115 @@ +package digitalocean + +import ( + "context" + "fmt" + "time" + + "github.com/digitalocean/godo" + "github.com/projectdiscovery/cloudlist/pkg/schema" +) + +// reservedIPProvider is a reserved IP provider for digitalocean API +type reservedIPProvider struct { + id string + client *godo.Client + extendedMetadata bool +} + +func (d *reservedIPProvider) name() string { + return "reservedip" +} + +// GetResource returns all the reserved IPv4 and IPv6 addresses, including +// unassigned ones, which the droplet listing never returns. +func (d *reservedIPProvider) GetResource(ctx context.Context) (*schema.Resources, error) { + list := schema.NewResources() + + opt := &godo.ListOptions{PerPage: 200} + for { + ips, resp, err := d.client.ReservedIPs.List(ctx, opt) + if err != nil { + return list, err + } + + for _, ip := range ips { + var metadata map[string]string + if d.extendedMetadata { + metadata = getReservedIPMetadata(ip.Region, ip.Droplet) + schema.AddMetadata(metadata, "project_id", &ip.ProjectID) + locked := fmt.Sprintf("%v", ip.Locked) + schema.AddMetadata(metadata, "locked", &locked) + } + + list.Append(&schema.Resource{ + Provider: providerName, + ID: d.id, + PublicIPv4: ip.IP, + Public: true, + Service: d.name(), + Metadata: metadata, + }) + } + if resp.Links == nil || resp.Links.IsLastPage() { + break + } + + page, err := resp.Links.CurrentPage() + if err != nil { + return list, err + } + opt.Page = page + 1 + } + + opt = &godo.ListOptions{PerPage: 200} + for { + ips, resp, err := d.client.ReservedIPV6s.List(ctx, opt) + if err != nil { + return list, err + } + + for _, ip := range ips { + var metadata map[string]string + if d.extendedMetadata { + metadata = getReservedIPMetadata(&godo.Region{Slug: ip.RegionSlug}, ip.Droplet) + if !ip.ReservedAt.IsZero() { + reservedAt := ip.ReservedAt.Format(time.RFC3339) + schema.AddMetadata(metadata, "reserved_at", &reservedAt) + } + } + + list.Append(&schema.Resource{ + Provider: providerName, + ID: d.id, + PublicIPv6: ip.IP, + Public: true, + Service: d.name(), + Metadata: metadata, + }) + } + if resp.Links == nil || resp.Links.IsLastPage() { + break + } + + page, err := resp.Links.CurrentPage() + if err != nil { + return list, err + } + opt.Page = page + 1 + } + return list, nil +} + +func getReservedIPMetadata(region *godo.Region, droplet *godo.Droplet) map[string]string { + metadata := make(map[string]string) + + if region != nil { + schema.AddMetadata(metadata, "region_slug", ®ion.Slug) + } + if droplet != nil { + dropletID := fmt.Sprintf("%d", droplet.ID) + schema.AddMetadata(metadata, "droplet_id", &dropletID) + schema.AddMetadata(metadata, "droplet_name", &droplet.Name) + } + return metadata +}