From 4070dc069e6604dee855562545b0582898deea23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 8 Oct 2026 16:36:36 +0300 Subject: [PATCH 1/3] gcp: add addresses and forwarding rules to compute --- pkg/providers/gcp/addresses.go | 173 ++++++++++++++++++++++++++++ pkg/providers/gcp/addresses_test.go | 83 +++++++++++++ pkg/providers/gcp/gcp.go | 2 +- pkg/providers/gcp/vms.go | 2 + 4 files changed, 259 insertions(+), 1 deletion(-) create mode 100644 pkg/providers/gcp/addresses.go create mode 100644 pkg/providers/gcp/addresses_test.go diff --git a/pkg/providers/gcp/addresses.go b/pkg/providers/gcp/addresses.go new file mode 100644 index 00000000..11015b30 --- /dev/null +++ b/pkg/providers/gcp/addresses.go @@ -0,0 +1,173 @@ +package gcp + +import ( + "context" + "fmt" + "log" + "strings" + + "github.com/projectdiscovery/cloudlist/pkg/schema" + "google.golang.org/api/compute/v1" +) + +// getAddressResources lists reserved addresses and forwarding rules (load +// balancer frontends), which are not attached to instances and so are missed +// by the instance listing. Internal IPs are classified as private on Append. +func (d *cloudVMProvider) getAddressResources(ctx context.Context, project string) *schema.Resources { + list := schema.NewResources() + + appendAddress := func(address *compute.Address) { + if address == nil || address.Address == "" { + return + } + var metadata map[string]string + if d.extendedMetadata { + metadata = getAddressMetadata(address, project) + } + list.Append(&schema.Resource{ + ID: d.id, + Public: true, + Provider: providerName, + PublicIPv4: address.Address, + Service: d.name(), + Metadata: metadata, + }) + } + + appendForwardingRule := func(rule *compute.ForwardingRule) { + if rule == nil || rule.IPAddress == "" { + return + } + var metadata map[string]string + if d.extendedMetadata { + metadata = getForwardingRuleMetadata(rule, project) + } + list.Append(&schema.Resource{ + ID: d.id, + Public: true, + Provider: providerName, + PublicIPv4: rule.IPAddress, + Service: d.name(), + Metadata: metadata, + }) + } + + err := d.compute.Addresses.AggregatedList(project).Pages(ctx, func(resp *compute.AddressAggregatedList) error { + for _, scoped := range resp.Items { + for _, address := range scoped.Addresses { + appendAddress(address) + } + } + return nil + }) + if err != nil { + log.Printf("Could not get all addresses for project %s: %s\n", project, err) + } + + err = d.compute.GlobalAddresses.List(project).Pages(ctx, func(resp *compute.AddressList) error { + for _, address := range resp.Items { + appendAddress(address) + } + return nil + }) + if err != nil { + log.Printf("Could not get all global addresses for project %s: %s\n", project, err) + } + + err = d.compute.ForwardingRules.AggregatedList(project).Pages(ctx, func(resp *compute.ForwardingRuleAggregatedList) error { + for _, scoped := range resp.Items { + for _, rule := range scoped.ForwardingRules { + appendForwardingRule(rule) + } + } + return nil + }) + if err != nil { + log.Printf("Could not get all forwarding rules for project %s: %s\n", project, err) + } + + err = d.compute.GlobalForwardingRules.List(project).Pages(ctx, func(resp *compute.ForwardingRuleList) error { + for _, rule := range resp.Items { + appendForwardingRule(rule) + } + return nil + }) + if err != nil { + log.Printf("Could not get all global forwarding rules for project %s: %s\n", project, err) + } + + return list +} + +func getAddressMetadata(address *compute.Address, project string) map[string]string { + metadata := make(map[string]string) + + schema.AddMetadata(metadata, "address_name", &address.Name) + if address.Id != 0 { + metadata["address_id"] = fmt.Sprintf("%d", address.Id) + } + schema.AddMetadata(metadata, "address_type", &address.AddressType) + schema.AddMetadata(metadata, "status", &address.Status) + schema.AddMetadata(metadata, "purpose", &address.Purpose) + schema.AddMetadata(metadata, "network_tier", &address.NetworkTier) + schema.AddMetadata(metadata, "ip_version", &address.IpVersion) + region := extractResourceName(address.Region) + schema.AddMetadata(metadata, "region", ®ion) + schema.AddMetadata(metadata, "creation_timestamp", &address.CreationTimestamp) + schema.AddMetadata(metadata, "description", &address.Description) + + if len(address.Users) > 0 { + users := make([]string, 0, len(address.Users)) + for _, user := range address.Users { + users = append(users, extractResourceName(user)) + } + metadata["users"] = strings.Join(users, ",") + } + if len(address.Labels) > 0 { + metadata["labels"] = joinLabels(address.Labels) + } + + metadata["project_id"] = project + metadata["owner_id"] = project + return metadata +} + +func getForwardingRuleMetadata(rule *compute.ForwardingRule, project string) map[string]string { + metadata := make(map[string]string) + + schema.AddMetadata(metadata, "forwarding_rule_name", &rule.Name) + if rule.Id != 0 { + metadata["forwarding_rule_id"] = fmt.Sprintf("%d", rule.Id) + } + schema.AddMetadata(metadata, "load_balancing_scheme", &rule.LoadBalancingScheme) + schema.AddMetadata(metadata, "ip_protocol", &rule.IPProtocol) + schema.AddMetadata(metadata, "port_range", &rule.PortRange) + if len(rule.Ports) > 0 { + metadata["ports"] = strings.Join(rule.Ports, ",") + } + schema.AddMetadata(metadata, "network_tier", &rule.NetworkTier) + target := extractResourceName(rule.Target) + schema.AddMetadata(metadata, "target", &target) + backendService := extractResourceName(rule.BackendService) + schema.AddMetadata(metadata, "backend_service", &backendService) + region := extractResourceName(rule.Region) + schema.AddMetadata(metadata, "region", ®ion) + schema.AddMetadata(metadata, "creation_timestamp", &rule.CreationTimestamp) + schema.AddMetadata(metadata, "description", &rule.Description) + + if len(rule.Labels) > 0 { + metadata["labels"] = joinLabels(rule.Labels) + } + + metadata["project_id"] = project + metadata["owner_id"] = project + return metadata +} + +func joinLabels(labels map[string]string) string { + pairs := make([]string, 0, len(labels)) + for key, value := range labels { + pairs = append(pairs, fmt.Sprintf("%s=%s", key, value)) + } + return strings.Join(pairs, ",") +} diff --git a/pkg/providers/gcp/addresses_test.go b/pkg/providers/gcp/addresses_test.go new file mode 100644 index 00000000..728c990b --- /dev/null +++ b/pkg/providers/gcp/addresses_test.go @@ -0,0 +1,83 @@ +package gcp + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/api/compute/v1" + "google.golang.org/api/option" +) + +var fakeComputeResponses = map[string]string{ + "/projects/p1/aggregated/instances": `{"items":{"zones/us-central1-a":{"instances":[ + {"name":"vm-1","networkInterfaces":[{"accessConfigs":[{"natIP":"34.10.0.1"}]}]}]}}}`, + "/projects/p1/aggregated/addresses": `{"items":{ + "regions/us-central1":{"addresses":[ + {"name":"vm-1-ip","address":"34.10.0.1","addressType":"EXTERNAL","status":"IN_USE"}, + {"name":"reserved","address":"34.10.0.2","addressType":"EXTERNAL","status":"RESERVED","region":"https://www.googleapis.com/compute/v1/projects/p1/regions/us-central1"}, + {"name":"internal","address":"10.128.0.5","addressType":"INTERNAL","status":"RESERVED"}]}, + "regions/europe-west1":{"warning":{"code":"NO_RESULTS_ON_PAGE"}}}}`, + "/projects/p1/global/addresses": `{"items":[{"name":"global-ip","address":"34.120.0.3","addressType":"EXTERNAL"}]}`, + "/projects/p1/aggregated/forwardingRules": `{"items":{"regions/us-central1":{"forwardingRules":[ + {"name":"nlb","IPAddress":"35.200.0.4","loadBalancingScheme":"EXTERNAL","IPProtocol":"TCP","portRange":"443-443","target":"https://www.googleapis.com/compute/v1/projects/p1/regions/us-central1/targetPools/pool-1"}]}}}`, + "/projects/p1/global/forwardingRules": `{"items":[{"name":"https-lb","IPAddress":"34.120.0.3","loadBalancingScheme":"EXTERNAL_MANAGED"}]}`, +} + +func newFakeComputeService(t *testing.T, responses map[string]string) *compute.Service { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, ok := responses[strings.TrimPrefix(r.URL.Path, "/compute/v1")] + if !ok { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + + svc, err := compute.NewService(context.Background(), + option.WithEndpoint(server.URL+"/compute/v1/"), + option.WithoutAuthentication(), + option.WithHTTPClient(server.Client()), + ) + require.NoError(t, err) + return svc +} + +func TestCloudVMProvider_IncludesAddressesAndForwardingRules(t *testing.T) { + provider := &cloudVMProvider{ + id: "test", + compute: newFakeComputeService(t, fakeComputeResponses), + projects: []string{"p1"}, + extendedMetadata: true, + } + + resources, err := provider.GetResource(context.Background()) + require.NoError(t, err) + + byIP := make(map[string]map[string]string) + var private []string + for _, r := range resources.Items { + switch { + case r.PublicIPv4 != "": + byIP[r.PublicIPv4] = r.Metadata + case r.PrivateIpv4 != "": + private = append(private, r.PrivateIpv4) + } + } + + assert.Len(t, byIP, 4) + assert.Equal(t, "vm-1", byIP["34.10.0.1"]["instance_name"], "in-use address must keep the instance entry") + assert.Equal(t, "RESERVED", byIP["34.10.0.2"]["status"]) + assert.Equal(t, "us-central1", byIP["34.10.0.2"]["region"]) + assert.Equal(t, "global-ip", byIP["34.120.0.3"]["address_name"], "global address and global forwarding rule share an IP") + assert.Equal(t, "nlb", byIP["35.200.0.4"]["forwarding_rule_name"]) + assert.Equal(t, "pool-1", byIP["35.200.0.4"]["target"]) + assert.Equal(t, []string{"10.128.0.5"}, private, "internal addresses are classified as private") +} diff --git a/pkg/providers/gcp/gcp.go b/pkg/providers/gcp/gcp.go index 5f26e557..84b34a97 100644 --- a/pkg/providers/gcp/gcp.go +++ b/pkg/providers/gcp/gcp.go @@ -66,7 +66,7 @@ type OrganizationProvider struct { // Services that provide IP addresses or DNS names only var Services = []string{ "dns", // DNS names, IPv4/IPv6 addresses from DNS records - "compute", // IPv4/IPv6 addresses from VM instances + "compute", // IPv4/IPv6 addresses from VM instances, reserved addresses and forwarding rules "gke", // DNS names and IPs from Kubernetes ingresses "cloud-function", // DNS names from function HTTPS URLs "cloud-run", // DNS names from service URLs diff --git a/pkg/providers/gcp/vms.go b/pkg/providers/gcp/vms.go index 79003e58..48a2a4db 100644 --- a/pkg/providers/gcp/vms.go +++ b/pkg/providers/gcp/vms.go @@ -63,6 +63,8 @@ func (d *cloudVMProvider) GetResource(ctx context.Context) (*schema.Resources, e log.Printf("Could not get all instances for project %s: %s\n", project, err) continue } + // Merged after instances so an in-use address keeps its instance metadata on dedup. + list.Merge(d.getAddressResources(ctx, project)) } return list, nil } From 90a857cd7e9b15d5d3e4a7b9f355325689423477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 8 Oct 2026 18:03:58 +0300 Subject: [PATCH 2/3] gcp: sort address labels --- pkg/providers/gcp/addresses.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/providers/gcp/addresses.go b/pkg/providers/gcp/addresses.go index 11015b30..035d49f5 100644 --- a/pkg/providers/gcp/addresses.go +++ b/pkg/providers/gcp/addresses.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log" + "sort" "strings" "github.com/projectdiscovery/cloudlist/pkg/schema" @@ -169,5 +170,6 @@ func joinLabels(labels map[string]string) string { for key, value := range labels { pairs = append(pairs, fmt.Sprintf("%s=%s", key, value)) } + sort.Strings(pairs) return strings.Join(pairs, ",") } From fec4f747f61a257acf2c5be2ef4e23b96c709e61 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Fri, 9 Oct 2026 00:01:22 +0200 Subject: [PATCH 3/3] keep addresses --- pkg/providers/gcp/addresses_test.go | 25 +++++++++++++++++++++++++ pkg/providers/gcp/vms.go | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/pkg/providers/gcp/addresses_test.go b/pkg/providers/gcp/addresses_test.go index 728c990b..0b29da58 100644 --- a/pkg/providers/gcp/addresses_test.go +++ b/pkg/providers/gcp/addresses_test.go @@ -81,3 +81,28 @@ func TestCloudVMProvider_IncludesAddressesAndForwardingRules(t *testing.T) { assert.Equal(t, "pool-1", byIP["35.200.0.4"]["target"]) assert.Equal(t, []string{"10.128.0.5"}, private, "internal addresses are classified as private") } + +func TestCloudVMProvider_KeepsAddressesWhenInstancesFail(t *testing.T) { + responses := make(map[string]string, len(fakeComputeResponses)) + for path, body := range fakeComputeResponses { + responses[path] = body + } + delete(responses, "/projects/p1/aggregated/instances") + + provider := &cloudVMProvider{ + id: "test", + compute: newFakeComputeService(t, responses), + projects: []string{"p1"}, + } + + resources, err := provider.GetResource(context.Background()) + require.NoError(t, err) + + var public []string + for _, item := range resources.Items { + if item.PublicIPv4 != "" { + public = append(public, item.PublicIPv4) + } + } + assert.ElementsMatch(t, []string{"34.10.0.1", "34.10.0.2", "34.120.0.3", "35.200.0.4"}, public) +} diff --git a/pkg/providers/gcp/vms.go b/pkg/providers/gcp/vms.go index 48a2a4db..a7270d0c 100644 --- a/pkg/providers/gcp/vms.go +++ b/pkg/providers/gcp/vms.go @@ -61,9 +61,9 @@ func (d *cloudVMProvider) GetResource(ctx context.Context) (*schema.Resources, e }) if err != nil { log.Printf("Could not get all instances for project %s: %s\n", project, err) - continue } // Merged after instances so an in-use address keeps its instance metadata on dedup. + // Still collected when the instance listing fails. list.Merge(d.getAddressResources(ctx, project)) } return list, nil