feat(gcp): add addresses and forwarding rules to compute

pkg/providers/gcp/addresses.go

@@ -0,0 +1,175 @@
+package gcp
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"sort"
+	"strings"
+
+	"github.com/projectdiscovery/cloudlist/pkg/schema"
+	"google.golang.org/api/compute/v1"
+)
+
+// getAddressResources lists reserved addresses and forwarding rules (load
+// balancer frontends), which are not attached to instances and so are missed
+// by the instance listing. Internal IPs are classified as private on Append.
+func (d *cloudVMProvider) getAddressResources(ctx context.Context, project string) *schema.Resources {
+	list := schema.NewResources()
+
+	appendAddress := func(address *compute.Address) {
+		if address == nil || address.Address == "" {
+			return
+		}
+		var metadata map[string]string
+		if d.extendedMetadata {
+			metadata = getAddressMetadata(address, project)
+		}
+		list.Append(&schema.Resource{
+			ID:         d.id,
+			Public:     true,
+			Provider:   providerName,
+			PublicIPv4: address.Address,
+			Service:    d.name(),
+			Metadata:   metadata,
+		})
+	}
+
+	appendForwardingRule := func(rule *compute.ForwardingRule) {
+		if rule == nil || rule.IPAddress == "" {
+			return
+		}
+		var metadata map[string]string
+		if d.extendedMetadata {
+			metadata = getForwardingRuleMetadata(rule, project)
+		}
+		list.Append(&schema.Resource{
+			ID:         d.id,
+			Public:     true,
+			Provider:   providerName,
+			PublicIPv4: rule.IPAddress,
+			Service:    d.name(),
+			Metadata:   metadata,
+		})
+	}
+
+	err := d.compute.Addresses.AggregatedList(project).Pages(ctx, func(resp *compute.AddressAggregatedList) error {
+		for _, scoped := range resp.Items {
+			for _, address := range scoped.Addresses {
+				appendAddress(address)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		log.Printf("Could not get all addresses for project %s: %s\n", project, err)
+	}
+
+	err = d.compute.GlobalAddresses.List(project).Pages(ctx, func(resp *compute.AddressList) error {
+		for _, address := range resp.Items {
+			appendAddress(address)
+		}
+		return nil
+	})
+	if err != nil {
+		log.Printf("Could not get all global addresses for project %s: %s\n", project, err)
+	}
+
+	err = d.compute.ForwardingRules.AggregatedList(project).Pages(ctx, func(resp *compute.ForwardingRuleAggregatedList) error {
+		for _, scoped := range resp.Items {
+			for _, rule := range scoped.ForwardingRules {
+				appendForwardingRule(rule)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		log.Printf("Could not get all forwarding rules for project %s: %s\n", project, err)
+	}
+
+	err = d.compute.GlobalForwardingRules.List(project).Pages(ctx, func(resp *compute.ForwardingRuleList) error {
+		for _, rule := range resp.Items {
+			appendForwardingRule(rule)
+		}
+		return nil
+	})
+	if err != nil {
+		log.Printf("Could not get all global forwarding rules for project %s: %s\n", project, err)
+	}
+
+	return list
+}
+
+func getAddressMetadata(address *compute.Address, project string) map[string]string {
+	metadata := make(map[string]string)
+
+	schema.AddMetadata(metadata, "address_name", &address.Name)
+	if address.Id != 0 {
+		metadata["address_id"] = fmt.Sprintf("%d", address.Id)
+	}
+	schema.AddMetadata(metadata, "address_type", &address.AddressType)
+	schema.AddMetadata(metadata, "status", &address.Status)
+	schema.AddMetadata(metadata, "purpose", &address.Purpose)
+	schema.AddMetadata(metadata, "network_tier", &address.NetworkTier)
+	schema.AddMetadata(metadata, "ip_version", &address.IpVersion)
+	region := extractResourceName(address.Region)
+	schema.AddMetadata(metadata, "region", &region)
+	schema.AddMetadata(metadata, "creation_timestamp", &address.CreationTimestamp)
+	schema.AddMetadata(metadata, "description", &address.Description)
+
+	if len(address.Users) > 0 {
+		users := make([]string, 0, len(address.Users))
+		for _, user := range address.Users {
+			users = append(users, extractResourceName(user))
+		}
+		metadata["users"] = strings.Join(users, ",")
+	}
+	if len(address.Labels) > 0 {
+		metadata["labels"] = joinLabels(address.Labels)
+	}
+
+	metadata["project_id"] = project
+	metadata["owner_id"] = project
+	return metadata
+}
+
+func getForwardingRuleMetadata(rule *compute.ForwardingRule, project string) map[string]string {
+	metadata := make(map[string]string)
+
+	schema.AddMetadata(metadata, "forwarding_rule_name", &rule.Name)
+	if rule.Id != 0 {
+		metadata["forwarding_rule_id"] = fmt.Sprintf("%d", rule.Id)
+	}
+	schema.AddMetadata(metadata, "load_balancing_scheme", &rule.LoadBalancingScheme)
+	schema.AddMetadata(metadata, "ip_protocol", &rule.IPProtocol)
+	schema.AddMetadata(metadata, "port_range", &rule.PortRange)
+	if len(rule.Ports) > 0 {
+		metadata["ports"] = strings.Join(rule.Ports, ",")
+	}
+	schema.AddMetadata(metadata, "network_tier", &rule.NetworkTier)
+	target := extractResourceName(rule.Target)
+	schema.AddMetadata(metadata, "target", &target)
+	backendService := extractResourceName(rule.BackendService)
+	schema.AddMetadata(metadata, "backend_service", &backendService)
+	region := extractResourceName(rule.Region)
+	schema.AddMetadata(metadata, "region", &region)
+	schema.AddMetadata(metadata, "creation_timestamp", &rule.CreationTimestamp)
+	schema.AddMetadata(metadata, "description", &rule.Description)
+
+	if len(rule.Labels) > 0 {
+		metadata["labels"] = joinLabels(rule.Labels)
+	}
+
+	metadata["project_id"] = project
+	metadata["owner_id"] = project
+	return metadata
+}
+
+func joinLabels(labels map[string]string) string {
+	pairs := make([]string, 0, len(labels))
+	for key, value := range labels {
+		pairs = append(pairs, fmt.Sprintf("%s=%s", key, value))
+	}
+	sort.Strings(pairs)
+	return strings.Join(pairs, ",")
+}

pkg/providers/gcp/addresses_test.go

@@ -0,0 +1,108 @@
+package gcp
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/api/compute/v1"
+	"google.golang.org/api/option"
+)
+
+var fakeComputeResponses = map[string]string{
+	"/projects/p1/aggregated/instances": `{"items":{"zones/us-central1-a":{"instances":[
+			{"name":"vm-1","networkInterfaces":[{"accessConfigs":[{"natIP":"34.10.0.1"}]}]}]}}}`,
+	"/projects/p1/aggregated/addresses": `{"items":{
+			"regions/us-central1":{"addresses":[
+				{"name":"vm-1-ip","address":"34.10.0.1","addressType":"EXTERNAL","status":"IN_USE"},
+				{"name":"reserved","address":"34.10.0.2","addressType":"EXTERNAL","status":"RESERVED","region":"https://www.googleapis.com/compute/v1/projects/p1/regions/us-central1"},
+				{"name":"internal","address":"10.128.0.5","addressType":"INTERNAL","status":"RESERVED"}]},
+			"regions/europe-west1":{"warning":{"code":"NO_RESULTS_ON_PAGE"}}}}`,
+	"/projects/p1/global/addresses": `{"items":[{"name":"global-ip","address":"34.120.0.3","addressType":"EXTERNAL"}]}`,
+	"/projects/p1/aggregated/forwardingRules": `{"items":{"regions/us-central1":{"forwardingRules":[
+			{"name":"nlb","IPAddress":"35.200.0.4","loadBalancingScheme":"EXTERNAL","IPProtocol":"TCP","portRange":"443-443","target":"https://www.googleapis.com/compute/v1/projects/p1/regions/us-central1/targetPools/pool-1"}]}}}`,
+	"/projects/p1/global/forwardingRules": `{"items":[{"name":"https-lb","IPAddress":"34.120.0.3","loadBalancingScheme":"EXTERNAL_MANAGED"}]}`,
+}
+
+func newFakeComputeService(t *testing.T, responses map[string]string) *compute.Service {
+	t.Helper()
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, ok := responses[strings.TrimPrefix(r.URL.Path, "/compute/v1")]
+		if !ok {
+			http.NotFound(w, r)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(body))
+	}))
+	t.Cleanup(server.Close)
+
+	svc, err := compute.NewService(context.Background(),
+		option.WithEndpoint(server.URL+"/compute/v1/"),
+		option.WithoutAuthentication(),
+		option.WithHTTPClient(server.Client()),
+	)
+	require.NoError(t, err)
+	return svc
+}
+
+func TestCloudVMProvider_IncludesAddressesAndForwardingRules(t *testing.T) {
+	provider := &cloudVMProvider{
+		id:               "test",
+		compute:          newFakeComputeService(t, fakeComputeResponses),
+		projects:         []string{"p1"},
+		extendedMetadata: true,
+	}
+
+	resources, err := provider.GetResource(context.Background())
+	require.NoError(t, err)
+
+	byIP := make(map[string]map[string]string)
+	var private []string
+	for _, r := range resources.Items {
+		switch {
+		case r.PublicIPv4 != "":
+			byIP[r.PublicIPv4] = r.Metadata
+		case r.PrivateIpv4 != "":
+			private = append(private, r.PrivateIpv4)
+		}
+	}
+
+	assert.Len(t, byIP, 4)
+	assert.Equal(t, "vm-1", byIP["34.10.0.1"]["instance_name"], "in-use address must keep the instance entry")
+	assert.Equal(t, "RESERVED", byIP["34.10.0.2"]["status"])
+	assert.Equal(t, "us-central1", byIP["34.10.0.2"]["region"])
+	assert.Equal(t, "global-ip", byIP["34.120.0.3"]["address_name"], "global address and global forwarding rule share an IP")
+	assert.Equal(t, "nlb", byIP["35.200.0.4"]["forwarding_rule_name"])
+	assert.Equal(t, "pool-1", byIP["35.200.0.4"]["target"])
+	assert.Equal(t, []string{"10.128.0.5"}, private, "internal addresses are classified as private")
+}
+
+func TestCloudVMProvider_KeepsAddressesWhenInstancesFail(t *testing.T) {
+	responses := make(map[string]string, len(fakeComputeResponses))
+	for path, body := range fakeComputeResponses {
+		responses[path] = body
+	}
+	delete(responses, "/projects/p1/aggregated/instances")
+
+	provider := &cloudVMProvider{
+		id:       "test",
+		compute:  newFakeComputeService(t, responses),
+		projects: []string{"p1"},
+	}
+
+	resources, err := provider.GetResource(context.Background())
+	require.NoError(t, err)
+
+	var public []string
+	for _, item := range resources.Items {
+		if item.PublicIPv4 != "" {
+			public = append(public, item.PublicIPv4)
+		}
+	}
+	assert.ElementsMatch(t, []string{"34.10.0.1", "34.10.0.2", "34.120.0.3", "35.200.0.4"}, public)
+}

pkg/providers/gcp/gcp.go

@@ -66,7 +66,7 @@ type OrganizationProvider struct {
 // Services that provide IP addresses or DNS names only
 var Services = []string{
 	"dns",            // DNS names, IPv4/IPv6 addresses from DNS records
-	"compute",        // IPv4/IPv6 addresses from VM instances
+	"compute",        // IPv4/IPv6 addresses from VM instances, reserved addresses and forwarding rules
 	"gke",            // DNS names and IPs from Kubernetes ingresses
 	"cloud-function", // DNS names from function HTTPS URLs
 	"cloud-run",      // DNS names from service URLs

pkg/providers/gcp/vms.go

@@ -61,8 +61,10 @@ func (d *cloudVMProvider) GetResource(ctx context.Context) (*schema.Resources, e
 		})
 		if err != nil {
 			log.Printf("Could not get all instances for project %s: %s\n", project, err)
-			continue
 		}
+		// Merged after instances so an in-use address keeps its instance metadata on dedup.
+		// Still collected when the instance listing fails.
+		list.Merge(d.getAddressResources(ctx, project))
 	}
 	return list, nil
 }