From 5d4916336ed8c1595106fd416ace8ca2929495b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 8 Oct 2026 16:36:42 +0300 Subject: [PATCH 1/3] openstack: add floating ips and load balancers --- pkg/providers/openstack/floatingips.go | 51 +++++++++++++++++ pkg/providers/openstack/loadbalancers.go | 52 ++++++++++++++++++ pkg/providers/openstack/openstack.go | 49 ++++++++++++++--- pkg/providers/openstack/openstack_test.go | 67 +++++++++++++++++++++++ 4 files changed, 211 insertions(+), 8 deletions(-) create mode 100644 pkg/providers/openstack/floatingips.go create mode 100644 pkg/providers/openstack/loadbalancers.go create mode 100644 pkg/providers/openstack/openstack_test.go diff --git a/pkg/providers/openstack/floatingips.go b/pkg/providers/openstack/floatingips.go new file mode 100644 index 00000000..27202c01 --- /dev/null +++ b/pkg/providers/openstack/floatingips.go @@ -0,0 +1,51 @@ +package openstack + +import ( + "context" + + "github.com/gophercloud/gophercloud" + "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips" + "github.com/gophercloud/gophercloud/pagination" + "github.com/projectdiscovery/cloudlist/pkg/schema" + "github.com/projectdiscovery/gologger" +) + +// floatingIPProvider lists Neutron floating IPs, including ones not attached +// to an instance, which the compute API never returns. +type floatingIPProvider struct { + id string + client *gophercloud.ServiceClient +} + +func (p *floatingIPProvider) name() string { + return "floatingip" +} + +// GetResource returns all the floating IPs in the store for a provider. +func (p *floatingIPProvider) GetResource(ctx context.Context) (*schema.Resources, error) { + list := schema.NewResources() + + err := floatingips.List(p.client, floatingips.ListOpts{}).EachPage(func(page pagination.Page) (bool, error) { + ips, err := floatingips.ExtractFloatingIPs(page) + if err != nil { + return false, err + } + + for _, ip := range ips { + list.Append(&schema.Resource{ + Provider: providerName, + ID: p.id, + PublicIPv4: ip.FloatingIP, + Service: p.name(), + }) + } + return true, nil + }) + + if err != nil { + gologger.Error().Msgf("Couldn't list Openstack floating IPs: %s\n", err) + return nil, err + } + + return list, nil +} diff --git a/pkg/providers/openstack/loadbalancers.go b/pkg/providers/openstack/loadbalancers.go new file mode 100644 index 00000000..071b38e7 --- /dev/null +++ b/pkg/providers/openstack/loadbalancers.go @@ -0,0 +1,52 @@ +package openstack + +import ( + "context" + + "github.com/gophercloud/gophercloud" + "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/loadbalancers" + "github.com/gophercloud/gophercloud/pagination" + "github.com/projectdiscovery/cloudlist/pkg/schema" + "github.com/projectdiscovery/gologger" +) + +// loadBalancerProvider lists Octavia load balancer VIPs +type loadBalancerProvider struct { + id string + client *gophercloud.ServiceClient +} + +func (p *loadBalancerProvider) name() string { + return "loadbalancer" +} + +// GetResource returns all the load balancers in the store for a provider. +func (p *loadBalancerProvider) GetResource(ctx context.Context) (*schema.Resources, error) { + list := schema.NewResources() + + err := loadbalancers.List(p.client, loadbalancers.ListOpts{}).EachPage(func(page pagination.Page) (bool, error) { + lbs, err := loadbalancers.ExtractLoadBalancers(page) + if err != nil { + return false, err + } + + // The VIP is usually private; a public address in front of it is a + // floating IP and is reported by the floatingip service. + for _, lb := range lbs { + list.Append(&schema.Resource{ + Provider: providerName, + ID: p.id, + PrivateIpv4: lb.VipAddress, + Service: p.name(), + }) + } + return true, nil + }) + + if err != nil { + gologger.Error().Msgf("Couldn't list Openstack load balancers: %s\n", err) + return nil, err + } + + return list, nil +} diff --git a/pkg/providers/openstack/openstack.go b/pkg/providers/openstack/openstack.go index eecb071a..000f2f8f 100644 --- a/pkg/providers/openstack/openstack.go +++ b/pkg/providers/openstack/openstack.go @@ -20,13 +20,15 @@ const ( providerName = "openstack" ) -var Services = []string{"instance"} +var Services = []string{"instance", "floatingip", "loadbalancer"} // Provider is a data provider for Openstack API type Provider struct { - id string - client *gophercloud.ServiceClient - services schema.ServiceMap + id string + client *gophercloud.ServiceClient + network *gophercloud.ServiceClient + loadBalancer *gophercloud.ServiceClient + services schema.ServiceMap } // New creates a new provider client for Openstack API @@ -72,9 +74,8 @@ func New(options schema.OptionBlock) (*Provider, error) { return nil, err } - client, err := openstack.NewComputeV2(provider, gophercloud.EndpointOpts{ - Region: "RegionOne", - }) + endpointOpts := gophercloud.EndpointOpts{Region: "RegionOne"} + client, err := openstack.NewComputeV2(provider, endpointOpts) if err != nil { gologger.Error().Msgf("Couldn't use Openstack region: %s\n", err) @@ -82,7 +83,27 @@ func New(options schema.OptionBlock) (*Provider, error) { } services := options.ResolveServices(Services) - return &Provider{id: id, client: client, services: services}, nil + p := &Provider{id: id, client: client, services: services} + + // Neutron and Octavia are optional on many clouds, so a missing catalog + // entry only disables that service instead of failing the provider. + if services.Has("floatingip") { + p.network = optionalClient(openstack.NewNetworkV2(provider, endpointOpts)) + } + if services.Has("loadbalancer") { + p.loadBalancer = optionalClient(openstack.NewLoadBalancerV2(provider, endpointOpts)) + } + return p, nil +} + +// optionalClient drops the client on error, since gophercloud returns a +// non-nil but unusable client when the endpoint is not in the catalog. +func optionalClient(client *gophercloud.ServiceClient, err error) *gophercloud.ServiceClient { + if err != nil { + gologger.Warning().Msgf("Couldn't use Openstack service: %s\n", err) + return nil + } + return client } // Name returns the name of the provider @@ -109,5 +130,17 @@ func (p *Provider) Resources(ctx context.Context) (*schema.Resources, error) { finalResources.Merge(resources) } } + if p.network != nil { + provider := &floatingIPProvider{id: p.id, client: p.network} + if resources, err := provider.GetResource(ctx); err == nil { + finalResources.Merge(resources) + } + } + if p.loadBalancer != nil { + provider := &loadBalancerProvider{id: p.id, client: p.loadBalancer} + if resources, err := provider.GetResource(ctx); err == nil { + finalResources.Merge(resources) + } + } return finalResources, nil } diff --git a/pkg/providers/openstack/openstack_test.go b/pkg/providers/openstack/openstack_test.go new file mode 100644 index 00000000..c72b44d4 --- /dev/null +++ b/pkg/providers/openstack/openstack_test.go @@ -0,0 +1,67 @@ +package openstack + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gophercloud/gophercloud" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newTestServiceClient(t *testing.T, path, body string) *gophercloud.ServiceClient { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, path, r.URL.Path) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + + return &gophercloud.ServiceClient{ + ProviderClient: &gophercloud.ProviderClient{TokenID: "test"}, + Endpoint: server.URL + "/", + ResourceBase: server.URL + "/v2.0/", + } +} + +func TestFloatingIPProvider(t *testing.T) { + client := newTestServiceClient(t, "/v2.0/floatingips", `{"floatingips": [ + {"id": "attached", "floating_ip_address": "185.12.64.10", "port_id": "port-1", "fixed_ip_address": "10.0.0.5"}, + {"id": "unattached", "floating_ip_address": "185.12.64.20", "port_id": null} + ]}`) + + resources, err := (&floatingIPProvider{id: "test", client: client}).GetResource(context.Background()) + require.NoError(t, err) + require.Len(t, resources.Items, 2) + + for i, ip := range []string{"185.12.64.10", "185.12.64.20"} { + assert.Equal(t, ip, resources.Items[i].PublicIPv4) + assert.True(t, resources.Items[i].Public) + assert.Equal(t, "floatingip", resources.Items[i].Service) + } +} + +func TestLoadBalancerProvider(t *testing.T) { + client := newTestServiceClient(t, "/v2.0/lbaas/loadbalancers", `{"loadbalancers": [ + {"id": "private", "name": "internal", "vip_address": "10.0.0.20"}, + {"id": "public", "name": "edge", "vip_address": "185.12.64.30"} + ]}`) + + resources, err := (&loadBalancerProvider{id: "test", client: client}).GetResource(context.Background()) + require.NoError(t, err) + require.Len(t, resources.Items, 2) + + assert.Equal(t, "10.0.0.20", resources.Items[0].PrivateIpv4) + assert.False(t, resources.Items[0].Public) + assert.Equal(t, "185.12.64.30", resources.Items[1].PublicIPv4) + assert.True(t, resources.Items[1].Public) + assert.Equal(t, "loadbalancer", resources.Items[1].Service) +} + +func TestOptionalClientDropsClientOnError(t *testing.T) { + assert.Nil(t, optionalClient(&gophercloud.ServiceClient{}, &gophercloud.ErrEndpointNotFound{})) + assert.NotNil(t, optionalClient(&gophercloud.ServiceClient{}, nil)) +} From 023977fd228efb35a9dd557cbabc80e94c6e4bbf Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Fri, 9 Oct 2026 01:16:53 +0200 Subject: [PATCH 2/3] keep pages --- pkg/providers/openstack/floatingips.go | 2 +- pkg/providers/openstack/loadbalancers.go | 2 +- pkg/providers/openstack/openstack.go | 9 +++++--- pkg/providers/openstack/openstack_test.go | 26 +++++++++++++++++++++++ 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/pkg/providers/openstack/floatingips.go b/pkg/providers/openstack/floatingips.go index 27202c01..de93eae8 100644 --- a/pkg/providers/openstack/floatingips.go +++ b/pkg/providers/openstack/floatingips.go @@ -44,7 +44,7 @@ func (p *floatingIPProvider) GetResource(ctx context.Context) (*schema.Resources if err != nil { gologger.Error().Msgf("Couldn't list Openstack floating IPs: %s\n", err) - return nil, err + return list, err } return list, nil diff --git a/pkg/providers/openstack/loadbalancers.go b/pkg/providers/openstack/loadbalancers.go index 071b38e7..303e4eae 100644 --- a/pkg/providers/openstack/loadbalancers.go +++ b/pkg/providers/openstack/loadbalancers.go @@ -45,7 +45,7 @@ func (p *loadBalancerProvider) GetResource(ctx context.Context) (*schema.Resourc if err != nil { gologger.Error().Msgf("Couldn't list Openstack load balancers: %s\n", err) - return nil, err + return list, err } return list, nil diff --git a/pkg/providers/openstack/openstack.go b/pkg/providers/openstack/openstack.go index 000f2f8f..bbb0fb2a 100644 --- a/pkg/providers/openstack/openstack.go +++ b/pkg/providers/openstack/openstack.go @@ -126,19 +126,22 @@ func (p *Provider) Resources(ctx context.Context) (*schema.Resources, error) { finalResources := schema.NewResources() if p.services.Has("instance") { provider := &instanceProvider{id: p.id, client: p.client} - if resources, err := provider.GetResource(ctx); err == nil { + resources, _ := provider.GetResource(ctx) + if resources != nil { finalResources.Merge(resources) } } if p.network != nil { provider := &floatingIPProvider{id: p.id, client: p.network} - if resources, err := provider.GetResource(ctx); err == nil { + resources, _ := provider.GetResource(ctx) + if resources != nil { finalResources.Merge(resources) } } if p.loadBalancer != nil { provider := &loadBalancerProvider{id: p.id, client: p.loadBalancer} - if resources, err := provider.GetResource(ctx); err == nil { + resources, _ := provider.GetResource(ctx) + if resources != nil { finalResources.Merge(resources) } } diff --git a/pkg/providers/openstack/openstack_test.go b/pkg/providers/openstack/openstack_test.go index c72b44d4..427bd31f 100644 --- a/pkg/providers/openstack/openstack_test.go +++ b/pkg/providers/openstack/openstack_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/gophercloud/gophercloud" + "github.com/projectdiscovery/cloudlist/pkg/schema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -65,3 +66,28 @@ func TestOptionalClientDropsClientOnError(t *testing.T) { assert.Nil(t, optionalClient(&gophercloud.ServiceClient{}, &gophercloud.ErrEndpointNotFound{})) assert.NotNil(t, optionalClient(&gophercloud.ServiceClient{}, nil)) } + +func TestFloatingIPProviderKeepsEarlierPage(t *testing.T) { + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.URL.Query().Get("marker") == "2" { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"error":{"message":"page failed"}}`)) + return + } + _, _ = w.Write([]byte(`{"floatingips":[{"id":"one","floating_ip_address":"185.12.64.10"}],"floatingips_links":[{"href":"` + server.URL + `/v2.0/floatingips?marker=2","rel":"next"}]}`)) + })) + t.Cleanup(server.Close) + + client := &gophercloud.ServiceClient{ + ProviderClient: &gophercloud.ProviderClient{TokenID: "test"}, + Endpoint: server.URL + "/", + ResourceBase: server.URL + "/v2.0/", + } + provider := &Provider{id: "test", network: client, services: schema.ServiceMap{"floatingip": {}}} + resources, err := provider.Resources(context.Background()) + require.NoError(t, err) + require.Len(t, resources.Items, 1) + assert.Equal(t, "185.12.64.10", resources.Items[0].PublicIPv4) +} From d2c10959b7cbef6c04990300010ba43f1b8ae020 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Fri, 9 Oct 2026 02:19:36 +0200 Subject: [PATCH 3/3] fix listings --- pkg/providers/openstack/openstack.go | 73 ++++++++++++++------- pkg/providers/openstack/openstack_test.go | 79 +++++++++++++++++++++++ 2 files changed, 130 insertions(+), 22 deletions(-) diff --git a/pkg/providers/openstack/openstack.go b/pkg/providers/openstack/openstack.go index bbb0fb2a..67cf01d5 100644 --- a/pkg/providers/openstack/openstack.go +++ b/pkg/providers/openstack/openstack.go @@ -74,17 +74,25 @@ func New(options schema.OptionBlock) (*Provider, error) { return nil, err } + services := options.ResolveServices(Services) endpointOpts := gophercloud.EndpointOpts{Region: "RegionOne"} - client, err := openstack.NewComputeV2(provider, endpointOpts) - - if err != nil { - gologger.Error().Msgf("Couldn't use Openstack region: %s\n", err) - return nil, err + p := &Provider{id: id, services: services} + + // Compute is optional when Neutron or Octavia can still answer the + // selected services. A missing Nova endpoint must not fail those. + if services.Has("instance") { + client, clientErr := openstack.NewComputeV2(provider, endpointOpts) + if clientErr != nil && !services.Has("floatingip") && !services.Has("loadbalancer") { + gologger.Error().Msgf("Couldn't use Openstack region: %s\n", clientErr) + return nil, clientErr + } + if clientErr != nil { + gologger.Warning().Msgf("Couldn't use Openstack compute: %s\n", clientErr) + } else { + p.client = client + } } - services := options.ResolveServices(Services) - p := &Provider{id: id, client: client, services: services} - // Neutron and Octavia are optional on many clouds, so a missing catalog // entry only disables that service instead of failing the provider. if services.Has("floatingip") { @@ -124,26 +132,47 @@ func (p *Provider) Services() []string { // Resources returns the provider for an resource func (p *Provider) Resources(ctx context.Context) (*schema.Resources, error) { finalResources := schema.NewResources() - if p.services.Has("instance") { - provider := &instanceProvider{id: p.id, client: p.client} - resources, _ := provider.GetResource(ctx) - if resources != nil { - finalResources.Merge(resources) - } - } + var failed error + + // Neutron owns the floatingip service identity. Nova also reports an + // attached floating address, and the first writer wins deduplication. if p.network != nil { provider := &floatingIPProvider{id: p.id, client: p.network} - resources, _ := provider.GetResource(ctx) - if resources != nil { - finalResources.Merge(resources) - } + resources, err := provider.GetResource(ctx) + failed = absorb(finalResources, resources, err) } if p.loadBalancer != nil { provider := &loadBalancerProvider{id: p.id, client: p.loadBalancer} - resources, _ := provider.GetResource(ctx) - if resources != nil { - finalResources.Merge(resources) + resources, err := provider.GetResource(ctx) + if err := absorb(finalResources, resources, err); err != nil { + failed = err + } + } + if p.services.Has("instance") && p.client != nil { + provider := &instanceProvider{id: p.id, client: p.client} + resources, err := provider.GetResource(ctx) + if err := absorb(finalResources, resources, err); err != nil { + failed = err } } + if failed != nil && len(finalResources.Items) == 0 { + return nil, failed + } + if failed != nil { + gologger.Warning().Msgf("Couldn't list Openstack resources: %s\n", failed) + } return finalResources, nil } + +// absorb keeps a partial listing. An empty failed listing is returned so the +// caller can surface it when nothing else was collected. +func absorb(final *schema.Resources, resources *schema.Resources, err error) error { + if resources != nil && len(resources.Items) > 0 { + final.Merge(resources) + if err != nil { + gologger.Warning().Msgf("Couldn't list all Openstack resources: %s\n", err) + } + return nil + } + return err +} diff --git a/pkg/providers/openstack/openstack_test.go b/pkg/providers/openstack/openstack_test.go index 427bd31f..a3ca9123 100644 --- a/pkg/providers/openstack/openstack_test.go +++ b/pkg/providers/openstack/openstack_test.go @@ -91,3 +91,82 @@ func TestFloatingIPProviderKeepsEarlierPage(t *testing.T) { require.Len(t, resources.Items, 1) assert.Equal(t, "185.12.64.10", resources.Items[0].PublicIPv4) } + +func TestFloatingIPFailureWithoutResults(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"error":{"message":"unavailable"}}`)) + })) + t.Cleanup(server.Close) + + client := &gophercloud.ServiceClient{ + ProviderClient: &gophercloud.ProviderClient{TokenID: "test"}, + Endpoint: server.URL + "/", + ResourceBase: server.URL + "/v2.0/", + } + provider := &Provider{id: "test", network: client, services: schema.ServiceMap{"floatingip": {}}} + resources, err := provider.Resources(context.Background()) + require.Error(t, err) + assert.Nil(t, resources) +} + +func TestFloatingIPServiceWinsDedup(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/servers/detail": + _, _ = w.Write([]byte(`{"servers":[{"id":"s1","addresses":{"public":[{"addr":"185.12.64.10","OS-EXT-IPS:type":"floating"}]}}]}`)) + default: + _, _ = w.Write([]byte(`{"floatingips":[{"id":"attached","floating_ip_address":"185.12.64.10"}]}`)) + } + })) + t.Cleanup(server.Close) + + base := &gophercloud.ProviderClient{TokenID: "test"} + provider := &Provider{ + id: "test", + client: &gophercloud.ServiceClient{ + ProviderClient: base, + Endpoint: server.URL + "/", + ResourceBase: server.URL + "/", + }, + network: &gophercloud.ServiceClient{ + ProviderClient: base, + Endpoint: server.URL + "/", + ResourceBase: server.URL + "/v2.0/", + }, + services: schema.ServiceMap{"instance": {}, "floatingip": {}}, + } + resources, err := provider.Resources(context.Background()) + require.NoError(t, err) + require.Len(t, resources.Items, 1) + assert.Equal(t, "floatingip", resources.Items[0].Service) + assert.Equal(t, "185.12.64.10", resources.Items[0].PublicIPv4) +} + +func TestNewSkipsMissingComputeWhenFloatingIPSelected(t *testing.T) { + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v3/auth/tokens" { + http.NotFound(w, r) + return + } + w.Header().Set("X-Subject-Token", "test-token") + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"token":{"expires_at":"2099-01-01T00:00:00.000000Z","catalog":[{"type":"network","name":"neutron","endpoints":[{"interface":"public","region":"RegionOne","url":"` + server.URL + `/v2.0"}]}]}}`)) + })) + t.Cleanup(server.Close) + + provider, err := New(schema.OptionBlock{ + "identity_endpoint": server.URL + "/v3", + "domain_name": "default", + "tenant_name": "demo", + "username": "user", + "password": "pass", + "services": "floatingip", + }) + require.NoError(t, err) + assert.Nil(t, provider.client) + assert.NotNil(t, provider.network) +}