diff --git a/README.md b/README.md index 298c8733..00793f79 100644 --- a/README.md +++ b/README.md @@ -255,6 +255,7 @@ DEBUG: -silent silent mode -v, -verbose verbose mode -si, -stats-interval int number of seconds to wait between showing a statistics update (default: 5) + -mp, -metrics-port int port to expose httpx metrics on (default 63636) -nc, -no-color disable colors in cli output -tr, -trace trace diff --git a/runner/metrics_port_test.go b/runner/metrics_port_test.go new file mode 100644 index 00000000..ffcdbb99 --- /dev/null +++ b/runner/metrics_port_test.go @@ -0,0 +1,92 @@ +package runner + +import ( + "fmt" + "net" + "net/http" + "testing" + + "github.com/projectdiscovery/clistats" + "github.com/stretchr/testify/require" +) + +func TestOptions_ValidateMetricsPort(t *testing.T) { + tests := []struct { + port int + wantErr bool + }{ + {port: -1, wantErr: true}, + {port: 0}, + {port: 8080}, + {port: 65535}, + {port: 65536, wantErr: true}, + } + + for _, test := range tests { + t.Run(fmt.Sprint(test.port), func(t *testing.T) { + options := &Options{MetricsPort: test.port} + err := options.ValidateOptions() + if test.wantErr { + require.ErrorContains(t, err, "invalid metrics port") + return + } + require.NoError(t, err) + }) + } +} + +func TestOptions_metricsPort(t *testing.T) { + // a library caller that leaves the field unset gets the clistats default + require.Equal(t, clistats.DefaultOptions.ListenPort, (&Options{}).metricsPort()) + require.Equal(t, 8080, (&Options{MetricsPort: 8080}).metricsPort()) +} + +func TestRunner_metricsPort(t *testing.T) { + defaultPort := clistats.DefaultOptions.ListenPort + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + freePort := listener.Addr().(*net.TCPAddr).Port + require.NoError(t, listener.Close()) + + busy, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer func() { _ = busy.Close() }() + busyPort := busy.Addr().(*net.TCPAddr).Port + + tests := []struct { + name string + port int + wantSame bool + }{ + {name: "requested port is used", port: freePort, wantSame: true}, + {name: "busy port falls back to a free one", port: busyPort, wantSame: false}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + r, err := New(&Options{ShowStatistics: true, MetricsPort: test.port}) + require.NoError(t, err) + defer r.Close() + + stats, ok := r.stats.(*clistats.Statistics) + require.True(t, ok, "stats should use clistats") + require.NoError(t, stats.Start()) + defer func() { _ = stats.Stop() }() + + if test.wantSame { + require.Equal(t, test.port, stats.Options.ListenPort) + } else { + require.NotEqual(t, test.port, stats.Options.ListenPort) + } + + resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/metrics", stats.Options.ListenPort)) + require.NoError(t, err) + _ = resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + // the runner works on a copy of the clistats defaults + require.Equal(t, defaultPort, clistats.DefaultOptions.ListenPort) + }) + } +} diff --git a/runner/options.go b/runner/options.go index f636b9d1..3174a959 100644 --- a/runner/options.go +++ b/runner/options.go @@ -12,6 +12,7 @@ import ( "golang.org/x/exp/maps" "github.com/projectdiscovery/cdncheck" + "github.com/projectdiscovery/clistats" "github.com/projectdiscovery/goconfig" "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/gologger" @@ -275,6 +276,7 @@ type Options struct { protocol string ShowStatistics bool StatsInterval int + MetricsPort int RandomAgent bool AutoReferer bool StoreChain bool @@ -568,6 +570,7 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.Silent, "silent", false, "silent mode"), flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "verbose mode"), flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 0, "number of seconds to wait between showing a statistics update (default: 5)"), + flagSet.IntVarP(&options.MetricsPort, "metrics-port", "mp", clistats.DefaultOptions.ListenPort, "port to expose httpx metrics on"), flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable colors in cli output"), flagSet.BoolVarP(&options.Trace, "trace", "tr", false, "trace"), ) @@ -757,6 +760,10 @@ func (options *Options) ValidateOptions() error { return errors.New("-im/-input-mode requires -l/-list to specify an input file") } + if options.MetricsPort < 0 || options.MetricsPort > 65535 { + return fmt.Errorf("invalid metrics port %d, must be between 0 and 65535", options.MetricsPort) + } + if options.Silent { incompatibleFlagsList := flagsIncompatibleWithSilent(options) if len(incompatibleFlagsList) > 0 { @@ -973,3 +980,12 @@ func flagsIncompatibleWithSilent(options *Options) []string { } return incompatibleFlagsList } + +// metricsPort returns the port the -stats metrics server is asked to listen +// on: the -metrics-port value, or the clistats default when it is not set. +func (options *Options) metricsPort() int { + if options.MetricsPort > 0 { + return options.MetricsPort + } + return clistats.DefaultOptions.ListenPort +} diff --git a/runner/runner.go b/runner/runner.go index c903054d..b84fd3bd 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -400,7 +400,10 @@ func New(options *Options) (*Runner, error) { runner.scanopts = scanopts if options.ShowStatistics { - runner.stats, err = clistats.New() + // a copy, as Start writes the port it falls back to into the options + statsOptions := clistats.DefaultOptions + statsOptions.ListenPort = options.metricsPort() + runner.stats, err = clistats.NewWithOptions(context.Background(), &statsOptions) if err != nil { return nil, err } @@ -617,6 +620,11 @@ func (r *Runner) prepareInput() { err := r.stats.Start() if err != nil { gologger.Warning().Msgf("Could not create statistics: %s\n", err) + } else if stats, ok := r.stats.(*clistats.Statistics); ok { + if requested := r.options.metricsPort(); stats.Options.ListenPort != requested { + gologger.Info().Msgf("Metrics port %d is not available, using %d\n", requested, stats.Options.ListenPort) + } + gologger.Verbose().Msgf("Started metrics server at %s\n", net.JoinHostPort(stats.Options.ListenHost, strconv.Itoa(stats.Options.ListenPort))) } r.stats.GetStatResponse(time.Duration(r.options.StatsInterval)*time.Second, func(s string, err error) error {