feat(runner): add -mp/-metrics-port for the -stats metrics server

README.md

@@ -255,6 +255,7 @@ DEBUG:
    -silent                   silent mode
    -v, -verbose              verbose mode
    -si, -stats-interval int  number of seconds to wait between showing a statistics update (default: 5)
+   -mp, -metrics-port int    port to expose httpx metrics on (default 63636)
    -nc, -no-color            disable colors in cli output
    -tr, -trace               trace
 

runner/metrics_port_test.go

@@ -0,0 +1,92 @@
+package runner
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"testing"
+
+	"github.com/projectdiscovery/clistats"
+	"github.com/stretchr/testify/require"
+)
+
+func TestOptions_ValidateMetricsPort(t *testing.T) {
+	tests := []struct {
+		port    int
+		wantErr bool
+	}{
+		{port: -1, wantErr: true},
+		{port: 0},
+		{port: 8080},
+		{port: 65535},
+		{port: 65536, wantErr: true},
+	}
+
+	for _, test := range tests {
+		t.Run(fmt.Sprint(test.port), func(t *testing.T) {
+			options := &Options{MetricsPort: test.port}
+			err := options.ValidateOptions()
+			if test.wantErr {
+				require.ErrorContains(t, err, "invalid metrics port")
+				return
+			}
+			require.NoError(t, err)
+		})
+	}
+}
+
+func TestOptions_metricsPort(t *testing.T) {
+	// a library caller that leaves the field unset gets the clistats default
+	require.Equal(t, clistats.DefaultOptions.ListenPort, (&Options{}).metricsPort())
+	require.Equal(t, 8080, (&Options{MetricsPort: 8080}).metricsPort())
+}
+
+func TestRunner_metricsPort(t *testing.T) {
+	defaultPort := clistats.DefaultOptions.ListenPort
+
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	freePort := listener.Addr().(*net.TCPAddr).Port
+	require.NoError(t, listener.Close())
+
+	busy, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	defer func() { _ = busy.Close() }()
+	busyPort := busy.Addr().(*net.TCPAddr).Port
+
+	tests := []struct {
+		name     string
+		port     int
+		wantSame bool
+	}{
+		{name: "requested port is used", port: freePort, wantSame: true},
+		{name: "busy port falls back to a free one", port: busyPort, wantSame: false},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			r, err := New(&Options{ShowStatistics: true, MetricsPort: test.port})
+			require.NoError(t, err)
+			defer r.Close()
+
+			stats, ok := r.stats.(*clistats.Statistics)
+			require.True(t, ok, "stats should use clistats")
+			require.NoError(t, stats.Start())
+			defer func() { _ = stats.Stop() }()
+
+			if test.wantSame {
+				require.Equal(t, test.port, stats.Options.ListenPort)
+			} else {
+				require.NotEqual(t, test.port, stats.Options.ListenPort)
+			}
+
+			resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/metrics", stats.Options.ListenPort))
+			require.NoError(t, err)
+			_ = resp.Body.Close()
+			require.Equal(t, http.StatusOK, resp.StatusCode)
+
+			// the runner works on a copy of the clistats defaults
+			require.Equal(t, defaultPort, clistats.DefaultOptions.ListenPort)
+		})
+	}
+}

runner/options.go

@@ -12,6 +12,7 @@ import (
 	"golang.org/x/exp/maps"
 
 	"github.com/projectdiscovery/cdncheck"
+	"github.com/projectdiscovery/clistats"
 	"github.com/projectdiscovery/goconfig"
 	"github.com/projectdiscovery/goflags"
 	"github.com/projectdiscovery/gologger"
@@ -275,6 +276,7 @@ type Options struct {
 	protocol                  string
 	ShowStatistics            bool
 	StatsInterval             int
+	MetricsPort               int
 	RandomAgent               bool
 	AutoReferer               bool
 	StoreChain                bool
@@ -568,6 +570,7 @@ func ParseOptions() *Options {
 		flagSet.BoolVar(&options.Silent, "silent", false, "silent mode"),
 		flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "verbose mode"),
 		flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 0, "number of seconds to wait between showing a statistics update (default: 5)"),
+		flagSet.IntVarP(&options.MetricsPort, "metrics-port", "mp", clistats.DefaultOptions.ListenPort, "port to expose httpx metrics on"),
 		flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable colors in cli output"),
 		flagSet.BoolVarP(&options.Trace, "trace", "tr", false, "trace"),
 	)
@@ -757,6 +760,10 @@ func (options *Options) ValidateOptions() error {
 		return errors.New("-im/-input-mode requires -l/-list to specify an input file")
 	}
 
+	if options.MetricsPort < 0 || options.MetricsPort > 65535 {
+		return fmt.Errorf("invalid metrics port %d, must be between 0 and 65535", options.MetricsPort)
+	}
+
 	if options.Silent {
 		incompatibleFlagsList := flagsIncompatibleWithSilent(options)
 		if len(incompatibleFlagsList) > 0 {
@@ -973,3 +980,12 @@ func flagsIncompatibleWithSilent(options *Options) []string {
 	}
 	return incompatibleFlagsList
 }
+
+// metricsPort returns the port the -stats metrics server is asked to listen
+// on: the -metrics-port value, or the clistats default when it is not set.
+func (options *Options) metricsPort() int {
+	if options.MetricsPort > 0 {
+		return options.MetricsPort
+	}
+	return clistats.DefaultOptions.ListenPort
+}

runner/runner.go

@@ -400,7 +400,10 @@ func New(options *Options) (*Runner, error) {
 	runner.scanopts = scanopts
 
 	if options.ShowStatistics {
-		runner.stats, err = clistats.New()
+		// a copy, as Start writes the port it falls back to into the options
+		statsOptions := clistats.DefaultOptions
+		statsOptions.ListenPort = options.metricsPort()
+		runner.stats, err = clistats.NewWithOptions(context.Background(), &statsOptions)
 		if err != nil {
 			return nil, err
 		}
@@ -617,6 +620,11 @@ func (r *Runner) prepareInput() {
 		err := r.stats.Start()
 		if err != nil {
 			gologger.Warning().Msgf("Could not create statistics: %s\n", err)
+		} else if stats, ok := r.stats.(*clistats.Statistics); ok {
+			if requested := r.options.metricsPort(); stats.Options.ListenPort != requested {
+				gologger.Info().Msgf("Metrics port %d is not available, using %d\n", requested, stats.Options.ListenPort)
+			}
+			gologger.Verbose().Msgf("Started metrics server at %s\n", net.JoinHostPort(stats.Options.ListenHost, strconv.Itoa(stats.Options.ListenPort)))
 		}
 
 		r.stats.GetStatResponse(time.Duration(r.options.StatsInterval)*time.Second, func(s string, err error) error {