Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,18 @@ See the [examples/](examples/) directory for runnable code samples.

See [docs/architecture.md](docs/architecture.md) for the package layout and data flow.

### Portable quality graph

`qualitygraph.Build` projects a completed `inspect.Report` into the shared
`hawk-core-contracts/graph` vocabulary. The projection contains a quality node
for the report, bounded quality nodes for findings, `contains` edges, and
immutable observation events.

The projection is metadata-only: scan targets, URLs, messages, DOM elements,
fixes, and evidence are represented by SHA-256 digests rather than copied into
graph attributes. Inspect remains the source of truth for scans and reports;
consumers such as Hawk may journal or compose the portable projection.

## Ecosystem

inspect is part of the hawk ecosystem:
Expand Down
6 changes: 6 additions & 0 deletions docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ inspect/
├── sarif.go 📊 GenerateSARIF — SARIF 2.1.0 output
├── browser.go 🌐 BrowserEngine interface + page-data types (no rod import)
├── browser_fetcher.go 🔌 Adapts a BrowserEngine into the crawler's fetcher
├── qualitygraph/ 🕸️ Privacy-safe shared quality graph projection
├── checks/ ✅ Built-in checks run against crawled responses
│ ├── headers.go 📋 Missing security headers (CSP, HSTS, …)
│ ├── cookies.go 🍪 Cookie Secure/HttpOnly/SameSite flags
Expand Down Expand Up @@ -165,6 +166,11 @@ and check, per-check durations) and a `FailOn` threshold; `Report.Failed()` and
`Report.MaxSeverity()` summarize the run. `GenerateSARIF` converts findings to
SARIF 2.1.0.

`qualitygraph.Build` is an explicit read-only projection boundary after a scan
completes. It maps the report and a bounded set of findings to shared quality
nodes, containment edges, and observation events. Sensitive report fields are
hashed, and the package neither runs scans nor persists graph state.

---

## 🛡️ Crawler Safeguards
Expand Down
2 changes: 1 addition & 1 deletion go.mod

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions go.sum

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

51 changes: 28 additions & 23 deletions internal/crawler/crawler.go
Original file line number Diff line number Diff line change
Expand Up @@ -556,37 +556,42 @@ func (c *Crawler) validateURL(rawURL string) error {
return nil
}

// privateCIDRs are the RFC 1918, loopback, link-local, and CGNAT ranges
// used for SSRF protection. Parsed once at init time; a malformed entry
// is a programming error caught at startup, not at scan time.
var privateCIDRs = func() []*net.IPNet {
cidrs := []string{
"10.0.0.0/8",
"172.16.0.0/12",
"192.168.0.0/16",
"127.0.0.0/8",
"169.254.0.0/16", // link-local, incl. cloud metadata (169.254.169.254)
"100.64.0.0/10", // CGNAT shared address space
"::1/128",
"fc00::/7",
"fe80::/10", // IPv6 link-local
}
nets := make([]*net.IPNet, 0, len(cidrs))
for _, s := range cidrs {
_, network, err := net.ParseCIDR(s)
if err != nil {
panic(fmt.Sprintf("inspect/crawler: invalid private CIDR %q: %v", s, err))
}
nets = append(nets, network)
}
return nets
}()

// isPrivateIP checks if an IP is in a private/loopback range.
func isPrivateIP(ip net.IP) bool {
privateRanges := []struct {
network *net.IPNet
}{
{mustParseCIDR("10.0.0.0/8")},
{mustParseCIDR("172.16.0.0/12")},
{mustParseCIDR("192.168.0.0/16")},
{mustParseCIDR("127.0.0.0/8")},
{mustParseCIDR("169.254.0.0/16")}, // link-local, incl. cloud metadata (169.254.169.254)
{mustParseCIDR("100.64.0.0/10")}, // CGNAT shared address space
{mustParseCIDR("::1/128")},
{mustParseCIDR("fc00::/7")},
{mustParseCIDR("fe80::/10")}, // IPv6 link-local
}
for _, r := range privateRanges {
if r.network.Contains(ip) {
for _, network := range privateCIDRs {
if network.Contains(ip) {
return true
}
}
return false
}

func mustParseCIDR(s string) *net.IPNet {
_, network, err := net.ParseCIDR(s)
if err != nil {
panic(err)
}
return network
}

func isRetryable(statusCode int) bool {
return statusCode == 429 || statusCode == 500 || statusCode == 502 ||
statusCode == 503 || statusCode == 504 || statusCode == 0
Expand Down
33 changes: 33 additions & 0 deletions mcp/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,16 @@ func (s *Server) registerTools() {
mcplib.WithDescription("Scan a local directory of HTML files"),
mcplib.WithString("path", mcplib.Required(), mcplib.Description("Local directory path")),
), s.handleScanDir)

s.kit.AddTool(mcplib.NewTool(
"inspect_checks",
mcplib.WithDescription("List all available audit checks and what they detect"),
), s.handleChecks)

s.kit.AddTool(mcplib.NewTool(
"inspect_status",
mcplib.WithDescription("Return the inspect server version and capabilities"),
), s.handleStatus)
}

func (s *Server) handleScan(ctx context.Context, req mcplib.CallToolRequest) (*mcplib.CallToolResult, error) {
Expand Down Expand Up @@ -84,3 +94,26 @@ func (s *Server) handleScanDir(ctx context.Context, req mcplib.CallToolRequest)
}
return mcpkit.JSONResult(report)
}

func (s *Server) handleChecks(_ context.Context, _ mcplib.CallToolRequest) (*mcplib.CallToolResult, error) {
checks := []map[string]string{
{"name": "links", "description": "Broken links, redirect chains, and dead anchors"},
{"name": "security", "description": "Security headers (CSP, HSTS, X-Frame-Options), mixed content"},
{"name": "forms", "description": "Form accessibility, autocomplete, and sensitive field handling"},
{"name": "a11y", "description": "Accessibility: alt text, ARIA roles, contrast, heading structure"},
{"name": "perf", "description": "Performance: render-blocking resources, image optimization hints"},
{"name": "seo", "description": "SEO: meta tags, canonical URLs, structured data"},
{"name": "sri", "description": "Subresource Integrity hashes on external scripts and styles"},
{"name": "aiready", "description": "AI-readiness: robots.txt, llms.txt, structured data for crawlers"},
{"name": "reachability", "description": "URL reachability and HTTP status codes"},
}
return mcpkit.JSONResult(map[string]interface{}{"checks": checks})
}

func (s *Server) handleStatus(_ context.Context, _ mcplib.CallToolRequest) (*mcplib.CallToolResult, error) {
return mcpkit.JSONResult(map[string]interface{}{
"name": "inspect",
"version": inspect.Version,
"tools": []string{"inspect_scan", "inspect_scan_dir", "inspect_checks", "inspect_status"},
})
}
151 changes: 134 additions & 17 deletions mcp/server_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -200,19 +200,18 @@ func TestProtocol_ToolsList(t *testing.T) {
t.Fatalf("unmarshal: %v", err)
}

if len(result.Result.Tools) != 2 {
t.Fatalf("expected 2 tools, got %d", len(result.Result.Tools))
if len(result.Result.Tools) != 4 {
t.Fatalf("expected 4 tools, got %d", len(result.Result.Tools))
}

names := make(map[string]bool)
for _, tool := range result.Result.Tools {
names[tool.Name] = true
}
if !names["inspect_scan"] {
t.Error("inspect_scan tool not found in tools/list")
}
if !names["inspect_scan_dir"] {
t.Error("inspect_scan_dir tool not found in tools/list")
for _, expected := range []string{"inspect_scan", "inspect_scan_dir", "inspect_checks", "inspect_status"} {
if !names[expected] {
t.Errorf("%s tool not found in tools/list", expected)
}
}
}

Expand Down Expand Up @@ -489,18 +488,15 @@ func TestHandleScanDir_NonexistentDir(t *testing.T) {
if result == nil {
t.Fatal("expected result")
}
// The ScanDir function starts a file server on the given directory.
// For a nonexistent path, the server may still start and serve 404s.
// The scanner reports these as findings, not as a tool error.
// ScanDir calls os.Stat on the path; a nonexistent directory returns an
// error, which handleScanDir surfaces as a tool-level error result.
if !result.IsError {
t.Fatal("expected tool-level error for nonexistent directory")
}
for _, c := range result.Content {
if tc, ok := c.(mcplib.TextContent); ok {
var report inspect.Report
if err := json.Unmarshal([]byte(tc.Text), &report); err != nil {
t.Fatalf("expected valid JSON report: %v", err)
}
// Report should exist with a target.
if report.Target == "" {
t.Error("expected non-empty target in report")
if tc.Text == "" {
t.Fatal("expected non-empty error message")
}
}
}
Expand Down Expand Up @@ -644,5 +640,126 @@ func TestErrorHandling_UnknownMethod(t *testing.T) {
}
}

// ---------------------------------------------------------------------------
// inspect_checks and inspect_status tools
// ---------------------------------------------------------------------------

func TestProtocol_ToolsCall_Checks(t *testing.T) {
ts, _ := newTestHTTPServer(t, inspect.Quick, inspect.WithAllowPrivateIPs())
sid := initAndSession(t, ts)

resp, err := postSessionJSON(ts.URL, sid, jsonRPCRequest(3, "tools/call", map[string]any{
"name": "inspect_checks",
"arguments": map[string]any{},
}))
if err != nil {
t.Fatalf("tools/call: %v", err)
}

body := readBody(t, resp)
if resp.StatusCode != http.StatusOK {
t.Fatalf("expected 200, got %d: %s", resp.StatusCode, body)
}

var rpcResp struct {
Result struct {
Content []struct {
Type string `json:"type"`
Text string `json:"text"`
} `json:"content"`
IsError bool `json:"isError"`
} `json:"result"`
}
if err := json.Unmarshal(body, &rpcResp); err != nil {
t.Fatalf("unmarshal: %v", err)
}
if rpcResp.Result.IsError {
t.Fatalf("expected success, got error: %s", rpcResp.Result.Content[0].Text)
}
if len(rpcResp.Result.Content) == 0 {
t.Fatal("expected at least one content item")
}

// The text content should be a JSON object with a "checks" array.
var payload struct {
Checks []struct {
Name string `json:"name"`
Description string `json:"description"`
} `json:"checks"`
}
if err := json.Unmarshal([]byte(rpcResp.Result.Content[0].Text), &payload); err != nil {
t.Fatalf("content is not valid JSON: %v\nraw: %s", err, rpcResp.Result.Content[0].Text)
}
if len(payload.Checks) == 0 {
t.Fatal("expected at least one audit check in the catalog")
}

// Spot-check that a known check is present.
found := map[string]bool{}
for _, c := range payload.Checks {
found[c.Name] = true
}
for _, want := range []string{"links", "security", "a11y"} {
if !found[want] {
t.Errorf("expected %q check in catalog, not found", want)
}
}
}

func TestProtocol_ToolsCall_Status(t *testing.T) {
ts, _ := newTestHTTPServer(t, inspect.Quick, inspect.WithAllowPrivateIPs())
sid := initAndSession(t, ts)

resp, err := postSessionJSON(ts.URL, sid, jsonRPCRequest(3, "tools/call", map[string]any{
"name": "inspect_status",
"arguments": map[string]any{},
}))
if err != nil {
t.Fatalf("tools/call: %v", err)
}

body := readBody(t, resp)
if resp.StatusCode != http.StatusOK {
t.Fatalf("expected 200, got %d: %s", resp.StatusCode, body)
}

var rpcResp struct {
Result struct {
Content []struct {
Type string `json:"type"`
Text string `json:"text"`
} `json:"content"`
IsError bool `json:"isError"`
} `json:"result"`
}
if err := json.Unmarshal(body, &rpcResp); err != nil {
t.Fatalf("unmarshal: %v", err)
}
if rpcResp.Result.IsError {
t.Fatalf("expected success, got error: %s", rpcResp.Result.Content[0].Text)
}
if len(rpcResp.Result.Content) == 0 {
t.Fatal("expected at least one content item")
}

var payload struct {
Name string `json:"name"`
Version string `json:"version"`
Tools []string `json:"tools"`
}
if err := json.Unmarshal([]byte(rpcResp.Result.Content[0].Text), &payload); err != nil {
t.Fatalf("content is not valid JSON: %v\nraw: %s", err, rpcResp.Result.Content[0].Text)
}
if payload.Name != "inspect" {
t.Errorf("name: want inspect, got %q", payload.Name)
}
if payload.Version != inspect.Version {
t.Errorf("version: want %s, got %s", inspect.Version, payload.Version)
}
if len(payload.Tools) == 0 {
t.Error("expected at least one tool in status output")
}
}

// Note: additional tests for the mcp package live in server_more_test.go
// (split out for file size/clarity). Shared test helpers remain in this file.
Loading
Loading