From 24312cb37657d931b0b9418301984d9f6fe1d193 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Migone?= Date: Fri, 9 Oct 2026 14:37:15 -0300 Subject: [PATCH] chore: add some alert definitions for escrow manager MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tomás Migone --- grafana/README.md | 18 +++++ grafana/alerts/escrow_manager.yml | 76 ++++++++++++++++++++ grafana/{ => dashboards}/aggregator.json | 0 grafana/{ => dashboards}/escrow_manager.json | 0 4 files changed, 94 insertions(+) create mode 100644 grafana/README.md create mode 100644 grafana/alerts/escrow_manager.yml rename grafana/{ => dashboards}/aggregator.json (100%) rename grafana/{ => dashboards}/escrow_manager.json (100%) diff --git a/grafana/README.md b/grafana/README.md new file mode 100644 index 0000000..470d2b0 --- /dev/null +++ b/grafana/README.md @@ -0,0 +1,18 @@ +# Grafana + +- `dashboards/`: dashboard exports. Import them in Grafana with **Dashboards → New → Import**. +- `alerts/`: alert rules in Prometheus rule format. + +## Importing alert rules + +The rules become Grafana-managed rules. Their `team` label routes them to Rootly through the +notification policies in graph-infra (`infra/terraform/grafana-cloud`). + +1. Go to **Alerting → Alert rules → Import to Grafana-managed rules**. +2. Select **Prometheus YAML file** and upload the file, for example `alerts/escrow_manager.yml`. +3. Select the VictoriaMetrics data source that the dashboards use. +4. In **Additional settings**, select a folder that holds only these rules, for example + `Escrow Manager`. An import can overwrite rules in the target folder. + +To change a rule, edit the file and import it again into the same folder. Changes made in the +Grafana UI are not written back to this repository. diff --git a/grafana/alerts/escrow_manager.yml b/grafana/alerts/escrow_manager.yml new file mode 100644 index 0000000..c75bccf --- /dev/null +++ b/grafana/alerts/escrow_manager.yml @@ -0,0 +1,76 @@ +groups: + - name: Escrow Manager + interval: 1m + labels: + team: Smart Contracts + environment: production + rules: + - alert: EscrowManagerDown + expr: >- + up{job="default/tap-escrow-manager-mainnet"} == 0 + or absent(up{job="default/tap-escrow-manager-mainnet"}) + for: 5m + labels: + priority: P2-High + annotations: + summary: Escrow manager is down or not being scraped + description: >- + No successful scrape of {{ $labels.job }} for 5 minutes. While it is down, no escrow + is deposited, so receivers become underfunded as debt grows. + __dashboardUid__: graph-tally-escrow-manager + __panelId__: "17" + + # The loop skips to the next cycle when it cannot read the network subgraph, before the + # duration is observed. The process stays up and the gauges keep their last values, so this + # is the only signal that the cycle is not running. + - alert: EscrowManagerStalled + expr: >- + sum by (job) (increase(escrow_loop_duration_seconds_count{job="default/tap-escrow-manager-mainnet"}[15m])) == 0 + for: 5m + labels: + priority: P2-High + annotations: + summary: Escrow manager is running but not completing cycles + description: >- + {{ $labels.job }} has completed no cycle in the last 15 minutes. Check the logs for + network subgraph errors. Escrow metrics on the dashboard are stale until this resolves. + __dashboardUid__: graph-tally-escrow-manager + __panelId__: "7" + + # Per-receiver gauges are never removed. A receiver that leaves the tracked set while this is + # firing keeps it firing until the pod restarts. + - alert: EscrowReceiverUnderfunded + expr: >- + ( + escrow_debt_grt{job="default/tap-escrow-manager-mainnet"} + - (escrow_balance_grt{job="default/tap-escrow-manager-mainnet"} - escrow_thawing_grt{job="default/tap-escrow-manager-mainnet"}) + ) > 0 + for: 15m + labels: + priority: P1-High + annotations: + summary: "Receiver {{ $labels.receiver }} has less escrow than debt" + description: >- + Receiver {{ $labels.receiver }} is owed {{ $value | printf "%.2f" }} GRT more than its + escrow balance (net of thawing) can cover, for 15 minutes. It cannot collect the full + amount. Check for failed or capped deposits. + __dashboardUid__: graph-tally-escrow-manager + __panelId__: "16" + + # Deposit counters start at 1 on process start, so a restart adds to both and does not + # match `ok == 0`. + - alert: EscrowDepositsFailing + expr: >- + sum by (job) (increase(escrow_deposit_err{job="default/tap-escrow-manager-mainnet"}[30m])) >= 2 + and + sum by (job) (increase(escrow_deposit_ok{job="default/tap-escrow-manager-mainnet"}[30m])) == 0 + labels: + priority: P1-High + annotations: + summary: Escrow deposits are failing + description: >- + {{ $labels.job }} had {{ $value | printf "%.0f" }} failed deposit transactions and no + successful one in the last 30 minutes. Check the payer wallet GRT and ETH balance, + the GRT allowance, and the RPC provider. + __dashboardUid__: graph-tally-escrow-manager + __panelId__: "9" diff --git a/grafana/aggregator.json b/grafana/dashboards/aggregator.json similarity index 100% rename from grafana/aggregator.json rename to grafana/dashboards/aggregator.json diff --git a/grafana/escrow_manager.json b/grafana/dashboards/escrow_manager.json similarity index 100% rename from grafana/escrow_manager.json rename to grafana/dashboards/escrow_manager.json