feat(blockchain): implement Stellar sequence-number manager for concurrent server submissions (#82)

docs/architecture/blockchain-layer.md

@@ -71,3 +71,26 @@ export class ReputationContractClient {
 4. Mobile signs the transaction
 5. Mobile sends signed XDR back
 6. API sends transaction to network using `soroban.client.ts`
+
+## Server-signed submissions
+
+Server-initiated transactions (default detection, admin ops, future
+loan-funding paths) use `SequenceManagerService` in
+`src/blockchain/sequence-manager/`. Submissions go through a managed
+source account whose sequence number is:
+
+- Refreshed from Horizon on first use and on every `tx_bad_seq`
+  rejection, so a stale local counter can never wedge the protocol.
+- Serialized per-account through an in-memory promise chain so two
+  concurrent callers cannot double-spend the same sequence.
+- Optionally fan out across a `STELLAR_CHANNEL_ACCOUNTS` pool for true
+  parallel in-flight transactions; each channel has its own counter.
+
+`BlockchainService.submitServerTransaction(spec)` is the public
+entrypoint. `spec.build` receives an `Account` already pre-populated
+with the source-account id and the next sequence number; the manager
+signs and submits, exposing Prometheus counters
+(`blockchain_source_submissions_in_flight`,
+`blockchain_source_bad_seq_retries_total`,
+`blockchain_source_submissions_total`,
+`blockchain_source_next_sequence`).

docs/setup/environment-variables.md

@@ -142,6 +142,28 @@ TX_STATUS_INTERVAL=15
 REMINDER_CRON=0 9 * * *
 ```
 
+### Server-signed Stellar submissions (sequence manager)
+
+These variables let the API itself submit Stellar transactions for loan
+funding, default detection, and admin operations. Without the source
+account secret, `BlockchainService.submitServerTransaction` is disabled
+and the API can still run in read-only / frontend-only mode.
+
+```env
+# Secret key (S…) of the protocol source account that signs server txs.
+# REQUIRED — leave unset only for read-only deployments.
+STELLAR_SOURCE_ACCOUNT_SECRET=S...
+
+# Optional comma-separated list of channel-account secret keys (S…) for
+# concurrent in-flight submissions. Each channel maintains its own
+# sequence number so they never collide. Funded separately.
+STELLAR_CHANNEL_ACCOUNTS=S...,S...
+
+# Max attempts (re-syncs from Horizon) before the manager gives up on
+# tx_bad_seq. Default 3 (one initial attempt + three retries).
+STELLAR_BAD_SEQ_MAX_RETRIES=3
+```
+
 ### Monitoring (Optional)
 
 ```env

src/app.module.ts

@@ -23,6 +23,7 @@ import { TransactionStatusCheckerModule } from './jobs/transaction-status-checke
 import { NonceCleanupModule } from './jobs/nonce-cleanup/nonce-cleanup.module';
 import { SupabaseKeepAliveModule } from './jobs/supabase-keepalive/supabase-keepalive.module';
 import { StellarModule } from './stellar/stellar.module';
+import { SequenceManagerModule } from './blockchain/sequence-manager/sequence-manager.module';
 import { LoggerModule } from './common/logger/logger.module';
 import { MetricsModule } from './modules/metrics/metrics.module';
 import { CreditScoringModule } from './modules/credit-scoring/credit-scoring.module';
@@ -65,6 +66,7 @@ import { StateReconciliationModule } from './jobs/state-reconciliation/state-rec
     CreditScoringModule,
     AdminModule,
     StellarModule,
+    SequenceManagerModule,
   ],
   controllers: [],
   providers: [

src/blockchain/sequence-manager/sequence-manager.metrics.ts

@@ -0,0 +1,115 @@
+import { Injectable, Optional } from '@nestjs/common';
+import { Counter, Gauge } from 'prom-client';
+import {
+  InjectMetric,
+  makeCounterProvider,
+  makeGaugeProvider,
+} from '@willsoto/nestjs-prometheus';
+
+/**
+ * Metric names exported via the global Prometheus registry.
+ * The local blockchain module only registers / observes these — the
+ * `MetricsModule` already exposes them on `/metrics` because the providers
+ * are imported by `SequenceManagerModule`.
+ */
+export const SOURCE_SUBMISSIONS_IN_FLIGHT =
+  'blockchain_source_submissions_in_flight';
+export const SOURCE_BAD_SEQ_RETRIES_TOTAL =
+  'blockchain_source_bad_seq_retries_total';
+export const SOURCE_SUBMISSIONS_TOTAL =
+  'blockchain_source_submissions_total';
+export const SOURCE_NEXT_SEQUENCE = 'blockchain_source_next_sequence';
+
+export const sequenceManagerMetricProviders = [
+  makeGaugeProvider({
+    name: SOURCE_SUBMISSIONS_IN_FLIGHT,
+    help: 'Number of server-signed Stellar transactions currently in flight, grouped by source account',
+    labelNames: ['source_account'] as const,
+  }),
+  makeCounterProvider({
+    name: SOURCE_BAD_SEQ_RETRIES_TOTAL,
+    help: 'Number of tx_bad_seq re-sync retries performed by the sequence manager, grouped by source account',
+    labelNames: ['source_account'] as const,
+  }),
+  makeCounterProvider({
+    name: SOURCE_SUBMISSIONS_TOTAL,
+    help: 'Number of server-signed Stellar transaction submission attempts, grouped by source account and outcome',
+    labelNames: ['source_account', 'outcome'] as const,
+  }),
+  makeGaugeProvider({
+    name: SOURCE_NEXT_SEQUENCE,
+    help: 'Next sequence number expected to be used by the sequence manager, grouped by source account',
+    labelNames: ['source_account'] as const,
+  }),
+];
+
+/**
+ * Thin helper for emitting sequence-manager metrics. Injectable so the
+ * service can be unit-tested without a Nest module, but registered as a
+ * global provider by `SequenceManagerModule`.
+ */
+@Injectable()
+export class SequenceMetrics {
+  constructor(
+    @Optional()
+    @InjectMetric(SOURCE_SUBMISSIONS_IN_FLIGHT)
+    private readonly inFlight?: Gauge<string>,
+    @Optional()
+    @InjectMetric(SOURCE_BAD_SEQ_RETRIES_TOTAL)
+    private readonly badSeqRetries?: Counter<string>,
+    @Optional()
+    @InjectMetric(SOURCE_SUBMISSIONS_TOTAL)
+    private readonly submissions?: Counter<string>,
+    @Optional()
+    @InjectMetric(SOURCE_NEXT_SEQUENCE)
+    private readonly nextSequence?: Gauge<string>,
+  ) {}
+
+  incInFlight(publicKey: string): void {
+    this.inFlight?.labels(publicKey).inc();
+  }
+
+  decInFlight(publicKey: string): void {
+    this.inFlight?.labels(publicKey).dec();
+  }
+
+  setInFlight(labeledCount: number): void {
+    this.inFlight?.set(labeledCount);
+  }
+
+  incBadSeqRetry(publicKey: string): void {
+    this.badSeqRetries?.labels(publicKey).inc();
+  }
+
+  resetBadSeqRetries(): void {
+    this.badSeqRetries?.reset();
+  }
+
+  incSubmitted(publicKey: string): void {
+    this.submissions?.labels(publicKey, 'success').inc();
+  }
+
+  incSubmissionError(publicKey: string, outcome: string): void {
+    this.submissions?.labels(publicKey, outcome).inc();
+  }
+
+  observeSequence(publicKey: string, sequence: bigint): void {
+    this.nextSequence?.labels(publicKey).set(Number(sequence));
+  }
+
+  /**
+   * Test helper — snapshots the labeled values of the bad-seq counter so
+   * unit tests can assert the metric actually incremented. Returns an
+   * empty array if the metric is unmocked in the test module.
+   */
+  async badSeqRetriesSnapshot(): Promise<
+    Array<{ labels: { source_account: string }; value: number }>
+  > {
+    if (!this.badSeqRetries) return [];
+    const metric = await this.badSeqRetries.get();
+    return metric.values.map((v) => ({
+      labels: v.labels as { source_account: string },
+      value: v.value,
+    }));
+  }
+}

src/blockchain/sequence-manager/sequence-manager.module.ts

@@ -0,0 +1,28 @@
+import { Global, Module } from '@nestjs/common';
+import { ConfigModule } from '@nestjs/config';
+import {
+  SequenceManagerService,
+} from './sequence-manager.service';
+import {
+  SequenceMetrics,
+  sequenceManagerMetricProviders,
+} from './sequence-manager.metrics';
+
+/**
+ * Global module so the sequence manager is available everywhere without a
+ * per-feature import dance. Metric providers attach to the prom-client
+ * global registry that `MetricsModule` exposes; we deliberately do NOT
+ * call `PrometheusModule.register()` here so we don't create a second
+ * Prometheus controller/registry.
+ */
+@Global()
+@Module({
+  imports: [ConfigModule],
+  providers: [
+    ...sequenceManagerMetricProviders,
+    SequenceMetrics,
+    SequenceManagerService,
+  ],
+  exports: [SequenceManagerService, SequenceMetrics],
+})
+export class SequenceManagerModule {}