core: implement Horizon failover with multiple RPC endpoints

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,30 +1,49 @@
-## 🔗 Related Issue
-Closes #issue-number
+## Summary
 
----
+Closes #[issue number]
 
-## 🔖 Title
-<!-- Brief and clear. Describe the specific task -->
+Briefly describe what this PR does in 2-3 sentences.
 
----
+## This repo is for the NestJS backend API only
 
-## 📝 Description
-<!-- Describe the changes made in this PR. What problem does it solve? -->
+Before submitting, confirm your changes belong here:
 
----
+- [ ] My changes are inside src/ or test/
+- [ ] I have NOT added React, React Native,
+      or frontend component files
+- [ ] I have NOT added Rust or Soroban contract code
+- [ ] This is NestJS/TypeScript backend work
 
-## 🔄 Changes Made
-<!-- List the main changes in this PR -->
-- [ ] <!-- Change 1 -->
-- [ ] <!-- Change 2 -->
-- [ ] <!-- Change 3 -->
+## Type of change
 
----
+- [ ] Bug fix
+- [ ] New endpoint
+- [ ] New service or module
+- [ ] Database migration
+- [ ] Background job
+- [ ] Test coverage
 
-## 📸 Screenshots (if applicable)
-<!-- Add screenshots or GIFs if the changes affect the UI -->
+## Testing
 
----
+- [ ] npm run build passes with zero TypeScript errors
+- [ ] npm test passes — all 184+ existing tests pass
+- [ ] No new `any` types introduced anywhere
+- [ ] Swagger decorators added to every new endpoint
+- [ ] Migration file created for any schema changes
+- [ ] New unit tests written for new service methods
 
-## 🗒️ Additional Notes
-<!-- Any additional information, concerns, or context for reviewers -->
+## Context files reviewed
+
+- [ ] context/architecture-context.md
+- [ ] context/code-standards.md
+- [ ] context/progress-tracker.md updated
+
+## Mandatory before requesting review
+
+Running these must all exit 0:
+npm run build
+npm test
+
+If either fails, fix it before opening this PR.
+PRs with failing CI checks will be closed without review.
+PRs that reduce the test count will be rejected.

src/modules/health/health.controller.ts

@@ -1,4 +1,4 @@
-import { Controller, Get } from '@nestjs/common';
+import { Controller, Get, ServiceUnavailableException } from '@nestjs/common';
 import { HealthService } from './health.service';
 import { ApiTags, ApiOperation, ApiResponse } from '@nestjs/swagger';
 
@@ -13,6 +13,9 @@ export class HealthController {
   @ApiResponse({ status: 503, description: 'One or more systems degraded' })
   async check() {
     const result = await this.healthService.check();
+    if (result.status !== 'ok') {
+      throw new ServiceUnavailableException(result);
+    }
     return result;
   }
 
@@ -21,7 +24,11 @@ export class HealthController {
   @ApiResponse({ status: 200, description: 'Database is connected' })
   @ApiResponse({ status: 503, description: 'Database connection failed' })
   async checkDatabase() {
-    return this.healthService.checkDatabaseMinimal();
+    const result = await this.healthService.checkDatabaseMinimal();
+    if (result.status !== 'ok') {
+      throw new ServiceUnavailableException(result);
+    }
+    return result;
   }
 
   @Get('sentry-test')

src/modules/health/health.module.ts

@@ -1,10 +1,14 @@
 import { Module } from '@nestjs/common';
+import { StellarModule } from '../../stellar/stellar.module';
 import { HealthController } from './health.controller';
 import { StellarTomlController } from './stellar-toml.controller';
 import { HealthService } from './health.service';
 import { SupabaseService } from '../../database/supabase.client';
 
 @Module({
+  imports: [
+    StellarModule,
+  ],
   controllers: [HealthController, StellarTomlController],
   providers: [HealthService, SupabaseService],
 })

src/modules/health/health.service.ts

@@ -1,27 +1,15 @@
 import { Injectable, Logger } from '@nestjs/common';
-import { ConfigService } from '@nestjs/config';
 import { SupabaseService } from '../../database/supabase.client';
-
-interface HorizonRoot {
-  horizon_version: string;
-  network: string;
-  core_version: string;
-  history_latest_ledger: number;
-}
+import { HorizonClientService } from '../../stellar/horizon-client.service';
 
 @Injectable()
 export class HealthService {
   private readonly logger = new Logger(HealthService.name);
-  private readonly horizonUrl: string;
 
   constructor(
-    private readonly configService: ConfigService,
+    private readonly horizonClientService: HorizonClientService,
     private readonly supabaseService: SupabaseService,
-  ) {
-    this.horizonUrl =
-      this.configService.get<string>('STELLAR_HORIZON_URL') ||
-      'https://horizon-testnet.stellar.org';
-  }
+  ) {}
 
   async check() {
     const [db, horizon, indexer] = await Promise.all([
@@ -49,57 +37,76 @@ export class HealthService {
   async checkDatabase() {
     try {
       const client = this.supabaseService.getClient();
+      if (!client) {
+        throw new Error('Supabase client is unavailable');
+      }
       const { error } = await client.auth.getSession();
       if (error && error.message !== 'Invalid Refresh Token' && !error.message.includes('JWT')) {
         throw error;
       }
-      return { status: 'ok', database: 'connected', message: 'Supabase reachable' };
+      return {
+        status: 'ok',
+        database: 'connected',
+        message: 'Supabase reachable',
+        timestamp: new Date().toISOString(),
+      };
     } catch (error) {
-      this.logger.error({ context: 'HealthService', action: 'checkDatabase', error: error.message });
-      return { status: 'error', database: 'disconnected', message: error.message };
+      const errorMessage =
+        error instanceof Error
+          ? error.message
+          : typeof error === 'object' && error && 'message' in error
+            ? String((error as { message: unknown }).message)
+            : String(error);
+      this.logger.error({ context: 'HealthService', action: 'checkDatabase', error: errorMessage });
+      return {
+        status: 'error',
+        database: 'disconnected',
+        message: errorMessage,
+        timestamp: new Date().toISOString(),
+      };
     }
   }
 
   async checkHorizon(): Promise<{ status: string; [key: string]: unknown }> {
     try {
-      const root = await this.fetchHorizonRoot();
+      const root = await this.horizonClientService.getRoot();
+      const endpoints = this.horizonClientService.getEndpointStatuses();
+      const healthyPrimary = endpoints.some((e) => e.isPrimary && e.status === 'closed');
       return {
-        status: 'ok',
+        status: healthyPrimary ? 'ok' : 'degraded',
         horizon: root.horizon_version,
         network: root.network,
         protocolVersion: root.core_version,
+        endpoints,
       };
     } catch (error) {
-      this.logger.error({ context: 'HealthService', action: 'checkHorizon', error: error.message });
-      return { status: 'error', horizon: 'unreachable', message: error.message };
+      this.logger.error({ context: 'HealthService', action: 'checkHorizon', error: (error as Error).message });
+      return {
+        status: 'error',
+        horizon: 'unreachable',
+        message: (error as Error).message,
+        endpoints: this.horizonClientService.getEndpointStatuses(),
+      };
     }
   }
 
   async checkIndexerLag(): Promise<{ status: string; [key: string]: unknown }> {
     try {
       const cursor = await this.getIndexerCursor();
-      const root = await this.fetchHorizonRoot();
+      const root = await this.horizonClientService.getRoot();
       const latestLedger = root.history_latest_ledger;
       const lag = latestLedger - cursor;
       const status = lag < 100 ? 'ok' : lag < 500 ? 'warning' : 'error';
       return { status, cursor, latestLedger, lag };
     } catch (error) {
-      return { status: 'unknown', message: error.message };
+      return { status: 'unknown', message: (error as Error).message };
     }
   }
 
   async checkDatabaseMinimal() {
     return this.checkDatabase();
   }
 
-  private async fetchHorizonRoot(): Promise<HorizonRoot> {
-    const response = await fetch(this.horizonUrl);
-    if (!response.ok) {
-      throw new Error(`Horizon returned ${response.status}`);
-    }
-    return response.json() as Promise<HorizonRoot>;
-  }
-
   private async getIndexerCursor(): Promise<number> {
     try {
       const db = this.supabaseService.getServiceRoleClient();
@@ -114,4 +121,4 @@ export class HealthService {
       return 0;
     }
   }
-}
+}