From 5625d73eb67e4d45b90be87180959f1c254656fc Mon Sep 17 00:00:00 2001 From: Victor Fusco <1221933+vfusco@users.noreply.github.com> Date: Wed, 12 Aug 2026 17:13:39 -0300 Subject: [PATCH 01/13] refactor(machine): model completed requests explicitly --- internal/manager/instance.go | 86 +++++--- internal/manager/instance_test.go | 118 +++++----- internal/model/models.go | 1 + pkg/machine/implementation.go | 336 ++++++++++++++++------------- pkg/machine/implementation_test.go | 333 ++++++++++++---------------- pkg/machine/machine.go | 59 +++-- pkg/machine/machine_test.go | 52 +++-- 7 files changed, 520 insertions(+), 465 deletions(-) diff --git a/internal/manager/instance.go b/internal/manager/instance.go index caed1509c..1df5d58aa 100644 --- a/internal/manager/instance.go +++ b/internal/manager/instance.go @@ -24,6 +24,8 @@ import ( var ( ErrMachineClosed = errors.New("machine is closed") ErrInvalidInputIndex = errors.New("invalid input index") + ErrIncompleteAdvance = errors.New("machine advance returned no completed result") + ErrIncompleteInspect = errors.New("machine inspect returned no completed result") ErrInvalidSnapshotPoint = errors.New("invalid snapshot point") ErrInvalidApplication = errors.New("application must not be nil") ErrInvalidAdvanceTimeout = errors.New("advance timeout must not be negative") @@ -301,7 +303,16 @@ func (m *MachineInstanceImpl) Advance(ctx context.Context, input []byte, epochIn // Process the input advanceResp, err := fork.Advance(advanceCtx, input, prevMachineHash, computeHashes) - status, err := toInputStatus(advanceResp.Accepted, err) + if err != nil { + return nil, errors.Join(err, fork.Close()) + } + if advanceResp == nil { + return nil, errors.Join(ErrIncompleteAdvance, fork.Close()) + } + if err := validateCompletionExceptionData(advanceResp.Status, advanceResp.ExceptionData); err != nil { + return nil, errors.Join(ErrIncompleteAdvance, err, fork.Close()) + } + status, err := toInputStatus(advanceResp.Status) if err != nil { return nil, errors.Join(err, fork.Close()) } @@ -313,6 +324,7 @@ func (m *MachineInstanceImpl) Advance(ctx context.Context, input []byte, epochIn Status: status, Outputs: advanceResp.Outputs, Reports: advanceResp.Reports, + ExceptionData: advanceResp.ExceptionData, Hashes: advanceResp.Hashes, RemainingMetaCycles: advanceResp.RemainingCycles, IsDaveConsensus: computeHashes, @@ -401,14 +413,31 @@ func (m *MachineInstanceImpl) Inspect(ctx context.Context, query []byte) (*Inspe defer cancel() // Process the query - accepted, reports, inspectErr := fork.Inspect(inspectCtx, query) + inspectResponse, inspectErr := fork.Inspect(inspectCtx, query) // Create the result result := &InspectResult{ ProcessedInputs: processedInputs, - Accepted: accepted, - Reports: reports, - Error: inspectErr, + } + if inspectResponse != nil { + result.Reports = inspectResponse.Reports + } + if inspectErr != nil { + result.Error = inspectErr + } else if inspectResponse == nil || !inspectResponse.Status.IsCompleted() { + result.Error = errors.Join(ErrIncompleteInspect, machine.ErrMachineInternal) + } else { + switch inspectResponse.Status { + case machine.CompletionStatusAccepted: + result.Accepted = true + case machine.CompletionStatusRejected: + case machine.CompletionStatusException: + result.Error = machine.ErrException + case machine.CompletionStatusHalted: + result.Error = machine.ErrHalted + default: + result.Error = errors.Join(ErrIncompleteInspect, machine.ErrMachineInternal) + } } // Close the fork @@ -719,34 +748,31 @@ func (f *SnapshotMachineRuntimeFactory) CreateMachineRuntime( // Default factory instance var defaultFactory MachineRuntimeFactory = &DefaultMachineRuntimeFactory{} -// Helper function to convert machine response to input status -func toInputStatus(accepted bool, err error) (status InputCompletionStatus, _ error) { - if err == nil { - if accepted { - return InputCompletionStatus_Accepted, nil - } else { - return InputCompletionStatus_Rejected, nil - } +// toInputStatus converts only completed machine statuses to input statuses. +func toInputStatus(status machine.CompletionStatus) (InputCompletionStatus, error) { + switch status { + case machine.CompletionStatusAccepted: + return InputCompletionStatus_Accepted, nil + case machine.CompletionStatusRejected: + return InputCompletionStatus_Rejected, nil + case machine.CompletionStatusException: + return InputCompletionStatus_Exception, nil + case machine.CompletionStatusHalted: + return InputCompletionStatus_MachineHalted, nil + default: + return InputCompletionStatus_None, fmt.Errorf( + "unknown completed machine status %d: %w", status, ErrIncompleteAdvance, + ) } +} +func validateCompletionExceptionData(status machine.CompletionStatus, data []byte) error { switch { - case errors.Is(err, machine.ErrException): - return InputCompletionStatus_Exception, nil - case errors.Is(err, machine.ErrHalted): - return InputCompletionStatus_MachineHalted, nil - case errors.Is(err, machine.ErrOutputsLimitExceeded): - return InputCompletionStatus_OutputsLimitExceeded, nil - case errors.Is(err, machine.ErrReportsLimitExceeded): - return InputCompletionStatus_ReportsLimitExceeded, nil - case errors.Is(err, machine.ErrReachedTargetMcycle): - return InputCompletionStatus_CycleLimitExceeded, nil - case errors.Is(err, machine.ErrPayloadLengthLimitExceeded): - return InputCompletionStatus_PayloadLengthLimitExceeded, nil - case errors.Is(err, machine.ErrDeadlineExceeded): - return InputCompletionStatus_TimeLimitExceeded, nil - case errors.Is(err, machine.ErrMachineInternal): - fallthrough + case status == machine.CompletionStatusException && data == nil: + return fmt.Errorf("completed exception has no exception data: %w", machine.ErrMachineInternal) + case status != machine.CompletionStatusException && data != nil: + return fmt.Errorf("completion status %d unexpectedly has exception data: %w", status, machine.ErrMachineInternal) default: - return status, err + return nil } } diff --git a/internal/manager/instance_test.go b/internal/manager/instance_test.go index 6f758d812..9233fe1b4 100644 --- a/internal/manager/instance_test.go +++ b/internal/manager/instance_test.go @@ -30,8 +30,8 @@ type MachineInstanceSuite struct{ suite.Suite } func (s *MachineInstanceSuite) TestMcycleOverflowRemainsIncomplete() { require := s.Require() - status, err := toInputStatus(false, machine.ErrReachedLimitMcycle) - require.ErrorIs(err, machine.ErrReachedLimitMcycle) + status, err := toInputStatus(machine.CompletionStatusUnknown) + require.ErrorIs(err, ErrIncompleteAdvance) require.Empty(status) } @@ -334,31 +334,34 @@ func (s *MachineInstanceSuite) TestAdvance() { s.Run("Reject", func() { require := s.Require() - inner, fork, machine := s.setupAdvance() - fork.AdvanceAcceptedReturn = false + inner, fork, machineInst := s.setupAdvance() + fork.AdvanceStatusReturn = machine.CompletionStatusRejected fork.CloseError = nil - res, err := machine.Advance(context.Background(), []byte{}, 0, 5, false) + res, err := machineInst.Advance(context.Background(), []byte{}, 0, 5, false) require.Nil(err) require.NotNil(res) - require.Same(inner, machine.runtime) + require.Same(inner, machineInst.runtime) require.Equal(model.InputCompletionStatus_Rejected, res.Status) require.Equal(expectedOutputs, res.Outputs) require.Equal(expectedReports1, res.Reports) require.Equal(newHash(1), res.OutputsHash) require.Equal(newHash(2), res.MachineHash) - require.Equal(uint64(6), machine.processedInputs.Load()) + require.Equal(uint64(6), machineInst.processedInputs.Load()) }) - testSoftError := func(name string, err error, status model.InputCompletionStatus) { + testCompletion := func(name string, completion machine.CompletionStatus, status model.InputCompletionStatus) { s.Run(name, func() { require := s.Require() - inner, fork, machine := s.setupAdvance() - fork.AdvanceError = err + inner, fork, machineInst := s.setupAdvance() + fork.AdvanceStatusReturn = completion + if completion == machine.CompletionStatusException { + fork.AdvanceExceptionData = []byte("exception data") + } fork.CloseError, inner.CloseError = inner.CloseError, fork.CloseError - res, err := machine.Advance(context.Background(), []byte{}, 0, 5, false) + res, err := machineInst.Advance(context.Background(), []byte{}, 0, 5, false) require.Nil(err) require.NotNil(res) @@ -367,37 +370,17 @@ func (s *MachineInstanceSuite) TestAdvance() { require.Equal(expectedReports1, res.Reports) require.Equal(newHash(1), res.OutputsHash) require.Equal(newHash(2), res.MachineHash) - require.Equal(uint64(6), machine.processedInputs.Load()) + require.Equal(uint64(6), machineInst.processedInputs.Load()) }) } - testSoftError("Exception", - machine.ErrException, + testCompletion("Exception", + machine.CompletionStatusException, model.InputCompletionStatus_Exception) - testSoftError("Halted", - machine.ErrHalted, + testCompletion("Halted", + machine.CompletionStatusHalted, model.InputCompletionStatus_MachineHalted) - - testSoftError("OutputsLimit", - machine.ErrOutputsLimitExceeded, - model.InputCompletionStatus_OutputsLimitExceeded) - - testSoftError("ReportsLimit", - machine.ErrReportsLimitExceeded, - model.InputCompletionStatus_ReportsLimitExceeded) - - testSoftError("ReachedTargetMcycle", - machine.ErrReachedTargetMcycle, - model.InputCompletionStatus_CycleLimitExceeded) - - testSoftError("TimeLimit", - machine.ErrDeadlineExceeded, - model.InputCompletionStatus_TimeLimitExceeded) - - testSoftError("PayloadLengthLimit", - machine.ErrPayloadLengthLimitExceeded, - model.InputCompletionStatus_PayloadLengthLimitExceeded) }) s.Run("Error", func() { @@ -498,7 +481,8 @@ func (s *MachineInstanceSuite) TestAdvance() { s.Run("Fork", func() { require := s.Require() _, fork, machineInst := s.setupAdvance() - fork.AdvanceError = machine.ErrException + fork.AdvanceStatusReturn = machine.CompletionStatusException + fork.AdvanceExceptionData = []byte("exception data") fork.CloseError = errors.New("Close error") // Close error on fork is logged, not propagated. @@ -513,16 +497,16 @@ func (s *MachineInstanceSuite) TestAdvance() { s.Run("CollectHashes", func() { require := s.Require() - inner, fork, machine := s.setupAdvance() + inner, fork, machineInst := s.setupAdvance() - res, err := machine.Advance(context.Background(), []byte{}, 0, 5, true) + res, err := machineInst.Advance(context.Background(), []byte{}, 0, 5, true) require.Nil(err) require.NotNil(res) - require.Same(fork, machine.runtime) + require.Same(fork, machineInst.runtime) require.Equal(model.InputCompletionStatus_Accepted, res.Status) require.True(res.IsDaveConsensus) - require.Equal(uint64(6), machine.processedInputs.Load()) + require.Equal(uint64(6), machineInst.processedInputs.Load()) // Verify the inner runtime was closed (accept path) _ = inner @@ -533,7 +517,7 @@ func (s *MachineInstanceSuite) TestAdvance() { // same machine never happens by design. This test verifies that two // sequential advances correctly increment processedInputs. require := s.Require() - inner, fork, machine := s.setupAdvance() + inner, fork, machineInst := s.setupAdvance() // Allow inner.Close to succeed (old runtime close on accept) inner.CloseError = nil @@ -542,7 +526,7 @@ func (s *MachineInstanceSuite) TestAdvance() { // After accept, fork becomes the new runtime. // Second advance: fork from fork (processedInputs=6), fork must also fork. fork2 := &MockRollupsMachine{} - fork2.AdvanceAcceptedReturn = true + fork2.AdvanceStatusReturn = machine.CompletionStatusAccepted fork2.AdvanceOutputsReturn = expectedOutputs fork2.AdvanceReportsReturn = expectedReports1 fork2.OutputsHashReturn = newHash(1) @@ -553,16 +537,16 @@ func (s *MachineInstanceSuite) TestAdvance() { fork.CloseError = nil // close of fork (now old runtime) in second advance // First advance at index 5 - res1, err := machine.Advance(context.Background(), []byte{}, 0, 5, false) + res1, err := machineInst.Advance(context.Background(), []byte{}, 0, 5, false) require.Nil(err) require.NotNil(res1) - require.Equal(uint64(6), machine.processedInputs.Load()) + require.Equal(uint64(6), machineInst.processedInputs.Load()) // Second advance at index 6 - res2, err := machine.Advance(context.Background(), []byte{}, 0, 6, false) + res2, err := machineInst.Advance(context.Background(), []byte{}, 0, 6, false) require.Nil(err) require.NotNil(res2) - require.Equal(uint64(7), machine.processedInputs.Load()) + require.Equal(uint64(7), machineInst.processedInputs.Load()) }) } @@ -585,14 +569,14 @@ func (s *MachineInstanceSuite) TestInspect() { s.Run("Reject", func() { require := s.Require() - _, fork, machine := s.setupInspect() - fork.InspectAcceptedReturn = false + _, fork, machineInst := s.setupInspect() + fork.InspectStatusReturn = machine.CompletionStatusRejected - res, err := machine.Inspect(context.Background(), []byte{}) + res, err := machineInst.Inspect(context.Background(), []byte{}) require.Nil(err) require.NotNil(res) - require.NotSame(fork, machine.runtime) + require.NotSame(fork, machineInst.runtime) require.Equal(uint64(55), res.ProcessedInputs) require.False(res.Accepted) require.Equal(expectedReports2, res.Reports) @@ -1036,7 +1020,7 @@ func (s *MachineInstanceSuite) setupAdvance() (*MockRollupsMachine, *MockRollups inner.ForkReturn = fork inner.CloseError = nil - fork.AdvanceAcceptedReturn = true + fork.AdvanceStatusReturn = machine.CompletionStatusAccepted fork.AdvanceOutputsReturn = []machine.Output{ newBytes(11, 100), newBytes(12, 100), @@ -1052,7 +1036,7 @@ func (s *MachineInstanceSuite) setupAdvance() (*MockRollupsMachine, *MockRollups fork.HashReturn = newHash(2) fork.HashError = nil - fork.InspectAcceptedReturn = true + fork.InspectStatusReturn = machine.CompletionStatusAccepted fork.InspectReportsReturn = []machine.Report{ newBytes(31, 300), newBytes(32, 300), @@ -1096,7 +1080,7 @@ func (s *MachineInstanceSuite) setupInspect() (*MockRollupsMachine, *MockRollups fork.AdvanceError = errUnreachable fork.HashError = errUnreachable - fork.InspectAcceptedReturn = true + fork.InspectStatusReturn = machine.CompletionStatusAccepted fork.InspectReportsReturn = []machine.Report{ newBytes(31, 300), newBytes(32, 300), @@ -1236,7 +1220,7 @@ func (r *mockSyncRepository) GetLastSnapshot( func newForkableMock() *MockRollupsMachine { m := &MockRollupsMachine{} m.CloseError = nil - m.AdvanceAcceptedReturn = true + m.AdvanceStatusReturn = machine.CompletionStatusAccepted m.HashReturn = newHash(1) m.OutputsHashReturn = newHash(2) m.ForkFunc = func(_ context.Context) (machine.Machine, error) { @@ -1466,7 +1450,8 @@ type MockRollupsMachine struct { HashReturn machine.Hash HashError error - AdvanceAcceptedReturn bool + AdvanceStatusReturn machine.CompletionStatus + AdvanceExceptionData []byte AdvanceOutputsReturn []machine.Output AdvanceReportsReturn []machine.Report AdvanceLeafsReturn []machine.Hash @@ -1477,9 +1462,9 @@ type MockRollupsMachine struct { OutputsHashProofError error AdvanceError error - InspectAcceptedReturn bool - InspectReportsReturn []machine.Report - InspectError error + InspectStatusReturn machine.CompletionStatus + InspectReportsReturn []machine.Report + InspectError error StoreError error @@ -1506,18 +1491,25 @@ func (m *MockRollupsMachine) OutputsHashProof(_ context.Context) ([]machine.Hash } func (m *MockRollupsMachine) Advance(_ context.Context, _ []byte, _ machine.Hash, _ bool) (*machine.AdvanceResponse, error) { + if m.AdvanceError != nil { + return nil, m.AdvanceError + } return &machine.AdvanceResponse{ - Accepted: m.AdvanceAcceptedReturn, + Status: m.AdvanceStatusReturn, Outputs: m.AdvanceOutputsReturn, Reports: m.AdvanceReportsReturn, + ExceptionData: m.AdvanceExceptionData, Hashes: m.AdvanceLeafsReturn, RemainingCycles: m.AdvanceRemainingReturn, OutputsHash: m.OutputsHashReturn, - }, m.AdvanceError + }, nil } -func (m *MockRollupsMachine) Inspect(_ context.Context, _ []byte) (bool, []machine.Report, error) { - return m.InspectAcceptedReturn, m.InspectReportsReturn, m.InspectError +func (m *MockRollupsMachine) Inspect(_ context.Context, _ []byte) (*machine.InspectResponse, error) { + if m.InspectError != nil { + return nil, m.InspectError + } + return &machine.InspectResponse{Status: m.InspectStatusReturn, Reports: m.InspectReportsReturn}, nil } func (m *MockRollupsMachine) Store(_ context.Context, _ string) error { diff --git a/internal/model/models.go b/internal/model/models.go index e902deedb..b818ca527 100644 --- a/internal/model/models.go +++ b/internal/model/models.go @@ -1340,6 +1340,7 @@ type AdvanceResult struct { Status InputCompletionStatus Outputs [][]byte Reports [][]byte + ExceptionData []byte Hashes [][32]byte RemainingMetaCycles uint64 IsDaveConsensus bool diff --git a/pkg/machine/implementation.go b/pkg/machine/implementation.go index 345580058..2db43b019 100644 --- a/pkg/machine/implementation.go +++ b/pkg/machine/implementation.go @@ -23,12 +23,27 @@ const ( InspectStateRequest requestType = 0x1 ) -type yieldType uint8 +type runResult struct { + outputs []Output + reports []Report + periodicStateHashes []Hash + paddingRepetitions uint64 +} -const ( - AutomaticYield yieldType = 0x0 - ManualYield yieldType = 0x1 -) +type processResult struct { + runResult + completion completionResult +} + +type incrementResult struct { + breakReason BreakReason + currentCycle Cycle +} + +type completionResult struct { + status CompletionStatus + data []byte +} type automaticYieldReason uint16 @@ -131,24 +146,36 @@ func (m *machineImpl) Hash(ctx context.Context) (Hash, error) { // OutputsHash returns the outputs hash stored in the cmio tx buffer func (m *machineImpl) OutputsHash(ctx context.Context) (Hash, error) { - accepted, data, err := m.wasLastRequestAccepted(ctx) + result, err := m.readManualYieldResult(ctx) if err != nil { err = fmt.Errorf("could not read the outputs hash: %w", err) return Hash{}, err } - if !accepted { - err = fmt.Errorf("could not read the outputs hash: machine manual yield reason is not accepted") - return Hash{}, err - } - - if length := len(data); length != HashSize { + switch result.status { + case CompletionStatusAccepted: + // Intentionally empty. + case CompletionStatusRejected: + return Hash{}, fmt.Errorf("could not read the outputs hash: %w", ErrRejected) + case CompletionStatusException: + return Hash{}, fmt.Errorf("could not read the outputs hash: %w", ErrException) + case CompletionStatusHalted: + return Hash{}, fmt.Errorf("could not read the outputs hash: %w", ErrHalted) + case CompletionStatusUnknown: + return Hash{}, fmt.Errorf( + "could not read the outputs hash with completion status %d: %w", + result.status, + ErrMachineInternal, + ) + } + + if length := len(result.data); length != HashSize { err = fmt.Errorf("invalid outputs hash: %w (it has %d bytes)", ErrHashLength, length) return Hash{}, err } var outputsHash Hash - copy(outputsHash[:], data) + copy(outputsHash[:], result.data) return outputsHash, nil } @@ -166,41 +193,59 @@ func (m *machineImpl) OutputsHashProof(ctx context.Context) ([]Hash, error) { // Advance sends an input to the machine and processes it func (m *machineImpl) Advance(ctx context.Context, input []byte, checkpointHash Hash, computeHashes bool) (*AdvanceResponse, error) { - // TODO: return the exception reason - accepted, outputs, reports, hashes, remaining, data, err := m.process(ctx, input, AdvanceStateRequest, &checkpointHash, computeHashes) + result, err := m.process(ctx, input, AdvanceStateRequest, &checkpointHash, computeHashes) if err != nil { - return &AdvanceResponse{ - Accepted: accepted, - Outputs: outputs, - Reports: reports, - Hashes: hashes, - RemainingCycles: remaining, - }, err + return nil, err + } + if !result.completion.status.IsCompleted() { + return nil, fmt.Errorf( + "invalid completed advance status %d: %w", + result.completion.status, + ErrMachineInternal, + ) } resp := &AdvanceResponse{ - Accepted: accepted, - Outputs: outputs, - Reports: reports, - Hashes: hashes, - RemainingCycles: remaining, + Status: result.completion.status, + Outputs: result.outputs, + Reports: result.reports, + Hashes: result.periodicStateHashes, + RemainingCycles: result.paddingRepetitions, } - if accepted { - if length := len(data); length != HashSize { - return resp, fmt.Errorf("%w (it has %d bytes)", ErrHashLength, length) + if resp.Status == CompletionStatusAccepted { + if length := len(result.completion.data); length != HashSize { + return nil, fmt.Errorf("%w (it has %d bytes)", ErrHashLength, length) } - copy(resp.OutputsHash[:], data) + copy(resp.OutputsHash[:], result.completion.data) + } else if resp.Status == CompletionStatusException { + resp.ExceptionData = append([]byte{}, result.completion.data...) } return resp, nil } -// Inspect sends a query to the machine and returns the results -func (m *machineImpl) Inspect(ctx context.Context, query []byte) (bool, []Report, error) { - // TODO: return the exception reason +// Inspect sends a query to the machine and returns the results. +func (m *machineImpl) Inspect(ctx context.Context, query []byte) (*InspectResponse, error) { // For inspect-state requests, revert_root_hash is not checked and can be NULL/empty - accepted, _, reports, _, _, _, err := m.process(ctx, query, InspectStateRequest, nil, false) - return accepted, reports, err + result, err := m.process(ctx, query, InspectStateRequest, nil, false) + if err != nil { + return nil, err + } + if !result.completion.status.IsCompleted() { + return nil, fmt.Errorf( + "invalid completed inspect status %d: %w", + result.completion.status, + ErrMachineInternal, + ) + } + response := &InspectResponse{ + Status: result.completion.status, + Reports: result.reports, + } + if response.Status == CompletionStatusException { + response.ExceptionData = append([]byte{}, result.completion.data...) + } + return response, nil } // Store saves the machine state to the specified path @@ -284,30 +329,28 @@ func (m *machineImpl) isAtManualYield(ctx context.Context) (bool, error) { return isAtManualYield, nil } -// wasLastRequestAccepted returns true if the last request was accepted and false otherwise. -// It returns the ErrException error if the last request yielded an exception. -// -// The machine MUST be at a manual yield when calling this function. -func (m *machineImpl) wasLastRequestAccepted(ctx context.Context) (bool, []byte, error) { +// readManualYieldResult returns the typed completion status and data reported by +// the current manual yield. The machine MUST be at a manual yield when called. +func (m *machineImpl) readManualYieldResult(ctx context.Context) (completionResult, error) { if err := checkContext(ctx); err != nil { - return false, nil, err + return completionResult{}, err } _, yieldReason, data, err := m.backend.ReceiveCmioRequest(m.params.FastDeadline) if err != nil { - return false, nil, err + return completionResult{}, err } switch manualYieldReason(yieldReason) { case ManualYieldReasonAccepted: - return true, data, nil + return completionResult{status: CompletionStatusAccepted, data: data}, nil case ManualYieldReasonRejected: - return false, data, nil + return completionResult{status: CompletionStatusRejected, data: data}, nil case ManualYieldReasonException: - return false, data, ErrException + return completionResult{status: CompletionStatusException, data: data}, nil default: err = fmt.Errorf("invalid manual yield reason: %d: %w", yieldReason, ErrMachineInternal) - return false, nil, err + return completionResult{}, err } } @@ -337,47 +380,64 @@ func (m *machineImpl) process( reqType requestType, checkpointHash *Hash, computeHashes bool, -) (bool, []Output, []Report, []Hash, uint64, []byte, error) { +) (processResult, error) { if err := checkContext(ctx); err != nil { - return false, nil, nil, nil, 0, nil, err + return processResult{}, err } // Check payload length limit if length := uint64(len(request)); length > m.backend.CmioRxBufferSize() { - return false, nil, nil, nil, 0, nil, ErrPayloadLengthLimitExceeded + return processResult{}, ErrPayloadLengthLimitExceeded } - err := m.backend.SendCmioResponse(uint16(reqType), request, checkpointHash, m.params.FastDeadline) + currentCycle, err := m.readMCycle(ctx) if err != nil { - return false, nil, nil, nil, 0, nil, err + return processResult{}, err + } + limitCycle := currentCycle + m.params.AdvanceMaxCycles + if reqType == InspectStateRequest { + limitCycle = currentCycle + m.params.InspectMaxCycles } - outputs, reports, hashes, remaining, err := m.run(ctx, reqType, computeHashes) + err = m.backend.SendCmioResponse(uint16(reqType), request, checkpointHash, m.params.FastDeadline) if err != nil { - return false, outputs, reports, nil, 0, nil, err + return processResult{}, err + } + + execution, err := m.run(ctx, reqType, computeHashes, currentCycle, limitCycle) + result := processResult{runResult: execution} + switch { + case err == nil: + manualResult, err := m.readManualYieldResult(ctx) + result.completion = manualResult + return result, err + case errors.Is(err, ErrMachineInternal): + return result, err + case errors.Is(err, ErrHalted): + result.completion.status = CompletionStatusHalted + return result, nil + default: + return result, err } - - accepted, data, err := m.wasLastRequestAccepted(ctx) - - return accepted, outputs, reports, hashes, remaining, data, err } -// run runs the machine until it manually yields. -// It returns any collected responses. -func (m *machineImpl) run(ctx context.Context, reqType requestType, computeHashes bool) ([]Output, []Report, []Hash, uint64, error) { +// run executes a request between explicit cycle bounds and returns any +// responses collected before it reaches a fixed point. +func (m *machineImpl) run( + ctx context.Context, + reqType requestType, + computeHashes bool, + currentCycle uint64, + limitCycle uint64, +) (runResult, error) { startTime := time.Now() - currentCycle, err := m.readMCycle(ctx) - if err != nil { - return nil, nil, nil, 0, err - } - - limitCycle := currentCycle + m.params.AdvanceMaxCycles stepTimeout := m.params.AdvanceIncDeadline runTimeout := m.params.AdvanceMaxDeadline + increment := m.params.AdvanceIncCycles if reqType == InspectStateRequest { - limitCycle = currentCycle + m.params.InspectMaxCycles stepTimeout = m.params.InspectIncDeadline runTimeout = m.params.InspectMaxDeadline + increment = m.params.InspectIncCycles } m.logger.Debug("run", @@ -385,8 +445,10 @@ func (m *machineImpl) run(ctx context.Context, reqType requestType, computeHashe "limitCycle", limitCycle, "leftover", limitCycle-currentCycle) - outputs := make([]Output, 0, 16) //nolint:mnd - reports := make([]Report, 0, 16) //nolint:mnd + result := runResult{ + outputs: make([]Output, 0, 16), //nolint:mnd + reports: make([]Report, 0, 16), //nolint:mnd + } var hashCollectorState *HashCollectorState if computeHashes { @@ -398,58 +460,54 @@ func (m *machineImpl) run(ctx context.Context, reqType requestType, computeHashe Hashes: []Hash{}, } } - hashes := func() []Hash { - if computeHashes { - return hashCollectorState.Hashes - } - return []Hash{} - } - remainingMetaCycles := func() uint64 { - if computeHashes { - return StrideCountInInput - uint64(len(hashCollectorState.Hashes)) + finish := func(runErr error) (runResult, error) { + if hashCollectorState != nil { + result.periodicStateHashes = hashCollectorState.Hashes + result.paddingRepetitions = StrideCountInInput - uint64(len(hashCollectorState.Hashes)) } - return 0 + return result, runErr } for { - var yt *yieldType - var err error - - // Steps the machine as many times as needed until it manually/automatically yields. - for yt == nil { - if err := checkContext(ctx); err != nil { - return outputs, reports, hashes(), remainingMetaCycles(), err - } - if time.Since(startTime) > runTimeout { - werr := fmt.Errorf("run operation timed out: %w", ErrDeadlineExceeded) - return outputs, reports, hashes(), remainingMetaCycles(), werr - } - yt, currentCycle, err = m.runIncrementInterval(ctx, currentCycle, limitCycle, hashCollectorState, stepTimeout) - if err != nil && err != ErrReachedTargetMcycle { - return outputs, reports, hashes(), remainingMetaCycles(), err - } + if err := checkContext(ctx); err != nil { + return finish(err) + } + if time.Since(startTime) > runTimeout { + return finish(fmt.Errorf("run operation timed out: %w", ErrDeadlineExceeded)) } - // Returns with the responses when the machine manually yields. - if *yt == ManualYield { - return outputs, reports, hashes(), remainingMetaCycles(), nil + interval, err := m.runIncrementInterval( + ctx, currentCycle, limitCycle, increment, hashCollectorState, stepTimeout, + ) + currentCycle = interval.currentCycle + if err != nil { + return finish(err) } - // Asserts the machine yielded automatically. - if *yt != AutomaticYield { - err := fmt.Errorf("invalid yield type: %d: %w", *yt, ErrMachineInternal) - return outputs, reports, hashes(), remainingMetaCycles(), err + switch interval.breakReason { + case YieldedManually: + return finish(nil) + case Halted: + return finish(ErrHalted) + case McycleOverflow: + return finish(ErrReachedLimitMcycle) + case ReachedTargetMcycle, YieldedSoftly: + continue + case Failed: + return finish(ErrMachineInternal) + case YieldedAutomatically: + default: + return finish(fmt.Errorf("invalid break reason: %d: %w", interval.breakReason, ErrMachineInternal)) } - yt = nil if err := checkContext(ctx); err != nil { - return outputs, reports, hashes(), remainingMetaCycles(), err + return finish(err) } _, yieldReason, data, err := m.backend.ReceiveCmioRequest(m.params.FastDeadline) if err != nil { werr := fmt.Errorf("could not read output/report: %w", err) - return outputs, reports, hashes(), remainingMetaCycles(), werr + return finish(werr) } switch automaticYieldReason(yieldReason) { @@ -457,55 +515,51 @@ func (m *machineImpl) run(ctx context.Context, reqType requestType, computeHashe m.logger.Debug("ignoring yield reason progress", "value", fmt.Sprintf("%v", data)) case AutomaticYieldReasonOutput: // TODO: should we remove this? - if len(outputs) == maxOutputs { - return outputs, reports, hashes(), remainingMetaCycles(), ErrOutputsLimitExceeded + if len(result.outputs) == maxOutputs { + return finish(ErrOutputsLimitExceeded) } - outputs = append(outputs, data) + result.outputs = append(result.outputs, data) case AutomaticYieldReasonReport: - if len(reports) == maxReports { - return outputs, reports, hashes(), remainingMetaCycles(), ErrReportsLimitExceeded + if len(result.reports) == maxReports { + return finish(ErrReportsLimitExceeded) } - reports = append(reports, data) + result.reports = append(result.reports, data) default: err := fmt.Errorf("invalid automatic yield reason: %d: %w", yieldReason, ErrMachineInternal) - return outputs, reports, hashes(), remainingMetaCycles(), err + return finish(err) } } } -// runIncrementInterval runs the machine for at most machine.inc cycles (or the amount of cycles left to reach -// limitCycle, whichever is the lowest). -// It returns the yield type and the machine cycle after the increment interval. -// If the machine did not manually/automatically yield, the yield type will be nil (meaning runIncrementInterval -// must be called again to complete the computation). +// runIncrementInterval runs the machine for at most incrementLimit mcycles and +// preserves the emulator's typed break reason for the request loop. func (m *machineImpl) runIncrementInterval(ctx context.Context, currentCycle Cycle, limitCycle Cycle, + incrementLimit Cycle, hashCollectorState *HashCollectorState, timeout time.Duration, -) (*yieldType, Cycle, error) { +) (incrementResult, error) { startingCycle := currentCycle - // Returns with an error if the next run would exceed limitCycle. - if currentCycle >= limitCycle && m.params.AdvanceIncCycles != 0 { - return nil, 0, ErrReachedLimitMcycle + if currentCycle >= limitCycle { + return incrementResult{currentCycle: currentCycle}, ErrReachedLimitMcycle } - // Calculates the increment. - increment := min(m.params.AdvanceIncCycles, limitCycle-currentCycle) + increment := min(incrementLimit, limitCycle-currentCycle) m.logger.Debug("machine step before run", "currentCycle", currentCycle, "increment", increment) // Runs the machine. - breakReason, err := m.backend_run(currentCycle+increment, hashCollectorState, timeout) + breakReason, err := m.backendRun(currentCycle+increment, hashCollectorState, timeout) if err != nil { - return nil, 0, err + return incrementResult{currentCycle: currentCycle}, err } // Gets the current cycle. currentCycle, err = m.readMCycle(ctx) if err != nil { - return nil, 0, err + return incrementResult{}, err } m.logger.Debug("machine step after run", @@ -516,29 +570,21 @@ func (m *machineImpl) runIncrementInterval(ctx context.Context, "breakReason", breakReason) switch breakReason { - case YieldedManually: - yt := ManualYield - return &yt, currentCycle, nil // returns with the yield type - case YieldedAutomatically: - yt := AutomaticYield - return &yt, currentCycle, nil // returns with the yield type - case YieldedSoftly: - return nil, currentCycle, nil // returns with no yield type - case ReachedTargetMcycle: - return nil, currentCycle, ErrReachedTargetMcycle - case McycleOverflow: - return nil, currentCycle, ErrReachedLimitMcycle - case Halted: - return nil, currentCycle, ErrHalted - case Failed: - return nil, currentCycle, ErrMachineInternal + case YieldedManually, + YieldedAutomatically, + YieldedSoftly, + ReachedTargetMcycle, + McycleOverflow, + Halted, + Failed: + return incrementResult{breakReason: breakReason, currentCycle: currentCycle}, nil default: err := fmt.Errorf("invalid break reason: %d: %w", breakReason, ErrMachineInternal) - return nil, currentCycle, err + return incrementResult{breakReason: breakReason, currentCycle: currentCycle}, err } } -func (m *machineImpl) backend_run(mcycleEnd uint64, hashCollectorState *HashCollectorState, timeout time.Duration) (BreakReason, error) { +func (m *machineImpl) backendRun(mcycleEnd uint64, hashCollectorState *HashCollectorState, timeout time.Duration) (BreakReason, error) { if hashCollectorState != nil { m.logger.Debug("Running with root hash collection") return m.backend.RunAndCollectRootHashes(mcycleEnd, hashCollectorState, timeout) diff --git a/pkg/machine/implementation_test.go b/pkg/machine/implementation_test.go index e732c538f..f40b08ebd 100644 --- a/pkg/machine/implementation_test.go +++ b/pkg/machine/implementation_test.go @@ -164,8 +164,7 @@ func (s *ImplementationSuite) TestOutputsHash() { }, } _, err = machine2.OutputsHash(ctx) - require.Error(err) - require.Contains(err.Error(), "machine manual yield reason is not accepted") + require.ErrorIs(err, ErrRejected) mockBackend2.AssertExpectations(s.T()) // Test outputs hash with invalid length @@ -274,7 +273,7 @@ func (s *ImplementationSuite) TestAdvance() { input := []byte("test input") resp, err := machine.Advance(ctx, input, expectedHash, false) require.NoError(err) - require.True(resp.Accepted) + require.Equal(CompletionStatusAccepted, resp.Status) require.Empty(resp.Outputs) require.Empty(resp.Reports) require.NotEqual(Hash{}, resp.OutputsHash) @@ -296,7 +295,7 @@ func (s *ImplementationSuite) TestAdvance() { } resp, err = machine2.Advance(ctx, input, expectedHash, false) require.NoError(err) - require.False(resp.Accepted) + require.Equal(CompletionStatusRejected, resp.Status) require.Empty(resp.Outputs) require.Empty(resp.Reports) require.Equal(Hash{}, resp.OutputsHash) @@ -317,8 +316,9 @@ func (s *ImplementationSuite) TestAdvance() { }, } resp, err = machine3.Advance(ctx, input, expectedHash, false) - require.ErrorIs(err, ErrException) - require.False(resp.Accepted) + require.NoError(err) + require.Equal(CompletionStatusException, resp.Status) + require.Equal([]byte("exception data"), resp.ExceptionData) require.Equal(Hash{}, resp.OutputsHash) mockBackend3.AssertExpectations(s.T()) @@ -337,8 +337,9 @@ func (s *ImplementationSuite) TestAdvance() { }, } largeInput := make([]byte, 10) - _, err = machine4.Advance(ctx, largeInput, expectedHash, false) + resp, err = machine4.Advance(ctx, largeInput, expectedHash, false) require.ErrorIs(err, ErrPayloadLengthLimitExceeded) + require.Nil(resp) mockBackend4.AssertExpectations(s.T()) // Test advance with invalid hash length @@ -360,9 +361,10 @@ func (s *ImplementationSuite) TestAdvance() { AdvanceMaxDeadline: time.Second * 10, }, } - _, err = machine5.Advance(ctx, input, expectedHash, false) + resp, err = machine5.Advance(ctx, input, expectedHash, false) require.Error(err) require.ErrorIs(err, ErrHashLength) + require.Nil(resp) mockBackend5.AssertExpectations(s.T()) } @@ -388,10 +390,10 @@ func (s *ImplementationSuite) TestInspect() { } query := []byte("test query") - accepted, reports, err := machine.Inspect(ctx, query) + response, err := machine.Inspect(ctx, query) require.NoError(err) - require.True(accepted) - require.Empty(reports) + require.Equal(CompletionStatusAccepted, response.Status) + require.Empty(response.Reports) mockBackend.AssertExpectations(s.T()) // Test inspect with rejection @@ -408,10 +410,10 @@ func (s *ImplementationSuite) TestInspect() { InspectMaxDeadline: time.Second * 10, }, } - accepted, reports, err = machine2.Inspect(ctx, query) + response, err = machine2.Inspect(ctx, query) require.NoError(err) - require.False(accepted) - require.Empty(reports) + require.Equal(CompletionStatusRejected, response.Status) + require.Empty(response.Reports) mockBackend2.AssertExpectations(s.T()) // Test inspect with exception @@ -428,10 +430,11 @@ func (s *ImplementationSuite) TestInspect() { InspectMaxDeadline: time.Second * 10, }, } - accepted, reports, err = machine3.Inspect(ctx, query) - require.ErrorIs(err, ErrException) - require.False(accepted) - require.Empty(reports) + response, err = machine3.Inspect(ctx, query) + require.NoError(err) + require.Equal(CompletionStatusException, response.Status) + require.Equal([]byte("exception data"), response.ExceptionData) + require.Empty(response.Reports) mockBackend3.AssertExpectations(s.T()) // Test inspect with payload too large @@ -449,8 +452,9 @@ func (s *ImplementationSuite) TestInspect() { }, } largeQuery := make([]byte, 10) - _, _, err = machine4.Inspect(ctx, largeQuery) + response, err = machine4.Inspect(ctx, largeQuery) require.ErrorIs(err, ErrPayloadLengthLimitExceeded) + require.Nil(response) mockBackend4.AssertExpectations(s.T()) } @@ -591,7 +595,7 @@ func (s *ImplementationSuite) TestHelperMethods() { require.ErrorIs(err, ErrMachineInternal) mockBackend2.AssertExpectations(s.T()) - // Test wasLastRequestAccepted + // Test readManualYieldResult mockBackend3 := NewMockBackend() expectedHash3 := randomFakeHash() mockBackend3.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( @@ -603,10 +607,10 @@ func (s *ImplementationSuite) TestHelperMethods() { FastDeadline: time.Second * 5, }, } - accepted, data, err := machine3.wasLastRequestAccepted(ctx) + manualResult, err := machine3.readManualYieldResult(ctx) require.NoError(err) - require.True(accepted) - require.NotNil(data) + require.Equal(CompletionStatusAccepted, manualResult.status) + require.NotNil(manualResult.data) mockBackend3.AssertExpectations(s.T()) mockBackend4 := NewMockBackend() @@ -620,10 +624,10 @@ func (s *ImplementationSuite) TestHelperMethods() { FastDeadline: time.Second * 5, }, } - accepted, data, err = machine4.wasLastRequestAccepted(ctx) + manualResult, err = machine4.readManualYieldResult(ctx) require.NoError(err) - require.False(accepted) - require.NotNil(data) + require.Equal(CompletionStatusRejected, manualResult.status) + require.NotNil(manualResult.data) mockBackend4.AssertExpectations(s.T()) mockBackend5 := NewMockBackend() @@ -636,10 +640,10 @@ func (s *ImplementationSuite) TestHelperMethods() { FastDeadline: time.Second * 5, }, } - accepted, data, err = machine5.wasLastRequestAccepted(ctx) - require.ErrorIs(err, ErrException) - require.False(accepted) - require.NotNil(data) + manualResult, err = machine5.readManualYieldResult(ctx) + require.NoError(err) + require.Equal(CompletionStatusException, manualResult.status) + require.NotNil(manualResult.data) mockBackend5.AssertExpectations(s.T()) // Test readMCycle @@ -678,7 +682,7 @@ func (s *ImplementationSuite) TestHelperMethods() { _, err = machine.isAtManualYield(canceledCtx) require.ErrorIs(err, ErrCanceled) - _, _, err = machine3.wasLastRequestAccepted(canceledCtx) + _, err = machine3.readManualYieldResult(canceledCtx) require.ErrorIs(err, ErrCanceled) _, err = machine6.readMCycle(canceledCtx) @@ -707,10 +711,10 @@ func (s *ImplementationSuite) TestRun() { }, } - outputs, reports, _, _, err := machine.run(ctx, AdvanceStateRequest, false) + result, err := machine.run(ctx, AdvanceStateRequest, false, 0, 1000) require.NoError(err) - require.Empty(outputs) - require.Empty(reports) + require.Empty(result.outputs) + require.Empty(result.reports) mockBackend.AssertExpectations(s.T()) // Test run with read cycle error @@ -727,7 +731,7 @@ func (s *ImplementationSuite) TestRun() { AdvanceMaxDeadline: time.Second * 10, }, } - _, _, _, _, err = machine2.run(ctx, AdvanceStateRequest, false) + _, err = machine2.run(ctx, AdvanceStateRequest, false, 0, 1000) require.Error(err) require.Contains(err.Error(), "read cycle failed") mockBackend2.AssertExpectations(s.T()) @@ -750,7 +754,7 @@ func (s *ImplementationSuite) TestRun() { }, } - _, _, _, _, err = machine3.run(ctx, AdvanceStateRequest, false) + _, err = machine3.run(ctx, AdvanceStateRequest, false, 0, 1000) require.NoError(err) mockBackend3.AssertExpectations(s.T()) @@ -773,7 +777,7 @@ func (s *ImplementationSuite) TestRun() { }, } - _, _, _, _, err = machine4.run(ctx, AdvanceStateRequest, false) + _, err = machine4.run(ctx, AdvanceStateRequest, false, 0, 1000) require.Error(err) require.Contains(err.Error(), "could not read output/report") require.Contains(err.Error(), "cmio request failed") @@ -801,11 +805,11 @@ func (s *ImplementationSuite) TestRun() { }, } - outputs5, reports5, _, _, err := machine5.run(ctx, AdvanceStateRequest, false) + result5, err := machine5.run(ctx, AdvanceStateRequest, false, 0, 1000) require.NoError(err) - require.Len(outputs5, 1) - require.Equal([]byte("output data"), []byte(outputs5[0])) - require.Empty(reports5) + require.Len(result5.outputs, 1) + require.Equal([]byte("output data"), []byte(result5.outputs[0])) + require.Empty(result5.reports) mockBackend5.AssertExpectations(s.T()) // Test run with automatic yield producing report then manual yield @@ -830,136 +834,53 @@ func (s *ImplementationSuite) TestRun() { }, } - outputs6, reports6, _, _, err := machine6.run(ctx, AdvanceStateRequest, false) + result6, err := machine6.run(ctx, AdvanceStateRequest, false, 0, 1000) require.NoError(err) - require.Empty(outputs6) - require.Len(reports6, 1) - require.Equal([]byte("report data"), []byte(reports6[0])) + require.Empty(result6.outputs) + require.Len(result6.reports, 1) + require.Equal([]byte("report data"), []byte(result6.reports[0])) mockBackend6.AssertExpectations(s.T()) } -// Test step method -func (s *ImplementationSuite) TestStep() { - require := s.Require() - ctx := context.Background() - - machine := &machineImpl{ - backend: nil, // Will be set per test - logger: s.logger, - params: model.ExecutionParameters{ - AdvanceIncCycles: 100, - }, +func (s *ImplementationSuite) TestRunIncrementIntervalPreservesBreakReason() { + for _, test := range []struct { + name string + breakReason BreakReason + cycle uint64 + }{ + {"manual", YieldedManually, 150}, + {"automatic", YieldedAutomatically, 200}, + {"soft", YieldedSoftly, 150}, + {"target", ReachedTargetMcycle, 200}, + {"overflow", McycleOverflow, 199}, + {"halt", Halted, 175}, + {"failed", Failed, 160}, + } { + s.Run(test.name, func() { + backend := NewMockBackend() + backend.On("Run", uint64(200), mock.AnythingOfType("time.Duration")).Return(test.breakReason, nil) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(test.cycle, nil) + machine := &machineImpl{backend: backend, logger: s.logger} + + result, err := machine.runIncrementInterval( + context.Background(), 100, 1000, 100, nil, time.Second, + ) + s.Require().NoError(err) + s.Equal(test.breakReason, result.breakReason) + s.Equal(test.cycle, result.currentCycle) + backend.AssertExpectations(s.T()) + }) } - // Test step with manual yield - mockBackend := NewMockBackend() - mockBackend.On("Run", mock.AnythingOfType("uint64"), mock.AnythingOfType("time.Duration")).Return(YieldedManually, nil) - mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(150), nil) - machine.backend = mockBackend - - yieldType, cycle, err := machine.runIncrementInterval(ctx, 100, 1000, nil, time.Second) - require.NoError(err) - require.NotNil(yieldType) - require.Equal(ManualYield, *yieldType) - require.Equal(uint64(150), cycle) - mockBackend.AssertExpectations(s.T()) - - // Test runIncrementInterval with automatic yield - mockBackend2 := NewMockBackend() - mockBackend2.On("Run", mock.AnythingOfType("uint64"), mock.AnythingOfType("time.Duration")).Return(YieldedAutomatically, nil) - mockBackend2.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(200), nil) - machine.backend = mockBackend2 - - yieldType, cycle, err = machine.runIncrementInterval(ctx, 100, 1000, nil, time.Second) - require.NoError(err) - require.NotNil(yieldType) - require.Equal(AutomaticYield, *yieldType) - require.Equal(uint64(200), cycle) - mockBackend2.AssertExpectations(s.T()) - - // Test runIncrementInterval with soft yield (no yield) - mockBackend3 := NewMockBackend() - mockBackend3.On("Run", mock.AnythingOfType("uint64"), mock.AnythingOfType("time.Duration")).Return(YieldedSoftly, nil) - mockBackend3.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(150), nil) - machine.backend = mockBackend3 - - yieldType, cycle, err = machine.runIncrementInterval(ctx, 100, 1000, nil, time.Second) - require.NoError(err) - require.Nil(yieldType) - require.Equal(uint64(150), cycle) - mockBackend3.AssertExpectations(s.T()) - - // Test runIncrementInterval with reached target mcycle - mockBackend4 := NewMockBackend() - mockBackend4.On("Run", mock.AnythingOfType("uint64"), mock.AnythingOfType("time.Duration")).Return(ReachedTargetMcycle, nil) - mockBackend4.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(1000), nil) - machine.backend = mockBackend4 - - yieldType, cycle, err = machine.runIncrementInterval(ctx, 100, 1000, nil, time.Second) - require.ErrorIs(err, ErrReachedTargetMcycle) - require.Nil(yieldType) - require.Equal(uint64(1000), cycle) - mockBackend4.AssertExpectations(s.T()) - - // Test runIncrementInterval with mcycle overflow - mockBackendOverflow := NewMockBackend() - mockBackendOverflow.On( - "Run", - mock.AnythingOfType("uint64"), - mock.AnythingOfType("time.Duration"), - ).Return(McycleOverflow, nil) - mockBackendOverflow.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(999), nil) - machine.backend = mockBackendOverflow - - yieldType, cycle, err = machine.runIncrementInterval(ctx, 100, 1000, nil, time.Second) - require.ErrorIs(err, ErrReachedLimitMcycle) - require.NotErrorIs(err, ErrMachineInternal) - require.Nil(yieldType) - require.Equal(uint64(999), cycle) - mockBackendOverflow.AssertExpectations(s.T()) - - // Test runIncrementInterval with halted - mockBackend5 := NewMockBackend() - mockBackend5.On("Run", mock.AnythingOfType("uint64"), mock.AnythingOfType("time.Duration")).Return(Halted, nil) - mockBackend5.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(500), nil) - machine.backend = mockBackend5 - - yieldType, cycle, err = machine.runIncrementInterval(ctx, 100, 1000, nil, time.Second) - require.ErrorIs(err, ErrHalted) - require.Nil(yieldType) - require.Equal(uint64(500), cycle) - - // Test runIncrementInterval already at limit cycle - yieldType, cycle, err = machine.runIncrementInterval(ctx, 1000, 1000, nil, time.Second) - require.ErrorIs(err, ErrReachedLimitMcycle) - require.Nil(yieldType) - require.Equal(uint64(0), cycle) - mockBackend5.AssertExpectations(s.T()) - - // Test runIncrementInterval with backend run error - mockBackend6 := NewMockBackend() - mockBackend6.On("Run", - mock.AnythingOfType("uint64"), - mock.AnythingOfType("time.Duration"), - ).Return(BreakReason(0), errors.New("run failed")) - machine.backend = mockBackend6 - yieldType, _, err = machine.runIncrementInterval(ctx, 100, 1000, nil, time.Second) - require.Error(err) - require.Contains(err.Error(), "run failed") - require.Nil(yieldType) - mockBackend6.AssertExpectations(s.T()) - - // Test runIncrementInterval with read cycle error - mockBackend7 := NewMockBackend() - mockBackend7.On("Run", mock.AnythingOfType("uint64"), mock.AnythingOfType("time.Duration")).Return(YieldedManually, nil) - mockBackend7.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(0), errors.New("read cycle failed")) - machine.backend = mockBackend7 - yieldType, _, err = machine.runIncrementInterval(ctx, 100, 1000, nil, time.Second) - require.Error(err) - require.Contains(err.Error(), "read cycle failed") - require.Nil(yieldType) - mockBackend7.AssertExpectations(s.T()) + s.Run("already at limit", func() { + machine := &machineImpl{backend: NewMockBackend(), logger: s.logger} + result, err := machine.runIncrementInterval( + context.Background(), 1000, 1000, 100, nil, time.Second, + ) + s.Require().ErrorIs(err, ErrReachedLimitMcycle) + s.Equal(uint64(1000), result.currentCycle) + }) } // Test process method @@ -990,14 +911,47 @@ func (s *ImplementationSuite) TestProcess() { } input := []byte("test input") - accepted, outputs, reports, _, _, data, err := machine.process(ctx, input, AdvanceStateRequest, &expectedHash, false) + result, err := machine.process(ctx, input, AdvanceStateRequest, &expectedHash, false) require.NoError(err) - require.True(accepted) - require.Empty(outputs) - require.Empty(reports) - require.NotNil(data) + require.Equal(CompletionStatusAccepted, result.completion.status) + require.Empty(result.outputs) + require.Empty(result.reports) + require.NotNil(result.completion.data) mockBackend.AssertExpectations(s.T()) + // A halt completes the request without producing a CMIO manual yield. + haltedBackend := NewMockBackend() + haltedBackend.On("CmioRxBufferSize").Return(uint64(1024)) + haltedBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(uint64(0), nil).Once() + haltedBackend.On( + "SendCmioResponse", + mock.AnythingOfType("uint16"), mock.Anything, expectedHash, + mock.AnythingOfType("time.Duration"), + ).Return(nil).Once() + haltedBackend.On("Run", uint64(100), mock.AnythingOfType("time.Duration")). + Return(Halted, nil).Once() + haltedBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(uint64(100), nil).Once() + haltedMachine := &machineImpl{ + backend: haltedBackend, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: 5 * time.Second, + AdvanceMaxCycles: 1000, + AdvanceIncCycles: 100, + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: 10 * time.Second, + }, + } + result, err = haltedMachine.process( + ctx, input, AdvanceStateRequest, &expectedHash, false, + ) + require.NoError(err) + require.Equal(CompletionStatusHalted, result.completion.status) + require.Nil(result.completion.data) + haltedBackend.AssertExpectations(s.T()) + // Test process with payload too large mockBackend2 := NewMockBackend() mockBackend2.On("CmioRxBufferSize").Return(uint64(5)) @@ -1012,7 +966,7 @@ func (s *ImplementationSuite) TestProcess() { AdvanceMaxDeadline: time.Second * 10, }, } - _, _, _, _, _, _, err = machine2.process(ctx, input, AdvanceStateRequest, &expectedHash, false) + _, err = machine2.process(ctx, input, AdvanceStateRequest, &expectedHash, false) require.ErrorIs(err, ErrPayloadLengthLimitExceeded) mockBackend2.AssertExpectations(s.T()) @@ -1036,7 +990,8 @@ func (s *ImplementationSuite) TestProcess() { AdvanceMaxDeadline: time.Second * 10, }, } - _, _, _, _, _, _, err = machine3.process(ctx, input, AdvanceStateRequest, &expectedHash, false) + mockBackend3.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(0), nil) + _, err = machine3.process(ctx, input, AdvanceStateRequest, &expectedHash, false) require.Error(err) require.Contains(err.Error(), "send failed") mockBackend3.AssertExpectations(s.T()) @@ -1057,7 +1012,7 @@ func (s *ImplementationSuite) TestProcess() { AdvanceMaxDeadline: time.Second * 10, }, } - _, _, _, _, _, _, err = machine4.process(ctx, input, AdvanceStateRequest, &expectedHash, false) + _, err = machine4.process(ctx, input, AdvanceStateRequest, &expectedHash, false) require.Error(err) require.Contains(err.Error(), "read cycle failed") mockBackend4.AssertExpectations(s.T()) @@ -1082,7 +1037,6 @@ func (s *ImplementationSuite) TestRunWithAutomaticYields() { } // Setup for automatic yield with output - mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(0), nil).Once() mockBackend.On("Run", uint64(100), mock.AnythingOfType("time.Duration")).Return(YieldedAutomatically, nil).Once() mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(50), nil).Once() mockBackend.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( @@ -1092,11 +1046,11 @@ func (s *ImplementationSuite) TestRunWithAutomaticYields() { mockBackend.On("Run", uint64(150), mock.AnythingOfType("time.Duration")).Return(YieldedManually, nil).Once() mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(100), nil).Once() - outputs, reports, _, _, err := machine.run(ctx, AdvanceStateRequest, false) + result, err := machine.run(ctx, AdvanceStateRequest, false, 0, 1000) require.NoError(err) - require.Len(outputs, 1) - require.Equal([]byte("test output"), outputs[0]) - require.Empty(reports) + require.Len(result.outputs, 1) + require.Equal([]byte("test output"), result.outputs[0]) + require.Empty(result.reports) mockBackend.AssertExpectations(s.T()) } @@ -1120,7 +1074,6 @@ func (s *ImplementationSuite) TestRunWithAutomaticYieldsReports() { } // Setup for automatic yield with report - mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(0), nil).Once() mockBackend.On("Run", uint64(100), mock.AnythingOfType("time.Duration")).Return(YieldedAutomatically, nil).Once() mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(50), nil).Once() mockBackend.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( @@ -1130,11 +1083,11 @@ func (s *ImplementationSuite) TestRunWithAutomaticYieldsReports() { mockBackend.On("Run", uint64(150), mock.AnythingOfType("time.Duration")).Return(YieldedManually, nil).Once() mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(100), nil).Once() - outputs, reports, _, _, err := machine.run(ctx, AdvanceStateRequest, false) + result, err := machine.run(ctx, AdvanceStateRequest, false, 0, 1000) require.NoError(err) - require.Empty(outputs) - require.Len(reports, 1) - require.Equal([]byte("test report"), reports[0]) + require.Empty(result.outputs) + require.Len(result.reports, 1) + require.Equal([]byte("test report"), result.reports[0]) mockBackend.AssertExpectations(s.T()) } @@ -1158,8 +1111,6 @@ func (s *ImplementationSuite) TestMultipleAutomaticYields() { } // Setup for multiple automatic yields followed by manual yield - mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(0), nil).Once() - // First automatic yield with output mockBackend.On("Run", uint64(100), mock.AnythingOfType("time.Duration")).Return(YieldedAutomatically, nil).Once() mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(10), nil).Once() @@ -1194,16 +1145,16 @@ func (s *ImplementationSuite) TestMultipleAutomaticYields() { mockBackend.On("Run", uint64(150), mock.AnythingOfType("time.Duration")).Return(YieldedManually, nil).Once() mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(60), nil).Once() - outputs, reports, _, _, err := machine.run(ctx, AdvanceStateRequest, false) + result, err := machine.run(ctx, AdvanceStateRequest, false, 0, 1000) require.NoError(err) - require.Len(outputs, 2) - require.Equal([]byte("output1"), outputs[0]) - require.Equal([]byte(""), outputs[1]) + require.Len(result.outputs, 2) + require.Equal([]byte("output1"), result.outputs[0]) + require.Equal([]byte(""), result.outputs[1]) - require.Len(reports, 2) - require.Equal([]byte("output2"), reports[0]) - require.Equal([]byte(""), reports[1]) + require.Len(result.reports, 2) + require.Equal([]byte("output2"), result.reports[0]) + require.Equal([]byte(""), result.reports[1]) mockBackend.AssertExpectations(s.T()) } diff --git a/pkg/machine/machine.go b/pkg/machine/machine.go index aaedb21d2..855c9e997 100644 --- a/pkg/machine/machine.go +++ b/pkg/machine/machine.go @@ -27,16 +27,49 @@ type ( Hash = [HashSize]byte ) -// AdvanceResponse contains the result of an advance operation. +// CompletionStatus identifies how a guest-machine request completed. If +// execution does not complete, the request returns an error instead. +type CompletionStatus uint8 + +const ( + CompletionStatusUnknown CompletionStatus = iota + CompletionStatusAccepted + CompletionStatusRejected + CompletionStatusException + CompletionStatusHalted +) + +// IsCompleted reports whether the status is a completed guest-machine outcome. +func (s CompletionStatus) IsCompleted() bool { + switch s { + case CompletionStatusAccepted, + CompletionStatusRejected, + CompletionStatusException, + CompletionStatusHalted: + return true + default: + return false + } +} + +// AdvanceResponse contains the result of a completed advance operation. type AdvanceResponse struct { - Accepted bool + Status CompletionStatus Outputs []Output Reports []Report + ExceptionData []byte Hashes []Hash RemainingCycles uint64 OutputsHash Hash } +// InspectResponse contains the result of a completed inspect operation. +type InspectResponse struct { + Status CompletionStatus + ExceptionData []byte + Reports []Report +} + // Common errors var ( ErrMachineInternal = errors.New("machine internal error") @@ -71,16 +104,12 @@ type Machine interface { // Advance sends an input to the machine. // The checkpointHash is the machine's root hash before processing the input, // sent along with the request so the machine can revert to it if needed. - // It always returns a non-nil AdvanceResponse, even on error paths. - // The response contains whether the request was accepted, - // the corresponding outputs, reports, and the hash of the outputs. - // In case the request is not accepted, the response does not contain outputs. + // A non-nil response and nil error mean execution completed with a typed + // status. Incomplete execution returns a nil response and a non-nil error. Advance(ctx context.Context, input []byte, checkpointHash Hash, computeHashes bool) (*AdvanceResponse, error) - // Inspect sends a query to the machine. - // It returns a boolean indicating whether or not the request was accepted - // It also returns the corresponding reports. - Inspect(ctx context.Context, query []byte) (bool, []Report, error) + // Inspect sends a query to the machine and returns its typed completion. + Inspect(ctx context.Context, query []byte) (*InspectResponse, error) // Store saves the machine state to the specified path. Store(ctx context.Context, path string) error @@ -196,13 +225,17 @@ func Load(ctx context.Context, logger *slog.Logger, config *MachineConfig) (Mach return nil, ErrNotAtManualYield } - // Ensures that the last request the machine received did not yield an exception. - accepted, _, err := machine.wasLastRequestAccepted(ctx) + // Ensures that the last request left the machine in an accepted manual yield. + manualResult, err := machine.readManualYieldResult(ctx) if err != nil { machine.Close() return nil, err } - if !accepted { + if manualResult.status == CompletionStatusException { + machine.Close() + return nil, ErrException + } + if manualResult.status != CompletionStatusAccepted { machine.Close() return nil, ErrRejected } diff --git a/pkg/machine/machine_test.go b/pkg/machine/machine_test.go index 2acdc1043..d35135fde 100644 --- a/pkg/machine/machine_test.go +++ b/pkg/machine/machine_test.go @@ -204,15 +204,15 @@ func (s *MachineSuite) TestMachineInterface() { // Create a mock machine mockMachine := &MockMachine{ - AddressReturn: "127.0.0.1:12345", - HashReturn: Hash{1, 2, 3, 4, 5}, - OutputsHashReturn: Hash{6, 7, 8, 9, 10}, - AdvanceAcceptedReturn: true, - AdvanceOutputsReturn: []Output{[]byte("output1"), []byte("output2")}, - AdvanceReportsReturn: []Report{[]byte("report1")}, - AdvanceHashReturn: Hash{11, 12, 13, 14, 15}, - InspectAcceptedReturn: true, - InspectReportsReturn: []Report{[]byte("inspect report")}, + AddressReturn: "127.0.0.1:12345", + HashReturn: Hash{1, 2, 3, 4, 5}, + OutputsHashReturn: Hash{6, 7, 8, 9, 10}, + AdvanceStatusReturn: CompletionStatusAccepted, + AdvanceOutputsReturn: []Output{[]byte("output1"), []byte("output2")}, + AdvanceReportsReturn: []Report{[]byte("report1")}, + AdvanceHashReturn: Hash{11, 12, 13, 14, 15}, + InspectStatusReturn: CompletionStatusAccepted, + InspectReportsReturn: []Report{[]byte("inspect report")}, } // Test that MockMachine implements Machine interface @@ -235,7 +235,7 @@ func (s *MachineSuite) TestMachineInterface() { // Test Advance advanceResp, err := machine.Advance(ctx, []byte("input"), Hash{}, false) require.NoError(err) - require.True(advanceResp.Accepted) + require.Equal(CompletionStatusAccepted, advanceResp.Status) require.Len(advanceResp.Outputs, 2) require.Equal([]byte("output1"), advanceResp.Outputs[0]) require.Equal([]byte("output2"), advanceResp.Outputs[1]) @@ -244,11 +244,11 @@ func (s *MachineSuite) TestMachineInterface() { require.Equal(Hash{11, 12, 13, 14, 15}, advanceResp.OutputsHash) // Test Inspect - accepted, inspectReports, err := machine.Inspect(ctx, []byte("query")) + inspectResp, err := machine.Inspect(ctx, []byte("query")) require.NoError(err) - require.True(accepted) - require.Len(inspectReports, 1) - require.Equal([]byte("inspect report"), inspectReports[0]) + require.Equal(CompletionStatusAccepted, inspectResp.Status) + require.Len(inspectResp.Reports, 1) + require.Equal([]byte("inspect report"), inspectResp.Reports[0]) // Test Store err = machine.Store(ctx, "/tmp/test") @@ -303,7 +303,7 @@ func (s *MachineSuite) TestMachineInterfaceErrors() { require.Contains(err.Error(), "advance error") // Test Inspect error - _, _, err = machine.Inspect(ctx, []byte("query")) + _, err = machine.Inspect(ctx, []byte("query")) require.Error(err) require.Contains(err.Error(), "inspect error") @@ -332,7 +332,7 @@ type MockMachine struct { OutputsHashProofReturn []Hash OutputsHashProofError error - AdvanceAcceptedReturn bool + AdvanceStatusReturn CompletionStatus AdvanceOutputsReturn []Output AdvanceReportsReturn []Report AdvanceHashesReturn []Hash @@ -340,9 +340,9 @@ type MockMachine struct { AdvanceHashReturn Hash AdvanceError error - InspectAcceptedReturn bool - InspectReportsReturn []Report - InspectError error + InspectStatusReturn CompletionStatus + InspectReportsReturn []Report + InspectError error StoreError error @@ -368,18 +368,24 @@ func (m *MockMachine) OutputsHashProof(_ context.Context) ([]Hash, error) { } func (m *MockMachine) Advance(_ context.Context, _ []byte, _ Hash, _ bool) (*AdvanceResponse, error) { + if m.AdvanceError != nil { + return nil, m.AdvanceError + } return &AdvanceResponse{ - Accepted: m.AdvanceAcceptedReturn, + Status: m.AdvanceStatusReturn, Outputs: m.AdvanceOutputsReturn, Reports: m.AdvanceReportsReturn, Hashes: m.AdvanceHashesReturn, RemainingCycles: m.AdvanceRemainingReturn, OutputsHash: m.AdvanceHashReturn, - }, m.AdvanceError + }, nil } -func (m *MockMachine) Inspect(_ context.Context, _ []byte) (bool, []Report, error) { - return m.InspectAcceptedReturn, m.InspectReportsReturn, m.InspectError +func (m *MockMachine) Inspect(_ context.Context, _ []byte) (*InspectResponse, error) { + if m.InspectError != nil { + return nil, m.InspectError + } + return &InspectResponse{Status: m.InspectStatusReturn, Reports: m.InspectReportsReturn}, nil } func (m *MockMachine) Store(_ context.Context, _ string) error { From 6fa353dae51cf29427a8698d75a556fc3f991d04 Mon Sep 17 00:00:00 2001 From: Victor Fusco <1221933+vfusco@users.noreply.github.com> Date: Wed, 12 Aug 2026 17:30:48 -0300 Subject: [PATCH 02/13] refactor(machine): enforce fixed execution bounds --- internal/model/models.go | 10 + pkg/machine/implementation.go | 199 +++++++-- pkg/machine/implementation_test.go | 630 ++++++++++++++++++++++++++++- pkg/machine/machine.go | 23 +- pkg/machine/machine_test.go | 27 +- 5 files changed, 843 insertions(+), 46 deletions(-) diff --git a/internal/model/models.go b/internal/model/models.go index b818ca527..2cab6409a 100644 --- a/internal/model/models.go +++ b/internal/model/models.go @@ -690,6 +690,16 @@ func (e *ExecutionParameters) UnmarshalJSON(data []byte) error { return nil } +// Log2MaxExecutionCycles is the node-side definition of the protocol +// execution window size. +const Log2MaxExecutionCycles uint64 = 48 + +// MaxExecutionCycles is the number of mcycles in one machine-enforced window. +const MaxExecutionCycles uint64 = 1 << Log2MaxExecutionCycles + +// MaxExecutionCycleSpan is the largest endpoint delta in that window. +const MaxExecutionCycleSpan uint64 = MaxExecutionCycles - 1 + // validateParameters constants const maxDuration = 24 * time.Hour const maxConcurrentInspects = 1000 diff --git a/pkg/machine/implementation.go b/pkg/machine/implementation.go index 2db43b019..65df9b70a 100644 --- a/pkg/machine/implementation.go +++ b/pkg/machine/implementation.go @@ -45,6 +45,27 @@ type completionResult struct { data []byte } +// executionBounds is the resolved cycle window for one request. A configured +// span of zero is converted to the fixed machine window exactly once here; +// downstream execution and diagnostics consume the same resolved values. +type executionBounds struct { + start Cycle + limit Cycle + span Cycle + configured bool +} + +func (r requestType) String() string { + switch r { + case AdvanceStateRequest: + return "advance" + case InspectStateRequest: + return "inspect" + default: + return fmt.Sprintf("request_type_%d", r) + } +} + type automaticYieldReason uint16 const ( @@ -228,20 +249,19 @@ func (m *machineImpl) Advance(ctx context.Context, input []byte, checkpointHash func (m *machineImpl) Inspect(ctx context.Context, query []byte) (*InspectResponse, error) { // For inspect-state requests, revert_root_hash is not checked and can be NULL/empty result, err := m.process(ctx, query, InspectStateRequest, nil, false) + response := &InspectResponse{Reports: result.reports} if err != nil { - return nil, err + return response, err } - if !result.completion.status.IsCompleted() { - return nil, fmt.Errorf( + + response.Status = result.completion.status + if !response.Status.IsCompleted() { + return response, fmt.Errorf( "invalid completed inspect status %d: %w", result.completion.status, ErrMachineInternal, ) } - response := &InspectResponse{ - Status: result.completion.status, - Reports: result.reports, - } if response.Status == CompletionStatusException { response.ExceptionData = append([]byte{}, result.completion.data...) } @@ -384,18 +404,20 @@ func (m *machineImpl) process( if err := checkContext(ctx); err != nil { return processResult{}, err } - // Check payload length limit - if length := uint64(len(request)); length > m.backend.CmioRxBufferSize() { - return processResult{}, ErrPayloadLengthLimitExceeded + if length, capacity := uint64(len(request)), m.backend.CmioRxBufferSize(); length > capacity { + return processResult{}, fmt.Errorf( + "%s request payload length %d exceeds CMIO receive buffer capacity %d: %w", + reqType, length, capacity, ErrPayloadLengthLimitExceeded, + ) } - currentCycle, err := m.readMCycle(ctx) - if err != nil { + // Validate execution parameters before SendCmioResponse mutates the machine. + if err := m.validateExecutionCycleIncrement(reqType); err != nil { return processResult{}, err } - limitCycle := currentCycle + m.params.AdvanceMaxCycles - if reqType == InspectStateRequest { - limitCycle = currentCycle + m.params.InspectMaxCycles + bounds, err := m.executionCycleBounds(ctx, reqType) + if err != nil { + return processResult{}, err } err = m.backend.SendCmioResponse(uint16(reqType), request, checkpointHash, m.params.FastDeadline) @@ -403,7 +425,7 @@ func (m *machineImpl) process( return processResult{}, err } - execution, err := m.run(ctx, reqType, computeHashes, currentCycle, limitCycle) + execution, err := m.run(ctx, reqType, computeHashes, bounds) result := processResult{runResult: execution} switch { case err == nil: @@ -426,24 +448,23 @@ func (m *machineImpl) run( ctx context.Context, reqType requestType, computeHashes bool, - currentCycle uint64, - limitCycle uint64, + bounds executionBounds, ) (runResult, error) { startTime := time.Now() + currentCycle := bounds.start stepTimeout := m.params.AdvanceIncDeadline runTimeout := m.params.AdvanceMaxDeadline - increment := m.params.AdvanceIncCycles + increment := m.executionCycleIncrement(reqType) if reqType == InspectStateRequest { stepTimeout = m.params.InspectIncDeadline runTimeout = m.params.InspectMaxDeadline - increment = m.params.InspectIncCycles } m.logger.Debug("run", "startingCycle", currentCycle, - "limitCycle", limitCycle, - "leftover", limitCycle-currentCycle) + "limitCycle", bounds.limit, + "leftover", bounds.limit-currentCycle) result := runResult{ outputs: make([]Output, 0, 16), //nolint:mnd @@ -477,10 +498,13 @@ func (m *machineImpl) run( } interval, err := m.runIncrementInterval( - ctx, currentCycle, limitCycle, increment, hashCollectorState, stepTimeout, + ctx, currentCycle, bounds.limit, increment, hashCollectorState, stepTimeout, ) currentCycle = interval.currentCycle if err != nil { + if errors.Is(err, ErrReachedLimitMcycle) { + return finish(executionLimitError(reqType, bounds, currentCycle, err)) + } return finish(err) } @@ -490,7 +514,7 @@ func (m *machineImpl) run( case Halted: return finish(ErrHalted) case McycleOverflow: - return finish(ErrReachedLimitMcycle) + return finish(executionLimitError(reqType, bounds, currentCycle, ErrMcycleOverflow)) case ReachedTargetMcycle, YieldedSoftly: continue case Failed: @@ -514,14 +538,17 @@ func (m *machineImpl) run( case AutomaticYieldReasonProgress: m.logger.Debug("ignoring yield reason progress", "value", fmt.Sprintf("%v", data)) case AutomaticYieldReasonOutput: - // TODO: should we remove this? if len(result.outputs) == maxOutputs { - return finish(ErrOutputsLimitExceeded) + return finish(executionResponseLimitError( + reqType, "output", len(result.outputs)+1, maxOutputs, ErrOutputsLimitExceeded, + )) } result.outputs = append(result.outputs, data) case AutomaticYieldReasonReport: if len(result.reports) == maxReports { - return finish(ErrReportsLimitExceeded) + return finish(executionResponseLimitError( + reqType, "report", len(result.reports)+1, maxReports, ErrReportsLimitExceeded, + )) } result.reports = append(result.reports, data) default: @@ -531,6 +558,114 @@ func (m *machineImpl) run( } } +func executionResponseLimitError( + reqType requestType, + responseKind string, + count int, + capacity int, + limitErr error, +) error { + return fmt.Errorf( + "%s %s count %d exceeds local operational capacity %d: %w", + reqType, responseKind, count, capacity, limitErr, + ) +} + +// executionCycleBounds applies the request-specific configured cycle span. +// Zero uses the machine's complete fixed window; non-representable endpoints +// saturate at MaxUint64 to mirror emulator mcycle arithmetic. +func (m *machineImpl) executionCycleBounds( + ctx context.Context, + reqType requestType, +) (executionBounds, error) { + executionCycleSpan := m.configuredCycleSpan(reqType) + if executionCycleSpan > model.MaxExecutionCycleSpan { + return executionBounds{}, fmt.Errorf( + "%s execution configured cycle span exceeds hard maximum: requested_span=%d maximum_span=%d: %w", + reqType, executionCycleSpan, model.MaxExecutionCycleSpan, ErrReachedLimitMcycle, + ) + } + + currentCycle, err := m.readMCycle(ctx) + if err != nil { + return executionBounds{}, err + } + configured := executionCycleSpan != 0 + if executionCycleSpan == 0 { + executionCycleSpan = BarchSpanToInput + } + limit := ^uint64(0) + if currentCycle <= ^uint64(0)-executionCycleSpan { + limit = currentCycle + executionCycleSpan + } + return executionBounds{ + start: currentCycle, limit: limit, span: executionCycleSpan, configured: configured, + }, nil +} + +func (m *machineImpl) validateExecutionCycleIncrement(reqType requestType) error { + if m.executionCycleIncrement(reqType) == 0 { + return fmt.Errorf("%s execution increment must be greater than zero: %w", reqType, ErrMachineInternal) + } + return nil +} + +func (m *machineImpl) configuredCycleSpan(reqType requestType) uint64 { + if reqType == InspectStateRequest { + return m.params.InspectMaxCycles + } + return m.params.AdvanceMaxCycles +} + +func (m *machineImpl) executionCycleIncrement(reqType requestType) uint64 { + if reqType == InspectStateRequest { + return m.params.InspectIncCycles + } + return m.params.AdvanceIncCycles +} + +func executionLimitError( + reqType requestType, + bounds executionBounds, + current uint64, + origin error, +) error { + targetSpan := bounds.limit - bounds.start + executedCycles := uint64(0) + if current >= bounds.start { + executedCycles = current - bounds.start + } + source := "fixed" + if bounds.configured { + source = "configured" + } + progress := fmt.Sprintf( + "start=%d requested_span=%d target_span=%d executed_cycles=%d", + bounds.start, bounds.span, targetSpan, executedCycles, + ) + + if current < bounds.limit { + return fmt.Errorf( + "%s execution stopped before %s cycle limit: %s: %w", + reqType, source, progress, origin, + ) + } + if errors.Is(origin, ErrMcycleOverflow) && bounds.configured { + return fmt.Errorf( + "%s execution stopped at machine imcyclemax coincident with configured target: %s: %w", + reqType, progress, origin, + ) + } + + if errors.Is(origin, ErrMcycleOverflow) { + source += " (machine imcyclemax)" + } + return fmt.Errorf( + "%s execution reached %s cycle limit without completing: %s: %w", + reqType, source, progress, origin, + ) +} + // runIncrementInterval runs the machine for at most incrementLimit mcycles and // preserves the emulator's typed break reason for the request loop. func (m *machineImpl) runIncrementInterval(ctx context.Context, @@ -542,7 +677,8 @@ func (m *machineImpl) runIncrementInterval(ctx context.Context, ) (incrementResult, error) { startingCycle := currentCycle - if currentCycle >= limitCycle { + atSaturatedEndpoint := currentCycle == ^uint64(0) && limitCycle == ^uint64(0) + if currentCycle >= limitCycle && !atSaturatedEndpoint { return incrementResult{currentCycle: currentCycle}, ErrReachedLimitMcycle } @@ -569,6 +705,13 @@ func (m *machineImpl) runIncrementInterval(ctx context.Context, "leftover", limitCycle-currentCycle, "breakReason", breakReason) + if atSaturatedEndpoint && breakReason != McycleOverflow { + return incrementResult{breakReason: breakReason, currentCycle: currentCycle}, fmt.Errorf( + "machine returned break reason %d instead of mcycle overflow at MaxUint64: %w", + breakReason, ErrMachineInternal, + ) + } + switch breakReason { case YieldedManually, YieldedAutomatically, diff --git a/pkg/machine/implementation_test.go b/pkg/machine/implementation_test.go index f40b08ebd..1529f40bc 100644 --- a/pkg/machine/implementation_test.go +++ b/pkg/machine/implementation_test.go @@ -6,6 +6,7 @@ package machine import ( "context" "errors" + "fmt" "io" "log/slog" "testing" @@ -453,11 +454,599 @@ func (s *ImplementationSuite) TestInspect() { } largeQuery := make([]byte, 10) response, err = machine4.Inspect(ctx, largeQuery) + require.NotNil(response) + require.Equal(CompletionStatusUnknown, response.Status) require.ErrorIs(err, ErrPayloadLengthLimitExceeded) - require.Nil(response) mockBackend4.AssertExpectations(s.T()) } +func (s *ImplementationSuite) TestRunUsesHardExecutionSpanWhenMaximumIsZero() { + const startCycle uint64 = 7 + const executionCycleSpan uint64 = BarchSpanToInput + + tests := []struct { + name string + reqType requestType + params model.ExecutionParameters + }{ + { + name: "advance", + reqType: AdvanceStateRequest, + params: model.ExecutionParameters{ + AdvanceIncCycles: ^uint64(0), + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: time.Second, + }, + }, + { + name: "inspect", + reqType: InspectStateRequest, + params: model.ExecutionParameters{ + InspectIncCycles: ^uint64(0), + InspectIncDeadline: time.Second, + InspectMaxDeadline: time.Second, + }, + }, + } + + for _, tt := range tests { + s.Run(tt.name, func() { + mockBackend := NewMockBackend() + mockBackend.On("Run", startCycle+executionCycleSpan, mock.AnythingOfType("time.Duration")). + Return(McycleOverflow, nil) + mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(startCycle+executionCycleSpan, nil).Once() + + machine := &machineImpl{backend: mockBackend, logger: s.logger, params: tt.params} + _, err := machine.run( + context.Background(), tt.reqType, false, + executionBounds{ + start: startCycle, limit: startCycle + executionCycleSpan, + span: executionCycleSpan, + }, + ) + s.Require().ErrorIs(err, ErrReachedLimitMcycle) + s.Require().ErrorIs(err, ErrMcycleOverflow) + s.Contains(err.Error(), fmt.Sprintf("executed_cycles=%d", executionCycleSpan)) + mockBackend.AssertExpectations(s.T()) + }) + } +} + +func (s *ImplementationSuite) TestExecutionCycleBoundsAcceptsMaxSafeStart() { + const maxSafeStart uint64 = ^uint64(0) - model.MaxExecutionCycleSpan + + mockBackend := NewMockBackend() + mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(maxSafeStart, nil) + machine := &machineImpl{ + backend: mockBackend, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second, + AdvanceIncCycles: 1, + }, + } + + bounds, err := machine.executionCycleBounds(context.Background(), AdvanceStateRequest) + s.Require().NoError(err) + s.Equal(maxSafeStart, bounds.start) + s.Equal(maxSafeStart+model.MaxExecutionCycleSpan, bounds.limit) + s.Equal(model.MaxExecutionCycleSpan, bounds.span) + s.False(bounds.configured) + mockBackend.AssertExpectations(s.T()) +} + +func (s *ImplementationSuite) TestExecutionCycleBoundsUsesRequestSpecificMaximum() { + const start uint64 = 17 + for _, test := range []struct { + name string + reqType requestType + span uint64 + params model.ExecutionParameters + }{ + { + name: "advance configured", reqType: AdvanceStateRequest, span: 101, + params: model.ExecutionParameters{AdvanceIncCycles: 1, AdvanceMaxCycles: 101}, + }, + { + name: "inspect configured", reqType: InspectStateRequest, span: 202, + params: model.ExecutionParameters{InspectIncCycles: 1, InspectMaxCycles: 202}, + }, + { + name: "advance zero uses fixed", reqType: AdvanceStateRequest, span: model.MaxExecutionCycleSpan, + params: model.ExecutionParameters{AdvanceIncCycles: 1}, + }, + { + name: "inspect zero uses fixed", reqType: InspectStateRequest, span: model.MaxExecutionCycleSpan, + params: model.ExecutionParameters{InspectIncCycles: 1}, + }, + } { + s.Run(test.name, func() { + backend := NewMockBackend() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start, nil) + machine := &machineImpl{backend: backend, logger: s.logger, params: test.params} + + bounds, err := machine.executionCycleBounds(context.Background(), test.reqType) + s.Require().NoError(err) + s.Equal(start, bounds.start) + s.Equal(start+test.span, bounds.limit) + s.Equal(test.span, bounds.span) + if test.params.AdvanceMaxCycles != 0 || test.params.InspectMaxCycles != 0 { + s.True(bounds.configured) + } else { + s.False(bounds.configured) + } + backend.AssertExpectations(s.T()) + }) + } +} + +func (s *ImplementationSuite) TestExecutionCycleBoundsRejectsMaximumAboveHardLimit() { + for _, reqType := range []requestType{AdvanceStateRequest, InspectStateRequest} { + s.Run(reqType.String(), func() { + backend := NewMockBackend() + params := model.ExecutionParameters{ + AdvanceIncCycles: 1, + AdvanceMaxCycles: model.MaxExecutionCycles, + InspectIncCycles: 1, + } + if reqType == InspectStateRequest { + params.AdvanceMaxCycles = 0 + params.InspectMaxCycles = model.MaxExecutionCycles + } + machine := &machineImpl{backend: backend, logger: s.logger, params: params} + + _, err := machine.executionCycleBounds(context.Background(), reqType) + s.Require().ErrorIs(err, ErrReachedLimitMcycle) + s.Contains(err.Error(), reqType.String()) + s.Contains(err.Error(), fmt.Sprint(model.MaxExecutionCycles)) + backend.AssertNotCalled(s.T(), "ReadMCycle", mock.Anything) + backend.AssertExpectations(s.T()) + }) + } +} + +func (s *ImplementationSuite) TestProcessRejectsInvalidMaximumBeforeCMIO() { + backend := NewMockBackend() + backend.On("CmioRxBufferSize").Return(uint64(1024)) + machine := &machineImpl{ + backend: backend, + logger: s.logger, + params: model.ExecutionParameters{ + AdvanceIncCycles: 1, + AdvanceMaxCycles: model.MaxExecutionCycles, + FastDeadline: time.Second, + }, + } + + response, err := machine.Advance(context.Background(), []byte("input"), Hash{}, false) + s.Require().Nil(response) + s.Require().ErrorIs(err, ErrReachedLimitMcycle) + backend.AssertNotCalled(s.T(), "ReadMCycle", mock.Anything) + backend.AssertNotCalled(s.T(), "SendCmioResponse", + mock.Anything, mock.Anything, mock.Anything, mock.Anything) + backend.AssertExpectations(s.T()) +} + +func (s *ImplementationSuite) TestProcessRejectsZeroIncrementBeforeCMIO() { + for _, test := range []struct { + name string + reqType requestType + invoke func(*machineImpl) error + }{ + { + name: "advance", + reqType: AdvanceStateRequest, + invoke: func(machine *machineImpl) error { + _, err := machine.Advance(context.Background(), []byte("input"), Hash{}, false) + return err + }, + }, + { + name: "inspect", + reqType: InspectStateRequest, + invoke: func(machine *machineImpl) error { + _, err := machine.Inspect(context.Background(), []byte("query")) + return err + }, + }, + } { + s.Run(test.name, func() { + backend := NewMockBackend() + backend.On("CmioRxBufferSize").Return(uint64(1024)).Maybe() + params := model.ExecutionParameters{ + FastDeadline: time.Second, + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: time.Second, + InspectIncDeadline: time.Second, + InspectMaxDeadline: time.Second, + } + machine := &machineImpl{backend: backend, logger: s.logger, params: params} + + err := test.invoke(machine) + s.Require().Error(err) + s.Contains(err.Error(), test.reqType.String()) + s.Contains(err.Error(), "increment") + backend.AssertNotCalled(s.T(), "ReadMCycle", mock.Anything) + backend.AssertNotCalled(s.T(), "SendCmioResponse", + mock.Anything, mock.Anything, mock.Anything, mock.Anything) + backend.AssertNotCalled(s.T(), "Run", mock.Anything, mock.Anything) + backend.AssertNotCalled(s.T(), "RunAndCollectRootHashes", mock.Anything, mock.Anything, mock.Anything) + backend.AssertExpectations(s.T()) + }) + } +} + +func (s *ImplementationSuite) TestConfiguredCycleEndpointCompletionSucceeds() { + const start, span = uint64(10), uint64(5) + expectedOutputsHash := randomFakeHash() + backend := NewMockBackend() + backend.On("CmioRxBufferSize").Return(uint64(1024)) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start, nil).Once() + backend.On( + "SendCmioResponse", + uint16(AdvanceStateRequest), []byte("input"), mock.Anything, mock.AnythingOfType("time.Duration"), + ).Return(nil) + backend.On("Run", start+span, mock.AnythingOfType("time.Duration")).Return(YieldedManually, nil) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start+span, nil).Once() + backend.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( + uint8(0), uint16(ManualYieldReasonAccepted), expectedOutputsHash[:], nil) + machine := &machineImpl{backend: backend, logger: s.logger, params: model.ExecutionParameters{ + AdvanceIncCycles: span + 100, AdvanceMaxCycles: span, + AdvanceIncDeadline: time.Second, AdvanceMaxDeadline: time.Second, + FastDeadline: time.Second, + }} + + response, err := machine.Advance(context.Background(), []byte("input"), Hash{}, false) + s.Require().NoError(err) + s.Require().Equal(CompletionStatusAccepted, response.Status) + backend.AssertExpectations(s.T()) +} + +func (s *ImplementationSuite) TestConfiguredCycleEndpointHaltSucceeds() { + const start, span = uint64(20), uint64(7) + backend := NewMockBackend() + backend.On("CmioRxBufferSize").Return(uint64(1024)) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start, nil).Once() + backend.On( + "SendCmioResponse", + uint16(AdvanceStateRequest), []byte("input"), mock.Anything, mock.AnythingOfType("time.Duration"), + ).Return(nil) + backend.On("Run", start+span, mock.AnythingOfType("time.Duration")).Return(Halted, nil) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start+span, nil).Once() + machine := &machineImpl{backend: backend, logger: s.logger, params: model.ExecutionParameters{ + AdvanceIncCycles: span + 100, AdvanceMaxCycles: span, + AdvanceIncDeadline: time.Second, AdvanceMaxDeadline: time.Second, + FastDeadline: time.Second, + }} + + response, err := machine.Advance(context.Background(), []byte("input"), Hash{}, false) + s.Require().NoError(err) + s.Require().Equal(CompletionStatusHalted, response.Status) + backend.AssertExpectations(s.T()) +} + +func (s *ImplementationSuite) TestConfiguredCycleExhaustionFails() { + const start, span = uint64(10), uint64(5) + backend := NewMockBackend() + backend.On("CmioRxBufferSize").Return(uint64(1024)) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start, nil).Once() + backend.On( + "SendCmioResponse", + uint16(InspectStateRequest), []byte("query"), mock.Anything, mock.AnythingOfType("time.Duration"), + ).Return(nil) + backend.On("Run", start+span, mock.AnythingOfType("time.Duration")).Return(ReachedTargetMcycle, nil) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start+span, nil).Once() + machine := &machineImpl{backend: backend, logger: s.logger, params: model.ExecutionParameters{ + InspectIncCycles: span + 100, InspectMaxCycles: span, + InspectIncDeadline: time.Second, InspectMaxDeadline: time.Second, + FastDeadline: time.Second, + }} + + response, err := machine.Inspect(context.Background(), []byte("query")) + s.Require().NotNil(response) + s.Require().Equal(CompletionStatusUnknown, response.Status) + s.Require().Empty(response.Reports) + s.Require().ErrorIs(err, ErrReachedLimitMcycle) + s.Contains(err.Error(), "inspect execution reached configured cycle limit") + s.Contains(err.Error(), "start=10 requested_span=5") + backend.AssertExpectations(s.T()) +} + +func (s *ImplementationSuite) TestAdvanceCycleExhaustionSources() { + for _, test := range []struct { + name string + configuredSpan uint64 + breakReason BreakReason + wantSource string + }{ + { + name: "configured endpoint", configuredSpan: 9, + breakReason: ReachedTargetMcycle, wantSource: "configured", + }, + { + name: "fixed window", breakReason: McycleOverflow, wantSource: "fixed", + }, + } { + s.Run(test.name, func() { + const start = uint64(30) + span := test.configuredSpan + if span == 0 { + span = model.MaxExecutionCycleSpan + } + backend := NewMockBackend() + backend.On("CmioRxBufferSize").Return(uint64(1024)) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start, nil).Once() + backend.On( + "SendCmioResponse", + uint16(AdvanceStateRequest), []byte("input"), mock.Anything, mock.AnythingOfType("time.Duration"), + ).Return(nil) + backend.On("Run", start+span, mock.AnythingOfType("time.Duration")).Return(test.breakReason, nil) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start+span, nil).Once() + machine := &machineImpl{backend: backend, logger: s.logger, params: model.ExecutionParameters{ + AdvanceIncCycles: ^uint64(0), + AdvanceMaxCycles: test.configuredSpan, + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: time.Second, + FastDeadline: time.Second, + }} + + response, err := machine.Advance(context.Background(), []byte("input"), Hash{}, false) + s.Require().Nil(response) + s.Require().ErrorIs(err, ErrReachedLimitMcycle) + if test.breakReason == McycleOverflow { + s.Require().ErrorIs(err, ErrMcycleOverflow) + } + s.Contains(err.Error(), test.wantSource) + s.Contains(err.Error(), "cycle limit") + s.Contains(err.Error(), fmt.Sprintf("requested_span=%d", span)) + backend.AssertExpectations(s.T()) + }) + } +} + +func (s *ImplementationSuite) TestConfiguredLimitTiePreservesMachineOverflowPrecedence() { + const start = uint64(30) + configuredSpan := model.MaxExecutionCycleSpan + limit := start + configuredSpan + backend := NewMockBackend() + backend.On("Run", limit, mock.AnythingOfType("time.Duration")).Return(McycleOverflow, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(limit, nil).Once() + machine := &machineImpl{backend: backend, logger: s.logger, params: model.ExecutionParameters{ + AdvanceIncCycles: ^uint64(0), + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: time.Second, + AdvanceMaxCycles: configuredSpan, + FastDeadline: time.Second, + }} + + _, err := machine.run(context.Background(), AdvanceStateRequest, false, executionBounds{ + start: start, limit: limit, span: configuredSpan, configured: true, + }) + s.Require().ErrorIs(err, ErrReachedLimitMcycle) + s.Require().ErrorIs(err, ErrMcycleOverflow) + s.Contains(err.Error(), "advance execution stopped at machine imcyclemax coincident with configured target") + backend.AssertExpectations(s.T()) +} + +func (s *ImplementationSuite) TestExecutionStartingAtMachineMaximumPreservesOverflowOrigin() { + for _, test := range []struct { + name string + configuredMax uint64 + }{ + {name: "no operator cap", configuredMax: 0}, + {name: "configured cap also saturates", configuredMax: 1}, + } { + s.Run(test.name, func() { + backend := NewMockBackend() + backend.On("CmioRxBufferSize").Return(uint64(1024)) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(^uint64(0), nil).Once() + backend.On( + "SendCmioResponse", + uint16(AdvanceStateRequest), []byte("input"), mock.Anything, mock.AnythingOfType("time.Duration"), + ).Return(nil).Once() + backend.On("Run", ^uint64(0), mock.AnythingOfType("time.Duration")). + Return(McycleOverflow, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(^uint64(0), nil).Once() + machine := &machineImpl{backend: backend, logger: s.logger, params: model.ExecutionParameters{ + AdvanceIncCycles: 1, + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: time.Second, + AdvanceMaxCycles: test.configuredMax, + FastDeadline: time.Second, + }} + + response, err := machine.Advance(context.Background(), []byte("input"), Hash{}, false) + s.Require().Nil(response) + s.Require().ErrorIs(err, ErrReachedLimitMcycle) + s.Require().ErrorIs(err, ErrMcycleOverflow) + s.Contains(err.Error(), "target_span=0") + s.Contains(err.Error(), "executed_cycles=0") + backend.AssertExpectations(s.T()) + }) + } +} + +func (s *ImplementationSuite) TestRunRejectsNonOverflowReasonAtMachineMaximum() { + for _, breakReason := range []BreakReason{ReachedTargetMcycle, YieldedSoftly} { + s.Run(fmt.Sprintf("break reason %d", breakReason), func() { + backend := NewMockBackend() + backend.On("Run", ^uint64(0), mock.AnythingOfType("time.Duration")). + Return(breakReason, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(^uint64(0), nil).Once() + machine := &machineImpl{backend: backend, logger: s.logger, params: model.ExecutionParameters{ + FastDeadline: time.Second, + }} + + result, err := machine.runIncrementInterval( + context.Background(), ^uint64(0), ^uint64(0), 1, nil, time.Second, + ) + + s.Equal(breakReason, result.breakReason) + s.Equal(^uint64(0), result.currentCycle) + s.Require().ErrorIs(err, ErrMachineInternal) + s.Contains(err.Error(), "instead of mcycle overflow at MaxUint64") + backend.AssertExpectations(s.T()) + }) + } +} + +func (s *ImplementationSuite) TestInspectInheritedMcycleOverflowDoesNotClaimLocalLimitExhaustion() { + const start = uint64(100) + + for _, test := range []struct { + name string + configuredMax uint64 + requestedSpan uint64 + executedCycles uint64 + stopDescription string + }{ + { + name: "before configured limit", + configuredMax: 50, + requestedSpan: 50, + executedCycles: 7, + stopDescription: "stopped before configured cycle limit", + }, + { + name: "at configured limit", + configuredMax: 7, + requestedSpan: 7, + executedCycles: 7, + stopDescription: "stopped at machine imcyclemax coincident with configured target", + }, + { + name: "before zero default fixed limit", + requestedSpan: model.MaxExecutionCycleSpan, + executedCycles: 7, + stopDescription: "stopped before fixed cycle limit", + }, + } { + s.Run(test.name, func() { + backend := NewMockBackend() + backend.On("CmioRxBufferSize").Return(uint64(1024)) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start, nil).Once() + backend.On( + "SendCmioResponse", + uint16(InspectStateRequest), []byte("query"), nil, mock.AnythingOfType("time.Duration"), + ).Return(nil).Once() + backend.On("Run", start+test.requestedSpan, mock.AnythingOfType("time.Duration")). + Return(McycleOverflow, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(start+test.executedCycles, nil).Once() + + machine := &machineImpl{backend: backend, logger: s.logger, params: model.ExecutionParameters{ + FastDeadline: time.Second, + InspectIncCycles: ^uint64(0), + InspectIncDeadline: time.Second, + InspectMaxDeadline: time.Second, + InspectMaxCycles: test.configuredMax, + }} + + response, err := machine.Inspect(context.Background(), []byte("query")) + s.Require().NotNil(response) + s.Equal(CompletionStatusUnknown, response.Status) + s.Empty(response.Reports) + s.Require().ErrorIs(err, ErrReachedLimitMcycle) + s.Require().ErrorIs(err, ErrMcycleOverflow) + s.Contains(err.Error(), test.stopDescription) + s.Contains(err.Error(), fmt.Sprintf("requested_span=%d", test.requestedSpan)) + s.Contains(err.Error(), fmt.Sprintf("executed_cycles=%d", test.executedCycles)) + backend.AssertExpectations(s.T()) + }) + } +} + +func (s *ImplementationSuite) TestExecutionCycleBoundsSaturatesAtMachineMaximum() { + for _, test := range []struct { + name string + start uint64 + configured uint64 + }{ + { + name: "configured cap beyond representable range", start: ^uint64(0) - 10, configured: 11, + }, + { + name: "zero cap", start: ^uint64(0) - model.MaxExecutionCycleSpan + 1, + }, + { + name: "maximum cap", start: ^uint64(0) - model.MaxExecutionCycleSpan + 1, + configured: model.MaxExecutionCycleSpan, + }, + } { + s.Run(test.name, func() { + backend := NewMockBackend() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(test.start, nil) + machine := &machineImpl{backend: backend, logger: s.logger, params: model.ExecutionParameters{ + AdvanceIncCycles: 1, + AdvanceMaxCycles: test.configured, + }} + + bounds, err := machine.executionCycleBounds(context.Background(), AdvanceStateRequest) + s.Require().NoError(err) + s.Equal(test.start, bounds.start) + s.Equal(^uint64(0), bounds.limit) + backend.AssertExpectations(s.T()) + }) + } +} + +func (s *ImplementationSuite) TestExecutionResponseLimitErrorsExposeOnlyCounts() { + for _, test := range []struct { + kind string + count int + capacity int + sentinel error + }{ + {kind: "output", count: maxOutputs + 1, capacity: maxOutputs, sentinel: ErrOutputsLimitExceeded}, + {kind: "report", count: maxReports + 1, capacity: maxReports, sentinel: ErrReportsLimitExceeded}, + } { + err := executionResponseLimitError( + AdvanceStateRequest, test.kind, test.count, test.capacity, test.sentinel, + ) + s.Require().ErrorIs(err, test.sentinel) + s.Contains(err.Error(), fmt.Sprintf("advance %s count %d", test.kind, test.count)) + s.Contains(err.Error(), fmt.Sprintf("capacity %d", test.capacity)) + } +} + +func (s *ImplementationSuite) TestInspectUsesConfiguredCycleIncrement() { + const startCycle uint64 = 7 + const inspectIncCycles uint64 = 137 + + mockBackend := NewMockBackend() + mockBackend.On("CmioRxBufferSize").Return(uint64(1024)) + mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(startCycle, nil).Once() + mockBackend.On( + "SendCmioResponse", + uint16(InspectStateRequest), mock.Anything, mock.Anything, mock.AnythingOfType("time.Duration"), + ).Return(nil) + mockBackend.On("Run", startCycle+inspectIncCycles, mock.AnythingOfType("time.Duration")). + Return(YieldedManually, nil) + mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(startCycle+inspectIncCycles, nil).Once() + mockBackend.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( + uint8(0), uint16(ManualYieldReasonAccepted), []byte(nil), nil) + + machine := &machineImpl{ + backend: mockBackend, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second, + InspectIncCycles: inspectIncCycles, + InspectIncDeadline: time.Second, + InspectMaxDeadline: time.Second, + }, + } + + response, err := machine.Inspect(context.Background(), []byte("query")) + s.Require().NoError(err) + s.Require().Equal(CompletionStatusAccepted, response.Status) + mockBackend.AssertExpectations(s.T()) +} + // Test Store method func (s *ImplementationSuite) TestStore() { require := s.Require() @@ -711,7 +1300,9 @@ func (s *ImplementationSuite) TestRun() { }, } - result, err := machine.run(ctx, AdvanceStateRequest, false, 0, 1000) + result, err := machine.run(ctx, AdvanceStateRequest, false, executionBounds{ + start: 0, limit: 1000, span: 1000, configured: true, + }) require.NoError(err) require.Empty(result.outputs) require.Empty(result.reports) @@ -719,6 +1310,7 @@ func (s *ImplementationSuite) TestRun() { // Test run with read cycle error mockBackend2 := NewMockBackend() + mockBackend2.On("Run", uint64(100), mock.AnythingOfType("time.Duration")).Return(YieldedManually, nil) mockBackend2.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(0), errors.New("read cycle failed")) machine2 := &machineImpl{ backend: mockBackend2, @@ -731,7 +1323,9 @@ func (s *ImplementationSuite) TestRun() { AdvanceMaxDeadline: time.Second * 10, }, } - _, err = machine2.run(ctx, AdvanceStateRequest, false, 0, 1000) + _, err = machine2.run(ctx, AdvanceStateRequest, false, executionBounds{ + start: 0, limit: 1000, span: 1000, configured: true, + }) require.Error(err) require.Contains(err.Error(), "read cycle failed") mockBackend2.AssertExpectations(s.T()) @@ -754,7 +1348,9 @@ func (s *ImplementationSuite) TestRun() { }, } - _, err = machine3.run(ctx, AdvanceStateRequest, false, 0, 1000) + _, err = machine3.run(ctx, AdvanceStateRequest, false, executionBounds{ + start: 0, limit: 1000, span: 1000, configured: true, + }) require.NoError(err) mockBackend3.AssertExpectations(s.T()) @@ -777,7 +1373,9 @@ func (s *ImplementationSuite) TestRun() { }, } - _, err = machine4.run(ctx, AdvanceStateRequest, false, 0, 1000) + _, err = machine4.run(ctx, AdvanceStateRequest, false, executionBounds{ + start: 0, limit: 1000, span: 1000, configured: true, + }) require.Error(err) require.Contains(err.Error(), "could not read output/report") require.Contains(err.Error(), "cmio request failed") @@ -805,7 +1403,9 @@ func (s *ImplementationSuite) TestRun() { }, } - result5, err := machine5.run(ctx, AdvanceStateRequest, false, 0, 1000) + result5, err := machine5.run(ctx, AdvanceStateRequest, false, executionBounds{ + start: 0, limit: 1000, span: 1000, configured: true, + }) require.NoError(err) require.Len(result5.outputs, 1) require.Equal([]byte("output data"), []byte(result5.outputs[0])) @@ -834,7 +1434,9 @@ func (s *ImplementationSuite) TestRun() { }, } - result6, err := machine6.run(ctx, AdvanceStateRequest, false, 0, 1000) + result6, err := machine6.run(ctx, AdvanceStateRequest, false, executionBounds{ + start: 0, limit: 1000, span: 1000, configured: true, + }) require.NoError(err) require.Empty(result6.outputs) require.Len(result6.reports, 1) @@ -999,7 +1601,6 @@ func (s *ImplementationSuite) TestProcess() { // Test process with run error mockBackend4 := NewMockBackend() mockBackend4.On("CmioRxBufferSize").Return(uint64(1024)) - mockBackend4.On("SendCmioResponse", mock.AnythingOfType("uint16"), mock.Anything, expectedHash, mock.AnythingOfType("time.Duration")).Return(nil) mockBackend4.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(0), errors.New("read cycle failed")) machine4 := &machineImpl{ backend: mockBackend4, @@ -1015,6 +1616,7 @@ func (s *ImplementationSuite) TestProcess() { _, err = machine4.process(ctx, input, AdvanceStateRequest, &expectedHash, false) require.Error(err) require.Contains(err.Error(), "read cycle failed") + mockBackend4.AssertNotCalled(s.T(), "SendCmioResponse", mock.Anything, mock.Anything, mock.Anything) mockBackend4.AssertExpectations(s.T()) } @@ -1046,7 +1648,9 @@ func (s *ImplementationSuite) TestRunWithAutomaticYields() { mockBackend.On("Run", uint64(150), mock.AnythingOfType("time.Duration")).Return(YieldedManually, nil).Once() mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(100), nil).Once() - result, err := machine.run(ctx, AdvanceStateRequest, false, 0, 1000) + result, err := machine.run(ctx, AdvanceStateRequest, false, executionBounds{ + start: 0, limit: 1000, span: 1000, configured: true, + }) require.NoError(err) require.Len(result.outputs, 1) require.Equal([]byte("test output"), result.outputs[0]) @@ -1083,7 +1687,9 @@ func (s *ImplementationSuite) TestRunWithAutomaticYieldsReports() { mockBackend.On("Run", uint64(150), mock.AnythingOfType("time.Duration")).Return(YieldedManually, nil).Once() mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(100), nil).Once() - result, err := machine.run(ctx, AdvanceStateRequest, false, 0, 1000) + result, err := machine.run(ctx, AdvanceStateRequest, false, executionBounds{ + start: 0, limit: 1000, span: 1000, configured: true, + }) require.NoError(err) require.Empty(result.outputs) require.Len(result.reports, 1) @@ -1145,7 +1751,9 @@ func (s *ImplementationSuite) TestMultipleAutomaticYields() { mockBackend.On("Run", uint64(150), mock.AnythingOfType("time.Duration")).Return(YieldedManually, nil).Once() mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(60), nil).Once() - result, err := machine.run(ctx, AdvanceStateRequest, false, 0, 1000) + result, err := machine.run(ctx, AdvanceStateRequest, false, executionBounds{ + start: 0, limit: 1000, span: 1000, configured: true, + }) require.NoError(err) require.Len(result.outputs, 2) diff --git a/pkg/machine/machine.go b/pkg/machine/machine.go index 855c9e997..c0e2956b9 100644 --- a/pkg/machine/machine.go +++ b/pkg/machine/machine.go @@ -82,12 +82,25 @@ var ( ErrHalted = errors.New("machine halted") ErrOutputsLimitExceeded = errors.New("outputs limit exceeded") ErrReportsLimitExceeded = errors.New("reports limit exceeded") - ErrReachedTargetMcycle = errors.New("machine reached target mcycle") ErrPayloadLengthLimitExceeded = errors.New("payload length limit exceeded") ErrHashLength = errors.New("hash does not have the exactly number of bytes") ErrReachedLimitMcycle = errors.New("machine reached limit mcycle") + + // ErrMcycleOverflow preserves the emulator-reported fact that the machine + // itself reached imcyclemax. It remains distinguishable from a node target + // because canonical overflow eligibility depends on the stop origin. + ErrMcycleOverflow = fmt.Errorf("machine reached imcyclemax: %w", ErrReachedLimitMcycle) ) +// IsExecutionLimitError reports whether err is an incomplete execution caused +// by a payload, response-count, or cycle ceiling. +func IsExecutionLimitError(err error) bool { + return errors.Is(err, ErrPayloadLengthLimitExceeded) || + errors.Is(err, ErrOutputsLimitExceeded) || + errors.Is(err, ErrReportsLimitExceeded) || + errors.Is(err, ErrReachedLimitMcycle) +} + // The Machine interface covers the core rollups-oriented functionalities of a cartesi // machine: forking, getting the merkle tree's root hash, sending advance-state requests, // sending inspect-state requests, and storing machine state. @@ -137,10 +150,10 @@ func DefaultConfig(path string) *MachineConfig { Address: "127.0.0.1:0", Path: path, ExecutionParameters: model.ExecutionParameters{ - AdvanceIncCycles: 1 << 22, // nolint: mnd - AdvanceMaxCycles: ^uint64(0) >> 2, // nolint: mnd - InspectIncCycles: 1 << 22, // nolint: mnd - InspectMaxCycles: ^uint64(0) >> 2, // nolint: mnd + AdvanceIncCycles: 1 << 22, //nolint:mnd + AdvanceMaxCycles: 0, + InspectIncCycles: 1 << 22, //nolint:mnd + InspectMaxCycles: 0, AdvanceIncDeadline: time.Second * 10, // nolint: mnd AdvanceMaxDeadline: time.Second * 180, // nolint: mnd InspectIncDeadline: time.Second * 10, // nolint: mnd diff --git a/pkg/machine/machine_test.go b/pkg/machine/machine_test.go index d35135fde..f632c3fdb 100644 --- a/pkg/machine/machine_test.go +++ b/pkg/machine/machine_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + "github.com/cartesi/rollups-node/internal/model" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" ) @@ -185,9 +186,9 @@ func (s *MachineSuite) TestDefaultConfig() { // Test execution parameters are set require.Greater(config.ExecutionParameters.AdvanceIncCycles, uint64(0)) - require.Greater(config.ExecutionParameters.AdvanceMaxCycles, uint64(0)) + require.Zero(config.ExecutionParameters.AdvanceMaxCycles) require.Greater(config.ExecutionParameters.InspectIncCycles, uint64(0)) - require.Greater(config.ExecutionParameters.InspectMaxCycles, uint64(0)) + require.Zero(config.ExecutionParameters.InspectMaxCycles) require.Greater(config.ExecutionParameters.AdvanceIncDeadline, time.Duration(0)) require.Greater(config.ExecutionParameters.AdvanceMaxDeadline, time.Duration(0)) require.Greater(config.ExecutionParameters.InspectIncDeadline, time.Duration(0)) @@ -197,6 +198,28 @@ func (s *MachineSuite) TestDefaultConfig() { require.Greater(config.ExecutionParameters.FastDeadline, time.Duration(0)) } +func (s *MachineSuite) TestConfiguredHardCycleCeilingMatchesMachineInputSpan() { + s.Require().Equal(BarchSpanToInput, model.MaxExecutionCycleSpan) + s.Require().Equal(uint64(1)< Date: Wed, 12 Aug 2026 17:40:38 -0300 Subject: [PATCH 03/13] refactor(machine): align PRT collection with emulator 0.21 --- internal/advancer/advancer.go | 4 +- internal/manager/instance.go | 85 +- internal/manager/instance_test.go | 399 +++-- internal/model/input_hash_collection.go | 50 + internal/model/input_hash_collection_test.go | 46 + internal/model/models.go | 8 +- internal/repository/postgres/bulk.go | 2 +- .../repository/repotest/bulk_test_cases.go | 10 +- .../repotest/state_hash_test_cases.go | 4 +- internal/validator/validator.go | 12 +- internal/validator/validator_test.go | 12 +- pkg/emulator/types.go | 7 + pkg/machine/backend.go | 12 +- pkg/machine/computation_hash.go | 115 ++ pkg/machine/computation_hash_test.go | 45 + pkg/machine/doc.go | 38 + pkg/machine/implementation.go | 265 ++- pkg/machine/implementation_test.go | 1461 +++++++++++++---- pkg/machine/libcartesi.go | 89 +- pkg/machine/libcartesi_test.go | 72 +- pkg/machine/machine.go | 57 +- pkg/machine/machine_test.go | 80 +- 22 files changed, 2172 insertions(+), 701 deletions(-) create mode 100644 internal/model/input_hash_collection.go create mode 100644 internal/model/input_hash_collection_test.go create mode 100644 pkg/machine/computation_hash.go create mode 100644 pkg/machine/computation_hash_test.go create mode 100644 pkg/machine/doc.go diff --git a/internal/advancer/advancer.go b/internal/advancer/advancer.go index 8efec4f63..f3bb375c1 100644 --- a/internal/advancer/advancer.go +++ b/internal/advancer/advancer.go @@ -297,8 +297,8 @@ func (s *Service) processInputs(ctx context.Context, app *Application, inputs [] "status", result.Status, "outputs", len(result.Outputs), "reports", len(result.Reports), - "hashes", len(result.Hashes), - "remaining_cycles", result.RemainingMetaCycles, + "periodic_state_hashes", len(result.PeriodicStateHashes), + "padding_repetitions", result.PaddingRepetitions, ) // Store the result in the database diff --git a/internal/manager/instance.go b/internal/manager/instance.go index 1df5d58aa..c8ecc7449 100644 --- a/internal/manager/instance.go +++ b/internal/manager/instance.go @@ -325,13 +325,16 @@ func (m *MachineInstanceImpl) Advance(ctx context.Context, input []byte, epochIn Outputs: advanceResp.Outputs, Reports: advanceResp.Reports, ExceptionData: advanceResp.ExceptionData, - Hashes: advanceResp.Hashes, - RemainingMetaCycles: advanceResp.RemainingCycles, + PeriodicStateHashes: advanceResp.PeriodicStateHashes, + PaddingRepetitions: advanceResp.PaddingRepetitions, IsDaveConsensus: computeHashes, } - // If the input was accepted, update the machine state - if result.Status == InputCompletionStatus_Accepted { + // Resolve the canonical result and fork disposition once. Validation below + // must succeed before the selected disposition mutates the live instance. + adoptFork := false + switch result.Status { + case InputCompletionStatus_Accepted: // Get the machine hash after processing result.MachineHash, err = fork.Hash(ctx) if err != nil { @@ -342,7 +345,37 @@ func (m *MachineInstanceImpl) Advance(ctx context.Context, input []byte, epochIn if err != nil { return nil, errors.Join(err, fork.Close()) } + adoptFork = true + case InputCompletionStatus_Rejected, + InputCompletionStatus_Exception, + InputCompletionStatus_MachineHalted: + // Use the previous state for currently nonaccepted inputs. + result.MachineHash = prevMachineHash + result.OutputsHash = prevOutputsHash + result.OutputsHashProof = prevOutputsHashProof + case InputCompletionStatus_None: + return nil, errors.Join( + fmt.Errorf("cannot resolve advance result for status %q", result.Status), + fork.Close(), + ) + default: + return nil, errors.Join( + fmt.Errorf("cannot resolve advance result for unknown status %q", result.Status), + fork.Close(), + ) + } + + if computeHashes { + err = validateCanonicalInputHashCollectionSpan( + uint64(len(result.PeriodicStateHashes)), + result.PaddingRepetitions, + ) + if err != nil { + return nil, errors.Join(err, fork.Close()) + } + } + if adoptFork { // Replace the current machine with the fork m.mutex.HLock() oldRuntime := m.runtime @@ -354,10 +387,6 @@ func (m *MachineInstanceImpl) Advance(ctx context.Context, input []byte, epochIn m.logger.Warn("Failed to close old machine runtime", "error", err) } } else { - // Use the previous state for rejected inputs - result.MachineHash = prevMachineHash - result.OutputsHash = prevOutputsHash - result.OutputsHashProof = prevOutputsHashProof // Close the fork since we're not using it if err := fork.Close(); err != nil { @@ -373,6 +402,21 @@ func (m *MachineInstanceImpl) Advance(ctx context.Context, input []byte, epochIn return result, nil } +func validateCanonicalInputHashCollectionSpan( + hashCount uint64, + paddingRepetitions uint64, +) error { + if err := machine.ValidateInputHashCollectionSpan( + hashCount, paddingRepetitions, + ); err != nil { + return fmt.Errorf("invalid canonical input hash collection span: %w", err) + } + if paddingRepetitions == 0 { + return errors.New("canonical input hash collection requires a positive final repetition tail") + } + return nil +} + // forkForInspect creates a copy of the machine for inspect operations // It returns the forked machine and the current processed inputs count func (m *MachineInstanceImpl) forkForInspect(ctx context.Context) (machine.Machine, uint64, error) { @@ -422,6 +466,10 @@ func (m *MachineInstanceImpl) Inspect(ctx context.Context, query []byte) (*Inspe if inspectResponse != nil { result.Reports = inspectResponse.Reports } + + // An execution error takes precedence over any status supplied by a Machine + // implementation. Inspection did not complete, but reports emitted before + // the failure remain useful to the caller. if inspectErr != nil { result.Error = inspectErr } else if inspectResponse == nil || !inspectResponse.Status.IsCompleted() { @@ -435,6 +483,8 @@ func (m *MachineInstanceImpl) Inspect(ctx context.Context, query []byte) (*Inspe result.Error = machine.ErrException case machine.CompletionStatusHalted: result.Error = machine.ErrHalted + case machine.CompletionStatusUnknown: + result.Error = errors.Join(ErrIncompleteInspect, machine.ErrMachineInternal) default: result.Error = errors.Join(ErrIncompleteInspect, machine.ErrMachineInternal) } @@ -748,7 +798,8 @@ func (f *SnapshotMachineRuntimeFactory) CreateMachineRuntime( // Default factory instance var defaultFactory MachineRuntimeFactory = &DefaultMachineRuntimeFactory{} -// toInputStatus converts only completed machine statuses to input statuses. +// toInputStatus converts only completed, deterministic machine statuses to +// canonical input statuses. Infrastructure interruptions never reach here. func toInputStatus(status machine.CompletionStatus) (InputCompletionStatus, error) { switch status { case machine.CompletionStatusAccepted: @@ -759,9 +810,17 @@ func toInputStatus(status machine.CompletionStatus) (InputCompletionStatus, erro return InputCompletionStatus_Exception, nil case machine.CompletionStatusHalted: return InputCompletionStatus_MachineHalted, nil + case machine.CompletionStatusUnknown: + return InputCompletionStatus_None, fmt.Errorf( + "unknown completed machine status %d: %w", + status, + ErrIncompleteAdvance, + ) default: return InputCompletionStatus_None, fmt.Errorf( - "unknown completed machine status %d: %w", status, ErrIncompleteAdvance, + "unknown completed machine status %d: %w", + status, + ErrIncompleteAdvance, ) } } @@ -771,7 +830,11 @@ func validateCompletionExceptionData(status machine.CompletionStatus, data []byt case status == machine.CompletionStatusException && data == nil: return fmt.Errorf("completed exception has no exception data: %w", machine.ErrMachineInternal) case status != machine.CompletionStatusException && data != nil: - return fmt.Errorf("completion status %d unexpectedly has exception data: %w", status, machine.ErrMachineInternal) + return fmt.Errorf( + "completion status %d unexpectedly has exception data: %w", + status, + machine.ErrMachineInternal, + ) default: return nil } diff --git a/internal/manager/instance_test.go b/internal/manager/instance_test.go index 9233fe1b4..30394f138 100644 --- a/internal/manager/instance_test.go +++ b/internal/manager/instance_test.go @@ -27,14 +27,6 @@ func TestMachineInstance(t *testing.T) { type MachineInstanceSuite struct{ suite.Suite } -func (s *MachineInstanceSuite) TestMcycleOverflowRemainsIncomplete() { - require := s.Require() - - status, err := toInputStatus(machine.CompletionStatusUnknown) - require.ErrorIs(err, ErrIncompleteAdvance) - require.Empty(status) -} - // MockMachineRuntimeFactory implements MachineRuntimeFactory for testing type MockMachineRuntimeFactory struct { RuntimeToReturn machine.Machine @@ -334,56 +326,127 @@ func (s *MachineInstanceSuite) TestAdvance() { s.Run("Reject", func() { require := s.Require() - inner, fork, machineInst := s.setupAdvance() - fork.AdvanceStatusReturn = machine.CompletionStatusRejected + inner, fork, instance := s.setupAdvance() + fork.CompletionStatusReturn = machine.CompletionStatusRejected fork.CloseError = nil - res, err := machineInst.Advance(context.Background(), []byte{}, 0, 5, false) + res, err := instance.Advance(context.Background(), []byte{}, 0, 5, false) require.Nil(err) require.NotNil(res) - require.Same(inner, machineInst.runtime) + require.Same(inner, instance.runtime) require.Equal(model.InputCompletionStatus_Rejected, res.Status) require.Equal(expectedOutputs, res.Outputs) require.Equal(expectedReports1, res.Reports) require.Equal(newHash(1), res.OutputsHash) require.Equal(newHash(2), res.MachineHash) - require.Equal(uint64(6), machineInst.processedInputs.Load()) + require.Equal(uint64(6), instance.processedInputs.Load()) }) - testCompletion := func(name string, completion machine.CompletionStatus, status model.InputCompletionStatus) { + testCompletedStatus := func( + name string, + machineStatus machine.CompletionStatus, + inputStatus model.InputCompletionStatus, + exceptionData []byte, + ) { s.Run(name, func() { require := s.Require() - inner, fork, machineInst := s.setupAdvance() - fork.AdvanceStatusReturn = completion - if completion == machine.CompletionStatusException { - fork.AdvanceExceptionData = []byte("exception data") - } - fork.CloseError, inner.CloseError = inner.CloseError, fork.CloseError + inner, fork, instance := s.setupAdvance() + fork.CompletionStatusReturn = machineStatus + fork.ExceptionDataReturn = exceptionData + fork.CloseError = nil - res, err := machineInst.Advance(context.Background(), []byte{}, 0, 5, false) + res, err := instance.Advance(context.Background(), []byte{}, 0, 5, false) require.Nil(err) require.NotNil(res) - require.Equal(status, res.Status) + require.Same(inner, instance.runtime) + require.Equal(inputStatus, res.Status) + require.Equal(exceptionData, res.ExceptionData) require.Equal(expectedOutputs, res.Outputs) require.Equal(expectedReports1, res.Reports) require.Equal(newHash(1), res.OutputsHash) require.Equal(newHash(2), res.MachineHash) - require.Equal(uint64(6), machineInst.processedInputs.Load()) + require.Equal(uint64(6), instance.processedInputs.Load()) }) } - testCompletion("Exception", + testCompletedStatus("Exception", machine.CompletionStatusException, - model.InputCompletionStatus_Exception) + model.InputCompletionStatus_Exception, + []byte("guest exception")) - testCompletion("Halted", + testCompletedStatus("Halted", machine.CompletionStatusHalted, - model.InputCompletionStatus_MachineHalted) + model.InputCompletionStatus_MachineHalted, + nil) }) s.Run("Error", func() { + interruptions := []struct { + name string + err error + }{ + {"OutputsLimit", machine.ErrOutputsLimitExceeded}, + {"ReportsLimit", machine.ErrReportsLimitExceeded}, + {"ReachedCycleLimit", machine.ErrReachedLimitMcycle}, + {"Deadline", machine.ErrDeadlineExceeded}, + {"Canceled", machine.ErrCanceled}, + {"PayloadLengthLimit", machine.ErrPayloadLengthLimitExceeded}, + {"MachineInternal", machine.ErrMachineInternal}, + } + for _, interruption := range interruptions { + s.Run(interruption.name, func() { + require := s.Require() + inner, fork, instance := s.setupAdvance() + fork.AdvanceError = interruption.err + fork.CloseError = nil + + res, err := instance.Advance(context.Background(), []byte{}, 0, 5, false) + require.ErrorIs(err, interruption.err) + require.Nil(res) + require.Same(inner, instance.runtime) + require.Equal(uint64(5), instance.processedInputs.Load()) + }) + } + + s.Run("UnknownStatus", func() { + require := s.Require() + inner, fork, instance := s.setupAdvance() + fork.CompletionStatusReturn = machine.CompletionStatusUnknown + fork.CloseError = nil + + res, err := instance.Advance(context.Background(), []byte{}, 0, 5, false) + require.ErrorIs(err, ErrIncompleteAdvance) + require.Nil(res) + require.Same(inner, instance.runtime) + require.Equal(uint64(5), instance.processedInputs.Load()) + }) + + for _, test := range []struct { + name string + status machine.CompletionStatus + exceptionData []byte + }{ + {"ExceptionWithoutData", machine.CompletionStatusException, nil}, + {"AcceptedWithExceptionData", machine.CompletionStatusAccepted, []byte("unexpected")}, + } { + s.Run(test.name, func() { + require := s.Require() + inner, fork, instance := s.setupAdvance() + fork.CompletionStatusReturn = test.status + fork.ExceptionDataReturn = test.exceptionData + fork.CloseError = nil + + res, err := instance.Advance(context.Background(), []byte{}, 0, 5, false) + require.ErrorIs(err, ErrIncompleteAdvance) + require.ErrorIs(err, machine.ErrMachineInternal) + require.Nil(res) + require.Same(inner, instance.runtime) + require.Equal(uint64(5), instance.processedInputs.Load()) + }) + } + s.Run("Fork", func() { require := s.Require() inner, _, machine := s.setupAdvance() @@ -481,8 +544,8 @@ func (s *MachineInstanceSuite) TestAdvance() { s.Run("Fork", func() { require := s.Require() _, fork, machineInst := s.setupAdvance() - fork.AdvanceStatusReturn = machine.CompletionStatusException - fork.AdvanceExceptionData = []byte("exception data") + fork.CompletionStatusReturn = machine.CompletionStatusException + fork.ExceptionDataReturn = []byte("guest exception") fork.CloseError = errors.New("Close error") // Close error on fork is logged, not propagated. @@ -497,16 +560,18 @@ func (s *MachineInstanceSuite) TestAdvance() { s.Run("CollectHashes", func() { require := s.Require() - inner, fork, machineInst := s.setupAdvance() + inner, fork, instance := s.setupAdvance() - res, err := machineInst.Advance(context.Background(), []byte{}, 0, 5, true) + fork.AdvanceRemainingReturn = machine.InputEntryCapacity + + res, err := instance.Advance(context.Background(), []byte{}, 0, 5, true) require.Nil(err) require.NotNil(res) - require.Same(fork, machineInst.runtime) + require.Same(fork, instance.runtime) require.Equal(model.InputCompletionStatus_Accepted, res.Status) require.True(res.IsDaveConsensus) - require.Equal(uint64(6), machineInst.processedInputs.Load()) + require.Equal(uint64(6), instance.processedInputs.Load()) // Verify the inner runtime was closed (accept path) _ = inner @@ -517,7 +582,7 @@ func (s *MachineInstanceSuite) TestAdvance() { // same machine never happens by design. This test verifies that two // sequential advances correctly increment processedInputs. require := s.Require() - inner, fork, machineInst := s.setupAdvance() + inner, fork, instance := s.setupAdvance() // Allow inner.Close to succeed (old runtime close on accept) inner.CloseError = nil @@ -526,7 +591,7 @@ func (s *MachineInstanceSuite) TestAdvance() { // After accept, fork becomes the new runtime. // Second advance: fork from fork (processedInputs=6), fork must also fork. fork2 := &MockRollupsMachine{} - fork2.AdvanceStatusReturn = machine.CompletionStatusAccepted + fork2.CompletionStatusReturn = machine.CompletionStatusAccepted fork2.AdvanceOutputsReturn = expectedOutputs fork2.AdvanceReportsReturn = expectedReports1 fork2.OutputsHashReturn = newHash(1) @@ -537,134 +602,139 @@ func (s *MachineInstanceSuite) TestAdvance() { fork.CloseError = nil // close of fork (now old runtime) in second advance // First advance at index 5 - res1, err := machineInst.Advance(context.Background(), []byte{}, 0, 5, false) + res1, err := instance.Advance(context.Background(), []byte{}, 0, 5, false) require.Nil(err) require.NotNil(res1) - require.Equal(uint64(6), machineInst.processedInputs.Load()) + require.Equal(uint64(6), instance.processedInputs.Load()) // Second advance at index 6 - res2, err := machineInst.Advance(context.Background(), []byte{}, 0, 6, false) + res2, err := instance.Advance(context.Background(), []byte{}, 0, 6, false) require.Nil(err) require.NotNil(res2) - require.Equal(uint64(7), machineInst.processedInputs.Load()) + require.Equal(uint64(7), instance.processedInputs.Load()) }) } func (s *MachineInstanceSuite) TestInspect() { - s.Run("Ok", func() { - s.Run("Accept", func() { + for _, test := range []struct { + name string + status machine.CompletionStatus + accepted bool + resultErr error + }{ + {"Accept", machine.CompletionStatusAccepted, true, nil}, + {"Reject", machine.CompletionStatusRejected, false, nil}, + {"Exception", machine.CompletionStatusException, false, machine.ErrException}, + {"Halted", machine.CompletionStatusHalted, false, machine.ErrHalted}, + } { + s.Run(test.name, func() { require := s.Require() - _, fork, machine := s.setupInspect() - - res, err := machine.Inspect(context.Background(), []byte{}) - require.Nil(err) - require.NotNil(res) - - require.NotSame(fork, machine.runtime) - require.Equal(uint64(55), res.ProcessedInputs) - require.True(res.Accepted) - require.Equal(expectedReports2, res.Reports) - require.Nil(res.Error) + _, fork, instance := s.setupInspect() + fork.InspectResponseReturn.Status = test.status + + result, err := instance.Inspect(context.Background(), []byte{}) + require.NoError(err) + require.NotNil(result) + require.NotSame(fork, instance.runtime) + require.Equal(uint64(55), result.ProcessedInputs) + require.Equal(test.accepted, result.Accepted) + require.Equal(expectedReports2, result.Reports) + if test.resultErr == nil { + require.NoError(result.Error) + } else { + require.ErrorIs(result.Error, test.resultErr) + } }) + } - s.Run("Reject", func() { - require := s.Require() - _, fork, machineInst := s.setupInspect() - fork.InspectStatusReturn = machine.CompletionStatusRejected - - res, err := machineInst.Inspect(context.Background(), []byte{}) - require.Nil(err) - require.NotNil(res) + s.Run("AtCapacity", func() { + require := s.Require() + _, _, instance := s.setupInspect() + instance.inspectSemaphore.TryAcquire(int64(instance.maxConcurrentInspects)) + defer instance.inspectSemaphore.Release(int64(instance.maxConcurrentInspects)) - require.NotSame(fork, machineInst.runtime) - require.Equal(uint64(55), res.ProcessedInputs) - require.False(res.Accepted) - require.Equal(expectedReports2, res.Reports) - require.Nil(res.Error) - }) + result, err := instance.Inspect(context.Background(), []byte{}) + require.ErrorIs(err, ErrInspectAtCapacity) + require.Nil(result) }) - s.Run("Error", func() { - s.Run("AtCapacity", func() { - require := s.Require() - _, _, machine := s.setupInspect() - - // Pre-fill all semaphore slots to simulate a saturated app - machine.inspectSemaphore.TryAcquire(int64(machine.maxConcurrentInspects)) - - // TryAcquire is non-blocking: the error is returned immediately, - // no context deadline required. - res, err := machine.Inspect(context.Background(), []byte{}) - require.Error(err) - require.Nil(res) - require.ErrorIs(err, ErrInspectAtCapacity) + s.Run("ForkError", func() { + require := s.Require() + inner, _, instance := s.setupInspect() + errFork := errors.New("Fork error") + inner.ForkError = errFork - // Release the semaphore for cleanup - machine.inspectSemaphore.Release(int64(machine.maxConcurrentInspects)) - }) + result, err := instance.Inspect(context.Background(), []byte{}) + require.ErrorIs(err, errFork) + require.Nil(result) + }) - s.Run("Fork", func() { - require := s.Require() - inner, _, machine := s.setupInspect() - errFork := errors.New("Fork error") - inner.ForkError = errFork + s.Run("ExecutionErrorPreservesPartialReports", func() { + require := s.Require() + _, fork, instance := s.setupInspect() + errInspect := errors.New("Inspect error") + fork.InspectError = errInspect - res, err := machine.Inspect(context.Background(), []byte{}) - require.Error(err) - require.Nil(res) - require.Equal(errFork, err) - }) + result, err := instance.Inspect(context.Background(), []byte{}) + require.NoError(err) + require.False(result.Accepted) + require.Equal(expectedReports2, result.Reports) + require.ErrorIs(result.Error, errInspect) + }) - s.Run("Inspect", func() { + for _, test := range []struct { + name string + response *machine.InspectResponse + }{ + {"NilResponse", nil}, + {"UnknownStatus", &machine.InspectResponse{Status: machine.CompletionStatusUnknown}}, + {"InvalidStatus", &machine.InspectResponse{Status: machine.CompletionStatus(255)}}, + } { + s.Run(test.name+"FailsClosed", func() { require := s.Require() - _, fork, machine := s.setupInspect() - errInspect := errors.New("Inspect error") - fork.InspectError = errInspect - - res, err := machine.Inspect(context.Background(), []byte{}) - require.Nil(err) - require.NotNil(res) - require.Equal(errInspect, res.Error) + _, fork, instance := s.setupInspect() + fork.InspectResponseReturn = test.response + + result, err := instance.Inspect(context.Background(), []byte{}) + require.NoError(err) + require.False(result.Accepted) + require.ErrorIs(result.Error, ErrIncompleteInspect) + require.ErrorIs(result.Error, machine.ErrMachineInternal) }) + } - s.Run("Close", func() { - require := s.Require() - _, fork, machine := s.setupInspect() - errClose := errors.New("Close error") - fork.CloseError = errClose + s.Run("CloseErrorOverridesResult", func() { + require := s.Require() + _, fork, instance := s.setupInspect() + errClose := errors.New("Close error") + fork.CloseError = errClose - res, err := machine.Inspect(context.Background(), []byte{}) - require.Error(err) - require.Nil(res) - require.Equal(errClose, err) - }) + result, err := instance.Inspect(context.Background(), []byte{}) + require.ErrorIs(err, errClose) + require.Nil(result) }) s.Run("Concurrency", func() { require := s.Require() - _, _, machine := s.setupInspect() - - // Test that we can run maxConcurrentInspects inspects concurrently + _, _, instance := s.setupInspect() var wg sync.WaitGroup - errors := make(chan error, machine.maxConcurrentInspects) + errs := make(chan error, instance.maxConcurrentInspects) - for range int(machine.maxConcurrentInspects) { + for range int(instance.maxConcurrentInspects) { wg.Add(1) go func() { defer wg.Done() - _, err := machine.Inspect(context.Background(), []byte{}) + _, err := instance.Inspect(context.Background(), []byte{}) if err != nil { - errors <- err + errs <- err } }() } wg.Wait() - close(errors) - - // Check if any errors occurred - for err := range errors { - require.Nil(err, "Concurrent inspect failed: %v", err) + close(errs) + for err := range errs { + require.NoError(err) } }) } @@ -1020,7 +1090,7 @@ func (s *MachineInstanceSuite) setupAdvance() (*MockRollupsMachine, *MockRollups inner.ForkReturn = fork inner.CloseError = nil - fork.AdvanceStatusReturn = machine.CompletionStatusAccepted + fork.CompletionStatusReturn = machine.CompletionStatusAccepted fork.AdvanceOutputsReturn = []machine.Output{ newBytes(11, 100), newBytes(12, 100), @@ -1036,12 +1106,14 @@ func (s *MachineInstanceSuite) setupAdvance() (*MockRollupsMachine, *MockRollups fork.HashReturn = newHash(2) fork.HashError = nil - fork.InspectStatusReturn = machine.CompletionStatusAccepted - fork.InspectReportsReturn = []machine.Report{ - newBytes(31, 300), - newBytes(32, 300), - newBytes(33, 300), - newBytes(34, 300), + fork.InspectResponseReturn = &machine.InspectResponse{ + Status: machine.CompletionStatusAccepted, + Reports: []machine.Report{ + newBytes(31, 300), + newBytes(32, 300), + newBytes(33, 300), + newBytes(34, 300), + }, } fork.InspectError = errUnreachable @@ -1080,12 +1152,14 @@ func (s *MachineInstanceSuite) setupInspect() (*MockRollupsMachine, *MockRollups fork.AdvanceError = errUnreachable fork.HashError = errUnreachable - fork.InspectStatusReturn = machine.CompletionStatusAccepted - fork.InspectReportsReturn = []machine.Report{ - newBytes(31, 300), - newBytes(32, 300), - newBytes(33, 300), - newBytes(34, 300), + fork.InspectResponseReturn = &machine.InspectResponse{ + Status: machine.CompletionStatusAccepted, + Reports: []machine.Report{ + newBytes(31, 300), + newBytes(32, 300), + newBytes(33, 300), + newBytes(34, 300), + }, } fork.InspectError = nil @@ -1220,7 +1294,7 @@ func (r *mockSyncRepository) GetLastSnapshot( func newForkableMock() *MockRollupsMachine { m := &MockRollupsMachine{} m.CloseError = nil - m.AdvanceStatusReturn = machine.CompletionStatusAccepted + m.CompletionStatusReturn = machine.CompletionStatusAccepted m.HashReturn = newHash(1) m.OutputsHashReturn = newHash(2) m.ForkFunc = func(_ context.Context) (machine.Machine, error) { @@ -1450,21 +1524,21 @@ type MockRollupsMachine struct { HashReturn machine.Hash HashError error - AdvanceStatusReturn machine.CompletionStatus - AdvanceExceptionData []byte - AdvanceOutputsReturn []machine.Output - AdvanceReportsReturn []machine.Report - AdvanceLeafsReturn []machine.Hash - AdvanceRemainingReturn uint64 - OutputsHashReturn machine.Hash - OutputsHashError error - OutputsHashProofReturn []machine.Hash - OutputsHashProofError error - AdvanceError error - - InspectStatusReturn machine.CompletionStatus - InspectReportsReturn []machine.Report - InspectError error + CompletionStatusReturn machine.CompletionStatus + ExceptionDataReturn []byte + AdvanceOutputsReturn []machine.Output + AdvanceReportsReturn []machine.Report + AdvanceLeafsReturn []machine.Hash + AdvanceRemainingReturn uint64 + OutputsHashReturn machine.Hash + OutputsHashError error + OutputsHashProofReturn []machine.Hash + OutputsHashProofError error + AdvanceError error + LastAdvanceComputeHashes bool + + InspectResponseReturn *machine.InspectResponse + InspectError error StoreError error @@ -1495,21 +1569,18 @@ func (m *MockRollupsMachine) Advance(_ context.Context, _ []byte, _ machine.Hash return nil, m.AdvanceError } return &machine.AdvanceResponse{ - Status: m.AdvanceStatusReturn, - Outputs: m.AdvanceOutputsReturn, - Reports: m.AdvanceReportsReturn, - ExceptionData: m.AdvanceExceptionData, - Hashes: m.AdvanceLeafsReturn, - RemainingCycles: m.AdvanceRemainingReturn, - OutputsHash: m.OutputsHashReturn, + Status: m.CompletionStatusReturn, + ExceptionData: m.ExceptionDataReturn, + Outputs: m.AdvanceOutputsReturn, + Reports: m.AdvanceReportsReturn, + PeriodicStateHashes: m.AdvanceLeafsReturn, + PaddingRepetitions: m.AdvanceRemainingReturn, + OutputsHash: m.OutputsHashReturn, }, nil } func (m *MockRollupsMachine) Inspect(_ context.Context, _ []byte) (*machine.InspectResponse, error) { - if m.InspectError != nil { - return nil, m.InspectError - } - return &machine.InspectResponse{Status: m.InspectStatusReturn, Reports: m.InspectReportsReturn}, nil + return m.InspectResponseReturn, m.InspectError } func (m *MockRollupsMachine) Store(_ context.Context, _ string) error { diff --git a/internal/model/input_hash_collection.go b/internal/model/input_hash_collection.go new file mode 100644 index 000000000..d35172691 --- /dev/null +++ b/internal/model/input_hash_collection.go @@ -0,0 +1,50 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package model + +import "fmt" + +const ( + // Log2MaxAdvanceStatesPerEpoch is the number of input-index bits reserved + // by the protocol computation-hash tree for one epoch. + Log2MaxAdvanceStatesPerEpoch uint64 = 24 + + // MaxAdvanceStatesPerEpoch is the number of input slots reserved by the + // protocol computation-hash tree for one epoch. + MaxAdvanceStatesPerEpoch uint64 = 1 << Log2MaxAdvanceStatesPerEpoch + + // Log2InputHashCollectionCapacity is the number of state-hash index bits + // reserved for one input in the protocol computation-hash tree. + Log2InputHashCollectionCapacity uint64 = 24 + + // InputHashCollectionCapacity is the number of computation-hash leaves + // reserved for one input. + InputHashCollectionCapacity uint64 = 1 << Log2InputHashCollectionCapacity + + // Log2EpochComputationHashLeafCount is the height of the epoch computation- + // hash tree: 2^24 input slots, each containing 2^24 state-hash entries. + Log2EpochComputationHashLeafCount uint64 = Log2MaxAdvanceStatesPerEpoch + Log2InputHashCollectionCapacity +) + +// ValidateInputHashCollectionSpan verifies that periodic state hashes and the +// final-state padding cover exactly one persisted input hash collection. +func ValidateInputHashCollectionSpan(hashCount, paddingRepetitions uint64) error { + if hashCount > InputHashCollectionCapacity { + return fmt.Errorf( + "collected state hash count %d exceeds input hash collection capacity %d", + hashCount, + InputHashCollectionCapacity, + ) + } + expectedPaddingRepetitions := InputHashCollectionCapacity - hashCount + if paddingRepetitions != expectedPaddingRepetitions { + return fmt.Errorf( + "collected state hash count %d with padding repetitions %d does not cover input hash collection capacity %d", + hashCount, + paddingRepetitions, + InputHashCollectionCapacity, + ) + } + return nil +} diff --git a/internal/model/input_hash_collection_test.go b/internal/model/input_hash_collection_test.go new file mode 100644 index 000000000..e727b9ab9 --- /dev/null +++ b/internal/model/input_hash_collection_test.go @@ -0,0 +1,46 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package model + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestInputHashCollectionDimensions(t *testing.T) { + t.Parallel() + require.Equal(t, uint64(24), Log2MaxAdvanceStatesPerEpoch) + require.Equal(t, uint64(1)<<24, MaxAdvanceStatesPerEpoch) + require.Equal(t, uint64(24), Log2InputHashCollectionCapacity) + require.Equal(t, uint64(1)<<24, InputHashCollectionCapacity) + require.Equal(t, uint64(48), Log2EpochComputationHashLeafCount) +} + +func TestValidateInputHashCollectionSpan(t *testing.T) { + t.Parallel() + tests := []struct { + name string + hashes uint64 + padding uint64 + wantErr bool + }{ + {name: "empty", padding: InputHashCollectionCapacity}, + {name: "partial", hashes: 2, padding: InputHashCollectionCapacity - 2}, + {name: "full", hashes: InputHashCollectionCapacity}, + {name: "short", hashes: 2, padding: InputHashCollectionCapacity - 3, wantErr: true}, + {name: "long", hashes: 2, padding: InputHashCollectionCapacity - 1, wantErr: true}, + {name: "overflow", hashes: InputHashCollectionCapacity + 1, wantErr: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := ValidateInputHashCollectionSpan(test.hashes, test.padding) + if test.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/internal/model/models.go b/internal/model/models.go index 2cab6409a..b80d01c31 100644 --- a/internal/model/models.go +++ b/internal/model/models.go @@ -690,8 +690,8 @@ func (e *ExecutionParameters) UnmarshalJSON(data []byte) error { return nil } -// Log2MaxExecutionCycles is the node-side definition of the protocol -// execution window size. +// Log2MaxExecutionCycles is the single node-side definition of the protocol +// execution window size. pkg/machine aliases it as Log2MaxMCyclesPerAdvanceState. const Log2MaxExecutionCycles uint64 = 48 // MaxExecutionCycles is the number of mcycles in one machine-enforced window. @@ -1351,8 +1351,8 @@ type AdvanceResult struct { Outputs [][]byte Reports [][]byte ExceptionData []byte - Hashes [][32]byte - RemainingMetaCycles uint64 + PeriodicStateHashes [][32]byte + PaddingRepetitions uint64 IsDaveConsensus bool } diff --git a/internal/repository/postgres/bulk.go b/internal/repository/postgres/bulk.go index e3596d7bd..fc0d33a57 100644 --- a/internal/repository/postgres/bulk.go +++ b/internal/repository/postgres/bulk.go @@ -373,7 +373,7 @@ func (r *PostgresRepository) StoreAdvanceResult( } if res.IsDaveConsensus { - err = insertStateHashes(ctx, tx, appID, res.EpochIndex, res.InputIndex, res.Hashes, res.MachineHash, res.RemainingMetaCycles) + err = insertStateHashes(ctx, tx, appID, res.EpochIndex, res.InputIndex, res.PeriodicStateHashes, res.MachineHash, res.PaddingRepetitions) if err != nil { return err } diff --git a/internal/repository/repotest/bulk_test_cases.go b/internal/repository/repotest/bulk_test_cases.go index 05e57afdf..9ee5c374f 100644 --- a/internal/repository/repotest/bulk_test_cases.go +++ b/internal/repository/repotest/bulk_test_cases.go @@ -174,8 +174,8 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { InputIndex: 0, Status: InputCompletionStatus_Accepted, Outputs: [][]byte{[]byte("dave-output")}, - Hashes: [][32]byte{hash1, hash2, hash3}, - RemainingMetaCycles: 42, + PeriodicStateHashes: [][32]byte{hash1, hash2, hash3}, + PaddingRepetitions: 42, IsDaveConsensus: true, OutputsProof: OutputsProof{ OutputsHash: outputsHash, @@ -204,7 +204,7 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { s.Equal(common.Hash(hash3), stateHashes[2].MachineHash) s.Equal(uint64(1), stateHashes[2].Repetitions) - // Verify final hash has RemainingMetaCycles as Repetitions + // Verify final hash has PaddingRepetitions as Repetitions s.Equal(machineHash, stateHashes[3].MachineHash) s.Equal(uint64(42), stateHashes[3].Repetitions) @@ -322,8 +322,8 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResultRollback() { InputIndex: 0, Status: InputCompletionStatus_Accepted, Outputs: [][]byte{[]byte("should-be-rolled-back")}, - Hashes: [][32]byte{{1}, {2}}, - RemainingMetaCycles: 10, + PeriodicStateHashes: [][32]byte{{1}, {2}}, + PaddingRepetitions: 10, IsDaveConsensus: true, OutputsProof: OutputsProof{ OutputsHash: UniqueHash(), diff --git a/internal/repository/repotest/state_hash_test_cases.go b/internal/repository/repotest/state_hash_test_cases.go index 1f9df2b1c..c13abfee2 100644 --- a/internal/repository/repotest/state_hash_test_cases.go +++ b/internal/repository/repotest/state_hash_test_cases.go @@ -54,8 +54,8 @@ func (s *StateHashSuite) TestListStateHashes() { EpochIndex: 0, InputIndex: 0, Status: InputCompletionStatus_Accepted, - Hashes: [][32]byte{hash1, hash2}, - RemainingMetaCycles: 10, + PeriodicStateHashes: [][32]byte{hash1, hash2}, + PaddingRepetitions: 10, IsDaveConsensus: true, OutputsProof: OutputsProof{ OutputsHash: outputsHash, diff --git a/internal/validator/validator.go b/internal/validator/validator.go index 0578d9fff..2bd74a94e 100644 --- a/internal/validator/validator.go +++ b/internal/validator/validator.go @@ -351,10 +351,10 @@ func (s *Service) buildCommitment(ctx context.Context, app *Application, epoch * } builder := merkle.Builder{} inputCount := epoch.InputIndexUpperBound - epoch.InputIndexLowerBound - if inputCount > pkgm.InputsPerEpoch { + if inputCount > pkgm.MaxAdvanceStatesPerEpoch { return nil, nil, s.setApplicationCorrupted(ctx, app, "input count is too large for epoch %v of application %v: max %v, got %v", - epoch.Index, app.Name, pkgm.InputsPerEpoch, inputCount) + epoch.Index, app.Name, pkgm.MaxAdvanceStatesPerEpoch, inputCount) } if inputCount > 0 { @@ -382,9 +382,9 @@ func (s *Service) buildCommitment(ctx context.Context, app *Application, epoch * } } - remainingInputs := pkgm.InputsPerEpoch - inputCount - // Safe: inputCount ≤ InputsPerEpoch enforced above, so remainingInputs << Log2StridesPerInput won't overflow. - remainingStrides := remainingInputs << pkgm.Log2StridesPerInput + remainingInputs := pkgm.MaxAdvanceStatesPerEpoch - inputCount + // Safe: inputCount ≤ MaxAdvanceStatesPerEpoch enforced above, so remainingInputs << Log2InputEntryCapacity won't overflow. + remainingStrides := remainingInputs << pkgm.Log2InputEntryCapacity if remainingStrides > 0 { if err := builder.AppendRepeatedUint64(merkle.TreeLeaf(*epoch.MachineHash), remainingStrides); err != nil { return nil, nil, s.setApplicationCorrupted(ctx, app, @@ -398,7 +398,7 @@ func (s *Service) buildCommitment(ctx context.Context, app *Application, epoch * "failed to build commitment for epoch %d of application %s with error: %v", epoch.Index, app.Name, err) } // The commitment geometry is fixed: 2²⁴ inputs × 2²⁴ strides ⇒ height 48. - const expectedHeight = pkgm.Log2InputSpanToEpoch + pkgm.Log2StridesPerInput // 48 + const expectedHeight = pkgm.Log2MaxAdvanceStatesPerEpoch + pkgm.Log2InputEntryCapacity // 48 if uint64(epochCommitmentTree.Height) != expectedHeight { return nil, nil, s.setApplicationCorrupted(ctx, app, "epoch %v commitment tree height %v, expected %v — state hash repetitions are inconsistent", diff --git a/internal/validator/validator_test.go b/internal/validator/validator_test.go index d866b584a..4d6428029 100644 --- a/internal/validator/validator_test.go +++ b/internal/validator/validator_test.go @@ -822,9 +822,9 @@ func (s *ValidatorSuite) TestBuildCommitment() { } // 5 inputs, each with one state hash covering the full - // strides-per-input count (1< InputEntryCapacity { + return 0, fmt.Errorf( + "collected state hash count %d exceeds input entry capacity %d", + hashCount, + InputEntryCapacity, + ) + } + return InputEntryCapacity - hashCount, nil +} diff --git a/pkg/machine/computation_hash_test.go b/pkg/machine/computation_hash_test.go new file mode 100644 index 000000000..d8f99efc9 --- /dev/null +++ b/pkg/machine/computation_hash_test.go @@ -0,0 +1,45 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package machine + +import ( + "testing" + + "github.com/cartesi/rollups-node/internal/model" + "github.com/stretchr/testify/require" +) + +func TestComputationHashDimensions(t *testing.T) { + require.Equal(t, uint64(20), Log2MaxUarchCyclesPerMCycle) + require.Equal(t, uint64(48), Log2MaxMCyclesPerAdvanceState) + require.Equal(t, uint64(24), Log2MaxAdvanceStatesPerEpoch) + require.Equal(t, uint64(44), Log2UarchCycleComputationHashPeriod) + require.Equal(t, uint64(68), Log2UarchCyclesPerAdvanceState) + require.Equal(t, uint64(1)<<20-1, MaxUarchCycle) + require.Equal(t, uint64(1)<<48-1, MaxMCycleDeltaPerAdvanceState) + require.Equal(t, uint64(1)<<24-1, MaxAdvanceStateIndexPerEpoch) + + require.Equal(t, uint64(24), Log2MCycleComputationHashPeriod) + require.Equal(t, uint64(1)<<24, MCycleComputationHashPeriod) + require.Equal(t, uint64(24), Log2InputEntryCapacity) + require.Equal(t, uint64(1)<<24, InputEntryCapacity) + require.Equal(t, uint64(48), Log2EpochComputationHashLeafCount) + require.Equal(t, uint64(1)<<48, EpochComputationHashLeafCount) + require.Equal(t, uint64(1)<<24, MaxAdvanceStatesPerEpoch) + require.Equal( + t, + model.Log2InputHashCollectionCapacity, + Log2MaxMCyclesPerAdvanceState-Log2MCycleComputationHashPeriod, + "the execution window and sampling period must derive the protocol input hash collection capacity", + ) +} + +func TestComputationHashCollectionChunkMatchesEmulatorCLI(t *testing.T) { + require.Equal( + t, + MCycleComputationHashPeriod<<8, + mcycleComputationHashChunkSize, + "cartesi-machine 0.21 targets 2^8 returned hashes per collection call", + ) +} diff --git a/pkg/machine/doc.go b/pkg/machine/doc.go new file mode 100644 index 000000000..cf70cfba7 --- /dev/null +++ b/pkg/machine/doc.go @@ -0,0 +1,38 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +// Package machine exposes the rollups-oriented operations of a Cartesi +// machine and translates emulator stops into a small Go contract. +// +// A request that reaches a deterministic guest completion returns a response +// value with CompletionStatusAccepted, CompletionStatusRejected, +// CompletionStatusException, or CompletionStatusHalted. Anything that prevents +// completion—including deadlines, local resource limits, backend failures, and +// cycle exhaustion—returns an error. Advance then returns no response; Inspect +// may return partial reports with CompletionStatusUnknown. In short: terminal +// guest outcomes travel as values; incomplete execution travels as an error. +// Callers decide how a completed outcome affects canonical application state. +// +// For a PRT advance, AdvanceResponse contains a canonical compressed input hash +// collection. PeriodicStateHashes contains the sampled machine root hashes +// before the final canonical root. PaddingRepetitions says how many times that +// final root completes the collection, so +// +// len(PeriodicStateHashes) + PaddingRepetitions == InputEntryCapacity +// +// and PaddingRepetitions is positive. The machine implementation is the sole +// canonicalizer of this representation; downstream layers validate it but do +// not repair it. +// +// Related terms describe different levels of the same construction. A root +// hash identifies one machine state. Periodic state hashes are sampled roots. +// An input hash collection is those samples plus the repeated final root. The +// DAVE computation hash commits to the larger sequence assembled from these +// collections. Persistence stores the compressed collection as state-hash rows. +// +// A configured cycle span of zero means that no operator cap shortens the +// emulator's fixed window. The resolved endpoint mirrors emulator 0.21's +// saturating imcyclemax arithmetic. ErrMcycleOverflow preserves the distinct +// emulator-reported fact that imcyclemax, rather than a node target, stopped the +// machine. +package machine diff --git a/pkg/machine/implementation.go b/pkg/machine/implementation.go index 65df9b70a..bceae6a61 100644 --- a/pkg/machine/implementation.go +++ b/pkg/machine/implementation.go @@ -82,39 +82,15 @@ const ( ManualYieldReasonException manualYieldReason = 0x4 ) -// Limits for outputs and reports per input +// maxOutputs and maxReports are local operational resource ceilings. They +// bound host memory consumption; exceeding them is an incomplete execution +// failure, never a protocol-level input completion status. const maxOutputs = 65536 // 2^16 const maxReports = 65536 // 2^16 const TxBufferAddress uint64 = 0x60800000 const HashLog2Size = 5 // 32 bytes -const ( - // log2 value of the maximal number of micro instructions that emulates a big instruction - Log2UarchSpanToBarch uint64 = 20 - // log2 value of the maximal number of big instructions that executes an input - Log2BarchSpanToInput uint64 = 48 - // log2 value of the maximal number of inputs that allowed in an epoch - Log2InputSpanToEpoch uint64 = 24 - // gap of each leaf in the commitment tree, should use the same value as ArbitrationConstants.sol:log2step(0) - Log2Stride uint64 = 44 - // log2 value of the maximal number of micro instructions that executes an input - Log2UarchSpanToInput uint64 = Log2BarchSpanToInput + Log2UarchSpanToBarch // 68 - - UarchSpanToBarch uint64 = (1 << Log2UarchSpanToBarch) - 1 // 1_048_575 - BarchSpanToInput uint64 = (1 << Log2BarchSpanToInput) - 1 // 281_474_976_710_655 - InputSpanToEpoch uint64 = (1 << Log2InputSpanToEpoch) - 1 // 16_777_215 - - BigStepsInStride uint64 = 1 << (Log2Stride - Log2UarchSpanToBarch) // 16_777_216 - StrideCountInInput uint64 = 1 << (Log2BarchSpanToInput + Log2UarchSpanToBarch - Log2Stride) // 16_777_216 - - StrideCountInEpoch uint64 = 1 << (Log2InputSpanToEpoch + Log2BarchSpanToInput + Log2UarchSpanToBarch - Log2Stride) - - Log2StridesPerInput uint64 = Log2BarchSpanToInput + Log2UarchSpanToBarch - Log2Stride - - InputsPerEpoch uint64 = 1 << Log2InputSpanToEpoch -) - // machineImpl implements the Machine interface by wrapping an emulator.RemoteMachine type machineImpl struct { backend Backend @@ -227,11 +203,11 @@ func (m *machineImpl) Advance(ctx context.Context, input []byte, checkpointHash } resp := &AdvanceResponse{ - Status: result.completion.status, - Outputs: result.outputs, - Reports: result.reports, - Hashes: result.periodicStateHashes, - RemainingCycles: result.paddingRepetitions, + Status: result.completion.status, + Outputs: result.outputs, + Reports: result.reports, + PeriodicStateHashes: result.periodicStateHashes, + PaddingRepetitions: result.paddingRepetitions, } if resp.Status == CompletionStatusAccepted { @@ -404,6 +380,11 @@ func (m *machineImpl) process( if err := checkContext(ctx); err != nil { return processResult{}, err } + // An L1 advance arrives as the full EvmAdvance encoding, which the canonical + // InputBox bounds to CanonicalMachine.INPUT_MAX_SIZE = 2^16 bytes (it reverts + // InputTooLarge above that), while the standard CMIO RX buffer holds 2^21 + // bytes. Advance overflow is therefore defensive on the supported path; + // inspect and custom input providers still need the runtime check. if length, capacity := uint64(len(request)), m.backend.CmioRxBufferSize(); length > capacity { return processResult{}, fmt.Errorf( "%s request payload length %d exceeds CMIO receive buffer capacity %d: %w", @@ -411,10 +392,15 @@ func (m *machineImpl) process( ) } - // Validate execution parameters before SendCmioResponse mutates the machine. + // Validate the request-specific increment before consulting or changing the + // machine. A zero increment would otherwise make the run loop spin forever. if err := m.validateExecutionCycleIncrement(reqType); err != nil { return processResult{}, err } + // Validate the execution target before sending the request. Sending a CMIO + // response mutates the candidate, so invalid configuration must fail before + // that point. A valid target that exceeds uint64 saturates at MaxUint64, + // matching the emulator's imcyclemax calculation. bounds, err := m.executionCycleBounds(ctx, reqType) if err != nil { return processResult{}, err @@ -442,8 +428,9 @@ func (m *machineImpl) process( } } -// run executes a request between explicit cycle bounds and returns any -// responses collected before it reaches a fixed point. +// run executes a request whose increment and cycle bounds were validated before +// SendCmioResponse mutated the machine. It returns collected responses and the +// canonical compressed input hash collection. func (m *machineImpl) run( ctx context.Context, reqType requestType, @@ -460,6 +447,9 @@ func (m *machineImpl) run( stepTimeout = m.params.InspectIncDeadline runTimeout = m.params.InspectMaxDeadline } + if computeHashes { + increment = min(increment, mcycleComputationHashChunkSize) + } m.logger.Debug("run", "startingCycle", currentCycle, @@ -467,71 +457,94 @@ func (m *machineImpl) run( "leftover", bounds.limit-currentCycle) result := runResult{ - outputs: make([]Output, 0, 16), //nolint:mnd - reports: make([]Report, 0, 16), //nolint:mnd + outputs: make([]Output, 0, 16), + reports: make([]Report, 0, 16), } var hashCollectorState *HashCollectorState if computeHashes { hashCollectorState = &HashCollectorState{ - Period: BigStepsInStride, - Phase: 0, - MaxHashes: 0, - BundleLog2: 0, - Hashes: []Hash{}, + MCycleSamplingPeriod: MCycleComputationHashPeriod, + MCyclePhase: 0, + Log2BundleMCycleCount: 0, + Hashes: []Hash{}, } } - finish := func(runErr error) (runResult, error) { - if hashCollectorState != nil { - result.periodicStateHashes = hashCollectorState.Hashes - result.paddingRepetitions = StrideCountInInput - uint64(len(hashCollectorState.Hashes)) - } + finish := func(runErr error, terminalHashAppended bool) (runResult, error) { + return finalizeRunResult(result, hashCollectorState, terminalHashAppended, runErr) + } + // Incomplete executions may expose partial outputs and reports, but their PRT + // collections are discarded rather than normalized into persistable evidence. + fail := func(runErr error) (runResult, error) { return result, runErr } for { if err := checkContext(ctx); err != nil { - return finish(err) + return fail(err) } if time.Since(startTime) > runTimeout { - return finish(fmt.Errorf("run operation timed out: %w", ErrDeadlineExceeded)) + return fail(fmt.Errorf("run operation timed out: %w", ErrDeadlineExceeded)) } - interval, err := m.runIncrementInterval( - ctx, currentCycle, bounds.limit, increment, hashCollectorState, stepTimeout, + hashCountBeforeRun := 0 + if hashCollectorState != nil { + hashCountBeforeRun = len(hashCollectorState.Hashes) + } + incrementResult, err := m.runIncrementInterval( + ctx, + currentCycle, + bounds.limit, + increment, + hashCollectorState, + stepTimeout, ) - currentCycle = interval.currentCycle + currentCycle = incrementResult.currentCycle if err != nil { if errors.Is(err, ErrReachedLimitMcycle) { - return finish(executionLimitError(reqType, bounds, currentCycle, err)) + return fail(executionLimitError(reqType, bounds, currentCycle, err)) } - return finish(err) + return fail(err) } - switch interval.breakReason { + terminalHashAppended := computeHashes && isFixedPointBreakReason(incrementResult.breakReason) + if terminalHashAppended && len(hashCollectorState.Hashes) <= hashCountBeforeRun { + return fail(fmt.Errorf( + "machine stopped at fixed point %d without appending its state hash: %w", + incrementResult.breakReason, + ErrMachineInternal, + )) + } + + switch incrementResult.breakReason { case YieldedManually: - return finish(nil) + return finish(nil, terminalHashAppended) case Halted: - return finish(ErrHalted) + return finish(ErrHalted, terminalHashAppended) case McycleOverflow: - return finish(executionLimitError(reqType, bounds, currentCycle, ErrMcycleOverflow)) + return fail(executionLimitError(reqType, bounds, currentCycle, ErrMcycleOverflow)) case ReachedTargetMcycle, YieldedSoftly: continue case Failed: - return finish(ErrMachineInternal) + return fail(ErrMachineInternal) case YieldedAutomatically: + // Service the CMIO request below, then continue execution. default: - return finish(fmt.Errorf("invalid break reason: %d: %w", interval.breakReason, ErrMachineInternal)) + return fail(fmt.Errorf( + "invalid break reason: %d: %w", + incrementResult.breakReason, + ErrMachineInternal, + )) } if err := checkContext(ctx); err != nil { - return finish(err) + return fail(err) } _, yieldReason, data, err := m.backend.ReceiveCmioRequest(m.params.FastDeadline) if err != nil { werr := fmt.Errorf("could not read output/report: %w", err) - return finish(werr) + return fail(werr) } switch automaticYieldReason(yieldReason) { @@ -539,25 +552,72 @@ func (m *machineImpl) run( m.logger.Debug("ignoring yield reason progress", "value", fmt.Sprintf("%v", data)) case AutomaticYieldReasonOutput: if len(result.outputs) == maxOutputs { - return finish(executionResponseLimitError( + return fail(executionResponseLimitError( reqType, "output", len(result.outputs)+1, maxOutputs, ErrOutputsLimitExceeded, )) } result.outputs = append(result.outputs, data) case AutomaticYieldReasonReport: if len(result.reports) == maxReports { - return finish(executionResponseLimitError( + return fail(executionResponseLimitError( reqType, "report", len(result.reports)+1, maxReports, ErrReportsLimitExceeded, )) } result.reports = append(result.reports, data) default: err := fmt.Errorf("invalid automatic yield reason: %d: %w", yieldReason, ErrMachineInternal) - return finish(err) + return fail(err) } } } +// finalizeRunResult converts the emulator collector's fixed-point-inclusive +// result into the node's compressed representation. Keeping this at the return +// boundary leaves the execution loop concerned only with execution policy. +func finalizeRunResult( + result runResult, + collector *HashCollectorState, + terminalHashAppended bool, + runErr error, +) (runResult, error) { + if collector == nil { + return result, runErr + } + + collected := collector.Hashes + canonicalCount, paddingRepetitions, spanErr := canonicalInputHashCollectionShape( + uint64(len(collected)), + terminalHashAppended, + ) + if spanErr != nil { + return result, invalidInputHashCollectionSpanError(runErr, spanErr) + } + + // Emulator 0.21 includes a fixed-point entry and pads the input's remaining + // entry capacity with it. The node stores that same sequence in compressed + // form: periodic entries are kept here, while the manager supplies the + // completed status's canonical final root for persistence to append with + // paddingRepetitions in place of the removed collector entry. + result.periodicStateHashes = collected[:canonicalCount] + result.paddingRepetitions = paddingRepetitions + return result, runErr +} + +// invalidInputHashCollectionSpanError makes collection integrity authoritative +// over a terminal execution status. The status remains useful context, but is +// deliberately not wrapped: callers must not persist a malformed collection as +// a completed exception or halt. +func invalidInputHashCollectionSpanError(runErr, spanErr error) error { + err := errors.Join( + ErrMachineInternal, + fmt.Errorf("invalid collected state hash span: %w", spanErr), + ) + if runErr != nil { + err = fmt.Errorf("execution ended with %v and invalid hash span: %w", runErr, err) + } + return err +} + func executionResponseLimitError( reqType requestType, responseKind string, @@ -571,9 +631,34 @@ func executionResponseLimitError( ) } +// canonicalInputHashCollectionShape validates the collector's raw count before +// optionally excluding a terminal fixed-point sample. Validating first prevents +// an overcollection of InputEntryCapacity+1 from being normalized into an +// apparently valid collection. +func canonicalInputHashCollectionShape( + hashCount uint64, + terminalHashAppended bool, +) (canonicalHashCount uint64, paddingRepetitions uint64, err error) { + if _, err := inputHashCollectionPaddingRepetitions(hashCount); err != nil { + return 0, 0, err + } + if terminalHashAppended { + if hashCount == 0 { + return 0, 0, errors.New("terminal fixed point was reported without a collected state hash") + } + hashCount-- + } + paddingRepetitions, err = inputHashCollectionPaddingRepetitions(hashCount) + if err != nil { + return 0, 0, err + } + return hashCount, paddingRepetitions, nil +} + // executionCycleBounds applies the request-specific configured cycle span. -// Zero uses the machine's complete fixed window; non-representable endpoints -// saturate at MaxUint64 to mirror emulator mcycle arithmetic. +// Zero uses the machine's complete 2^48-cycle window. Every valid target uses +// the machine's saturating mcycle arithmetic: when start+span is not +// representable, MaxUint64 is both the local target and imcyclemax. func (m *machineImpl) executionCycleBounds( ctx context.Context, reqType requestType, @@ -590,9 +675,15 @@ func (m *machineImpl) executionCycleBounds( if err != nil { return executionBounds{}, err } + configured := executionCycleSpan != 0 if executionCycleSpan == 0 { - executionCycleSpan = BarchSpanToInput + // Emulator 0.21 arms imcyclemax = mcycle + 2^48 - 1 (saturating) at + // input delivery and classifies reaching it as mcycle overflow with + // priority over yield and halt. Mirror that exact window - endpoint + // and saturation - so the node's target can never disagree with the + // machine's own enforcement. + executionCycleSpan = model.MaxExecutionCycleSpan } limit := ^uint64(0) if currentCycle <= ^uint64(0)-executionCycleSpan { @@ -603,6 +694,9 @@ func (m *machineImpl) executionCycleBounds( }, nil } +// validateExecutionCycleIncrement is a runtime defense for values loaded from +// persistent storage or supplied through a custom MachineConfig. process calls +// it before SendCmioResponse changes the candidate machine. func (m *machineImpl) validateExecutionCycleIncrement(reqType requestType) error { if m.executionCycleIncrement(reqType) == 0 { return fmt.Errorf("%s execution increment must be greater than zero: %w", reqType, ErrMachineInternal) @@ -624,6 +718,8 @@ func (m *machineImpl) executionCycleIncrement(reqType requestType) uint64 { return m.params.AdvanceIncCycles } +// executionLimitError formats the resolved request window while preserving the +// emulator's typed imcyclemax origin when it—not a node target—stopped the run. func executionLimitError( reqType requestType, bounds executionBounds, @@ -644,6 +740,10 @@ func executionLimitError( bounds.start, bounds.span, targetSpan, executedCycles, ) + // A machine overflow may stop execution before its local target. This most + // commonly happens because an inspect does not re-arm imcyclemax and inherits + // the ceiling from the preceding advance. Preserve the machine-origin + // sentinel, but do not claim that a local span was exhausted. if current < bounds.limit { return fmt.Errorf( "%s execution stopped before %s cycle limit: %s: %w", @@ -651,6 +751,9 @@ func executionLimitError( ) } if errors.Is(origin, ErrMcycleOverflow) && bounds.configured { + // imcyclemax has precedence over reaching the requested run target. Even + // when both cycles are equal, the machine-enforced ceiling—not the local + // configured cap—stopped execution, so raising that cap cannot help. return fmt.Errorf( "%s execution stopped at machine imcyclemax coincident with configured target: %s: %w", reqType, progress, origin, @@ -666,8 +769,9 @@ func executionLimitError( ) } -// runIncrementInterval runs the machine for at most incrementLimit mcycles and -// preserves the emulator's typed break reason for the request loop. +// runIncrementInterval runs the machine for at most incrementLimit mcycles (or +// the distance left to limitCycle) and preserves the emulator's typed break +// reason for the request loop to classify. func (m *machineImpl) runIncrementInterval(ctx context.Context, currentCycle Cycle, limitCycle Cycle, @@ -677,11 +781,18 @@ func (m *machineImpl) runIncrementInterval(ctx context.Context, ) (incrementResult, error) { startingCycle := currentCycle + // Return without calling the backend after an ordinary local target has + // already been reached. MaxUint64 is different: saturating arithmetic can + // make both the node target and the emulator's imcyclemax equal the current + // cycle. Calling the emulator once at that fixed point preserves its + // authoritative McycleOverflow reason instead of mislabeling the stop as an + // operator-cap failure. atSaturatedEndpoint := currentCycle == ^uint64(0) && limitCycle == ^uint64(0) if currentCycle >= limitCycle && !atSaturatedEndpoint { return incrementResult{currentCycle: currentCycle}, ErrReachedLimitMcycle } + // Calculates the increment. increment := min(incrementLimit, limitCycle-currentCycle) m.logger.Debug("machine step before run", "currentCycle", currentCycle, "increment", increment) @@ -691,6 +802,13 @@ func (m *machineImpl) runIncrementInterval(ctx context.Context, if err != nil { return incrementResult{currentCycle: currentCycle}, err } + if hashCollectorState != nil && hashCollectorState.ConsoleIOError != "" { + m.logger.Warn( + "machine console I/O error while collecting computation-hash entries", + "error", hashCollectorState.ConsoleIOError, + ) + hashCollectorState.ConsoleIOError = "" + } // Gets the current cycle. currentCycle, err = m.readMCycle(ctx) @@ -706,6 +824,9 @@ func (m *machineImpl) runIncrementInterval(ctx context.Context, "breakReason", breakReason) if atSaturatedEndpoint && breakReason != McycleOverflow { + // Emulator 0.21 gives mcycle overflow priority over target, yield, and + // halt when mcycle == imcyclemax. Accepting any other reason here could + // either hide that terminal origin or repeat zero-progress runs forever. return incrementResult{breakReason: breakReason, currentCycle: currentCycle}, fmt.Errorf( "machine returned break reason %d instead of mcycle overflow at MaxUint64: %w", breakReason, ErrMachineInternal, @@ -735,6 +856,10 @@ func (m *machineImpl) backendRun(mcycleEnd uint64, hashCollectorState *HashColle return m.backend.Run(mcycleEnd, timeout) } +func isFixedPointBreakReason(reason BreakReason) bool { + return reason == YieldedManually || reason == Halted || reason == McycleOverflow +} + // Helper functions func checkContext(ctx context.Context) error { diff --git a/pkg/machine/implementation_test.go b/pkg/machine/implementation_test.go index 1529f40bc..39df3c38a 100644 --- a/pkg/machine/implementation_test.go +++ b/pkg/machine/implementation_test.go @@ -26,6 +26,12 @@ type ImplementationSuite struct { logger *slog.Logger } +func testExecutionBounds(start, limit uint64) executionBounds { + return executionBounds{ + start: start, limit: limit, span: limit - start, configured: true, + } +} + func (s *ImplementationSuite) SetupSuite() { s.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) } @@ -168,6 +174,21 @@ func (s *ImplementationSuite) TestOutputsHash() { require.ErrorIs(err, ErrRejected) mockBackend2.AssertExpectations(s.T()) + // Exception remains distinguishable through the public error taxonomy. + mockBackendException := NewMockBackend() + mockBackendException.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( + uint8(0), uint16(ManualYieldReasonException), []byte("exception"), nil) + machineException := &machineImpl{ + backend: mockBackendException, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second * 5, + }, + } + _, err = machineException.OutputsHash(ctx) + require.ErrorIs(err, ErrException) + mockBackendException.AssertExpectations(s.T()) + // Test outputs hash with invalid length mockBackend3 := NewMockBackend() mockBackend3.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( @@ -264,7 +285,6 @@ func (s *ImplementationSuite) TestAdvance() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, @@ -274,6 +294,7 @@ func (s *ImplementationSuite) TestAdvance() { input := []byte("test input") resp, err := machine.Advance(ctx, input, expectedHash, false) require.NoError(err) + require.NotNil(resp) require.Equal(CompletionStatusAccepted, resp.Status) require.Empty(resp.Outputs) require.Empty(resp.Reports) @@ -288,7 +309,6 @@ func (s *ImplementationSuite) TestAdvance() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, @@ -296,6 +316,7 @@ func (s *ImplementationSuite) TestAdvance() { } resp, err = machine2.Advance(ctx, input, expectedHash, false) require.NoError(err) + require.NotNil(resp) require.Equal(CompletionStatusRejected, resp.Status) require.Empty(resp.Outputs) require.Empty(resp.Reports) @@ -310,7 +331,6 @@ func (s *ImplementationSuite) TestAdvance() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, @@ -318,11 +338,90 @@ func (s *ImplementationSuite) TestAdvance() { } resp, err = machine3.Advance(ctx, input, expectedHash, false) require.NoError(err) + require.NotNil(resp) require.Equal(CompletionStatusException, resp.Status) require.Equal([]byte("exception data"), resp.ExceptionData) require.Equal(Hash{}, resp.OutputsHash) mockBackend3.AssertExpectations(s.T()) + // Halting is also a completed deterministic status and retains + // the PRT input hash collection produced before the halt. + expectedHaltHash := randomFakeHash() + mockBackendHalted := NewMockBackend() + mockBackendHalted.On("CmioRxBufferSize").Return(uint64(1024)) + mockBackendHalted.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(0), nil).Once() + mockBackendHalted.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(100), nil).Once() + mockBackendHalted.On( + "SendCmioResponse", + uint16(AdvanceStateRequest), mock.Anything, mock.Anything, mock.AnythingOfType("time.Duration"), + ).Return(nil) + mockBackendHalted.On( + "RunAndCollectRootHashes", + mock.AnythingOfType("uint64"), + mock.AnythingOfType("*machine.HashCollectorState"), + mock.AnythingOfType("time.Duration"), + ).Run(func(args mock.Arguments) { + state := args.Get(1).(*HashCollectorState) + state.Hashes = append(state.Hashes, expectedHaltHash) + state.MCyclePhase = 0 // halt occurred exactly at the collected boundary + }).Return(Halted, nil) + machineHalted := &machineImpl{ + backend: mockBackendHalted, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second * 5, + AdvanceIncCycles: 100, + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: time.Second * 10, + }, + } + resp, err = machineHalted.Advance(ctx, input, expectedHash, true) + require.NoError(err) + require.NotNil(resp) + require.Equal(CompletionStatusHalted, resp.Status) + require.Empty(resp.PeriodicStateHashes) + require.Equal(InputEntryCapacity, resp.PaddingRepetitions) + mockBackendHalted.AssertExpectations(s.T()) + + // Test advance halted off a sampling boundary: 0.21 still appends the + // fixed-point sample (nonzero phase) and the node must still drop it. + offBoundaryHaltHash := randomFakeHash() + mockBackendHaltedOff := NewMockBackend() + mockBackendHaltedOff.On("CmioRxBufferSize").Return(uint64(1024)) + mockBackendHaltedOff.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(0), nil).Once() + mockBackendHaltedOff.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(101), nil).Once() + mockBackendHaltedOff.On( + "SendCmioResponse", + uint16(AdvanceStateRequest), mock.Anything, mock.Anything, mock.AnythingOfType("time.Duration"), + ).Return(nil) + mockBackendHaltedOff.On( + "RunAndCollectRootHashes", + mock.AnythingOfType("uint64"), + mock.AnythingOfType("*machine.HashCollectorState"), + mock.AnythingOfType("time.Duration"), + ).Run(func(args mock.Arguments) { + state := args.Get(1).(*HashCollectorState) + state.Hashes = append(state.Hashes, offBoundaryHaltHash) + state.MCyclePhase = 5 // halt occurred between sampling boundaries + }).Return(Halted, nil) + machineHaltedOff := &machineImpl{ + backend: mockBackendHaltedOff, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second * 5, + AdvanceIncCycles: 100, + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: time.Second * 10, + }, + } + resp, err = machineHaltedOff.Advance(ctx, input, expectedHash, true) + require.NoError(err) + require.NotNil(resp) + require.Equal(CompletionStatusHalted, resp.Status) + require.Empty(resp.PeriodicStateHashes) + require.Equal(InputEntryCapacity, resp.PaddingRepetitions) + mockBackendHaltedOff.AssertExpectations(s.T()) + // Test advance with payload too large mockBackend4 := NewMockBackend() mockBackend4.On("CmioRxBufferSize").Return(uint64(5)) @@ -331,7 +430,6 @@ func (s *ImplementationSuite) TestAdvance() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, @@ -343,6 +441,39 @@ func (s *ImplementationSuite) TestAdvance() { require.Nil(resp) mockBackend4.AssertExpectations(s.T()) + // A configured target beyond uint64 saturates at MaxUint64, just like the + // emulator's imcyclemax. If the machine reaches that endpoint, the machine + // overflow—not the configured-cap sentinel—explains why execution stopped. + mockBackendOverflow := NewMockBackend() + mockBackendOverflow.On("CmioRxBufferSize").Return(uint64(1024)) + mockBackendOverflow.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(^uint64(0)-uint64(999), nil).Once() + mockBackendOverflow.On("SendCmioResponse", uint16(AdvanceStateRequest), input, expectedHash, + mock.AnythingOfType("time.Duration")).Return(nil).Once() + mockBackendOverflow.On("Run", ^uint64(0), mock.AnythingOfType("time.Duration")). + Return(McycleOverflow, nil).Once() + mockBackendOverflow.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(^uint64(0), nil).Once() + machineOverflow := &machineImpl{ + backend: mockBackendOverflow, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second * 5, + AdvanceIncCycles: ^uint64(0), + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: time.Second, + AdvanceMaxCycles: 1000, + }, + } + resp, err = machineOverflow.Advance(ctx, input, expectedHash, false) + require.ErrorIs(err, ErrReachedLimitMcycle) + require.ErrorIs(err, ErrMcycleOverflow) + require.Contains(err.Error(), "requested_span=1000") + require.Contains(err.Error(), "target_span=999") + require.Contains(err.Error(), "executed_cycles=999") + require.Nil(resp) + mockBackendOverflow.AssertExpectations(s.T()) + // Test advance with invalid hash length mockBackend5 := NewMockBackend() mockBackend5.On("CmioRxBufferSize").Return(uint64(1024)) @@ -356,7 +487,6 @@ func (s *ImplementationSuite) TestAdvance() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, @@ -369,100 +499,249 @@ func (s *ImplementationSuite) TestAdvance() { mockBackend5.AssertExpectations(s.T()) } -// Test Inspect method -func (s *ImplementationSuite) TestInspect() { - require := s.Require() - ctx := context.Background() +func (s *ImplementationSuite) TestCompletedAdvanceIsIndependentOfAdequateIncrementSettings() { + expectedOutputsHash := randomFakeHash() + expectedHashes := []Hash{randomFakeHash(), randomFakeHash()} + configs := []model.ExecutionParameters{ + { + FastDeadline: time.Second, + AdvanceIncCycles: 2, + AdvanceIncDeadline: 10 * time.Millisecond, + AdvanceMaxDeadline: time.Second, + }, + { + FastDeadline: 2 * time.Second, + AdvanceIncCycles: 4, + AdvanceIncDeadline: 100 * time.Millisecond, + AdvanceMaxDeadline: 2 * time.Second, + }, + { + FastDeadline: 3 * time.Second, + AdvanceIncCycles: 100, + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: 3 * time.Second, + }, + } + expectedTargets := [][]uint64{ + {2, 4, 5, 7, 8, 10}, + {4, 7, 10}, + {100, 103, 106}, + } - // Test successful inspect (accepted) - mockBackend := NewMockBackend() - mockBackend.SetupAccepted(InspectStateRequest) + var baseline *AdvanceResponse + for i, params := range configs { + s.Run(fmt.Sprintf("configuration-%d", i), func() { + backend := &statefulAdvanceBackend{ + outputsHash: expectedOutputsHash, + hashes: expectedHashes, + } + machine := &machineImpl{backend: backend, logger: s.logger, params: params} - machine := &machineImpl{ - backend: mockBackend, - logger: s.logger, - params: model.ExecutionParameters{ - FastDeadline: time.Second * 5, - InspectIncCycles: 100, - InspectMaxCycles: 1000, - InspectIncDeadline: time.Second * 1, - InspectMaxDeadline: time.Second * 10, - }, + result, err := machine.Advance(context.Background(), []byte("stable-input"), Hash{}, true) + s.Require().NoError(err) + s.Require().NotNil(result) + s.Require().Equal(expectedTargets[i], backend.runTargets) + if baseline == nil { + baseline = result + } else { + s.Require().Equal(baseline, result) + } + }) } +} - query := []byte("test query") - response, err := machine.Inspect(ctx, query) - require.NoError(err) - require.Equal(CompletionStatusAccepted, response.Status) - require.Empty(response.Reports) - mockBackend.AssertExpectations(s.T()) +func (s *ImplementationSuite) TestPRTHashCollectionIsIndependentOfConfiguredCycleCap() { + const configuredCap uint64 = 10 + expectedOutputsHash := randomFakeHash() + expectedHashes := []Hash{randomFakeHash(), randomFakeHash()} + + runAdvance := func(maxCycles uint64) (*AdvanceResponse, []uint64) { + backend := &statefulAdvanceBackend{ + outputsHash: expectedOutputsHash, + hashes: expectedHashes, + } + machine := &machineImpl{ + backend: backend, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second, + AdvanceIncCycles: 100, + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: time.Second, + AdvanceMaxCycles: maxCycles, + }, + } - // Test inspect with rejection - mockBackend2 := NewMockBackend() - mockBackend2.SetupRejected(InspectStateRequest) - machine2 := &machineImpl{ - backend: mockBackend2, - logger: s.logger, - params: model.ExecutionParameters{ - FastDeadline: time.Second * 5, - InspectIncCycles: 100, - InspectMaxCycles: 1000, - InspectIncDeadline: time.Second * 1, - InspectMaxDeadline: time.Second * 10, - }, + result, err := machine.Advance(context.Background(), []byte("stable-input"), Hash{}, true) + s.Require().NoError(err) + s.Require().NotNil(result) + return result, backend.runTargets } - response, err = machine2.Inspect(ctx, query) - require.NoError(err) - require.Equal(CompletionStatusRejected, response.Status) - require.Empty(response.Reports) - mockBackend2.AssertExpectations(s.T()) - // Test inspect with exception - mockBackend3 := NewMockBackend() - mockBackend3.SetupException(InspectStateRequest) - machine3 := &machineImpl{ - backend: mockBackend3, - logger: s.logger, - params: model.ExecutionParameters{ - FastDeadline: time.Second * 5, - InspectIncCycles: 100, - InspectMaxCycles: 1000, - InspectIncDeadline: time.Second * 1, - InspectMaxDeadline: time.Second * 10, - }, + uncapped, uncappedTargets := runAdvance(0) + capped, cappedTargets := runAdvance(configuredCap) + maximumCap, maximumCapTargets := runAdvance(model.MaxExecutionCycleSpan) + + s.Equal([]uint64{100, 103, 106}, uncappedTargets) + s.Equal([]uint64{configuredCap, configuredCap, configuredCap}, cappedTargets) + s.NotEqual(uncappedTargets, cappedTargets, "configured cap must alter the backend run targets") + s.Equal(uncapped, capped) + s.Equal(uncapped.PeriodicStateHashes, capped.PeriodicStateHashes) + s.Equal(uncapped.PaddingRepetitions, capped.PaddingRepetitions) + s.Equal(expectedHashes, capped.PeriodicStateHashes) + s.Equal(InputEntryCapacity-uint64(len(expectedHashes)), capped.PaddingRepetitions) + s.Equal(uncappedTargets, maximumCapTargets) + s.Equal(uncapped, maximumCap, + "the largest valid configured span must be canonically identical to no operator cap") +} + +// statefulAdvanceBackend models a guest that produces an output at mcycle 3, +// a report at mcycle 6, and its final accepted yield at mcycle 9. Small +// increments reach intermediate targets; large increments stop early at the +// same guest events. This makes the scheduling-invariance test exercise the +// real run loop rather than preprogramming one yield per call. +type statefulAdvanceBackend struct { + cycle uint64 + runTargets []uint64 + outputsHash Hash + hashes []Hash +} + +func (b *statefulAdvanceBackend) run(mcycleEnd uint64) (BreakReason, error) { + b.runTargets = append(b.runTargets, mcycleEnd) + if mcycleEnd <= b.cycle { + return Failed, fmt.Errorf("run target %d is not ahead of cycle %d", mcycleEnd, b.cycle) } - response, err = machine3.Inspect(ctx, query) - require.NoError(err) - require.Equal(CompletionStatusException, response.Status) - require.Equal([]byte("exception data"), response.ExceptionData) - require.Empty(response.Reports) - mockBackend3.AssertExpectations(s.T()) + var eventCycle uint64 + var reason BreakReason + switch { + case b.cycle < 3: + eventCycle, reason = 3, YieldedAutomatically + case b.cycle < 6: + eventCycle, reason = 6, YieldedAutomatically + case b.cycle < 9: + eventCycle, reason = 9, YieldedManually + default: + return Failed, errors.New("scripted advance already completed") + } + if mcycleEnd < eventCycle { + b.cycle = mcycleEnd + return ReachedTargetMcycle, nil + } + b.cycle = eventCycle + return reason, nil +} - // Test inspect with payload too large - mockBackend4 := NewMockBackend() - mockBackend4.On("CmioRxBufferSize").Return(uint64(5)) - machine4 := &machineImpl{ - backend: mockBackend4, +func (b *statefulAdvanceBackend) Load(string, string, time.Duration) error { return nil } +func (b *statefulAdvanceBackend) Store(string, time.Duration) error { return nil } +func (b *statefulAdvanceBackend) Run(end uint64, _ time.Duration) (BreakReason, error) { + return b.run(end) +} +func (b *statefulAdvanceBackend) RunAndCollectRootHashes( + end uint64, + state *HashCollectorState, + _ time.Duration, +) (BreakReason, error) { + reason, err := b.run(end) + if err == nil && reason == YieldedAutomatically { + switch b.cycle { + case 3: + state.Hashes = append(state.Hashes, b.hashes[0]) + case 6: + state.Hashes = append(state.Hashes, b.hashes[1]) + } + } + if err == nil && reason == YieldedManually { + // Emulator 0.21 appends the fixed-point sample whenever a collecting + // run stops at a manual yield, regardless of phase; the node drops + // exactly this final entry. + state.Hashes = append(state.Hashes, Hash{0xF1, 0xED}) + } + return reason, err +} +func (b *statefulAdvanceBackend) IsAtManualYield(time.Duration) (bool, error) { + return b.cycle == 9, nil +} +func (b *statefulAdvanceBackend) ReadMCycle(time.Duration) (uint64, error) { return b.cycle, nil } +func (b *statefulAdvanceBackend) SendCmioResponse(reason uint16, data []byte, _ *Hash, _ time.Duration) error { + if reason != uint16(AdvanceStateRequest) || string(data) != "stable-input" { + return fmt.Errorf("unexpected CMIO response reason=%d data_len=%d", reason, len(data)) + } + return nil +} +func (b *statefulAdvanceBackend) ReceiveCmioRequest(time.Duration) (uint8, uint16, []byte, error) { + switch b.cycle { + case 3: + return 0, uint16(AutomaticYieldReasonOutput), []byte("stable-output"), nil + case 6: + return 0, uint16(AutomaticYieldReasonReport), []byte("stable-report"), nil + case 9: + return 0, uint16(ManualYieldReasonAccepted), b.outputsHash[:], nil + default: + return 0, 0, nil, fmt.Errorf("no CMIO request at cycle %d", b.cycle) + } +} +func (b *statefulAdvanceBackend) WriteMemory(uint64, []byte, time.Duration) error { return nil } +func (b *statefulAdvanceBackend) GetRootHash(time.Duration) (Hash, error) { return Hash{}, nil } +func (b *statefulAdvanceBackend) GetProof(uint64, int32, time.Duration) ([]Hash, error) { + return nil, nil +} +func (b *statefulAdvanceBackend) Delete() {} +func (b *statefulAdvanceBackend) ForkServer(time.Duration) (Backend, string, uint32, error) { + return nil, "", 0, errors.New("not implemented") +} +func (b *statefulAdvanceBackend) ShutdownServer(time.Duration) error { return nil } +func (b *statefulAdvanceBackend) NewMachineRuntimeConfig() (string, error) { + return "{}", nil +} +func (b *statefulAdvanceBackend) CmioRxBufferSize() uint64 { return 1024 } + +func (s *ImplementationSuite) TestInterruptedAdvanceReturnsNilAndCanBeRetried() { + expectedOutputsHash := randomFakeHash() + backend := NewMockBackend() + backend.On("CmioRxBufferSize").Return(uint64(1024)) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(0), nil) + backend.On( + "SendCmioResponse", + uint16(AdvanceStateRequest), + []byte("retry-input"), + mock.Anything, + mock.AnythingOfType("time.Duration"), + ).Return(nil) + backend.On("Run", mock.AnythingOfType("uint64"), mock.AnythingOfType("time.Duration")). + Return(YieldedManually, nil) + backend.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( + uint8(0), uint16(ManualYieldReasonAccepted), expectedOutputsHash[:], nil, + ) + machine := &machineImpl{ + backend: backend, logger: s.logger, params: model.ExecutionParameters{ - FastDeadline: time.Second * 5, - InspectIncCycles: 100, - InspectMaxCycles: 1000, - InspectIncDeadline: time.Second * 1, - InspectMaxDeadline: time.Second * 10, + FastDeadline: time.Second, + AdvanceIncCycles: 100, + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: time.Second, }, } - largeQuery := make([]byte, 10) - response, err = machine4.Inspect(ctx, largeQuery) - require.NotNil(response) - require.Equal(CompletionStatusUnknown, response.Status) - require.ErrorIs(err, ErrPayloadLengthLimitExceeded) - mockBackend4.AssertExpectations(s.T()) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + result, err := machine.Advance(ctx, []byte("retry-input"), Hash{}, false) + s.Require().ErrorIs(err, ErrCanceled) + s.Require().Nil(result) + + result, err = machine.Advance(context.Background(), []byte("retry-input"), Hash{}, false) + s.Require().NoError(err) + s.Require().NotNil(result) + s.Require().Equal(CompletionStatusAccepted, result.Status) + s.Require().Equal(expectedOutputsHash, result.OutputsHash) + backend.AssertExpectations(s.T()) } func (s *ImplementationSuite) TestRunUsesHardExecutionSpanWhenMaximumIsZero() { const startCycle uint64 = 7 - const executionCycleSpan uint64 = BarchSpanToInput + // The fixed window mirrors the emulator's imcyclemax: 2^48 - 1 ahead. + const executionCycleSpan uint64 = (1 << Log2MaxMCyclesPerAdvanceState) - 1 tests := []struct { name string @@ -497,7 +776,12 @@ func (s *ImplementationSuite) TestRunUsesHardExecutionSpanWhenMaximumIsZero() { mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). Return(startCycle+executionCycleSpan, nil).Once() - machine := &machineImpl{backend: mockBackend, logger: s.logger, params: tt.params} + machine := &machineImpl{ + backend: mockBackend, + logger: s.logger, + params: tt.params, + } + _, err := machine.run( context.Background(), tt.reqType, false, executionBounds{ @@ -654,6 +938,7 @@ func (s *ImplementationSuite) TestProcessRejectsZeroIncrementBeforeCMIO() { s.Run(test.name, func() { backend := NewMockBackend() backend.On("CmioRxBufferSize").Return(uint64(1024)).Maybe() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(1), nil).Maybe() params := model.ExecutionParameters{ FastDeadline: time.Second, AdvanceIncDeadline: time.Second, @@ -667,7 +952,6 @@ func (s *ImplementationSuite) TestProcessRejectsZeroIncrementBeforeCMIO() { s.Require().Error(err) s.Contains(err.Error(), test.reqType.String()) s.Contains(err.Error(), "increment") - backend.AssertNotCalled(s.T(), "ReadMCycle", mock.Anything) backend.AssertNotCalled(s.T(), "SendCmioResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything) backend.AssertNotCalled(s.T(), "Run", mock.Anything, mock.Anything) @@ -753,63 +1037,71 @@ func (s *ImplementationSuite) TestConfiguredCycleExhaustionFails() { backend.AssertExpectations(s.T()) } -func (s *ImplementationSuite) TestAdvanceCycleExhaustionSources() { - for _, test := range []struct { - name string - configuredSpan uint64 - breakReason BreakReason - wantSource string - }{ - { - name: "configured endpoint", configuredSpan: 9, - breakReason: ReachedTargetMcycle, wantSource: "configured", - }, - { - name: "fixed window", breakReason: McycleOverflow, wantSource: "fixed", - }, - } { - s.Run(test.name, func() { - const start = uint64(30) - span := test.configuredSpan - if span == 0 { - span = model.MaxExecutionCycleSpan - } - backend := NewMockBackend() - backend.On("CmioRxBufferSize").Return(uint64(1024)) - backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start, nil).Once() - backend.On( - "SendCmioResponse", - uint16(AdvanceStateRequest), []byte("input"), mock.Anything, mock.AnythingOfType("time.Duration"), - ).Return(nil) - backend.On("Run", start+span, mock.AnythingOfType("time.Duration")).Return(test.breakReason, nil) - backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start+span, nil).Once() - machine := &machineImpl{backend: backend, logger: s.logger, params: model.ExecutionParameters{ - AdvanceIncCycles: ^uint64(0), - AdvanceMaxCycles: test.configuredSpan, - AdvanceIncDeadline: time.Second, - AdvanceMaxDeadline: time.Second, - FastDeadline: time.Second, - }} +func (s *ImplementationSuite) TestAdvanceConfiguredCycleExhaustionReturnsNoResult() { + const start, span = uint64(30), uint64(9) + backend := NewMockBackend() + backend.On("CmioRxBufferSize").Return(uint64(1024)) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start, nil).Once() + backend.On( + "SendCmioResponse", + uint16(AdvanceStateRequest), []byte("input"), mock.Anything, mock.AnythingOfType("time.Duration"), + ).Return(nil) + backend.On("Run", start+span, mock.AnythingOfType("time.Duration")).Return(ReachedTargetMcycle, nil) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start+span, nil).Once() + machine := &machineImpl{backend: backend, logger: s.logger, params: model.ExecutionParameters{ + AdvanceIncCycles: span + 100, AdvanceMaxCycles: span, + AdvanceIncDeadline: time.Second, AdvanceMaxDeadline: time.Second, + FastDeadline: time.Second, + }} - response, err := machine.Advance(context.Background(), []byte("input"), Hash{}, false) - s.Require().Nil(response) - s.Require().ErrorIs(err, ErrReachedLimitMcycle) - if test.breakReason == McycleOverflow { - s.Require().ErrorIs(err, ErrMcycleOverflow) - } - s.Contains(err.Error(), test.wantSource) - s.Contains(err.Error(), "cycle limit") - s.Contains(err.Error(), fmt.Sprintf("requested_span=%d", span)) - backend.AssertExpectations(s.T()) - }) - } + response, err := machine.Advance(context.Background(), []byte("input"), Hash{}, false) + s.Require().Nil(response) + s.Require().ErrorIs(err, ErrReachedLimitMcycle) + s.Contains(err.Error(), "advance execution reached configured cycle limit") + s.Contains(err.Error(), "start=30 requested_span=9") + backend.AssertExpectations(s.T()) +} + +func (s *ImplementationSuite) TestAdvanceFixedSpanExhaustionPreservesMachineOverflow() { + const start = uint64(30) + backend := NewMockBackend() + backend.On("CmioRxBufferSize").Return(uint64(1024)) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start, nil).Once() + backend.On( + "SendCmioResponse", + uint16(AdvanceStateRequest), []byte("input"), mock.Anything, mock.AnythingOfType("time.Duration"), + ).Return(nil) + // With no configured cap, the emulator enforces the span itself and reports + // the mcycle-overflow fixed point as a break reason. + backend.On("Run", mock.AnythingOfType("uint64"), mock.AnythingOfType("time.Duration")). + Return(McycleOverflow, nil) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(start+model.MaxExecutionCycleSpan, nil).Once() + machine := &machineImpl{backend: backend, logger: s.logger, params: model.ExecutionParameters{ + AdvanceIncCycles: ^uint64(0), + AdvanceIncDeadline: time.Second, AdvanceMaxDeadline: time.Second, + FastDeadline: time.Second, + }} + + response, err := machine.Advance(context.Background(), []byte("input"), Hash{}, false) + s.Require().Nil(response) + s.Require().ErrorIs(err, ErrReachedLimitMcycle) + s.Require().ErrorIs(err, ErrMcycleOverflow) + s.Contains(err.Error(), "advance execution reached fixed (machine imcyclemax) cycle limit") + backend.AssertExpectations(s.T()) } -func (s *ImplementationSuite) TestConfiguredLimitTiePreservesMachineOverflowPrecedence() { +func (s *ImplementationSuite) TestAdvanceConfiguredLimitTiePreservesMachineOverflowPrecedence() { const start = uint64(30) configuredSpan := model.MaxExecutionCycleSpan limit := start + configuredSpan backend := NewMockBackend() + backend.On("CmioRxBufferSize").Return(uint64(1024)) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start, nil).Once() + backend.On( + "SendCmioResponse", + uint16(AdvanceStateRequest), []byte("input"), mock.Anything, mock.AnythingOfType("time.Duration"), + ).Return(nil).Once() backend.On("Run", limit, mock.AnythingOfType("time.Duration")).Return(McycleOverflow, nil).Once() backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(limit, nil).Once() machine := &machineImpl{backend: backend, logger: s.logger, params: model.ExecutionParameters{ @@ -820,16 +1112,17 @@ func (s *ImplementationSuite) TestConfiguredLimitTiePreservesMachineOverflowPrec FastDeadline: time.Second, }} - _, err := machine.run(context.Background(), AdvanceStateRequest, false, executionBounds{ - start: start, limit: limit, span: configuredSpan, configured: true, - }) + response, err := machine.Advance(context.Background(), []byte("input"), Hash{}, false) + s.Require().Nil(response) s.Require().ErrorIs(err, ErrReachedLimitMcycle) s.Require().ErrorIs(err, ErrMcycleOverflow) s.Contains(err.Error(), "advance execution stopped at machine imcyclemax coincident with configured target") + s.Contains(err.Error(), fmt.Sprintf("requested_span=%d", configuredSpan)) + s.Contains(err.Error(), fmt.Sprintf("executed_cycles=%d", configuredSpan)) backend.AssertExpectations(s.T()) } -func (s *ImplementationSuite) TestExecutionStartingAtMachineMaximumPreservesOverflowOrigin() { +func (s *ImplementationSuite) TestAdvanceStartingAtMachineMaximumPreservesOverflowOrigin() { for _, test := range []struct { name string configuredMax uint64 @@ -840,7 +1133,8 @@ func (s *ImplementationSuite) TestExecutionStartingAtMachineMaximumPreservesOver s.Run(test.name, func() { backend := NewMockBackend() backend.On("CmioRxBufferSize").Return(uint64(1024)) - backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(^uint64(0), nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(^uint64(0), nil).Once() backend.On( "SendCmioResponse", uint16(AdvanceStateRequest), []byte("input"), mock.Anything, mock.AnythingOfType("time.Duration"), @@ -861,6 +1155,11 @@ func (s *ImplementationSuite) TestExecutionStartingAtMachineMaximumPreservesOver s.Require().Nil(response) s.Require().ErrorIs(err, ErrReachedLimitMcycle) s.Require().ErrorIs(err, ErrMcycleOverflow) + expectedRequestedSpan := test.configuredMax + if expectedRequestedSpan == 0 { + expectedRequestedSpan = model.MaxExecutionCycleSpan + } + s.Contains(err.Error(), fmt.Sprintf("requested_span=%d", expectedRequestedSpan)) s.Contains(err.Error(), "target_span=0") s.Contains(err.Error(), "executed_cycles=0") backend.AssertExpectations(s.T()) @@ -945,106 +1244,601 @@ func (s *ImplementationSuite) TestInspectInheritedMcycleOverflowDoesNotClaimLoca InspectMaxCycles: test.configuredMax, }} - response, err := machine.Inspect(context.Background(), []byte("query")) - s.Require().NotNil(response) - s.Equal(CompletionStatusUnknown, response.Status) - s.Empty(response.Reports) - s.Require().ErrorIs(err, ErrReachedLimitMcycle) - s.Require().ErrorIs(err, ErrMcycleOverflow) - s.Contains(err.Error(), test.stopDescription) - s.Contains(err.Error(), fmt.Sprintf("requested_span=%d", test.requestedSpan)) - s.Contains(err.Error(), fmt.Sprintf("executed_cycles=%d", test.executedCycles)) - backend.AssertExpectations(s.T()) - }) + response, err := machine.Inspect(context.Background(), []byte("query")) + s.Require().NotNil(response) + s.Equal(CompletionStatusUnknown, response.Status) + s.Empty(response.Reports) + s.Require().ErrorIs(err, ErrReachedLimitMcycle) + s.Require().ErrorIs(err, ErrMcycleOverflow) + s.Contains(err.Error(), test.stopDescription) + s.Contains(err.Error(), fmt.Sprintf("requested_span=%d", test.requestedSpan)) + s.Contains(err.Error(), fmt.Sprintf("executed_cycles=%d", test.executedCycles)) + backend.AssertExpectations(s.T()) + }) + } +} + +func (s *ImplementationSuite) TestExecutionCycleBoundsSaturatesAtMachineMaximum() { + for _, test := range []struct { + name string + start uint64 + configured uint64 + }{ + { + name: "configured cap beyond representable range", + start: ^uint64(0) - 10, configured: 11, + }, + { + name: "zero cap", + start: ^uint64(0) - model.MaxExecutionCycleSpan + 1, configured: 0, + }, + { + name: "maximum cap", + start: ^uint64(0) - model.MaxExecutionCycleSpan + 1, + configured: model.MaxExecutionCycleSpan, + }, + } { + s.Run(test.name, func() { + backend := NewMockBackend() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(test.start, nil) + machine := &machineImpl{backend: backend, logger: s.logger, params: model.ExecutionParameters{ + AdvanceIncCycles: 1, + AdvanceMaxCycles: test.configured, + }} + + bounds, err := machine.executionCycleBounds(context.Background(), AdvanceStateRequest) + s.Require().NoError(err) + s.Equal(test.start, bounds.start) + s.Equal(^uint64(0), bounds.limit) + backend.AssertExpectations(s.T()) + }) + } +} + +func (s *ImplementationSuite) TestExecutionCycleBoundsUsesFullWindowWhenUncapped() { + + s.Run("window endpoint is start plus 2^48 minus 1", func() { + const start = uint64(1000) + backend := NewMockBackend() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start, nil) + machine := &machineImpl{backend: backend, logger: s.logger, params: model.ExecutionParameters{ + AdvanceIncCycles: 1, + }} + + bounds, err := machine.executionCycleBounds(context.Background(), AdvanceStateRequest) + s.Require().NoError(err) + s.Equal(start, bounds.start) + s.Equal(start+model.MaxExecutionCycleSpan, bounds.limit) + backend.AssertExpectations(s.T()) + }) +} + +func (s *ImplementationSuite) TestExecutionResponseLimitErrorsExposeOnlyCounts() { + for _, test := range []struct { + kind string + count int + capacity int + sentinel error + }{ + {kind: "output", count: maxOutputs + 1, capacity: maxOutputs, sentinel: ErrOutputsLimitExceeded}, + {kind: "report", count: maxReports + 1, capacity: maxReports, sentinel: ErrReportsLimitExceeded}, + } { + err := executionResponseLimitError( + AdvanceStateRequest, test.kind, test.count, test.capacity, test.sentinel, + ) + s.Require().ErrorIs(err, test.sentinel) + s.Contains(err.Error(), fmt.Sprintf("advance %s count %d", test.kind, test.count)) + s.Contains(err.Error(), fmt.Sprintf("capacity %d", test.capacity)) + } +} + +func (s *ImplementationSuite) TestInspectUsesConfiguredCycleIncrement() { + require := s.Require() + const startCycle uint64 = 7 + const inspectIncCycles uint64 = 137 + + mockBackend := NewMockBackend() + mockBackend.On("CmioRxBufferSize").Return(uint64(1024)) + mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(startCycle, nil).Once() + mockBackend.On( + "SendCmioResponse", + uint16(InspectStateRequest), mock.Anything, mock.Anything, mock.AnythingOfType("time.Duration"), + ).Return(nil) + mockBackend.On("Run", startCycle+inspectIncCycles, mock.AnythingOfType("time.Duration")).Return(YieldedManually, nil) + mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(startCycle+inspectIncCycles, nil).Once() + mockBackend.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( + uint8(0), uint16(ManualYieldReasonAccepted), []byte(nil), nil) + + machine := &machineImpl{ + backend: mockBackend, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second, + InspectIncCycles: inspectIncCycles, + InspectIncDeadline: time.Second, + InspectMaxDeadline: time.Second, + }, + } + + response, err := machine.Inspect(context.Background(), []byte("query")) + require.NoError(err) + require.Equal(CompletionStatusAccepted, response.Status) + mockBackend.AssertExpectations(s.T()) +} + +// Test Inspect method +func (s *ImplementationSuite) TestInspect() { + require := s.Require() + ctx := context.Background() + + // Test successful inspect (accepted) + mockBackend := NewMockBackend() + mockBackend.SetupAccepted(InspectStateRequest) + + machine := &machineImpl{ + backend: mockBackend, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second * 5, + InspectIncCycles: 100, + InspectIncDeadline: time.Second * 1, + InspectMaxDeadline: time.Second * 10, + }, + } + + query := []byte("test query") + response, err := machine.Inspect(ctx, query) + require.NoError(err) + require.Equal(CompletionStatusAccepted, response.Status) + require.Empty(response.Reports) + mockBackend.AssertExpectations(s.T()) + + // Test inspect with rejection + mockBackend2 := NewMockBackend() + mockBackend2.SetupRejected(InspectStateRequest) + machine2 := &machineImpl{ + backend: mockBackend2, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second * 5, + InspectIncCycles: 100, + InspectIncDeadline: time.Second * 1, + InspectMaxDeadline: time.Second * 10, + }, + } + response, err = machine2.Inspect(ctx, query) + require.NoError(err) + require.Equal(CompletionStatusRejected, response.Status) + require.Empty(response.Reports) + mockBackend2.AssertExpectations(s.T()) + + // Test inspect with exception + mockBackend3 := NewMockBackend() + mockBackend3.SetupException(InspectStateRequest) + machine3 := &machineImpl{ + backend: mockBackend3, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second * 5, + InspectIncCycles: 100, + InspectIncDeadline: time.Second * 1, + InspectMaxDeadline: time.Second * 10, + }, + } + response, err = machine3.Inspect(ctx, query) + require.NoError(err) + require.Equal(CompletionStatusException, response.Status) + require.Equal([]byte("exception data"), response.ExceptionData) + require.Empty(response.Reports) + mockBackend3.AssertExpectations(s.T()) + + // A halt is a completed guest outcome. Reports emitted before the halt are + // part of that completed inspect response. + mockBackendHalted := NewMockBackend() + mockBackendHalted.On("CmioRxBufferSize").Return(uint64(1024)) + mockBackendHalted.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(0), nil).Once() + mockBackendHalted.On("SendCmioResponse", uint16(InspectStateRequest), query, nil, + mock.AnythingOfType("time.Duration")).Return(nil).Once() + mockBackendHalted.On("Run", uint64(100), mock.AnythingOfType("time.Duration")). + Return(YieldedAutomatically, nil).Once() + mockBackendHalted.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(50), nil).Once() + mockBackendHalted.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")). + Return(uint8(0), uint16(AutomaticYieldReasonReport), []byte("before halt"), nil).Once() + mockBackendHalted.On("Run", uint64(150), mock.AnythingOfType("time.Duration")). + Return(Halted, nil).Once() + mockBackendHalted.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(75), nil).Once() + machineHalted := &machineImpl{ + backend: mockBackendHalted, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second * 5, + InspectIncCycles: 100, + InspectIncDeadline: time.Second, + InspectMaxDeadline: time.Second * 10, + }, + } + response, err = machineHalted.Inspect(ctx, query) + require.NoError(err) + require.Equal(CompletionStatusHalted, response.Status) + require.Equal([]Report{[]byte("before halt")}, response.Reports) + mockBackendHalted.AssertExpectations(s.T()) + + // An internal stop is not a completed guest outcome. It returns Unknown + // with the partial report prefix and a typed execution error. + mockBackendFailed := NewMockBackend() + mockBackendFailed.On("CmioRxBufferSize").Return(uint64(1024)) + mockBackendFailed.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(0), nil).Once() + mockBackendFailed.On("SendCmioResponse", uint16(InspectStateRequest), query, nil, + mock.AnythingOfType("time.Duration")).Return(nil).Once() + mockBackendFailed.On("Run", uint64(100), mock.AnythingOfType("time.Duration")). + Return(YieldedAutomatically, nil).Once() + mockBackendFailed.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(50), nil).Once() + mockBackendFailed.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")). + Return(uint8(0), uint16(AutomaticYieldReasonReport), []byte("partial"), nil).Once() + mockBackendFailed.On("Run", uint64(150), mock.AnythingOfType("time.Duration")). + Return(Failed, nil).Once() + mockBackendFailed.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(75), nil).Once() + machineFailed := &machineImpl{ + backend: mockBackendFailed, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second * 5, + InspectIncCycles: 100, + InspectIncDeadline: time.Second, + InspectMaxDeadline: time.Second * 10, + }, + } + response, err = machineFailed.Inspect(ctx, query) + require.ErrorIs(err, ErrMachineInternal) + require.Equal(CompletionStatusUnknown, response.Status) + require.Equal([]Report{[]byte("partial")}, response.Reports) + mockBackendFailed.AssertExpectations(s.T()) + + // Test inspect with payload too large + mockBackend4 := NewMockBackend() + mockBackend4.On("CmioRxBufferSize").Return(uint64(5)) + machine4 := &machineImpl{ + backend: mockBackend4, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second * 5, + InspectIncCycles: 100, + InspectIncDeadline: time.Second * 1, + InspectMaxDeadline: time.Second * 10, + }, + } + largeQuery := make([]byte, 10) + response, err = machine4.Inspect(ctx, largeQuery) + require.NotNil(response) + require.Equal(CompletionStatusUnknown, response.Status) + require.ErrorIs(err, ErrPayloadLengthLimitExceeded) + mockBackend4.AssertExpectations(s.T()) + + // Inspect uses the same saturating target as advance. Reaching MaxUint64 is + // reported as a machine overflow and never as a completed input result. + mockBackendOverflow := NewMockBackend() + mockBackendOverflow.On("CmioRxBufferSize").Return(uint64(1024)) + mockBackendOverflow.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(^uint64(0)-uint64(999), nil).Once() + mockBackendOverflow.On("SendCmioResponse", uint16(InspectStateRequest), query, nil, + mock.AnythingOfType("time.Duration")).Return(nil).Once() + mockBackendOverflow.On("Run", ^uint64(0), mock.AnythingOfType("time.Duration")). + Return(McycleOverflow, nil).Once() + mockBackendOverflow.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(^uint64(0), nil).Once() + machineOverflow := &machineImpl{ + backend: mockBackendOverflow, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second, + InspectIncCycles: ^uint64(0), + InspectIncDeadline: time.Second, + InspectMaxDeadline: time.Second, + InspectMaxCycles: 1000, + }, } + response, err = machineOverflow.Inspect(ctx, query) + require.NotNil(response) + require.Equal(CompletionStatusUnknown, response.Status) + require.Empty(response.Reports) + require.ErrorIs(err, ErrReachedLimitMcycle) + require.ErrorIs(err, ErrMcycleOverflow) + require.Contains(err.Error(), "requested_span=1000") + require.Contains(err.Error(), "target_span=999") + require.Contains(err.Error(), "executed_cycles=999") + mockBackendOverflow.AssertExpectations(s.T()) } -func (s *ImplementationSuite) TestExecutionCycleBoundsSaturatesAtMachineMaximum() { +func (s *ImplementationSuite) TestAdvanceCanonicalizesTerminalBoundaryHash() { for _, test := range []struct { - name string - start uint64 - configured uint64 + name string + yieldReason manualYieldReason + status CompletionStatus + terminalAtBoundary bool + sameRunPriorBoundary bool + immediateYield bool }{ { - name: "configured cap beyond representable range", start: ^uint64(0) - 10, configured: 11, + name: "accepted at first boundary", + yieldReason: ManualYieldReasonAccepted, + status: CompletionStatusAccepted, + terminalAtBoundary: true, }, { - name: "zero cap", start: ^uint64(0) - model.MaxExecutionCycleSpan + 1, + name: "rejected at first boundary", + yieldReason: ManualYieldReasonRejected, + status: CompletionStatusRejected, + terminalAtBoundary: true, }, { - name: "maximum cap", start: ^uint64(0) - model.MaxExecutionCycleSpan + 1, - configured: model.MaxExecutionCycleSpan, + name: "exception at first boundary", + yieldReason: ManualYieldReasonException, + status: CompletionStatusException, + terminalAtBoundary: true, + }, + { + name: "off-boundary terminal keeps prior boundary", + yieldReason: ManualYieldReasonAccepted, + status: CompletionStatusAccepted, + terminalAtBoundary: false, + }, + { + name: "same run prior boundary survives off-boundary terminal", + yieldReason: ManualYieldReasonAccepted, + status: CompletionStatusAccepted, + sameRunPriorBoundary: true, + }, + { + name: "immediate off-boundary yield drops its fixed-point sample", + yieldReason: ManualYieldReasonAccepted, + status: CompletionStatusAccepted, + immediateYield: true, }, } { s.Run(test.name, func() { + collectedHash := randomFakeHash() + terminalSample := randomFakeHash() + outputsHash := randomFakeHash() + backend := NewMockBackend() - backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(test.start, nil) - machine := &machineImpl{backend: backend, logger: s.logger, params: model.ExecutionParameters{ - AdvanceIncCycles: 1, - AdvanceMaxCycles: test.configured, - }} + backend.On("CmioRxBufferSize").Return(uint64(1024)) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(uint64(0), nil).Once() + backend.On( + "SendCmioResponse", + uint16(AdvanceStateRequest), []byte("input"), mock.Anything, mock.AnythingOfType("time.Duration"), + ). + Return(nil).Once() + switch { + case test.terminalAtBoundary: + backend.On( + "RunAndCollectRootHashes", + MCycleComputationHashPeriod, + mock.AnythingOfType("*machine.HashCollectorState"), + mock.AnythingOfType("time.Duration"), + ).Run(func(args mock.Arguments) { + state := args.Get(1).(*HashCollectorState) + state.Hashes = append(state.Hashes, collectedHash) + state.MCyclePhase = 0 + }).Return(YieldedManually, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(MCycleComputationHashPeriod, nil).Once() + case test.sameRunPriorBoundary: + // RunAndCollectRootHashes may cross and collect a prior + // boundary, then stop at a manual yield one cycle later in the + // same call. Emulator 0.21 also appends the off-boundary + // fixed-point sample; only that final entry may be dropped. + backend.On( + "RunAndCollectRootHashes", + 2*MCycleComputationHashPeriod, + mock.AnythingOfType("*machine.HashCollectorState"), + mock.AnythingOfType("time.Duration"), + ).Run(func(args mock.Arguments) { + state := args.Get(1).(*HashCollectorState) + state.Hashes = append(state.Hashes, collectedHash, terminalSample) + state.MCyclePhase = 1 + }).Return(YieldedManually, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(MCycleComputationHashPeriod+1, nil).Once() + case test.immediateYield: + // Even a yield right after delivery is a fixed point: 0.21 + // appends its sample at a nonzero phase, and the node must + // still drop it. + backend.On( + "RunAndCollectRootHashes", + MCycleComputationHashPeriod, + mock.AnythingOfType("*machine.HashCollectorState"), + mock.AnythingOfType("time.Duration"), + ).Run(func(args mock.Arguments) { + state := args.Get(1).(*HashCollectorState) + state.Hashes = append(state.Hashes, terminalSample) + state.MCyclePhase = 1 + }).Return(YieldedManually, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(uint64(1), nil).Once() + default: + // The first call reaches and records a real prior boundary. + backend.On( + "RunAndCollectRootHashes", + MCycleComputationHashPeriod, + mock.AnythingOfType("*machine.HashCollectorState"), + mock.AnythingOfType("time.Duration"), + ).Run(func(args mock.Arguments) { + state := args.Get(1).(*HashCollectorState) + state.Hashes = append(state.Hashes, collectedHash) + state.MCyclePhase = 0 + }).Return(ReachedTargetMcycle, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(MCycleComputationHashPeriod, nil).Once() + // The second call manually yields one cycle later; 0.21 appends + // the off-boundary fixed-point sample, which is dropped while + // the prior boundary remains in the collection. + backend.On( + "RunAndCollectRootHashes", + 2*MCycleComputationHashPeriod, + mock.AnythingOfType("*machine.HashCollectorState"), + mock.AnythingOfType("time.Duration"), + ).Run(func(args mock.Arguments) { + state := args.Get(1).(*HashCollectorState) + state.Hashes = append(state.Hashes, terminalSample) + state.MCyclePhase = 1 + }).Return(YieldedManually, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(MCycleComputationHashPeriod+1, nil).Once() + } + backend.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")). + Return(uint8(0), uint16(test.yieldReason), outputsHash[:], nil).Once() - bounds, err := machine.executionCycleBounds(context.Background(), AdvanceStateRequest) + advanceIncrement := MCycleComputationHashPeriod + if test.sameRunPriorBoundary { + advanceIncrement = 2 * MCycleComputationHashPeriod + } + machine := &machineImpl{ + backend: backend, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second, + AdvanceIncCycles: advanceIncrement, + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: time.Second, + }, + } + + response, err := machine.Advance(context.Background(), []byte("input"), Hash{}, true) s.Require().NoError(err) - s.Equal(test.start, bounds.start) - s.Equal(^uint64(0), bounds.limit) + s.Require().NotNil(response) + s.Equal(test.status, response.Status) + if test.terminalAtBoundary || test.immediateYield { + s.Empty(response.PeriodicStateHashes) + s.Equal(InputEntryCapacity, response.PaddingRepetitions) + } else { + s.Equal([]Hash{collectedHash}, response.PeriodicStateHashes) + s.Equal(InputEntryCapacity-1, response.PaddingRepetitions) + } backend.AssertExpectations(s.T()) }) } } -func (s *ImplementationSuite) TestExecutionResponseLimitErrorsExposeOnlyCounts() { - for _, test := range []struct { - kind string - count int - capacity int - sentinel error - }{ - {kind: "output", count: maxOutputs + 1, capacity: maxOutputs, sentinel: ErrOutputsLimitExceeded}, - {kind: "report", count: maxReports + 1, capacity: maxReports, sentinel: ErrReportsLimitExceeded}, - } { - err := executionResponseLimitError( - AdvanceStateRequest, test.kind, test.count, test.capacity, test.sentinel, - ) - s.Require().ErrorIs(err, test.sentinel) - s.Contains(err.Error(), fmt.Sprintf("advance %s count %d", test.kind, test.count)) - s.Contains(err.Error(), fmt.Sprintf("capacity %d", test.capacity)) +func (s *ImplementationSuite) TestRunDiscardsOverflowCollection() { + const fixedEndpoint uint64 = model.MaxExecutionCycleSpan + const startCycle uint64 = fixedEndpoint - mcycleComputationHashChunkSize + terminalSample := randomFakeHash() + backend := NewMockBackend() + backend.On( + "RunAndCollectRootHashes", + fixedEndpoint, + mock.AnythingOfType("*machine.HashCollectorState"), + mock.AnythingOfType("time.Duration"), + ).Run(func(args mock.Arguments) { + state := args.Get(1).(*HashCollectorState) + state.Hashes = append(state.Hashes, terminalSample) + state.MCyclePhase = 7 + }).Return(McycleOverflow, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(fixedEndpoint, nil).Once() + machine := &machineImpl{ + backend: backend, + logger: s.logger, + params: model.ExecutionParameters{ + AdvanceIncCycles: fixedEndpoint, + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: time.Second, + }, } -} -func (s *ImplementationSuite) TestInspectUsesConfiguredCycleIncrement() { - const startCycle uint64 = 7 - const inspectIncCycles uint64 = 137 + result, err := machine.run( + context.Background(), AdvanceStateRequest, true, testExecutionBounds(startCycle, fixedEndpoint), + ) - mockBackend := NewMockBackend() - mockBackend.On("CmioRxBufferSize").Return(uint64(1024)) - mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(startCycle, nil).Once() - mockBackend.On( - "SendCmioResponse", - uint16(InspectStateRequest), mock.Anything, mock.Anything, mock.AnythingOfType("time.Duration"), - ).Return(nil) - mockBackend.On("Run", startCycle+inspectIncCycles, mock.AnythingOfType("time.Duration")). - Return(YieldedManually, nil) - mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). - Return(startCycle+inspectIncCycles, nil).Once() - mockBackend.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( - uint8(0), uint16(ManualYieldReasonAccepted), []byte(nil), nil) + s.Require().ErrorIs(err, ErrMcycleOverflow) + s.Empty(result.outputs) + s.Empty(result.reports) + s.Empty(result.periodicStateHashes) + s.Zero(result.paddingRepetitions) + backend.AssertExpectations(s.T()) +} +func (s *ImplementationSuite) TestRunCapsCollectorChunksAndPreservesCollectionState() { + firstPeriodicHash := randomFakeHash() + terminalHash := randomFakeHash() + backend := NewMockBackend() + backend.On( + "RunAndCollectRootHashes", + mcycleComputationHashChunkSize, + mock.AnythingOfType("*machine.HashCollectorState"), + mock.AnythingOfType("time.Duration"), + ).Run(func(args mock.Arguments) { + state := args.Get(1).(*HashCollectorState) + s.Zero(state.MCyclePhase) + s.Empty(state.Hashes) + state.Hashes = append(state.Hashes, firstPeriodicHash) + state.MCyclePhase = 7 + }).Return(ReachedTargetMcycle, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(mcycleComputationHashChunkSize, nil).Once() + backend.On( + "RunAndCollectRootHashes", + 2*mcycleComputationHashChunkSize, + mock.AnythingOfType("*machine.HashCollectorState"), + mock.AnythingOfType("time.Duration"), + ).Run(func(args mock.Arguments) { + state := args.Get(1).(*HashCollectorState) + s.Equal(uint64(7), state.MCyclePhase) + s.Equal([]Hash{firstPeriodicHash}, state.Hashes) + state.Hashes = append(state.Hashes, terminalHash) + state.MCyclePhase = 11 + }).Return(YieldedManually, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(2*mcycleComputationHashChunkSize, nil).Once() machine := &machineImpl{ - backend: mockBackend, + backend: backend, logger: s.logger, params: model.ExecutionParameters{ - FastDeadline: time.Second, - InspectIncCycles: inspectIncCycles, - InspectIncDeadline: time.Second, - InspectMaxDeadline: time.Second, + AdvanceIncCycles: 3 * mcycleComputationHashChunkSize, + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: time.Second, }, } - response, err := machine.Inspect(context.Background(), []byte("query")) + result, err := machine.run( + context.Background(), AdvanceStateRequest, true, + testExecutionBounds(0, 3*mcycleComputationHashChunkSize), + ) + s.Require().NoError(err) - s.Require().Equal(CompletionStatusAccepted, response.Status) - mockBackend.AssertExpectations(s.T()) + s.Equal([]Hash{firstPeriodicHash}, result.periodicStateHashes) + s.Equal(InputEntryCapacity-1, result.paddingRepetitions) + backend.AssertExpectations(s.T()) +} + +func (s *ImplementationSuite) TestRunRejectsFixedPointWithoutAppendedHash() { + backend := NewMockBackend() + backend.On( + "RunAndCollectRootHashes", + MCycleComputationHashPeriod, + mock.AnythingOfType("*machine.HashCollectorState"), + mock.AnythingOfType("time.Duration"), + ).Return(YieldedManually, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(MCycleComputationHashPeriod, nil).Once() + machine := &machineImpl{ + backend: backend, + logger: s.logger, + params: model.ExecutionParameters{ + AdvanceIncCycles: MCycleComputationHashPeriod, + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: time.Second, + }, + } + + _, err := machine.run( + context.Background(), AdvanceStateRequest, true, + testExecutionBounds(0, model.MaxExecutionCycleSpan), + ) + + s.Require().ErrorIs(err, ErrMachineInternal) + s.Contains(err.Error(), "without appending its state hash") + backend.AssertExpectations(s.T()) } // Test Store method @@ -1293,16 +2087,15 @@ func (s *ImplementationSuite) TestRun() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, }, } - result, err := machine.run(ctx, AdvanceStateRequest, false, executionBounds{ - start: 0, limit: 1000, span: 1000, configured: true, - }) + result, err := machine.run( + ctx, AdvanceStateRequest, false, testExecutionBounds(0, model.MaxExecutionCycleSpan), + ) require.NoError(err) require.Empty(result.outputs) require.Empty(result.reports) @@ -1317,15 +2110,14 @@ func (s *ImplementationSuite) TestRun() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, }, } - _, err = machine2.run(ctx, AdvanceStateRequest, false, executionBounds{ - start: 0, limit: 1000, span: 1000, configured: true, - }) + _, err = machine2.run( + ctx, AdvanceStateRequest, false, testExecutionBounds(0, model.MaxExecutionCycleSpan), + ) require.Error(err) require.Contains(err.Error(), "read cycle failed") mockBackend2.AssertExpectations(s.T()) @@ -1341,16 +2133,15 @@ func (s *ImplementationSuite) TestRun() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, }, } - _, err = machine3.run(ctx, AdvanceStateRequest, false, executionBounds{ - start: 0, limit: 1000, span: 1000, configured: true, - }) + _, err = machine3.run( + ctx, AdvanceStateRequest, false, testExecutionBounds(0, model.MaxExecutionCycleSpan), + ) require.NoError(err) mockBackend3.AssertExpectations(s.T()) @@ -1366,16 +2157,15 @@ func (s *ImplementationSuite) TestRun() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, }, } - _, err = machine4.run(ctx, AdvanceStateRequest, false, executionBounds{ - start: 0, limit: 1000, span: 1000, configured: true, - }) + _, err = machine4.run( + ctx, AdvanceStateRequest, false, testExecutionBounds(0, model.MaxExecutionCycleSpan), + ) require.Error(err) require.Contains(err.Error(), "could not read output/report") require.Contains(err.Error(), "cmio request failed") @@ -1396,16 +2186,15 @@ func (s *ImplementationSuite) TestRun() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, }, } - result5, err := machine5.run(ctx, AdvanceStateRequest, false, executionBounds{ - start: 0, limit: 1000, span: 1000, configured: true, - }) + result5, err := machine5.run( + ctx, AdvanceStateRequest, false, testExecutionBounds(0, model.MaxExecutionCycleSpan), + ) require.NoError(err) require.Len(result5.outputs, 1) require.Equal([]byte("output data"), []byte(result5.outputs[0])) @@ -1427,16 +2216,15 @@ func (s *ImplementationSuite) TestRun() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, }, } - result6, err := machine6.run(ctx, AdvanceStateRequest, false, executionBounds{ - start: 0, limit: 1000, span: 1000, configured: true, - }) + result6, err := machine6.run( + ctx, AdvanceStateRequest, false, testExecutionBounds(0, model.MaxExecutionCycleSpan), + ) require.NoError(err) require.Empty(result6.outputs) require.Len(result6.reports, 1) @@ -1447,34 +2235,39 @@ func (s *ImplementationSuite) TestRun() { func (s *ImplementationSuite) TestRunIncrementIntervalPreservesBreakReason() { for _, test := range []struct { - name string - breakReason BreakReason - cycle uint64 + name string + breakReason BreakReason + currentCycle uint64 }{ - {"manual", YieldedManually, 150}, - {"automatic", YieldedAutomatically, 200}, - {"soft", YieldedSoftly, 150}, + {"manual yield", YieldedManually, 150}, + {"automatic yield", YieldedAutomatically, 200}, + {"soft yield", YieldedSoftly, 150}, {"target", ReachedTargetMcycle, 200}, - {"overflow", McycleOverflow, 199}, + {"mcycle overflow", McycleOverflow, 199}, {"halt", Halted, 175}, - {"failed", Failed, 160}, + {"failure", Failed, 160}, } { s.Run(test.name, func() { backend := NewMockBackend() - backend.On("Run", uint64(200), mock.AnythingOfType("time.Duration")).Return(test.breakReason, nil) - backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(test.cycle, nil) + backend.On("Run", uint64(200), mock.AnythingOfType("time.Duration")). + Return(test.breakReason, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(test.currentCycle, nil).Once() machine := &machineImpl{backend: backend, logger: s.logger} result, err := machine.runIncrementInterval( context.Background(), 100, 1000, 100, nil, time.Second, ) + s.Require().NoError(err) s.Equal(test.breakReason, result.breakReason) - s.Equal(test.cycle, result.currentCycle) + s.Equal(test.currentCycle, result.currentCycle) backend.AssertExpectations(s.T()) }) } +} +func (s *ImplementationSuite) TestRunIncrementIntervalErrors() { s.Run("already at limit", func() { machine := &machineImpl{backend: NewMockBackend(), logger: s.logger} result, err := machine.runIncrementInterval( @@ -1483,6 +2276,47 @@ func (s *ImplementationSuite) TestRunIncrementIntervalPreservesBreakReason() { s.Require().ErrorIs(err, ErrReachedLimitMcycle) s.Equal(uint64(1000), result.currentCycle) }) + + s.Run("backend error", func() { + backend := NewMockBackend() + backend.On("Run", uint64(200), mock.AnythingOfType("time.Duration")). + Return(Failed, errors.New("run failed")).Once() + machine := &machineImpl{backend: backend, logger: s.logger} + _, err := machine.runIncrementInterval( + context.Background(), 100, 1000, 100, nil, time.Second, + ) + s.Require().ErrorContains(err, "run failed") + backend.AssertExpectations(s.T()) + }) + + s.Run("read cycle error", func() { + backend := NewMockBackend() + backend.On("Run", uint64(200), mock.AnythingOfType("time.Duration")). + Return(YieldedManually, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(uint64(0), errors.New("read cycle failed")).Once() + machine := &machineImpl{backend: backend, logger: s.logger} + _, err := machine.runIncrementInterval( + context.Background(), 100, 1000, 100, nil, time.Second, + ) + s.Require().ErrorContains(err, "read cycle failed") + backend.AssertExpectations(s.T()) + }) + + s.Run("unknown break reason", func() { + backend := NewMockBackend() + backend.On("Run", uint64(200), mock.AnythingOfType("time.Duration")). + Return(BreakReason(99), nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(uint64(150), nil).Once() + machine := &machineImpl{backend: backend, logger: s.logger} + result, err := machine.runIncrementInterval( + context.Background(), 100, 1000, 100, nil, time.Second, + ) + s.Require().ErrorIs(err, ErrMachineInternal) + s.Equal(BreakReason(99), result.breakReason) + backend.AssertExpectations(s.T()) + }) } // Test process method @@ -1505,7 +2339,6 @@ func (s *ImplementationSuite) TestProcess() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, @@ -1562,7 +2395,6 @@ func (s *ImplementationSuite) TestProcess() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, @@ -1570,11 +2402,15 @@ func (s *ImplementationSuite) TestProcess() { } _, err = machine2.process(ctx, input, AdvanceStateRequest, &expectedHash, false) require.ErrorIs(err, ErrPayloadLengthLimitExceeded) + require.Contains(err.Error(), "advance request payload length 10") + require.Contains(err.Error(), "capacity 5") + require.NotContains(err.Error(), string(input)) mockBackend2.AssertExpectations(s.T()) // Test process with send error mockBackend3 := NewMockBackend() mockBackend3.On("CmioRxBufferSize").Return(uint64(1024)) + mockBackend3.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(0), nil) mockBackend3.On("SendCmioResponse", mock.AnythingOfType("uint16"), mock.Anything, @@ -1586,13 +2422,11 @@ func (s *ImplementationSuite) TestProcess() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, }, } - mockBackend3.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(0), nil) _, err = machine3.process(ctx, input, AdvanceStateRequest, &expectedHash, false) require.Error(err) require.Contains(err.Error(), "send failed") @@ -1607,7 +2441,6 @@ func (s *ImplementationSuite) TestProcess() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, @@ -1620,84 +2453,6 @@ func (s *ImplementationSuite) TestProcess() { mockBackend4.AssertExpectations(s.T()) } -// Test run method with automatic yields and outputs/reports -func (s *ImplementationSuite) TestRunWithAutomaticYields() { - require := s.Require() - ctx := context.Background() - - mockBackend := NewMockBackend() - machine := &machineImpl{ - backend: mockBackend, - logger: s.logger, - params: model.ExecutionParameters{ - FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, - AdvanceIncCycles: 100, - AdvanceIncDeadline: time.Second * 1, - AdvanceMaxDeadline: time.Second * 10, - }, - } - - // Setup for automatic yield with output - mockBackend.On("Run", uint64(100), mock.AnythingOfType("time.Duration")).Return(YieldedAutomatically, nil).Once() - mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(50), nil).Once() - mockBackend.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( - uint8(0), uint16(AutomaticYieldReasonOutput), []byte("test output"), nil).Once() - - // Setup for manual yield after automatic yield - mockBackend.On("Run", uint64(150), mock.AnythingOfType("time.Duration")).Return(YieldedManually, nil).Once() - mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(100), nil).Once() - - result, err := machine.run(ctx, AdvanceStateRequest, false, executionBounds{ - start: 0, limit: 1000, span: 1000, configured: true, - }) - require.NoError(err) - require.Len(result.outputs, 1) - require.Equal([]byte("test output"), result.outputs[0]) - require.Empty(result.reports) - - mockBackend.AssertExpectations(s.T()) -} - -// Test run method with automatic yields and reports -func (s *ImplementationSuite) TestRunWithAutomaticYieldsReports() { - require := s.Require() - ctx := context.Background() - - mockBackend := NewMockBackend() - machine := &machineImpl{ - backend: mockBackend, - logger: s.logger, - params: model.ExecutionParameters{ - FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, - AdvanceIncCycles: 100, - AdvanceIncDeadline: time.Second * 1, - AdvanceMaxDeadline: time.Second * 10, - }, - } - - // Setup for automatic yield with report - mockBackend.On("Run", uint64(100), mock.AnythingOfType("time.Duration")).Return(YieldedAutomatically, nil).Once() - mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(50), nil).Once() - mockBackend.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( - uint8(0), uint16(AutomaticYieldReasonReport), []byte("test report"), nil).Once() - - // Setup for manual yield after automatic yield - mockBackend.On("Run", uint64(150), mock.AnythingOfType("time.Duration")).Return(YieldedManually, nil).Once() - mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(100), nil).Once() - - result, err := machine.run(ctx, AdvanceStateRequest, false, executionBounds{ - start: 0, limit: 1000, span: 1000, configured: true, - }) - require.NoError(err) - require.Empty(result.outputs) - require.Len(result.reports, 1) - require.Equal([]byte("test report"), result.reports[0]) - - mockBackend.AssertExpectations(s.T()) -} - // Test multiple automatic yields in sequence func (s *ImplementationSuite) TestMultipleAutomaticYields() { require := s.Require() @@ -1709,7 +2464,6 @@ func (s *ImplementationSuite) TestMultipleAutomaticYields() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, @@ -1751,9 +2505,9 @@ func (s *ImplementationSuite) TestMultipleAutomaticYields() { mockBackend.On("Run", uint64(150), mock.AnythingOfType("time.Duration")).Return(YieldedManually, nil).Once() mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(60), nil).Once() - result, err := machine.run(ctx, AdvanceStateRequest, false, executionBounds{ - start: 0, limit: 1000, span: 1000, configured: true, - }) + result, err := machine.run( + ctx, AdvanceStateRequest, false, testExecutionBounds(0, model.MaxExecutionCycleSpan), + ) require.NoError(err) require.Len(result.outputs, 2) @@ -1793,3 +2547,52 @@ func (s *ImplementationSuite) TestCheckContext() { err = checkContext(nil) // nolint require.NoError(err) // nil context is valid in Go } + +func (s *ImplementationSuite) TestInputHashCollectionSpan() { + require := s.Require() + + for _, test := range []struct { + name string + hashes uint64 + padding uint64 + wantErr bool + }{ + {name: "empty collection", hashes: 0, padding: InputEntryCapacity}, + {name: "partial collection", hashes: 2, padding: InputEntryCapacity - 2}, + {name: "exact boundary", hashes: InputEntryCapacity, padding: 0}, + {name: "short span", hashes: 2, padding: InputEntryCapacity - 3, wantErr: true}, + {name: "long span", hashes: 2, padding: InputEntryCapacity - 1, wantErr: true}, + {name: "collector overflow", hashes: InputEntryCapacity + 1, wantErr: true}, + } { + s.Run(test.name, func() { + err := ValidateInputHashCollectionSpan(test.hashes, test.padding) + require.Equal(test.wantErr, err != nil) + }) + } + + padding, err := inputHashCollectionPaddingRepetitions(InputEntryCapacity) + require.NoError(err) + require.Zero(padding) + _, err = inputHashCollectionPaddingRepetitions(InputEntryCapacity + 1) + require.ErrorContains(err, "exceeds input entry capacity") + + canonicalCount, padding, err := canonicalInputHashCollectionShape( + InputEntryCapacity, + true, + ) + require.NoError(err) + require.Equal(InputEntryCapacity-1, canonicalCount) + require.Equal(uint64(1), padding) + + // Validate the raw collector count before removing the terminal sample. + // Otherwise S+1 would be incorrectly normalized into a valid S shape. + _, _, err = canonicalInputHashCollectionShape(InputEntryCapacity+1, true) + require.ErrorContains(err, "exceeds input entry capacity") + + // A malformed collection is an internal failure even when execution also + // reached a completed terminal state. The terminal status is context only. + err = invalidInputHashCollectionSpanError(ErrHalted, errors.New("overcollection")) + require.ErrorIs(err, ErrMachineInternal) + require.NotErrorIs(err, ErrHalted) + require.ErrorContains(err, ErrHalted.Error()) +} diff --git a/pkg/machine/libcartesi.go b/pkg/machine/libcartesi.go index 2414aa7a7..8044f013c 100644 --- a/pkg/machine/libcartesi.go +++ b/pkg/machine/libcartesi.go @@ -103,6 +103,11 @@ func (p *proofJson) UnmarshalJSON(data []byte) error { } func NewLibCartesiBackend(address string, timeout time.Duration) (Backend, string, uint32, error) { + // Keep this defensive check even though the advancer validates the constants + // at startup. Other callers can construct a backend directly. + if err := ValidateEmulatorComputationHashLimits(); err != nil { + return nil, "", 0, err + } rm, address, pid, err := emulator.SpawnServer(address, timeout) if err != nil { return nil, address, pid, err @@ -110,6 +115,53 @@ func NewLibCartesiBackend(address string, timeout time.Duration) (Backend, strin return &LibCartesiBackend{inner: rm}, address, pid, nil } +// ValidateEmulatorComputationHashLimits verifies that the node and compiled +// emulator agree on the three exported rollup limits checked here. A mismatch +// would change per-input or per-epoch computation-hash dimensions. This does +// not validate the selected sampling period, bundle exponent, runtime library, +// or hash algorithm. +func ValidateEmulatorComputationHashLimits() error { + checks := []struct { + name string + node uint64 + emulator uint64 + }{ + { + name: "LOG2_MAX_UARCH_CYCLES_PER_MCYCLE", + node: Log2MaxUarchCyclesPerMCycle, + emulator: emulator.Log2MaxUarchCyclesPerMCycle, + }, + { + name: "LOG2_MAX_MCYCLES_PER_ADVANCE_STATE", + node: Log2MaxMCyclesPerAdvanceState, + emulator: emulator.Log2MaxMCyclesPerAdvanceState, + }, + { + name: "LOG2_MAX_ADVANCE_STATES_PER_EPOCH", + node: Log2MaxAdvanceStatesPerEpoch, + emulator: emulator.Log2MaxAdvanceStatesPerEpoch, + }, + } + for _, check := range checks { + if err := validateEmulatorComputationHashLimit(check.name, check.node, check.emulator); err != nil { + return err + } + } + return nil +} + +// validateEmulatorComputationHashLimit takes explicit values so the mismatch +// path can be tested without requiring a differently compiled emulator library. +func validateEmulatorComputationHashLimit(name string, nodeLog2, emulatorLog2 uint64) error { + if nodeLog2 == emulatorLog2 { + return nil + } + return fmt.Errorf( + "node computation-hash dimension 2^%d does not match emulator CM_ROLLUP_%s=%d: %w", + nodeLog2, name, emulatorLog2, ErrMachineInternal, + ) +} + // LibCartesiBackend is an adapter that implements Backend by wrapping a RemoteMachineInterface. type LibCartesiBackend struct { inner RemoteMachineInterface @@ -268,15 +320,20 @@ func (e *LibCartesiBackend) RunAndCollectRootHashes( if state == nil { return Failed, errors.New("nil state") } - if state.Period == 0 { + if state.MCycleSamplingPeriod == 0 { return Failed, errors.New("state period must be greater than zero") } - log2Period := uint64(bits.Len64(state.Period) - 1) - if uint64(1)<= state.Period { - return Failed, fmt.Errorf("phase must be less than period, got phase %v and period %v", state.Phase, state.Period) + if state.MCyclePhase >= state.MCycleSamplingPeriod { + return Failed, fmt.Errorf( + "phase must be less than period, got phase %v and period %v", + state.MCyclePhase, + state.MCycleSamplingPeriod, + ) } if err := e.inner.SetTimeout(timeout.Milliseconds()); err != nil { return Failed, fmt.Errorf("failed to set operation timeout: %w", err) @@ -285,18 +342,19 @@ func (e *LibCartesiBackend) RunAndCollectRootHashes( rawResult, err := e.inner.CollectMCycleRootHashes( mcycleEnd, log2Period, - state.Phase, - state.BundleLog2, + state.MCyclePhase, + state.Log2BundleMCycleCount, state.PartialBundle, ) if err != nil { return Failed, err } result := struct { - RootHashes []string `json:"hashes"` - MCyclePhase uint64 `json:"mcycle_phase"` - BreakReason string `json:"break_reason"` - PartialBundle json.RawMessage `json:"partial_bundle,omitempty"` + RootHashes []string `json:"hashes"` + MCyclePhase uint64 `json:"mcycle_phase"` + BreakReason string `json:"break_reason"` + PartialBundle json.RawMessage `json:"partial_bundle,omitempty"` + ConsoleIOError string `json:"console_io_error,omitempty"` }{} err = json.Unmarshal(rawResult, &result) if err != nil { @@ -306,11 +364,11 @@ func (e *LibCartesiBackend) RunAndCollectRootHashes( if err != nil { return Failed, fmt.Errorf("invalid CollectMCycleRootHashes result: %w", err) } - if result.MCyclePhase >= state.Period { + if result.MCyclePhase >= state.MCycleSamplingPeriod { return Failed, fmt.Errorf( "invalid CollectMCycleRootHashes result: phase %v must be less than period %v", result.MCyclePhase, - state.Period, + state.MCycleSamplingPeriod, ) } @@ -322,8 +380,9 @@ func (e *LibCartesiBackend) RunAndCollectRootHashes( } state.Hashes = append(state.Hashes, decodedHashes...) - state.Phase = result.MCyclePhase + state.MCyclePhase = result.MCyclePhase state.PartialBundle = result.PartialBundle + state.ConsoleIOError = result.ConsoleIOError return reason, nil } diff --git a/pkg/machine/libcartesi_test.go b/pkg/machine/libcartesi_test.go index 82819384a..aafd798f1 100644 --- a/pkg/machine/libcartesi_test.go +++ b/pkg/machine/libcartesi_test.go @@ -12,6 +12,7 @@ import ( "github.com/cartesi/rollups-node/pkg/emulator" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) @@ -19,6 +20,16 @@ func TestLibCartesi(t *testing.T) { suite.Run(t, new(LibCartesiSuite)) } +func TestValidateEmulatorComputationHashLimits(t *testing.T) { + require.NoError(t, ValidateEmulatorComputationHashLimits()) + require.NoError(t, validateEmulatorComputationHashLimit("LOG2_MAX_MCYCLES_PER_ADVANCE_STATE", 48, 48)) + + err := validateEmulatorComputationHashLimit("LOG2_MAX_MCYCLES_PER_ADVANCE_STATE", 48, 47) + require.ErrorIs(t, err, ErrMachineInternal) + require.ErrorContains(t, err, "node computation-hash dimension 2^48") + require.ErrorContains(t, err, "CM_ROLLUP_LOG2_MAX_MCYCLES_PER_ADVANCE_STATE=47") +} + type LibCartesiSuite struct { suite.Suite mockRemoteMachine *MockRemoteMachine @@ -111,7 +122,7 @@ func (s *LibCartesiSuite) TestRunMcycleOverflow() { func (s *LibCartesiSuite) TestRunAndCollectRootHashesMcycleOverflow() { require := s.Require() - state := &HashCollectorState{Period: 1} + state := &HashCollectorState{MCycleSamplingPeriod: 1} result := []byte(`{"hashes":[],"mcycle_phase":0,"break_reason":"mcycle_overflow"}`) s.mockRemoteMachine.On("SetTimeout", int64(5000)).Return(nil) @@ -142,22 +153,24 @@ func (s *LibCartesiSuite) TestRunAndCollectRootHashesContinuesPartialBundle() { previousHash := Hash{0x11} collectedHash := Hash{0x22} state := &HashCollectorState{ - Period: 4, - Phase: 3, - BundleLog2: 2, - Hashes: []Hash{previousHash}, - PartialBundle: previousPartialBundle, + MCycleSamplingPeriod: 4, + MCyclePhase: 3, + Log2BundleMCycleCount: 2, + Hashes: []Hash{previousHash}, + PartialBundle: previousPartialBundle, } result, err := json.Marshal(struct { - Hashes []string `json:"hashes"` - MCyclePhase uint64 `json:"mcycle_phase"` - BreakReason string `json:"break_reason"` - PartialBundle json.RawMessage `json:"partial_bundle"` + Hashes []string `json:"hashes"` + MCyclePhase uint64 `json:"mcycle_phase"` + BreakReason string `json:"break_reason"` + PartialBundle json.RawMessage `json:"partial_bundle"` + ConsoleIOError string `json:"console_io_error"` }{ - Hashes: []string{base64.StdEncoding.EncodeToString(collectedHash[:])}, - MCyclePhase: 1, - BreakReason: "reached_target_mcycle", - PartialBundle: nextPartialBundle, + Hashes: []string{base64.StdEncoding.EncodeToString(collectedHash[:])}, + MCyclePhase: 1, + BreakReason: "reached_target_mcycle", + PartialBundle: nextPartialBundle, + ConsoleIOError: "console write failed", }) require.NoError(err) @@ -174,9 +187,10 @@ func (s *LibCartesiSuite) TestRunAndCollectRootHashesContinuesPartialBundle() { breakReason, err := s.backend.RunAndCollectRootHashes(1000, state, 5*time.Second) require.NoError(err) require.Equal(ReachedTargetMcycle, breakReason) - require.Equal(uint64(1), state.Phase) + require.Equal(uint64(1), state.MCyclePhase) require.Equal([]Hash{previousHash, collectedHash}, state.Hashes) require.Equal(nextPartialBundle, state.PartialBundle) + require.Equal("console write failed", state.ConsoleIOError) s.mockRemoteMachine.AssertExpectations(s.T()) } @@ -186,10 +200,10 @@ func (s *LibCartesiSuite) TestRunAndCollectRootHashesClearsPartialBundleAtFixedP `{"log2_max_leaves":2,"hash_function":"keccak256","leaf_count":3,"context":["AwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="]}`, ) state := &HashCollectorState{ - Period: 4, - Phase: 2, - BundleLog2: 2, - PartialBundle: previousPartialBundle, + MCycleSamplingPeriod: 4, + MCyclePhase: 2, + Log2BundleMCycleCount: 2, + PartialBundle: previousPartialBundle, } result := []byte(`{"hashes":[],"mcycle_phase":2,"break_reason":"halted"}`) @@ -221,11 +235,11 @@ func (s *LibCartesiSuite) TestRunAndCollectRootHashesInvalidHashLeavesStateUncha previousHash := Hash{0x11} validHash := Hash{0x22} state := &HashCollectorState{ - Period: 4, - Phase: 3, - BundleLog2: 2, - Hashes: []Hash{previousHash}, - PartialBundle: previousPartialBundle, + MCycleSamplingPeriod: 4, + MCyclePhase: 3, + Log2BundleMCycleCount: 2, + Hashes: []Hash{previousHash}, + PartialBundle: previousPartialBundle, } originalState := *state originalState.Hashes = append([]Hash(nil), state.Hashes...) @@ -267,9 +281,9 @@ func (s *LibCartesiSuite) TestRunAndCollectRootHashesInvalidBreakReasonLeavesSta require := s.Require() previousHash := Hash{0x11} state := &HashCollectorState{ - Period: 4, - Phase: 3, - Hashes: []Hash{previousHash}, + MCycleSamplingPeriod: 4, + MCyclePhase: 3, + Hashes: []Hash{previousHash}, } originalState := *state originalState.Hashes = append([]Hash(nil), state.Hashes...) @@ -294,7 +308,7 @@ func (s *LibCartesiSuite) TestRunAndCollectRootHashesInvalidBreakReasonLeavesSta func (s *LibCartesiSuite) TestRunAndCollectRootHashesInvalidResultPhaseLeavesStateUnchanged() { require := s.Require() - state := &HashCollectorState{Period: 4, Phase: 3} + state := &HashCollectorState{MCycleSamplingPeriod: 4, MCyclePhase: 3} originalState := *state result := []byte(`{"hashes":[],"mcycle_phase":4,"break_reason":"reached_target_mcycle"}`) @@ -317,7 +331,7 @@ func (s *LibCartesiSuite) TestRunAndCollectRootHashesInvalidResultPhaseLeavesSta func (s *LibCartesiSuite) TestRunAndCollectRootHashesRejectsInvalidInputPhase() { require := s.Require() - state := &HashCollectorState{Period: 4, Phase: 4} + state := &HashCollectorState{MCycleSamplingPeriod: 4, MCyclePhase: 4} breakReason, err := s.backend.RunAndCollectRootHashes(1000, state, 5*time.Second) require.ErrorContains(err, "phase must be less than period") diff --git a/pkg/machine/machine.go b/pkg/machine/machine.go index c0e2956b9..6dc582c8b 100644 --- a/pkg/machine/machine.go +++ b/pkg/machine/machine.go @@ -1,8 +1,6 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -// Package machine provides a unified interface for interacting with Cartesi machines. -// It consolidates functionality from the previous rollupsmachine and cartesimachine packages. package machine import ( @@ -27,11 +25,16 @@ type ( Hash = [HashSize]byte ) -// CompletionStatus identifies how a guest-machine request completed. If -// execution does not complete, the request returns an error instead. +// CompletionStatus identifies how a guest-machine request completed. Advance +// and Inspect have the same completion outcomes; their callers decide whether +// and how those outcomes affect canonical state. If execution does not +// complete, the operation returns CompletionStatusUnknown together with an +// error instead. type CompletionStatus uint8 const ( + // CompletionStatusUnknown is the zero-value sentinel. A successful request + // never returns it. CompletionStatusUnknown CompletionStatus = iota CompletionStatusAccepted CompletionStatusRejected @@ -47,6 +50,8 @@ func (s CompletionStatus) IsCompleted() bool { CompletionStatusException, CompletionStatusHalted: return true + case CompletionStatusUnknown: + return false default: return false } @@ -54,18 +59,25 @@ func (s CompletionStatus) IsCompleted() bool { // AdvanceResponse contains the result of a completed advance operation. type AdvanceResponse struct { - Status CompletionStatus - Outputs []Output - Reports []Report - ExceptionData []byte - Hashes []Hash - RemainingCycles uint64 - OutputsHash Hash + Status CompletionStatus + Outputs []Output + Reports []Report + // ExceptionData is the raw CMIO payload supplied by the guest when Status + // is CompletionStatusException. It is nil for every other status; an + // exception with an empty payload is represented by a non-nil empty slice. + ExceptionData []byte + PeriodicStateHashes []Hash + PaddingRepetitions uint64 + OutputsHash Hash } -// InspectResponse contains the result of a completed inspect operation. +// InspectResponse contains the result of an inspect operation. On incomplete +// execution, Reports contains any reports emitted before the failure, Status +// is CompletionStatusUnknown, and Inspect returns a non-nil error. type InspectResponse struct { - Status CompletionStatus + Status CompletionStatus + // ExceptionData has the same status-dependent meaning as + // AdvanceResponse.ExceptionData. ExceptionData []byte Reports []Report } @@ -87,13 +99,14 @@ var ( ErrReachedLimitMcycle = errors.New("machine reached limit mcycle") // ErrMcycleOverflow preserves the emulator-reported fact that the machine - // itself reached imcyclemax. It remains distinguishable from a node target - // because canonical overflow eligibility depends on the stop origin. + // itself reached imcyclemax, rather than a node target. Canonical overflow + // eligibility depends on that stop origin. It wraps ErrReachedLimitMcycle so + // existing execution-limit classification remains unchanged. ErrMcycleOverflow = fmt.Errorf("machine reached imcyclemax: %w", ErrReachedLimitMcycle) ) // IsExecutionLimitError reports whether err is an incomplete execution caused -// by a payload, response-count, or cycle ceiling. +// by a local payload, response-count, or cycle ceiling. func IsExecutionLimitError(err error) bool { return errors.Is(err, ErrPayloadLengthLimitExceeded) || errors.Is(err, ErrOutputsLimitExceeded) || @@ -117,11 +130,17 @@ type Machine interface { // Advance sends an input to the machine. // The checkpointHash is the machine's root hash before processing the input, // sent along with the request so the machine can revert to it if needed. - // A non-nil response and nil error mean execution completed with a typed - // status. Incomplete execution returns a nil response and a non-nil error. + // A non-nil response and nil error mean the machine completed with one of + // the four completed CompletionStatus values. Any incomplete execution—input + // validation, an operational limit, deadline/cancellation, or infrastructure + // failure—returns a nil response and a non-nil error. CompletionStatusUnknown + // is never returned by a successful call. Advance(ctx context.Context, input []byte, checkpointHash Hash, computeHashes bool) (*AdvanceResponse, error) - // Inspect sends a query to the machine and returns its typed completion. + // Inspect sends a query to the machine. A nil error means the guest completed + // with one of the four non-unknown CompletionStatus values. On incomplete + // execution, the response preserves reports emitted before the failure and + // the error identifies why inspection could not complete. Inspect(ctx context.Context, query []byte) (*InspectResponse, error) // Store saves the machine state to the specified path. diff --git a/pkg/machine/machine_test.go b/pkg/machine/machine_test.go index f632c3fdb..7f002a1af 100644 --- a/pkg/machine/machine_test.go +++ b/pkg/machine/machine_test.go @@ -6,12 +6,14 @@ package machine import ( "context" "errors" + "fmt" "io" "log/slog" "testing" "time" "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/pkg/emulator" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" ) @@ -199,27 +201,43 @@ func (s *MachineSuite) TestDefaultConfig() { } func (s *MachineSuite) TestConfiguredHardCycleCeilingMatchesMachineInputSpan() { - s.Require().Equal(BarchSpanToInput, model.MaxExecutionCycleSpan) - s.Require().Equal(uint64(1)< Date: Wed, 12 Aug 2026 17:50:15 -0300 Subject: [PATCH 04/13] fix(advancer): preserve machine-database alignment after failures --- internal/advancer/advancer.go | 49 +- internal/advancer/advancer_test.go | 219 +++++- internal/advancer/determinism_test.go | 947 ++++++++++++++++++++++++++ 3 files changed, 1183 insertions(+), 32 deletions(-) create mode 100644 internal/advancer/determinism_test.go diff --git a/internal/advancer/advancer.go b/internal/advancer/advancer.go index f3bb375c1..0599754f6 100644 --- a/internal/advancer/advancer.go +++ b/internal/advancer/advancer.go @@ -248,26 +248,24 @@ func (s *Service) processInputs(ctx context.Context, app *Application, inputs [] result, err := machine.Advance(ctx, input.RawData, input.EpochIndex, input.Index, app.IsDaveConsensus()) input.RawData = nil // allow GC to collect payload while batch continues if err != nil { - // Graceful shutdown: bail out quietly without marking FAILED. - if errors.Is(err, context.Canceled) { - s.Logger.Debug("Advance cancelled due to shutdown", + // Cancellation of this service context is the normal shutdown path, + // so it does not change the application's status. A returned error + // that happens to include context.Canceled is still a failure while + // the service context remains active. + if errors.Is(ctx.Err(), context.Canceled) { + s.Logger.Debug("Advance stopped because the service is shutting down", "application", app.Name, - "index", input.Index) + "index", input.Index, + "error", err) return err } - // Anything else (including DeadlineExceeded) is a real failure. + // Anything else, including a deadline, is an execution failure. s.Logger.Error("Error executing advance", "application", app.Name, "index", input.Index, "error", err) - // DeadlineExceeded is a real failure but not a state-corruption - // signal — let the upper layer retry rather than marking FAILED. - if errors.Is(err, context.DeadlineExceeded) { - return err - } - if dbErr := appstatus.SetFailed(ctx, s.Logger, s.repository, app, err.Error()); dbErr != nil { s.Logger.Error("Failed to persist FAILED status — machine will be closed "+ "but the app status remains unchanged in DB; it may be re-created "+ @@ -304,19 +302,34 @@ func (s *Service) processInputs(ctx context.Context, app *Application, inputs [] // Store the result in the database err = s.repository.StoreAdvanceResult(ctx, input.EpochApplicationID, result) if err != nil { - // Machine state is now ahead of the database. This desync is - // unrecoverable without a restart — regardless of whether the - // failure was a DB error or a context timeout. Shut down the - // node so it can restart cleanly from the last snapshot. + // Advance has already changed the live machine, but the transaction + // did not confirm that its result was saved. The database may still + // show this input as pending. Reusing this machine could then execute + // the input again from the wrong state, so StoreAdvanceResult is not + // retried against this live machine. s.Logger.Error( - "FATAL: failed to store advance result after machine state "+ - "was already updated — shutting down to prevent permanent desync", + "Could not confirm that the advance result was saved; "+ + "the live machine has already advanced, so services will stop; "+ + "after the node is restarted, execution will use persisted state", "application", app.Name, "epoch", input.EpochIndex, "index", input.Index, "error", err) + + // Try to close the machine now so the already-advanced runtime cannot + // be used again. Cancel services even if Close fails. After the node + // is restarted, the machine is rebuilt from persisted state, and the + // database decides whether this input is still pending and needs a + // safe retry. + closeErr := machine.Close() s.Cancel() // triggers graceful shutdown of all services - return err + if closeErr != nil { + s.Logger.Error("Could not close the machine after its advance result "+ + "was not confirmed saved; service shutdown is still required", + "application", app.Name, + "error", closeErr) + } + return errors.Join(err, closeErr) } // Create a snapshot if needed diff --git a/internal/advancer/advancer_test.go b/internal/advancer/advancer_test.go index 0f8c9aa7c..b1ac371d6 100644 --- a/internal/advancer/advancer_test.go +++ b/internal/advancer/advancer_test.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "io/fs" + "log/slog" mrand "math/rand" "os" "path/filepath" @@ -22,6 +23,7 @@ import ( . "github.com/cartesi/rollups-node/internal/model" "github.com/cartesi/rollups-node/internal/repository" "github.com/cartesi/rollups-node/internal/repository/repotest" + pkgmachine "github.com/cartesi/rollups-node/pkg/machine" "github.com/cartesi/rollups-node/pkg/service" "github.com/ethereum/go-ethereum/common" @@ -34,6 +36,32 @@ func TestAdvancer(t *testing.T) { type AdvancerSuite struct{ suite.Suite } +type advancerLogCapture struct { + mu sync.Mutex + records []slog.Record +} + +func (h *advancerLogCapture) Enabled(context.Context, slog.Level) bool { return true } +func (h *advancerLogCapture) Handle(_ context.Context, record slog.Record) error { + h.mu.Lock() + defer h.mu.Unlock() + h.records = append(h.records, record.Clone()) + return nil +} +func (h *advancerLogCapture) WithAttrs([]slog.Attr) slog.Handler { return h } +func (h *advancerLogCapture) WithGroup(string) slog.Handler { return h } + +func (h *advancerLogCapture) contains(level slog.Level, message string) bool { + h.mu.Lock() + defer h.mu.Unlock() + for _, record := range h.records { + if record.Level == level && record.Message == message { + return true + } + } + return false +} + func newMockAdvancerService(machineManager *MockMachineManager, repo *MockRepository) (*Service, error) { return newMockAdvancerServiceWithBatchSize(machineManager, repo, 500) } @@ -336,6 +364,37 @@ func (s *AdvancerSuite) TestProcess() { require.Contains(err.Error(), "advance error") }) + s.Run("IncompleteAdvanceMarksFailedWithoutCanonicalResult", func() { + interruptions := []struct { + name string + err error + }{ + {"InternalWatchdog", pkgmachine.ErrDeadlineExceeded}, + {"OutputLimit", pkgmachine.ErrOutputsLimitExceeded}, + {"ReportLimit", pkgmachine.ErrReportsLimitExceeded}, + {"PayloadLimit", pkgmachine.ErrPayloadLengthLimitExceeded}, + {"CycleLimit", pkgmachine.ErrReachedLimitMcycle}, + } + + for _, interruption := range interruptions { + s.Run(interruption.name, func() { + require := s.Require() + env := s.setupOneApp() + env.app.AdvanceError = interruption.err + inputs := []*Input{ + newInput(env.app.Application.ID, 0, 0, []byte("input")), + } + + err := env.service.processInputs(context.Background(), env.app.Application, inputs) + require.ErrorIs(err, interruption.err) + require.Empty(env.repo.StoredResults) + require.Equal(1, env.repo.ApplicationStatusUpdates) + require.Equal(ApplicationStatus_Failed, env.repo.LastApplicationStatus) + require.Equal(1, env.mm.Map[env.app.Application.ID].closeCalls) + }) + } + }) + s.Run("Ok", func() { require := s.Require() env := s.setupOneApp() @@ -394,20 +453,61 @@ func (s *AdvancerSuite) TestProcess() { s.Run("StoreAdvance", func() { require := s.Require() env := s.setupOneApp() + pending := newInput( + env.app.Application.ID, 0, 0, marshal(randomAdvanceResult(0)), + ) inputs := []*Input{ - newInput(env.app.Application.ID, 0, 0, marshal(randomAdvanceResult(0))), + pending, newInput(env.app.Application.ID, 0, 1, []byte("unreachable")), } + address := env.app.Application.IApplicationAddress + env.repo.GetInputsReturn = map[common.Address][]*Input{ + address: {pending}, + } env.repo.StoreAdvanceError = errors.New("store-advance error") err := env.service.processInputs(context.Background(), env.app.Application, inputs) require.Error(err) require.Contains(err.Error(), "store-advance error") - require.Len(env.repo.StoredResults, 1) + require.Empty(env.repo.StoredResults) + require.Empty(env.repo.StoredAppIDs) + require.Equal([]*Input{pending}, env.repo.GetInputsReturn[address], + "an atomic store failure must leave the input pending") // Verify that the node shutdown was triggered (context cancelled) require.Error(env.service.Context.Err(), "shared context should be cancelled") }) + + s.Run("StoreAdvanceCommitResponseLost", func() { + require := s.Require() + env := s.setupOneApp() + pending := newInput( + env.app.Application.ID, 0, 0, marshal(randomAdvanceResult(0)), + ) + address := env.app.Application.IApplicationAddress + env.repo.GetInputsReturn = map[common.Address][]*Input{ + address: {pending}, + } + env.repo.StoreAdvanceCommitError = errors.New("commit response lost") + + err := env.service.processInputs( + context.Background(), env.app.Application, []*Input{pending}, + ) + require.ErrorContains(err, "commit response lost") + require.Len(env.repo.StoredResults, 1) + require.Equal([]int64{env.app.Application.ID}, env.repo.StoredAppIDs) + require.Empty(env.repo.GetInputsReturn[address], + "a committed result and its input cursor must advance atomically") + require.Error(env.service.Context.Err(), "shared context should be cancelled") + require.Equal(1, env.mm.Map[env.app.Application.ID].closeCalls) + + unprocessed, _, listErr := getUnprocessedInputs( + context.Background(), env.repo, address.String(), pending.EpochIndex, 500, + ) + require.NoError(listErr) + require.Empty(unprocessed, + "a restart must not execute an input whose result already committed") + }) }) } @@ -436,6 +536,9 @@ func (s *AdvancerSuite) TestContextCancellation() { case err := <-errCh: require.Error(err) require.ErrorIs(err, context.Canceled) + require.Empty(env.repo.StoredResults) + require.Zero(env.repo.ApplicationStatusUpdates) + require.Zero(env.mm.Map[env.app.Application.ID].closeCalls) case <-time.After(100 * time.Millisecond): require.Fail("Step operation did not respect context cancellation") } @@ -445,6 +548,8 @@ func (s *AdvancerSuite) TestContextCancellation() { require := s.Require() env := s.setupOneApp() env.app.AdvanceBlock = true + logs := &advancerLogCapture{} + env.service.Logger = slog.New(logs) inputs := []*Input{ newInput(env.app.Application.ID, 0, 0, marshal(randomAdvanceResult(0))), @@ -465,10 +570,84 @@ func (s *AdvancerSuite) TestContextCancellation() { case err := <-errCh: require.Error(err) require.ErrorIs(err, context.Canceled) + require.Empty(env.repo.StoredResults) + require.Zero(env.repo.ApplicationStatusUpdates) + require.Zero(env.mm.Map[env.app.Application.ID].closeCalls) + require.True(logs.contains( + slog.LevelDebug, + "Advance stopped because the service is shutting down", + )) + require.False(logs.contains(slog.LevelError, "Error executing advance")) case <-time.After(100 * time.Millisecond): require.Fail("processInputs operation did not respect context cancellation") } }) + + s.Run("DeadlineDuringProcessInputs", func() { + require := s.Require() + env := s.setupOneApp() + env.app.AdvanceBlock = true + logs := &advancerLogCapture{} + env.service.Logger = slog.New(logs) + inputs := []*Input{ + newInput(env.app.Application.ID, 0, 0, marshal(randomAdvanceResult(0))), + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + + err := env.service.processInputs(ctx, env.app.Application, inputs) + require.ErrorIs(err, context.DeadlineExceeded) + require.Empty(env.repo.StoredResults) + // The expired context prevents the immediate database write, so the + // application failure is queued for a later status-write retry. + require.Zero(env.repo.ApplicationStatusUpdates) + require.Equal(1, env.mm.Map[env.app.Application.ID].closeCalls) + require.True(logs.contains(slog.LevelError, "Error executing advance")) + require.False(logs.contains( + slog.LevelDebug, + "Advance stopped because the service is shutting down", + )) + }) + + for _, test := range []struct { + name string + err error + }{ + { + name: "LiveContextWithJoinedDeadlineAndCancellation", + err: errors.Join(context.DeadlineExceeded, context.Canceled), + }, + { + name: "LiveContextWithJoinedMachineFailureAndCancellation", + err: errors.Join(pkgmachine.ErrMachineInternal, context.Canceled), + }, + } { + s.Run(test.name, func() { + require := s.Require() + env := s.setupOneApp() + env.app.AdvanceError = test.err + logs := &advancerLogCapture{} + env.service.Logger = slog.New(logs) + inputs := []*Input{ + newInput(env.app.Application.ID, 0, 0, []byte("input")), + } + + ctx := context.Background() + err := env.service.processInputs(ctx, env.app.Application, inputs) + + require.ErrorIs(err, context.Canceled) + require.NoError(ctx.Err(), "the caller context must remain active") + require.Empty(env.repo.StoredResults) + require.Equal(1, env.repo.ApplicationStatusUpdates) + require.Equal(ApplicationStatus_Failed, env.repo.LastApplicationStatus) + require.Equal(1, env.mm.Map[env.app.Application.ID].closeCalls) + require.True(logs.contains(slog.LevelError, "Error executing advance")) + require.False(logs.contains( + slog.LevelDebug, + "Advance stopped because the service is shutting down", + )) + }) + } } // TestLargeNumberOfInputs how the advancer handles large volumes of inputs @@ -504,7 +683,11 @@ func (s *AdvancerSuite) TestErrorRecovery() { err := env.service.processInputs(context.Background(), env.app.Application, inputs) require.Error(err) require.Contains(err.Error(), "temporary failure") + require.Empty(env.repo.StoredResults) + require.Empty(env.repo.StoredAppIDs) require.Error(env.service.Context.Err(), "shared context should be cancelled") + require.Equal(1, env.mm.Map[env.app.Application.ID].closeCalls, + "the already-advanced machine must be closed before processInputs returns") }) } @@ -1819,6 +2002,7 @@ type MockMachineInstance struct { application *Application machineImpl *MockMachineImpl createSnapshotError error + closeCalls int } // Advance implements the MachineInstance interface for testing @@ -1870,7 +2054,7 @@ func (m *MockMachineInstance) Hash(ctx context.Context) ([32]byte, error) { // Close implements the MachineInstance interface for testing func (m *MockMachineInstance) Close() error { - // Not used in advancer tests, but needed to satisfy the interface + m.closeCalls++ return nil } @@ -1884,6 +2068,7 @@ type MockRepository struct { GetInputsError error GetInputsBlock bool StoreAdvanceError error + StoreAdvanceCommitError error StoreAdvanceFailCount int UpdateApplicationStatusError error UpdateEpochsError error @@ -2004,27 +2189,33 @@ func (mock *MockRepository) StoreAdvanceResult( return errors.New("temporary failure") } + // Model an atomic transaction failure before commit: neither the canonical + // result nor the pending-input cursor changes. + if mock.StoreAdvanceError != nil { + return mock.StoreAdvanceError + } + mock.StoredResults = append(mock.StoredResults, res) mock.StoredAppIDs = append(mock.StoredAppIDs, appID) // Simulate real behavior: processed inputs change status and are no longer // returned by queries filtering for unprocessed (Status_None) inputs. // This prevents infinite loops in batched fetching. - if mock.StoreAdvanceError == nil { - for addr, inputs := range mock.GetInputsReturn { - for i, inp := range inputs { - if inp.EpochApplicationID == appID && inp.Index == res.InputIndex { - newInputs := make([]*Input, 0, len(inputs)-1) - newInputs = append(newInputs, inputs[:i]...) - newInputs = append(newInputs, inputs[i+1:]...) - mock.GetInputsReturn[addr] = newInputs - break - } + for addr, inputs := range mock.GetInputsReturn { + for i, inp := range inputs { + if inp.EpochApplicationID == appID && inp.Index == res.InputIndex { + newInputs := make([]*Input, 0, len(inputs)-1) + newInputs = append(newInputs, inputs[:i]...) + newInputs = append(newInputs, inputs[i+1:]...) + mock.GetInputsReturn[addr] = newInputs + break } } } - return mock.StoreAdvanceError + // Model a committed transaction whose success response was lost. The + // canonical result and pending-input cursor changed together. + return mock.StoreAdvanceCommitError } func (mock *MockRepository) UpdateEpochOutputsProof(ctx context.Context, appID int64, epochIndex uint64, proof *OutputsProof) error { diff --git a/internal/advancer/determinism_test.go b/internal/advancer/determinism_test.go new file mode 100644 index 000000000..c665941e8 --- /dev/null +++ b/internal/advancer/determinism_test.go @@ -0,0 +1,947 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package advancer + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "io" + "log/slog" + "sync" + "testing" + "time" + + "github.com/cartesi/rollups-node/internal/manager" + "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/pkg/machine" + pkgservice "github.com/cartesi/rollups-node/pkg/service" + "github.com/ethereum/go-ethereum/common" + "github.com/stretchr/testify/require" +) + +const determinismWaitTimeout = 5 * time.Second + +var errDeterminismRuntimeClosed = errors.New("determinism runtime is closed") + +func TestProcessInputs_RetryFromNonzeroPredecessorMatchesUninterruptedResult(t *testing.T) { + tests := []struct { + name string + targetPayload []byte + wantStatus model.InputCompletionStatus + }{ + { + name: "accepted", + targetPayload: []byte("deposit:alice:17"), + wantStatus: model.InputCompletionStatus_Accepted, + }, + { + name: "rejected", + targetPayload: []byte("reject:malformed-deposit"), + wantStatus: model.InputCompletionStatus_Rejected, + }, + { + name: "exception", + targetPayload: []byte("exception:application-error"), + wantStatus: model.InputCompletionStatus_Exception, + }, + { + name: "machine halted", + targetPayload: []byte("halt:application-finished"), + wantStatus: model.InputCompletionStatus_MachineHalted, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + prefix, predecessor, target := determinismBaseline(t, tt.targetPayload) + requireDeterminismTarget(t, prefix, target, tt.targetPayload, tt.wantStatus) + + t.Run("pre-canceled calls do not fork", func(t *testing.T) { + testDeterminismEarlyInterruptions(t, prefix, target, tt.targetPayload) + }) + t.Run("caller cancellation after mutation", func(t *testing.T) { + testDeterminismCallerInterruptions( + t, prefix, target, tt.targetPayload, context.Canceled, 1, + ) + }) + t.Run("caller deadline reconstructs predecessor across retries", func(t *testing.T) { + testDeterminismCallerDeadlines( + t, prefix, predecessor, target, tt.targetPayload, 3, + ) + }) + t.Run("infrastructure retry reconstructs predecessor", func(t *testing.T) { + testDeterminismInfrastructureRetries( + t, prefix, predecessor, target, tt.targetPayload, 2, + ) + }) + for _, deadline := range []time.Duration{100 * time.Millisecond, 250 * time.Millisecond} { + t.Run("manager watchdog "+deadline.String(), func(t *testing.T) { + testDeterminismManagerWatchdog( + t, prefix, predecessor, target, tt.targetPayload, deadline, + ) + }) + } + }) + } +} + +func determinismBaseline( + t *testing.T, + targetPayload []byte, +) (*model.AdvanceResult, determinismMachineState, *model.AdvanceResult) { + t.Helper() + repo := &MockRepository{} + harness := newTemplateDeterminismHarness( + t, repo, determinismWaitTimeout, + determinismAdvanceSuccess, + determinismAdvanceSuccess, + ) + + require.NoError(t, harness.process(context.Background(), 0, []byte("deposit:prefix:11"))) + require.Len(t, repo.StoredResults, 1) + prefix := cloneDeterminismResult(repo.StoredResults[0]) + predecessor := harness.liveState(t) + require.Equal(t, uint64(1), predecessor.step) + + require.NoError(t, harness.process(context.Background(), 1, targetPayload)) + require.Len(t, repo.StoredResults, 2) + target := cloneDeterminismResult(repo.StoredResults[1]) + require.Equal(t, uint64(2), harness.instance.ProcessedInputs()) + require.Equal(t, machine.Hash(target.MachineHash), harness.runtimeHash(t)) + return prefix, predecessor, target +} + +func testDeterminismEarlyInterruptions( + t *testing.T, + wantPrefix *model.AdvanceResult, + wantTarget *model.AdvanceResult, + targetPayload []byte, +) { + t.Helper() + repo := &MockRepository{} + harness := newTemplateDeterminismHarness( + t, repo, determinismWaitTimeout, + determinismAdvanceSuccess, + determinismAdvanceSuccess, + ) + require.NoError(t, harness.process(context.Background(), 0, []byte("deposit:prefix:11"))) + require.Equal(t, wantPrefix, repo.StoredResults[0]) + predecessor := harness.liveRuntime(t) + predecessorState := predecessor.snapshot() + forksBefore := harness.factory.forkCount() + + canceled, cancel := context.WithCancel(context.Background()) + cancel() + require.ErrorIs(t, harness.process(canceled, 1, targetPayload), context.Canceled) + + expired, cancelDeadline := context.WithDeadline( + context.Background(), time.Now().Add(-time.Second), + ) + defer cancelDeadline() + require.ErrorIs(t, harness.process(expired, 1, targetPayload), context.DeadlineExceeded) + + require.Equal(t, forksBefore, harness.factory.forkCount(), + "an already-finished caller context must be rejected before Fork") + requireCanonicalPrefix(t, repo, wantPrefix) + require.Equal(t, uint64(1), harness.instance.ProcessedInputs()) + require.False(t, predecessor.isClosed()) + require.Equal(t, predecessorState, harness.liveState(t)) + + require.NoError(t, harness.process(context.Background(), 1, targetPayload)) + requireCanonicalHistory(t, repo, wantPrefix, wantTarget) + requireSuccessfulRetryState(t, harness, predecessor, wantTarget) +} + +func testDeterminismCallerInterruptions( + t *testing.T, + wantPrefix *model.AdvanceResult, + wantTarget *model.AdvanceResult, + targetPayload []byte, + wantErr error, + attempts int, +) { + t.Helper() + behaviors := []determinismAdvanceBehavior{determinismAdvanceSuccess} + for range attempts { + behaviors = append(behaviors, determinismAdvanceWaitForContext) + } + behaviors = append(behaviors, determinismAdvanceSuccess) + repo := &MockRepository{} + harness := newTemplateDeterminismHarness( + t, repo, determinismWaitTimeout, behaviors..., + ) + require.NoError(t, harness.process(context.Background(), 0, []byte("deposit:prefix:11"))) + require.Equal(t, wantPrefix, repo.StoredResults[0]) + predecessor := harness.liveRuntime(t) + predecessorState := predecessor.snapshot() + + for range attempts { + controlled := newDeterminismContext(context.Background()) + defer controlled.finish(wantErr) + errCh := make(chan error, 1) + go func() { errCh <- harness.process(controlled, 1, targetPayload) }() + + fork := harness.waitForMutation(t) + controlled.finish(wantErr) + require.ErrorIs(t, waitDeterminismError(t, errCh), wantErr) + requireCanonicalPrefix(t, repo, wantPrefix) + require.Equal(t, uint64(1), harness.instance.ProcessedInputs()) + require.Zero(t, repo.ApplicationStatusUpdates, + "caller-owned interruptions must not mark the application failed") + require.False(t, predecessor.isClosed(), + "caller interruption must leave the reusable predecessor open") + require.Equal(t, predecessorState, harness.liveState(t)) + requireDiscardedMutatedFork(t, fork, predecessorState) + } + + require.NoError(t, harness.process(context.Background(), 1, targetPayload)) + requireCanonicalHistory(t, repo, wantPrefix, wantTarget) + requireSuccessfulRetryState(t, harness, predecessor, wantTarget) +} + +func testDeterminismCallerDeadlines( + t *testing.T, + wantPrefix *model.AdvanceResult, + predecessor determinismMachineState, + wantTarget *model.AdvanceResult, + targetPayload []byte, + attempts int, +) { + t.Helper() + repo := repositoryWithDeterminismPrefix(wantPrefix) + + for range attempts { + harness := newSnapshotDeterminismHarness( + t, repo, predecessor, determinismWaitTimeout, + determinismAdvanceWaitForContext, + ) + controlled := newDeterminismContext(context.Background()) + errCh := make(chan error, 1) + go func() { errCh <- harness.process(controlled, 1, targetPayload) }() + + fork := harness.waitForMutation(t) + controlled.finish(context.DeadlineExceeded) + require.ErrorIs(t, waitDeterminismError(t, errCh), context.DeadlineExceeded) + requireCanonicalPrefix(t, repo, wantPrefix) + require.Equal(t, uint64(1), harness.instance.ProcessedInputs()) + require.Zero(t, repo.ApplicationStatusUpdates, + "the expired context prevents the immediate FAILED status write") + require.Equal(t, predecessor, harness.factory.base.snapshot()) + require.True(t, harness.factory.base.isClosed(), + "a timed-out advance must close the changed live machine") + _, err := harness.factory.base.Fork(context.Background()) + require.ErrorIs(t, err, errDeterminismRuntimeClosed) + requireDiscardedMutatedFork(t, fork, predecessor) + } + + recovered := newSnapshotDeterminismHarness( + t, repo, predecessor, determinismWaitTimeout, determinismAdvanceSuccess, + ) + require.NoError(t, recovered.process(context.Background(), 1, targetPayload)) + requireCanonicalHistory(t, repo, wantPrefix, wantTarget) + requireSuccessfulRetryState(t, recovered, recovered.factory.base, wantTarget) +} + +func testDeterminismInfrastructureRetries( + t *testing.T, + wantPrefix *model.AdvanceResult, + predecessor determinismMachineState, + wantTarget *model.AdvanceResult, + targetPayload []byte, + attempts int, +) { + t.Helper() + repo := repositoryWithDeterminismPrefix(wantPrefix) + + for range attempts { + harness := newSnapshotDeterminismHarness( + t, repo, predecessor, determinismWaitTimeout, + determinismAdvanceInfrastructureFailure, + ) + err := harness.process(context.Background(), 1, targetPayload) + require.ErrorIs(t, err, machine.ErrMachineInternal) + requireCanonicalPrefix(t, repo, wantPrefix) + require.Equal(t, uint64(1), harness.instance.ProcessedInputs()) + require.Equal(t, predecessor, harness.factory.base.snapshot()) + require.True(t, harness.factory.base.isClosed(), + "advancer must close the failed live runtime") + _, err = harness.factory.base.Fork(context.Background()) + require.ErrorIs(t, err, errDeterminismRuntimeClosed) + requireDiscardedMutatedFork(t, harness.lastFork(t), predecessor) + } + require.Equal(t, attempts, repo.ApplicationStatusUpdates) + require.Equal(t, model.ApplicationStatus_Failed, repo.LastApplicationStatus) + + recovered := newSnapshotDeterminismHarness( + t, repo, predecessor, determinismWaitTimeout, determinismAdvanceSuccess, + ) + require.NoError(t, recovered.process(context.Background(), 1, targetPayload)) + requireCanonicalHistory(t, repo, wantPrefix, wantTarget) + requireSuccessfulRetryState(t, recovered, recovered.factory.base, wantTarget) +} + +func testDeterminismManagerWatchdog( + t *testing.T, + wantPrefix *model.AdvanceResult, + predecessor determinismMachineState, + wantTarget *model.AdvanceResult, + targetPayload []byte, + deadline time.Duration, +) { + t.Helper() + repo := repositoryWithDeterminismPrefix(wantPrefix) + harness := newSnapshotDeterminismHarness( + t, repo, predecessor, deadline, determinismAdvanceWaitForContext, + ) + parent := context.Background() + errCh := make(chan error, 1) + go func() { errCh <- harness.process(parent, 1, targetPayload) }() + + err := waitDeterminismError(t, errCh) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.NoError(t, parent.Err(), "the manager watchdog, not the caller, must expire") + requireCanonicalPrefix(t, repo, wantPrefix) + require.Equal(t, uint64(1), harness.instance.ProcessedInputs()) + require.Equal(t, 1, repo.ApplicationStatusUpdates) + require.Equal(t, model.ApplicationStatus_Failed, repo.LastApplicationStatus) + require.Equal(t, predecessor, harness.factory.base.snapshot()) + require.True(t, harness.factory.base.isClosed()) + requireDiscardedMutatedFork(t, harness.lastFork(t), predecessor) + + recovered := newSnapshotDeterminismHarness( + t, repo, predecessor, determinismWaitTimeout, determinismAdvanceSuccess, + ) + require.NoError(t, recovered.process(context.Background(), 1, targetPayload)) + requireCanonicalHistory(t, repo, wantPrefix, wantTarget) + requireSuccessfulRetryState(t, recovered, recovered.factory.base, wantTarget) +} + +func requireDeterminismTarget( + t *testing.T, + prefix *model.AdvanceResult, + target *model.AdvanceResult, + targetPayload []byte, + wantStatus model.InputCompletionStatus, +) { + t.Helper() + require.Equal(t, uint64(7), target.EpochIndex) + require.Equal(t, uint64(1), target.InputIndex) + require.Equal(t, wantStatus, target.Status) + require.True(t, target.IsDaveConsensus) + require.NotEmpty(t, target.OutputsHashProof) + require.Len(t, target.PeriodicStateHashes, 2, "a PRT result must retain its periodic state hashes") + require.Equal(t, machine.InputEntryCapacity-uint64(len(target.PeriodicStateHashes)), target.PaddingRepetitions) + + if wantStatus == model.InputCompletionStatus_Accepted { + require.Equal(t, [][]byte{append([]byte("output:"), targetPayload...)}, target.Outputs) + require.Equal(t, [][]byte{append([]byte("report:"), targetPayload...)}, target.Reports) + require.NotEqual(t, prefix.MachineHash, target.MachineHash) + require.NotEqual(t, prefix.OutputsHash, target.OutputsHash) + return + } + + require.Empty(t, target.Outputs, "effects are canonical only for accepted inputs") + require.Empty(t, target.Reports, "effects are canonical only for accepted inputs") + require.Equal(t, prefix.MachineHash, target.MachineHash, + "a nonaccepted candidate must not replace the predecessor") + require.Equal(t, prefix.OutputsHash, target.OutputsHash) + require.Equal(t, prefix.OutputsHashProof, target.OutputsHashProof) +} + +func requireDiscardedMutatedFork( + t *testing.T, + fork *determinismRuntime, + predecessor determinismMachineState, +) { + t.Helper() + require.True(t, fork.isClosed(), "the mutated candidate must be discarded") + require.NotEqual(t, predecessor.machineHash, fork.snapshot().machineHash, + "the injected failure must happen after candidate state changes") + _, err := fork.Hash(context.Background()) + require.ErrorIs(t, err, errDeterminismRuntimeClosed) +} + +func requireSuccessfulRetryState( + t *testing.T, + harness *determinismHarness, + predecessor *determinismRuntime, + wantTarget *model.AdvanceResult, +) { + t.Helper() + require.Equal(t, uint64(2), harness.instance.ProcessedInputs()) + require.Equal(t, machine.Hash(wantTarget.MachineHash), harness.runtimeHash(t)) + lastCandidate := harness.lastFork(t) + if wantTarget.Status == model.InputCompletionStatus_Accepted { + require.True(t, predecessor.isClosed()) + require.False(t, lastCandidate.isClosed(), "the accepted candidate must be adopted") + return + } + require.False(t, predecessor.isClosed(), "rejection must keep the predecessor live") + require.True(t, lastCandidate.isClosed(), "the rejected candidate must be discarded") +} + +func requireCanonicalPrefix( + t *testing.T, + repo *MockRepository, + wantPrefix *model.AdvanceResult, +) { + t.Helper() + require.Len(t, repo.StoredResults, 1, + "an interrupted input must not add a canonical result") + require.Equal(t, wantPrefix, repo.StoredResults[0]) +} + +func requireCanonicalHistory( + t *testing.T, + repo *MockRepository, + wantPrefix *model.AdvanceResult, + wantTarget *model.AdvanceResult, +) { + t.Helper() + require.Equal(t, []*model.AdvanceResult{wantPrefix, wantTarget}, repo.StoredResults) +} + +func repositoryWithDeterminismPrefix(prefix *model.AdvanceResult) *MockRepository { + return &MockRepository{StoredResults: []*model.AdvanceResult{cloneDeterminismResult(prefix)}} +} + +func waitDeterminismError(t *testing.T, errCh <-chan error) error { + t.Helper() + select { + case err := <-errCh: + return err + case <-time.After(determinismWaitTimeout): + t.Fatal("advance did not finish before the test timeout") + return nil + } +} + +func cloneDeterminismResult(result *model.AdvanceResult) *model.AdvanceResult { + clone := *result + clone.OutputsHashProof = append([][32]byte(nil), result.OutputsHashProof...) + clone.Outputs = cloneDeterminismBytes(result.Outputs) + clone.Reports = cloneDeterminismBytes(result.Reports) + clone.ExceptionData = append([]byte(nil), result.ExceptionData...) + clone.PeriodicStateHashes = append([][32]byte(nil), result.PeriodicStateHashes...) + return &clone +} + +func cloneDeterminismBytes(values [][]byte) [][]byte { + if values == nil { + return nil + } + clone := make([][]byte, len(values)) + for i := range values { + clone[i] = append([]byte(nil), values[i]...) + } + return clone +} + +type determinismHarness struct { + app *model.Application + instance manager.MachineInstance + service *Service + factory *determinismRuntimeFactory + provider *determinismMachineProvider +} + +func newTemplateDeterminismHarness( + t *testing.T, + repo *MockRepository, + advanceDeadline time.Duration, + behaviors ...determinismAdvanceBehavior, +) *determinismHarness { + t.Helper() + return newDeterminismHarness( + t, repo, newDeterminismMachineState(), 0, advanceDeadline, behaviors..., + ) +} + +func newSnapshotDeterminismHarness( + t *testing.T, + repo *MockRepository, + predecessor determinismMachineState, + advanceDeadline time.Duration, + behaviors ...determinismAdvanceBehavior, +) *determinismHarness { + t.Helper() + return newDeterminismHarness( + t, repo, predecessor, 1, advanceDeadline, behaviors..., + ) +} + +func newDeterminismHarness( + t *testing.T, + repo *MockRepository, + start determinismMachineState, + processedInputs uint64, + advanceDeadline time.Duration, + behaviors ...determinismAdvanceBehavior, +) *determinismHarness { + t.Helper() + app := &model.Application{ + ID: 1, + Name: "deterministic-input-statuses", + IApplicationAddress: common.HexToAddress("0x1"), + ConsensusType: model.Consensus_PRT, + Enabled: true, + Status: model.ApplicationStatus_OK, + ProcessedInputs: processedInputs, + ExecutionParameters: model.ExecutionParameters{ + SnapshotPolicy: model.SnapshotPolicy_None, + AdvanceMaxDeadline: advanceDeadline, + InspectMaxDeadline: determinismWaitTimeout, + LoadDeadline: determinismWaitTimeout, + StoreDeadline: determinismWaitTimeout, + MaxConcurrentInspects: 1, + }, + } + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + factory := newDeterminismRuntimeFactory(start, behaviors...) + instance, err := manager.NewMachineInstanceWithFactory( + context.Background(), app, processedInputs, logger, false, factory, + ) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, instance.Close()) }) + provider := &determinismMachineProvider{app: app, instance: instance} + service := &Service{ + Service: pkgservice.Service{ + Logger: logger, + Cancel: func() {}, + }, + inputBatchSize: 500, + machineManager: provider, + repository: repo, + } + return &determinismHarness{ + app: app, instance: instance, service: service, factory: factory, provider: provider, + } +} + +func (h *determinismHarness) process(ctx context.Context, index uint64, payload []byte) error { + return h.service.processInputs(ctx, h.app, []*model.Input{{ + EpochApplicationID: h.app.ID, + EpochIndex: 7, + Index: index, + RawData: append([]byte(nil), payload...), + }}) +} + +func (h *determinismHarness) waitForMutation(t *testing.T) *determinismRuntime { + t.Helper() + select { + case fork := <-h.factory.mutated: + return fork + case <-time.After(determinismWaitTimeout): + t.Fatal("candidate did not reach the injected interruption point") + return nil + } +} + +func (h *determinismHarness) liveRuntime(t *testing.T) *determinismRuntime { + t.Helper() + if h.instance.ProcessedInputs() == 0 { + return h.factory.base + } + return h.factory.acceptedRuntime(t) +} + +func (h *determinismHarness) liveState(t *testing.T) determinismMachineState { + t.Helper() + return h.liveRuntime(t).snapshot() +} + +func (h *determinismHarness) lastFork(t *testing.T) *determinismRuntime { + t.Helper() + return h.factory.lastFork(t) +} + +func (h *determinismHarness) runtimeHash(t *testing.T) machine.Hash { + t.Helper() + hash, err := h.instance.Hash(context.Background()) + require.NoError(t, err) + return hash +} + +type determinismAdvanceBehavior uint8 + +const ( + determinismAdvanceSuccess determinismAdvanceBehavior = iota + determinismAdvanceWaitForContext + determinismAdvanceInfrastructureFailure +) + +type determinismMachineState struct { + step uint64 + machineHash machine.Hash + outputsHash machine.Hash + outputsProof []machine.Hash + checkpointHash machine.Hash +} + +func newDeterminismMachineState() determinismMachineState { + machineHash := determinismHash("base-machine") + outputsHash := determinismHash("base-outputs") + return determinismMachineState{ + machineHash: machineHash, + outputsHash: outputsHash, + outputsProof: determinismProof(outputsHash), + } +} + +func (s determinismMachineState) clone() determinismMachineState { + s.outputsProof = append([]machine.Hash(nil), s.outputsProof...) + return s +} + +type determinismRuntimeFactory struct { + mu sync.Mutex + behaviors []determinismAdvanceBehavior + next int + start determinismMachineState + base *determinismRuntime + forks []*determinismRuntime + accepted *determinismRuntime + mutated chan *determinismRuntime +} + +func newDeterminismRuntimeFactory( + start determinismMachineState, + behaviors ...determinismAdvanceBehavior, +) *determinismRuntimeFactory { + return &determinismRuntimeFactory{ + behaviors: append([]determinismAdvanceBehavior(nil), behaviors...), + start: start.clone(), + mutated: make(chan *determinismRuntime, len(behaviors)+1), + } +} + +func (f *determinismRuntimeFactory) CreateMachineRuntime( + ctx context.Context, + _ *model.Application, + _ *slog.Logger, + _ bool, +) (machine.Machine, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + f.mu.Lock() + defer f.mu.Unlock() + f.base = &determinismRuntime{factory: f, state: f.start.clone()} + return f.base, nil +} + +func (f *determinismRuntimeFactory) fork(state determinismMachineState) *determinismRuntime { + f.mu.Lock() + defer f.mu.Unlock() + behavior := determinismAdvanceSuccess + if f.next < len(f.behaviors) { + behavior = f.behaviors[f.next] + } + f.next++ + child := &determinismRuntime{ + factory: f, + behavior: behavior, + state: state.clone(), + } + f.forks = append(f.forks, child) + return child +} + +func (f *determinismRuntimeFactory) recordAccepted(runtime *determinismRuntime) { + f.mu.Lock() + defer f.mu.Unlock() + f.accepted = runtime +} + +func (f *determinismRuntimeFactory) acceptedRuntime(t *testing.T) *determinismRuntime { + t.Helper() + f.mu.Lock() + defer f.mu.Unlock() + require.NotNil(t, f.accepted) + return f.accepted +} + +func (f *determinismRuntimeFactory) lastFork(t *testing.T) *determinismRuntime { + t.Helper() + f.mu.Lock() + defer f.mu.Unlock() + require.NotEmpty(t, f.forks) + return f.forks[len(f.forks)-1] +} + +func (f *determinismRuntimeFactory) forkCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.forks) +} + +type determinismRuntime struct { + mu sync.Mutex + factory *determinismRuntimeFactory + behavior determinismAdvanceBehavior + state determinismMachineState + closed bool +} + +func (m *determinismRuntime) Fork(ctx context.Context) (machine.Machine, error) { + m.mu.Lock() + defer m.mu.Unlock() + if err := m.checkOpenLocked(ctx); err != nil { + return nil, err + } + return m.factory.fork(m.state), nil +} + +func (m *determinismRuntime) Hash(ctx context.Context) (machine.Hash, error) { + m.mu.Lock() + defer m.mu.Unlock() + if err := m.checkOpenLocked(ctx); err != nil { + return machine.Hash{}, err + } + return m.state.machineHash, nil +} + +func (m *determinismRuntime) OutputsHash(ctx context.Context) (machine.Hash, error) { + m.mu.Lock() + defer m.mu.Unlock() + if err := m.checkOpenLocked(ctx); err != nil { + return machine.Hash{}, err + } + return m.state.outputsHash, nil +} + +func (m *determinismRuntime) OutputsHashProof(ctx context.Context) ([]machine.Hash, error) { + m.mu.Lock() + defer m.mu.Unlock() + if err := m.checkOpenLocked(ctx); err != nil { + return nil, err + } + return append([]machine.Hash(nil), m.state.outputsProof...), nil +} + +func (m *determinismRuntime) Advance( + ctx context.Context, + input []byte, + checkpointHash machine.Hash, + computeHashes bool, +) (*machine.AdvanceResponse, error) { + m.mu.Lock() + if err := m.checkOpenLocked(ctx); err != nil { + m.mu.Unlock() + return nil, err + } + if !computeHashes { + m.mu.Unlock() + return nil, errors.New("determinism test requires PRT input hash collection") + } + if len(input) == 0 { + m.mu.Unlock() + return nil, errors.New("determinism test input must not be empty") + } + + if checkpointHash != m.state.machineHash { + m.mu.Unlock() + return nil, errors.New("determinism test requires the checkpoint hash to equal the pre-input machine root") + } + previous := m.state.clone() + m.state.checkpointHash = checkpointHash + status := machine.CompletionStatusAccepted + switch { + case bytes.HasPrefix(input, []byte("reject:")): + status = machine.CompletionStatusRejected + case bytes.HasPrefix(input, []byte("exception:")): + status = machine.CompletionStatusException + case bytes.HasPrefix(input, []byte("halt:")): + status = machine.CompletionStatusHalted + } + output := append([]byte("output:"), input...) + report := append([]byte("report:"), input...) + firstHash := determinismHash("trace-before", previous.machineHash[:], input) + finalHash := determinismHash("trace-after", firstHash[:], input) + m.state.step++ + m.state.machineHash = determinismHash( + "machine", previous.machineHash[:], checkpointHash[:], input, + ) + m.state.outputsHash = determinismHash("outputs", previous.outputsHash[:], output) + m.state.outputsProof = determinismProof(m.state.outputsHash) + hashes := []machine.Hash{firstHash, finalHash} + response := &machine.AdvanceResponse{ + Status: status, + PeriodicStateHashes: hashes, + PaddingRepetitions: machine.InputEntryCapacity - uint64(len(hashes)), + OutputsHash: m.state.outputsHash, + } + if status == machine.CompletionStatusAccepted { + response.Outputs = []machine.Output{output} + response.Reports = []machine.Report{report} + } else if status == machine.CompletionStatusException { + response.ExceptionData = append([]byte{}, input...) + } + behavior := m.behavior + m.mu.Unlock() + + if behavior == determinismAdvanceWaitForContext { + m.factory.mutated <- m + } + + switch behavior { + case determinismAdvanceSuccess: + if status == machine.CompletionStatusAccepted { + m.factory.recordAccepted(m) + } + return response, nil + case determinismAdvanceWaitForContext: + <-ctx.Done() + return nil, ctx.Err() + case determinismAdvanceInfrastructureFailure: + return nil, machine.ErrMachineInternal + default: + return nil, errors.New("unknown determinism advance behavior") + } +} + +func (m *determinismRuntime) Inspect( + ctx context.Context, + _ []byte, +) (*machine.InspectResponse, error) { + m.mu.Lock() + defer m.mu.Unlock() + if err := m.checkOpenLocked(ctx); err != nil { + return &machine.InspectResponse{}, err + } + return &machine.InspectResponse{Status: machine.CompletionStatusRejected}, nil +} + +func (m *determinismRuntime) Store(ctx context.Context, _ string) error { + m.mu.Lock() + defer m.mu.Unlock() + return m.checkOpenLocked(ctx) +} + +func (m *determinismRuntime) Close() error { + m.mu.Lock() + defer m.mu.Unlock() + m.closed = true + return nil +} + +func (m *determinismRuntime) Address() string { return "determinism-runtime" } + +func (m *determinismRuntime) checkOpenLocked(ctx context.Context) error { + if m.closed { + return errDeterminismRuntimeClosed + } + return ctx.Err() +} + +// snapshot is test-only diagnostics; unlike the machine interface it remains +// readable after Close so tests can prove which state was closed or discarded. +func (m *determinismRuntime) snapshot() determinismMachineState { + m.mu.Lock() + defer m.mu.Unlock() + return m.state.clone() +} + +func (m *determinismRuntime) isClosed() bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.closed +} + +func determinismHash(label string, values ...[]byte) machine.Hash { + h := sha256.New() + _, _ = h.Write([]byte(label)) + for _, value := range values { + _, _ = h.Write(value) + } + var result machine.Hash + copy(result[:], h.Sum(nil)) + return result +} + +func determinismProof(outputsHash machine.Hash) []machine.Hash { + return []machine.Hash{ + determinismHash("proof-0", outputsHash[:]), + determinismHash("proof-1", outputsHash[:]), + } +} + +type determinismMachineProvider struct { + app *model.Application + instance manager.MachineInstance + mu sync.Mutex + failures map[int64]string +} + +func (p *determinismMachineProvider) GetMachine(appID int64) (manager.MachineInstance, bool) { + if appID != p.app.ID { + return nil, false + } + return p.instance, true +} + +func (p *determinismMachineProvider) Applications() []*model.Application { + return []*model.Application{p.app} +} + +func (p *determinismMachineProvider) UpdateMachines(context.Context) error { return nil } + +func (p *determinismMachineProvider) RecordApplicationFailure(app *model.Application, reason string) { + p.mu.Lock() + defer p.mu.Unlock() + if p.failures == nil { + p.failures = map[int64]string{} + } + p.failures[app.ID] = reason +} + +func (p *determinismMachineProvider) HasPendingApplicationFailures() bool { + p.mu.Lock() + defer p.mu.Unlock() + return len(p.failures) != 0 +} + +func (p *determinismMachineProvider) failureReason(appID int64) string { + p.mu.Lock() + defer p.mu.Unlock() + return p.failures[appID] +} + +func (p *determinismMachineProvider) HasMachine(appID int64) bool { return appID == p.app.ID } + +func (p *determinismMachineProvider) Close() error { return p.instance.Close() } + +// determinismContext makes cancellation and deadline propagation controllable +// without relying on scheduler timing or short wall-clock deadlines. +type determinismContext struct { + parent context.Context + done chan struct{} + once sync.Once + mu sync.RWMutex + err error +} + +func newDeterminismContext(parent context.Context) *determinismContext { + return &determinismContext{parent: parent, done: make(chan struct{})} +} + +func (c *determinismContext) Deadline() (time.Time, bool) { return time.Time{}, false } +func (c *determinismContext) Done() <-chan struct{} { return c.done } +func (c *determinismContext) Err() error { + c.mu.RLock() + defer c.mu.RUnlock() + return c.err +} +func (c *determinismContext) Value(key any) any { return c.parent.Value(key) } +func (c *determinismContext) finish(err error) { + c.once.Do(func() { + c.mu.Lock() + c.err = err + c.mu.Unlock() + close(c.done) + }) +} From f1e94753a57732adedd939a63abe9afedcc77e8b Mon Sep 17 00:00:00 2001 From: Victor Fusco <1221933+vfusco@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:26:35 -0300 Subject: [PATCH 05/13] refactor(execution): define fixed cycle-window configuration Treat zero max-cycle settings as no operator-imposed cap while retaining the machine-enforced 2^48 execution window. Validate increments and ceilings consistently across the model, CLI, schema, and JSON-RPC discovery, and refuse startup when node and emulator computation-hash constants disagree. --- .../execution_parameters.go | 30 +++-- .../execution_parameters_test.go | 38 ++++++ internal/advancer/service.go | 7 + internal/jsonrpc/jsonrpc-discover.json | 2 + internal/model/execution_parameters_test.go | 126 ++++++++++++++++++ internal/model/models.go | 29 +++- .../000001_create_initial_schema.up.sql | 4 +- .../repotest/application_test_cases.go | 24 ++-- 8 files changed, 232 insertions(+), 28 deletions(-) create mode 100644 cmd/cartesi-rollups-cli/root/app/execution-parameters/execution_parameters_test.go create mode 100644 internal/model/execution_parameters_test.go diff --git a/cmd/cartesi-rollups-cli/root/app/execution-parameters/execution_parameters.go b/cmd/cartesi-rollups-cli/root/app/execution-parameters/execution_parameters.go index fa5ad0f25..9f6330d34 100644 --- a/cmd/cartesi-rollups-cli/root/app/execution-parameters/execution_parameters.go +++ b/cmd/cartesi-rollups-cli/root/app/execution-parameters/execution_parameters.go @@ -449,17 +449,21 @@ func setParameterValue(params *model.ExecutionParameters, parameter, value strin } func printParameters(params *model.ExecutionParameters) { - fmt.Printf("snapshot_policy: %s\n", params.SnapshotPolicy) - fmt.Printf("advance_inc_cycles: %d\n", params.AdvanceIncCycles) - fmt.Printf("advance_max_cycles: %d\n", params.AdvanceMaxCycles) - fmt.Printf("inspect_inc_cycles: %d\n", params.InspectIncCycles) - fmt.Printf("inspect_max_cycles: %d\n", params.InspectMaxCycles) - fmt.Printf("advance_inc_deadline: %s\n", params.AdvanceIncDeadline) - fmt.Printf("advance_max_deadline: %s\n", params.AdvanceMaxDeadline) - fmt.Printf("inspect_inc_deadline: %s\n", params.InspectIncDeadline) - fmt.Printf("inspect_max_deadline: %s\n", params.InspectMaxDeadline) - fmt.Printf("load_deadline: %s\n", params.LoadDeadline) - fmt.Printf("store_deadline: %s\n", params.StoreDeadline) - fmt.Printf("fast_deadline: %s\n", params.FastDeadline) - fmt.Printf("max_concurrent_inspects: %d\n", params.MaxConcurrentInspects) + writeParameters(os.Stdout, params) +} + +func writeParameters(w io.Writer, params *model.ExecutionParameters) { + fmt.Fprintf(w, "snapshot_policy: %s\n", params.SnapshotPolicy) + fmt.Fprintf(w, "advance_inc_cycles: %d\n", params.AdvanceIncCycles) + fmt.Fprintf(w, "advance_max_cycles: %d\n", params.AdvanceMaxCycles) + fmt.Fprintf(w, "inspect_inc_cycles: %d\n", params.InspectIncCycles) + fmt.Fprintf(w, "inspect_max_cycles: %d\n", params.InspectMaxCycles) + fmt.Fprintf(w, "advance_inc_deadline: %s\n", params.AdvanceIncDeadline) + fmt.Fprintf(w, "advance_max_deadline: %s\n", params.AdvanceMaxDeadline) + fmt.Fprintf(w, "inspect_inc_deadline: %s\n", params.InspectIncDeadline) + fmt.Fprintf(w, "inspect_max_deadline: %s\n", params.InspectMaxDeadline) + fmt.Fprintf(w, "load_deadline: %s\n", params.LoadDeadline) + fmt.Fprintf(w, "store_deadline: %s\n", params.StoreDeadline) + fmt.Fprintf(w, "fast_deadline: %s\n", params.FastDeadline) + fmt.Fprintf(w, "max_concurrent_inspects: %d\n", params.MaxConcurrentInspects) } diff --git a/cmd/cartesi-rollups-cli/root/app/execution-parameters/execution_parameters_test.go b/cmd/cartesi-rollups-cli/root/app/execution-parameters/execution_parameters_test.go new file mode 100644 index 000000000..b261e81a1 --- /dev/null +++ b/cmd/cartesi-rollups-cli/root/app/execution-parameters/execution_parameters_test.go @@ -0,0 +1,38 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package execution + +import ( + "bytes" + "testing" + + "github.com/cartesi/rollups-node/internal/model" + "github.com/stretchr/testify/require" +) + +func TestCycleMaximumParameters(t *testing.T) { + params := &model.ExecutionParameters{} + require.NoError(t, setParameterValue(params, "advance_max_cycles", "123")) + require.NoError(t, setParameterValue(params, "inspect_max_cycles", "456")) + value, err := getParameterValue(params, "advance_max_cycles") + require.NoError(t, err) + require.Equal(t, "123", value) + value, err = getParameterValue(params, "inspect_max_cycles") + require.NoError(t, err) + require.Equal(t, "456", value) +} + +func TestWriteParametersIncludesCycleMaximums(t *testing.T) { + params := &model.ExecutionParameters{ + AdvanceIncCycles: 11, AdvanceMaxCycles: 12, + InspectIncCycles: 21, InspectMaxCycles: 22, + } + var output bytes.Buffer + writeParameters(&output, params) + + require.Contains(t, output.String(), "advance_inc_cycles: 11") + require.Contains(t, output.String(), "advance_max_cycles: 12") + require.Contains(t, output.String(), "inspect_inc_cycles: 21") + require.Contains(t, output.String(), "inspect_max_cycles: 22") +} diff --git a/internal/advancer/service.go b/internal/advancer/service.go index 3e657d5eb..68ee3f9fd 100644 --- a/internal/advancer/service.go +++ b/internal/advancer/service.go @@ -15,6 +15,7 @@ import ( "github.com/cartesi/rollups-node/internal/inspect" "github.com/cartesi/rollups-node/internal/manager" "github.com/cartesi/rollups-node/internal/repository" + "github.com/cartesi/rollups-node/pkg/machine" "github.com/cartesi/rollups-node/pkg/service" ) @@ -50,6 +51,12 @@ func Create(ctx context.Context, c *CreateInfo) (*Service, error) { if err = ctx.Err(); err != nil { return nil, err // This returns context.Canceled or context.DeadlineExceeded. } + // This is a process-wide compatibility invariant, not an application + // failure. Refuse advancer startup instead of retrying every application on + // every tick while reporting a misleading healthy service. + if err = machine.ValidateEmulatorComputationHashLimits(); err != nil { + return nil, fmt.Errorf("invalid machine execution constants: %w", err) + } s := &Service{} c.Impl = s diff --git a/internal/jsonrpc/jsonrpc-discover.json b/internal/jsonrpc/jsonrpc-discover.json index ea21d239e..d4e38fb55 100644 --- a/internal/jsonrpc/jsonrpc-discover.json +++ b/internal/jsonrpc/jsonrpc-discover.json @@ -2056,12 +2056,14 @@ "$ref": "#/components/schemas/UnsignedInteger" }, "advance_max_cycles": { + "description": "Optional advance execution delta from the starting mcycle in range 0..2^48-1; 0 imposes no operator cap and the machine's imcyclemax governs", "$ref": "#/components/schemas/UnsignedInteger" }, "inspect_inc_cycles": { "$ref": "#/components/schemas/UnsignedInteger" }, "inspect_max_cycles": { + "description": "Optional inspect execution delta from the starting mcycle in range 0..2^48-1; 0 imposes no operator cap and the machine's imcyclemax governs", "$ref": "#/components/schemas/UnsignedInteger" }, "advance_inc_deadline": { diff --git a/internal/model/execution_parameters_test.go b/internal/model/execution_parameters_test.go new file mode 100644 index 000000000..4945f77e0 --- /dev/null +++ b/internal/model/execution_parameters_test.go @@ -0,0 +1,126 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package model + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestExecutionParametersJSONRoundtrip(t *testing.T) { + original := ExecutionParameters{ + SnapshotPolicy: SnapshotPolicy_EveryInput, + AdvanceIncCycles: 101, + AdvanceMaxCycles: 111, + InspectIncCycles: 202, + InspectMaxCycles: 222, + AdvanceIncDeadline: 3 * time.Second, + AdvanceMaxDeadline: 4 * time.Second, + InspectIncDeadline: 5 * time.Second, + InspectMaxDeadline: 6 * time.Second, + LoadDeadline: 7 * time.Second, + StoreDeadline: 8 * time.Second, + FastDeadline: 9 * time.Second, + MaxConcurrentInspects: 10, + } + + data, err := json.Marshal(&original) + require.NoError(t, err) + require.Contains(t, string(data), `"advance_inc_cycles":"0x65"`) + require.Contains(t, string(data), `"advance_max_cycles":"0x6f"`) + require.Contains(t, string(data), `"inspect_inc_cycles":"0xca"`) + require.Contains(t, string(data), `"inspect_max_cycles":"0xde"`) + + var decoded ExecutionParameters + require.NoError(t, json.Unmarshal(data, &decoded)) + require.Equal(t, original.SnapshotPolicy, decoded.SnapshotPolicy) + require.Equal(t, original.AdvanceIncCycles, decoded.AdvanceIncCycles) + require.Equal(t, original.AdvanceMaxCycles, decoded.AdvanceMaxCycles) + require.Equal(t, original.InspectIncCycles, decoded.InspectIncCycles) + require.Equal(t, original.InspectMaxCycles, decoded.InspectMaxCycles) + require.Equal(t, original.AdvanceIncDeadline, decoded.AdvanceIncDeadline) + require.Equal(t, original.AdvanceMaxDeadline, decoded.AdvanceMaxDeadline) + require.Equal(t, original.InspectIncDeadline, decoded.InspectIncDeadline) + require.Equal(t, original.InspectMaxDeadline, decoded.InspectMaxDeadline) + require.Equal(t, original.LoadDeadline, decoded.LoadDeadline) + require.Equal(t, original.StoreDeadline, decoded.StoreDeadline) + require.Equal(t, original.FastDeadline, decoded.FastDeadline) + require.Equal(t, original.MaxConcurrentInspects, decoded.MaxConcurrentInspects) +} + +func TestExecutionParametersZeroCycleMaximumsRoundtrip(t *testing.T) { + data, err := json.Marshal(&ExecutionParameters{}) + require.NoError(t, err) + require.Contains(t, string(data), `"advance_max_cycles":"0x0"`) + require.Contains(t, string(data), `"inspect_max_cycles":"0x0"`) + + var decoded ExecutionParameters + require.NoError(t, json.Unmarshal(data, &decoded)) + require.Zero(t, decoded.AdvanceMaxCycles) + require.Zero(t, decoded.InspectMaxCycles) +} + +func TestExecutionParametersCycleMaximumBounds(t *testing.T) { + for _, value := range []uint64{0, 1, MaxExecutionCycleSpan} { + params := ExecutionParameters{ + SnapshotPolicy: SnapshotPolicy_None, + AdvanceIncCycles: 1, + AdvanceMaxCycles: value, + InspectIncCycles: 1, + InspectMaxCycles: value, + } + require.NoError(t, params.Validate()) + } + + for _, test := range []struct { + name string + mutate func(*ExecutionParameters) + }{ + {"advance", func(p *ExecutionParameters) { p.AdvanceMaxCycles = MaxExecutionCycles }}, + {"inspect", func(p *ExecutionParameters) { p.InspectMaxCycles = MaxExecutionCycles }}, + } { + t.Run(test.name, func(t *testing.T) { + params := ExecutionParameters{ + SnapshotPolicy: SnapshotPolicy_None, + AdvanceIncCycles: 1, + InspectIncCycles: 1, + } + test.mutate(¶ms) + require.ErrorContains(t, params.Validate(), "must be between 0 and 281474976710655") + }) + } +} + +func TestExecutionParametersRejectZeroCycleIncrements(t *testing.T) { + for _, test := range []struct { + name string + field string + mutate func(*ExecutionParameters) + }{ + { + name: "advance", + field: "advance_inc_cycles", + mutate: func(p *ExecutionParameters) { p.AdvanceIncCycles = 0 }, + }, + { + name: "inspect", + field: "inspect_inc_cycles", + mutate: func(p *ExecutionParameters) { p.InspectIncCycles = 0 }, + }, + } { + t.Run(test.name, func(t *testing.T) { + params := ExecutionParameters{ + SnapshotPolicy: SnapshotPolicy_None, + AdvanceIncCycles: 1, + InspectIncCycles: 1, + } + test.mutate(¶ms) + + require.ErrorContains(t, params.Validate(), test.field) + }) + } +} diff --git a/internal/model/models.go b/internal/model/models.go index b80d01c31..fa8e8dfb7 100644 --- a/internal/model/models.go +++ b/internal/model/models.go @@ -606,7 +606,6 @@ func (e *ExecutionParameters) UnmarshalJSON(data []byte) error { } e.AdvanceIncCycles = val } - if aux.AdvanceMaxCycles != "" { val, err := ParseHexUint64(aux.AdvanceMaxCycles) if err != nil { @@ -622,7 +621,6 @@ func (e *ExecutionParameters) UnmarshalJSON(data []byte) error { } e.InspectIncCycles = val } - if aux.InspectMaxCycles != "" { val, err := ParseHexUint64(aux.InspectMaxCycles) if err != nil { @@ -691,13 +689,21 @@ func (e *ExecutionParameters) UnmarshalJSON(data []byte) error { } // Log2MaxExecutionCycles is the single node-side definition of the protocol -// execution window size. pkg/machine aliases it as Log2MaxMCyclesPerAdvanceState. +// execution window size. pkg/machine aliases it as Log2MaxMCyclesPerAdvanceState, +// so the execution ceiling and computation-hash dimensions cannot diverge. The +// emulator exposes the same value as +// CM_ROLLUP_LOG2_MAX_MCYCLES_PER_ADVANCE_STATE. const Log2MaxExecutionCycles uint64 = 48 -// MaxExecutionCycles is the number of mcycles in one machine-enforced window. +// MaxExecutionCycles is the number of cycles in one machine-enforced execution +// window. It is the mcycle window covered by one input hash collection. const MaxExecutionCycles uint64 = 1 << Log2MaxExecutionCycles -// MaxExecutionCycleSpan is the largest endpoint delta in that window. +// MaxExecutionCycleSpan is the largest configurable distance between the +// starting mcycle and the execution endpoint. The endpoint itself is included +// in the MaxExecutionCycles-wide window, hence the subtraction by one. A +// configured maximum of zero means no operator-imposed cap; the machine's +// MaxExecutionCycleSpan ceiling applies instead. const MaxExecutionCycleSpan uint64 = MaxExecutionCycles - 1 // validateParameters constants @@ -706,6 +712,19 @@ const maxConcurrentInspects = 1000 // validateParameters performs validation on the loaded parameters func (e *ExecutionParameters) Validate() error { + if e.AdvanceIncCycles == 0 { + return errors.New("advance_inc_cycles must be greater than 0") + } + if e.AdvanceMaxCycles > MaxExecutionCycleSpan { + return fmt.Errorf("advance_max_cycles must be between 0 and %d", MaxExecutionCycleSpan) + } + if e.InspectIncCycles == 0 { + return errors.New("inspect_inc_cycles must be greater than 0") + } + if e.InspectMaxCycles > MaxExecutionCycleSpan { + return fmt.Errorf("inspect_max_cycles must be between 0 and %d", MaxExecutionCycleSpan) + } + // Validate durations are reasonable if e.AdvanceIncDeadline < 0 || e.AdvanceIncDeadline > maxDuration { return fmt.Errorf("advance_inc_deadline must be between 0 and 24h") diff --git a/internal/repository/postgres/schema/migrations/000001_create_initial_schema.up.sql b/internal/repository/postgres/schema/migrations/000001_create_initial_schema.up.sql index 4bc48857c..82e79ae7a 100644 --- a/internal/repository/postgres/schema/migrations/000001_create_initial_schema.up.sql +++ b/internal/repository/postgres/schema/migrations/000001_create_initial_schema.up.sql @@ -192,9 +192,9 @@ CREATE TABLE "execution_parameters" ( "application_id" INT PRIMARY KEY, "snapshot_policy" "SnapshotPolicy" NOT NULL DEFAULT 'NONE', "advance_inc_cycles" BIGINT NOT NULL CHECK ("advance_inc_cycles" > 0) DEFAULT 4194304, -- 1 << 22 - "advance_max_cycles" BIGINT NOT NULL CHECK ("advance_max_cycles" > 0) DEFAULT 4611686018427387903, -- uint64 max >> 2 + "advance_max_cycles" BIGINT NOT NULL CHECK ("advance_max_cycles" >= 0 AND "advance_max_cycles" <= 281474976710655) DEFAULT 0, -- 0 uses the machine's fixed (1 << 48) - 1 cycle span "inspect_inc_cycles" BIGINT NOT NULL CHECK ("inspect_inc_cycles" > 0) DEFAULT 4194304, -- 1 << 22 - "inspect_max_cycles" BIGINT NOT NULL CHECK ("inspect_max_cycles" > 0) DEFAULT 4611686018427387903, + "inspect_max_cycles" BIGINT NOT NULL CHECK ("inspect_max_cycles" >= 0 AND "inspect_max_cycles" <= 281474976710655) DEFAULT 0, -- 0 uses the machine's fixed (1 << 48) - 1 cycle span "advance_inc_deadline" BIGINT NOT NULL CHECK ("advance_inc_deadline" > 0) DEFAULT 10000000000, -- 10s "advance_max_deadline" BIGINT NOT NULL CHECK ("advance_max_deadline" > 0) DEFAULT 180000000000, -- 180s "inspect_inc_deadline" BIGINT NOT NULL CHECK ("inspect_inc_deadline" > 0) DEFAULT 10000000000, --10s diff --git a/internal/repository/repotest/application_test_cases.go b/internal/repository/repotest/application_test_cases.go index c6ffb668e..694f24805 100644 --- a/internal/repository/repotest/application_test_cases.go +++ b/internal/repository/repotest/application_test_cases.go @@ -31,9 +31,9 @@ func (s *ApplicationSuite) TestCreateApplication() { ep := ExecutionParameters{ SnapshotPolicy: SnapshotPolicy_EveryEpoch, AdvanceIncCycles: 1000, - AdvanceMaxCycles: 5000, - InspectIncCycles: 1000, - InspectMaxCycles: 5000, + AdvanceMaxCycles: 1500, //nolint:mnd + InspectIncCycles: 2000, + InspectMaxCycles: 2500, //nolint:mnd AdvanceIncDeadline: 10 * time.Second, AdvanceMaxDeadline: 60 * time.Second, InspectIncDeadline: 10 * time.Second, @@ -53,6 +53,8 @@ func (s *ApplicationSuite) TestCreateApplication() { s.Equal(ep.SnapshotPolicy, got.SnapshotPolicy) s.Equal(ep.AdvanceIncCycles, got.AdvanceIncCycles) s.Equal(ep.AdvanceMaxCycles, got.AdvanceMaxCycles) + s.Equal(ep.InspectIncCycles, got.InspectIncCycles) + s.Equal(ep.InspectMaxCycles, got.InspectMaxCycles) s.Equal(ep.MaxConcurrentInspects, got.MaxConcurrentInspects) }) } @@ -754,6 +756,8 @@ func (s *ApplicationSuite) TestGetExecutionParameters() { ep, err := s.Repo.GetExecutionParameters(s.Ctx, app.ID) s.Require().NoError(err) s.NotNil(ep) + s.Zero(ep.AdvanceMaxCycles) + s.Zero(ep.InspectMaxCycles) }) } @@ -762,9 +766,9 @@ func (s *ApplicationSuite) TestUpdateExecutionParameters() { ep := ExecutionParameters{ SnapshotPolicy: SnapshotPolicy_EveryInput, AdvanceIncCycles: 2000, - AdvanceMaxCycles: 10000, - InspectIncCycles: 2000, - InspectMaxCycles: 10000, + AdvanceMaxCycles: 2500, //nolint:mnd + InspectIncCycles: 3000, + InspectMaxCycles: 3500, //nolint:mnd AdvanceIncDeadline: 20 * time.Second, AdvanceMaxDeadline: 120 * time.Second, InspectIncDeadline: 20 * time.Second, @@ -779,13 +783,17 @@ func (s *ApplicationSuite) TestUpdateExecutionParameters() { Create(s.Ctx, s.T(), s.Repo) ep.ApplicationID = app.ID - ep.AdvanceMaxCycles = 99999 + ep.AdvanceIncCycles = 99999 + ep.AdvanceMaxCycles = MaxExecutionCycleSpan err := s.Repo.UpdateExecutionParameters(s.Ctx, &ep) s.Require().NoError(err) got, err := s.Repo.GetExecutionParameters(s.Ctx, app.ID) s.Require().NoError(err) - s.Equal(uint64(99999), got.AdvanceMaxCycles) + s.Equal(uint64(99999), got.AdvanceIncCycles) + s.Equal(MaxExecutionCycleSpan, got.AdvanceMaxCycles) + s.Equal(ep.InspectIncCycles, got.InspectIncCycles) + s.Equal(ep.InspectMaxCycles, got.InspectMaxCycles) }) } From 2e7e697600b0521dcdf832abe810c43b0d09d92e Mon Sep 17 00:00:00 2001 From: Victor Fusco <1221933+vfusco@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:30:56 -0300 Subject: [PATCH 06/13] refactor(repository): persist only completed input outcomes Restrict canonical input history to deterministic completed statuses, reject incomplete results before opening a transaction, and persist guest exception payloads with matching model, JSON-RPC, schema, and repository invariants. --- .../execution_outcome_contract_test.go | 59 ++++++++++++ internal/jsonrpc/jsonrpc-discover.json | 18 ++-- internal/model/execution_parameters_test.go | 40 ++++++++ internal/model/models.go | 95 +++++++++++-------- internal/model/models_json_test.go | 22 ++++- internal/repository/postgres/application.go | 2 + internal/repository/postgres/bulk.go | 34 ++++++- .../public/enum/inputcompletionstatus.go | 30 ++---- .../db/rollupsdb/public/table/input.go | 7 +- internal/repository/postgres/input.go | 8 ++ .../postgres/input_exception_data_test.go | 68 +++++++++++++ .../repository/postgres/postgres_repo_test.go | 90 ++++++++++++++++++ .../000001_create_initial_schema.up.sql | 13 +-- .../repository/repotest/bulk_test_cases.go | 88 +++++++++++++++++ internal/repository/repotest/repotest.go | 3 + 15 files changed, 499 insertions(+), 78 deletions(-) create mode 100644 internal/jsonrpc/execution_outcome_contract_test.go create mode 100644 internal/repository/postgres/input_exception_data_test.go diff --git a/internal/jsonrpc/execution_outcome_contract_test.go b/internal/jsonrpc/execution_outcome_contract_test.go new file mode 100644 index 000000000..811da54be --- /dev/null +++ b/internal/jsonrpc/execution_outcome_contract_test.go @@ -0,0 +1,59 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package jsonrpc + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDiscoverySchemaExecutionOutcomeContract(t *testing.T) { + data, err := discoverSpec.ReadFile("jsonrpc-discover.json") + require.NoError(t, err) + + var spec struct { + Components struct { + Schemas map[string]json.RawMessage `json:"schemas"` + } `json:"components"` + } + require.NoError(t, json.Unmarshal(data, &spec)) + + var completionStatus struct { + Enum []string `json:"enum"` + } + require.NoError(t, json.Unmarshal(spec.Components.Schemas["InputCompletionStatus"], &completionStatus)) + require.Equal(t, []string{"NONE", "ACCEPTED", "REJECTED", "EXCEPTION", "MACHINE_HALTED"}, completionStatus.Enum) + + var input struct { + Properties map[string]json.RawMessage `json:"properties"` + } + require.NoError(t, json.Unmarshal(spec.Components.Schemas["Input"], &input)) + require.Contains(t, input.Properties, "exception_data") + + var executionParameters struct { + Properties map[string]json.RawMessage `json:"properties"` + } + require.NoError(t, json.Unmarshal(spec.Components.Schemas["ExecutionParameters"], &executionParameters)) + require.Contains(t, executionParameters.Properties, "advance_inc_cycles") + require.Contains(t, executionParameters.Properties, "inspect_inc_cycles") + require.Contains(t, executionParameters.Properties, "advance_max_cycles") + require.Contains(t, executionParameters.Properties, "inspect_max_cycles") + for field, operation := range map[string]string{ + "advance_max_cycles": "advance", + "inspect_max_cycles": "inspect", + } { + var property struct { + Description string `json:"description"` + } + require.NoError(t, json.Unmarshal(executionParameters.Properties[field], &property)) + require.Equal(t, + "Optional "+operation+ + " execution delta from the starting mcycle in range 0..2^48-1; "+ + "0 imposes no operator cap and the machine's imcyclemax governs", + property.Description, + ) + } +} diff --git a/internal/jsonrpc/jsonrpc-discover.json b/internal/jsonrpc/jsonrpc-discover.json index d4e38fb55..ea2b43c12 100644 --- a/internal/jsonrpc/jsonrpc-discover.json +++ b/internal/jsonrpc/jsonrpc-discover.json @@ -1657,12 +1657,7 @@ "ACCEPTED", "REJECTED", "EXCEPTION", - "MACHINE_HALTED", - "OUTPUTS_LIMIT_EXCEEDED", - "REPORTS_LIMIT_EXCEEDED", - "CYCLE_LIMIT_EXCEEDED", - "TIME_LIMIT_EXCEEDED", - "PAYLOAD_LENGTH_LIMIT_EXCEEDED" + "MACHINE_HALTED" ] }, "Input": { @@ -1693,6 +1688,17 @@ "status": { "$ref": "#/components/schemas/InputCompletionStatus" }, + "exception_data": { + "description": "Raw guest-provided CMIO exception payload. It is non-null only when status is EXCEPTION; an empty payload is encoded as 0x.", + "oneOf": [ + { + "$ref": "#/components/schemas/ByteArray" + }, + { + "type": "null" + } + ] + }, "machine_hash": { "oneOf": [ { diff --git a/internal/model/execution_parameters_test.go b/internal/model/execution_parameters_test.go index 4945f77e0..7a22256e3 100644 --- a/internal/model/execution_parameters_test.go +++ b/internal/model/execution_parameters_test.go @@ -124,3 +124,43 @@ func TestExecutionParametersRejectZeroCycleIncrements(t *testing.T) { }) } } + +func TestInputCompletionStatusContract(t *testing.T) { + expected := []InputCompletionStatus{ + InputCompletionStatus_None, + InputCompletionStatus_Accepted, + InputCompletionStatus_Rejected, + InputCompletionStatus_Exception, + InputCompletionStatus_MachineHalted, + } + require.Equal(t, expected, InputCompletionStatusAllValues) + + for _, value := range expected { + t.Run(value.String(), func(t *testing.T) { + var fromString InputCompletionStatus + require.NoError(t, fromString.Scan(value.String())) + require.Equal(t, value, fromString) + + var fromBytes InputCompletionStatus + require.NoError(t, fromBytes.Scan([]byte(value.String()))) + require.Equal(t, value, fromBytes) + require.Equal(t, value != InputCompletionStatus_None, value.IsCompleted()) + }) + } + + removedOrInvalid := []string{ + "OUTPUTS_LIMIT_EXCEEDED", + "REPORTS_LIMIT_EXCEEDED", + "CYCLE_LIMIT_EXCEEDED", + "TIME_LIMIT_EXCEEDED", + "PAYLOAD_LENGTH_LIMIT_EXCEEDED", + "INVALID", + } + for _, value := range removedOrInvalid { + t.Run(value, func(t *testing.T) { + var status InputCompletionStatus + require.Error(t, status.Scan(value)) + require.False(t, InputCompletionStatus(value).IsCompleted()) + }) + } +} diff --git a/internal/model/models.go b/internal/model/models.go index fa8e8dfb7..a660fc9d8 100644 --- a/internal/model/models.go +++ b/internal/model/models.go @@ -986,6 +986,7 @@ type Input struct { BlockNumber uint64 `json:"block_number"` RawData []byte `json:"raw_data"` Status InputCompletionStatus `json:"status"` + ExceptionData []byte `json:"-"` MachineHash *common.Hash `json:"machine_hash"` OutputsHash *common.Hash `json:"outputs_hash"` TransactionHash common.Hash `json:"transaction_hash"` @@ -999,20 +1000,27 @@ func (i *Input) MarshalJSON() ([]byte, error) { // Create an alias to avoid infinite recursion in MarshalJSON. type Alias Input // Define a new structure that embeds the alias but overrides the hex fields. + var exceptionData *string + if i.ExceptionData != nil { + encoded := hexutil.Encode(i.ExceptionData) + exceptionData = &encoded + } aux := &struct { - EpochIndex string `json:"epoch_index"` - Index string `json:"index"` - BlockNumber string `json:"block_number"` - RawData string `json:"raw_data"` - LogIndex string `json:"log_index"` + EpochIndex string `json:"epoch_index"` + Index string `json:"index"` + BlockNumber string `json:"block_number"` + RawData string `json:"raw_data"` + ExceptionData *string `json:"exception_data"` + LogIndex string `json:"log_index"` *Alias }{ - EpochIndex: fmt.Sprintf("0x%x", i.EpochIndex), - Index: fmt.Sprintf("0x%x", i.Index), - BlockNumber: fmt.Sprintf("0x%x", i.BlockNumber), - RawData: "0x" + hex.EncodeToString(i.RawData), - LogIndex: fmt.Sprintf("0x%x", i.LogIndex), - Alias: (*Alias)(i), + EpochIndex: fmt.Sprintf("0x%x", i.EpochIndex), + Index: fmt.Sprintf("0x%x", i.Index), + BlockNumber: fmt.Sprintf("0x%x", i.BlockNumber), + RawData: "0x" + hex.EncodeToString(i.RawData), + ExceptionData: exceptionData, + LogIndex: fmt.Sprintf("0x%x", i.LogIndex), + Alias: (*Alias)(i), } return json.Marshal(aux) } @@ -1020,11 +1028,12 @@ func (i *Input) MarshalJSON() ([]byte, error) { func (i *Input) UnmarshalJSON(in []byte) error { type Alias Input aux := &struct { - EpochIndex string `json:"epoch_index"` - Index string `json:"index"` - BlockNumber string `json:"block_number"` - RawData string `json:"raw_data"` - LogIndex string `json:"log_index"` + EpochIndex string `json:"epoch_index"` + Index string `json:"index"` + BlockNumber string `json:"block_number"` + RawData string `json:"raw_data"` + ExceptionData *string `json:"exception_data"` + LogIndex string `json:"log_index"` *Alias }{Alias: (*Alias)(i)} @@ -1052,6 +1061,14 @@ func (i *Input) UnmarshalJSON(in []byte) error { if err != nil { return fmt.Errorf("error on RawData: %w", err) } + if aux.ExceptionData == nil { + i.ExceptionData = nil + } else { + i.ExceptionData, err = hexutil.Decode(*aux.ExceptionData) + if err != nil { + return fmt.Errorf("error on ExceptionData: %w", err) + } + } i.LogIndex, err = ParseHexUint64(aux.LogIndex) if err != nil { @@ -1064,16 +1081,11 @@ func (i *Input) UnmarshalJSON(in []byte) error { type InputCompletionStatus string const ( - InputCompletionStatus_None InputCompletionStatus = "NONE" - InputCompletionStatus_Accepted InputCompletionStatus = "ACCEPTED" - InputCompletionStatus_Rejected InputCompletionStatus = "REJECTED" - InputCompletionStatus_Exception InputCompletionStatus = "EXCEPTION" - InputCompletionStatus_MachineHalted InputCompletionStatus = "MACHINE_HALTED" - InputCompletionStatus_OutputsLimitExceeded InputCompletionStatus = "OUTPUTS_LIMIT_EXCEEDED" - InputCompletionStatus_ReportsLimitExceeded InputCompletionStatus = "REPORTS_LIMIT_EXCEEDED" - InputCompletionStatus_CycleLimitExceeded InputCompletionStatus = "CYCLE_LIMIT_EXCEEDED" - InputCompletionStatus_TimeLimitExceeded InputCompletionStatus = "TIME_LIMIT_EXCEEDED" - InputCompletionStatus_PayloadLengthLimitExceeded InputCompletionStatus = "PAYLOAD_LENGTH_LIMIT_EXCEEDED" + InputCompletionStatus_None InputCompletionStatus = "NONE" + InputCompletionStatus_Accepted InputCompletionStatus = "ACCEPTED" + InputCompletionStatus_Rejected InputCompletionStatus = "REJECTED" + InputCompletionStatus_Exception InputCompletionStatus = "EXCEPTION" + InputCompletionStatus_MachineHalted InputCompletionStatus = "MACHINE_HALTED" ) var InputCompletionStatusAllValues = []InputCompletionStatus{ @@ -1082,11 +1094,22 @@ var InputCompletionStatusAllValues = []InputCompletionStatus{ InputCompletionStatus_Rejected, InputCompletionStatus_Exception, InputCompletionStatus_MachineHalted, - InputCompletionStatus_OutputsLimitExceeded, - InputCompletionStatus_ReportsLimitExceeded, - InputCompletionStatus_CycleLimitExceeded, - InputCompletionStatus_TimeLimitExceeded, - InputCompletionStatus_PayloadLengthLimitExceeded, +} + +// IsCompleted reports whether the status is a deterministic completed result +// of an advance execution. NONE represents an input that has not completed. +func (e InputCompletionStatus) IsCompleted() bool { + switch e { + case InputCompletionStatus_Accepted, + InputCompletionStatus_Rejected, + InputCompletionStatus_Exception, + InputCompletionStatus_MachineHalted: + return true + case InputCompletionStatus_None: + return false + default: + return false + } } func (e *InputCompletionStatus) Scan(value any) error { @@ -1111,16 +1134,6 @@ func (e *InputCompletionStatus) Scan(value any) error { *e = InputCompletionStatus_Exception case "MACHINE_HALTED": *e = InputCompletionStatus_MachineHalted - case "OUTPUTS_LIMIT_EXCEEDED": - *e = InputCompletionStatus_OutputsLimitExceeded - case "REPORTS_LIMIT_EXCEEDED": - *e = InputCompletionStatus_ReportsLimitExceeded - case "CYCLE_LIMIT_EXCEEDED": - *e = InputCompletionStatus_CycleLimitExceeded - case "TIME_LIMIT_EXCEEDED": - *e = InputCompletionStatus_TimeLimitExceeded - case "PAYLOAD_LENGTH_LIMIT_EXCEEDED": - *e = InputCompletionStatus_PayloadLengthLimitExceeded default: return errors.New("invalid value '" + enumValue + "' for InputCompletionStatus enum") } diff --git a/internal/model/models_json_test.go b/internal/model/models_json_test.go index a4ebd9eeb..929d23dae 100644 --- a/internal/model/models_json_test.go +++ b/internal/model/models_json_test.go @@ -55,7 +55,8 @@ func TestInputJSONRoundtrip(t *testing.T) { Index: 7, BlockNumber: 12345, RawData: []byte{0xde, 0xad, 0xbe, 0xef}, - Status: InputCompletionStatus_Accepted, + Status: InputCompletionStatus_Exception, + ExceptionData: []byte{0xff, 0x00, 0x80}, MachineHash: &machineHash, TransactionHash: common.HexToHash("0x5678"), LogIndex: 11, @@ -68,6 +69,7 @@ func TestInputJSONRoundtrip(t *testing.T) { // LogIndex must be hex-encoded like the other uint64 fields. require.Contains(t, string(data), `"log_index":"0xb"`) + require.Contains(t, string(data), `"exception_data":"0xff0080"`) var decoded Input err = json.Unmarshal(data, &decoded) @@ -80,11 +82,24 @@ func TestInputJSONRoundtrip(t *testing.T) { require.Equal(t, original.BlockNumber, decoded.BlockNumber) require.Equal(t, original.RawData, decoded.RawData) require.Equal(t, original.Status, decoded.Status) + require.Equal(t, original.ExceptionData, decoded.ExceptionData) require.Equal(t, original.MachineHash, decoded.MachineHash) require.Equal(t, original.TransactionHash, decoded.TransactionHash) require.Equal(t, original.LogIndex, decoded.LogIndex) } +func TestInputJSONDistinguishesMissingAndEmptyExceptionData(t *testing.T) { + base := Input{} + data, err := json.Marshal(&base) + require.NoError(t, err) + require.Contains(t, string(data), `"exception_data":null`) + + base.ExceptionData = []byte{} + data, err = json.Marshal(&base) + require.NoError(t, err) + require.Contains(t, string(data), `"exception_data":"0x"`) +} + func TestInputUnmarshalJSONInvalidHex(t *testing.T) { validJSON := `{"epoch_index":"0x0","index":"0x0","block_number":"0x0","raw_data":"0x","log_index":"0x0"}` tests := []struct { @@ -102,6 +117,11 @@ func TestInputUnmarshalJSONInvalidHex(t *testing.T) { json: `{"epoch_index":"0x0","index":"0x0","block_number":"0x0","raw_data":"0x","log_index":"bad"}`, wantErr: "LogIndex", }, + { + name: "invalid ExceptionData", + json: `{"epoch_index":"0x0","index":"0x0","block_number":"0x0","raw_data":"0x","exception_data":"not-hex","log_index":"0x0"}`, + wantErr: "ExceptionData", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/internal/repository/postgres/application.go b/internal/repository/postgres/application.go index c62c34f43..c372c9db9 100644 --- a/internal/repository/postgres/application.go +++ b/internal/repository/postgres/application.go @@ -741,6 +741,7 @@ func (r *PostgresRepository) GetLastSnapshot(ctx context.Context, nameOrAddress table.Input.BlockNumber, table.Input.RawData, table.Input.Status, + table.Input.ExceptionData, table.Input.MachineHash, table.Input.OutputsHash, table.Input.TransactionHash, @@ -774,6 +775,7 @@ func (r *PostgresRepository) GetLastSnapshot(ctx context.Context, nameOrAddress &inp.BlockNumber, &inp.RawData, &inp.Status, + &inp.ExceptionData, &inp.MachineHash, &inp.OutputsHash, &inp.TransactionHash, diff --git a/internal/repository/postgres/bulk.go b/internal/repository/postgres/bulk.go index fc0d33a57..21f0423eb 100644 --- a/internal/repository/postgres/bulk.go +++ b/internal/repository/postgres/bulk.go @@ -5,6 +5,7 @@ package postgres import ( "context" + "errors" "fmt" "unsafe" @@ -251,6 +252,7 @@ func updateInput( appID int64, inputIndex uint64, status model.InputCompletionStatus, + exceptionData []byte, outputsHash common.Hash, machineHash common.Hash, ) error { @@ -258,11 +260,13 @@ func updateInput( updStmt := table.Input. UPDATE( table.Input.Status, + table.Input.ExceptionData, table.Input.MachineHash, table.Input.OutputsHash, ). SET( status, + exceptionData, machineHash[:], outputsHash[:], ). @@ -354,6 +358,16 @@ func (r *PostgresRepository) StoreAdvanceResult( appID int64, res *model.AdvanceResult, ) error { + if res == nil { + return errors.New("advance result must not be nil") + } + if !res.Status.IsCompleted() { + return fmt.Errorf("cannot store advance result with noncompleted status %q", res.Status) + } + if err := validateAdvanceExceptionData(res.Status, res.ExceptionData); err != nil { + return err + } + tx, err := r.db.Begin(ctx) if err != nil { return err @@ -373,13 +387,18 @@ func (r *PostgresRepository) StoreAdvanceResult( } if res.IsDaveConsensus { - err = insertStateHashes(ctx, tx, appID, res.EpochIndex, res.InputIndex, res.PeriodicStateHashes, res.MachineHash, res.PaddingRepetitions) + err = insertStateHashes( + ctx, tx, appID, res.EpochIndex, res.InputIndex, + res.PeriodicStateHashes, res.MachineHash, res.PaddingRepetitions, + ) if err != nil { return err } } - err = updateInput(ctx, tx, appID, res.InputIndex, res.Status, res.OutputsHash, res.MachineHash) + err = updateInput( + ctx, tx, appID, res.InputIndex, res.Status, res.ExceptionData, res.OutputsHash, res.MachineHash, + ) if err != nil { return err } @@ -398,6 +417,17 @@ func (r *PostgresRepository) StoreAdvanceResult( return tx.Commit(ctx) } +func validateAdvanceExceptionData(status model.InputCompletionStatus, data []byte) error { + switch { + case status == model.InputCompletionStatus_Exception && data == nil: + return errors.New("exception advance result must include exception data") + case status != model.InputCompletionStatus_Exception && data != nil: + return fmt.Errorf("advance result with status %q must not include exception data", status) + default: + return nil + } +} + func updateEpochClaim( ctx context.Context, tx pgx.Tx, diff --git a/internal/repository/postgres/db/rollupsdb/public/enum/inputcompletionstatus.go b/internal/repository/postgres/db/rollupsdb/public/enum/inputcompletionstatus.go index a9624fcb2..ce3410837 100644 --- a/internal/repository/postgres/db/rollupsdb/public/enum/inputcompletionstatus.go +++ b/internal/repository/postgres/db/rollupsdb/public/enum/inputcompletionstatus.go @@ -10,25 +10,15 @@ package enum import "github.com/go-jet/jet/v2/postgres" var InputCompletionStatus = &struct { - None postgres.StringExpression - Accepted postgres.StringExpression - Rejected postgres.StringExpression - Exception postgres.StringExpression - MachineHalted postgres.StringExpression - OutputsLimitExceeded postgres.StringExpression - ReportsLimitExceeded postgres.StringExpression - CycleLimitExceeded postgres.StringExpression - TimeLimitExceeded postgres.StringExpression - PayloadLengthLimitExceeded postgres.StringExpression + None postgres.StringExpression + Accepted postgres.StringExpression + Rejected postgres.StringExpression + Exception postgres.StringExpression + MachineHalted postgres.StringExpression }{ - None: postgres.NewEnumValue("NONE"), - Accepted: postgres.NewEnumValue("ACCEPTED"), - Rejected: postgres.NewEnumValue("REJECTED"), - Exception: postgres.NewEnumValue("EXCEPTION"), - MachineHalted: postgres.NewEnumValue("MACHINE_HALTED"), - OutputsLimitExceeded: postgres.NewEnumValue("OUTPUTS_LIMIT_EXCEEDED"), - ReportsLimitExceeded: postgres.NewEnumValue("REPORTS_LIMIT_EXCEEDED"), - CycleLimitExceeded: postgres.NewEnumValue("CYCLE_LIMIT_EXCEEDED"), - TimeLimitExceeded: postgres.NewEnumValue("TIME_LIMIT_EXCEEDED"), - PayloadLengthLimitExceeded: postgres.NewEnumValue("PAYLOAD_LENGTH_LIMIT_EXCEEDED"), + None: postgres.NewEnumValue("NONE"), + Accepted: postgres.NewEnumValue("ACCEPTED"), + Rejected: postgres.NewEnumValue("REJECTED"), + Exception: postgres.NewEnumValue("EXCEPTION"), + MachineHalted: postgres.NewEnumValue("MACHINE_HALTED"), } diff --git a/internal/repository/postgres/db/rollupsdb/public/table/input.go b/internal/repository/postgres/db/rollupsdb/public/table/input.go index 3d86c30a5..11c33e20c 100644 --- a/internal/repository/postgres/db/rollupsdb/public/table/input.go +++ b/internal/repository/postgres/db/rollupsdb/public/table/input.go @@ -23,6 +23,7 @@ type inputTable struct { BlockNumber postgres.ColumnFloat RawData postgres.ColumnBytea Status postgres.ColumnString + ExceptionData postgres.ColumnBytea MachineHash postgres.ColumnBytea OutputsHash postgres.ColumnBytea TransactionHash postgres.ColumnBytea @@ -77,6 +78,7 @@ func newInputTableImpl(schemaName, tableName, alias string) inputTable { BlockNumberColumn = postgres.FloatColumn("block_number") RawDataColumn = postgres.ByteaColumn("raw_data") StatusColumn = postgres.StringColumn("status") + ExceptionDataColumn = postgres.ByteaColumn("exception_data") MachineHashColumn = postgres.ByteaColumn("machine_hash") OutputsHashColumn = postgres.ByteaColumn("outputs_hash") TransactionHashColumn = postgres.ByteaColumn("transaction_hash") @@ -84,8 +86,8 @@ func newInputTableImpl(schemaName, tableName, alias string) inputTable { SnapshotURIColumn = postgres.StringColumn("snapshot_uri") CreatedAtColumn = postgres.TimestampzColumn("created_at") UpdatedAtColumn = postgres.TimestampzColumn("updated_at") - allColumns = postgres.ColumnList{EpochApplicationIDColumn, EpochIndexColumn, IndexColumn, BlockNumberColumn, RawDataColumn, StatusColumn, MachineHashColumn, OutputsHashColumn, TransactionHashColumn, LogIndexColumn, SnapshotURIColumn, CreatedAtColumn, UpdatedAtColumn} - mutableColumns = postgres.ColumnList{EpochIndexColumn, BlockNumberColumn, RawDataColumn, StatusColumn, MachineHashColumn, OutputsHashColumn, TransactionHashColumn, LogIndexColumn, SnapshotURIColumn, CreatedAtColumn, UpdatedAtColumn} + allColumns = postgres.ColumnList{EpochApplicationIDColumn, EpochIndexColumn, IndexColumn, BlockNumberColumn, RawDataColumn, StatusColumn, ExceptionDataColumn, MachineHashColumn, OutputsHashColumn, TransactionHashColumn, LogIndexColumn, SnapshotURIColumn, CreatedAtColumn, UpdatedAtColumn} + mutableColumns = postgres.ColumnList{EpochIndexColumn, BlockNumberColumn, RawDataColumn, StatusColumn, ExceptionDataColumn, MachineHashColumn, OutputsHashColumn, TransactionHashColumn, LogIndexColumn, SnapshotURIColumn, CreatedAtColumn, UpdatedAtColumn} defaultColumns = postgres.ColumnList{CreatedAtColumn, UpdatedAtColumn} ) @@ -99,6 +101,7 @@ func newInputTableImpl(schemaName, tableName, alias string) inputTable { BlockNumber: BlockNumberColumn, RawData: RawDataColumn, Status: StatusColumn, + ExceptionData: ExceptionDataColumn, MachineHash: MachineHashColumn, OutputsHash: OutputsHashColumn, TransactionHash: TransactionHashColumn, diff --git a/internal/repository/postgres/input.go b/internal/repository/postgres/input.go index 5d944d4c2..c826387a1 100644 --- a/internal/repository/postgres/input.go +++ b/internal/repository/postgres/input.go @@ -31,6 +31,7 @@ func (r *PostgresRepository) GetInput( table.Input.BlockNumber, table.Input.RawData, table.Input.Status, + table.Input.ExceptionData, table.Input.MachineHash, table.Input.OutputsHash, table.Input.TransactionHash, @@ -61,6 +62,7 @@ func (r *PostgresRepository) GetInput( &inp.BlockNumber, &inp.RawData, &inp.Status, + &inp.ExceptionData, &inp.MachineHash, &inp.OutputsHash, &inp.TransactionHash, @@ -94,6 +96,7 @@ func (r *PostgresRepository) GetLastInput( table.Input.BlockNumber, table.Input.RawData, table.Input.Status, + table.Input.ExceptionData, table.Input.MachineHash, table.Input.OutputsHash, table.Input.TransactionHash, @@ -126,6 +129,7 @@ func (r *PostgresRepository) GetLastInput( &inp.BlockNumber, &inp.RawData, &inp.Status, + &inp.ExceptionData, &inp.MachineHash, &inp.OutputsHash, &inp.TransactionHash, @@ -158,6 +162,7 @@ func (r *PostgresRepository) GetLastProcessedInput( table.Input.BlockNumber, table.Input.RawData, table.Input.Status, + table.Input.ExceptionData, table.Input.MachineHash, table.Input.OutputsHash, table.Input.TransactionHash, @@ -190,6 +195,7 @@ func (r *PostgresRepository) GetLastProcessedInput( &inp.BlockNumber, &inp.RawData, &inp.Status, + &inp.ExceptionData, &inp.MachineHash, &inp.OutputsHash, &inp.TransactionHash, @@ -267,6 +273,7 @@ func (r *PostgresRepository) ListInputs( table.Input.BlockNumber, table.Input.RawData, table.Input.Status, + table.Input.ExceptionData, table.Input.MachineHash, table.Input.OutputsHash, table.Input.TransactionHash, @@ -307,6 +314,7 @@ func (r *PostgresRepository) ListInputs( &in.BlockNumber, &in.RawData, &in.Status, + &in.ExceptionData, &in.MachineHash, &in.OutputsHash, &in.TransactionHash, diff --git a/internal/repository/postgres/input_exception_data_test.go b/internal/repository/postgres/input_exception_data_test.go new file mode 100644 index 000000000..8bb741452 --- /dev/null +++ b/internal/repository/postgres/input_exception_data_test.go @@ -0,0 +1,68 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package postgres_test + +import ( + "context" + "testing" + + "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/internal/repository/factory" + "github.com/cartesi/rollups-node/internal/repository/repotest" + "github.com/cartesi/rollups-node/test/tooling/db" + "github.com/jackc/pgx/v5" + "github.com/stretchr/testify/require" +) + +func TestPostgresInputExceptionDataContract(t *testing.T) { + endpoint, err := db.GetTestDatabaseEndpoint() + if err != nil { + t.Skipf("Skipping: %v", err) + } + require.NoError(t, db.SetupTestPostgres(endpoint)) + + ctx := context.Background() + repo, err := factory.NewRepositoryFromConnectionString(ctx, endpoint) + require.NoError(t, err) + t.Cleanup(repo.Close) + + conn, err := pgx.Connect(ctx, endpoint) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, conn.Close(ctx)) }) + + seed := repotest.Seed(ctx, t, repo) + machineHash := repotest.UniqueHash() + outputsHash := repotest.UniqueHash() + + _, err = conn.Exec(ctx, ` + UPDATE input + SET status = 'EXCEPTION', machine_hash = $2, outputs_hash = $3 + WHERE epoch_application_id = $1 AND index = 0`, + seed.App.ID, machineHash[:], outputsHash[:], + ) + requirePostgresConstraint(t, err, "input_exception_data_check") + + _, err = conn.Exec(ctx, ` + UPDATE input + SET status = 'ACCEPTED', exception_data = '\x01', machine_hash = $2, outputs_hash = $3 + WHERE epoch_application_id = $1 AND index = 0`, + seed.App.ID, machineHash[:], outputsHash[:], + ) + requirePostgresConstraint(t, err, "input_exception_data_check") + + require.NoError(t, repo.StoreAdvanceResult(ctx, seed.App.ID, &model.AdvanceResult{ + EpochIndex: seed.Epoch.Index, + InputIndex: seed.Input.Index, + Status: model.InputCompletionStatus_Exception, + ExceptionData: []byte{}, + OutputsProof: model.OutputsProof{ + MachineHash: machineHash, + OutputsHash: outputsHash, + }, + })) + completed, err := repo.GetInput(ctx, seed.App.IApplicationAddress.String(), seed.Input.Index) + require.NoError(t, err) + require.NotNil(t, completed.ExceptionData) + require.Empty(t, completed.ExceptionData) +} diff --git a/internal/repository/postgres/postgres_repo_test.go b/internal/repository/postgres/postgres_repo_test.go index 96e3a8e0d..642bacb0c 100644 --- a/internal/repository/postgres/postgres_repo_test.go +++ b/internal/repository/postgres/postgres_repo_test.go @@ -5,12 +5,16 @@ package postgres_test import ( "context" + "fmt" "testing" + "github.com/cartesi/rollups-node/internal/model" "github.com/cartesi/rollups-node/internal/repository" "github.com/cartesi/rollups-node/internal/repository/factory" "github.com/cartesi/rollups-node/internal/repository/repotest" "github.com/cartesi/rollups-node/test/tooling/db" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" "github.com/stretchr/testify/require" ) @@ -31,3 +35,89 @@ func TestPostgresRepository(t *testing.T) { return repo, func() { repo.Close() } }) } + +func TestPostgresSchemaExecutionOutcomeContract(t *testing.T) { + endpoint, err := db.GetTestDatabaseEndpoint() + if err != nil { + t.Skipf("Skipping: %v", err) + } + require.NoError(t, db.SetupTestPostgres(endpoint)) + + ctx := context.Background() + conn, err := pgx.Connect(ctx, endpoint) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, conn.Close(ctx)) }) + + rows, err := conn.Query(ctx, ` + SELECT enumlabel + FROM pg_enum + JOIN pg_type ON pg_type.oid = pg_enum.enumtypid + WHERE pg_type.typname = 'InputCompletionStatus' + ORDER BY enumsortorder`) + require.NoError(t, err) + labels, err := pgx.CollectRows(rows, pgx.RowTo[string]) + require.NoError(t, err) + require.Equal(t, []string{"NONE", "ACCEPTED", "REJECTED", "EXCEPTION", "MACHINE_HALTED"}, labels) + + rows, err = conn.Query(ctx, ` + SELECT column_name + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'execution_parameters'`) + require.NoError(t, err) + columns, err := pgx.CollectRows(rows, pgx.RowTo[string]) + require.NoError(t, err) + require.Contains(t, columns, "advance_max_cycles") + require.Contains(t, columns, "inspect_max_cycles") + require.Contains(t, columns, "advance_inc_cycles") + require.Contains(t, columns, "inspect_inc_cycles") + + for _, column := range []string{"advance_max_cycles", "inspect_max_cycles"} { + var defaultValue string + err := conn.QueryRow(ctx, ` + SELECT column_default + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'execution_parameters' + AND column_name = $1`, column).Scan(&defaultValue) + require.NoError(t, err) + require.Equal(t, "0", defaultValue) + } + + repo, err := factory.NewRepositoryFromConnectionString(ctx, endpoint) + require.NoError(t, err) + t.Cleanup(repo.Close) + app := repotest.NewApplicationBuilder().Create(ctx, t, repo) + for _, column := range []string{"advance_max_cycles", "inspect_max_cycles"} { + for _, value := range []int64{0, int64(model.MaxExecutionCycleSpan)} { + _, err := conn.Exec(ctx, fmt.Sprintf( + `UPDATE execution_parameters SET %s = $1 WHERE application_id = $2`, + column, + ), value, app.ID) + require.NoError(t, err, "%s must accept %d", column, value) + } + } + for _, test := range []struct { + column string + value int64 + }{ + {"advance_inc_cycles", 0}, + {"inspect_inc_cycles", 0}, + {"advance_max_cycles", -1}, + {"inspect_max_cycles", -1}, + {"advance_max_cycles", int64(model.MaxExecutionCycles)}, + {"inspect_max_cycles", int64(model.MaxExecutionCycles)}, + } { + _, err := conn.Exec(ctx, fmt.Sprintf( + `UPDATE execution_parameters SET %s = $1 WHERE application_id = $2`, + test.column, + ), test.value, app.ID) + requirePostgresConstraint(t, err, "execution_parameters_"+test.column+"_check") + } +} + +func requirePostgresConstraint(t *testing.T, err error, constraint string) { + t.Helper() + require.Error(t, err) + var pgErr *pgconn.PgError + require.ErrorAs(t, err, &pgErr) + require.Equal(t, constraint, pgErr.ConstraintName) +} diff --git a/internal/repository/postgres/schema/migrations/000001_create_initial_schema.up.sql b/internal/repository/postgres/schema/migrations/000001_create_initial_schema.up.sql index 82e79ae7a..7eb4a5808 100644 --- a/internal/repository/postgres/schema/migrations/000001_create_initial_schema.up.sql +++ b/internal/repository/postgres/schema/migrations/000001_create_initial_schema.up.sql @@ -15,12 +15,7 @@ CREATE TYPE "InputCompletionStatus" AS ENUM ( 'ACCEPTED', 'REJECTED', 'EXCEPTION', - 'MACHINE_HALTED', - 'OUTPUTS_LIMIT_EXCEEDED', - 'REPORTS_LIMIT_EXCEEDED', - 'CYCLE_LIMIT_EXCEEDED', - 'TIME_LIMIT_EXCEEDED', - 'PAYLOAD_LENGTH_LIMIT_EXCEEDED'); + 'MACHINE_HALTED'); CREATE TYPE "DefaultBlock" AS ENUM ('FINALIZED', 'LATEST', 'PENDING', 'SAFE'); @@ -390,6 +385,7 @@ CREATE TABLE "input" "block_number" uint64 NOT NULL, "raw_data" BYTEA NOT NULL, "status" "InputCompletionStatus" NOT NULL, + "exception_data" BYTEA, "machine_hash" hash, "outputs_hash" hash, "transaction_hash" hash NOT NULL, @@ -397,6 +393,11 @@ CREATE TABLE "input" "snapshot_uri" VARCHAR(4096), "created_at" TIMESTAMPTZ NOT NULL DEFAULT NOW(), "updated_at" TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT "input_exception_data_check" CHECK ( + ("status" = 'EXCEPTION' AND "exception_data" IS NOT NULL) + OR + ("status" <> 'EXCEPTION' AND "exception_data" IS NULL) + ), CONSTRAINT "input_pkey" PRIMARY KEY ("epoch_application_id", "index"), CONSTRAINT "input_epoch_index_unique" UNIQUE ("epoch_application_id", "epoch_index", "index"), CONSTRAINT "input_application_id_tx_hash_log_index_unique" UNIQUE ("epoch_application_id", "transaction_hash", "log_index"), diff --git a/internal/repository/repotest/bulk_test_cases.go b/internal/repository/repotest/bulk_test_cases.go index 9ee5c374f..69c8e6e4b 100644 --- a/internal/repository/repotest/bulk_test_cases.go +++ b/internal/repository/repotest/bulk_test_cases.go @@ -24,6 +24,11 @@ func NewBulkOperationsSuite(factory RepositoryFactory) *BulkOperationsSuite { } func (s *BulkOperationsSuite) TestStoreAdvanceResult() { + s.Run("RejectsNilResult", func() { + err := s.Repo.StoreAdvanceResult(s.Ctx, 0, nil) + s.Require().EqualError(err, "advance result must not be nil") + }) + s.Run("AcceptedInput", func() { seed := Seed(s.Ctx, s.T(), s.Repo) machineHash := crypto.Keccak256Hash([]byte("machine")) @@ -89,6 +94,89 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { s.Equal(InputCompletionStatus_Rejected, input.Status) }) + for _, status := range []InputCompletionStatus{ + InputCompletionStatus_Exception, + InputCompletionStatus_MachineHalted, + } { + s.Run("CompletedStatus/"+status.String(), func() { + seed := Seed(s.Ctx, s.T(), s.Repo) + var exceptionData []byte + if status == InputCompletionStatus_Exception { + exceptionData = []byte{0xff, 0x00, 0x80} + } + result := &AdvanceResult{ + EpochIndex: 0, + InputIndex: 0, + Status: status, + ExceptionData: exceptionData, + OutputsProof: OutputsProof{ + MachineHash: UniqueHash(), + }, + } + + err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result) + s.Require().NoError(err) + input, err := s.Repo.GetInput(s.Ctx, seed.App.IApplicationAddress.String(), 0) + s.Require().NoError(err) + s.Equal(status, input.Status) + s.Equal(exceptionData, input.ExceptionData) + }) + } + + for _, test := range []struct { + name string + status InputCompletionStatus + exceptionData []byte + }{ + {"ExceptionWithoutData", InputCompletionStatus_Exception, nil}, + {"AcceptedWithExceptionData", InputCompletionStatus_Accepted, []byte("unexpected")}, + } { + s.Run("Rejects"+test.name, func() { + seed := Seed(s.Ctx, s.T(), s.Repo) + err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, &AdvanceResult{ + EpochIndex: 0, + InputIndex: 0, + Status: test.status, + ExceptionData: test.exceptionData, + OutputsProof: OutputsProof{ + MachineHash: UniqueHash(), + OutputsHash: UniqueHash(), + }, + }) + s.Require().ErrorContains(err, "exception data") + }) + } + + invalidStatuses := []InputCompletionStatus{ + InputCompletionStatus_None, + "OUTPUTS_LIMIT_EXCEEDED", + "REPORTS_LIMIT_EXCEEDED", + "CYCLE_LIMIT_EXCEEDED", + "TIME_LIMIT_EXCEEDED", + "PAYLOAD_LENGTH_LIMIT_EXCEEDED", + "INVALID", + } + for _, status := range invalidStatuses { + s.Run("RejectsNoncompletedStatus/"+status.String(), func() { + seed := Seed(s.Ctx, s.T(), s.Repo) + result := &AdvanceResult{ + EpochIndex: 0, + InputIndex: 0, + Status: status, + Outputs: [][]byte{[]byte("must-not-be-stored")}, + OutputsProof: OutputsProof{ + MachineHash: UniqueHash(), + }, + } + + err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result) + s.Require().ErrorContains(err, "noncompleted status") + input, err := s.Repo.GetInput(s.Ctx, seed.App.IApplicationAddress.String(), 0) + s.Require().NoError(err) + s.Equal(InputCompletionStatus_None, input.Status) + }) + } + s.Run("WithNoOutputsOrReports", func() { seed := Seed(s.Ctx, s.T(), s.Repo) machineHash := crypto.Keccak256Hash([]byte("machine-empty")) diff --git a/internal/repository/repotest/repotest.go b/internal/repository/repotest/repotest.go index 282317eef..4cbb529e5 100644 --- a/internal/repository/repotest/repotest.go +++ b/internal/repository/repotest/repotest.go @@ -54,6 +54,9 @@ func StoreAdvanceResult( MachineHash: UniqueHash(), }, } + if status == InputCompletionStatus_Exception { + result.ExceptionData = []byte{} + } err := repo.StoreAdvanceResult(ctx, appID, result) require.NoError(t, err) } From a35fc06856d19e59e07192809f914e394f687018 Mon Sep 17 00:00:00 2001 From: Victor Fusco <1221933+vfusco@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:34:33 -0300 Subject: [PATCH 07/13] refactor(repository): validate and stream PRT state hashes Validate canonical input hash-collection spans and index ranges before persistence, then stream rows through PostgreSQL COPY so full-span collections do not allocate a second multi-million-row statement representation. --- internal/repository/postgres/bulk.go | 155 +++++++++++++----- .../repository/repotest/bulk_test_cases.go | 121 +++++++++++++- .../repotest/state_hash_test_cases.go | 21 ++- internal/validator/validator.go | 19 +-- internal/validator/validator_test.go | 17 +- 5 files changed, 252 insertions(+), 81 deletions(-) diff --git a/internal/repository/postgres/bulk.go b/internal/repository/postgres/bulk.go index 21f0423eb..cf57379a8 100644 --- a/internal/repository/postgres/bulk.go +++ b/internal/repository/postgres/bulk.go @@ -7,6 +7,7 @@ import ( "context" "errors" "fmt" + "math" "unsafe" "github.com/ethereum/go-ethereum/common" @@ -96,23 +97,24 @@ func getStateHashNextIndex( epochIndex uint64, ) (uint64, error) { - query := table.StateHashes.SELECT( - postgres.COALESCE( - postgres.Float(1).ADD(postgres.MAXf(table.StateHashes.Index)), - postgres.Float(0), - ), - ).WHERE( - table.StateHashes.InputEpochApplicationID.EQ(postgres.Int64(appID)). - AND(table.StateHashes.EpochIndex.EQ(uint64Expr(epochIndex))), - ) - + query := table.StateHashes.SELECT(table.StateHashes.Index). + WHERE( + table.StateHashes.InputEpochApplicationID.EQ(postgres.Int64(appID)). + AND(table.StateHashes.EpochIndex.EQ(uint64Expr(epochIndex))), + ). + ORDER_BY(table.StateHashes.Index.DESC()). + LIMIT(1) queryStr, args := query.Sql() - var currentIndex uint64 - err := tx.QueryRow(ctx, queryStr, args...).Scan(¤tIndex) - if err != nil { - return 0, fmt.Errorf("failed to get the next state hash index: %w", err) + var maxIndex uint64 + if err := tx.QueryRow(ctx, queryStr, args...).Scan(&maxIndex); errors.Is(err, pgx.ErrNoRows) { + return 0, nil + } else if err != nil { + return 0, fmt.Errorf("failed to get the last state hash index: %w", err) } - return currentIndex, nil + if maxIndex == math.MaxUint64 { + return 0, errors.New("state hash index space is exhausted") + } + return maxIndex + 1, nil } func insertOutputs( @@ -201,51 +203,116 @@ func insertStateHashes( inputIndex uint64, hashes [][32]byte, machineHash common.Hash, - remainingMetaCycles uint64, + paddingRepetitions uint64, ) error { + rowCount, err := stateHashInsertShape(uint64(len(hashes)), paddingRepetitions) + if err != nil { + return err + } nextIndex, err := getStateHashNextIndex(ctx, tx, appID, epochIndex) if err != nil { return err } - - stmt := table.StateHashes.INSERT( - table.StateHashes.InputEpochApplicationID, - table.StateHashes.EpochIndex, - table.StateHashes.InputIndex, - table.StateHashes.Index, - table.StateHashes.MachineHash, - table.StateHashes.Repetitions, - ) - - for i, h := range hashes { - stmt = stmt.VALUES( - appID, - epochIndex, - inputIndex, - nextIndex+uint64(i), - h[:], - 1, + if rowCount-1 > math.MaxUint64-nextIndex { + return fmt.Errorf( + "state hash index range overflows uint64: start=%d rows=%d", + nextIndex, + rowCount, ) } - stmt = stmt.VALUES( - appID, - epochIndex, - inputIndex, - nextIndex+uint64(len(hashes)), - machineHash[:], - remainingMetaCycles, + source := &stateHashCopySource{ + appID: appID, + epochIndex: epochIndex, + inputIndex: inputIndex, + nextIndex: nextIndex, + hashes: hashes, + machineHash: machineHash, + paddingRepetitions: paddingRepetitions, + rowCount: rowCount, + } + copied, err := tx.CopyFrom( + ctx, + pgx.Identifier{"public", "state_hashes"}, + []string{ + "input_epoch_application_id", + "epoch_index", + "input_index", + "index", + "machine_hash", + "repetitions", + }, + source, ) - - sqlStr, args := stmt.Sql() - _, err = tx.Exec(ctx, sqlStr, args...) if err != nil { return err } + expectedCopied := int64(rowCount) //nolint:gosec // Span validation bounds rowCount far below MaxInt64. + if copied != expectedCopied { + return fmt.Errorf("copied %d state hashes, expected %d", copied, expectedCopied) + } return nil } +// stateHashCopySource streams one input hash collection directly into PostgreSQL. It +// holds only the caller's hash slice and one row of CopyFrom values, so even a +// full input-span collection does not create a second multi-million-row object. +type stateHashCopySource struct { + appID int64 + epochIndex uint64 + inputIndex uint64 + nextIndex uint64 + hashes [][32]byte + machineHash common.Hash + paddingRepetitions uint64 + rowCount uint64 + nextRow uint64 + currentRow uint64 + values [6]any +} + +func (source *stateHashCopySource) Next() bool { + if source.nextRow >= source.rowCount { + return false + } + source.currentRow = source.nextRow + source.nextRow++ + return true +} + +func (source *stateHashCopySource) Values() ([]any, error) { + source.values[0] = source.appID + source.values[1] = source.epochIndex + source.values[2] = source.inputIndex + source.values[3] = source.nextIndex + source.currentRow + if source.currentRow < uint64(len(source.hashes)) { + source.values[4] = source.hashes[source.currentRow][:] + source.values[5] = int64(1) + return source.values[:], nil + } + source.values[4] = source.machineHash[:] + source.values[5] = source.paddingRepetitions + return source.values[:], nil +} + +func (*stateHashCopySource) Err() error { return nil } + +func stateHashInsertShape( + hashCount uint64, + paddingRepetitions uint64, +) (rowCount uint64, err error) { + if err := model.ValidateInputHashCollectionSpan(hashCount, paddingRepetitions); err != nil { + return 0, fmt.Errorf("invalid input hash collection span: %w", err) + } + if paddingRepetitions == 0 { + return 0, errors.New("canonical input hash collection requires a positive final repetition tail") + } + // Span validation bounds hashCount by the collection capacity, so appending the + // required canonical tail row cannot overflow uint64. + return hashCount + 1, nil +} + func updateInput( ctx context.Context, tx pgx.Tx, diff --git a/internal/repository/repotest/bulk_test_cases.go b/internal/repository/repotest/bulk_test_cases.go index 69c8e6e4b..001e996d1 100644 --- a/internal/repository/repotest/bulk_test_cases.go +++ b/internal/repository/repotest/bulk_test_cases.go @@ -256,14 +256,15 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { hash1 := [32]byte(crypto.Keccak256Hash([]byte("state-1"))) hash2 := [32]byte(crypto.Keccak256Hash([]byte("state-2"))) hash3 := [32]byte(crypto.Keccak256Hash([]byte("state-3"))) + hashes := [][32]byte{hash1, hash2, hash3} result := &AdvanceResult{ EpochIndex: 0, InputIndex: 0, Status: InputCompletionStatus_Accepted, Outputs: [][]byte{[]byte("dave-output")}, - PeriodicStateHashes: [][32]byte{hash1, hash2, hash3}, - PaddingRepetitions: 42, + PeriodicStateHashes: hashes, + PaddingRepetitions: InputHashCollectionCapacity - uint64(len(hashes)), IsDaveConsensus: true, OutputsProof: OutputsProof{ OutputsHash: outputsHash, @@ -274,15 +275,16 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result) s.Require().NoError(err) - // Verify state hashes were created (3 intermediate + 1 final = 4) + // Verify one row per intermediate hash plus the final repetition tail. epochIdx := uint64(0) stateHashes, total, err := s.Repo.ListStateHashes( s.Ctx, seed.App.IApplicationAddress.String(), repository.StateHashFilter{EpochIndex: &epochIdx}, repository.Pagination{Limit: 10}, false) s.Require().NoError(err) - s.Len(stateHashes, 4) - s.Equal(uint64(4), total) + expectedStateHashRows := len(hashes) + 1 + s.Len(stateHashes, expectedStateHashRows) + s.Equal(uint64(expectedStateHashRows), total) // Verify intermediate hashes have Repetitions=1 s.Equal(common.Hash(hash1), stateHashes[0].MachineHash) @@ -293,8 +295,9 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { s.Equal(uint64(1), stateHashes[2].Repetitions) // Verify final hash has PaddingRepetitions as Repetitions - s.Equal(machineHash, stateHashes[3].MachineHash) - s.Equal(uint64(42), stateHashes[3].Repetitions) + tail := stateHashes[len(hashes)] + s.Equal(machineHash, tail.MachineHash) + s.Equal(result.PaddingRepetitions, tail.Repetitions) // Verify outputs were also created outputs, _, err := s.Repo.ListOutputs( @@ -308,6 +311,105 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { s.Require().NoError(err) s.Equal(InputCompletionStatus_Accepted, input.Status) }) + + const resultPageLimit = 10 + const hashesAboveExtendedProtocolParameterLimit = 11_000 + + s.Run("DaveConsensusStreamsStateHashesAboveParameterLimit", func() { + seed := Seed(s.Ctx, s.T(), s.Repo) + hashes := make([][32]byte, hashesAboveExtendedProtocolParameterLimit) + machineHash := UniqueHash() + result := &AdvanceResult{ + EpochIndex: 0, + InputIndex: 0, + Status: InputCompletionStatus_Accepted, + PeriodicStateHashes: hashes, + PaddingRepetitions: InputHashCollectionCapacity - uint64(len(hashes)), + IsDaveConsensus: true, + OutputsProof: OutputsProof{ + OutputsHash: UniqueHash(), + MachineHash: machineHash, + }, + } + + s.Require().NoError(s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result)) + epochIndex := uint64(0) + expectedRows := len(hashes) + 1 + stateHashes, total, err := s.Repo.ListStateHashes( + s.Ctx, + seed.App.IApplicationAddress.String(), + repository.StateHashFilter{EpochIndex: &epochIndex}, + repository.Pagination{Limit: uint64(expectedRows)}, + false, + ) + s.Require().NoError(err) + s.Equal(uint64(expectedRows), total) + s.Require().Len(stateHashes, expectedRows) + for _, index := range []int{0, len(hashes) / 2, len(hashes) - 1} { + s.Equal(common.Hash(hashes[index]), stateHashes[index].MachineHash) + s.Equal(uint64(index), stateHashes[index].Index) + s.Equal(uint64(1), stateHashes[index].Repetitions) + } + tail := stateHashes[len(hashes)] + s.Equal(machineHash, tail.MachineHash) + s.Equal(uint64(len(hashes)), tail.Index) + s.Equal(InputHashCollectionCapacity-uint64(len(hashes)), tail.Repetitions) + }) + + s.Run("PRTConsensusRejectsUnnormalizedExactBoundaryHashCollection", func() { + seed := Seed(s.Ctx, s.T(), s.Repo) + result := &AdvanceResult{ + EpochIndex: 0, + InputIndex: 0, + Status: InputCompletionStatus_Accepted, + Outputs: [][]byte{[]byte("must-roll-back")}, + Reports: [][]byte{[]byte("must-roll-back")}, + PaddingRepetitions: 0, + IsDaveConsensus: true, + OutputsProof: OutputsProof{ + OutputsHash: UniqueHash(), + MachineHash: UniqueHash(), + }, + } + + err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result) + s.Require().Error(err) + + input, err := s.Repo.GetInput(s.Ctx, seed.App.IApplicationAddress.String(), 0) + s.Require().NoError(err) + s.Equal(InputCompletionStatus_None, input.Status) + outputs, outputCount, err := s.Repo.ListOutputs( + s.Ctx, + seed.App.IApplicationAddress.String(), + repository.OutputFilter{}, + repository.Pagination{Limit: resultPageLimit}, + false, + ) + s.Require().NoError(err) + s.Empty(outputs) + s.Zero(outputCount) + reports, reportCount, err := s.Repo.ListReports( + s.Ctx, + seed.App.IApplicationAddress.String(), + repository.ReportFilter{}, + repository.Pagination{Limit: resultPageLimit}, + false, + ) + s.Require().NoError(err) + s.Empty(reports) + s.Zero(reportCount) + epochIndex := uint64(0) + stateHashes, stateHashCount, err := s.Repo.ListStateHashes( + s.Ctx, + seed.App.IApplicationAddress.String(), + repository.StateHashFilter{EpochIndex: &epochIndex}, + repository.Pagination{Limit: resultPageLimit}, + false, + ) + s.Require().NoError(err) + s.Empty(stateHashes) + s.Zero(stateHashCount) + }) } func (s *BulkOperationsSuite) TestStoreAdvanceResultRollback() { @@ -404,14 +506,15 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResultRollback() { // a bad epoch index), all prior work (outputs, reports) is rolled back. s.Run("DaveConsensusRollbackOnStateHashFailure", func() { seed := Seed(s.Ctx, s.T(), s.Repo) + hashes := [][32]byte{{1}, {2}} result := &AdvanceResult{ EpochIndex: 99, // non-existent epoch InputIndex: 0, Status: InputCompletionStatus_Accepted, Outputs: [][]byte{[]byte("should-be-rolled-back")}, - PeriodicStateHashes: [][32]byte{{1}, {2}}, - PaddingRepetitions: 10, + PeriodicStateHashes: hashes, + PaddingRepetitions: InputHashCollectionCapacity - uint64(len(hashes)), IsDaveConsensus: true, OutputsProof: OutputsProof{ OutputsHash: UniqueHash(), diff --git a/internal/repository/repotest/state_hash_test_cases.go b/internal/repository/repotest/state_hash_test_cases.go index c13abfee2..59992c708 100644 --- a/internal/repository/repotest/state_hash_test_cases.go +++ b/internal/repository/repotest/state_hash_test_cases.go @@ -49,13 +49,14 @@ func (s *StateHashSuite) TestListStateHashes() { hash1 := [32]byte(crypto.Keccak256Hash([]byte("list-state-1"))) hash2 := [32]byte(crypto.Keccak256Hash([]byte("list-state-2"))) + collectedHashes := [][32]byte{hash1, hash2} result := &AdvanceResult{ EpochIndex: 0, InputIndex: 0, Status: InputCompletionStatus_Accepted, - PeriodicStateHashes: [][32]byte{hash1, hash2}, - PaddingRepetitions: 10, + PeriodicStateHashes: collectedHashes, + PaddingRepetitions: InputHashCollectionCapacity - uint64(len(collectedHashes)), IsDaveConsensus: true, OutputsProof: OutputsProof{ OutputsHash: outputsHash, @@ -67,12 +68,13 @@ func (s *StateHashSuite) TestListStateHashes() { s.Require().NoError(err) // List all state hashes + expectedRows := len(collectedHashes) + 1 hashes, total, err := s.Repo.ListStateHashes( s.Ctx, seed.App.IApplicationAddress.String(), repository.StateHashFilter{}, repository.Pagination{Limit: 10}, false) s.Require().NoError(err) - s.Len(hashes, 3) // 2 intermediate + 1 final - s.Equal(uint64(3), total) + s.Len(hashes, expectedRows) + s.Equal(uint64(expectedRows), total) // List with epoch filter epochIdx := uint64(0) @@ -81,16 +83,17 @@ func (s *StateHashSuite) TestListStateHashes() { repository.StateHashFilter{EpochIndex: &epochIdx}, repository.Pagination{Limit: 10}, false) s.Require().NoError(err) - s.Len(hashes, 3) - s.Equal(uint64(3), total) + s.Len(hashes, expectedRows) + s.Equal(uint64(expectedRows), total) // Verify pagination + pageSize := uint64(len(collectedHashes)) hashes, total, err = s.Repo.ListStateHashes( s.Ctx, seed.App.IApplicationAddress.String(), repository.StateHashFilter{}, - repository.Pagination{Limit: 2, Offset: 0}, false) + repository.Pagination{Limit: pageSize, Offset: 0}, false) s.Require().NoError(err) - s.Len(hashes, 2) - s.Equal(uint64(3), total) + s.Len(hashes, len(collectedHashes)) + s.Equal(uint64(expectedRows), total) }) } diff --git a/internal/validator/validator.go b/internal/validator/validator.go index 2bd74a94e..dade2609d 100644 --- a/internal/validator/validator.go +++ b/internal/validator/validator.go @@ -18,7 +18,6 @@ import ( "github.com/cartesi/rollups-node/internal/merkle" . "github.com/cartesi/rollups-node/internal/model" "github.com/cartesi/rollups-node/internal/repository" - pkgm "github.com/cartesi/rollups-node/pkg/machine" "github.com/cartesi/rollups-node/pkg/service" ) @@ -351,10 +350,10 @@ func (s *Service) buildCommitment(ctx context.Context, app *Application, epoch * } builder := merkle.Builder{} inputCount := epoch.InputIndexUpperBound - epoch.InputIndexLowerBound - if inputCount > pkgm.MaxAdvanceStatesPerEpoch { + if inputCount > MaxAdvanceStatesPerEpoch { return nil, nil, s.setApplicationCorrupted(ctx, app, "input count is too large for epoch %v of application %v: max %v, got %v", - epoch.Index, app.Name, pkgm.MaxAdvanceStatesPerEpoch, inputCount) + epoch.Index, app.Name, MaxAdvanceStatesPerEpoch, inputCount) } if inputCount > 0 { @@ -382,11 +381,11 @@ func (s *Service) buildCommitment(ctx context.Context, app *Application, epoch * } } - remainingInputs := pkgm.MaxAdvanceStatesPerEpoch - inputCount - // Safe: inputCount ≤ MaxAdvanceStatesPerEpoch enforced above, so remainingInputs << Log2InputEntryCapacity won't overflow. - remainingStrides := remainingInputs << pkgm.Log2InputEntryCapacity - if remainingStrides > 0 { - if err := builder.AppendRepeatedUint64(merkle.TreeLeaf(*epoch.MachineHash), remainingStrides); err != nil { + remainingInputs := MaxAdvanceStatesPerEpoch - inputCount + // Safe: inputCount is bounded above, so the remaining-input shift cannot overflow. + remainingEntries := remainingInputs << Log2InputHashCollectionCapacity + if remainingEntries > 0 { + if err := builder.AppendRepeatedUint64(merkle.TreeLeaf(*epoch.MachineHash), remainingEntries); err != nil { return nil, nil, s.setApplicationCorrupted(ctx, app, "failed to append state hash to builder for epoch %d of application %s with error: %v", epoch.Index, app.Name, err) } @@ -397,8 +396,8 @@ func (s *Service) buildCommitment(ctx context.Context, app *Application, epoch * return nil, nil, s.setApplicationCorrupted(ctx, app, "failed to build commitment for epoch %d of application %s with error: %v", epoch.Index, app.Name, err) } - // The commitment geometry is fixed: 2²⁴ inputs × 2²⁴ strides ⇒ height 48. - const expectedHeight = pkgm.Log2MaxAdvanceStatesPerEpoch + pkgm.Log2InputEntryCapacity // 48 + // The commitment geometry is fixed: 2²⁴ inputs × 2²⁴ entries ⇒ height 48. + const expectedHeight = Log2EpochComputationHashLeafCount if uint64(epochCommitmentTree.Height) != expectedHeight { return nil, nil, s.setApplicationCorrupted(ctx, app, "epoch %v commitment tree height %v, expected %v — state hash repetitions are inconsistent", diff --git a/internal/validator/validator_test.go b/internal/validator/validator_test.go index 4d6428029..5f9888259 100644 --- a/internal/validator/validator_test.go +++ b/internal/validator/validator_test.go @@ -12,7 +12,6 @@ import ( "github.com/cartesi/rollups-node/internal/merkle" . "github.com/cartesi/rollups-node/internal/model" "github.com/cartesi/rollups-node/internal/repository" - pkgm "github.com/cartesi/rollups-node/pkg/machine" "github.com/cartesi/rollups-node/pkg/service" "github.com/ethereum/go-ethereum/common" "github.com/stretchr/testify/mock" @@ -822,14 +821,14 @@ func (s *ValidatorSuite) TestBuildCommitment() { } // 5 inputs, each with one state hash covering the full - // strides-per-input count (1< Date: Wed, 12 Aug 2026 16:35:39 -0300 Subject: [PATCH 08/13] refactor(inspect): expose typed completion outcomes Carry machine completion status and exception payloads through the manager and HTTP API. Distinguish deterministic guest outcomes from incomplete execution failures while preserving partial reports and sanitizing internal diagnostics. --- api/openapi/inspect.yaml | 31 ++--- internal/advancer/advancer_test.go | 2 +- internal/inspect/hardening_test.go | 2 +- internal/inspect/inspect.go | 118 +++++++++++++------ internal/inspect/inspect_test.go | 180 ++++++++++++++++++++++++++++- internal/manager/instance.go | 23 ++-- internal/manager/instance_test.go | 64 +++++++--- internal/manager/manager_test.go | 2 +- internal/manager/types.go | 12 ++ internal/model/models.go | 7 -- pkg/inspectclient/generated.go | 39 ++++--- 11 files changed, 369 insertions(+), 111 deletions(-) diff --git a/api/openapi/inspect.yaml b/api/openapi/inspect.yaml index 65eeef0fc..2a88cf557 100644 --- a/api/openapi/inspect.yaml +++ b/api/openapi/inspect.yaml @@ -18,7 +18,7 @@ paths: description: | This POST method sends an inspect-state request to the DApp backend, using the body contents as a binary payload for the inspect method. - The response includes a status string and reports generated by the DApp backend. If an exception occurs, the `exception_payload` field will contain the exception details; otherwise, this field will be null. + The response includes the inspection completion status, reports generated by the DApp backend, and the number of previously processed inputs. The optional `error` field contains a sanitized description when the status is `Exception` or `Failed`. For `Exception`, `exception_data` contains the raw CMIO payload supplied by the guest. The inspect operation is executed on a temporary fork of the machine created upon request arrival, which is discarded afterward. Note that this method is synchronous and not recommended for resource-intensive operations. @@ -88,10 +88,17 @@ components: properties: status: $ref: "#/components/schemas/CompletionStatus" - exception_payload: - $ref: "#/components/schemas/Payload" + error: + type: string + description: Sanitized error description, present only when status is Exception or Failed + example: "The node could not complete the inspection" + exception_data: + allOf: + - $ref: "#/components/schemas/Payload" + description: Raw guest-provided CMIO exception payload, present only when status is Exception reports: type: array + description: Reports emitted before completion; for Failed, this may be only a partial prefix items: $ref: "#/components/schemas/Report" processed_input_count: @@ -100,22 +107,18 @@ components: example: 0 required: - status - - exception_payload - reports - processed_input_count CompletionStatus: type: string - description: Whether inspection completed or not (and why not) - enum: - [ - Accepted, - Rejected, - Exception, - MachineHalted, - CycleLimitExceeded, - TimeLimitExceeded, - ] + description: | + How inspection completed. MachineHalted means the temporary inspect + execution halted; the canonical application machine is unchanged. + Failed means the node could not complete inspection because of an + operational limit, cancellation, timeout, protocol error, or internal + failure. Reports returned with Failed may be partial. + enum: [Accepted, Rejected, Exception, MachineHalted, Failed] example: "Accepted" Payload: diff --git a/internal/advancer/advancer_test.go b/internal/advancer/advancer_test.go index b1ac371d6..50936bfe6 100644 --- a/internal/advancer/advancer_test.go +++ b/internal/advancer/advancer_test.go @@ -2011,7 +2011,7 @@ func (m *MockMachineInstance) Advance(ctx context.Context, input []byte, epochIn } // Inspect implements the MachineInstance interface for testing -func (m *MockMachineInstance) Inspect(ctx context.Context, query []byte) (*InspectResult, error) { +func (m *MockMachineInstance) Inspect(ctx context.Context, query []byte) (*manager.InspectResult, error) { // Not used in advancer tests, but needed to satisfy the interface return nil, nil } diff --git a/internal/inspect/hardening_test.go b/internal/inspect/hardening_test.go index 63e107465..c088a9c56 100644 --- a/internal/inspect/hardening_test.go +++ b/internal/inspect/hardening_test.go @@ -83,7 +83,7 @@ type erroringMachine struct { err error } -func (m *erroringMachine) Inspect(_ context.Context, _ []byte) (*InspectResult, error) { +func (m *erroringMachine) Inspect(_ context.Context, _ []byte) (*manager.InspectResult, error) { if m.err == errPanicSentinel { panic("boom-from-machine") } diff --git a/internal/inspect/inspect.go b/internal/inspect/inspect.go index d5c378197..54c3e4c2e 100644 --- a/internal/inspect/inspect.go +++ b/internal/inspect/inspect.go @@ -17,6 +17,7 @@ import ( "github.com/cartesi/rollups-node/internal/manager" . "github.com/cartesi/rollups-node/internal/model" + pkgmachine "github.com/cartesi/rollups-node/pkg/machine" "github.com/cartesi/rollups-node/pkg/service" "github.com/ethereum/go-ethereum/common/hexutil" ) @@ -31,6 +32,11 @@ const maxPayloadSize = 1 << 21 // 2 MiB // serialization after the Cartesi Machine's inspect deadline fires. const inspectResponseHeadroom = 30 * time.Second +const ( + inspectStatusFailed = "Failed" + inspectFailureMessage = "The node could not complete the inspection" +) + var ( ErrInvalidMachines = errors.New("machines must not be nil") ErrNoApp = errors.New("no application") @@ -67,7 +73,8 @@ type ReportResponse struct { type InspectResponse struct { Status string `json:"status"` - Exception string `json:"exception,omitempty"` + Error string `json:"error,omitempty"` + ExceptionData string `json:"exception_data,omitempty"` Reports []ReportResponse `json:"reports"` ProcessedInputs uint64 `json:"processed_input_count"` } @@ -157,23 +164,16 @@ func (inspect *Inspector) Admission() *service.SemaphoreAdmission { } func (inspect *Inspector) ServeHTTP(w http.ResponseWriter, r *http.Request) { - var ( - dapp string - payload []byte - err error - reports []ReportResponse - status string - errorMessage string - ) + requestID := service.RequestIDFromContext(r.Context()) + dapp := r.PathValue("dapp") - if r.PathValue("dapp") == "" { + if dapp == "" { inspect.Logger.Info("Bad request", "err", "Missing application address") http.Error(w, "Missing application address", http.StatusBadRequest) return } - dapp = r.PathValue("dapp") if r.Method != http.MethodPost { inspect.Logger.Info("HTTP method not allowed", "application", dapp, "method", r.Method) w.Header().Set("Allow", http.MethodPost) @@ -185,7 +185,7 @@ func (inspect *Inspector) ServeHTTP(w http.ResponseWriter, r *http.Request) { // both enforces the limit and signals the server to close the connection // on over-limit so clients can't pipeline further requests on it. r.Body = http.MaxBytesReader(w, r.Body, maxPayloadSize) - payload, err = io.ReadAll(r.Body) + payload, err := io.ReadAll(r.Body) if err != nil { var maxErr *http.MaxBytesError if errors.As(err, &maxErr) { @@ -244,27 +244,7 @@ func (inspect *Inspector) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - for _, report := range result.Reports { - reports = append(reports, ReportResponse{Payload: hexutil.Encode(report)}) - } - - if result.Accepted { - status = "Accepted" - } else { - status = "Rejected" - } - - if result.Error != nil { - status = "Exception" - errorMessage = fmt.Sprintf("Error on the machine while inspecting: %s", result.Error) - } - - response := InspectResponse{ - Status: status, - Exception: errorMessage, - Reports: reports, - ProcessedInputs: result.ProcessedInputs, - } + response := inspect.buildInspectResponse(dapp, requestID, result) w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(response); err != nil { @@ -274,15 +254,83 @@ func (inspect *Inspector) ServeHTTP(w http.ResponseWriter, r *http.Request) { inspect.Logger.Error("failed to encode inspect response", "err", err, "application", dapp, - "request_id", service.RequestIDFromContext(r.Context()), + "request_id", requestID, ) return } inspect.Logger.Info("Request executed", - "status", status, + "status", response.Status, "application", dapp) } +// buildInspectResponse maps the manager result to the public inspect contract. +// Machine execution details remain in trusted logs; anonymous clients receive +// a stable sanitized failure message and any reports emitted before the stop. +func (inspect *Inspector) buildInspectResponse( + dapp string, + requestID string, + result *manager.InspectResult, +) InspectResponse { + response := InspectResponse{ + Reports: make([]ReportResponse, 0, len(result.Reports)), + ProcessedInputs: result.ProcessedInputs, + } + for _, report := range result.Reports { + response.Reports = append(response.Reports, ReportResponse{Payload: hexutil.Encode(report)}) + } + + if result.Error != nil { + response.Status = inspectStatusFailed + response.Error = inspectFailureMessage + // Inspect is an anonymous endpoint. Keep machine positions and local + // policy values in operator logs so clients cannot discover configured + // limits or price a resource-exhaustion input. + inspect.Logger.Warn("Machine failed while inspecting", + "application", dapp, + "error", result.Error, + "request_id", requestID, + ) + return response + } + + switch result.Status { + case pkgmachine.CompletionStatusAccepted: + response.Status = "Accepted" + case pkgmachine.CompletionStatusRejected: + response.Status = "Rejected" + case pkgmachine.CompletionStatusException: + response.Status = "Exception" + response.Error = "The machine raised an exception while inspecting" + response.ExceptionData = hexutil.Encode(result.ExceptionData) + inspect.Logger.Debug("Machine returned a guest inspect exception", + "application", dapp, + "request_id", requestID, + ) + case pkgmachine.CompletionStatusHalted: + response.Status = "MachineHalted" + inspect.Logger.Debug("Machine halted while inspecting", + "application", dapp, + "request_id", requestID, + ) + case pkgmachine.CompletionStatusUnknown: + response.Status = inspectStatusFailed + response.Error = inspectFailureMessage + inspect.Logger.Warn("Machine returned an incomplete inspect result", + "application", dapp, + "request_id", requestID, + ) + default: + response.Status = inspectStatusFailed + response.Error = inspectFailureMessage + inspect.Logger.Warn("Machine returned an unknown inspect status", + "application", dapp, + "status", result.Status, + "request_id", requestID, + ) + } + return response +} + func (inspect *Inspector) warnDeadlineExceedsWriteTimeout(app *Application, deadline time.Duration) { inspect.deadlineWarnedMu.Lock() defer inspect.deadlineWarnedMu.Unlock() diff --git a/internal/inspect/inspect_test.go b/internal/inspect/inspect_test.go index ff099a5fc..67b35bc84 100644 --- a/internal/inspect/inspect_test.go +++ b/internal/inspect/inspect_test.go @@ -8,6 +8,7 @@ import ( "context" crand "crypto/rand" "encoding/json" + "errors" "fmt" "io" "log/slog" @@ -18,6 +19,8 @@ import ( "github.com/cartesi/rollups-node/internal/manager" . "github.com/cartesi/rollups-node/internal/model" + inspectclient "github.com/cartesi/rollups-node/pkg/inspectclient" + pkgmachine "github.com/cartesi/rollups-node/pkg/machine" "github.com/cartesi/rollups-node/pkg/service" "github.com/ethereum/go-ethereum/common" @@ -194,6 +197,156 @@ func (s *InspectSuite) TestPostPayloadTooLarge() { s.Equal(http.StatusRequestEntityTooLarge, resp.StatusCode) } +func (s *InspectSuite) TestPostResponseMatchesGeneratedClientContract() { + tests := []struct { + name string + result manager.InspectResult + wantStatus inspectclient.CompletionStatus + wantError string + wantExceptionData []byte + }{ + { + name: "accepted", + result: manager.InspectResult{ + Status: pkgmachine.CompletionStatusAccepted, + Reports: [][]byte{{0xde, 0xad}, {}}, + ProcessedInputs: 17, + }, + wantStatus: inspectclient.Accepted, + }, + { + name: "rejected", + result: manager.InspectResult{ + Status: pkgmachine.CompletionStatusRejected, + Reports: [][]byte{{0xbe, 0xef}}, + ProcessedInputs: 23, + }, + wantStatus: inspectclient.Rejected, + }, + { + name: "exception", + result: manager.InspectResult{ + Status: pkgmachine.CompletionStatusException, + ExceptionData: []byte{0xff, 0x00, 0x80}, + Reports: [][]byte{{0xca, 0xfe}}, + ProcessedInputs: 42, + }, + wantStatus: inspectclient.Exception, + wantError: "The machine raised an exception while inspecting", + wantExceptionData: []byte{0xff, 0x00, 0x80}, + }, + { + name: "halted", + result: manager.InspectResult{ + Status: pkgmachine.CompletionStatusHalted, + Reports: [][]byte{{0xfa, 0xce}}, + ProcessedInputs: 51, + }, + wantStatus: inspectclient.MachineHalted, + }, + { + name: "failed", + result: manager.InspectResult{ + Status: pkgmachine.CompletionStatusUnknown, + Reports: [][]byte{{0xba, 0xdd}}, + ProcessedInputs: 63, + Error: errors.New("backend disconnected"), + }, + wantStatus: inspectclient.Failed, + wantError: "The node could not complete the inspection", + }, + } + + for _, test := range tests { + s.Run(test.name, func() { + inspect, app := s.setupWithInspectResult(test.result) + app.Status = ApplicationStatus_OK + request := httptest.NewRequest(http.MethodPost, "/inspect/"+app.Name, + bytes.NewBufferString("query")) + request.SetPathValue("dapp", app.Name) + recorder := httptest.NewRecorder() + inspect.ServeHTTP(recorder, request) + s.Equal(http.StatusOK, recorder.Code) + + var got inspectclient.InspectResult + s.Require().NoError(json.NewDecoder(recorder.Body).Decode(&got)) + s.Equal(test.wantStatus, got.Status) + s.EqualValues(test.result.ProcessedInputs, got.ProcessedInputCount) + s.Require().Len(got.Reports, len(test.result.Reports)) + for i, report := range test.result.Reports { + s.Equal(fmt.Sprintf("0x%x", report), got.Reports[i].Payload) + } + if test.wantError == "" { + s.Nil(got.Error) + } else { + s.Require().NotNil(got.Error) + s.Equal(test.wantError, *got.Error) + } + if test.wantExceptionData == nil { + s.Nil(got.ExceptionData) + } else { + s.Require().NotNil(got.ExceptionData) + s.Equal(fmt.Sprintf("0x%x", test.wantExceptionData), *got.ExceptionData) + } + s.Equal(ApplicationStatus_OK, app.Status, "inspect limits must not change application status") + }) + } +} + +func (s *InspectSuite) TestCycleLimitIsSanitizedFailedResultWithoutApplicationFailure() { + detailedErr := fmt.Errorf( + "inspect stopped at absolute_mcycle=123456 with configured_cap=789: %w", + pkgmachine.ErrReachedLimitMcycle, + ) + inspect, app := s.setupWithInspectResult(manager.InspectResult{ + Status: pkgmachine.CompletionStatusUnknown, + ProcessedInputs: 42, + Error: detailedErr, + }) + var logs bytes.Buffer + inspect.Logger = slog.New(slog.NewTextHandler(&logs, nil)) + app.Status = ApplicationStatus_OK + req := httptest.NewRequest(http.MethodPost, "/inspect/"+app.Name, bytes.NewBufferString("query")) + req.SetPathValue("dapp", app.Name) + recorder := httptest.NewRecorder() + + inspect.ServeHTTP(recorder, req) + + s.Equal(http.StatusOK, recorder.Code) + var got inspectclient.InspectResult + s.Require().NoError(json.NewDecoder(recorder.Body).Decode(&got)) + s.Equal(inspectclient.Failed, got.Status) + s.Require().NotNil(got.Error) + s.Equal("The node could not complete the inspection", *got.Error) + s.NotContains(*got.Error, "absolute_mcycle") + s.NotContains(*got.Error, "configured_cap") + s.Contains(logs.String(), "absolute_mcycle=123456") + s.Contains(logs.String(), "configured_cap=789") + s.EqualValues(42, got.ProcessedInputCount) + s.Equal(ApplicationStatus_OK, app.Status) +} + +func (s *InspectSuite) TestGuestExceptionUsesDebugLogLevel() { + inspect, app := s.setupWithInspectResult(manager.InspectResult{ + Status: pkgmachine.CompletionStatusException, + ExceptionData: []byte("guest exception details"), + ProcessedInputs: 42, + }) + var logs bytes.Buffer + inspect.Logger = slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})) + req := httptest.NewRequest(http.MethodPost, "/inspect/"+app.Name, bytes.NewBufferString("query")) + req.SetPathValue("dapp", app.Name) + recorder := httptest.NewRecorder() + + inspect.ServeHTTP(recorder, req) + + s.Equal(http.StatusOK, recorder.Code) + s.Contains(logs.String(), "level=DEBUG") + s.Contains(logs.String(), "Machine returned a guest inspect exception") + s.NotContains(logs.String(), "level=WARN") + s.NotContains(logs.String(), "guest exception details") +} + func (s *InspectSuite) startServer(inspect *Inspector) *httptest.Server { router := http.NewServeMux() router.Handle("/inspect/{dapp}", inspect) @@ -201,7 +354,17 @@ func (s *InspectSuite) startServer(inspect *Inspector) *httptest.Server { } func (s *InspectSuite) setup() (*Inspector, *Application, common.Hash) { + payload := randomHash() + inspect, app := s.setupWithInspectResult(manager.InspectResult{ + Status: pkgmachine.CompletionStatusAccepted, + Reports: [][]byte{payload.Bytes()}, + }) + return inspect, app, payload +} + +func (s *InspectSuite) setupWithInspectResult(result manager.InspectResult) (*Inspector, *Application) { m := newMockMachine(1) + m.inspectResult = &result repo := newMockRepository() repo.apps = append(repo.apps, m.application) machines := newMockMachines() @@ -211,8 +374,7 @@ func (s *InspectSuite) setup() (*Inspector, *Application, common.Hash) { IInspectMachines: machines, Logger: service.NewLogger(slog.LevelDebug, true), } - payload := randomHash() - return inspect, m.application, payload + return inspect, m.application } func (s *InspectSuite) assertResponse(resp *http.Response, payload string) { @@ -251,18 +413,24 @@ func (mock *MachinesMock) GetMachine(appId int64) (manager.MachineInstance, bool // ------------------------------------------------------------------------------------------------ type MockMachine struct { - application *Application + application *Application + inspectResult *manager.InspectResult } func (mock *MockMachine) Inspect( _ context.Context, query []byte, -) (*InspectResult, error) { - var res InspectResult +) (*manager.InspectResult, error) { + if mock.inspectResult != nil { + result := *mock.inspectResult + return &result, nil + } + + var res manager.InspectResult var reports [][]byte reports = append(reports, query) - res.Accepted = true + res.Status = pkgmachine.CompletionStatusAccepted res.ProcessedInputs = 0 res.Error = nil res.Reports = reports diff --git a/internal/manager/instance.go b/internal/manager/instance.go index c8ecc7449..00e45746b 100644 --- a/internal/manager/instance.go +++ b/internal/manager/instance.go @@ -464,6 +464,8 @@ func (m *MachineInstanceImpl) Inspect(ctx context.Context, query []byte) (*Inspe ProcessedInputs: processedInputs, } if inspectResponse != nil { + result.Status = inspectResponse.Status + result.ExceptionData = inspectResponse.ExceptionData result.Reports = inspectResponse.Reports } @@ -471,23 +473,14 @@ func (m *MachineInstanceImpl) Inspect(ctx context.Context, query []byte) (*Inspe // implementation. Inspection did not complete, but reports emitted before // the failure remain useful to the caller. if inspectErr != nil { + result.Status = machine.CompletionStatusUnknown result.Error = inspectErr - } else if inspectResponse == nil || !inspectResponse.Status.IsCompleted() { + } else if inspectResponse == nil || !result.Status.IsCompleted() { + result.Status = machine.CompletionStatusUnknown result.Error = errors.Join(ErrIncompleteInspect, machine.ErrMachineInternal) - } else { - switch inspectResponse.Status { - case machine.CompletionStatusAccepted: - result.Accepted = true - case machine.CompletionStatusRejected: - case machine.CompletionStatusException: - result.Error = machine.ErrException - case machine.CompletionStatusHalted: - result.Error = machine.ErrHalted - case machine.CompletionStatusUnknown: - result.Error = errors.Join(ErrIncompleteInspect, machine.ErrMachineInternal) - default: - result.Error = errors.Join(ErrIncompleteInspect, machine.ErrMachineInternal) - } + } else if err := validateCompletionExceptionData(result.Status, result.ExceptionData); err != nil { + result.Status = machine.CompletionStatusUnknown + result.Error = errors.Join(ErrIncompleteInspect, err) } // Close the fork diff --git a/internal/manager/instance_test.go b/internal/manager/instance_test.go index 30394f138..419d80a91 100644 --- a/internal/manager/instance_test.go +++ b/internal/manager/instance_test.go @@ -27,6 +27,18 @@ func TestMachineInstance(t *testing.T) { type MachineInstanceSuite struct{ suite.Suite } +func (s *MachineInstanceSuite) TestMcycleOverflowRemainsIncomplete() { + require := s.Require() + + // Cycle exhaustion surfaces as an error from machine.Advance, never as a + // completed CompletionStatus, so no input completion status can exist for it. + // The zero-value status is the closest representable input and must be + // rejected rather than mapped to a completed status. + status, err := toInputStatus(machine.CompletionStatusUnknown) + require.ErrorIs(err, ErrIncompleteAdvance) + require.Equal(model.InputCompletionStatus_None, status) +} + // MockMachineRuntimeFactory implements MachineRuntimeFactory for testing type MockMachineRuntimeFactory struct { RuntimeToReturn machine.Machine @@ -617,33 +629,53 @@ func (s *MachineInstanceSuite) TestAdvance() { func (s *MachineInstanceSuite) TestInspect() { for _, test := range []struct { - name string - status machine.CompletionStatus - accepted bool - resultErr error + name string + status machine.CompletionStatus }{ - {"Accept", machine.CompletionStatusAccepted, true, nil}, - {"Reject", machine.CompletionStatusRejected, false, nil}, - {"Exception", machine.CompletionStatusException, false, machine.ErrException}, - {"Halted", machine.CompletionStatusHalted, false, machine.ErrHalted}, + {"Accept", machine.CompletionStatusAccepted}, + {"Reject", machine.CompletionStatusRejected}, + {"Exception", machine.CompletionStatusException}, + {"Halted", machine.CompletionStatusHalted}, } { s.Run(test.name, func() { require := s.Require() _, fork, instance := s.setupInspect() fork.InspectResponseReturn.Status = test.status + if test.status == machine.CompletionStatusException { + fork.InspectResponseReturn.ExceptionData = []byte("guest exception") + } result, err := instance.Inspect(context.Background(), []byte{}) require.NoError(err) require.NotNil(result) require.NotSame(fork, instance.runtime) require.Equal(uint64(55), result.ProcessedInputs) - require.Equal(test.accepted, result.Accepted) + require.Equal(test.status, result.Status) + require.Equal(fork.InspectResponseReturn.ExceptionData, result.ExceptionData) require.Equal(expectedReports2, result.Reports) - if test.resultErr == nil { - require.NoError(result.Error) - } else { - require.ErrorIs(result.Error, test.resultErr) - } + require.NoError(result.Error) + }) + } + + for _, test := range []struct { + name string + status machine.CompletionStatus + exceptionData []byte + }{ + {"ExceptionWithoutData", machine.CompletionStatusException, nil}, + {"RejectedWithExceptionData", machine.CompletionStatusRejected, []byte("unexpected")}, + } { + s.Run(test.name+"FailsClosed", func() { + require := s.Require() + _, fork, instance := s.setupInspect() + fork.InspectResponseReturn.Status = test.status + fork.InspectResponseReturn.ExceptionData = test.exceptionData + + result, err := instance.Inspect(context.Background(), []byte{}) + require.NoError(err) + require.Equal(machine.CompletionStatusUnknown, result.Status) + require.ErrorIs(result.Error, ErrIncompleteInspect) + require.ErrorIs(result.Error, machine.ErrMachineInternal) }) } @@ -677,7 +709,7 @@ func (s *MachineInstanceSuite) TestInspect() { result, err := instance.Inspect(context.Background(), []byte{}) require.NoError(err) - require.False(result.Accepted) + require.Equal(machine.CompletionStatusUnknown, result.Status) require.Equal(expectedReports2, result.Reports) require.ErrorIs(result.Error, errInspect) }) @@ -697,7 +729,7 @@ func (s *MachineInstanceSuite) TestInspect() { result, err := instance.Inspect(context.Background(), []byte{}) require.NoError(err) - require.False(result.Accepted) + require.Equal(machine.CompletionStatusUnknown, result.Status) require.ErrorIs(result.Error, ErrIncompleteInspect) require.ErrorIs(result.Error, machine.ErrMachineInternal) }) diff --git a/internal/manager/manager_test.go b/internal/manager/manager_test.go index c6a16dae7..2b61120e7 100644 --- a/internal/manager/manager_test.go +++ b/internal/manager/manager_test.go @@ -617,7 +617,7 @@ func (m *DummyMachineInstanceMock) Advance(_ context.Context, _ []byte, _ uint64 return nil, nil } -func (m *DummyMachineInstanceMock) Inspect(_ context.Context, _ []byte) (*model.InspectResult, error) { +func (m *DummyMachineInstanceMock) Inspect(_ context.Context, _ []byte) (*InspectResult, error) { return nil, nil } diff --git a/internal/manager/types.go b/internal/manager/types.go index d72e1f97a..272904c7c 100644 --- a/internal/manager/types.go +++ b/internal/manager/types.go @@ -7,8 +7,20 @@ import ( "context" . "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/pkg/machine" ) +// InspectResult carries a typed machine completion or an incomplete execution +// failure. Error is non-nil only when Status is machine.CompletionStatusUnknown; +// Reports then contains any reports emitted before the failure. +type InspectResult struct { + ProcessedInputs uint64 + Status machine.CompletionStatus + ExceptionData []byte + Reports [][]byte + Error error +} + // MachineInstance defines the interface for a machine instance type MachineInstance interface { Application() *Application diff --git a/internal/model/models.go b/internal/model/models.go index a660fc9d8..c932450b2 100644 --- a/internal/model/models.go +++ b/internal/model/models.go @@ -1388,13 +1388,6 @@ type AdvanceResult struct { IsDaveConsensus bool } -type InspectResult struct { - ProcessedInputs uint64 - Accepted bool - Reports [][]byte - Error error -} - // FIXME: remove this type. Migrate claim to use Application + Epoch type ClaimRow struct { Epoch diff --git a/pkg/inspectclient/generated.go b/pkg/inspectclient/generated.go index 6718bc10e..549403c10 100644 --- a/pkg/inspectclient/generated.go +++ b/pkg/inspectclient/generated.go @@ -17,15 +17,18 @@ import ( // Defines values for CompletionStatus. const ( - Accepted CompletionStatus = "Accepted" - CycleLimitExceeded CompletionStatus = "CycleLimitExceeded" - Exception CompletionStatus = "Exception" - MachineHalted CompletionStatus = "MachineHalted" - Rejected CompletionStatus = "Rejected" - TimeLimitExceeded CompletionStatus = "TimeLimitExceeded" + Accepted CompletionStatus = "Accepted" + Exception CompletionStatus = "Exception" + Failed CompletionStatus = "Failed" + MachineHalted CompletionStatus = "MachineHalted" + Rejected CompletionStatus = "Rejected" ) -// CompletionStatus Whether inspection completed or not (and why not) +// CompletionStatus How inspection completed. MachineHalted means the temporary inspect +// execution halted; the canonical application machine is unchanged. +// Failed means the node could not complete inspection because of an +// operational limit, cancellation, timeout, protocol error, or internal +// failure. Reports returned with Failed may be partial. type CompletionStatus string // Error Detailed error message. @@ -33,17 +36,23 @@ type Error = string // InspectResult defines model for InspectResult. type InspectResult struct { - // ExceptionPayload Payload in the Ethereum hex binary format. - // The first two characters are '0x' followed by pairs of hexadecimal numbers that correspond to one byte. - // For instance, '0xdeadbeef' corresponds to a payload with length 4 and bytes 222, 173, 190, 175. - // An empty payload is represented by the string '0x'. - ExceptionPayload Payload `json:"exception_payload"` + // Error Sanitized error description, present only when status is Exception or Failed + Error *string `json:"error,omitempty"` + + // ExceptionData Raw guest-provided CMIO exception payload, present only when status is Exception + ExceptionData *Payload `json:"exception_data,omitempty"` // ProcessedInputCount Number of processed inputs since genesis - ProcessedInputCount int `json:"processed_input_count"` - Reports []Report `json:"reports"` + ProcessedInputCount int `json:"processed_input_count"` + + // Reports Reports emitted before completion; for Failed, this may be only a partial prefix + Reports []Report `json:"reports"` - // Status Whether inspection completed or not (and why not) + // Status How inspection completed. MachineHalted means the temporary inspect + // execution halted; the canonical application machine is unchanged. + // Failed means the node could not complete inspection because of an + // operational limit, cancellation, timeout, protocol error, or internal + // failure. Reports returned with Failed may be partial. Status CompletionStatus `json:"status"` } From 2614e90523500ded283c12565bc629871f2e526d Mon Sep 17 00:00:00 2001 From: Victor Fusco <1221933+vfusco@users.noreply.github.com> Date: Fri, 14 Aug 2026 10:36:33 -0300 Subject: [PATCH 09/13] feat(repository): provide canonical replay data --- internal/model/models.go | 40 ++ .../postgres/input_exception_data_test.go | 15 + internal/repository/postgres/replay.go | 543 ++++++++++++++++++ .../repository/postgres/replay_source_test.go | 323 +++++++++++ internal/repository/postgres/replay_test.go | 222 +++++++ .../000001_create_initial_schema.down.sql | 5 + .../000001_create_initial_schema.up.sql | 38 ++ internal/repository/replay.go | 212 +++++++ internal/repository/replay_test.go | 39 ++ internal/repository/repository.go | 1 + .../repository/repotest/epoch_test_cases.go | 11 +- .../repository/repotest/input_test_cases.go | 34 +- 12 files changed, 1469 insertions(+), 14 deletions(-) create mode 100644 internal/repository/postgres/replay.go create mode 100644 internal/repository/postgres/replay_source_test.go create mode 100644 internal/repository/postgres/replay_test.go create mode 100644 internal/repository/replay.go create mode 100644 internal/repository/replay_test.go diff --git a/internal/model/models.go b/internal/model/models.go index c932450b2..432650f62 100644 --- a/internal/model/models.go +++ b/internal/model/models.go @@ -1388,6 +1388,46 @@ type AdvanceResult struct { IsDaveConsensus bool } +// ReplaySummary identifies the immutable completed-input prefix selected for +// one machine replay. +type ReplaySummary struct { + ApplicationID int64 + ProcessedInputs uint64 + Consensus Consensus +} + +// ReplayInput is the canonical input evidence needed to verify one replayed +// machine execution. It is deliberately narrower than Input: L1 metadata, +// timestamps, and snapshot location do not participate in the comparison. +type ReplayInput struct { + ApplicationID int64 + EpochIndex uint64 + InputIndex uint64 + RawData []byte + Status InputCompletionStatus + ExceptionData []byte + MachineHash *common.Hash + OutputsHash *common.Hash +} + +// ReplayStateHash is one persisted row of a PRT input hash collection. Keeping +// this projection narrow matters because one input may contain millions of +// rows. +type ReplayStateHash struct { + Index uint64 + MachineHash common.Hash + Repetitions uint64 +} + +// ReplayRecord contains one completed input and its requested verification +// evidence. Canonical records leave Outputs, Reports, and StateHashes empty. +type ReplayRecord struct { + Input ReplayInput + Outputs [][]byte + Reports [][]byte + StateHashes []ReplayStateHash +} + // FIXME: remove this type. Migrate claim to use Application + Epoch type ClaimRow struct { Epoch diff --git a/internal/repository/postgres/input_exception_data_test.go b/internal/repository/postgres/input_exception_data_test.go index 8bb741452..602b83543 100644 --- a/internal/repository/postgres/input_exception_data_test.go +++ b/internal/repository/postgres/input_exception_data_test.go @@ -65,4 +65,19 @@ func TestPostgresInputExceptionDataContract(t *testing.T) { require.NoError(t, err) require.NotNil(t, completed.ExceptionData) require.Empty(t, completed.ExceptionData) + + _, err = conn.Exec(ctx, ` + UPDATE input SET exception_data = '\x02' + WHERE epoch_application_id = $1 AND index = 0`, seed.App.ID) + require.ErrorContains(t, err, "completed input result is immutable") + + _, err = conn.Exec(ctx, ` + UPDATE input SET raw_data = '\x03' + WHERE epoch_application_id = $1 AND index = 0`, seed.App.ID) + require.ErrorContains(t, err, "completed input result is immutable") + + _, err = conn.Exec(ctx, ` + UPDATE input SET snapshot_uri = '/tmp/snapshot' + WHERE epoch_application_id = $1 AND index = 0`, seed.App.ID) + require.NoError(t, err, "snapshot metadata remains mutable after completion") } diff --git a/internal/repository/postgres/replay.go b/internal/repository/postgres/replay.go new file mode 100644 index 000000000..540b0eb55 --- /dev/null +++ b/internal/repository/postgres/replay.go @@ -0,0 +1,543 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package postgres + +import ( + "context" + "errors" + "fmt" + "math" + + "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/internal/repository" + "github.com/cartesi/rollups-node/internal/repository/postgres/db/rollupsdb/public/table" + "github.com/ethereum/go-ethereum/common" + "github.com/go-jet/jet/v2/postgres" + "github.com/jackc/pgx/v5" +) + +var replayCompletedStatuses = []postgres.Expression{ + postgres.NewEnumValue(model.InputCompletionStatus_Accepted.String()), + postgres.NewEnumValue(model.InputCompletionStatus_Rejected.String()), + postgres.NewEnumValue(model.InputCompletionStatus_Exception.String()), + postgres.NewEnumValue(model.InputCompletionStatus_MachineHalted.String()), +} + +func replayCompletedStatus(status postgres.StringExpression) postgres.BoolExpression { + return status.IN(replayCompletedStatuses...) +} + +// replayChildSpec captures the schema shared by replay outputs and reports. +// State hashes deliberately use a separate path because their ordering and +// repetition invariants are different. +type replayChildSpec struct { + table postgres.Table + applicationID postgres.IntegerExpression + inputIndex postgres.FloatExpression + index postgres.FloatExpression + rawData postgres.ByteaExpression + childKind repository.ReplayEvidenceKind + appendToRecord func(*model.ReplayRecord, []byte) +} + +var ( + replayOutputSpec = replayChildSpec{ + table: table.Output, + applicationID: table.Output.InputEpochApplicationID, + inputIndex: table.Output.InputIndex, + index: table.Output.Index, + rawData: table.Output.RawData, + childKind: repository.ReplayEvidenceOutput, + appendToRecord: func(record *model.ReplayRecord, data []byte) { + record.Outputs = append(record.Outputs, data) + }, + } + replayReportSpec = replayChildSpec{ + table: table.Report, + applicationID: table.Report.InputEpochApplicationID, + inputIndex: table.Report.InputIndex, + index: table.Report.Index, + rawData: table.Report.RawData, + childKind: repository.ReplayEvidenceReport, + appendToRecord: func(record *model.ReplayRecord, data []byte) { + record.Reports = append(record.Reports, data) + }, + } +) + +// ReplaySummary reads the application identity, consensus, and processed-input +// count from one database snapshot. It checks that the count matches the number +// of completed inputs and that completed input indexes are contiguous from zero. +// In Full mode, it also checks PRT state-hash ordering. For non-PRT applications, +// it checks that no state hashes exist. All queries run in the same short +// repeatable-read transaction. +func (r *PostgresRepository) ReplaySummary( + ctx context.Context, + applicationAddress common.Address, + verification repository.ReplayVerificationLevel, +) (model.ReplaySummary, error) { + if !verification.IsValid() { + return model.ReplaySummary{}, fmt.Errorf("unsupported replay verification level %d", verification) + } + whereApp := table.Application.IapplicationAddress.EQ(postgres.Bytea(applicationAddress.Bytes())) + tx, err := beginReadTx(ctx, r.db) + if err != nil { + return model.ReplaySummary{}, err + } + defer tx.Rollback(ctx) //nolint:errcheck + + appStmt := table.Application.SELECT( + table.Application.ID, + table.Application.ConsensusType, + table.Application.ProcessedInputs, + ).WHERE(whereApp) + appSQL, appArgs := appStmt.Sql() + var summary model.ReplaySummary + if err := tx.QueryRow(ctx, appSQL, appArgs...).Scan( + &summary.ApplicationID, + &summary.Consensus, + &summary.ProcessedInputs, + ); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return model.ReplaySummary{}, repository.ErrNotFound + } + return model.ReplaySummary{}, err + } + + whereInputApp := table.Input.EpochApplicationID.EQ(postgres.Int64(summary.ApplicationID)) + countStmt := table.Input.SELECT(postgres.COUNT(postgres.STAR)). + WHERE(whereInputApp.AND(replayCompletedStatus(table.Input.Status))) + completedInputCount, err := countFromTx(ctx, tx, countStmt) + if err != nil { + return model.ReplaySummary{}, err + } + if completedInputCount != summary.ProcessedInputs { + return model.ReplaySummary{}, &repository.ReplayStructureViolationError{ + Kind: repository.ReplayStructureProcessedInputCount, + InputIndex: completedInputCount, + ApplicationProcessedInputs: summary.ProcessedInputs, + CompletedInputCount: completedInputCount, + } + } + if err := validateReplayCompletedPrefix(ctx, tx, whereInputApp, completedInputCount); err != nil { + return model.ReplaySummary{}, err + } + if verification == repository.ReplayVerificationFull { + whereStateHashApp := table.StateHashes.InputEpochApplicationID.EQ( + postgres.Int64(summary.ApplicationID), + ) + var err error + switch summary.Consensus { + case model.Consensus_PRT: + err = validateReplayStateHashOrdering(ctx, tx, whereStateHashApp) + case model.Consensus_Authority, model.Consensus_Quorum: + err = validateReplayStateHashesAbsent(ctx, tx, whereStateHashApp) + default: + err = fmt.Errorf("unsupported replay consensus %q", summary.Consensus) + } + if err != nil { + return model.ReplaySummary{}, err + } + } + if err := tx.Commit(ctx); err != nil { + return model.ReplaySummary{}, err + } + return summary, nil +} + +func scanOptionalRow( + ctx context.Context, + tx pgx.Tx, + stmt postgres.SelectStatement, + dest ...any, +) (bool, error) { + sqlStr, args := stmt.Sql() + err := tx.QueryRow(ctx, sqlStr, args...).Scan(dest...) + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + return err == nil, err +} + +func replayStructureViolation( + kind repository.ReplayStructureViolationKind, + epochIndex uint64, + inputIndex uint64, + evidenceIndex uint64, +) *repository.ReplayStructureViolationError { + return &repository.ReplayStructureViolationError{ + Kind: kind, + EpochIndex: &epochIndex, + InputIndex: inputIndex, + EvidenceIndex: evidenceIndex, + } +} + +func validateReplayCompletedPrefix( + ctx context.Context, + tx pgx.Tx, + whereInputApp postgres.BoolExpression, + total uint64, +) error { + if total == 0 { + return nil + } + stmt := table.Input.SELECT(table.Input.Index). + WHERE(whereInputApp.AND(replayCompletedStatus(table.Input.Status))). + ORDER_BY(table.Input.Index.DESC()). + LIMIT(1) + sqlStr, args := stmt.Sql() + var lastInputIndex uint64 + if err := tx.QueryRow(ctx, sqlStr, args...).Scan(&lastInputIndex); err != nil { + return err + } + expectedLastInputIndex := total - 1 + // Input indexes are non-negative and unique by primary key. Therefore, + // total distinct rows whose maximum index is total-1 must be exactly the + // contiguous sequence 0..total-1; a gap would force a larger maximum. + if lastInputIndex != expectedLastInputIndex { + return &repository.ReplayStructureViolationError{ + Kind: repository.ReplayStructureCompletedInputSequence, + InputIndex: lastInputIndex, + ExpectedIndex: expectedLastInputIndex, + } + } + return nil +} + +func validateReplayStateHashOrdering( + ctx context.Context, + tx pgx.Tx, + whereStateHashApp postgres.BoolExpression, +) error { + orderedStateHashes := table.StateHashes.SELECT( + table.StateHashes.EpochIndex.AS("epoch_index"), + table.StateHashes.Index.AS("state_hash_index"), + table.StateHashes.InputIndex.AS("input_index"), + postgres.ROW_NUMBER().OVER( + postgres.PARTITION_BY(table.StateHashes.EpochIndex). + ORDER_BY(table.StateHashes.Index), + ).AS("ordinal"), + postgres.LAG(table.StateHashes.InputIndex).OVER( + postgres.PARTITION_BY(table.StateHashes.EpochIndex). + ORDER_BY(table.StateHashes.Index), + ).AS("previous_input_index"), + ).WHERE(whereStateHashApp).AsTable("ordered_state_hash") + + epochIndexColumn := postgres.FloatColumn("epoch_index").From(orderedStateHashes) + stateHashIndexColumn := postgres.FloatColumn("state_hash_index").From(orderedStateHashes) + inputIndexColumn := postgres.FloatColumn("input_index").From(orderedStateHashes) + ordinalColumn := postgres.FloatColumn("ordinal").From(orderedStateHashes) + previousInputIndexColumn := postgres.FloatColumn("previous_input_index").From(orderedStateHashes) + stmt := orderedStateHashes.SELECT( + epochIndexColumn, + stateHashIndexColumn, + inputIndexColumn, + ordinalColumn, + postgres.COALESCE(previousInputIndexColumn, inputIndexColumn), + previousInputIndexColumn.IS_NOT_NULL(), + ).WHERE( + stateHashIndexColumn.NOT_EQ(ordinalColumn.SUB(postgres.Float(1))).OR( + previousInputIndexColumn.IS_NOT_NULL().AND( + inputIndexColumn.LT(previousInputIndexColumn), + ), + ), + ).ORDER_BY(epochIndexColumn.ASC(), stateHashIndexColumn.ASC()).LIMIT(1) + + var epochIndex, stateHashIndex, inputIndex, ordinal, previousInputIndex uint64 + var hasPreviousInput bool + found, err := scanOptionalRow(ctx, tx, stmt, + &epochIndex, + &stateHashIndex, + &inputIndex, + &ordinal, + &previousInputIndex, + &hasPreviousInput, + ) + if err != nil { + return err + } + if !found { + return nil + } + if stateHashIndex != ordinal-1 { + violation := replayStructureViolation( + repository.ReplayStructureStateHashIndexSequence, + epochIndex, + inputIndex, + stateHashIndex, + ) + violation.ExpectedIndex = ordinal - 1 + return violation + } + violation := replayStructureViolation( + repository.ReplayStructureStateHashInputOrder, + epochIndex, + inputIndex, + stateHashIndex, + ) + if hasPreviousInput { + violation.PreviousInputIndex = previousInputIndex + } + return violation +} + +func validateReplayStateHashesAbsent( + ctx context.Context, + tx pgx.Tx, + whereStateHashApp postgres.BoolExpression, +) error { + stmt := table.StateHashes.SELECT( + table.StateHashes.EpochIndex, + table.StateHashes.InputIndex, + table.StateHashes.Index, + ).WHERE(whereStateHashApp). + ORDER_BY(table.StateHashes.EpochIndex.ASC(), table.StateHashes.Index.ASC()). + LIMIT(1) + + var epochIndex, inputIndex, stateHashIndex uint64 + found, err := scanOptionalRow( + ctx, tx, stmt, + &epochIndex, + &inputIndex, + &stateHashIndex, + ) + if err != nil { + return err + } + if !found { + return nil + } + return replayStructureViolation( + repository.ReplayStructureUnexpectedStateHash, + epochIndex, + inputIndex, + stateHashIndex, + ) +} + +// ReplayPage returns one keyset page from the summary's fixed application and +// high-water mark. Canonical pages query inputs only. Full pages load child +// evidence in the same short repeatable-read transaction. +func (r *PostgresRepository) ReplayPage( + ctx context.Context, + request repository.ReplayPageRequest, +) ([]*model.ReplayRecord, error) { + if !request.Verification.IsValid() { + return nil, fmt.Errorf("unsupported replay verification level %d", request.Verification) + } + if request.ApplicationID <= 0 { + return nil, fmt.Errorf("replay application ID must be greater than zero") + } + if request.Limit == 0 { + return nil, fmt.Errorf("replay page limit must be greater than zero") + } + if request.Limit > math.MaxInt64 { + return nil, fmt.Errorf("replay page limit %d exceeds maximum supported value %d", request.Limit, int64(math.MaxInt64)) + } + if request.FromInput > request.ToInputExclusive { + return nil, fmt.Errorf( + "replay input range is invalid: lower bound %d exceeds upper bound %d", + request.FromInput, request.ToInputExclusive, + ) + } + if request.FromInput == request.ToInputExclusive { + return []*model.ReplayRecord{}, nil + } + + whereInputApp := table.Input.EpochApplicationID.EQ(postgres.Int64(request.ApplicationID)) + tx, err := beginReadTx(ctx, r.db) + if err != nil { + return nil, err + } + defer tx.Rollback(ctx) //nolint:errcheck + + inputStmt := table.Input.SELECT( + table.Input.EpochApplicationID, + table.Input.EpochIndex, + table.Input.Index, + table.Input.RawData, + table.Input.Status, + table.Input.ExceptionData, + table.Input.MachineHash, + table.Input.OutputsHash, + ). + WHERE( + whereInputApp. + AND(replayCompletedStatus(table.Input.Status)). + AND(table.Input.Index.GT_EQ(uint64Expr(request.FromInput))). + AND(table.Input.Index.LT(uint64Expr(request.ToInputExclusive))), + ). + ORDER_BY(table.Input.Index.ASC()). + LIMIT(int64(request.Limit)) + + records, err := selectReplayInputs(ctx, tx, inputStmt) + if err != nil { + return nil, err + } + + byInput := make(map[uint64]*model.ReplayRecord, len(records)) + for _, record := range records { + byInput[record.Input.InputIndex] = record + } + if request.Verification == repository.ReplayVerificationCanonical { + if err := tx.Commit(ctx); err != nil { + return nil, err + } + return records, nil + } + + childToInputIndexExclusive := request.ToInputExclusive + if uint64(len(records)) == request.Limit { + // A full page covers only through its last returned input. Evidence for + // later completed inputs belongs to a subsequent page and must not be + // mistaken for an unmatched child in this page. + childToInputIndexExclusive = records[len(records)-1].Input.InputIndex + 1 + } + + if err := selectReplayChildren( + ctx, tx, request.ApplicationID, request.FromInput, childToInputIndexExclusive, byInput, replayOutputSpec, + ); err != nil { + return nil, err + } + if err := selectReplayChildren( + ctx, tx, request.ApplicationID, request.FromInput, childToInputIndexExclusive, byInput, replayReportSpec, + ); err != nil { + return nil, err + } + if err := selectReplayStateHashes( + ctx, tx, request.ApplicationID, request.FromInput, childToInputIndexExclusive, byInput, + ); err != nil { + return nil, err + } + if err := tx.Commit(ctx); err != nil { + return nil, err + } + return records, nil +} + +func selectReplayInputs( + ctx context.Context, + tx pgx.Tx, + stmt postgres.SelectStatement, +) ([]*model.ReplayRecord, error) { + sqlStr, args := stmt.Sql() + rows, err := tx.Query(ctx, sqlStr, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + var records []*model.ReplayRecord + for rows.Next() { + record := new(model.ReplayRecord) + in := &record.Input + if err := rows.Scan( + &in.ApplicationID, + &in.EpochIndex, + &in.InputIndex, + &in.RawData, + &in.Status, + &in.ExceptionData, + &in.MachineHash, + &in.OutputsHash, + ); err != nil { + return nil, err + } + records = append(records, record) + } + return records, rows.Err() +} + +func replayRecordForChild( + byInput map[uint64]*model.ReplayRecord, + childKind repository.ReplayEvidenceKind, + inputIndex uint64, +) (*model.ReplayRecord, error) { + record := byInput[inputIndex] + if record == nil { + return nil, &repository.ReplayInconsistentEvidenceError{ + Kind: childKind, + InputIndex: inputIndex, + } + } + return record, nil +} + +func selectReplayChildren( + ctx context.Context, + tx pgx.Tx, + applicationID int64, + fromInputIndex, toInputIndexExclusive uint64, + byInput map[uint64]*model.ReplayRecord, + spec replayChildSpec, +) error { + stmt := spec.table.SELECT( + spec.inputIndex, + spec.rawData, + ).WHERE( + spec.applicationID.EQ(postgres.Int64(applicationID)). + AND(spec.inputIndex.GT_EQ(uint64Expr(fromInputIndex))). + AND(spec.inputIndex.LT(uint64Expr(toInputIndexExclusive))), + ).ORDER_BY(spec.inputIndex.ASC(), spec.index.ASC()) + + sqlStr, args := stmt.Sql() + rows, err := tx.Query(ctx, sqlStr, args...) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var inputIndex uint64 + var data []byte + if err := rows.Scan(&inputIndex, &data); err != nil { + return err + } + record, err := replayRecordForChild(byInput, spec.childKind, inputIndex) + if err != nil { + return err + } + spec.appendToRecord(record, data) + } + return rows.Err() +} + +func selectReplayStateHashes( + ctx context.Context, + tx pgx.Tx, + applicationID int64, + fromInputIndex, toInputIndexExclusive uint64, + byInput map[uint64]*model.ReplayRecord, +) error { + stmt := table.StateHashes.SELECT( + table.StateHashes.InputIndex, + table.StateHashes.Index, + table.StateHashes.MachineHash, + table.StateHashes.Repetitions, + ).WHERE( + table.StateHashes.InputEpochApplicationID.EQ(postgres.Int64(applicationID)). + AND(table.StateHashes.InputIndex.GT_EQ(uint64Expr(fromInputIndex))). + AND(table.StateHashes.InputIndex.LT(uint64Expr(toInputIndexExclusive))), + ).ORDER_BY(table.StateHashes.InputIndex.ASC(), table.StateHashes.Index.ASC()) + + sqlStr, args := stmt.Sql() + rows, err := tx.Query(ctx, sqlStr, args...) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var inputIndex uint64 + var row model.ReplayStateHash + if err := rows.Scan(&inputIndex, &row.Index, &row.MachineHash, &row.Repetitions); err != nil { + return err + } + record, err := replayRecordForChild(byInput, repository.ReplayEvidenceStateHash, inputIndex) + if err != nil { + return err + } + record.StateHashes = append(record.StateHashes, row) + } + return rows.Err() +} diff --git a/internal/repository/postgres/replay_source_test.go b/internal/repository/postgres/replay_source_test.go new file mode 100644 index 000000000..461685cf2 --- /dev/null +++ b/internal/repository/postgres/replay_source_test.go @@ -0,0 +1,323 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package postgres_test + +import ( + "context" + "testing" + + "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/internal/repository" + "github.com/cartesi/rollups-node/internal/repository/factory" + "github.com/cartesi/rollups-node/internal/repository/repotest" + "github.com/cartesi/rollups-node/test/tooling/db" + "github.com/ethereum/go-ethereum/common" + "github.com/jackc/pgx/v5" + "github.com/stretchr/testify/require" +) + +func TestPostgresReplayVerificationLevels(t *testing.T) { + endpoint, err := db.GetTestDatabaseEndpoint() + if err != nil { + t.Skipf("Skipping: %v", err) + } + require.NoError(t, db.SetupTestPostgres(endpoint)) + + ctx := context.Background() + repo, err := factory.NewRepositoryFromConnectionString(ctx, endpoint) + require.NoError(t, err) + t.Cleanup(repo.Close) + conn, err := pgx.Connect(ctx, endpoint) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, conn.Close(ctx)) }) + + app := repotest.NewApplicationBuilder().Create(ctx, t, repo) + epoch := repotest.NewEpochBuilder(app.ID). + WithStatus(model.EpochStatus_Closed). + WithInputBounds(0, 1). + Build() + inputs := []*model.Input{ + repotest.NewInputBuilder().WithIndex(0).Build(), + repotest.NewInputBuilder().WithIndex(1).Build(), + } + require.NoError(t, repo.CreateEpochsAndInputs( + ctx, + app.IApplicationAddress.String(), + map[*model.Epoch][]*model.Input{epoch: inputs}, + 10, + )) + require.NoError(t, repo.StoreAdvanceResult(ctx, app.ID, &model.AdvanceResult{ + EpochIndex: 0, + InputIndex: 0, + Status: model.InputCompletionStatus_Accepted, + Outputs: [][]byte{[]byte("output")}, + Reports: [][]byte{[]byte("report")}, + OutputsProof: model.OutputsProof{ + MachineHash: repotest.UniqueHash(), + OutputsHash: repotest.UniqueHash(), + }, + })) + require.NoError(t, repo.StoreAdvanceResult(ctx, app.ID, &model.AdvanceResult{ + EpochIndex: 0, + InputIndex: 1, + Status: model.InputCompletionStatus_Rejected, + OutputsProof: model.OutputsProof{ + MachineHash: repotest.UniqueHash(), + OutputsHash: repotest.UniqueHash(), + }, + })) + + canonical, err := repo.ReplaySummary( + ctx, app.IApplicationAddress, repository.ReplayVerificationCanonical, + ) + require.NoError(t, err) + require.Equal(t, app.ID, canonical.ApplicationID) + require.Equal(t, uint64(2), canonical.ProcessedInputs) + require.Equal(t, model.Consensus_Authority, canonical.Consensus) + + canonicalPage, err := repo.ReplayPage(ctx, repository.ReplayPageRequest{ + ApplicationID: canonical.ApplicationID, + FromInput: 0, + ToInputExclusive: canonical.ProcessedInputs, + Limit: canonical.ProcessedInputs, + Verification: repository.ReplayVerificationCanonical, + }) + require.NoError(t, err) + require.Len(t, canonicalPage, 2) + for _, record := range canonicalPage { + require.Empty(t, record.Outputs) + require.Empty(t, record.Reports) + require.Empty(t, record.StateHashes) + } + + _, err = repo.ReplaySummary( + ctx, app.IApplicationAddress, repository.ReplayVerificationFull, + ) + require.NoError(t, err) + fullPage, err := repo.ReplayPage(ctx, repository.ReplayPageRequest{ + ApplicationID: canonical.ApplicationID, + FromInput: 0, + ToInputExclusive: canonical.ProcessedInputs, + Limit: canonical.ProcessedInputs, + Verification: repository.ReplayVerificationFull, + }) + require.NoError(t, err) + require.Equal(t, [][]byte{[]byte("output")}, fullPage[0].Outputs) + require.Equal(t, [][]byte{[]byte("report")}, fullPage[0].Reports) + + // Poison only full evidence. Canonical reconstruction remains independent + // of child evidence, while a full page exposes it to replay comparison. + _, err = conn.Exec(ctx, ` + INSERT INTO output (input_epoch_application_id, input_index, index, raw_data) + VALUES ($1, 1, 1, $2)`, app.ID, []byte("illegal-output")) + require.NoError(t, err) + _, err = repo.ReplaySummary( + ctx, app.IApplicationAddress, repository.ReplayVerificationCanonical, + ) + require.NoError(t, err) + fullPage, err = repo.ReplayPage(ctx, repository.ReplayPageRequest{ + ApplicationID: canonical.ApplicationID, + FromInput: 0, + ToInputExclusive: canonical.ProcessedInputs, + Limit: canonical.ProcessedInputs, + Verification: repository.ReplayVerificationFull, + }) + require.NoError(t, err) + require.Equal(t, [][]byte{[]byte("illegal-output")}, fullPage[1].Outputs) + + // State hashes are invalid for every Authority input, including completed + // inputs whose other Full evidence is otherwise valid. + illegalStateHash := repotest.UniqueHash() + _, err = conn.Exec(ctx, `INSERT INTO state_hashes ( + input_epoch_application_id, epoch_index, input_index, index, machine_hash, repetitions + ) VALUES ($1, 0, 1, 0, $2, 1)`, app.ID, illegalStateHash.Bytes()) + require.NoError(t, err) + _, err = repo.ReplaySummary( + ctx, app.IApplicationAddress, repository.ReplayVerificationCanonical, + ) + require.NoError(t, err) + _, err = repo.ReplaySummary( + ctx, app.IApplicationAddress, repository.ReplayVerificationFull, + ) + violation := requireReplayViolationKind( + t, err, repository.ReplayStructureUnexpectedStateHash, + ) + require.Equal(t, uint64(1), violation.InputIndex) + require.Equal(t, uint64(0), violation.EvidenceIndex) + _, err = conn.Exec(ctx, `DELETE FROM state_hashes WHERE input_epoch_application_id = $1`, app.ID) + require.NoError(t, err) + + // The persisted application counter and completed input rows are one + // invariant and must agree in the same repository snapshot. + _, err = conn.Exec(ctx, `UPDATE application SET processed_inputs = 1 WHERE id = $1`, app.ID) + require.NoError(t, err) + _, err = repo.ReplaySummary( + ctx, app.IApplicationAddress, repository.ReplayVerificationCanonical, + ) + violation = requireReplayViolationKind( + t, err, repository.ReplayStructureProcessedInputCount, + ) + require.Equal(t, uint64(1), violation.ApplicationProcessedInputs) + require.Equal(t, uint64(2), violation.CompletedInputCount) +} + +func TestPostgresReplayRejectsCompletedInputGap(t *testing.T) { + endpoint, err := db.GetTestDatabaseEndpoint() + if err != nil { + t.Skipf("Skipping: %v", err) + } + require.NoError(t, db.SetupTestPostgres(endpoint)) + + ctx := context.Background() + repo, err := factory.NewRepositoryFromConnectionString(ctx, endpoint) + require.NoError(t, err) + t.Cleanup(repo.Close) + conn, err := pgx.Connect(ctx, endpoint) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, conn.Close(ctx)) }) + + app := repotest.NewApplicationBuilder().Create(ctx, t, repo) + epoch := repotest.NewEpochBuilder(app.ID). + WithStatus(model.EpochStatus_Closed). + WithInputBounds(0, 1). + Build() + inputs := []*model.Input{ + repotest.NewInputBuilder().WithIndex(0).Build(), + repotest.NewInputBuilder().WithIndex(1).Build(), + } + require.NoError(t, repo.CreateEpochsAndInputs( + ctx, + app.IApplicationAddress.String(), + map[*model.Epoch][]*model.Input{epoch: inputs}, + 10, + )) + + // Create two completed rows at indexes 0 and 2. Changing the index while + // the input is still pending is allowed by the completion immutability + // trigger and models a malformed completed prefix without disabling it. + _, err = conn.Exec(ctx, `UPDATE input SET index = 2 + WHERE epoch_application_id = $1 AND index = 1`, app.ID) + require.NoError(t, err) + machineHash := repotest.UniqueHash() + outputsHash := repotest.UniqueHash() + _, err = conn.Exec(ctx, `UPDATE input + SET status = 'ACCEPTED', machine_hash = $2, outputs_hash = $3 + WHERE epoch_application_id = $1`, app.ID, machineHash.Bytes(), outputsHash.Bytes()) + require.NoError(t, err) + _, err = conn.Exec(ctx, `UPDATE application SET processed_inputs = 2 WHERE id = $1`, app.ID) + require.NoError(t, err) + + _, err = repo.ReplaySummary( + ctx, app.IApplicationAddress, repository.ReplayVerificationCanonical, + ) + violation := requireReplayViolationKind( + t, err, repository.ReplayStructureCompletedInputSequence, + ) + require.Equal(t, uint64(2), violation.InputIndex) + require.Equal(t, uint64(1), violation.ExpectedIndex) +} + +func TestPostgresReplayRejectsInvalidStateHashOrdering(t *testing.T) { + endpoint, err := db.GetTestDatabaseEndpoint() + if err != nil { + t.Skipf("Skipping: %v", err) + } + require.NoError(t, db.SetupTestPostgres(endpoint)) + + ctx := context.Background() + repo, err := factory.NewRepositoryFromConnectionString(ctx, endpoint) + require.NoError(t, err) + t.Cleanup(repo.Close) + conn, err := pgx.Connect(ctx, endpoint) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, conn.Close(ctx)) }) + + app := repotest.NewApplicationBuilder().WithConsensus(model.Consensus_PRT).Create(ctx, t, repo) + epoch := repotest.NewEpochBuilder(app.ID). + WithStatus(model.EpochStatus_Closed). + WithInputBounds(0, 1). + Build() + inputs := []*model.Input{ + repotest.NewInputBuilder().WithIndex(0).Build(), + repotest.NewInputBuilder().WithIndex(1).Build(), + } + require.NoError(t, repo.CreateEpochsAndInputs( + ctx, + app.IApplicationAddress.String(), + map[*model.Epoch][]*model.Input{epoch: inputs}, + 10, + )) + for inputIndex := range uint64(2) { + require.NoError(t, repo.StoreAdvanceResult(ctx, app.ID, &model.AdvanceResult{ + EpochIndex: 0, + InputIndex: inputIndex, + Status: model.InputCompletionStatus_Accepted, + IsDaveConsensus: true, + PaddingRepetitions: 1 << 24, + OutputsProof: model.OutputsProof{ + MachineHash: repotest.UniqueHash(), + OutputsHash: repotest.UniqueHash(), + }, + })) + } + _, err = repo.ReplaySummary(ctx, app.IApplicationAddress, repository.ReplayVerificationFull) + require.NoError(t, err) + + _, err = conn.Exec(ctx, `UPDATE state_hashes SET index = 2 WHERE index = 1`) + require.NoError(t, err) + _, err = repo.ReplaySummary(ctx, app.IApplicationAddress, repository.ReplayVerificationFull) + _ = requireReplayViolationKind(t, err, repository.ReplayStructureStateHashIndexSequence) + _, err = conn.Exec(ctx, `UPDATE state_hashes SET index = 1 WHERE index = 2`) + require.NoError(t, err) + + _, err = conn.Exec(ctx, `UPDATE state_hashes + SET input_index = CASE index WHEN 0 THEN 1 ELSE 0 END`) + require.NoError(t, err) + _, err = repo.ReplaySummary(ctx, app.IApplicationAddress, repository.ReplayVerificationFull) + _ = requireReplayViolationKind(t, err, repository.ReplayStructureStateHashInputOrder) +} + +func requireReplayViolationKind( + t *testing.T, + err error, + want repository.ReplayStructureViolationKind, +) *repository.ReplayStructureViolationError { + t.Helper() + require.ErrorIs(t, err, repository.ErrReplayInvalidStructure) + var violation *repository.ReplayStructureViolationError + require.ErrorAs(t, err, &violation) + require.Equal(t, want, violation.Kind) + return violation +} + +func TestPostgresReplayRejectsInvalidRequests(t *testing.T) { + endpoint, err := db.GetTestDatabaseEndpoint() + if err != nil { + t.Skipf("Skipping: %v", err) + } + require.NoError(t, db.SetupTestPostgres(endpoint)) + + ctx := context.Background() + repo, err := factory.NewRepositoryFromConnectionString(ctx, endpoint) + require.NoError(t, err) + t.Cleanup(repo.Close) + app := repotest.NewApplicationBuilder().Create(ctx, t, repo) + + _, err = repo.ReplaySummary(ctx, app.IApplicationAddress, repository.ReplayVerificationLevel(255)) + require.ErrorContains(t, err, "unsupported replay verification level") + _, err = repo.ReplaySummary( + ctx, + common.HexToAddress("0xdead"), + repository.ReplayVerificationCanonical, + ) + require.ErrorIs(t, err, repository.ErrNotFound) + _, err = repo.ReplayPage(ctx, repository.ReplayPageRequest{ + ApplicationID: app.ID, + FromInput: 1, + ToInputExclusive: 0, + Limit: 1, + Verification: repository.ReplayVerificationCanonical, + }) + require.ErrorContains(t, err, "replay input range is invalid") +} diff --git a/internal/repository/postgres/replay_test.go b/internal/repository/postgres/replay_test.go new file mode 100644 index 000000000..3fd48e483 --- /dev/null +++ b/internal/repository/postgres/replay_test.go @@ -0,0 +1,222 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package postgres + +import ( + "context" + "errors" + "testing" + + "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/internal/repository" + "github.com/cartesi/rollups-node/internal/repository/repotest" + "github.com/cartesi/rollups-node/test/tooling/db" + "github.com/ethereum/go-ethereum/common" + "github.com/jackc/pgx/v5" + "github.com/stretchr/testify/require" +) + +func TestStateHashInsertShape(t *testing.T) { + t.Parallel() + + t.Run("positive padding appends final row", func(t *testing.T) { + rows, err := stateHashInsertShape( + 1, model.InputHashCollectionCapacity-1, + ) + require.NoError(t, err) + require.Equal(t, uint64(2), rows) + }) + + t.Run("empty collection uses one positive padding row", func(t *testing.T) { + rows, err := stateHashInsertShape(0, model.InputHashCollectionCapacity) + require.NoError(t, err) + require.Equal(t, uint64(1), rows) + }) + + t.Run("exact boundary requires normalization before persistence", func(t *testing.T) { + _, err := stateHashInsertShape(model.InputHashCollectionCapacity, 0) + require.ErrorContains(t, err, "positive final repetition tail") + }) + + t.Run("rejects incomplete span", func(t *testing.T) { + _, err := stateHashInsertShape(1, 7) + require.ErrorContains(t, err, "does not cover input hash collection capacity") + }) + +} + +func TestStateHashCopySource(t *testing.T) { + t.Parallel() + + hashes := [][32]byte{common.HexToHash("0x11"), common.HexToHash("0x12")} + finalHash := common.HexToHash("0x13") + source := &stateHashCopySource{ + appID: 7, + epochIndex: 8, + inputIndex: 9, + nextIndex: 10, + hashes: hashes, + machineHash: finalHash, + paddingRepetitions: model.InputHashCollectionCapacity - uint64(len(hashes)), + rowCount: uint64(len(hashes)) + 1, + } + + for index, expectedHash := range append(hashes, finalHash) { + require.True(t, source.Next()) + values, err := source.Values() + require.NoError(t, err) + require.Equal(t, int64(7), values[0]) + require.Equal(t, uint64(8), values[1]) + require.Equal(t, uint64(9), values[2]) + require.Equal(t, uint64(10+index), values[3]) + require.Equal(t, expectedHash[:], values[4]) + if index < len(hashes) { + require.Equal(t, int64(1), values[5]) + } else { + require.Equal(t, model.InputHashCollectionCapacity-uint64(len(hashes)), values[5]) + } + } + require.False(t, source.Next()) + require.NoError(t, source.Err()) +} + +func TestInsertStateHashesQualifiesPublicSchema(t *testing.T) { + endpoint, err := db.GetTestDatabaseEndpoint() + if err != nil { + t.Skipf("Skipping: %v", err) + } + ctx := context.Background() + + // Remove a shadow schema left by an interrupted prior run before schema + // migrations try to drop the uint64/hash domains it may reference. + bootstrap, err := pgx.Connect(ctx, endpoint) + require.NoError(t, err) + _, err = bootstrap.Exec(ctx, `DROP SCHEMA IF EXISTS state_hash_copy_shadow CASCADE`) + require.NoError(t, err) + require.NoError(t, bootstrap.Close(ctx)) + require.NoError(t, db.SetupTestPostgres(endpoint)) + + repositoryInterface, err := NewPostgresRepository(ctx, endpoint, 1, 0) + require.NoError(t, err) + repo := repositoryInterface.(*PostgresRepository) + t.Cleanup(func() { + _, _ = repo.db.Exec(ctx, `DROP SCHEMA IF EXISTS state_hash_copy_shadow CASCADE`) + repo.Close() + }) + + app := repotest.NewApplicationBuilder().WithConsensus(model.Consensus_PRT).Create( + ctx, t, repo, + ) + epoch := repotest.NewEpochBuilder(app.ID). + WithStatus(model.EpochStatus_Closed). + WithInputBounds(0, 0). + Build() + input := repotest.NewInputBuilder().WithIndex(0).Build() + require.NoError(t, repo.CreateEpochsAndInputs( + ctx, + app.IApplicationAddress.String(), + map[*model.Epoch][]*model.Input{epoch: {input}}, + 1, + )) + + _, err = repo.db.Exec(ctx, ` + CREATE SCHEMA state_hash_copy_shadow; + CREATE TABLE state_hash_copy_shadow.state_hashes + (LIKE public.state_hashes INCLUDING ALL)`) + require.NoError(t, err) + tx, err := repo.db.Begin(ctx) + require.NoError(t, err) + defer tx.Rollback(ctx) //nolint:errcheck + _, err = tx.Exec(ctx, `SET LOCAL search_path TO state_hash_copy_shadow, public`) + require.NoError(t, err) + machineHash := repotest.UniqueHash() + require.NoError(t, insertStateHashes( + ctx, + tx, + app.ID, + 0, + 0, + nil, + machineHash, + model.InputHashCollectionCapacity, + )) + require.NoError(t, tx.Commit(ctx)) + + var publicCount, shadowCount uint64 + require.NoError(t, repo.db.QueryRow(ctx, `SELECT count(*) FROM public.state_hashes`).Scan(&publicCount)) + require.NoError(t, repo.db.QueryRow( + ctx, + `SELECT count(*) FROM state_hash_copy_shadow.state_hashes`, + ).Scan(&shadowCount)) + require.Equal(t, uint64(1), publicCount) + require.Zero(t, shadowCount) +} + +func TestReplayRecordForChild(t *testing.T) { + t.Parallel() + + record := new(model.ReplayRecord) + got, err := replayRecordForChild( + map[uint64]*model.ReplayRecord{4: record}, + repository.ReplayEvidenceOutput, + 4, + ) + require.NoError(t, err) + require.Same(t, record, got) + + for _, childKind := range []repository.ReplayEvidenceKind{ + repository.ReplayEvidenceOutput, + repository.ReplayEvidenceReport, + repository.ReplayEvidenceStateHash, + } { + t.Run(childKind.String(), func(t *testing.T) { + const inputWithoutCompletedRecord = uint64(7) + _, err := replayRecordForChild(nil, childKind, inputWithoutCompletedRecord) + + require.ErrorIs(t, err, repository.ErrReplayInconsistentEvidence) + var detail *repository.ReplayInconsistentEvidenceError + require.ErrorAs(t, err, &detail) + require.Equal(t, childKind, detail.Kind) + require.Equal(t, inputWithoutCompletedRecord, detail.InputIndex) + require.Contains(t, err.Error(), childKind.String()) + require.Contains(t, err.Error(), "input 7") + require.NotContains(t, err.Error(), "payload-must-stay-private") + }) + } +} + +func TestReplayInconsistencyErrorIdentity(t *testing.T) { + t.Parallel() + err := &repository.ReplayInconsistentEvidenceError{ + Kind: repository.ReplayEvidenceReport, + InputIndex: 11, + } + require.True(t, errors.Is(err, repository.ErrReplayInconsistentEvidence)) +} + +func TestReplayStructureViolationErrorIdentityAndPayloadHygiene(t *testing.T) { + t.Parallel() + epochIndex := uint64(3) + for _, kind := range []repository.ReplayStructureViolationKind{ + repository.ReplayStructureCompletedInputSequence, + repository.ReplayStructureStateHashIndexSequence, + repository.ReplayStructureStateHashInputOrder, + repository.ReplayStructureProcessedInputCount, + repository.ReplayStructureUnexpectedStateHash, + } { + t.Run(kind.String(), func(t *testing.T) { + err := &repository.ReplayStructureViolationError{ + Kind: kind, + EpochIndex: &epochIndex, + InputIndex: 4, + EvidenceIndex: 7, + ExpectedIndex: 6, + PreviousInputIndex: 5, + } + require.ErrorIs(t, err, repository.ErrReplayInvalidStructure) + require.Contains(t, err.Error(), "kind="+kind.String()) + require.NotContains(t, err.Error(), "payload-must-stay-private") + }) + } +} diff --git a/internal/repository/postgres/schema/migrations/000001_create_initial_schema.down.sql b/internal/repository/postgres/schema/migrations/000001_create_initial_schema.down.sql index f08086a54..dc333f594 100644 --- a/internal/repository/postgres/schema/migrations/000001_create_initial_schema.down.sql +++ b/internal/repository/postgres/schema/migrations/000001_create_initial_schema.down.sql @@ -4,6 +4,7 @@ BEGIN; DROP TRIGGER IF EXISTS "state_hashes_set_updated_at" ON "state_hashes"; +DROP INDEX IF EXISTS "state_hashes_input_index_idx"; DROP TABLE IF EXISTS "state_hashes"; DROP TRIGGER IF EXISTS "match_advances_set_updated_at" ON "match_advances"; @@ -29,6 +30,7 @@ DROP TRIGGER IF EXISTS "config_set_updated_at" ON "node_config"; DROP TABLE IF EXISTS "node_config"; DROP TRIGGER IF EXISTS "report_set_updated_at" ON "report"; +DROP INDEX IF EXISTS "report_input_index_idx"; DROP TABLE IF EXISTS "report"; DROP TRIGGER IF EXISTS "withdrawal_set_updated_at" ON "withdrawal"; @@ -36,16 +38,19 @@ DROP INDEX IF EXISTS "withdrawal_block_number_idx"; DROP TABLE IF EXISTS "withdrawal"; DROP TRIGGER IF EXISTS "output_set_updated_at" ON "output"; +DROP INDEX IF EXISTS "output_input_index_idx"; DROP INDEX IF EXISTS "output_raw_data_address_idx"; DROP INDEX IF EXISTS "output_raw_data_type_idx"; DROP TABLE IF EXISTS "output"; DROP TRIGGER IF EXISTS "input_set_updated_at" ON "input"; +DROP TRIGGER IF EXISTS "input_completion_immutability_check" ON "input"; DROP INDEX IF EXISTS "input_sender_idx"; DROP INDEX IF EXISTS "input_unprocessed_idx"; DROP INDEX IF EXISTS "input_status_idx"; DROP INDEX IF EXISTS "input_block_number_idx"; DROP TABLE IF EXISTS "input"; +DROP FUNCTION IF EXISTS "enforce_input_completion_immutability"; DROP TRIGGER IF EXISTS "epoch_status_transition_check" ON "epoch"; DROP TRIGGER IF EXISTS "epoch_set_updated_at" ON "epoch"; diff --git a/internal/repository/postgres/schema/migrations/000001_create_initial_schema.up.sql b/internal/repository/postgres/schema/migrations/000001_create_initial_schema.up.sql index 7eb4a5808..2d2da98fe 100644 --- a/internal/repository/postgres/schema/migrations/000001_create_initial_schema.up.sql +++ b/internal/repository/postgres/schema/migrations/000001_create_initial_schema.up.sql @@ -401,6 +401,11 @@ CREATE TABLE "input" CONSTRAINT "input_pkey" PRIMARY KEY ("epoch_application_id", "index"), CONSTRAINT "input_epoch_index_unique" UNIQUE ("epoch_application_id", "epoch_index", "index"), CONSTRAINT "input_application_id_tx_hash_log_index_unique" UNIQUE ("epoch_application_id", "transaction_hash", "log_index"), + CONSTRAINT "input_completed_hashes_check" CHECK ( + ("status" = 'NONE' AND "machine_hash" IS NULL AND "outputs_hash" IS NULL) + OR + ("status" <> 'NONE' AND "machine_hash" IS NOT NULL AND "outputs_hash" IS NOT NULL) + ), CONSTRAINT "input_epoch_id_fkey" FOREIGN KEY ("epoch_application_id", "epoch_index") REFERENCES "epoch"("application_id", "index") ON DELETE CASCADE ); @@ -416,6 +421,33 @@ CREATE INDEX "input_sender_idx" ON "input" ("epoch_application_id", substring("r -- dead tuples (and the shrinking partial index) are reclaimed promptly. ALTER TABLE "input" SET (autovacuum_vacuum_scale_factor = 0.02); +CREATE FUNCTION enforce_input_completion_immutability() +RETURNS TRIGGER AS $$ +BEGIN + IF OLD.status <> 'NONE' AND ( + NEW.epoch_application_id IS DISTINCT FROM OLD.epoch_application_id + OR NEW.epoch_index IS DISTINCT FROM OLD.epoch_index + OR NEW.index IS DISTINCT FROM OLD.index + OR NEW.raw_data IS DISTINCT FROM OLD.raw_data + OR NEW.status IS DISTINCT FROM OLD.status + OR NEW.exception_data IS DISTINCT FROM OLD.exception_data + OR NEW.machine_hash IS DISTINCT FROM OLD.machine_hash + OR NEW.outputs_hash IS DISTINCT FROM OLD.outputs_hash + ) THEN + RAISE EXCEPTION + 'completed input result is immutable'; + END IF; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER "input_completion_immutability_check" + BEFORE UPDATE OF "epoch_application_id", "epoch_index", "index", "raw_data", + "status", "exception_data", "machine_hash", "outputs_hash" ON "input" + FOR EACH ROW + EXECUTE FUNCTION enforce_input_completion_immutability(); + CREATE TRIGGER "input_set_updated_at" BEFORE UPDATE ON "input" FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); @@ -437,6 +469,8 @@ CREATE TABLE "output" CREATE INDEX "output_raw_data_type_idx" ON "output" ("input_epoch_application_id", substring("raw_data" FROM 1 FOR 4)); +CREATE INDEX "output_input_index_idx" ON "output" ("input_epoch_application_id", "input_index", "index"); + CREATE INDEX "output_raw_data_address_idx" ON "output" ("input_epoch_application_id", substring("raw_data" FROM 17 FOR 20)) WHERE SUBSTRING("raw_data" FROM 1 FOR 4) IN ( E'\\x10321e8b', -- DelegateCallVoucher @@ -486,6 +520,8 @@ CREATE TABLE "report" CONSTRAINT "report_input_id_fkey" FOREIGN KEY ("input_epoch_application_id", "input_index") REFERENCES "input"("epoch_application_id", "index") ON DELETE CASCADE ); +CREATE INDEX "report_input_index_idx" ON "report" ("input_epoch_application_id", "input_index", "index"); + CREATE TRIGGER "report_set_updated_at" BEFORE UPDATE ON "report" FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); @@ -660,6 +696,8 @@ CREATE TABLE "state_hashes" CONSTRAINT "state_hashes_input_id_fkey" FOREIGN KEY ("input_epoch_application_id", "epoch_index", "input_index") REFERENCES "input"("epoch_application_id", "epoch_index", "index") ON DELETE CASCADE ); +CREATE INDEX "state_hashes_input_index_idx" ON "state_hashes" ("input_epoch_application_id", "input_index", "index"); + CREATE TRIGGER "state_hashes_set_updated_at" BEFORE UPDATE ON "state_hashes" FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); diff --git a/internal/repository/replay.go b/internal/repository/replay.go new file mode 100644 index 000000000..d66c65ee6 --- /dev/null +++ b/internal/repository/replay.go @@ -0,0 +1,212 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package repository + +import ( + "context" + "errors" + "fmt" + + "github.com/cartesi/rollups-node/internal/model" + "github.com/ethereum/go-ethereum/common" +) + +var ( + // ErrReplayInconsistentEvidence identifies persisted child evidence whose + // parent is not one of the completed inputs selected for the same page. + ErrReplayInconsistentEvidence = errors.New("persisted replay evidence is inconsistent") + + // ErrReplayInvalidStructure identifies malformed persisted replay + // coordinates or relationships. Typed details never contain payload bytes. + ErrReplayInvalidStructure = errors.New("persisted replay structure is invalid") +) + +const unknownReplayValue = "unknown" + +// ReplayVerificationLevel selects the persisted evidence read during replay. +type ReplayVerificationLevel uint8 + +const ( + // ReplayVerificationCanonical reads the completion outcome and canonical + // roots. It is the inexpensive default for normal machine reconstruction. + ReplayVerificationCanonical ReplayVerificationLevel = iota + + // ReplayVerificationFull additionally reads accepted outputs and reports. + // For PRT applications, it also reads the compressed per-input state-hash + // collections used to reconstruct and verify the epoch computation hash. + // Outputs and reports are audit evidence and do not participate in that hash. + // It is intended for explicit audits and the forthcoming + // machine-tool/libcartesi adapter, not normal reconstruction. + // ReplayPage evidence is scoped to the requested half-open input range; + // ReplaySummary's completed-prefix and structural checks are application-wide. + // A maximal PRT collection is large, so callers must provision memory for + // all evidence associated with at least one input. + ReplayVerificationFull +) + +// IsValid reports whether level is a supported verification policy. +func (level ReplayVerificationLevel) IsValid() bool { + return level == ReplayVerificationCanonical || level == ReplayVerificationFull +} + +func (level ReplayVerificationLevel) String() string { + switch level { + case ReplayVerificationCanonical: + return "canonical" + case ReplayVerificationFull: + return "full" + default: + return unknownReplayValue + } +} + +// ReplayPageRequest selects an absolute, application-local, half-open input +// range and the evidence that must accompany it. +type ReplayPageRequest struct { + ApplicationID int64 + FromInput uint64 + ToInputExclusive uint64 + Limit uint64 + Verification ReplayVerificationLevel +} + +// ReplayRepository provides a stable completed-input prefix and keyset- +// paginated replay evidence. Implementations validate persisted invariants for +// the requested verification level. +type ReplayRepository interface { + ReplaySummary( + ctx context.Context, + applicationAddress common.Address, + verification ReplayVerificationLevel, + ) (model.ReplaySummary, error) + ReplayPage(ctx context.Context, request ReplayPageRequest) ([]*model.ReplayRecord, error) +} + +// ReplayStructureViolationKind identifies one payload-free replay invariant. +type ReplayStructureViolationKind uint8 + +const ( + // Zero is reserved so an uninitialized kind cannot identify a real + // structural violation and instead formats as unknown. + ReplayStructureCompletedInputSequence ReplayStructureViolationKind = iota + 1 + ReplayStructureStateHashIndexSequence + ReplayStructureStateHashInputOrder + ReplayStructureProcessedInputCount + ReplayStructureUnexpectedStateHash +) + +func (kind ReplayStructureViolationKind) String() string { + switch kind { + case ReplayStructureCompletedInputSequence: + return "completed_input.index_sequence" + case ReplayStructureStateHashIndexSequence: + return "state_hash.index_sequence" + case ReplayStructureStateHashInputOrder: + return "state_hash.input_order" + case ReplayStructureProcessedInputCount: + return "application.processed_input_count" + case ReplayStructureUnexpectedStateHash: + return "state_hash.unexpected_for_consensus" + default: + return unknownReplayValue + } +} + +// ReplayStructureViolationError contains only persisted counts and coordinates. +type ReplayStructureViolationError struct { + Kind ReplayStructureViolationKind + EpochIndex *uint64 + InputIndex uint64 + EvidenceIndex uint64 + ExpectedIndex uint64 + PreviousInputIndex uint64 + ApplicationProcessedInputs uint64 + CompletedInputCount uint64 +} + +func (e *ReplayStructureViolationError) Error() string { + epoch := "" + if e.EpochIndex != nil { + epoch = fmt.Sprint(*e.EpochIndex) + } + switch e.Kind { + case ReplayStructureCompletedInputSequence: + return fmt.Sprintf( + "%v: kind=%s expected_last_input=%d actual_last_input=%d", + ErrReplayInvalidStructure, e.Kind, e.ExpectedIndex, e.InputIndex, + ) + case ReplayStructureStateHashIndexSequence: + return fmt.Sprintf( + "%v: kind=%s epoch=%s input=%d expected_index=%d actual_index=%d", + ErrReplayInvalidStructure, e.Kind, epoch, e.InputIndex, + e.ExpectedIndex, e.EvidenceIndex, + ) + case ReplayStructureStateHashInputOrder: + return fmt.Sprintf( + "%v: kind=%s epoch=%s state_hash_index=%d previous_input=%d input=%d", + ErrReplayInvalidStructure, e.Kind, epoch, e.EvidenceIndex, + e.PreviousInputIndex, e.InputIndex, + ) + case ReplayStructureProcessedInputCount: + return fmt.Sprintf( + "%v: kind=%s application_processed_inputs=%d completed_input_count=%d", + ErrReplayInvalidStructure, e.Kind, + e.ApplicationProcessedInputs, e.CompletedInputCount, + ) + case ReplayStructureUnexpectedStateHash: + return fmt.Sprintf( + "%v: kind=%s epoch=%s input=%d state_hash_index=%d", + ErrReplayInvalidStructure, e.Kind, epoch, e.InputIndex, e.EvidenceIndex, + ) + default: + return fmt.Sprintf("%v: kind=%s", ErrReplayInvalidStructure, unknownReplayValue) + } +} + +func (e *ReplayStructureViolationError) Unwrap() error { return ErrReplayInvalidStructure } + +// ReplayEvidenceKind identifies persisted child evidence without exposing +// payloads. +type ReplayEvidenceKind uint8 + +const ( + // Zero is reserved so an uninitialized kind cannot identify real evidence + // and instead formats as unknown. + ReplayEvidenceOutput ReplayEvidenceKind = iota + 1 + ReplayEvidenceReport + ReplayEvidenceStateHash +) + +func (kind ReplayEvidenceKind) String() string { + switch kind { + case ReplayEvidenceOutput: + return "output" + case ReplayEvidenceReport: + return "report" + case ReplayEvidenceStateHash: + return "state_hash" + default: + return unknownReplayValue + } +} + +// ReplayInconsistentEvidenceError reports evidence whose input is not among +// the completed inputs selected for the page. +type ReplayInconsistentEvidenceError struct { + Kind ReplayEvidenceKind + InputIndex uint64 +} + +func (e *ReplayInconsistentEvidenceError) Error() string { + return fmt.Sprintf( + "%v: %s row for input %d has no completed replay input in the selected page", + ErrReplayInconsistentEvidence, + e.Kind, + e.InputIndex, + ) +} + +func (e *ReplayInconsistentEvidenceError) Unwrap() error { + return ErrReplayInconsistentEvidence +} diff --git a/internal/repository/replay_test.go b/internal/repository/replay_test.go new file mode 100644 index 000000000..678905023 --- /dev/null +++ b/internal/repository/replay_test.go @@ -0,0 +1,39 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package repository + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestReplayVerificationLevel(t *testing.T) { + require.True(t, ReplayVerificationCanonical.IsValid()) + require.True(t, ReplayVerificationFull.IsValid()) + require.False(t, ReplayVerificationLevel(255).IsValid()) + require.Equal(t, "canonical", ReplayVerificationCanonical.String()) + require.Equal(t, "full", ReplayVerificationFull.String()) + require.Equal(t, "unknown", ReplayVerificationLevel(255).String()) +} + +func TestReplaySourceErrorsPreserveIdentityWithoutPayloads(t *testing.T) { + inconsistency := &ReplayInconsistentEvidenceError{ + Kind: ReplayEvidenceReport, InputIndex: 7, + } + require.True(t, errors.Is(inconsistency, ErrReplayInconsistentEvidence)) + require.NotContains(t, inconsistency.Error(), "payload") + + violation := &ReplayStructureViolationError{ + Kind: ReplayStructureProcessedInputCount, + InputIndex: 7, + ApplicationProcessedInputs: 8, + CompletedInputCount: 9, + } + require.True(t, errors.Is(violation, ErrReplayInvalidStructure)) + require.Contains(t, violation.Error(), "application_processed_inputs=8") + require.Contains(t, violation.Error(), "completed_input_count=9") + require.NotContains(t, violation.Error(), "payload") +} diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 44f0d6115..f17d621fe 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -406,6 +406,7 @@ type ClaimerRepository interface { } type Repository interface { + ReplayRepository ApplicationRepository EpochRepository InputRepository diff --git a/internal/repository/repotest/epoch_test_cases.go b/internal/repository/repotest/epoch_test_cases.go index 3128fc4f4..bc91275de 100644 --- a/internal/repository/repotest/epoch_test_cases.go +++ b/internal/repository/repotest/epoch_test_cases.go @@ -1252,9 +1252,12 @@ func (s *EpochSuite) TestDrainGates() { s.Ctx, app.IApplicationAddress.String(), map[*Epoch][]*Input{ep: { NewInputBuilder().WithIndex(idx).WithEpochIndex(idx). - WithBlockNumber(first + 1).WithStatus(inputStatus).Build(), + WithBlockNumber(first + 1).Build(), }}, last+1) s.Require().NoError(err) + if inputStatus.IsCompleted() { + StoreAdvanceResult(s.Ctx, s.T(), s.Repo, app.ID, idx, idx, inputStatus, nil, nil) + } if target != EpochStatus_Closed { AdvanceEpochStatus(s.Ctx, s.T(), @@ -1494,11 +1497,11 @@ func (s *EpochSuite) TestDrainGates() { s.Ctx, app.IApplicationAddress.String(), map[*Epoch][]*Input{ep: { NewInputBuilder().WithIndex(0).WithEpochIndex(0). - WithBlockNumber(forecloseBlock). - WithStatus(InputCompletionStatus_Accepted). - Build(), + WithBlockNumber(forecloseBlock).Build(), }}, forecloseBlock+10) s.Require().NoError(err) + StoreAdvanceResult(s.Ctx, s.T(), s.Repo, app.ID, 0, 0, + InputCompletionStatus_Accepted, nil, nil) AdvanceEpochStatus(s.Ctx, s.T(), s.Repo, app.IApplicationAddress.String(), ep, EpochStatus_ClaimComputed) s.Require().NoError(s.Repo.UpdateApplicationForeclosure( diff --git a/internal/repository/repotest/input_test_cases.go b/internal/repository/repotest/input_test_cases.go index 5e725d68e..ddd55fd0a 100644 --- a/internal/repository/repotest/input_test_cases.go +++ b/internal/repository/repotest/input_test_cases.go @@ -68,22 +68,28 @@ func (s *InputSuite) TestGetLastInput() { func (s *InputSuite) TestGetLastProcessedInput() { s.Run("ReturnsLastProcessed", func() { + const ( + processedInputBlock uint64 = 5 + pendingInputBlock uint64 = 10 + ) + app := NewApplicationBuilder().Create(s.Ctx, s.T(), s.Repo) epoch := NewEpochBuilder(app.ID). WithIndex(0).WithStatus(EpochStatus_Closed). WithBlocks(0, 19).WithInputBounds(0, 1).Build() input0 := NewInputBuilder(). - WithIndex(0).WithBlockNumber(5). - WithStatus(InputCompletionStatus_Accepted).Build() + WithIndex(0).WithBlockNumber(processedInputBlock).Build() input1 := NewInputBuilder(). - WithIndex(1).WithBlockNumber(10). + WithIndex(1).WithBlockNumber(pendingInputBlock). WithStatus(InputCompletionStatus_None).Build() err := s.Repo.CreateEpochsAndInputs( s.Ctx, app.IApplicationAddress.String(), map[*Epoch][]*Input{epoch: {input0, input1}}, 20) s.Require().NoError(err) + StoreAdvanceResult(s.Ctx, s.T(), s.Repo, app.ID, 0, 0, + InputCompletionStatus_Accepted, nil, nil) got, err := s.Repo.GetLastProcessedInput(s.Ctx, app.IApplicationAddress.String()) s.Require().NoError(err) @@ -218,6 +224,11 @@ func (s *InputSuite) TestCreateEpochsAndInputsWithSameTransactionHash() { } func (s *InputSuite) TestListInputs() { + const ( + firstFilteredInputBlock uint64 = 5 + secondFilteredInputBlock uint64 = 10 + ) + s.Run("EmptyResult", func() { app := NewApplicationBuilder().Create(s.Ctx, s.T(), s.Repo) inputs, total, err := s.Repo.ListInputs( @@ -287,16 +298,17 @@ func (s *InputSuite) TestListInputs() { WithBlocks(0, 19).WithInputBounds(0, 1).Build() input0 := NewInputBuilder(). - WithIndex(0).WithBlockNumber(5). - WithStatus(InputCompletionStatus_Accepted).Build() + WithIndex(0).WithBlockNumber(firstFilteredInputBlock).Build() input1 := NewInputBuilder(). - WithIndex(1).WithBlockNumber(10). + WithIndex(1).WithBlockNumber(secondFilteredInputBlock). WithStatus(InputCompletionStatus_None).Build() err := s.Repo.CreateEpochsAndInputs( s.Ctx, app.IApplicationAddress.String(), map[*Epoch][]*Input{epoch: {input0, input1}}, 20) s.Require().NoError(err) + StoreAdvanceResult(s.Ctx, s.T(), s.Repo, app.ID, 0, 0, + InputCompletionStatus_Accepted, nil, nil) status := InputCompletionStatus_Accepted inputs, total, err := s.Repo.ListInputs( @@ -316,11 +328,9 @@ func (s *InputSuite) TestListInputs() { WithBlocks(0, 19).WithInputBounds(0, 2).Build() input0 := NewInputBuilder(). - WithIndex(0).WithBlockNumber(5). - WithStatus(InputCompletionStatus_Accepted).Build() + WithIndex(0).WithBlockNumber(firstFilteredInputBlock).Build() input1 := NewInputBuilder(). - WithIndex(1).WithBlockNumber(10). - WithStatus(InputCompletionStatus_Rejected).Build() + WithIndex(1).WithBlockNumber(secondFilteredInputBlock).Build() input2 := NewInputBuilder(). WithIndex(2).WithBlockNumber(15). WithStatus(InputCompletionStatus_None).Build() @@ -329,6 +339,10 @@ func (s *InputSuite) TestListInputs() { s.Ctx, app.IApplicationAddress.String(), map[*Epoch][]*Input{epoch: {input0, input1, input2}}, 20) s.Require().NoError(err) + StoreAdvanceResult(s.Ctx, s.T(), s.Repo, app.ID, 0, 0, + InputCompletionStatus_Accepted, nil, nil) + StoreAdvanceResult(s.Ctx, s.T(), s.Repo, app.ID, 0, 1, + InputCompletionStatus_Rejected, nil, nil) notStatus := InputCompletionStatus_None inputs, total, err := s.Repo.ListInputs( From f7a47c23c5713e2fe8b69d480834cfb39c005eb1 Mon Sep 17 00:00:00 2001 From: Victor Fusco <1221933+vfusco@users.noreply.github.com> Date: Fri, 14 Aug 2026 10:40:39 -0300 Subject: [PATCH 10/13] feat(replay): verify deterministic execution --- internal/replay/compare.go | 238 ++++++++++++++++++++ internal/replay/compare_test.go | 374 ++++++++++++++++++++++++++++++++ internal/replay/run.go | 253 +++++++++++++++++++++ internal/replay/run_test.go | 301 +++++++++++++++++++++++++ internal/replay/types.go | 55 +++++ 5 files changed, 1221 insertions(+) create mode 100644 internal/replay/compare.go create mode 100644 internal/replay/compare_test.go create mode 100644 internal/replay/run.go create mode 100644 internal/replay/run_test.go create mode 100644 internal/replay/types.go diff --git a/internal/replay/compare.go b/internal/replay/compare.go new file mode 100644 index 000000000..6a97e17a7 --- /dev/null +++ b/internal/replay/compare.go @@ -0,0 +1,238 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package replay + +import ( + "bytes" + "crypto/sha256" + "fmt" + "math" + + "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/internal/repository" + "github.com/ethereum/go-ethereum/common" +) + +type contradiction func(field string, expected, actual any) error + +func newContradiction( + application string, + epochIndex *uint64, + inputIndex uint64, + field string, + expected, actual any, +) *ContradictionError { + if application == "" { + application = "" + } + return &ContradictionError{ + Application: application, + EpochIndex: epochIndex, + InputIndex: inputIndex, + Field: field, + Expected: fmt.Sprint(expected), + Actual: fmt.Sprint(actual), + } +} + +func knownEpochIndex(epochIndex uint64) *uint64 { return &epochIndex } + +func compareRecord( + application string, + applicationID int64, + isPRT bool, + verification repository.ReplayVerificationLevel, + record *model.ReplayRecord, + actual *model.AdvanceResult, +) error { + if record == nil { + return newContradiction(application, nil, 0, "record", "persisted replay record", "nil") + } + contradiction := func(field string, expected, got any) error { + return newContradiction( + application, + knownEpochIndex(record.Input.EpochIndex), + record.Input.InputIndex, + field, + expected, + got, + ) + } + if actual == nil { + return contradiction("result", "completed result", "nil") + } + if record.Input.ApplicationID != applicationID { + return contradiction("application_id", applicationID, record.Input.ApplicationID) + } + if actual.EpochIndex != record.Input.EpochIndex { + return contradiction("epoch_index", record.Input.EpochIndex, actual.EpochIndex) + } + if actual.InputIndex != record.Input.InputIndex { + return contradiction("input_index", record.Input.InputIndex, actual.InputIndex) + } + if actual.Status != record.Input.Status { + return contradiction("status", record.Input.Status, actual.Status) + } + if !record.Input.Status.IsCompleted() { + return contradiction("status", "completed status", record.Input.Status) + } + if err := compareExceptionData(record.Input, actual, contradiction); err != nil { + return err + } + if record.Input.MachineHash == nil { + return contradiction("machine_hash", "persisted hash", "missing") + } + if actual.MachineHash != *record.Input.MachineHash { + return contradiction("machine_hash", record.Input.MachineHash.Hex(), actual.MachineHash.Hex()) + } + if record.Input.OutputsHash == nil { + return contradiction("outputs_hash", "persisted hash", "missing") + } + if actual.OutputsHash != *record.Input.OutputsHash { + return contradiction("outputs_hash", record.Input.OutputsHash.Hex(), actual.OutputsHash.Hex()) + } + if verification == repository.ReplayVerificationCanonical { + return nil + } + + if record.Input.Status == model.InputCompletionStatus_Accepted { + if err := compareBytes("outputs", record.Outputs, actual.Outputs, contradiction); err != nil { + return err + } + if err := compareBytes("reports", record.Reports, actual.Reports, contradiction); err != nil { + return err + } + } else { + // Effects of nonaccepted executions are diagnostics, not canonical. + if len(record.Outputs) != 0 { + return contradiction("outputs.count", 0, len(record.Outputs)) + } + if len(record.Reports) != 0 { + return contradiction("reports.count", 0, len(record.Reports)) + } + } + + if !isPRT { + if len(record.StateHashes) != 0 { + return contradiction("state_hashes.count", 0, len(record.StateHashes)) + } + if len(actual.PeriodicStateHashes) != 0 || actual.PaddingRepetitions != 0 { + return contradiction( + "replay_hash_collection", + "none", + fmt.Sprintf("hashes=%d padding=%d", len(actual.PeriodicStateHashes), actual.PaddingRepetitions), + ) + } + return nil + } + return compareHashCollection(record, actual, contradiction) +} + +func compareExceptionData( + input model.ReplayInput, + actual *model.AdvanceResult, + contradiction contradiction, +) error { + if input.Status == model.InputCompletionStatus_Exception { + if input.ExceptionData == nil { + return contradiction("exception_data", "persisted payload", "missing") + } + if actual.ExceptionData == nil { + return contradiction("exception_data", compactBytes(input.ExceptionData), "missing") + } + if !bytes.Equal(input.ExceptionData, actual.ExceptionData) { + return contradiction( + "exception_data", + compactBytes(input.ExceptionData), + compactBytes(actual.ExceptionData), + ) + } + return nil + } + if input.ExceptionData != nil { + return contradiction("exception_data", "none", compactBytes(input.ExceptionData)) + } + if actual.ExceptionData != nil { + return contradiction("exception_data", "none", compactBytes(actual.ExceptionData)) + } + return nil +} + +func compareBytes(field string, expected, actual [][]byte, contradiction contradiction) error { + if len(expected) != len(actual) { + return contradiction(field+".count", len(expected), len(actual)) + } + for i := range expected { + if !bytes.Equal(expected[i], actual[i]) { + return contradiction( + fmt.Sprintf("%s[%d]", field, i), + compactBytes(expected[i]), + compactBytes(actual[i]), + ) + } + } + return nil +} + +func compactBytes(data []byte) string { + digest := sha256.Sum256(data) + return fmt.Sprintf("len=%d sha256=%x", len(data), digest[:8]) +} + +func compareHashCollection( + record *model.ReplayRecord, + actual *model.AdvanceResult, + contradiction contradiction, +) error { + if err := model.ValidateInputHashCollectionSpan( + uint64(len(actual.PeriodicStateHashes)), + actual.PaddingRepetitions, + ); err != nil { + return contradiction( + "state_hashes.span", + model.InputHashCollectionCapacity, + fmt.Sprintf("hashes=%d padding=%d", len(actual.PeriodicStateHashes), actual.PaddingRepetitions), + ) + } + if actual.PaddingRepetitions == 0 { + return contradiction("state_hashes.final.repetitions", ">0", 0) + } + actualCount := len(actual.PeriodicStateHashes) + 1 + if len(record.StateHashes) != actualCount { + return contradiction("state_hashes.count", len(record.StateHashes), actualCount) + } + for i := 1; i < len(record.StateHashes); i++ { + previousIndex := record.StateHashes[i-1].Index + if previousIndex == math.MaxUint64 { + return contradiction("state_hashes.index", "index after uint64 maximum", record.StateHashes[i].Index) + } + if record.StateHashes[i].Index != previousIndex+1 { + return contradiction("state_hashes.index", previousIndex+1, record.StateHashes[i].Index) + } + } + for i, hash := range actual.PeriodicStateHashes { + row := record.StateHashes[i] + if row.Repetitions != 1 { + return contradiction(fmt.Sprintf("state_hashes[%d].repetitions", i), 1, row.Repetitions) + } + if row.MachineHash != common.Hash(hash) { + return contradiction( + fmt.Sprintf("state_hashes[%d].machine_hash", i), + row.MachineHash.Hex(), + common.Hash(hash).Hex(), + ) + } + } + final := record.StateHashes[len(record.StateHashes)-1] + if final.MachineHash != *record.Input.MachineHash { + return contradiction("state_hashes.final.machine_hash", record.Input.MachineHash.Hex(), final.MachineHash.Hex()) + } + if final.MachineHash != actual.MachineHash { + return contradiction("state_hashes.final.replay_hash", final.MachineHash.Hex(), actual.MachineHash.Hex()) + } + if final.Repetitions != actual.PaddingRepetitions { + return contradiction("state_hashes.final.repetitions", final.Repetitions, actual.PaddingRepetitions) + } + return nil +} diff --git a/internal/replay/compare_test.go b/internal/replay/compare_test.go new file mode 100644 index 000000000..830ec186c --- /dev/null +++ b/internal/replay/compare_test.go @@ -0,0 +1,374 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package replay + +import ( + "errors" + "fmt" + "testing" + + "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/internal/repository" + "github.com/ethereum/go-ethereum/common" + "github.com/stretchr/testify/require" +) + +func replayFixture(status model.InputCompletionStatus, consensus model.Consensus) ( + *model.Application, + *model.ReplayRecord, + *model.AdvanceResult, +) { + machineHash := common.HexToHash("0x11") + outputsHash := common.HexToHash("0x22") + app := &model.Application{ID: 7, Name: "replay-app", ConsensusType: consensus} + record := &model.ReplayRecord{ + Input: model.ReplayInput{ + ApplicationID: app.ID, + EpochIndex: 3, + InputIndex: 9, + RawData: []byte("input"), + Status: status, + MachineHash: &machineHash, + OutputsHash: &outputsHash, + }, + } + actual := &model.AdvanceResult{ + EpochIndex: 3, + InputIndex: 9, + Status: status, + OutputsProof: model.OutputsProof{ + MachineHash: machineHash, + OutputsHash: outputsHash, + }, + } + if status == model.InputCompletionStatus_Exception { + record.Input.ExceptionData = []byte("guest exception") + actual.ExceptionData = []byte("guest exception") + } + if status == model.InputCompletionStatus_Accepted { + record.Outputs = [][]byte{[]byte("output-a"), []byte("output-b")} + record.Reports = [][]byte{[]byte("report-a"), []byte("report-b")} + actual.Outputs = [][]byte{[]byte("output-a"), []byte("output-b")} + actual.Reports = [][]byte{[]byte("report-a"), []byte("report-b")} + } + return app, record, actual +} + +func TestCompareRecordCanonicalIgnoresFullEvidence(t *testing.T) { + app, record, actual := replayFixture( + model.InputCompletionStatus_Accepted, + model.Consensus_PRT, + ) + record.Outputs = [][]byte{[]byte("persisted-output")} + record.Reports = [][]byte{[]byte("persisted-report")} + record.StateHashes = []model.ReplayStateHash{{MachineHash: *record.Input.MachineHash, Repetitions: 1}} + actual.Outputs = [][]byte{[]byte("different-output")} + actual.Reports = [][]byte{[]byte("different-report")} + actual.PeriodicStateHashes = [][32]byte{{1}} + actual.PaddingRepetitions = model.InputHashCollectionCapacity - 1 + + require.NoError(t, compareRecord( + app.Name, + app.ID, + app.IsDaveConsensus(), + repository.ReplayVerificationCanonical, + record, + actual, + )) + + actual.MachineHash[0]++ + require.ErrorIs(t, compareRecord( + app.Name, + app.ID, + app.IsDaveConsensus(), + repository.ReplayVerificationCanonical, + record, + actual, + ), ErrContradiction) +} + +func TestCompareReplayRecordExceptionData(t *testing.T) { + t.Parallel() + + t.Run("payload mismatch", func(t *testing.T) { + app, record, actual := replayFixture( + model.InputCompletionStatus_Exception, + model.Consensus_Authority, + ) + actual.ExceptionData = []byte("different guest exception") + err := compareRecord(app.Name, app.ID, app.IsDaveConsensus(), repository.ReplayVerificationFull, record, actual) + var detail *ContradictionError + require.ErrorAs(t, err, &detail) + require.Equal(t, "exception_data", detail.Field) + require.NotContains(t, err.Error(), "guest exception") + }) + + for _, test := range []struct { + name string + status model.InputCompletionStatus + mutate func(*model.ReplayRecord, *model.AdvanceResult) + }{ + {"missing persisted exception payload", model.InputCompletionStatus_Exception, func(r *model.ReplayRecord, _ *model.AdvanceResult) { + r.Input.ExceptionData = nil + }}, + {"missing replayed exception payload", model.InputCompletionStatus_Exception, func(_ *model.ReplayRecord, a *model.AdvanceResult) { + a.ExceptionData = nil + }}, + {"unexpected persisted payload", model.InputCompletionStatus_Rejected, func(r *model.ReplayRecord, _ *model.AdvanceResult) { + r.Input.ExceptionData = []byte("unexpected") + }}, + {"unexpected replayed payload", model.InputCompletionStatus_Rejected, func(_ *model.ReplayRecord, a *model.AdvanceResult) { + a.ExceptionData = []byte("unexpected") + }}, + } { + t.Run(test.name, func(t *testing.T) { + app, record, actual := replayFixture(test.status, model.Consensus_Authority) + test.mutate(record, actual) + require.ErrorIs(t, + compareRecord(app.Name, app.ID, app.IsDaveConsensus(), repository.ReplayVerificationFull, record, actual), + ErrContradiction, + ) + }) + } +} + +func TestCompareReplayRecordCompletionMatrix(t *testing.T) { + t.Parallel() + + statuses := []model.InputCompletionStatus{ + model.InputCompletionStatus_Accepted, + model.InputCompletionStatus_Rejected, + model.InputCompletionStatus_Exception, + model.InputCompletionStatus_MachineHalted, + } + consensuses := []model.Consensus{ + model.Consensus_Authority, + model.Consensus_Quorum, + model.Consensus_PRT, + } + for _, consensus := range consensuses { + for _, status := range statuses { + t.Run(consensus.String()+"/"+status.String(), func(t *testing.T) { + app, record, actual := replayFixture(status, consensus) + if status != model.InputCompletionStatus_Accepted { + actual.Outputs = [][]byte{[]byte("noncanonical diagnostic output")} + actual.Reports = [][]byte{[]byte("noncanonical diagnostic report")} + } + if consensus == model.Consensus_PRT { + checkpoint := common.HexToHash("0x33") + actual.IsDaveConsensus = true + actual.PeriodicStateHashes = [][32]byte{checkpoint} + actual.PaddingRepetitions = model.InputHashCollectionCapacity - 1 + record.StateHashes = []model.ReplayStateHash{ + {Index: 20, MachineHash: checkpoint, Repetitions: 1}, + {Index: 21, MachineHash: actual.MachineHash, Repetitions: model.InputHashCollectionCapacity - 1}, + } + } else { + require.False(t, actual.IsDaveConsensus) + require.Empty(t, actual.PeriodicStateHashes) + require.Zero(t, actual.PaddingRepetitions) + require.Empty(t, record.StateHashes) + } + + require.NoError(t, + compareRecord(app.Name, app.ID, app.IsDaveConsensus(), repository.ReplayVerificationFull, record, actual), + ) + mismatchStatus := model.InputCompletionStatus_Accepted + if status == model.InputCompletionStatus_Accepted { + mismatchStatus = model.InputCompletionStatus_Rejected + } + actual.Status = mismatchStatus + require.ErrorIs(t, + compareRecord(app.Name, app.ID, app.IsDaveConsensus(), repository.ReplayVerificationFull, record, actual), + ErrContradiction, + ) + }) + } + } +} + +func TestCompareReplayRecordAcceptedMutationTable(t *testing.T) { + t.Parallel() + + mutations := []struct { + name string + mutate func(*model.Application, *model.ReplayRecord, *model.AdvanceResult) + }{ + {"application", func(_ *model.Application, r *model.ReplayRecord, _ *model.AdvanceResult) { + r.Input.ApplicationID++ + }}, + {"status", func(_ *model.Application, _ *model.ReplayRecord, a *model.AdvanceResult) { + a.Status = model.InputCompletionStatus_Rejected + }}, + {"machine-root", func(_ *model.Application, _ *model.ReplayRecord, a *model.AdvanceResult) { a.MachineHash[0]++ }}, + {"outputs-root", func(_ *model.Application, _ *model.ReplayRecord, a *model.AdvanceResult) { a.OutputsHash[0]++ }}, + {"outputs-count", func(_ *model.Application, _ *model.ReplayRecord, a *model.AdvanceResult) { + a.Outputs = a.Outputs[:1] + }}, + {"outputs-content", func(_ *model.Application, _ *model.ReplayRecord, a *model.AdvanceResult) { + a.Outputs[0] = []byte("changed") + }}, + {"outputs-order", func(_ *model.Application, _ *model.ReplayRecord, a *model.AdvanceResult) { + a.Outputs[0], a.Outputs[1] = a.Outputs[1], a.Outputs[0] + }}, + {"reports-count", func(_ *model.Application, _ *model.ReplayRecord, a *model.AdvanceResult) { + a.Reports = a.Reports[:1] + }}, + {"reports-content", func(_ *model.Application, _ *model.ReplayRecord, a *model.AdvanceResult) { + a.Reports[0] = []byte("changed") + }}, + {"reports-order", func(_ *model.Application, _ *model.ReplayRecord, a *model.AdvanceResult) { + a.Reports[0], a.Reports[1] = a.Reports[1], a.Reports[0] + }}, + {"unexpected-hash-collection", func(_ *model.Application, _ *model.ReplayRecord, a *model.AdvanceResult) { + a.PeriodicStateHashes = append(a.PeriodicStateHashes, common.HexToHash("0x33")) + }}, + } + for _, mutation := range mutations { + t.Run(mutation.name, func(t *testing.T) { + app, record, actual := replayFixture(model.InputCompletionStatus_Accepted, model.Consensus_Authority) + mutation.mutate(app, record, actual) + err := compareRecord(app.Name, app.ID, app.IsDaveConsensus(), repository.ReplayVerificationFull, record, actual) + require.ErrorIs(t, err, ErrContradiction) + var detail *ContradictionError + require.ErrorAs(t, err, &detail) + require.NotEmpty(t, detail.Field) + require.NotNil(t, detail.EpochIndex) + require.Equal(t, uint64(3), *detail.EpochIndex) + require.Contains(t, err.Error(), "epoch=3") + require.NotContains(t, err.Error(), "output-a") + }) + } + +} + +func TestCompareReplayRecordPersistedRecordValidation(t *testing.T) { + t.Parallel() + + t.Run("missing machine root", func(t *testing.T) { + app, record, actual := replayFixture(model.InputCompletionStatus_Rejected, model.Consensus_Authority) + record.Input.MachineHash = nil + require.ErrorIs(t, + compareRecord(app.Name, app.ID, app.IsDaveConsensus(), repository.ReplayVerificationFull, record, actual), + ErrContradiction, + ) + }) + t.Run("missing outputs root", func(t *testing.T) { + app, record, actual := replayFixture(model.InputCompletionStatus_Rejected, model.Consensus_Authority) + record.Input.OutputsHash = nil + require.ErrorIs(t, + compareRecord(app.Name, app.ID, app.IsDaveConsensus(), repository.ReplayVerificationFull, record, actual), + ErrContradiction, + ) + }) + t.Run("unknown persisted status", func(t *testing.T) { + app, record, actual := replayFixture(model.InputCompletionStatus_Rejected, model.Consensus_Authority) + record.Input.Status = model.InputCompletionStatus("UNKNOWN") + require.ErrorIs(t, + compareRecord(app.Name, app.ID, app.IsDaveConsensus(), repository.ReplayVerificationFull, record, actual), + ErrContradiction, + ) + }) + for _, effect := range []string{"output", "report"} { + t.Run("nonaccepted persisted "+effect, func(t *testing.T) { + app, record, actual := replayFixture(model.InputCompletionStatus_Rejected, model.Consensus_Authority) + if effect == "output" { + record.Outputs = [][]byte{[]byte("illegal")} + } else { + record.Reports = [][]byte{[]byte("illegal")} + } + // Replay diagnostics are ignored, but persisted effects are corruption. + actual.Outputs = [][]byte{[]byte("diagnostic")} + actual.Reports = [][]byte{[]byte("diagnostic")} + require.ErrorIs(t, + compareRecord(app.Name, app.ID, app.IsDaveConsensus(), repository.ReplayVerificationFull, record, actual), + ErrContradiction, + ) + }) + } + t.Run("nonaccepted replay diagnostics ignored", func(t *testing.T) { + app, record, actual := replayFixture(model.InputCompletionStatus_Exception, model.Consensus_Authority) + actual.Outputs = [][]byte{[]byte("diagnostic")} + actual.Reports = [][]byte{[]byte("diagnostic")} + require.NoError(t, compareRecord(app.Name, app.ID, app.IsDaveConsensus(), repository.ReplayVerificationFull, record, actual)) + }) +} + +func TestCompareReplayRecordPRTMutationTable(t *testing.T) { + t.Parallel() + + fixture := func() (*model.Application, *model.ReplayRecord, *model.AdvanceResult) { + app, record, actual := replayFixture(model.InputCompletionStatus_Accepted, model.Consensus_PRT) + hash0 := common.HexToHash("0x31") + hash1 := common.HexToHash("0x32") + actual.IsDaveConsensus = true + actual.PeriodicStateHashes = [][32]byte{hash0, hash1} + actual.PaddingRepetitions = model.InputHashCollectionCapacity - 2 + record.StateHashes = []model.ReplayStateHash{ + {Index: 40, MachineHash: hash0, Repetitions: 1}, + {Index: 41, MachineHash: hash1, Repetitions: 1}, + {Index: 42, MachineHash: actual.MachineHash, Repetitions: model.InputHashCollectionCapacity - 2}, + } + return app, record, actual + } + + app, record, actual := fixture() + require.NoError(t, compareRecord(app.Name, app.ID, app.IsDaveConsensus(), repository.ReplayVerificationFull, record, actual)) + + mutations := []struct { + name string + mutate func(*model.ReplayRecord, *model.AdvanceResult) + }{ + {"count", func(r *model.ReplayRecord, _ *model.AdvanceResult) { r.StateHashes = r.StateHashes[:2] }}, + {"hash", func(r *model.ReplayRecord, _ *model.AdvanceResult) { r.StateHashes[0].MachineHash[0]++ }}, + {"order", func(r *model.ReplayRecord, _ *model.AdvanceResult) { + r.StateHashes[0], r.StateHashes[1] = r.StateHashes[1], r.StateHashes[0] + }}, + {"index-order", func(r *model.ReplayRecord, _ *model.AdvanceResult) { r.StateHashes[1].Index++ }}, + {"repetition", func(r *model.ReplayRecord, _ *model.AdvanceResult) { r.StateHashes[0].Repetitions++ }}, + {"final-row", func(r *model.ReplayRecord, _ *model.AdvanceResult) { r.StateHashes[2].MachineHash[0]++ }}, + {"padding", func(r *model.ReplayRecord, _ *model.AdvanceResult) { r.StateHashes[2].Repetitions++ }}, + {"result-padding", func(_ *model.ReplayRecord, a *model.AdvanceResult) { a.PaddingRepetitions++ }}, + } + for _, mutation := range mutations { + t.Run(mutation.name, func(t *testing.T) { + app, record, actual := fixture() + mutation.mutate(record, actual) + require.ErrorIs(t, + compareRecord(app.Name, app.ID, app.IsDaveConsensus(), repository.ReplayVerificationFull, record, actual), + ErrContradiction, + ) + }) + } + + t.Run("diagnostics keep persisted expected and replay actual", func(t *testing.T) { + app, record, actual := fixture() + actual.PaddingRepetitions = model.InputHashCollectionCapacity - 1 + err := compareRecord(app.Name, app.ID, app.IsDaveConsensus(), repository.ReplayVerificationFull, record, actual) + var detail *ContradictionError + require.ErrorAs(t, err, &detail) + require.Equal(t, "state_hashes.span", detail.Field) + require.Equal(t, fmt.Sprint(model.InputHashCollectionCapacity), detail.Expected) + require.Equal(t, fmt.Sprintf("hashes=2 padding=%d", model.InputHashCollectionCapacity-1), detail.Actual) + }) +} + +func TestContradictionErrorIdentity(t *testing.T) { + t.Parallel() + t.Run("unknown epoch", func(t *testing.T) { + err := &ContradictionError{Application: "app", Field: "status"} + require.True(t, errors.Is(err, ErrContradiction)) + require.Contains(t, err.Error(), "epoch=") + }) + t.Run("known nonzero epoch", func(t *testing.T) { + epochIndex := uint64(17) + err := &ContradictionError{ + Application: "app", + EpochIndex: &epochIndex, + Field: "status", + } + require.Contains(t, err.Error(), "epoch=17") + require.NotContains(t, err.Error(), "epoch=") + }) +} diff --git a/internal/replay/run.go b/internal/replay/run.go new file mode 100644 index 000000000..c89b31f86 --- /dev/null +++ b/internal/replay/run.go @@ -0,0 +1,253 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package replay + +import ( + "context" + "errors" + "fmt" + + "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/internal/repository" +) + +// Executor is the machine behavior required by Run. Machine lifecycle and +// persistence remain the caller's responsibility. +type Executor interface { + ProcessedInputs() uint64 + Advance( + ctx context.Context, + input []byte, + epochIndex uint64, + inputIndex uint64, + computeHashes bool, + ) (*model.AdvanceResult, error) +} + +// Options selects an absolute, application-local, half-open replay range. +type Options struct { + Application *model.Application + FromInput uint64 + ToInputExclusive uint64 + BatchSize uint64 + Verification repository.ReplayVerificationLevel +} + +// Result summarizes a successfully verified replay. +type Result struct { + ReplayedInputs uint64 +} + +// Run reconstructs and verifies a machine over the requested input range. It +// never closes or stores executor; callers discard it after any error. +func Run( + ctx context.Context, + source repository.ReplayRepository, + executor Executor, + opts Options, +) (Result, error) { + if err := validateOptions(source, executor, opts); err != nil { + return Result{}, err + } + applicationLabel := opts.Application.Name + if applicationLabel == "" { + applicationLabel = opts.Application.IApplicationAddress.String() + } + summary, err := source.ReplaySummary( + ctx, + opts.Application.IApplicationAddress, + opts.Verification, + ) + if err != nil { + return Result{}, classifySourceError(applicationLabel, err) + } + if err := verifySummary(applicationLabel, opts.Application, summary, opts.FromInput); err != nil { + return Result{}, err + } + + isPRT := opts.Application.IsDaveConsensus() + computeHashes := opts.Verification == repository.ReplayVerificationFull && isPRT + contradiction := func(inputIndex uint64, field string, expected, actual any) error { + return newContradiction(applicationLabel, nil, inputIndex, field, expected, actual) + } + var replayed uint64 + for opts.FromInput+replayed < opts.ToInputExclusive { + from := opts.FromInput + replayed + remaining := opts.ToInputExclusive - from + limit := min(opts.BatchSize, remaining) + records, err := source.ReplayPage(ctx, repository.ReplayPageRequest{ + ApplicationID: summary.ApplicationID, + FromInput: from, + ToInputExclusive: opts.ToInputExclusive, + Limit: limit, + Verification: opts.Verification, + }) + if err != nil { + return Result{}, classifySourceError(applicationLabel, err) + } + if err := validatePage(applicationLabel, records, from, limit, remaining); err != nil { + return Result{}, err + } + for _, record := range records { + input := &record.Input + actual, err := executor.Advance( + ctx, + input.RawData, + input.EpochIndex, + input.InputIndex, + computeHashes, + ) + if err != nil { + return Result{}, fmt.Errorf( + "%w: input %d: %w", + ErrExecution, + input.InputIndex, + err, + ) + } + if err := compareRecord( + applicationLabel, + opts.Application.ID, + isPRT, + opts.Verification, + record, + actual, + ); err != nil { + return Result{}, err + } + replayed++ + if expected := input.InputIndex + 1; executor.ProcessedInputs() != expected { + return Result{}, contradiction( + input.InputIndex, + "executor.processed_inputs", + expected, + executor.ProcessedInputs(), + ) + } + } + } + return Result{ReplayedInputs: replayed}, nil +} + +func verifySummary( + application string, + expected *model.Application, + actual model.ReplaySummary, + firstInput uint64, +) error { + contradiction := func(field string, expected, actual any) error { + return newContradiction(application, nil, firstInput, field, expected, actual) + } + if actual.ApplicationID != expected.ID { + return contradiction("application_id", expected.ID, actual.ApplicationID) + } + if actual.Consensus != expected.ConsensusType { + return contradiction("consensus", expected.ConsensusType, actual.Consensus) + } + if actual.ProcessedInputs != expected.ProcessedInputs { + return contradiction( + "processed_inputs.count", + expected.ProcessedInputs, + actual.ProcessedInputs, + ) + } + return nil +} + +func validateOptions(source repository.ReplayRepository, executor Executor, opts Options) error { + switch { + case source == nil: + return fmt.Errorf("%w: replay source is nil", ErrInvalidOptions) + case executor == nil: + return fmt.Errorf("%w: replay executor is nil", ErrInvalidOptions) + case opts.Application == nil: + return fmt.Errorf("%w: application is nil", ErrInvalidOptions) + case !opts.Verification.IsValid(): + return fmt.Errorf("%w: unsupported verification level %d", ErrInvalidOptions, opts.Verification) + case opts.BatchSize == 0: + return fmt.Errorf("%w: replay batch size must be greater than zero", ErrInvalidOptions) + case opts.FromInput > opts.ToInputExclusive: + return fmt.Errorf( + "%w: lower bound %d exceeds upper bound %d", + ErrInvalidOptions, + opts.FromInput, + opts.ToInputExclusive, + ) + case opts.ToInputExclusive > opts.Application.ProcessedInputs: + return fmt.Errorf( + "%w: upper bound %d exceeds application processed input count %d", + ErrInvalidOptions, + opts.ToInputExclusive, + opts.Application.ProcessedInputs, + ) + case executor.ProcessedInputs() != opts.FromInput: + return fmt.Errorf( + "%w: executor has %d processed inputs; replay starts at %d", + ErrInvalidOptions, + executor.ProcessedInputs(), + opts.FromInput, + ) + default: + return nil + } +} + +func validatePage( + application string, + records []*model.ReplayRecord, + firstInput uint64, + limit uint64, + remaining uint64, +) error { + contradiction := func(inputIndex uint64, field string, expected, actual any) error { + return newContradiction(application, nil, inputIndex, field, expected, actual) + } + if uint64(len(records)) > limit { + return contradiction(firstInput, "replay_page.count", limit, len(records)) + } + if len(records) == 0 { + return contradiction(firstInput, "replay_sequence.remaining_records", remaining, 0) + } + for offset, record := range records { + expectedInput := firstInput + uint64(offset) + if record == nil { + return contradiction(expectedInput, "replay_record", "record", "nil") + } + if record.Input.InputIndex != expectedInput { + return contradiction( + expectedInput, + "input_index.sequence", + expectedInput, + record.Input.InputIndex, + ) + } + } + return nil +} + +func classifySourceError(application string, err error) error { + var structure *repository.ReplayStructureViolationError + if errors.As(err, &structure) { + return newContradiction( + application, + structure.EpochIndex, + structure.InputIndex, + "source."+structure.Kind.String(), + "valid persisted replay structure", + structure.Error(), + ) + } + var inconsistency *repository.ReplayInconsistentEvidenceError + if errors.As(err, &inconsistency) { + return newContradiction( + application, + nil, + inconsistency.InputIndex, + "source."+inconsistency.Kind.String()+".completed_input", + "completed replay input in page", + "not present", + ) + } + return fmt.Errorf("%w: source: %w", ErrExecution, err) +} diff --git a/internal/replay/run_test.go b/internal/replay/run_test.go new file mode 100644 index 000000000..69c59a5f2 --- /dev/null +++ b/internal/replay/run_test.go @@ -0,0 +1,301 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package replay + +import ( + "context" + "errors" + "math/big" + "testing" + + "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/internal/repository" + "github.com/ethereum/go-ethereum/common" + "github.com/stretchr/testify/require" +) + +type fakeSource struct { + summary model.ReplaySummary + summaryErr error + records []*model.ReplayRecord + pageErr error + pageRequests []repository.ReplayPageRequest + summaryLevels []repository.ReplayVerificationLevel + summaryApps []common.Address + pageOverride func(repository.ReplayPageRequest) ([]*model.ReplayRecord, error) +} + +func (source *fakeSource) ReplaySummary( + _ context.Context, + applicationAddress common.Address, + verification repository.ReplayVerificationLevel, +) (model.ReplaySummary, error) { + source.summaryApps = append(source.summaryApps, applicationAddress) + source.summaryLevels = append(source.summaryLevels, verification) + return source.summary, source.summaryErr +} + +func (source *fakeSource) ReplayPage( + _ context.Context, + request repository.ReplayPageRequest, +) ([]*model.ReplayRecord, error) { + source.pageRequests = append(source.pageRequests, request) + if source.pageOverride != nil { + return source.pageOverride(request) + } + if source.pageErr != nil { + return nil, source.pageErr + } + page := make([]*model.ReplayRecord, 0, request.Limit) + for _, record := range source.records { + if record == nil || record.Input.InputIndex >= request.FromInput && + record.Input.InputIndex < request.ToInputExclusive { + page = append(page, record) + if uint64(len(page)) == request.Limit { + break + } + } + } + return page, nil +} + +type fakeExecutor struct { + processed uint64 + advanceErr error + advanceCalls []model.ReplayInput + computeHashes []bool + wrongResultPos bool + fullPRTResult bool +} + +func (executor *fakeExecutor) ProcessedInputs() uint64 { return executor.processed } + +func (executor *fakeExecutor) Advance( + _ context.Context, + data []byte, + epochIndex uint64, + inputIndex uint64, + computeHashes bool, +) (*model.AdvanceResult, error) { + if executor.advanceErr != nil { + return nil, executor.advanceErr + } + executor.advanceCalls = append(executor.advanceCalls, model.ReplayInput{ + EpochIndex: epochIndex, + InputIndex: inputIndex, + RawData: data, + }) + executor.computeHashes = append(executor.computeHashes, computeHashes) + executor.processed++ + resultIndex := inputIndex + if executor.wrongResultPos { + resultIndex++ + } + result := &model.AdvanceResult{ + EpochIndex: epochIndex, + InputIndex: resultIndex, + Status: model.InputCompletionStatus_Accepted, + OutputsProof: model.OutputsProof{ + MachineHash: common.BigToHash(newBig(inputIndex + 1)), + OutputsHash: common.BigToHash(newBig(inputIndex + 100)), + }, + } + if executor.fullPRTResult { + result.PaddingRepetitions = 1 << 24 + } + return result, nil +} + +func newBig(value uint64) *big.Int { return new(big.Int).SetUint64(value) } + +func replayRecords(count uint64) []*model.ReplayRecord { + records := make([]*model.ReplayRecord, count) + for index := range count { + machineHash := common.BigToHash(newBig(index + 1)) + outputsHash := common.BigToHash(newBig(index + 100)) + records[index] = &model.ReplayRecord{Input: model.ReplayInput{ + ApplicationID: 7, + EpochIndex: index / 2, + InputIndex: index, + RawData: []byte{byte(index)}, + Status: model.InputCompletionStatus_Accepted, + MachineHash: &machineHash, + OutputsHash: &outputsHash, + }} + } + return records +} + +func replayOptions(consensus model.Consensus, from, to uint64) Options { + return Options{ + Application: &model.Application{ + ID: 7, + Name: "app", + IApplicationAddress: common.HexToAddress("0x1234"), + ConsensusType: consensus, + ProcessedInputs: to, + }, + FromInput: from, + ToInputExclusive: to, + BatchSize: 2, + Verification: repository.ReplayVerificationCanonical, + } +} + +func TestRunCanonicalRangeAndPagination(t *testing.T) { + records := replayRecords(5) + source := &fakeSource{ + summary: model.ReplaySummary{ApplicationID: 7, ProcessedInputs: 5, Consensus: model.Consensus_PRT}, + records: records, + } + executor := &fakeExecutor{processed: 2} + opts := replayOptions(model.Consensus_PRT, 2, 5) + + result, err := Run(context.Background(), source, executor, opts) + require.NoError(t, err) + require.Equal(t, uint64(3), result.ReplayedInputs) + require.Equal(t, uint64(5), executor.ProcessedInputs()) + require.Equal(t, []bool{false, false, false}, executor.computeHashes) + require.Equal(t, []common.Address{opts.Application.IApplicationAddress}, source.summaryApps) + require.Equal(t, []repository.ReplayPageRequest{ + {ApplicationID: 7, FromInput: 2, ToInputExclusive: 5, Limit: 2, Verification: repository.ReplayVerificationCanonical}, + {ApplicationID: 7, FromInput: 4, ToInputExclusive: 5, Limit: 1, Verification: repository.ReplayVerificationCanonical}, + }, source.pageRequests) +} + +func TestRunSelectsReplaySummaryByTypedAddress(t *testing.T) { + source := &fakeSource{ + summary: model.ReplaySummary{ + ApplicationID: 7, ProcessedInputs: 0, Consensus: model.Consensus_Authority, + }, + } + opts := replayOptions(model.Consensus_Authority, 0, 0) + // A valid application name may itself look like another application's + // address. Replay identity must never be inferred from that string. + opts.Application.Name = common.HexToAddress("0x9999").String() + + _, err := Run(context.Background(), source, &fakeExecutor{}, opts) + require.NoError(t, err) + require.Equal(t, []common.Address{opts.Application.IApplicationAddress}, source.summaryApps) +} + +func TestRunFullPRTComputesHashes(t *testing.T) { + record := replayRecords(1)[0] + record.StateHashes = []model.ReplayStateHash{{MachineHash: *record.Input.MachineHash, Repetitions: 1 << 24}} + source := &fakeSource{ + summary: model.ReplaySummary{ApplicationID: 7, ProcessedInputs: 1, Consensus: model.Consensus_PRT}, + records: []*model.ReplayRecord{record}, + } + executor := &fakeExecutor{fullPRTResult: true} + opts := replayOptions(model.Consensus_PRT, 0, 1) + opts.Verification = repository.ReplayVerificationFull + + _, err := Run(context.Background(), source, executor, opts) + require.NoError(t, err) + require.Equal(t, []bool{true}, executor.computeHashes) +} + +func TestRunCaughtUpStillValidatesSummary(t *testing.T) { + source := &fakeSource{ + summary: model.ReplaySummary{ApplicationID: 7, ProcessedInputs: 2, Consensus: model.Consensus_Authority}, + } + executor := &fakeExecutor{processed: 2} + opts := replayOptions(model.Consensus_Authority, 2, 2) + + result, err := Run(context.Background(), source, executor, opts) + require.NoError(t, err) + require.Zero(t, result.ReplayedInputs) + require.Equal(t, []repository.ReplayVerificationLevel{repository.ReplayVerificationCanonical}, source.summaryLevels) + require.Empty(t, source.pageRequests) +} + +func TestRunRejectsMalformedPagesBeforeExecution(t *testing.T) { + tests := []struct { + name string + records []*model.ReplayRecord + }{ + {name: "empty", records: nil}, + {name: "nil record", records: []*model.ReplayRecord{nil}}, + {name: "gap", records: replayRecords(2)[1:]}, + {name: "oversized", records: replayRecords(3)}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + source := &fakeSource{ + summary: model.ReplaySummary{ApplicationID: 7, ProcessedInputs: 2, Consensus: model.Consensus_Authority}, + records: test.records, + } + if test.name == "oversized" { + source.pageOverride = func(repository.ReplayPageRequest) ([]*model.ReplayRecord, error) { + return test.records, nil + } + } + executor := &fakeExecutor{} + opts := replayOptions(model.Consensus_Authority, 0, 2) + + _, err := Run(context.Background(), source, executor, opts) + require.ErrorIs(t, err, ErrContradiction) + require.Empty(t, executor.advanceCalls) + }) + } +} + +func TestRunErrorClassification(t *testing.T) { + opts := replayOptions(model.Consensus_Authority, 0, 1) + t.Run("source transport", func(t *testing.T) { + sourceErr := errors.New("database unavailable") + source := &fakeSource{summaryErr: sourceErr} + _, err := Run(context.Background(), source, &fakeExecutor{}, opts) + require.ErrorIs(t, err, ErrExecution) + require.ErrorIs(t, err, sourceErr) + require.NotErrorIs(t, err, ErrContradiction) + }) + t.Run("source structure", func(t *testing.T) { + source := &fakeSource{summaryErr: &repository.ReplayStructureViolationError{ + Kind: repository.ReplayStructureCompletedInputSequence, + InputIndex: 0, + }} + _, err := Run(context.Background(), source, &fakeExecutor{}, opts) + require.ErrorIs(t, err, ErrContradiction) + }) + t.Run("executor", func(t *testing.T) { + executionErr := errors.New("emulator unavailable") + source := &fakeSource{ + summary: model.ReplaySummary{ApplicationID: 7, ProcessedInputs: 1, Consensus: model.Consensus_Authority}, + records: replayRecords(1), + } + _, err := Run(context.Background(), source, &fakeExecutor{advanceErr: executionErr}, opts) + require.ErrorIs(t, err, ErrExecution) + require.ErrorIs(t, err, executionErr) + }) +} + +func TestRunRejectsInvalidOptions(t *testing.T) { + source := &fakeSource{} + executor := &fakeExecutor{} + opts := replayOptions(model.Consensus_Authority, 0, 1) + tests := []struct { + name string + source repository.ReplayRepository + exec Executor + mutate func(*Options) + }{ + {"nil source", nil, executor, func(*Options) {}}, + {"nil executor", source, nil, func(*Options) {}}, + {"nil app", source, executor, func(o *Options) { o.Application = nil }}, + {"invalid level", source, executor, func(o *Options) { o.Verification = 255 }}, + {"zero batch", source, executor, func(o *Options) { o.BatchSize = 0 }}, + {"inverted range", source, executor, func(o *Options) { o.FromInput = 2 }}, + {"upper beyond app", source, executor, func(o *Options) { o.ToInputExclusive = 2 }}, + {"executor position", source, &fakeExecutor{processed: 1}, func(*Options) {}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + current := opts + test.mutate(¤t) + _, err := Run(context.Background(), test.source, test.exec, current) + require.ErrorIs(t, err, ErrInvalidOptions) + }) + } +} diff --git a/internal/replay/types.go b/internal/replay/types.go new file mode 100644 index 000000000..716672da2 --- /dev/null +++ b/internal/replay/types.go @@ -0,0 +1,55 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +// Package replay reconstructs and verifies Cartesi machines from persisted +// canonical input outcomes. Repository evidence access is defined by +// repository.ReplayRepository; machine lifecycle remains caller responsibility. +package replay + +import ( + "errors" + "fmt" +) + +var ( + // ErrContradiction identifies a deterministic difference between a + // persisted result and its replay. + ErrContradiction = errors.New("replay contradicts persisted canonical result") + + // ErrExecution identifies a failure to reconstruct the requested range. + ErrExecution = errors.New("failed to replay machine execution") + + // ErrInvalidOptions identifies an invalid replay request. + ErrInvalidOptions = errors.New("invalid replay options") +) + +// ContradictionError identifies the first deterministic difference between a +// persisted result and its replay. Byte values are represented by lengths and +// digests, so the diagnostic never exposes input, output, or report payloads. +type ContradictionError struct { + Application string + EpochIndex *uint64 + InputIndex uint64 + Field string + Expected string + Actual string +} + +func (e *ContradictionError) Error() string { + epochIndex := "" + if e.EpochIndex != nil { + epochIndex = fmt.Sprint(*e.EpochIndex) + } + return fmt.Sprintf( + "%v: app=%q epoch=%s input=%d field=%s expected=%s actual=%s", + ErrContradiction, + e.Application, + epochIndex, + e.InputIndex, + e.Field, + e.Expected, + e.Actual, + ) +} + +func (e *ContradictionError) Unwrap() error { return ErrContradiction } From 0debf9c3bd9a34758799d1da0cc5d866d5a021ae Mon Sep 17 00:00:00 2001 From: Victor Fusco <1221933+vfusco@users.noreply.github.com> Date: Fri, 14 Aug 2026 10:41:58 -0300 Subject: [PATCH 11/13] fix(manager): reject contradictory reconstruction --- internal/advancer/advancer_test.go | 6 - internal/advancer/determinism_test.go | 8 +- internal/appstatus/appstatus.go | 13 +- internal/inspect/hardening_test.go | 3 - internal/inspect/inspect_test.go | 5 - internal/manager/errors.go | 75 ++ internal/manager/instance.go | 197 ++-- internal/manager/instance_test.go | 464 +++----- internal/manager/manager.go | 563 +++++++--- internal/manager/manager_test.go | 1400 ++++++++++++++++++++++++- internal/manager/types.go | 1 - 11 files changed, 2063 insertions(+), 672 deletions(-) create mode 100644 internal/manager/errors.go diff --git a/internal/advancer/advancer_test.go b/internal/advancer/advancer_test.go index 50936bfe6..1d2d6545b 100644 --- a/internal/advancer/advancer_test.go +++ b/internal/advancer/advancer_test.go @@ -2035,12 +2035,6 @@ func (m *MockMachineInstance) OutputsProof(ctx context.Context) (*OutputsProof, }, nil } -// Synchronize implements the MachineInstance interface for testing -func (m *MockMachineInstance) Synchronize(ctx context.Context, repo manager.MachineRepository, batchSize uint64) error { - // Not used in advancer tests, but needed to satisfy the interface - return nil -} - // CreateSnapshot implements the MachineInstance interface for testing func (m *MockMachineInstance) CreateSnapshot(ctx context.Context, processInputs uint64, path string) error { return m.createSnapshotError diff --git a/internal/advancer/determinism_test.go b/internal/advancer/determinism_test.go index c665941e8..766478aff 100644 --- a/internal/advancer/determinism_test.go +++ b/internal/advancer/determinism_test.go @@ -502,7 +502,7 @@ func newDeterminismHarness( logger := slog.New(slog.NewTextHandler(io.Discard, nil)) factory := newDeterminismRuntimeFactory(start, behaviors...) instance, err := manager.NewMachineInstanceWithFactory( - context.Background(), app, processedInputs, logger, false, factory, + context.Background(), app, processedInputs, logger, factory, ) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, instance.Close()) }) @@ -623,7 +623,6 @@ func (f *determinismRuntimeFactory) CreateMachineRuntime( ctx context.Context, _ *model.Application, _ *slog.Logger, - _ bool, ) (machine.Machine, error) { if err := ctx.Err(); err != nil { return nil, err @@ -743,11 +742,16 @@ func (m *determinismRuntime) Advance( return nil, errors.New("determinism test input must not be empty") } + // The revert root rides with the advance request and must be the machine's + // pre-input root — the instance always passes fork.Hash(). A mismatch is a + // harness (or caller) bug, not a determinism scenario. if checkpointHash != m.state.machineHash { m.mu.Unlock() return nil, errors.New("determinism test requires the checkpoint hash to equal the pre-input machine root") } previous := m.state.clone() + // Recording the request's own revert root is the first mutation of the + // candidate, exactly like the CMIO response. m.state.checkpointHash = checkpointHash status := machine.CompletionStatusAccepted switch { diff --git a/internal/appstatus/appstatus.go b/internal/appstatus/appstatus.go index 34eba94be..29ba5f3cc 100644 --- a/internal/appstatus/appstatus.go +++ b/internal/appstatus/appstatus.go @@ -30,7 +30,7 @@ type Repository interface { // Recovery assumptions — FAILED is safe to re-enable only when: // - The failure was a machine runtime error (not a DB desync). // - The last snapshot is consistent with the database state. -// - Synchronize() will correctly replay inputs from the snapshot point. +// - replay.Run will correctly verify inputs from the snapshot point. // // The reason parameter must be a pre-formatted string describing the failure. // Returns the database error if the status update fails; returns nil on success. @@ -134,7 +134,7 @@ func setTerminalStatus( status ApplicationStatus, reason string, ) error { - reason = truncateReason(reason) + reason = NormalizeReason(reason) dbErr := setApplicationStatus(ctx, logger, repo, app, status, reason) reasonErr := errors.New(reason) if dbErr != nil { @@ -143,9 +143,10 @@ func setTerminalStatus( return reasonErr } -// truncateReason truncates a reason string to maxReasonLength to avoid -// exceeding the database VARCHAR(4096) constraint. -func truncateReason(reason string) string { +// NormalizeReason returns the exact reason representation persisted by status +// helpers. Callers that compare a later readback must normalize before keeping +// their expected value. +func NormalizeReason(reason string) string { if len(reason) > maxReasonLength { return reason[:maxReasonLength] + "... (truncated)" } @@ -160,7 +161,7 @@ func setApplicationStatus( status ApplicationStatus, reason string, ) error { - reason = truncateReason(reason) + reason = NormalizeReason(reason) switch status { case ApplicationStatus_Failed: diff --git a/internal/inspect/hardening_test.go b/internal/inspect/hardening_test.go index c088a9c56..fa6991609 100644 --- a/internal/inspect/hardening_test.go +++ b/internal/inspect/hardening_test.go @@ -106,9 +106,6 @@ func (m *erroringMachine) ProcessedInputs() uint64 { return m.inner.ProcessedI func (m *erroringMachine) OutputsProof(ctx context.Context) (*OutputsProof, error) { return m.inner.OutputsProof(ctx) } -func (m *erroringMachine) Synchronize(ctx context.Context, repo manager.MachineRepository, batchSize uint64) error { - return m.inner.Synchronize(ctx, repo, batchSize) -} func (m *erroringMachine) CreateSnapshot(ctx context.Context, processedInputs uint64, path string) error { return m.inner.CreateSnapshot(ctx, processedInputs, path) } diff --git a/internal/inspect/inspect_test.go b/internal/inspect/inspect_test.go index 67b35bc84..9abb0f0a5 100644 --- a/internal/inspect/inspect_test.go +++ b/internal/inspect/inspect_test.go @@ -461,11 +461,6 @@ func (m *MockMachine) OutputsProof(ctx context.Context) (*OutputsProof, error) { return nil, nil } -// Not used in inspect tests, but needed to satisfy the interface -func (mock *MockMachine) Synchronize(ctx context.Context, repo manager.MachineRepository, batchSize uint64) error { - return nil -} - // Not used in inspect tests, but needed to satisfy the interface func (mock *MockMachine) CreateSnapshot(ctx context.Context, processedInputs uint64, path string) error { return nil diff --git a/internal/manager/errors.go b/internal/manager/errors.go new file mode 100644 index 000000000..e94adc7a0 --- /dev/null +++ b/internal/manager/errors.go @@ -0,0 +1,75 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package manager + +import ( + "errors" + "fmt" +) + +var ErrApplicationFailureNotDurable = errors.New("application failure status is not durably confirmed") + +// ApplicationFailurePersistenceError means local machine work is fenced but +// the repository has not confirmed the corresponding FAILED (or a stronger +// terminal/deleted) application state. +type ApplicationFailurePersistenceError struct { + ApplicationID int64 + WriteErr error + ReadErr error +} + +func (e *ApplicationFailurePersistenceError) Error() string { + if e.ReadErr != nil { + return fmt.Sprintf( + "%v: application=%d write_error=%v read_error=%v", + ErrApplicationFailureNotDurable, e.ApplicationID, e.WriteErr, e.ReadErr, + ) + } + return fmt.Sprintf( + "%v: application=%d write_error=%v durable status remains unconfirmed", + ErrApplicationFailureNotDurable, e.ApplicationID, e.WriteErr, + ) +} + +func (e *ApplicationFailurePersistenceError) Unwrap() []error { + errList := []error{ErrApplicationFailureNotDurable} + if e.WriteErr != nil { + errList = append(errList, e.WriteErr) + } + if e.ReadErr != nil { + errList = append(errList, e.ReadErr) + } + return errList +} + +// IsOnlyApplicationFailurePersistenceErrors reports whether err is one or +// more application-local durability failures and contains no global failure. +// It deliberately examines joined top-level errors without descending into a +// persistence error's write/read causes. +func IsOnlyApplicationFailurePersistenceErrors(err error) bool { + if err == nil { + return false + } + if _, ok := err.(*ApplicationFailurePersistenceError); ok { + return true + } + + switch wrapped := err.(type) { + case interface{ Unwrap() []error }: + children := wrapped.Unwrap() + if len(children) == 0 { + return false + } + for _, child := range children { + if !IsOnlyApplicationFailurePersistenceErrors(child) { + return false + } + } + return true + case interface{ Unwrap() error }: + return IsOnlyApplicationFailurePersistenceErrors(wrapped.Unwrap()) + default: + return false + } +} diff --git a/internal/manager/instance.go b/internal/manager/instance.go index 00e45746b..53ef50392 100644 --- a/internal/manager/instance.go +++ b/internal/manager/instance.go @@ -9,13 +9,13 @@ import ( "errors" "fmt" "log/slog" + "math" "sync" "sync/atomic" "time" "github.com/cartesi/rollups-node/internal/manager/pmutex" . "github.com/cartesi/rollups-node/internal/model" - "github.com/cartesi/rollups-node/internal/repository" "github.com/cartesi/rollups-node/pkg/machine" "github.com/ethereum/go-ethereum/common" "golang.org/x/sync/semaphore" @@ -84,9 +84,10 @@ func NewMachineInstance( ctx context.Context, app *Application, logger *slog.Logger, - checkHash bool, + checkTemplateHash bool, ) (MachineInstance, error) { - return NewMachineInstanceWithFactory(ctx, app, 0, logger, checkHash, defaultFactory) + factory := &DefaultMachineRuntimeFactory{CheckTemplateHash: checkTemplateHash} + return NewMachineInstanceWithFactory(ctx, app, 0, logger, factory) } // NewMachineInstanceFromSnapshot creates a new machine instance from a snapshot @@ -94,16 +95,33 @@ func NewMachineInstanceFromSnapshot( ctx context.Context, app *Application, logger *slog.Logger, - checkHash bool, snapshotPath string, - machineHash *common.Hash, + expectedHash common.Hash, inputIndex uint64, ) (MachineInstance, error) { + return newMachineInstanceFromSnapshot( + ctx, app, logger, snapshotPath, expectedHash, inputIndex, nil, + ) +} + +func newMachineInstanceFromSnapshot( + ctx context.Context, + app *Application, + logger *slog.Logger, + snapshotPath string, + expectedHash common.Hash, + inputIndex uint64, + loader machineLoader, +) (MachineInstance, error) { + if inputIndex == math.MaxUint64 { + return nil, fmt.Errorf("%w: snapshot input index cannot be incremented", ErrInvalidSnapshotPoint) + } factory := &SnapshotMachineRuntimeFactory{ SnapshotPath: snapshotPath, - MachineHash: machineHash, + ExpectedHash: expectedHash, + loader: loader, } - return NewMachineInstanceWithFactory(ctx, app, inputIndex+1, logger, checkHash, factory) + return NewMachineInstanceWithFactory(ctx, app, inputIndex+1, logger, factory) } // NewMachineInstanceWithFactory creates a new machine instance with a custom factory @@ -112,7 +130,6 @@ func NewMachineInstanceWithFactory( app *Application, processedInputs uint64, logger *slog.Logger, - checkHash bool, factory MachineRuntimeFactory, ) (MachineInstance, error) { // Validate parameters @@ -138,9 +155,15 @@ func NewMachineInstanceWithFactory( } // Create the machine server and runtime - runtime, err := factory.CreateMachineRuntime(ctx, app, logger, checkHash) + runtime, err := factory.CreateMachineRuntime(ctx, app, logger) if err != nil { - return nil, fmt.Errorf("%w: %v", ErrMachineCreation, err) + if runtime != nil { + err = errors.Join(err, runtime.Close()) + } + return nil, fmt.Errorf("%w: %w", ErrMachineCreation, err) + } + if runtime == nil { + return nil, fmt.Errorf("%w: runtime factory returned nil runtime", ErrMachineCreation) } // Create the machine instance @@ -169,82 +192,6 @@ func (m *MachineInstanceImpl) ProcessedInputs() uint64 { return m.processedInputs.Load() } -// Synchronize brings the machine up to date with processed inputs. -// It handles both template-based instances (processedInputs == 0, replays all) -// and snapshot-based instances (processedInputs > 0, replays only remaining). -// Inputs are fetched in batches to bound memory usage. -func (m *MachineInstanceImpl) Synchronize(ctx context.Context, repo MachineRepository, batchSize uint64) error { - appAddress := m.application.IApplicationAddress.String() - currentProcessed := m.processedInputs.Load() - m.logger.Info("Synchronizing machine with processed inputs", - "address", appAddress, - "app_processed_inputs", m.application.ProcessedInputs, - "machine_processed_inputs", currentProcessed) - - initialProcessedInputs := currentProcessed - replayed := uint64(0) - toReplay := uint64(0) - - for { - p := repository.Pagination{ - Limit: batchSize, - Offset: initialProcessedInputs + replayed, - } - inputs, totalCount, err := getProcessedInputs(ctx, repo, appAddress, p) - if err != nil { - return fmt.Errorf("%w: %w", ErrMachineSynchronization, err) - } - - // Validate count on the first batch - if replayed == 0 { - if totalCount != m.application.ProcessedInputs { - errorMsg := fmt.Sprintf( - "processed inputs count mismatch: expected %d, got %d", - m.application.ProcessedInputs, totalCount) - m.logger.Error(errorMsg, "address", appAddress) - return fmt.Errorf("%w: %s", ErrMachineSynchronization, errorMsg) - } - if currentProcessed > totalCount { - return fmt.Errorf( - "%w: machine has processed %d inputs but DB only has %d", - ErrMachineSynchronization, currentProcessed, totalCount) - } - toReplay = totalCount - currentProcessed - if toReplay == 0 { - m.logger.Info("No inputs to replay during synchronization", - "address", appAddress) - return nil - } - } - - for _, input := range inputs { - m.logger.Info("Replaying input during synchronization", - "address", appAddress, - "epoch_index", input.EpochIndex, - "input_index", input.Index, - "progress", fmt.Sprintf("%d/%d", replayed+1, toReplay)) - - _, err := m.Advance(ctx, input.RawData, input.EpochIndex, input.Index, false) - if err != nil { - return fmt.Errorf("%w: failed to replay input %d: %w", - ErrMachineSynchronization, input.Index, err) - } - replayed++ - } - - if replayed >= toReplay { - break - } - if len(inputs) == 0 { - return fmt.Errorf( - "%w: expected to replay %d inputs but only replayed %d", - ErrMachineSynchronization, toReplay, replayed) - } - } - - return nil -} - // forkForAdvance creates a copy of the machine for advance operations // It verifies the input index and returns a forked machine func (m *MachineInstanceImpl) forkForAdvance(ctx context.Context, index uint64) (machine.Machine, error) { @@ -266,7 +213,12 @@ func (m *MachineInstanceImpl) forkForAdvance(ctx context.Context, index uint64) return m.runtime.Fork(ctx) } -// Advance processes an input and advances the machine state +// Advance treats a machine fork as the execution transaction for one input. +// It executes on the fork, selects the canonical root from the typed completion +// status, and advances processedInputs exactly once for every completed input. +// Accepted adopts the fork; currently nonaccepted completions keep the +// predecessor runtime and close the fork. Incomplete execution returns an error +// and adopts neither the fork nor a canonical result. func (m *MachineInstanceImpl) Advance(ctx context.Context, input []byte, epochIndex uint64, index uint64, computeHashes bool) (*AdvanceResult, error) { // Only one advance can be active at a time m.advanceMutex.Lock() @@ -668,19 +620,25 @@ type MachineRuntimeFactory interface { ctx context.Context, app *Application, logger *slog.Logger, - checkHash bool, ) (machine.Machine, error) } +type machineLoader func( + ctx context.Context, + logger *slog.Logger, + config *machine.MachineConfig, +) (machine.Machine, error) + // createMachineRuntimeCommon contains the shared logic for creating machine runtimes func createMachineRuntimeCommon( ctx context.Context, app *Application, logger *slog.Logger, - checkHash bool, + verifyExpectedHash bool, machinePath string, sourceType string, expectedHash common.Hash, + loader machineLoader, ) (machine.Machine, error) { if logger == nil { return nil, ErrInvalidLogger @@ -699,9 +657,16 @@ func createMachineRuntimeCommon( config.ExecutionParameters = app.ExecutionParameters // Create the machine - m, err := machine.Load(ctx, logger, config) + if loader == nil { + loader = machine.Load + } + m, err := loader(ctx, logger, config) if err != nil { - return nil, err + // Preserve a partially created runtime so the caller can close it. + return m, err + } + if m == nil { + return nil, errors.New("machine loader returned nil runtime without an error") } logger.Debug(fmt.Sprintf("Machine loaded from %s", sourceType), @@ -711,7 +676,7 @@ func createMachineRuntimeCommon( "path", machinePath) // Verify the machine hash if required - if checkHash { + if verifyExpectedHash { logger.Debug("Verifying machine hash", "application", app.Name, "address", appAddress) @@ -737,31 +702,37 @@ func createMachineRuntimeCommon( return m, nil } -// DefaultMachineRuntimeFactory is the standard implementation of MachineRuntimeFactory -type DefaultMachineRuntimeFactory struct{} +// DefaultMachineRuntimeFactory is the standard template implementation of +// MachineRuntimeFactory. Template verification remains configurable for +// backwards compatibility; snapshot verification is always mandatory. +type DefaultMachineRuntimeFactory struct { + CheckTemplateHash bool + loader machineLoader +} // CreateMachineRuntime creates a new machine runtime for an application func (f *DefaultMachineRuntimeFactory) CreateMachineRuntime( ctx context.Context, app *Application, logger *slog.Logger, - checkHash bool, ) (machine.Machine, error) { return createMachineRuntimeCommon( ctx, app, logger, - checkHash, + f.CheckTemplateHash, app.TemplateURI, "template", app.TemplateHash, + f.loader, ) } // SnapshotMachineRuntimeFactory creates machine runtimes from snapshots type SnapshotMachineRuntimeFactory struct { SnapshotPath string - MachineHash *common.Hash // The hash to check against (from the input's machine_hash) + ExpectedHash common.Hash + loader machineLoader } // CreateMachineRuntime creates a new machine runtime from a snapshot @@ -769,28 +740,19 @@ func (f *SnapshotMachineRuntimeFactory) CreateMachineRuntime( ctx context.Context, app *Application, logger *slog.Logger, - checkHash bool, ) (machine.Machine, error) { - // Determine which hash to check against - expectedHash := app.TemplateHash - if f.MachineHash != nil { - expectedHash = *f.MachineHash - } - return createMachineRuntimeCommon( ctx, app, logger, - checkHash, + true, f.SnapshotPath, "snapshot", - expectedHash, + f.ExpectedHash, + f.loader, ) } -// Default factory instance -var defaultFactory MachineRuntimeFactory = &DefaultMachineRuntimeFactory{} - // toInputStatus converts only completed, deterministic machine statuses to // canonical input statuses. Infrastructure interruptions never reach here. func toInputStatus(status machine.CompletionStatus) (InputCompletionStatus, error) { @@ -804,18 +766,13 @@ func toInputStatus(status machine.CompletionStatus) (InputCompletionStatus, erro case machine.CompletionStatusHalted: return InputCompletionStatus_MachineHalted, nil case machine.CompletionStatusUnknown: - return InputCompletionStatus_None, fmt.Errorf( - "unknown completed machine status %d: %w", - status, - ErrIncompleteAdvance, - ) - default: - return InputCompletionStatus_None, fmt.Errorf( - "unknown completed machine status %d: %w", - status, - ErrIncompleteAdvance, - ) + // Intentionally empty. } + return InputCompletionStatus_None, fmt.Errorf( + "unknown completed machine status %d: %w", + status, + ErrIncompleteAdvance, + ) } func validateCompletionExceptionData(status machine.CompletionStatus, data []byte) error { diff --git a/internal/manager/instance_test.go b/internal/manager/instance_test.go index 419d80a91..d40cbf025 100644 --- a/internal/manager/instance_test.go +++ b/internal/manager/instance_test.go @@ -8,13 +8,14 @@ import ( "errors" "io" "log/slog" + "math" "sync" + "sync/atomic" "testing" "time" "github.com/cartesi/rollups-node/internal/manager/pmutex" "github.com/cartesi/rollups-node/internal/model" - "github.com/cartesi/rollups-node/internal/repository" "github.com/cartesi/rollups-node/pkg/machine" "github.com/ethereum/go-ethereum/common" "github.com/stretchr/testify/suite" @@ -49,7 +50,6 @@ func (f *MockMachineRuntimeFactory) CreateMachineRuntime( _ context.Context, _ *model.Application, _ *slog.Logger, - _ bool, ) (machine.Machine, error) { return f.RuntimeToReturn, f.ErrorToReturn } @@ -80,7 +80,6 @@ func (s *MachineInstanceSuite) TestNewMachineInstance() { app, 0, testLogger, - false, mockFactory, ) require.Nil(err) @@ -108,7 +107,6 @@ func (s *MachineInstanceSuite) TestNewMachineInstance() { app, 0, testLogger, - false, mockFactory, ) require.Error(err) @@ -134,7 +132,6 @@ func (s *MachineInstanceSuite) TestNewMachineInstance() { app, 0, testLogger, - false, mockFactory, ) require.Error(err) @@ -160,7 +157,6 @@ func (s *MachineInstanceSuite) TestNewMachineInstance() { app, 0, testLogger, - false, mockFactory, ) require.Error(err) @@ -184,7 +180,6 @@ func (s *MachineInstanceSuite) TestNewMachineInstance() { app, 0, nil, - false, mockFactory, ) require.Error(err) @@ -208,19 +203,56 @@ func (s *MachineInstanceSuite) TestNewMachineInstance() { app, 0, testLogger, - false, nil, ) require.Error(err) require.Nil(machine) require.Contains(err.Error(), "factory must not be nil") }) + + s.Run("FactoryErrorClosesPartialRuntime", func() { + require := s.Require() + factoryErr := errors.New("factory failed after creating runtime") + closeErr := errors.New("partial runtime close failed") + partial := &MockRollupsMachine{CloseError: closeErr} + factory := &MockMachineRuntimeFactory{RuntimeToReturn: partial, ErrorToReturn: factoryErr} + app := &model.Application{ExecutionParameters: model.ExecutionParameters{ + AdvanceMaxDeadline: decisecond, InspectMaxDeadline: centisecond, MaxConcurrentInspects: 1, + }} + + instance, err := NewMachineInstanceWithFactory( + context.Background(), app, 0, + slog.New(slog.NewTextHandler(io.Discard, nil)), factory, + ) + + require.Nil(instance) + require.ErrorIs(err, ErrMachineCreation) + require.ErrorIs(err, factoryErr) + require.ErrorIs(err, closeErr) + require.Equal(int64(1), partial.CloseCalls.Load()) + }) + + s.Run("FactoryNilRuntimeIsRejected", func() { + require := s.Require() + app := &model.Application{ExecutionParameters: model.ExecutionParameters{ + AdvanceMaxDeadline: decisecond, InspectMaxDeadline: centisecond, MaxConcurrentInspects: 1, + }} + factory := &MockMachineRuntimeFactory{} + + instance, err := NewMachineInstanceWithFactory( + context.Background(), app, 0, + slog.New(slog.NewTextHandler(io.Discard, nil)), factory, + ) + + require.Nil(instance) + require.ErrorIs(err, ErrMachineCreation) + require.Contains(err.Error(), "nil runtime") + }) } func (s *MachineInstanceSuite) TestNewMachineInstanceFromSnapshot() { - s.Run("Ok", func() { - require := s.Require() - app := &model.Application{ + newApp := func() *model.Application { + return &model.Application{ Name: "TestApp", ExecutionParameters: model.ExecutionParameters{ AdvanceMaxDeadline: decisecond, @@ -228,65 +260,95 @@ func (s *MachineInstanceSuite) TestNewMachineInstanceFromSnapshot() { MaxConcurrentInspects: 3, }, } + } + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - testLogger := slog.New(slog.NewTextHandler(io.Discard, nil)) - mockRuntime := &MockRollupsMachine{} - mockFactory := &MockMachineRuntimeFactory{ - RuntimeToReturn: mockRuntime, - ErrorToReturn: nil, + s.Run("MatchingHashCreatesInstanceAtNextInput", func() { + require := s.Require() + expected := newHash(7) + runtime := &MockRollupsMachine{HashReturn: expected} + loader := func( + _ context.Context, _ *slog.Logger, config *machine.MachineConfig, + ) (machine.Machine, error) { + require.Equal("snapshot-dir", config.Path) + return runtime, nil } - // NewMachineInstanceFromSnapshot creates a SnapshotMachineRuntimeFactory - // internally, so we use NewMachineInstanceWithFactory to test the same - // logic with a controlled factory. - inputIndex := uint64(5) - - // The function sets processedInputs = inputIndex + 1 - // Use the mock factory to avoid actual machine loading - inst, err := NewMachineInstanceWithFactory( - context.Background(), - app, - inputIndex+1, - testLogger, - false, - mockFactory, + instance, err := newMachineInstanceFromSnapshot( + context.Background(), newApp(), logger, "snapshot-dir", expected, 5, loader, ) + require.NoError(err) - require.NotNil(inst) - require.Equal(inputIndex+1, inst.ProcessedInputs()) + require.Equal(uint64(6), instance.ProcessedInputs()) + require.Equal(int64(1), runtime.HashCalls.Load()) + require.Zero(runtime.CloseCalls.Load()) + require.NoError(instance.Close()) + }) + + s.Run("MismatchingHashClosesRuntime", func() { + require := s.Require() + runtime := &MockRollupsMachine{HashReturn: newHash(8)} + factory := &SnapshotMachineRuntimeFactory{ + SnapshotPath: "snapshot-dir", + ExpectedHash: newHash(9), + loader: func(context.Context, *slog.Logger, *machine.MachineConfig) (machine.Machine, error) { + return runtime, nil + }, + } - inst.Close() + got, err := factory.CreateMachineRuntime(context.Background(), newApp(), logger) + + require.Nil(got) + require.Error(err) + require.Contains(err.Error(), "machine hash mismatch") + require.Equal(int64(1), runtime.HashCalls.Load()) + require.Equal(int64(1), runtime.CloseCalls.Load()) }) - s.Run("FactoryError", func() { + s.Run("HashReadErrorClosesRuntime", func() { require := s.Require() - app := &model.Application{ - Name: "TestApp", - ExecutionParameters: model.ExecutionParameters{ - AdvanceMaxDeadline: decisecond, - InspectMaxDeadline: centisecond, - MaxConcurrentInspects: 3, + hashErr := errors.New("hash unavailable") + runtime := &MockRollupsMachine{HashError: hashErr} + factory := &SnapshotMachineRuntimeFactory{ + SnapshotPath: "snapshot-dir", + ExpectedHash: newHash(9), + loader: func(context.Context, *slog.Logger, *machine.MachineConfig) (machine.Machine, error) { + return runtime, nil }, } - testLogger := slog.New(slog.NewTextHandler(io.Discard, nil)) - mockFactory := &MockMachineRuntimeFactory{ - RuntimeToReturn: nil, - ErrorToReturn: errors.New("snapshot load failed"), + got, err := factory.CreateMachineRuntime(context.Background(), newApp(), logger) + + require.Nil(got) + require.ErrorIs(err, hashErr) + require.Equal(int64(1), runtime.HashCalls.Load()) + require.Equal(int64(1), runtime.CloseCalls.Load()) + }) + + s.Run("NilRuntimeFromLoaderIsRejected", func() { + require := s.Require() + loader := func(context.Context, *slog.Logger, *machine.MachineConfig) (machine.Machine, error) { + return nil, nil } - inst, err := NewMachineInstanceWithFactory( - context.Background(), - app, - 6, - testLogger, - false, - mockFactory, + instance, err := newMachineInstanceFromSnapshot( + context.Background(), newApp(), logger, "snapshot-dir", newHash(1), 0, loader, ) - require.Error(err) - require.Nil(inst) + + require.Nil(instance) require.ErrorIs(err, ErrMachineCreation) - require.Contains(err.Error(), "snapshot load failed") + require.Contains(err.Error(), "machine loader returned nil runtime") + }) + + s.Run("InputIndexOverflowIsRejectedBeforeLoad", func() { + require := s.Require() + + instance, err := NewMachineInstanceFromSnapshot( + context.Background(), newApp(), logger, "snapshot-dir", newHash(1), math.MaxUint64, + ) + + require.Nil(instance) + require.ErrorIs(err, ErrInvalidSnapshotPoint) }) } @@ -309,7 +371,7 @@ func (s *MachineInstanceSuite) TestApplicationAndProcessedInputs() { } inst, err := NewMachineInstanceWithFactory( - context.Background(), app, 42, testLogger, false, mockFactory, + context.Background(), app, 42, testLogger, mockFactory, ) require.NoError(err) require.Same(app, inst.Application()) @@ -1261,291 +1323,6 @@ func newBytes(n byte, size int) []byte { return bytes } -// ------------------------------------------------------------------------------------------------ -// Synchronize tests -// ------------------------------------------------------------------------------------------------ - -// mockSyncRepository is a lightweight mock for Synchronize tests. -// It simulates pagination over a slice of inputs. -type mockSyncRepository struct { - inputs []*model.Input - totalCount uint64 - listErr error -} - -func (r *mockSyncRepository) ListApplications( - _ context.Context, - _ repository.ApplicationFilter, - _ repository.Pagination, - _ bool, -) ([]*model.Application, uint64, error) { - return nil, 0, nil -} - -func (r *mockSyncRepository) HasUndrainedEpochsBeforeBlock( - _ context.Context, - _ int64, - _ uint64, -) (bool, error) { - return false, nil -} - -func (r *mockSyncRepository) ListInputs( - ctx context.Context, - _ string, - _ repository.InputFilter, - p repository.Pagination, - _ bool, -) ([]*model.Input, uint64, error) { - if err := ctx.Err(); err != nil { - return nil, 0, err - } - if r.listErr != nil { - return nil, 0, r.listErr - } - start := p.Offset - if start >= uint64(len(r.inputs)) { - return nil, r.totalCount, nil - } - end := start + p.Limit - if p.Limit == 0 || end > uint64(len(r.inputs)) { - end = uint64(len(r.inputs)) - } - return r.inputs[start:end], r.totalCount, nil -} - -func (r *mockSyncRepository) GetLastSnapshot( - _ context.Context, - _ string, -) (*model.Input, error) { - return nil, nil -} - -// newForkableMock creates a mock where Fork returns a fresh mock each time, -// properly exercising the fork/replace lifecycle in Synchronize tests. -func newForkableMock() *MockRollupsMachine { - m := &MockRollupsMachine{} - m.CloseError = nil - m.CompletionStatusReturn = machine.CompletionStatusAccepted - m.HashReturn = newHash(1) - m.OutputsHashReturn = newHash(2) - m.ForkFunc = func(_ context.Context) (machine.Machine, error) { - return newForkableMock(), nil - } - return m -} - -func (s *MachineInstanceSuite) newSyncMachine(processedInputs uint64, appProcessedInputs uint64) *MachineInstanceImpl { - runtime := newForkableMock() - - inst := &MachineInstanceImpl{ - application: &model.Application{ - ProcessedInputs: appProcessedInputs, - ExecutionParameters: model.ExecutionParameters{ - AdvanceMaxDeadline: decisecond, - InspectMaxDeadline: centisecond, - MaxConcurrentInspects: 3, - }, - }, - runtime: runtime, - advanceTimeout: decisecond, - inspectTimeout: centisecond, - maxConcurrentInspects: 3, - closeTimeout: defaultCloseTimeout, - mutex: pmutex.New(), - inspectSemaphore: semaphore.NewWeighted(3), - logger: slog.New(slog.NewTextHandler(io.Discard, nil)), - } - inst.processedInputs.Store(processedInputs) - return inst -} - -func makeInputs(startIndex, count uint64) []*model.Input { - inputs := make([]*model.Input, count) - for i := uint64(0); i < count; i++ { - inputs[i] = &model.Input{ - Index: startIndex + i, - EpochIndex: 0, - RawData: []byte{byte(startIndex + i)}, - } - } - return inputs -} - -func (s *MachineInstanceSuite) TestSynchronize() { - s.Run("TemplateSyncAllInputs", func() { - require := s.Require() - inst := s.newSyncMachine(0, 3) - originalRuntime := inst.runtime - repo := &mockSyncRepository{ - inputs: makeInputs(0, 3), - totalCount: 3, - } - - err := inst.Synchronize(context.Background(), repo, 1000) - require.NoError(err) - require.Equal(uint64(3), inst.processedInputs.Load()) - // Verify the runtime was actually replaced (not self-fork) - require.NotSame(originalRuntime, inst.runtime) - }) - - s.Run("SnapshotSyncRemainingInputs", func() { - require := s.Require() - // Snapshot was at index 2, so processedInputs=3, but app has 5 total - inst := s.newSyncMachine(3, 5) - repo := &mockSyncRepository{ - inputs: makeInputs(0, 5), - totalCount: 5, - } - - err := inst.Synchronize(context.Background(), repo, 1000) - require.NoError(err) - require.Equal(uint64(5), inst.processedInputs.Load()) - }) - - s.Run("NoInputsToReplay", func() { - require := s.Require() - inst := s.newSyncMachine(0, 0) - repo := &mockSyncRepository{ - inputs: nil, - totalCount: 0, - } - - err := inst.Synchronize(context.Background(), repo, 1000) - require.NoError(err) - require.Equal(uint64(0), inst.processedInputs.Load()) - }) - - s.Run("SnapshotAlreadyCaughtUp", func() { - require := s.Require() - // Snapshot at last input — nothing to replay - inst := s.newSyncMachine(5, 5) - repo := &mockSyncRepository{ - inputs: makeInputs(0, 5), - totalCount: 5, - } - - err := inst.Synchronize(context.Background(), repo, 1000) - require.NoError(err) - require.Equal(uint64(5), inst.processedInputs.Load()) - }) - - s.Run("MachineAheadOfDB", func() { - require := s.Require() - // Machine has processed 5 inputs but DB only has 3 - inst := s.newSyncMachine(5, 3) - repo := &mockSyncRepository{ - inputs: makeInputs(0, 3), - totalCount: 3, - } - - err := inst.Synchronize(context.Background(), repo, 1000) - require.Error(err) - require.ErrorIs(err, ErrMachineSynchronization) - require.Contains(err.Error(), "machine has processed 5 inputs but DB only has 3") - }) - - s.Run("CountMismatch", func() { - require := s.Require() - inst := s.newSyncMachine(0, 5) - repo := &mockSyncRepository{ - inputs: makeInputs(0, 3), - totalCount: 3, // DB says 3 but app expects 5 - } - - err := inst.Synchronize(context.Background(), repo, 1000) - require.Error(err) - require.ErrorIs(err, ErrMachineSynchronization) - require.Contains(err.Error(), "count mismatch") - }) - - s.Run("ListInputsError", func() { - require := s.Require() - inst := s.newSyncMachine(0, 3) - listErr := errors.New("database connection lost") - repo := &mockSyncRepository{ - listErr: listErr, - } - - err := inst.Synchronize(context.Background(), repo, 1000) - require.Error(err) - require.ErrorIs(err, ErrMachineSynchronization) - require.Contains(err.Error(), "database connection lost") - }) - - s.Run("AdvanceErrorMidReplay", func() { - require := s.Require() - inst := s.newSyncMachine(0, 3) - // Make each fork return a hard error on Advance. - runtime := inst.runtime.(*MockRollupsMachine) - runtime.ForkFunc = func(_ context.Context) (machine.Machine, error) { - fork := newForkableMock() - fork.AdvanceError = errors.New("advance failed during replay") - return fork, nil - } - - repo := &mockSyncRepository{ - inputs: makeInputs(0, 3), - totalCount: 3, - } - - err := inst.Synchronize(context.Background(), repo, 1000) - require.Error(err) - require.ErrorIs(err, ErrMachineSynchronization) - require.Contains(err.Error(), "failed to replay input") - }) - - s.Run("BatchBoundaryCrossing", func() { - require := s.Require() - // Use batchSize=2 with 3 inputs so the loop must fetch two batches - // (batch 1: inputs 0-1, batch 2: input 2), exercising pagination. - inst := s.newSyncMachine(0, 3) - repo := &mockSyncRepository{ - inputs: makeInputs(0, 3), - totalCount: 3, - } - - err := inst.Synchronize(context.Background(), repo, 2) - require.NoError(err) - require.Equal(uint64(3), inst.processedInputs.Load()) - }) - - s.Run("PartialSyncDetected", func() { - require := s.Require() - // Machine has 0 processed, app has 5. But the mock only has 2 inputs, - // simulating rows disappearing between batches. - // Use batchSize=2 so the first batch returns inputs [0,1] (replayed=2), - // then the second batch returns 0 rows (offset=2 >= len=2). - // The loop must detect replayed(2) != toReplay(5) and return an error. - inst := s.newSyncMachine(0, 5) - repo := &mockSyncRepository{ - inputs: makeInputs(0, 2), // only 2 inputs exist - totalCount: 5, // but totalCount says 5 - } - - err := inst.Synchronize(context.Background(), repo, 2) - require.Error(err) - require.ErrorIs(err, ErrMachineSynchronization) - require.Contains(err.Error(), "expected to replay 5 inputs but only replayed 2") - }) - - s.Run("ContextCancellation", func() { - require := s.Require() - inst := s.newSyncMachine(0, 3) - - ctx, cancel := context.WithCancel(context.Background()) - cancel() // Cancel immediately - - repo := &mockSyncRepository{ - inputs: makeInputs(0, 3), - totalCount: 3, - } - - err := inst.Synchronize(ctx, repo, 1000) - require.Error(err) - }) -} - // ------------------------------------------------------------------------------------------------ type MockRollupsMachine struct { @@ -1555,6 +1332,7 @@ type MockRollupsMachine struct { HashReturn machine.Hash HashError error + HashCalls atomic.Int64 CompletionStatusReturn machine.CompletionStatus ExceptionDataReturn []byte @@ -1575,6 +1353,7 @@ type MockRollupsMachine struct { StoreError error CloseError error + CloseCalls atomic.Int64 } func (m *MockRollupsMachine) Fork(ctx context.Context) (machine.Machine, error) { @@ -1585,6 +1364,7 @@ func (m *MockRollupsMachine) Fork(ctx context.Context) (machine.Machine, error) } func (m *MockRollupsMachine) Hash(_ context.Context) (machine.Hash, error) { + m.HashCalls.Add(1) return m.HashReturn, m.HashError } @@ -1596,7 +1376,8 @@ func (m *MockRollupsMachine) OutputsHashProof(_ context.Context) ([]machine.Hash return m.OutputsHashProofReturn, m.OutputsHashProofError } -func (m *MockRollupsMachine) Advance(_ context.Context, _ []byte, _ machine.Hash, _ bool) (*machine.AdvanceResponse, error) { +func (m *MockRollupsMachine) Advance(_ context.Context, _ []byte, _ machine.Hash, computeHashes bool) (*machine.AdvanceResponse, error) { + m.LastAdvanceComputeHashes = computeHashes if m.AdvanceError != nil { return nil, m.AdvanceError } @@ -1620,6 +1401,7 @@ func (m *MockRollupsMachine) Store(_ context.Context, _ string) error { } func (m *MockRollupsMachine) Close() error { + m.CloseCalls.Add(1) return m.CloseError } diff --git a/internal/manager/manager.go b/internal/manager/manager.go index ce82f9271..fc8fb921e 100644 --- a/internal/manager/manager.go +++ b/internal/manager/manager.go @@ -8,68 +8,84 @@ import ( "errors" "fmt" "log/slog" + "math" "os" "sort" "sync" + "github.com/cartesi/rollups-node/internal/appstatus" . "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/internal/replay" "github.com/cartesi/rollups-node/internal/repository" + "github.com/cartesi/rollups-node/pkg/machine" "github.com/ethereum/go-ethereum/common" ) var ( - ErrApplicationNotFound = errors.New("application not found") - ErrMachineCreation = errors.New("failed to create machine") - ErrMachineSynchronization = errors.New("failed to synchronize machine") + ErrApplicationNotFound = errors.New("application not found") + ErrMachineCreation = errors.New("failed to create machine") ) // MachineRepository defines the repository interface needed by the MachineManager type MachineRepository interface { + repository.ReplayRepository + // ListApplications retrieves applications based on filter criteria ListApplications(ctx context.Context, f repository.ApplicationFilter, p repository.Pagination, descending bool) ([]*Application, uint64, error) HasUndrainedEpochsBeforeBlock(ctx context.Context, appID int64, blockBound uint64) (bool, error) - // ListInputs retrieves inputs based on filter criteria - ListInputs(ctx context.Context, nameOrAddress string, f repository.InputFilter, p repository.Pagination, descending bool) ([]*Input, uint64, error) - // GetLastSnapshot retrieves the most recent input with a snapshot for the given application GetLastSnapshot(ctx context.Context, nameOrAddress string) (*Input, error) + GetApplication(ctx context.Context, nameOrAddress string) (*Application, error) + + // UpdateApplicationStatus persists an application's health status. + UpdateApplicationStatus(ctx context.Context, appID int64, status ApplicationStatus, reason *string) error } // MachineInstanceFactory creates MachineInstance values from applications. // Implementations decide whether to load from a template or snapshot. type MachineInstanceFactory interface { - NewFromTemplate(ctx context.Context, app *Application, logger *slog.Logger, checkHash bool) (MachineInstance, error) - NewFromSnapshot(ctx context.Context, app *Application, logger *slog.Logger, checkHash bool, - snapshotPath string, machineHash *common.Hash, inputIndex uint64) (MachineInstance, error) + NewFromTemplate(ctx context.Context, app *Application, logger *slog.Logger, checkTemplateHash bool) (MachineInstance, error) + NewFromSnapshot(ctx context.Context, app *Application, logger *slog.Logger, + snapshotPath string, expectedHash common.Hash, inputIndex uint64) (MachineInstance, error) } // DefaultMachineInstanceFactory delegates to NewMachineInstance / NewMachineInstanceFromSnapshot. type DefaultMachineInstanceFactory struct{} func (f *DefaultMachineInstanceFactory) NewFromTemplate( - ctx context.Context, app *Application, logger *slog.Logger, checkHash bool, + ctx context.Context, app *Application, logger *slog.Logger, checkTemplateHash bool, ) (MachineInstance, error) { - return NewMachineInstance(ctx, app, logger, checkHash) + return NewMachineInstance(ctx, app, logger, checkTemplateHash) } func (f *DefaultMachineInstanceFactory) NewFromSnapshot( - ctx context.Context, app *Application, logger *slog.Logger, checkHash bool, - snapshotPath string, machineHash *common.Hash, inputIndex uint64, + ctx context.Context, app *Application, logger *slog.Logger, + snapshotPath string, expectedHash common.Hash, inputIndex uint64, ) (MachineInstance, error) { - return NewMachineInstanceFromSnapshot(ctx, app, logger, checkHash, snapshotPath, machineHash, inputIndex) + return NewMachineInstanceFromSnapshot(ctx, app, logger, snapshotPath, expectedHash, inputIndex) } // MachineManager manages the lifecycle of machine instances for applications type MachineManager struct { - mutex sync.RWMutex - machines map[int64]MachineInstance - closed bool - repository MachineRepository - checkHash bool - inputBatchSize uint64 - logger *slog.Logger - instanceFactory MachineInstanceFactory + mutex sync.RWMutex + machines map[int64]MachineInstance + pendingApplicationFailures map[int64]*pendingApplicationFailure + closed bool + repository MachineRepository + checkTemplateHash bool + inputBatchSize uint64 + logger *slog.Logger + instanceFactory MachineInstanceFactory + replayRun func(context.Context, repository.ReplayRepository, replay.Executor, replay.Options) (replay.Result, error) +} + +// pendingApplicationFailure fences an application after a FAILED status write +// cannot be confirmed. It is deliberately private: this is a short-lived retry +// queue, not process-local application health state. +type pendingApplicationFailure struct { + application *Application + reason string } // Option configures a MachineManager. @@ -80,21 +96,207 @@ func WithInstanceFactory(f MachineInstanceFactory) Option { return func(m *MachineManager) { m.instanceFactory = f } } +// withReplayRun overrides replay execution for manager policy tests. +func withReplayRun( + run func(context.Context, repository.ReplayRepository, replay.Executor, replay.Options) (replay.Result, error), +) Option { + return func(m *MachineManager) { m.replayRun = run } +} + +func snapshotProcessedInputs(app *Application, snapshot *Input) (uint64, error) { + if snapshot.Index == math.MaxUint64 { + return 0, fmt.Errorf("%w: snapshot input index cannot be incremented", ErrInvalidSnapshotPoint) + } + processedInputs := snapshot.Index + 1 + if processedInputs > app.ProcessedInputs { + return 0, fmt.Errorf( + "%w: snapshot represents %d processed inputs, application has %d", + ErrInvalidSnapshotPoint, processedInputs, app.ProcessedInputs, + ) + } + return processedInputs, nil +} + +func closeMachineCandidate( + logger *slog.Logger, + candidate MachineInstance, + failureMessage string, +) { + if candidate == nil { + return + } + if err := candidate.Close(); err != nil { + logger.Warn(failureMessage, "error", err) + } +} + +func (m *MachineManager) tryLoadSnapshotInstance( + ctx context.Context, + app *Application, + logger *slog.Logger, +) MachineInstance { + snapshot, err := m.repository.GetLastSnapshot(ctx, app.IApplicationAddress.String()) + if err != nil { + // Shutdown cancels the query mid-flight; deadlines and repository + // failures still require operator attention. + if errors.Is(err, context.Canceled) { + logger.Debug("GetLastSnapshot canceled during shutdown", "error", err) + } else { + logger.Error("Failed to find latest snapshot", "error", err) + } + return nil + } + if snapshot == nil || snapshot.SnapshotURI == nil { + return nil + } + + snapshotLogger := logger.With( + "snapshot", *snapshot.SnapshotURI, + "input_index", snapshot.Index, + ) + if snapshot.MachineHash == nil { + snapshotLogger.Warn("Snapshot input has no persisted machine hash; falling back to template") + return nil + } + + expectedProcessedInputs, err := snapshotProcessedInputs(app, snapshot) + if err != nil { + snapshotLogger.Warn( + "Snapshot input is not a valid application starting point; falling back to template", + "app_processed_inputs", app.ProcessedInputs, + "error", err, + ) + return nil + } + + if _, err := os.Stat(*snapshot.SnapshotURI); err != nil { + if errors.Is(err, os.ErrNotExist) { + snapshotLogger.Warn("Snapshot path does not exist") + } else { + snapshotLogger.Error("Failed to access snapshot path", "error", err) + } + return nil + } + + snapshotLogger.Info("Creating machine instance from snapshot") + candidate, err := m.instanceFactory.NewFromSnapshot( + ctx, app, m.logger, + *snapshot.SnapshotURI, *snapshot.MachineHash, snapshot.Index, + ) + if err != nil { + snapshotLogger.Error("Failed to create machine instance from snapshot", "error", err) + closeMachineCandidate(snapshotLogger, candidate, "Failed to close partial snapshot machine") + return nil + } + if candidate == nil { + snapshotLogger.Error("Snapshot factory returned no machine instance") + return nil + } + if candidate.ProcessedInputs() != expectedProcessedInputs { + snapshotLogger.Error("Snapshot machine has an invalid processed-input count", + "expected_processed_inputs", expectedProcessedInputs, + "actual_processed_inputs", candidate.ProcessedInputs(), + ) + closeMachineCandidate(snapshotLogger, candidate, "Failed to close invalid snapshot machine") + return nil + } + + // The default snapshot factory verifies this hash while loading. Verify it + // again at the manager boundary because tests and alternative factories are + // explicit injection seams and are not required to duplicate that policy. + actualHash, err := candidate.Hash(ctx) + if err != nil { + snapshotLogger.Error("Failed to verify snapshot machine hash", "error", err) + closeMachineCandidate(snapshotLogger, candidate, "Failed to close unverifiable snapshot machine") + return nil + } + if common.Hash(actualHash) != *snapshot.MachineHash { + snapshotLogger.Error("Snapshot machine hash mismatch", + "expected_hash", snapshot.MachineHash.Hex(), + "actual_hash", common.Hash(actualHash).Hex(), + ) + closeMachineCandidate(snapshotLogger, candidate, "Failed to close mismatched snapshot machine") + return nil + } + return candidate +} + +func (m *MachineManager) tryLoadTemplateInstance( + ctx context.Context, + app *Application, + logger *slog.Logger, +) MachineInstance { + candidate, err := m.instanceFactory.NewFromTemplate(ctx, app, m.logger, m.checkTemplateHash) + if err != nil { + if errors.Is(err, context.Canceled) { + logger.Debug("NewFromTemplate canceled during shutdown", "error", err) + } else { + logger.Error("Failed to create machine instance", "error", err) + } + closeMachineCandidate(logger, candidate, "Failed to close partial template machine") + return nil + } + if candidate == nil { + logger.Error("Template factory returned no machine instance") + return nil + } + return candidate +} + +func replayContradictionReason(err error) string { + reason := replay.ErrContradiction.Error() + var detail *replay.ContradictionError + if errors.As(err, &detail) { + // ContradictionError represents byte values only as lengths and + // digests, so its text is safe to persist for operator diagnostics. + reason = detail.Error() + } + return reason +} + +func classifyReplayFailure(err error) (reason, logMessage string, shouldFence bool) { + switch { + case errors.Is(err, replay.ErrContradiction): + // A contradiction proves that this local reconstruction is incompatible + // with the stored result, but not that the database itself is corrupt. A + // repaired template or runtime may recover it, so fence it as FAILED. An + // unresolved contradiction is detected and fenced again on re-enable. + return replayContradictionReason(err), + "Machine replay contradicts stored input result; marking application failed", true + case machine.IsExecutionLimitError(err): + return fmt.Sprintf("machine replay reached execution limit: %v", err), + "Machine replay reached an execution limit; marking application failed", true + case errors.Is(err, machine.ErrDeadlineExceeded): + return fmt.Sprintf("machine replay exceeded execution deadline: %v", err), + "Machine replay exceeded its execution deadline; marking application failed", true + case errors.Is(err, ErrIncompleteAdvance): + return fmt.Sprintf("machine replay returned an incomplete advance result: %v", err), + "Machine replay returned an incomplete advance result; marking application failed", true + case errors.Is(err, machine.ErrMachineInternal): + return fmt.Sprintf("machine replay failed internally: %v", err), + "Machine replay failed internally; marking application failed", true + default: + return "", "Failed to replay machine", false + } +} + // NewMachineManager creates a new machine manager. func NewMachineManager( repo MachineRepository, logger *slog.Logger, - checkHash bool, + checkTemplateHash bool, inputBatchSize uint64, opts ...Option, ) *MachineManager { m := &MachineManager{ - machines: map[int64]MachineInstance{}, - repository: repo, - checkHash: checkHash, - inputBatchSize: inputBatchSize, - logger: logger, - instanceFactory: &DefaultMachineInstanceFactory{}, + machines: map[int64]MachineInstance{}, + pendingApplicationFailures: map[int64]*pendingApplicationFailure{}, + repository: repo, + checkTemplateHash: checkTemplateHash, + inputBatchSize: inputBatchSize, + logger: logger, + instanceFactory: &DefaultMachineInstanceFactory{}, + replayRun: replay.Run, } for _, opt := range opts { opt(m) @@ -105,9 +307,20 @@ func NewMachineManager( // UpdateMachines refreshes the list of machines based on applications that // still need local machine work. func (m *MachineManager) UpdateMachines(ctx context.Context) error { - apps, _, err := getMachineApplications(ctx, m.repository) + // Retry failed status writes before doing any more machine work. Every app + // returned here remains fenced for this entire update, including when the + // retry succeeds but a repository read observes a stale executable row. + fenced, persistenceErr := m.persistPendingApplicationFailures(ctx) + if ctxErr := ctx.Err(); ctxErr != nil { + return errors.Join(persistenceErr, ctxErr) + } + if persistenceErr != nil && !IsOnlyApplicationFailurePersistenceErrors(persistenceErr) { + return persistenceErr + } + + apps, err := getMachineApplications(ctx, m.repository) if err != nil { - return err + return errors.Join(persistenceErr, err) } // Create machines for new applications @@ -115,95 +328,53 @@ func (m *MachineManager) UpdateMachines(ctx context.Context) error { if m.HasMachine(app.ID) { continue } + if _, pending := fenced[app.ID]; pending { + continue + } - m.logger.Info("Creating new machine instance", + appLogger := m.logger.With( "application", app.Name, - "address", app.IApplicationAddress) - - // Check if we have a snapshot to load from - var instance MachineInstance - - // Find the latest snapshot for this application - snapshot, err := m.repository.GetLastSnapshot(ctx, app.IApplicationAddress.String()) - if err != nil { - // Shutdown cancels the ctx mid-query; downgrade to Debug so - // operators don't see spurious ERR lines during a graceful - // stop. DeadlineExceeded would still flow through the Error - // branch and demand investigation. - if errors.Is(err, context.Canceled) { - m.logger.Debug("GetLastSnapshot canceled during shutdown", - "application", app.Name, - "error", err) - } else { - m.logger.Error("Failed to find latest snapshot", - "application", app.Name, - "error", err) - } - // Continue with template-based initialization - } + "address", app.IApplicationAddress, + ) + appLogger.Info("Creating new machine instance") - if snapshot != nil && snapshot.SnapshotURI != nil { - // Verify the snapshot path exists - if _, statErr := os.Stat(*snapshot.SnapshotURI); statErr == nil { - m.logger.Info("Creating machine instance from snapshot", - "application", app.Name, - "snapshot", *snapshot.SnapshotURI) - - instance, err = m.instanceFactory.NewFromSnapshot( - ctx, app, m.logger, m.checkHash, - *snapshot.SnapshotURI, snapshot.MachineHash, snapshot.Index) - if err != nil { - m.logger.Error("Failed to create machine instance from snapshot", - "application", app.Name, - "snapshot", *snapshot.SnapshotURI, - "error", err) - // Fall back to template-based initialization below - } - } else if errors.Is(statErr, os.ErrNotExist) { - m.logger.Warn("Snapshot path does not exist", - "application", app.Name, - "snapshot", *snapshot.SnapshotURI) - } else { - m.logger.Error("Failed to access snapshot path", - "application", app.Name, - "snapshot", *snapshot.SnapshotURI, - "error", statErr) - } + // Prefer a verified snapshot and fall back to the template whenever the + // snapshot is absent, stale, inaccessible, or invalid. + instance := m.tryLoadSnapshotInstance(ctx, app, appLogger) + if instance == nil { + instance = m.tryLoadTemplateInstance(ctx, app, appLogger) } - - // Fall back to template if snapshot loading failed or was unavailable if instance == nil { - instance, err = m.instanceFactory.NewFromTemplate(ctx, app, m.logger, m.checkHash) - if err != nil { - // Shutdown cancels the ctx mid-spawn; the partially - // constructed machine is torn down by NewFromTemplate - // itself. Downgrade to Debug for the graceful-stop case - // so the noise doesn't drown out real spawn failures. - if errors.Is(err, context.Canceled) { - m.logger.Debug("NewFromTemplate canceled during shutdown", - "application", app.IApplicationAddress, - "error", err) - } else { - m.logger.Error("Failed to create machine instance", - "application", app.IApplicationAddress, - "error", err) - } - continue - } + continue } - // Synchronize the machine with processed inputs. - // For template instances (processedInputs=0) this replays all inputs. - // For snapshot instances (processedInputs=snapshotIndex+1) this replays - // only inputs after the snapshot. - err = instance.Synchronize(ctx, m.repository, m.inputBatchSize) + // Canonical verification is the normal reconstruction policy: replay all + // completed inputs after the candidate's starting point and compare each + // status, exception payload, machine root, and cumulative outputs root. + _, err = m.replayRun(ctx, m.repository, instance, replay.Options{ + Application: app, + FromInput: instance.ProcessedInputs(), + ToInputExclusive: app.ProcessedInputs, + BatchSize: m.inputBatchSize, + Verification: repository.ReplayVerificationCanonical, + }) if err != nil { - m.logger.Error("Failed to synchronize machine", - "application", app.IApplicationAddress, - "error", err) + var appPersistenceErr error + reason, logMessage, shouldFence := classifyReplayFailure(err) + appLogger.Error(logMessage, "error", err) + if shouldFence { + m.FenceApplicationFailure(app, reason) + fenced[app.ID] = struct{}{} + appPersistenceErr = m.persistApplicationFailure(ctx, app.ID) + } if err := instance.Close(); err != nil { - m.logger.Warn("Failed to close machine after synchronization failure", - "application", app.Name, "error", err) + appLogger.Warn("Failed to close machine after replay failure", "error", err) + } + if appPersistenceErr != nil { + persistenceErr = errors.Join(persistenceErr, appPersistenceErr) + } + if ctxErr := ctx.Err(); ctxErr != nil { + return errors.Join(persistenceErr, ctxErr) } continue } @@ -211,18 +382,148 @@ func (m *MachineManager) UpdateMachines(ctx context.Context) error { // Add the machine to the manager; close if it fails if !m.addMachine(app.ID, instance) { if err := instance.Close(); err != nil { - m.logger.Warn("Failed to close duplicate machine instance", - "application", app.Name, "error", err) + appLogger.Warn("Failed to close duplicate machine instance", "error", err) } } } // Remove machines for non-enabled applications (disabled, failed, etc.) - m.removeMachines(apps) + m.removeMachines(excludeFencedApplications(apps, fenced)) + + return persistenceErr +} + +func excludeFencedApplications(apps []*Application, fenced map[int64]struct{}) []*Application { + if len(fenced) == 0 { + return apps + } + active := make([]*Application, 0, len(apps)) + for _, app := range apps { + if _, excluded := fenced[app.ID]; !excluded { + active = append(active, app) + } + } + return active +} + +// FenceApplicationFailure immediately fences app and queues the exact +// normalized FAILED reason for a later durability retry. Callers use this only +// after their initial status write failed; this method does not write the +// repository itself. +func (m *MachineManager) FenceApplicationFailure(app *Application, reason string) { + reason = appstatus.NormalizeReason(reason) + pending := &pendingApplicationFailure{ + application: app, + reason: reason, + } + m.mutex.Lock() + defer m.mutex.Unlock() + if _, exists := m.pendingApplicationFailures[app.ID]; !exists { + m.pendingApplicationFailures[app.ID] = pending + } +} +// persistPendingApplicationFailures retries every pending FAILED write and +// returns the application IDs that must remain fenced for this update. +func (m *MachineManager) persistPendingApplicationFailures( + ctx context.Context, +) (map[int64]struct{}, error) { + m.mutex.RLock() + appIDs := make([]int64, 0, len(m.pendingApplicationFailures)) + for appID := range m.pendingApplicationFailures { + appIDs = append(appIDs, appID) + } + m.mutex.RUnlock() + sort.Slice(appIDs, func(i, j int) bool { return appIDs[i] < appIDs[j] }) + + fenced := make(map[int64]struct{}, len(appIDs)) + var persistenceErrors []error + for _, appID := range appIDs { + fenced[appID] = struct{}{} + if ctxErr := ctx.Err(); ctxErr != nil { + return fenced, errors.Join(errors.Join(persistenceErrors...), ctxErr) + } + if err := m.persistApplicationFailure(ctx, appID); err != nil { + persistenceErrors = append(persistenceErrors, err) + } + if ctxErr := ctx.Err(); ctxErr != nil { + return fenced, errors.Join(errors.Join(persistenceErrors...), ctxErr) + } + } + return fenced, errors.Join(persistenceErrors...) +} + +func (m *MachineManager) persistApplicationFailure(ctx context.Context, appID int64) error { + m.mutex.RLock() + pending := m.pendingApplicationFailures[appID] + m.mutex.RUnlock() + if pending == nil { + return nil + } + + if writeErr := appstatus.SetFailed(ctx, m.logger, m.repository, pending.application, pending.reason); writeErr != nil { + if errors.Is(writeErr, repository.ErrNotFound) { + m.deletePendingApplicationFailure(appID, pending) + return nil + } + // Another process may have installed a stronger terminal status, or the + // write may have committed even though its result was lost. Retire the + // fence only when the durable row proves one of those outcomes. An + // unrelated FAILED reason remains fenced: it does not prove that this + // application failure was recorded. Unknown/read failures and OK + // (including disabled+OK) also remain pending. + current, readErr := m.repository.GetApplication( + ctx, pending.application.IApplicationAddress.String(), + ) + retire := errors.Is(readErr, repository.ErrNotFound) + if readErr == nil { + retire = current == nil || + current.ID != pending.application.ID || + applicationFailureAlreadyResolved(current, pending.reason) + } + if retire { + m.deletePendingApplicationFailure(appID, pending) + return nil + } + return &ApplicationFailurePersistenceError{ + ApplicationID: appID, + WriteErr: writeErr, + ReadErr: readErr, + } + } + + m.deletePendingApplicationFailure(appID, pending) return nil } +func applicationFailureAlreadyResolved(app *Application, reason string) bool { + if app == nil { + return false + } + switch app.Status { + case ApplicationStatus_Diverged, ApplicationStatus_Corrupted: + return true + case ApplicationStatus_Failed: + return app.Reason != nil && *app.Reason == reason + case ApplicationStatus_OK: + return false + default: + return false + } +} + +func (m *MachineManager) deletePendingApplicationFailure( + appID int64, + pending *pendingApplicationFailure, +) { + m.mutex.Lock() + // Remove only the pending record that this persistence attempt observed. + if m.pendingApplicationFailures[appID] == pending { + delete(m.pendingApplicationFailures, appID) + } + m.mutex.Unlock() +} + // GetMachine retrieves a machine instance for an application func (m *MachineManager) GetMachine(appID int64) (MachineInstance, bool) { m.mutex.RLock() @@ -239,6 +540,14 @@ func (m *MachineManager) HasMachine(appID int64) bool { return exists } +// HasPendingApplicationFailures reports whether any local machine work is +// fenced while its durable application status remains unconfirmed. +func (m *MachineManager) HasPendingApplicationFailures() bool { + m.mutex.RLock() + defer m.mutex.RUnlock() + return len(m.pendingApplicationFailures) != 0 +} + // addMachine adds a machine to the manager. // Returns false if the manager is closed or the appID already exists. func (m *MachineManager) addMachine(appID int64, machine MachineInstance) bool { @@ -337,17 +646,16 @@ func (m *MachineManager) Close() error { return errors.Join(errs...) } -func getMachineApplications(ctx context.Context, repo MachineRepository) ([]*Application, uint64, error) { - apps, total, err := repo.ListApplications(ctx, repository.ExecutableApplicationsFilter(), repository.Pagination{}, false) +func getMachineApplications(ctx context.Context, repo MachineRepository) ([]*Application, error) { + apps, _, err := repo.ListApplications(ctx, repository.ExecutableApplicationsFilter(), repository.Pagination{}, false) if err != nil { - return nil, 0, err + return nil, err } - foreclosedApps, foreclosedTotal, err := repo.ListApplications(ctx, foreclosedMachineDrainFilter(), repository.Pagination{}, false) + foreclosedApps, _, err := repo.ListApplications(ctx, foreclosedMachineDrainFilter(), repository.Pagination{}, false) if err != nil { - return nil, 0, err + return nil, err } - total += foreclosedTotal for _, app := range foreclosedApps { if app.ForecloseBlock == 0 { continue @@ -364,13 +672,13 @@ func getMachineApplications(ctx context.Context, repo MachineRepository) ([]*App } undrained, err := repo.HasUndrainedEpochsBeforeBlock(ctx, app.ID, app.ForecloseBlock) if err != nil { - return nil, 0, err + return nil, err } if undrained { apps = append(apps, app) } } - return apps, total, nil + return apps, nil } func foreclosedMachineDrainFilter() repository.ApplicationFilter { @@ -380,14 +688,3 @@ func foreclosedMachineDrainFilter() repository.ApplicationFilter { ForeclosureRecorded: new(true), } } - -// getProcessedInputs retrieves processed inputs with pagination support. -func getProcessedInputs( - ctx context.Context, - repo MachineRepository, - appAddress string, - p repository.Pagination, -) ([]*Input, uint64, error) { - f := repository.InputFilter{NotStatus: Pointer(InputCompletionStatus_None)} - return repo.ListInputs(ctx, appAddress, f, p, false) -} diff --git a/internal/manager/manager_test.go b/internal/manager/manager_test.go index 2b61120e7..9fd6665ff 100644 --- a/internal/manager/manager_test.go +++ b/internal/manager/manager_test.go @@ -6,14 +6,23 @@ package manager import ( "context" "errors" + "fmt" "io" "log/slog" + "math" + "math/big" + "strings" "testing" + "time" + "github.com/cartesi/rollups-node/internal/appstatus" "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/internal/replay" "github.com/cartesi/rollups-node/internal/repository" + "github.com/cartesi/rollups-node/pkg/machine" "github.com/ethereum/go-ethereum/common" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) @@ -25,17 +34,923 @@ type MachineManagerSuite struct { suite.Suite } +func newForkableMock() *MockRollupsMachine { + runtime := &MockRollupsMachine{ + CompletionStatusReturn: machine.CompletionStatusAccepted, + HashReturn: newHash(1), + OutputsHashReturn: newHash(2), + } + runtime.ForkFunc = func(context.Context) (machine.Machine, error) { + return newForkableMock(), nil + } + return runtime +} + +func newTestMachineManager( + repo MachineRepository, + logger *slog.Logger, + checkTemplateHash bool, + inputBatchSize uint64, + opts ...Option, +) *MachineManager { + testRun := withReplayRun(func( + _ context.Context, + _ repository.ReplayRepository, + executor replay.Executor, + _ replay.Options, + ) (replay.Result, error) { + if instance, ok := executor.(*DummyMachineInstanceMock); ok { + instance.replayCalls++ + return replay.Result{}, instance.replayErr + } + return replay.Result{}, nil + }) + return NewMachineManager( + repo, logger, checkTemplateHash, inputBatchSize, append(opts, testRun)..., + ) +} + +type nilSingleUnwrapperError struct{} + +func (nilSingleUnwrapperError) Error() string { return "nil single unwrapper" } +func (nilSingleUnwrapperError) Unwrap() error { return nil } + +type emptyMultiUnwrapperError struct{} + +func (emptyMultiUnwrapperError) Error() string { return "empty multi unwrapper" } +func (emptyMultiUnwrapperError) Unwrap() []error { return []error{} } + func (s *MachineManagerSuite) TestNewMachineManager() { require := s.Require() repo := &MockMachineRepository{} testLogger := slog.New(slog.NewTextHandler(io.Discard, nil)) - manager := NewMachineManager(repo, testLogger, false, 500) + manager := newTestMachineManager(repo, testLogger, false, 500) require.NotNil(manager) require.Empty(manager.machines) require.Equal(repo, manager.repository) } +func (s *MachineManagerSuite) TestUpdateMachinesUsesCanonicalReplayPolicy() { + require := s.Require() + app := &model.Application{ + ID: 47, + Name: "ReplayPolicy", + IApplicationAddress: common.HexToAddress("0x47"), + Status: model.ApplicationStatus_OK, + ProcessedInputs: 7, + } + repo := &MockMachineRepository{} + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && !*f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{app}, uint64(1), nil).Once() + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && *f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{}, uint64(0), nil).Once() + repo.On("GetLastSnapshot", mock.Anything, app.IApplicationAddress.String()).Return(nil, nil).Once() + + instance := &DummyMachineInstanceMock{application: app, processedInputs: 3} + factory := &MockMachineInstanceFactory{Instance: instance} + var capturedSource repository.ReplayRepository + var capturedExecutor replay.Executor + var capturedOptions replay.Options + run := func( + _ context.Context, + source repository.ReplayRepository, + executor replay.Executor, + options replay.Options, + ) (replay.Result, error) { + capturedSource = source + capturedExecutor = executor + capturedOptions = options + return replay.Result{ReplayedInputs: options.ToInputExclusive - options.FromInput}, nil + } + manager := NewMachineManager( + repo, + slog.New(slog.NewTextHandler(io.Discard, nil)), + false, + 23, + WithInstanceFactory(factory), + withReplayRun(run), + ) + + require.NoError(manager.UpdateMachines(context.Background())) + require.Same(repo, capturedSource) + require.Same(instance, capturedExecutor) + require.Same(app, capturedOptions.Application) + require.Equal(uint64(3), capturedOptions.FromInput) + require.Equal(uint64(7), capturedOptions.ToInputExclusive) + require.Equal(uint64(23), capturedOptions.BatchSize) + require.Equal(repository.ReplayVerificationCanonical, capturedOptions.Verification) + require.True(manager.HasMachine(app.ID)) + repo.AssertExpectations(s.T()) + require.NoError(manager.Close()) +} + +func (s *MachineManagerSuite) TestIsOnlyApplicationFailurePersistenceErrors() { + localFailure := func(applicationID int64) error { + return &ApplicationFailurePersistenceError{ + ApplicationID: applicationID, + WriteErr: errors.New("repository unavailable"), + } + } + wrap := func(message string, err error) error { + return fmt.Errorf("%s: %w", message, err) + } + + tests := []struct { + name string + err error + want bool + }{ + { + name: "nil", + err: nil, + want: false, + }, + { + name: "plain non-wrapping error", + err: errors.New("global repository failure"), + want: false, + }, + { + name: "single unwrapper returning nil", + err: nilSingleUnwrapperError{}, + want: false, + }, + { + name: "multi unwrapper returning empty slice", + err: emptyMultiUnwrapperError{}, + want: false, + }, + { + name: "single wrapper", + err: wrap("update machines", localFailure(1)), + want: true, + }, + { + name: "multiple wrappers", + err: wrap("advancer step", wrap("update machines", localFailure(2))), + want: true, + }, + { + name: "nested joins of wrapped local failures", + err: errors.Join( + wrap("first application", localFailure(3)), + errors.Join( + wrap("second application", wrap("status persistence", localFailure(4))), + wrap("third application", localFailure(5)), + ), + ), + want: true, + }, + { + name: "mixed join with global failure", + err: errors.Join( + wrap("application", localFailure(6)), + errors.Join( + wrap("another application", localFailure(7)), + errors.New("global repository failure"), + ), + ), + want: false, + }, + } + + for _, test := range tests { + s.Run(test.name, func() { + s.Equal(test.want, IsOnlyApplicationFailurePersistenceErrors(test.err)) + }) + } +} + func (s *MachineManagerSuite) TestUpdateMachines() { + s.Run("ReplayContradictionClosesAndDoesNotRegisterMachine", func() { + require := s.Require() + repo := &MockMachineRepository{} + app := &model.Application{ + ID: 71, + Name: "ReplayMismatch", + IApplicationAddress: common.HexToAddress("0x71"), + Enabled: true, + Status: model.ApplicationStatus_OK, + } + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && !*f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{app}, uint64(1), nil).Once() + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && *f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{}, uint64(0), nil).Once() + repo.On("GetLastSnapshot", mock.Anything, app.IApplicationAddress.String()).Return(nil, nil).Once() + const expectedReason = "replay contradicts persisted canonical result: " + + "app=\"ReplayMismatch\" epoch= input=0 " + + "field=machine_hash expected= actual=" + repo.On("UpdateApplicationStatus", mock.Anything, app.ID, model.ApplicationStatus_Failed, + mock.MatchedBy(func(reason *string) bool { + return reason != nil && *reason == expectedReason + })).Return(nil).Once() + + instance := &DummyMachineInstanceMock{ + application: app, + replayErr: &replay.ContradictionError{Application: app.Name, Field: "machine_hash"}, + } + factory := &MockMachineInstanceFactory{Instance: instance} + manager := newTestMachineManager( + repo, + slog.New(slog.NewTextHandler(io.Discard, nil)), + false, + 10, + WithInstanceFactory(factory), + ) + + require.NoError(manager.UpdateMachines(context.Background())) + require.Equal(1, instance.closeCalls) + require.Equal(1, factory.TemplateCalls) + require.False(manager.HasMachine(app.ID)) + require.Empty(manager.Applications()) + require.Equal(model.ApplicationStatus_Failed, app.Status) + require.NotNil(app.Reason) + require.Empty(manager.pendingApplicationFailures) + require.False(manager.HasPendingApplicationFailures()) + repo.AssertExpectations(s.T()) + + // A fresh manager sees no executable row once FAILED is durable, so a + // process restart cannot replay the contradictory history again. + restartRepo := &MockMachineRepository{} + restartRepo.On("ListApplications", mock.Anything, mock.Anything, repository.Pagination{}, false). + Return([]*model.Application{}, uint64(0), nil).Twice() + restartFactory := &MockMachineInstanceFactory{} + restarted := newTestMachineManager( + restartRepo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + WithInstanceFactory(restartFactory), + ) + require.NoError(restarted.UpdateMachines(context.Background())) + require.Zero(restartFactory.TemplateCalls) + restartRepo.AssertExpectations(s.T()) + }) + + s.Run("ApplicationFailureStatusWriteRetriesWithoutReplay", func() { + require := s.Require() + repo := &MockMachineRepository{} + app := &model.Application{ + ID: 75, Name: "RetryFailedStatus", IApplicationAddress: common.HexToAddress("0x75"), + Enabled: true, Status: model.ApplicationStatus_OK, + } + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && !*f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{app}, uint64(1), nil).Twice() + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && *f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{}, uint64(0), nil).Twice() + repo.On("GetLastSnapshot", mock.Anything, app.IApplicationAddress.String()).Return(nil, nil).Once() + writeErr := errors.New("temporary status write failure") + repo.On("UpdateApplicationStatus", mock.Anything, app.ID, model.ApplicationStatus_Failed, mock.Anything). + Return(writeErr).Once() + repo.On("GetApplication", mock.Anything, app.IApplicationAddress.String()).Return(app, nil).Once() + repo.On("UpdateApplicationStatus", mock.Anything, app.ID, model.ApplicationStatus_Failed, mock.Anything). + Return(nil).Once() + + instance := &DummyMachineInstanceMock{ + application: app, + replayErr: &replay.ContradictionError{ + Application: app.Name, InputIndex: 9, Field: "outputs_hash", + Expected: "0x01", Actual: "0x02", + }, + } + factory := &MockMachineInstanceFactory{Instance: instance} + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + WithInstanceFactory(factory), + ) + + err := manager.UpdateMachines(context.Background()) + require.ErrorIs(err, ErrApplicationFailureNotDurable) + var persistenceErr *ApplicationFailurePersistenceError + require.ErrorAs(err, &persistenceErr) + require.ErrorIs(err, writeErr) + require.Equal(app.ID, persistenceErr.ApplicationID) + require.Equal(model.ApplicationStatus_OK, app.Status) + require.Len(manager.pendingApplicationFailures, 1) + require.True(manager.HasPendingApplicationFailures()) + require.Equal(1, factory.TemplateCalls) + require.Equal(1, instance.closeCalls) + require.False(manager.HasMachine(app.ID)) + + // The next update retries only the FAILED write. The app remains fenced + // even though this mock returns its stale executable row. + require.NoError(manager.UpdateMachines(context.Background())) + require.Equal(model.ApplicationStatus_Failed, app.Status) + require.Empty(manager.pendingApplicationFailures) + require.False(manager.HasPendingApplicationFailures()) + require.Equal(1, factory.TemplateCalls) + require.Equal(1, instance.closeCalls) + require.False(manager.HasMachine(app.ID)) + repo.AssertExpectations(s.T()) + }) + + s.Run("ApplicationFailureFenceClearsWhenTerminalStatusWins", func() { + require := s.Require() + repo := &MockMachineRepository{} + app := &model.Application{ + ID: 76, Name: "TerminalStatusWins", IApplicationAddress: common.HexToAddress("0x76"), + Enabled: true, Status: model.ApplicationStatus_OK, + } + terminalApp := *app + terminalApp.Status = model.ApplicationStatus_Corrupted + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && !*f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{app}, uint64(1), nil).Once() + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && *f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{}, uint64(0), nil).Once() + repo.On("GetLastSnapshot", mock.Anything, app.IApplicationAddress.String()).Return(nil, nil).Once() + repo.On("UpdateApplicationStatus", mock.Anything, app.ID, model.ApplicationStatus_Failed, mock.Anything). + Return(errors.New("terminal status transition rejected")).Once() + repo.On("GetApplication", mock.Anything, app.IApplicationAddress.String()). + Return(&terminalApp, nil).Once() + + instance := &DummyMachineInstanceMock{ + application: app, + replayErr: &replay.ContradictionError{Application: app.Name, Field: "machine_hash"}, + } + factory := &MockMachineInstanceFactory{Instance: instance} + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + WithInstanceFactory(factory), + ) + + require.NoError(manager.UpdateMachines(context.Background())) + require.Empty(manager.pendingApplicationFailures) + require.False(manager.HasPendingApplicationFailures()) + require.Equal(1, factory.TemplateCalls) + require.Equal(1, instance.closeCalls) + require.False(manager.HasMachine(app.ID)) + repo.AssertExpectations(s.T()) + }) + + s.Run("ApplicationFailureFenceRetainsUnrelatedFailedStatus", func() { + require := s.Require() + repo := &MockMachineRepository{} + app := &model.Application{ + ID: 77, Name: "UnrelatedFailure", IApplicationAddress: common.HexToAddress("0x77"), + Status: model.ApplicationStatus_OK, + } + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + ) + manager.FenceApplicationFailure(app, replayContradictionReason(&replay.ContradictionError{ + Application: app.Name, Field: "machine_hash", + })) + + unrelatedReason := "machine process crashed" + durableApp := *app + durableApp.Status = model.ApplicationStatus_Failed + durableApp.Reason = &unrelatedReason + repo.On("UpdateApplicationStatus", mock.Anything, app.ID, model.ApplicationStatus_Failed, mock.Anything). + Return(errors.New("status transition rejected")).Once() + repo.On("GetApplication", mock.Anything, app.IApplicationAddress.String()). + Return(&durableApp, nil).Once() + + err := manager.persistApplicationFailure(context.Background(), app.ID) + + require.ErrorIs(err, ErrApplicationFailureNotDurable) + require.Len(manager.pendingApplicationFailures, 1) + require.True(manager.HasPendingApplicationFailures()) + repo.AssertExpectations(s.T()) + }) + + s.Run("AmbiguousWriteClearsFenceWhenExactFailedStatusIsDurable", func() { + require := s.Require() + repo := &MockMachineRepository{} + app := &model.Application{ + ID: 78, Name: "MatchingFailure", IApplicationAddress: common.HexToAddress("0x78"), + Status: model.ApplicationStatus_OK, + } + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + ) + manager.FenceApplicationFailure(app, replayContradictionReason(&replay.ContradictionError{ + Application: app.Name, Field: "machine_hash", + })) + pending := manager.pendingApplicationFailures[app.ID] + + durableApp := *app + durableApp.Status = model.ApplicationStatus_Failed + durableApp.Reason = &pending.reason + repo.On("UpdateApplicationStatus", mock.Anything, app.ID, model.ApplicationStatus_Failed, mock.Anything). + Return(errors.New("write result lost")).Once() + repo.On("GetApplication", mock.Anything, app.IApplicationAddress.String()). + Return(&durableApp, nil).Once() + + require.NoError(manager.persistApplicationFailure(context.Background(), app.ID)) + + require.Empty(manager.pendingApplicationFailures) + require.False(manager.HasPendingApplicationFailures()) + repo.AssertExpectations(s.T()) + }) + + s.Run("AmbiguousWriteClearsFenceForNormalizedLongReason", func() { + require := s.Require() + repo := &MockMachineRepository{} + app := &model.Application{ + ID: 81, Name: "LongReason", IApplicationAddress: common.HexToAddress("0x81"), + Status: model.ApplicationStatus_OK, + } + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + ) + detail := &replay.ContradictionError{ + Application: app.Name, + Field: "machine_hash", + Expected: strings.Repeat("e", 5000), + Actual: "different", + } + expectedReason := appstatus.NormalizeReason(detail.Error()) + manager.FenceApplicationFailure(app, replayContradictionReason(detail)) + pending := manager.pendingApplicationFailures[app.ID] + require.Equal(expectedReason, pending.reason) + + durableApp := *app + durableApp.Status = model.ApplicationStatus_Failed + durableApp.Reason = &expectedReason + repo.On( + "UpdateApplicationStatus", + mock.Anything, + app.ID, + model.ApplicationStatus_Failed, + mock.MatchedBy(func(reason *string) bool { + return reason != nil && *reason == expectedReason + }), + ).Return(errors.New("write result lost")).Once() + repo.On("GetApplication", mock.Anything, app.IApplicationAddress.String()). + Return(&durableApp, nil).Once() + + require.NoError(manager.persistApplicationFailure(context.Background(), app.ID)) + require.False(manager.HasPendingApplicationFailures()) + repo.AssertExpectations(s.T()) + }) + + s.Run("ApplicationFailureFenceClearsWhenStatusWriteFindsDeletedApplication", func() { + require := s.Require() + repo := &MockMachineRepository{} + app := &model.Application{ + ID: 79, Name: "DeletedApplication", IApplicationAddress: common.HexToAddress("0x79"), + Status: model.ApplicationStatus_OK, + } + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + ) + manager.FenceApplicationFailure(app, replayContradictionReason(&replay.ContradictionError{ + Application: app.Name, Field: "machine_hash", + })) + + repo.On("UpdateApplicationStatus", mock.Anything, app.ID, model.ApplicationStatus_Failed, mock.Anything). + Return(repository.ErrNotFound).Once() + + require.NoError(manager.persistApplicationFailure(context.Background(), app.ID)) + + require.Empty(manager.pendingApplicationFailures) + require.False(manager.HasPendingApplicationFailures()) + repo.AssertNotCalled(s.T(), "GetApplication", mock.Anything, mock.Anything) + repo.AssertExpectations(s.T()) + }) + + s.Run("ApplicationFailureFenceClearsWhenReadFindsDeletedApplication", func() { + require := s.Require() + repo := &MockMachineRepository{} + app := &model.Application{ + ID: 82, Name: "DeletedApplicationReadback", IApplicationAddress: common.HexToAddress("0x82"), + Status: model.ApplicationStatus_OK, + } + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + ) + manager.FenceApplicationFailure(app, "machine execution failed") + repo.On("UpdateApplicationStatus", mock.Anything, app.ID, model.ApplicationStatus_Failed, mock.Anything). + Return(errors.New("write result unavailable")).Once() + repo.On("GetApplication", mock.Anything, app.IApplicationAddress.String()). + Return(nil, nil).Once() + + require.NoError(manager.persistApplicationFailure(context.Background(), app.ID)) + require.False(manager.HasPendingApplicationFailures()) + repo.AssertExpectations(s.T()) + }) + + s.Run("ApplicationFailureFenceClearsWhenAddressBelongsToReplacement", func() { + require := s.Require() + repo := &MockMachineRepository{} + oldApp := &model.Application{ + ID: 82, Name: "OldApplication", IApplicationAddress: common.HexToAddress("0x82"), + Status: model.ApplicationStatus_OK, + } + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + ) + manager.FenceApplicationFailure(oldApp, replayContradictionReason(&replay.ContradictionError{ + Application: oldApp.Name, Field: "machine_hash", + })) + replacement := *oldApp + replacement.ID = 83 + replacement.Name = "ReplacementApplication" + + repo.On( + "UpdateApplicationStatus", + mock.Anything, + oldApp.ID, + model.ApplicationStatus_Failed, + mock.Anything, + ).Return(errors.New("write result unavailable")).Once() + repo.On("GetApplication", mock.Anything, oldApp.IApplicationAddress.String()). + Return(&replacement, nil).Once() + + require.NoError(manager.persistApplicationFailure(context.Background(), oldApp.ID)) + require.False(manager.HasPendingApplicationFailures()) + require.Equal(model.ApplicationStatus_OK, replacement.Status) + require.Nil(replacement.Reason) + repo.AssertNotCalled( + s.T(), + "UpdateApplicationStatus", + mock.Anything, + replacement.ID, + mock.Anything, + mock.Anything, + ) + repo.AssertExpectations(s.T()) + }) + + s.Run("ApplicationFailureFencePropagatesWriteAndReadFailure", func() { + require := s.Require() + repo := &MockMachineRepository{} + app := &model.Application{ + ID: 80, Name: "UnconfirmedFailure", IApplicationAddress: common.HexToAddress("0x80"), + Status: model.ApplicationStatus_OK, + } + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + ) + manager.FenceApplicationFailure(app, replayContradictionReason(&replay.ContradictionError{ + Application: app.Name, Field: "machine_hash", + })) + writeErr := errors.New("status write unavailable") + readErr := errors.New("status read unavailable") + repo.On("UpdateApplicationStatus", mock.Anything, app.ID, model.ApplicationStatus_Failed, mock.Anything). + Return(writeErr).Once() + repo.On("GetApplication", mock.Anything, app.IApplicationAddress.String()). + Return(nil, readErr).Once() + + err := manager.persistApplicationFailure(context.Background(), app.ID) + + require.ErrorIs(err, ErrApplicationFailureNotDurable) + require.ErrorIs(err, writeErr) + require.ErrorIs(err, readErr) + require.True(manager.HasPendingApplicationFailures()) + repo.AssertExpectations(s.T()) + }) + + s.Run("ApplicationFailureLocalDeadlineDoesNotBlockHealthySibling", func() { + require := s.Require() + repo := &MockMachineRepository{} + failed := &model.Application{ + ID: 91, Name: "LocalDeadline", IApplicationAddress: common.HexToAddress("0x91"), + Enabled: true, Status: model.ApplicationStatus_OK, + } + healthy := &model.Application{ + ID: 92, Name: "HealthyAfterDeadline", IApplicationAddress: common.HexToAddress("0x92"), + Enabled: true, Status: model.ApplicationStatus_OK, + } + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + WithInstanceFactory(&MockMachineInstanceFactory{TemplateFn: func(app *model.Application) (MachineInstance, error) { + return &DummyMachineInstanceMock{application: app}, nil + }}), + ) + manager.FenceApplicationFailure(failed, "machine execution failed") + localDeadline := fmt.Errorf("repository statement timeout: %w", context.DeadlineExceeded) + repo.On("UpdateApplicationStatus", mock.Anything, failed.ID, model.ApplicationStatus_Failed, mock.Anything). + Return(localDeadline).Once() + repo.On("GetApplication", mock.Anything, failed.IApplicationAddress.String()). + Return(failed, nil).Once() + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && !*f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{failed, healthy}, uint64(2), nil).Once() + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && *f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{}, uint64(0), nil).Once() + repo.On("GetLastSnapshot", mock.Anything, healthy.IApplicationAddress.String()).Return(nil, nil).Once() + + err := manager.UpdateMachines(context.Background()) + + require.ErrorIs(err, context.DeadlineExceeded) + require.True(IsOnlyApplicationFailurePersistenceErrors(err)) + require.True(manager.HasPendingApplicationFailures()) + require.False(manager.HasMachine(failed.ID)) + require.True(manager.HasMachine(healthy.ID), "a repository-local timeout must not block the sibling") + repo.AssertExpectations(s.T()) + }) + + s.Run("PendingApplicationFailureCancellationPreservesErrorChainAndStops", func() { + require := s.Require() + repo := &MockMachineRepository{} + manager := newTestMachineManager(repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10) + apps := []*model.Application{ + {ID: 101, Name: "First", IApplicationAddress: common.HexToAddress("0x101")}, + {ID: 102, Name: "Canceled", IApplicationAddress: common.HexToAddress("0x102")}, + {ID: 103, Name: "MustNotRun", IApplicationAddress: common.HexToAddress("0x103")}, + } + for _, app := range apps { + manager.FenceApplicationFailure(app, "machine execution failed") + } + firstWriteErr := errors.New("first write unavailable") + firstReadErr := errors.New("first read unavailable") + repo.On("UpdateApplicationStatus", mock.Anything, apps[0].ID, model.ApplicationStatus_Failed, mock.Anything). + Return(firstWriteErr).Once() + repo.On("GetApplication", mock.Anything, apps[0].IApplicationAddress.String()). + Return(nil, firstReadErr).Once() + ctx, cancel := context.WithCancel(context.Background()) + repo.On("UpdateApplicationStatus", mock.Anything, apps[1].ID, model.ApplicationStatus_Failed, mock.Anything). + Run(func(mock.Arguments) { cancel() }).Return(context.Canceled).Once() + repo.On("GetApplication", mock.Anything, apps[1].IApplicationAddress.String()). + Return(nil, context.Canceled).Once() + + _, err := manager.persistPendingApplicationFailures(ctx) + + require.ErrorIs(err, firstWriteErr) + require.ErrorIs(err, firstReadErr) + require.ErrorIs(err, context.Canceled) + var persistenceIDs []int64 + var collect func(error) + collect = func(current error) { + if current == nil { + return + } + if typed, ok := current.(*ApplicationFailurePersistenceError); ok { + persistenceIDs = append(persistenceIDs, typed.ApplicationID) + return + } + if joined, ok := current.(interface{ Unwrap() []error }); ok { + for _, child := range joined.Unwrap() { + collect(child) + } + } + } + collect(err) + require.ElementsMatch([]int64{apps[0].ID, apps[1].ID}, persistenceIDs) + repo.AssertNotCalled(s.T(), "UpdateApplicationStatus", mock.Anything, apps[2].ID, mock.Anything, mock.Anything) + require.Len(manager.pendingApplicationFailures, 3) + repo.AssertExpectations(s.T()) + }) + + s.Run("QueuedLiveFailureRetryClearsFenceAndRemovesOnlyAffectedMachine", func() { + require := s.Require() + repo := &MockMachineRepository{} + failed := &model.Application{ + ID: 111, Name: "LiveFailure", IApplicationAddress: common.HexToAddress("0x111"), + Enabled: true, Status: model.ApplicationStatus_OK, + } + healthy := &model.Application{ + ID: 112, Name: "Healthy", IApplicationAddress: common.HexToAddress("0x112"), + Enabled: true, Status: model.ApplicationStatus_OK, + } + manager := newTestMachineManager(repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10) + failedMachine := &DummyMachineInstanceMock{application: failed} + healthyMachine := &DummyMachineInstanceMock{application: healthy} + require.True(manager.addMachine(failed.ID, failedMachine)) + require.True(manager.addMachine(healthy.ID, healthyMachine)) + const reason = "advance execution reached configured cycle limit" + manager.FenceApplicationFailure(failed, reason) + repo.On("UpdateApplicationStatus", mock.Anything, failed.ID, model.ApplicationStatus_Failed, + mock.MatchedBy(func(got *string) bool { return got != nil && *got == reason })).Return(nil).Once() + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && !*f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{failed, healthy}, uint64(2), nil).Once() + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && *f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{}, uint64(0), nil).Once() + + require.NoError(manager.UpdateMachines(context.Background())) + + require.False(manager.HasPendingApplicationFailures()) + require.False(manager.HasMachine(failed.ID)) + require.True(manager.HasMachine(healthy.ID)) + require.Equal(1, failedMachine.closeCalls) + require.Zero(healthyMachine.closeCalls) + repo.AssertExpectations(s.T()) + }) + + s.Run("TransientReplayErrorRetries", func() { + require := s.Require() + repo := &MockMachineRepository{} + app := &model.Application{ + ID: 72, Name: "TransientReplay", IApplicationAddress: common.HexToAddress("0x72"), + Enabled: true, Status: model.ApplicationStatus_OK, + } + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && !*f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{app}, uint64(1), nil).Twice() + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && *f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{}, uint64(0), nil).Twice() + repo.On("GetLastSnapshot", mock.Anything, app.IApplicationAddress.String()).Return(nil, nil).Twice() + + var instances []*DummyMachineInstanceMock + factory := &MockMachineInstanceFactory{TemplateFn: func(got *model.Application) (MachineInstance, error) { + instance := &DummyMachineInstanceMock{application: got} + if len(instances) == 0 { + instance.replayErr = errors.New("temporary repository outage") + } + instances = append(instances, instance) + return instance, nil + }} + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + WithInstanceFactory(factory), + ) + + require.NoError(manager.UpdateMachines(context.Background())) + require.Len(instances, 1) + require.Equal(1, instances[0].closeCalls) + require.Empty(manager.pendingApplicationFailures) + require.False(manager.HasMachine(app.ID)) + + require.NoError(manager.UpdateMachines(context.Background())) + require.Len(instances, 2) + require.Equal(2, factory.TemplateCalls) + require.True(manager.HasMachine(app.ID)) + require.Empty(manager.pendingApplicationFailures) + repo.AssertExpectations(s.T()) + }) + + s.Run("ReplayContradictionDoesNotBlockHealthySibling", func() { + require := s.Require() + repo := &MockMachineRepository{} + badApp := &model.Application{ + ID: 73, Name: "BadReplay", IApplicationAddress: common.HexToAddress("0x73"), + Enabled: true, Status: model.ApplicationStatus_OK, + } + goodApp := &model.Application{ + ID: 74, Name: "GoodReplay", IApplicationAddress: common.HexToAddress("0x74"), + Enabled: true, Status: model.ApplicationStatus_OK, + } + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && !*f.ForeclosureRecorded + }), repository.Pagination{}, false). + Return([]*model.Application{badApp, goodApp}, uint64(2), nil).Once() + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && *f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{}, uint64(0), nil).Once() + repo.On("GetLastSnapshot", mock.Anything, mock.Anything).Return(nil, nil).Twice() + repo.On("UpdateApplicationStatus", mock.Anything, badApp.ID, model.ApplicationStatus_Failed, + mock.Anything).Return(nil).Once() + + instances := map[int64]*DummyMachineInstanceMock{} + factory := &MockMachineInstanceFactory{TemplateFn: func(app *model.Application) (MachineInstance, error) { + instance := &DummyMachineInstanceMock{application: app} + if app.ID == badApp.ID { + instance.replayErr = &replay.ContradictionError{ + Application: app.Name, InputIndex: 4, Field: "outputs_hash", + Expected: "0x01", Actual: "0x02", + } + } + instances[app.ID] = instance + return instance, nil + }} + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + WithInstanceFactory(factory), + ) + + require.NoError(manager.UpdateMachines(context.Background())) + require.False(manager.HasMachine(badApp.ID)) + require.True(manager.HasMachine(goodApp.ID)) + require.Equal(1, instances[badApp.ID].closeCalls) + require.Zero(instances[goodApp.ID].closeCalls) + require.Equal(model.ApplicationStatus_Failed, badApp.Status) + require.Equal(model.ApplicationStatus_OK, goodApp.Status) + require.Empty(manager.pendingApplicationFailures) + require.False(manager.HasPendingApplicationFailures()) + repo.AssertExpectations(s.T()) + }) + + s.Run("UnconfirmedApplicationFailureDoesNotBlockHealthySibling", func() { + require := s.Require() + repo := &MockMachineRepository{} + badApp := &model.Application{ + ID: 83, Name: "BadUndurableReplay", IApplicationAddress: common.HexToAddress("0x83"), + Enabled: true, Status: model.ApplicationStatus_OK, + } + goodApp := &model.Application{ + ID: 84, Name: "HealthyReplay", IApplicationAddress: common.HexToAddress("0x84"), + Enabled: true, Status: model.ApplicationStatus_OK, + } + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && !*f.ForeclosureRecorded + }), repository.Pagination{}, false). + Return([]*model.Application{badApp, goodApp}, uint64(2), nil).Once() + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && *f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{}, uint64(0), nil).Once() + repo.On("GetLastSnapshot", mock.Anything, mock.Anything).Return(nil, nil).Twice() + writeErr := errors.New("status write unavailable") + readErr := errors.New("status read unavailable") + repo.On("UpdateApplicationStatus", mock.Anything, badApp.ID, model.ApplicationStatus_Failed, mock.Anything). + Return(writeErr).Once() + repo.On("GetApplication", mock.Anything, badApp.IApplicationAddress.String()). + Return(nil, readErr).Once() + + instances := map[int64]*DummyMachineInstanceMock{} + factory := &MockMachineInstanceFactory{TemplateFn: func(app *model.Application) (MachineInstance, error) { + instance := &DummyMachineInstanceMock{application: app} + if app.ID == badApp.ID { + instance.replayErr = &replay.ContradictionError{ + Application: app.Name, InputIndex: 4, Field: "outputs_hash", + } + } + instances[app.ID] = instance + return instance, nil + }} + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + WithInstanceFactory(factory), + ) + + err := manager.UpdateMachines(context.Background()) + require.ErrorIs(err, ErrApplicationFailureNotDurable) + require.ErrorIs(err, writeErr) + require.ErrorIs(err, readErr) + require.False(manager.HasMachine(badApp.ID)) + require.True(manager.HasMachine(goodApp.ID)) + require.Equal(1, instances[badApp.ID].closeCalls) + require.Zero(instances[goodApp.ID].closeCalls) + require.True(manager.HasPendingApplicationFailures()) + repo.AssertExpectations(s.T()) + }) + + replayFailures := []struct { + name string + err error + reasonContains string + }{ + {"PayloadLengthLimit", machine.ErrPayloadLengthLimitExceeded, "execution limit"}, + {"OutputsLimit", machine.ErrOutputsLimitExceeded, "execution limit"}, + {"ReportsLimit", machine.ErrReportsLimitExceeded, "execution limit"}, + {"McycleLimit", machine.ErrReachedLimitMcycle, "execution limit"}, + {"Deadline", machine.ErrDeadlineExceeded, "execution deadline"}, + {"IncompleteAdvance", ErrIncompleteAdvance, "incomplete advance result"}, + {"MachineInternal", machine.ErrMachineInternal, "failed internally"}, + } + for index, failure := range replayFailures { + s.Run("ReplayFailure/"+failure.name, func() { + require := s.Require() + repo := &MockMachineRepository{} + failedID := int64(85 + index*2) + failed := &model.Application{ + ID: failedID, Name: "FailedReplay", + IApplicationAddress: common.BigToAddress(big.NewInt(failedID)), + Enabled: true, Status: model.ApplicationStatus_OK, + } + healthy := &model.Application{ + ID: failedID + 1, Name: "HealthyReplay", + IApplicationAddress: common.BigToAddress(big.NewInt(failedID + 1)), + Enabled: true, Status: model.ApplicationStatus_OK, + } + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && !*f.ForeclosureRecorded + }), repository.Pagination{}, false). + Return([]*model.Application{failed, healthy}, uint64(2), nil).Once() + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && *f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{}, uint64(0), nil).Once() + repo.On("GetLastSnapshot", mock.Anything, mock.Anything).Return(nil, nil).Twice() + repo.On("UpdateApplicationStatus", mock.Anything, failed.ID, model.ApplicationStatus_Failed, + mock.MatchedBy(func(reason *string) bool { + return reason != nil && strings.Contains(*reason, failure.reasonContains) + })).Return(nil).Once() + + instances := map[int64]*DummyMachineInstanceMock{} + factory := &MockMachineInstanceFactory{TemplateFn: func(app *model.Application) (MachineInstance, error) { + instance := &DummyMachineInstanceMock{application: app} + if app.ID == failed.ID { + instance.replayErr = fmt.Errorf("replay input: %w", failure.err) + } + instances[app.ID] = instance + return instance, nil + }} + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + WithInstanceFactory(factory), + ) + + require.NoError(manager.UpdateMachines(context.Background())) + require.False(manager.HasMachine(failed.ID)) + require.True(manager.HasMachine(healthy.ID)) + require.Equal(model.ApplicationStatus_Failed, failed.Status) + require.Equal(model.ApplicationStatus_OK, healthy.Status) + require.Equal(1, instances[failed.ID].closeCalls) + require.Zero(instances[healthy.ID].closeCalls) + require.False(manager.HasPendingApplicationFailures()) + repo.AssertExpectations(s.T()) + }) + } + s.Run("AddNewMachines", func() { require := s.Require() @@ -56,10 +971,6 @@ func (s *MachineManagerSuite) TestUpdateMachines() { repo.On("ListApplications", mock.Anything, mock.Anything, mock.Anything, false). Return([]*model.Application{app1}, uint64(1), nil) - // Empty inputs for synchronization - repo.On("ListInputs", mock.Anything, mock.Anything, mock.Anything, mock.Anything, false). - Return([]*model.Input{}, uint64(0), nil) - // Mock GetLastSnapshot to return nil (no snapshot available) repo.On("GetLastSnapshot", mock.Anything, mock.Anything). Return(nil, nil) @@ -68,7 +979,7 @@ func (s *MachineManagerSuite) TestUpdateMachines() { testLogger := slog.New(slog.NewTextHandler(io.Discard, nil)) mockInstance := &DummyMachineInstanceMock{application: app1} factory := &MockMachineInstanceFactory{Instance: mockInstance} - manager := NewMachineManager(repo, testLogger, false, 500, WithInstanceFactory(factory)) + manager := newTestMachineManager(repo, testLogger, false, 500, WithInstanceFactory(factory)) err := manager.UpdateMachines(context.Background()) require.NoError(err) @@ -116,7 +1027,7 @@ func (s *MachineManagerSuite) TestUpdateMachines() { testLogger := slog.New(slog.NewTextHandler(io.Discard, nil)) mockInstance := &DummyMachineInstanceMock{application: app} factory := &MockMachineInstanceFactory{Instance: mockInstance} - manager := NewMachineManager(repo, testLogger, false, 500, WithInstanceFactory(factory)) + manager := newTestMachineManager(repo, testLogger, false, 500, WithInstanceFactory(factory)) err := manager.UpdateMachines(context.Background()) require.NoError(err) @@ -161,7 +1072,7 @@ func (s *MachineManagerSuite) TestUpdateMachines() { testLogger := slog.New(slog.NewTextHandler(io.Discard, nil)) mockInstance := &DummyMachineInstanceMock{application: app} factory := &MockMachineInstanceFactory{Instance: mockInstance} - manager := NewMachineManager(repo, testLogger, false, 500, WithInstanceFactory(factory)) + manager := newTestMachineManager(repo, testLogger, false, 500, WithInstanceFactory(factory)) err := manager.UpdateMachines(context.Background()) require.NoError(err) @@ -180,7 +1091,7 @@ func (s *MachineManagerSuite) TestUpdateMachines() { // Create a test logger testLogger := slog.New(slog.NewTextHandler(io.Discard, nil)) - manager := NewMachineManager(repo, testLogger, false, 500) + manager := newTestMachineManager(repo, testLogger, false, 500) // Add mock machines app1 := &model.Application{ID: 1, Name: "App1"} @@ -206,6 +1117,316 @@ func (s *MachineManagerSuite) TestUpdateMachines() { }) } +func (s *MachineManagerSuite) TestSnapshotStartingStateVerification() { + newApp := func(id int64, processed uint64) *model.Application { + return &model.Application{ + ID: id, + Name: "SnapshotApp", + IApplicationAddress: common.BigToAddress(new(big.Int).SetInt64(id)), + Enabled: true, + Status: model.ApplicationStatus_OK, + ProcessedInputs: processed, + ExecutionParameters: model.ExecutionParameters{ + AdvanceMaxDeadline: time.Second, + InspectMaxDeadline: time.Second, + LoadDeadline: time.Second, + MaxConcurrentInspects: 3, + }, + } + } + prepareRepo := func( + app *model.Application, + snapshot *model.Input, + snapshotErr error, + ) *MockMachineRepository { + repo := &MockMachineRepository{ + replayApplicationID: app.ID, + replayConsensus: app.ConsensusType, + } + repo.On("ListApplications", mock.Anything, mock.Anything, mock.Anything, false). + Return([]*model.Application{app}, uint64(1), nil) + repo.On("GetLastSnapshot", mock.Anything, app.IApplicationAddress.String()). + Return(snapshot, snapshotErr).Once() + return repo + } + configureReplay := func(repo *MockMachineRepository, app *model.Application) common.Hash { + machineHash := newHash(1) + outputsHash := newHash(2) + repo.replayCount = app.ProcessedInputs + repo.replayRecords = []*model.ReplayRecord{{ + Input: model.ReplayInput{ + ApplicationID: app.ID, + EpochIndex: 0, + InputIndex: 0, + RawData: []byte("replayed input"), + Status: model.InputCompletionStatus_Accepted, + MachineHash: &machineHash, + OutputsHash: &outputsHash, + }, + }} + return machineHash + } + newReplayTemplate := func(app *model.Application) MachineInstance { + base := &MockRollupsMachine{ + HashReturn: newHash(0), + OutputsHashReturn: newHash(0), + } + base.ForkReturn = newForkableMock() + instance, err := NewMachineInstanceWithFactory( + context.Background(), app, 0, + slog.New(slog.NewTextHandler(io.Discard, nil)), + &MockMachineRuntimeFactory{RuntimeToReturn: base}, + ) + s.Require().NoError(err) + return instance + } + assertTemplateReplay := func( + require *require.Assertions, + manager *MachineManager, + repo *MockMachineRepository, + app *model.Application, + expectedHash common.Hash, + ) { + machineInstance, exists := manager.GetMachine(app.ID) + require.True(exists) + require.Equal(app.ProcessedInputs, machineInstance.ProcessedInputs()) + actualHash, err := machineInstance.Hash(context.Background()) + require.NoError(err) + require.Equal([32]byte(expectedHash), actualHash) + require.Positive(repo.replayListCalls, "template fallback must replay canonical inputs") + } + + s.Run("MatchingCaughtUpSnapshotIsAccepted", func() { + require := s.Require() + app := newApp(81, 3) + path := s.T().TempDir() + hash := common.HexToHash("0x81") + snapshot := &model.Input{Index: 2, SnapshotURI: &path, MachineHash: &hash} + repo := prepareRepo(app, snapshot, nil) + repo.replayCount = 3 + snapshotInstance := &DummyMachineInstanceMock{ + application: app, processedInputs: 3, hashReturn: [32]byte(hash), + } + factory := &MockMachineInstanceFactory{SnapshotInstance: snapshotInstance} + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + WithInstanceFactory(factory), + ) + + require.NoError(manager.UpdateMachines(context.Background())) + require.Equal(1, factory.SnapshotCalls) + require.Zero(factory.TemplateCalls) + require.Equal(1, snapshotInstance.hashCalls) + require.Equal(1, snapshotInstance.replayCalls) + require.True(manager.HasMachine(app.ID)) + require.NoError(manager.Close()) + }) + + fallbackTests := []struct { + name string + snapshot func(path string, expectedHash common.Hash) *model.Input + repositoryErr error + candidate func(app *model.Application, expectedHash common.Hash) *DummyMachineInstanceMock + factoryErr error + wantCalls int + wantHashCalls int + }{ + { + name: "missing persisted hash", + snapshot: func(path string, _ common.Hash) *model.Input { + return &model.Input{Index: 0, SnapshotURI: &path} + }, + wantCalls: 0, + }, + { + name: "repository result accompanied by error", + snapshot: func(path string, expectedHash common.Hash) *model.Input { + return &model.Input{Index: 0, SnapshotURI: &path, MachineHash: &expectedHash} + }, + repositoryErr: errors.New("snapshot query failed"), + wantCalls: 0, + }, + { + name: "snapshot input index cannot be incremented", + snapshot: func(path string, expectedHash common.Hash) *model.Input { + return &model.Input{Index: math.MaxUint64, SnapshotURI: &path, MachineHash: &expectedHash} + }, + wantCalls: 0, + }, + { + name: "snapshot is beyond application progress", + snapshot: func(path string, expectedHash common.Hash) *model.Input { + return &model.Input{Index: 1, SnapshotURI: &path, MachineHash: &expectedHash} + }, + wantCalls: 0, + }, + { + name: "matching hash with wrong processed input count", + snapshot: func(path string, expectedHash common.Hash) *model.Input { + return &model.Input{Index: 0, SnapshotURI: &path, MachineHash: &expectedHash} + }, + candidate: func(app *model.Application, expectedHash common.Hash) *DummyMachineInstanceMock { + return &DummyMachineInstanceMock{ + application: app, processedInputs: 0, hashReturn: [32]byte(expectedHash), + } + }, + wantCalls: 1, + }, + { + name: "manager boundary hash mismatch", + snapshot: func(path string, expectedHash common.Hash) *model.Input { + return &model.Input{Index: 0, SnapshotURI: &path, MachineHash: &expectedHash} + }, + candidate: func(app *model.Application, _ common.Hash) *DummyMachineInstanceMock { + return &DummyMachineInstanceMock{application: app, processedInputs: 1, hashReturn: newHash(99)} + }, + wantCalls: 1, + wantHashCalls: 1, + }, + { + name: "manager boundary hash read error", + snapshot: func(path string, expectedHash common.Hash) *model.Input { + return &model.Input{Index: 0, SnapshotURI: &path, MachineHash: &expectedHash} + }, + candidate: func(app *model.Application, expectedHash common.Hash) *DummyMachineInstanceMock { + return &DummyMachineInstanceMock{ + application: app, processedInputs: 1, hashReturn: [32]byte(expectedHash), + hashError: errors.New("snapshot hash unavailable"), + } + }, + wantCalls: 1, + wantHashCalls: 1, + }, + { + name: "factory returns partial candidate with error", + snapshot: func(path string, expectedHash common.Hash) *model.Input { + return &model.Input{Index: 0, SnapshotURI: &path, MachineHash: &expectedHash} + }, + candidate: func(app *model.Application, expectedHash common.Hash) *DummyMachineInstanceMock { + return &DummyMachineInstanceMock{ + application: app, processedInputs: 1, hashReturn: [32]byte(expectedHash), + } + }, + factoryErr: errors.New("factory failed after creating candidate"), + wantCalls: 1, + }, + } + + for i, tt := range fallbackTests { + s.Run(tt.name, func() { + require := s.Require() + app := newApp(82+int64(i), 1) + path := s.T().TempDir() + expectedHash := newHash(1) + snapshot := tt.snapshot(path, expectedHash) + repo := prepareRepo(app, snapshot, tt.repositoryErr) + require.Equal(expectedHash, configureReplay(repo, app)) + candidate := (*DummyMachineInstanceMock)(nil) + if tt.candidate != nil { + candidate = tt.candidate(app, expectedHash) + } + template := newReplayTemplate(app) + factory := &MockMachineInstanceFactory{ + Instance: template, SnapshotInstance: candidate, SnapshotErr: tt.factoryErr, + } + manager := NewMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + WithInstanceFactory(factory), + ) + + require.NoError(manager.UpdateMachines(context.Background())) + require.Equal(tt.wantCalls, factory.SnapshotCalls) + require.Equal(1, factory.TemplateCalls) + require.Equal([]bool{false}, factory.TemplateChecks) + if candidate != nil { + require.Equal(1, candidate.closeCalls) + require.Equal(tt.wantHashCalls, candidate.hashCalls) + require.Zero(candidate.replayCalls) + } + assertTemplateReplay(require, manager, repo, app, expectedHash) + require.NoError(manager.Close()) + }) + } + + s.Run("SnapshotFactoryNilResultFallsBackToTemplate", func() { + require := s.Require() + app := newApp(90, 1) + path := s.T().TempDir() + expectedHash := common.Hash{} + snapshot := &model.Input{Index: 0, SnapshotURI: &path, MachineHash: &expectedHash} + repo := prepareRepo(app, snapshot, nil) + expectedHash = configureReplay(repo, app) + snapshot.MachineHash = &expectedHash + template := newReplayTemplate(app) + factory := &MockMachineInstanceFactory{Instance: template, ReturnNilSnapshot: true} + manager := NewMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + WithInstanceFactory(factory), + ) + + require.NoError(manager.UpdateMachines(context.Background())) + require.Equal(1, factory.SnapshotCalls) + require.Equal(1, factory.TemplateCalls) + assertTemplateReplay(require, manager, repo, app, expectedHash) + require.NoError(manager.Close()) + }) +} + +func (s *MachineManagerSuite) TestTemplateFactoryResultHandling() { + tests := []struct { + name string + candidate *DummyMachineInstanceMock + factoryErr error + }{ + {name: "nil instance without error"}, + { + name: "partial instance with error", + candidate: &DummyMachineInstanceMock{ + closeError: errors.New("partial template close failed"), + }, + factoryErr: errors.New("template factory failed"), + }, + } + + for i, tt := range tests { + s.Run(tt.name, func() { + require := s.Require() + app := &model.Application{ + ID: 91 + int64(i), + Name: "TemplateFactoryResultApp", + IApplicationAddress: common.BigToAddress(big.NewInt(91 + int64(i))), + Enabled: true, + Status: model.ApplicationStatus_OK, + } + if tt.candidate != nil { + tt.candidate.application = app + } + repo := &MockMachineRepository{} + repo.On("ListApplications", mock.Anything, mock.Anything, mock.Anything, false). + Return([]*model.Application{app}, uint64(1), nil) + repo.On("GetLastSnapshot", mock.Anything, app.IApplicationAddress.String()). + Return(nil, nil).Once() + factory := &MockMachineInstanceFactory{Err: tt.factoryErr} + if tt.candidate != nil { + factory.Instance = tt.candidate + } + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + WithInstanceFactory(factory), + ) + + require.NoError(manager.UpdateMachines(context.Background())) + require.Equal(1, factory.TemplateCalls) + require.False(manager.HasMachine(app.ID)) + if tt.candidate != nil { + require.Equal(1, tt.candidate.closeCalls) + require.Zero(tt.candidate.replayCalls) + } + require.NoError(manager.Close()) + }) + } +} + func (s *MachineManagerSuite) TestGetMachine() { require := s.Require() @@ -213,7 +1434,7 @@ func (s *MachineManagerSuite) TestGetMachine() { repo.On("GetLastSnapshot", mock.Anything, mock.Anything). Return(nil, nil) - manager := NewMachineManager(repo, nil, false, 500) + manager := newTestMachineManager(repo, nil, false, 500) machine := &DummyMachineInstanceMock{application: &model.Application{ID: 1}} // Add a machine @@ -236,7 +1457,7 @@ func (s *MachineManagerSuite) TestHasMachine() { repo.On("GetLastSnapshot", mock.Anything, mock.Anything). Return(nil, nil) - manager := NewMachineManager(repo, nil, false, 500) + manager := newTestMachineManager(repo, nil, false, 500) machine := &DummyMachineInstanceMock{application: &model.Application{ID: 1}} // Add a machine @@ -256,7 +1477,7 @@ func (s *MachineManagerSuite) TestAddMachine() { repo.On("GetLastSnapshot", mock.Anything, mock.Anything). Return(nil, nil) - manager := NewMachineManager(repo, nil, false, 500) + manager := newTestMachineManager(repo, nil, false, 500) machine1 := &DummyMachineInstanceMock{application: &model.Application{ID: 1}} machine2 := &DummyMachineInstanceMock{application: &model.Application{ID: 2}} @@ -287,7 +1508,7 @@ func (s *MachineManagerSuite) TestAddMachine() { func (s *MachineManagerSuite) TestRemoveDisabledMachines() { require := s.Require() - manager := NewMachineManager(nil, nil, false, 500) + manager := newTestMachineManager(nil, nil, false, 500) // Add machines app1 := &model.Application{ID: 1} @@ -321,7 +1542,7 @@ func (s *MachineManagerSuite) TestUpdateMachinesErrors() { Return(([]*model.Application)(nil), uint64(0), errors.New("db error")) testLogger := slog.New(slog.NewTextHandler(io.Discard, nil)) - manager := NewMachineManager(repo, testLogger, false, 500) + manager := newTestMachineManager(repo, testLogger, false, 500) err := manager.UpdateMachines(context.Background()) require.Error(err) @@ -354,15 +1575,11 @@ func (s *MachineManagerSuite) TestUpdateMachinesErrors() { Return([]*model.Application{app}, uint64(1), nil) repo.On("GetLastSnapshot", mock.Anything, mock.Anything). Return(snapshotInput, nil) - // ListInputs for synchronization (no inputs to replay) - repo.On("ListInputs", mock.Anything, mock.Anything, mock.Anything, mock.Anything, false). - Return([]*model.Input{}, uint64(0), nil) - // The snapshot path doesn't exist, so it should fall back to template testLogger := slog.New(slog.NewTextHandler(io.Discard, nil)) mockInstance := &DummyMachineInstanceMock{application: app} factory := &MockMachineInstanceFactory{Instance: mockInstance} - manager := NewMachineManager(repo, testLogger, false, 500, WithInstanceFactory(factory)) + manager := newTestMachineManager(repo, testLogger, false, 500, WithInstanceFactory(factory)) err := manager.UpdateMachines(context.Background()) require.NoError(err) @@ -392,7 +1609,7 @@ func (s *MachineManagerSuite) TestUpdateMachinesErrors() { testLogger := slog.New(slog.NewTextHandler(io.Discard, nil)) factory := &MockMachineInstanceFactory{Err: errors.New("machine creation failed")} - manager := NewMachineManager(repo, testLogger, false, 500, WithInstanceFactory(factory)) + manager := newTestMachineManager(repo, testLogger, false, 500, WithInstanceFactory(factory)) err := manager.UpdateMachines(context.Background()) // UpdateMachines should not return an error; it logs and skips @@ -421,9 +1638,7 @@ func (s *MachineManagerSuite) TestUpdateMachinesErrors() { Return([]*model.Application{app}, uint64(1), nil) repo.On("GetLastSnapshot", mock.Anything, mock.Anything). Return(nil, nil) - // ListInputs returns an error so the real Synchronize method propagates the failure. - repo.On("ListInputs", mock.Anything, mock.Anything, mock.Anything, mock.Anything, false). - Return(([]*model.Input)(nil), uint64(0), errors.New("db connection lost")) + repo.replayCountError = errors.New("db connection lost") testLogger := slog.New(slog.NewTextHandler(io.Discard, nil)) @@ -447,7 +1662,7 @@ func (s *MachineManagerSuite) TestUpdateMachinesErrors() { func (s *MachineManagerSuite) TestCloseAggregatesErrors() { require := s.Require() - manager := NewMachineManager(nil, nil, false, 500) + manager := newTestMachineManager(nil, nil, false, 500) machine1 := &DummyMachineInstanceMock{application: &model.Application{ID: 1}} machine2 := &DummyMachineInstanceMock{ @@ -476,7 +1691,7 @@ func (s *MachineManagerSuite) TestApplications() { repo.On("GetLastSnapshot", mock.Anything, mock.Anything). Return(nil, nil) - manager := NewMachineManager(repo, nil, false, 500) + manager := newTestMachineManager(repo, nil, false, 500) // Add machines app1 := &model.Application{ID: 1, Name: "App1"} @@ -507,6 +1722,43 @@ func (s *MachineManagerSuite) TestApplications() { // Mock repository for testing type MockMachineRepository struct { mock.Mock + replayApplicationID int64 + replayConsensus model.Consensus + replayCount uint64 + replayCountError error + replayRecords []*model.ReplayRecord + replayListCalls int +} + +func (m *MockMachineRepository) ReplaySummary( + _ context.Context, + _ common.Address, + _ repository.ReplayVerificationLevel, +) (model.ReplaySummary, error) { + return model.ReplaySummary{ + ApplicationID: m.replayApplicationID, + ProcessedInputs: m.replayCount, + Consensus: m.replayConsensus, + }, m.replayCountError +} + +func (m *MockMachineRepository) ReplayPage( + _ context.Context, + request repository.ReplayPageRequest, +) ([]*model.ReplayRecord, error) { + m.replayListCalls++ + records := make([]*model.ReplayRecord, 0, min(request.Limit, uint64(len(m.replayRecords)))) + for _, record := range m.replayRecords { + index := record.Input.InputIndex + if index < request.FromInput || index >= request.ToInputExclusive { + continue + } + records = append(records, record) + if uint64(len(records)) == request.Limit { + break + } + } + return records, nil } func (m *MockMachineRepository) ListApplications( @@ -528,17 +1780,6 @@ func (m *MockMachineRepository) HasUndrainedEpochsBeforeBlock( return args.Bool(0), args.Error(1) } -func (m *MockMachineRepository) ListInputs( - ctx context.Context, - nameOrAddress string, - f repository.InputFilter, - p repository.Pagination, - descending bool, -) ([]*model.Input, uint64, error) { - args := m.Called(ctx, nameOrAddress, f, p, descending) - return args.Get(0).([]*model.Input), args.Get(1).(uint64), args.Error(2) -} - func (m *MockMachineRepository) GetLastSnapshot( ctx context.Context, nameOrAddress string) (*model.Input, error) { @@ -549,25 +1790,67 @@ func (m *MockMachineRepository) GetLastSnapshot( return args.Get(0).(*model.Input), args.Error(1) } +func (m *MockMachineRepository) GetApplication( + ctx context.Context, + nameOrAddress string, +) (*model.Application, error) { + args := m.Called(ctx, nameOrAddress) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).(*model.Application), args.Error(1) +} + +func (m *MockMachineRepository) UpdateApplicationStatus( + ctx context.Context, + appID int64, + status model.ApplicationStatus, + reason *string, +) error { + return m.Called(ctx, appID, status, reason).Error(0) +} + // ------------------------------------------------------------------------------------------------ // MockMachineInstanceFactory implements MachineInstanceFactory for testing. // It returns the same instance for every call, ignoring the app/path arguments. type MockMachineInstanceFactory struct { - Instance MachineInstance - Err error + Instance MachineInstance + SnapshotInstance MachineInstance + Err error + SnapshotErr error + TemplateFn func(*model.Application) (MachineInstance, error) + ReturnNilSnapshot bool + TemplateCalls int + SnapshotCalls int + TemplateChecks []bool } func (f *MockMachineInstanceFactory) NewFromTemplate( - _ context.Context, _ *model.Application, _ *slog.Logger, _ bool, + _ context.Context, app *model.Application, _ *slog.Logger, checkTemplateHash bool, ) (MachineInstance, error) { + f.TemplateCalls++ + f.TemplateChecks = append(f.TemplateChecks, checkTemplateHash) + if f.TemplateFn != nil { + return f.TemplateFn(app) + } return f.Instance, f.Err } func (f *MockMachineInstanceFactory) NewFromSnapshot( - _ context.Context, _ *model.Application, _ *slog.Logger, _ bool, - _ string, _ *common.Hash, _ uint64, + _ context.Context, _ *model.Application, _ *slog.Logger, + _ string, _ common.Hash, _ uint64, ) (MachineInstance, error) { + f.SnapshotCalls++ + if f.ReturnNilSnapshot { + return nil, nil + } + if f.SnapshotErr != nil { + return f.SnapshotInstance, f.SnapshotErr + } + if f.SnapshotInstance != nil { + return f.SnapshotInstance, nil + } return f.Instance, f.Err } @@ -580,25 +1863,34 @@ type realMachineInstanceFactory struct { } func (f *realMachineInstanceFactory) NewFromTemplate( - ctx context.Context, app *model.Application, logger *slog.Logger, checkHash bool, + ctx context.Context, app *model.Application, logger *slog.Logger, _ bool, ) (MachineInstance, error) { - return NewMachineInstanceWithFactory(ctx, app, 0, logger, checkHash, f.runtimeFactory) + return NewMachineInstanceWithFactory(ctx, app, 0, logger, f.runtimeFactory) } func (f *realMachineInstanceFactory) NewFromSnapshot( - ctx context.Context, app *model.Application, logger *slog.Logger, checkHash bool, - _ string, _ *common.Hash, inputIndex uint64, + ctx context.Context, app *model.Application, logger *slog.Logger, + _ string, _ common.Hash, inputIndex uint64, ) (MachineInstance, error) { - return NewMachineInstanceWithFactory(ctx, app, inputIndex+1, logger, checkHash, f.runtimeFactory) + if inputIndex == math.MaxUint64 { + return nil, ErrInvalidSnapshotPoint + } + return NewMachineInstanceWithFactory(ctx, app, inputIndex+1, logger, f.runtimeFactory) } // ------------------------------------------------------------------------------------------------ // DummyMachineInstanceMock implements the MachineInstance interface for testing type DummyMachineInstanceMock struct { - application *model.Application - closeError error - synchronizeErr error + application *model.Application + closeError error + hashError error + hashReturn [32]byte + replayErr error + closeCalls int + hashCalls int + processedInputs uint64 + replayCalls int } func (m *DummyMachineInstanceMock) Application() *model.Application { @@ -606,7 +1898,7 @@ func (m *DummyMachineInstanceMock) Application() *model.Application { } func (m *DummyMachineInstanceMock) ProcessedInputs() uint64 { - return 0 + return m.processedInputs } func (m *DummyMachineInstanceMock) OutputsProof(ctx context.Context) (*model.OutputsProof, error) { @@ -621,18 +1913,16 @@ func (m *DummyMachineInstanceMock) Inspect(_ context.Context, _ []byte) (*Inspec return nil, nil } -func (m *DummyMachineInstanceMock) Synchronize(_ context.Context, _ MachineRepository, _ uint64) error { - return m.synchronizeErr -} - func (m *DummyMachineInstanceMock) CreateSnapshot(_ context.Context, _ uint64, _ string) error { return nil } func (m *DummyMachineInstanceMock) Hash(_ context.Context) ([32]byte, error) { - return [32]byte{}, nil + m.hashCalls++ + return m.hashReturn, m.hashError } func (m *DummyMachineInstanceMock) Close() error { + m.closeCalls++ return m.closeError } diff --git a/internal/manager/types.go b/internal/manager/types.go index 272904c7c..2c03f26d5 100644 --- a/internal/manager/types.go +++ b/internal/manager/types.go @@ -26,7 +26,6 @@ type MachineInstance interface { Application() *Application Advance(ctx context.Context, input []byte, epochIndex uint64, inputIndex uint64, computeHashes bool) (*AdvanceResult, error) Inspect(ctx context.Context, query []byte) (*InspectResult, error) - Synchronize(ctx context.Context, repo MachineRepository, batchSize uint64) error CreateSnapshot(ctx context.Context, processedInputs uint64, path string) error ProcessedInputs() uint64 Hash(ctx context.Context) ([32]byte, error) From e22fed42a5b8140f214ce91341fbb8a2bf6333c0 Mon Sep 17 00:00:00 2001 From: Victor Fusco <1221933+vfusco@users.noreply.github.com> Date: Fri, 14 Aug 2026 10:42:52 -0300 Subject: [PATCH 12/13] fix(advancer): preserve application failure fences --- internal/advancer/advancer.go | 37 ++++---- internal/advancer/advancer_test.go | 124 +++++++++++++++++++++++++- internal/advancer/determinism_test.go | 8 +- internal/advancer/service.go | 9 +- internal/manager/types.go | 8 ++ 5 files changed, 164 insertions(+), 22 deletions(-) diff --git a/internal/advancer/advancer.go b/internal/advancer/advancer.go index 0599754f6..30f8aa5c6 100644 --- a/internal/advancer/advancer.go +++ b/internal/advancer/advancer.go @@ -81,18 +81,18 @@ func (s *Service) Step(ctx context.Context) (bool, error) { } // Update the machine manager with any new or disabled applications - err := s.machineManager.UpdateMachines(ctx) - if err != nil { - return false, err + updateErr := s.machineManager.UpdateMachines(ctx) + if updateErr != nil && !manager.IsOnlyApplicationFailurePersistenceErrors(updateErr) { + return false, updateErr } // Get all applications with active machines (returned sorted by ID). apps := s.machineManager.Applications() if len(apps) == 0 { - return false, nil + return false, updateErr } anyWork := false - var errs []error + errs := []error{updateErr} for _, app := range apps { hadWork, err := s.stepApp(ctx, app) if err != nil { @@ -266,13 +266,7 @@ func (s *Service) processInputs(ctx context.Context, app *Application, inputs [] "index", input.Index, "error", err) - if dbErr := appstatus.SetFailed(ctx, s.Logger, s.repository, app, err.Error()); dbErr != nil { - s.Logger.Error("Failed to persist FAILED status — machine will be closed "+ - "but the app status remains unchanged in DB; it may be re-created "+ - "from the last snapshot on the next tick. If the root cause "+ - "persists, this may loop.", - "application", app.Name, "db_error", dbErr) - } + s.markApplicationFailed(ctx, app, err.Error()) // Eagerly close the machine to release the child process. // The app has failed, so no further operations will succeed. @@ -354,6 +348,20 @@ func (s *Service) processInputs(ctx context.Context, app *Application, inputs [] return nil } +// markApplicationFailed persists FAILED or installs a local fence when the +// status write cannot be confirmed. Keeping those operations together prevents +// a failed application from being handed more work while durability is retried. +func (s *Service) markApplicationFailed(ctx context.Context, app *Application, reason string) { + if err := appstatus.SetFailed(ctx, s.Logger, s.repository, app, reason); err != nil { + s.machineManager.FenceApplicationFailure(app, reason) + s.Logger.Error( + "Could not persist FAILED application status; the application remains fenced until the write is retried", + "application", app.Name, + "db_error", err, + ) + } +} + func (s *Service) isEpochLastInput(ctx context.Context, app *Application, input *Input) (bool, error) { if app == nil || input == nil { return false, fmt.Errorf("application and input must not be nil") @@ -421,10 +429,7 @@ func (s *Service) handleEpochAfterInputsProcessed(ctx context.Context, app *Appl // If the runtime was destroyed (e.g., child process crashed), // mark the app as failed to avoid an infinite retry loop. if errors.Is(err, manager.ErrMachineClosed) { - if dbErr := appstatus.SetFailed(ctx, s.Logger, s.repository, app, err.Error()); dbErr != nil { - s.Logger.Error("Failed to persist FAILED status for crashed machine", - "application", app.Name, "db_error", dbErr) - } + s.markApplicationFailed(ctx, app, err.Error()) } return fmt.Errorf("failed to get outputs proof from machine: %w", err) } diff --git a/internal/advancer/advancer_test.go b/internal/advancer/advancer_test.go index 1d2d6545b..efa70f142 100644 --- a/internal/advancer/advancer_test.go +++ b/internal/advancer/advancer_test.go @@ -19,6 +19,7 @@ import ( "testing" "time" + "github.com/cartesi/rollups-node/internal/appstatus" "github.com/cartesi/rollups-node/internal/manager" . "github.com/cartesi/rollups-node/internal/model" "github.com/cartesi/rollups-node/internal/repository" @@ -118,6 +119,10 @@ func (s *AdvancerSuite) TestServiceInterface() { // Test service interface methods require.True(advancer.Alive()) require.True(advancer.Ready()) + machineManager.PendingApplicationFailures = true + require.False(advancer.Ready()) + machineManager.PendingApplicationFailures = false + require.True(advancer.Ready()) require.Empty(advancer.Reload()) require.Equal(advancer.Name, advancer.String()) @@ -224,6 +229,40 @@ func (s *AdvancerSuite) TestStep() { require.Contains(err.Error(), "update machines error") }) + s.Run("Error/UnconfirmedApplicationFailureStatus", func() { + require := s.Require() + persistenceErr := &manager.ApplicationFailurePersistenceError{ + ApplicationID: 7, + WriteErr: errors.New("status write unavailable"), + ReadErr: errors.New("status read unavailable"), + } + healthy := newMockMachine(8) + machineManager := &MockMachineManager{ + Map: map[int64]*MockMachineInstance{ + healthy.Application.ID: newMockInstance(healthy), + }, + UpdateMachinesError: persistenceErr, + PendingApplicationFailures: true, + } + repo := &MockRepository{ + GetEpochsReturn: map[common.Address][]*Epoch{ + healthy.Application.IApplicationAddress: {{Index: 0, Status: EpochStatus_Open}}, + }, + GetInputsReturn: map[common.Address][]*Input{ + healthy.Application.IApplicationAddress: { + newInput(healthy.Application.ID, 0, 0, marshal(randomAdvanceResult(0))), + }, + }, + } + advancer, err := newMockAdvancerService(machineManager, repo) + require.NoError(err) + + _, err = advancer.Step(context.Background()) + require.ErrorIs(err, manager.ErrApplicationFailureNotDurable) + require.Len(repo.StoredResults, 1, "the healthy application must still advance") + require.False(advancer.Ready()) + }) + s.Run("Error/GetInputs", func() { require := s.Require() env := s.setupOneApp() @@ -295,6 +334,48 @@ func (s *AdvancerSuite) TestStep() { // app2's input was processed despite app1's failure require.Len(repo.StoredResults, 1) }) + + s.Run("LiveCycleLimitWriteFailureFencesAppWithoutBlockingHealthySibling", func() { + require := s.Require() + mm := newMockMachineManager() + limited := newMockMachine(1) + healthy := newMockMachine(2) + limitErr := fmt.Errorf( + "advance execution reached configured cycle limit: %w", + pkgmachine.ErrReachedLimitMcycle, + ) + limited.AdvanceError = limitErr + mm.Map[limited.Application.ID] = newMockInstance(limited) + mm.Map[healthy.Application.ID] = newMockInstance(healthy) + repo := &MockRepository{ + GetEpochsReturn: map[common.Address][]*Epoch{ + limited.Application.IApplicationAddress: {{Index: 0, Status: EpochStatus_Open}}, + healthy.Application.IApplicationAddress: {{Index: 0, Status: EpochStatus_Open}}, + }, + GetInputsReturn: map[common.Address][]*Input{ + limited.Application.IApplicationAddress: { + newInput(limited.Application.ID, 0, 0, []byte("limited input")), + }, + healthy.Application.IApplicationAddress: { + newInput(healthy.Application.ID, 0, 0, marshal(randomAdvanceResult(0))), + }, + }, + UpdateApplicationStatusError: errors.New("FAILED write unavailable"), + } + svc, err := newMockAdvancerService(mm, repo) + require.NoError(err) + + _, err = svc.Step(context.Background()) + + require.ErrorIs(err, pkgmachine.ErrReachedLimitMcycle) + require.Equal( + appstatus.NormalizeReason(limitErr.Error()), + mm.RecordedApplicationFailures[limited.Application.ID], + ) + require.False(svc.Ready(), "an unconfirmed FAILED write must fail readiness immediately") + require.Len(repo.StoredResults, 1, "the healthy sibling must still advance") + require.Equal(healthy.Application.ID, repo.StoredAppIDs[0]) + }) } func (s *AdvancerSuite) TestGetUnprocessedInputs() { @@ -601,6 +682,11 @@ func (s *AdvancerSuite) TestContextCancellation() { // The expired context prevents the immediate database write, so the // application failure is queued for a later status-write retry. require.Zero(env.repo.ApplicationStatusUpdates) + require.Equal( + context.DeadlineExceeded.Error(), + env.mm.RecordedApplicationFailures[env.app.Application.ID], + ) + require.True(env.mm.PendingApplicationFailures) require.Equal(1, env.mm.Map[env.app.Application.ID].closeCalls) require.True(logs.contains(slog.LevelError, "Error executing advance")) require.False(logs.contains( @@ -893,6 +979,23 @@ func (s *AdvancerSuite) TestHandleEpochAfterInputsProcessed() { require.Equal(ApplicationStatus_Failed, env.repo.LastApplicationStatus) }) + s.Run("EmptyEpochIndex0ErrMachineClosedWriteFailureQueuesDurableFence", func() { + require := s.Require() + env := s.setupOneApp() + env.app.OutputsProofError = manager.ErrMachineClosed + env.repo.UpdateApplicationStatusError = errors.New("FAILED write unavailable") + epoch := &Epoch{Index: 0, Status: EpochStatus_Closed, InputIndexLowerBound: 0, InputIndexUpperBound: 0} + + err := env.service.handleEpochAfterInputsProcessed(context.Background(), env.app.Application, epoch) + + require.ErrorIs(err, manager.ErrMachineClosed) + require.Equal( + appstatus.NormalizeReason(manager.ErrMachineClosed.Error()), + env.mm.RecordedApplicationFailures[env.app.Application.ID], + ) + require.False(env.service.Ready()) + }) + s.Run("EmptyEpochIndexGt0RepeatsPreviousProof", func() { require := s.Require() env := s.setupOneApp() @@ -1957,13 +2060,16 @@ func newMockInstance(impl *MockMachineImpl) *MockMachineInstance { // ------------------------------------------------------------------------------------------------ type MockMachineManager struct { - Map map[int64]*MockMachineInstance - UpdateMachinesError error + Map map[int64]*MockMachineInstance + UpdateMachinesError error + PendingApplicationFailures bool + RecordedApplicationFailures map[int64]string } func newMockMachineManager() *MockMachineManager { return &MockMachineManager{ - Map: map[int64]*MockMachineInstance{}, + Map: map[int64]*MockMachineInstance{}, + RecordedApplicationFailures: map[int64]string{}, } } @@ -1979,6 +2085,14 @@ func (mock *MockMachineManager) UpdateMachines(ctx context.Context) error { return mock.UpdateMachinesError } +func (mock *MockMachineManager) FenceApplicationFailure(app *Application, reason string) { + if mock.RecordedApplicationFailures == nil { + mock.RecordedApplicationFailures = map[int64]string{} + } + mock.RecordedApplicationFailures[app.ID] = appstatus.NormalizeReason(reason) + mock.PendingApplicationFailures = true +} + func (mock *MockMachineManager) Applications() []*Application { apps := make([]*Application, 0, len(mock.Map)) for _, v := range mock.Map { @@ -1993,6 +2107,10 @@ func (mock *MockMachineManager) HasMachine(appID int64) bool { return exists } +func (mock *MockMachineManager) HasPendingApplicationFailures() bool { + return mock.PendingApplicationFailures +} + func (mock *MockMachineManager) Close() error { return nil } diff --git a/internal/advancer/determinism_test.go b/internal/advancer/determinism_test.go index 766478aff..eb8bf99ed 100644 --- a/internal/advancer/determinism_test.go +++ b/internal/advancer/determinism_test.go @@ -229,6 +229,12 @@ func testDeterminismCallerDeadlines( require.Equal(t, uint64(1), harness.instance.ProcessedInputs()) require.Zero(t, repo.ApplicationStatusUpdates, "the expired context prevents the immediate FAILED status write") + require.True(t, harness.provider.HasPendingApplicationFailures()) + require.Equal( + t, + context.DeadlineExceeded.Error(), + harness.provider.failureReason(harness.app.ID), + ) require.Equal(t, predecessor, harness.factory.base.snapshot()) require.True(t, harness.factory.base.isClosed(), "a timed-out advance must close the changed live machine") @@ -894,7 +900,7 @@ func (p *determinismMachineProvider) Applications() []*model.Application { func (p *determinismMachineProvider) UpdateMachines(context.Context) error { return nil } -func (p *determinismMachineProvider) RecordApplicationFailure(app *model.Application, reason string) { +func (p *determinismMachineProvider) FenceApplicationFailure(app *model.Application, reason string) { p.mu.Lock() defer p.mu.Unlock() if p.failures == nil { diff --git a/internal/advancer/service.go b/internal/advancer/service.go index 68ee3f9fd..6a3479a51 100644 --- a/internal/advancer/service.go +++ b/internal/advancer/service.go @@ -113,8 +113,13 @@ func Create(ctx context.Context, c *CreateInfo) (*Service, error) { } // Service interface implementation -func (s *Service) Alive() bool { return true } -func (s *Service) Ready() bool { return true } +func (s *Service) Alive() bool { return true } +func (s *Service) Ready() bool { + // This is a local fail-closed signal while application-failure durability + // is unresolved. It cannot atomically revoke work already selected by a + // separate process before that durable status becomes visible. + return s.machineManager != nil && !s.machineManager.HasPendingApplicationFailures() +} func (s *Service) Reload() []error { return nil } func (s *Service) Tick() []error { hadWork, err := s.Step(s.Context) diff --git a/internal/manager/types.go b/internal/manager/types.go index 2c03f26d5..91570fff8 100644 --- a/internal/manager/types.go +++ b/internal/manager/types.go @@ -44,9 +44,17 @@ type MachineProvider interface { // UpdateMachines refreshes the list of machines UpdateMachines(ctx context.Context) error + // FenceApplicationFailure fences an application whose initial FAILED + // status write could not be confirmed. It queues a later durability retry + // without duplicating the initial repository write. + FenceApplicationFailure(app *Application, reason string) + // HasMachine checks if a machine exists for the given application ID HasMachine(appID int64) bool + // HasPendingApplicationFailures reports an unresolved durable-status fence. + HasPendingApplicationFailures() bool + // Close shuts down all machine instances and releases resources Close() error } From 68b210f4cf54bdcbde09b3ea5e8028a432c05017 Mon Sep 17 00:00:00 2001 From: Victor Fusco <1221933+vfusco@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:22:35 -0300 Subject: [PATCH 13/13] build: isolate machine CGo artifacts --- Makefile | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 6a361709e..936749638 100644 --- a/Makefile +++ b/Makefile @@ -61,6 +61,12 @@ endif # Go artifacts GO_ARTIFACTS := $(addprefix cartesi-rollups-,node cli evm-reader advancer validator claimer jsonrpc-api prt machine-tool) +# These artifacts embed the machine runtime and therefore require libcartesi. +# Keep this list explicit: every other artifact is built with CGO_ENABLED=0, so +# the normal build fails if a C dependency leaks into a pure service or tool. +MACHINE_GO_ARTIFACTS := cartesi-rollups-node cartesi-rollups-advancer +PURE_GO_ARTIFACTS := $(filter-out $(MACHINE_GO_ARTIFACTS),$(GO_ARTIFACTS)) + # fixme(vfusco): path on all oses CGO_CFLAGS:= -I$(PREFIX)/include CGO_LDFLAGS:= -L$(PREFIX)/lib @@ -70,17 +76,25 @@ export CGO_LDFLAGS CARTESI_TEST_MACHINE_IMAGES_PATH:= $(PREFIX)/share/cartesi-machine/images/ export CARTESI_TEST_MACHINE_IMAGES_PATH -GO_BUILD_PARAMS := -ldflags "-s -w -X 'github.com/cartesi/rollups-node/internal/version.BuildVersion=$(ROLLUPS_NODE_VERSION)' -r $(PREFIX)/lib" +GO_VERSION_LDFLAGS := -s -w -X 'github.com/cartesi/rollups-node/internal/version.BuildVersion=$(ROLLUPS_NODE_VERSION)' +PURE_GO_BUILD_PARAMS := -ldflags "$(GO_VERSION_LDFLAGS)" +MACHINE_GO_BUILD_PARAMS := -ldflags "$(GO_VERSION_LDFLAGS) -r $(PREFIX)/lib" ifeq ($(BUILD_TYPE),debug) - GO_BUILD_PARAMS += -gcflags "all=-N -l" + PURE_GO_BUILD_PARAMS += -gcflags "all=-N -l" + MACHINE_GO_BUILD_PARAMS += -gcflags "all=-N -l" endif +# Tests and Go development tools cover machine packages, so retain the +# machine-capable parameters for those existing recipes. +GO_BUILD_PARAMS = $(MACHINE_GO_BUILD_PARAMS) + GO_TEST_PACKAGES ?= ./... GO_TEST_FLAGS ?= VERBOSE ?= ifeq ($(VERBOSE),true) - GO_BUILD_PARAMS += -v + PURE_GO_BUILD_PARAMS += -v + MACHINE_GO_BUILD_PARAMS += -v GO_TEST_FLAGS += -v endif @@ -147,9 +161,13 @@ env: # ============================================================================= # Artifacts # ============================================================================= -$(GO_ARTIFACTS): +$(PURE_GO_ARTIFACTS): + @echo "Building pure Go artifact $@" + CGO_ENABLED=0 go build $(PURE_GO_BUILD_PARAMS) ./cmd/$@ + +$(MACHINE_GO_ARTIFACTS): @echo "Building Go artifact $@" - go build $(GO_BUILD_PARAMS) ./cmd/$@ + CGO_ENABLED=1 go build $(MACHINE_GO_BUILD_PARAMS) ./cmd/$@ tidy-go: @go mod tidy