diff --git a/Makefile b/Makefile index 6a361709e..936749638 100644 --- a/Makefile +++ b/Makefile @@ -61,6 +61,12 @@ endif # Go artifacts GO_ARTIFACTS := $(addprefix cartesi-rollups-,node cli evm-reader advancer validator claimer jsonrpc-api prt machine-tool) +# These artifacts embed the machine runtime and therefore require libcartesi. +# Keep this list explicit: every other artifact is built with CGO_ENABLED=0, so +# the normal build fails if a C dependency leaks into a pure service or tool. +MACHINE_GO_ARTIFACTS := cartesi-rollups-node cartesi-rollups-advancer +PURE_GO_ARTIFACTS := $(filter-out $(MACHINE_GO_ARTIFACTS),$(GO_ARTIFACTS)) + # fixme(vfusco): path on all oses CGO_CFLAGS:= -I$(PREFIX)/include CGO_LDFLAGS:= -L$(PREFIX)/lib @@ -70,17 +76,25 @@ export CGO_LDFLAGS CARTESI_TEST_MACHINE_IMAGES_PATH:= $(PREFIX)/share/cartesi-machine/images/ export CARTESI_TEST_MACHINE_IMAGES_PATH -GO_BUILD_PARAMS := -ldflags "-s -w -X 'github.com/cartesi/rollups-node/internal/version.BuildVersion=$(ROLLUPS_NODE_VERSION)' -r $(PREFIX)/lib" +GO_VERSION_LDFLAGS := -s -w -X 'github.com/cartesi/rollups-node/internal/version.BuildVersion=$(ROLLUPS_NODE_VERSION)' +PURE_GO_BUILD_PARAMS := -ldflags "$(GO_VERSION_LDFLAGS)" +MACHINE_GO_BUILD_PARAMS := -ldflags "$(GO_VERSION_LDFLAGS) -r $(PREFIX)/lib" ifeq ($(BUILD_TYPE),debug) - GO_BUILD_PARAMS += -gcflags "all=-N -l" + PURE_GO_BUILD_PARAMS += -gcflags "all=-N -l" + MACHINE_GO_BUILD_PARAMS += -gcflags "all=-N -l" endif +# Tests and Go development tools cover machine packages, so retain the +# machine-capable parameters for those existing recipes. +GO_BUILD_PARAMS = $(MACHINE_GO_BUILD_PARAMS) + GO_TEST_PACKAGES ?= ./... GO_TEST_FLAGS ?= VERBOSE ?= ifeq ($(VERBOSE),true) - GO_BUILD_PARAMS += -v + PURE_GO_BUILD_PARAMS += -v + MACHINE_GO_BUILD_PARAMS += -v GO_TEST_FLAGS += -v endif @@ -147,9 +161,13 @@ env: # ============================================================================= # Artifacts # ============================================================================= -$(GO_ARTIFACTS): +$(PURE_GO_ARTIFACTS): + @echo "Building pure Go artifact $@" + CGO_ENABLED=0 go build $(PURE_GO_BUILD_PARAMS) ./cmd/$@ + +$(MACHINE_GO_ARTIFACTS): @echo "Building Go artifact $@" - go build $(GO_BUILD_PARAMS) ./cmd/$@ + CGO_ENABLED=1 go build $(MACHINE_GO_BUILD_PARAMS) ./cmd/$@ tidy-go: @go mod tidy diff --git a/api/openapi/inspect.yaml b/api/openapi/inspect.yaml index 65eeef0fc..2a88cf557 100644 --- a/api/openapi/inspect.yaml +++ b/api/openapi/inspect.yaml @@ -18,7 +18,7 @@ paths: description: | This POST method sends an inspect-state request to the DApp backend, using the body contents as a binary payload for the inspect method. - The response includes a status string and reports generated by the DApp backend. If an exception occurs, the `exception_payload` field will contain the exception details; otherwise, this field will be null. + The response includes the inspection completion status, reports generated by the DApp backend, and the number of previously processed inputs. The optional `error` field contains a sanitized description when the status is `Exception` or `Failed`. For `Exception`, `exception_data` contains the raw CMIO payload supplied by the guest. The inspect operation is executed on a temporary fork of the machine created upon request arrival, which is discarded afterward. Note that this method is synchronous and not recommended for resource-intensive operations. @@ -88,10 +88,17 @@ components: properties: status: $ref: "#/components/schemas/CompletionStatus" - exception_payload: - $ref: "#/components/schemas/Payload" + error: + type: string + description: Sanitized error description, present only when status is Exception or Failed + example: "The node could not complete the inspection" + exception_data: + allOf: + - $ref: "#/components/schemas/Payload" + description: Raw guest-provided CMIO exception payload, present only when status is Exception reports: type: array + description: Reports emitted before completion; for Failed, this may be only a partial prefix items: $ref: "#/components/schemas/Report" processed_input_count: @@ -100,22 +107,18 @@ components: example: 0 required: - status - - exception_payload - reports - processed_input_count CompletionStatus: type: string - description: Whether inspection completed or not (and why not) - enum: - [ - Accepted, - Rejected, - Exception, - MachineHalted, - CycleLimitExceeded, - TimeLimitExceeded, - ] + description: | + How inspection completed. MachineHalted means the temporary inspect + execution halted; the canonical application machine is unchanged. + Failed means the node could not complete inspection because of an + operational limit, cancellation, timeout, protocol error, or internal + failure. Reports returned with Failed may be partial. + enum: [Accepted, Rejected, Exception, MachineHalted, Failed] example: "Accepted" Payload: diff --git a/cmd/cartesi-rollups-cli/root/app/execution-parameters/execution_parameters.go b/cmd/cartesi-rollups-cli/root/app/execution-parameters/execution_parameters.go index fa5ad0f25..9f6330d34 100644 --- a/cmd/cartesi-rollups-cli/root/app/execution-parameters/execution_parameters.go +++ b/cmd/cartesi-rollups-cli/root/app/execution-parameters/execution_parameters.go @@ -449,17 +449,21 @@ func setParameterValue(params *model.ExecutionParameters, parameter, value strin } func printParameters(params *model.ExecutionParameters) { - fmt.Printf("snapshot_policy: %s\n", params.SnapshotPolicy) - fmt.Printf("advance_inc_cycles: %d\n", params.AdvanceIncCycles) - fmt.Printf("advance_max_cycles: %d\n", params.AdvanceMaxCycles) - fmt.Printf("inspect_inc_cycles: %d\n", params.InspectIncCycles) - fmt.Printf("inspect_max_cycles: %d\n", params.InspectMaxCycles) - fmt.Printf("advance_inc_deadline: %s\n", params.AdvanceIncDeadline) - fmt.Printf("advance_max_deadline: %s\n", params.AdvanceMaxDeadline) - fmt.Printf("inspect_inc_deadline: %s\n", params.InspectIncDeadline) - fmt.Printf("inspect_max_deadline: %s\n", params.InspectMaxDeadline) - fmt.Printf("load_deadline: %s\n", params.LoadDeadline) - fmt.Printf("store_deadline: %s\n", params.StoreDeadline) - fmt.Printf("fast_deadline: %s\n", params.FastDeadline) - fmt.Printf("max_concurrent_inspects: %d\n", params.MaxConcurrentInspects) + writeParameters(os.Stdout, params) +} + +func writeParameters(w io.Writer, params *model.ExecutionParameters) { + fmt.Fprintf(w, "snapshot_policy: %s\n", params.SnapshotPolicy) + fmt.Fprintf(w, "advance_inc_cycles: %d\n", params.AdvanceIncCycles) + fmt.Fprintf(w, "advance_max_cycles: %d\n", params.AdvanceMaxCycles) + fmt.Fprintf(w, "inspect_inc_cycles: %d\n", params.InspectIncCycles) + fmt.Fprintf(w, "inspect_max_cycles: %d\n", params.InspectMaxCycles) + fmt.Fprintf(w, "advance_inc_deadline: %s\n", params.AdvanceIncDeadline) + fmt.Fprintf(w, "advance_max_deadline: %s\n", params.AdvanceMaxDeadline) + fmt.Fprintf(w, "inspect_inc_deadline: %s\n", params.InspectIncDeadline) + fmt.Fprintf(w, "inspect_max_deadline: %s\n", params.InspectMaxDeadline) + fmt.Fprintf(w, "load_deadline: %s\n", params.LoadDeadline) + fmt.Fprintf(w, "store_deadline: %s\n", params.StoreDeadline) + fmt.Fprintf(w, "fast_deadline: %s\n", params.FastDeadline) + fmt.Fprintf(w, "max_concurrent_inspects: %d\n", params.MaxConcurrentInspects) } diff --git a/cmd/cartesi-rollups-cli/root/app/execution-parameters/execution_parameters_test.go b/cmd/cartesi-rollups-cli/root/app/execution-parameters/execution_parameters_test.go new file mode 100644 index 000000000..b261e81a1 --- /dev/null +++ b/cmd/cartesi-rollups-cli/root/app/execution-parameters/execution_parameters_test.go @@ -0,0 +1,38 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package execution + +import ( + "bytes" + "testing" + + "github.com/cartesi/rollups-node/internal/model" + "github.com/stretchr/testify/require" +) + +func TestCycleMaximumParameters(t *testing.T) { + params := &model.ExecutionParameters{} + require.NoError(t, setParameterValue(params, "advance_max_cycles", "123")) + require.NoError(t, setParameterValue(params, "inspect_max_cycles", "456")) + value, err := getParameterValue(params, "advance_max_cycles") + require.NoError(t, err) + require.Equal(t, "123", value) + value, err = getParameterValue(params, "inspect_max_cycles") + require.NoError(t, err) + require.Equal(t, "456", value) +} + +func TestWriteParametersIncludesCycleMaximums(t *testing.T) { + params := &model.ExecutionParameters{ + AdvanceIncCycles: 11, AdvanceMaxCycles: 12, + InspectIncCycles: 21, InspectMaxCycles: 22, + } + var output bytes.Buffer + writeParameters(&output, params) + + require.Contains(t, output.String(), "advance_inc_cycles: 11") + require.Contains(t, output.String(), "advance_max_cycles: 12") + require.Contains(t, output.String(), "inspect_inc_cycles: 21") + require.Contains(t, output.String(), "inspect_max_cycles: 22") +} diff --git a/internal/advancer/advancer.go b/internal/advancer/advancer.go index 8efec4f63..30f8aa5c6 100644 --- a/internal/advancer/advancer.go +++ b/internal/advancer/advancer.go @@ -81,18 +81,18 @@ func (s *Service) Step(ctx context.Context) (bool, error) { } // Update the machine manager with any new or disabled applications - err := s.machineManager.UpdateMachines(ctx) - if err != nil { - return false, err + updateErr := s.machineManager.UpdateMachines(ctx) + if updateErr != nil && !manager.IsOnlyApplicationFailurePersistenceErrors(updateErr) { + return false, updateErr } // Get all applications with active machines (returned sorted by ID). apps := s.machineManager.Applications() if len(apps) == 0 { - return false, nil + return false, updateErr } anyWork := false - var errs []error + errs := []error{updateErr} for _, app := range apps { hadWork, err := s.stepApp(ctx, app) if err != nil { @@ -248,33 +248,25 @@ func (s *Service) processInputs(ctx context.Context, app *Application, inputs [] result, err := machine.Advance(ctx, input.RawData, input.EpochIndex, input.Index, app.IsDaveConsensus()) input.RawData = nil // allow GC to collect payload while batch continues if err != nil { - // Graceful shutdown: bail out quietly without marking FAILED. - if errors.Is(err, context.Canceled) { - s.Logger.Debug("Advance cancelled due to shutdown", + // Cancellation of this service context is the normal shutdown path, + // so it does not change the application's status. A returned error + // that happens to include context.Canceled is still a failure while + // the service context remains active. + if errors.Is(ctx.Err(), context.Canceled) { + s.Logger.Debug("Advance stopped because the service is shutting down", "application", app.Name, - "index", input.Index) + "index", input.Index, + "error", err) return err } - // Anything else (including DeadlineExceeded) is a real failure. + // Anything else, including a deadline, is an execution failure. s.Logger.Error("Error executing advance", "application", app.Name, "index", input.Index, "error", err) - // DeadlineExceeded is a real failure but not a state-corruption - // signal — let the upper layer retry rather than marking FAILED. - if errors.Is(err, context.DeadlineExceeded) { - return err - } - - if dbErr := appstatus.SetFailed(ctx, s.Logger, s.repository, app, err.Error()); dbErr != nil { - s.Logger.Error("Failed to persist FAILED status — machine will be closed "+ - "but the app status remains unchanged in DB; it may be re-created "+ - "from the last snapshot on the next tick. If the root cause "+ - "persists, this may loop.", - "application", app.Name, "db_error", dbErr) - } + s.markApplicationFailed(ctx, app, err.Error()) // Eagerly close the machine to release the child process. // The app has failed, so no further operations will succeed. @@ -297,26 +289,41 @@ func (s *Service) processInputs(ctx context.Context, app *Application, inputs [] "status", result.Status, "outputs", len(result.Outputs), "reports", len(result.Reports), - "hashes", len(result.Hashes), - "remaining_cycles", result.RemainingMetaCycles, + "periodic_state_hashes", len(result.PeriodicStateHashes), + "padding_repetitions", result.PaddingRepetitions, ) // Store the result in the database err = s.repository.StoreAdvanceResult(ctx, input.EpochApplicationID, result) if err != nil { - // Machine state is now ahead of the database. This desync is - // unrecoverable without a restart — regardless of whether the - // failure was a DB error or a context timeout. Shut down the - // node so it can restart cleanly from the last snapshot. + // Advance has already changed the live machine, but the transaction + // did not confirm that its result was saved. The database may still + // show this input as pending. Reusing this machine could then execute + // the input again from the wrong state, so StoreAdvanceResult is not + // retried against this live machine. s.Logger.Error( - "FATAL: failed to store advance result after machine state "+ - "was already updated — shutting down to prevent permanent desync", + "Could not confirm that the advance result was saved; "+ + "the live machine has already advanced, so services will stop; "+ + "after the node is restarted, execution will use persisted state", "application", app.Name, "epoch", input.EpochIndex, "index", input.Index, "error", err) + + // Try to close the machine now so the already-advanced runtime cannot + // be used again. Cancel services even if Close fails. After the node + // is restarted, the machine is rebuilt from persisted state, and the + // database decides whether this input is still pending and needs a + // safe retry. + closeErr := machine.Close() s.Cancel() // triggers graceful shutdown of all services - return err + if closeErr != nil { + s.Logger.Error("Could not close the machine after its advance result "+ + "was not confirmed saved; service shutdown is still required", + "application", app.Name, + "error", closeErr) + } + return errors.Join(err, closeErr) } // Create a snapshot if needed @@ -341,6 +348,20 @@ func (s *Service) processInputs(ctx context.Context, app *Application, inputs [] return nil } +// markApplicationFailed persists FAILED or installs a local fence when the +// status write cannot be confirmed. Keeping those operations together prevents +// a failed application from being handed more work while durability is retried. +func (s *Service) markApplicationFailed(ctx context.Context, app *Application, reason string) { + if err := appstatus.SetFailed(ctx, s.Logger, s.repository, app, reason); err != nil { + s.machineManager.FenceApplicationFailure(app, reason) + s.Logger.Error( + "Could not persist FAILED application status; the application remains fenced until the write is retried", + "application", app.Name, + "db_error", err, + ) + } +} + func (s *Service) isEpochLastInput(ctx context.Context, app *Application, input *Input) (bool, error) { if app == nil || input == nil { return false, fmt.Errorf("application and input must not be nil") @@ -408,10 +429,7 @@ func (s *Service) handleEpochAfterInputsProcessed(ctx context.Context, app *Appl // If the runtime was destroyed (e.g., child process crashed), // mark the app as failed to avoid an infinite retry loop. if errors.Is(err, manager.ErrMachineClosed) { - if dbErr := appstatus.SetFailed(ctx, s.Logger, s.repository, app, err.Error()); dbErr != nil { - s.Logger.Error("Failed to persist FAILED status for crashed machine", - "application", app.Name, "db_error", dbErr) - } + s.markApplicationFailed(ctx, app, err.Error()) } return fmt.Errorf("failed to get outputs proof from machine: %w", err) } diff --git a/internal/advancer/advancer_test.go b/internal/advancer/advancer_test.go index 0f8c9aa7c..efa70f142 100644 --- a/internal/advancer/advancer_test.go +++ b/internal/advancer/advancer_test.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "io/fs" + "log/slog" mrand "math/rand" "os" "path/filepath" @@ -18,10 +19,12 @@ import ( "testing" "time" + "github.com/cartesi/rollups-node/internal/appstatus" "github.com/cartesi/rollups-node/internal/manager" . "github.com/cartesi/rollups-node/internal/model" "github.com/cartesi/rollups-node/internal/repository" "github.com/cartesi/rollups-node/internal/repository/repotest" + pkgmachine "github.com/cartesi/rollups-node/pkg/machine" "github.com/cartesi/rollups-node/pkg/service" "github.com/ethereum/go-ethereum/common" @@ -34,6 +37,32 @@ func TestAdvancer(t *testing.T) { type AdvancerSuite struct{ suite.Suite } +type advancerLogCapture struct { + mu sync.Mutex + records []slog.Record +} + +func (h *advancerLogCapture) Enabled(context.Context, slog.Level) bool { return true } +func (h *advancerLogCapture) Handle(_ context.Context, record slog.Record) error { + h.mu.Lock() + defer h.mu.Unlock() + h.records = append(h.records, record.Clone()) + return nil +} +func (h *advancerLogCapture) WithAttrs([]slog.Attr) slog.Handler { return h } +func (h *advancerLogCapture) WithGroup(string) slog.Handler { return h } + +func (h *advancerLogCapture) contains(level slog.Level, message string) bool { + h.mu.Lock() + defer h.mu.Unlock() + for _, record := range h.records { + if record.Level == level && record.Message == message { + return true + } + } + return false +} + func newMockAdvancerService(machineManager *MockMachineManager, repo *MockRepository) (*Service, error) { return newMockAdvancerServiceWithBatchSize(machineManager, repo, 500) } @@ -90,6 +119,10 @@ func (s *AdvancerSuite) TestServiceInterface() { // Test service interface methods require.True(advancer.Alive()) require.True(advancer.Ready()) + machineManager.PendingApplicationFailures = true + require.False(advancer.Ready()) + machineManager.PendingApplicationFailures = false + require.True(advancer.Ready()) require.Empty(advancer.Reload()) require.Equal(advancer.Name, advancer.String()) @@ -196,6 +229,40 @@ func (s *AdvancerSuite) TestStep() { require.Contains(err.Error(), "update machines error") }) + s.Run("Error/UnconfirmedApplicationFailureStatus", func() { + require := s.Require() + persistenceErr := &manager.ApplicationFailurePersistenceError{ + ApplicationID: 7, + WriteErr: errors.New("status write unavailable"), + ReadErr: errors.New("status read unavailable"), + } + healthy := newMockMachine(8) + machineManager := &MockMachineManager{ + Map: map[int64]*MockMachineInstance{ + healthy.Application.ID: newMockInstance(healthy), + }, + UpdateMachinesError: persistenceErr, + PendingApplicationFailures: true, + } + repo := &MockRepository{ + GetEpochsReturn: map[common.Address][]*Epoch{ + healthy.Application.IApplicationAddress: {{Index: 0, Status: EpochStatus_Open}}, + }, + GetInputsReturn: map[common.Address][]*Input{ + healthy.Application.IApplicationAddress: { + newInput(healthy.Application.ID, 0, 0, marshal(randomAdvanceResult(0))), + }, + }, + } + advancer, err := newMockAdvancerService(machineManager, repo) + require.NoError(err) + + _, err = advancer.Step(context.Background()) + require.ErrorIs(err, manager.ErrApplicationFailureNotDurable) + require.Len(repo.StoredResults, 1, "the healthy application must still advance") + require.False(advancer.Ready()) + }) + s.Run("Error/GetInputs", func() { require := s.Require() env := s.setupOneApp() @@ -267,6 +334,48 @@ func (s *AdvancerSuite) TestStep() { // app2's input was processed despite app1's failure require.Len(repo.StoredResults, 1) }) + + s.Run("LiveCycleLimitWriteFailureFencesAppWithoutBlockingHealthySibling", func() { + require := s.Require() + mm := newMockMachineManager() + limited := newMockMachine(1) + healthy := newMockMachine(2) + limitErr := fmt.Errorf( + "advance execution reached configured cycle limit: %w", + pkgmachine.ErrReachedLimitMcycle, + ) + limited.AdvanceError = limitErr + mm.Map[limited.Application.ID] = newMockInstance(limited) + mm.Map[healthy.Application.ID] = newMockInstance(healthy) + repo := &MockRepository{ + GetEpochsReturn: map[common.Address][]*Epoch{ + limited.Application.IApplicationAddress: {{Index: 0, Status: EpochStatus_Open}}, + healthy.Application.IApplicationAddress: {{Index: 0, Status: EpochStatus_Open}}, + }, + GetInputsReturn: map[common.Address][]*Input{ + limited.Application.IApplicationAddress: { + newInput(limited.Application.ID, 0, 0, []byte("limited input")), + }, + healthy.Application.IApplicationAddress: { + newInput(healthy.Application.ID, 0, 0, marshal(randomAdvanceResult(0))), + }, + }, + UpdateApplicationStatusError: errors.New("FAILED write unavailable"), + } + svc, err := newMockAdvancerService(mm, repo) + require.NoError(err) + + _, err = svc.Step(context.Background()) + + require.ErrorIs(err, pkgmachine.ErrReachedLimitMcycle) + require.Equal( + appstatus.NormalizeReason(limitErr.Error()), + mm.RecordedApplicationFailures[limited.Application.ID], + ) + require.False(svc.Ready(), "an unconfirmed FAILED write must fail readiness immediately") + require.Len(repo.StoredResults, 1, "the healthy sibling must still advance") + require.Equal(healthy.Application.ID, repo.StoredAppIDs[0]) + }) } func (s *AdvancerSuite) TestGetUnprocessedInputs() { @@ -336,6 +445,37 @@ func (s *AdvancerSuite) TestProcess() { require.Contains(err.Error(), "advance error") }) + s.Run("IncompleteAdvanceMarksFailedWithoutCanonicalResult", func() { + interruptions := []struct { + name string + err error + }{ + {"InternalWatchdog", pkgmachine.ErrDeadlineExceeded}, + {"OutputLimit", pkgmachine.ErrOutputsLimitExceeded}, + {"ReportLimit", pkgmachine.ErrReportsLimitExceeded}, + {"PayloadLimit", pkgmachine.ErrPayloadLengthLimitExceeded}, + {"CycleLimit", pkgmachine.ErrReachedLimitMcycle}, + } + + for _, interruption := range interruptions { + s.Run(interruption.name, func() { + require := s.Require() + env := s.setupOneApp() + env.app.AdvanceError = interruption.err + inputs := []*Input{ + newInput(env.app.Application.ID, 0, 0, []byte("input")), + } + + err := env.service.processInputs(context.Background(), env.app.Application, inputs) + require.ErrorIs(err, interruption.err) + require.Empty(env.repo.StoredResults) + require.Equal(1, env.repo.ApplicationStatusUpdates) + require.Equal(ApplicationStatus_Failed, env.repo.LastApplicationStatus) + require.Equal(1, env.mm.Map[env.app.Application.ID].closeCalls) + }) + } + }) + s.Run("Ok", func() { require := s.Require() env := s.setupOneApp() @@ -394,20 +534,61 @@ func (s *AdvancerSuite) TestProcess() { s.Run("StoreAdvance", func() { require := s.Require() env := s.setupOneApp() + pending := newInput( + env.app.Application.ID, 0, 0, marshal(randomAdvanceResult(0)), + ) inputs := []*Input{ - newInput(env.app.Application.ID, 0, 0, marshal(randomAdvanceResult(0))), + pending, newInput(env.app.Application.ID, 0, 1, []byte("unreachable")), } + address := env.app.Application.IApplicationAddress + env.repo.GetInputsReturn = map[common.Address][]*Input{ + address: {pending}, + } env.repo.StoreAdvanceError = errors.New("store-advance error") err := env.service.processInputs(context.Background(), env.app.Application, inputs) require.Error(err) require.Contains(err.Error(), "store-advance error") - require.Len(env.repo.StoredResults, 1) + require.Empty(env.repo.StoredResults) + require.Empty(env.repo.StoredAppIDs) + require.Equal([]*Input{pending}, env.repo.GetInputsReturn[address], + "an atomic store failure must leave the input pending") // Verify that the node shutdown was triggered (context cancelled) require.Error(env.service.Context.Err(), "shared context should be cancelled") }) + + s.Run("StoreAdvanceCommitResponseLost", func() { + require := s.Require() + env := s.setupOneApp() + pending := newInput( + env.app.Application.ID, 0, 0, marshal(randomAdvanceResult(0)), + ) + address := env.app.Application.IApplicationAddress + env.repo.GetInputsReturn = map[common.Address][]*Input{ + address: {pending}, + } + env.repo.StoreAdvanceCommitError = errors.New("commit response lost") + + err := env.service.processInputs( + context.Background(), env.app.Application, []*Input{pending}, + ) + require.ErrorContains(err, "commit response lost") + require.Len(env.repo.StoredResults, 1) + require.Equal([]int64{env.app.Application.ID}, env.repo.StoredAppIDs) + require.Empty(env.repo.GetInputsReturn[address], + "a committed result and its input cursor must advance atomically") + require.Error(env.service.Context.Err(), "shared context should be cancelled") + require.Equal(1, env.mm.Map[env.app.Application.ID].closeCalls) + + unprocessed, _, listErr := getUnprocessedInputs( + context.Background(), env.repo, address.String(), pending.EpochIndex, 500, + ) + require.NoError(listErr) + require.Empty(unprocessed, + "a restart must not execute an input whose result already committed") + }) }) } @@ -436,6 +617,9 @@ func (s *AdvancerSuite) TestContextCancellation() { case err := <-errCh: require.Error(err) require.ErrorIs(err, context.Canceled) + require.Empty(env.repo.StoredResults) + require.Zero(env.repo.ApplicationStatusUpdates) + require.Zero(env.mm.Map[env.app.Application.ID].closeCalls) case <-time.After(100 * time.Millisecond): require.Fail("Step operation did not respect context cancellation") } @@ -445,6 +629,8 @@ func (s *AdvancerSuite) TestContextCancellation() { require := s.Require() env := s.setupOneApp() env.app.AdvanceBlock = true + logs := &advancerLogCapture{} + env.service.Logger = slog.New(logs) inputs := []*Input{ newInput(env.app.Application.ID, 0, 0, marshal(randomAdvanceResult(0))), @@ -465,10 +651,89 @@ func (s *AdvancerSuite) TestContextCancellation() { case err := <-errCh: require.Error(err) require.ErrorIs(err, context.Canceled) + require.Empty(env.repo.StoredResults) + require.Zero(env.repo.ApplicationStatusUpdates) + require.Zero(env.mm.Map[env.app.Application.ID].closeCalls) + require.True(logs.contains( + slog.LevelDebug, + "Advance stopped because the service is shutting down", + )) + require.False(logs.contains(slog.LevelError, "Error executing advance")) case <-time.After(100 * time.Millisecond): require.Fail("processInputs operation did not respect context cancellation") } }) + + s.Run("DeadlineDuringProcessInputs", func() { + require := s.Require() + env := s.setupOneApp() + env.app.AdvanceBlock = true + logs := &advancerLogCapture{} + env.service.Logger = slog.New(logs) + inputs := []*Input{ + newInput(env.app.Application.ID, 0, 0, marshal(randomAdvanceResult(0))), + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + + err := env.service.processInputs(ctx, env.app.Application, inputs) + require.ErrorIs(err, context.DeadlineExceeded) + require.Empty(env.repo.StoredResults) + // The expired context prevents the immediate database write, so the + // application failure is queued for a later status-write retry. + require.Zero(env.repo.ApplicationStatusUpdates) + require.Equal( + context.DeadlineExceeded.Error(), + env.mm.RecordedApplicationFailures[env.app.Application.ID], + ) + require.True(env.mm.PendingApplicationFailures) + require.Equal(1, env.mm.Map[env.app.Application.ID].closeCalls) + require.True(logs.contains(slog.LevelError, "Error executing advance")) + require.False(logs.contains( + slog.LevelDebug, + "Advance stopped because the service is shutting down", + )) + }) + + for _, test := range []struct { + name string + err error + }{ + { + name: "LiveContextWithJoinedDeadlineAndCancellation", + err: errors.Join(context.DeadlineExceeded, context.Canceled), + }, + { + name: "LiveContextWithJoinedMachineFailureAndCancellation", + err: errors.Join(pkgmachine.ErrMachineInternal, context.Canceled), + }, + } { + s.Run(test.name, func() { + require := s.Require() + env := s.setupOneApp() + env.app.AdvanceError = test.err + logs := &advancerLogCapture{} + env.service.Logger = slog.New(logs) + inputs := []*Input{ + newInput(env.app.Application.ID, 0, 0, []byte("input")), + } + + ctx := context.Background() + err := env.service.processInputs(ctx, env.app.Application, inputs) + + require.ErrorIs(err, context.Canceled) + require.NoError(ctx.Err(), "the caller context must remain active") + require.Empty(env.repo.StoredResults) + require.Equal(1, env.repo.ApplicationStatusUpdates) + require.Equal(ApplicationStatus_Failed, env.repo.LastApplicationStatus) + require.Equal(1, env.mm.Map[env.app.Application.ID].closeCalls) + require.True(logs.contains(slog.LevelError, "Error executing advance")) + require.False(logs.contains( + slog.LevelDebug, + "Advance stopped because the service is shutting down", + )) + }) + } } // TestLargeNumberOfInputs how the advancer handles large volumes of inputs @@ -504,7 +769,11 @@ func (s *AdvancerSuite) TestErrorRecovery() { err := env.service.processInputs(context.Background(), env.app.Application, inputs) require.Error(err) require.Contains(err.Error(), "temporary failure") + require.Empty(env.repo.StoredResults) + require.Empty(env.repo.StoredAppIDs) require.Error(env.service.Context.Err(), "shared context should be cancelled") + require.Equal(1, env.mm.Map[env.app.Application.ID].closeCalls, + "the already-advanced machine must be closed before processInputs returns") }) } @@ -710,6 +979,23 @@ func (s *AdvancerSuite) TestHandleEpochAfterInputsProcessed() { require.Equal(ApplicationStatus_Failed, env.repo.LastApplicationStatus) }) + s.Run("EmptyEpochIndex0ErrMachineClosedWriteFailureQueuesDurableFence", func() { + require := s.Require() + env := s.setupOneApp() + env.app.OutputsProofError = manager.ErrMachineClosed + env.repo.UpdateApplicationStatusError = errors.New("FAILED write unavailable") + epoch := &Epoch{Index: 0, Status: EpochStatus_Closed, InputIndexLowerBound: 0, InputIndexUpperBound: 0} + + err := env.service.handleEpochAfterInputsProcessed(context.Background(), env.app.Application, epoch) + + require.ErrorIs(err, manager.ErrMachineClosed) + require.Equal( + appstatus.NormalizeReason(manager.ErrMachineClosed.Error()), + env.mm.RecordedApplicationFailures[env.app.Application.ID], + ) + require.False(env.service.Ready()) + }) + s.Run("EmptyEpochIndexGt0RepeatsPreviousProof", func() { require := s.Require() env := s.setupOneApp() @@ -1774,13 +2060,16 @@ func newMockInstance(impl *MockMachineImpl) *MockMachineInstance { // ------------------------------------------------------------------------------------------------ type MockMachineManager struct { - Map map[int64]*MockMachineInstance - UpdateMachinesError error + Map map[int64]*MockMachineInstance + UpdateMachinesError error + PendingApplicationFailures bool + RecordedApplicationFailures map[int64]string } func newMockMachineManager() *MockMachineManager { return &MockMachineManager{ - Map: map[int64]*MockMachineInstance{}, + Map: map[int64]*MockMachineInstance{}, + RecordedApplicationFailures: map[int64]string{}, } } @@ -1796,6 +2085,14 @@ func (mock *MockMachineManager) UpdateMachines(ctx context.Context) error { return mock.UpdateMachinesError } +func (mock *MockMachineManager) FenceApplicationFailure(app *Application, reason string) { + if mock.RecordedApplicationFailures == nil { + mock.RecordedApplicationFailures = map[int64]string{} + } + mock.RecordedApplicationFailures[app.ID] = appstatus.NormalizeReason(reason) + mock.PendingApplicationFailures = true +} + func (mock *MockMachineManager) Applications() []*Application { apps := make([]*Application, 0, len(mock.Map)) for _, v := range mock.Map { @@ -1810,6 +2107,10 @@ func (mock *MockMachineManager) HasMachine(appID int64) bool { return exists } +func (mock *MockMachineManager) HasPendingApplicationFailures() bool { + return mock.PendingApplicationFailures +} + func (mock *MockMachineManager) Close() error { return nil } @@ -1819,6 +2120,7 @@ type MockMachineInstance struct { application *Application machineImpl *MockMachineImpl createSnapshotError error + closeCalls int } // Advance implements the MachineInstance interface for testing @@ -1827,7 +2129,7 @@ func (m *MockMachineInstance) Advance(ctx context.Context, input []byte, epochIn } // Inspect implements the MachineInstance interface for testing -func (m *MockMachineInstance) Inspect(ctx context.Context, query []byte) (*InspectResult, error) { +func (m *MockMachineInstance) Inspect(ctx context.Context, query []byte) (*manager.InspectResult, error) { // Not used in advancer tests, but needed to satisfy the interface return nil, nil } @@ -1851,12 +2153,6 @@ func (m *MockMachineInstance) OutputsProof(ctx context.Context) (*OutputsProof, }, nil } -// Synchronize implements the MachineInstance interface for testing -func (m *MockMachineInstance) Synchronize(ctx context.Context, repo manager.MachineRepository, batchSize uint64) error { - // Not used in advancer tests, but needed to satisfy the interface - return nil -} - // CreateSnapshot implements the MachineInstance interface for testing func (m *MockMachineInstance) CreateSnapshot(ctx context.Context, processInputs uint64, path string) error { return m.createSnapshotError @@ -1870,7 +2166,7 @@ func (m *MockMachineInstance) Hash(ctx context.Context) ([32]byte, error) { // Close implements the MachineInstance interface for testing func (m *MockMachineInstance) Close() error { - // Not used in advancer tests, but needed to satisfy the interface + m.closeCalls++ return nil } @@ -1884,6 +2180,7 @@ type MockRepository struct { GetInputsError error GetInputsBlock bool StoreAdvanceError error + StoreAdvanceCommitError error StoreAdvanceFailCount int UpdateApplicationStatusError error UpdateEpochsError error @@ -2004,27 +2301,33 @@ func (mock *MockRepository) StoreAdvanceResult( return errors.New("temporary failure") } + // Model an atomic transaction failure before commit: neither the canonical + // result nor the pending-input cursor changes. + if mock.StoreAdvanceError != nil { + return mock.StoreAdvanceError + } + mock.StoredResults = append(mock.StoredResults, res) mock.StoredAppIDs = append(mock.StoredAppIDs, appID) // Simulate real behavior: processed inputs change status and are no longer // returned by queries filtering for unprocessed (Status_None) inputs. // This prevents infinite loops in batched fetching. - if mock.StoreAdvanceError == nil { - for addr, inputs := range mock.GetInputsReturn { - for i, inp := range inputs { - if inp.EpochApplicationID == appID && inp.Index == res.InputIndex { - newInputs := make([]*Input, 0, len(inputs)-1) - newInputs = append(newInputs, inputs[:i]...) - newInputs = append(newInputs, inputs[i+1:]...) - mock.GetInputsReturn[addr] = newInputs - break - } + for addr, inputs := range mock.GetInputsReturn { + for i, inp := range inputs { + if inp.EpochApplicationID == appID && inp.Index == res.InputIndex { + newInputs := make([]*Input, 0, len(inputs)-1) + newInputs = append(newInputs, inputs[:i]...) + newInputs = append(newInputs, inputs[i+1:]...) + mock.GetInputsReturn[addr] = newInputs + break } } } - return mock.StoreAdvanceError + // Model a committed transaction whose success response was lost. The + // canonical result and pending-input cursor changed together. + return mock.StoreAdvanceCommitError } func (mock *MockRepository) UpdateEpochOutputsProof(ctx context.Context, appID int64, epochIndex uint64, proof *OutputsProof) error { diff --git a/internal/advancer/determinism_test.go b/internal/advancer/determinism_test.go new file mode 100644 index 000000000..eb8bf99ed --- /dev/null +++ b/internal/advancer/determinism_test.go @@ -0,0 +1,957 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package advancer + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "io" + "log/slog" + "sync" + "testing" + "time" + + "github.com/cartesi/rollups-node/internal/manager" + "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/pkg/machine" + pkgservice "github.com/cartesi/rollups-node/pkg/service" + "github.com/ethereum/go-ethereum/common" + "github.com/stretchr/testify/require" +) + +const determinismWaitTimeout = 5 * time.Second + +var errDeterminismRuntimeClosed = errors.New("determinism runtime is closed") + +func TestProcessInputs_RetryFromNonzeroPredecessorMatchesUninterruptedResult(t *testing.T) { + tests := []struct { + name string + targetPayload []byte + wantStatus model.InputCompletionStatus + }{ + { + name: "accepted", + targetPayload: []byte("deposit:alice:17"), + wantStatus: model.InputCompletionStatus_Accepted, + }, + { + name: "rejected", + targetPayload: []byte("reject:malformed-deposit"), + wantStatus: model.InputCompletionStatus_Rejected, + }, + { + name: "exception", + targetPayload: []byte("exception:application-error"), + wantStatus: model.InputCompletionStatus_Exception, + }, + { + name: "machine halted", + targetPayload: []byte("halt:application-finished"), + wantStatus: model.InputCompletionStatus_MachineHalted, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + prefix, predecessor, target := determinismBaseline(t, tt.targetPayload) + requireDeterminismTarget(t, prefix, target, tt.targetPayload, tt.wantStatus) + + t.Run("pre-canceled calls do not fork", func(t *testing.T) { + testDeterminismEarlyInterruptions(t, prefix, target, tt.targetPayload) + }) + t.Run("caller cancellation after mutation", func(t *testing.T) { + testDeterminismCallerInterruptions( + t, prefix, target, tt.targetPayload, context.Canceled, 1, + ) + }) + t.Run("caller deadline reconstructs predecessor across retries", func(t *testing.T) { + testDeterminismCallerDeadlines( + t, prefix, predecessor, target, tt.targetPayload, 3, + ) + }) + t.Run("infrastructure retry reconstructs predecessor", func(t *testing.T) { + testDeterminismInfrastructureRetries( + t, prefix, predecessor, target, tt.targetPayload, 2, + ) + }) + for _, deadline := range []time.Duration{100 * time.Millisecond, 250 * time.Millisecond} { + t.Run("manager watchdog "+deadline.String(), func(t *testing.T) { + testDeterminismManagerWatchdog( + t, prefix, predecessor, target, tt.targetPayload, deadline, + ) + }) + } + }) + } +} + +func determinismBaseline( + t *testing.T, + targetPayload []byte, +) (*model.AdvanceResult, determinismMachineState, *model.AdvanceResult) { + t.Helper() + repo := &MockRepository{} + harness := newTemplateDeterminismHarness( + t, repo, determinismWaitTimeout, + determinismAdvanceSuccess, + determinismAdvanceSuccess, + ) + + require.NoError(t, harness.process(context.Background(), 0, []byte("deposit:prefix:11"))) + require.Len(t, repo.StoredResults, 1) + prefix := cloneDeterminismResult(repo.StoredResults[0]) + predecessor := harness.liveState(t) + require.Equal(t, uint64(1), predecessor.step) + + require.NoError(t, harness.process(context.Background(), 1, targetPayload)) + require.Len(t, repo.StoredResults, 2) + target := cloneDeterminismResult(repo.StoredResults[1]) + require.Equal(t, uint64(2), harness.instance.ProcessedInputs()) + require.Equal(t, machine.Hash(target.MachineHash), harness.runtimeHash(t)) + return prefix, predecessor, target +} + +func testDeterminismEarlyInterruptions( + t *testing.T, + wantPrefix *model.AdvanceResult, + wantTarget *model.AdvanceResult, + targetPayload []byte, +) { + t.Helper() + repo := &MockRepository{} + harness := newTemplateDeterminismHarness( + t, repo, determinismWaitTimeout, + determinismAdvanceSuccess, + determinismAdvanceSuccess, + ) + require.NoError(t, harness.process(context.Background(), 0, []byte("deposit:prefix:11"))) + require.Equal(t, wantPrefix, repo.StoredResults[0]) + predecessor := harness.liveRuntime(t) + predecessorState := predecessor.snapshot() + forksBefore := harness.factory.forkCount() + + canceled, cancel := context.WithCancel(context.Background()) + cancel() + require.ErrorIs(t, harness.process(canceled, 1, targetPayload), context.Canceled) + + expired, cancelDeadline := context.WithDeadline( + context.Background(), time.Now().Add(-time.Second), + ) + defer cancelDeadline() + require.ErrorIs(t, harness.process(expired, 1, targetPayload), context.DeadlineExceeded) + + require.Equal(t, forksBefore, harness.factory.forkCount(), + "an already-finished caller context must be rejected before Fork") + requireCanonicalPrefix(t, repo, wantPrefix) + require.Equal(t, uint64(1), harness.instance.ProcessedInputs()) + require.False(t, predecessor.isClosed()) + require.Equal(t, predecessorState, harness.liveState(t)) + + require.NoError(t, harness.process(context.Background(), 1, targetPayload)) + requireCanonicalHistory(t, repo, wantPrefix, wantTarget) + requireSuccessfulRetryState(t, harness, predecessor, wantTarget) +} + +func testDeterminismCallerInterruptions( + t *testing.T, + wantPrefix *model.AdvanceResult, + wantTarget *model.AdvanceResult, + targetPayload []byte, + wantErr error, + attempts int, +) { + t.Helper() + behaviors := []determinismAdvanceBehavior{determinismAdvanceSuccess} + for range attempts { + behaviors = append(behaviors, determinismAdvanceWaitForContext) + } + behaviors = append(behaviors, determinismAdvanceSuccess) + repo := &MockRepository{} + harness := newTemplateDeterminismHarness( + t, repo, determinismWaitTimeout, behaviors..., + ) + require.NoError(t, harness.process(context.Background(), 0, []byte("deposit:prefix:11"))) + require.Equal(t, wantPrefix, repo.StoredResults[0]) + predecessor := harness.liveRuntime(t) + predecessorState := predecessor.snapshot() + + for range attempts { + controlled := newDeterminismContext(context.Background()) + defer controlled.finish(wantErr) + errCh := make(chan error, 1) + go func() { errCh <- harness.process(controlled, 1, targetPayload) }() + + fork := harness.waitForMutation(t) + controlled.finish(wantErr) + require.ErrorIs(t, waitDeterminismError(t, errCh), wantErr) + requireCanonicalPrefix(t, repo, wantPrefix) + require.Equal(t, uint64(1), harness.instance.ProcessedInputs()) + require.Zero(t, repo.ApplicationStatusUpdates, + "caller-owned interruptions must not mark the application failed") + require.False(t, predecessor.isClosed(), + "caller interruption must leave the reusable predecessor open") + require.Equal(t, predecessorState, harness.liveState(t)) + requireDiscardedMutatedFork(t, fork, predecessorState) + } + + require.NoError(t, harness.process(context.Background(), 1, targetPayload)) + requireCanonicalHistory(t, repo, wantPrefix, wantTarget) + requireSuccessfulRetryState(t, harness, predecessor, wantTarget) +} + +func testDeterminismCallerDeadlines( + t *testing.T, + wantPrefix *model.AdvanceResult, + predecessor determinismMachineState, + wantTarget *model.AdvanceResult, + targetPayload []byte, + attempts int, +) { + t.Helper() + repo := repositoryWithDeterminismPrefix(wantPrefix) + + for range attempts { + harness := newSnapshotDeterminismHarness( + t, repo, predecessor, determinismWaitTimeout, + determinismAdvanceWaitForContext, + ) + controlled := newDeterminismContext(context.Background()) + errCh := make(chan error, 1) + go func() { errCh <- harness.process(controlled, 1, targetPayload) }() + + fork := harness.waitForMutation(t) + controlled.finish(context.DeadlineExceeded) + require.ErrorIs(t, waitDeterminismError(t, errCh), context.DeadlineExceeded) + requireCanonicalPrefix(t, repo, wantPrefix) + require.Equal(t, uint64(1), harness.instance.ProcessedInputs()) + require.Zero(t, repo.ApplicationStatusUpdates, + "the expired context prevents the immediate FAILED status write") + require.True(t, harness.provider.HasPendingApplicationFailures()) + require.Equal( + t, + context.DeadlineExceeded.Error(), + harness.provider.failureReason(harness.app.ID), + ) + require.Equal(t, predecessor, harness.factory.base.snapshot()) + require.True(t, harness.factory.base.isClosed(), + "a timed-out advance must close the changed live machine") + _, err := harness.factory.base.Fork(context.Background()) + require.ErrorIs(t, err, errDeterminismRuntimeClosed) + requireDiscardedMutatedFork(t, fork, predecessor) + } + + recovered := newSnapshotDeterminismHarness( + t, repo, predecessor, determinismWaitTimeout, determinismAdvanceSuccess, + ) + require.NoError(t, recovered.process(context.Background(), 1, targetPayload)) + requireCanonicalHistory(t, repo, wantPrefix, wantTarget) + requireSuccessfulRetryState(t, recovered, recovered.factory.base, wantTarget) +} + +func testDeterminismInfrastructureRetries( + t *testing.T, + wantPrefix *model.AdvanceResult, + predecessor determinismMachineState, + wantTarget *model.AdvanceResult, + targetPayload []byte, + attempts int, +) { + t.Helper() + repo := repositoryWithDeterminismPrefix(wantPrefix) + + for range attempts { + harness := newSnapshotDeterminismHarness( + t, repo, predecessor, determinismWaitTimeout, + determinismAdvanceInfrastructureFailure, + ) + err := harness.process(context.Background(), 1, targetPayload) + require.ErrorIs(t, err, machine.ErrMachineInternal) + requireCanonicalPrefix(t, repo, wantPrefix) + require.Equal(t, uint64(1), harness.instance.ProcessedInputs()) + require.Equal(t, predecessor, harness.factory.base.snapshot()) + require.True(t, harness.factory.base.isClosed(), + "advancer must close the failed live runtime") + _, err = harness.factory.base.Fork(context.Background()) + require.ErrorIs(t, err, errDeterminismRuntimeClosed) + requireDiscardedMutatedFork(t, harness.lastFork(t), predecessor) + } + require.Equal(t, attempts, repo.ApplicationStatusUpdates) + require.Equal(t, model.ApplicationStatus_Failed, repo.LastApplicationStatus) + + recovered := newSnapshotDeterminismHarness( + t, repo, predecessor, determinismWaitTimeout, determinismAdvanceSuccess, + ) + require.NoError(t, recovered.process(context.Background(), 1, targetPayload)) + requireCanonicalHistory(t, repo, wantPrefix, wantTarget) + requireSuccessfulRetryState(t, recovered, recovered.factory.base, wantTarget) +} + +func testDeterminismManagerWatchdog( + t *testing.T, + wantPrefix *model.AdvanceResult, + predecessor determinismMachineState, + wantTarget *model.AdvanceResult, + targetPayload []byte, + deadline time.Duration, +) { + t.Helper() + repo := repositoryWithDeterminismPrefix(wantPrefix) + harness := newSnapshotDeterminismHarness( + t, repo, predecessor, deadline, determinismAdvanceWaitForContext, + ) + parent := context.Background() + errCh := make(chan error, 1) + go func() { errCh <- harness.process(parent, 1, targetPayload) }() + + err := waitDeterminismError(t, errCh) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.NoError(t, parent.Err(), "the manager watchdog, not the caller, must expire") + requireCanonicalPrefix(t, repo, wantPrefix) + require.Equal(t, uint64(1), harness.instance.ProcessedInputs()) + require.Equal(t, 1, repo.ApplicationStatusUpdates) + require.Equal(t, model.ApplicationStatus_Failed, repo.LastApplicationStatus) + require.Equal(t, predecessor, harness.factory.base.snapshot()) + require.True(t, harness.factory.base.isClosed()) + requireDiscardedMutatedFork(t, harness.lastFork(t), predecessor) + + recovered := newSnapshotDeterminismHarness( + t, repo, predecessor, determinismWaitTimeout, determinismAdvanceSuccess, + ) + require.NoError(t, recovered.process(context.Background(), 1, targetPayload)) + requireCanonicalHistory(t, repo, wantPrefix, wantTarget) + requireSuccessfulRetryState(t, recovered, recovered.factory.base, wantTarget) +} + +func requireDeterminismTarget( + t *testing.T, + prefix *model.AdvanceResult, + target *model.AdvanceResult, + targetPayload []byte, + wantStatus model.InputCompletionStatus, +) { + t.Helper() + require.Equal(t, uint64(7), target.EpochIndex) + require.Equal(t, uint64(1), target.InputIndex) + require.Equal(t, wantStatus, target.Status) + require.True(t, target.IsDaveConsensus) + require.NotEmpty(t, target.OutputsHashProof) + require.Len(t, target.PeriodicStateHashes, 2, "a PRT result must retain its periodic state hashes") + require.Equal(t, machine.InputEntryCapacity-uint64(len(target.PeriodicStateHashes)), target.PaddingRepetitions) + + if wantStatus == model.InputCompletionStatus_Accepted { + require.Equal(t, [][]byte{append([]byte("output:"), targetPayload...)}, target.Outputs) + require.Equal(t, [][]byte{append([]byte("report:"), targetPayload...)}, target.Reports) + require.NotEqual(t, prefix.MachineHash, target.MachineHash) + require.NotEqual(t, prefix.OutputsHash, target.OutputsHash) + return + } + + require.Empty(t, target.Outputs, "effects are canonical only for accepted inputs") + require.Empty(t, target.Reports, "effects are canonical only for accepted inputs") + require.Equal(t, prefix.MachineHash, target.MachineHash, + "a nonaccepted candidate must not replace the predecessor") + require.Equal(t, prefix.OutputsHash, target.OutputsHash) + require.Equal(t, prefix.OutputsHashProof, target.OutputsHashProof) +} + +func requireDiscardedMutatedFork( + t *testing.T, + fork *determinismRuntime, + predecessor determinismMachineState, +) { + t.Helper() + require.True(t, fork.isClosed(), "the mutated candidate must be discarded") + require.NotEqual(t, predecessor.machineHash, fork.snapshot().machineHash, + "the injected failure must happen after candidate state changes") + _, err := fork.Hash(context.Background()) + require.ErrorIs(t, err, errDeterminismRuntimeClosed) +} + +func requireSuccessfulRetryState( + t *testing.T, + harness *determinismHarness, + predecessor *determinismRuntime, + wantTarget *model.AdvanceResult, +) { + t.Helper() + require.Equal(t, uint64(2), harness.instance.ProcessedInputs()) + require.Equal(t, machine.Hash(wantTarget.MachineHash), harness.runtimeHash(t)) + lastCandidate := harness.lastFork(t) + if wantTarget.Status == model.InputCompletionStatus_Accepted { + require.True(t, predecessor.isClosed()) + require.False(t, lastCandidate.isClosed(), "the accepted candidate must be adopted") + return + } + require.False(t, predecessor.isClosed(), "rejection must keep the predecessor live") + require.True(t, lastCandidate.isClosed(), "the rejected candidate must be discarded") +} + +func requireCanonicalPrefix( + t *testing.T, + repo *MockRepository, + wantPrefix *model.AdvanceResult, +) { + t.Helper() + require.Len(t, repo.StoredResults, 1, + "an interrupted input must not add a canonical result") + require.Equal(t, wantPrefix, repo.StoredResults[0]) +} + +func requireCanonicalHistory( + t *testing.T, + repo *MockRepository, + wantPrefix *model.AdvanceResult, + wantTarget *model.AdvanceResult, +) { + t.Helper() + require.Equal(t, []*model.AdvanceResult{wantPrefix, wantTarget}, repo.StoredResults) +} + +func repositoryWithDeterminismPrefix(prefix *model.AdvanceResult) *MockRepository { + return &MockRepository{StoredResults: []*model.AdvanceResult{cloneDeterminismResult(prefix)}} +} + +func waitDeterminismError(t *testing.T, errCh <-chan error) error { + t.Helper() + select { + case err := <-errCh: + return err + case <-time.After(determinismWaitTimeout): + t.Fatal("advance did not finish before the test timeout") + return nil + } +} + +func cloneDeterminismResult(result *model.AdvanceResult) *model.AdvanceResult { + clone := *result + clone.OutputsHashProof = append([][32]byte(nil), result.OutputsHashProof...) + clone.Outputs = cloneDeterminismBytes(result.Outputs) + clone.Reports = cloneDeterminismBytes(result.Reports) + clone.ExceptionData = append([]byte(nil), result.ExceptionData...) + clone.PeriodicStateHashes = append([][32]byte(nil), result.PeriodicStateHashes...) + return &clone +} + +func cloneDeterminismBytes(values [][]byte) [][]byte { + if values == nil { + return nil + } + clone := make([][]byte, len(values)) + for i := range values { + clone[i] = append([]byte(nil), values[i]...) + } + return clone +} + +type determinismHarness struct { + app *model.Application + instance manager.MachineInstance + service *Service + factory *determinismRuntimeFactory + provider *determinismMachineProvider +} + +func newTemplateDeterminismHarness( + t *testing.T, + repo *MockRepository, + advanceDeadline time.Duration, + behaviors ...determinismAdvanceBehavior, +) *determinismHarness { + t.Helper() + return newDeterminismHarness( + t, repo, newDeterminismMachineState(), 0, advanceDeadline, behaviors..., + ) +} + +func newSnapshotDeterminismHarness( + t *testing.T, + repo *MockRepository, + predecessor determinismMachineState, + advanceDeadline time.Duration, + behaviors ...determinismAdvanceBehavior, +) *determinismHarness { + t.Helper() + return newDeterminismHarness( + t, repo, predecessor, 1, advanceDeadline, behaviors..., + ) +} + +func newDeterminismHarness( + t *testing.T, + repo *MockRepository, + start determinismMachineState, + processedInputs uint64, + advanceDeadline time.Duration, + behaviors ...determinismAdvanceBehavior, +) *determinismHarness { + t.Helper() + app := &model.Application{ + ID: 1, + Name: "deterministic-input-statuses", + IApplicationAddress: common.HexToAddress("0x1"), + ConsensusType: model.Consensus_PRT, + Enabled: true, + Status: model.ApplicationStatus_OK, + ProcessedInputs: processedInputs, + ExecutionParameters: model.ExecutionParameters{ + SnapshotPolicy: model.SnapshotPolicy_None, + AdvanceMaxDeadline: advanceDeadline, + InspectMaxDeadline: determinismWaitTimeout, + LoadDeadline: determinismWaitTimeout, + StoreDeadline: determinismWaitTimeout, + MaxConcurrentInspects: 1, + }, + } + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + factory := newDeterminismRuntimeFactory(start, behaviors...) + instance, err := manager.NewMachineInstanceWithFactory( + context.Background(), app, processedInputs, logger, factory, + ) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, instance.Close()) }) + provider := &determinismMachineProvider{app: app, instance: instance} + service := &Service{ + Service: pkgservice.Service{ + Logger: logger, + Cancel: func() {}, + }, + inputBatchSize: 500, + machineManager: provider, + repository: repo, + } + return &determinismHarness{ + app: app, instance: instance, service: service, factory: factory, provider: provider, + } +} + +func (h *determinismHarness) process(ctx context.Context, index uint64, payload []byte) error { + return h.service.processInputs(ctx, h.app, []*model.Input{{ + EpochApplicationID: h.app.ID, + EpochIndex: 7, + Index: index, + RawData: append([]byte(nil), payload...), + }}) +} + +func (h *determinismHarness) waitForMutation(t *testing.T) *determinismRuntime { + t.Helper() + select { + case fork := <-h.factory.mutated: + return fork + case <-time.After(determinismWaitTimeout): + t.Fatal("candidate did not reach the injected interruption point") + return nil + } +} + +func (h *determinismHarness) liveRuntime(t *testing.T) *determinismRuntime { + t.Helper() + if h.instance.ProcessedInputs() == 0 { + return h.factory.base + } + return h.factory.acceptedRuntime(t) +} + +func (h *determinismHarness) liveState(t *testing.T) determinismMachineState { + t.Helper() + return h.liveRuntime(t).snapshot() +} + +func (h *determinismHarness) lastFork(t *testing.T) *determinismRuntime { + t.Helper() + return h.factory.lastFork(t) +} + +func (h *determinismHarness) runtimeHash(t *testing.T) machine.Hash { + t.Helper() + hash, err := h.instance.Hash(context.Background()) + require.NoError(t, err) + return hash +} + +type determinismAdvanceBehavior uint8 + +const ( + determinismAdvanceSuccess determinismAdvanceBehavior = iota + determinismAdvanceWaitForContext + determinismAdvanceInfrastructureFailure +) + +type determinismMachineState struct { + step uint64 + machineHash machine.Hash + outputsHash machine.Hash + outputsProof []machine.Hash + checkpointHash machine.Hash +} + +func newDeterminismMachineState() determinismMachineState { + machineHash := determinismHash("base-machine") + outputsHash := determinismHash("base-outputs") + return determinismMachineState{ + machineHash: machineHash, + outputsHash: outputsHash, + outputsProof: determinismProof(outputsHash), + } +} + +func (s determinismMachineState) clone() determinismMachineState { + s.outputsProof = append([]machine.Hash(nil), s.outputsProof...) + return s +} + +type determinismRuntimeFactory struct { + mu sync.Mutex + behaviors []determinismAdvanceBehavior + next int + start determinismMachineState + base *determinismRuntime + forks []*determinismRuntime + accepted *determinismRuntime + mutated chan *determinismRuntime +} + +func newDeterminismRuntimeFactory( + start determinismMachineState, + behaviors ...determinismAdvanceBehavior, +) *determinismRuntimeFactory { + return &determinismRuntimeFactory{ + behaviors: append([]determinismAdvanceBehavior(nil), behaviors...), + start: start.clone(), + mutated: make(chan *determinismRuntime, len(behaviors)+1), + } +} + +func (f *determinismRuntimeFactory) CreateMachineRuntime( + ctx context.Context, + _ *model.Application, + _ *slog.Logger, +) (machine.Machine, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + f.mu.Lock() + defer f.mu.Unlock() + f.base = &determinismRuntime{factory: f, state: f.start.clone()} + return f.base, nil +} + +func (f *determinismRuntimeFactory) fork(state determinismMachineState) *determinismRuntime { + f.mu.Lock() + defer f.mu.Unlock() + behavior := determinismAdvanceSuccess + if f.next < len(f.behaviors) { + behavior = f.behaviors[f.next] + } + f.next++ + child := &determinismRuntime{ + factory: f, + behavior: behavior, + state: state.clone(), + } + f.forks = append(f.forks, child) + return child +} + +func (f *determinismRuntimeFactory) recordAccepted(runtime *determinismRuntime) { + f.mu.Lock() + defer f.mu.Unlock() + f.accepted = runtime +} + +func (f *determinismRuntimeFactory) acceptedRuntime(t *testing.T) *determinismRuntime { + t.Helper() + f.mu.Lock() + defer f.mu.Unlock() + require.NotNil(t, f.accepted) + return f.accepted +} + +func (f *determinismRuntimeFactory) lastFork(t *testing.T) *determinismRuntime { + t.Helper() + f.mu.Lock() + defer f.mu.Unlock() + require.NotEmpty(t, f.forks) + return f.forks[len(f.forks)-1] +} + +func (f *determinismRuntimeFactory) forkCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.forks) +} + +type determinismRuntime struct { + mu sync.Mutex + factory *determinismRuntimeFactory + behavior determinismAdvanceBehavior + state determinismMachineState + closed bool +} + +func (m *determinismRuntime) Fork(ctx context.Context) (machine.Machine, error) { + m.mu.Lock() + defer m.mu.Unlock() + if err := m.checkOpenLocked(ctx); err != nil { + return nil, err + } + return m.factory.fork(m.state), nil +} + +func (m *determinismRuntime) Hash(ctx context.Context) (machine.Hash, error) { + m.mu.Lock() + defer m.mu.Unlock() + if err := m.checkOpenLocked(ctx); err != nil { + return machine.Hash{}, err + } + return m.state.machineHash, nil +} + +func (m *determinismRuntime) OutputsHash(ctx context.Context) (machine.Hash, error) { + m.mu.Lock() + defer m.mu.Unlock() + if err := m.checkOpenLocked(ctx); err != nil { + return machine.Hash{}, err + } + return m.state.outputsHash, nil +} + +func (m *determinismRuntime) OutputsHashProof(ctx context.Context) ([]machine.Hash, error) { + m.mu.Lock() + defer m.mu.Unlock() + if err := m.checkOpenLocked(ctx); err != nil { + return nil, err + } + return append([]machine.Hash(nil), m.state.outputsProof...), nil +} + +func (m *determinismRuntime) Advance( + ctx context.Context, + input []byte, + checkpointHash machine.Hash, + computeHashes bool, +) (*machine.AdvanceResponse, error) { + m.mu.Lock() + if err := m.checkOpenLocked(ctx); err != nil { + m.mu.Unlock() + return nil, err + } + if !computeHashes { + m.mu.Unlock() + return nil, errors.New("determinism test requires PRT input hash collection") + } + if len(input) == 0 { + m.mu.Unlock() + return nil, errors.New("determinism test input must not be empty") + } + + // The revert root rides with the advance request and must be the machine's + // pre-input root — the instance always passes fork.Hash(). A mismatch is a + // harness (or caller) bug, not a determinism scenario. + if checkpointHash != m.state.machineHash { + m.mu.Unlock() + return nil, errors.New("determinism test requires the checkpoint hash to equal the pre-input machine root") + } + previous := m.state.clone() + // Recording the request's own revert root is the first mutation of the + // candidate, exactly like the CMIO response. + m.state.checkpointHash = checkpointHash + status := machine.CompletionStatusAccepted + switch { + case bytes.HasPrefix(input, []byte("reject:")): + status = machine.CompletionStatusRejected + case bytes.HasPrefix(input, []byte("exception:")): + status = machine.CompletionStatusException + case bytes.HasPrefix(input, []byte("halt:")): + status = machine.CompletionStatusHalted + } + output := append([]byte("output:"), input...) + report := append([]byte("report:"), input...) + firstHash := determinismHash("trace-before", previous.machineHash[:], input) + finalHash := determinismHash("trace-after", firstHash[:], input) + m.state.step++ + m.state.machineHash = determinismHash( + "machine", previous.machineHash[:], checkpointHash[:], input, + ) + m.state.outputsHash = determinismHash("outputs", previous.outputsHash[:], output) + m.state.outputsProof = determinismProof(m.state.outputsHash) + hashes := []machine.Hash{firstHash, finalHash} + response := &machine.AdvanceResponse{ + Status: status, + PeriodicStateHashes: hashes, + PaddingRepetitions: machine.InputEntryCapacity - uint64(len(hashes)), + OutputsHash: m.state.outputsHash, + } + if status == machine.CompletionStatusAccepted { + response.Outputs = []machine.Output{output} + response.Reports = []machine.Report{report} + } else if status == machine.CompletionStatusException { + response.ExceptionData = append([]byte{}, input...) + } + behavior := m.behavior + m.mu.Unlock() + + if behavior == determinismAdvanceWaitForContext { + m.factory.mutated <- m + } + + switch behavior { + case determinismAdvanceSuccess: + if status == machine.CompletionStatusAccepted { + m.factory.recordAccepted(m) + } + return response, nil + case determinismAdvanceWaitForContext: + <-ctx.Done() + return nil, ctx.Err() + case determinismAdvanceInfrastructureFailure: + return nil, machine.ErrMachineInternal + default: + return nil, errors.New("unknown determinism advance behavior") + } +} + +func (m *determinismRuntime) Inspect( + ctx context.Context, + _ []byte, +) (*machine.InspectResponse, error) { + m.mu.Lock() + defer m.mu.Unlock() + if err := m.checkOpenLocked(ctx); err != nil { + return &machine.InspectResponse{}, err + } + return &machine.InspectResponse{Status: machine.CompletionStatusRejected}, nil +} + +func (m *determinismRuntime) Store(ctx context.Context, _ string) error { + m.mu.Lock() + defer m.mu.Unlock() + return m.checkOpenLocked(ctx) +} + +func (m *determinismRuntime) Close() error { + m.mu.Lock() + defer m.mu.Unlock() + m.closed = true + return nil +} + +func (m *determinismRuntime) Address() string { return "determinism-runtime" } + +func (m *determinismRuntime) checkOpenLocked(ctx context.Context) error { + if m.closed { + return errDeterminismRuntimeClosed + } + return ctx.Err() +} + +// snapshot is test-only diagnostics; unlike the machine interface it remains +// readable after Close so tests can prove which state was closed or discarded. +func (m *determinismRuntime) snapshot() determinismMachineState { + m.mu.Lock() + defer m.mu.Unlock() + return m.state.clone() +} + +func (m *determinismRuntime) isClosed() bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.closed +} + +func determinismHash(label string, values ...[]byte) machine.Hash { + h := sha256.New() + _, _ = h.Write([]byte(label)) + for _, value := range values { + _, _ = h.Write(value) + } + var result machine.Hash + copy(result[:], h.Sum(nil)) + return result +} + +func determinismProof(outputsHash machine.Hash) []machine.Hash { + return []machine.Hash{ + determinismHash("proof-0", outputsHash[:]), + determinismHash("proof-1", outputsHash[:]), + } +} + +type determinismMachineProvider struct { + app *model.Application + instance manager.MachineInstance + mu sync.Mutex + failures map[int64]string +} + +func (p *determinismMachineProvider) GetMachine(appID int64) (manager.MachineInstance, bool) { + if appID != p.app.ID { + return nil, false + } + return p.instance, true +} + +func (p *determinismMachineProvider) Applications() []*model.Application { + return []*model.Application{p.app} +} + +func (p *determinismMachineProvider) UpdateMachines(context.Context) error { return nil } + +func (p *determinismMachineProvider) FenceApplicationFailure(app *model.Application, reason string) { + p.mu.Lock() + defer p.mu.Unlock() + if p.failures == nil { + p.failures = map[int64]string{} + } + p.failures[app.ID] = reason +} + +func (p *determinismMachineProvider) HasPendingApplicationFailures() bool { + p.mu.Lock() + defer p.mu.Unlock() + return len(p.failures) != 0 +} + +func (p *determinismMachineProvider) failureReason(appID int64) string { + p.mu.Lock() + defer p.mu.Unlock() + return p.failures[appID] +} + +func (p *determinismMachineProvider) HasMachine(appID int64) bool { return appID == p.app.ID } + +func (p *determinismMachineProvider) Close() error { return p.instance.Close() } + +// determinismContext makes cancellation and deadline propagation controllable +// without relying on scheduler timing or short wall-clock deadlines. +type determinismContext struct { + parent context.Context + done chan struct{} + once sync.Once + mu sync.RWMutex + err error +} + +func newDeterminismContext(parent context.Context) *determinismContext { + return &determinismContext{parent: parent, done: make(chan struct{})} +} + +func (c *determinismContext) Deadline() (time.Time, bool) { return time.Time{}, false } +func (c *determinismContext) Done() <-chan struct{} { return c.done } +func (c *determinismContext) Err() error { + c.mu.RLock() + defer c.mu.RUnlock() + return c.err +} +func (c *determinismContext) Value(key any) any { return c.parent.Value(key) } +func (c *determinismContext) finish(err error) { + c.once.Do(func() { + c.mu.Lock() + c.err = err + c.mu.Unlock() + close(c.done) + }) +} diff --git a/internal/advancer/service.go b/internal/advancer/service.go index 3e657d5eb..6a3479a51 100644 --- a/internal/advancer/service.go +++ b/internal/advancer/service.go @@ -15,6 +15,7 @@ import ( "github.com/cartesi/rollups-node/internal/inspect" "github.com/cartesi/rollups-node/internal/manager" "github.com/cartesi/rollups-node/internal/repository" + "github.com/cartesi/rollups-node/pkg/machine" "github.com/cartesi/rollups-node/pkg/service" ) @@ -50,6 +51,12 @@ func Create(ctx context.Context, c *CreateInfo) (*Service, error) { if err = ctx.Err(); err != nil { return nil, err // This returns context.Canceled or context.DeadlineExceeded. } + // This is a process-wide compatibility invariant, not an application + // failure. Refuse advancer startup instead of retrying every application on + // every tick while reporting a misleading healthy service. + if err = machine.ValidateEmulatorComputationHashLimits(); err != nil { + return nil, fmt.Errorf("invalid machine execution constants: %w", err) + } s := &Service{} c.Impl = s @@ -106,8 +113,13 @@ func Create(ctx context.Context, c *CreateInfo) (*Service, error) { } // Service interface implementation -func (s *Service) Alive() bool { return true } -func (s *Service) Ready() bool { return true } +func (s *Service) Alive() bool { return true } +func (s *Service) Ready() bool { + // This is a local fail-closed signal while application-failure durability + // is unresolved. It cannot atomically revoke work already selected by a + // separate process before that durable status becomes visible. + return s.machineManager != nil && !s.machineManager.HasPendingApplicationFailures() +} func (s *Service) Reload() []error { return nil } func (s *Service) Tick() []error { hadWork, err := s.Step(s.Context) diff --git a/internal/appstatus/appstatus.go b/internal/appstatus/appstatus.go index 34eba94be..29ba5f3cc 100644 --- a/internal/appstatus/appstatus.go +++ b/internal/appstatus/appstatus.go @@ -30,7 +30,7 @@ type Repository interface { // Recovery assumptions — FAILED is safe to re-enable only when: // - The failure was a machine runtime error (not a DB desync). // - The last snapshot is consistent with the database state. -// - Synchronize() will correctly replay inputs from the snapshot point. +// - replay.Run will correctly verify inputs from the snapshot point. // // The reason parameter must be a pre-formatted string describing the failure. // Returns the database error if the status update fails; returns nil on success. @@ -134,7 +134,7 @@ func setTerminalStatus( status ApplicationStatus, reason string, ) error { - reason = truncateReason(reason) + reason = NormalizeReason(reason) dbErr := setApplicationStatus(ctx, logger, repo, app, status, reason) reasonErr := errors.New(reason) if dbErr != nil { @@ -143,9 +143,10 @@ func setTerminalStatus( return reasonErr } -// truncateReason truncates a reason string to maxReasonLength to avoid -// exceeding the database VARCHAR(4096) constraint. -func truncateReason(reason string) string { +// NormalizeReason returns the exact reason representation persisted by status +// helpers. Callers that compare a later readback must normalize before keeping +// their expected value. +func NormalizeReason(reason string) string { if len(reason) > maxReasonLength { return reason[:maxReasonLength] + "... (truncated)" } @@ -160,7 +161,7 @@ func setApplicationStatus( status ApplicationStatus, reason string, ) error { - reason = truncateReason(reason) + reason = NormalizeReason(reason) switch status { case ApplicationStatus_Failed: diff --git a/internal/inspect/hardening_test.go b/internal/inspect/hardening_test.go index 63e107465..fa6991609 100644 --- a/internal/inspect/hardening_test.go +++ b/internal/inspect/hardening_test.go @@ -83,7 +83,7 @@ type erroringMachine struct { err error } -func (m *erroringMachine) Inspect(_ context.Context, _ []byte) (*InspectResult, error) { +func (m *erroringMachine) Inspect(_ context.Context, _ []byte) (*manager.InspectResult, error) { if m.err == errPanicSentinel { panic("boom-from-machine") } @@ -106,9 +106,6 @@ func (m *erroringMachine) ProcessedInputs() uint64 { return m.inner.ProcessedI func (m *erroringMachine) OutputsProof(ctx context.Context) (*OutputsProof, error) { return m.inner.OutputsProof(ctx) } -func (m *erroringMachine) Synchronize(ctx context.Context, repo manager.MachineRepository, batchSize uint64) error { - return m.inner.Synchronize(ctx, repo, batchSize) -} func (m *erroringMachine) CreateSnapshot(ctx context.Context, processedInputs uint64, path string) error { return m.inner.CreateSnapshot(ctx, processedInputs, path) } diff --git a/internal/inspect/inspect.go b/internal/inspect/inspect.go index d5c378197..54c3e4c2e 100644 --- a/internal/inspect/inspect.go +++ b/internal/inspect/inspect.go @@ -17,6 +17,7 @@ import ( "github.com/cartesi/rollups-node/internal/manager" . "github.com/cartesi/rollups-node/internal/model" + pkgmachine "github.com/cartesi/rollups-node/pkg/machine" "github.com/cartesi/rollups-node/pkg/service" "github.com/ethereum/go-ethereum/common/hexutil" ) @@ -31,6 +32,11 @@ const maxPayloadSize = 1 << 21 // 2 MiB // serialization after the Cartesi Machine's inspect deadline fires. const inspectResponseHeadroom = 30 * time.Second +const ( + inspectStatusFailed = "Failed" + inspectFailureMessage = "The node could not complete the inspection" +) + var ( ErrInvalidMachines = errors.New("machines must not be nil") ErrNoApp = errors.New("no application") @@ -67,7 +73,8 @@ type ReportResponse struct { type InspectResponse struct { Status string `json:"status"` - Exception string `json:"exception,omitempty"` + Error string `json:"error,omitempty"` + ExceptionData string `json:"exception_data,omitempty"` Reports []ReportResponse `json:"reports"` ProcessedInputs uint64 `json:"processed_input_count"` } @@ -157,23 +164,16 @@ func (inspect *Inspector) Admission() *service.SemaphoreAdmission { } func (inspect *Inspector) ServeHTTP(w http.ResponseWriter, r *http.Request) { - var ( - dapp string - payload []byte - err error - reports []ReportResponse - status string - errorMessage string - ) + requestID := service.RequestIDFromContext(r.Context()) + dapp := r.PathValue("dapp") - if r.PathValue("dapp") == "" { + if dapp == "" { inspect.Logger.Info("Bad request", "err", "Missing application address") http.Error(w, "Missing application address", http.StatusBadRequest) return } - dapp = r.PathValue("dapp") if r.Method != http.MethodPost { inspect.Logger.Info("HTTP method not allowed", "application", dapp, "method", r.Method) w.Header().Set("Allow", http.MethodPost) @@ -185,7 +185,7 @@ func (inspect *Inspector) ServeHTTP(w http.ResponseWriter, r *http.Request) { // both enforces the limit and signals the server to close the connection // on over-limit so clients can't pipeline further requests on it. r.Body = http.MaxBytesReader(w, r.Body, maxPayloadSize) - payload, err = io.ReadAll(r.Body) + payload, err := io.ReadAll(r.Body) if err != nil { var maxErr *http.MaxBytesError if errors.As(err, &maxErr) { @@ -244,27 +244,7 @@ func (inspect *Inspector) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - for _, report := range result.Reports { - reports = append(reports, ReportResponse{Payload: hexutil.Encode(report)}) - } - - if result.Accepted { - status = "Accepted" - } else { - status = "Rejected" - } - - if result.Error != nil { - status = "Exception" - errorMessage = fmt.Sprintf("Error on the machine while inspecting: %s", result.Error) - } - - response := InspectResponse{ - Status: status, - Exception: errorMessage, - Reports: reports, - ProcessedInputs: result.ProcessedInputs, - } + response := inspect.buildInspectResponse(dapp, requestID, result) w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(response); err != nil { @@ -274,15 +254,83 @@ func (inspect *Inspector) ServeHTTP(w http.ResponseWriter, r *http.Request) { inspect.Logger.Error("failed to encode inspect response", "err", err, "application", dapp, - "request_id", service.RequestIDFromContext(r.Context()), + "request_id", requestID, ) return } inspect.Logger.Info("Request executed", - "status", status, + "status", response.Status, "application", dapp) } +// buildInspectResponse maps the manager result to the public inspect contract. +// Machine execution details remain in trusted logs; anonymous clients receive +// a stable sanitized failure message and any reports emitted before the stop. +func (inspect *Inspector) buildInspectResponse( + dapp string, + requestID string, + result *manager.InspectResult, +) InspectResponse { + response := InspectResponse{ + Reports: make([]ReportResponse, 0, len(result.Reports)), + ProcessedInputs: result.ProcessedInputs, + } + for _, report := range result.Reports { + response.Reports = append(response.Reports, ReportResponse{Payload: hexutil.Encode(report)}) + } + + if result.Error != nil { + response.Status = inspectStatusFailed + response.Error = inspectFailureMessage + // Inspect is an anonymous endpoint. Keep machine positions and local + // policy values in operator logs so clients cannot discover configured + // limits or price a resource-exhaustion input. + inspect.Logger.Warn("Machine failed while inspecting", + "application", dapp, + "error", result.Error, + "request_id", requestID, + ) + return response + } + + switch result.Status { + case pkgmachine.CompletionStatusAccepted: + response.Status = "Accepted" + case pkgmachine.CompletionStatusRejected: + response.Status = "Rejected" + case pkgmachine.CompletionStatusException: + response.Status = "Exception" + response.Error = "The machine raised an exception while inspecting" + response.ExceptionData = hexutil.Encode(result.ExceptionData) + inspect.Logger.Debug("Machine returned a guest inspect exception", + "application", dapp, + "request_id", requestID, + ) + case pkgmachine.CompletionStatusHalted: + response.Status = "MachineHalted" + inspect.Logger.Debug("Machine halted while inspecting", + "application", dapp, + "request_id", requestID, + ) + case pkgmachine.CompletionStatusUnknown: + response.Status = inspectStatusFailed + response.Error = inspectFailureMessage + inspect.Logger.Warn("Machine returned an incomplete inspect result", + "application", dapp, + "request_id", requestID, + ) + default: + response.Status = inspectStatusFailed + response.Error = inspectFailureMessage + inspect.Logger.Warn("Machine returned an unknown inspect status", + "application", dapp, + "status", result.Status, + "request_id", requestID, + ) + } + return response +} + func (inspect *Inspector) warnDeadlineExceedsWriteTimeout(app *Application, deadline time.Duration) { inspect.deadlineWarnedMu.Lock() defer inspect.deadlineWarnedMu.Unlock() diff --git a/internal/inspect/inspect_test.go b/internal/inspect/inspect_test.go index ff099a5fc..9abb0f0a5 100644 --- a/internal/inspect/inspect_test.go +++ b/internal/inspect/inspect_test.go @@ -8,6 +8,7 @@ import ( "context" crand "crypto/rand" "encoding/json" + "errors" "fmt" "io" "log/slog" @@ -18,6 +19,8 @@ import ( "github.com/cartesi/rollups-node/internal/manager" . "github.com/cartesi/rollups-node/internal/model" + inspectclient "github.com/cartesi/rollups-node/pkg/inspectclient" + pkgmachine "github.com/cartesi/rollups-node/pkg/machine" "github.com/cartesi/rollups-node/pkg/service" "github.com/ethereum/go-ethereum/common" @@ -194,6 +197,156 @@ func (s *InspectSuite) TestPostPayloadTooLarge() { s.Equal(http.StatusRequestEntityTooLarge, resp.StatusCode) } +func (s *InspectSuite) TestPostResponseMatchesGeneratedClientContract() { + tests := []struct { + name string + result manager.InspectResult + wantStatus inspectclient.CompletionStatus + wantError string + wantExceptionData []byte + }{ + { + name: "accepted", + result: manager.InspectResult{ + Status: pkgmachine.CompletionStatusAccepted, + Reports: [][]byte{{0xde, 0xad}, {}}, + ProcessedInputs: 17, + }, + wantStatus: inspectclient.Accepted, + }, + { + name: "rejected", + result: manager.InspectResult{ + Status: pkgmachine.CompletionStatusRejected, + Reports: [][]byte{{0xbe, 0xef}}, + ProcessedInputs: 23, + }, + wantStatus: inspectclient.Rejected, + }, + { + name: "exception", + result: manager.InspectResult{ + Status: pkgmachine.CompletionStatusException, + ExceptionData: []byte{0xff, 0x00, 0x80}, + Reports: [][]byte{{0xca, 0xfe}}, + ProcessedInputs: 42, + }, + wantStatus: inspectclient.Exception, + wantError: "The machine raised an exception while inspecting", + wantExceptionData: []byte{0xff, 0x00, 0x80}, + }, + { + name: "halted", + result: manager.InspectResult{ + Status: pkgmachine.CompletionStatusHalted, + Reports: [][]byte{{0xfa, 0xce}}, + ProcessedInputs: 51, + }, + wantStatus: inspectclient.MachineHalted, + }, + { + name: "failed", + result: manager.InspectResult{ + Status: pkgmachine.CompletionStatusUnknown, + Reports: [][]byte{{0xba, 0xdd}}, + ProcessedInputs: 63, + Error: errors.New("backend disconnected"), + }, + wantStatus: inspectclient.Failed, + wantError: "The node could not complete the inspection", + }, + } + + for _, test := range tests { + s.Run(test.name, func() { + inspect, app := s.setupWithInspectResult(test.result) + app.Status = ApplicationStatus_OK + request := httptest.NewRequest(http.MethodPost, "/inspect/"+app.Name, + bytes.NewBufferString("query")) + request.SetPathValue("dapp", app.Name) + recorder := httptest.NewRecorder() + inspect.ServeHTTP(recorder, request) + s.Equal(http.StatusOK, recorder.Code) + + var got inspectclient.InspectResult + s.Require().NoError(json.NewDecoder(recorder.Body).Decode(&got)) + s.Equal(test.wantStatus, got.Status) + s.EqualValues(test.result.ProcessedInputs, got.ProcessedInputCount) + s.Require().Len(got.Reports, len(test.result.Reports)) + for i, report := range test.result.Reports { + s.Equal(fmt.Sprintf("0x%x", report), got.Reports[i].Payload) + } + if test.wantError == "" { + s.Nil(got.Error) + } else { + s.Require().NotNil(got.Error) + s.Equal(test.wantError, *got.Error) + } + if test.wantExceptionData == nil { + s.Nil(got.ExceptionData) + } else { + s.Require().NotNil(got.ExceptionData) + s.Equal(fmt.Sprintf("0x%x", test.wantExceptionData), *got.ExceptionData) + } + s.Equal(ApplicationStatus_OK, app.Status, "inspect limits must not change application status") + }) + } +} + +func (s *InspectSuite) TestCycleLimitIsSanitizedFailedResultWithoutApplicationFailure() { + detailedErr := fmt.Errorf( + "inspect stopped at absolute_mcycle=123456 with configured_cap=789: %w", + pkgmachine.ErrReachedLimitMcycle, + ) + inspect, app := s.setupWithInspectResult(manager.InspectResult{ + Status: pkgmachine.CompletionStatusUnknown, + ProcessedInputs: 42, + Error: detailedErr, + }) + var logs bytes.Buffer + inspect.Logger = slog.New(slog.NewTextHandler(&logs, nil)) + app.Status = ApplicationStatus_OK + req := httptest.NewRequest(http.MethodPost, "/inspect/"+app.Name, bytes.NewBufferString("query")) + req.SetPathValue("dapp", app.Name) + recorder := httptest.NewRecorder() + + inspect.ServeHTTP(recorder, req) + + s.Equal(http.StatusOK, recorder.Code) + var got inspectclient.InspectResult + s.Require().NoError(json.NewDecoder(recorder.Body).Decode(&got)) + s.Equal(inspectclient.Failed, got.Status) + s.Require().NotNil(got.Error) + s.Equal("The node could not complete the inspection", *got.Error) + s.NotContains(*got.Error, "absolute_mcycle") + s.NotContains(*got.Error, "configured_cap") + s.Contains(logs.String(), "absolute_mcycle=123456") + s.Contains(logs.String(), "configured_cap=789") + s.EqualValues(42, got.ProcessedInputCount) + s.Equal(ApplicationStatus_OK, app.Status) +} + +func (s *InspectSuite) TestGuestExceptionUsesDebugLogLevel() { + inspect, app := s.setupWithInspectResult(manager.InspectResult{ + Status: pkgmachine.CompletionStatusException, + ExceptionData: []byte("guest exception details"), + ProcessedInputs: 42, + }) + var logs bytes.Buffer + inspect.Logger = slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})) + req := httptest.NewRequest(http.MethodPost, "/inspect/"+app.Name, bytes.NewBufferString("query")) + req.SetPathValue("dapp", app.Name) + recorder := httptest.NewRecorder() + + inspect.ServeHTTP(recorder, req) + + s.Equal(http.StatusOK, recorder.Code) + s.Contains(logs.String(), "level=DEBUG") + s.Contains(logs.String(), "Machine returned a guest inspect exception") + s.NotContains(logs.String(), "level=WARN") + s.NotContains(logs.String(), "guest exception details") +} + func (s *InspectSuite) startServer(inspect *Inspector) *httptest.Server { router := http.NewServeMux() router.Handle("/inspect/{dapp}", inspect) @@ -201,7 +354,17 @@ func (s *InspectSuite) startServer(inspect *Inspector) *httptest.Server { } func (s *InspectSuite) setup() (*Inspector, *Application, common.Hash) { + payload := randomHash() + inspect, app := s.setupWithInspectResult(manager.InspectResult{ + Status: pkgmachine.CompletionStatusAccepted, + Reports: [][]byte{payload.Bytes()}, + }) + return inspect, app, payload +} + +func (s *InspectSuite) setupWithInspectResult(result manager.InspectResult) (*Inspector, *Application) { m := newMockMachine(1) + m.inspectResult = &result repo := newMockRepository() repo.apps = append(repo.apps, m.application) machines := newMockMachines() @@ -211,8 +374,7 @@ func (s *InspectSuite) setup() (*Inspector, *Application, common.Hash) { IInspectMachines: machines, Logger: service.NewLogger(slog.LevelDebug, true), } - payload := randomHash() - return inspect, m.application, payload + return inspect, m.application } func (s *InspectSuite) assertResponse(resp *http.Response, payload string) { @@ -251,18 +413,24 @@ func (mock *MachinesMock) GetMachine(appId int64) (manager.MachineInstance, bool // ------------------------------------------------------------------------------------------------ type MockMachine struct { - application *Application + application *Application + inspectResult *manager.InspectResult } func (mock *MockMachine) Inspect( _ context.Context, query []byte, -) (*InspectResult, error) { - var res InspectResult +) (*manager.InspectResult, error) { + if mock.inspectResult != nil { + result := *mock.inspectResult + return &result, nil + } + + var res manager.InspectResult var reports [][]byte reports = append(reports, query) - res.Accepted = true + res.Status = pkgmachine.CompletionStatusAccepted res.ProcessedInputs = 0 res.Error = nil res.Reports = reports @@ -293,11 +461,6 @@ func (m *MockMachine) OutputsProof(ctx context.Context) (*OutputsProof, error) { return nil, nil } -// Not used in inspect tests, but needed to satisfy the interface -func (mock *MockMachine) Synchronize(ctx context.Context, repo manager.MachineRepository, batchSize uint64) error { - return nil -} - // Not used in inspect tests, but needed to satisfy the interface func (mock *MockMachine) CreateSnapshot(ctx context.Context, processedInputs uint64, path string) error { return nil diff --git a/internal/jsonrpc/execution_outcome_contract_test.go b/internal/jsonrpc/execution_outcome_contract_test.go new file mode 100644 index 000000000..811da54be --- /dev/null +++ b/internal/jsonrpc/execution_outcome_contract_test.go @@ -0,0 +1,59 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package jsonrpc + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDiscoverySchemaExecutionOutcomeContract(t *testing.T) { + data, err := discoverSpec.ReadFile("jsonrpc-discover.json") + require.NoError(t, err) + + var spec struct { + Components struct { + Schemas map[string]json.RawMessage `json:"schemas"` + } `json:"components"` + } + require.NoError(t, json.Unmarshal(data, &spec)) + + var completionStatus struct { + Enum []string `json:"enum"` + } + require.NoError(t, json.Unmarshal(spec.Components.Schemas["InputCompletionStatus"], &completionStatus)) + require.Equal(t, []string{"NONE", "ACCEPTED", "REJECTED", "EXCEPTION", "MACHINE_HALTED"}, completionStatus.Enum) + + var input struct { + Properties map[string]json.RawMessage `json:"properties"` + } + require.NoError(t, json.Unmarshal(spec.Components.Schemas["Input"], &input)) + require.Contains(t, input.Properties, "exception_data") + + var executionParameters struct { + Properties map[string]json.RawMessage `json:"properties"` + } + require.NoError(t, json.Unmarshal(spec.Components.Schemas["ExecutionParameters"], &executionParameters)) + require.Contains(t, executionParameters.Properties, "advance_inc_cycles") + require.Contains(t, executionParameters.Properties, "inspect_inc_cycles") + require.Contains(t, executionParameters.Properties, "advance_max_cycles") + require.Contains(t, executionParameters.Properties, "inspect_max_cycles") + for field, operation := range map[string]string{ + "advance_max_cycles": "advance", + "inspect_max_cycles": "inspect", + } { + var property struct { + Description string `json:"description"` + } + require.NoError(t, json.Unmarshal(executionParameters.Properties[field], &property)) + require.Equal(t, + "Optional "+operation+ + " execution delta from the starting mcycle in range 0..2^48-1; "+ + "0 imposes no operator cap and the machine's imcyclemax governs", + property.Description, + ) + } +} diff --git a/internal/jsonrpc/jsonrpc-discover.json b/internal/jsonrpc/jsonrpc-discover.json index ea21d239e..ea2b43c12 100644 --- a/internal/jsonrpc/jsonrpc-discover.json +++ b/internal/jsonrpc/jsonrpc-discover.json @@ -1657,12 +1657,7 @@ "ACCEPTED", "REJECTED", "EXCEPTION", - "MACHINE_HALTED", - "OUTPUTS_LIMIT_EXCEEDED", - "REPORTS_LIMIT_EXCEEDED", - "CYCLE_LIMIT_EXCEEDED", - "TIME_LIMIT_EXCEEDED", - "PAYLOAD_LENGTH_LIMIT_EXCEEDED" + "MACHINE_HALTED" ] }, "Input": { @@ -1693,6 +1688,17 @@ "status": { "$ref": "#/components/schemas/InputCompletionStatus" }, + "exception_data": { + "description": "Raw guest-provided CMIO exception payload. It is non-null only when status is EXCEPTION; an empty payload is encoded as 0x.", + "oneOf": [ + { + "$ref": "#/components/schemas/ByteArray" + }, + { + "type": "null" + } + ] + }, "machine_hash": { "oneOf": [ { @@ -2056,12 +2062,14 @@ "$ref": "#/components/schemas/UnsignedInteger" }, "advance_max_cycles": { + "description": "Optional advance execution delta from the starting mcycle in range 0..2^48-1; 0 imposes no operator cap and the machine's imcyclemax governs", "$ref": "#/components/schemas/UnsignedInteger" }, "inspect_inc_cycles": { "$ref": "#/components/schemas/UnsignedInteger" }, "inspect_max_cycles": { + "description": "Optional inspect execution delta from the starting mcycle in range 0..2^48-1; 0 imposes no operator cap and the machine's imcyclemax governs", "$ref": "#/components/schemas/UnsignedInteger" }, "advance_inc_deadline": { diff --git a/internal/manager/errors.go b/internal/manager/errors.go new file mode 100644 index 000000000..e94adc7a0 --- /dev/null +++ b/internal/manager/errors.go @@ -0,0 +1,75 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package manager + +import ( + "errors" + "fmt" +) + +var ErrApplicationFailureNotDurable = errors.New("application failure status is not durably confirmed") + +// ApplicationFailurePersistenceError means local machine work is fenced but +// the repository has not confirmed the corresponding FAILED (or a stronger +// terminal/deleted) application state. +type ApplicationFailurePersistenceError struct { + ApplicationID int64 + WriteErr error + ReadErr error +} + +func (e *ApplicationFailurePersistenceError) Error() string { + if e.ReadErr != nil { + return fmt.Sprintf( + "%v: application=%d write_error=%v read_error=%v", + ErrApplicationFailureNotDurable, e.ApplicationID, e.WriteErr, e.ReadErr, + ) + } + return fmt.Sprintf( + "%v: application=%d write_error=%v durable status remains unconfirmed", + ErrApplicationFailureNotDurable, e.ApplicationID, e.WriteErr, + ) +} + +func (e *ApplicationFailurePersistenceError) Unwrap() []error { + errList := []error{ErrApplicationFailureNotDurable} + if e.WriteErr != nil { + errList = append(errList, e.WriteErr) + } + if e.ReadErr != nil { + errList = append(errList, e.ReadErr) + } + return errList +} + +// IsOnlyApplicationFailurePersistenceErrors reports whether err is one or +// more application-local durability failures and contains no global failure. +// It deliberately examines joined top-level errors without descending into a +// persistence error's write/read causes. +func IsOnlyApplicationFailurePersistenceErrors(err error) bool { + if err == nil { + return false + } + if _, ok := err.(*ApplicationFailurePersistenceError); ok { + return true + } + + switch wrapped := err.(type) { + case interface{ Unwrap() []error }: + children := wrapped.Unwrap() + if len(children) == 0 { + return false + } + for _, child := range children { + if !IsOnlyApplicationFailurePersistenceErrors(child) { + return false + } + } + return true + case interface{ Unwrap() error }: + return IsOnlyApplicationFailurePersistenceErrors(wrapped.Unwrap()) + default: + return false + } +} diff --git a/internal/manager/instance.go b/internal/manager/instance.go index caed1509c..53ef50392 100644 --- a/internal/manager/instance.go +++ b/internal/manager/instance.go @@ -9,13 +9,13 @@ import ( "errors" "fmt" "log/slog" + "math" "sync" "sync/atomic" "time" "github.com/cartesi/rollups-node/internal/manager/pmutex" . "github.com/cartesi/rollups-node/internal/model" - "github.com/cartesi/rollups-node/internal/repository" "github.com/cartesi/rollups-node/pkg/machine" "github.com/ethereum/go-ethereum/common" "golang.org/x/sync/semaphore" @@ -24,6 +24,8 @@ import ( var ( ErrMachineClosed = errors.New("machine is closed") ErrInvalidInputIndex = errors.New("invalid input index") + ErrIncompleteAdvance = errors.New("machine advance returned no completed result") + ErrIncompleteInspect = errors.New("machine inspect returned no completed result") ErrInvalidSnapshotPoint = errors.New("invalid snapshot point") ErrInvalidApplication = errors.New("application must not be nil") ErrInvalidAdvanceTimeout = errors.New("advance timeout must not be negative") @@ -82,9 +84,10 @@ func NewMachineInstance( ctx context.Context, app *Application, logger *slog.Logger, - checkHash bool, + checkTemplateHash bool, ) (MachineInstance, error) { - return NewMachineInstanceWithFactory(ctx, app, 0, logger, checkHash, defaultFactory) + factory := &DefaultMachineRuntimeFactory{CheckTemplateHash: checkTemplateHash} + return NewMachineInstanceWithFactory(ctx, app, 0, logger, factory) } // NewMachineInstanceFromSnapshot creates a new machine instance from a snapshot @@ -92,16 +95,33 @@ func NewMachineInstanceFromSnapshot( ctx context.Context, app *Application, logger *slog.Logger, - checkHash bool, snapshotPath string, - machineHash *common.Hash, + expectedHash common.Hash, + inputIndex uint64, +) (MachineInstance, error) { + return newMachineInstanceFromSnapshot( + ctx, app, logger, snapshotPath, expectedHash, inputIndex, nil, + ) +} + +func newMachineInstanceFromSnapshot( + ctx context.Context, + app *Application, + logger *slog.Logger, + snapshotPath string, + expectedHash common.Hash, inputIndex uint64, + loader machineLoader, ) (MachineInstance, error) { + if inputIndex == math.MaxUint64 { + return nil, fmt.Errorf("%w: snapshot input index cannot be incremented", ErrInvalidSnapshotPoint) + } factory := &SnapshotMachineRuntimeFactory{ SnapshotPath: snapshotPath, - MachineHash: machineHash, + ExpectedHash: expectedHash, + loader: loader, } - return NewMachineInstanceWithFactory(ctx, app, inputIndex+1, logger, checkHash, factory) + return NewMachineInstanceWithFactory(ctx, app, inputIndex+1, logger, factory) } // NewMachineInstanceWithFactory creates a new machine instance with a custom factory @@ -110,7 +130,6 @@ func NewMachineInstanceWithFactory( app *Application, processedInputs uint64, logger *slog.Logger, - checkHash bool, factory MachineRuntimeFactory, ) (MachineInstance, error) { // Validate parameters @@ -136,9 +155,15 @@ func NewMachineInstanceWithFactory( } // Create the machine server and runtime - runtime, err := factory.CreateMachineRuntime(ctx, app, logger, checkHash) + runtime, err := factory.CreateMachineRuntime(ctx, app, logger) if err != nil { - return nil, fmt.Errorf("%w: %v", ErrMachineCreation, err) + if runtime != nil { + err = errors.Join(err, runtime.Close()) + } + return nil, fmt.Errorf("%w: %w", ErrMachineCreation, err) + } + if runtime == nil { + return nil, fmt.Errorf("%w: runtime factory returned nil runtime", ErrMachineCreation) } // Create the machine instance @@ -167,82 +192,6 @@ func (m *MachineInstanceImpl) ProcessedInputs() uint64 { return m.processedInputs.Load() } -// Synchronize brings the machine up to date with processed inputs. -// It handles both template-based instances (processedInputs == 0, replays all) -// and snapshot-based instances (processedInputs > 0, replays only remaining). -// Inputs are fetched in batches to bound memory usage. -func (m *MachineInstanceImpl) Synchronize(ctx context.Context, repo MachineRepository, batchSize uint64) error { - appAddress := m.application.IApplicationAddress.String() - currentProcessed := m.processedInputs.Load() - m.logger.Info("Synchronizing machine with processed inputs", - "address", appAddress, - "app_processed_inputs", m.application.ProcessedInputs, - "machine_processed_inputs", currentProcessed) - - initialProcessedInputs := currentProcessed - replayed := uint64(0) - toReplay := uint64(0) - - for { - p := repository.Pagination{ - Limit: batchSize, - Offset: initialProcessedInputs + replayed, - } - inputs, totalCount, err := getProcessedInputs(ctx, repo, appAddress, p) - if err != nil { - return fmt.Errorf("%w: %w", ErrMachineSynchronization, err) - } - - // Validate count on the first batch - if replayed == 0 { - if totalCount != m.application.ProcessedInputs { - errorMsg := fmt.Sprintf( - "processed inputs count mismatch: expected %d, got %d", - m.application.ProcessedInputs, totalCount) - m.logger.Error(errorMsg, "address", appAddress) - return fmt.Errorf("%w: %s", ErrMachineSynchronization, errorMsg) - } - if currentProcessed > totalCount { - return fmt.Errorf( - "%w: machine has processed %d inputs but DB only has %d", - ErrMachineSynchronization, currentProcessed, totalCount) - } - toReplay = totalCount - currentProcessed - if toReplay == 0 { - m.logger.Info("No inputs to replay during synchronization", - "address", appAddress) - return nil - } - } - - for _, input := range inputs { - m.logger.Info("Replaying input during synchronization", - "address", appAddress, - "epoch_index", input.EpochIndex, - "input_index", input.Index, - "progress", fmt.Sprintf("%d/%d", replayed+1, toReplay)) - - _, err := m.Advance(ctx, input.RawData, input.EpochIndex, input.Index, false) - if err != nil { - return fmt.Errorf("%w: failed to replay input %d: %w", - ErrMachineSynchronization, input.Index, err) - } - replayed++ - } - - if replayed >= toReplay { - break - } - if len(inputs) == 0 { - return fmt.Errorf( - "%w: expected to replay %d inputs but only replayed %d", - ErrMachineSynchronization, toReplay, replayed) - } - } - - return nil -} - // forkForAdvance creates a copy of the machine for advance operations // It verifies the input index and returns a forked machine func (m *MachineInstanceImpl) forkForAdvance(ctx context.Context, index uint64) (machine.Machine, error) { @@ -264,7 +213,12 @@ func (m *MachineInstanceImpl) forkForAdvance(ctx context.Context, index uint64) return m.runtime.Fork(ctx) } -// Advance processes an input and advances the machine state +// Advance treats a machine fork as the execution transaction for one input. +// It executes on the fork, selects the canonical root from the typed completion +// status, and advances processedInputs exactly once for every completed input. +// Accepted adopts the fork; currently nonaccepted completions keep the +// predecessor runtime and close the fork. Incomplete execution returns an error +// and adopts neither the fork nor a canonical result. func (m *MachineInstanceImpl) Advance(ctx context.Context, input []byte, epochIndex uint64, index uint64, computeHashes bool) (*AdvanceResult, error) { // Only one advance can be active at a time m.advanceMutex.Lock() @@ -301,7 +255,16 @@ func (m *MachineInstanceImpl) Advance(ctx context.Context, input []byte, epochIn // Process the input advanceResp, err := fork.Advance(advanceCtx, input, prevMachineHash, computeHashes) - status, err := toInputStatus(advanceResp.Accepted, err) + if err != nil { + return nil, errors.Join(err, fork.Close()) + } + if advanceResp == nil { + return nil, errors.Join(ErrIncompleteAdvance, fork.Close()) + } + if err := validateCompletionExceptionData(advanceResp.Status, advanceResp.ExceptionData); err != nil { + return nil, errors.Join(ErrIncompleteAdvance, err, fork.Close()) + } + status, err := toInputStatus(advanceResp.Status) if err != nil { return nil, errors.Join(err, fork.Close()) } @@ -313,13 +276,17 @@ func (m *MachineInstanceImpl) Advance(ctx context.Context, input []byte, epochIn Status: status, Outputs: advanceResp.Outputs, Reports: advanceResp.Reports, - Hashes: advanceResp.Hashes, - RemainingMetaCycles: advanceResp.RemainingCycles, + ExceptionData: advanceResp.ExceptionData, + PeriodicStateHashes: advanceResp.PeriodicStateHashes, + PaddingRepetitions: advanceResp.PaddingRepetitions, IsDaveConsensus: computeHashes, } - // If the input was accepted, update the machine state - if result.Status == InputCompletionStatus_Accepted { + // Resolve the canonical result and fork disposition once. Validation below + // must succeed before the selected disposition mutates the live instance. + adoptFork := false + switch result.Status { + case InputCompletionStatus_Accepted: // Get the machine hash after processing result.MachineHash, err = fork.Hash(ctx) if err != nil { @@ -330,7 +297,37 @@ func (m *MachineInstanceImpl) Advance(ctx context.Context, input []byte, epochIn if err != nil { return nil, errors.Join(err, fork.Close()) } + adoptFork = true + case InputCompletionStatus_Rejected, + InputCompletionStatus_Exception, + InputCompletionStatus_MachineHalted: + // Use the previous state for currently nonaccepted inputs. + result.MachineHash = prevMachineHash + result.OutputsHash = prevOutputsHash + result.OutputsHashProof = prevOutputsHashProof + case InputCompletionStatus_None: + return nil, errors.Join( + fmt.Errorf("cannot resolve advance result for status %q", result.Status), + fork.Close(), + ) + default: + return nil, errors.Join( + fmt.Errorf("cannot resolve advance result for unknown status %q", result.Status), + fork.Close(), + ) + } + + if computeHashes { + err = validateCanonicalInputHashCollectionSpan( + uint64(len(result.PeriodicStateHashes)), + result.PaddingRepetitions, + ) + if err != nil { + return nil, errors.Join(err, fork.Close()) + } + } + if adoptFork { // Replace the current machine with the fork m.mutex.HLock() oldRuntime := m.runtime @@ -342,10 +339,6 @@ func (m *MachineInstanceImpl) Advance(ctx context.Context, input []byte, epochIn m.logger.Warn("Failed to close old machine runtime", "error", err) } } else { - // Use the previous state for rejected inputs - result.MachineHash = prevMachineHash - result.OutputsHash = prevOutputsHash - result.OutputsHashProof = prevOutputsHashProof // Close the fork since we're not using it if err := fork.Close(); err != nil { @@ -361,6 +354,21 @@ func (m *MachineInstanceImpl) Advance(ctx context.Context, input []byte, epochIn return result, nil } +func validateCanonicalInputHashCollectionSpan( + hashCount uint64, + paddingRepetitions uint64, +) error { + if err := machine.ValidateInputHashCollectionSpan( + hashCount, paddingRepetitions, + ); err != nil { + return fmt.Errorf("invalid canonical input hash collection span: %w", err) + } + if paddingRepetitions == 0 { + return errors.New("canonical input hash collection requires a positive final repetition tail") + } + return nil +} + // forkForInspect creates a copy of the machine for inspect operations // It returns the forked machine and the current processed inputs count func (m *MachineInstanceImpl) forkForInspect(ctx context.Context) (machine.Machine, uint64, error) { @@ -401,14 +409,30 @@ func (m *MachineInstanceImpl) Inspect(ctx context.Context, query []byte) (*Inspe defer cancel() // Process the query - accepted, reports, inspectErr := fork.Inspect(inspectCtx, query) + inspectResponse, inspectErr := fork.Inspect(inspectCtx, query) // Create the result result := &InspectResult{ ProcessedInputs: processedInputs, - Accepted: accepted, - Reports: reports, - Error: inspectErr, + } + if inspectResponse != nil { + result.Status = inspectResponse.Status + result.ExceptionData = inspectResponse.ExceptionData + result.Reports = inspectResponse.Reports + } + + // An execution error takes precedence over any status supplied by a Machine + // implementation. Inspection did not complete, but reports emitted before + // the failure remain useful to the caller. + if inspectErr != nil { + result.Status = machine.CompletionStatusUnknown + result.Error = inspectErr + } else if inspectResponse == nil || !result.Status.IsCompleted() { + result.Status = machine.CompletionStatusUnknown + result.Error = errors.Join(ErrIncompleteInspect, machine.ErrMachineInternal) + } else if err := validateCompletionExceptionData(result.Status, result.ExceptionData); err != nil { + result.Status = machine.CompletionStatusUnknown + result.Error = errors.Join(ErrIncompleteInspect, err) } // Close the fork @@ -596,19 +620,25 @@ type MachineRuntimeFactory interface { ctx context.Context, app *Application, logger *slog.Logger, - checkHash bool, ) (machine.Machine, error) } +type machineLoader func( + ctx context.Context, + logger *slog.Logger, + config *machine.MachineConfig, +) (machine.Machine, error) + // createMachineRuntimeCommon contains the shared logic for creating machine runtimes func createMachineRuntimeCommon( ctx context.Context, app *Application, logger *slog.Logger, - checkHash bool, + verifyExpectedHash bool, machinePath string, sourceType string, expectedHash common.Hash, + loader machineLoader, ) (machine.Machine, error) { if logger == nil { return nil, ErrInvalidLogger @@ -627,9 +657,16 @@ func createMachineRuntimeCommon( config.ExecutionParameters = app.ExecutionParameters // Create the machine - m, err := machine.Load(ctx, logger, config) + if loader == nil { + loader = machine.Load + } + m, err := loader(ctx, logger, config) if err != nil { - return nil, err + // Preserve a partially created runtime so the caller can close it. + return m, err + } + if m == nil { + return nil, errors.New("machine loader returned nil runtime without an error") } logger.Debug(fmt.Sprintf("Machine loaded from %s", sourceType), @@ -639,7 +676,7 @@ func createMachineRuntimeCommon( "path", machinePath) // Verify the machine hash if required - if checkHash { + if verifyExpectedHash { logger.Debug("Verifying machine hash", "application", app.Name, "address", appAddress) @@ -665,31 +702,37 @@ func createMachineRuntimeCommon( return m, nil } -// DefaultMachineRuntimeFactory is the standard implementation of MachineRuntimeFactory -type DefaultMachineRuntimeFactory struct{} +// DefaultMachineRuntimeFactory is the standard template implementation of +// MachineRuntimeFactory. Template verification remains configurable for +// backwards compatibility; snapshot verification is always mandatory. +type DefaultMachineRuntimeFactory struct { + CheckTemplateHash bool + loader machineLoader +} // CreateMachineRuntime creates a new machine runtime for an application func (f *DefaultMachineRuntimeFactory) CreateMachineRuntime( ctx context.Context, app *Application, logger *slog.Logger, - checkHash bool, ) (machine.Machine, error) { return createMachineRuntimeCommon( ctx, app, logger, - checkHash, + f.CheckTemplateHash, app.TemplateURI, "template", app.TemplateHash, + f.loader, ) } // SnapshotMachineRuntimeFactory creates machine runtimes from snapshots type SnapshotMachineRuntimeFactory struct { SnapshotPath string - MachineHash *common.Hash // The hash to check against (from the input's machine_hash) + ExpectedHash common.Hash + loader machineLoader } // CreateMachineRuntime creates a new machine runtime from a snapshot @@ -697,56 +740,52 @@ func (f *SnapshotMachineRuntimeFactory) CreateMachineRuntime( ctx context.Context, app *Application, logger *slog.Logger, - checkHash bool, ) (machine.Machine, error) { - // Determine which hash to check against - expectedHash := app.TemplateHash - if f.MachineHash != nil { - expectedHash = *f.MachineHash - } - return createMachineRuntimeCommon( ctx, app, logger, - checkHash, + true, f.SnapshotPath, "snapshot", - expectedHash, + f.ExpectedHash, + f.loader, ) } -// Default factory instance -var defaultFactory MachineRuntimeFactory = &DefaultMachineRuntimeFactory{} - -// Helper function to convert machine response to input status -func toInputStatus(accepted bool, err error) (status InputCompletionStatus, _ error) { - if err == nil { - if accepted { - return InputCompletionStatus_Accepted, nil - } else { - return InputCompletionStatus_Rejected, nil - } +// toInputStatus converts only completed, deterministic machine statuses to +// canonical input statuses. Infrastructure interruptions never reach here. +func toInputStatus(status machine.CompletionStatus) (InputCompletionStatus, error) { + switch status { + case machine.CompletionStatusAccepted: + return InputCompletionStatus_Accepted, nil + case machine.CompletionStatusRejected: + return InputCompletionStatus_Rejected, nil + case machine.CompletionStatusException: + return InputCompletionStatus_Exception, nil + case machine.CompletionStatusHalted: + return InputCompletionStatus_MachineHalted, nil + case machine.CompletionStatusUnknown: + // Intentionally empty. } + return InputCompletionStatus_None, fmt.Errorf( + "unknown completed machine status %d: %w", + status, + ErrIncompleteAdvance, + ) +} +func validateCompletionExceptionData(status machine.CompletionStatus, data []byte) error { switch { - case errors.Is(err, machine.ErrException): - return InputCompletionStatus_Exception, nil - case errors.Is(err, machine.ErrHalted): - return InputCompletionStatus_MachineHalted, nil - case errors.Is(err, machine.ErrOutputsLimitExceeded): - return InputCompletionStatus_OutputsLimitExceeded, nil - case errors.Is(err, machine.ErrReportsLimitExceeded): - return InputCompletionStatus_ReportsLimitExceeded, nil - case errors.Is(err, machine.ErrReachedTargetMcycle): - return InputCompletionStatus_CycleLimitExceeded, nil - case errors.Is(err, machine.ErrPayloadLengthLimitExceeded): - return InputCompletionStatus_PayloadLengthLimitExceeded, nil - case errors.Is(err, machine.ErrDeadlineExceeded): - return InputCompletionStatus_TimeLimitExceeded, nil - case errors.Is(err, machine.ErrMachineInternal): - fallthrough + case status == machine.CompletionStatusException && data == nil: + return fmt.Errorf("completed exception has no exception data: %w", machine.ErrMachineInternal) + case status != machine.CompletionStatusException && data != nil: + return fmt.Errorf( + "completion status %d unexpectedly has exception data: %w", + status, + machine.ErrMachineInternal, + ) default: - return status, err + return nil } } diff --git a/internal/manager/instance_test.go b/internal/manager/instance_test.go index 6f758d812..d40cbf025 100644 --- a/internal/manager/instance_test.go +++ b/internal/manager/instance_test.go @@ -8,13 +8,14 @@ import ( "errors" "io" "log/slog" + "math" "sync" + "sync/atomic" "testing" "time" "github.com/cartesi/rollups-node/internal/manager/pmutex" "github.com/cartesi/rollups-node/internal/model" - "github.com/cartesi/rollups-node/internal/repository" "github.com/cartesi/rollups-node/pkg/machine" "github.com/ethereum/go-ethereum/common" "github.com/stretchr/testify/suite" @@ -30,9 +31,13 @@ type MachineInstanceSuite struct{ suite.Suite } func (s *MachineInstanceSuite) TestMcycleOverflowRemainsIncomplete() { require := s.Require() - status, err := toInputStatus(false, machine.ErrReachedLimitMcycle) - require.ErrorIs(err, machine.ErrReachedLimitMcycle) - require.Empty(status) + // Cycle exhaustion surfaces as an error from machine.Advance, never as a + // completed CompletionStatus, so no input completion status can exist for it. + // The zero-value status is the closest representable input and must be + // rejected rather than mapped to a completed status. + status, err := toInputStatus(machine.CompletionStatusUnknown) + require.ErrorIs(err, ErrIncompleteAdvance) + require.Equal(model.InputCompletionStatus_None, status) } // MockMachineRuntimeFactory implements MachineRuntimeFactory for testing @@ -45,7 +50,6 @@ func (f *MockMachineRuntimeFactory) CreateMachineRuntime( _ context.Context, _ *model.Application, _ *slog.Logger, - _ bool, ) (machine.Machine, error) { return f.RuntimeToReturn, f.ErrorToReturn } @@ -76,7 +80,6 @@ func (s *MachineInstanceSuite) TestNewMachineInstance() { app, 0, testLogger, - false, mockFactory, ) require.Nil(err) @@ -104,7 +107,6 @@ func (s *MachineInstanceSuite) TestNewMachineInstance() { app, 0, testLogger, - false, mockFactory, ) require.Error(err) @@ -130,7 +132,6 @@ func (s *MachineInstanceSuite) TestNewMachineInstance() { app, 0, testLogger, - false, mockFactory, ) require.Error(err) @@ -156,7 +157,6 @@ func (s *MachineInstanceSuite) TestNewMachineInstance() { app, 0, testLogger, - false, mockFactory, ) require.Error(err) @@ -180,7 +180,6 @@ func (s *MachineInstanceSuite) TestNewMachineInstance() { app, 0, nil, - false, mockFactory, ) require.Error(err) @@ -204,19 +203,56 @@ func (s *MachineInstanceSuite) TestNewMachineInstance() { app, 0, testLogger, - false, nil, ) require.Error(err) require.Nil(machine) require.Contains(err.Error(), "factory must not be nil") }) + + s.Run("FactoryErrorClosesPartialRuntime", func() { + require := s.Require() + factoryErr := errors.New("factory failed after creating runtime") + closeErr := errors.New("partial runtime close failed") + partial := &MockRollupsMachine{CloseError: closeErr} + factory := &MockMachineRuntimeFactory{RuntimeToReturn: partial, ErrorToReturn: factoryErr} + app := &model.Application{ExecutionParameters: model.ExecutionParameters{ + AdvanceMaxDeadline: decisecond, InspectMaxDeadline: centisecond, MaxConcurrentInspects: 1, + }} + + instance, err := NewMachineInstanceWithFactory( + context.Background(), app, 0, + slog.New(slog.NewTextHandler(io.Discard, nil)), factory, + ) + + require.Nil(instance) + require.ErrorIs(err, ErrMachineCreation) + require.ErrorIs(err, factoryErr) + require.ErrorIs(err, closeErr) + require.Equal(int64(1), partial.CloseCalls.Load()) + }) + + s.Run("FactoryNilRuntimeIsRejected", func() { + require := s.Require() + app := &model.Application{ExecutionParameters: model.ExecutionParameters{ + AdvanceMaxDeadline: decisecond, InspectMaxDeadline: centisecond, MaxConcurrentInspects: 1, + }} + factory := &MockMachineRuntimeFactory{} + + instance, err := NewMachineInstanceWithFactory( + context.Background(), app, 0, + slog.New(slog.NewTextHandler(io.Discard, nil)), factory, + ) + + require.Nil(instance) + require.ErrorIs(err, ErrMachineCreation) + require.Contains(err.Error(), "nil runtime") + }) } func (s *MachineInstanceSuite) TestNewMachineInstanceFromSnapshot() { - s.Run("Ok", func() { - require := s.Require() - app := &model.Application{ + newApp := func() *model.Application { + return &model.Application{ Name: "TestApp", ExecutionParameters: model.ExecutionParameters{ AdvanceMaxDeadline: decisecond, @@ -224,65 +260,95 @@ func (s *MachineInstanceSuite) TestNewMachineInstanceFromSnapshot() { MaxConcurrentInspects: 3, }, } + } + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - testLogger := slog.New(slog.NewTextHandler(io.Discard, nil)) - mockRuntime := &MockRollupsMachine{} - mockFactory := &MockMachineRuntimeFactory{ - RuntimeToReturn: mockRuntime, - ErrorToReturn: nil, + s.Run("MatchingHashCreatesInstanceAtNextInput", func() { + require := s.Require() + expected := newHash(7) + runtime := &MockRollupsMachine{HashReturn: expected} + loader := func( + _ context.Context, _ *slog.Logger, config *machine.MachineConfig, + ) (machine.Machine, error) { + require.Equal("snapshot-dir", config.Path) + return runtime, nil } - // NewMachineInstanceFromSnapshot creates a SnapshotMachineRuntimeFactory - // internally, so we use NewMachineInstanceWithFactory to test the same - // logic with a controlled factory. - inputIndex := uint64(5) - - // The function sets processedInputs = inputIndex + 1 - // Use the mock factory to avoid actual machine loading - inst, err := NewMachineInstanceWithFactory( - context.Background(), - app, - inputIndex+1, - testLogger, - false, - mockFactory, + instance, err := newMachineInstanceFromSnapshot( + context.Background(), newApp(), logger, "snapshot-dir", expected, 5, loader, ) + require.NoError(err) - require.NotNil(inst) - require.Equal(inputIndex+1, inst.ProcessedInputs()) + require.Equal(uint64(6), instance.ProcessedInputs()) + require.Equal(int64(1), runtime.HashCalls.Load()) + require.Zero(runtime.CloseCalls.Load()) + require.NoError(instance.Close()) + }) + + s.Run("MismatchingHashClosesRuntime", func() { + require := s.Require() + runtime := &MockRollupsMachine{HashReturn: newHash(8)} + factory := &SnapshotMachineRuntimeFactory{ + SnapshotPath: "snapshot-dir", + ExpectedHash: newHash(9), + loader: func(context.Context, *slog.Logger, *machine.MachineConfig) (machine.Machine, error) { + return runtime, nil + }, + } - inst.Close() + got, err := factory.CreateMachineRuntime(context.Background(), newApp(), logger) + + require.Nil(got) + require.Error(err) + require.Contains(err.Error(), "machine hash mismatch") + require.Equal(int64(1), runtime.HashCalls.Load()) + require.Equal(int64(1), runtime.CloseCalls.Load()) }) - s.Run("FactoryError", func() { + s.Run("HashReadErrorClosesRuntime", func() { require := s.Require() - app := &model.Application{ - Name: "TestApp", - ExecutionParameters: model.ExecutionParameters{ - AdvanceMaxDeadline: decisecond, - InspectMaxDeadline: centisecond, - MaxConcurrentInspects: 3, + hashErr := errors.New("hash unavailable") + runtime := &MockRollupsMachine{HashError: hashErr} + factory := &SnapshotMachineRuntimeFactory{ + SnapshotPath: "snapshot-dir", + ExpectedHash: newHash(9), + loader: func(context.Context, *slog.Logger, *machine.MachineConfig) (machine.Machine, error) { + return runtime, nil }, } - testLogger := slog.New(slog.NewTextHandler(io.Discard, nil)) - mockFactory := &MockMachineRuntimeFactory{ - RuntimeToReturn: nil, - ErrorToReturn: errors.New("snapshot load failed"), + got, err := factory.CreateMachineRuntime(context.Background(), newApp(), logger) + + require.Nil(got) + require.ErrorIs(err, hashErr) + require.Equal(int64(1), runtime.HashCalls.Load()) + require.Equal(int64(1), runtime.CloseCalls.Load()) + }) + + s.Run("NilRuntimeFromLoaderIsRejected", func() { + require := s.Require() + loader := func(context.Context, *slog.Logger, *machine.MachineConfig) (machine.Machine, error) { + return nil, nil } - inst, err := NewMachineInstanceWithFactory( - context.Background(), - app, - 6, - testLogger, - false, - mockFactory, + instance, err := newMachineInstanceFromSnapshot( + context.Background(), newApp(), logger, "snapshot-dir", newHash(1), 0, loader, ) - require.Error(err) - require.Nil(inst) + + require.Nil(instance) require.ErrorIs(err, ErrMachineCreation) - require.Contains(err.Error(), "snapshot load failed") + require.Contains(err.Error(), "machine loader returned nil runtime") + }) + + s.Run("InputIndexOverflowIsRejectedBeforeLoad", func() { + require := s.Require() + + instance, err := NewMachineInstanceFromSnapshot( + context.Background(), newApp(), logger, "snapshot-dir", newHash(1), math.MaxUint64, + ) + + require.Nil(instance) + require.ErrorIs(err, ErrInvalidSnapshotPoint) }) } @@ -305,7 +371,7 @@ func (s *MachineInstanceSuite) TestApplicationAndProcessedInputs() { } inst, err := NewMachineInstanceWithFactory( - context.Background(), app, 42, testLogger, false, mockFactory, + context.Background(), app, 42, testLogger, mockFactory, ) require.NoError(err) require.Same(app, inst.Application()) @@ -334,73 +400,127 @@ func (s *MachineInstanceSuite) TestAdvance() { s.Run("Reject", func() { require := s.Require() - inner, fork, machine := s.setupAdvance() - fork.AdvanceAcceptedReturn = false + inner, fork, instance := s.setupAdvance() + fork.CompletionStatusReturn = machine.CompletionStatusRejected fork.CloseError = nil - res, err := machine.Advance(context.Background(), []byte{}, 0, 5, false) + res, err := instance.Advance(context.Background(), []byte{}, 0, 5, false) require.Nil(err) require.NotNil(res) - require.Same(inner, machine.runtime) + require.Same(inner, instance.runtime) require.Equal(model.InputCompletionStatus_Rejected, res.Status) require.Equal(expectedOutputs, res.Outputs) require.Equal(expectedReports1, res.Reports) require.Equal(newHash(1), res.OutputsHash) require.Equal(newHash(2), res.MachineHash) - require.Equal(uint64(6), machine.processedInputs.Load()) + require.Equal(uint64(6), instance.processedInputs.Load()) }) - testSoftError := func(name string, err error, status model.InputCompletionStatus) { + testCompletedStatus := func( + name string, + machineStatus machine.CompletionStatus, + inputStatus model.InputCompletionStatus, + exceptionData []byte, + ) { s.Run(name, func() { require := s.Require() - inner, fork, machine := s.setupAdvance() - fork.AdvanceError = err - fork.CloseError, inner.CloseError = inner.CloseError, fork.CloseError + inner, fork, instance := s.setupAdvance() + fork.CompletionStatusReturn = machineStatus + fork.ExceptionDataReturn = exceptionData + fork.CloseError = nil - res, err := machine.Advance(context.Background(), []byte{}, 0, 5, false) + res, err := instance.Advance(context.Background(), []byte{}, 0, 5, false) require.Nil(err) require.NotNil(res) - require.Equal(status, res.Status) + require.Same(inner, instance.runtime) + require.Equal(inputStatus, res.Status) + require.Equal(exceptionData, res.ExceptionData) require.Equal(expectedOutputs, res.Outputs) require.Equal(expectedReports1, res.Reports) require.Equal(newHash(1), res.OutputsHash) require.Equal(newHash(2), res.MachineHash) - require.Equal(uint64(6), machine.processedInputs.Load()) + require.Equal(uint64(6), instance.processedInputs.Load()) }) } - testSoftError("Exception", - machine.ErrException, - model.InputCompletionStatus_Exception) + testCompletedStatus("Exception", + machine.CompletionStatusException, + model.InputCompletionStatus_Exception, + []byte("guest exception")) - testSoftError("Halted", - machine.ErrHalted, - model.InputCompletionStatus_MachineHalted) - - testSoftError("OutputsLimit", - machine.ErrOutputsLimitExceeded, - model.InputCompletionStatus_OutputsLimitExceeded) + testCompletedStatus("Halted", + machine.CompletionStatusHalted, + model.InputCompletionStatus_MachineHalted, + nil) + }) - testSoftError("ReportsLimit", - machine.ErrReportsLimitExceeded, - model.InputCompletionStatus_ReportsLimitExceeded) + s.Run("Error", func() { + interruptions := []struct { + name string + err error + }{ + {"OutputsLimit", machine.ErrOutputsLimitExceeded}, + {"ReportsLimit", machine.ErrReportsLimitExceeded}, + {"ReachedCycleLimit", machine.ErrReachedLimitMcycle}, + {"Deadline", machine.ErrDeadlineExceeded}, + {"Canceled", machine.ErrCanceled}, + {"PayloadLengthLimit", machine.ErrPayloadLengthLimitExceeded}, + {"MachineInternal", machine.ErrMachineInternal}, + } + for _, interruption := range interruptions { + s.Run(interruption.name, func() { + require := s.Require() + inner, fork, instance := s.setupAdvance() + fork.AdvanceError = interruption.err + fork.CloseError = nil + + res, err := instance.Advance(context.Background(), []byte{}, 0, 5, false) + require.ErrorIs(err, interruption.err) + require.Nil(res) + require.Same(inner, instance.runtime) + require.Equal(uint64(5), instance.processedInputs.Load()) + }) + } - testSoftError("ReachedTargetMcycle", - machine.ErrReachedTargetMcycle, - model.InputCompletionStatus_CycleLimitExceeded) + s.Run("UnknownStatus", func() { + require := s.Require() + inner, fork, instance := s.setupAdvance() + fork.CompletionStatusReturn = machine.CompletionStatusUnknown + fork.CloseError = nil - testSoftError("TimeLimit", - machine.ErrDeadlineExceeded, - model.InputCompletionStatus_TimeLimitExceeded) + res, err := instance.Advance(context.Background(), []byte{}, 0, 5, false) + require.ErrorIs(err, ErrIncompleteAdvance) + require.Nil(res) + require.Same(inner, instance.runtime) + require.Equal(uint64(5), instance.processedInputs.Load()) + }) - testSoftError("PayloadLengthLimit", - machine.ErrPayloadLengthLimitExceeded, - model.InputCompletionStatus_PayloadLengthLimitExceeded) - }) + for _, test := range []struct { + name string + status machine.CompletionStatus + exceptionData []byte + }{ + {"ExceptionWithoutData", machine.CompletionStatusException, nil}, + {"AcceptedWithExceptionData", machine.CompletionStatusAccepted, []byte("unexpected")}, + } { + s.Run(test.name, func() { + require := s.Require() + inner, fork, instance := s.setupAdvance() + fork.CompletionStatusReturn = test.status + fork.ExceptionDataReturn = test.exceptionData + fork.CloseError = nil + + res, err := instance.Advance(context.Background(), []byte{}, 0, 5, false) + require.ErrorIs(err, ErrIncompleteAdvance) + require.ErrorIs(err, machine.ErrMachineInternal) + require.Nil(res) + require.Same(inner, instance.runtime) + require.Equal(uint64(5), instance.processedInputs.Load()) + }) + } - s.Run("Error", func() { s.Run("Fork", func() { require := s.Require() inner, _, machine := s.setupAdvance() @@ -498,7 +618,8 @@ func (s *MachineInstanceSuite) TestAdvance() { s.Run("Fork", func() { require := s.Require() _, fork, machineInst := s.setupAdvance() - fork.AdvanceError = machine.ErrException + fork.CompletionStatusReturn = machine.CompletionStatusException + fork.ExceptionDataReturn = []byte("guest exception") fork.CloseError = errors.New("Close error") // Close error on fork is logged, not propagated. @@ -513,16 +634,18 @@ func (s *MachineInstanceSuite) TestAdvance() { s.Run("CollectHashes", func() { require := s.Require() - inner, fork, machine := s.setupAdvance() + inner, fork, instance := s.setupAdvance() + + fork.AdvanceRemainingReturn = machine.InputEntryCapacity - res, err := machine.Advance(context.Background(), []byte{}, 0, 5, true) + res, err := instance.Advance(context.Background(), []byte{}, 0, 5, true) require.Nil(err) require.NotNil(res) - require.Same(fork, machine.runtime) + require.Same(fork, instance.runtime) require.Equal(model.InputCompletionStatus_Accepted, res.Status) require.True(res.IsDaveConsensus) - require.Equal(uint64(6), machine.processedInputs.Load()) + require.Equal(uint64(6), instance.processedInputs.Load()) // Verify the inner runtime was closed (accept path) _ = inner @@ -533,7 +656,7 @@ func (s *MachineInstanceSuite) TestAdvance() { // same machine never happens by design. This test verifies that two // sequential advances correctly increment processedInputs. require := s.Require() - inner, fork, machine := s.setupAdvance() + inner, fork, instance := s.setupAdvance() // Allow inner.Close to succeed (old runtime close on accept) inner.CloseError = nil @@ -542,7 +665,7 @@ func (s *MachineInstanceSuite) TestAdvance() { // After accept, fork becomes the new runtime. // Second advance: fork from fork (processedInputs=6), fork must also fork. fork2 := &MockRollupsMachine{} - fork2.AdvanceAcceptedReturn = true + fork2.CompletionStatusReturn = machine.CompletionStatusAccepted fork2.AdvanceOutputsReturn = expectedOutputs fork2.AdvanceReportsReturn = expectedReports1 fork2.OutputsHashReturn = newHash(1) @@ -553,134 +676,159 @@ func (s *MachineInstanceSuite) TestAdvance() { fork.CloseError = nil // close of fork (now old runtime) in second advance // First advance at index 5 - res1, err := machine.Advance(context.Background(), []byte{}, 0, 5, false) + res1, err := instance.Advance(context.Background(), []byte{}, 0, 5, false) require.Nil(err) require.NotNil(res1) - require.Equal(uint64(6), machine.processedInputs.Load()) + require.Equal(uint64(6), instance.processedInputs.Load()) // Second advance at index 6 - res2, err := machine.Advance(context.Background(), []byte{}, 0, 6, false) + res2, err := instance.Advance(context.Background(), []byte{}, 0, 6, false) require.Nil(err) require.NotNil(res2) - require.Equal(uint64(7), machine.processedInputs.Load()) + require.Equal(uint64(7), instance.processedInputs.Load()) }) } func (s *MachineInstanceSuite) TestInspect() { - s.Run("Ok", func() { - s.Run("Accept", func() { + for _, test := range []struct { + name string + status machine.CompletionStatus + }{ + {"Accept", machine.CompletionStatusAccepted}, + {"Reject", machine.CompletionStatusRejected}, + {"Exception", machine.CompletionStatusException}, + {"Halted", machine.CompletionStatusHalted}, + } { + s.Run(test.name, func() { require := s.Require() - _, fork, machine := s.setupInspect() - - res, err := machine.Inspect(context.Background(), []byte{}) - require.Nil(err) - require.NotNil(res) - - require.NotSame(fork, machine.runtime) - require.Equal(uint64(55), res.ProcessedInputs) - require.True(res.Accepted) - require.Equal(expectedReports2, res.Reports) - require.Nil(res.Error) + _, fork, instance := s.setupInspect() + fork.InspectResponseReturn.Status = test.status + if test.status == machine.CompletionStatusException { + fork.InspectResponseReturn.ExceptionData = []byte("guest exception") + } + + result, err := instance.Inspect(context.Background(), []byte{}) + require.NoError(err) + require.NotNil(result) + require.NotSame(fork, instance.runtime) + require.Equal(uint64(55), result.ProcessedInputs) + require.Equal(test.status, result.Status) + require.Equal(fork.InspectResponseReturn.ExceptionData, result.ExceptionData) + require.Equal(expectedReports2, result.Reports) + require.NoError(result.Error) }) + } - s.Run("Reject", func() { + for _, test := range []struct { + name string + status machine.CompletionStatus + exceptionData []byte + }{ + {"ExceptionWithoutData", machine.CompletionStatusException, nil}, + {"RejectedWithExceptionData", machine.CompletionStatusRejected, []byte("unexpected")}, + } { + s.Run(test.name+"FailsClosed", func() { require := s.Require() - _, fork, machine := s.setupInspect() - fork.InspectAcceptedReturn = false - - res, err := machine.Inspect(context.Background(), []byte{}) - require.Nil(err) - require.NotNil(res) - - require.NotSame(fork, machine.runtime) - require.Equal(uint64(55), res.ProcessedInputs) - require.False(res.Accepted) - require.Equal(expectedReports2, res.Reports) - require.Nil(res.Error) + _, fork, instance := s.setupInspect() + fork.InspectResponseReturn.Status = test.status + fork.InspectResponseReturn.ExceptionData = test.exceptionData + + result, err := instance.Inspect(context.Background(), []byte{}) + require.NoError(err) + require.Equal(machine.CompletionStatusUnknown, result.Status) + require.ErrorIs(result.Error, ErrIncompleteInspect) + require.ErrorIs(result.Error, machine.ErrMachineInternal) }) - }) + } - s.Run("Error", func() { - s.Run("AtCapacity", func() { - require := s.Require() - _, _, machine := s.setupInspect() + s.Run("AtCapacity", func() { + require := s.Require() + _, _, instance := s.setupInspect() + instance.inspectSemaphore.TryAcquire(int64(instance.maxConcurrentInspects)) + defer instance.inspectSemaphore.Release(int64(instance.maxConcurrentInspects)) - // Pre-fill all semaphore slots to simulate a saturated app - machine.inspectSemaphore.TryAcquire(int64(machine.maxConcurrentInspects)) + result, err := instance.Inspect(context.Background(), []byte{}) + require.ErrorIs(err, ErrInspectAtCapacity) + require.Nil(result) + }) - // TryAcquire is non-blocking: the error is returned immediately, - // no context deadline required. - res, err := machine.Inspect(context.Background(), []byte{}) - require.Error(err) - require.Nil(res) - require.ErrorIs(err, ErrInspectAtCapacity) + s.Run("ForkError", func() { + require := s.Require() + inner, _, instance := s.setupInspect() + errFork := errors.New("Fork error") + inner.ForkError = errFork - // Release the semaphore for cleanup - machine.inspectSemaphore.Release(int64(machine.maxConcurrentInspects)) - }) + result, err := instance.Inspect(context.Background(), []byte{}) + require.ErrorIs(err, errFork) + require.Nil(result) + }) - s.Run("Fork", func() { - require := s.Require() - inner, _, machine := s.setupInspect() - errFork := errors.New("Fork error") - inner.ForkError = errFork + s.Run("ExecutionErrorPreservesPartialReports", func() { + require := s.Require() + _, fork, instance := s.setupInspect() + errInspect := errors.New("Inspect error") + fork.InspectError = errInspect - res, err := machine.Inspect(context.Background(), []byte{}) - require.Error(err) - require.Nil(res) - require.Equal(errFork, err) - }) + result, err := instance.Inspect(context.Background(), []byte{}) + require.NoError(err) + require.Equal(machine.CompletionStatusUnknown, result.Status) + require.Equal(expectedReports2, result.Reports) + require.ErrorIs(result.Error, errInspect) + }) - s.Run("Inspect", func() { + for _, test := range []struct { + name string + response *machine.InspectResponse + }{ + {"NilResponse", nil}, + {"UnknownStatus", &machine.InspectResponse{Status: machine.CompletionStatusUnknown}}, + {"InvalidStatus", &machine.InspectResponse{Status: machine.CompletionStatus(255)}}, + } { + s.Run(test.name+"FailsClosed", func() { require := s.Require() - _, fork, machine := s.setupInspect() - errInspect := errors.New("Inspect error") - fork.InspectError = errInspect - - res, err := machine.Inspect(context.Background(), []byte{}) - require.Nil(err) - require.NotNil(res) - require.Equal(errInspect, res.Error) + _, fork, instance := s.setupInspect() + fork.InspectResponseReturn = test.response + + result, err := instance.Inspect(context.Background(), []byte{}) + require.NoError(err) + require.Equal(machine.CompletionStatusUnknown, result.Status) + require.ErrorIs(result.Error, ErrIncompleteInspect) + require.ErrorIs(result.Error, machine.ErrMachineInternal) }) + } - s.Run("Close", func() { - require := s.Require() - _, fork, machine := s.setupInspect() - errClose := errors.New("Close error") - fork.CloseError = errClose + s.Run("CloseErrorOverridesResult", func() { + require := s.Require() + _, fork, instance := s.setupInspect() + errClose := errors.New("Close error") + fork.CloseError = errClose - res, err := machine.Inspect(context.Background(), []byte{}) - require.Error(err) - require.Nil(res) - require.Equal(errClose, err) - }) + result, err := instance.Inspect(context.Background(), []byte{}) + require.ErrorIs(err, errClose) + require.Nil(result) }) s.Run("Concurrency", func() { require := s.Require() - _, _, machine := s.setupInspect() - - // Test that we can run maxConcurrentInspects inspects concurrently + _, _, instance := s.setupInspect() var wg sync.WaitGroup - errors := make(chan error, machine.maxConcurrentInspects) + errs := make(chan error, instance.maxConcurrentInspects) - for range int(machine.maxConcurrentInspects) { + for range int(instance.maxConcurrentInspects) { wg.Add(1) go func() { defer wg.Done() - _, err := machine.Inspect(context.Background(), []byte{}) + _, err := instance.Inspect(context.Background(), []byte{}) if err != nil { - errors <- err + errs <- err } }() } wg.Wait() - close(errors) - - // Check if any errors occurred - for err := range errors { - require.Nil(err, "Concurrent inspect failed: %v", err) + close(errs) + for err := range errs { + require.NoError(err) } }) } @@ -1036,7 +1184,7 @@ func (s *MachineInstanceSuite) setupAdvance() (*MockRollupsMachine, *MockRollups inner.ForkReturn = fork inner.CloseError = nil - fork.AdvanceAcceptedReturn = true + fork.CompletionStatusReturn = machine.CompletionStatusAccepted fork.AdvanceOutputsReturn = []machine.Output{ newBytes(11, 100), newBytes(12, 100), @@ -1052,12 +1200,14 @@ func (s *MachineInstanceSuite) setupAdvance() (*MockRollupsMachine, *MockRollups fork.HashReturn = newHash(2) fork.HashError = nil - fork.InspectAcceptedReturn = true - fork.InspectReportsReturn = []machine.Report{ - newBytes(31, 300), - newBytes(32, 300), - newBytes(33, 300), - newBytes(34, 300), + fork.InspectResponseReturn = &machine.InspectResponse{ + Status: machine.CompletionStatusAccepted, + Reports: []machine.Report{ + newBytes(31, 300), + newBytes(32, 300), + newBytes(33, 300), + newBytes(34, 300), + }, } fork.InspectError = errUnreachable @@ -1096,12 +1246,14 @@ func (s *MachineInstanceSuite) setupInspect() (*MockRollupsMachine, *MockRollups fork.AdvanceError = errUnreachable fork.HashError = errUnreachable - fork.InspectAcceptedReturn = true - fork.InspectReportsReturn = []machine.Report{ - newBytes(31, 300), - newBytes(32, 300), - newBytes(33, 300), - newBytes(34, 300), + fork.InspectResponseReturn = &machine.InspectResponse{ + Status: machine.CompletionStatusAccepted, + Reports: []machine.Report{ + newBytes(31, 300), + newBytes(32, 300), + newBytes(33, 300), + newBytes(34, 300), + }, } fork.InspectError = nil @@ -1171,291 +1323,6 @@ func newBytes(n byte, size int) []byte { return bytes } -// ------------------------------------------------------------------------------------------------ -// Synchronize tests -// ------------------------------------------------------------------------------------------------ - -// mockSyncRepository is a lightweight mock for Synchronize tests. -// It simulates pagination over a slice of inputs. -type mockSyncRepository struct { - inputs []*model.Input - totalCount uint64 - listErr error -} - -func (r *mockSyncRepository) ListApplications( - _ context.Context, - _ repository.ApplicationFilter, - _ repository.Pagination, - _ bool, -) ([]*model.Application, uint64, error) { - return nil, 0, nil -} - -func (r *mockSyncRepository) HasUndrainedEpochsBeforeBlock( - _ context.Context, - _ int64, - _ uint64, -) (bool, error) { - return false, nil -} - -func (r *mockSyncRepository) ListInputs( - ctx context.Context, - _ string, - _ repository.InputFilter, - p repository.Pagination, - _ bool, -) ([]*model.Input, uint64, error) { - if err := ctx.Err(); err != nil { - return nil, 0, err - } - if r.listErr != nil { - return nil, 0, r.listErr - } - start := p.Offset - if start >= uint64(len(r.inputs)) { - return nil, r.totalCount, nil - } - end := start + p.Limit - if p.Limit == 0 || end > uint64(len(r.inputs)) { - end = uint64(len(r.inputs)) - } - return r.inputs[start:end], r.totalCount, nil -} - -func (r *mockSyncRepository) GetLastSnapshot( - _ context.Context, - _ string, -) (*model.Input, error) { - return nil, nil -} - -// newForkableMock creates a mock where Fork returns a fresh mock each time, -// properly exercising the fork/replace lifecycle in Synchronize tests. -func newForkableMock() *MockRollupsMachine { - m := &MockRollupsMachine{} - m.CloseError = nil - m.AdvanceAcceptedReturn = true - m.HashReturn = newHash(1) - m.OutputsHashReturn = newHash(2) - m.ForkFunc = func(_ context.Context) (machine.Machine, error) { - return newForkableMock(), nil - } - return m -} - -func (s *MachineInstanceSuite) newSyncMachine(processedInputs uint64, appProcessedInputs uint64) *MachineInstanceImpl { - runtime := newForkableMock() - - inst := &MachineInstanceImpl{ - application: &model.Application{ - ProcessedInputs: appProcessedInputs, - ExecutionParameters: model.ExecutionParameters{ - AdvanceMaxDeadline: decisecond, - InspectMaxDeadline: centisecond, - MaxConcurrentInspects: 3, - }, - }, - runtime: runtime, - advanceTimeout: decisecond, - inspectTimeout: centisecond, - maxConcurrentInspects: 3, - closeTimeout: defaultCloseTimeout, - mutex: pmutex.New(), - inspectSemaphore: semaphore.NewWeighted(3), - logger: slog.New(slog.NewTextHandler(io.Discard, nil)), - } - inst.processedInputs.Store(processedInputs) - return inst -} - -func makeInputs(startIndex, count uint64) []*model.Input { - inputs := make([]*model.Input, count) - for i := uint64(0); i < count; i++ { - inputs[i] = &model.Input{ - Index: startIndex + i, - EpochIndex: 0, - RawData: []byte{byte(startIndex + i)}, - } - } - return inputs -} - -func (s *MachineInstanceSuite) TestSynchronize() { - s.Run("TemplateSyncAllInputs", func() { - require := s.Require() - inst := s.newSyncMachine(0, 3) - originalRuntime := inst.runtime - repo := &mockSyncRepository{ - inputs: makeInputs(0, 3), - totalCount: 3, - } - - err := inst.Synchronize(context.Background(), repo, 1000) - require.NoError(err) - require.Equal(uint64(3), inst.processedInputs.Load()) - // Verify the runtime was actually replaced (not self-fork) - require.NotSame(originalRuntime, inst.runtime) - }) - - s.Run("SnapshotSyncRemainingInputs", func() { - require := s.Require() - // Snapshot was at index 2, so processedInputs=3, but app has 5 total - inst := s.newSyncMachine(3, 5) - repo := &mockSyncRepository{ - inputs: makeInputs(0, 5), - totalCount: 5, - } - - err := inst.Synchronize(context.Background(), repo, 1000) - require.NoError(err) - require.Equal(uint64(5), inst.processedInputs.Load()) - }) - - s.Run("NoInputsToReplay", func() { - require := s.Require() - inst := s.newSyncMachine(0, 0) - repo := &mockSyncRepository{ - inputs: nil, - totalCount: 0, - } - - err := inst.Synchronize(context.Background(), repo, 1000) - require.NoError(err) - require.Equal(uint64(0), inst.processedInputs.Load()) - }) - - s.Run("SnapshotAlreadyCaughtUp", func() { - require := s.Require() - // Snapshot at last input — nothing to replay - inst := s.newSyncMachine(5, 5) - repo := &mockSyncRepository{ - inputs: makeInputs(0, 5), - totalCount: 5, - } - - err := inst.Synchronize(context.Background(), repo, 1000) - require.NoError(err) - require.Equal(uint64(5), inst.processedInputs.Load()) - }) - - s.Run("MachineAheadOfDB", func() { - require := s.Require() - // Machine has processed 5 inputs but DB only has 3 - inst := s.newSyncMachine(5, 3) - repo := &mockSyncRepository{ - inputs: makeInputs(0, 3), - totalCount: 3, - } - - err := inst.Synchronize(context.Background(), repo, 1000) - require.Error(err) - require.ErrorIs(err, ErrMachineSynchronization) - require.Contains(err.Error(), "machine has processed 5 inputs but DB only has 3") - }) - - s.Run("CountMismatch", func() { - require := s.Require() - inst := s.newSyncMachine(0, 5) - repo := &mockSyncRepository{ - inputs: makeInputs(0, 3), - totalCount: 3, // DB says 3 but app expects 5 - } - - err := inst.Synchronize(context.Background(), repo, 1000) - require.Error(err) - require.ErrorIs(err, ErrMachineSynchronization) - require.Contains(err.Error(), "count mismatch") - }) - - s.Run("ListInputsError", func() { - require := s.Require() - inst := s.newSyncMachine(0, 3) - listErr := errors.New("database connection lost") - repo := &mockSyncRepository{ - listErr: listErr, - } - - err := inst.Synchronize(context.Background(), repo, 1000) - require.Error(err) - require.ErrorIs(err, ErrMachineSynchronization) - require.Contains(err.Error(), "database connection lost") - }) - - s.Run("AdvanceErrorMidReplay", func() { - require := s.Require() - inst := s.newSyncMachine(0, 3) - // Make each fork return a hard error on Advance. - runtime := inst.runtime.(*MockRollupsMachine) - runtime.ForkFunc = func(_ context.Context) (machine.Machine, error) { - fork := newForkableMock() - fork.AdvanceError = errors.New("advance failed during replay") - return fork, nil - } - - repo := &mockSyncRepository{ - inputs: makeInputs(0, 3), - totalCount: 3, - } - - err := inst.Synchronize(context.Background(), repo, 1000) - require.Error(err) - require.ErrorIs(err, ErrMachineSynchronization) - require.Contains(err.Error(), "failed to replay input") - }) - - s.Run("BatchBoundaryCrossing", func() { - require := s.Require() - // Use batchSize=2 with 3 inputs so the loop must fetch two batches - // (batch 1: inputs 0-1, batch 2: input 2), exercising pagination. - inst := s.newSyncMachine(0, 3) - repo := &mockSyncRepository{ - inputs: makeInputs(0, 3), - totalCount: 3, - } - - err := inst.Synchronize(context.Background(), repo, 2) - require.NoError(err) - require.Equal(uint64(3), inst.processedInputs.Load()) - }) - - s.Run("PartialSyncDetected", func() { - require := s.Require() - // Machine has 0 processed, app has 5. But the mock only has 2 inputs, - // simulating rows disappearing between batches. - // Use batchSize=2 so the first batch returns inputs [0,1] (replayed=2), - // then the second batch returns 0 rows (offset=2 >= len=2). - // The loop must detect replayed(2) != toReplay(5) and return an error. - inst := s.newSyncMachine(0, 5) - repo := &mockSyncRepository{ - inputs: makeInputs(0, 2), // only 2 inputs exist - totalCount: 5, // but totalCount says 5 - } - - err := inst.Synchronize(context.Background(), repo, 2) - require.Error(err) - require.ErrorIs(err, ErrMachineSynchronization) - require.Contains(err.Error(), "expected to replay 5 inputs but only replayed 2") - }) - - s.Run("ContextCancellation", func() { - require := s.Require() - inst := s.newSyncMachine(0, 3) - - ctx, cancel := context.WithCancel(context.Background()) - cancel() // Cancel immediately - - repo := &mockSyncRepository{ - inputs: makeInputs(0, 3), - totalCount: 3, - } - - err := inst.Synchronize(ctx, repo, 1000) - require.Error(err) - }) -} - // ------------------------------------------------------------------------------------------------ type MockRollupsMachine struct { @@ -1465,25 +1332,28 @@ type MockRollupsMachine struct { HashReturn machine.Hash HashError error - - AdvanceAcceptedReturn bool - AdvanceOutputsReturn []machine.Output - AdvanceReportsReturn []machine.Report - AdvanceLeafsReturn []machine.Hash - AdvanceRemainingReturn uint64 - OutputsHashReturn machine.Hash - OutputsHashError error - OutputsHashProofReturn []machine.Hash - OutputsHashProofError error - AdvanceError error - - InspectAcceptedReturn bool - InspectReportsReturn []machine.Report + HashCalls atomic.Int64 + + CompletionStatusReturn machine.CompletionStatus + ExceptionDataReturn []byte + AdvanceOutputsReturn []machine.Output + AdvanceReportsReturn []machine.Report + AdvanceLeafsReturn []machine.Hash + AdvanceRemainingReturn uint64 + OutputsHashReturn machine.Hash + OutputsHashError error + OutputsHashProofReturn []machine.Hash + OutputsHashProofError error + AdvanceError error + LastAdvanceComputeHashes bool + + InspectResponseReturn *machine.InspectResponse InspectError error StoreError error CloseError error + CloseCalls atomic.Int64 } func (m *MockRollupsMachine) Fork(ctx context.Context) (machine.Machine, error) { @@ -1494,6 +1364,7 @@ func (m *MockRollupsMachine) Fork(ctx context.Context) (machine.Machine, error) } func (m *MockRollupsMachine) Hash(_ context.Context) (machine.Hash, error) { + m.HashCalls.Add(1) return m.HashReturn, m.HashError } @@ -1505,19 +1376,24 @@ func (m *MockRollupsMachine) OutputsHashProof(_ context.Context) ([]machine.Hash return m.OutputsHashProofReturn, m.OutputsHashProofError } -func (m *MockRollupsMachine) Advance(_ context.Context, _ []byte, _ machine.Hash, _ bool) (*machine.AdvanceResponse, error) { +func (m *MockRollupsMachine) Advance(_ context.Context, _ []byte, _ machine.Hash, computeHashes bool) (*machine.AdvanceResponse, error) { + m.LastAdvanceComputeHashes = computeHashes + if m.AdvanceError != nil { + return nil, m.AdvanceError + } return &machine.AdvanceResponse{ - Accepted: m.AdvanceAcceptedReturn, - Outputs: m.AdvanceOutputsReturn, - Reports: m.AdvanceReportsReturn, - Hashes: m.AdvanceLeafsReturn, - RemainingCycles: m.AdvanceRemainingReturn, - OutputsHash: m.OutputsHashReturn, - }, m.AdvanceError + Status: m.CompletionStatusReturn, + ExceptionData: m.ExceptionDataReturn, + Outputs: m.AdvanceOutputsReturn, + Reports: m.AdvanceReportsReturn, + PeriodicStateHashes: m.AdvanceLeafsReturn, + PaddingRepetitions: m.AdvanceRemainingReturn, + OutputsHash: m.OutputsHashReturn, + }, nil } -func (m *MockRollupsMachine) Inspect(_ context.Context, _ []byte) (bool, []machine.Report, error) { - return m.InspectAcceptedReturn, m.InspectReportsReturn, m.InspectError +func (m *MockRollupsMachine) Inspect(_ context.Context, _ []byte) (*machine.InspectResponse, error) { + return m.InspectResponseReturn, m.InspectError } func (m *MockRollupsMachine) Store(_ context.Context, _ string) error { @@ -1525,6 +1401,7 @@ func (m *MockRollupsMachine) Store(_ context.Context, _ string) error { } func (m *MockRollupsMachine) Close() error { + m.CloseCalls.Add(1) return m.CloseError } diff --git a/internal/manager/manager.go b/internal/manager/manager.go index ce82f9271..fc8fb921e 100644 --- a/internal/manager/manager.go +++ b/internal/manager/manager.go @@ -8,68 +8,84 @@ import ( "errors" "fmt" "log/slog" + "math" "os" "sort" "sync" + "github.com/cartesi/rollups-node/internal/appstatus" . "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/internal/replay" "github.com/cartesi/rollups-node/internal/repository" + "github.com/cartesi/rollups-node/pkg/machine" "github.com/ethereum/go-ethereum/common" ) var ( - ErrApplicationNotFound = errors.New("application not found") - ErrMachineCreation = errors.New("failed to create machine") - ErrMachineSynchronization = errors.New("failed to synchronize machine") + ErrApplicationNotFound = errors.New("application not found") + ErrMachineCreation = errors.New("failed to create machine") ) // MachineRepository defines the repository interface needed by the MachineManager type MachineRepository interface { + repository.ReplayRepository + // ListApplications retrieves applications based on filter criteria ListApplications(ctx context.Context, f repository.ApplicationFilter, p repository.Pagination, descending bool) ([]*Application, uint64, error) HasUndrainedEpochsBeforeBlock(ctx context.Context, appID int64, blockBound uint64) (bool, error) - // ListInputs retrieves inputs based on filter criteria - ListInputs(ctx context.Context, nameOrAddress string, f repository.InputFilter, p repository.Pagination, descending bool) ([]*Input, uint64, error) - // GetLastSnapshot retrieves the most recent input with a snapshot for the given application GetLastSnapshot(ctx context.Context, nameOrAddress string) (*Input, error) + GetApplication(ctx context.Context, nameOrAddress string) (*Application, error) + + // UpdateApplicationStatus persists an application's health status. + UpdateApplicationStatus(ctx context.Context, appID int64, status ApplicationStatus, reason *string) error } // MachineInstanceFactory creates MachineInstance values from applications. // Implementations decide whether to load from a template or snapshot. type MachineInstanceFactory interface { - NewFromTemplate(ctx context.Context, app *Application, logger *slog.Logger, checkHash bool) (MachineInstance, error) - NewFromSnapshot(ctx context.Context, app *Application, logger *slog.Logger, checkHash bool, - snapshotPath string, machineHash *common.Hash, inputIndex uint64) (MachineInstance, error) + NewFromTemplate(ctx context.Context, app *Application, logger *slog.Logger, checkTemplateHash bool) (MachineInstance, error) + NewFromSnapshot(ctx context.Context, app *Application, logger *slog.Logger, + snapshotPath string, expectedHash common.Hash, inputIndex uint64) (MachineInstance, error) } // DefaultMachineInstanceFactory delegates to NewMachineInstance / NewMachineInstanceFromSnapshot. type DefaultMachineInstanceFactory struct{} func (f *DefaultMachineInstanceFactory) NewFromTemplate( - ctx context.Context, app *Application, logger *slog.Logger, checkHash bool, + ctx context.Context, app *Application, logger *slog.Logger, checkTemplateHash bool, ) (MachineInstance, error) { - return NewMachineInstance(ctx, app, logger, checkHash) + return NewMachineInstance(ctx, app, logger, checkTemplateHash) } func (f *DefaultMachineInstanceFactory) NewFromSnapshot( - ctx context.Context, app *Application, logger *slog.Logger, checkHash bool, - snapshotPath string, machineHash *common.Hash, inputIndex uint64, + ctx context.Context, app *Application, logger *slog.Logger, + snapshotPath string, expectedHash common.Hash, inputIndex uint64, ) (MachineInstance, error) { - return NewMachineInstanceFromSnapshot(ctx, app, logger, checkHash, snapshotPath, machineHash, inputIndex) + return NewMachineInstanceFromSnapshot(ctx, app, logger, snapshotPath, expectedHash, inputIndex) } // MachineManager manages the lifecycle of machine instances for applications type MachineManager struct { - mutex sync.RWMutex - machines map[int64]MachineInstance - closed bool - repository MachineRepository - checkHash bool - inputBatchSize uint64 - logger *slog.Logger - instanceFactory MachineInstanceFactory + mutex sync.RWMutex + machines map[int64]MachineInstance + pendingApplicationFailures map[int64]*pendingApplicationFailure + closed bool + repository MachineRepository + checkTemplateHash bool + inputBatchSize uint64 + logger *slog.Logger + instanceFactory MachineInstanceFactory + replayRun func(context.Context, repository.ReplayRepository, replay.Executor, replay.Options) (replay.Result, error) +} + +// pendingApplicationFailure fences an application after a FAILED status write +// cannot be confirmed. It is deliberately private: this is a short-lived retry +// queue, not process-local application health state. +type pendingApplicationFailure struct { + application *Application + reason string } // Option configures a MachineManager. @@ -80,21 +96,207 @@ func WithInstanceFactory(f MachineInstanceFactory) Option { return func(m *MachineManager) { m.instanceFactory = f } } +// withReplayRun overrides replay execution for manager policy tests. +func withReplayRun( + run func(context.Context, repository.ReplayRepository, replay.Executor, replay.Options) (replay.Result, error), +) Option { + return func(m *MachineManager) { m.replayRun = run } +} + +func snapshotProcessedInputs(app *Application, snapshot *Input) (uint64, error) { + if snapshot.Index == math.MaxUint64 { + return 0, fmt.Errorf("%w: snapshot input index cannot be incremented", ErrInvalidSnapshotPoint) + } + processedInputs := snapshot.Index + 1 + if processedInputs > app.ProcessedInputs { + return 0, fmt.Errorf( + "%w: snapshot represents %d processed inputs, application has %d", + ErrInvalidSnapshotPoint, processedInputs, app.ProcessedInputs, + ) + } + return processedInputs, nil +} + +func closeMachineCandidate( + logger *slog.Logger, + candidate MachineInstance, + failureMessage string, +) { + if candidate == nil { + return + } + if err := candidate.Close(); err != nil { + logger.Warn(failureMessage, "error", err) + } +} + +func (m *MachineManager) tryLoadSnapshotInstance( + ctx context.Context, + app *Application, + logger *slog.Logger, +) MachineInstance { + snapshot, err := m.repository.GetLastSnapshot(ctx, app.IApplicationAddress.String()) + if err != nil { + // Shutdown cancels the query mid-flight; deadlines and repository + // failures still require operator attention. + if errors.Is(err, context.Canceled) { + logger.Debug("GetLastSnapshot canceled during shutdown", "error", err) + } else { + logger.Error("Failed to find latest snapshot", "error", err) + } + return nil + } + if snapshot == nil || snapshot.SnapshotURI == nil { + return nil + } + + snapshotLogger := logger.With( + "snapshot", *snapshot.SnapshotURI, + "input_index", snapshot.Index, + ) + if snapshot.MachineHash == nil { + snapshotLogger.Warn("Snapshot input has no persisted machine hash; falling back to template") + return nil + } + + expectedProcessedInputs, err := snapshotProcessedInputs(app, snapshot) + if err != nil { + snapshotLogger.Warn( + "Snapshot input is not a valid application starting point; falling back to template", + "app_processed_inputs", app.ProcessedInputs, + "error", err, + ) + return nil + } + + if _, err := os.Stat(*snapshot.SnapshotURI); err != nil { + if errors.Is(err, os.ErrNotExist) { + snapshotLogger.Warn("Snapshot path does not exist") + } else { + snapshotLogger.Error("Failed to access snapshot path", "error", err) + } + return nil + } + + snapshotLogger.Info("Creating machine instance from snapshot") + candidate, err := m.instanceFactory.NewFromSnapshot( + ctx, app, m.logger, + *snapshot.SnapshotURI, *snapshot.MachineHash, snapshot.Index, + ) + if err != nil { + snapshotLogger.Error("Failed to create machine instance from snapshot", "error", err) + closeMachineCandidate(snapshotLogger, candidate, "Failed to close partial snapshot machine") + return nil + } + if candidate == nil { + snapshotLogger.Error("Snapshot factory returned no machine instance") + return nil + } + if candidate.ProcessedInputs() != expectedProcessedInputs { + snapshotLogger.Error("Snapshot machine has an invalid processed-input count", + "expected_processed_inputs", expectedProcessedInputs, + "actual_processed_inputs", candidate.ProcessedInputs(), + ) + closeMachineCandidate(snapshotLogger, candidate, "Failed to close invalid snapshot machine") + return nil + } + + // The default snapshot factory verifies this hash while loading. Verify it + // again at the manager boundary because tests and alternative factories are + // explicit injection seams and are not required to duplicate that policy. + actualHash, err := candidate.Hash(ctx) + if err != nil { + snapshotLogger.Error("Failed to verify snapshot machine hash", "error", err) + closeMachineCandidate(snapshotLogger, candidate, "Failed to close unverifiable snapshot machine") + return nil + } + if common.Hash(actualHash) != *snapshot.MachineHash { + snapshotLogger.Error("Snapshot machine hash mismatch", + "expected_hash", snapshot.MachineHash.Hex(), + "actual_hash", common.Hash(actualHash).Hex(), + ) + closeMachineCandidate(snapshotLogger, candidate, "Failed to close mismatched snapshot machine") + return nil + } + return candidate +} + +func (m *MachineManager) tryLoadTemplateInstance( + ctx context.Context, + app *Application, + logger *slog.Logger, +) MachineInstance { + candidate, err := m.instanceFactory.NewFromTemplate(ctx, app, m.logger, m.checkTemplateHash) + if err != nil { + if errors.Is(err, context.Canceled) { + logger.Debug("NewFromTemplate canceled during shutdown", "error", err) + } else { + logger.Error("Failed to create machine instance", "error", err) + } + closeMachineCandidate(logger, candidate, "Failed to close partial template machine") + return nil + } + if candidate == nil { + logger.Error("Template factory returned no machine instance") + return nil + } + return candidate +} + +func replayContradictionReason(err error) string { + reason := replay.ErrContradiction.Error() + var detail *replay.ContradictionError + if errors.As(err, &detail) { + // ContradictionError represents byte values only as lengths and + // digests, so its text is safe to persist for operator diagnostics. + reason = detail.Error() + } + return reason +} + +func classifyReplayFailure(err error) (reason, logMessage string, shouldFence bool) { + switch { + case errors.Is(err, replay.ErrContradiction): + // A contradiction proves that this local reconstruction is incompatible + // with the stored result, but not that the database itself is corrupt. A + // repaired template or runtime may recover it, so fence it as FAILED. An + // unresolved contradiction is detected and fenced again on re-enable. + return replayContradictionReason(err), + "Machine replay contradicts stored input result; marking application failed", true + case machine.IsExecutionLimitError(err): + return fmt.Sprintf("machine replay reached execution limit: %v", err), + "Machine replay reached an execution limit; marking application failed", true + case errors.Is(err, machine.ErrDeadlineExceeded): + return fmt.Sprintf("machine replay exceeded execution deadline: %v", err), + "Machine replay exceeded its execution deadline; marking application failed", true + case errors.Is(err, ErrIncompleteAdvance): + return fmt.Sprintf("machine replay returned an incomplete advance result: %v", err), + "Machine replay returned an incomplete advance result; marking application failed", true + case errors.Is(err, machine.ErrMachineInternal): + return fmt.Sprintf("machine replay failed internally: %v", err), + "Machine replay failed internally; marking application failed", true + default: + return "", "Failed to replay machine", false + } +} + // NewMachineManager creates a new machine manager. func NewMachineManager( repo MachineRepository, logger *slog.Logger, - checkHash bool, + checkTemplateHash bool, inputBatchSize uint64, opts ...Option, ) *MachineManager { m := &MachineManager{ - machines: map[int64]MachineInstance{}, - repository: repo, - checkHash: checkHash, - inputBatchSize: inputBatchSize, - logger: logger, - instanceFactory: &DefaultMachineInstanceFactory{}, + machines: map[int64]MachineInstance{}, + pendingApplicationFailures: map[int64]*pendingApplicationFailure{}, + repository: repo, + checkTemplateHash: checkTemplateHash, + inputBatchSize: inputBatchSize, + logger: logger, + instanceFactory: &DefaultMachineInstanceFactory{}, + replayRun: replay.Run, } for _, opt := range opts { opt(m) @@ -105,9 +307,20 @@ func NewMachineManager( // UpdateMachines refreshes the list of machines based on applications that // still need local machine work. func (m *MachineManager) UpdateMachines(ctx context.Context) error { - apps, _, err := getMachineApplications(ctx, m.repository) + // Retry failed status writes before doing any more machine work. Every app + // returned here remains fenced for this entire update, including when the + // retry succeeds but a repository read observes a stale executable row. + fenced, persistenceErr := m.persistPendingApplicationFailures(ctx) + if ctxErr := ctx.Err(); ctxErr != nil { + return errors.Join(persistenceErr, ctxErr) + } + if persistenceErr != nil && !IsOnlyApplicationFailurePersistenceErrors(persistenceErr) { + return persistenceErr + } + + apps, err := getMachineApplications(ctx, m.repository) if err != nil { - return err + return errors.Join(persistenceErr, err) } // Create machines for new applications @@ -115,95 +328,53 @@ func (m *MachineManager) UpdateMachines(ctx context.Context) error { if m.HasMachine(app.ID) { continue } + if _, pending := fenced[app.ID]; pending { + continue + } - m.logger.Info("Creating new machine instance", + appLogger := m.logger.With( "application", app.Name, - "address", app.IApplicationAddress) - - // Check if we have a snapshot to load from - var instance MachineInstance - - // Find the latest snapshot for this application - snapshot, err := m.repository.GetLastSnapshot(ctx, app.IApplicationAddress.String()) - if err != nil { - // Shutdown cancels the ctx mid-query; downgrade to Debug so - // operators don't see spurious ERR lines during a graceful - // stop. DeadlineExceeded would still flow through the Error - // branch and demand investigation. - if errors.Is(err, context.Canceled) { - m.logger.Debug("GetLastSnapshot canceled during shutdown", - "application", app.Name, - "error", err) - } else { - m.logger.Error("Failed to find latest snapshot", - "application", app.Name, - "error", err) - } - // Continue with template-based initialization - } + "address", app.IApplicationAddress, + ) + appLogger.Info("Creating new machine instance") - if snapshot != nil && snapshot.SnapshotURI != nil { - // Verify the snapshot path exists - if _, statErr := os.Stat(*snapshot.SnapshotURI); statErr == nil { - m.logger.Info("Creating machine instance from snapshot", - "application", app.Name, - "snapshot", *snapshot.SnapshotURI) - - instance, err = m.instanceFactory.NewFromSnapshot( - ctx, app, m.logger, m.checkHash, - *snapshot.SnapshotURI, snapshot.MachineHash, snapshot.Index) - if err != nil { - m.logger.Error("Failed to create machine instance from snapshot", - "application", app.Name, - "snapshot", *snapshot.SnapshotURI, - "error", err) - // Fall back to template-based initialization below - } - } else if errors.Is(statErr, os.ErrNotExist) { - m.logger.Warn("Snapshot path does not exist", - "application", app.Name, - "snapshot", *snapshot.SnapshotURI) - } else { - m.logger.Error("Failed to access snapshot path", - "application", app.Name, - "snapshot", *snapshot.SnapshotURI, - "error", statErr) - } + // Prefer a verified snapshot and fall back to the template whenever the + // snapshot is absent, stale, inaccessible, or invalid. + instance := m.tryLoadSnapshotInstance(ctx, app, appLogger) + if instance == nil { + instance = m.tryLoadTemplateInstance(ctx, app, appLogger) } - - // Fall back to template if snapshot loading failed or was unavailable if instance == nil { - instance, err = m.instanceFactory.NewFromTemplate(ctx, app, m.logger, m.checkHash) - if err != nil { - // Shutdown cancels the ctx mid-spawn; the partially - // constructed machine is torn down by NewFromTemplate - // itself. Downgrade to Debug for the graceful-stop case - // so the noise doesn't drown out real spawn failures. - if errors.Is(err, context.Canceled) { - m.logger.Debug("NewFromTemplate canceled during shutdown", - "application", app.IApplicationAddress, - "error", err) - } else { - m.logger.Error("Failed to create machine instance", - "application", app.IApplicationAddress, - "error", err) - } - continue - } + continue } - // Synchronize the machine with processed inputs. - // For template instances (processedInputs=0) this replays all inputs. - // For snapshot instances (processedInputs=snapshotIndex+1) this replays - // only inputs after the snapshot. - err = instance.Synchronize(ctx, m.repository, m.inputBatchSize) + // Canonical verification is the normal reconstruction policy: replay all + // completed inputs after the candidate's starting point and compare each + // status, exception payload, machine root, and cumulative outputs root. + _, err = m.replayRun(ctx, m.repository, instance, replay.Options{ + Application: app, + FromInput: instance.ProcessedInputs(), + ToInputExclusive: app.ProcessedInputs, + BatchSize: m.inputBatchSize, + Verification: repository.ReplayVerificationCanonical, + }) if err != nil { - m.logger.Error("Failed to synchronize machine", - "application", app.IApplicationAddress, - "error", err) + var appPersistenceErr error + reason, logMessage, shouldFence := classifyReplayFailure(err) + appLogger.Error(logMessage, "error", err) + if shouldFence { + m.FenceApplicationFailure(app, reason) + fenced[app.ID] = struct{}{} + appPersistenceErr = m.persistApplicationFailure(ctx, app.ID) + } if err := instance.Close(); err != nil { - m.logger.Warn("Failed to close machine after synchronization failure", - "application", app.Name, "error", err) + appLogger.Warn("Failed to close machine after replay failure", "error", err) + } + if appPersistenceErr != nil { + persistenceErr = errors.Join(persistenceErr, appPersistenceErr) + } + if ctxErr := ctx.Err(); ctxErr != nil { + return errors.Join(persistenceErr, ctxErr) } continue } @@ -211,18 +382,148 @@ func (m *MachineManager) UpdateMachines(ctx context.Context) error { // Add the machine to the manager; close if it fails if !m.addMachine(app.ID, instance) { if err := instance.Close(); err != nil { - m.logger.Warn("Failed to close duplicate machine instance", - "application", app.Name, "error", err) + appLogger.Warn("Failed to close duplicate machine instance", "error", err) } } } // Remove machines for non-enabled applications (disabled, failed, etc.) - m.removeMachines(apps) + m.removeMachines(excludeFencedApplications(apps, fenced)) + + return persistenceErr +} + +func excludeFencedApplications(apps []*Application, fenced map[int64]struct{}) []*Application { + if len(fenced) == 0 { + return apps + } + active := make([]*Application, 0, len(apps)) + for _, app := range apps { + if _, excluded := fenced[app.ID]; !excluded { + active = append(active, app) + } + } + return active +} + +// FenceApplicationFailure immediately fences app and queues the exact +// normalized FAILED reason for a later durability retry. Callers use this only +// after their initial status write failed; this method does not write the +// repository itself. +func (m *MachineManager) FenceApplicationFailure(app *Application, reason string) { + reason = appstatus.NormalizeReason(reason) + pending := &pendingApplicationFailure{ + application: app, + reason: reason, + } + m.mutex.Lock() + defer m.mutex.Unlock() + if _, exists := m.pendingApplicationFailures[app.ID]; !exists { + m.pendingApplicationFailures[app.ID] = pending + } +} +// persistPendingApplicationFailures retries every pending FAILED write and +// returns the application IDs that must remain fenced for this update. +func (m *MachineManager) persistPendingApplicationFailures( + ctx context.Context, +) (map[int64]struct{}, error) { + m.mutex.RLock() + appIDs := make([]int64, 0, len(m.pendingApplicationFailures)) + for appID := range m.pendingApplicationFailures { + appIDs = append(appIDs, appID) + } + m.mutex.RUnlock() + sort.Slice(appIDs, func(i, j int) bool { return appIDs[i] < appIDs[j] }) + + fenced := make(map[int64]struct{}, len(appIDs)) + var persistenceErrors []error + for _, appID := range appIDs { + fenced[appID] = struct{}{} + if ctxErr := ctx.Err(); ctxErr != nil { + return fenced, errors.Join(errors.Join(persistenceErrors...), ctxErr) + } + if err := m.persistApplicationFailure(ctx, appID); err != nil { + persistenceErrors = append(persistenceErrors, err) + } + if ctxErr := ctx.Err(); ctxErr != nil { + return fenced, errors.Join(errors.Join(persistenceErrors...), ctxErr) + } + } + return fenced, errors.Join(persistenceErrors...) +} + +func (m *MachineManager) persistApplicationFailure(ctx context.Context, appID int64) error { + m.mutex.RLock() + pending := m.pendingApplicationFailures[appID] + m.mutex.RUnlock() + if pending == nil { + return nil + } + + if writeErr := appstatus.SetFailed(ctx, m.logger, m.repository, pending.application, pending.reason); writeErr != nil { + if errors.Is(writeErr, repository.ErrNotFound) { + m.deletePendingApplicationFailure(appID, pending) + return nil + } + // Another process may have installed a stronger terminal status, or the + // write may have committed even though its result was lost. Retire the + // fence only when the durable row proves one of those outcomes. An + // unrelated FAILED reason remains fenced: it does not prove that this + // application failure was recorded. Unknown/read failures and OK + // (including disabled+OK) also remain pending. + current, readErr := m.repository.GetApplication( + ctx, pending.application.IApplicationAddress.String(), + ) + retire := errors.Is(readErr, repository.ErrNotFound) + if readErr == nil { + retire = current == nil || + current.ID != pending.application.ID || + applicationFailureAlreadyResolved(current, pending.reason) + } + if retire { + m.deletePendingApplicationFailure(appID, pending) + return nil + } + return &ApplicationFailurePersistenceError{ + ApplicationID: appID, + WriteErr: writeErr, + ReadErr: readErr, + } + } + + m.deletePendingApplicationFailure(appID, pending) return nil } +func applicationFailureAlreadyResolved(app *Application, reason string) bool { + if app == nil { + return false + } + switch app.Status { + case ApplicationStatus_Diverged, ApplicationStatus_Corrupted: + return true + case ApplicationStatus_Failed: + return app.Reason != nil && *app.Reason == reason + case ApplicationStatus_OK: + return false + default: + return false + } +} + +func (m *MachineManager) deletePendingApplicationFailure( + appID int64, + pending *pendingApplicationFailure, +) { + m.mutex.Lock() + // Remove only the pending record that this persistence attempt observed. + if m.pendingApplicationFailures[appID] == pending { + delete(m.pendingApplicationFailures, appID) + } + m.mutex.Unlock() +} + // GetMachine retrieves a machine instance for an application func (m *MachineManager) GetMachine(appID int64) (MachineInstance, bool) { m.mutex.RLock() @@ -239,6 +540,14 @@ func (m *MachineManager) HasMachine(appID int64) bool { return exists } +// HasPendingApplicationFailures reports whether any local machine work is +// fenced while its durable application status remains unconfirmed. +func (m *MachineManager) HasPendingApplicationFailures() bool { + m.mutex.RLock() + defer m.mutex.RUnlock() + return len(m.pendingApplicationFailures) != 0 +} + // addMachine adds a machine to the manager. // Returns false if the manager is closed or the appID already exists. func (m *MachineManager) addMachine(appID int64, machine MachineInstance) bool { @@ -337,17 +646,16 @@ func (m *MachineManager) Close() error { return errors.Join(errs...) } -func getMachineApplications(ctx context.Context, repo MachineRepository) ([]*Application, uint64, error) { - apps, total, err := repo.ListApplications(ctx, repository.ExecutableApplicationsFilter(), repository.Pagination{}, false) +func getMachineApplications(ctx context.Context, repo MachineRepository) ([]*Application, error) { + apps, _, err := repo.ListApplications(ctx, repository.ExecutableApplicationsFilter(), repository.Pagination{}, false) if err != nil { - return nil, 0, err + return nil, err } - foreclosedApps, foreclosedTotal, err := repo.ListApplications(ctx, foreclosedMachineDrainFilter(), repository.Pagination{}, false) + foreclosedApps, _, err := repo.ListApplications(ctx, foreclosedMachineDrainFilter(), repository.Pagination{}, false) if err != nil { - return nil, 0, err + return nil, err } - total += foreclosedTotal for _, app := range foreclosedApps { if app.ForecloseBlock == 0 { continue @@ -364,13 +672,13 @@ func getMachineApplications(ctx context.Context, repo MachineRepository) ([]*App } undrained, err := repo.HasUndrainedEpochsBeforeBlock(ctx, app.ID, app.ForecloseBlock) if err != nil { - return nil, 0, err + return nil, err } if undrained { apps = append(apps, app) } } - return apps, total, nil + return apps, nil } func foreclosedMachineDrainFilter() repository.ApplicationFilter { @@ -380,14 +688,3 @@ func foreclosedMachineDrainFilter() repository.ApplicationFilter { ForeclosureRecorded: new(true), } } - -// getProcessedInputs retrieves processed inputs with pagination support. -func getProcessedInputs( - ctx context.Context, - repo MachineRepository, - appAddress string, - p repository.Pagination, -) ([]*Input, uint64, error) { - f := repository.InputFilter{NotStatus: Pointer(InputCompletionStatus_None)} - return repo.ListInputs(ctx, appAddress, f, p, false) -} diff --git a/internal/manager/manager_test.go b/internal/manager/manager_test.go index c6a16dae7..9fd6665ff 100644 --- a/internal/manager/manager_test.go +++ b/internal/manager/manager_test.go @@ -6,14 +6,23 @@ package manager import ( "context" "errors" + "fmt" "io" "log/slog" + "math" + "math/big" + "strings" "testing" + "time" + "github.com/cartesi/rollups-node/internal/appstatus" "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/internal/replay" "github.com/cartesi/rollups-node/internal/repository" + "github.com/cartesi/rollups-node/pkg/machine" "github.com/ethereum/go-ethereum/common" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) @@ -25,17 +34,923 @@ type MachineManagerSuite struct { suite.Suite } +func newForkableMock() *MockRollupsMachine { + runtime := &MockRollupsMachine{ + CompletionStatusReturn: machine.CompletionStatusAccepted, + HashReturn: newHash(1), + OutputsHashReturn: newHash(2), + } + runtime.ForkFunc = func(context.Context) (machine.Machine, error) { + return newForkableMock(), nil + } + return runtime +} + +func newTestMachineManager( + repo MachineRepository, + logger *slog.Logger, + checkTemplateHash bool, + inputBatchSize uint64, + opts ...Option, +) *MachineManager { + testRun := withReplayRun(func( + _ context.Context, + _ repository.ReplayRepository, + executor replay.Executor, + _ replay.Options, + ) (replay.Result, error) { + if instance, ok := executor.(*DummyMachineInstanceMock); ok { + instance.replayCalls++ + return replay.Result{}, instance.replayErr + } + return replay.Result{}, nil + }) + return NewMachineManager( + repo, logger, checkTemplateHash, inputBatchSize, append(opts, testRun)..., + ) +} + +type nilSingleUnwrapperError struct{} + +func (nilSingleUnwrapperError) Error() string { return "nil single unwrapper" } +func (nilSingleUnwrapperError) Unwrap() error { return nil } + +type emptyMultiUnwrapperError struct{} + +func (emptyMultiUnwrapperError) Error() string { return "empty multi unwrapper" } +func (emptyMultiUnwrapperError) Unwrap() []error { return []error{} } + func (s *MachineManagerSuite) TestNewMachineManager() { require := s.Require() repo := &MockMachineRepository{} testLogger := slog.New(slog.NewTextHandler(io.Discard, nil)) - manager := NewMachineManager(repo, testLogger, false, 500) + manager := newTestMachineManager(repo, testLogger, false, 500) require.NotNil(manager) require.Empty(manager.machines) require.Equal(repo, manager.repository) } +func (s *MachineManagerSuite) TestUpdateMachinesUsesCanonicalReplayPolicy() { + require := s.Require() + app := &model.Application{ + ID: 47, + Name: "ReplayPolicy", + IApplicationAddress: common.HexToAddress("0x47"), + Status: model.ApplicationStatus_OK, + ProcessedInputs: 7, + } + repo := &MockMachineRepository{} + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && !*f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{app}, uint64(1), nil).Once() + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && *f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{}, uint64(0), nil).Once() + repo.On("GetLastSnapshot", mock.Anything, app.IApplicationAddress.String()).Return(nil, nil).Once() + + instance := &DummyMachineInstanceMock{application: app, processedInputs: 3} + factory := &MockMachineInstanceFactory{Instance: instance} + var capturedSource repository.ReplayRepository + var capturedExecutor replay.Executor + var capturedOptions replay.Options + run := func( + _ context.Context, + source repository.ReplayRepository, + executor replay.Executor, + options replay.Options, + ) (replay.Result, error) { + capturedSource = source + capturedExecutor = executor + capturedOptions = options + return replay.Result{ReplayedInputs: options.ToInputExclusive - options.FromInput}, nil + } + manager := NewMachineManager( + repo, + slog.New(slog.NewTextHandler(io.Discard, nil)), + false, + 23, + WithInstanceFactory(factory), + withReplayRun(run), + ) + + require.NoError(manager.UpdateMachines(context.Background())) + require.Same(repo, capturedSource) + require.Same(instance, capturedExecutor) + require.Same(app, capturedOptions.Application) + require.Equal(uint64(3), capturedOptions.FromInput) + require.Equal(uint64(7), capturedOptions.ToInputExclusive) + require.Equal(uint64(23), capturedOptions.BatchSize) + require.Equal(repository.ReplayVerificationCanonical, capturedOptions.Verification) + require.True(manager.HasMachine(app.ID)) + repo.AssertExpectations(s.T()) + require.NoError(manager.Close()) +} + +func (s *MachineManagerSuite) TestIsOnlyApplicationFailurePersistenceErrors() { + localFailure := func(applicationID int64) error { + return &ApplicationFailurePersistenceError{ + ApplicationID: applicationID, + WriteErr: errors.New("repository unavailable"), + } + } + wrap := func(message string, err error) error { + return fmt.Errorf("%s: %w", message, err) + } + + tests := []struct { + name string + err error + want bool + }{ + { + name: "nil", + err: nil, + want: false, + }, + { + name: "plain non-wrapping error", + err: errors.New("global repository failure"), + want: false, + }, + { + name: "single unwrapper returning nil", + err: nilSingleUnwrapperError{}, + want: false, + }, + { + name: "multi unwrapper returning empty slice", + err: emptyMultiUnwrapperError{}, + want: false, + }, + { + name: "single wrapper", + err: wrap("update machines", localFailure(1)), + want: true, + }, + { + name: "multiple wrappers", + err: wrap("advancer step", wrap("update machines", localFailure(2))), + want: true, + }, + { + name: "nested joins of wrapped local failures", + err: errors.Join( + wrap("first application", localFailure(3)), + errors.Join( + wrap("second application", wrap("status persistence", localFailure(4))), + wrap("third application", localFailure(5)), + ), + ), + want: true, + }, + { + name: "mixed join with global failure", + err: errors.Join( + wrap("application", localFailure(6)), + errors.Join( + wrap("another application", localFailure(7)), + errors.New("global repository failure"), + ), + ), + want: false, + }, + } + + for _, test := range tests { + s.Run(test.name, func() { + s.Equal(test.want, IsOnlyApplicationFailurePersistenceErrors(test.err)) + }) + } +} + func (s *MachineManagerSuite) TestUpdateMachines() { + s.Run("ReplayContradictionClosesAndDoesNotRegisterMachine", func() { + require := s.Require() + repo := &MockMachineRepository{} + app := &model.Application{ + ID: 71, + Name: "ReplayMismatch", + IApplicationAddress: common.HexToAddress("0x71"), + Enabled: true, + Status: model.ApplicationStatus_OK, + } + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && !*f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{app}, uint64(1), nil).Once() + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && *f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{}, uint64(0), nil).Once() + repo.On("GetLastSnapshot", mock.Anything, app.IApplicationAddress.String()).Return(nil, nil).Once() + const expectedReason = "replay contradicts persisted canonical result: " + + "app=\"ReplayMismatch\" epoch= input=0 " + + "field=machine_hash expected= actual=" + repo.On("UpdateApplicationStatus", mock.Anything, app.ID, model.ApplicationStatus_Failed, + mock.MatchedBy(func(reason *string) bool { + return reason != nil && *reason == expectedReason + })).Return(nil).Once() + + instance := &DummyMachineInstanceMock{ + application: app, + replayErr: &replay.ContradictionError{Application: app.Name, Field: "machine_hash"}, + } + factory := &MockMachineInstanceFactory{Instance: instance} + manager := newTestMachineManager( + repo, + slog.New(slog.NewTextHandler(io.Discard, nil)), + false, + 10, + WithInstanceFactory(factory), + ) + + require.NoError(manager.UpdateMachines(context.Background())) + require.Equal(1, instance.closeCalls) + require.Equal(1, factory.TemplateCalls) + require.False(manager.HasMachine(app.ID)) + require.Empty(manager.Applications()) + require.Equal(model.ApplicationStatus_Failed, app.Status) + require.NotNil(app.Reason) + require.Empty(manager.pendingApplicationFailures) + require.False(manager.HasPendingApplicationFailures()) + repo.AssertExpectations(s.T()) + + // A fresh manager sees no executable row once FAILED is durable, so a + // process restart cannot replay the contradictory history again. + restartRepo := &MockMachineRepository{} + restartRepo.On("ListApplications", mock.Anything, mock.Anything, repository.Pagination{}, false). + Return([]*model.Application{}, uint64(0), nil).Twice() + restartFactory := &MockMachineInstanceFactory{} + restarted := newTestMachineManager( + restartRepo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + WithInstanceFactory(restartFactory), + ) + require.NoError(restarted.UpdateMachines(context.Background())) + require.Zero(restartFactory.TemplateCalls) + restartRepo.AssertExpectations(s.T()) + }) + + s.Run("ApplicationFailureStatusWriteRetriesWithoutReplay", func() { + require := s.Require() + repo := &MockMachineRepository{} + app := &model.Application{ + ID: 75, Name: "RetryFailedStatus", IApplicationAddress: common.HexToAddress("0x75"), + Enabled: true, Status: model.ApplicationStatus_OK, + } + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && !*f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{app}, uint64(1), nil).Twice() + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && *f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{}, uint64(0), nil).Twice() + repo.On("GetLastSnapshot", mock.Anything, app.IApplicationAddress.String()).Return(nil, nil).Once() + writeErr := errors.New("temporary status write failure") + repo.On("UpdateApplicationStatus", mock.Anything, app.ID, model.ApplicationStatus_Failed, mock.Anything). + Return(writeErr).Once() + repo.On("GetApplication", mock.Anything, app.IApplicationAddress.String()).Return(app, nil).Once() + repo.On("UpdateApplicationStatus", mock.Anything, app.ID, model.ApplicationStatus_Failed, mock.Anything). + Return(nil).Once() + + instance := &DummyMachineInstanceMock{ + application: app, + replayErr: &replay.ContradictionError{ + Application: app.Name, InputIndex: 9, Field: "outputs_hash", + Expected: "0x01", Actual: "0x02", + }, + } + factory := &MockMachineInstanceFactory{Instance: instance} + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + WithInstanceFactory(factory), + ) + + err := manager.UpdateMachines(context.Background()) + require.ErrorIs(err, ErrApplicationFailureNotDurable) + var persistenceErr *ApplicationFailurePersistenceError + require.ErrorAs(err, &persistenceErr) + require.ErrorIs(err, writeErr) + require.Equal(app.ID, persistenceErr.ApplicationID) + require.Equal(model.ApplicationStatus_OK, app.Status) + require.Len(manager.pendingApplicationFailures, 1) + require.True(manager.HasPendingApplicationFailures()) + require.Equal(1, factory.TemplateCalls) + require.Equal(1, instance.closeCalls) + require.False(manager.HasMachine(app.ID)) + + // The next update retries only the FAILED write. The app remains fenced + // even though this mock returns its stale executable row. + require.NoError(manager.UpdateMachines(context.Background())) + require.Equal(model.ApplicationStatus_Failed, app.Status) + require.Empty(manager.pendingApplicationFailures) + require.False(manager.HasPendingApplicationFailures()) + require.Equal(1, factory.TemplateCalls) + require.Equal(1, instance.closeCalls) + require.False(manager.HasMachine(app.ID)) + repo.AssertExpectations(s.T()) + }) + + s.Run("ApplicationFailureFenceClearsWhenTerminalStatusWins", func() { + require := s.Require() + repo := &MockMachineRepository{} + app := &model.Application{ + ID: 76, Name: "TerminalStatusWins", IApplicationAddress: common.HexToAddress("0x76"), + Enabled: true, Status: model.ApplicationStatus_OK, + } + terminalApp := *app + terminalApp.Status = model.ApplicationStatus_Corrupted + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && !*f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{app}, uint64(1), nil).Once() + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && *f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{}, uint64(0), nil).Once() + repo.On("GetLastSnapshot", mock.Anything, app.IApplicationAddress.String()).Return(nil, nil).Once() + repo.On("UpdateApplicationStatus", mock.Anything, app.ID, model.ApplicationStatus_Failed, mock.Anything). + Return(errors.New("terminal status transition rejected")).Once() + repo.On("GetApplication", mock.Anything, app.IApplicationAddress.String()). + Return(&terminalApp, nil).Once() + + instance := &DummyMachineInstanceMock{ + application: app, + replayErr: &replay.ContradictionError{Application: app.Name, Field: "machine_hash"}, + } + factory := &MockMachineInstanceFactory{Instance: instance} + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + WithInstanceFactory(factory), + ) + + require.NoError(manager.UpdateMachines(context.Background())) + require.Empty(manager.pendingApplicationFailures) + require.False(manager.HasPendingApplicationFailures()) + require.Equal(1, factory.TemplateCalls) + require.Equal(1, instance.closeCalls) + require.False(manager.HasMachine(app.ID)) + repo.AssertExpectations(s.T()) + }) + + s.Run("ApplicationFailureFenceRetainsUnrelatedFailedStatus", func() { + require := s.Require() + repo := &MockMachineRepository{} + app := &model.Application{ + ID: 77, Name: "UnrelatedFailure", IApplicationAddress: common.HexToAddress("0x77"), + Status: model.ApplicationStatus_OK, + } + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + ) + manager.FenceApplicationFailure(app, replayContradictionReason(&replay.ContradictionError{ + Application: app.Name, Field: "machine_hash", + })) + + unrelatedReason := "machine process crashed" + durableApp := *app + durableApp.Status = model.ApplicationStatus_Failed + durableApp.Reason = &unrelatedReason + repo.On("UpdateApplicationStatus", mock.Anything, app.ID, model.ApplicationStatus_Failed, mock.Anything). + Return(errors.New("status transition rejected")).Once() + repo.On("GetApplication", mock.Anything, app.IApplicationAddress.String()). + Return(&durableApp, nil).Once() + + err := manager.persistApplicationFailure(context.Background(), app.ID) + + require.ErrorIs(err, ErrApplicationFailureNotDurable) + require.Len(manager.pendingApplicationFailures, 1) + require.True(manager.HasPendingApplicationFailures()) + repo.AssertExpectations(s.T()) + }) + + s.Run("AmbiguousWriteClearsFenceWhenExactFailedStatusIsDurable", func() { + require := s.Require() + repo := &MockMachineRepository{} + app := &model.Application{ + ID: 78, Name: "MatchingFailure", IApplicationAddress: common.HexToAddress("0x78"), + Status: model.ApplicationStatus_OK, + } + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + ) + manager.FenceApplicationFailure(app, replayContradictionReason(&replay.ContradictionError{ + Application: app.Name, Field: "machine_hash", + })) + pending := manager.pendingApplicationFailures[app.ID] + + durableApp := *app + durableApp.Status = model.ApplicationStatus_Failed + durableApp.Reason = &pending.reason + repo.On("UpdateApplicationStatus", mock.Anything, app.ID, model.ApplicationStatus_Failed, mock.Anything). + Return(errors.New("write result lost")).Once() + repo.On("GetApplication", mock.Anything, app.IApplicationAddress.String()). + Return(&durableApp, nil).Once() + + require.NoError(manager.persistApplicationFailure(context.Background(), app.ID)) + + require.Empty(manager.pendingApplicationFailures) + require.False(manager.HasPendingApplicationFailures()) + repo.AssertExpectations(s.T()) + }) + + s.Run("AmbiguousWriteClearsFenceForNormalizedLongReason", func() { + require := s.Require() + repo := &MockMachineRepository{} + app := &model.Application{ + ID: 81, Name: "LongReason", IApplicationAddress: common.HexToAddress("0x81"), + Status: model.ApplicationStatus_OK, + } + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + ) + detail := &replay.ContradictionError{ + Application: app.Name, + Field: "machine_hash", + Expected: strings.Repeat("e", 5000), + Actual: "different", + } + expectedReason := appstatus.NormalizeReason(detail.Error()) + manager.FenceApplicationFailure(app, replayContradictionReason(detail)) + pending := manager.pendingApplicationFailures[app.ID] + require.Equal(expectedReason, pending.reason) + + durableApp := *app + durableApp.Status = model.ApplicationStatus_Failed + durableApp.Reason = &expectedReason + repo.On( + "UpdateApplicationStatus", + mock.Anything, + app.ID, + model.ApplicationStatus_Failed, + mock.MatchedBy(func(reason *string) bool { + return reason != nil && *reason == expectedReason + }), + ).Return(errors.New("write result lost")).Once() + repo.On("GetApplication", mock.Anything, app.IApplicationAddress.String()). + Return(&durableApp, nil).Once() + + require.NoError(manager.persistApplicationFailure(context.Background(), app.ID)) + require.False(manager.HasPendingApplicationFailures()) + repo.AssertExpectations(s.T()) + }) + + s.Run("ApplicationFailureFenceClearsWhenStatusWriteFindsDeletedApplication", func() { + require := s.Require() + repo := &MockMachineRepository{} + app := &model.Application{ + ID: 79, Name: "DeletedApplication", IApplicationAddress: common.HexToAddress("0x79"), + Status: model.ApplicationStatus_OK, + } + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + ) + manager.FenceApplicationFailure(app, replayContradictionReason(&replay.ContradictionError{ + Application: app.Name, Field: "machine_hash", + })) + + repo.On("UpdateApplicationStatus", mock.Anything, app.ID, model.ApplicationStatus_Failed, mock.Anything). + Return(repository.ErrNotFound).Once() + + require.NoError(manager.persistApplicationFailure(context.Background(), app.ID)) + + require.Empty(manager.pendingApplicationFailures) + require.False(manager.HasPendingApplicationFailures()) + repo.AssertNotCalled(s.T(), "GetApplication", mock.Anything, mock.Anything) + repo.AssertExpectations(s.T()) + }) + + s.Run("ApplicationFailureFenceClearsWhenReadFindsDeletedApplication", func() { + require := s.Require() + repo := &MockMachineRepository{} + app := &model.Application{ + ID: 82, Name: "DeletedApplicationReadback", IApplicationAddress: common.HexToAddress("0x82"), + Status: model.ApplicationStatus_OK, + } + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + ) + manager.FenceApplicationFailure(app, "machine execution failed") + repo.On("UpdateApplicationStatus", mock.Anything, app.ID, model.ApplicationStatus_Failed, mock.Anything). + Return(errors.New("write result unavailable")).Once() + repo.On("GetApplication", mock.Anything, app.IApplicationAddress.String()). + Return(nil, nil).Once() + + require.NoError(manager.persistApplicationFailure(context.Background(), app.ID)) + require.False(manager.HasPendingApplicationFailures()) + repo.AssertExpectations(s.T()) + }) + + s.Run("ApplicationFailureFenceClearsWhenAddressBelongsToReplacement", func() { + require := s.Require() + repo := &MockMachineRepository{} + oldApp := &model.Application{ + ID: 82, Name: "OldApplication", IApplicationAddress: common.HexToAddress("0x82"), + Status: model.ApplicationStatus_OK, + } + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + ) + manager.FenceApplicationFailure(oldApp, replayContradictionReason(&replay.ContradictionError{ + Application: oldApp.Name, Field: "machine_hash", + })) + replacement := *oldApp + replacement.ID = 83 + replacement.Name = "ReplacementApplication" + + repo.On( + "UpdateApplicationStatus", + mock.Anything, + oldApp.ID, + model.ApplicationStatus_Failed, + mock.Anything, + ).Return(errors.New("write result unavailable")).Once() + repo.On("GetApplication", mock.Anything, oldApp.IApplicationAddress.String()). + Return(&replacement, nil).Once() + + require.NoError(manager.persistApplicationFailure(context.Background(), oldApp.ID)) + require.False(manager.HasPendingApplicationFailures()) + require.Equal(model.ApplicationStatus_OK, replacement.Status) + require.Nil(replacement.Reason) + repo.AssertNotCalled( + s.T(), + "UpdateApplicationStatus", + mock.Anything, + replacement.ID, + mock.Anything, + mock.Anything, + ) + repo.AssertExpectations(s.T()) + }) + + s.Run("ApplicationFailureFencePropagatesWriteAndReadFailure", func() { + require := s.Require() + repo := &MockMachineRepository{} + app := &model.Application{ + ID: 80, Name: "UnconfirmedFailure", IApplicationAddress: common.HexToAddress("0x80"), + Status: model.ApplicationStatus_OK, + } + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + ) + manager.FenceApplicationFailure(app, replayContradictionReason(&replay.ContradictionError{ + Application: app.Name, Field: "machine_hash", + })) + writeErr := errors.New("status write unavailable") + readErr := errors.New("status read unavailable") + repo.On("UpdateApplicationStatus", mock.Anything, app.ID, model.ApplicationStatus_Failed, mock.Anything). + Return(writeErr).Once() + repo.On("GetApplication", mock.Anything, app.IApplicationAddress.String()). + Return(nil, readErr).Once() + + err := manager.persistApplicationFailure(context.Background(), app.ID) + + require.ErrorIs(err, ErrApplicationFailureNotDurable) + require.ErrorIs(err, writeErr) + require.ErrorIs(err, readErr) + require.True(manager.HasPendingApplicationFailures()) + repo.AssertExpectations(s.T()) + }) + + s.Run("ApplicationFailureLocalDeadlineDoesNotBlockHealthySibling", func() { + require := s.Require() + repo := &MockMachineRepository{} + failed := &model.Application{ + ID: 91, Name: "LocalDeadline", IApplicationAddress: common.HexToAddress("0x91"), + Enabled: true, Status: model.ApplicationStatus_OK, + } + healthy := &model.Application{ + ID: 92, Name: "HealthyAfterDeadline", IApplicationAddress: common.HexToAddress("0x92"), + Enabled: true, Status: model.ApplicationStatus_OK, + } + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + WithInstanceFactory(&MockMachineInstanceFactory{TemplateFn: func(app *model.Application) (MachineInstance, error) { + return &DummyMachineInstanceMock{application: app}, nil + }}), + ) + manager.FenceApplicationFailure(failed, "machine execution failed") + localDeadline := fmt.Errorf("repository statement timeout: %w", context.DeadlineExceeded) + repo.On("UpdateApplicationStatus", mock.Anything, failed.ID, model.ApplicationStatus_Failed, mock.Anything). + Return(localDeadline).Once() + repo.On("GetApplication", mock.Anything, failed.IApplicationAddress.String()). + Return(failed, nil).Once() + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && !*f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{failed, healthy}, uint64(2), nil).Once() + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && *f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{}, uint64(0), nil).Once() + repo.On("GetLastSnapshot", mock.Anything, healthy.IApplicationAddress.String()).Return(nil, nil).Once() + + err := manager.UpdateMachines(context.Background()) + + require.ErrorIs(err, context.DeadlineExceeded) + require.True(IsOnlyApplicationFailurePersistenceErrors(err)) + require.True(manager.HasPendingApplicationFailures()) + require.False(manager.HasMachine(failed.ID)) + require.True(manager.HasMachine(healthy.ID), "a repository-local timeout must not block the sibling") + repo.AssertExpectations(s.T()) + }) + + s.Run("PendingApplicationFailureCancellationPreservesErrorChainAndStops", func() { + require := s.Require() + repo := &MockMachineRepository{} + manager := newTestMachineManager(repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10) + apps := []*model.Application{ + {ID: 101, Name: "First", IApplicationAddress: common.HexToAddress("0x101")}, + {ID: 102, Name: "Canceled", IApplicationAddress: common.HexToAddress("0x102")}, + {ID: 103, Name: "MustNotRun", IApplicationAddress: common.HexToAddress("0x103")}, + } + for _, app := range apps { + manager.FenceApplicationFailure(app, "machine execution failed") + } + firstWriteErr := errors.New("first write unavailable") + firstReadErr := errors.New("first read unavailable") + repo.On("UpdateApplicationStatus", mock.Anything, apps[0].ID, model.ApplicationStatus_Failed, mock.Anything). + Return(firstWriteErr).Once() + repo.On("GetApplication", mock.Anything, apps[0].IApplicationAddress.String()). + Return(nil, firstReadErr).Once() + ctx, cancel := context.WithCancel(context.Background()) + repo.On("UpdateApplicationStatus", mock.Anything, apps[1].ID, model.ApplicationStatus_Failed, mock.Anything). + Run(func(mock.Arguments) { cancel() }).Return(context.Canceled).Once() + repo.On("GetApplication", mock.Anything, apps[1].IApplicationAddress.String()). + Return(nil, context.Canceled).Once() + + _, err := manager.persistPendingApplicationFailures(ctx) + + require.ErrorIs(err, firstWriteErr) + require.ErrorIs(err, firstReadErr) + require.ErrorIs(err, context.Canceled) + var persistenceIDs []int64 + var collect func(error) + collect = func(current error) { + if current == nil { + return + } + if typed, ok := current.(*ApplicationFailurePersistenceError); ok { + persistenceIDs = append(persistenceIDs, typed.ApplicationID) + return + } + if joined, ok := current.(interface{ Unwrap() []error }); ok { + for _, child := range joined.Unwrap() { + collect(child) + } + } + } + collect(err) + require.ElementsMatch([]int64{apps[0].ID, apps[1].ID}, persistenceIDs) + repo.AssertNotCalled(s.T(), "UpdateApplicationStatus", mock.Anything, apps[2].ID, mock.Anything, mock.Anything) + require.Len(manager.pendingApplicationFailures, 3) + repo.AssertExpectations(s.T()) + }) + + s.Run("QueuedLiveFailureRetryClearsFenceAndRemovesOnlyAffectedMachine", func() { + require := s.Require() + repo := &MockMachineRepository{} + failed := &model.Application{ + ID: 111, Name: "LiveFailure", IApplicationAddress: common.HexToAddress("0x111"), + Enabled: true, Status: model.ApplicationStatus_OK, + } + healthy := &model.Application{ + ID: 112, Name: "Healthy", IApplicationAddress: common.HexToAddress("0x112"), + Enabled: true, Status: model.ApplicationStatus_OK, + } + manager := newTestMachineManager(repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10) + failedMachine := &DummyMachineInstanceMock{application: failed} + healthyMachine := &DummyMachineInstanceMock{application: healthy} + require.True(manager.addMachine(failed.ID, failedMachine)) + require.True(manager.addMachine(healthy.ID, healthyMachine)) + const reason = "advance execution reached configured cycle limit" + manager.FenceApplicationFailure(failed, reason) + repo.On("UpdateApplicationStatus", mock.Anything, failed.ID, model.ApplicationStatus_Failed, + mock.MatchedBy(func(got *string) bool { return got != nil && *got == reason })).Return(nil).Once() + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && !*f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{failed, healthy}, uint64(2), nil).Once() + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && *f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{}, uint64(0), nil).Once() + + require.NoError(manager.UpdateMachines(context.Background())) + + require.False(manager.HasPendingApplicationFailures()) + require.False(manager.HasMachine(failed.ID)) + require.True(manager.HasMachine(healthy.ID)) + require.Equal(1, failedMachine.closeCalls) + require.Zero(healthyMachine.closeCalls) + repo.AssertExpectations(s.T()) + }) + + s.Run("TransientReplayErrorRetries", func() { + require := s.Require() + repo := &MockMachineRepository{} + app := &model.Application{ + ID: 72, Name: "TransientReplay", IApplicationAddress: common.HexToAddress("0x72"), + Enabled: true, Status: model.ApplicationStatus_OK, + } + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && !*f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{app}, uint64(1), nil).Twice() + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && *f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{}, uint64(0), nil).Twice() + repo.On("GetLastSnapshot", mock.Anything, app.IApplicationAddress.String()).Return(nil, nil).Twice() + + var instances []*DummyMachineInstanceMock + factory := &MockMachineInstanceFactory{TemplateFn: func(got *model.Application) (MachineInstance, error) { + instance := &DummyMachineInstanceMock{application: got} + if len(instances) == 0 { + instance.replayErr = errors.New("temporary repository outage") + } + instances = append(instances, instance) + return instance, nil + }} + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + WithInstanceFactory(factory), + ) + + require.NoError(manager.UpdateMachines(context.Background())) + require.Len(instances, 1) + require.Equal(1, instances[0].closeCalls) + require.Empty(manager.pendingApplicationFailures) + require.False(manager.HasMachine(app.ID)) + + require.NoError(manager.UpdateMachines(context.Background())) + require.Len(instances, 2) + require.Equal(2, factory.TemplateCalls) + require.True(manager.HasMachine(app.ID)) + require.Empty(manager.pendingApplicationFailures) + repo.AssertExpectations(s.T()) + }) + + s.Run("ReplayContradictionDoesNotBlockHealthySibling", func() { + require := s.Require() + repo := &MockMachineRepository{} + badApp := &model.Application{ + ID: 73, Name: "BadReplay", IApplicationAddress: common.HexToAddress("0x73"), + Enabled: true, Status: model.ApplicationStatus_OK, + } + goodApp := &model.Application{ + ID: 74, Name: "GoodReplay", IApplicationAddress: common.HexToAddress("0x74"), + Enabled: true, Status: model.ApplicationStatus_OK, + } + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && !*f.ForeclosureRecorded + }), repository.Pagination{}, false). + Return([]*model.Application{badApp, goodApp}, uint64(2), nil).Once() + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && *f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{}, uint64(0), nil).Once() + repo.On("GetLastSnapshot", mock.Anything, mock.Anything).Return(nil, nil).Twice() + repo.On("UpdateApplicationStatus", mock.Anything, badApp.ID, model.ApplicationStatus_Failed, + mock.Anything).Return(nil).Once() + + instances := map[int64]*DummyMachineInstanceMock{} + factory := &MockMachineInstanceFactory{TemplateFn: func(app *model.Application) (MachineInstance, error) { + instance := &DummyMachineInstanceMock{application: app} + if app.ID == badApp.ID { + instance.replayErr = &replay.ContradictionError{ + Application: app.Name, InputIndex: 4, Field: "outputs_hash", + Expected: "0x01", Actual: "0x02", + } + } + instances[app.ID] = instance + return instance, nil + }} + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + WithInstanceFactory(factory), + ) + + require.NoError(manager.UpdateMachines(context.Background())) + require.False(manager.HasMachine(badApp.ID)) + require.True(manager.HasMachine(goodApp.ID)) + require.Equal(1, instances[badApp.ID].closeCalls) + require.Zero(instances[goodApp.ID].closeCalls) + require.Equal(model.ApplicationStatus_Failed, badApp.Status) + require.Equal(model.ApplicationStatus_OK, goodApp.Status) + require.Empty(manager.pendingApplicationFailures) + require.False(manager.HasPendingApplicationFailures()) + repo.AssertExpectations(s.T()) + }) + + s.Run("UnconfirmedApplicationFailureDoesNotBlockHealthySibling", func() { + require := s.Require() + repo := &MockMachineRepository{} + badApp := &model.Application{ + ID: 83, Name: "BadUndurableReplay", IApplicationAddress: common.HexToAddress("0x83"), + Enabled: true, Status: model.ApplicationStatus_OK, + } + goodApp := &model.Application{ + ID: 84, Name: "HealthyReplay", IApplicationAddress: common.HexToAddress("0x84"), + Enabled: true, Status: model.ApplicationStatus_OK, + } + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && !*f.ForeclosureRecorded + }), repository.Pagination{}, false). + Return([]*model.Application{badApp, goodApp}, uint64(2), nil).Once() + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && *f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{}, uint64(0), nil).Once() + repo.On("GetLastSnapshot", mock.Anything, mock.Anything).Return(nil, nil).Twice() + writeErr := errors.New("status write unavailable") + readErr := errors.New("status read unavailable") + repo.On("UpdateApplicationStatus", mock.Anything, badApp.ID, model.ApplicationStatus_Failed, mock.Anything). + Return(writeErr).Once() + repo.On("GetApplication", mock.Anything, badApp.IApplicationAddress.String()). + Return(nil, readErr).Once() + + instances := map[int64]*DummyMachineInstanceMock{} + factory := &MockMachineInstanceFactory{TemplateFn: func(app *model.Application) (MachineInstance, error) { + instance := &DummyMachineInstanceMock{application: app} + if app.ID == badApp.ID { + instance.replayErr = &replay.ContradictionError{ + Application: app.Name, InputIndex: 4, Field: "outputs_hash", + } + } + instances[app.ID] = instance + return instance, nil + }} + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + WithInstanceFactory(factory), + ) + + err := manager.UpdateMachines(context.Background()) + require.ErrorIs(err, ErrApplicationFailureNotDurable) + require.ErrorIs(err, writeErr) + require.ErrorIs(err, readErr) + require.False(manager.HasMachine(badApp.ID)) + require.True(manager.HasMachine(goodApp.ID)) + require.Equal(1, instances[badApp.ID].closeCalls) + require.Zero(instances[goodApp.ID].closeCalls) + require.True(manager.HasPendingApplicationFailures()) + repo.AssertExpectations(s.T()) + }) + + replayFailures := []struct { + name string + err error + reasonContains string + }{ + {"PayloadLengthLimit", machine.ErrPayloadLengthLimitExceeded, "execution limit"}, + {"OutputsLimit", machine.ErrOutputsLimitExceeded, "execution limit"}, + {"ReportsLimit", machine.ErrReportsLimitExceeded, "execution limit"}, + {"McycleLimit", machine.ErrReachedLimitMcycle, "execution limit"}, + {"Deadline", machine.ErrDeadlineExceeded, "execution deadline"}, + {"IncompleteAdvance", ErrIncompleteAdvance, "incomplete advance result"}, + {"MachineInternal", machine.ErrMachineInternal, "failed internally"}, + } + for index, failure := range replayFailures { + s.Run("ReplayFailure/"+failure.name, func() { + require := s.Require() + repo := &MockMachineRepository{} + failedID := int64(85 + index*2) + failed := &model.Application{ + ID: failedID, Name: "FailedReplay", + IApplicationAddress: common.BigToAddress(big.NewInt(failedID)), + Enabled: true, Status: model.ApplicationStatus_OK, + } + healthy := &model.Application{ + ID: failedID + 1, Name: "HealthyReplay", + IApplicationAddress: common.BigToAddress(big.NewInt(failedID + 1)), + Enabled: true, Status: model.ApplicationStatus_OK, + } + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && !*f.ForeclosureRecorded + }), repository.Pagination{}, false). + Return([]*model.Application{failed, healthy}, uint64(2), nil).Once() + repo.On("ListApplications", mock.Anything, mock.MatchedBy(func(f repository.ApplicationFilter) bool { + return f.ForeclosureRecorded != nil && *f.ForeclosureRecorded + }), repository.Pagination{}, false).Return([]*model.Application{}, uint64(0), nil).Once() + repo.On("GetLastSnapshot", mock.Anything, mock.Anything).Return(nil, nil).Twice() + repo.On("UpdateApplicationStatus", mock.Anything, failed.ID, model.ApplicationStatus_Failed, + mock.MatchedBy(func(reason *string) bool { + return reason != nil && strings.Contains(*reason, failure.reasonContains) + })).Return(nil).Once() + + instances := map[int64]*DummyMachineInstanceMock{} + factory := &MockMachineInstanceFactory{TemplateFn: func(app *model.Application) (MachineInstance, error) { + instance := &DummyMachineInstanceMock{application: app} + if app.ID == failed.ID { + instance.replayErr = fmt.Errorf("replay input: %w", failure.err) + } + instances[app.ID] = instance + return instance, nil + }} + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + WithInstanceFactory(factory), + ) + + require.NoError(manager.UpdateMachines(context.Background())) + require.False(manager.HasMachine(failed.ID)) + require.True(manager.HasMachine(healthy.ID)) + require.Equal(model.ApplicationStatus_Failed, failed.Status) + require.Equal(model.ApplicationStatus_OK, healthy.Status) + require.Equal(1, instances[failed.ID].closeCalls) + require.Zero(instances[healthy.ID].closeCalls) + require.False(manager.HasPendingApplicationFailures()) + repo.AssertExpectations(s.T()) + }) + } + s.Run("AddNewMachines", func() { require := s.Require() @@ -56,10 +971,6 @@ func (s *MachineManagerSuite) TestUpdateMachines() { repo.On("ListApplications", mock.Anything, mock.Anything, mock.Anything, false). Return([]*model.Application{app1}, uint64(1), nil) - // Empty inputs for synchronization - repo.On("ListInputs", mock.Anything, mock.Anything, mock.Anything, mock.Anything, false). - Return([]*model.Input{}, uint64(0), nil) - // Mock GetLastSnapshot to return nil (no snapshot available) repo.On("GetLastSnapshot", mock.Anything, mock.Anything). Return(nil, nil) @@ -68,7 +979,7 @@ func (s *MachineManagerSuite) TestUpdateMachines() { testLogger := slog.New(slog.NewTextHandler(io.Discard, nil)) mockInstance := &DummyMachineInstanceMock{application: app1} factory := &MockMachineInstanceFactory{Instance: mockInstance} - manager := NewMachineManager(repo, testLogger, false, 500, WithInstanceFactory(factory)) + manager := newTestMachineManager(repo, testLogger, false, 500, WithInstanceFactory(factory)) err := manager.UpdateMachines(context.Background()) require.NoError(err) @@ -116,7 +1027,7 @@ func (s *MachineManagerSuite) TestUpdateMachines() { testLogger := slog.New(slog.NewTextHandler(io.Discard, nil)) mockInstance := &DummyMachineInstanceMock{application: app} factory := &MockMachineInstanceFactory{Instance: mockInstance} - manager := NewMachineManager(repo, testLogger, false, 500, WithInstanceFactory(factory)) + manager := newTestMachineManager(repo, testLogger, false, 500, WithInstanceFactory(factory)) err := manager.UpdateMachines(context.Background()) require.NoError(err) @@ -161,7 +1072,7 @@ func (s *MachineManagerSuite) TestUpdateMachines() { testLogger := slog.New(slog.NewTextHandler(io.Discard, nil)) mockInstance := &DummyMachineInstanceMock{application: app} factory := &MockMachineInstanceFactory{Instance: mockInstance} - manager := NewMachineManager(repo, testLogger, false, 500, WithInstanceFactory(factory)) + manager := newTestMachineManager(repo, testLogger, false, 500, WithInstanceFactory(factory)) err := manager.UpdateMachines(context.Background()) require.NoError(err) @@ -180,7 +1091,7 @@ func (s *MachineManagerSuite) TestUpdateMachines() { // Create a test logger testLogger := slog.New(slog.NewTextHandler(io.Discard, nil)) - manager := NewMachineManager(repo, testLogger, false, 500) + manager := newTestMachineManager(repo, testLogger, false, 500) // Add mock machines app1 := &model.Application{ID: 1, Name: "App1"} @@ -206,6 +1117,316 @@ func (s *MachineManagerSuite) TestUpdateMachines() { }) } +func (s *MachineManagerSuite) TestSnapshotStartingStateVerification() { + newApp := func(id int64, processed uint64) *model.Application { + return &model.Application{ + ID: id, + Name: "SnapshotApp", + IApplicationAddress: common.BigToAddress(new(big.Int).SetInt64(id)), + Enabled: true, + Status: model.ApplicationStatus_OK, + ProcessedInputs: processed, + ExecutionParameters: model.ExecutionParameters{ + AdvanceMaxDeadline: time.Second, + InspectMaxDeadline: time.Second, + LoadDeadline: time.Second, + MaxConcurrentInspects: 3, + }, + } + } + prepareRepo := func( + app *model.Application, + snapshot *model.Input, + snapshotErr error, + ) *MockMachineRepository { + repo := &MockMachineRepository{ + replayApplicationID: app.ID, + replayConsensus: app.ConsensusType, + } + repo.On("ListApplications", mock.Anything, mock.Anything, mock.Anything, false). + Return([]*model.Application{app}, uint64(1), nil) + repo.On("GetLastSnapshot", mock.Anything, app.IApplicationAddress.String()). + Return(snapshot, snapshotErr).Once() + return repo + } + configureReplay := func(repo *MockMachineRepository, app *model.Application) common.Hash { + machineHash := newHash(1) + outputsHash := newHash(2) + repo.replayCount = app.ProcessedInputs + repo.replayRecords = []*model.ReplayRecord{{ + Input: model.ReplayInput{ + ApplicationID: app.ID, + EpochIndex: 0, + InputIndex: 0, + RawData: []byte("replayed input"), + Status: model.InputCompletionStatus_Accepted, + MachineHash: &machineHash, + OutputsHash: &outputsHash, + }, + }} + return machineHash + } + newReplayTemplate := func(app *model.Application) MachineInstance { + base := &MockRollupsMachine{ + HashReturn: newHash(0), + OutputsHashReturn: newHash(0), + } + base.ForkReturn = newForkableMock() + instance, err := NewMachineInstanceWithFactory( + context.Background(), app, 0, + slog.New(slog.NewTextHandler(io.Discard, nil)), + &MockMachineRuntimeFactory{RuntimeToReturn: base}, + ) + s.Require().NoError(err) + return instance + } + assertTemplateReplay := func( + require *require.Assertions, + manager *MachineManager, + repo *MockMachineRepository, + app *model.Application, + expectedHash common.Hash, + ) { + machineInstance, exists := manager.GetMachine(app.ID) + require.True(exists) + require.Equal(app.ProcessedInputs, machineInstance.ProcessedInputs()) + actualHash, err := machineInstance.Hash(context.Background()) + require.NoError(err) + require.Equal([32]byte(expectedHash), actualHash) + require.Positive(repo.replayListCalls, "template fallback must replay canonical inputs") + } + + s.Run("MatchingCaughtUpSnapshotIsAccepted", func() { + require := s.Require() + app := newApp(81, 3) + path := s.T().TempDir() + hash := common.HexToHash("0x81") + snapshot := &model.Input{Index: 2, SnapshotURI: &path, MachineHash: &hash} + repo := prepareRepo(app, snapshot, nil) + repo.replayCount = 3 + snapshotInstance := &DummyMachineInstanceMock{ + application: app, processedInputs: 3, hashReturn: [32]byte(hash), + } + factory := &MockMachineInstanceFactory{SnapshotInstance: snapshotInstance} + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + WithInstanceFactory(factory), + ) + + require.NoError(manager.UpdateMachines(context.Background())) + require.Equal(1, factory.SnapshotCalls) + require.Zero(factory.TemplateCalls) + require.Equal(1, snapshotInstance.hashCalls) + require.Equal(1, snapshotInstance.replayCalls) + require.True(manager.HasMachine(app.ID)) + require.NoError(manager.Close()) + }) + + fallbackTests := []struct { + name string + snapshot func(path string, expectedHash common.Hash) *model.Input + repositoryErr error + candidate func(app *model.Application, expectedHash common.Hash) *DummyMachineInstanceMock + factoryErr error + wantCalls int + wantHashCalls int + }{ + { + name: "missing persisted hash", + snapshot: func(path string, _ common.Hash) *model.Input { + return &model.Input{Index: 0, SnapshotURI: &path} + }, + wantCalls: 0, + }, + { + name: "repository result accompanied by error", + snapshot: func(path string, expectedHash common.Hash) *model.Input { + return &model.Input{Index: 0, SnapshotURI: &path, MachineHash: &expectedHash} + }, + repositoryErr: errors.New("snapshot query failed"), + wantCalls: 0, + }, + { + name: "snapshot input index cannot be incremented", + snapshot: func(path string, expectedHash common.Hash) *model.Input { + return &model.Input{Index: math.MaxUint64, SnapshotURI: &path, MachineHash: &expectedHash} + }, + wantCalls: 0, + }, + { + name: "snapshot is beyond application progress", + snapshot: func(path string, expectedHash common.Hash) *model.Input { + return &model.Input{Index: 1, SnapshotURI: &path, MachineHash: &expectedHash} + }, + wantCalls: 0, + }, + { + name: "matching hash with wrong processed input count", + snapshot: func(path string, expectedHash common.Hash) *model.Input { + return &model.Input{Index: 0, SnapshotURI: &path, MachineHash: &expectedHash} + }, + candidate: func(app *model.Application, expectedHash common.Hash) *DummyMachineInstanceMock { + return &DummyMachineInstanceMock{ + application: app, processedInputs: 0, hashReturn: [32]byte(expectedHash), + } + }, + wantCalls: 1, + }, + { + name: "manager boundary hash mismatch", + snapshot: func(path string, expectedHash common.Hash) *model.Input { + return &model.Input{Index: 0, SnapshotURI: &path, MachineHash: &expectedHash} + }, + candidate: func(app *model.Application, _ common.Hash) *DummyMachineInstanceMock { + return &DummyMachineInstanceMock{application: app, processedInputs: 1, hashReturn: newHash(99)} + }, + wantCalls: 1, + wantHashCalls: 1, + }, + { + name: "manager boundary hash read error", + snapshot: func(path string, expectedHash common.Hash) *model.Input { + return &model.Input{Index: 0, SnapshotURI: &path, MachineHash: &expectedHash} + }, + candidate: func(app *model.Application, expectedHash common.Hash) *DummyMachineInstanceMock { + return &DummyMachineInstanceMock{ + application: app, processedInputs: 1, hashReturn: [32]byte(expectedHash), + hashError: errors.New("snapshot hash unavailable"), + } + }, + wantCalls: 1, + wantHashCalls: 1, + }, + { + name: "factory returns partial candidate with error", + snapshot: func(path string, expectedHash common.Hash) *model.Input { + return &model.Input{Index: 0, SnapshotURI: &path, MachineHash: &expectedHash} + }, + candidate: func(app *model.Application, expectedHash common.Hash) *DummyMachineInstanceMock { + return &DummyMachineInstanceMock{ + application: app, processedInputs: 1, hashReturn: [32]byte(expectedHash), + } + }, + factoryErr: errors.New("factory failed after creating candidate"), + wantCalls: 1, + }, + } + + for i, tt := range fallbackTests { + s.Run(tt.name, func() { + require := s.Require() + app := newApp(82+int64(i), 1) + path := s.T().TempDir() + expectedHash := newHash(1) + snapshot := tt.snapshot(path, expectedHash) + repo := prepareRepo(app, snapshot, tt.repositoryErr) + require.Equal(expectedHash, configureReplay(repo, app)) + candidate := (*DummyMachineInstanceMock)(nil) + if tt.candidate != nil { + candidate = tt.candidate(app, expectedHash) + } + template := newReplayTemplate(app) + factory := &MockMachineInstanceFactory{ + Instance: template, SnapshotInstance: candidate, SnapshotErr: tt.factoryErr, + } + manager := NewMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + WithInstanceFactory(factory), + ) + + require.NoError(manager.UpdateMachines(context.Background())) + require.Equal(tt.wantCalls, factory.SnapshotCalls) + require.Equal(1, factory.TemplateCalls) + require.Equal([]bool{false}, factory.TemplateChecks) + if candidate != nil { + require.Equal(1, candidate.closeCalls) + require.Equal(tt.wantHashCalls, candidate.hashCalls) + require.Zero(candidate.replayCalls) + } + assertTemplateReplay(require, manager, repo, app, expectedHash) + require.NoError(manager.Close()) + }) + } + + s.Run("SnapshotFactoryNilResultFallsBackToTemplate", func() { + require := s.Require() + app := newApp(90, 1) + path := s.T().TempDir() + expectedHash := common.Hash{} + snapshot := &model.Input{Index: 0, SnapshotURI: &path, MachineHash: &expectedHash} + repo := prepareRepo(app, snapshot, nil) + expectedHash = configureReplay(repo, app) + snapshot.MachineHash = &expectedHash + template := newReplayTemplate(app) + factory := &MockMachineInstanceFactory{Instance: template, ReturnNilSnapshot: true} + manager := NewMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + WithInstanceFactory(factory), + ) + + require.NoError(manager.UpdateMachines(context.Background())) + require.Equal(1, factory.SnapshotCalls) + require.Equal(1, factory.TemplateCalls) + assertTemplateReplay(require, manager, repo, app, expectedHash) + require.NoError(manager.Close()) + }) +} + +func (s *MachineManagerSuite) TestTemplateFactoryResultHandling() { + tests := []struct { + name string + candidate *DummyMachineInstanceMock + factoryErr error + }{ + {name: "nil instance without error"}, + { + name: "partial instance with error", + candidate: &DummyMachineInstanceMock{ + closeError: errors.New("partial template close failed"), + }, + factoryErr: errors.New("template factory failed"), + }, + } + + for i, tt := range tests { + s.Run(tt.name, func() { + require := s.Require() + app := &model.Application{ + ID: 91 + int64(i), + Name: "TemplateFactoryResultApp", + IApplicationAddress: common.BigToAddress(big.NewInt(91 + int64(i))), + Enabled: true, + Status: model.ApplicationStatus_OK, + } + if tt.candidate != nil { + tt.candidate.application = app + } + repo := &MockMachineRepository{} + repo.On("ListApplications", mock.Anything, mock.Anything, mock.Anything, false). + Return([]*model.Application{app}, uint64(1), nil) + repo.On("GetLastSnapshot", mock.Anything, app.IApplicationAddress.String()). + Return(nil, nil).Once() + factory := &MockMachineInstanceFactory{Err: tt.factoryErr} + if tt.candidate != nil { + factory.Instance = tt.candidate + } + manager := newTestMachineManager( + repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, + WithInstanceFactory(factory), + ) + + require.NoError(manager.UpdateMachines(context.Background())) + require.Equal(1, factory.TemplateCalls) + require.False(manager.HasMachine(app.ID)) + if tt.candidate != nil { + require.Equal(1, tt.candidate.closeCalls) + require.Zero(tt.candidate.replayCalls) + } + require.NoError(manager.Close()) + }) + } +} + func (s *MachineManagerSuite) TestGetMachine() { require := s.Require() @@ -213,7 +1434,7 @@ func (s *MachineManagerSuite) TestGetMachine() { repo.On("GetLastSnapshot", mock.Anything, mock.Anything). Return(nil, nil) - manager := NewMachineManager(repo, nil, false, 500) + manager := newTestMachineManager(repo, nil, false, 500) machine := &DummyMachineInstanceMock{application: &model.Application{ID: 1}} // Add a machine @@ -236,7 +1457,7 @@ func (s *MachineManagerSuite) TestHasMachine() { repo.On("GetLastSnapshot", mock.Anything, mock.Anything). Return(nil, nil) - manager := NewMachineManager(repo, nil, false, 500) + manager := newTestMachineManager(repo, nil, false, 500) machine := &DummyMachineInstanceMock{application: &model.Application{ID: 1}} // Add a machine @@ -256,7 +1477,7 @@ func (s *MachineManagerSuite) TestAddMachine() { repo.On("GetLastSnapshot", mock.Anything, mock.Anything). Return(nil, nil) - manager := NewMachineManager(repo, nil, false, 500) + manager := newTestMachineManager(repo, nil, false, 500) machine1 := &DummyMachineInstanceMock{application: &model.Application{ID: 1}} machine2 := &DummyMachineInstanceMock{application: &model.Application{ID: 2}} @@ -287,7 +1508,7 @@ func (s *MachineManagerSuite) TestAddMachine() { func (s *MachineManagerSuite) TestRemoveDisabledMachines() { require := s.Require() - manager := NewMachineManager(nil, nil, false, 500) + manager := newTestMachineManager(nil, nil, false, 500) // Add machines app1 := &model.Application{ID: 1} @@ -321,7 +1542,7 @@ func (s *MachineManagerSuite) TestUpdateMachinesErrors() { Return(([]*model.Application)(nil), uint64(0), errors.New("db error")) testLogger := slog.New(slog.NewTextHandler(io.Discard, nil)) - manager := NewMachineManager(repo, testLogger, false, 500) + manager := newTestMachineManager(repo, testLogger, false, 500) err := manager.UpdateMachines(context.Background()) require.Error(err) @@ -354,15 +1575,11 @@ func (s *MachineManagerSuite) TestUpdateMachinesErrors() { Return([]*model.Application{app}, uint64(1), nil) repo.On("GetLastSnapshot", mock.Anything, mock.Anything). Return(snapshotInput, nil) - // ListInputs for synchronization (no inputs to replay) - repo.On("ListInputs", mock.Anything, mock.Anything, mock.Anything, mock.Anything, false). - Return([]*model.Input{}, uint64(0), nil) - // The snapshot path doesn't exist, so it should fall back to template testLogger := slog.New(slog.NewTextHandler(io.Discard, nil)) mockInstance := &DummyMachineInstanceMock{application: app} factory := &MockMachineInstanceFactory{Instance: mockInstance} - manager := NewMachineManager(repo, testLogger, false, 500, WithInstanceFactory(factory)) + manager := newTestMachineManager(repo, testLogger, false, 500, WithInstanceFactory(factory)) err := manager.UpdateMachines(context.Background()) require.NoError(err) @@ -392,7 +1609,7 @@ func (s *MachineManagerSuite) TestUpdateMachinesErrors() { testLogger := slog.New(slog.NewTextHandler(io.Discard, nil)) factory := &MockMachineInstanceFactory{Err: errors.New("machine creation failed")} - manager := NewMachineManager(repo, testLogger, false, 500, WithInstanceFactory(factory)) + manager := newTestMachineManager(repo, testLogger, false, 500, WithInstanceFactory(factory)) err := manager.UpdateMachines(context.Background()) // UpdateMachines should not return an error; it logs and skips @@ -421,9 +1638,7 @@ func (s *MachineManagerSuite) TestUpdateMachinesErrors() { Return([]*model.Application{app}, uint64(1), nil) repo.On("GetLastSnapshot", mock.Anything, mock.Anything). Return(nil, nil) - // ListInputs returns an error so the real Synchronize method propagates the failure. - repo.On("ListInputs", mock.Anything, mock.Anything, mock.Anything, mock.Anything, false). - Return(([]*model.Input)(nil), uint64(0), errors.New("db connection lost")) + repo.replayCountError = errors.New("db connection lost") testLogger := slog.New(slog.NewTextHandler(io.Discard, nil)) @@ -447,7 +1662,7 @@ func (s *MachineManagerSuite) TestUpdateMachinesErrors() { func (s *MachineManagerSuite) TestCloseAggregatesErrors() { require := s.Require() - manager := NewMachineManager(nil, nil, false, 500) + manager := newTestMachineManager(nil, nil, false, 500) machine1 := &DummyMachineInstanceMock{application: &model.Application{ID: 1}} machine2 := &DummyMachineInstanceMock{ @@ -476,7 +1691,7 @@ func (s *MachineManagerSuite) TestApplications() { repo.On("GetLastSnapshot", mock.Anything, mock.Anything). Return(nil, nil) - manager := NewMachineManager(repo, nil, false, 500) + manager := newTestMachineManager(repo, nil, false, 500) // Add machines app1 := &model.Application{ID: 1, Name: "App1"} @@ -507,6 +1722,43 @@ func (s *MachineManagerSuite) TestApplications() { // Mock repository for testing type MockMachineRepository struct { mock.Mock + replayApplicationID int64 + replayConsensus model.Consensus + replayCount uint64 + replayCountError error + replayRecords []*model.ReplayRecord + replayListCalls int +} + +func (m *MockMachineRepository) ReplaySummary( + _ context.Context, + _ common.Address, + _ repository.ReplayVerificationLevel, +) (model.ReplaySummary, error) { + return model.ReplaySummary{ + ApplicationID: m.replayApplicationID, + ProcessedInputs: m.replayCount, + Consensus: m.replayConsensus, + }, m.replayCountError +} + +func (m *MockMachineRepository) ReplayPage( + _ context.Context, + request repository.ReplayPageRequest, +) ([]*model.ReplayRecord, error) { + m.replayListCalls++ + records := make([]*model.ReplayRecord, 0, min(request.Limit, uint64(len(m.replayRecords)))) + for _, record := range m.replayRecords { + index := record.Input.InputIndex + if index < request.FromInput || index >= request.ToInputExclusive { + continue + } + records = append(records, record) + if uint64(len(records)) == request.Limit { + break + } + } + return records, nil } func (m *MockMachineRepository) ListApplications( @@ -528,17 +1780,6 @@ func (m *MockMachineRepository) HasUndrainedEpochsBeforeBlock( return args.Bool(0), args.Error(1) } -func (m *MockMachineRepository) ListInputs( - ctx context.Context, - nameOrAddress string, - f repository.InputFilter, - p repository.Pagination, - descending bool, -) ([]*model.Input, uint64, error) { - args := m.Called(ctx, nameOrAddress, f, p, descending) - return args.Get(0).([]*model.Input), args.Get(1).(uint64), args.Error(2) -} - func (m *MockMachineRepository) GetLastSnapshot( ctx context.Context, nameOrAddress string) (*model.Input, error) { @@ -549,25 +1790,67 @@ func (m *MockMachineRepository) GetLastSnapshot( return args.Get(0).(*model.Input), args.Error(1) } +func (m *MockMachineRepository) GetApplication( + ctx context.Context, + nameOrAddress string, +) (*model.Application, error) { + args := m.Called(ctx, nameOrAddress) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).(*model.Application), args.Error(1) +} + +func (m *MockMachineRepository) UpdateApplicationStatus( + ctx context.Context, + appID int64, + status model.ApplicationStatus, + reason *string, +) error { + return m.Called(ctx, appID, status, reason).Error(0) +} + // ------------------------------------------------------------------------------------------------ // MockMachineInstanceFactory implements MachineInstanceFactory for testing. // It returns the same instance for every call, ignoring the app/path arguments. type MockMachineInstanceFactory struct { - Instance MachineInstance - Err error + Instance MachineInstance + SnapshotInstance MachineInstance + Err error + SnapshotErr error + TemplateFn func(*model.Application) (MachineInstance, error) + ReturnNilSnapshot bool + TemplateCalls int + SnapshotCalls int + TemplateChecks []bool } func (f *MockMachineInstanceFactory) NewFromTemplate( - _ context.Context, _ *model.Application, _ *slog.Logger, _ bool, + _ context.Context, app *model.Application, _ *slog.Logger, checkTemplateHash bool, ) (MachineInstance, error) { + f.TemplateCalls++ + f.TemplateChecks = append(f.TemplateChecks, checkTemplateHash) + if f.TemplateFn != nil { + return f.TemplateFn(app) + } return f.Instance, f.Err } func (f *MockMachineInstanceFactory) NewFromSnapshot( - _ context.Context, _ *model.Application, _ *slog.Logger, _ bool, - _ string, _ *common.Hash, _ uint64, + _ context.Context, _ *model.Application, _ *slog.Logger, + _ string, _ common.Hash, _ uint64, ) (MachineInstance, error) { + f.SnapshotCalls++ + if f.ReturnNilSnapshot { + return nil, nil + } + if f.SnapshotErr != nil { + return f.SnapshotInstance, f.SnapshotErr + } + if f.SnapshotInstance != nil { + return f.SnapshotInstance, nil + } return f.Instance, f.Err } @@ -580,25 +1863,34 @@ type realMachineInstanceFactory struct { } func (f *realMachineInstanceFactory) NewFromTemplate( - ctx context.Context, app *model.Application, logger *slog.Logger, checkHash bool, + ctx context.Context, app *model.Application, logger *slog.Logger, _ bool, ) (MachineInstance, error) { - return NewMachineInstanceWithFactory(ctx, app, 0, logger, checkHash, f.runtimeFactory) + return NewMachineInstanceWithFactory(ctx, app, 0, logger, f.runtimeFactory) } func (f *realMachineInstanceFactory) NewFromSnapshot( - ctx context.Context, app *model.Application, logger *slog.Logger, checkHash bool, - _ string, _ *common.Hash, inputIndex uint64, + ctx context.Context, app *model.Application, logger *slog.Logger, + _ string, _ common.Hash, inputIndex uint64, ) (MachineInstance, error) { - return NewMachineInstanceWithFactory(ctx, app, inputIndex+1, logger, checkHash, f.runtimeFactory) + if inputIndex == math.MaxUint64 { + return nil, ErrInvalidSnapshotPoint + } + return NewMachineInstanceWithFactory(ctx, app, inputIndex+1, logger, f.runtimeFactory) } // ------------------------------------------------------------------------------------------------ // DummyMachineInstanceMock implements the MachineInstance interface for testing type DummyMachineInstanceMock struct { - application *model.Application - closeError error - synchronizeErr error + application *model.Application + closeError error + hashError error + hashReturn [32]byte + replayErr error + closeCalls int + hashCalls int + processedInputs uint64 + replayCalls int } func (m *DummyMachineInstanceMock) Application() *model.Application { @@ -606,7 +1898,7 @@ func (m *DummyMachineInstanceMock) Application() *model.Application { } func (m *DummyMachineInstanceMock) ProcessedInputs() uint64 { - return 0 + return m.processedInputs } func (m *DummyMachineInstanceMock) OutputsProof(ctx context.Context) (*model.OutputsProof, error) { @@ -617,22 +1909,20 @@ func (m *DummyMachineInstanceMock) Advance(_ context.Context, _ []byte, _ uint64 return nil, nil } -func (m *DummyMachineInstanceMock) Inspect(_ context.Context, _ []byte) (*model.InspectResult, error) { +func (m *DummyMachineInstanceMock) Inspect(_ context.Context, _ []byte) (*InspectResult, error) { return nil, nil } -func (m *DummyMachineInstanceMock) Synchronize(_ context.Context, _ MachineRepository, _ uint64) error { - return m.synchronizeErr -} - func (m *DummyMachineInstanceMock) CreateSnapshot(_ context.Context, _ uint64, _ string) error { return nil } func (m *DummyMachineInstanceMock) Hash(_ context.Context) ([32]byte, error) { - return [32]byte{}, nil + m.hashCalls++ + return m.hashReturn, m.hashError } func (m *DummyMachineInstanceMock) Close() error { + m.closeCalls++ return m.closeError } diff --git a/internal/manager/types.go b/internal/manager/types.go index d72e1f97a..91570fff8 100644 --- a/internal/manager/types.go +++ b/internal/manager/types.go @@ -7,14 +7,25 @@ import ( "context" . "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/pkg/machine" ) +// InspectResult carries a typed machine completion or an incomplete execution +// failure. Error is non-nil only when Status is machine.CompletionStatusUnknown; +// Reports then contains any reports emitted before the failure. +type InspectResult struct { + ProcessedInputs uint64 + Status machine.CompletionStatus + ExceptionData []byte + Reports [][]byte + Error error +} + // MachineInstance defines the interface for a machine instance type MachineInstance interface { Application() *Application Advance(ctx context.Context, input []byte, epochIndex uint64, inputIndex uint64, computeHashes bool) (*AdvanceResult, error) Inspect(ctx context.Context, query []byte) (*InspectResult, error) - Synchronize(ctx context.Context, repo MachineRepository, batchSize uint64) error CreateSnapshot(ctx context.Context, processedInputs uint64, path string) error ProcessedInputs() uint64 Hash(ctx context.Context) ([32]byte, error) @@ -33,9 +44,17 @@ type MachineProvider interface { // UpdateMachines refreshes the list of machines UpdateMachines(ctx context.Context) error + // FenceApplicationFailure fences an application whose initial FAILED + // status write could not be confirmed. It queues a later durability retry + // without duplicating the initial repository write. + FenceApplicationFailure(app *Application, reason string) + // HasMachine checks if a machine exists for the given application ID HasMachine(appID int64) bool + // HasPendingApplicationFailures reports an unresolved durable-status fence. + HasPendingApplicationFailures() bool + // Close shuts down all machine instances and releases resources Close() error } diff --git a/internal/model/execution_parameters_test.go b/internal/model/execution_parameters_test.go new file mode 100644 index 000000000..7a22256e3 --- /dev/null +++ b/internal/model/execution_parameters_test.go @@ -0,0 +1,166 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package model + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestExecutionParametersJSONRoundtrip(t *testing.T) { + original := ExecutionParameters{ + SnapshotPolicy: SnapshotPolicy_EveryInput, + AdvanceIncCycles: 101, + AdvanceMaxCycles: 111, + InspectIncCycles: 202, + InspectMaxCycles: 222, + AdvanceIncDeadline: 3 * time.Second, + AdvanceMaxDeadline: 4 * time.Second, + InspectIncDeadline: 5 * time.Second, + InspectMaxDeadline: 6 * time.Second, + LoadDeadline: 7 * time.Second, + StoreDeadline: 8 * time.Second, + FastDeadline: 9 * time.Second, + MaxConcurrentInspects: 10, + } + + data, err := json.Marshal(&original) + require.NoError(t, err) + require.Contains(t, string(data), `"advance_inc_cycles":"0x65"`) + require.Contains(t, string(data), `"advance_max_cycles":"0x6f"`) + require.Contains(t, string(data), `"inspect_inc_cycles":"0xca"`) + require.Contains(t, string(data), `"inspect_max_cycles":"0xde"`) + + var decoded ExecutionParameters + require.NoError(t, json.Unmarshal(data, &decoded)) + require.Equal(t, original.SnapshotPolicy, decoded.SnapshotPolicy) + require.Equal(t, original.AdvanceIncCycles, decoded.AdvanceIncCycles) + require.Equal(t, original.AdvanceMaxCycles, decoded.AdvanceMaxCycles) + require.Equal(t, original.InspectIncCycles, decoded.InspectIncCycles) + require.Equal(t, original.InspectMaxCycles, decoded.InspectMaxCycles) + require.Equal(t, original.AdvanceIncDeadline, decoded.AdvanceIncDeadline) + require.Equal(t, original.AdvanceMaxDeadline, decoded.AdvanceMaxDeadline) + require.Equal(t, original.InspectIncDeadline, decoded.InspectIncDeadline) + require.Equal(t, original.InspectMaxDeadline, decoded.InspectMaxDeadline) + require.Equal(t, original.LoadDeadline, decoded.LoadDeadline) + require.Equal(t, original.StoreDeadline, decoded.StoreDeadline) + require.Equal(t, original.FastDeadline, decoded.FastDeadline) + require.Equal(t, original.MaxConcurrentInspects, decoded.MaxConcurrentInspects) +} + +func TestExecutionParametersZeroCycleMaximumsRoundtrip(t *testing.T) { + data, err := json.Marshal(&ExecutionParameters{}) + require.NoError(t, err) + require.Contains(t, string(data), `"advance_max_cycles":"0x0"`) + require.Contains(t, string(data), `"inspect_max_cycles":"0x0"`) + + var decoded ExecutionParameters + require.NoError(t, json.Unmarshal(data, &decoded)) + require.Zero(t, decoded.AdvanceMaxCycles) + require.Zero(t, decoded.InspectMaxCycles) +} + +func TestExecutionParametersCycleMaximumBounds(t *testing.T) { + for _, value := range []uint64{0, 1, MaxExecutionCycleSpan} { + params := ExecutionParameters{ + SnapshotPolicy: SnapshotPolicy_None, + AdvanceIncCycles: 1, + AdvanceMaxCycles: value, + InspectIncCycles: 1, + InspectMaxCycles: value, + } + require.NoError(t, params.Validate()) + } + + for _, test := range []struct { + name string + mutate func(*ExecutionParameters) + }{ + {"advance", func(p *ExecutionParameters) { p.AdvanceMaxCycles = MaxExecutionCycles }}, + {"inspect", func(p *ExecutionParameters) { p.InspectMaxCycles = MaxExecutionCycles }}, + } { + t.Run(test.name, func(t *testing.T) { + params := ExecutionParameters{ + SnapshotPolicy: SnapshotPolicy_None, + AdvanceIncCycles: 1, + InspectIncCycles: 1, + } + test.mutate(¶ms) + require.ErrorContains(t, params.Validate(), "must be between 0 and 281474976710655") + }) + } +} + +func TestExecutionParametersRejectZeroCycleIncrements(t *testing.T) { + for _, test := range []struct { + name string + field string + mutate func(*ExecutionParameters) + }{ + { + name: "advance", + field: "advance_inc_cycles", + mutate: func(p *ExecutionParameters) { p.AdvanceIncCycles = 0 }, + }, + { + name: "inspect", + field: "inspect_inc_cycles", + mutate: func(p *ExecutionParameters) { p.InspectIncCycles = 0 }, + }, + } { + t.Run(test.name, func(t *testing.T) { + params := ExecutionParameters{ + SnapshotPolicy: SnapshotPolicy_None, + AdvanceIncCycles: 1, + InspectIncCycles: 1, + } + test.mutate(¶ms) + + require.ErrorContains(t, params.Validate(), test.field) + }) + } +} + +func TestInputCompletionStatusContract(t *testing.T) { + expected := []InputCompletionStatus{ + InputCompletionStatus_None, + InputCompletionStatus_Accepted, + InputCompletionStatus_Rejected, + InputCompletionStatus_Exception, + InputCompletionStatus_MachineHalted, + } + require.Equal(t, expected, InputCompletionStatusAllValues) + + for _, value := range expected { + t.Run(value.String(), func(t *testing.T) { + var fromString InputCompletionStatus + require.NoError(t, fromString.Scan(value.String())) + require.Equal(t, value, fromString) + + var fromBytes InputCompletionStatus + require.NoError(t, fromBytes.Scan([]byte(value.String()))) + require.Equal(t, value, fromBytes) + require.Equal(t, value != InputCompletionStatus_None, value.IsCompleted()) + }) + } + + removedOrInvalid := []string{ + "OUTPUTS_LIMIT_EXCEEDED", + "REPORTS_LIMIT_EXCEEDED", + "CYCLE_LIMIT_EXCEEDED", + "TIME_LIMIT_EXCEEDED", + "PAYLOAD_LENGTH_LIMIT_EXCEEDED", + "INVALID", + } + for _, value := range removedOrInvalid { + t.Run(value, func(t *testing.T) { + var status InputCompletionStatus + require.Error(t, status.Scan(value)) + require.False(t, InputCompletionStatus(value).IsCompleted()) + }) + } +} diff --git a/internal/model/input_hash_collection.go b/internal/model/input_hash_collection.go new file mode 100644 index 000000000..d35172691 --- /dev/null +++ b/internal/model/input_hash_collection.go @@ -0,0 +1,50 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package model + +import "fmt" + +const ( + // Log2MaxAdvanceStatesPerEpoch is the number of input-index bits reserved + // by the protocol computation-hash tree for one epoch. + Log2MaxAdvanceStatesPerEpoch uint64 = 24 + + // MaxAdvanceStatesPerEpoch is the number of input slots reserved by the + // protocol computation-hash tree for one epoch. + MaxAdvanceStatesPerEpoch uint64 = 1 << Log2MaxAdvanceStatesPerEpoch + + // Log2InputHashCollectionCapacity is the number of state-hash index bits + // reserved for one input in the protocol computation-hash tree. + Log2InputHashCollectionCapacity uint64 = 24 + + // InputHashCollectionCapacity is the number of computation-hash leaves + // reserved for one input. + InputHashCollectionCapacity uint64 = 1 << Log2InputHashCollectionCapacity + + // Log2EpochComputationHashLeafCount is the height of the epoch computation- + // hash tree: 2^24 input slots, each containing 2^24 state-hash entries. + Log2EpochComputationHashLeafCount uint64 = Log2MaxAdvanceStatesPerEpoch + Log2InputHashCollectionCapacity +) + +// ValidateInputHashCollectionSpan verifies that periodic state hashes and the +// final-state padding cover exactly one persisted input hash collection. +func ValidateInputHashCollectionSpan(hashCount, paddingRepetitions uint64) error { + if hashCount > InputHashCollectionCapacity { + return fmt.Errorf( + "collected state hash count %d exceeds input hash collection capacity %d", + hashCount, + InputHashCollectionCapacity, + ) + } + expectedPaddingRepetitions := InputHashCollectionCapacity - hashCount + if paddingRepetitions != expectedPaddingRepetitions { + return fmt.Errorf( + "collected state hash count %d with padding repetitions %d does not cover input hash collection capacity %d", + hashCount, + paddingRepetitions, + InputHashCollectionCapacity, + ) + } + return nil +} diff --git a/internal/model/input_hash_collection_test.go b/internal/model/input_hash_collection_test.go new file mode 100644 index 000000000..e727b9ab9 --- /dev/null +++ b/internal/model/input_hash_collection_test.go @@ -0,0 +1,46 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package model + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestInputHashCollectionDimensions(t *testing.T) { + t.Parallel() + require.Equal(t, uint64(24), Log2MaxAdvanceStatesPerEpoch) + require.Equal(t, uint64(1)<<24, MaxAdvanceStatesPerEpoch) + require.Equal(t, uint64(24), Log2InputHashCollectionCapacity) + require.Equal(t, uint64(1)<<24, InputHashCollectionCapacity) + require.Equal(t, uint64(48), Log2EpochComputationHashLeafCount) +} + +func TestValidateInputHashCollectionSpan(t *testing.T) { + t.Parallel() + tests := []struct { + name string + hashes uint64 + padding uint64 + wantErr bool + }{ + {name: "empty", padding: InputHashCollectionCapacity}, + {name: "partial", hashes: 2, padding: InputHashCollectionCapacity - 2}, + {name: "full", hashes: InputHashCollectionCapacity}, + {name: "short", hashes: 2, padding: InputHashCollectionCapacity - 3, wantErr: true}, + {name: "long", hashes: 2, padding: InputHashCollectionCapacity - 1, wantErr: true}, + {name: "overflow", hashes: InputHashCollectionCapacity + 1, wantErr: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := ValidateInputHashCollectionSpan(test.hashes, test.padding) + if test.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/internal/model/models.go b/internal/model/models.go index e902deedb..432650f62 100644 --- a/internal/model/models.go +++ b/internal/model/models.go @@ -606,7 +606,6 @@ func (e *ExecutionParameters) UnmarshalJSON(data []byte) error { } e.AdvanceIncCycles = val } - if aux.AdvanceMaxCycles != "" { val, err := ParseHexUint64(aux.AdvanceMaxCycles) if err != nil { @@ -622,7 +621,6 @@ func (e *ExecutionParameters) UnmarshalJSON(data []byte) error { } e.InspectIncCycles = val } - if aux.InspectMaxCycles != "" { val, err := ParseHexUint64(aux.InspectMaxCycles) if err != nil { @@ -690,12 +688,43 @@ func (e *ExecutionParameters) UnmarshalJSON(data []byte) error { return nil } +// Log2MaxExecutionCycles is the single node-side definition of the protocol +// execution window size. pkg/machine aliases it as Log2MaxMCyclesPerAdvanceState, +// so the execution ceiling and computation-hash dimensions cannot diverge. The +// emulator exposes the same value as +// CM_ROLLUP_LOG2_MAX_MCYCLES_PER_ADVANCE_STATE. +const Log2MaxExecutionCycles uint64 = 48 + +// MaxExecutionCycles is the number of cycles in one machine-enforced execution +// window. It is the mcycle window covered by one input hash collection. +const MaxExecutionCycles uint64 = 1 << Log2MaxExecutionCycles + +// MaxExecutionCycleSpan is the largest configurable distance between the +// starting mcycle and the execution endpoint. The endpoint itself is included +// in the MaxExecutionCycles-wide window, hence the subtraction by one. A +// configured maximum of zero means no operator-imposed cap; the machine's +// MaxExecutionCycleSpan ceiling applies instead. +const MaxExecutionCycleSpan uint64 = MaxExecutionCycles - 1 + // validateParameters constants const maxDuration = 24 * time.Hour const maxConcurrentInspects = 1000 // validateParameters performs validation on the loaded parameters func (e *ExecutionParameters) Validate() error { + if e.AdvanceIncCycles == 0 { + return errors.New("advance_inc_cycles must be greater than 0") + } + if e.AdvanceMaxCycles > MaxExecutionCycleSpan { + return fmt.Errorf("advance_max_cycles must be between 0 and %d", MaxExecutionCycleSpan) + } + if e.InspectIncCycles == 0 { + return errors.New("inspect_inc_cycles must be greater than 0") + } + if e.InspectMaxCycles > MaxExecutionCycleSpan { + return fmt.Errorf("inspect_max_cycles must be between 0 and %d", MaxExecutionCycleSpan) + } + // Validate durations are reasonable if e.AdvanceIncDeadline < 0 || e.AdvanceIncDeadline > maxDuration { return fmt.Errorf("advance_inc_deadline must be between 0 and 24h") @@ -957,6 +986,7 @@ type Input struct { BlockNumber uint64 `json:"block_number"` RawData []byte `json:"raw_data"` Status InputCompletionStatus `json:"status"` + ExceptionData []byte `json:"-"` MachineHash *common.Hash `json:"machine_hash"` OutputsHash *common.Hash `json:"outputs_hash"` TransactionHash common.Hash `json:"transaction_hash"` @@ -970,20 +1000,27 @@ func (i *Input) MarshalJSON() ([]byte, error) { // Create an alias to avoid infinite recursion in MarshalJSON. type Alias Input // Define a new structure that embeds the alias but overrides the hex fields. + var exceptionData *string + if i.ExceptionData != nil { + encoded := hexutil.Encode(i.ExceptionData) + exceptionData = &encoded + } aux := &struct { - EpochIndex string `json:"epoch_index"` - Index string `json:"index"` - BlockNumber string `json:"block_number"` - RawData string `json:"raw_data"` - LogIndex string `json:"log_index"` + EpochIndex string `json:"epoch_index"` + Index string `json:"index"` + BlockNumber string `json:"block_number"` + RawData string `json:"raw_data"` + ExceptionData *string `json:"exception_data"` + LogIndex string `json:"log_index"` *Alias }{ - EpochIndex: fmt.Sprintf("0x%x", i.EpochIndex), - Index: fmt.Sprintf("0x%x", i.Index), - BlockNumber: fmt.Sprintf("0x%x", i.BlockNumber), - RawData: "0x" + hex.EncodeToString(i.RawData), - LogIndex: fmt.Sprintf("0x%x", i.LogIndex), - Alias: (*Alias)(i), + EpochIndex: fmt.Sprintf("0x%x", i.EpochIndex), + Index: fmt.Sprintf("0x%x", i.Index), + BlockNumber: fmt.Sprintf("0x%x", i.BlockNumber), + RawData: "0x" + hex.EncodeToString(i.RawData), + ExceptionData: exceptionData, + LogIndex: fmt.Sprintf("0x%x", i.LogIndex), + Alias: (*Alias)(i), } return json.Marshal(aux) } @@ -991,11 +1028,12 @@ func (i *Input) MarshalJSON() ([]byte, error) { func (i *Input) UnmarshalJSON(in []byte) error { type Alias Input aux := &struct { - EpochIndex string `json:"epoch_index"` - Index string `json:"index"` - BlockNumber string `json:"block_number"` - RawData string `json:"raw_data"` - LogIndex string `json:"log_index"` + EpochIndex string `json:"epoch_index"` + Index string `json:"index"` + BlockNumber string `json:"block_number"` + RawData string `json:"raw_data"` + ExceptionData *string `json:"exception_data"` + LogIndex string `json:"log_index"` *Alias }{Alias: (*Alias)(i)} @@ -1023,6 +1061,14 @@ func (i *Input) UnmarshalJSON(in []byte) error { if err != nil { return fmt.Errorf("error on RawData: %w", err) } + if aux.ExceptionData == nil { + i.ExceptionData = nil + } else { + i.ExceptionData, err = hexutil.Decode(*aux.ExceptionData) + if err != nil { + return fmt.Errorf("error on ExceptionData: %w", err) + } + } i.LogIndex, err = ParseHexUint64(aux.LogIndex) if err != nil { @@ -1035,16 +1081,11 @@ func (i *Input) UnmarshalJSON(in []byte) error { type InputCompletionStatus string const ( - InputCompletionStatus_None InputCompletionStatus = "NONE" - InputCompletionStatus_Accepted InputCompletionStatus = "ACCEPTED" - InputCompletionStatus_Rejected InputCompletionStatus = "REJECTED" - InputCompletionStatus_Exception InputCompletionStatus = "EXCEPTION" - InputCompletionStatus_MachineHalted InputCompletionStatus = "MACHINE_HALTED" - InputCompletionStatus_OutputsLimitExceeded InputCompletionStatus = "OUTPUTS_LIMIT_EXCEEDED" - InputCompletionStatus_ReportsLimitExceeded InputCompletionStatus = "REPORTS_LIMIT_EXCEEDED" - InputCompletionStatus_CycleLimitExceeded InputCompletionStatus = "CYCLE_LIMIT_EXCEEDED" - InputCompletionStatus_TimeLimitExceeded InputCompletionStatus = "TIME_LIMIT_EXCEEDED" - InputCompletionStatus_PayloadLengthLimitExceeded InputCompletionStatus = "PAYLOAD_LENGTH_LIMIT_EXCEEDED" + InputCompletionStatus_None InputCompletionStatus = "NONE" + InputCompletionStatus_Accepted InputCompletionStatus = "ACCEPTED" + InputCompletionStatus_Rejected InputCompletionStatus = "REJECTED" + InputCompletionStatus_Exception InputCompletionStatus = "EXCEPTION" + InputCompletionStatus_MachineHalted InputCompletionStatus = "MACHINE_HALTED" ) var InputCompletionStatusAllValues = []InputCompletionStatus{ @@ -1053,11 +1094,22 @@ var InputCompletionStatusAllValues = []InputCompletionStatus{ InputCompletionStatus_Rejected, InputCompletionStatus_Exception, InputCompletionStatus_MachineHalted, - InputCompletionStatus_OutputsLimitExceeded, - InputCompletionStatus_ReportsLimitExceeded, - InputCompletionStatus_CycleLimitExceeded, - InputCompletionStatus_TimeLimitExceeded, - InputCompletionStatus_PayloadLengthLimitExceeded, +} + +// IsCompleted reports whether the status is a deterministic completed result +// of an advance execution. NONE represents an input that has not completed. +func (e InputCompletionStatus) IsCompleted() bool { + switch e { + case InputCompletionStatus_Accepted, + InputCompletionStatus_Rejected, + InputCompletionStatus_Exception, + InputCompletionStatus_MachineHalted: + return true + case InputCompletionStatus_None: + return false + default: + return false + } } func (e *InputCompletionStatus) Scan(value any) error { @@ -1082,16 +1134,6 @@ func (e *InputCompletionStatus) Scan(value any) error { *e = InputCompletionStatus_Exception case "MACHINE_HALTED": *e = InputCompletionStatus_MachineHalted - case "OUTPUTS_LIMIT_EXCEEDED": - *e = InputCompletionStatus_OutputsLimitExceeded - case "REPORTS_LIMIT_EXCEEDED": - *e = InputCompletionStatus_ReportsLimitExceeded - case "CYCLE_LIMIT_EXCEEDED": - *e = InputCompletionStatus_CycleLimitExceeded - case "TIME_LIMIT_EXCEEDED": - *e = InputCompletionStatus_TimeLimitExceeded - case "PAYLOAD_LENGTH_LIMIT_EXCEEDED": - *e = InputCompletionStatus_PayloadLengthLimitExceeded default: return errors.New("invalid value '" + enumValue + "' for InputCompletionStatus enum") } @@ -1340,16 +1382,50 @@ type AdvanceResult struct { Status InputCompletionStatus Outputs [][]byte Reports [][]byte - Hashes [][32]byte - RemainingMetaCycles uint64 + ExceptionData []byte + PeriodicStateHashes [][32]byte + PaddingRepetitions uint64 IsDaveConsensus bool } -type InspectResult struct { +// ReplaySummary identifies the immutable completed-input prefix selected for +// one machine replay. +type ReplaySummary struct { + ApplicationID int64 ProcessedInputs uint64 - Accepted bool - Reports [][]byte - Error error + Consensus Consensus +} + +// ReplayInput is the canonical input evidence needed to verify one replayed +// machine execution. It is deliberately narrower than Input: L1 metadata, +// timestamps, and snapshot location do not participate in the comparison. +type ReplayInput struct { + ApplicationID int64 + EpochIndex uint64 + InputIndex uint64 + RawData []byte + Status InputCompletionStatus + ExceptionData []byte + MachineHash *common.Hash + OutputsHash *common.Hash +} + +// ReplayStateHash is one persisted row of a PRT input hash collection. Keeping +// this projection narrow matters because one input may contain millions of +// rows. +type ReplayStateHash struct { + Index uint64 + MachineHash common.Hash + Repetitions uint64 +} + +// ReplayRecord contains one completed input and its requested verification +// evidence. Canonical records leave Outputs, Reports, and StateHashes empty. +type ReplayRecord struct { + Input ReplayInput + Outputs [][]byte + Reports [][]byte + StateHashes []ReplayStateHash } // FIXME: remove this type. Migrate claim to use Application + Epoch diff --git a/internal/model/models_json_test.go b/internal/model/models_json_test.go index a4ebd9eeb..929d23dae 100644 --- a/internal/model/models_json_test.go +++ b/internal/model/models_json_test.go @@ -55,7 +55,8 @@ func TestInputJSONRoundtrip(t *testing.T) { Index: 7, BlockNumber: 12345, RawData: []byte{0xde, 0xad, 0xbe, 0xef}, - Status: InputCompletionStatus_Accepted, + Status: InputCompletionStatus_Exception, + ExceptionData: []byte{0xff, 0x00, 0x80}, MachineHash: &machineHash, TransactionHash: common.HexToHash("0x5678"), LogIndex: 11, @@ -68,6 +69,7 @@ func TestInputJSONRoundtrip(t *testing.T) { // LogIndex must be hex-encoded like the other uint64 fields. require.Contains(t, string(data), `"log_index":"0xb"`) + require.Contains(t, string(data), `"exception_data":"0xff0080"`) var decoded Input err = json.Unmarshal(data, &decoded) @@ -80,11 +82,24 @@ func TestInputJSONRoundtrip(t *testing.T) { require.Equal(t, original.BlockNumber, decoded.BlockNumber) require.Equal(t, original.RawData, decoded.RawData) require.Equal(t, original.Status, decoded.Status) + require.Equal(t, original.ExceptionData, decoded.ExceptionData) require.Equal(t, original.MachineHash, decoded.MachineHash) require.Equal(t, original.TransactionHash, decoded.TransactionHash) require.Equal(t, original.LogIndex, decoded.LogIndex) } +func TestInputJSONDistinguishesMissingAndEmptyExceptionData(t *testing.T) { + base := Input{} + data, err := json.Marshal(&base) + require.NoError(t, err) + require.Contains(t, string(data), `"exception_data":null`) + + base.ExceptionData = []byte{} + data, err = json.Marshal(&base) + require.NoError(t, err) + require.Contains(t, string(data), `"exception_data":"0x"`) +} + func TestInputUnmarshalJSONInvalidHex(t *testing.T) { validJSON := `{"epoch_index":"0x0","index":"0x0","block_number":"0x0","raw_data":"0x","log_index":"0x0"}` tests := []struct { @@ -102,6 +117,11 @@ func TestInputUnmarshalJSONInvalidHex(t *testing.T) { json: `{"epoch_index":"0x0","index":"0x0","block_number":"0x0","raw_data":"0x","log_index":"bad"}`, wantErr: "LogIndex", }, + { + name: "invalid ExceptionData", + json: `{"epoch_index":"0x0","index":"0x0","block_number":"0x0","raw_data":"0x","exception_data":"not-hex","log_index":"0x0"}`, + wantErr: "ExceptionData", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/internal/replay/compare.go b/internal/replay/compare.go new file mode 100644 index 000000000..6a97e17a7 --- /dev/null +++ b/internal/replay/compare.go @@ -0,0 +1,238 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package replay + +import ( + "bytes" + "crypto/sha256" + "fmt" + "math" + + "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/internal/repository" + "github.com/ethereum/go-ethereum/common" +) + +type contradiction func(field string, expected, actual any) error + +func newContradiction( + application string, + epochIndex *uint64, + inputIndex uint64, + field string, + expected, actual any, +) *ContradictionError { + if application == "" { + application = "" + } + return &ContradictionError{ + Application: application, + EpochIndex: epochIndex, + InputIndex: inputIndex, + Field: field, + Expected: fmt.Sprint(expected), + Actual: fmt.Sprint(actual), + } +} + +func knownEpochIndex(epochIndex uint64) *uint64 { return &epochIndex } + +func compareRecord( + application string, + applicationID int64, + isPRT bool, + verification repository.ReplayVerificationLevel, + record *model.ReplayRecord, + actual *model.AdvanceResult, +) error { + if record == nil { + return newContradiction(application, nil, 0, "record", "persisted replay record", "nil") + } + contradiction := func(field string, expected, got any) error { + return newContradiction( + application, + knownEpochIndex(record.Input.EpochIndex), + record.Input.InputIndex, + field, + expected, + got, + ) + } + if actual == nil { + return contradiction("result", "completed result", "nil") + } + if record.Input.ApplicationID != applicationID { + return contradiction("application_id", applicationID, record.Input.ApplicationID) + } + if actual.EpochIndex != record.Input.EpochIndex { + return contradiction("epoch_index", record.Input.EpochIndex, actual.EpochIndex) + } + if actual.InputIndex != record.Input.InputIndex { + return contradiction("input_index", record.Input.InputIndex, actual.InputIndex) + } + if actual.Status != record.Input.Status { + return contradiction("status", record.Input.Status, actual.Status) + } + if !record.Input.Status.IsCompleted() { + return contradiction("status", "completed status", record.Input.Status) + } + if err := compareExceptionData(record.Input, actual, contradiction); err != nil { + return err + } + if record.Input.MachineHash == nil { + return contradiction("machine_hash", "persisted hash", "missing") + } + if actual.MachineHash != *record.Input.MachineHash { + return contradiction("machine_hash", record.Input.MachineHash.Hex(), actual.MachineHash.Hex()) + } + if record.Input.OutputsHash == nil { + return contradiction("outputs_hash", "persisted hash", "missing") + } + if actual.OutputsHash != *record.Input.OutputsHash { + return contradiction("outputs_hash", record.Input.OutputsHash.Hex(), actual.OutputsHash.Hex()) + } + if verification == repository.ReplayVerificationCanonical { + return nil + } + + if record.Input.Status == model.InputCompletionStatus_Accepted { + if err := compareBytes("outputs", record.Outputs, actual.Outputs, contradiction); err != nil { + return err + } + if err := compareBytes("reports", record.Reports, actual.Reports, contradiction); err != nil { + return err + } + } else { + // Effects of nonaccepted executions are diagnostics, not canonical. + if len(record.Outputs) != 0 { + return contradiction("outputs.count", 0, len(record.Outputs)) + } + if len(record.Reports) != 0 { + return contradiction("reports.count", 0, len(record.Reports)) + } + } + + if !isPRT { + if len(record.StateHashes) != 0 { + return contradiction("state_hashes.count", 0, len(record.StateHashes)) + } + if len(actual.PeriodicStateHashes) != 0 || actual.PaddingRepetitions != 0 { + return contradiction( + "replay_hash_collection", + "none", + fmt.Sprintf("hashes=%d padding=%d", len(actual.PeriodicStateHashes), actual.PaddingRepetitions), + ) + } + return nil + } + return compareHashCollection(record, actual, contradiction) +} + +func compareExceptionData( + input model.ReplayInput, + actual *model.AdvanceResult, + contradiction contradiction, +) error { + if input.Status == model.InputCompletionStatus_Exception { + if input.ExceptionData == nil { + return contradiction("exception_data", "persisted payload", "missing") + } + if actual.ExceptionData == nil { + return contradiction("exception_data", compactBytes(input.ExceptionData), "missing") + } + if !bytes.Equal(input.ExceptionData, actual.ExceptionData) { + return contradiction( + "exception_data", + compactBytes(input.ExceptionData), + compactBytes(actual.ExceptionData), + ) + } + return nil + } + if input.ExceptionData != nil { + return contradiction("exception_data", "none", compactBytes(input.ExceptionData)) + } + if actual.ExceptionData != nil { + return contradiction("exception_data", "none", compactBytes(actual.ExceptionData)) + } + return nil +} + +func compareBytes(field string, expected, actual [][]byte, contradiction contradiction) error { + if len(expected) != len(actual) { + return contradiction(field+".count", len(expected), len(actual)) + } + for i := range expected { + if !bytes.Equal(expected[i], actual[i]) { + return contradiction( + fmt.Sprintf("%s[%d]", field, i), + compactBytes(expected[i]), + compactBytes(actual[i]), + ) + } + } + return nil +} + +func compactBytes(data []byte) string { + digest := sha256.Sum256(data) + return fmt.Sprintf("len=%d sha256=%x", len(data), digest[:8]) +} + +func compareHashCollection( + record *model.ReplayRecord, + actual *model.AdvanceResult, + contradiction contradiction, +) error { + if err := model.ValidateInputHashCollectionSpan( + uint64(len(actual.PeriodicStateHashes)), + actual.PaddingRepetitions, + ); err != nil { + return contradiction( + "state_hashes.span", + model.InputHashCollectionCapacity, + fmt.Sprintf("hashes=%d padding=%d", len(actual.PeriodicStateHashes), actual.PaddingRepetitions), + ) + } + if actual.PaddingRepetitions == 0 { + return contradiction("state_hashes.final.repetitions", ">0", 0) + } + actualCount := len(actual.PeriodicStateHashes) + 1 + if len(record.StateHashes) != actualCount { + return contradiction("state_hashes.count", len(record.StateHashes), actualCount) + } + for i := 1; i < len(record.StateHashes); i++ { + previousIndex := record.StateHashes[i-1].Index + if previousIndex == math.MaxUint64 { + return contradiction("state_hashes.index", "index after uint64 maximum", record.StateHashes[i].Index) + } + if record.StateHashes[i].Index != previousIndex+1 { + return contradiction("state_hashes.index", previousIndex+1, record.StateHashes[i].Index) + } + } + for i, hash := range actual.PeriodicStateHashes { + row := record.StateHashes[i] + if row.Repetitions != 1 { + return contradiction(fmt.Sprintf("state_hashes[%d].repetitions", i), 1, row.Repetitions) + } + if row.MachineHash != common.Hash(hash) { + return contradiction( + fmt.Sprintf("state_hashes[%d].machine_hash", i), + row.MachineHash.Hex(), + common.Hash(hash).Hex(), + ) + } + } + final := record.StateHashes[len(record.StateHashes)-1] + if final.MachineHash != *record.Input.MachineHash { + return contradiction("state_hashes.final.machine_hash", record.Input.MachineHash.Hex(), final.MachineHash.Hex()) + } + if final.MachineHash != actual.MachineHash { + return contradiction("state_hashes.final.replay_hash", final.MachineHash.Hex(), actual.MachineHash.Hex()) + } + if final.Repetitions != actual.PaddingRepetitions { + return contradiction("state_hashes.final.repetitions", final.Repetitions, actual.PaddingRepetitions) + } + return nil +} diff --git a/internal/replay/compare_test.go b/internal/replay/compare_test.go new file mode 100644 index 000000000..830ec186c --- /dev/null +++ b/internal/replay/compare_test.go @@ -0,0 +1,374 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package replay + +import ( + "errors" + "fmt" + "testing" + + "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/internal/repository" + "github.com/ethereum/go-ethereum/common" + "github.com/stretchr/testify/require" +) + +func replayFixture(status model.InputCompletionStatus, consensus model.Consensus) ( + *model.Application, + *model.ReplayRecord, + *model.AdvanceResult, +) { + machineHash := common.HexToHash("0x11") + outputsHash := common.HexToHash("0x22") + app := &model.Application{ID: 7, Name: "replay-app", ConsensusType: consensus} + record := &model.ReplayRecord{ + Input: model.ReplayInput{ + ApplicationID: app.ID, + EpochIndex: 3, + InputIndex: 9, + RawData: []byte("input"), + Status: status, + MachineHash: &machineHash, + OutputsHash: &outputsHash, + }, + } + actual := &model.AdvanceResult{ + EpochIndex: 3, + InputIndex: 9, + Status: status, + OutputsProof: model.OutputsProof{ + MachineHash: machineHash, + OutputsHash: outputsHash, + }, + } + if status == model.InputCompletionStatus_Exception { + record.Input.ExceptionData = []byte("guest exception") + actual.ExceptionData = []byte("guest exception") + } + if status == model.InputCompletionStatus_Accepted { + record.Outputs = [][]byte{[]byte("output-a"), []byte("output-b")} + record.Reports = [][]byte{[]byte("report-a"), []byte("report-b")} + actual.Outputs = [][]byte{[]byte("output-a"), []byte("output-b")} + actual.Reports = [][]byte{[]byte("report-a"), []byte("report-b")} + } + return app, record, actual +} + +func TestCompareRecordCanonicalIgnoresFullEvidence(t *testing.T) { + app, record, actual := replayFixture( + model.InputCompletionStatus_Accepted, + model.Consensus_PRT, + ) + record.Outputs = [][]byte{[]byte("persisted-output")} + record.Reports = [][]byte{[]byte("persisted-report")} + record.StateHashes = []model.ReplayStateHash{{MachineHash: *record.Input.MachineHash, Repetitions: 1}} + actual.Outputs = [][]byte{[]byte("different-output")} + actual.Reports = [][]byte{[]byte("different-report")} + actual.PeriodicStateHashes = [][32]byte{{1}} + actual.PaddingRepetitions = model.InputHashCollectionCapacity - 1 + + require.NoError(t, compareRecord( + app.Name, + app.ID, + app.IsDaveConsensus(), + repository.ReplayVerificationCanonical, + record, + actual, + )) + + actual.MachineHash[0]++ + require.ErrorIs(t, compareRecord( + app.Name, + app.ID, + app.IsDaveConsensus(), + repository.ReplayVerificationCanonical, + record, + actual, + ), ErrContradiction) +} + +func TestCompareReplayRecordExceptionData(t *testing.T) { + t.Parallel() + + t.Run("payload mismatch", func(t *testing.T) { + app, record, actual := replayFixture( + model.InputCompletionStatus_Exception, + model.Consensus_Authority, + ) + actual.ExceptionData = []byte("different guest exception") + err := compareRecord(app.Name, app.ID, app.IsDaveConsensus(), repository.ReplayVerificationFull, record, actual) + var detail *ContradictionError + require.ErrorAs(t, err, &detail) + require.Equal(t, "exception_data", detail.Field) + require.NotContains(t, err.Error(), "guest exception") + }) + + for _, test := range []struct { + name string + status model.InputCompletionStatus + mutate func(*model.ReplayRecord, *model.AdvanceResult) + }{ + {"missing persisted exception payload", model.InputCompletionStatus_Exception, func(r *model.ReplayRecord, _ *model.AdvanceResult) { + r.Input.ExceptionData = nil + }}, + {"missing replayed exception payload", model.InputCompletionStatus_Exception, func(_ *model.ReplayRecord, a *model.AdvanceResult) { + a.ExceptionData = nil + }}, + {"unexpected persisted payload", model.InputCompletionStatus_Rejected, func(r *model.ReplayRecord, _ *model.AdvanceResult) { + r.Input.ExceptionData = []byte("unexpected") + }}, + {"unexpected replayed payload", model.InputCompletionStatus_Rejected, func(_ *model.ReplayRecord, a *model.AdvanceResult) { + a.ExceptionData = []byte("unexpected") + }}, + } { + t.Run(test.name, func(t *testing.T) { + app, record, actual := replayFixture(test.status, model.Consensus_Authority) + test.mutate(record, actual) + require.ErrorIs(t, + compareRecord(app.Name, app.ID, app.IsDaveConsensus(), repository.ReplayVerificationFull, record, actual), + ErrContradiction, + ) + }) + } +} + +func TestCompareReplayRecordCompletionMatrix(t *testing.T) { + t.Parallel() + + statuses := []model.InputCompletionStatus{ + model.InputCompletionStatus_Accepted, + model.InputCompletionStatus_Rejected, + model.InputCompletionStatus_Exception, + model.InputCompletionStatus_MachineHalted, + } + consensuses := []model.Consensus{ + model.Consensus_Authority, + model.Consensus_Quorum, + model.Consensus_PRT, + } + for _, consensus := range consensuses { + for _, status := range statuses { + t.Run(consensus.String()+"/"+status.String(), func(t *testing.T) { + app, record, actual := replayFixture(status, consensus) + if status != model.InputCompletionStatus_Accepted { + actual.Outputs = [][]byte{[]byte("noncanonical diagnostic output")} + actual.Reports = [][]byte{[]byte("noncanonical diagnostic report")} + } + if consensus == model.Consensus_PRT { + checkpoint := common.HexToHash("0x33") + actual.IsDaveConsensus = true + actual.PeriodicStateHashes = [][32]byte{checkpoint} + actual.PaddingRepetitions = model.InputHashCollectionCapacity - 1 + record.StateHashes = []model.ReplayStateHash{ + {Index: 20, MachineHash: checkpoint, Repetitions: 1}, + {Index: 21, MachineHash: actual.MachineHash, Repetitions: model.InputHashCollectionCapacity - 1}, + } + } else { + require.False(t, actual.IsDaveConsensus) + require.Empty(t, actual.PeriodicStateHashes) + require.Zero(t, actual.PaddingRepetitions) + require.Empty(t, record.StateHashes) + } + + require.NoError(t, + compareRecord(app.Name, app.ID, app.IsDaveConsensus(), repository.ReplayVerificationFull, record, actual), + ) + mismatchStatus := model.InputCompletionStatus_Accepted + if status == model.InputCompletionStatus_Accepted { + mismatchStatus = model.InputCompletionStatus_Rejected + } + actual.Status = mismatchStatus + require.ErrorIs(t, + compareRecord(app.Name, app.ID, app.IsDaveConsensus(), repository.ReplayVerificationFull, record, actual), + ErrContradiction, + ) + }) + } + } +} + +func TestCompareReplayRecordAcceptedMutationTable(t *testing.T) { + t.Parallel() + + mutations := []struct { + name string + mutate func(*model.Application, *model.ReplayRecord, *model.AdvanceResult) + }{ + {"application", func(_ *model.Application, r *model.ReplayRecord, _ *model.AdvanceResult) { + r.Input.ApplicationID++ + }}, + {"status", func(_ *model.Application, _ *model.ReplayRecord, a *model.AdvanceResult) { + a.Status = model.InputCompletionStatus_Rejected + }}, + {"machine-root", func(_ *model.Application, _ *model.ReplayRecord, a *model.AdvanceResult) { a.MachineHash[0]++ }}, + {"outputs-root", func(_ *model.Application, _ *model.ReplayRecord, a *model.AdvanceResult) { a.OutputsHash[0]++ }}, + {"outputs-count", func(_ *model.Application, _ *model.ReplayRecord, a *model.AdvanceResult) { + a.Outputs = a.Outputs[:1] + }}, + {"outputs-content", func(_ *model.Application, _ *model.ReplayRecord, a *model.AdvanceResult) { + a.Outputs[0] = []byte("changed") + }}, + {"outputs-order", func(_ *model.Application, _ *model.ReplayRecord, a *model.AdvanceResult) { + a.Outputs[0], a.Outputs[1] = a.Outputs[1], a.Outputs[0] + }}, + {"reports-count", func(_ *model.Application, _ *model.ReplayRecord, a *model.AdvanceResult) { + a.Reports = a.Reports[:1] + }}, + {"reports-content", func(_ *model.Application, _ *model.ReplayRecord, a *model.AdvanceResult) { + a.Reports[0] = []byte("changed") + }}, + {"reports-order", func(_ *model.Application, _ *model.ReplayRecord, a *model.AdvanceResult) { + a.Reports[0], a.Reports[1] = a.Reports[1], a.Reports[0] + }}, + {"unexpected-hash-collection", func(_ *model.Application, _ *model.ReplayRecord, a *model.AdvanceResult) { + a.PeriodicStateHashes = append(a.PeriodicStateHashes, common.HexToHash("0x33")) + }}, + } + for _, mutation := range mutations { + t.Run(mutation.name, func(t *testing.T) { + app, record, actual := replayFixture(model.InputCompletionStatus_Accepted, model.Consensus_Authority) + mutation.mutate(app, record, actual) + err := compareRecord(app.Name, app.ID, app.IsDaveConsensus(), repository.ReplayVerificationFull, record, actual) + require.ErrorIs(t, err, ErrContradiction) + var detail *ContradictionError + require.ErrorAs(t, err, &detail) + require.NotEmpty(t, detail.Field) + require.NotNil(t, detail.EpochIndex) + require.Equal(t, uint64(3), *detail.EpochIndex) + require.Contains(t, err.Error(), "epoch=3") + require.NotContains(t, err.Error(), "output-a") + }) + } + +} + +func TestCompareReplayRecordPersistedRecordValidation(t *testing.T) { + t.Parallel() + + t.Run("missing machine root", func(t *testing.T) { + app, record, actual := replayFixture(model.InputCompletionStatus_Rejected, model.Consensus_Authority) + record.Input.MachineHash = nil + require.ErrorIs(t, + compareRecord(app.Name, app.ID, app.IsDaveConsensus(), repository.ReplayVerificationFull, record, actual), + ErrContradiction, + ) + }) + t.Run("missing outputs root", func(t *testing.T) { + app, record, actual := replayFixture(model.InputCompletionStatus_Rejected, model.Consensus_Authority) + record.Input.OutputsHash = nil + require.ErrorIs(t, + compareRecord(app.Name, app.ID, app.IsDaveConsensus(), repository.ReplayVerificationFull, record, actual), + ErrContradiction, + ) + }) + t.Run("unknown persisted status", func(t *testing.T) { + app, record, actual := replayFixture(model.InputCompletionStatus_Rejected, model.Consensus_Authority) + record.Input.Status = model.InputCompletionStatus("UNKNOWN") + require.ErrorIs(t, + compareRecord(app.Name, app.ID, app.IsDaveConsensus(), repository.ReplayVerificationFull, record, actual), + ErrContradiction, + ) + }) + for _, effect := range []string{"output", "report"} { + t.Run("nonaccepted persisted "+effect, func(t *testing.T) { + app, record, actual := replayFixture(model.InputCompletionStatus_Rejected, model.Consensus_Authority) + if effect == "output" { + record.Outputs = [][]byte{[]byte("illegal")} + } else { + record.Reports = [][]byte{[]byte("illegal")} + } + // Replay diagnostics are ignored, but persisted effects are corruption. + actual.Outputs = [][]byte{[]byte("diagnostic")} + actual.Reports = [][]byte{[]byte("diagnostic")} + require.ErrorIs(t, + compareRecord(app.Name, app.ID, app.IsDaveConsensus(), repository.ReplayVerificationFull, record, actual), + ErrContradiction, + ) + }) + } + t.Run("nonaccepted replay diagnostics ignored", func(t *testing.T) { + app, record, actual := replayFixture(model.InputCompletionStatus_Exception, model.Consensus_Authority) + actual.Outputs = [][]byte{[]byte("diagnostic")} + actual.Reports = [][]byte{[]byte("diagnostic")} + require.NoError(t, compareRecord(app.Name, app.ID, app.IsDaveConsensus(), repository.ReplayVerificationFull, record, actual)) + }) +} + +func TestCompareReplayRecordPRTMutationTable(t *testing.T) { + t.Parallel() + + fixture := func() (*model.Application, *model.ReplayRecord, *model.AdvanceResult) { + app, record, actual := replayFixture(model.InputCompletionStatus_Accepted, model.Consensus_PRT) + hash0 := common.HexToHash("0x31") + hash1 := common.HexToHash("0x32") + actual.IsDaveConsensus = true + actual.PeriodicStateHashes = [][32]byte{hash0, hash1} + actual.PaddingRepetitions = model.InputHashCollectionCapacity - 2 + record.StateHashes = []model.ReplayStateHash{ + {Index: 40, MachineHash: hash0, Repetitions: 1}, + {Index: 41, MachineHash: hash1, Repetitions: 1}, + {Index: 42, MachineHash: actual.MachineHash, Repetitions: model.InputHashCollectionCapacity - 2}, + } + return app, record, actual + } + + app, record, actual := fixture() + require.NoError(t, compareRecord(app.Name, app.ID, app.IsDaveConsensus(), repository.ReplayVerificationFull, record, actual)) + + mutations := []struct { + name string + mutate func(*model.ReplayRecord, *model.AdvanceResult) + }{ + {"count", func(r *model.ReplayRecord, _ *model.AdvanceResult) { r.StateHashes = r.StateHashes[:2] }}, + {"hash", func(r *model.ReplayRecord, _ *model.AdvanceResult) { r.StateHashes[0].MachineHash[0]++ }}, + {"order", func(r *model.ReplayRecord, _ *model.AdvanceResult) { + r.StateHashes[0], r.StateHashes[1] = r.StateHashes[1], r.StateHashes[0] + }}, + {"index-order", func(r *model.ReplayRecord, _ *model.AdvanceResult) { r.StateHashes[1].Index++ }}, + {"repetition", func(r *model.ReplayRecord, _ *model.AdvanceResult) { r.StateHashes[0].Repetitions++ }}, + {"final-row", func(r *model.ReplayRecord, _ *model.AdvanceResult) { r.StateHashes[2].MachineHash[0]++ }}, + {"padding", func(r *model.ReplayRecord, _ *model.AdvanceResult) { r.StateHashes[2].Repetitions++ }}, + {"result-padding", func(_ *model.ReplayRecord, a *model.AdvanceResult) { a.PaddingRepetitions++ }}, + } + for _, mutation := range mutations { + t.Run(mutation.name, func(t *testing.T) { + app, record, actual := fixture() + mutation.mutate(record, actual) + require.ErrorIs(t, + compareRecord(app.Name, app.ID, app.IsDaveConsensus(), repository.ReplayVerificationFull, record, actual), + ErrContradiction, + ) + }) + } + + t.Run("diagnostics keep persisted expected and replay actual", func(t *testing.T) { + app, record, actual := fixture() + actual.PaddingRepetitions = model.InputHashCollectionCapacity - 1 + err := compareRecord(app.Name, app.ID, app.IsDaveConsensus(), repository.ReplayVerificationFull, record, actual) + var detail *ContradictionError + require.ErrorAs(t, err, &detail) + require.Equal(t, "state_hashes.span", detail.Field) + require.Equal(t, fmt.Sprint(model.InputHashCollectionCapacity), detail.Expected) + require.Equal(t, fmt.Sprintf("hashes=2 padding=%d", model.InputHashCollectionCapacity-1), detail.Actual) + }) +} + +func TestContradictionErrorIdentity(t *testing.T) { + t.Parallel() + t.Run("unknown epoch", func(t *testing.T) { + err := &ContradictionError{Application: "app", Field: "status"} + require.True(t, errors.Is(err, ErrContradiction)) + require.Contains(t, err.Error(), "epoch=") + }) + t.Run("known nonzero epoch", func(t *testing.T) { + epochIndex := uint64(17) + err := &ContradictionError{ + Application: "app", + EpochIndex: &epochIndex, + Field: "status", + } + require.Contains(t, err.Error(), "epoch=17") + require.NotContains(t, err.Error(), "epoch=") + }) +} diff --git a/internal/replay/run.go b/internal/replay/run.go new file mode 100644 index 000000000..c89b31f86 --- /dev/null +++ b/internal/replay/run.go @@ -0,0 +1,253 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package replay + +import ( + "context" + "errors" + "fmt" + + "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/internal/repository" +) + +// Executor is the machine behavior required by Run. Machine lifecycle and +// persistence remain the caller's responsibility. +type Executor interface { + ProcessedInputs() uint64 + Advance( + ctx context.Context, + input []byte, + epochIndex uint64, + inputIndex uint64, + computeHashes bool, + ) (*model.AdvanceResult, error) +} + +// Options selects an absolute, application-local, half-open replay range. +type Options struct { + Application *model.Application + FromInput uint64 + ToInputExclusive uint64 + BatchSize uint64 + Verification repository.ReplayVerificationLevel +} + +// Result summarizes a successfully verified replay. +type Result struct { + ReplayedInputs uint64 +} + +// Run reconstructs and verifies a machine over the requested input range. It +// never closes or stores executor; callers discard it after any error. +func Run( + ctx context.Context, + source repository.ReplayRepository, + executor Executor, + opts Options, +) (Result, error) { + if err := validateOptions(source, executor, opts); err != nil { + return Result{}, err + } + applicationLabel := opts.Application.Name + if applicationLabel == "" { + applicationLabel = opts.Application.IApplicationAddress.String() + } + summary, err := source.ReplaySummary( + ctx, + opts.Application.IApplicationAddress, + opts.Verification, + ) + if err != nil { + return Result{}, classifySourceError(applicationLabel, err) + } + if err := verifySummary(applicationLabel, opts.Application, summary, opts.FromInput); err != nil { + return Result{}, err + } + + isPRT := opts.Application.IsDaveConsensus() + computeHashes := opts.Verification == repository.ReplayVerificationFull && isPRT + contradiction := func(inputIndex uint64, field string, expected, actual any) error { + return newContradiction(applicationLabel, nil, inputIndex, field, expected, actual) + } + var replayed uint64 + for opts.FromInput+replayed < opts.ToInputExclusive { + from := opts.FromInput + replayed + remaining := opts.ToInputExclusive - from + limit := min(opts.BatchSize, remaining) + records, err := source.ReplayPage(ctx, repository.ReplayPageRequest{ + ApplicationID: summary.ApplicationID, + FromInput: from, + ToInputExclusive: opts.ToInputExclusive, + Limit: limit, + Verification: opts.Verification, + }) + if err != nil { + return Result{}, classifySourceError(applicationLabel, err) + } + if err := validatePage(applicationLabel, records, from, limit, remaining); err != nil { + return Result{}, err + } + for _, record := range records { + input := &record.Input + actual, err := executor.Advance( + ctx, + input.RawData, + input.EpochIndex, + input.InputIndex, + computeHashes, + ) + if err != nil { + return Result{}, fmt.Errorf( + "%w: input %d: %w", + ErrExecution, + input.InputIndex, + err, + ) + } + if err := compareRecord( + applicationLabel, + opts.Application.ID, + isPRT, + opts.Verification, + record, + actual, + ); err != nil { + return Result{}, err + } + replayed++ + if expected := input.InputIndex + 1; executor.ProcessedInputs() != expected { + return Result{}, contradiction( + input.InputIndex, + "executor.processed_inputs", + expected, + executor.ProcessedInputs(), + ) + } + } + } + return Result{ReplayedInputs: replayed}, nil +} + +func verifySummary( + application string, + expected *model.Application, + actual model.ReplaySummary, + firstInput uint64, +) error { + contradiction := func(field string, expected, actual any) error { + return newContradiction(application, nil, firstInput, field, expected, actual) + } + if actual.ApplicationID != expected.ID { + return contradiction("application_id", expected.ID, actual.ApplicationID) + } + if actual.Consensus != expected.ConsensusType { + return contradiction("consensus", expected.ConsensusType, actual.Consensus) + } + if actual.ProcessedInputs != expected.ProcessedInputs { + return contradiction( + "processed_inputs.count", + expected.ProcessedInputs, + actual.ProcessedInputs, + ) + } + return nil +} + +func validateOptions(source repository.ReplayRepository, executor Executor, opts Options) error { + switch { + case source == nil: + return fmt.Errorf("%w: replay source is nil", ErrInvalidOptions) + case executor == nil: + return fmt.Errorf("%w: replay executor is nil", ErrInvalidOptions) + case opts.Application == nil: + return fmt.Errorf("%w: application is nil", ErrInvalidOptions) + case !opts.Verification.IsValid(): + return fmt.Errorf("%w: unsupported verification level %d", ErrInvalidOptions, opts.Verification) + case opts.BatchSize == 0: + return fmt.Errorf("%w: replay batch size must be greater than zero", ErrInvalidOptions) + case opts.FromInput > opts.ToInputExclusive: + return fmt.Errorf( + "%w: lower bound %d exceeds upper bound %d", + ErrInvalidOptions, + opts.FromInput, + opts.ToInputExclusive, + ) + case opts.ToInputExclusive > opts.Application.ProcessedInputs: + return fmt.Errorf( + "%w: upper bound %d exceeds application processed input count %d", + ErrInvalidOptions, + opts.ToInputExclusive, + opts.Application.ProcessedInputs, + ) + case executor.ProcessedInputs() != opts.FromInput: + return fmt.Errorf( + "%w: executor has %d processed inputs; replay starts at %d", + ErrInvalidOptions, + executor.ProcessedInputs(), + opts.FromInput, + ) + default: + return nil + } +} + +func validatePage( + application string, + records []*model.ReplayRecord, + firstInput uint64, + limit uint64, + remaining uint64, +) error { + contradiction := func(inputIndex uint64, field string, expected, actual any) error { + return newContradiction(application, nil, inputIndex, field, expected, actual) + } + if uint64(len(records)) > limit { + return contradiction(firstInput, "replay_page.count", limit, len(records)) + } + if len(records) == 0 { + return contradiction(firstInput, "replay_sequence.remaining_records", remaining, 0) + } + for offset, record := range records { + expectedInput := firstInput + uint64(offset) + if record == nil { + return contradiction(expectedInput, "replay_record", "record", "nil") + } + if record.Input.InputIndex != expectedInput { + return contradiction( + expectedInput, + "input_index.sequence", + expectedInput, + record.Input.InputIndex, + ) + } + } + return nil +} + +func classifySourceError(application string, err error) error { + var structure *repository.ReplayStructureViolationError + if errors.As(err, &structure) { + return newContradiction( + application, + structure.EpochIndex, + structure.InputIndex, + "source."+structure.Kind.String(), + "valid persisted replay structure", + structure.Error(), + ) + } + var inconsistency *repository.ReplayInconsistentEvidenceError + if errors.As(err, &inconsistency) { + return newContradiction( + application, + nil, + inconsistency.InputIndex, + "source."+inconsistency.Kind.String()+".completed_input", + "completed replay input in page", + "not present", + ) + } + return fmt.Errorf("%w: source: %w", ErrExecution, err) +} diff --git a/internal/replay/run_test.go b/internal/replay/run_test.go new file mode 100644 index 000000000..69c59a5f2 --- /dev/null +++ b/internal/replay/run_test.go @@ -0,0 +1,301 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package replay + +import ( + "context" + "errors" + "math/big" + "testing" + + "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/internal/repository" + "github.com/ethereum/go-ethereum/common" + "github.com/stretchr/testify/require" +) + +type fakeSource struct { + summary model.ReplaySummary + summaryErr error + records []*model.ReplayRecord + pageErr error + pageRequests []repository.ReplayPageRequest + summaryLevels []repository.ReplayVerificationLevel + summaryApps []common.Address + pageOverride func(repository.ReplayPageRequest) ([]*model.ReplayRecord, error) +} + +func (source *fakeSource) ReplaySummary( + _ context.Context, + applicationAddress common.Address, + verification repository.ReplayVerificationLevel, +) (model.ReplaySummary, error) { + source.summaryApps = append(source.summaryApps, applicationAddress) + source.summaryLevels = append(source.summaryLevels, verification) + return source.summary, source.summaryErr +} + +func (source *fakeSource) ReplayPage( + _ context.Context, + request repository.ReplayPageRequest, +) ([]*model.ReplayRecord, error) { + source.pageRequests = append(source.pageRequests, request) + if source.pageOverride != nil { + return source.pageOverride(request) + } + if source.pageErr != nil { + return nil, source.pageErr + } + page := make([]*model.ReplayRecord, 0, request.Limit) + for _, record := range source.records { + if record == nil || record.Input.InputIndex >= request.FromInput && + record.Input.InputIndex < request.ToInputExclusive { + page = append(page, record) + if uint64(len(page)) == request.Limit { + break + } + } + } + return page, nil +} + +type fakeExecutor struct { + processed uint64 + advanceErr error + advanceCalls []model.ReplayInput + computeHashes []bool + wrongResultPos bool + fullPRTResult bool +} + +func (executor *fakeExecutor) ProcessedInputs() uint64 { return executor.processed } + +func (executor *fakeExecutor) Advance( + _ context.Context, + data []byte, + epochIndex uint64, + inputIndex uint64, + computeHashes bool, +) (*model.AdvanceResult, error) { + if executor.advanceErr != nil { + return nil, executor.advanceErr + } + executor.advanceCalls = append(executor.advanceCalls, model.ReplayInput{ + EpochIndex: epochIndex, + InputIndex: inputIndex, + RawData: data, + }) + executor.computeHashes = append(executor.computeHashes, computeHashes) + executor.processed++ + resultIndex := inputIndex + if executor.wrongResultPos { + resultIndex++ + } + result := &model.AdvanceResult{ + EpochIndex: epochIndex, + InputIndex: resultIndex, + Status: model.InputCompletionStatus_Accepted, + OutputsProof: model.OutputsProof{ + MachineHash: common.BigToHash(newBig(inputIndex + 1)), + OutputsHash: common.BigToHash(newBig(inputIndex + 100)), + }, + } + if executor.fullPRTResult { + result.PaddingRepetitions = 1 << 24 + } + return result, nil +} + +func newBig(value uint64) *big.Int { return new(big.Int).SetUint64(value) } + +func replayRecords(count uint64) []*model.ReplayRecord { + records := make([]*model.ReplayRecord, count) + for index := range count { + machineHash := common.BigToHash(newBig(index + 1)) + outputsHash := common.BigToHash(newBig(index + 100)) + records[index] = &model.ReplayRecord{Input: model.ReplayInput{ + ApplicationID: 7, + EpochIndex: index / 2, + InputIndex: index, + RawData: []byte{byte(index)}, + Status: model.InputCompletionStatus_Accepted, + MachineHash: &machineHash, + OutputsHash: &outputsHash, + }} + } + return records +} + +func replayOptions(consensus model.Consensus, from, to uint64) Options { + return Options{ + Application: &model.Application{ + ID: 7, + Name: "app", + IApplicationAddress: common.HexToAddress("0x1234"), + ConsensusType: consensus, + ProcessedInputs: to, + }, + FromInput: from, + ToInputExclusive: to, + BatchSize: 2, + Verification: repository.ReplayVerificationCanonical, + } +} + +func TestRunCanonicalRangeAndPagination(t *testing.T) { + records := replayRecords(5) + source := &fakeSource{ + summary: model.ReplaySummary{ApplicationID: 7, ProcessedInputs: 5, Consensus: model.Consensus_PRT}, + records: records, + } + executor := &fakeExecutor{processed: 2} + opts := replayOptions(model.Consensus_PRT, 2, 5) + + result, err := Run(context.Background(), source, executor, opts) + require.NoError(t, err) + require.Equal(t, uint64(3), result.ReplayedInputs) + require.Equal(t, uint64(5), executor.ProcessedInputs()) + require.Equal(t, []bool{false, false, false}, executor.computeHashes) + require.Equal(t, []common.Address{opts.Application.IApplicationAddress}, source.summaryApps) + require.Equal(t, []repository.ReplayPageRequest{ + {ApplicationID: 7, FromInput: 2, ToInputExclusive: 5, Limit: 2, Verification: repository.ReplayVerificationCanonical}, + {ApplicationID: 7, FromInput: 4, ToInputExclusive: 5, Limit: 1, Verification: repository.ReplayVerificationCanonical}, + }, source.pageRequests) +} + +func TestRunSelectsReplaySummaryByTypedAddress(t *testing.T) { + source := &fakeSource{ + summary: model.ReplaySummary{ + ApplicationID: 7, ProcessedInputs: 0, Consensus: model.Consensus_Authority, + }, + } + opts := replayOptions(model.Consensus_Authority, 0, 0) + // A valid application name may itself look like another application's + // address. Replay identity must never be inferred from that string. + opts.Application.Name = common.HexToAddress("0x9999").String() + + _, err := Run(context.Background(), source, &fakeExecutor{}, opts) + require.NoError(t, err) + require.Equal(t, []common.Address{opts.Application.IApplicationAddress}, source.summaryApps) +} + +func TestRunFullPRTComputesHashes(t *testing.T) { + record := replayRecords(1)[0] + record.StateHashes = []model.ReplayStateHash{{MachineHash: *record.Input.MachineHash, Repetitions: 1 << 24}} + source := &fakeSource{ + summary: model.ReplaySummary{ApplicationID: 7, ProcessedInputs: 1, Consensus: model.Consensus_PRT}, + records: []*model.ReplayRecord{record}, + } + executor := &fakeExecutor{fullPRTResult: true} + opts := replayOptions(model.Consensus_PRT, 0, 1) + opts.Verification = repository.ReplayVerificationFull + + _, err := Run(context.Background(), source, executor, opts) + require.NoError(t, err) + require.Equal(t, []bool{true}, executor.computeHashes) +} + +func TestRunCaughtUpStillValidatesSummary(t *testing.T) { + source := &fakeSource{ + summary: model.ReplaySummary{ApplicationID: 7, ProcessedInputs: 2, Consensus: model.Consensus_Authority}, + } + executor := &fakeExecutor{processed: 2} + opts := replayOptions(model.Consensus_Authority, 2, 2) + + result, err := Run(context.Background(), source, executor, opts) + require.NoError(t, err) + require.Zero(t, result.ReplayedInputs) + require.Equal(t, []repository.ReplayVerificationLevel{repository.ReplayVerificationCanonical}, source.summaryLevels) + require.Empty(t, source.pageRequests) +} + +func TestRunRejectsMalformedPagesBeforeExecution(t *testing.T) { + tests := []struct { + name string + records []*model.ReplayRecord + }{ + {name: "empty", records: nil}, + {name: "nil record", records: []*model.ReplayRecord{nil}}, + {name: "gap", records: replayRecords(2)[1:]}, + {name: "oversized", records: replayRecords(3)}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + source := &fakeSource{ + summary: model.ReplaySummary{ApplicationID: 7, ProcessedInputs: 2, Consensus: model.Consensus_Authority}, + records: test.records, + } + if test.name == "oversized" { + source.pageOverride = func(repository.ReplayPageRequest) ([]*model.ReplayRecord, error) { + return test.records, nil + } + } + executor := &fakeExecutor{} + opts := replayOptions(model.Consensus_Authority, 0, 2) + + _, err := Run(context.Background(), source, executor, opts) + require.ErrorIs(t, err, ErrContradiction) + require.Empty(t, executor.advanceCalls) + }) + } +} + +func TestRunErrorClassification(t *testing.T) { + opts := replayOptions(model.Consensus_Authority, 0, 1) + t.Run("source transport", func(t *testing.T) { + sourceErr := errors.New("database unavailable") + source := &fakeSource{summaryErr: sourceErr} + _, err := Run(context.Background(), source, &fakeExecutor{}, opts) + require.ErrorIs(t, err, ErrExecution) + require.ErrorIs(t, err, sourceErr) + require.NotErrorIs(t, err, ErrContradiction) + }) + t.Run("source structure", func(t *testing.T) { + source := &fakeSource{summaryErr: &repository.ReplayStructureViolationError{ + Kind: repository.ReplayStructureCompletedInputSequence, + InputIndex: 0, + }} + _, err := Run(context.Background(), source, &fakeExecutor{}, opts) + require.ErrorIs(t, err, ErrContradiction) + }) + t.Run("executor", func(t *testing.T) { + executionErr := errors.New("emulator unavailable") + source := &fakeSource{ + summary: model.ReplaySummary{ApplicationID: 7, ProcessedInputs: 1, Consensus: model.Consensus_Authority}, + records: replayRecords(1), + } + _, err := Run(context.Background(), source, &fakeExecutor{advanceErr: executionErr}, opts) + require.ErrorIs(t, err, ErrExecution) + require.ErrorIs(t, err, executionErr) + }) +} + +func TestRunRejectsInvalidOptions(t *testing.T) { + source := &fakeSource{} + executor := &fakeExecutor{} + opts := replayOptions(model.Consensus_Authority, 0, 1) + tests := []struct { + name string + source repository.ReplayRepository + exec Executor + mutate func(*Options) + }{ + {"nil source", nil, executor, func(*Options) {}}, + {"nil executor", source, nil, func(*Options) {}}, + {"nil app", source, executor, func(o *Options) { o.Application = nil }}, + {"invalid level", source, executor, func(o *Options) { o.Verification = 255 }}, + {"zero batch", source, executor, func(o *Options) { o.BatchSize = 0 }}, + {"inverted range", source, executor, func(o *Options) { o.FromInput = 2 }}, + {"upper beyond app", source, executor, func(o *Options) { o.ToInputExclusive = 2 }}, + {"executor position", source, &fakeExecutor{processed: 1}, func(*Options) {}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + current := opts + test.mutate(¤t) + _, err := Run(context.Background(), test.source, test.exec, current) + require.ErrorIs(t, err, ErrInvalidOptions) + }) + } +} diff --git a/internal/replay/types.go b/internal/replay/types.go new file mode 100644 index 000000000..716672da2 --- /dev/null +++ b/internal/replay/types.go @@ -0,0 +1,55 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +// Package replay reconstructs and verifies Cartesi machines from persisted +// canonical input outcomes. Repository evidence access is defined by +// repository.ReplayRepository; machine lifecycle remains caller responsibility. +package replay + +import ( + "errors" + "fmt" +) + +var ( + // ErrContradiction identifies a deterministic difference between a + // persisted result and its replay. + ErrContradiction = errors.New("replay contradicts persisted canonical result") + + // ErrExecution identifies a failure to reconstruct the requested range. + ErrExecution = errors.New("failed to replay machine execution") + + // ErrInvalidOptions identifies an invalid replay request. + ErrInvalidOptions = errors.New("invalid replay options") +) + +// ContradictionError identifies the first deterministic difference between a +// persisted result and its replay. Byte values are represented by lengths and +// digests, so the diagnostic never exposes input, output, or report payloads. +type ContradictionError struct { + Application string + EpochIndex *uint64 + InputIndex uint64 + Field string + Expected string + Actual string +} + +func (e *ContradictionError) Error() string { + epochIndex := "" + if e.EpochIndex != nil { + epochIndex = fmt.Sprint(*e.EpochIndex) + } + return fmt.Sprintf( + "%v: app=%q epoch=%s input=%d field=%s expected=%s actual=%s", + ErrContradiction, + e.Application, + epochIndex, + e.InputIndex, + e.Field, + e.Expected, + e.Actual, + ) +} + +func (e *ContradictionError) Unwrap() error { return ErrContradiction } diff --git a/internal/repository/postgres/application.go b/internal/repository/postgres/application.go index c62c34f43..c372c9db9 100644 --- a/internal/repository/postgres/application.go +++ b/internal/repository/postgres/application.go @@ -741,6 +741,7 @@ func (r *PostgresRepository) GetLastSnapshot(ctx context.Context, nameOrAddress table.Input.BlockNumber, table.Input.RawData, table.Input.Status, + table.Input.ExceptionData, table.Input.MachineHash, table.Input.OutputsHash, table.Input.TransactionHash, @@ -774,6 +775,7 @@ func (r *PostgresRepository) GetLastSnapshot(ctx context.Context, nameOrAddress &inp.BlockNumber, &inp.RawData, &inp.Status, + &inp.ExceptionData, &inp.MachineHash, &inp.OutputsHash, &inp.TransactionHash, diff --git a/internal/repository/postgres/bulk.go b/internal/repository/postgres/bulk.go index e3596d7bd..cf57379a8 100644 --- a/internal/repository/postgres/bulk.go +++ b/internal/repository/postgres/bulk.go @@ -5,7 +5,9 @@ package postgres import ( "context" + "errors" "fmt" + "math" "unsafe" "github.com/ethereum/go-ethereum/common" @@ -95,23 +97,24 @@ func getStateHashNextIndex( epochIndex uint64, ) (uint64, error) { - query := table.StateHashes.SELECT( - postgres.COALESCE( - postgres.Float(1).ADD(postgres.MAXf(table.StateHashes.Index)), - postgres.Float(0), - ), - ).WHERE( - table.StateHashes.InputEpochApplicationID.EQ(postgres.Int64(appID)). - AND(table.StateHashes.EpochIndex.EQ(uint64Expr(epochIndex))), - ) - + query := table.StateHashes.SELECT(table.StateHashes.Index). + WHERE( + table.StateHashes.InputEpochApplicationID.EQ(postgres.Int64(appID)). + AND(table.StateHashes.EpochIndex.EQ(uint64Expr(epochIndex))), + ). + ORDER_BY(table.StateHashes.Index.DESC()). + LIMIT(1) queryStr, args := query.Sql() - var currentIndex uint64 - err := tx.QueryRow(ctx, queryStr, args...).Scan(¤tIndex) - if err != nil { - return 0, fmt.Errorf("failed to get the next state hash index: %w", err) + var maxIndex uint64 + if err := tx.QueryRow(ctx, queryStr, args...).Scan(&maxIndex); errors.Is(err, pgx.ErrNoRows) { + return 0, nil + } else if err != nil { + return 0, fmt.Errorf("failed to get the last state hash index: %w", err) } - return currentIndex, nil + if maxIndex == math.MaxUint64 { + return 0, errors.New("state hash index space is exhausted") + } + return maxIndex + 1, nil } func insertOutputs( @@ -200,57 +203,123 @@ func insertStateHashes( inputIndex uint64, hashes [][32]byte, machineHash common.Hash, - remainingMetaCycles uint64, + paddingRepetitions uint64, ) error { + rowCount, err := stateHashInsertShape(uint64(len(hashes)), paddingRepetitions) + if err != nil { + return err + } nextIndex, err := getStateHashNextIndex(ctx, tx, appID, epochIndex) if err != nil { return err } - - stmt := table.StateHashes.INSERT( - table.StateHashes.InputEpochApplicationID, - table.StateHashes.EpochIndex, - table.StateHashes.InputIndex, - table.StateHashes.Index, - table.StateHashes.MachineHash, - table.StateHashes.Repetitions, - ) - - for i, h := range hashes { - stmt = stmt.VALUES( - appID, - epochIndex, - inputIndex, - nextIndex+uint64(i), - h[:], - 1, + if rowCount-1 > math.MaxUint64-nextIndex { + return fmt.Errorf( + "state hash index range overflows uint64: start=%d rows=%d", + nextIndex, + rowCount, ) } - stmt = stmt.VALUES( - appID, - epochIndex, - inputIndex, - nextIndex+uint64(len(hashes)), - machineHash[:], - remainingMetaCycles, + source := &stateHashCopySource{ + appID: appID, + epochIndex: epochIndex, + inputIndex: inputIndex, + nextIndex: nextIndex, + hashes: hashes, + machineHash: machineHash, + paddingRepetitions: paddingRepetitions, + rowCount: rowCount, + } + copied, err := tx.CopyFrom( + ctx, + pgx.Identifier{"public", "state_hashes"}, + []string{ + "input_epoch_application_id", + "epoch_index", + "input_index", + "index", + "machine_hash", + "repetitions", + }, + source, ) - - sqlStr, args := stmt.Sql() - _, err = tx.Exec(ctx, sqlStr, args...) if err != nil { return err } + expectedCopied := int64(rowCount) //nolint:gosec // Span validation bounds rowCount far below MaxInt64. + if copied != expectedCopied { + return fmt.Errorf("copied %d state hashes, expected %d", copied, expectedCopied) + } return nil } +// stateHashCopySource streams one input hash collection directly into PostgreSQL. It +// holds only the caller's hash slice and one row of CopyFrom values, so even a +// full input-span collection does not create a second multi-million-row object. +type stateHashCopySource struct { + appID int64 + epochIndex uint64 + inputIndex uint64 + nextIndex uint64 + hashes [][32]byte + machineHash common.Hash + paddingRepetitions uint64 + rowCount uint64 + nextRow uint64 + currentRow uint64 + values [6]any +} + +func (source *stateHashCopySource) Next() bool { + if source.nextRow >= source.rowCount { + return false + } + source.currentRow = source.nextRow + source.nextRow++ + return true +} + +func (source *stateHashCopySource) Values() ([]any, error) { + source.values[0] = source.appID + source.values[1] = source.epochIndex + source.values[2] = source.inputIndex + source.values[3] = source.nextIndex + source.currentRow + if source.currentRow < uint64(len(source.hashes)) { + source.values[4] = source.hashes[source.currentRow][:] + source.values[5] = int64(1) + return source.values[:], nil + } + source.values[4] = source.machineHash[:] + source.values[5] = source.paddingRepetitions + return source.values[:], nil +} + +func (*stateHashCopySource) Err() error { return nil } + +func stateHashInsertShape( + hashCount uint64, + paddingRepetitions uint64, +) (rowCount uint64, err error) { + if err := model.ValidateInputHashCollectionSpan(hashCount, paddingRepetitions); err != nil { + return 0, fmt.Errorf("invalid input hash collection span: %w", err) + } + if paddingRepetitions == 0 { + return 0, errors.New("canonical input hash collection requires a positive final repetition tail") + } + // Span validation bounds hashCount by the collection capacity, so appending the + // required canonical tail row cannot overflow uint64. + return hashCount + 1, nil +} + func updateInput( ctx context.Context, tx pgx.Tx, appID int64, inputIndex uint64, status model.InputCompletionStatus, + exceptionData []byte, outputsHash common.Hash, machineHash common.Hash, ) error { @@ -258,11 +327,13 @@ func updateInput( updStmt := table.Input. UPDATE( table.Input.Status, + table.Input.ExceptionData, table.Input.MachineHash, table.Input.OutputsHash, ). SET( status, + exceptionData, machineHash[:], outputsHash[:], ). @@ -354,6 +425,16 @@ func (r *PostgresRepository) StoreAdvanceResult( appID int64, res *model.AdvanceResult, ) error { + if res == nil { + return errors.New("advance result must not be nil") + } + if !res.Status.IsCompleted() { + return fmt.Errorf("cannot store advance result with noncompleted status %q", res.Status) + } + if err := validateAdvanceExceptionData(res.Status, res.ExceptionData); err != nil { + return err + } + tx, err := r.db.Begin(ctx) if err != nil { return err @@ -373,13 +454,18 @@ func (r *PostgresRepository) StoreAdvanceResult( } if res.IsDaveConsensus { - err = insertStateHashes(ctx, tx, appID, res.EpochIndex, res.InputIndex, res.Hashes, res.MachineHash, res.RemainingMetaCycles) + err = insertStateHashes( + ctx, tx, appID, res.EpochIndex, res.InputIndex, + res.PeriodicStateHashes, res.MachineHash, res.PaddingRepetitions, + ) if err != nil { return err } } - err = updateInput(ctx, tx, appID, res.InputIndex, res.Status, res.OutputsHash, res.MachineHash) + err = updateInput( + ctx, tx, appID, res.InputIndex, res.Status, res.ExceptionData, res.OutputsHash, res.MachineHash, + ) if err != nil { return err } @@ -398,6 +484,17 @@ func (r *PostgresRepository) StoreAdvanceResult( return tx.Commit(ctx) } +func validateAdvanceExceptionData(status model.InputCompletionStatus, data []byte) error { + switch { + case status == model.InputCompletionStatus_Exception && data == nil: + return errors.New("exception advance result must include exception data") + case status != model.InputCompletionStatus_Exception && data != nil: + return fmt.Errorf("advance result with status %q must not include exception data", status) + default: + return nil + } +} + func updateEpochClaim( ctx context.Context, tx pgx.Tx, diff --git a/internal/repository/postgres/db/rollupsdb/public/enum/inputcompletionstatus.go b/internal/repository/postgres/db/rollupsdb/public/enum/inputcompletionstatus.go index a9624fcb2..ce3410837 100644 --- a/internal/repository/postgres/db/rollupsdb/public/enum/inputcompletionstatus.go +++ b/internal/repository/postgres/db/rollupsdb/public/enum/inputcompletionstatus.go @@ -10,25 +10,15 @@ package enum import "github.com/go-jet/jet/v2/postgres" var InputCompletionStatus = &struct { - None postgres.StringExpression - Accepted postgres.StringExpression - Rejected postgres.StringExpression - Exception postgres.StringExpression - MachineHalted postgres.StringExpression - OutputsLimitExceeded postgres.StringExpression - ReportsLimitExceeded postgres.StringExpression - CycleLimitExceeded postgres.StringExpression - TimeLimitExceeded postgres.StringExpression - PayloadLengthLimitExceeded postgres.StringExpression + None postgres.StringExpression + Accepted postgres.StringExpression + Rejected postgres.StringExpression + Exception postgres.StringExpression + MachineHalted postgres.StringExpression }{ - None: postgres.NewEnumValue("NONE"), - Accepted: postgres.NewEnumValue("ACCEPTED"), - Rejected: postgres.NewEnumValue("REJECTED"), - Exception: postgres.NewEnumValue("EXCEPTION"), - MachineHalted: postgres.NewEnumValue("MACHINE_HALTED"), - OutputsLimitExceeded: postgres.NewEnumValue("OUTPUTS_LIMIT_EXCEEDED"), - ReportsLimitExceeded: postgres.NewEnumValue("REPORTS_LIMIT_EXCEEDED"), - CycleLimitExceeded: postgres.NewEnumValue("CYCLE_LIMIT_EXCEEDED"), - TimeLimitExceeded: postgres.NewEnumValue("TIME_LIMIT_EXCEEDED"), - PayloadLengthLimitExceeded: postgres.NewEnumValue("PAYLOAD_LENGTH_LIMIT_EXCEEDED"), + None: postgres.NewEnumValue("NONE"), + Accepted: postgres.NewEnumValue("ACCEPTED"), + Rejected: postgres.NewEnumValue("REJECTED"), + Exception: postgres.NewEnumValue("EXCEPTION"), + MachineHalted: postgres.NewEnumValue("MACHINE_HALTED"), } diff --git a/internal/repository/postgres/db/rollupsdb/public/table/input.go b/internal/repository/postgres/db/rollupsdb/public/table/input.go index 3d86c30a5..11c33e20c 100644 --- a/internal/repository/postgres/db/rollupsdb/public/table/input.go +++ b/internal/repository/postgres/db/rollupsdb/public/table/input.go @@ -23,6 +23,7 @@ type inputTable struct { BlockNumber postgres.ColumnFloat RawData postgres.ColumnBytea Status postgres.ColumnString + ExceptionData postgres.ColumnBytea MachineHash postgres.ColumnBytea OutputsHash postgres.ColumnBytea TransactionHash postgres.ColumnBytea @@ -77,6 +78,7 @@ func newInputTableImpl(schemaName, tableName, alias string) inputTable { BlockNumberColumn = postgres.FloatColumn("block_number") RawDataColumn = postgres.ByteaColumn("raw_data") StatusColumn = postgres.StringColumn("status") + ExceptionDataColumn = postgres.ByteaColumn("exception_data") MachineHashColumn = postgres.ByteaColumn("machine_hash") OutputsHashColumn = postgres.ByteaColumn("outputs_hash") TransactionHashColumn = postgres.ByteaColumn("transaction_hash") @@ -84,8 +86,8 @@ func newInputTableImpl(schemaName, tableName, alias string) inputTable { SnapshotURIColumn = postgres.StringColumn("snapshot_uri") CreatedAtColumn = postgres.TimestampzColumn("created_at") UpdatedAtColumn = postgres.TimestampzColumn("updated_at") - allColumns = postgres.ColumnList{EpochApplicationIDColumn, EpochIndexColumn, IndexColumn, BlockNumberColumn, RawDataColumn, StatusColumn, MachineHashColumn, OutputsHashColumn, TransactionHashColumn, LogIndexColumn, SnapshotURIColumn, CreatedAtColumn, UpdatedAtColumn} - mutableColumns = postgres.ColumnList{EpochIndexColumn, BlockNumberColumn, RawDataColumn, StatusColumn, MachineHashColumn, OutputsHashColumn, TransactionHashColumn, LogIndexColumn, SnapshotURIColumn, CreatedAtColumn, UpdatedAtColumn} + allColumns = postgres.ColumnList{EpochApplicationIDColumn, EpochIndexColumn, IndexColumn, BlockNumberColumn, RawDataColumn, StatusColumn, ExceptionDataColumn, MachineHashColumn, OutputsHashColumn, TransactionHashColumn, LogIndexColumn, SnapshotURIColumn, CreatedAtColumn, UpdatedAtColumn} + mutableColumns = postgres.ColumnList{EpochIndexColumn, BlockNumberColumn, RawDataColumn, StatusColumn, ExceptionDataColumn, MachineHashColumn, OutputsHashColumn, TransactionHashColumn, LogIndexColumn, SnapshotURIColumn, CreatedAtColumn, UpdatedAtColumn} defaultColumns = postgres.ColumnList{CreatedAtColumn, UpdatedAtColumn} ) @@ -99,6 +101,7 @@ func newInputTableImpl(schemaName, tableName, alias string) inputTable { BlockNumber: BlockNumberColumn, RawData: RawDataColumn, Status: StatusColumn, + ExceptionData: ExceptionDataColumn, MachineHash: MachineHashColumn, OutputsHash: OutputsHashColumn, TransactionHash: TransactionHashColumn, diff --git a/internal/repository/postgres/input.go b/internal/repository/postgres/input.go index 5d944d4c2..c826387a1 100644 --- a/internal/repository/postgres/input.go +++ b/internal/repository/postgres/input.go @@ -31,6 +31,7 @@ func (r *PostgresRepository) GetInput( table.Input.BlockNumber, table.Input.RawData, table.Input.Status, + table.Input.ExceptionData, table.Input.MachineHash, table.Input.OutputsHash, table.Input.TransactionHash, @@ -61,6 +62,7 @@ func (r *PostgresRepository) GetInput( &inp.BlockNumber, &inp.RawData, &inp.Status, + &inp.ExceptionData, &inp.MachineHash, &inp.OutputsHash, &inp.TransactionHash, @@ -94,6 +96,7 @@ func (r *PostgresRepository) GetLastInput( table.Input.BlockNumber, table.Input.RawData, table.Input.Status, + table.Input.ExceptionData, table.Input.MachineHash, table.Input.OutputsHash, table.Input.TransactionHash, @@ -126,6 +129,7 @@ func (r *PostgresRepository) GetLastInput( &inp.BlockNumber, &inp.RawData, &inp.Status, + &inp.ExceptionData, &inp.MachineHash, &inp.OutputsHash, &inp.TransactionHash, @@ -158,6 +162,7 @@ func (r *PostgresRepository) GetLastProcessedInput( table.Input.BlockNumber, table.Input.RawData, table.Input.Status, + table.Input.ExceptionData, table.Input.MachineHash, table.Input.OutputsHash, table.Input.TransactionHash, @@ -190,6 +195,7 @@ func (r *PostgresRepository) GetLastProcessedInput( &inp.BlockNumber, &inp.RawData, &inp.Status, + &inp.ExceptionData, &inp.MachineHash, &inp.OutputsHash, &inp.TransactionHash, @@ -267,6 +273,7 @@ func (r *PostgresRepository) ListInputs( table.Input.BlockNumber, table.Input.RawData, table.Input.Status, + table.Input.ExceptionData, table.Input.MachineHash, table.Input.OutputsHash, table.Input.TransactionHash, @@ -307,6 +314,7 @@ func (r *PostgresRepository) ListInputs( &in.BlockNumber, &in.RawData, &in.Status, + &in.ExceptionData, &in.MachineHash, &in.OutputsHash, &in.TransactionHash, diff --git a/internal/repository/postgres/input_exception_data_test.go b/internal/repository/postgres/input_exception_data_test.go new file mode 100644 index 000000000..602b83543 --- /dev/null +++ b/internal/repository/postgres/input_exception_data_test.go @@ -0,0 +1,83 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package postgres_test + +import ( + "context" + "testing" + + "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/internal/repository/factory" + "github.com/cartesi/rollups-node/internal/repository/repotest" + "github.com/cartesi/rollups-node/test/tooling/db" + "github.com/jackc/pgx/v5" + "github.com/stretchr/testify/require" +) + +func TestPostgresInputExceptionDataContract(t *testing.T) { + endpoint, err := db.GetTestDatabaseEndpoint() + if err != nil { + t.Skipf("Skipping: %v", err) + } + require.NoError(t, db.SetupTestPostgres(endpoint)) + + ctx := context.Background() + repo, err := factory.NewRepositoryFromConnectionString(ctx, endpoint) + require.NoError(t, err) + t.Cleanup(repo.Close) + + conn, err := pgx.Connect(ctx, endpoint) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, conn.Close(ctx)) }) + + seed := repotest.Seed(ctx, t, repo) + machineHash := repotest.UniqueHash() + outputsHash := repotest.UniqueHash() + + _, err = conn.Exec(ctx, ` + UPDATE input + SET status = 'EXCEPTION', machine_hash = $2, outputs_hash = $3 + WHERE epoch_application_id = $1 AND index = 0`, + seed.App.ID, machineHash[:], outputsHash[:], + ) + requirePostgresConstraint(t, err, "input_exception_data_check") + + _, err = conn.Exec(ctx, ` + UPDATE input + SET status = 'ACCEPTED', exception_data = '\x01', machine_hash = $2, outputs_hash = $3 + WHERE epoch_application_id = $1 AND index = 0`, + seed.App.ID, machineHash[:], outputsHash[:], + ) + requirePostgresConstraint(t, err, "input_exception_data_check") + + require.NoError(t, repo.StoreAdvanceResult(ctx, seed.App.ID, &model.AdvanceResult{ + EpochIndex: seed.Epoch.Index, + InputIndex: seed.Input.Index, + Status: model.InputCompletionStatus_Exception, + ExceptionData: []byte{}, + OutputsProof: model.OutputsProof{ + MachineHash: machineHash, + OutputsHash: outputsHash, + }, + })) + completed, err := repo.GetInput(ctx, seed.App.IApplicationAddress.String(), seed.Input.Index) + require.NoError(t, err) + require.NotNil(t, completed.ExceptionData) + require.Empty(t, completed.ExceptionData) + + _, err = conn.Exec(ctx, ` + UPDATE input SET exception_data = '\x02' + WHERE epoch_application_id = $1 AND index = 0`, seed.App.ID) + require.ErrorContains(t, err, "completed input result is immutable") + + _, err = conn.Exec(ctx, ` + UPDATE input SET raw_data = '\x03' + WHERE epoch_application_id = $1 AND index = 0`, seed.App.ID) + require.ErrorContains(t, err, "completed input result is immutable") + + _, err = conn.Exec(ctx, ` + UPDATE input SET snapshot_uri = '/tmp/snapshot' + WHERE epoch_application_id = $1 AND index = 0`, seed.App.ID) + require.NoError(t, err, "snapshot metadata remains mutable after completion") +} diff --git a/internal/repository/postgres/postgres_repo_test.go b/internal/repository/postgres/postgres_repo_test.go index 96e3a8e0d..642bacb0c 100644 --- a/internal/repository/postgres/postgres_repo_test.go +++ b/internal/repository/postgres/postgres_repo_test.go @@ -5,12 +5,16 @@ package postgres_test import ( "context" + "fmt" "testing" + "github.com/cartesi/rollups-node/internal/model" "github.com/cartesi/rollups-node/internal/repository" "github.com/cartesi/rollups-node/internal/repository/factory" "github.com/cartesi/rollups-node/internal/repository/repotest" "github.com/cartesi/rollups-node/test/tooling/db" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" "github.com/stretchr/testify/require" ) @@ -31,3 +35,89 @@ func TestPostgresRepository(t *testing.T) { return repo, func() { repo.Close() } }) } + +func TestPostgresSchemaExecutionOutcomeContract(t *testing.T) { + endpoint, err := db.GetTestDatabaseEndpoint() + if err != nil { + t.Skipf("Skipping: %v", err) + } + require.NoError(t, db.SetupTestPostgres(endpoint)) + + ctx := context.Background() + conn, err := pgx.Connect(ctx, endpoint) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, conn.Close(ctx)) }) + + rows, err := conn.Query(ctx, ` + SELECT enumlabel + FROM pg_enum + JOIN pg_type ON pg_type.oid = pg_enum.enumtypid + WHERE pg_type.typname = 'InputCompletionStatus' + ORDER BY enumsortorder`) + require.NoError(t, err) + labels, err := pgx.CollectRows(rows, pgx.RowTo[string]) + require.NoError(t, err) + require.Equal(t, []string{"NONE", "ACCEPTED", "REJECTED", "EXCEPTION", "MACHINE_HALTED"}, labels) + + rows, err = conn.Query(ctx, ` + SELECT column_name + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'execution_parameters'`) + require.NoError(t, err) + columns, err := pgx.CollectRows(rows, pgx.RowTo[string]) + require.NoError(t, err) + require.Contains(t, columns, "advance_max_cycles") + require.Contains(t, columns, "inspect_max_cycles") + require.Contains(t, columns, "advance_inc_cycles") + require.Contains(t, columns, "inspect_inc_cycles") + + for _, column := range []string{"advance_max_cycles", "inspect_max_cycles"} { + var defaultValue string + err := conn.QueryRow(ctx, ` + SELECT column_default + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'execution_parameters' + AND column_name = $1`, column).Scan(&defaultValue) + require.NoError(t, err) + require.Equal(t, "0", defaultValue) + } + + repo, err := factory.NewRepositoryFromConnectionString(ctx, endpoint) + require.NoError(t, err) + t.Cleanup(repo.Close) + app := repotest.NewApplicationBuilder().Create(ctx, t, repo) + for _, column := range []string{"advance_max_cycles", "inspect_max_cycles"} { + for _, value := range []int64{0, int64(model.MaxExecutionCycleSpan)} { + _, err := conn.Exec(ctx, fmt.Sprintf( + `UPDATE execution_parameters SET %s = $1 WHERE application_id = $2`, + column, + ), value, app.ID) + require.NoError(t, err, "%s must accept %d", column, value) + } + } + for _, test := range []struct { + column string + value int64 + }{ + {"advance_inc_cycles", 0}, + {"inspect_inc_cycles", 0}, + {"advance_max_cycles", -1}, + {"inspect_max_cycles", -1}, + {"advance_max_cycles", int64(model.MaxExecutionCycles)}, + {"inspect_max_cycles", int64(model.MaxExecutionCycles)}, + } { + _, err := conn.Exec(ctx, fmt.Sprintf( + `UPDATE execution_parameters SET %s = $1 WHERE application_id = $2`, + test.column, + ), test.value, app.ID) + requirePostgresConstraint(t, err, "execution_parameters_"+test.column+"_check") + } +} + +func requirePostgresConstraint(t *testing.T, err error, constraint string) { + t.Helper() + require.Error(t, err) + var pgErr *pgconn.PgError + require.ErrorAs(t, err, &pgErr) + require.Equal(t, constraint, pgErr.ConstraintName) +} diff --git a/internal/repository/postgres/replay.go b/internal/repository/postgres/replay.go new file mode 100644 index 000000000..540b0eb55 --- /dev/null +++ b/internal/repository/postgres/replay.go @@ -0,0 +1,543 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package postgres + +import ( + "context" + "errors" + "fmt" + "math" + + "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/internal/repository" + "github.com/cartesi/rollups-node/internal/repository/postgres/db/rollupsdb/public/table" + "github.com/ethereum/go-ethereum/common" + "github.com/go-jet/jet/v2/postgres" + "github.com/jackc/pgx/v5" +) + +var replayCompletedStatuses = []postgres.Expression{ + postgres.NewEnumValue(model.InputCompletionStatus_Accepted.String()), + postgres.NewEnumValue(model.InputCompletionStatus_Rejected.String()), + postgres.NewEnumValue(model.InputCompletionStatus_Exception.String()), + postgres.NewEnumValue(model.InputCompletionStatus_MachineHalted.String()), +} + +func replayCompletedStatus(status postgres.StringExpression) postgres.BoolExpression { + return status.IN(replayCompletedStatuses...) +} + +// replayChildSpec captures the schema shared by replay outputs and reports. +// State hashes deliberately use a separate path because their ordering and +// repetition invariants are different. +type replayChildSpec struct { + table postgres.Table + applicationID postgres.IntegerExpression + inputIndex postgres.FloatExpression + index postgres.FloatExpression + rawData postgres.ByteaExpression + childKind repository.ReplayEvidenceKind + appendToRecord func(*model.ReplayRecord, []byte) +} + +var ( + replayOutputSpec = replayChildSpec{ + table: table.Output, + applicationID: table.Output.InputEpochApplicationID, + inputIndex: table.Output.InputIndex, + index: table.Output.Index, + rawData: table.Output.RawData, + childKind: repository.ReplayEvidenceOutput, + appendToRecord: func(record *model.ReplayRecord, data []byte) { + record.Outputs = append(record.Outputs, data) + }, + } + replayReportSpec = replayChildSpec{ + table: table.Report, + applicationID: table.Report.InputEpochApplicationID, + inputIndex: table.Report.InputIndex, + index: table.Report.Index, + rawData: table.Report.RawData, + childKind: repository.ReplayEvidenceReport, + appendToRecord: func(record *model.ReplayRecord, data []byte) { + record.Reports = append(record.Reports, data) + }, + } +) + +// ReplaySummary reads the application identity, consensus, and processed-input +// count from one database snapshot. It checks that the count matches the number +// of completed inputs and that completed input indexes are contiguous from zero. +// In Full mode, it also checks PRT state-hash ordering. For non-PRT applications, +// it checks that no state hashes exist. All queries run in the same short +// repeatable-read transaction. +func (r *PostgresRepository) ReplaySummary( + ctx context.Context, + applicationAddress common.Address, + verification repository.ReplayVerificationLevel, +) (model.ReplaySummary, error) { + if !verification.IsValid() { + return model.ReplaySummary{}, fmt.Errorf("unsupported replay verification level %d", verification) + } + whereApp := table.Application.IapplicationAddress.EQ(postgres.Bytea(applicationAddress.Bytes())) + tx, err := beginReadTx(ctx, r.db) + if err != nil { + return model.ReplaySummary{}, err + } + defer tx.Rollback(ctx) //nolint:errcheck + + appStmt := table.Application.SELECT( + table.Application.ID, + table.Application.ConsensusType, + table.Application.ProcessedInputs, + ).WHERE(whereApp) + appSQL, appArgs := appStmt.Sql() + var summary model.ReplaySummary + if err := tx.QueryRow(ctx, appSQL, appArgs...).Scan( + &summary.ApplicationID, + &summary.Consensus, + &summary.ProcessedInputs, + ); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return model.ReplaySummary{}, repository.ErrNotFound + } + return model.ReplaySummary{}, err + } + + whereInputApp := table.Input.EpochApplicationID.EQ(postgres.Int64(summary.ApplicationID)) + countStmt := table.Input.SELECT(postgres.COUNT(postgres.STAR)). + WHERE(whereInputApp.AND(replayCompletedStatus(table.Input.Status))) + completedInputCount, err := countFromTx(ctx, tx, countStmt) + if err != nil { + return model.ReplaySummary{}, err + } + if completedInputCount != summary.ProcessedInputs { + return model.ReplaySummary{}, &repository.ReplayStructureViolationError{ + Kind: repository.ReplayStructureProcessedInputCount, + InputIndex: completedInputCount, + ApplicationProcessedInputs: summary.ProcessedInputs, + CompletedInputCount: completedInputCount, + } + } + if err := validateReplayCompletedPrefix(ctx, tx, whereInputApp, completedInputCount); err != nil { + return model.ReplaySummary{}, err + } + if verification == repository.ReplayVerificationFull { + whereStateHashApp := table.StateHashes.InputEpochApplicationID.EQ( + postgres.Int64(summary.ApplicationID), + ) + var err error + switch summary.Consensus { + case model.Consensus_PRT: + err = validateReplayStateHashOrdering(ctx, tx, whereStateHashApp) + case model.Consensus_Authority, model.Consensus_Quorum: + err = validateReplayStateHashesAbsent(ctx, tx, whereStateHashApp) + default: + err = fmt.Errorf("unsupported replay consensus %q", summary.Consensus) + } + if err != nil { + return model.ReplaySummary{}, err + } + } + if err := tx.Commit(ctx); err != nil { + return model.ReplaySummary{}, err + } + return summary, nil +} + +func scanOptionalRow( + ctx context.Context, + tx pgx.Tx, + stmt postgres.SelectStatement, + dest ...any, +) (bool, error) { + sqlStr, args := stmt.Sql() + err := tx.QueryRow(ctx, sqlStr, args...).Scan(dest...) + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + return err == nil, err +} + +func replayStructureViolation( + kind repository.ReplayStructureViolationKind, + epochIndex uint64, + inputIndex uint64, + evidenceIndex uint64, +) *repository.ReplayStructureViolationError { + return &repository.ReplayStructureViolationError{ + Kind: kind, + EpochIndex: &epochIndex, + InputIndex: inputIndex, + EvidenceIndex: evidenceIndex, + } +} + +func validateReplayCompletedPrefix( + ctx context.Context, + tx pgx.Tx, + whereInputApp postgres.BoolExpression, + total uint64, +) error { + if total == 0 { + return nil + } + stmt := table.Input.SELECT(table.Input.Index). + WHERE(whereInputApp.AND(replayCompletedStatus(table.Input.Status))). + ORDER_BY(table.Input.Index.DESC()). + LIMIT(1) + sqlStr, args := stmt.Sql() + var lastInputIndex uint64 + if err := tx.QueryRow(ctx, sqlStr, args...).Scan(&lastInputIndex); err != nil { + return err + } + expectedLastInputIndex := total - 1 + // Input indexes are non-negative and unique by primary key. Therefore, + // total distinct rows whose maximum index is total-1 must be exactly the + // contiguous sequence 0..total-1; a gap would force a larger maximum. + if lastInputIndex != expectedLastInputIndex { + return &repository.ReplayStructureViolationError{ + Kind: repository.ReplayStructureCompletedInputSequence, + InputIndex: lastInputIndex, + ExpectedIndex: expectedLastInputIndex, + } + } + return nil +} + +func validateReplayStateHashOrdering( + ctx context.Context, + tx pgx.Tx, + whereStateHashApp postgres.BoolExpression, +) error { + orderedStateHashes := table.StateHashes.SELECT( + table.StateHashes.EpochIndex.AS("epoch_index"), + table.StateHashes.Index.AS("state_hash_index"), + table.StateHashes.InputIndex.AS("input_index"), + postgres.ROW_NUMBER().OVER( + postgres.PARTITION_BY(table.StateHashes.EpochIndex). + ORDER_BY(table.StateHashes.Index), + ).AS("ordinal"), + postgres.LAG(table.StateHashes.InputIndex).OVER( + postgres.PARTITION_BY(table.StateHashes.EpochIndex). + ORDER_BY(table.StateHashes.Index), + ).AS("previous_input_index"), + ).WHERE(whereStateHashApp).AsTable("ordered_state_hash") + + epochIndexColumn := postgres.FloatColumn("epoch_index").From(orderedStateHashes) + stateHashIndexColumn := postgres.FloatColumn("state_hash_index").From(orderedStateHashes) + inputIndexColumn := postgres.FloatColumn("input_index").From(orderedStateHashes) + ordinalColumn := postgres.FloatColumn("ordinal").From(orderedStateHashes) + previousInputIndexColumn := postgres.FloatColumn("previous_input_index").From(orderedStateHashes) + stmt := orderedStateHashes.SELECT( + epochIndexColumn, + stateHashIndexColumn, + inputIndexColumn, + ordinalColumn, + postgres.COALESCE(previousInputIndexColumn, inputIndexColumn), + previousInputIndexColumn.IS_NOT_NULL(), + ).WHERE( + stateHashIndexColumn.NOT_EQ(ordinalColumn.SUB(postgres.Float(1))).OR( + previousInputIndexColumn.IS_NOT_NULL().AND( + inputIndexColumn.LT(previousInputIndexColumn), + ), + ), + ).ORDER_BY(epochIndexColumn.ASC(), stateHashIndexColumn.ASC()).LIMIT(1) + + var epochIndex, stateHashIndex, inputIndex, ordinal, previousInputIndex uint64 + var hasPreviousInput bool + found, err := scanOptionalRow(ctx, tx, stmt, + &epochIndex, + &stateHashIndex, + &inputIndex, + &ordinal, + &previousInputIndex, + &hasPreviousInput, + ) + if err != nil { + return err + } + if !found { + return nil + } + if stateHashIndex != ordinal-1 { + violation := replayStructureViolation( + repository.ReplayStructureStateHashIndexSequence, + epochIndex, + inputIndex, + stateHashIndex, + ) + violation.ExpectedIndex = ordinal - 1 + return violation + } + violation := replayStructureViolation( + repository.ReplayStructureStateHashInputOrder, + epochIndex, + inputIndex, + stateHashIndex, + ) + if hasPreviousInput { + violation.PreviousInputIndex = previousInputIndex + } + return violation +} + +func validateReplayStateHashesAbsent( + ctx context.Context, + tx pgx.Tx, + whereStateHashApp postgres.BoolExpression, +) error { + stmt := table.StateHashes.SELECT( + table.StateHashes.EpochIndex, + table.StateHashes.InputIndex, + table.StateHashes.Index, + ).WHERE(whereStateHashApp). + ORDER_BY(table.StateHashes.EpochIndex.ASC(), table.StateHashes.Index.ASC()). + LIMIT(1) + + var epochIndex, inputIndex, stateHashIndex uint64 + found, err := scanOptionalRow( + ctx, tx, stmt, + &epochIndex, + &inputIndex, + &stateHashIndex, + ) + if err != nil { + return err + } + if !found { + return nil + } + return replayStructureViolation( + repository.ReplayStructureUnexpectedStateHash, + epochIndex, + inputIndex, + stateHashIndex, + ) +} + +// ReplayPage returns one keyset page from the summary's fixed application and +// high-water mark. Canonical pages query inputs only. Full pages load child +// evidence in the same short repeatable-read transaction. +func (r *PostgresRepository) ReplayPage( + ctx context.Context, + request repository.ReplayPageRequest, +) ([]*model.ReplayRecord, error) { + if !request.Verification.IsValid() { + return nil, fmt.Errorf("unsupported replay verification level %d", request.Verification) + } + if request.ApplicationID <= 0 { + return nil, fmt.Errorf("replay application ID must be greater than zero") + } + if request.Limit == 0 { + return nil, fmt.Errorf("replay page limit must be greater than zero") + } + if request.Limit > math.MaxInt64 { + return nil, fmt.Errorf("replay page limit %d exceeds maximum supported value %d", request.Limit, int64(math.MaxInt64)) + } + if request.FromInput > request.ToInputExclusive { + return nil, fmt.Errorf( + "replay input range is invalid: lower bound %d exceeds upper bound %d", + request.FromInput, request.ToInputExclusive, + ) + } + if request.FromInput == request.ToInputExclusive { + return []*model.ReplayRecord{}, nil + } + + whereInputApp := table.Input.EpochApplicationID.EQ(postgres.Int64(request.ApplicationID)) + tx, err := beginReadTx(ctx, r.db) + if err != nil { + return nil, err + } + defer tx.Rollback(ctx) //nolint:errcheck + + inputStmt := table.Input.SELECT( + table.Input.EpochApplicationID, + table.Input.EpochIndex, + table.Input.Index, + table.Input.RawData, + table.Input.Status, + table.Input.ExceptionData, + table.Input.MachineHash, + table.Input.OutputsHash, + ). + WHERE( + whereInputApp. + AND(replayCompletedStatus(table.Input.Status)). + AND(table.Input.Index.GT_EQ(uint64Expr(request.FromInput))). + AND(table.Input.Index.LT(uint64Expr(request.ToInputExclusive))), + ). + ORDER_BY(table.Input.Index.ASC()). + LIMIT(int64(request.Limit)) + + records, err := selectReplayInputs(ctx, tx, inputStmt) + if err != nil { + return nil, err + } + + byInput := make(map[uint64]*model.ReplayRecord, len(records)) + for _, record := range records { + byInput[record.Input.InputIndex] = record + } + if request.Verification == repository.ReplayVerificationCanonical { + if err := tx.Commit(ctx); err != nil { + return nil, err + } + return records, nil + } + + childToInputIndexExclusive := request.ToInputExclusive + if uint64(len(records)) == request.Limit { + // A full page covers only through its last returned input. Evidence for + // later completed inputs belongs to a subsequent page and must not be + // mistaken for an unmatched child in this page. + childToInputIndexExclusive = records[len(records)-1].Input.InputIndex + 1 + } + + if err := selectReplayChildren( + ctx, tx, request.ApplicationID, request.FromInput, childToInputIndexExclusive, byInput, replayOutputSpec, + ); err != nil { + return nil, err + } + if err := selectReplayChildren( + ctx, tx, request.ApplicationID, request.FromInput, childToInputIndexExclusive, byInput, replayReportSpec, + ); err != nil { + return nil, err + } + if err := selectReplayStateHashes( + ctx, tx, request.ApplicationID, request.FromInput, childToInputIndexExclusive, byInput, + ); err != nil { + return nil, err + } + if err := tx.Commit(ctx); err != nil { + return nil, err + } + return records, nil +} + +func selectReplayInputs( + ctx context.Context, + tx pgx.Tx, + stmt postgres.SelectStatement, +) ([]*model.ReplayRecord, error) { + sqlStr, args := stmt.Sql() + rows, err := tx.Query(ctx, sqlStr, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + var records []*model.ReplayRecord + for rows.Next() { + record := new(model.ReplayRecord) + in := &record.Input + if err := rows.Scan( + &in.ApplicationID, + &in.EpochIndex, + &in.InputIndex, + &in.RawData, + &in.Status, + &in.ExceptionData, + &in.MachineHash, + &in.OutputsHash, + ); err != nil { + return nil, err + } + records = append(records, record) + } + return records, rows.Err() +} + +func replayRecordForChild( + byInput map[uint64]*model.ReplayRecord, + childKind repository.ReplayEvidenceKind, + inputIndex uint64, +) (*model.ReplayRecord, error) { + record := byInput[inputIndex] + if record == nil { + return nil, &repository.ReplayInconsistentEvidenceError{ + Kind: childKind, + InputIndex: inputIndex, + } + } + return record, nil +} + +func selectReplayChildren( + ctx context.Context, + tx pgx.Tx, + applicationID int64, + fromInputIndex, toInputIndexExclusive uint64, + byInput map[uint64]*model.ReplayRecord, + spec replayChildSpec, +) error { + stmt := spec.table.SELECT( + spec.inputIndex, + spec.rawData, + ).WHERE( + spec.applicationID.EQ(postgres.Int64(applicationID)). + AND(spec.inputIndex.GT_EQ(uint64Expr(fromInputIndex))). + AND(spec.inputIndex.LT(uint64Expr(toInputIndexExclusive))), + ).ORDER_BY(spec.inputIndex.ASC(), spec.index.ASC()) + + sqlStr, args := stmt.Sql() + rows, err := tx.Query(ctx, sqlStr, args...) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var inputIndex uint64 + var data []byte + if err := rows.Scan(&inputIndex, &data); err != nil { + return err + } + record, err := replayRecordForChild(byInput, spec.childKind, inputIndex) + if err != nil { + return err + } + spec.appendToRecord(record, data) + } + return rows.Err() +} + +func selectReplayStateHashes( + ctx context.Context, + tx pgx.Tx, + applicationID int64, + fromInputIndex, toInputIndexExclusive uint64, + byInput map[uint64]*model.ReplayRecord, +) error { + stmt := table.StateHashes.SELECT( + table.StateHashes.InputIndex, + table.StateHashes.Index, + table.StateHashes.MachineHash, + table.StateHashes.Repetitions, + ).WHERE( + table.StateHashes.InputEpochApplicationID.EQ(postgres.Int64(applicationID)). + AND(table.StateHashes.InputIndex.GT_EQ(uint64Expr(fromInputIndex))). + AND(table.StateHashes.InputIndex.LT(uint64Expr(toInputIndexExclusive))), + ).ORDER_BY(table.StateHashes.InputIndex.ASC(), table.StateHashes.Index.ASC()) + + sqlStr, args := stmt.Sql() + rows, err := tx.Query(ctx, sqlStr, args...) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var inputIndex uint64 + var row model.ReplayStateHash + if err := rows.Scan(&inputIndex, &row.Index, &row.MachineHash, &row.Repetitions); err != nil { + return err + } + record, err := replayRecordForChild(byInput, repository.ReplayEvidenceStateHash, inputIndex) + if err != nil { + return err + } + record.StateHashes = append(record.StateHashes, row) + } + return rows.Err() +} diff --git a/internal/repository/postgres/replay_source_test.go b/internal/repository/postgres/replay_source_test.go new file mode 100644 index 000000000..461685cf2 --- /dev/null +++ b/internal/repository/postgres/replay_source_test.go @@ -0,0 +1,323 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package postgres_test + +import ( + "context" + "testing" + + "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/internal/repository" + "github.com/cartesi/rollups-node/internal/repository/factory" + "github.com/cartesi/rollups-node/internal/repository/repotest" + "github.com/cartesi/rollups-node/test/tooling/db" + "github.com/ethereum/go-ethereum/common" + "github.com/jackc/pgx/v5" + "github.com/stretchr/testify/require" +) + +func TestPostgresReplayVerificationLevels(t *testing.T) { + endpoint, err := db.GetTestDatabaseEndpoint() + if err != nil { + t.Skipf("Skipping: %v", err) + } + require.NoError(t, db.SetupTestPostgres(endpoint)) + + ctx := context.Background() + repo, err := factory.NewRepositoryFromConnectionString(ctx, endpoint) + require.NoError(t, err) + t.Cleanup(repo.Close) + conn, err := pgx.Connect(ctx, endpoint) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, conn.Close(ctx)) }) + + app := repotest.NewApplicationBuilder().Create(ctx, t, repo) + epoch := repotest.NewEpochBuilder(app.ID). + WithStatus(model.EpochStatus_Closed). + WithInputBounds(0, 1). + Build() + inputs := []*model.Input{ + repotest.NewInputBuilder().WithIndex(0).Build(), + repotest.NewInputBuilder().WithIndex(1).Build(), + } + require.NoError(t, repo.CreateEpochsAndInputs( + ctx, + app.IApplicationAddress.String(), + map[*model.Epoch][]*model.Input{epoch: inputs}, + 10, + )) + require.NoError(t, repo.StoreAdvanceResult(ctx, app.ID, &model.AdvanceResult{ + EpochIndex: 0, + InputIndex: 0, + Status: model.InputCompletionStatus_Accepted, + Outputs: [][]byte{[]byte("output")}, + Reports: [][]byte{[]byte("report")}, + OutputsProof: model.OutputsProof{ + MachineHash: repotest.UniqueHash(), + OutputsHash: repotest.UniqueHash(), + }, + })) + require.NoError(t, repo.StoreAdvanceResult(ctx, app.ID, &model.AdvanceResult{ + EpochIndex: 0, + InputIndex: 1, + Status: model.InputCompletionStatus_Rejected, + OutputsProof: model.OutputsProof{ + MachineHash: repotest.UniqueHash(), + OutputsHash: repotest.UniqueHash(), + }, + })) + + canonical, err := repo.ReplaySummary( + ctx, app.IApplicationAddress, repository.ReplayVerificationCanonical, + ) + require.NoError(t, err) + require.Equal(t, app.ID, canonical.ApplicationID) + require.Equal(t, uint64(2), canonical.ProcessedInputs) + require.Equal(t, model.Consensus_Authority, canonical.Consensus) + + canonicalPage, err := repo.ReplayPage(ctx, repository.ReplayPageRequest{ + ApplicationID: canonical.ApplicationID, + FromInput: 0, + ToInputExclusive: canonical.ProcessedInputs, + Limit: canonical.ProcessedInputs, + Verification: repository.ReplayVerificationCanonical, + }) + require.NoError(t, err) + require.Len(t, canonicalPage, 2) + for _, record := range canonicalPage { + require.Empty(t, record.Outputs) + require.Empty(t, record.Reports) + require.Empty(t, record.StateHashes) + } + + _, err = repo.ReplaySummary( + ctx, app.IApplicationAddress, repository.ReplayVerificationFull, + ) + require.NoError(t, err) + fullPage, err := repo.ReplayPage(ctx, repository.ReplayPageRequest{ + ApplicationID: canonical.ApplicationID, + FromInput: 0, + ToInputExclusive: canonical.ProcessedInputs, + Limit: canonical.ProcessedInputs, + Verification: repository.ReplayVerificationFull, + }) + require.NoError(t, err) + require.Equal(t, [][]byte{[]byte("output")}, fullPage[0].Outputs) + require.Equal(t, [][]byte{[]byte("report")}, fullPage[0].Reports) + + // Poison only full evidence. Canonical reconstruction remains independent + // of child evidence, while a full page exposes it to replay comparison. + _, err = conn.Exec(ctx, ` + INSERT INTO output (input_epoch_application_id, input_index, index, raw_data) + VALUES ($1, 1, 1, $2)`, app.ID, []byte("illegal-output")) + require.NoError(t, err) + _, err = repo.ReplaySummary( + ctx, app.IApplicationAddress, repository.ReplayVerificationCanonical, + ) + require.NoError(t, err) + fullPage, err = repo.ReplayPage(ctx, repository.ReplayPageRequest{ + ApplicationID: canonical.ApplicationID, + FromInput: 0, + ToInputExclusive: canonical.ProcessedInputs, + Limit: canonical.ProcessedInputs, + Verification: repository.ReplayVerificationFull, + }) + require.NoError(t, err) + require.Equal(t, [][]byte{[]byte("illegal-output")}, fullPage[1].Outputs) + + // State hashes are invalid for every Authority input, including completed + // inputs whose other Full evidence is otherwise valid. + illegalStateHash := repotest.UniqueHash() + _, err = conn.Exec(ctx, `INSERT INTO state_hashes ( + input_epoch_application_id, epoch_index, input_index, index, machine_hash, repetitions + ) VALUES ($1, 0, 1, 0, $2, 1)`, app.ID, illegalStateHash.Bytes()) + require.NoError(t, err) + _, err = repo.ReplaySummary( + ctx, app.IApplicationAddress, repository.ReplayVerificationCanonical, + ) + require.NoError(t, err) + _, err = repo.ReplaySummary( + ctx, app.IApplicationAddress, repository.ReplayVerificationFull, + ) + violation := requireReplayViolationKind( + t, err, repository.ReplayStructureUnexpectedStateHash, + ) + require.Equal(t, uint64(1), violation.InputIndex) + require.Equal(t, uint64(0), violation.EvidenceIndex) + _, err = conn.Exec(ctx, `DELETE FROM state_hashes WHERE input_epoch_application_id = $1`, app.ID) + require.NoError(t, err) + + // The persisted application counter and completed input rows are one + // invariant and must agree in the same repository snapshot. + _, err = conn.Exec(ctx, `UPDATE application SET processed_inputs = 1 WHERE id = $1`, app.ID) + require.NoError(t, err) + _, err = repo.ReplaySummary( + ctx, app.IApplicationAddress, repository.ReplayVerificationCanonical, + ) + violation = requireReplayViolationKind( + t, err, repository.ReplayStructureProcessedInputCount, + ) + require.Equal(t, uint64(1), violation.ApplicationProcessedInputs) + require.Equal(t, uint64(2), violation.CompletedInputCount) +} + +func TestPostgresReplayRejectsCompletedInputGap(t *testing.T) { + endpoint, err := db.GetTestDatabaseEndpoint() + if err != nil { + t.Skipf("Skipping: %v", err) + } + require.NoError(t, db.SetupTestPostgres(endpoint)) + + ctx := context.Background() + repo, err := factory.NewRepositoryFromConnectionString(ctx, endpoint) + require.NoError(t, err) + t.Cleanup(repo.Close) + conn, err := pgx.Connect(ctx, endpoint) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, conn.Close(ctx)) }) + + app := repotest.NewApplicationBuilder().Create(ctx, t, repo) + epoch := repotest.NewEpochBuilder(app.ID). + WithStatus(model.EpochStatus_Closed). + WithInputBounds(0, 1). + Build() + inputs := []*model.Input{ + repotest.NewInputBuilder().WithIndex(0).Build(), + repotest.NewInputBuilder().WithIndex(1).Build(), + } + require.NoError(t, repo.CreateEpochsAndInputs( + ctx, + app.IApplicationAddress.String(), + map[*model.Epoch][]*model.Input{epoch: inputs}, + 10, + )) + + // Create two completed rows at indexes 0 and 2. Changing the index while + // the input is still pending is allowed by the completion immutability + // trigger and models a malformed completed prefix without disabling it. + _, err = conn.Exec(ctx, `UPDATE input SET index = 2 + WHERE epoch_application_id = $1 AND index = 1`, app.ID) + require.NoError(t, err) + machineHash := repotest.UniqueHash() + outputsHash := repotest.UniqueHash() + _, err = conn.Exec(ctx, `UPDATE input + SET status = 'ACCEPTED', machine_hash = $2, outputs_hash = $3 + WHERE epoch_application_id = $1`, app.ID, machineHash.Bytes(), outputsHash.Bytes()) + require.NoError(t, err) + _, err = conn.Exec(ctx, `UPDATE application SET processed_inputs = 2 WHERE id = $1`, app.ID) + require.NoError(t, err) + + _, err = repo.ReplaySummary( + ctx, app.IApplicationAddress, repository.ReplayVerificationCanonical, + ) + violation := requireReplayViolationKind( + t, err, repository.ReplayStructureCompletedInputSequence, + ) + require.Equal(t, uint64(2), violation.InputIndex) + require.Equal(t, uint64(1), violation.ExpectedIndex) +} + +func TestPostgresReplayRejectsInvalidStateHashOrdering(t *testing.T) { + endpoint, err := db.GetTestDatabaseEndpoint() + if err != nil { + t.Skipf("Skipping: %v", err) + } + require.NoError(t, db.SetupTestPostgres(endpoint)) + + ctx := context.Background() + repo, err := factory.NewRepositoryFromConnectionString(ctx, endpoint) + require.NoError(t, err) + t.Cleanup(repo.Close) + conn, err := pgx.Connect(ctx, endpoint) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, conn.Close(ctx)) }) + + app := repotest.NewApplicationBuilder().WithConsensus(model.Consensus_PRT).Create(ctx, t, repo) + epoch := repotest.NewEpochBuilder(app.ID). + WithStatus(model.EpochStatus_Closed). + WithInputBounds(0, 1). + Build() + inputs := []*model.Input{ + repotest.NewInputBuilder().WithIndex(0).Build(), + repotest.NewInputBuilder().WithIndex(1).Build(), + } + require.NoError(t, repo.CreateEpochsAndInputs( + ctx, + app.IApplicationAddress.String(), + map[*model.Epoch][]*model.Input{epoch: inputs}, + 10, + )) + for inputIndex := range uint64(2) { + require.NoError(t, repo.StoreAdvanceResult(ctx, app.ID, &model.AdvanceResult{ + EpochIndex: 0, + InputIndex: inputIndex, + Status: model.InputCompletionStatus_Accepted, + IsDaveConsensus: true, + PaddingRepetitions: 1 << 24, + OutputsProof: model.OutputsProof{ + MachineHash: repotest.UniqueHash(), + OutputsHash: repotest.UniqueHash(), + }, + })) + } + _, err = repo.ReplaySummary(ctx, app.IApplicationAddress, repository.ReplayVerificationFull) + require.NoError(t, err) + + _, err = conn.Exec(ctx, `UPDATE state_hashes SET index = 2 WHERE index = 1`) + require.NoError(t, err) + _, err = repo.ReplaySummary(ctx, app.IApplicationAddress, repository.ReplayVerificationFull) + _ = requireReplayViolationKind(t, err, repository.ReplayStructureStateHashIndexSequence) + _, err = conn.Exec(ctx, `UPDATE state_hashes SET index = 1 WHERE index = 2`) + require.NoError(t, err) + + _, err = conn.Exec(ctx, `UPDATE state_hashes + SET input_index = CASE index WHEN 0 THEN 1 ELSE 0 END`) + require.NoError(t, err) + _, err = repo.ReplaySummary(ctx, app.IApplicationAddress, repository.ReplayVerificationFull) + _ = requireReplayViolationKind(t, err, repository.ReplayStructureStateHashInputOrder) +} + +func requireReplayViolationKind( + t *testing.T, + err error, + want repository.ReplayStructureViolationKind, +) *repository.ReplayStructureViolationError { + t.Helper() + require.ErrorIs(t, err, repository.ErrReplayInvalidStructure) + var violation *repository.ReplayStructureViolationError + require.ErrorAs(t, err, &violation) + require.Equal(t, want, violation.Kind) + return violation +} + +func TestPostgresReplayRejectsInvalidRequests(t *testing.T) { + endpoint, err := db.GetTestDatabaseEndpoint() + if err != nil { + t.Skipf("Skipping: %v", err) + } + require.NoError(t, db.SetupTestPostgres(endpoint)) + + ctx := context.Background() + repo, err := factory.NewRepositoryFromConnectionString(ctx, endpoint) + require.NoError(t, err) + t.Cleanup(repo.Close) + app := repotest.NewApplicationBuilder().Create(ctx, t, repo) + + _, err = repo.ReplaySummary(ctx, app.IApplicationAddress, repository.ReplayVerificationLevel(255)) + require.ErrorContains(t, err, "unsupported replay verification level") + _, err = repo.ReplaySummary( + ctx, + common.HexToAddress("0xdead"), + repository.ReplayVerificationCanonical, + ) + require.ErrorIs(t, err, repository.ErrNotFound) + _, err = repo.ReplayPage(ctx, repository.ReplayPageRequest{ + ApplicationID: app.ID, + FromInput: 1, + ToInputExclusive: 0, + Limit: 1, + Verification: repository.ReplayVerificationCanonical, + }) + require.ErrorContains(t, err, "replay input range is invalid") +} diff --git a/internal/repository/postgres/replay_test.go b/internal/repository/postgres/replay_test.go new file mode 100644 index 000000000..3fd48e483 --- /dev/null +++ b/internal/repository/postgres/replay_test.go @@ -0,0 +1,222 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package postgres + +import ( + "context" + "errors" + "testing" + + "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/internal/repository" + "github.com/cartesi/rollups-node/internal/repository/repotest" + "github.com/cartesi/rollups-node/test/tooling/db" + "github.com/ethereum/go-ethereum/common" + "github.com/jackc/pgx/v5" + "github.com/stretchr/testify/require" +) + +func TestStateHashInsertShape(t *testing.T) { + t.Parallel() + + t.Run("positive padding appends final row", func(t *testing.T) { + rows, err := stateHashInsertShape( + 1, model.InputHashCollectionCapacity-1, + ) + require.NoError(t, err) + require.Equal(t, uint64(2), rows) + }) + + t.Run("empty collection uses one positive padding row", func(t *testing.T) { + rows, err := stateHashInsertShape(0, model.InputHashCollectionCapacity) + require.NoError(t, err) + require.Equal(t, uint64(1), rows) + }) + + t.Run("exact boundary requires normalization before persistence", func(t *testing.T) { + _, err := stateHashInsertShape(model.InputHashCollectionCapacity, 0) + require.ErrorContains(t, err, "positive final repetition tail") + }) + + t.Run("rejects incomplete span", func(t *testing.T) { + _, err := stateHashInsertShape(1, 7) + require.ErrorContains(t, err, "does not cover input hash collection capacity") + }) + +} + +func TestStateHashCopySource(t *testing.T) { + t.Parallel() + + hashes := [][32]byte{common.HexToHash("0x11"), common.HexToHash("0x12")} + finalHash := common.HexToHash("0x13") + source := &stateHashCopySource{ + appID: 7, + epochIndex: 8, + inputIndex: 9, + nextIndex: 10, + hashes: hashes, + machineHash: finalHash, + paddingRepetitions: model.InputHashCollectionCapacity - uint64(len(hashes)), + rowCount: uint64(len(hashes)) + 1, + } + + for index, expectedHash := range append(hashes, finalHash) { + require.True(t, source.Next()) + values, err := source.Values() + require.NoError(t, err) + require.Equal(t, int64(7), values[0]) + require.Equal(t, uint64(8), values[1]) + require.Equal(t, uint64(9), values[2]) + require.Equal(t, uint64(10+index), values[3]) + require.Equal(t, expectedHash[:], values[4]) + if index < len(hashes) { + require.Equal(t, int64(1), values[5]) + } else { + require.Equal(t, model.InputHashCollectionCapacity-uint64(len(hashes)), values[5]) + } + } + require.False(t, source.Next()) + require.NoError(t, source.Err()) +} + +func TestInsertStateHashesQualifiesPublicSchema(t *testing.T) { + endpoint, err := db.GetTestDatabaseEndpoint() + if err != nil { + t.Skipf("Skipping: %v", err) + } + ctx := context.Background() + + // Remove a shadow schema left by an interrupted prior run before schema + // migrations try to drop the uint64/hash domains it may reference. + bootstrap, err := pgx.Connect(ctx, endpoint) + require.NoError(t, err) + _, err = bootstrap.Exec(ctx, `DROP SCHEMA IF EXISTS state_hash_copy_shadow CASCADE`) + require.NoError(t, err) + require.NoError(t, bootstrap.Close(ctx)) + require.NoError(t, db.SetupTestPostgres(endpoint)) + + repositoryInterface, err := NewPostgresRepository(ctx, endpoint, 1, 0) + require.NoError(t, err) + repo := repositoryInterface.(*PostgresRepository) + t.Cleanup(func() { + _, _ = repo.db.Exec(ctx, `DROP SCHEMA IF EXISTS state_hash_copy_shadow CASCADE`) + repo.Close() + }) + + app := repotest.NewApplicationBuilder().WithConsensus(model.Consensus_PRT).Create( + ctx, t, repo, + ) + epoch := repotest.NewEpochBuilder(app.ID). + WithStatus(model.EpochStatus_Closed). + WithInputBounds(0, 0). + Build() + input := repotest.NewInputBuilder().WithIndex(0).Build() + require.NoError(t, repo.CreateEpochsAndInputs( + ctx, + app.IApplicationAddress.String(), + map[*model.Epoch][]*model.Input{epoch: {input}}, + 1, + )) + + _, err = repo.db.Exec(ctx, ` + CREATE SCHEMA state_hash_copy_shadow; + CREATE TABLE state_hash_copy_shadow.state_hashes + (LIKE public.state_hashes INCLUDING ALL)`) + require.NoError(t, err) + tx, err := repo.db.Begin(ctx) + require.NoError(t, err) + defer tx.Rollback(ctx) //nolint:errcheck + _, err = tx.Exec(ctx, `SET LOCAL search_path TO state_hash_copy_shadow, public`) + require.NoError(t, err) + machineHash := repotest.UniqueHash() + require.NoError(t, insertStateHashes( + ctx, + tx, + app.ID, + 0, + 0, + nil, + machineHash, + model.InputHashCollectionCapacity, + )) + require.NoError(t, tx.Commit(ctx)) + + var publicCount, shadowCount uint64 + require.NoError(t, repo.db.QueryRow(ctx, `SELECT count(*) FROM public.state_hashes`).Scan(&publicCount)) + require.NoError(t, repo.db.QueryRow( + ctx, + `SELECT count(*) FROM state_hash_copy_shadow.state_hashes`, + ).Scan(&shadowCount)) + require.Equal(t, uint64(1), publicCount) + require.Zero(t, shadowCount) +} + +func TestReplayRecordForChild(t *testing.T) { + t.Parallel() + + record := new(model.ReplayRecord) + got, err := replayRecordForChild( + map[uint64]*model.ReplayRecord{4: record}, + repository.ReplayEvidenceOutput, + 4, + ) + require.NoError(t, err) + require.Same(t, record, got) + + for _, childKind := range []repository.ReplayEvidenceKind{ + repository.ReplayEvidenceOutput, + repository.ReplayEvidenceReport, + repository.ReplayEvidenceStateHash, + } { + t.Run(childKind.String(), func(t *testing.T) { + const inputWithoutCompletedRecord = uint64(7) + _, err := replayRecordForChild(nil, childKind, inputWithoutCompletedRecord) + + require.ErrorIs(t, err, repository.ErrReplayInconsistentEvidence) + var detail *repository.ReplayInconsistentEvidenceError + require.ErrorAs(t, err, &detail) + require.Equal(t, childKind, detail.Kind) + require.Equal(t, inputWithoutCompletedRecord, detail.InputIndex) + require.Contains(t, err.Error(), childKind.String()) + require.Contains(t, err.Error(), "input 7") + require.NotContains(t, err.Error(), "payload-must-stay-private") + }) + } +} + +func TestReplayInconsistencyErrorIdentity(t *testing.T) { + t.Parallel() + err := &repository.ReplayInconsistentEvidenceError{ + Kind: repository.ReplayEvidenceReport, + InputIndex: 11, + } + require.True(t, errors.Is(err, repository.ErrReplayInconsistentEvidence)) +} + +func TestReplayStructureViolationErrorIdentityAndPayloadHygiene(t *testing.T) { + t.Parallel() + epochIndex := uint64(3) + for _, kind := range []repository.ReplayStructureViolationKind{ + repository.ReplayStructureCompletedInputSequence, + repository.ReplayStructureStateHashIndexSequence, + repository.ReplayStructureStateHashInputOrder, + repository.ReplayStructureProcessedInputCount, + repository.ReplayStructureUnexpectedStateHash, + } { + t.Run(kind.String(), func(t *testing.T) { + err := &repository.ReplayStructureViolationError{ + Kind: kind, + EpochIndex: &epochIndex, + InputIndex: 4, + EvidenceIndex: 7, + ExpectedIndex: 6, + PreviousInputIndex: 5, + } + require.ErrorIs(t, err, repository.ErrReplayInvalidStructure) + require.Contains(t, err.Error(), "kind="+kind.String()) + require.NotContains(t, err.Error(), "payload-must-stay-private") + }) + } +} diff --git a/internal/repository/postgres/schema/migrations/000001_create_initial_schema.down.sql b/internal/repository/postgres/schema/migrations/000001_create_initial_schema.down.sql index f08086a54..dc333f594 100644 --- a/internal/repository/postgres/schema/migrations/000001_create_initial_schema.down.sql +++ b/internal/repository/postgres/schema/migrations/000001_create_initial_schema.down.sql @@ -4,6 +4,7 @@ BEGIN; DROP TRIGGER IF EXISTS "state_hashes_set_updated_at" ON "state_hashes"; +DROP INDEX IF EXISTS "state_hashes_input_index_idx"; DROP TABLE IF EXISTS "state_hashes"; DROP TRIGGER IF EXISTS "match_advances_set_updated_at" ON "match_advances"; @@ -29,6 +30,7 @@ DROP TRIGGER IF EXISTS "config_set_updated_at" ON "node_config"; DROP TABLE IF EXISTS "node_config"; DROP TRIGGER IF EXISTS "report_set_updated_at" ON "report"; +DROP INDEX IF EXISTS "report_input_index_idx"; DROP TABLE IF EXISTS "report"; DROP TRIGGER IF EXISTS "withdrawal_set_updated_at" ON "withdrawal"; @@ -36,16 +38,19 @@ DROP INDEX IF EXISTS "withdrawal_block_number_idx"; DROP TABLE IF EXISTS "withdrawal"; DROP TRIGGER IF EXISTS "output_set_updated_at" ON "output"; +DROP INDEX IF EXISTS "output_input_index_idx"; DROP INDEX IF EXISTS "output_raw_data_address_idx"; DROP INDEX IF EXISTS "output_raw_data_type_idx"; DROP TABLE IF EXISTS "output"; DROP TRIGGER IF EXISTS "input_set_updated_at" ON "input"; +DROP TRIGGER IF EXISTS "input_completion_immutability_check" ON "input"; DROP INDEX IF EXISTS "input_sender_idx"; DROP INDEX IF EXISTS "input_unprocessed_idx"; DROP INDEX IF EXISTS "input_status_idx"; DROP INDEX IF EXISTS "input_block_number_idx"; DROP TABLE IF EXISTS "input"; +DROP FUNCTION IF EXISTS "enforce_input_completion_immutability"; DROP TRIGGER IF EXISTS "epoch_status_transition_check" ON "epoch"; DROP TRIGGER IF EXISTS "epoch_set_updated_at" ON "epoch"; diff --git a/internal/repository/postgres/schema/migrations/000001_create_initial_schema.up.sql b/internal/repository/postgres/schema/migrations/000001_create_initial_schema.up.sql index 4bc48857c..2d2da98fe 100644 --- a/internal/repository/postgres/schema/migrations/000001_create_initial_schema.up.sql +++ b/internal/repository/postgres/schema/migrations/000001_create_initial_schema.up.sql @@ -15,12 +15,7 @@ CREATE TYPE "InputCompletionStatus" AS ENUM ( 'ACCEPTED', 'REJECTED', 'EXCEPTION', - 'MACHINE_HALTED', - 'OUTPUTS_LIMIT_EXCEEDED', - 'REPORTS_LIMIT_EXCEEDED', - 'CYCLE_LIMIT_EXCEEDED', - 'TIME_LIMIT_EXCEEDED', - 'PAYLOAD_LENGTH_LIMIT_EXCEEDED'); + 'MACHINE_HALTED'); CREATE TYPE "DefaultBlock" AS ENUM ('FINALIZED', 'LATEST', 'PENDING', 'SAFE'); @@ -192,9 +187,9 @@ CREATE TABLE "execution_parameters" ( "application_id" INT PRIMARY KEY, "snapshot_policy" "SnapshotPolicy" NOT NULL DEFAULT 'NONE', "advance_inc_cycles" BIGINT NOT NULL CHECK ("advance_inc_cycles" > 0) DEFAULT 4194304, -- 1 << 22 - "advance_max_cycles" BIGINT NOT NULL CHECK ("advance_max_cycles" > 0) DEFAULT 4611686018427387903, -- uint64 max >> 2 + "advance_max_cycles" BIGINT NOT NULL CHECK ("advance_max_cycles" >= 0 AND "advance_max_cycles" <= 281474976710655) DEFAULT 0, -- 0 uses the machine's fixed (1 << 48) - 1 cycle span "inspect_inc_cycles" BIGINT NOT NULL CHECK ("inspect_inc_cycles" > 0) DEFAULT 4194304, -- 1 << 22 - "inspect_max_cycles" BIGINT NOT NULL CHECK ("inspect_max_cycles" > 0) DEFAULT 4611686018427387903, + "inspect_max_cycles" BIGINT NOT NULL CHECK ("inspect_max_cycles" >= 0 AND "inspect_max_cycles" <= 281474976710655) DEFAULT 0, -- 0 uses the machine's fixed (1 << 48) - 1 cycle span "advance_inc_deadline" BIGINT NOT NULL CHECK ("advance_inc_deadline" > 0) DEFAULT 10000000000, -- 10s "advance_max_deadline" BIGINT NOT NULL CHECK ("advance_max_deadline" > 0) DEFAULT 180000000000, -- 180s "inspect_inc_deadline" BIGINT NOT NULL CHECK ("inspect_inc_deadline" > 0) DEFAULT 10000000000, --10s @@ -390,6 +385,7 @@ CREATE TABLE "input" "block_number" uint64 NOT NULL, "raw_data" BYTEA NOT NULL, "status" "InputCompletionStatus" NOT NULL, + "exception_data" BYTEA, "machine_hash" hash, "outputs_hash" hash, "transaction_hash" hash NOT NULL, @@ -397,9 +393,19 @@ CREATE TABLE "input" "snapshot_uri" VARCHAR(4096), "created_at" TIMESTAMPTZ NOT NULL DEFAULT NOW(), "updated_at" TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT "input_exception_data_check" CHECK ( + ("status" = 'EXCEPTION' AND "exception_data" IS NOT NULL) + OR + ("status" <> 'EXCEPTION' AND "exception_data" IS NULL) + ), CONSTRAINT "input_pkey" PRIMARY KEY ("epoch_application_id", "index"), CONSTRAINT "input_epoch_index_unique" UNIQUE ("epoch_application_id", "epoch_index", "index"), CONSTRAINT "input_application_id_tx_hash_log_index_unique" UNIQUE ("epoch_application_id", "transaction_hash", "log_index"), + CONSTRAINT "input_completed_hashes_check" CHECK ( + ("status" = 'NONE' AND "machine_hash" IS NULL AND "outputs_hash" IS NULL) + OR + ("status" <> 'NONE' AND "machine_hash" IS NOT NULL AND "outputs_hash" IS NOT NULL) + ), CONSTRAINT "input_epoch_id_fkey" FOREIGN KEY ("epoch_application_id", "epoch_index") REFERENCES "epoch"("application_id", "index") ON DELETE CASCADE ); @@ -415,6 +421,33 @@ CREATE INDEX "input_sender_idx" ON "input" ("epoch_application_id", substring("r -- dead tuples (and the shrinking partial index) are reclaimed promptly. ALTER TABLE "input" SET (autovacuum_vacuum_scale_factor = 0.02); +CREATE FUNCTION enforce_input_completion_immutability() +RETURNS TRIGGER AS $$ +BEGIN + IF OLD.status <> 'NONE' AND ( + NEW.epoch_application_id IS DISTINCT FROM OLD.epoch_application_id + OR NEW.epoch_index IS DISTINCT FROM OLD.epoch_index + OR NEW.index IS DISTINCT FROM OLD.index + OR NEW.raw_data IS DISTINCT FROM OLD.raw_data + OR NEW.status IS DISTINCT FROM OLD.status + OR NEW.exception_data IS DISTINCT FROM OLD.exception_data + OR NEW.machine_hash IS DISTINCT FROM OLD.machine_hash + OR NEW.outputs_hash IS DISTINCT FROM OLD.outputs_hash + ) THEN + RAISE EXCEPTION + 'completed input result is immutable'; + END IF; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER "input_completion_immutability_check" + BEFORE UPDATE OF "epoch_application_id", "epoch_index", "index", "raw_data", + "status", "exception_data", "machine_hash", "outputs_hash" ON "input" + FOR EACH ROW + EXECUTE FUNCTION enforce_input_completion_immutability(); + CREATE TRIGGER "input_set_updated_at" BEFORE UPDATE ON "input" FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); @@ -436,6 +469,8 @@ CREATE TABLE "output" CREATE INDEX "output_raw_data_type_idx" ON "output" ("input_epoch_application_id", substring("raw_data" FROM 1 FOR 4)); +CREATE INDEX "output_input_index_idx" ON "output" ("input_epoch_application_id", "input_index", "index"); + CREATE INDEX "output_raw_data_address_idx" ON "output" ("input_epoch_application_id", substring("raw_data" FROM 17 FOR 20)) WHERE SUBSTRING("raw_data" FROM 1 FOR 4) IN ( E'\\x10321e8b', -- DelegateCallVoucher @@ -485,6 +520,8 @@ CREATE TABLE "report" CONSTRAINT "report_input_id_fkey" FOREIGN KEY ("input_epoch_application_id", "input_index") REFERENCES "input"("epoch_application_id", "index") ON DELETE CASCADE ); +CREATE INDEX "report_input_index_idx" ON "report" ("input_epoch_application_id", "input_index", "index"); + CREATE TRIGGER "report_set_updated_at" BEFORE UPDATE ON "report" FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); @@ -659,6 +696,8 @@ CREATE TABLE "state_hashes" CONSTRAINT "state_hashes_input_id_fkey" FOREIGN KEY ("input_epoch_application_id", "epoch_index", "input_index") REFERENCES "input"("epoch_application_id", "epoch_index", "index") ON DELETE CASCADE ); +CREATE INDEX "state_hashes_input_index_idx" ON "state_hashes" ("input_epoch_application_id", "input_index", "index"); + CREATE TRIGGER "state_hashes_set_updated_at" BEFORE UPDATE ON "state_hashes" FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); diff --git a/internal/repository/replay.go b/internal/repository/replay.go new file mode 100644 index 000000000..d66c65ee6 --- /dev/null +++ b/internal/repository/replay.go @@ -0,0 +1,212 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package repository + +import ( + "context" + "errors" + "fmt" + + "github.com/cartesi/rollups-node/internal/model" + "github.com/ethereum/go-ethereum/common" +) + +var ( + // ErrReplayInconsistentEvidence identifies persisted child evidence whose + // parent is not one of the completed inputs selected for the same page. + ErrReplayInconsistentEvidence = errors.New("persisted replay evidence is inconsistent") + + // ErrReplayInvalidStructure identifies malformed persisted replay + // coordinates or relationships. Typed details never contain payload bytes. + ErrReplayInvalidStructure = errors.New("persisted replay structure is invalid") +) + +const unknownReplayValue = "unknown" + +// ReplayVerificationLevel selects the persisted evidence read during replay. +type ReplayVerificationLevel uint8 + +const ( + // ReplayVerificationCanonical reads the completion outcome and canonical + // roots. It is the inexpensive default for normal machine reconstruction. + ReplayVerificationCanonical ReplayVerificationLevel = iota + + // ReplayVerificationFull additionally reads accepted outputs and reports. + // For PRT applications, it also reads the compressed per-input state-hash + // collections used to reconstruct and verify the epoch computation hash. + // Outputs and reports are audit evidence and do not participate in that hash. + // It is intended for explicit audits and the forthcoming + // machine-tool/libcartesi adapter, not normal reconstruction. + // ReplayPage evidence is scoped to the requested half-open input range; + // ReplaySummary's completed-prefix and structural checks are application-wide. + // A maximal PRT collection is large, so callers must provision memory for + // all evidence associated with at least one input. + ReplayVerificationFull +) + +// IsValid reports whether level is a supported verification policy. +func (level ReplayVerificationLevel) IsValid() bool { + return level == ReplayVerificationCanonical || level == ReplayVerificationFull +} + +func (level ReplayVerificationLevel) String() string { + switch level { + case ReplayVerificationCanonical: + return "canonical" + case ReplayVerificationFull: + return "full" + default: + return unknownReplayValue + } +} + +// ReplayPageRequest selects an absolute, application-local, half-open input +// range and the evidence that must accompany it. +type ReplayPageRequest struct { + ApplicationID int64 + FromInput uint64 + ToInputExclusive uint64 + Limit uint64 + Verification ReplayVerificationLevel +} + +// ReplayRepository provides a stable completed-input prefix and keyset- +// paginated replay evidence. Implementations validate persisted invariants for +// the requested verification level. +type ReplayRepository interface { + ReplaySummary( + ctx context.Context, + applicationAddress common.Address, + verification ReplayVerificationLevel, + ) (model.ReplaySummary, error) + ReplayPage(ctx context.Context, request ReplayPageRequest) ([]*model.ReplayRecord, error) +} + +// ReplayStructureViolationKind identifies one payload-free replay invariant. +type ReplayStructureViolationKind uint8 + +const ( + // Zero is reserved so an uninitialized kind cannot identify a real + // structural violation and instead formats as unknown. + ReplayStructureCompletedInputSequence ReplayStructureViolationKind = iota + 1 + ReplayStructureStateHashIndexSequence + ReplayStructureStateHashInputOrder + ReplayStructureProcessedInputCount + ReplayStructureUnexpectedStateHash +) + +func (kind ReplayStructureViolationKind) String() string { + switch kind { + case ReplayStructureCompletedInputSequence: + return "completed_input.index_sequence" + case ReplayStructureStateHashIndexSequence: + return "state_hash.index_sequence" + case ReplayStructureStateHashInputOrder: + return "state_hash.input_order" + case ReplayStructureProcessedInputCount: + return "application.processed_input_count" + case ReplayStructureUnexpectedStateHash: + return "state_hash.unexpected_for_consensus" + default: + return unknownReplayValue + } +} + +// ReplayStructureViolationError contains only persisted counts and coordinates. +type ReplayStructureViolationError struct { + Kind ReplayStructureViolationKind + EpochIndex *uint64 + InputIndex uint64 + EvidenceIndex uint64 + ExpectedIndex uint64 + PreviousInputIndex uint64 + ApplicationProcessedInputs uint64 + CompletedInputCount uint64 +} + +func (e *ReplayStructureViolationError) Error() string { + epoch := "" + if e.EpochIndex != nil { + epoch = fmt.Sprint(*e.EpochIndex) + } + switch e.Kind { + case ReplayStructureCompletedInputSequence: + return fmt.Sprintf( + "%v: kind=%s expected_last_input=%d actual_last_input=%d", + ErrReplayInvalidStructure, e.Kind, e.ExpectedIndex, e.InputIndex, + ) + case ReplayStructureStateHashIndexSequence: + return fmt.Sprintf( + "%v: kind=%s epoch=%s input=%d expected_index=%d actual_index=%d", + ErrReplayInvalidStructure, e.Kind, epoch, e.InputIndex, + e.ExpectedIndex, e.EvidenceIndex, + ) + case ReplayStructureStateHashInputOrder: + return fmt.Sprintf( + "%v: kind=%s epoch=%s state_hash_index=%d previous_input=%d input=%d", + ErrReplayInvalidStructure, e.Kind, epoch, e.EvidenceIndex, + e.PreviousInputIndex, e.InputIndex, + ) + case ReplayStructureProcessedInputCount: + return fmt.Sprintf( + "%v: kind=%s application_processed_inputs=%d completed_input_count=%d", + ErrReplayInvalidStructure, e.Kind, + e.ApplicationProcessedInputs, e.CompletedInputCount, + ) + case ReplayStructureUnexpectedStateHash: + return fmt.Sprintf( + "%v: kind=%s epoch=%s input=%d state_hash_index=%d", + ErrReplayInvalidStructure, e.Kind, epoch, e.InputIndex, e.EvidenceIndex, + ) + default: + return fmt.Sprintf("%v: kind=%s", ErrReplayInvalidStructure, unknownReplayValue) + } +} + +func (e *ReplayStructureViolationError) Unwrap() error { return ErrReplayInvalidStructure } + +// ReplayEvidenceKind identifies persisted child evidence without exposing +// payloads. +type ReplayEvidenceKind uint8 + +const ( + // Zero is reserved so an uninitialized kind cannot identify real evidence + // and instead formats as unknown. + ReplayEvidenceOutput ReplayEvidenceKind = iota + 1 + ReplayEvidenceReport + ReplayEvidenceStateHash +) + +func (kind ReplayEvidenceKind) String() string { + switch kind { + case ReplayEvidenceOutput: + return "output" + case ReplayEvidenceReport: + return "report" + case ReplayEvidenceStateHash: + return "state_hash" + default: + return unknownReplayValue + } +} + +// ReplayInconsistentEvidenceError reports evidence whose input is not among +// the completed inputs selected for the page. +type ReplayInconsistentEvidenceError struct { + Kind ReplayEvidenceKind + InputIndex uint64 +} + +func (e *ReplayInconsistentEvidenceError) Error() string { + return fmt.Sprintf( + "%v: %s row for input %d has no completed replay input in the selected page", + ErrReplayInconsistentEvidence, + e.Kind, + e.InputIndex, + ) +} + +func (e *ReplayInconsistentEvidenceError) Unwrap() error { + return ErrReplayInconsistentEvidence +} diff --git a/internal/repository/replay_test.go b/internal/repository/replay_test.go new file mode 100644 index 000000000..678905023 --- /dev/null +++ b/internal/repository/replay_test.go @@ -0,0 +1,39 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package repository + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestReplayVerificationLevel(t *testing.T) { + require.True(t, ReplayVerificationCanonical.IsValid()) + require.True(t, ReplayVerificationFull.IsValid()) + require.False(t, ReplayVerificationLevel(255).IsValid()) + require.Equal(t, "canonical", ReplayVerificationCanonical.String()) + require.Equal(t, "full", ReplayVerificationFull.String()) + require.Equal(t, "unknown", ReplayVerificationLevel(255).String()) +} + +func TestReplaySourceErrorsPreserveIdentityWithoutPayloads(t *testing.T) { + inconsistency := &ReplayInconsistentEvidenceError{ + Kind: ReplayEvidenceReport, InputIndex: 7, + } + require.True(t, errors.Is(inconsistency, ErrReplayInconsistentEvidence)) + require.NotContains(t, inconsistency.Error(), "payload") + + violation := &ReplayStructureViolationError{ + Kind: ReplayStructureProcessedInputCount, + InputIndex: 7, + ApplicationProcessedInputs: 8, + CompletedInputCount: 9, + } + require.True(t, errors.Is(violation, ErrReplayInvalidStructure)) + require.Contains(t, violation.Error(), "application_processed_inputs=8") + require.Contains(t, violation.Error(), "completed_input_count=9") + require.NotContains(t, violation.Error(), "payload") +} diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 44f0d6115..f17d621fe 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -406,6 +406,7 @@ type ClaimerRepository interface { } type Repository interface { + ReplayRepository ApplicationRepository EpochRepository InputRepository diff --git a/internal/repository/repotest/application_test_cases.go b/internal/repository/repotest/application_test_cases.go index c6ffb668e..694f24805 100644 --- a/internal/repository/repotest/application_test_cases.go +++ b/internal/repository/repotest/application_test_cases.go @@ -31,9 +31,9 @@ func (s *ApplicationSuite) TestCreateApplication() { ep := ExecutionParameters{ SnapshotPolicy: SnapshotPolicy_EveryEpoch, AdvanceIncCycles: 1000, - AdvanceMaxCycles: 5000, - InspectIncCycles: 1000, - InspectMaxCycles: 5000, + AdvanceMaxCycles: 1500, //nolint:mnd + InspectIncCycles: 2000, + InspectMaxCycles: 2500, //nolint:mnd AdvanceIncDeadline: 10 * time.Second, AdvanceMaxDeadline: 60 * time.Second, InspectIncDeadline: 10 * time.Second, @@ -53,6 +53,8 @@ func (s *ApplicationSuite) TestCreateApplication() { s.Equal(ep.SnapshotPolicy, got.SnapshotPolicy) s.Equal(ep.AdvanceIncCycles, got.AdvanceIncCycles) s.Equal(ep.AdvanceMaxCycles, got.AdvanceMaxCycles) + s.Equal(ep.InspectIncCycles, got.InspectIncCycles) + s.Equal(ep.InspectMaxCycles, got.InspectMaxCycles) s.Equal(ep.MaxConcurrentInspects, got.MaxConcurrentInspects) }) } @@ -754,6 +756,8 @@ func (s *ApplicationSuite) TestGetExecutionParameters() { ep, err := s.Repo.GetExecutionParameters(s.Ctx, app.ID) s.Require().NoError(err) s.NotNil(ep) + s.Zero(ep.AdvanceMaxCycles) + s.Zero(ep.InspectMaxCycles) }) } @@ -762,9 +766,9 @@ func (s *ApplicationSuite) TestUpdateExecutionParameters() { ep := ExecutionParameters{ SnapshotPolicy: SnapshotPolicy_EveryInput, AdvanceIncCycles: 2000, - AdvanceMaxCycles: 10000, - InspectIncCycles: 2000, - InspectMaxCycles: 10000, + AdvanceMaxCycles: 2500, //nolint:mnd + InspectIncCycles: 3000, + InspectMaxCycles: 3500, //nolint:mnd AdvanceIncDeadline: 20 * time.Second, AdvanceMaxDeadline: 120 * time.Second, InspectIncDeadline: 20 * time.Second, @@ -779,13 +783,17 @@ func (s *ApplicationSuite) TestUpdateExecutionParameters() { Create(s.Ctx, s.T(), s.Repo) ep.ApplicationID = app.ID - ep.AdvanceMaxCycles = 99999 + ep.AdvanceIncCycles = 99999 + ep.AdvanceMaxCycles = MaxExecutionCycleSpan err := s.Repo.UpdateExecutionParameters(s.Ctx, &ep) s.Require().NoError(err) got, err := s.Repo.GetExecutionParameters(s.Ctx, app.ID) s.Require().NoError(err) - s.Equal(uint64(99999), got.AdvanceMaxCycles) + s.Equal(uint64(99999), got.AdvanceIncCycles) + s.Equal(MaxExecutionCycleSpan, got.AdvanceMaxCycles) + s.Equal(ep.InspectIncCycles, got.InspectIncCycles) + s.Equal(ep.InspectMaxCycles, got.InspectMaxCycles) }) } diff --git a/internal/repository/repotest/bulk_test_cases.go b/internal/repository/repotest/bulk_test_cases.go index 05e57afdf..001e996d1 100644 --- a/internal/repository/repotest/bulk_test_cases.go +++ b/internal/repository/repotest/bulk_test_cases.go @@ -24,6 +24,11 @@ func NewBulkOperationsSuite(factory RepositoryFactory) *BulkOperationsSuite { } func (s *BulkOperationsSuite) TestStoreAdvanceResult() { + s.Run("RejectsNilResult", func() { + err := s.Repo.StoreAdvanceResult(s.Ctx, 0, nil) + s.Require().EqualError(err, "advance result must not be nil") + }) + s.Run("AcceptedInput", func() { seed := Seed(s.Ctx, s.T(), s.Repo) machineHash := crypto.Keccak256Hash([]byte("machine")) @@ -89,6 +94,89 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { s.Equal(InputCompletionStatus_Rejected, input.Status) }) + for _, status := range []InputCompletionStatus{ + InputCompletionStatus_Exception, + InputCompletionStatus_MachineHalted, + } { + s.Run("CompletedStatus/"+status.String(), func() { + seed := Seed(s.Ctx, s.T(), s.Repo) + var exceptionData []byte + if status == InputCompletionStatus_Exception { + exceptionData = []byte{0xff, 0x00, 0x80} + } + result := &AdvanceResult{ + EpochIndex: 0, + InputIndex: 0, + Status: status, + ExceptionData: exceptionData, + OutputsProof: OutputsProof{ + MachineHash: UniqueHash(), + }, + } + + err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result) + s.Require().NoError(err) + input, err := s.Repo.GetInput(s.Ctx, seed.App.IApplicationAddress.String(), 0) + s.Require().NoError(err) + s.Equal(status, input.Status) + s.Equal(exceptionData, input.ExceptionData) + }) + } + + for _, test := range []struct { + name string + status InputCompletionStatus + exceptionData []byte + }{ + {"ExceptionWithoutData", InputCompletionStatus_Exception, nil}, + {"AcceptedWithExceptionData", InputCompletionStatus_Accepted, []byte("unexpected")}, + } { + s.Run("Rejects"+test.name, func() { + seed := Seed(s.Ctx, s.T(), s.Repo) + err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, &AdvanceResult{ + EpochIndex: 0, + InputIndex: 0, + Status: test.status, + ExceptionData: test.exceptionData, + OutputsProof: OutputsProof{ + MachineHash: UniqueHash(), + OutputsHash: UniqueHash(), + }, + }) + s.Require().ErrorContains(err, "exception data") + }) + } + + invalidStatuses := []InputCompletionStatus{ + InputCompletionStatus_None, + "OUTPUTS_LIMIT_EXCEEDED", + "REPORTS_LIMIT_EXCEEDED", + "CYCLE_LIMIT_EXCEEDED", + "TIME_LIMIT_EXCEEDED", + "PAYLOAD_LENGTH_LIMIT_EXCEEDED", + "INVALID", + } + for _, status := range invalidStatuses { + s.Run("RejectsNoncompletedStatus/"+status.String(), func() { + seed := Seed(s.Ctx, s.T(), s.Repo) + result := &AdvanceResult{ + EpochIndex: 0, + InputIndex: 0, + Status: status, + Outputs: [][]byte{[]byte("must-not-be-stored")}, + OutputsProof: OutputsProof{ + MachineHash: UniqueHash(), + }, + } + + err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result) + s.Require().ErrorContains(err, "noncompleted status") + input, err := s.Repo.GetInput(s.Ctx, seed.App.IApplicationAddress.String(), 0) + s.Require().NoError(err) + s.Equal(InputCompletionStatus_None, input.Status) + }) + } + s.Run("WithNoOutputsOrReports", func() { seed := Seed(s.Ctx, s.T(), s.Repo) machineHash := crypto.Keccak256Hash([]byte("machine-empty")) @@ -168,14 +256,15 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { hash1 := [32]byte(crypto.Keccak256Hash([]byte("state-1"))) hash2 := [32]byte(crypto.Keccak256Hash([]byte("state-2"))) hash3 := [32]byte(crypto.Keccak256Hash([]byte("state-3"))) + hashes := [][32]byte{hash1, hash2, hash3} result := &AdvanceResult{ EpochIndex: 0, InputIndex: 0, Status: InputCompletionStatus_Accepted, Outputs: [][]byte{[]byte("dave-output")}, - Hashes: [][32]byte{hash1, hash2, hash3}, - RemainingMetaCycles: 42, + PeriodicStateHashes: hashes, + PaddingRepetitions: InputHashCollectionCapacity - uint64(len(hashes)), IsDaveConsensus: true, OutputsProof: OutputsProof{ OutputsHash: outputsHash, @@ -186,15 +275,16 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result) s.Require().NoError(err) - // Verify state hashes were created (3 intermediate + 1 final = 4) + // Verify one row per intermediate hash plus the final repetition tail. epochIdx := uint64(0) stateHashes, total, err := s.Repo.ListStateHashes( s.Ctx, seed.App.IApplicationAddress.String(), repository.StateHashFilter{EpochIndex: &epochIdx}, repository.Pagination{Limit: 10}, false) s.Require().NoError(err) - s.Len(stateHashes, 4) - s.Equal(uint64(4), total) + expectedStateHashRows := len(hashes) + 1 + s.Len(stateHashes, expectedStateHashRows) + s.Equal(uint64(expectedStateHashRows), total) // Verify intermediate hashes have Repetitions=1 s.Equal(common.Hash(hash1), stateHashes[0].MachineHash) @@ -204,9 +294,10 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { s.Equal(common.Hash(hash3), stateHashes[2].MachineHash) s.Equal(uint64(1), stateHashes[2].Repetitions) - // Verify final hash has RemainingMetaCycles as Repetitions - s.Equal(machineHash, stateHashes[3].MachineHash) - s.Equal(uint64(42), stateHashes[3].Repetitions) + // Verify final hash has PaddingRepetitions as Repetitions + tail := stateHashes[len(hashes)] + s.Equal(machineHash, tail.MachineHash) + s.Equal(result.PaddingRepetitions, tail.Repetitions) // Verify outputs were also created outputs, _, err := s.Repo.ListOutputs( @@ -220,6 +311,105 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { s.Require().NoError(err) s.Equal(InputCompletionStatus_Accepted, input.Status) }) + + const resultPageLimit = 10 + const hashesAboveExtendedProtocolParameterLimit = 11_000 + + s.Run("DaveConsensusStreamsStateHashesAboveParameterLimit", func() { + seed := Seed(s.Ctx, s.T(), s.Repo) + hashes := make([][32]byte, hashesAboveExtendedProtocolParameterLimit) + machineHash := UniqueHash() + result := &AdvanceResult{ + EpochIndex: 0, + InputIndex: 0, + Status: InputCompletionStatus_Accepted, + PeriodicStateHashes: hashes, + PaddingRepetitions: InputHashCollectionCapacity - uint64(len(hashes)), + IsDaveConsensus: true, + OutputsProof: OutputsProof{ + OutputsHash: UniqueHash(), + MachineHash: machineHash, + }, + } + + s.Require().NoError(s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result)) + epochIndex := uint64(0) + expectedRows := len(hashes) + 1 + stateHashes, total, err := s.Repo.ListStateHashes( + s.Ctx, + seed.App.IApplicationAddress.String(), + repository.StateHashFilter{EpochIndex: &epochIndex}, + repository.Pagination{Limit: uint64(expectedRows)}, + false, + ) + s.Require().NoError(err) + s.Equal(uint64(expectedRows), total) + s.Require().Len(stateHashes, expectedRows) + for _, index := range []int{0, len(hashes) / 2, len(hashes) - 1} { + s.Equal(common.Hash(hashes[index]), stateHashes[index].MachineHash) + s.Equal(uint64(index), stateHashes[index].Index) + s.Equal(uint64(1), stateHashes[index].Repetitions) + } + tail := stateHashes[len(hashes)] + s.Equal(machineHash, tail.MachineHash) + s.Equal(uint64(len(hashes)), tail.Index) + s.Equal(InputHashCollectionCapacity-uint64(len(hashes)), tail.Repetitions) + }) + + s.Run("PRTConsensusRejectsUnnormalizedExactBoundaryHashCollection", func() { + seed := Seed(s.Ctx, s.T(), s.Repo) + result := &AdvanceResult{ + EpochIndex: 0, + InputIndex: 0, + Status: InputCompletionStatus_Accepted, + Outputs: [][]byte{[]byte("must-roll-back")}, + Reports: [][]byte{[]byte("must-roll-back")}, + PaddingRepetitions: 0, + IsDaveConsensus: true, + OutputsProof: OutputsProof{ + OutputsHash: UniqueHash(), + MachineHash: UniqueHash(), + }, + } + + err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result) + s.Require().Error(err) + + input, err := s.Repo.GetInput(s.Ctx, seed.App.IApplicationAddress.String(), 0) + s.Require().NoError(err) + s.Equal(InputCompletionStatus_None, input.Status) + outputs, outputCount, err := s.Repo.ListOutputs( + s.Ctx, + seed.App.IApplicationAddress.String(), + repository.OutputFilter{}, + repository.Pagination{Limit: resultPageLimit}, + false, + ) + s.Require().NoError(err) + s.Empty(outputs) + s.Zero(outputCount) + reports, reportCount, err := s.Repo.ListReports( + s.Ctx, + seed.App.IApplicationAddress.String(), + repository.ReportFilter{}, + repository.Pagination{Limit: resultPageLimit}, + false, + ) + s.Require().NoError(err) + s.Empty(reports) + s.Zero(reportCount) + epochIndex := uint64(0) + stateHashes, stateHashCount, err := s.Repo.ListStateHashes( + s.Ctx, + seed.App.IApplicationAddress.String(), + repository.StateHashFilter{EpochIndex: &epochIndex}, + repository.Pagination{Limit: resultPageLimit}, + false, + ) + s.Require().NoError(err) + s.Empty(stateHashes) + s.Zero(stateHashCount) + }) } func (s *BulkOperationsSuite) TestStoreAdvanceResultRollback() { @@ -316,14 +506,15 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResultRollback() { // a bad epoch index), all prior work (outputs, reports) is rolled back. s.Run("DaveConsensusRollbackOnStateHashFailure", func() { seed := Seed(s.Ctx, s.T(), s.Repo) + hashes := [][32]byte{{1}, {2}} result := &AdvanceResult{ EpochIndex: 99, // non-existent epoch InputIndex: 0, Status: InputCompletionStatus_Accepted, Outputs: [][]byte{[]byte("should-be-rolled-back")}, - Hashes: [][32]byte{{1}, {2}}, - RemainingMetaCycles: 10, + PeriodicStateHashes: hashes, + PaddingRepetitions: InputHashCollectionCapacity - uint64(len(hashes)), IsDaveConsensus: true, OutputsProof: OutputsProof{ OutputsHash: UniqueHash(), diff --git a/internal/repository/repotest/epoch_test_cases.go b/internal/repository/repotest/epoch_test_cases.go index 3128fc4f4..bc91275de 100644 --- a/internal/repository/repotest/epoch_test_cases.go +++ b/internal/repository/repotest/epoch_test_cases.go @@ -1252,9 +1252,12 @@ func (s *EpochSuite) TestDrainGates() { s.Ctx, app.IApplicationAddress.String(), map[*Epoch][]*Input{ep: { NewInputBuilder().WithIndex(idx).WithEpochIndex(idx). - WithBlockNumber(first + 1).WithStatus(inputStatus).Build(), + WithBlockNumber(first + 1).Build(), }}, last+1) s.Require().NoError(err) + if inputStatus.IsCompleted() { + StoreAdvanceResult(s.Ctx, s.T(), s.Repo, app.ID, idx, idx, inputStatus, nil, nil) + } if target != EpochStatus_Closed { AdvanceEpochStatus(s.Ctx, s.T(), @@ -1494,11 +1497,11 @@ func (s *EpochSuite) TestDrainGates() { s.Ctx, app.IApplicationAddress.String(), map[*Epoch][]*Input{ep: { NewInputBuilder().WithIndex(0).WithEpochIndex(0). - WithBlockNumber(forecloseBlock). - WithStatus(InputCompletionStatus_Accepted). - Build(), + WithBlockNumber(forecloseBlock).Build(), }}, forecloseBlock+10) s.Require().NoError(err) + StoreAdvanceResult(s.Ctx, s.T(), s.Repo, app.ID, 0, 0, + InputCompletionStatus_Accepted, nil, nil) AdvanceEpochStatus(s.Ctx, s.T(), s.Repo, app.IApplicationAddress.String(), ep, EpochStatus_ClaimComputed) s.Require().NoError(s.Repo.UpdateApplicationForeclosure( diff --git a/internal/repository/repotest/input_test_cases.go b/internal/repository/repotest/input_test_cases.go index 5e725d68e..ddd55fd0a 100644 --- a/internal/repository/repotest/input_test_cases.go +++ b/internal/repository/repotest/input_test_cases.go @@ -68,22 +68,28 @@ func (s *InputSuite) TestGetLastInput() { func (s *InputSuite) TestGetLastProcessedInput() { s.Run("ReturnsLastProcessed", func() { + const ( + processedInputBlock uint64 = 5 + pendingInputBlock uint64 = 10 + ) + app := NewApplicationBuilder().Create(s.Ctx, s.T(), s.Repo) epoch := NewEpochBuilder(app.ID). WithIndex(0).WithStatus(EpochStatus_Closed). WithBlocks(0, 19).WithInputBounds(0, 1).Build() input0 := NewInputBuilder(). - WithIndex(0).WithBlockNumber(5). - WithStatus(InputCompletionStatus_Accepted).Build() + WithIndex(0).WithBlockNumber(processedInputBlock).Build() input1 := NewInputBuilder(). - WithIndex(1).WithBlockNumber(10). + WithIndex(1).WithBlockNumber(pendingInputBlock). WithStatus(InputCompletionStatus_None).Build() err := s.Repo.CreateEpochsAndInputs( s.Ctx, app.IApplicationAddress.String(), map[*Epoch][]*Input{epoch: {input0, input1}}, 20) s.Require().NoError(err) + StoreAdvanceResult(s.Ctx, s.T(), s.Repo, app.ID, 0, 0, + InputCompletionStatus_Accepted, nil, nil) got, err := s.Repo.GetLastProcessedInput(s.Ctx, app.IApplicationAddress.String()) s.Require().NoError(err) @@ -218,6 +224,11 @@ func (s *InputSuite) TestCreateEpochsAndInputsWithSameTransactionHash() { } func (s *InputSuite) TestListInputs() { + const ( + firstFilteredInputBlock uint64 = 5 + secondFilteredInputBlock uint64 = 10 + ) + s.Run("EmptyResult", func() { app := NewApplicationBuilder().Create(s.Ctx, s.T(), s.Repo) inputs, total, err := s.Repo.ListInputs( @@ -287,16 +298,17 @@ func (s *InputSuite) TestListInputs() { WithBlocks(0, 19).WithInputBounds(0, 1).Build() input0 := NewInputBuilder(). - WithIndex(0).WithBlockNumber(5). - WithStatus(InputCompletionStatus_Accepted).Build() + WithIndex(0).WithBlockNumber(firstFilteredInputBlock).Build() input1 := NewInputBuilder(). - WithIndex(1).WithBlockNumber(10). + WithIndex(1).WithBlockNumber(secondFilteredInputBlock). WithStatus(InputCompletionStatus_None).Build() err := s.Repo.CreateEpochsAndInputs( s.Ctx, app.IApplicationAddress.String(), map[*Epoch][]*Input{epoch: {input0, input1}}, 20) s.Require().NoError(err) + StoreAdvanceResult(s.Ctx, s.T(), s.Repo, app.ID, 0, 0, + InputCompletionStatus_Accepted, nil, nil) status := InputCompletionStatus_Accepted inputs, total, err := s.Repo.ListInputs( @@ -316,11 +328,9 @@ func (s *InputSuite) TestListInputs() { WithBlocks(0, 19).WithInputBounds(0, 2).Build() input0 := NewInputBuilder(). - WithIndex(0).WithBlockNumber(5). - WithStatus(InputCompletionStatus_Accepted).Build() + WithIndex(0).WithBlockNumber(firstFilteredInputBlock).Build() input1 := NewInputBuilder(). - WithIndex(1).WithBlockNumber(10). - WithStatus(InputCompletionStatus_Rejected).Build() + WithIndex(1).WithBlockNumber(secondFilteredInputBlock).Build() input2 := NewInputBuilder(). WithIndex(2).WithBlockNumber(15). WithStatus(InputCompletionStatus_None).Build() @@ -329,6 +339,10 @@ func (s *InputSuite) TestListInputs() { s.Ctx, app.IApplicationAddress.String(), map[*Epoch][]*Input{epoch: {input0, input1, input2}}, 20) s.Require().NoError(err) + StoreAdvanceResult(s.Ctx, s.T(), s.Repo, app.ID, 0, 0, + InputCompletionStatus_Accepted, nil, nil) + StoreAdvanceResult(s.Ctx, s.T(), s.Repo, app.ID, 0, 1, + InputCompletionStatus_Rejected, nil, nil) notStatus := InputCompletionStatus_None inputs, total, err := s.Repo.ListInputs( diff --git a/internal/repository/repotest/repotest.go b/internal/repository/repotest/repotest.go index 282317eef..4cbb529e5 100644 --- a/internal/repository/repotest/repotest.go +++ b/internal/repository/repotest/repotest.go @@ -54,6 +54,9 @@ func StoreAdvanceResult( MachineHash: UniqueHash(), }, } + if status == InputCompletionStatus_Exception { + result.ExceptionData = []byte{} + } err := repo.StoreAdvanceResult(ctx, appID, result) require.NoError(t, err) } diff --git a/internal/repository/repotest/state_hash_test_cases.go b/internal/repository/repotest/state_hash_test_cases.go index 1f9df2b1c..59992c708 100644 --- a/internal/repository/repotest/state_hash_test_cases.go +++ b/internal/repository/repotest/state_hash_test_cases.go @@ -49,13 +49,14 @@ func (s *StateHashSuite) TestListStateHashes() { hash1 := [32]byte(crypto.Keccak256Hash([]byte("list-state-1"))) hash2 := [32]byte(crypto.Keccak256Hash([]byte("list-state-2"))) + collectedHashes := [][32]byte{hash1, hash2} result := &AdvanceResult{ EpochIndex: 0, InputIndex: 0, Status: InputCompletionStatus_Accepted, - Hashes: [][32]byte{hash1, hash2}, - RemainingMetaCycles: 10, + PeriodicStateHashes: collectedHashes, + PaddingRepetitions: InputHashCollectionCapacity - uint64(len(collectedHashes)), IsDaveConsensus: true, OutputsProof: OutputsProof{ OutputsHash: outputsHash, @@ -67,12 +68,13 @@ func (s *StateHashSuite) TestListStateHashes() { s.Require().NoError(err) // List all state hashes + expectedRows := len(collectedHashes) + 1 hashes, total, err := s.Repo.ListStateHashes( s.Ctx, seed.App.IApplicationAddress.String(), repository.StateHashFilter{}, repository.Pagination{Limit: 10}, false) s.Require().NoError(err) - s.Len(hashes, 3) // 2 intermediate + 1 final - s.Equal(uint64(3), total) + s.Len(hashes, expectedRows) + s.Equal(uint64(expectedRows), total) // List with epoch filter epochIdx := uint64(0) @@ -81,16 +83,17 @@ func (s *StateHashSuite) TestListStateHashes() { repository.StateHashFilter{EpochIndex: &epochIdx}, repository.Pagination{Limit: 10}, false) s.Require().NoError(err) - s.Len(hashes, 3) - s.Equal(uint64(3), total) + s.Len(hashes, expectedRows) + s.Equal(uint64(expectedRows), total) // Verify pagination + pageSize := uint64(len(collectedHashes)) hashes, total, err = s.Repo.ListStateHashes( s.Ctx, seed.App.IApplicationAddress.String(), repository.StateHashFilter{}, - repository.Pagination{Limit: 2, Offset: 0}, false) + repository.Pagination{Limit: pageSize, Offset: 0}, false) s.Require().NoError(err) - s.Len(hashes, 2) - s.Equal(uint64(3), total) + s.Len(hashes, len(collectedHashes)) + s.Equal(uint64(expectedRows), total) }) } diff --git a/internal/validator/validator.go b/internal/validator/validator.go index 0578d9fff..dade2609d 100644 --- a/internal/validator/validator.go +++ b/internal/validator/validator.go @@ -18,7 +18,6 @@ import ( "github.com/cartesi/rollups-node/internal/merkle" . "github.com/cartesi/rollups-node/internal/model" "github.com/cartesi/rollups-node/internal/repository" - pkgm "github.com/cartesi/rollups-node/pkg/machine" "github.com/cartesi/rollups-node/pkg/service" ) @@ -351,10 +350,10 @@ func (s *Service) buildCommitment(ctx context.Context, app *Application, epoch * } builder := merkle.Builder{} inputCount := epoch.InputIndexUpperBound - epoch.InputIndexLowerBound - if inputCount > pkgm.InputsPerEpoch { + if inputCount > MaxAdvanceStatesPerEpoch { return nil, nil, s.setApplicationCorrupted(ctx, app, "input count is too large for epoch %v of application %v: max %v, got %v", - epoch.Index, app.Name, pkgm.InputsPerEpoch, inputCount) + epoch.Index, app.Name, MaxAdvanceStatesPerEpoch, inputCount) } if inputCount > 0 { @@ -382,11 +381,11 @@ func (s *Service) buildCommitment(ctx context.Context, app *Application, epoch * } } - remainingInputs := pkgm.InputsPerEpoch - inputCount - // Safe: inputCount ≤ InputsPerEpoch enforced above, so remainingInputs << Log2StridesPerInput won't overflow. - remainingStrides := remainingInputs << pkgm.Log2StridesPerInput - if remainingStrides > 0 { - if err := builder.AppendRepeatedUint64(merkle.TreeLeaf(*epoch.MachineHash), remainingStrides); err != nil { + remainingInputs := MaxAdvanceStatesPerEpoch - inputCount + // Safe: inputCount is bounded above, so the remaining-input shift cannot overflow. + remainingEntries := remainingInputs << Log2InputHashCollectionCapacity + if remainingEntries > 0 { + if err := builder.AppendRepeatedUint64(merkle.TreeLeaf(*epoch.MachineHash), remainingEntries); err != nil { return nil, nil, s.setApplicationCorrupted(ctx, app, "failed to append state hash to builder for epoch %d of application %s with error: %v", epoch.Index, app.Name, err) } @@ -397,8 +396,8 @@ func (s *Service) buildCommitment(ctx context.Context, app *Application, epoch * return nil, nil, s.setApplicationCorrupted(ctx, app, "failed to build commitment for epoch %d of application %s with error: %v", epoch.Index, app.Name, err) } - // The commitment geometry is fixed: 2²⁴ inputs × 2²⁴ strides ⇒ height 48. - const expectedHeight = pkgm.Log2InputSpanToEpoch + pkgm.Log2StridesPerInput // 48 + // The commitment geometry is fixed: 2²⁴ inputs × 2²⁴ entries ⇒ height 48. + const expectedHeight = Log2EpochComputationHashLeafCount if uint64(epochCommitmentTree.Height) != expectedHeight { return nil, nil, s.setApplicationCorrupted(ctx, app, "epoch %v commitment tree height %v, expected %v — state hash repetitions are inconsistent", diff --git a/internal/validator/validator_test.go b/internal/validator/validator_test.go index d866b584a..5f9888259 100644 --- a/internal/validator/validator_test.go +++ b/internal/validator/validator_test.go @@ -12,7 +12,6 @@ import ( "github.com/cartesi/rollups-node/internal/merkle" . "github.com/cartesi/rollups-node/internal/model" "github.com/cartesi/rollups-node/internal/repository" - pkgm "github.com/cartesi/rollups-node/pkg/machine" "github.com/cartesi/rollups-node/pkg/service" "github.com/ethereum/go-ethereum/common" "github.com/stretchr/testify/mock" @@ -822,14 +821,14 @@ func (s *ValidatorSuite) TestBuildCommitment() { } // 5 inputs, each with one state hash covering the full - // strides-per-input count (1< InputEntryCapacity { + return 0, fmt.Errorf( + "collected state hash count %d exceeds input entry capacity %d", + hashCount, + InputEntryCapacity, + ) + } + return InputEntryCapacity - hashCount, nil +} diff --git a/pkg/machine/computation_hash_test.go b/pkg/machine/computation_hash_test.go new file mode 100644 index 000000000..d8f99efc9 --- /dev/null +++ b/pkg/machine/computation_hash_test.go @@ -0,0 +1,45 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +package machine + +import ( + "testing" + + "github.com/cartesi/rollups-node/internal/model" + "github.com/stretchr/testify/require" +) + +func TestComputationHashDimensions(t *testing.T) { + require.Equal(t, uint64(20), Log2MaxUarchCyclesPerMCycle) + require.Equal(t, uint64(48), Log2MaxMCyclesPerAdvanceState) + require.Equal(t, uint64(24), Log2MaxAdvanceStatesPerEpoch) + require.Equal(t, uint64(44), Log2UarchCycleComputationHashPeriod) + require.Equal(t, uint64(68), Log2UarchCyclesPerAdvanceState) + require.Equal(t, uint64(1)<<20-1, MaxUarchCycle) + require.Equal(t, uint64(1)<<48-1, MaxMCycleDeltaPerAdvanceState) + require.Equal(t, uint64(1)<<24-1, MaxAdvanceStateIndexPerEpoch) + + require.Equal(t, uint64(24), Log2MCycleComputationHashPeriod) + require.Equal(t, uint64(1)<<24, MCycleComputationHashPeriod) + require.Equal(t, uint64(24), Log2InputEntryCapacity) + require.Equal(t, uint64(1)<<24, InputEntryCapacity) + require.Equal(t, uint64(48), Log2EpochComputationHashLeafCount) + require.Equal(t, uint64(1)<<48, EpochComputationHashLeafCount) + require.Equal(t, uint64(1)<<24, MaxAdvanceStatesPerEpoch) + require.Equal( + t, + model.Log2InputHashCollectionCapacity, + Log2MaxMCyclesPerAdvanceState-Log2MCycleComputationHashPeriod, + "the execution window and sampling period must derive the protocol input hash collection capacity", + ) +} + +func TestComputationHashCollectionChunkMatchesEmulatorCLI(t *testing.T) { + require.Equal( + t, + MCycleComputationHashPeriod<<8, + mcycleComputationHashChunkSize, + "cartesi-machine 0.21 targets 2^8 returned hashes per collection call", + ) +} diff --git a/pkg/machine/doc.go b/pkg/machine/doc.go new file mode 100644 index 000000000..cf70cfba7 --- /dev/null +++ b/pkg/machine/doc.go @@ -0,0 +1,38 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +// Package machine exposes the rollups-oriented operations of a Cartesi +// machine and translates emulator stops into a small Go contract. +// +// A request that reaches a deterministic guest completion returns a response +// value with CompletionStatusAccepted, CompletionStatusRejected, +// CompletionStatusException, or CompletionStatusHalted. Anything that prevents +// completion—including deadlines, local resource limits, backend failures, and +// cycle exhaustion—returns an error. Advance then returns no response; Inspect +// may return partial reports with CompletionStatusUnknown. In short: terminal +// guest outcomes travel as values; incomplete execution travels as an error. +// Callers decide how a completed outcome affects canonical application state. +// +// For a PRT advance, AdvanceResponse contains a canonical compressed input hash +// collection. PeriodicStateHashes contains the sampled machine root hashes +// before the final canonical root. PaddingRepetitions says how many times that +// final root completes the collection, so +// +// len(PeriodicStateHashes) + PaddingRepetitions == InputEntryCapacity +// +// and PaddingRepetitions is positive. The machine implementation is the sole +// canonicalizer of this representation; downstream layers validate it but do +// not repair it. +// +// Related terms describe different levels of the same construction. A root +// hash identifies one machine state. Periodic state hashes are sampled roots. +// An input hash collection is those samples plus the repeated final root. The +// DAVE computation hash commits to the larger sequence assembled from these +// collections. Persistence stores the compressed collection as state-hash rows. +// +// A configured cycle span of zero means that no operator cap shortens the +// emulator's fixed window. The resolved endpoint mirrors emulator 0.21's +// saturating imcyclemax arithmetic. ErrMcycleOverflow preserves the distinct +// emulator-reported fact that imcyclemax, rather than a node target, stopped the +// machine. +package machine diff --git a/pkg/machine/implementation.go b/pkg/machine/implementation.go index 345580058..bceae6a61 100644 --- a/pkg/machine/implementation.go +++ b/pkg/machine/implementation.go @@ -23,12 +23,48 @@ const ( InspectStateRequest requestType = 0x1 ) -type yieldType uint8 +type runResult struct { + outputs []Output + reports []Report + periodicStateHashes []Hash + paddingRepetitions uint64 +} -const ( - AutomaticYield yieldType = 0x0 - ManualYield yieldType = 0x1 -) +type processResult struct { + runResult + completion completionResult +} + +type incrementResult struct { + breakReason BreakReason + currentCycle Cycle +} + +type completionResult struct { + status CompletionStatus + data []byte +} + +// executionBounds is the resolved cycle window for one request. A configured +// span of zero is converted to the fixed machine window exactly once here; +// downstream execution and diagnostics consume the same resolved values. +type executionBounds struct { + start Cycle + limit Cycle + span Cycle + configured bool +} + +func (r requestType) String() string { + switch r { + case AdvanceStateRequest: + return "advance" + case InspectStateRequest: + return "inspect" + default: + return fmt.Sprintf("request_type_%d", r) + } +} type automaticYieldReason uint16 @@ -46,39 +82,15 @@ const ( ManualYieldReasonException manualYieldReason = 0x4 ) -// Limits for outputs and reports per input +// maxOutputs and maxReports are local operational resource ceilings. They +// bound host memory consumption; exceeding them is an incomplete execution +// failure, never a protocol-level input completion status. const maxOutputs = 65536 // 2^16 const maxReports = 65536 // 2^16 const TxBufferAddress uint64 = 0x60800000 const HashLog2Size = 5 // 32 bytes -const ( - // log2 value of the maximal number of micro instructions that emulates a big instruction - Log2UarchSpanToBarch uint64 = 20 - // log2 value of the maximal number of big instructions that executes an input - Log2BarchSpanToInput uint64 = 48 - // log2 value of the maximal number of inputs that allowed in an epoch - Log2InputSpanToEpoch uint64 = 24 - // gap of each leaf in the commitment tree, should use the same value as ArbitrationConstants.sol:log2step(0) - Log2Stride uint64 = 44 - // log2 value of the maximal number of micro instructions that executes an input - Log2UarchSpanToInput uint64 = Log2BarchSpanToInput + Log2UarchSpanToBarch // 68 - - UarchSpanToBarch uint64 = (1 << Log2UarchSpanToBarch) - 1 // 1_048_575 - BarchSpanToInput uint64 = (1 << Log2BarchSpanToInput) - 1 // 281_474_976_710_655 - InputSpanToEpoch uint64 = (1 << Log2InputSpanToEpoch) - 1 // 16_777_215 - - BigStepsInStride uint64 = 1 << (Log2Stride - Log2UarchSpanToBarch) // 16_777_216 - StrideCountInInput uint64 = 1 << (Log2BarchSpanToInput + Log2UarchSpanToBarch - Log2Stride) // 16_777_216 - - StrideCountInEpoch uint64 = 1 << (Log2InputSpanToEpoch + Log2BarchSpanToInput + Log2UarchSpanToBarch - Log2Stride) - - Log2StridesPerInput uint64 = Log2BarchSpanToInput + Log2UarchSpanToBarch - Log2Stride - - InputsPerEpoch uint64 = 1 << Log2InputSpanToEpoch -) - // machineImpl implements the Machine interface by wrapping an emulator.RemoteMachine type machineImpl struct { backend Backend @@ -131,24 +143,36 @@ func (m *machineImpl) Hash(ctx context.Context) (Hash, error) { // OutputsHash returns the outputs hash stored in the cmio tx buffer func (m *machineImpl) OutputsHash(ctx context.Context) (Hash, error) { - accepted, data, err := m.wasLastRequestAccepted(ctx) + result, err := m.readManualYieldResult(ctx) if err != nil { err = fmt.Errorf("could not read the outputs hash: %w", err) return Hash{}, err } - if !accepted { - err = fmt.Errorf("could not read the outputs hash: machine manual yield reason is not accepted") - return Hash{}, err - } - - if length := len(data); length != HashSize { + switch result.status { + case CompletionStatusAccepted: + // Intentionally empty. + case CompletionStatusRejected: + return Hash{}, fmt.Errorf("could not read the outputs hash: %w", ErrRejected) + case CompletionStatusException: + return Hash{}, fmt.Errorf("could not read the outputs hash: %w", ErrException) + case CompletionStatusHalted: + return Hash{}, fmt.Errorf("could not read the outputs hash: %w", ErrHalted) + case CompletionStatusUnknown: + return Hash{}, fmt.Errorf( + "could not read the outputs hash with completion status %d: %w", + result.status, + ErrMachineInternal, + ) + } + + if length := len(result.data); length != HashSize { err = fmt.Errorf("invalid outputs hash: %w (it has %d bytes)", ErrHashLength, length) return Hash{}, err } var outputsHash Hash - copy(outputsHash[:], data) + copy(outputsHash[:], result.data) return outputsHash, nil } @@ -166,41 +190,58 @@ func (m *machineImpl) OutputsHashProof(ctx context.Context) ([]Hash, error) { // Advance sends an input to the machine and processes it func (m *machineImpl) Advance(ctx context.Context, input []byte, checkpointHash Hash, computeHashes bool) (*AdvanceResponse, error) { - // TODO: return the exception reason - accepted, outputs, reports, hashes, remaining, data, err := m.process(ctx, input, AdvanceStateRequest, &checkpointHash, computeHashes) + result, err := m.process(ctx, input, AdvanceStateRequest, &checkpointHash, computeHashes) if err != nil { - return &AdvanceResponse{ - Accepted: accepted, - Outputs: outputs, - Reports: reports, - Hashes: hashes, - RemainingCycles: remaining, - }, err + return nil, err + } + if !result.completion.status.IsCompleted() { + return nil, fmt.Errorf( + "invalid completed advance status %d: %w", + result.completion.status, + ErrMachineInternal, + ) } resp := &AdvanceResponse{ - Accepted: accepted, - Outputs: outputs, - Reports: reports, - Hashes: hashes, - RemainingCycles: remaining, + Status: result.completion.status, + Outputs: result.outputs, + Reports: result.reports, + PeriodicStateHashes: result.periodicStateHashes, + PaddingRepetitions: result.paddingRepetitions, } - if accepted { - if length := len(data); length != HashSize { - return resp, fmt.Errorf("%w (it has %d bytes)", ErrHashLength, length) + if resp.Status == CompletionStatusAccepted { + if length := len(result.completion.data); length != HashSize { + return nil, fmt.Errorf("%w (it has %d bytes)", ErrHashLength, length) } - copy(resp.OutputsHash[:], data) + copy(resp.OutputsHash[:], result.completion.data) + } else if resp.Status == CompletionStatusException { + resp.ExceptionData = append([]byte{}, result.completion.data...) } return resp, nil } -// Inspect sends a query to the machine and returns the results -func (m *machineImpl) Inspect(ctx context.Context, query []byte) (bool, []Report, error) { - // TODO: return the exception reason +// Inspect sends a query to the machine and returns the results. +func (m *machineImpl) Inspect(ctx context.Context, query []byte) (*InspectResponse, error) { // For inspect-state requests, revert_root_hash is not checked and can be NULL/empty - accepted, _, reports, _, _, _, err := m.process(ctx, query, InspectStateRequest, nil, false) - return accepted, reports, err + result, err := m.process(ctx, query, InspectStateRequest, nil, false) + response := &InspectResponse{Reports: result.reports} + if err != nil { + return response, err + } + + response.Status = result.completion.status + if !response.Status.IsCompleted() { + return response, fmt.Errorf( + "invalid completed inspect status %d: %w", + result.completion.status, + ErrMachineInternal, + ) + } + if response.Status == CompletionStatusException { + response.ExceptionData = append([]byte{}, result.completion.data...) + } + return response, nil } // Store saves the machine state to the specified path @@ -284,30 +325,28 @@ func (m *machineImpl) isAtManualYield(ctx context.Context) (bool, error) { return isAtManualYield, nil } -// wasLastRequestAccepted returns true if the last request was accepted and false otherwise. -// It returns the ErrException error if the last request yielded an exception. -// -// The machine MUST be at a manual yield when calling this function. -func (m *machineImpl) wasLastRequestAccepted(ctx context.Context) (bool, []byte, error) { +// readManualYieldResult returns the typed completion status and data reported by +// the current manual yield. The machine MUST be at a manual yield when called. +func (m *machineImpl) readManualYieldResult(ctx context.Context) (completionResult, error) { if err := checkContext(ctx); err != nil { - return false, nil, err + return completionResult{}, err } _, yieldReason, data, err := m.backend.ReceiveCmioRequest(m.params.FastDeadline) if err != nil { - return false, nil, err + return completionResult{}, err } switch manualYieldReason(yieldReason) { case ManualYieldReasonAccepted: - return true, data, nil + return completionResult{status: CompletionStatusAccepted, data: data}, nil case ManualYieldReasonRejected: - return false, data, nil + return completionResult{status: CompletionStatusRejected, data: data}, nil case ManualYieldReasonException: - return false, data, ErrException + return completionResult{status: CompletionStatusException, data: data}, nil default: err = fmt.Errorf("invalid manual yield reason: %d: %w", yieldReason, ErrMachineInternal) - return false, nil, err + return completionResult{}, err } } @@ -337,175 +376,444 @@ func (m *machineImpl) process( reqType requestType, checkpointHash *Hash, computeHashes bool, -) (bool, []Output, []Report, []Hash, uint64, []byte, error) { +) (processResult, error) { if err := checkContext(ctx); err != nil { - return false, nil, nil, nil, 0, nil, err - } - // Check payload length limit - if length := uint64(len(request)); length > m.backend.CmioRxBufferSize() { - return false, nil, nil, nil, 0, nil, ErrPayloadLengthLimitExceeded - } - - err := m.backend.SendCmioResponse(uint16(reqType), request, checkpointHash, m.params.FastDeadline) + return processResult{}, err + } + // An L1 advance arrives as the full EvmAdvance encoding, which the canonical + // InputBox bounds to CanonicalMachine.INPUT_MAX_SIZE = 2^16 bytes (it reverts + // InputTooLarge above that), while the standard CMIO RX buffer holds 2^21 + // bytes. Advance overflow is therefore defensive on the supported path; + // inspect and custom input providers still need the runtime check. + if length, capacity := uint64(len(request)), m.backend.CmioRxBufferSize(); length > capacity { + return processResult{}, fmt.Errorf( + "%s request payload length %d exceeds CMIO receive buffer capacity %d: %w", + reqType, length, capacity, ErrPayloadLengthLimitExceeded, + ) + } + + // Validate the request-specific increment before consulting or changing the + // machine. A zero increment would otherwise make the run loop spin forever. + if err := m.validateExecutionCycleIncrement(reqType); err != nil { + return processResult{}, err + } + // Validate the execution target before sending the request. Sending a CMIO + // response mutates the candidate, so invalid configuration must fail before + // that point. A valid target that exceeds uint64 saturates at MaxUint64, + // matching the emulator's imcyclemax calculation. + bounds, err := m.executionCycleBounds(ctx, reqType) if err != nil { - return false, nil, nil, nil, 0, nil, err + return processResult{}, err } - outputs, reports, hashes, remaining, err := m.run(ctx, reqType, computeHashes) + err = m.backend.SendCmioResponse(uint16(reqType), request, checkpointHash, m.params.FastDeadline) if err != nil { - return false, outputs, reports, nil, 0, nil, err + return processResult{}, err + } + + execution, err := m.run(ctx, reqType, computeHashes, bounds) + result := processResult{runResult: execution} + switch { + case err == nil: + manualResult, err := m.readManualYieldResult(ctx) + result.completion = manualResult + return result, err + case errors.Is(err, ErrMachineInternal): + return result, err + case errors.Is(err, ErrHalted): + result.completion.status = CompletionStatusHalted + return result, nil + default: + return result, err } - - accepted, data, err := m.wasLastRequestAccepted(ctx) - - return accepted, outputs, reports, hashes, remaining, data, err } -// run runs the machine until it manually yields. -// It returns any collected responses. -func (m *machineImpl) run(ctx context.Context, reqType requestType, computeHashes bool) ([]Output, []Report, []Hash, uint64, error) { +// run executes a request whose increment and cycle bounds were validated before +// SendCmioResponse mutated the machine. It returns collected responses and the +// canonical compressed input hash collection. +func (m *machineImpl) run( + ctx context.Context, + reqType requestType, + computeHashes bool, + bounds executionBounds, +) (runResult, error) { startTime := time.Now() + currentCycle := bounds.start - currentCycle, err := m.readMCycle(ctx) - if err != nil { - return nil, nil, nil, 0, err - } - - limitCycle := currentCycle + m.params.AdvanceMaxCycles stepTimeout := m.params.AdvanceIncDeadline runTimeout := m.params.AdvanceMaxDeadline + increment := m.executionCycleIncrement(reqType) if reqType == InspectStateRequest { - limitCycle = currentCycle + m.params.InspectMaxCycles stepTimeout = m.params.InspectIncDeadline runTimeout = m.params.InspectMaxDeadline } + if computeHashes { + increment = min(increment, mcycleComputationHashChunkSize) + } m.logger.Debug("run", "startingCycle", currentCycle, - "limitCycle", limitCycle, - "leftover", limitCycle-currentCycle) + "limitCycle", bounds.limit, + "leftover", bounds.limit-currentCycle) - outputs := make([]Output, 0, 16) //nolint:mnd - reports := make([]Report, 0, 16) //nolint:mnd + result := runResult{ + outputs: make([]Output, 0, 16), + reports: make([]Report, 0, 16), + } var hashCollectorState *HashCollectorState if computeHashes { hashCollectorState = &HashCollectorState{ - Period: BigStepsInStride, - Phase: 0, - MaxHashes: 0, - BundleLog2: 0, - Hashes: []Hash{}, + MCycleSamplingPeriod: MCycleComputationHashPeriod, + MCyclePhase: 0, + Log2BundleMCycleCount: 0, + Hashes: []Hash{}, } } - hashes := func() []Hash { - if computeHashes { - return hashCollectorState.Hashes - } - return []Hash{} + finish := func(runErr error, terminalHashAppended bool) (runResult, error) { + return finalizeRunResult(result, hashCollectorState, terminalHashAppended, runErr) } - remainingMetaCycles := func() uint64 { - if computeHashes { - return StrideCountInInput - uint64(len(hashCollectorState.Hashes)) - } - return 0 + // Incomplete executions may expose partial outputs and reports, but their PRT + // collections are discarded rather than normalized into persistable evidence. + fail := func(runErr error) (runResult, error) { + return result, runErr } for { - var yt *yieldType - var err error + if err := checkContext(ctx); err != nil { + return fail(err) + } + if time.Since(startTime) > runTimeout { + return fail(fmt.Errorf("run operation timed out: %w", ErrDeadlineExceeded)) + } - // Steps the machine as many times as needed until it manually/automatically yields. - for yt == nil { - if err := checkContext(ctx); err != nil { - return outputs, reports, hashes(), remainingMetaCycles(), err - } - if time.Since(startTime) > runTimeout { - werr := fmt.Errorf("run operation timed out: %w", ErrDeadlineExceeded) - return outputs, reports, hashes(), remainingMetaCycles(), werr - } - yt, currentCycle, err = m.runIncrementInterval(ctx, currentCycle, limitCycle, hashCollectorState, stepTimeout) - if err != nil && err != ErrReachedTargetMcycle { - return outputs, reports, hashes(), remainingMetaCycles(), err + hashCountBeforeRun := 0 + if hashCollectorState != nil { + hashCountBeforeRun = len(hashCollectorState.Hashes) + } + incrementResult, err := m.runIncrementInterval( + ctx, + currentCycle, + bounds.limit, + increment, + hashCollectorState, + stepTimeout, + ) + currentCycle = incrementResult.currentCycle + if err != nil { + if errors.Is(err, ErrReachedLimitMcycle) { + return fail(executionLimitError(reqType, bounds, currentCycle, err)) } + return fail(err) } - // Returns with the responses when the machine manually yields. - if *yt == ManualYield { - return outputs, reports, hashes(), remainingMetaCycles(), nil + terminalHashAppended := computeHashes && isFixedPointBreakReason(incrementResult.breakReason) + if terminalHashAppended && len(hashCollectorState.Hashes) <= hashCountBeforeRun { + return fail(fmt.Errorf( + "machine stopped at fixed point %d without appending its state hash: %w", + incrementResult.breakReason, + ErrMachineInternal, + )) } - // Asserts the machine yielded automatically. - if *yt != AutomaticYield { - err := fmt.Errorf("invalid yield type: %d: %w", *yt, ErrMachineInternal) - return outputs, reports, hashes(), remainingMetaCycles(), err + switch incrementResult.breakReason { + case YieldedManually: + return finish(nil, terminalHashAppended) + case Halted: + return finish(ErrHalted, terminalHashAppended) + case McycleOverflow: + return fail(executionLimitError(reqType, bounds, currentCycle, ErrMcycleOverflow)) + case ReachedTargetMcycle, YieldedSoftly: + continue + case Failed: + return fail(ErrMachineInternal) + case YieldedAutomatically: + // Service the CMIO request below, then continue execution. + default: + return fail(fmt.Errorf( + "invalid break reason: %d: %w", + incrementResult.breakReason, + ErrMachineInternal, + )) } - yt = nil if err := checkContext(ctx); err != nil { - return outputs, reports, hashes(), remainingMetaCycles(), err + return fail(err) } _, yieldReason, data, err := m.backend.ReceiveCmioRequest(m.params.FastDeadline) if err != nil { werr := fmt.Errorf("could not read output/report: %w", err) - return outputs, reports, hashes(), remainingMetaCycles(), werr + return fail(werr) } switch automaticYieldReason(yieldReason) { case AutomaticYieldReasonProgress: m.logger.Debug("ignoring yield reason progress", "value", fmt.Sprintf("%v", data)) case AutomaticYieldReasonOutput: - // TODO: should we remove this? - if len(outputs) == maxOutputs { - return outputs, reports, hashes(), remainingMetaCycles(), ErrOutputsLimitExceeded + if len(result.outputs) == maxOutputs { + return fail(executionResponseLimitError( + reqType, "output", len(result.outputs)+1, maxOutputs, ErrOutputsLimitExceeded, + )) } - outputs = append(outputs, data) + result.outputs = append(result.outputs, data) case AutomaticYieldReasonReport: - if len(reports) == maxReports { - return outputs, reports, hashes(), remainingMetaCycles(), ErrReportsLimitExceeded + if len(result.reports) == maxReports { + return fail(executionResponseLimitError( + reqType, "report", len(result.reports)+1, maxReports, ErrReportsLimitExceeded, + )) } - reports = append(reports, data) + result.reports = append(result.reports, data) default: err := fmt.Errorf("invalid automatic yield reason: %d: %w", yieldReason, ErrMachineInternal) - return outputs, reports, hashes(), remainingMetaCycles(), err + return fail(err) } } } -// runIncrementInterval runs the machine for at most machine.inc cycles (or the amount of cycles left to reach -// limitCycle, whichever is the lowest). -// It returns the yield type and the machine cycle after the increment interval. -// If the machine did not manually/automatically yield, the yield type will be nil (meaning runIncrementInterval -// must be called again to complete the computation). +// finalizeRunResult converts the emulator collector's fixed-point-inclusive +// result into the node's compressed representation. Keeping this at the return +// boundary leaves the execution loop concerned only with execution policy. +func finalizeRunResult( + result runResult, + collector *HashCollectorState, + terminalHashAppended bool, + runErr error, +) (runResult, error) { + if collector == nil { + return result, runErr + } + + collected := collector.Hashes + canonicalCount, paddingRepetitions, spanErr := canonicalInputHashCollectionShape( + uint64(len(collected)), + terminalHashAppended, + ) + if spanErr != nil { + return result, invalidInputHashCollectionSpanError(runErr, spanErr) + } + + // Emulator 0.21 includes a fixed-point entry and pads the input's remaining + // entry capacity with it. The node stores that same sequence in compressed + // form: periodic entries are kept here, while the manager supplies the + // completed status's canonical final root for persistence to append with + // paddingRepetitions in place of the removed collector entry. + result.periodicStateHashes = collected[:canonicalCount] + result.paddingRepetitions = paddingRepetitions + return result, runErr +} + +// invalidInputHashCollectionSpanError makes collection integrity authoritative +// over a terminal execution status. The status remains useful context, but is +// deliberately not wrapped: callers must not persist a malformed collection as +// a completed exception or halt. +func invalidInputHashCollectionSpanError(runErr, spanErr error) error { + err := errors.Join( + ErrMachineInternal, + fmt.Errorf("invalid collected state hash span: %w", spanErr), + ) + if runErr != nil { + err = fmt.Errorf("execution ended with %v and invalid hash span: %w", runErr, err) + } + return err +} + +func executionResponseLimitError( + reqType requestType, + responseKind string, + count int, + capacity int, + limitErr error, +) error { + return fmt.Errorf( + "%s %s count %d exceeds local operational capacity %d: %w", + reqType, responseKind, count, capacity, limitErr, + ) +} + +// canonicalInputHashCollectionShape validates the collector's raw count before +// optionally excluding a terminal fixed-point sample. Validating first prevents +// an overcollection of InputEntryCapacity+1 from being normalized into an +// apparently valid collection. +func canonicalInputHashCollectionShape( + hashCount uint64, + terminalHashAppended bool, +) (canonicalHashCount uint64, paddingRepetitions uint64, err error) { + if _, err := inputHashCollectionPaddingRepetitions(hashCount); err != nil { + return 0, 0, err + } + if terminalHashAppended { + if hashCount == 0 { + return 0, 0, errors.New("terminal fixed point was reported without a collected state hash") + } + hashCount-- + } + paddingRepetitions, err = inputHashCollectionPaddingRepetitions(hashCount) + if err != nil { + return 0, 0, err + } + return hashCount, paddingRepetitions, nil +} + +// executionCycleBounds applies the request-specific configured cycle span. +// Zero uses the machine's complete 2^48-cycle window. Every valid target uses +// the machine's saturating mcycle arithmetic: when start+span is not +// representable, MaxUint64 is both the local target and imcyclemax. +func (m *machineImpl) executionCycleBounds( + ctx context.Context, + reqType requestType, +) (executionBounds, error) { + executionCycleSpan := m.configuredCycleSpan(reqType) + if executionCycleSpan > model.MaxExecutionCycleSpan { + return executionBounds{}, fmt.Errorf( + "%s execution configured cycle span exceeds hard maximum: requested_span=%d maximum_span=%d: %w", + reqType, executionCycleSpan, model.MaxExecutionCycleSpan, ErrReachedLimitMcycle, + ) + } + + currentCycle, err := m.readMCycle(ctx) + if err != nil { + return executionBounds{}, err + } + + configured := executionCycleSpan != 0 + if executionCycleSpan == 0 { + // Emulator 0.21 arms imcyclemax = mcycle + 2^48 - 1 (saturating) at + // input delivery and classifies reaching it as mcycle overflow with + // priority over yield and halt. Mirror that exact window - endpoint + // and saturation - so the node's target can never disagree with the + // machine's own enforcement. + executionCycleSpan = model.MaxExecutionCycleSpan + } + limit := ^uint64(0) + if currentCycle <= ^uint64(0)-executionCycleSpan { + limit = currentCycle + executionCycleSpan + } + return executionBounds{ + start: currentCycle, limit: limit, span: executionCycleSpan, configured: configured, + }, nil +} + +// validateExecutionCycleIncrement is a runtime defense for values loaded from +// persistent storage or supplied through a custom MachineConfig. process calls +// it before SendCmioResponse changes the candidate machine. +func (m *machineImpl) validateExecutionCycleIncrement(reqType requestType) error { + if m.executionCycleIncrement(reqType) == 0 { + return fmt.Errorf("%s execution increment must be greater than zero: %w", reqType, ErrMachineInternal) + } + return nil +} + +func (m *machineImpl) configuredCycleSpan(reqType requestType) uint64 { + if reqType == InspectStateRequest { + return m.params.InspectMaxCycles + } + return m.params.AdvanceMaxCycles +} + +func (m *machineImpl) executionCycleIncrement(reqType requestType) uint64 { + if reqType == InspectStateRequest { + return m.params.InspectIncCycles + } + return m.params.AdvanceIncCycles +} + +// executionLimitError formats the resolved request window while preserving the +// emulator's typed imcyclemax origin when it—not a node target—stopped the run. +func executionLimitError( + reqType requestType, + bounds executionBounds, + current uint64, + origin error, +) error { + targetSpan := bounds.limit - bounds.start + executedCycles := uint64(0) + if current >= bounds.start { + executedCycles = current - bounds.start + } + source := "fixed" + if bounds.configured { + source = "configured" + } + progress := fmt.Sprintf( + "start=%d requested_span=%d target_span=%d executed_cycles=%d", + bounds.start, bounds.span, targetSpan, executedCycles, + ) + + // A machine overflow may stop execution before its local target. This most + // commonly happens because an inspect does not re-arm imcyclemax and inherits + // the ceiling from the preceding advance. Preserve the machine-origin + // sentinel, but do not claim that a local span was exhausted. + if current < bounds.limit { + return fmt.Errorf( + "%s execution stopped before %s cycle limit: %s: %w", + reqType, source, progress, origin, + ) + } + if errors.Is(origin, ErrMcycleOverflow) && bounds.configured { + // imcyclemax has precedence over reaching the requested run target. Even + // when both cycles are equal, the machine-enforced ceiling—not the local + // configured cap—stopped execution, so raising that cap cannot help. + return fmt.Errorf( + "%s execution stopped at machine imcyclemax coincident with configured target: %s: %w", + reqType, progress, origin, + ) + } + + if errors.Is(origin, ErrMcycleOverflow) { + source += " (machine imcyclemax)" + } + return fmt.Errorf( + "%s execution reached %s cycle limit without completing: %s: %w", + reqType, source, progress, origin, + ) +} + +// runIncrementInterval runs the machine for at most incrementLimit mcycles (or +// the distance left to limitCycle) and preserves the emulator's typed break +// reason for the request loop to classify. func (m *machineImpl) runIncrementInterval(ctx context.Context, currentCycle Cycle, limitCycle Cycle, + incrementLimit Cycle, hashCollectorState *HashCollectorState, timeout time.Duration, -) (*yieldType, Cycle, error) { +) (incrementResult, error) { startingCycle := currentCycle - // Returns with an error if the next run would exceed limitCycle. - if currentCycle >= limitCycle && m.params.AdvanceIncCycles != 0 { - return nil, 0, ErrReachedLimitMcycle + // Return without calling the backend after an ordinary local target has + // already been reached. MaxUint64 is different: saturating arithmetic can + // make both the node target and the emulator's imcyclemax equal the current + // cycle. Calling the emulator once at that fixed point preserves its + // authoritative McycleOverflow reason instead of mislabeling the stop as an + // operator-cap failure. + atSaturatedEndpoint := currentCycle == ^uint64(0) && limitCycle == ^uint64(0) + if currentCycle >= limitCycle && !atSaturatedEndpoint { + return incrementResult{currentCycle: currentCycle}, ErrReachedLimitMcycle } // Calculates the increment. - increment := min(m.params.AdvanceIncCycles, limitCycle-currentCycle) + increment := min(incrementLimit, limitCycle-currentCycle) m.logger.Debug("machine step before run", "currentCycle", currentCycle, "increment", increment) // Runs the machine. - breakReason, err := m.backend_run(currentCycle+increment, hashCollectorState, timeout) + breakReason, err := m.backendRun(currentCycle+increment, hashCollectorState, timeout) if err != nil { - return nil, 0, err + return incrementResult{currentCycle: currentCycle}, err + } + if hashCollectorState != nil && hashCollectorState.ConsoleIOError != "" { + m.logger.Warn( + "machine console I/O error while collecting computation-hash entries", + "error", hashCollectorState.ConsoleIOError, + ) + hashCollectorState.ConsoleIOError = "" } // Gets the current cycle. currentCycle, err = m.readMCycle(ctx) if err != nil { - return nil, 0, err + return incrementResult{}, err } m.logger.Debug("machine step after run", @@ -515,30 +823,32 @@ func (m *machineImpl) runIncrementInterval(ctx context.Context, "leftover", limitCycle-currentCycle, "breakReason", breakReason) + if atSaturatedEndpoint && breakReason != McycleOverflow { + // Emulator 0.21 gives mcycle overflow priority over target, yield, and + // halt when mcycle == imcyclemax. Accepting any other reason here could + // either hide that terminal origin or repeat zero-progress runs forever. + return incrementResult{breakReason: breakReason, currentCycle: currentCycle}, fmt.Errorf( + "machine returned break reason %d instead of mcycle overflow at MaxUint64: %w", + breakReason, ErrMachineInternal, + ) + } + switch breakReason { - case YieldedManually: - yt := ManualYield - return &yt, currentCycle, nil // returns with the yield type - case YieldedAutomatically: - yt := AutomaticYield - return &yt, currentCycle, nil // returns with the yield type - case YieldedSoftly: - return nil, currentCycle, nil // returns with no yield type - case ReachedTargetMcycle: - return nil, currentCycle, ErrReachedTargetMcycle - case McycleOverflow: - return nil, currentCycle, ErrReachedLimitMcycle - case Halted: - return nil, currentCycle, ErrHalted - case Failed: - return nil, currentCycle, ErrMachineInternal + case YieldedManually, + YieldedAutomatically, + YieldedSoftly, + ReachedTargetMcycle, + McycleOverflow, + Halted, + Failed: + return incrementResult{breakReason: breakReason, currentCycle: currentCycle}, nil default: err := fmt.Errorf("invalid break reason: %d: %w", breakReason, ErrMachineInternal) - return nil, currentCycle, err + return incrementResult{breakReason: breakReason, currentCycle: currentCycle}, err } } -func (m *machineImpl) backend_run(mcycleEnd uint64, hashCollectorState *HashCollectorState, timeout time.Duration) (BreakReason, error) { +func (m *machineImpl) backendRun(mcycleEnd uint64, hashCollectorState *HashCollectorState, timeout time.Duration) (BreakReason, error) { if hashCollectorState != nil { m.logger.Debug("Running with root hash collection") return m.backend.RunAndCollectRootHashes(mcycleEnd, hashCollectorState, timeout) @@ -546,6 +856,10 @@ func (m *machineImpl) backend_run(mcycleEnd uint64, hashCollectorState *HashColl return m.backend.Run(mcycleEnd, timeout) } +func isFixedPointBreakReason(reason BreakReason) bool { + return reason == YieldedManually || reason == Halted || reason == McycleOverflow +} + // Helper functions func checkContext(ctx context.Context) error { diff --git a/pkg/machine/implementation_test.go b/pkg/machine/implementation_test.go index e732c538f..39df3c38a 100644 --- a/pkg/machine/implementation_test.go +++ b/pkg/machine/implementation_test.go @@ -6,6 +6,7 @@ package machine import ( "context" "errors" + "fmt" "io" "log/slog" "testing" @@ -25,6 +26,12 @@ type ImplementationSuite struct { logger *slog.Logger } +func testExecutionBounds(start, limit uint64) executionBounds { + return executionBounds{ + start: start, limit: limit, span: limit - start, configured: true, + } +} + func (s *ImplementationSuite) SetupSuite() { s.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) } @@ -164,10 +171,24 @@ func (s *ImplementationSuite) TestOutputsHash() { }, } _, err = machine2.OutputsHash(ctx) - require.Error(err) - require.Contains(err.Error(), "machine manual yield reason is not accepted") + require.ErrorIs(err, ErrRejected) mockBackend2.AssertExpectations(s.T()) + // Exception remains distinguishable through the public error taxonomy. + mockBackendException := NewMockBackend() + mockBackendException.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( + uint8(0), uint16(ManualYieldReasonException), []byte("exception"), nil) + machineException := &machineImpl{ + backend: mockBackendException, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second * 5, + }, + } + _, err = machineException.OutputsHash(ctx) + require.ErrorIs(err, ErrException) + mockBackendException.AssertExpectations(s.T()) + // Test outputs hash with invalid length mockBackend3 := NewMockBackend() mockBackend3.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( @@ -264,7 +285,6 @@ func (s *ImplementationSuite) TestAdvance() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, @@ -274,7 +294,8 @@ func (s *ImplementationSuite) TestAdvance() { input := []byte("test input") resp, err := machine.Advance(ctx, input, expectedHash, false) require.NoError(err) - require.True(resp.Accepted) + require.NotNil(resp) + require.Equal(CompletionStatusAccepted, resp.Status) require.Empty(resp.Outputs) require.Empty(resp.Reports) require.NotEqual(Hash{}, resp.OutputsHash) @@ -288,7 +309,6 @@ func (s *ImplementationSuite) TestAdvance() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, @@ -296,7 +316,8 @@ func (s *ImplementationSuite) TestAdvance() { } resp, err = machine2.Advance(ctx, input, expectedHash, false) require.NoError(err) - require.False(resp.Accepted) + require.NotNil(resp) + require.Equal(CompletionStatusRejected, resp.Status) require.Empty(resp.Outputs) require.Empty(resp.Reports) require.Equal(Hash{}, resp.OutputsHash) @@ -310,18 +331,97 @@ func (s *ImplementationSuite) TestAdvance() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, }, } resp, err = machine3.Advance(ctx, input, expectedHash, false) - require.ErrorIs(err, ErrException) - require.False(resp.Accepted) + require.NoError(err) + require.NotNil(resp) + require.Equal(CompletionStatusException, resp.Status) + require.Equal([]byte("exception data"), resp.ExceptionData) require.Equal(Hash{}, resp.OutputsHash) mockBackend3.AssertExpectations(s.T()) + // Halting is also a completed deterministic status and retains + // the PRT input hash collection produced before the halt. + expectedHaltHash := randomFakeHash() + mockBackendHalted := NewMockBackend() + mockBackendHalted.On("CmioRxBufferSize").Return(uint64(1024)) + mockBackendHalted.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(0), nil).Once() + mockBackendHalted.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(100), nil).Once() + mockBackendHalted.On( + "SendCmioResponse", + uint16(AdvanceStateRequest), mock.Anything, mock.Anything, mock.AnythingOfType("time.Duration"), + ).Return(nil) + mockBackendHalted.On( + "RunAndCollectRootHashes", + mock.AnythingOfType("uint64"), + mock.AnythingOfType("*machine.HashCollectorState"), + mock.AnythingOfType("time.Duration"), + ).Run(func(args mock.Arguments) { + state := args.Get(1).(*HashCollectorState) + state.Hashes = append(state.Hashes, expectedHaltHash) + state.MCyclePhase = 0 // halt occurred exactly at the collected boundary + }).Return(Halted, nil) + machineHalted := &machineImpl{ + backend: mockBackendHalted, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second * 5, + AdvanceIncCycles: 100, + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: time.Second * 10, + }, + } + resp, err = machineHalted.Advance(ctx, input, expectedHash, true) + require.NoError(err) + require.NotNil(resp) + require.Equal(CompletionStatusHalted, resp.Status) + require.Empty(resp.PeriodicStateHashes) + require.Equal(InputEntryCapacity, resp.PaddingRepetitions) + mockBackendHalted.AssertExpectations(s.T()) + + // Test advance halted off a sampling boundary: 0.21 still appends the + // fixed-point sample (nonzero phase) and the node must still drop it. + offBoundaryHaltHash := randomFakeHash() + mockBackendHaltedOff := NewMockBackend() + mockBackendHaltedOff.On("CmioRxBufferSize").Return(uint64(1024)) + mockBackendHaltedOff.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(0), nil).Once() + mockBackendHaltedOff.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(101), nil).Once() + mockBackendHaltedOff.On( + "SendCmioResponse", + uint16(AdvanceStateRequest), mock.Anything, mock.Anything, mock.AnythingOfType("time.Duration"), + ).Return(nil) + mockBackendHaltedOff.On( + "RunAndCollectRootHashes", + mock.AnythingOfType("uint64"), + mock.AnythingOfType("*machine.HashCollectorState"), + mock.AnythingOfType("time.Duration"), + ).Run(func(args mock.Arguments) { + state := args.Get(1).(*HashCollectorState) + state.Hashes = append(state.Hashes, offBoundaryHaltHash) + state.MCyclePhase = 5 // halt occurred between sampling boundaries + }).Return(Halted, nil) + machineHaltedOff := &machineImpl{ + backend: mockBackendHaltedOff, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second * 5, + AdvanceIncCycles: 100, + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: time.Second * 10, + }, + } + resp, err = machineHaltedOff.Advance(ctx, input, expectedHash, true) + require.NoError(err) + require.NotNil(resp) + require.Equal(CompletionStatusHalted, resp.Status) + require.Empty(resp.PeriodicStateHashes) + require.Equal(InputEntryCapacity, resp.PaddingRepetitions) + mockBackendHaltedOff.AssertExpectations(s.T()) + // Test advance with payload too large mockBackend4 := NewMockBackend() mockBackend4.On("CmioRxBufferSize").Return(uint64(5)) @@ -330,17 +430,50 @@ func (s *ImplementationSuite) TestAdvance() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, }, } largeInput := make([]byte, 10) - _, err = machine4.Advance(ctx, largeInput, expectedHash, false) + resp, err = machine4.Advance(ctx, largeInput, expectedHash, false) require.ErrorIs(err, ErrPayloadLengthLimitExceeded) + require.Nil(resp) mockBackend4.AssertExpectations(s.T()) + // A configured target beyond uint64 saturates at MaxUint64, just like the + // emulator's imcyclemax. If the machine reaches that endpoint, the machine + // overflow—not the configured-cap sentinel—explains why execution stopped. + mockBackendOverflow := NewMockBackend() + mockBackendOverflow.On("CmioRxBufferSize").Return(uint64(1024)) + mockBackendOverflow.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(^uint64(0)-uint64(999), nil).Once() + mockBackendOverflow.On("SendCmioResponse", uint16(AdvanceStateRequest), input, expectedHash, + mock.AnythingOfType("time.Duration")).Return(nil).Once() + mockBackendOverflow.On("Run", ^uint64(0), mock.AnythingOfType("time.Duration")). + Return(McycleOverflow, nil).Once() + mockBackendOverflow.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(^uint64(0), nil).Once() + machineOverflow := &machineImpl{ + backend: mockBackendOverflow, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second * 5, + AdvanceIncCycles: ^uint64(0), + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: time.Second, + AdvanceMaxCycles: 1000, + }, + } + resp, err = machineOverflow.Advance(ctx, input, expectedHash, false) + require.ErrorIs(err, ErrReachedLimitMcycle) + require.ErrorIs(err, ErrMcycleOverflow) + require.Contains(err.Error(), "requested_span=1000") + require.Contains(err.Error(), "target_span=999") + require.Contains(err.Error(), "executed_cycles=999") + require.Nil(resp) + mockBackendOverflow.AssertExpectations(s.T()) + // Test advance with invalid hash length mockBackend5 := NewMockBackend() mockBackend5.On("CmioRxBufferSize").Return(uint64(1024)) @@ -354,18 +487,885 @@ func (s *ImplementationSuite) TestAdvance() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, }, } - _, err = machine5.Advance(ctx, input, expectedHash, false) + resp, err = machine5.Advance(ctx, input, expectedHash, false) require.Error(err) require.ErrorIs(err, ErrHashLength) + require.Nil(resp) mockBackend5.AssertExpectations(s.T()) } +func (s *ImplementationSuite) TestCompletedAdvanceIsIndependentOfAdequateIncrementSettings() { + expectedOutputsHash := randomFakeHash() + expectedHashes := []Hash{randomFakeHash(), randomFakeHash()} + configs := []model.ExecutionParameters{ + { + FastDeadline: time.Second, + AdvanceIncCycles: 2, + AdvanceIncDeadline: 10 * time.Millisecond, + AdvanceMaxDeadline: time.Second, + }, + { + FastDeadline: 2 * time.Second, + AdvanceIncCycles: 4, + AdvanceIncDeadline: 100 * time.Millisecond, + AdvanceMaxDeadline: 2 * time.Second, + }, + { + FastDeadline: 3 * time.Second, + AdvanceIncCycles: 100, + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: 3 * time.Second, + }, + } + expectedTargets := [][]uint64{ + {2, 4, 5, 7, 8, 10}, + {4, 7, 10}, + {100, 103, 106}, + } + + var baseline *AdvanceResponse + for i, params := range configs { + s.Run(fmt.Sprintf("configuration-%d", i), func() { + backend := &statefulAdvanceBackend{ + outputsHash: expectedOutputsHash, + hashes: expectedHashes, + } + machine := &machineImpl{backend: backend, logger: s.logger, params: params} + + result, err := machine.Advance(context.Background(), []byte("stable-input"), Hash{}, true) + s.Require().NoError(err) + s.Require().NotNil(result) + s.Require().Equal(expectedTargets[i], backend.runTargets) + if baseline == nil { + baseline = result + } else { + s.Require().Equal(baseline, result) + } + }) + } +} + +func (s *ImplementationSuite) TestPRTHashCollectionIsIndependentOfConfiguredCycleCap() { + const configuredCap uint64 = 10 + expectedOutputsHash := randomFakeHash() + expectedHashes := []Hash{randomFakeHash(), randomFakeHash()} + + runAdvance := func(maxCycles uint64) (*AdvanceResponse, []uint64) { + backend := &statefulAdvanceBackend{ + outputsHash: expectedOutputsHash, + hashes: expectedHashes, + } + machine := &machineImpl{ + backend: backend, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second, + AdvanceIncCycles: 100, + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: time.Second, + AdvanceMaxCycles: maxCycles, + }, + } + + result, err := machine.Advance(context.Background(), []byte("stable-input"), Hash{}, true) + s.Require().NoError(err) + s.Require().NotNil(result) + return result, backend.runTargets + } + + uncapped, uncappedTargets := runAdvance(0) + capped, cappedTargets := runAdvance(configuredCap) + maximumCap, maximumCapTargets := runAdvance(model.MaxExecutionCycleSpan) + + s.Equal([]uint64{100, 103, 106}, uncappedTargets) + s.Equal([]uint64{configuredCap, configuredCap, configuredCap}, cappedTargets) + s.NotEqual(uncappedTargets, cappedTargets, "configured cap must alter the backend run targets") + s.Equal(uncapped, capped) + s.Equal(uncapped.PeriodicStateHashes, capped.PeriodicStateHashes) + s.Equal(uncapped.PaddingRepetitions, capped.PaddingRepetitions) + s.Equal(expectedHashes, capped.PeriodicStateHashes) + s.Equal(InputEntryCapacity-uint64(len(expectedHashes)), capped.PaddingRepetitions) + s.Equal(uncappedTargets, maximumCapTargets) + s.Equal(uncapped, maximumCap, + "the largest valid configured span must be canonically identical to no operator cap") +} + +// statefulAdvanceBackend models a guest that produces an output at mcycle 3, +// a report at mcycle 6, and its final accepted yield at mcycle 9. Small +// increments reach intermediate targets; large increments stop early at the +// same guest events. This makes the scheduling-invariance test exercise the +// real run loop rather than preprogramming one yield per call. +type statefulAdvanceBackend struct { + cycle uint64 + runTargets []uint64 + outputsHash Hash + hashes []Hash +} + +func (b *statefulAdvanceBackend) run(mcycleEnd uint64) (BreakReason, error) { + b.runTargets = append(b.runTargets, mcycleEnd) + if mcycleEnd <= b.cycle { + return Failed, fmt.Errorf("run target %d is not ahead of cycle %d", mcycleEnd, b.cycle) + } + var eventCycle uint64 + var reason BreakReason + switch { + case b.cycle < 3: + eventCycle, reason = 3, YieldedAutomatically + case b.cycle < 6: + eventCycle, reason = 6, YieldedAutomatically + case b.cycle < 9: + eventCycle, reason = 9, YieldedManually + default: + return Failed, errors.New("scripted advance already completed") + } + if mcycleEnd < eventCycle { + b.cycle = mcycleEnd + return ReachedTargetMcycle, nil + } + b.cycle = eventCycle + return reason, nil +} + +func (b *statefulAdvanceBackend) Load(string, string, time.Duration) error { return nil } +func (b *statefulAdvanceBackend) Store(string, time.Duration) error { return nil } +func (b *statefulAdvanceBackend) Run(end uint64, _ time.Duration) (BreakReason, error) { + return b.run(end) +} +func (b *statefulAdvanceBackend) RunAndCollectRootHashes( + end uint64, + state *HashCollectorState, + _ time.Duration, +) (BreakReason, error) { + reason, err := b.run(end) + if err == nil && reason == YieldedAutomatically { + switch b.cycle { + case 3: + state.Hashes = append(state.Hashes, b.hashes[0]) + case 6: + state.Hashes = append(state.Hashes, b.hashes[1]) + } + } + if err == nil && reason == YieldedManually { + // Emulator 0.21 appends the fixed-point sample whenever a collecting + // run stops at a manual yield, regardless of phase; the node drops + // exactly this final entry. + state.Hashes = append(state.Hashes, Hash{0xF1, 0xED}) + } + return reason, err +} +func (b *statefulAdvanceBackend) IsAtManualYield(time.Duration) (bool, error) { + return b.cycle == 9, nil +} +func (b *statefulAdvanceBackend) ReadMCycle(time.Duration) (uint64, error) { return b.cycle, nil } +func (b *statefulAdvanceBackend) SendCmioResponse(reason uint16, data []byte, _ *Hash, _ time.Duration) error { + if reason != uint16(AdvanceStateRequest) || string(data) != "stable-input" { + return fmt.Errorf("unexpected CMIO response reason=%d data_len=%d", reason, len(data)) + } + return nil +} +func (b *statefulAdvanceBackend) ReceiveCmioRequest(time.Duration) (uint8, uint16, []byte, error) { + switch b.cycle { + case 3: + return 0, uint16(AutomaticYieldReasonOutput), []byte("stable-output"), nil + case 6: + return 0, uint16(AutomaticYieldReasonReport), []byte("stable-report"), nil + case 9: + return 0, uint16(ManualYieldReasonAccepted), b.outputsHash[:], nil + default: + return 0, 0, nil, fmt.Errorf("no CMIO request at cycle %d", b.cycle) + } +} +func (b *statefulAdvanceBackend) WriteMemory(uint64, []byte, time.Duration) error { return nil } +func (b *statefulAdvanceBackend) GetRootHash(time.Duration) (Hash, error) { return Hash{}, nil } +func (b *statefulAdvanceBackend) GetProof(uint64, int32, time.Duration) ([]Hash, error) { + return nil, nil +} +func (b *statefulAdvanceBackend) Delete() {} +func (b *statefulAdvanceBackend) ForkServer(time.Duration) (Backend, string, uint32, error) { + return nil, "", 0, errors.New("not implemented") +} +func (b *statefulAdvanceBackend) ShutdownServer(time.Duration) error { return nil } +func (b *statefulAdvanceBackend) NewMachineRuntimeConfig() (string, error) { + return "{}", nil +} +func (b *statefulAdvanceBackend) CmioRxBufferSize() uint64 { return 1024 } + +func (s *ImplementationSuite) TestInterruptedAdvanceReturnsNilAndCanBeRetried() { + expectedOutputsHash := randomFakeHash() + backend := NewMockBackend() + backend.On("CmioRxBufferSize").Return(uint64(1024)) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(0), nil) + backend.On( + "SendCmioResponse", + uint16(AdvanceStateRequest), + []byte("retry-input"), + mock.Anything, + mock.AnythingOfType("time.Duration"), + ).Return(nil) + backend.On("Run", mock.AnythingOfType("uint64"), mock.AnythingOfType("time.Duration")). + Return(YieldedManually, nil) + backend.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( + uint8(0), uint16(ManualYieldReasonAccepted), expectedOutputsHash[:], nil, + ) + machine := &machineImpl{ + backend: backend, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second, + AdvanceIncCycles: 100, + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: time.Second, + }, + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + result, err := machine.Advance(ctx, []byte("retry-input"), Hash{}, false) + s.Require().ErrorIs(err, ErrCanceled) + s.Require().Nil(result) + + result, err = machine.Advance(context.Background(), []byte("retry-input"), Hash{}, false) + s.Require().NoError(err) + s.Require().NotNil(result) + s.Require().Equal(CompletionStatusAccepted, result.Status) + s.Require().Equal(expectedOutputsHash, result.OutputsHash) + backend.AssertExpectations(s.T()) +} + +func (s *ImplementationSuite) TestRunUsesHardExecutionSpanWhenMaximumIsZero() { + const startCycle uint64 = 7 + // The fixed window mirrors the emulator's imcyclemax: 2^48 - 1 ahead. + const executionCycleSpan uint64 = (1 << Log2MaxMCyclesPerAdvanceState) - 1 + + tests := []struct { + name string + reqType requestType + params model.ExecutionParameters + }{ + { + name: "advance", + reqType: AdvanceStateRequest, + params: model.ExecutionParameters{ + AdvanceIncCycles: ^uint64(0), + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: time.Second, + }, + }, + { + name: "inspect", + reqType: InspectStateRequest, + params: model.ExecutionParameters{ + InspectIncCycles: ^uint64(0), + InspectIncDeadline: time.Second, + InspectMaxDeadline: time.Second, + }, + }, + } + + for _, tt := range tests { + s.Run(tt.name, func() { + mockBackend := NewMockBackend() + mockBackend.On("Run", startCycle+executionCycleSpan, mock.AnythingOfType("time.Duration")). + Return(McycleOverflow, nil) + mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(startCycle+executionCycleSpan, nil).Once() + + machine := &machineImpl{ + backend: mockBackend, + logger: s.logger, + params: tt.params, + } + + _, err := machine.run( + context.Background(), tt.reqType, false, + executionBounds{ + start: startCycle, limit: startCycle + executionCycleSpan, + span: executionCycleSpan, + }, + ) + s.Require().ErrorIs(err, ErrReachedLimitMcycle) + s.Require().ErrorIs(err, ErrMcycleOverflow) + s.Contains(err.Error(), fmt.Sprintf("executed_cycles=%d", executionCycleSpan)) + mockBackend.AssertExpectations(s.T()) + }) + } +} + +func (s *ImplementationSuite) TestExecutionCycleBoundsAcceptsMaxSafeStart() { + const maxSafeStart uint64 = ^uint64(0) - model.MaxExecutionCycleSpan + + mockBackend := NewMockBackend() + mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(maxSafeStart, nil) + machine := &machineImpl{ + backend: mockBackend, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second, + AdvanceIncCycles: 1, + }, + } + + bounds, err := machine.executionCycleBounds(context.Background(), AdvanceStateRequest) + s.Require().NoError(err) + s.Equal(maxSafeStart, bounds.start) + s.Equal(maxSafeStart+model.MaxExecutionCycleSpan, bounds.limit) + s.Equal(model.MaxExecutionCycleSpan, bounds.span) + s.False(bounds.configured) + mockBackend.AssertExpectations(s.T()) +} + +func (s *ImplementationSuite) TestExecutionCycleBoundsUsesRequestSpecificMaximum() { + const start uint64 = 17 + for _, test := range []struct { + name string + reqType requestType + span uint64 + params model.ExecutionParameters + }{ + { + name: "advance configured", reqType: AdvanceStateRequest, span: 101, + params: model.ExecutionParameters{AdvanceIncCycles: 1, AdvanceMaxCycles: 101}, + }, + { + name: "inspect configured", reqType: InspectStateRequest, span: 202, + params: model.ExecutionParameters{InspectIncCycles: 1, InspectMaxCycles: 202}, + }, + { + name: "advance zero uses fixed", reqType: AdvanceStateRequest, span: model.MaxExecutionCycleSpan, + params: model.ExecutionParameters{AdvanceIncCycles: 1}, + }, + { + name: "inspect zero uses fixed", reqType: InspectStateRequest, span: model.MaxExecutionCycleSpan, + params: model.ExecutionParameters{InspectIncCycles: 1}, + }, + } { + s.Run(test.name, func() { + backend := NewMockBackend() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start, nil) + machine := &machineImpl{backend: backend, logger: s.logger, params: test.params} + + bounds, err := machine.executionCycleBounds(context.Background(), test.reqType) + s.Require().NoError(err) + s.Equal(start, bounds.start) + s.Equal(start+test.span, bounds.limit) + s.Equal(test.span, bounds.span) + if test.params.AdvanceMaxCycles != 0 || test.params.InspectMaxCycles != 0 { + s.True(bounds.configured) + } else { + s.False(bounds.configured) + } + backend.AssertExpectations(s.T()) + }) + } +} + +func (s *ImplementationSuite) TestExecutionCycleBoundsRejectsMaximumAboveHardLimit() { + for _, reqType := range []requestType{AdvanceStateRequest, InspectStateRequest} { + s.Run(reqType.String(), func() { + backend := NewMockBackend() + params := model.ExecutionParameters{ + AdvanceIncCycles: 1, + AdvanceMaxCycles: model.MaxExecutionCycles, + InspectIncCycles: 1, + } + if reqType == InspectStateRequest { + params.AdvanceMaxCycles = 0 + params.InspectMaxCycles = model.MaxExecutionCycles + } + machine := &machineImpl{backend: backend, logger: s.logger, params: params} + + _, err := machine.executionCycleBounds(context.Background(), reqType) + s.Require().ErrorIs(err, ErrReachedLimitMcycle) + s.Contains(err.Error(), reqType.String()) + s.Contains(err.Error(), fmt.Sprint(model.MaxExecutionCycles)) + backend.AssertNotCalled(s.T(), "ReadMCycle", mock.Anything) + backend.AssertExpectations(s.T()) + }) + } +} + +func (s *ImplementationSuite) TestProcessRejectsInvalidMaximumBeforeCMIO() { + backend := NewMockBackend() + backend.On("CmioRxBufferSize").Return(uint64(1024)) + machine := &machineImpl{ + backend: backend, + logger: s.logger, + params: model.ExecutionParameters{ + AdvanceIncCycles: 1, + AdvanceMaxCycles: model.MaxExecutionCycles, + FastDeadline: time.Second, + }, + } + + response, err := machine.Advance(context.Background(), []byte("input"), Hash{}, false) + s.Require().Nil(response) + s.Require().ErrorIs(err, ErrReachedLimitMcycle) + backend.AssertNotCalled(s.T(), "ReadMCycle", mock.Anything) + backend.AssertNotCalled(s.T(), "SendCmioResponse", + mock.Anything, mock.Anything, mock.Anything, mock.Anything) + backend.AssertExpectations(s.T()) +} + +func (s *ImplementationSuite) TestProcessRejectsZeroIncrementBeforeCMIO() { + for _, test := range []struct { + name string + reqType requestType + invoke func(*machineImpl) error + }{ + { + name: "advance", + reqType: AdvanceStateRequest, + invoke: func(machine *machineImpl) error { + _, err := machine.Advance(context.Background(), []byte("input"), Hash{}, false) + return err + }, + }, + { + name: "inspect", + reqType: InspectStateRequest, + invoke: func(machine *machineImpl) error { + _, err := machine.Inspect(context.Background(), []byte("query")) + return err + }, + }, + } { + s.Run(test.name, func() { + backend := NewMockBackend() + backend.On("CmioRxBufferSize").Return(uint64(1024)).Maybe() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(1), nil).Maybe() + params := model.ExecutionParameters{ + FastDeadline: time.Second, + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: time.Second, + InspectIncDeadline: time.Second, + InspectMaxDeadline: time.Second, + } + machine := &machineImpl{backend: backend, logger: s.logger, params: params} + + err := test.invoke(machine) + s.Require().Error(err) + s.Contains(err.Error(), test.reqType.String()) + s.Contains(err.Error(), "increment") + backend.AssertNotCalled(s.T(), "SendCmioResponse", + mock.Anything, mock.Anything, mock.Anything, mock.Anything) + backend.AssertNotCalled(s.T(), "Run", mock.Anything, mock.Anything) + backend.AssertNotCalled(s.T(), "RunAndCollectRootHashes", mock.Anything, mock.Anything, mock.Anything) + backend.AssertExpectations(s.T()) + }) + } +} + +func (s *ImplementationSuite) TestConfiguredCycleEndpointCompletionSucceeds() { + const start, span = uint64(10), uint64(5) + expectedOutputsHash := randomFakeHash() + backend := NewMockBackend() + backend.On("CmioRxBufferSize").Return(uint64(1024)) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start, nil).Once() + backend.On( + "SendCmioResponse", + uint16(AdvanceStateRequest), []byte("input"), mock.Anything, mock.AnythingOfType("time.Duration"), + ).Return(nil) + backend.On("Run", start+span, mock.AnythingOfType("time.Duration")).Return(YieldedManually, nil) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start+span, nil).Once() + backend.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( + uint8(0), uint16(ManualYieldReasonAccepted), expectedOutputsHash[:], nil) + machine := &machineImpl{backend: backend, logger: s.logger, params: model.ExecutionParameters{ + AdvanceIncCycles: span + 100, AdvanceMaxCycles: span, + AdvanceIncDeadline: time.Second, AdvanceMaxDeadline: time.Second, + FastDeadline: time.Second, + }} + + response, err := machine.Advance(context.Background(), []byte("input"), Hash{}, false) + s.Require().NoError(err) + s.Require().Equal(CompletionStatusAccepted, response.Status) + backend.AssertExpectations(s.T()) +} + +func (s *ImplementationSuite) TestConfiguredCycleEndpointHaltSucceeds() { + const start, span = uint64(20), uint64(7) + backend := NewMockBackend() + backend.On("CmioRxBufferSize").Return(uint64(1024)) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start, nil).Once() + backend.On( + "SendCmioResponse", + uint16(AdvanceStateRequest), []byte("input"), mock.Anything, mock.AnythingOfType("time.Duration"), + ).Return(nil) + backend.On("Run", start+span, mock.AnythingOfType("time.Duration")).Return(Halted, nil) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start+span, nil).Once() + machine := &machineImpl{backend: backend, logger: s.logger, params: model.ExecutionParameters{ + AdvanceIncCycles: span + 100, AdvanceMaxCycles: span, + AdvanceIncDeadline: time.Second, AdvanceMaxDeadline: time.Second, + FastDeadline: time.Second, + }} + + response, err := machine.Advance(context.Background(), []byte("input"), Hash{}, false) + s.Require().NoError(err) + s.Require().Equal(CompletionStatusHalted, response.Status) + backend.AssertExpectations(s.T()) +} + +func (s *ImplementationSuite) TestConfiguredCycleExhaustionFails() { + const start, span = uint64(10), uint64(5) + backend := NewMockBackend() + backend.On("CmioRxBufferSize").Return(uint64(1024)) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start, nil).Once() + backend.On( + "SendCmioResponse", + uint16(InspectStateRequest), []byte("query"), mock.Anything, mock.AnythingOfType("time.Duration"), + ).Return(nil) + backend.On("Run", start+span, mock.AnythingOfType("time.Duration")).Return(ReachedTargetMcycle, nil) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start+span, nil).Once() + machine := &machineImpl{backend: backend, logger: s.logger, params: model.ExecutionParameters{ + InspectIncCycles: span + 100, InspectMaxCycles: span, + InspectIncDeadline: time.Second, InspectMaxDeadline: time.Second, + FastDeadline: time.Second, + }} + + response, err := machine.Inspect(context.Background(), []byte("query")) + s.Require().NotNil(response) + s.Require().Equal(CompletionStatusUnknown, response.Status) + s.Require().Empty(response.Reports) + s.Require().ErrorIs(err, ErrReachedLimitMcycle) + s.Contains(err.Error(), "inspect execution reached configured cycle limit") + s.Contains(err.Error(), "start=10 requested_span=5") + backend.AssertExpectations(s.T()) +} + +func (s *ImplementationSuite) TestAdvanceConfiguredCycleExhaustionReturnsNoResult() { + const start, span = uint64(30), uint64(9) + backend := NewMockBackend() + backend.On("CmioRxBufferSize").Return(uint64(1024)) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start, nil).Once() + backend.On( + "SendCmioResponse", + uint16(AdvanceStateRequest), []byte("input"), mock.Anything, mock.AnythingOfType("time.Duration"), + ).Return(nil) + backend.On("Run", start+span, mock.AnythingOfType("time.Duration")).Return(ReachedTargetMcycle, nil) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start+span, nil).Once() + machine := &machineImpl{backend: backend, logger: s.logger, params: model.ExecutionParameters{ + AdvanceIncCycles: span + 100, AdvanceMaxCycles: span, + AdvanceIncDeadline: time.Second, AdvanceMaxDeadline: time.Second, + FastDeadline: time.Second, + }} + + response, err := machine.Advance(context.Background(), []byte("input"), Hash{}, false) + s.Require().Nil(response) + s.Require().ErrorIs(err, ErrReachedLimitMcycle) + s.Contains(err.Error(), "advance execution reached configured cycle limit") + s.Contains(err.Error(), "start=30 requested_span=9") + backend.AssertExpectations(s.T()) +} + +func (s *ImplementationSuite) TestAdvanceFixedSpanExhaustionPreservesMachineOverflow() { + const start = uint64(30) + backend := NewMockBackend() + backend.On("CmioRxBufferSize").Return(uint64(1024)) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start, nil).Once() + backend.On( + "SendCmioResponse", + uint16(AdvanceStateRequest), []byte("input"), mock.Anything, mock.AnythingOfType("time.Duration"), + ).Return(nil) + // With no configured cap, the emulator enforces the span itself and reports + // the mcycle-overflow fixed point as a break reason. + backend.On("Run", mock.AnythingOfType("uint64"), mock.AnythingOfType("time.Duration")). + Return(McycleOverflow, nil) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(start+model.MaxExecutionCycleSpan, nil).Once() + machine := &machineImpl{backend: backend, logger: s.logger, params: model.ExecutionParameters{ + AdvanceIncCycles: ^uint64(0), + AdvanceIncDeadline: time.Second, AdvanceMaxDeadline: time.Second, + FastDeadline: time.Second, + }} + + response, err := machine.Advance(context.Background(), []byte("input"), Hash{}, false) + s.Require().Nil(response) + s.Require().ErrorIs(err, ErrReachedLimitMcycle) + s.Require().ErrorIs(err, ErrMcycleOverflow) + s.Contains(err.Error(), "advance execution reached fixed (machine imcyclemax) cycle limit") + backend.AssertExpectations(s.T()) +} + +func (s *ImplementationSuite) TestAdvanceConfiguredLimitTiePreservesMachineOverflowPrecedence() { + const start = uint64(30) + configuredSpan := model.MaxExecutionCycleSpan + limit := start + configuredSpan + backend := NewMockBackend() + backend.On("CmioRxBufferSize").Return(uint64(1024)) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start, nil).Once() + backend.On( + "SendCmioResponse", + uint16(AdvanceStateRequest), []byte("input"), mock.Anything, mock.AnythingOfType("time.Duration"), + ).Return(nil).Once() + backend.On("Run", limit, mock.AnythingOfType("time.Duration")).Return(McycleOverflow, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(limit, nil).Once() + machine := &machineImpl{backend: backend, logger: s.logger, params: model.ExecutionParameters{ + AdvanceIncCycles: ^uint64(0), + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: time.Second, + AdvanceMaxCycles: configuredSpan, + FastDeadline: time.Second, + }} + + response, err := machine.Advance(context.Background(), []byte("input"), Hash{}, false) + s.Require().Nil(response) + s.Require().ErrorIs(err, ErrReachedLimitMcycle) + s.Require().ErrorIs(err, ErrMcycleOverflow) + s.Contains(err.Error(), "advance execution stopped at machine imcyclemax coincident with configured target") + s.Contains(err.Error(), fmt.Sprintf("requested_span=%d", configuredSpan)) + s.Contains(err.Error(), fmt.Sprintf("executed_cycles=%d", configuredSpan)) + backend.AssertExpectations(s.T()) +} + +func (s *ImplementationSuite) TestAdvanceStartingAtMachineMaximumPreservesOverflowOrigin() { + for _, test := range []struct { + name string + configuredMax uint64 + }{ + {name: "no operator cap", configuredMax: 0}, + {name: "configured cap also saturates", configuredMax: 1}, + } { + s.Run(test.name, func() { + backend := NewMockBackend() + backend.On("CmioRxBufferSize").Return(uint64(1024)) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(^uint64(0), nil).Once() + backend.On( + "SendCmioResponse", + uint16(AdvanceStateRequest), []byte("input"), mock.Anything, mock.AnythingOfType("time.Duration"), + ).Return(nil).Once() + backend.On("Run", ^uint64(0), mock.AnythingOfType("time.Duration")). + Return(McycleOverflow, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(^uint64(0), nil).Once() + machine := &machineImpl{backend: backend, logger: s.logger, params: model.ExecutionParameters{ + AdvanceIncCycles: 1, + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: time.Second, + AdvanceMaxCycles: test.configuredMax, + FastDeadline: time.Second, + }} + + response, err := machine.Advance(context.Background(), []byte("input"), Hash{}, false) + s.Require().Nil(response) + s.Require().ErrorIs(err, ErrReachedLimitMcycle) + s.Require().ErrorIs(err, ErrMcycleOverflow) + expectedRequestedSpan := test.configuredMax + if expectedRequestedSpan == 0 { + expectedRequestedSpan = model.MaxExecutionCycleSpan + } + s.Contains(err.Error(), fmt.Sprintf("requested_span=%d", expectedRequestedSpan)) + s.Contains(err.Error(), "target_span=0") + s.Contains(err.Error(), "executed_cycles=0") + backend.AssertExpectations(s.T()) + }) + } +} + +func (s *ImplementationSuite) TestRunRejectsNonOverflowReasonAtMachineMaximum() { + for _, breakReason := range []BreakReason{ReachedTargetMcycle, YieldedSoftly} { + s.Run(fmt.Sprintf("break reason %d", breakReason), func() { + backend := NewMockBackend() + backend.On("Run", ^uint64(0), mock.AnythingOfType("time.Duration")). + Return(breakReason, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(^uint64(0), nil).Once() + machine := &machineImpl{backend: backend, logger: s.logger, params: model.ExecutionParameters{ + FastDeadline: time.Second, + }} + + result, err := machine.runIncrementInterval( + context.Background(), ^uint64(0), ^uint64(0), 1, nil, time.Second, + ) + + s.Equal(breakReason, result.breakReason) + s.Equal(^uint64(0), result.currentCycle) + s.Require().ErrorIs(err, ErrMachineInternal) + s.Contains(err.Error(), "instead of mcycle overflow at MaxUint64") + backend.AssertExpectations(s.T()) + }) + } +} + +func (s *ImplementationSuite) TestInspectInheritedMcycleOverflowDoesNotClaimLocalLimitExhaustion() { + const start = uint64(100) + + for _, test := range []struct { + name string + configuredMax uint64 + requestedSpan uint64 + executedCycles uint64 + stopDescription string + }{ + { + name: "before configured limit", + configuredMax: 50, + requestedSpan: 50, + executedCycles: 7, + stopDescription: "stopped before configured cycle limit", + }, + { + name: "at configured limit", + configuredMax: 7, + requestedSpan: 7, + executedCycles: 7, + stopDescription: "stopped at machine imcyclemax coincident with configured target", + }, + { + name: "before zero default fixed limit", + requestedSpan: model.MaxExecutionCycleSpan, + executedCycles: 7, + stopDescription: "stopped before fixed cycle limit", + }, + } { + s.Run(test.name, func() { + backend := NewMockBackend() + backend.On("CmioRxBufferSize").Return(uint64(1024)) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start, nil).Once() + backend.On( + "SendCmioResponse", + uint16(InspectStateRequest), []byte("query"), nil, mock.AnythingOfType("time.Duration"), + ).Return(nil).Once() + backend.On("Run", start+test.requestedSpan, mock.AnythingOfType("time.Duration")). + Return(McycleOverflow, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(start+test.executedCycles, nil).Once() + + machine := &machineImpl{backend: backend, logger: s.logger, params: model.ExecutionParameters{ + FastDeadline: time.Second, + InspectIncCycles: ^uint64(0), + InspectIncDeadline: time.Second, + InspectMaxDeadline: time.Second, + InspectMaxCycles: test.configuredMax, + }} + + response, err := machine.Inspect(context.Background(), []byte("query")) + s.Require().NotNil(response) + s.Equal(CompletionStatusUnknown, response.Status) + s.Empty(response.Reports) + s.Require().ErrorIs(err, ErrReachedLimitMcycle) + s.Require().ErrorIs(err, ErrMcycleOverflow) + s.Contains(err.Error(), test.stopDescription) + s.Contains(err.Error(), fmt.Sprintf("requested_span=%d", test.requestedSpan)) + s.Contains(err.Error(), fmt.Sprintf("executed_cycles=%d", test.executedCycles)) + backend.AssertExpectations(s.T()) + }) + } +} + +func (s *ImplementationSuite) TestExecutionCycleBoundsSaturatesAtMachineMaximum() { + for _, test := range []struct { + name string + start uint64 + configured uint64 + }{ + { + name: "configured cap beyond representable range", + start: ^uint64(0) - 10, configured: 11, + }, + { + name: "zero cap", + start: ^uint64(0) - model.MaxExecutionCycleSpan + 1, configured: 0, + }, + { + name: "maximum cap", + start: ^uint64(0) - model.MaxExecutionCycleSpan + 1, + configured: model.MaxExecutionCycleSpan, + }, + } { + s.Run(test.name, func() { + backend := NewMockBackend() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(test.start, nil) + machine := &machineImpl{backend: backend, logger: s.logger, params: model.ExecutionParameters{ + AdvanceIncCycles: 1, + AdvanceMaxCycles: test.configured, + }} + + bounds, err := machine.executionCycleBounds(context.Background(), AdvanceStateRequest) + s.Require().NoError(err) + s.Equal(test.start, bounds.start) + s.Equal(^uint64(0), bounds.limit) + backend.AssertExpectations(s.T()) + }) + } +} + +func (s *ImplementationSuite) TestExecutionCycleBoundsUsesFullWindowWhenUncapped() { + + s.Run("window endpoint is start plus 2^48 minus 1", func() { + const start = uint64(1000) + backend := NewMockBackend() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(start, nil) + machine := &machineImpl{backend: backend, logger: s.logger, params: model.ExecutionParameters{ + AdvanceIncCycles: 1, + }} + + bounds, err := machine.executionCycleBounds(context.Background(), AdvanceStateRequest) + s.Require().NoError(err) + s.Equal(start, bounds.start) + s.Equal(start+model.MaxExecutionCycleSpan, bounds.limit) + backend.AssertExpectations(s.T()) + }) +} + +func (s *ImplementationSuite) TestExecutionResponseLimitErrorsExposeOnlyCounts() { + for _, test := range []struct { + kind string + count int + capacity int + sentinel error + }{ + {kind: "output", count: maxOutputs + 1, capacity: maxOutputs, sentinel: ErrOutputsLimitExceeded}, + {kind: "report", count: maxReports + 1, capacity: maxReports, sentinel: ErrReportsLimitExceeded}, + } { + err := executionResponseLimitError( + AdvanceStateRequest, test.kind, test.count, test.capacity, test.sentinel, + ) + s.Require().ErrorIs(err, test.sentinel) + s.Contains(err.Error(), fmt.Sprintf("advance %s count %d", test.kind, test.count)) + s.Contains(err.Error(), fmt.Sprintf("capacity %d", test.capacity)) + } +} + +func (s *ImplementationSuite) TestInspectUsesConfiguredCycleIncrement() { + require := s.Require() + const startCycle uint64 = 7 + const inspectIncCycles uint64 = 137 + + mockBackend := NewMockBackend() + mockBackend.On("CmioRxBufferSize").Return(uint64(1024)) + mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(startCycle, nil).Once() + mockBackend.On( + "SendCmioResponse", + uint16(InspectStateRequest), mock.Anything, mock.Anything, mock.AnythingOfType("time.Duration"), + ).Return(nil) + mockBackend.On("Run", startCycle+inspectIncCycles, mock.AnythingOfType("time.Duration")).Return(YieldedManually, nil) + mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(startCycle+inspectIncCycles, nil).Once() + mockBackend.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( + uint8(0), uint16(ManualYieldReasonAccepted), []byte(nil), nil) + + machine := &machineImpl{ + backend: mockBackend, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second, + InspectIncCycles: inspectIncCycles, + InspectIncDeadline: time.Second, + InspectMaxDeadline: time.Second, + }, + } + + response, err := machine.Inspect(context.Background(), []byte("query")) + require.NoError(err) + require.Equal(CompletionStatusAccepted, response.Status) + mockBackend.AssertExpectations(s.T()) +} + // Test Inspect method func (s *ImplementationSuite) TestInspect() { require := s.Require() @@ -381,17 +1381,16 @@ func (s *ImplementationSuite) TestInspect() { params: model.ExecutionParameters{ FastDeadline: time.Second * 5, InspectIncCycles: 100, - InspectMaxCycles: 1000, InspectIncDeadline: time.Second * 1, InspectMaxDeadline: time.Second * 10, }, } query := []byte("test query") - accepted, reports, err := machine.Inspect(ctx, query) + response, err := machine.Inspect(ctx, query) require.NoError(err) - require.True(accepted) - require.Empty(reports) + require.Equal(CompletionStatusAccepted, response.Status) + require.Empty(response.Reports) mockBackend.AssertExpectations(s.T()) // Test inspect with rejection @@ -403,15 +1402,14 @@ func (s *ImplementationSuite) TestInspect() { params: model.ExecutionParameters{ FastDeadline: time.Second * 5, InspectIncCycles: 100, - InspectMaxCycles: 1000, InspectIncDeadline: time.Second * 1, InspectMaxDeadline: time.Second * 10, }, } - accepted, reports, err = machine2.Inspect(ctx, query) + response, err = machine2.Inspect(ctx, query) require.NoError(err) - require.False(accepted) - require.Empty(reports) + require.Equal(CompletionStatusRejected, response.Status) + require.Empty(response.Reports) mockBackend2.AssertExpectations(s.T()) // Test inspect with exception @@ -423,17 +1421,79 @@ func (s *ImplementationSuite) TestInspect() { params: model.ExecutionParameters{ FastDeadline: time.Second * 5, InspectIncCycles: 100, - InspectMaxCycles: 1000, InspectIncDeadline: time.Second * 1, InspectMaxDeadline: time.Second * 10, }, } - accepted, reports, err = machine3.Inspect(ctx, query) - require.ErrorIs(err, ErrException) - require.False(accepted) - require.Empty(reports) + response, err = machine3.Inspect(ctx, query) + require.NoError(err) + require.Equal(CompletionStatusException, response.Status) + require.Equal([]byte("exception data"), response.ExceptionData) + require.Empty(response.Reports) mockBackend3.AssertExpectations(s.T()) + // A halt is a completed guest outcome. Reports emitted before the halt are + // part of that completed inspect response. + mockBackendHalted := NewMockBackend() + mockBackendHalted.On("CmioRxBufferSize").Return(uint64(1024)) + mockBackendHalted.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(0), nil).Once() + mockBackendHalted.On("SendCmioResponse", uint16(InspectStateRequest), query, nil, + mock.AnythingOfType("time.Duration")).Return(nil).Once() + mockBackendHalted.On("Run", uint64(100), mock.AnythingOfType("time.Duration")). + Return(YieldedAutomatically, nil).Once() + mockBackendHalted.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(50), nil).Once() + mockBackendHalted.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")). + Return(uint8(0), uint16(AutomaticYieldReasonReport), []byte("before halt"), nil).Once() + mockBackendHalted.On("Run", uint64(150), mock.AnythingOfType("time.Duration")). + Return(Halted, nil).Once() + mockBackendHalted.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(75), nil).Once() + machineHalted := &machineImpl{ + backend: mockBackendHalted, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second * 5, + InspectIncCycles: 100, + InspectIncDeadline: time.Second, + InspectMaxDeadline: time.Second * 10, + }, + } + response, err = machineHalted.Inspect(ctx, query) + require.NoError(err) + require.Equal(CompletionStatusHalted, response.Status) + require.Equal([]Report{[]byte("before halt")}, response.Reports) + mockBackendHalted.AssertExpectations(s.T()) + + // An internal stop is not a completed guest outcome. It returns Unknown + // with the partial report prefix and a typed execution error. + mockBackendFailed := NewMockBackend() + mockBackendFailed.On("CmioRxBufferSize").Return(uint64(1024)) + mockBackendFailed.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(0), nil).Once() + mockBackendFailed.On("SendCmioResponse", uint16(InspectStateRequest), query, nil, + mock.AnythingOfType("time.Duration")).Return(nil).Once() + mockBackendFailed.On("Run", uint64(100), mock.AnythingOfType("time.Duration")). + Return(YieldedAutomatically, nil).Once() + mockBackendFailed.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(50), nil).Once() + mockBackendFailed.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")). + Return(uint8(0), uint16(AutomaticYieldReasonReport), []byte("partial"), nil).Once() + mockBackendFailed.On("Run", uint64(150), mock.AnythingOfType("time.Duration")). + Return(Failed, nil).Once() + mockBackendFailed.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(75), nil).Once() + machineFailed := &machineImpl{ + backend: mockBackendFailed, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second * 5, + InspectIncCycles: 100, + InspectIncDeadline: time.Second, + InspectMaxDeadline: time.Second * 10, + }, + } + response, err = machineFailed.Inspect(ctx, query) + require.ErrorIs(err, ErrMachineInternal) + require.Equal(CompletionStatusUnknown, response.Status) + require.Equal([]Report{[]byte("partial")}, response.Reports) + mockBackendFailed.AssertExpectations(s.T()) + // Test inspect with payload too large mockBackend4 := NewMockBackend() mockBackend4.On("CmioRxBufferSize").Return(uint64(5)) @@ -443,15 +1503,342 @@ func (s *ImplementationSuite) TestInspect() { params: model.ExecutionParameters{ FastDeadline: time.Second * 5, InspectIncCycles: 100, - InspectMaxCycles: 1000, InspectIncDeadline: time.Second * 1, InspectMaxDeadline: time.Second * 10, }, } largeQuery := make([]byte, 10) - _, _, err = machine4.Inspect(ctx, largeQuery) + response, err = machine4.Inspect(ctx, largeQuery) + require.NotNil(response) + require.Equal(CompletionStatusUnknown, response.Status) require.ErrorIs(err, ErrPayloadLengthLimitExceeded) mockBackend4.AssertExpectations(s.T()) + + // Inspect uses the same saturating target as advance. Reaching MaxUint64 is + // reported as a machine overflow and never as a completed input result. + mockBackendOverflow := NewMockBackend() + mockBackendOverflow.On("CmioRxBufferSize").Return(uint64(1024)) + mockBackendOverflow.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(^uint64(0)-uint64(999), nil).Once() + mockBackendOverflow.On("SendCmioResponse", uint16(InspectStateRequest), query, nil, + mock.AnythingOfType("time.Duration")).Return(nil).Once() + mockBackendOverflow.On("Run", ^uint64(0), mock.AnythingOfType("time.Duration")). + Return(McycleOverflow, nil).Once() + mockBackendOverflow.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(^uint64(0), nil).Once() + machineOverflow := &machineImpl{ + backend: mockBackendOverflow, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second, + InspectIncCycles: ^uint64(0), + InspectIncDeadline: time.Second, + InspectMaxDeadline: time.Second, + InspectMaxCycles: 1000, + }, + } + response, err = machineOverflow.Inspect(ctx, query) + require.NotNil(response) + require.Equal(CompletionStatusUnknown, response.Status) + require.Empty(response.Reports) + require.ErrorIs(err, ErrReachedLimitMcycle) + require.ErrorIs(err, ErrMcycleOverflow) + require.Contains(err.Error(), "requested_span=1000") + require.Contains(err.Error(), "target_span=999") + require.Contains(err.Error(), "executed_cycles=999") + mockBackendOverflow.AssertExpectations(s.T()) +} + +func (s *ImplementationSuite) TestAdvanceCanonicalizesTerminalBoundaryHash() { + for _, test := range []struct { + name string + yieldReason manualYieldReason + status CompletionStatus + terminalAtBoundary bool + sameRunPriorBoundary bool + immediateYield bool + }{ + { + name: "accepted at first boundary", + yieldReason: ManualYieldReasonAccepted, + status: CompletionStatusAccepted, + terminalAtBoundary: true, + }, + { + name: "rejected at first boundary", + yieldReason: ManualYieldReasonRejected, + status: CompletionStatusRejected, + terminalAtBoundary: true, + }, + { + name: "exception at first boundary", + yieldReason: ManualYieldReasonException, + status: CompletionStatusException, + terminalAtBoundary: true, + }, + { + name: "off-boundary terminal keeps prior boundary", + yieldReason: ManualYieldReasonAccepted, + status: CompletionStatusAccepted, + terminalAtBoundary: false, + }, + { + name: "same run prior boundary survives off-boundary terminal", + yieldReason: ManualYieldReasonAccepted, + status: CompletionStatusAccepted, + sameRunPriorBoundary: true, + }, + { + name: "immediate off-boundary yield drops its fixed-point sample", + yieldReason: ManualYieldReasonAccepted, + status: CompletionStatusAccepted, + immediateYield: true, + }, + } { + s.Run(test.name, func() { + collectedHash := randomFakeHash() + terminalSample := randomFakeHash() + outputsHash := randomFakeHash() + + backend := NewMockBackend() + backend.On("CmioRxBufferSize").Return(uint64(1024)) + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(uint64(0), nil).Once() + backend.On( + "SendCmioResponse", + uint16(AdvanceStateRequest), []byte("input"), mock.Anything, mock.AnythingOfType("time.Duration"), + ). + Return(nil).Once() + switch { + case test.terminalAtBoundary: + backend.On( + "RunAndCollectRootHashes", + MCycleComputationHashPeriod, + mock.AnythingOfType("*machine.HashCollectorState"), + mock.AnythingOfType("time.Duration"), + ).Run(func(args mock.Arguments) { + state := args.Get(1).(*HashCollectorState) + state.Hashes = append(state.Hashes, collectedHash) + state.MCyclePhase = 0 + }).Return(YieldedManually, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(MCycleComputationHashPeriod, nil).Once() + case test.sameRunPriorBoundary: + // RunAndCollectRootHashes may cross and collect a prior + // boundary, then stop at a manual yield one cycle later in the + // same call. Emulator 0.21 also appends the off-boundary + // fixed-point sample; only that final entry may be dropped. + backend.On( + "RunAndCollectRootHashes", + 2*MCycleComputationHashPeriod, + mock.AnythingOfType("*machine.HashCollectorState"), + mock.AnythingOfType("time.Duration"), + ).Run(func(args mock.Arguments) { + state := args.Get(1).(*HashCollectorState) + state.Hashes = append(state.Hashes, collectedHash, terminalSample) + state.MCyclePhase = 1 + }).Return(YieldedManually, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(MCycleComputationHashPeriod+1, nil).Once() + case test.immediateYield: + // Even a yield right after delivery is a fixed point: 0.21 + // appends its sample at a nonzero phase, and the node must + // still drop it. + backend.On( + "RunAndCollectRootHashes", + MCycleComputationHashPeriod, + mock.AnythingOfType("*machine.HashCollectorState"), + mock.AnythingOfType("time.Duration"), + ).Run(func(args mock.Arguments) { + state := args.Get(1).(*HashCollectorState) + state.Hashes = append(state.Hashes, terminalSample) + state.MCyclePhase = 1 + }).Return(YieldedManually, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(uint64(1), nil).Once() + default: + // The first call reaches and records a real prior boundary. + backend.On( + "RunAndCollectRootHashes", + MCycleComputationHashPeriod, + mock.AnythingOfType("*machine.HashCollectorState"), + mock.AnythingOfType("time.Duration"), + ).Run(func(args mock.Arguments) { + state := args.Get(1).(*HashCollectorState) + state.Hashes = append(state.Hashes, collectedHash) + state.MCyclePhase = 0 + }).Return(ReachedTargetMcycle, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(MCycleComputationHashPeriod, nil).Once() + // The second call manually yields one cycle later; 0.21 appends + // the off-boundary fixed-point sample, which is dropped while + // the prior boundary remains in the collection. + backend.On( + "RunAndCollectRootHashes", + 2*MCycleComputationHashPeriod, + mock.AnythingOfType("*machine.HashCollectorState"), + mock.AnythingOfType("time.Duration"), + ).Run(func(args mock.Arguments) { + state := args.Get(1).(*HashCollectorState) + state.Hashes = append(state.Hashes, terminalSample) + state.MCyclePhase = 1 + }).Return(YieldedManually, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(MCycleComputationHashPeriod+1, nil).Once() + } + backend.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")). + Return(uint8(0), uint16(test.yieldReason), outputsHash[:], nil).Once() + + advanceIncrement := MCycleComputationHashPeriod + if test.sameRunPriorBoundary { + advanceIncrement = 2 * MCycleComputationHashPeriod + } + machine := &machineImpl{ + backend: backend, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second, + AdvanceIncCycles: advanceIncrement, + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: time.Second, + }, + } + + response, err := machine.Advance(context.Background(), []byte("input"), Hash{}, true) + s.Require().NoError(err) + s.Require().NotNil(response) + s.Equal(test.status, response.Status) + if test.terminalAtBoundary || test.immediateYield { + s.Empty(response.PeriodicStateHashes) + s.Equal(InputEntryCapacity, response.PaddingRepetitions) + } else { + s.Equal([]Hash{collectedHash}, response.PeriodicStateHashes) + s.Equal(InputEntryCapacity-1, response.PaddingRepetitions) + } + backend.AssertExpectations(s.T()) + }) + } +} + +func (s *ImplementationSuite) TestRunDiscardsOverflowCollection() { + const fixedEndpoint uint64 = model.MaxExecutionCycleSpan + const startCycle uint64 = fixedEndpoint - mcycleComputationHashChunkSize + terminalSample := randomFakeHash() + backend := NewMockBackend() + backend.On( + "RunAndCollectRootHashes", + fixedEndpoint, + mock.AnythingOfType("*machine.HashCollectorState"), + mock.AnythingOfType("time.Duration"), + ).Run(func(args mock.Arguments) { + state := args.Get(1).(*HashCollectorState) + state.Hashes = append(state.Hashes, terminalSample) + state.MCyclePhase = 7 + }).Return(McycleOverflow, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(fixedEndpoint, nil).Once() + machine := &machineImpl{ + backend: backend, + logger: s.logger, + params: model.ExecutionParameters{ + AdvanceIncCycles: fixedEndpoint, + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: time.Second, + }, + } + + result, err := machine.run( + context.Background(), AdvanceStateRequest, true, testExecutionBounds(startCycle, fixedEndpoint), + ) + + s.Require().ErrorIs(err, ErrMcycleOverflow) + s.Empty(result.outputs) + s.Empty(result.reports) + s.Empty(result.periodicStateHashes) + s.Zero(result.paddingRepetitions) + backend.AssertExpectations(s.T()) +} + +func (s *ImplementationSuite) TestRunCapsCollectorChunksAndPreservesCollectionState() { + firstPeriodicHash := randomFakeHash() + terminalHash := randomFakeHash() + backend := NewMockBackend() + backend.On( + "RunAndCollectRootHashes", + mcycleComputationHashChunkSize, + mock.AnythingOfType("*machine.HashCollectorState"), + mock.AnythingOfType("time.Duration"), + ).Run(func(args mock.Arguments) { + state := args.Get(1).(*HashCollectorState) + s.Zero(state.MCyclePhase) + s.Empty(state.Hashes) + state.Hashes = append(state.Hashes, firstPeriodicHash) + state.MCyclePhase = 7 + }).Return(ReachedTargetMcycle, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(mcycleComputationHashChunkSize, nil).Once() + backend.On( + "RunAndCollectRootHashes", + 2*mcycleComputationHashChunkSize, + mock.AnythingOfType("*machine.HashCollectorState"), + mock.AnythingOfType("time.Duration"), + ).Run(func(args mock.Arguments) { + state := args.Get(1).(*HashCollectorState) + s.Equal(uint64(7), state.MCyclePhase) + s.Equal([]Hash{firstPeriodicHash}, state.Hashes) + state.Hashes = append(state.Hashes, terminalHash) + state.MCyclePhase = 11 + }).Return(YieldedManually, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(2*mcycleComputationHashChunkSize, nil).Once() + machine := &machineImpl{ + backend: backend, + logger: s.logger, + params: model.ExecutionParameters{ + AdvanceIncCycles: 3 * mcycleComputationHashChunkSize, + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: time.Second, + }, + } + + result, err := machine.run( + context.Background(), AdvanceStateRequest, true, + testExecutionBounds(0, 3*mcycleComputationHashChunkSize), + ) + + s.Require().NoError(err) + s.Equal([]Hash{firstPeriodicHash}, result.periodicStateHashes) + s.Equal(InputEntryCapacity-1, result.paddingRepetitions) + backend.AssertExpectations(s.T()) +} + +func (s *ImplementationSuite) TestRunRejectsFixedPointWithoutAppendedHash() { + backend := NewMockBackend() + backend.On( + "RunAndCollectRootHashes", + MCycleComputationHashPeriod, + mock.AnythingOfType("*machine.HashCollectorState"), + mock.AnythingOfType("time.Duration"), + ).Return(YieldedManually, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(MCycleComputationHashPeriod, nil).Once() + machine := &machineImpl{ + backend: backend, + logger: s.logger, + params: model.ExecutionParameters{ + AdvanceIncCycles: MCycleComputationHashPeriod, + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: time.Second, + }, + } + + _, err := machine.run( + context.Background(), AdvanceStateRequest, true, + testExecutionBounds(0, model.MaxExecutionCycleSpan), + ) + + s.Require().ErrorIs(err, ErrMachineInternal) + s.Contains(err.Error(), "without appending its state hash") + backend.AssertExpectations(s.T()) } // Test Store method @@ -591,7 +1978,7 @@ func (s *ImplementationSuite) TestHelperMethods() { require.ErrorIs(err, ErrMachineInternal) mockBackend2.AssertExpectations(s.T()) - // Test wasLastRequestAccepted + // Test readManualYieldResult mockBackend3 := NewMockBackend() expectedHash3 := randomFakeHash() mockBackend3.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( @@ -603,10 +1990,10 @@ func (s *ImplementationSuite) TestHelperMethods() { FastDeadline: time.Second * 5, }, } - accepted, data, err := machine3.wasLastRequestAccepted(ctx) + manualResult, err := machine3.readManualYieldResult(ctx) require.NoError(err) - require.True(accepted) - require.NotNil(data) + require.Equal(CompletionStatusAccepted, manualResult.status) + require.NotNil(manualResult.data) mockBackend3.AssertExpectations(s.T()) mockBackend4 := NewMockBackend() @@ -620,10 +2007,10 @@ func (s *ImplementationSuite) TestHelperMethods() { FastDeadline: time.Second * 5, }, } - accepted, data, err = machine4.wasLastRequestAccepted(ctx) + manualResult, err = machine4.readManualYieldResult(ctx) require.NoError(err) - require.False(accepted) - require.NotNil(data) + require.Equal(CompletionStatusRejected, manualResult.status) + require.NotNil(manualResult.data) mockBackend4.AssertExpectations(s.T()) mockBackend5 := NewMockBackend() @@ -636,10 +2023,10 @@ func (s *ImplementationSuite) TestHelperMethods() { FastDeadline: time.Second * 5, }, } - accepted, data, err = machine5.wasLastRequestAccepted(ctx) - require.ErrorIs(err, ErrException) - require.False(accepted) - require.NotNil(data) + manualResult, err = machine5.readManualYieldResult(ctx) + require.NoError(err) + require.Equal(CompletionStatusException, manualResult.status) + require.NotNil(manualResult.data) mockBackend5.AssertExpectations(s.T()) // Test readMCycle @@ -678,7 +2065,7 @@ func (s *ImplementationSuite) TestHelperMethods() { _, err = machine.isAtManualYield(canceledCtx) require.ErrorIs(err, ErrCanceled) - _, _, err = machine3.wasLastRequestAccepted(canceledCtx) + _, err = machine3.readManualYieldResult(canceledCtx) require.ErrorIs(err, ErrCanceled) _, err = machine6.readMCycle(canceledCtx) @@ -700,34 +2087,37 @@ func (s *ImplementationSuite) TestRun() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, }, } - outputs, reports, _, _, err := machine.run(ctx, AdvanceStateRequest, false) + result, err := machine.run( + ctx, AdvanceStateRequest, false, testExecutionBounds(0, model.MaxExecutionCycleSpan), + ) require.NoError(err) - require.Empty(outputs) - require.Empty(reports) + require.Empty(result.outputs) + require.Empty(result.reports) mockBackend.AssertExpectations(s.T()) // Test run with read cycle error mockBackend2 := NewMockBackend() + mockBackend2.On("Run", uint64(100), mock.AnythingOfType("time.Duration")).Return(YieldedManually, nil) mockBackend2.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(0), errors.New("read cycle failed")) machine2 := &machineImpl{ backend: mockBackend2, logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, }, } - _, _, _, _, err = machine2.run(ctx, AdvanceStateRequest, false) + _, err = machine2.run( + ctx, AdvanceStateRequest, false, testExecutionBounds(0, model.MaxExecutionCycleSpan), + ) require.Error(err) require.Contains(err.Error(), "read cycle failed") mockBackend2.AssertExpectations(s.T()) @@ -743,14 +2133,15 @@ func (s *ImplementationSuite) TestRun() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, }, } - _, _, _, _, err = machine3.run(ctx, AdvanceStateRequest, false) + _, err = machine3.run( + ctx, AdvanceStateRequest, false, testExecutionBounds(0, model.MaxExecutionCycleSpan), + ) require.NoError(err) mockBackend3.AssertExpectations(s.T()) @@ -766,14 +2157,15 @@ func (s *ImplementationSuite) TestRun() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, }, } - _, _, _, _, err = machine4.run(ctx, AdvanceStateRequest, false) + _, err = machine4.run( + ctx, AdvanceStateRequest, false, testExecutionBounds(0, model.MaxExecutionCycleSpan), + ) require.Error(err) require.Contains(err.Error(), "could not read output/report") require.Contains(err.Error(), "cmio request failed") @@ -794,18 +2186,19 @@ func (s *ImplementationSuite) TestRun() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, }, } - outputs5, reports5, _, _, err := machine5.run(ctx, AdvanceStateRequest, false) + result5, err := machine5.run( + ctx, AdvanceStateRequest, false, testExecutionBounds(0, model.MaxExecutionCycleSpan), + ) require.NoError(err) - require.Len(outputs5, 1) - require.Equal([]byte("output data"), []byte(outputs5[0])) - require.Empty(reports5) + require.Len(result5.outputs, 1) + require.Equal([]byte("output data"), []byte(result5.outputs[0])) + require.Empty(result5.reports) mockBackend5.AssertExpectations(s.T()) // Test run with automatic yield producing report then manual yield @@ -823,143 +2216,107 @@ func (s *ImplementationSuite) TestRun() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, }, } - outputs6, reports6, _, _, err := machine6.run(ctx, AdvanceStateRequest, false) + result6, err := machine6.run( + ctx, AdvanceStateRequest, false, testExecutionBounds(0, model.MaxExecutionCycleSpan), + ) require.NoError(err) - require.Empty(outputs6) - require.Len(reports6, 1) - require.Equal([]byte("report data"), []byte(reports6[0])) + require.Empty(result6.outputs) + require.Len(result6.reports, 1) + require.Equal([]byte("report data"), []byte(result6.reports[0])) mockBackend6.AssertExpectations(s.T()) } -// Test step method -func (s *ImplementationSuite) TestStep() { - require := s.Require() - ctx := context.Background() - - machine := &machineImpl{ - backend: nil, // Will be set per test - logger: s.logger, - params: model.ExecutionParameters{ - AdvanceIncCycles: 100, - }, +func (s *ImplementationSuite) TestRunIncrementIntervalPreservesBreakReason() { + for _, test := range []struct { + name string + breakReason BreakReason + currentCycle uint64 + }{ + {"manual yield", YieldedManually, 150}, + {"automatic yield", YieldedAutomatically, 200}, + {"soft yield", YieldedSoftly, 150}, + {"target", ReachedTargetMcycle, 200}, + {"mcycle overflow", McycleOverflow, 199}, + {"halt", Halted, 175}, + {"failure", Failed, 160}, + } { + s.Run(test.name, func() { + backend := NewMockBackend() + backend.On("Run", uint64(200), mock.AnythingOfType("time.Duration")). + Return(test.breakReason, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(test.currentCycle, nil).Once() + machine := &machineImpl{backend: backend, logger: s.logger} + + result, err := machine.runIncrementInterval( + context.Background(), 100, 1000, 100, nil, time.Second, + ) + + s.Require().NoError(err) + s.Equal(test.breakReason, result.breakReason) + s.Equal(test.currentCycle, result.currentCycle) + backend.AssertExpectations(s.T()) + }) } +} - // Test step with manual yield - mockBackend := NewMockBackend() - mockBackend.On("Run", mock.AnythingOfType("uint64"), mock.AnythingOfType("time.Duration")).Return(YieldedManually, nil) - mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(150), nil) - machine.backend = mockBackend - - yieldType, cycle, err := machine.runIncrementInterval(ctx, 100, 1000, nil, time.Second) - require.NoError(err) - require.NotNil(yieldType) - require.Equal(ManualYield, *yieldType) - require.Equal(uint64(150), cycle) - mockBackend.AssertExpectations(s.T()) - - // Test runIncrementInterval with automatic yield - mockBackend2 := NewMockBackend() - mockBackend2.On("Run", mock.AnythingOfType("uint64"), mock.AnythingOfType("time.Duration")).Return(YieldedAutomatically, nil) - mockBackend2.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(200), nil) - machine.backend = mockBackend2 - - yieldType, cycle, err = machine.runIncrementInterval(ctx, 100, 1000, nil, time.Second) - require.NoError(err) - require.NotNil(yieldType) - require.Equal(AutomaticYield, *yieldType) - require.Equal(uint64(200), cycle) - mockBackend2.AssertExpectations(s.T()) - - // Test runIncrementInterval with soft yield (no yield) - mockBackend3 := NewMockBackend() - mockBackend3.On("Run", mock.AnythingOfType("uint64"), mock.AnythingOfType("time.Duration")).Return(YieldedSoftly, nil) - mockBackend3.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(150), nil) - machine.backend = mockBackend3 - - yieldType, cycle, err = machine.runIncrementInterval(ctx, 100, 1000, nil, time.Second) - require.NoError(err) - require.Nil(yieldType) - require.Equal(uint64(150), cycle) - mockBackend3.AssertExpectations(s.T()) - - // Test runIncrementInterval with reached target mcycle - mockBackend4 := NewMockBackend() - mockBackend4.On("Run", mock.AnythingOfType("uint64"), mock.AnythingOfType("time.Duration")).Return(ReachedTargetMcycle, nil) - mockBackend4.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(1000), nil) - machine.backend = mockBackend4 - - yieldType, cycle, err = machine.runIncrementInterval(ctx, 100, 1000, nil, time.Second) - require.ErrorIs(err, ErrReachedTargetMcycle) - require.Nil(yieldType) - require.Equal(uint64(1000), cycle) - mockBackend4.AssertExpectations(s.T()) - - // Test runIncrementInterval with mcycle overflow - mockBackendOverflow := NewMockBackend() - mockBackendOverflow.On( - "Run", - mock.AnythingOfType("uint64"), - mock.AnythingOfType("time.Duration"), - ).Return(McycleOverflow, nil) - mockBackendOverflow.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(999), nil) - machine.backend = mockBackendOverflow - - yieldType, cycle, err = machine.runIncrementInterval(ctx, 100, 1000, nil, time.Second) - require.ErrorIs(err, ErrReachedLimitMcycle) - require.NotErrorIs(err, ErrMachineInternal) - require.Nil(yieldType) - require.Equal(uint64(999), cycle) - mockBackendOverflow.AssertExpectations(s.T()) - - // Test runIncrementInterval with halted - mockBackend5 := NewMockBackend() - mockBackend5.On("Run", mock.AnythingOfType("uint64"), mock.AnythingOfType("time.Duration")).Return(Halted, nil) - mockBackend5.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(500), nil) - machine.backend = mockBackend5 - - yieldType, cycle, err = machine.runIncrementInterval(ctx, 100, 1000, nil, time.Second) - require.ErrorIs(err, ErrHalted) - require.Nil(yieldType) - require.Equal(uint64(500), cycle) - - // Test runIncrementInterval already at limit cycle - yieldType, cycle, err = machine.runIncrementInterval(ctx, 1000, 1000, nil, time.Second) - require.ErrorIs(err, ErrReachedLimitMcycle) - require.Nil(yieldType) - require.Equal(uint64(0), cycle) - mockBackend5.AssertExpectations(s.T()) - - // Test runIncrementInterval with backend run error - mockBackend6 := NewMockBackend() - mockBackend6.On("Run", - mock.AnythingOfType("uint64"), - mock.AnythingOfType("time.Duration"), - ).Return(BreakReason(0), errors.New("run failed")) - machine.backend = mockBackend6 - yieldType, _, err = machine.runIncrementInterval(ctx, 100, 1000, nil, time.Second) - require.Error(err) - require.Contains(err.Error(), "run failed") - require.Nil(yieldType) - mockBackend6.AssertExpectations(s.T()) - - // Test runIncrementInterval with read cycle error - mockBackend7 := NewMockBackend() - mockBackend7.On("Run", mock.AnythingOfType("uint64"), mock.AnythingOfType("time.Duration")).Return(YieldedManually, nil) - mockBackend7.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(0), errors.New("read cycle failed")) - machine.backend = mockBackend7 - yieldType, _, err = machine.runIncrementInterval(ctx, 100, 1000, nil, time.Second) - require.Error(err) - require.Contains(err.Error(), "read cycle failed") - require.Nil(yieldType) - mockBackend7.AssertExpectations(s.T()) +func (s *ImplementationSuite) TestRunIncrementIntervalErrors() { + s.Run("already at limit", func() { + machine := &machineImpl{backend: NewMockBackend(), logger: s.logger} + result, err := machine.runIncrementInterval( + context.Background(), 1000, 1000, 100, nil, time.Second, + ) + s.Require().ErrorIs(err, ErrReachedLimitMcycle) + s.Equal(uint64(1000), result.currentCycle) + }) + + s.Run("backend error", func() { + backend := NewMockBackend() + backend.On("Run", uint64(200), mock.AnythingOfType("time.Duration")). + Return(Failed, errors.New("run failed")).Once() + machine := &machineImpl{backend: backend, logger: s.logger} + _, err := machine.runIncrementInterval( + context.Background(), 100, 1000, 100, nil, time.Second, + ) + s.Require().ErrorContains(err, "run failed") + backend.AssertExpectations(s.T()) + }) + + s.Run("read cycle error", func() { + backend := NewMockBackend() + backend.On("Run", uint64(200), mock.AnythingOfType("time.Duration")). + Return(YieldedManually, nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(uint64(0), errors.New("read cycle failed")).Once() + machine := &machineImpl{backend: backend, logger: s.logger} + _, err := machine.runIncrementInterval( + context.Background(), 100, 1000, 100, nil, time.Second, + ) + s.Require().ErrorContains(err, "read cycle failed") + backend.AssertExpectations(s.T()) + }) + + s.Run("unknown break reason", func() { + backend := NewMockBackend() + backend.On("Run", uint64(200), mock.AnythingOfType("time.Duration")). + Return(BreakReason(99), nil).Once() + backend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(uint64(150), nil).Once() + machine := &machineImpl{backend: backend, logger: s.logger} + result, err := machine.runIncrementInterval( + context.Background(), 100, 1000, 100, nil, time.Second, + ) + s.Require().ErrorIs(err, ErrMachineInternal) + s.Equal(BreakReason(99), result.breakReason) + backend.AssertExpectations(s.T()) + }) } // Test process method @@ -982,7 +2339,6 @@ func (s *ImplementationSuite) TestProcess() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, @@ -990,14 +2346,47 @@ func (s *ImplementationSuite) TestProcess() { } input := []byte("test input") - accepted, outputs, reports, _, _, data, err := machine.process(ctx, input, AdvanceStateRequest, &expectedHash, false) + result, err := machine.process(ctx, input, AdvanceStateRequest, &expectedHash, false) require.NoError(err) - require.True(accepted) - require.Empty(outputs) - require.Empty(reports) - require.NotNil(data) + require.Equal(CompletionStatusAccepted, result.completion.status) + require.Empty(result.outputs) + require.Empty(result.reports) + require.NotNil(result.completion.data) mockBackend.AssertExpectations(s.T()) + // A halt completes the request without producing a CMIO manual yield. + haltedBackend := NewMockBackend() + haltedBackend.On("CmioRxBufferSize").Return(uint64(1024)) + haltedBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(uint64(0), nil).Once() + haltedBackend.On( + "SendCmioResponse", + mock.AnythingOfType("uint16"), mock.Anything, expectedHash, + mock.AnythingOfType("time.Duration"), + ).Return(nil).Once() + haltedBackend.On("Run", uint64(100), mock.AnythingOfType("time.Duration")). + Return(Halted, nil).Once() + haltedBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")). + Return(uint64(100), nil).Once() + haltedMachine := &machineImpl{ + backend: haltedBackend, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: 5 * time.Second, + AdvanceMaxCycles: 1000, + AdvanceIncCycles: 100, + AdvanceIncDeadline: time.Second, + AdvanceMaxDeadline: 10 * time.Second, + }, + } + result, err = haltedMachine.process( + ctx, input, AdvanceStateRequest, &expectedHash, false, + ) + require.NoError(err) + require.Equal(CompletionStatusHalted, result.completion.status) + require.Nil(result.completion.data) + haltedBackend.AssertExpectations(s.T()) + // Test process with payload too large mockBackend2 := NewMockBackend() mockBackend2.On("CmioRxBufferSize").Return(uint64(5)) @@ -1006,19 +2395,22 @@ func (s *ImplementationSuite) TestProcess() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, }, } - _, _, _, _, _, _, err = machine2.process(ctx, input, AdvanceStateRequest, &expectedHash, false) + _, err = machine2.process(ctx, input, AdvanceStateRequest, &expectedHash, false) require.ErrorIs(err, ErrPayloadLengthLimitExceeded) + require.Contains(err.Error(), "advance request payload length 10") + require.Contains(err.Error(), "capacity 5") + require.NotContains(err.Error(), string(input)) mockBackend2.AssertExpectations(s.T()) // Test process with send error mockBackend3 := NewMockBackend() mockBackend3.On("CmioRxBufferSize").Return(uint64(1024)) + mockBackend3.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(0), nil) mockBackend3.On("SendCmioResponse", mock.AnythingOfType("uint16"), mock.Anything, @@ -1030,13 +2422,12 @@ func (s *ImplementationSuite) TestProcess() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, }, } - _, _, _, _, _, _, err = machine3.process(ctx, input, AdvanceStateRequest, &expectedHash, false) + _, err = machine3.process(ctx, input, AdvanceStateRequest, &expectedHash, false) require.Error(err) require.Contains(err.Error(), "send failed") mockBackend3.AssertExpectations(s.T()) @@ -1044,101 +2435,24 @@ func (s *ImplementationSuite) TestProcess() { // Test process with run error mockBackend4 := NewMockBackend() mockBackend4.On("CmioRxBufferSize").Return(uint64(1024)) - mockBackend4.On("SendCmioResponse", mock.AnythingOfType("uint16"), mock.Anything, expectedHash, mock.AnythingOfType("time.Duration")).Return(nil) mockBackend4.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(0), errors.New("read cycle failed")) machine4 := &machineImpl{ backend: mockBackend4, logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, }, } - _, _, _, _, _, _, err = machine4.process(ctx, input, AdvanceStateRequest, &expectedHash, false) + _, err = machine4.process(ctx, input, AdvanceStateRequest, &expectedHash, false) require.Error(err) require.Contains(err.Error(), "read cycle failed") + mockBackend4.AssertNotCalled(s.T(), "SendCmioResponse", mock.Anything, mock.Anything, mock.Anything) mockBackend4.AssertExpectations(s.T()) } -// Test run method with automatic yields and outputs/reports -func (s *ImplementationSuite) TestRunWithAutomaticYields() { - require := s.Require() - ctx := context.Background() - - mockBackend := NewMockBackend() - machine := &machineImpl{ - backend: mockBackend, - logger: s.logger, - params: model.ExecutionParameters{ - FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, - AdvanceIncCycles: 100, - AdvanceIncDeadline: time.Second * 1, - AdvanceMaxDeadline: time.Second * 10, - }, - } - - // Setup for automatic yield with output - mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(0), nil).Once() - mockBackend.On("Run", uint64(100), mock.AnythingOfType("time.Duration")).Return(YieldedAutomatically, nil).Once() - mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(50), nil).Once() - mockBackend.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( - uint8(0), uint16(AutomaticYieldReasonOutput), []byte("test output"), nil).Once() - - // Setup for manual yield after automatic yield - mockBackend.On("Run", uint64(150), mock.AnythingOfType("time.Duration")).Return(YieldedManually, nil).Once() - mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(100), nil).Once() - - outputs, reports, _, _, err := machine.run(ctx, AdvanceStateRequest, false) - require.NoError(err) - require.Len(outputs, 1) - require.Equal([]byte("test output"), outputs[0]) - require.Empty(reports) - - mockBackend.AssertExpectations(s.T()) -} - -// Test run method with automatic yields and reports -func (s *ImplementationSuite) TestRunWithAutomaticYieldsReports() { - require := s.Require() - ctx := context.Background() - - mockBackend := NewMockBackend() - machine := &machineImpl{ - backend: mockBackend, - logger: s.logger, - params: model.ExecutionParameters{ - FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, - AdvanceIncCycles: 100, - AdvanceIncDeadline: time.Second * 1, - AdvanceMaxDeadline: time.Second * 10, - }, - } - - // Setup for automatic yield with report - mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(0), nil).Once() - mockBackend.On("Run", uint64(100), mock.AnythingOfType("time.Duration")).Return(YieldedAutomatically, nil).Once() - mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(50), nil).Once() - mockBackend.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( - uint8(0), uint16(AutomaticYieldReasonReport), []byte("test report"), nil).Once() - - // Setup for manual yield after automatic yield - mockBackend.On("Run", uint64(150), mock.AnythingOfType("time.Duration")).Return(YieldedManually, nil).Once() - mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(100), nil).Once() - - outputs, reports, _, _, err := machine.run(ctx, AdvanceStateRequest, false) - require.NoError(err) - require.Empty(outputs) - require.Len(reports, 1) - require.Equal([]byte("test report"), reports[0]) - - mockBackend.AssertExpectations(s.T()) -} - // Test multiple automatic yields in sequence func (s *ImplementationSuite) TestMultipleAutomaticYields() { require := s.Require() @@ -1150,7 +2464,6 @@ func (s *ImplementationSuite) TestMultipleAutomaticYields() { logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, - AdvanceMaxCycles: 1000, AdvanceIncCycles: 100, AdvanceIncDeadline: time.Second * 1, AdvanceMaxDeadline: time.Second * 10, @@ -1158,8 +2471,6 @@ func (s *ImplementationSuite) TestMultipleAutomaticYields() { } // Setup for multiple automatic yields followed by manual yield - mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(0), nil).Once() - // First automatic yield with output mockBackend.On("Run", uint64(100), mock.AnythingOfType("time.Duration")).Return(YieldedAutomatically, nil).Once() mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(10), nil).Once() @@ -1194,16 +2505,18 @@ func (s *ImplementationSuite) TestMultipleAutomaticYields() { mockBackend.On("Run", uint64(150), mock.AnythingOfType("time.Duration")).Return(YieldedManually, nil).Once() mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(60), nil).Once() - outputs, reports, _, _, err := machine.run(ctx, AdvanceStateRequest, false) + result, err := machine.run( + ctx, AdvanceStateRequest, false, testExecutionBounds(0, model.MaxExecutionCycleSpan), + ) require.NoError(err) - require.Len(outputs, 2) - require.Equal([]byte("output1"), outputs[0]) - require.Equal([]byte(""), outputs[1]) + require.Len(result.outputs, 2) + require.Equal([]byte("output1"), result.outputs[0]) + require.Equal([]byte(""), result.outputs[1]) - require.Len(reports, 2) - require.Equal([]byte("output2"), reports[0]) - require.Equal([]byte(""), reports[1]) + require.Len(result.reports, 2) + require.Equal([]byte("output2"), result.reports[0]) + require.Equal([]byte(""), result.reports[1]) mockBackend.AssertExpectations(s.T()) } @@ -1234,3 +2547,52 @@ func (s *ImplementationSuite) TestCheckContext() { err = checkContext(nil) // nolint require.NoError(err) // nil context is valid in Go } + +func (s *ImplementationSuite) TestInputHashCollectionSpan() { + require := s.Require() + + for _, test := range []struct { + name string + hashes uint64 + padding uint64 + wantErr bool + }{ + {name: "empty collection", hashes: 0, padding: InputEntryCapacity}, + {name: "partial collection", hashes: 2, padding: InputEntryCapacity - 2}, + {name: "exact boundary", hashes: InputEntryCapacity, padding: 0}, + {name: "short span", hashes: 2, padding: InputEntryCapacity - 3, wantErr: true}, + {name: "long span", hashes: 2, padding: InputEntryCapacity - 1, wantErr: true}, + {name: "collector overflow", hashes: InputEntryCapacity + 1, wantErr: true}, + } { + s.Run(test.name, func() { + err := ValidateInputHashCollectionSpan(test.hashes, test.padding) + require.Equal(test.wantErr, err != nil) + }) + } + + padding, err := inputHashCollectionPaddingRepetitions(InputEntryCapacity) + require.NoError(err) + require.Zero(padding) + _, err = inputHashCollectionPaddingRepetitions(InputEntryCapacity + 1) + require.ErrorContains(err, "exceeds input entry capacity") + + canonicalCount, padding, err := canonicalInputHashCollectionShape( + InputEntryCapacity, + true, + ) + require.NoError(err) + require.Equal(InputEntryCapacity-1, canonicalCount) + require.Equal(uint64(1), padding) + + // Validate the raw collector count before removing the terminal sample. + // Otherwise S+1 would be incorrectly normalized into a valid S shape. + _, _, err = canonicalInputHashCollectionShape(InputEntryCapacity+1, true) + require.ErrorContains(err, "exceeds input entry capacity") + + // A malformed collection is an internal failure even when execution also + // reached a completed terminal state. The terminal status is context only. + err = invalidInputHashCollectionSpanError(ErrHalted, errors.New("overcollection")) + require.ErrorIs(err, ErrMachineInternal) + require.NotErrorIs(err, ErrHalted) + require.ErrorContains(err, ErrHalted.Error()) +} diff --git a/pkg/machine/libcartesi.go b/pkg/machine/libcartesi.go index 2414aa7a7..8044f013c 100644 --- a/pkg/machine/libcartesi.go +++ b/pkg/machine/libcartesi.go @@ -103,6 +103,11 @@ func (p *proofJson) UnmarshalJSON(data []byte) error { } func NewLibCartesiBackend(address string, timeout time.Duration) (Backend, string, uint32, error) { + // Keep this defensive check even though the advancer validates the constants + // at startup. Other callers can construct a backend directly. + if err := ValidateEmulatorComputationHashLimits(); err != nil { + return nil, "", 0, err + } rm, address, pid, err := emulator.SpawnServer(address, timeout) if err != nil { return nil, address, pid, err @@ -110,6 +115,53 @@ func NewLibCartesiBackend(address string, timeout time.Duration) (Backend, strin return &LibCartesiBackend{inner: rm}, address, pid, nil } +// ValidateEmulatorComputationHashLimits verifies that the node and compiled +// emulator agree on the three exported rollup limits checked here. A mismatch +// would change per-input or per-epoch computation-hash dimensions. This does +// not validate the selected sampling period, bundle exponent, runtime library, +// or hash algorithm. +func ValidateEmulatorComputationHashLimits() error { + checks := []struct { + name string + node uint64 + emulator uint64 + }{ + { + name: "LOG2_MAX_UARCH_CYCLES_PER_MCYCLE", + node: Log2MaxUarchCyclesPerMCycle, + emulator: emulator.Log2MaxUarchCyclesPerMCycle, + }, + { + name: "LOG2_MAX_MCYCLES_PER_ADVANCE_STATE", + node: Log2MaxMCyclesPerAdvanceState, + emulator: emulator.Log2MaxMCyclesPerAdvanceState, + }, + { + name: "LOG2_MAX_ADVANCE_STATES_PER_EPOCH", + node: Log2MaxAdvanceStatesPerEpoch, + emulator: emulator.Log2MaxAdvanceStatesPerEpoch, + }, + } + for _, check := range checks { + if err := validateEmulatorComputationHashLimit(check.name, check.node, check.emulator); err != nil { + return err + } + } + return nil +} + +// validateEmulatorComputationHashLimit takes explicit values so the mismatch +// path can be tested without requiring a differently compiled emulator library. +func validateEmulatorComputationHashLimit(name string, nodeLog2, emulatorLog2 uint64) error { + if nodeLog2 == emulatorLog2 { + return nil + } + return fmt.Errorf( + "node computation-hash dimension 2^%d does not match emulator CM_ROLLUP_%s=%d: %w", + nodeLog2, name, emulatorLog2, ErrMachineInternal, + ) +} + // LibCartesiBackend is an adapter that implements Backend by wrapping a RemoteMachineInterface. type LibCartesiBackend struct { inner RemoteMachineInterface @@ -268,15 +320,20 @@ func (e *LibCartesiBackend) RunAndCollectRootHashes( if state == nil { return Failed, errors.New("nil state") } - if state.Period == 0 { + if state.MCycleSamplingPeriod == 0 { return Failed, errors.New("state period must be greater than zero") } - log2Period := uint64(bits.Len64(state.Period) - 1) - if uint64(1)<= state.Period { - return Failed, fmt.Errorf("phase must be less than period, got phase %v and period %v", state.Phase, state.Period) + if state.MCyclePhase >= state.MCycleSamplingPeriod { + return Failed, fmt.Errorf( + "phase must be less than period, got phase %v and period %v", + state.MCyclePhase, + state.MCycleSamplingPeriod, + ) } if err := e.inner.SetTimeout(timeout.Milliseconds()); err != nil { return Failed, fmt.Errorf("failed to set operation timeout: %w", err) @@ -285,18 +342,19 @@ func (e *LibCartesiBackend) RunAndCollectRootHashes( rawResult, err := e.inner.CollectMCycleRootHashes( mcycleEnd, log2Period, - state.Phase, - state.BundleLog2, + state.MCyclePhase, + state.Log2BundleMCycleCount, state.PartialBundle, ) if err != nil { return Failed, err } result := struct { - RootHashes []string `json:"hashes"` - MCyclePhase uint64 `json:"mcycle_phase"` - BreakReason string `json:"break_reason"` - PartialBundle json.RawMessage `json:"partial_bundle,omitempty"` + RootHashes []string `json:"hashes"` + MCyclePhase uint64 `json:"mcycle_phase"` + BreakReason string `json:"break_reason"` + PartialBundle json.RawMessage `json:"partial_bundle,omitempty"` + ConsoleIOError string `json:"console_io_error,omitempty"` }{} err = json.Unmarshal(rawResult, &result) if err != nil { @@ -306,11 +364,11 @@ func (e *LibCartesiBackend) RunAndCollectRootHashes( if err != nil { return Failed, fmt.Errorf("invalid CollectMCycleRootHashes result: %w", err) } - if result.MCyclePhase >= state.Period { + if result.MCyclePhase >= state.MCycleSamplingPeriod { return Failed, fmt.Errorf( "invalid CollectMCycleRootHashes result: phase %v must be less than period %v", result.MCyclePhase, - state.Period, + state.MCycleSamplingPeriod, ) } @@ -322,8 +380,9 @@ func (e *LibCartesiBackend) RunAndCollectRootHashes( } state.Hashes = append(state.Hashes, decodedHashes...) - state.Phase = result.MCyclePhase + state.MCyclePhase = result.MCyclePhase state.PartialBundle = result.PartialBundle + state.ConsoleIOError = result.ConsoleIOError return reason, nil } diff --git a/pkg/machine/libcartesi_test.go b/pkg/machine/libcartesi_test.go index 82819384a..aafd798f1 100644 --- a/pkg/machine/libcartesi_test.go +++ b/pkg/machine/libcartesi_test.go @@ -12,6 +12,7 @@ import ( "github.com/cartesi/rollups-node/pkg/emulator" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) @@ -19,6 +20,16 @@ func TestLibCartesi(t *testing.T) { suite.Run(t, new(LibCartesiSuite)) } +func TestValidateEmulatorComputationHashLimits(t *testing.T) { + require.NoError(t, ValidateEmulatorComputationHashLimits()) + require.NoError(t, validateEmulatorComputationHashLimit("LOG2_MAX_MCYCLES_PER_ADVANCE_STATE", 48, 48)) + + err := validateEmulatorComputationHashLimit("LOG2_MAX_MCYCLES_PER_ADVANCE_STATE", 48, 47) + require.ErrorIs(t, err, ErrMachineInternal) + require.ErrorContains(t, err, "node computation-hash dimension 2^48") + require.ErrorContains(t, err, "CM_ROLLUP_LOG2_MAX_MCYCLES_PER_ADVANCE_STATE=47") +} + type LibCartesiSuite struct { suite.Suite mockRemoteMachine *MockRemoteMachine @@ -111,7 +122,7 @@ func (s *LibCartesiSuite) TestRunMcycleOverflow() { func (s *LibCartesiSuite) TestRunAndCollectRootHashesMcycleOverflow() { require := s.Require() - state := &HashCollectorState{Period: 1} + state := &HashCollectorState{MCycleSamplingPeriod: 1} result := []byte(`{"hashes":[],"mcycle_phase":0,"break_reason":"mcycle_overflow"}`) s.mockRemoteMachine.On("SetTimeout", int64(5000)).Return(nil) @@ -142,22 +153,24 @@ func (s *LibCartesiSuite) TestRunAndCollectRootHashesContinuesPartialBundle() { previousHash := Hash{0x11} collectedHash := Hash{0x22} state := &HashCollectorState{ - Period: 4, - Phase: 3, - BundleLog2: 2, - Hashes: []Hash{previousHash}, - PartialBundle: previousPartialBundle, + MCycleSamplingPeriod: 4, + MCyclePhase: 3, + Log2BundleMCycleCount: 2, + Hashes: []Hash{previousHash}, + PartialBundle: previousPartialBundle, } result, err := json.Marshal(struct { - Hashes []string `json:"hashes"` - MCyclePhase uint64 `json:"mcycle_phase"` - BreakReason string `json:"break_reason"` - PartialBundle json.RawMessage `json:"partial_bundle"` + Hashes []string `json:"hashes"` + MCyclePhase uint64 `json:"mcycle_phase"` + BreakReason string `json:"break_reason"` + PartialBundle json.RawMessage `json:"partial_bundle"` + ConsoleIOError string `json:"console_io_error"` }{ - Hashes: []string{base64.StdEncoding.EncodeToString(collectedHash[:])}, - MCyclePhase: 1, - BreakReason: "reached_target_mcycle", - PartialBundle: nextPartialBundle, + Hashes: []string{base64.StdEncoding.EncodeToString(collectedHash[:])}, + MCyclePhase: 1, + BreakReason: "reached_target_mcycle", + PartialBundle: nextPartialBundle, + ConsoleIOError: "console write failed", }) require.NoError(err) @@ -174,9 +187,10 @@ func (s *LibCartesiSuite) TestRunAndCollectRootHashesContinuesPartialBundle() { breakReason, err := s.backend.RunAndCollectRootHashes(1000, state, 5*time.Second) require.NoError(err) require.Equal(ReachedTargetMcycle, breakReason) - require.Equal(uint64(1), state.Phase) + require.Equal(uint64(1), state.MCyclePhase) require.Equal([]Hash{previousHash, collectedHash}, state.Hashes) require.Equal(nextPartialBundle, state.PartialBundle) + require.Equal("console write failed", state.ConsoleIOError) s.mockRemoteMachine.AssertExpectations(s.T()) } @@ -186,10 +200,10 @@ func (s *LibCartesiSuite) TestRunAndCollectRootHashesClearsPartialBundleAtFixedP `{"log2_max_leaves":2,"hash_function":"keccak256","leaf_count":3,"context":["AwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="]}`, ) state := &HashCollectorState{ - Period: 4, - Phase: 2, - BundleLog2: 2, - PartialBundle: previousPartialBundle, + MCycleSamplingPeriod: 4, + MCyclePhase: 2, + Log2BundleMCycleCount: 2, + PartialBundle: previousPartialBundle, } result := []byte(`{"hashes":[],"mcycle_phase":2,"break_reason":"halted"}`) @@ -221,11 +235,11 @@ func (s *LibCartesiSuite) TestRunAndCollectRootHashesInvalidHashLeavesStateUncha previousHash := Hash{0x11} validHash := Hash{0x22} state := &HashCollectorState{ - Period: 4, - Phase: 3, - BundleLog2: 2, - Hashes: []Hash{previousHash}, - PartialBundle: previousPartialBundle, + MCycleSamplingPeriod: 4, + MCyclePhase: 3, + Log2BundleMCycleCount: 2, + Hashes: []Hash{previousHash}, + PartialBundle: previousPartialBundle, } originalState := *state originalState.Hashes = append([]Hash(nil), state.Hashes...) @@ -267,9 +281,9 @@ func (s *LibCartesiSuite) TestRunAndCollectRootHashesInvalidBreakReasonLeavesSta require := s.Require() previousHash := Hash{0x11} state := &HashCollectorState{ - Period: 4, - Phase: 3, - Hashes: []Hash{previousHash}, + MCycleSamplingPeriod: 4, + MCyclePhase: 3, + Hashes: []Hash{previousHash}, } originalState := *state originalState.Hashes = append([]Hash(nil), state.Hashes...) @@ -294,7 +308,7 @@ func (s *LibCartesiSuite) TestRunAndCollectRootHashesInvalidBreakReasonLeavesSta func (s *LibCartesiSuite) TestRunAndCollectRootHashesInvalidResultPhaseLeavesStateUnchanged() { require := s.Require() - state := &HashCollectorState{Period: 4, Phase: 3} + state := &HashCollectorState{MCycleSamplingPeriod: 4, MCyclePhase: 3} originalState := *state result := []byte(`{"hashes":[],"mcycle_phase":4,"break_reason":"reached_target_mcycle"}`) @@ -317,7 +331,7 @@ func (s *LibCartesiSuite) TestRunAndCollectRootHashesInvalidResultPhaseLeavesSta func (s *LibCartesiSuite) TestRunAndCollectRootHashesRejectsInvalidInputPhase() { require := s.Require() - state := &HashCollectorState{Period: 4, Phase: 4} + state := &HashCollectorState{MCycleSamplingPeriod: 4, MCyclePhase: 4} breakReason, err := s.backend.RunAndCollectRootHashes(1000, state, 5*time.Second) require.ErrorContains(err, "phase must be less than period") diff --git a/pkg/machine/machine.go b/pkg/machine/machine.go index aaedb21d2..6dc582c8b 100644 --- a/pkg/machine/machine.go +++ b/pkg/machine/machine.go @@ -1,8 +1,6 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -// Package machine provides a unified interface for interacting with Cartesi machines. -// It consolidates functionality from the previous rollupsmachine and cartesimachine packages. package machine import ( @@ -27,14 +25,61 @@ type ( Hash = [HashSize]byte ) -// AdvanceResponse contains the result of an advance operation. +// CompletionStatus identifies how a guest-machine request completed. Advance +// and Inspect have the same completion outcomes; their callers decide whether +// and how those outcomes affect canonical state. If execution does not +// complete, the operation returns CompletionStatusUnknown together with an +// error instead. +type CompletionStatus uint8 + +const ( + // CompletionStatusUnknown is the zero-value sentinel. A successful request + // never returns it. + CompletionStatusUnknown CompletionStatus = iota + CompletionStatusAccepted + CompletionStatusRejected + CompletionStatusException + CompletionStatusHalted +) + +// IsCompleted reports whether the status is a completed guest-machine outcome. +func (s CompletionStatus) IsCompleted() bool { + switch s { + case CompletionStatusAccepted, + CompletionStatusRejected, + CompletionStatusException, + CompletionStatusHalted: + return true + case CompletionStatusUnknown: + return false + default: + return false + } +} + +// AdvanceResponse contains the result of a completed advance operation. type AdvanceResponse struct { - Accepted bool - Outputs []Output - Reports []Report - Hashes []Hash - RemainingCycles uint64 - OutputsHash Hash + Status CompletionStatus + Outputs []Output + Reports []Report + // ExceptionData is the raw CMIO payload supplied by the guest when Status + // is CompletionStatusException. It is nil for every other status; an + // exception with an empty payload is represented by a non-nil empty slice. + ExceptionData []byte + PeriodicStateHashes []Hash + PaddingRepetitions uint64 + OutputsHash Hash +} + +// InspectResponse contains the result of an inspect operation. On incomplete +// execution, Reports contains any reports emitted before the failure, Status +// is CompletionStatusUnknown, and Inspect returns a non-nil error. +type InspectResponse struct { + Status CompletionStatus + // ExceptionData has the same status-dependent meaning as + // AdvanceResponse.ExceptionData. + ExceptionData []byte + Reports []Report } // Common errors @@ -49,12 +94,26 @@ var ( ErrHalted = errors.New("machine halted") ErrOutputsLimitExceeded = errors.New("outputs limit exceeded") ErrReportsLimitExceeded = errors.New("reports limit exceeded") - ErrReachedTargetMcycle = errors.New("machine reached target mcycle") ErrPayloadLengthLimitExceeded = errors.New("payload length limit exceeded") ErrHashLength = errors.New("hash does not have the exactly number of bytes") ErrReachedLimitMcycle = errors.New("machine reached limit mcycle") + + // ErrMcycleOverflow preserves the emulator-reported fact that the machine + // itself reached imcyclemax, rather than a node target. Canonical overflow + // eligibility depends on that stop origin. It wraps ErrReachedLimitMcycle so + // existing execution-limit classification remains unchanged. + ErrMcycleOverflow = fmt.Errorf("machine reached imcyclemax: %w", ErrReachedLimitMcycle) ) +// IsExecutionLimitError reports whether err is an incomplete execution caused +// by a local payload, response-count, or cycle ceiling. +func IsExecutionLimitError(err error) bool { + return errors.Is(err, ErrPayloadLengthLimitExceeded) || + errors.Is(err, ErrOutputsLimitExceeded) || + errors.Is(err, ErrReportsLimitExceeded) || + errors.Is(err, ErrReachedLimitMcycle) +} + // The Machine interface covers the core rollups-oriented functionalities of a cartesi // machine: forking, getting the merkle tree's root hash, sending advance-state requests, // sending inspect-state requests, and storing machine state. @@ -71,16 +130,18 @@ type Machine interface { // Advance sends an input to the machine. // The checkpointHash is the machine's root hash before processing the input, // sent along with the request so the machine can revert to it if needed. - // It always returns a non-nil AdvanceResponse, even on error paths. - // The response contains whether the request was accepted, - // the corresponding outputs, reports, and the hash of the outputs. - // In case the request is not accepted, the response does not contain outputs. + // A non-nil response and nil error mean the machine completed with one of + // the four completed CompletionStatus values. Any incomplete execution—input + // validation, an operational limit, deadline/cancellation, or infrastructure + // failure—returns a nil response and a non-nil error. CompletionStatusUnknown + // is never returned by a successful call. Advance(ctx context.Context, input []byte, checkpointHash Hash, computeHashes bool) (*AdvanceResponse, error) - // Inspect sends a query to the machine. - // It returns a boolean indicating whether or not the request was accepted - // It also returns the corresponding reports. - Inspect(ctx context.Context, query []byte) (bool, []Report, error) + // Inspect sends a query to the machine. A nil error means the guest completed + // with one of the four non-unknown CompletionStatus values. On incomplete + // execution, the response preserves reports emitted before the failure and + // the error identifies why inspection could not complete. + Inspect(ctx context.Context, query []byte) (*InspectResponse, error) // Store saves the machine state to the specified path. Store(ctx context.Context, path string) error @@ -108,10 +169,10 @@ func DefaultConfig(path string) *MachineConfig { Address: "127.0.0.1:0", Path: path, ExecutionParameters: model.ExecutionParameters{ - AdvanceIncCycles: 1 << 22, // nolint: mnd - AdvanceMaxCycles: ^uint64(0) >> 2, // nolint: mnd - InspectIncCycles: 1 << 22, // nolint: mnd - InspectMaxCycles: ^uint64(0) >> 2, // nolint: mnd + AdvanceIncCycles: 1 << 22, //nolint:mnd + AdvanceMaxCycles: 0, + InspectIncCycles: 1 << 22, //nolint:mnd + InspectMaxCycles: 0, AdvanceIncDeadline: time.Second * 10, // nolint: mnd AdvanceMaxDeadline: time.Second * 180, // nolint: mnd InspectIncDeadline: time.Second * 10, // nolint: mnd @@ -196,13 +257,17 @@ func Load(ctx context.Context, logger *slog.Logger, config *MachineConfig) (Mach return nil, ErrNotAtManualYield } - // Ensures that the last request the machine received did not yield an exception. - accepted, _, err := machine.wasLastRequestAccepted(ctx) + // Ensures that the last request left the machine in an accepted manual yield. + manualResult, err := machine.readManualYieldResult(ctx) if err != nil { machine.Close() return nil, err } - if !accepted { + if manualResult.status == CompletionStatusException { + machine.Close() + return nil, ErrException + } + if manualResult.status != CompletionStatusAccepted { machine.Close() return nil, ErrRejected } diff --git a/pkg/machine/machine_test.go b/pkg/machine/machine_test.go index 2acdc1043..7f002a1af 100644 --- a/pkg/machine/machine_test.go +++ b/pkg/machine/machine_test.go @@ -6,11 +6,14 @@ package machine import ( "context" "errors" + "fmt" "io" "log/slog" "testing" "time" + "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/pkg/emulator" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" ) @@ -185,9 +188,9 @@ func (s *MachineSuite) TestDefaultConfig() { // Test execution parameters are set require.Greater(config.ExecutionParameters.AdvanceIncCycles, uint64(0)) - require.Greater(config.ExecutionParameters.AdvanceMaxCycles, uint64(0)) + require.Zero(config.ExecutionParameters.AdvanceMaxCycles) require.Greater(config.ExecutionParameters.InspectIncCycles, uint64(0)) - require.Greater(config.ExecutionParameters.InspectMaxCycles, uint64(0)) + require.Zero(config.ExecutionParameters.InspectMaxCycles) require.Greater(config.ExecutionParameters.AdvanceIncDeadline, time.Duration(0)) require.Greater(config.ExecutionParameters.AdvanceMaxDeadline, time.Duration(0)) require.Greater(config.ExecutionParameters.InspectIncDeadline, time.Duration(0)) @@ -197,6 +200,44 @@ func (s *MachineSuite) TestDefaultConfig() { require.Greater(config.ExecutionParameters.FastDeadline, time.Duration(0)) } +func (s *MachineSuite) TestConfiguredHardCycleCeilingMatchesMachineInputSpan() { + s.Require().Equal(uint64(1)<